@webcrawl/mcp 1.4.2 → 1.4.3

package/dist/core/parse/media.js

@@ -0,0 +1,238 @@
+/**
+ * Audio / video -> timestamped transcript.
+ * 1. Captions first: if the caller passes captions (WebVTT or SRT), use them. No model call.
+ * 2. Otherwise Groq `whisper-large-v3-turbo` if GROQ_API_KEY is set, falling back to the AI Gateway
+ *    (`spacexai/grok-stt` by default, billed at 2 credits/min) when Groq is missing or fails.
+ * 3. Optional visual description for video via Gemini Flash-Lite (AI Gateway).
+ * Nothing is invented: without a provider the transcript is empty with a warning.
+ */
+import { createGateway } from "@ai-sdk/gateway";
+import { experimental_transcribe, generateText } from "ai";
+const GROQ_URL = "https://api.groq.com/openai/v1/audio/transcriptions";
+const DEFAULT_TRANSCRIBE_MODEL = "whisper-large-v3-turbo";
+const DEFAULT_VISION_MODEL = "google/gemini-2.5-flash-lite";
+/** Gemini inline video limit through the gateway; larger videos skip the description. */
+const MAX_VIDEO_DESCRIBE_BYTES = 20 * 1024 * 1024;
+export function transcriptionAvailable() {
+    return Boolean(process.env.GROQ_API_KEY || gatewayKey());
+}
+function gatewayKey() {
+    return process.env.AI_GATEWAY_API_KEY || process.env.VERCEL_OIDC_TOKEN;
+}
+/** ~$0.0017/min with segment timestamps; whisper-1 is $0.006/min. Override with WEBCRAWL_GATEWAY_TRANSCRIBE_MODEL. */
+const DEFAULT_GATEWAY_TRANSCRIBE_MODEL = "spacexai/grok-stt";
+/** Transcribes through the Vercel AI Gateway (no extra provider key needed). */
+export async function gatewayTranscribe(bytes, options = {}) {
+    const key = gatewayKey();
+    if (!key)
+        throw new Error("no transcription provider configured");
+    const gw = createGateway({ apiKey: key });
+    const r = await experimental_transcribe({
+        model: gw.transcription(process.env.WEBCRAWL_GATEWAY_TRANSCRIBE_MODEL || DEFAULT_GATEWAY_TRANSCRIBE_MODEL),
+        audio: bytes,
+        abortSignal: options.signal ?? AbortSignal.timeout(300_000),
+        ...(options.language ? { providerOptions: { openai: { language: options.language } } } : {}),
+    });
+    const segments = r.segments
+        .map((s) => ({ start: s.startSecond, end: s.endSecond, text: s.text.trim() }))
+        .filter((s) => s.text);
+    if (!segments.length && r.text.trim())
+        segments.push({ start: 0, end: r.durationInSeconds ?? 0, text: r.text.trim() });
+    return { segments, duration: r.durationInSeconds ?? segments[segments.length - 1]?.end ?? 0, language: r.language };
+}
+/** 3725.4 -> "1:02:05"; 65 -> "1:05". */
+export function formatTimestamp(seconds) {
+    const s = Math.max(0, Math.floor(seconds));
+    const h = Math.floor(s / 3600);
+    const m = Math.floor((s % 3600) / 60);
+    const sec = String(s % 60).padStart(2, "0");
+    return h > 0 ? `${h}:${String(m).padStart(2, "0")}:${sec}` : `${m}:${sec}`;
+}
+function parseClock(t) {
+    const parts = t.trim().replace(",", ".").split(":").map(Number);
+    return parts.reduce((acc, n) => acc * 60 + n, 0);
+}
+/** Parses WebVTT or SRT cues. Drops styling tags and repeated rolling-caption lines. */
+export function parseCaptions(text) {
+    const cues = [];
+    const blocks = text.replace(/\r/g, "").split(/\n{2,}/);
+    for (const block of blocks) {
+        const lines = block.split("\n");
+        const i = lines.findIndex((l) => l.includes("-->"));
+        if (i === -1)
+            continue;
+        const [a, b] = lines[i].split("-->");
+        const body = lines
+            .slice(i + 1)
+            .join(" ")
+            .replace(/<[^>]+>/g, "")
+            .replace(/\s+/g, " ")
+            .trim();
+        if (!body)
+            continue;
+        const start = parseClock(a);
+        const end = parseClock(b.trim().split(/\s/)[0]);
+        const prev = cues[cues.length - 1];
+        if (prev && prev.text === body) {
+            prev.end = end;
+            continue;
+        }
+        cues.push({ start, end, text: body });
+    }
+    return cues;
+}
+/** Groups segments into ~30 s paragraphs, each prefixed with its start time. */
+export function transcriptMarkdown(segments, paragraphSeconds = 30) {
+    const out = [];
+    let buf = [];
+    let bufStart = 0;
+    for (const seg of segments) {
+        if (buf.length === 0)
+            bufStart = seg.start;
+        buf.push(seg.text.trim());
+        if (seg.end - bufStart >= paragraphSeconds) {
+            out.push(`**[${formatTimestamp(bufStart)}]** ${buf.join(" ")}`);
+            buf = [];
+        }
+    }
+    if (buf.length)
+        out.push(`**[${formatTimestamp(bufStart)}]** ${buf.join(" ")}`);
+    return out.join("\n\n");
+}
+export async function groqTranscribe(bytes, mime, options = {}) {
+    const key = process.env.GROQ_API_KEY;
+    if (!key)
+        throw new Error("GROQ_API_KEY not set");
+    const form = new FormData();
+    const name = options.fileName?.split("/").pop() || `media.${mime.split("/")[1] || "bin"}`;
+    form.append("file", new Blob([bytes], { type: mime }), name);
+    form.append("model", process.env.WEBCRAWL_TRANSCRIBE_MODEL || DEFAULT_TRANSCRIBE_MODEL);
+    form.append("response_format", "verbose_json");
+    form.append("timestamp_granularities[]", "segment");
+    if (options.language)
+        form.append("language", options.language);
+    const res = await fetch(GROQ_URL, {
+        method: "POST",
+        headers: { Authorization: `Bearer ${key}` },
+        body: form,
+        signal: options.signal ?? AbortSignal.timeout(300_000),
+    });
+    if (!res.ok) {
+        const detail = (await res.text().catch(() => "")).slice(0, 300);
+        throw new Error(`Groq transcription ${res.status}: ${detail}`);
+    }
+    const json = (await res.json());
+    const segments = (json.segments ?? []).map((s) => ({ start: s.start, end: s.end, text: s.text.trim() }));
+    if (segments.length === 0 && json.text?.trim()) {
+        segments.push({ start: 0, end: json.duration ?? 0, text: json.text.trim() });
+    }
+    const duration = json.duration ?? segments[segments.length - 1]?.end ?? 0;
+    return { segments, duration, language: json.language };
+}
+async function describeVideo(bytes, mime, signal) {
+    const key = gatewayKey();
+    if (!key)
+        throw new Error("no vision model configured");
+    if (bytes.byteLength > MAX_VIDEO_DESCRIBE_BYTES)
+        throw new Error("video too large to describe");
+    const gw = createGateway({ apiKey: key });
+    const { text } = await generateText({
+        model: gw(process.env.WEBCRAWL_VISION_MODEL || DEFAULT_VISION_MODEL),
+        maxOutputTokens: 1_500,
+        abortSignal: signal ?? AbortSignal.timeout(120_000),
+        // Low resolution is ~66 tokens/frame instead of ~258: plenty for a scene summary, ~4x cheaper.
+        providerOptions: { google: { mediaResolution: "MEDIA_RESOLUTION_LOW" } },
+        messages: [
+            {
+                role: "user",
+                content: [
+                    { type: "file", data: bytes, mediaType: mime },
+                    {
+                        type: "text",
+                        text: "Describe what is shown on screen in this video in 3-6 plain sentences: setting, people, on-screen text, slides or charts. Do not transcribe speech.",
+                    },
+                ],
+            },
+        ],
+    });
+    return text.trim();
+}
+export async function parseMedia(bytes, kind, mime, options = {}) {
+    const started = performance.now();
+    const warnings = [];
+    const title = options.fileName?.split("/").pop()?.replace(/\.[a-z0-9]+$/i, "") || "";
+    let segments = [];
+    let duration;
+    let provider;
+    if (options.captions?.trim()) {
+        segments = parseCaptions(options.captions);
+        if (segments.length) {
+            provider = "captions";
+            duration = segments[segments.length - 1].end;
+        }
+        else {
+            warnings.push("Captions were provided but contained no cues; transcribing instead");
+        }
+    }
+    if (!provider) {
+        if (!transcriptionAvailable()) {
+            warnings.push("Transcript unavailable: no transcription provider configured");
+        }
+        else {
+            // Groq turbo (if configured) is ~2.5x cheaper than gateway grok-stt; the gateway is billed 2 cr/min.
+            const attempts = [];
+            if (process.env.GROQ_API_KEY)
+                attempts.push({ name: "groq-whisper-turbo", run: () => groqTranscribe(bytes, mime, options) });
+            if (gatewayKey())
+                attempts.push({ name: "gateway-stt", run: () => gatewayTranscribe(bytes, options) });
+            const errors = [];
+            for (const a of attempts) {
+                try {
+                    const r = await a.run();
+                    segments = r.segments;
+                    duration = r.duration;
+                    provider = a.name;
+                    if (!segments.length)
+                        warnings.push("No speech detected");
+                    break;
+                }
+                catch (err) {
+                    errors.push(`${a.name}: ${err instanceof Error ? err.message : String(err)}`);
+                }
+            }
+            if (!provider)
+                warnings.push(`Transcription failed: ${errors.join("; ")}`);
+        }
+    }
+    const parts = [];
+    let visionProvider;
+    if (kind === "video" && options.describeVideo) {
+        try {
+            const desc = await describeVideo(bytes, mime, options.signal);
+            if (desc) {
+                parts.push(`## What's on screen\n\n${desc}`);
+                visionProvider = "gemini-flash-lite";
+            }
+        }
+        catch (err) {
+            warnings.push(`Video description skipped: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    if (segments.length)
+        parts.push(`## Transcript\n\n${transcriptMarkdown(segments)}`);
+    return {
+        title,
+        markdown: parts.join("\n\n"),
+        meta: {
+            kind,
+            mime,
+            method: "transcript",
+            bytes: bytes.byteLength,
+            ...(duration !== undefined ? { durationSeconds: Math.round(duration * 10) / 10 } : {}),
+            ...(provider ? { transcriptProvider: provider } : {}),
+            ...(visionProvider ? { visionProvider } : {}),
+            warnings,
+            processingMs: Math.round(performance.now() - started),
+        },
+    };
+}

package/dist/core/parse/merge.d.ts

@@ -0,0 +1,19 @@
+/** Pure helpers for the durable parse workflow: batch PDF pages, then merge results. */
+import type { ParseMeta } from "./types.js";
+export interface Chunk {
+    title: string;
+    markdown: string;
+    meta: ParseMeta;
+    cached: boolean;
+}
+/** Splits the pages to read into fixed-size batches. */
+export declare function pdfBatches(pageCount: number, selected: number[] | undefined, size: number): number[][];
+/** Merges per-batch PDF results back into one document, in page order. */
+export declare function mergePdfChunks(chunks: Chunk[], info: {
+    pageCount?: number;
+    title?: string;
+}): {
+    title: string;
+    markdown: string;
+    meta: ParseMeta;
+};

package/dist/core/parse/merge.js

@@ -0,0 +1,48 @@
+/** Splits the pages to read into fixed-size batches. */
+export function pdfBatches(pageCount, selected, size) {
+    const pages = selected?.length
+        ? [...new Set(selected)].filter((p) => p >= 1 && p <= pageCount).sort((a, b) => a - b)
+        : Array.from({ length: pageCount }, (_, i) => i + 1);
+    const out = [];
+    for (let i = 0; i < pages.length; i += size)
+        out.push(pages.slice(i, i + size));
+    return out;
+}
+/** Merges per-batch PDF results back into one document, in page order. */
+export function mergePdfChunks(chunks, info) {
+    const nums = (pick) => chunks.flatMap((c) => pick(c.meta) ?? []).sort((a, b) => a - b);
+    const skipped = chunks.flatMap((c) => c.meta.pagesSkipped ?? []).sort((a, b) => a.page - b.page);
+    const pagesNative = nums((m) => m.pagesNative);
+    const pagesOcr = nums((m) => m.pagesOcr);
+    const parts = chunks.map((c) => c.markdown).filter(Boolean);
+    // Single-page batches have no page marker; add one so the merged document is navigable.
+    const markdown = chunks
+        .map((c) => {
+        if (!c.markdown)
+            return "";
+        if (c.markdown.startsWith("<!-- page "))
+            return c.markdown;
+        const p = [...(c.meta.pagesNative ?? []), ...(c.meta.pagesOcr ?? [])].sort((a, b) => a - b)[0];
+        return parts.length > 1 && p ? `<!-- page ${p} -->\n\n${c.markdown}` : c.markdown;
+    })
+        .filter(Boolean)
+        .join("\n\n");
+    const first = chunks[0]?.meta;
+    return {
+        title: info.title || chunks.find((c) => c.title)?.title || "",
+        markdown,
+        meta: {
+            kind: "pdf",
+            mime: "application/pdf",
+            method: pagesOcr.length > pagesNative.length ? "ocr" : "native",
+            bytes: first?.bytes ?? 0,
+            pages: info.pageCount,
+            pagesNative,
+            pagesOcr,
+            pagesSkipped: skipped,
+            ocrProvider: chunks.map((c) => c.meta.ocrProvider).find(Boolean),
+            warnings: [...new Set(chunks.flatMap((c) => c.meta.warnings))],
+            processingMs: chunks.reduce((n, c) => n + c.meta.processingMs, 0),
+        },
+    };
+}

package/dist/core/parse/ocr.d.ts

@@ -0,0 +1,25 @@
+export interface OcrPage {
+    /** 1-indexed. */
+    page: number;
+    markdown: string;
+}
+export interface OcrResult {
+    provider: string;
+    pages: OcrPage[];
+}
+export interface OcrInput {
+    bytes: Uint8Array;
+    mime: string;
+    /** 1-indexed pages to read (PDF only). Omit for images or the whole file. */
+    pages?: number[];
+    signal?: AbortSignal;
+}
+export declare function ocrAvailable(): boolean;
+export declare function mistralOcr(input: OcrInput): Promise<OcrResult>;
+/** Splits "<<<PAGE n>>>" sections; keeps only pages that were asked for. */
+export declare function splitPageMarkers(text: string, wanted?: number[]): OcrPage[];
+export declare function geminiOcr(input: OcrInput): Promise<OcrResult>;
+/** Tries Mistral, then Gemini. Throws only if every configured provider fails. */
+export declare function runOcr(input: OcrInput): Promise<OcrResult & {
+    warnings: string[];
+}>;

package/dist/core/parse/ocr.js

@@ -0,0 +1,121 @@
+/**
+ * OCR for scanned PDF pages and images.
+ * Primary: Mistral OCR (MISTRAL_API_KEY). Fallback: Gemini Flash-Lite via AI Gateway.
+ * Each provider returns only the pages it actually read; callers report the rest as skipped.
+ */
+import { createGateway } from "@ai-sdk/gateway";
+import { generateText } from "ai";
+const MISTRAL_URL = "https://api.mistral.ai/v1/ocr";
+const DEFAULT_MISTRAL_MODEL = "mistral-ocr-latest";
+const DEFAULT_VISION_MODEL = "google/gemini-2.5-flash-lite";
+function gatewayKey() {
+    return process.env.AI_GATEWAY_API_KEY || process.env.VERCEL_OIDC_TOKEN;
+}
+export function ocrAvailable() {
+    return Boolean(process.env.MISTRAL_API_KEY || gatewayKey());
+}
+function toBase64(bytes) {
+    return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("base64");
+}
+/** Mistral replaces figures with `![img-0.jpeg](img-0.jpeg)` placeholders we can't serve. */
+function stripPlaceholders(md) {
+    return md.replace(/!\[[^\]]*\]\((img-\d+\.\w+)\)\s*/g, "").trim();
+}
+export async function mistralOcr(input) {
+    const key = process.env.MISTRAL_API_KEY;
+    if (!key)
+        throw new Error("MISTRAL_API_KEY not set");
+    const isImage = input.mime.startsWith("image/");
+    const dataUrl = `data:${input.mime};base64,${toBase64(input.bytes)}`;
+    const body = {
+        model: process.env.WEBCRAWL_OCR_MODEL || DEFAULT_MISTRAL_MODEL,
+        document: isImage ? { type: "image_url", image_url: dataUrl } : { type: "document_url", document_url: dataUrl },
+        include_image_base64: false,
+    };
+    if (!isImage && input.pages?.length)
+        body.pages = input.pages.map((p) => p - 1);
+    const res = await fetch(MISTRAL_URL, {
+        method: "POST",
+        headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json" },
+        body: JSON.stringify(body),
+        signal: input.signal ?? AbortSignal.timeout(120_000),
+    });
+    if (!res.ok) {
+        const detail = (await res.text().catch(() => "")).slice(0, 300);
+        throw new Error(`Mistral OCR ${res.status}: ${detail}`);
+    }
+    const json = (await res.json());
+    return {
+        provider: "mistral",
+        pages: (json.pages ?? []).map((p) => ({ page: p.index + 1, markdown: stripPlaceholders(p.markdown || "") })),
+    };
+}
+const PAGE_MARK = /<<<PAGE (\d+)>>>/g;
+/** Splits "<<<PAGE n>>>" sections; keeps only pages that were asked for. */
+export function splitPageMarkers(text, wanted) {
+    const marks = [...text.matchAll(PAGE_MARK)];
+    if (marks.length === 0) {
+        const md = text.trim();
+        return md && (!wanted || wanted.length === 1) ? [{ page: wanted?.[0] ?? 1, markdown: md }] : [];
+    }
+    const allowed = wanted ? new Set(wanted) : null;
+    const out = [];
+    marks.forEach((m, i) => {
+        const page = Number(m[1]);
+        const end = i + 1 < marks.length ? marks[i + 1].index : text.length;
+        const md = text.slice((m.index ?? 0) + m[0].length, end).trim();
+        if (md && (!allowed || allowed.has(page)))
+            out.push({ page, markdown: md });
+    });
+    return out;
+}
+export async function geminiOcr(input) {
+    const key = gatewayKey();
+    if (!key)
+        throw new Error("AI gateway credentials not set");
+    const gw = createGateway({ apiKey: key });
+    const isImage = input.mime.startsWith("image/");
+    const pageList = input.pages?.length ? input.pages.join(", ") : "all";
+    const instruction = isImage
+        ? "Transcribe all text in this image as GitHub-flavoured Markdown. Keep headings, lists and tables. Output only the transcription; output nothing if there is no text."
+        : `Transcribe pages ${pageList} of this PDF as GitHub-flavoured Markdown. Keep headings, lists and tables (as Markdown tables). ` +
+            "Start each page with a line <<<PAGE n>>> where n is the 1-based page number. Output only the transcription. Do not summarise or invent text.";
+    const { text } = await generateText({
+        model: gw(process.env.WEBCRAWL_VISION_MODEL || DEFAULT_VISION_MODEL),
+        maxOutputTokens: 32_000,
+        abortSignal: input.signal ?? AbortSignal.timeout(120_000),
+        messages: [
+            {
+                role: "user",
+                content: [
+                    { type: "file", data: input.bytes, mediaType: input.mime },
+                    { type: "text", text: instruction },
+                ],
+            },
+        ],
+    });
+    const pages = isImage ? (text.trim() ? [{ page: 1, markdown: text.trim() }] : []) : splitPageMarkers(text, input.pages);
+    return { provider: "gemini-flash-lite", pages };
+}
+/** Tries Mistral, then Gemini. Throws only if every configured provider fails. */
+export async function runOcr(input) {
+    const warnings = [];
+    const providers = [];
+    if (process.env.MISTRAL_API_KEY)
+        providers.push(() => mistralOcr(input));
+    if (gatewayKey())
+        providers.push(() => geminiOcr(input));
+    if (providers.length === 0)
+        throw new Error("No OCR provider configured");
+    let lastErr;
+    for (const run of providers) {
+        try {
+            return { ...(await run()), warnings };
+        }
+        catch (err) {
+            lastErr = err;
+            warnings.push(err instanceof Error ? err.message : String(err));
+        }
+    }
+    throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
+}

package/dist/core/parse/office.d.ts

@@ -0,0 +1,28 @@
+/**
+ * DOCX / XLSX / PPTX -> Markdown, locally with officeparser (no network, no model).
+ * Sheets and slides get their own headings so a reader can tell where content came from.
+ */
+import type { FileKind } from "./detect.js";
+import type { ParsedFile } from "./types.js";
+type OfficeKind = Extract<FileKind, "docx" | "xlsx" | "pptx">;
+interface AstNode {
+    type: string;
+    metadata?: {
+        sheetName?: string;
+        slideNumber?: number;
+    };
+}
+/** Removes YAML front matter and `{#anchor}` heading ids officeparser adds. */
+export declare function cleanOfficeMarkdown(md: string): string;
+/**
+ * officeparser separates slides / sheets with `---` lines. Split on those so each
+ * section can be labelled with the slide number or sheet name from the AST.
+ */
+export declare function splitSections(md: string): string[];
+export declare function labelSections(kind: OfficeKind, sections: string[], nodes: AstNode[]): {
+    markdown: string;
+    units: number;
+    empty: string[];
+};
+export declare function parseOffice(bytes: Uint8Array, kind: OfficeKind, mime: string): Promise<ParsedFile>;
+export {};

package/dist/core/parse/office.js

@@ -0,0 +1,83 @@
+/** Removes YAML front matter and `{#anchor}` heading ids officeparser adds. */
+export function cleanOfficeMarkdown(md) {
+    return md
+        // Only strip a leading block that is really YAML (every line `key: value`); PPTX output
+        // also starts with `---`, and there the block is slide 1.
+        .replace(/^---\n((?:[\w:-]+:.*\n)+)---\n?/, "")
+        .replace(/^(#{1,6} .*?)[ \t]*\{#[^}]+\}[ \t]*$/gm, "$1")
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+}
+/**
+ * officeparser separates slides / sheets with `---` lines. Split on those so each
+ * section can be labelled with the slide number or sheet name from the AST.
+ */
+export function splitSections(md) {
+    return md
+        .split(/^---\s*$/m)
+        .map((s) => s.trim())
+        .filter(Boolean);
+}
+export function labelSections(kind, sections, nodes) {
+    if (kind === "docx")
+        return { markdown: sections.join("\n\n---\n\n"), units: 1, empty: [] };
+    const unitType = kind === "xlsx" ? "sheet" : "slide";
+    const units = nodes.filter((n) => n.type === unitType);
+    const empty = [];
+    const out = [];
+    // Empty sheets/slides produce no section, so match non-empty sections to units in order.
+    let si = 0;
+    units.forEach((u, i) => {
+        const label = kind === "xlsx"
+            ? `## Sheet: ${u.metadata?.sheetName || `Sheet ${i + 1}`}`
+            : `<!-- slide ${u.metadata?.slideNumber ?? i + 1} -->`;
+        const isEmpty = kind === "xlsx" ? u.children?.length === 0 : false;
+        if (isEmpty) {
+            empty.push(u.metadata?.sheetName || `Sheet ${i + 1}`);
+            return;
+        }
+        const body = sections[si++];
+        if (body !== undefined)
+            out.push(`${label}\n\n${body}`);
+    });
+    // Anything left over (AST and Markdown disagree) is kept rather than dropped.
+    while (si < sections.length)
+        out.push(sections[si++]);
+    return { markdown: out.join("\n\n"), units: units.length || sections.length, empty };
+}
+export async function parseOffice(bytes, kind, mime) {
+    const started = performance.now();
+    const { OfficeConverter, OfficeParser } = await import("officeparser");
+    const buf = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    const parseConfig = { fileType: kind, ignoreNotes: false };
+    const [converted, ast] = await Promise.all([
+        OfficeConverter.convert(buf, "md", { parseConfig }),
+        OfficeParser.parseOffice(buf, parseConfig),
+    ]);
+    const cleaned = cleanOfficeMarkdown(String(converted.value ?? ""));
+    const nodes = (ast.content ?? []);
+    const { markdown, units, empty } = labelSections(kind, splitSections(cleaned), nodes);
+    const warnings = (converted.messages ?? [])
+        .filter((m) => m.type === "warning")
+        .map((m) => String(m.message ?? m))
+        .slice(0, 10);
+    if (empty.length)
+        warnings.push(`Empty sheets: ${empty.join(", ")}`);
+    const title = ast.metadata?.title?.trim() ||
+        markdown.match(/^#\s+(.+)$/m)?.[1]?.trim() ||
+        "";
+    return {
+        title,
+        markdown,
+        meta: {
+            kind,
+            mime,
+            method: "native",
+            bytes: bytes.byteLength,
+            // "pages" means sheets for XLSX and slides for PPTX; DOCX has no reliable page count.
+            ...(kind === "docx" ? {} : { pages: units }),
+            warnings,
+            processingMs: Math.round(performance.now() - started),
+        },
+    };
+}

package/dist/core/parse/pdf.d.ts

@@ -0,0 +1,27 @@
+import type { ParsedFile, PdfParseOptions, SkippedPage } from "./types.js";
+export interface PdfPlan {
+    pageCount: number;
+    /** 1-indexed pages to read natively. */
+    native: number[];
+    /** 1-indexed pages to OCR. */
+    ocr: number[];
+    skipped: SkippedPage[];
+}
+/**
+ * Decides which pages go where. Pure, so it can be unit-tested without a PDF.
+ */
+export declare function planPdfPages(input: {
+    pageCount: number;
+    needsOcr: number[];
+    options: PdfParseOptions;
+    ocrConfigured: boolean;
+}): PdfPlan;
+/** First real-looking h1/h2 (skipping arXiv/DOI stamps and page furniture), else first h3. */
+export declare function pickTitle(markdown: string): string;
+/** Page count and pages needing OCR, without extracting text (cheap: ~1 ms/page). */
+export declare function pdfPageInfo(bytes: Uint8Array): {
+    pageCount: number;
+    pagesNeedingOcr: number[];
+    title?: string;
+};
+export declare function parsePdf(bytes: Uint8Array, options?: PdfParseOptions): Promise<ParsedFile>;