@webcrawl/mcp 1.4.2 → 1.4.3

package/dist/core/parse/detect.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Works out what kind of file a response or upload is, from magic bytes first,
+ * then the declared content type, then the file extension.
+ */
+export type FileKind = "pdf" | "docx" | "xlsx" | "pptx" | "image" | "audio" | "video" | "html" | "text";
+/** Kinds that go through the file parser instead of the HTML pipeline. */
+export declare const BINARY_KINDS: ReadonlySet<FileKind>;
+/** Content type to send to providers when the declared one is missing or generic. */
+export declare const KIND_DEFAULT_MIME: Partial<Record<FileKind, string>>;
+/** Returns the kind and a concrete mime from the first bytes, or null when unknown. */
+export declare function sniffMagic(bytes: Uint8Array): {
+    kind: FileKind;
+    mime: string;
+} | null;
+export interface DetectInput {
+    bytes?: Uint8Array;
+    contentType?: string;
+    fileName?: string;
+}
+export interface DetectedFile {
+    kind: FileKind;
+    /** Best concrete mime for the bytes (used when calling providers). */
+    mime: string;
+}
+/**
+ * Magic bytes win over the declared type (servers often send PDFs as
+ * octet-stream or text/html); then the declared type; then the extension.
+ */
+export declare function detectFileKind({ bytes, contentType, fileName }: DetectInput): DetectedFile;

package/dist/core/parse/detect.js

@@ -0,0 +1,152 @@
+/**
+ * Works out what kind of file a response or upload is, from magic bytes first,
+ * then the declared content type, then the file extension.
+ */
+/** Kinds that go through the file parser instead of the HTML pipeline. */
+export const BINARY_KINDS = new Set(["pdf", "docx", "xlsx", "pptx", "image", "audio", "video"]);
+const MIME_KINDS = [
+    [/^application\/pdf$/, "pdf"],
+    [/^application\/vnd\.openxmlformats-officedocument\.wordprocessingml/, "docx"],
+    [/^application\/vnd\.openxmlformats-officedocument\.spreadsheetml/, "xlsx"],
+    [/^application\/vnd\.openxmlformats-officedocument\.presentationml/, "pptx"],
+    [/^image\/svg\+xml$/, "text"],
+    [/^image\//, "image"],
+    [/^audio\//, "audio"],
+    [/^video\//, "video"],
+    [/^(text\/html|application\/xhtml\+xml)$/, "html"],
+    [/^(text\/|application\/(json|xml|csv|x-ndjson|ld\+json|rss\+xml|atom\+xml|javascript))/, "text"],
+];
+const EXT_KINDS = {
+    pdf: "pdf",
+    docx: "docx",
+    xlsx: "xlsx",
+    pptx: "pptx",
+    png: "image",
+    jpg: "image",
+    jpeg: "image",
+    webp: "image",
+    gif: "image",
+    heic: "image",
+    mp3: "audio",
+    wav: "audio",
+    m4a: "audio",
+    ogg: "audio",
+    oga: "audio",
+    flac: "audio",
+    opus: "audio",
+    aac: "audio",
+    mp4: "video",
+    m4v: "video",
+    mov: "video",
+    webm: "video",
+    mkv: "video",
+    html: "html",
+    htm: "html",
+    txt: "text",
+    md: "text",
+    csv: "text",
+    json: "text",
+    xml: "text",
+};
+/** Content type to send to providers when the declared one is missing or generic. */
+export const KIND_DEFAULT_MIME = {
+    pdf: "application/pdf",
+    docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    pptx: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+};
+function ascii(bytes, start, length) {
+    let s = "";
+    for (let i = start; i < Math.min(bytes.length, start + length); i++)
+        s += String.fromCharCode(bytes[i]);
+    return s;
+}
+/** Finds which OOXML package a ZIP is by the part names in its local file headers. */
+function sniffOoxml(bytes) {
+    const head = ascii(bytes, 0, Math.min(bytes.length, 64 * 1024));
+    if (head.includes("word/"))
+        return "docx";
+    if (head.includes("xl/"))
+        return "xlsx";
+    if (head.includes("ppt/"))
+        return "pptx";
+    return null;
+}
+/** Returns the kind and a concrete mime from the first bytes, or null when unknown. */
+export function sniffMagic(bytes) {
+    if (bytes.length < 4)
+        return null;
+    const b = bytes;
+    if (ascii(b, 0, 5) === "%PDF-")
+        return { kind: "pdf", mime: "application/pdf" };
+    if (b[0] === 0x50 && b[1] === 0x4b && b[2] === 0x03 && b[3] === 0x04) {
+        const kind = sniffOoxml(b);
+        return kind ? { kind, mime: KIND_DEFAULT_MIME[kind] } : null;
+    }
+    if (b[0] === 0x89 && ascii(b, 1, 3) === "PNG")
+        return { kind: "image", mime: "image/png" };
+    if (b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff)
+        return { kind: "image", mime: "image/jpeg" };
+    if (ascii(b, 0, 4) === "GIF8")
+        return { kind: "image", mime: "image/gif" };
+    if (ascii(b, 0, 4) === "RIFF" && ascii(b, 8, 4) === "WEBP")
+        return { kind: "image", mime: "image/webp" };
+    if (ascii(b, 0, 4) === "RIFF" && ascii(b, 8, 4) === "WAVE")
+        return { kind: "audio", mime: "audio/wav" };
+    if (ascii(b, 0, 3) === "ID3" || (b[0] === 0xff && (b[1] & 0xe0) === 0xe0 && (b[1] & 0x06) !== 0)) {
+        return { kind: "audio", mime: "audio/mpeg" };
+    }
+    if (ascii(b, 0, 4) === "fLaC")
+        return { kind: "audio", mime: "audio/flac" };
+    if (ascii(b, 0, 4) === "OggS")
+        return { kind: "audio", mime: "audio/ogg" };
+    if (b[0] === 0x1a && b[1] === 0x45 && b[2] === 0xdf && b[3] === 0xa3)
+        return { kind: "video", mime: "video/webm" };
+    if (ascii(b, 4, 4) === "ftyp") {
+        const brand = ascii(b, 8, 4);
+        if (/^(heic|heix|mif1|avif)/.test(brand))
+            return { kind: "image", mime: brand === "avif" ? "image/avif" : "image/heic" };
+        if (/^(M4A |M4B )/.test(brand))
+            return { kind: "audio", mime: "audio/mp4" };
+        if (/^qt/.test(brand))
+            return { kind: "video", mime: "video/quicktime" };
+        return { kind: "video", mime: "video/mp4" };
+    }
+    return null;
+}
+function extensionOf(name) {
+    if (!name)
+        return "";
+    const clean = name.split(/[?#]/)[0];
+    const dot = clean.lastIndexOf(".");
+    return dot >= 0 ? clean.slice(dot + 1).toLowerCase() : "";
+}
+/**
+ * Magic bytes win over the declared type (servers often send PDFs as
+ * octet-stream or text/html); then the declared type; then the extension.
+ */
+export function detectFileKind({ bytes, contentType, fileName }) {
+    const declared = (contentType || "").split(";")[0].trim().toLowerCase();
+    const magic = bytes ? sniffMagic(bytes) : null;
+    if (magic)
+        return magic;
+    const generic = !declared || /^(application\/octet-stream|binary\/octet-stream|application\/zip|application\/x-zip-compressed)$/.test(declared);
+    if (!generic) {
+        for (const [re, kind] of MIME_KINDS)
+            if (re.test(declared))
+                return { kind, mime: declared };
+    }
+    const ext = extensionOf(fileName);
+    const byExt = EXT_KINDS[ext];
+    if (byExt) {
+        const media = byExt === "image" || byExt === "audio" || byExt === "video";
+        const sub = { mp3: "mpeg", jpg: "jpeg", mov: "quicktime", m4a: "mp4", m4v: "mp4", oga: "ogg" }[ext] || ext;
+        return { kind: byExt, mime: KIND_DEFAULT_MIME[byExt] || (media ? `${byExt}/${sub}` : declared || "text/plain") };
+    }
+    if (bytes) {
+        const head = ascii(bytes, 0, 512).toLowerCase();
+        if (/<!doctype html|<html[\s>]|<head[\s>]|<body[\s>]/.test(head))
+            return { kind: "html", mime: "text/html" };
+    }
+    return { kind: "text", mime: declared || "text/plain" };
+}

package/dist/core/parse/fetch.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Fetch a URL as bytes and parse it if it is a binary file (PDF, Office, image, media).
+ * Returns null when the URL is HTML/text so callers fall back to the HTML pipeline.
+ */
+import { type FetchedResource } from "../engine/fetcher.js";
+import { type FileKind } from "./detect.js";
+import { type ParseFileOptions, type ParseFileResult } from "./index.js";
+/** Cheap check before any network: does the URL path end in a binary file extension? */
+export declare function looksLikeBinaryUrl(url: string): boolean;
+/** Does a response content type mean we should take the binary path? */
+export declare function isBinaryContentType(contentType: string | undefined): boolean;
+export interface FetchedParse {
+    resource: FetchedResource;
+    kind: FileKind;
+    result: ParseFileResult;
+}
+export declare function fetchAndParse(url: string, options?: ParseFileOptions & {
+    timeoutMs?: number;
+    resource?: FetchedResource;
+}): Promise<FetchedParse | null>;

package/dist/core/parse/fetch.js

@@ -0,0 +1,39 @@
+/**
+ * Fetch a URL as bytes and parse it if it is a binary file (PDF, Office, image, media).
+ * Returns null when the URL is HTML/text so callers fall back to the HTML pipeline.
+ */
+import { fetchResource } from "../engine/fetcher.js";
+import { detectFileKind } from "./detect.js";
+import { maxDownloadBytes } from "./limits.js";
+import { isBinaryKind, parseFile } from "./index.js";
+const BINARY_EXT = /\.(pdf|docx|xlsx|pptx|png|jpe?g|webp|gif|heic|avif|mp3|wav|m4a|ogg|oga|flac|opus|aac|mp4|m4v|mov|webm|mkv)$/i;
+/** Cheap check before any network: does the URL path end in a binary file extension? */
+export function looksLikeBinaryUrl(url) {
+    try {
+        return BINARY_EXT.test(new URL(url).pathname);
+    }
+    catch {
+        return BINARY_EXT.test(url.split(/[?#]/)[0]);
+    }
+}
+/** Does a response content type mean we should take the binary path? */
+export function isBinaryContentType(contentType) {
+    const ct = (contentType || "").split(";")[0].trim().toLowerCase();
+    if (!ct)
+        return false;
+    const { kind } = detectFileKind({ contentType: ct });
+    return isBinaryKind(kind) || ct === "application/octet-stream";
+}
+export async function fetchAndParse(url, options = {}) {
+    const resource = options.resource ??
+        (await fetchResource(url, { timeoutMs: options.timeoutMs ?? 30_000, maxBytes: maxDownloadBytes() }));
+    const fileName = new URL(resource.url || url).pathname.split("/").pop() || undefined;
+    const { kind } = detectFileKind({ bytes: resource.body, contentType: resource.contentType, fileName });
+    if (!isBinaryKind(kind))
+        return null;
+    if (resource.status >= 400) {
+        throw new Error(`File request failed with HTTP ${resource.status}`);
+    }
+    const result = await parseFile({ bytes: resource.body, contentType: resource.contentType, fileName }, options);
+    return { resource, kind, result };
+}

package/dist/core/parse/image.d.ts

@@ -0,0 +1,13 @@
+import type { ParsedFile } from "./types.js";
+export declare const VISION_PROVIDER = "gemini-flash-lite";
+export declare function visionAvailable(): boolean;
+/** Splits the model reply into description and text. Tolerates a missing TEXT marker. */
+export declare function splitVisionReply(reply: string): {
+    description: string;
+    text: string;
+};
+export declare function imageMarkdown(description: string, text: string): string;
+export declare function parseImage(bytes: Uint8Array, mime: string, options?: {
+    fileName?: string;
+    signal?: AbortSignal;
+}): Promise<ParsedFile>;

package/dist/core/parse/image.js

@@ -0,0 +1,100 @@
+/**
+ * Images -> a short description plus any text in the image, from Gemini Flash-Lite
+ * through AI Gateway. One call per image. Without credentials nothing is invented:
+ * the result is empty with a warning.
+ */
+import { createGateway } from "@ai-sdk/gateway";
+import { generateText } from "ai";
+const DEFAULT_VISION_MODEL = "google/gemini-2.5-flash-lite";
+export const VISION_PROVIDER = "gemini-flash-lite";
+function gatewayKey() {
+    return process.env.AI_GATEWAY_API_KEY || process.env.VERCEL_OIDC_TOKEN;
+}
+export function visionAvailable() {
+    return Boolean(gatewayKey());
+}
+const PROMPT = [
+    "Describe this image for someone who cannot see it, then transcribe any text in it.",
+    "Reply in exactly this format:",
+    "<<<DESCRIPTION>>>",
+    "Two to four plain sentences: what the image shows, and the key facts a reader needs (numbers in charts, labels in diagrams).",
+    "<<<TEXT>>>",
+    "All legible text in the image as GitHub-flavoured Markdown (keep tables as tables). Write NONE if there is no text.",
+    "Do not guess at text you cannot read.",
+].join("\n");
+/** Splits the model reply into description and text. Tolerates a missing TEXT marker. */
+export function splitVisionReply(reply) {
+    const d = reply.indexOf("<<<DESCRIPTION>>>");
+    const t = reply.indexOf("<<<TEXT>>>");
+    if (d === -1 && t === -1)
+        return { description: reply.trim(), text: "" };
+    const description = (t === -1 ? reply.slice(d + 17) : reply.slice(d === -1 ? 0 : d + 17, t)).trim();
+    let text = t === -1 ? "" : reply.slice(t + 10).trim();
+    if (/^none\.?$/i.test(text))
+        text = "";
+    return { description, text };
+}
+export function imageMarkdown(description, text) {
+    const parts = [];
+    if (description)
+        parts.push(`## Description\n\n${description}`);
+    if (text)
+        parts.push(`## Text in image\n\n${text}`);
+    return parts.join("\n\n");
+}
+export async function parseImage(bytes, mime, options = {}) {
+    const started = performance.now();
+    const warnings = [];
+    const base = {
+        kind: "image",
+        mime,
+        method: "vision",
+        bytes: bytes.byteLength,
+    };
+    const title = options.fileName?.split("/").pop()?.replace(/\.[a-z0-9]+$/i, "") || "";
+    const key = gatewayKey();
+    if (!key) {
+        warnings.push("Image description unavailable: no vision model configured");
+        return {
+            title,
+            markdown: "",
+            meta: { ...base, warnings, processingMs: Math.round(performance.now() - started) },
+        };
+    }
+    try {
+        const gw = createGateway({ apiKey: key });
+        const { text: reply } = await generateText({
+            model: gw(process.env.WEBCRAWL_VISION_MODEL || DEFAULT_VISION_MODEL),
+            maxOutputTokens: 4_000,
+            abortSignal: options.signal ?? AbortSignal.timeout(60_000),
+            messages: [
+                {
+                    role: "user",
+                    content: [
+                        { type: "image", image: bytes, mediaType: mime },
+                        { type: "text", text: PROMPT },
+                    ],
+                },
+            ],
+        });
+        const { description, text } = splitVisionReply(reply);
+        return {
+            title,
+            markdown: imageMarkdown(description, text),
+            meta: {
+                ...base,
+                visionProvider: VISION_PROVIDER,
+                warnings,
+                processingMs: Math.round(performance.now() - started),
+            },
+        };
+    }
+    catch (err) {
+        warnings.push(`Image description failed: ${err instanceof Error ? err.message : String(err)}`);
+        return {
+            title,
+            markdown: "",
+            meta: { ...base, warnings, processingMs: Math.round(performance.now() - started) },
+        };
+    }
+}

package/dist/core/parse/index.d.ts

@@ -0,0 +1,53 @@
+import { type FileKind } from "./detect.js";
+import type { ParsedFile, ParseMeta } from "./types.js";
+export { detectFileKind, BINARY_KINDS, type FileKind } from "./detect.js";
+export type { ParsedFile, ParseMeta } from "./types.js";
+export interface ParseFileOptions {
+    ocr?: "auto" | "force" | "off";
+    pages?: number[];
+    maxOcrPages?: number;
+    captions?: string;
+    language?: string;
+    describeVideo?: boolean;
+    bypassCache?: boolean;
+    signal?: AbortSignal;
+}
+export interface ParseFileInput {
+    bytes: Uint8Array;
+    contentType?: string;
+    fileName?: string;
+}
+export declare class FileTooLargeError extends Error {
+    readonly kind: FileKind;
+    readonly bytes: number;
+    readonly maxBytes: number;
+    constructor(kind: FileKind, bytes: number, maxBytes: number);
+}
+export declare class UnsupportedFileError extends Error {
+    readonly kind: FileKind;
+    constructor(kind: FileKind);
+}
+/** Converts parse metadata into billable usage. Cache hits are billed like the original parse. */
+export declare function parseUsageFromMeta(meta: ParseMeta): {
+    kind: FileKind;
+    method: import("./types.js").ParseMethod;
+    pagesNative: number | undefined;
+    pagesOcr: number | undefined;
+    durationSeconds: number | undefined;
+    transcribed: boolean;
+    transcriptProvider: string | undefined;
+    videoDescribed: boolean;
+    imageDescribed: boolean;
+};
+/** Public shape of `metadata.parse` in API responses. */
+export declare function publicParseMeta(r: ParseFileResult): Record<string, unknown>;
+export declare function isBinaryKind(kind: FileKind): boolean;
+export declare function contentHash(bytes: Uint8Array): string;
+/** Only options that change the output go into the cache key. */
+export declare function parseCacheKey(hash: string, kind: FileKind, o: ParseFileOptions): string;
+export interface ParseFileResult extends ParsedFile {
+    contentHash: string;
+    cached: boolean;
+    cacheLayer: "memory" | "disk" | "postgres" | "miss";
+}
+export declare function parseFile(input: ParseFileInput, options?: ParseFileOptions): Promise<ParseFileResult>;

package/dist/core/parse/index.js

@@ -0,0 +1,153 @@
+/**
+ * One entry point for binary files: detect the kind, check the size cap, route to the
+ * right parser, and cache by content hash (same bytes => same result, whatever the URL).
+ */
+import crypto from "node:crypto";
+import { getCachedPageAsync, setCachedPage } from "../engine/cache.js";
+import { BINARY_KINDS, detectFileKind } from "./detect.js";
+import { maxBytesFor } from "./limits.js";
+import { parseImage } from "./image.js";
+import { parseMedia } from "./media.js";
+import { parseOffice } from "./office.js";
+import { parsePdf } from "./pdf.js";
+export { detectFileKind, BINARY_KINDS } from "./detect.js";
+export class FileTooLargeError extends Error {
+    kind;
+    bytes;
+    maxBytes;
+    constructor(kind, bytes, maxBytes) {
+        super(`${kind.toUpperCase()} is ${(bytes / 1e6).toFixed(1)} MB; the limit is ${(maxBytes / 1e6).toFixed(0)} MB`);
+        this.kind = kind;
+        this.bytes = bytes;
+        this.maxBytes = maxBytes;
+        this.name = "FileTooLargeError";
+    }
+}
+export class UnsupportedFileError extends Error {
+    kind;
+    constructor(kind) {
+        super(`Not a binary file kind (${kind}); use the HTML/text pipeline`);
+        this.kind = kind;
+        this.name = "UnsupportedFileError";
+    }
+}
+/** Converts parse metadata into billable usage. Cache hits are billed like the original parse. */
+export function parseUsageFromMeta(meta) {
+    const isOffice = meta.kind === "docx" || meta.kind === "xlsx" || meta.kind === "pptx";
+    return {
+        kind: meta.kind,
+        method: meta.method,
+        pagesNative: meta.pagesNative?.length ?? (isOffice ? meta.pages ?? 1 : undefined),
+        pagesOcr: meta.pagesOcr?.length,
+        durationSeconds: meta.durationSeconds,
+        transcribed: Boolean(meta.transcriptProvider && meta.transcriptProvider !== "captions"),
+        transcriptProvider: meta.transcriptProvider,
+        videoDescribed: meta.kind === "video" && Boolean(meta.visionProvider),
+        imageDescribed: meta.kind === "image" && Boolean(meta.visionProvider),
+    };
+}
+/** Public shape of `metadata.parse` in API responses. */
+export function publicParseMeta(r) {
+    return { ...r.meta, contentHash: r.contentHash, cached: r.cached };
+}
+export function isBinaryKind(kind) {
+    return BINARY_KINDS.has(kind);
+}
+export function contentHash(bytes) {
+    return crypto.createHash("sha256").update(bytes).digest("hex");
+}
+/** Only options that change the output go into the cache key. */
+export function parseCacheKey(hash, kind, o) {
+    const parts = [];
+    if (kind === "pdf") {
+        parts.push(`ocr=${o.ocr ?? "auto"}`);
+        if (o.pages?.length)
+            parts.push(`pages=${o.pages.join(",")}`);
+        if (o.maxOcrPages)
+            parts.push(`max=${o.maxOcrPages}`);
+    }
+    if (kind === "audio" || kind === "video") {
+        if (o.captions)
+            parts.push(`cap=${crypto.createHash("sha256").update(o.captions).digest("hex").slice(0, 16)}`);
+        if (o.language)
+            parts.push(`lang=${o.language}`);
+        if (kind === "video" && o.describeVideo)
+            parts.push("describe");
+    }
+    return `wc-parse://${hash}/${parts.join("&") || "default"}`;
+}
+/** A result is worth caching only if nothing failed transiently. */
+function cacheable(meta) {
+    if (meta.pagesSkipped?.some((s) => s.reason === "ocr_failed" || s.reason === "ocr_unavailable"))
+        return false;
+    return !meta.warnings.some((w) => /failed|unavailable/i.test(w));
+}
+export async function parseFile(input, options = {}) {
+    const { kind, mime } = detectFileKind(input);
+    if (!isBinaryKind(kind))
+        throw new UnsupportedFileError(kind);
+    const max = maxBytesFor(kind);
+    if (input.bytes.byteLength > max)
+        throw new FileTooLargeError(kind, input.bytes.byteLength, max);
+    const hash = contentHash(input.bytes);
+    const key = parseCacheKey(hash, kind, options);
+    if (!options.bypassCache) {
+        const hit = await getCachedPageAsync(key, "parse");
+        const meta = hit?.structuredData?.parse;
+        if (hit && meta) {
+            return {
+                title: hit.title,
+                markdown: hit.markdown,
+                meta,
+                contentHash: hash,
+                cached: true,
+                cacheLayer: hit.cacheLayer || "postgres",
+            };
+        }
+    }
+    let parsed;
+    switch (kind) {
+        case "pdf":
+            parsed = await parsePdf(input.bytes, {
+                ocr: options.ocr,
+                pages: options.pages,
+                maxOcrPages: options.maxOcrPages,
+                signal: options.signal,
+            });
+            break;
+        case "docx":
+        case "xlsx":
+        case "pptx":
+            parsed = await parseOffice(input.bytes, kind, mime);
+            break;
+        case "image":
+            parsed = await parseImage(input.bytes, mime, { fileName: input.fileName, signal: options.signal });
+            break;
+        case "audio":
+        case "video":
+            parsed = await parseMedia(input.bytes, kind, mime, {
+                fileName: input.fileName,
+                captions: options.captions,
+                language: options.language,
+                describeVideo: options.describeVideo,
+                signal: options.signal,
+            });
+            break;
+        default:
+            throw new UnsupportedFileError(kind);
+    }
+    if (!parsed.title && input.fileName) {
+        parsed.title = input.fileName.split(/[?#]/)[0].split("/").pop() || "";
+    }
+    if (cacheable(parsed.meta) && parsed.markdown.length > 0) {
+        setCachedPage(key, "parse", {
+            title: parsed.title,
+            markdown: parsed.markdown,
+            links: [],
+            structuredData: { parse: parsed.meta },
+            statusCode: 200,
+            contentHash: hash,
+        });
+    }
+    return { ...parsed, contentHash: hash, cached: false, cacheLayer: "miss" };
+}

package/dist/core/parse/limits.d.ts

@@ -0,0 +1,12 @@
+import type { FileKind } from "./detect.js";
+/** Per-kind size caps. Override with WEBCRAWL_PARSE_MAX_{DOC,IMAGE,MEDIA}_MB. */
+export declare function maxBytesFor(kind: FileKind): number;
+/** Upper bound for any download before the kind is known. */
+export declare function maxDownloadBytes(): number;
+/** Jobs bigger than these run as a background workflow instead of inline. */
+export declare const ASYNC_THRESHOLDS: {
+    pdfPages: number;
+    mediaSeconds: number;
+    pdfBatchPages: number;
+    mediaChunkSeconds: number;
+};

package/dist/core/parse/limits.js

@@ -0,0 +1,29 @@
+const MB = 1024 * 1024;
+function envMb(name, fallbackMb) {
+    const raw = Number(process.env[name]);
+    return (Number.isFinite(raw) && raw > 0 ? raw : fallbackMb) * MB;
+}
+/** Per-kind size caps. Override with WEBCRAWL_PARSE_MAX_{DOC,IMAGE,MEDIA}_MB. */
+export function maxBytesFor(kind) {
+    switch (kind) {
+        case "image":
+            return envMb("WEBCRAWL_PARSE_MAX_IMAGE_MB", 20);
+        case "audio":
+        case "video":
+            // Groq's transcription upload limit.
+            return envMb("WEBCRAWL_PARSE_MAX_MEDIA_MB", 100);
+        default:
+            return envMb("WEBCRAWL_PARSE_MAX_DOC_MB", 50);
+    }
+}
+/** Upper bound for any download before the kind is known. */
+export function maxDownloadBytes() {
+    return Math.max(maxBytesFor("pdf"), maxBytesFor("image"), maxBytesFor("audio"));
+}
+/** Jobs bigger than these run as a background workflow instead of inline. */
+export const ASYNC_THRESHOLDS = {
+    pdfPages: Number(process.env.WEBCRAWL_PARSE_ASYNC_PAGES) || 50,
+    mediaSeconds: Number(process.env.WEBCRAWL_PARSE_ASYNC_SECONDS) || 600,
+    pdfBatchPages: 20,
+    mediaChunkSeconds: 600,
+};

package/dist/core/parse/media.d.ts

@@ -0,0 +1,42 @@
+import type { ParsedFile } from "./types.js";
+export interface Segment {
+    start: number;
+    end: number;
+    text: string;
+}
+export interface MediaParseOptions {
+    fileName?: string;
+    /** WebVTT or SRT text. Used instead of transcription when present. */
+    captions?: string;
+    /** BCP-47 / ISO-639-1 hint for Whisper, e.g. "en". */
+    language?: string;
+    /** Video only: add a short description of what is on screen. */
+    describeVideo?: boolean;
+    signal?: AbortSignal;
+}
+export declare function transcriptionAvailable(): boolean;
+/** Transcribes through the Vercel AI Gateway (no extra provider key needed). */
+export declare function gatewayTranscribe(bytes: Uint8Array, options?: {
+    language?: string;
+    signal?: AbortSignal;
+}): Promise<{
+    segments: Segment[];
+    duration: number;
+    language?: string;
+}>;
+/** 3725.4 -> "1:02:05"; 65 -> "1:05". */
+export declare function formatTimestamp(seconds: number): string;
+/** Parses WebVTT or SRT cues. Drops styling tags and repeated rolling-caption lines. */
+export declare function parseCaptions(text: string): Segment[];
+/** Groups segments into ~30 s paragraphs, each prefixed with its start time. */
+export declare function transcriptMarkdown(segments: Segment[], paragraphSeconds?: number): string;
+export declare function groqTranscribe(bytes: Uint8Array, mime: string, options?: {
+    fileName?: string;
+    language?: string;
+    signal?: AbortSignal;
+}): Promise<{
+    segments: Segment[];
+    duration: number;
+    language?: string;
+}>;
+export declare function parseMedia(bytes: Uint8Array, kind: "audio" | "video", mime: string, options?: MediaParseOptions): Promise<ParsedFile>;