@retinue/agentkit 0.1.0

package/dist/documents/extraction.js

@@ -0,0 +1,293 @@
+/**
+ * The extraction pipeline (#131).
+ *
+ * The parsers do the reading; this decides *when*, *within what bounds*, and *what is recorded when it goes
+ * wrong* — which is the part that makes extraction safe to run on a shared worker tier.
+ *
+ * Four properties, each answering an AC:
+ *
+ * - **It never throws for a document problem** (AC-4). Every outcome is recorded on the file: `extracted` with
+ *   a reference, or `failed` with a typed reason and a sentence the user reads. An extractor that threw would
+ *   leave the file looking unextracted, which is indistinguishable from "not got round to it yet" — and the
+ *   user would see an empty document rather than a reason.
+ * - **It is bounded in every dimension a document can grow** (AC-3): bytes read, pages, extracted text,
+ *   blocks, and wall clock. The timeout is enforced *here* rather than trusted to the parser, because a
+ *   parser stuck in a loop is exactly the case a parser-side check does not catch.
+ * - **It is enqueued, not awaited** (AC-2). `requestExtraction` puts a job on the queue and returns; an
+ *   enqueue failure marks the file `pending` and is reported, never propagated — an attachment that uploaded
+ *   fine must not fail because the extraction queue was briefly unreachable.
+ * - **The result is stored by reference and read in windows** (AC-6). The `ExtractedDocument` goes to
+ *   `BlobStore` and the file carries the ref. Nothing loads it into context; `read_document` reads a bounded
+ *   window, exactly as #130 established for the raw bytes.
+ */
+import { AgentPlatformError } from "../core/errors.js";
+import { DEFAULT_EXTRACTION_LIMITS, isExtractionFailure } from "./index.js";
+import {} from "./index.js";
+/** The media type without its parameters, lowercased. `text/plain; charset=utf-8` is `text/plain`. */
+const normaliseMediaType = (mediaType) => mediaType.split(";")[0]?.trim().toLowerCase() ?? "";
+/**
+ * Read at most `maxBytes` from the content store.
+ *
+ * Refuses past the limit rather than truncating, because a half-read document is not a shorter document: a
+ * PDF's structure is at the end, and half of one is malformed rather than partial. The distinction matters
+ * enough that `too-large` is its own failure reason.
+ */
+const readBounded = async (bytes, maxBytes) => {
+    const chunks = [];
+    let size = 0;
+    for await (const chunk of bytes) {
+        size += chunk.byteLength;
+        // Stops pulling, so a hostile object is not drained into memory before being rejected.
+        if (size > maxBytes)
+            return "too-large";
+        chunks.push(chunk);
+    }
+    const joined = new Uint8Array(size);
+    let offset = 0;
+    for (const chunk of chunks) {
+        joined.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return joined;
+};
+/**
+ * Run a parser under a wall-clock ceiling.
+ *
+ * The race is here rather than inside the parser because a parser caught in a loop cannot check its own
+ * clock. This does not *stop* the runaway work — nothing in a single-threaded runtime can — but it stops the
+ * pipeline waiting on it, so the file gets a `timed-out` record and the queue moves on. Said plainly because
+ * the alternative reading of this code is that it kills the parser, and it does not.
+ */
+const withTimeout = async (work, timeoutMs) => {
+    let timer;
+    const timeout = new Promise((resolve) => {
+        timer = setTimeout(() => resolve({
+            reason: "timed-out",
+            message: `Extraction took longer than ${Math.round(timeoutMs / 1000)} seconds and was stopped.`,
+        }), timeoutMs);
+    });
+    try {
+        return await Promise.race([work, timeout]);
+    }
+    finally {
+        // Without this the timer holds the event loop open for the full duration on every successful extraction,
+        // which turns a fast worker into one that exits slowly.
+        if (timer !== undefined)
+            clearTimeout(timer);
+    }
+};
+export const createExtractionService = (deps) => {
+    const limits = deps.limits ?? DEFAULT_EXTRACTION_LIMITS;
+    const clock = deps.clock ?? (() => new Date().toISOString());
+    const log = deps.log ?? (() => { });
+    /** Built once. Answers "is this type supported" without reading a byte, which is what makes AC-4's `unsupported-type` a cheap decision. */
+    const byMediaType = new Map();
+    for (const parser of deps.parsers)
+        for (const mediaType of parser.mediaTypes)
+            byMediaType.set(normaliseMediaType(mediaType), parser);
+    const fallbackByMediaType = new Map();
+    for (const parser of deps.fallbackParsers ?? [])
+        for (const mediaType of parser.mediaTypes)
+            fallbackByMediaType.set(normaliseMediaType(mediaType), parser);
+    const record = async (tenantId, id, extraction) => {
+        const { recorded } = await deps.metadata.recordExtraction({ tenantId, id, extraction });
+        // Not recorded means the file went away mid-extraction. Logged rather than thrown: the work is wasted,
+        // nothing is inconsistent, and a worker that threw here would retry a file that no longer exists.
+        if (!recorded)
+            log("extraction outcome dropped: file no longer exists", { tenantId, id });
+        return extraction;
+    };
+    return {
+        limits,
+        /** Types this pipeline can read at all. Exposed so an upload path can decide not to enqueue at all. */
+        supportedMediaTypes() {
+            return [...byMediaType.keys()];
+        },
+        /**
+         * Ask for extraction, without waiting for it — AC-2.
+         *
+         * Marks the file `pending` first, then enqueues. That order matters: the reverse would let a worker pick
+         * the job up, extract, and record its outcome *before* this function overwrote it with `pending`, losing a
+         * completed extraction to a race that only shows up under load.
+         */
+        async requestExtraction(context, id, mediaType) {
+            if (!byMediaType.has(normaliseMediaType(mediaType))) {
+                // Not a failure: nobody asked for this type to be readable. `skipped` says "we are not going to", which
+                // is a different sentence from "we tried and could not".
+                await record(context.tenantId, id, {
+                    state: "skipped",
+                    failureMessage: `${mediaType} is not a document type this platform extracts text from.`,
+                    at: clock(),
+                });
+                return { enqueued: false, state: "skipped" };
+            }
+            await record(context.tenantId, id, { state: "pending", at: clock() });
+            if (deps.dispatcher === undefined)
+                return { enqueued: false, state: "pending" };
+            try {
+                await deps.dispatcher.enqueueExtraction({ tenantId: context.tenantId, fileId: id });
+                return { enqueued: true, state: "pending" };
+            }
+            catch (error) {
+                // Swallowed on purpose, and this is the one place in the module where that is right: the upload
+                // succeeded, the file is safe, and the row stays `pending` for `sweepStuckExtractions` to re-enqueue.
+                // Propagating would fail an upload because a queue was briefly unreachable.
+                log("extraction enqueue failed; left pending for the sweep", { tenantId: context.tenantId, id, error });
+                return { enqueued: false, state: "pending" };
+            }
+        },
+        /**
+         * Do the extraction. Called by the worker, and directly by a single-process host.
+         *
+         * Returns the record it wrote rather than throwing, for every document-shaped problem. The only throw is
+         * for a *file* that is not there, which is a caller error rather than a document one.
+         */
+        async extract(job) {
+            const { tenantId, fileId } = job;
+            const file = await deps.metadata.get({ tenantId, id: fileId });
+            if (file === null)
+                throw new AgentPlatformError({ code: "not_found", message: "no such file", retryable: false });
+            if (file.state !== "stored")
+                // A file mid-upload or mid-delete has no stable bytes to read. Not an error and not a failure — the
+                // sweep will find it again if it settles.
+                return record(tenantId, fileId, {
+                    state: "skipped",
+                    failureMessage: "The file was not in a readable state when extraction ran.",
+                    at: clock(),
+                });
+            const mediaType = normaliseMediaType(file.mediaType);
+            const parser = byMediaType.get(mediaType);
+            if (parser === undefined)
+                return record(tenantId, fileId, {
+                    state: "skipped",
+                    failureMessage: `${file.mediaType} is not a document type this platform extracts text from.`,
+                    at: clock(),
+                });
+            // `running` before the work, so a crash leaves evidence. Without it a worker that died mid-parse would
+            // leave the file `pending` and the sweep would re-enqueue it forever, poisoning the queue with the one
+            // document that reliably kills a worker.
+            await record(tenantId, fileId, { state: "running", at: clock() });
+            const stream = await deps.content.readFile({ tenantId, contentKey: file.contentKey });
+            if (stream === null)
+                return record(tenantId, fileId, {
+                    state: "failed",
+                    failureReason: "malformed",
+                    failureMessage: "The file's contents are missing from storage.",
+                    at: clock(),
+                });
+            const bytes = await readBounded(stream, limits.maxBytes);
+            if (bytes === "too-large")
+                return record(tenantId, fileId, {
+                    state: "failed",
+                    failureReason: "too-large",
+                    // The limit named, so the user knows what to do rather than guessing.
+                    failureMessage: `That document is larger than the ${limits.maxBytes} byte extraction limit.`,
+                    at: clock(),
+                });
+            let outcome;
+            try {
+                outcome = await withTimeout(parser.parse({ bytes, mediaType, limits }), limits.timeoutMs);
+            }
+            catch (error) {
+                // A *thrown* error is the parser being broken, not the document being unreadable. Recorded as
+                // `malformed` with a generic sentence, and logged with the detail — a stack trace is not a user's
+                // problem, and the two must not be confused in a report.
+                log("parser threw", { tenantId, fileId, parser: parser.id, error });
+                return record(tenantId, fileId, {
+                    state: "failed",
+                    failureReason: "malformed",
+                    failureMessage: "That document could not be read.",
+                    at: clock(),
+                });
+            }
+            // #132. A scan is the one failure worth a second, more expensive attempt — and only that one. Retrying
+            // an encrypted or malformed document through OCR would spend money to reach the same conclusion.
+            if (isExtractionFailure(outcome) && outcome.reason === "no-text-layer") {
+                const fallback = fallbackByMediaType.get(mediaType);
+                if (fallback !== undefined && fallback.id !== parser.id) {
+                    try {
+                        outcome = await withTimeout(fallback.parse({ bytes, mediaType, limits }), limits.timeoutMs);
+                    }
+                    catch (error) {
+                        log("fallback parser threw", { tenantId, fileId, parser: fallback.id, error });
+                    }
+                }
+            }
+            if (isExtractionFailure(outcome))
+                return record(tenantId, fileId, {
+                    state: "failed",
+                    failureReason: outcome.reason,
+                    failureMessage: outcome.message,
+                    at: clock(),
+                });
+            // Stored by reference (AC-6). The blob is written *before* the ref is recorded: the reverse would leave
+            // a file pointing at a blob that does not exist, and a dangling ref reads as corruption while an
+            // unreferenced blob is merely waste.
+            // AC-4. Reported before the blob is written, and *stripped* from what is stored: a priced operation
+            // already happened, so a crash between here and the record must still have billed it — and a stored
+            // document has no business carrying billing data.
+            const { usage, ...document } = outcome;
+            for (const entry of usage ?? []) {
+                try {
+                    await deps.onPricedOperation?.({ tenantId, fileId, usage: entry });
+                }
+                catch (error) {
+                    // Logged, not thrown. A ledger write that fails must not discard an extraction that succeeded, and
+                    // an unbilled call is a smaller problem than a document the user paid for and cannot read.
+                    log("failed to record extraction usage", { tenantId, fileId, kind: entry.kind, error });
+                }
+            }
+            const ref = await deps.blobs.put({ tenantId, value: document });
+            return record(tenantId, fileId, {
+                state: "extracted",
+                ref,
+                pageCount: document.pageCount ?? 0,
+                blockCount: document.blocks.length,
+                truncated: document.truncated,
+                // On the record as well as in the document, so a listing can flag a low-confidence extraction
+                // without fetching the blob to find out.
+                ...(document.confidence === undefined ? {} : { confidence: document.confidence }),
+                at: clock(),
+            });
+        },
+        /** The extracted document, or null when there is none to read. */
+        async getExtracted(context, id) {
+            const file = await deps.metadata.get({ tenantId: context.tenantId, id });
+            if (file === null || file.extraction === undefined)
+                return { document: null };
+            if (file.extraction.state !== "extracted" || file.extraction.ref === undefined)
+                // The record comes back even with no document, because *why* there is nothing is the useful part —
+                // AC-4's whole point is that the assistant says "that PDF is a scan" rather than nothing.
+                return { document: null, extraction: file.extraction };
+            const value = await deps.blobs.get({ tenantId: context.tenantId, ref: file.extraction.ref });
+            return { document: value ?? null, extraction: file.extraction };
+        },
+        /**
+         * Re-enqueue extractions that never finished.
+         *
+         * Two shapes, and both are silent without this: a `pending` file whose enqueue was lost, and a `running`
+         * file whose worker died. Both are re-requested rather than repaired — the pipeline is idempotent, so
+         * running it again is the repair.
+         */
+        async sweepStuckExtractions(context, input) {
+            const states = {};
+            let requeued = 0;
+            for (const state of ["pending", "running"]) {
+                const page = await deps.metadata.listByExtractionState({
+                    tenantId: context.tenantId,
+                    state,
+                    olderThan: input.olderThan,
+                    limit: input.limit,
+                });
+                states[state] = page.items.length;
+                for (const file of page.items) {
+                    const result = await this.requestExtraction(context, file.id, file.mediaType);
+                    if (result.enqueued)
+                        requeued += 1;
+                }
+            }
+            return { requeued, states };
+        },
+    };
+};
+//# sourceMappingURL=extraction.js.map

package/dist/documents/index.d.ts

@@ -0,0 +1,199 @@
+/**
+ * Document extraction — `docs/05-knowledge-and-documents.md`, REQ-027 (#131).
+ *
+ * Extraction turns an opaque attachment into something the model can reason about. The shape it produces is
+ * the design decision here, and it is deliberately **not a flat string**.
+ *
+ * A flat string is what every quick extractor produces and it destroys exactly the information a question is
+ * usually about. "What was Q3 revenue?" is answerable from a table and unanswerable from that table flattened
+ * into prose — the row and column that gave a number its meaning are gone, and the model has no way to know
+ * they were ever there. So the intermediate is a block list: headings keep their level, tables keep their
+ * cells, lists keep their items.
+ *
+ * The second decision is that **failure is a value, not an exception**. A document that cannot be read is an
+ * ordinary outcome — a scan with no text layer, an encrypted file, something past a limit — and the
+ * assistant has to be able to say which. An extractor that threw would leave the caller with a stack trace
+ * and the user with a file that silently behaves as if it were empty, which is the failure mode AC-4 names.
+ */
+import type { BlobRef, FileId, TenantId } from "../core/ids.js";
+/** Heading depth, as in HTML. Beyond six the source is not using headings for structure. */
+export type HeadingLevel = 1 | 2 | 3 | 4 | 5 | 6;
+/**
+ * One piece of a document.
+ *
+ * A closed union rather than a bag with optional fields: a `table` with no `rows` and a `heading` with no
+ * `level` are both unrepresentable, so a renderer never has to decide what to do about one.
+ */
+export type DocumentBlock = {
+    readonly kind: "heading";
+    readonly level: HeadingLevel;
+    readonly text: string;
+    readonly page?: number;
+} | {
+    readonly kind: "paragraph";
+    readonly text: string;
+    readonly page?: number;
+} | {
+    readonly kind: "table";
+    /** Rows of cells. The first row is the header when `hasHeader` is true. */
+    readonly rows: readonly (readonly string[])[];
+    readonly hasHeader: boolean;
+    readonly page?: number;
+} | {
+    readonly kind: "list";
+    readonly items: readonly string[];
+    readonly ordered: boolean;
+    readonly page?: number;
+};
+/**
+ * Why a document could not be read.
+ *
+ * Each value exists because the user-facing sentence differs. `no-text-layer` is not `malformed`: the first
+ * means "this is a scan, OCR would help" and the second means "this file is broken". Collapsing them would
+ * send someone to re-export a file that needed a different pipeline.
+ */
+export declare const EXTRACTION_FAILURES: readonly ["unsupported-type", "too-large", "too-many-pages", "timed-out", "encrypted", "no-text-layer", "malformed"];
+export type ExtractionFailureReason = (typeof EXTRACTION_FAILURES)[number];
+export type ExtractionFailure = {
+    readonly reason: ExtractionFailureReason;
+    /** What the user is told. Names the limit when there is one, because "too large" sends someone to guess. */
+    readonly message: string;
+};
+/** What a parser produces when it succeeds. */
+export type ExtractedDocument = {
+    readonly blocks: readonly DocumentBlock[];
+    /** Pages seen. Absent for formats with no pagination — a CSV does not have pages. */
+    readonly pageCount?: number;
+    /** True when a limit stopped extraction early. The content is usable; it is just not all of it. */
+    readonly truncated: boolean;
+    /**
+     * Things the caller should know that are not failures.
+     *
+     * Surfaced rather than logged: "three pages had no text layer" changes how much a user should trust an
+     * answer, and a log line does not reach them.
+     */
+    readonly warnings: readonly string[];
+    /**
+     * How confident the extraction is, 0–1 (#132).
+     *
+     * Present only for extraction that *has* a confidence — OCR and vision. A PDF's text layer is not
+     * recognised, it is read, so a confidence there would be a number with nothing behind it. Absent therefore
+     * means "not a probabilistic extraction", which is a different fact from "confidence unknown".
+     */
+    readonly confidence?: number;
+    /**
+     * Priced operations this extraction performed (#132, AC-4).
+     *
+     * **Transport only.** The pipeline reports each entry to `onPricedOperation` and then strips the field
+     * before storing the blob, so a stored document never carries billing data. It lives on the result rather
+     * than in a callback because only the pipeline knows which file and tenant a parse belonged to — a callback
+     * handed to the parser at construction is shared across every file it ever parses, and correlating one
+     * would mean trusting that parses never interleave.
+     */
+    readonly usage?: readonly {
+        readonly kind: "vision" | "ocr";
+        readonly modelId: string;
+        readonly inputTokens: number;
+        readonly outputTokens: number;
+        readonly cachedInputTokens: number;
+    }[];
+};
+/**
+ * The bounds a parser runs inside — AC-3.
+ *
+ * Every one of these exists because a document is attacker-controlled input. A page limit alone is not
+ * enough: a single-page PDF can carry a content stream that decompresses to gigabytes, which is why
+ * `maxTextBytes` is separate from `maxBytes`, and why `maxBlocks` is separate again — a million empty
+ * paragraphs cost little text and a great deal of everything else.
+ */
+export type ExtractionLimits = {
+    /** Bytes read from storage. Past this the document is refused rather than truncated: a half-read PDF is not a shorter PDF. */
+    readonly maxBytes: number;
+    readonly maxPages: number;
+    /** Extracted text kept, across all blocks. Truncates rather than refuses — half a document is still useful. */
+    readonly maxTextBytes: number;
+    readonly maxBlocks: number;
+    /** Wall-clock ceiling. A parser that ignores it is stopped by the pipeline's own race. */
+    readonly timeoutMs: number;
+};
+/**
+ * Conservative defaults.
+ *
+ * `maxPages: 500` is a long report, not a book; `maxTextBytes: 2 MiB` is roughly 500k tokens of source text,
+ * far more than any context window, so the ceiling never binds before the read bounds do. `timeoutMs` is
+ * short on purpose — a worker occupied for a minute by one document is a worker not serving anyone else, and
+ * a document that slow is a document to reject rather than wait for.
+ */
+export declare const DEFAULT_EXTRACTION_LIMITS: ExtractionLimits;
+/**
+ * A parser for one family of media types.
+ *
+ * `mediaTypes` is data rather than a `canParse(type)` method so the pipeline can build its dispatch table
+ * once and answer "is this type supported at all" without running anything — which is what makes
+ * `unsupported-type` a decision taken before a byte is read.
+ */
+export interface DocumentParser {
+    readonly id: string;
+    readonly mediaTypes: readonly string[];
+    /**
+     * Parse, or return a typed failure.
+     *
+     * Returns rather than throws for document problems; a thrown error means the parser itself is broken, and
+     * the pipeline records that separately so the two are never confused in a report.
+     */
+    parse(input: {
+        readonly bytes: Uint8Array;
+        /**
+         * The normalised media type, without parameters.
+         *
+         * Passed in rather than re-derived: a parser serving five types has to know which one it was handed, and
+         * a parser that guessed from the bytes would disagree with the dispatch table that chose it.
+         */
+        readonly mediaType: string;
+        readonly limits: ExtractionLimits;
+    }): Promise<ExtractedDocument | ExtractionFailure>;
+}
+/** Discriminates a parser's two return shapes without a tag on the success side. */
+export declare const isExtractionFailure: (value: ExtractedDocument | ExtractionFailure) => value is ExtractionFailure;
+/**
+ * Extraction state on a file.
+ *
+ * `pending` and `failed` are both terminal-ish states a user can see, and they are separate from the file's
+ * own `state`: a file is perfectly `stored` while its extraction is `failed`, and conflating the two would
+ * make an unreadable document look like a lost upload.
+ */
+export declare const EXTRACTION_STATES: readonly ["pending", "running", "extracted", "failed", "skipped"];
+export type ExtractionState = (typeof EXTRACTION_STATES)[number];
+/** What the file metadata records about its derived text. */
+export type ExtractionRecord = {
+    readonly state: ExtractionState;
+    /** Where the `ExtractedDocument` lives. `BlobStore` holds JSON, which is exactly what this is. */
+    readonly ref?: BlobRef;
+    readonly failure?: ExtractionFailure;
+    readonly pageCount?: number;
+    readonly blockCount?: number;
+    readonly truncated?: boolean;
+    readonly at?: string;
+};
+/** A unit of extraction work. Mirrors `RunJob`'s shape so the two queues read the same way. */
+export type ExtractionJob = {
+    readonly tenantId: TenantId;
+    readonly fileId: FileId;
+};
+/**
+ * The queue's producing side.
+ *
+ * Separate from `JobDispatcher` rather than a method added to it: the run path depends on that interface, and
+ * widening it would make every existing implementation — including test doubles — incomplete for a job type
+ * they have nothing to do with.
+ */
+export interface ExtractionDispatcher {
+    enqueueExtraction(job: ExtractionJob): Promise<void>;
+}
+export * from "./parsers/text.js";
+export * from "./parsers/pdf.js";
+export * from "./extraction.js";
+export * from "./render.js";
+export * from "./read-tool.js";
+export * from "./vision.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/documents/index.js

@@ -0,0 +1,65 @@
+/**
+ * Document extraction — `docs/05-knowledge-and-documents.md`, REQ-027 (#131).
+ *
+ * Extraction turns an opaque attachment into something the model can reason about. The shape it produces is
+ * the design decision here, and it is deliberately **not a flat string**.
+ *
+ * A flat string is what every quick extractor produces and it destroys exactly the information a question is
+ * usually about. "What was Q3 revenue?" is answerable from a table and unanswerable from that table flattened
+ * into prose — the row and column that gave a number its meaning are gone, and the model has no way to know
+ * they were ever there. So the intermediate is a block list: headings keep their level, tables keep their
+ * cells, lists keep their items.
+ *
+ * The second decision is that **failure is a value, not an exception**. A document that cannot be read is an
+ * ordinary outcome — a scan with no text layer, an encrypted file, something past a limit — and the
+ * assistant has to be able to say which. An extractor that threw would leave the caller with a stack trace
+ * and the user with a file that silently behaves as if it were empty, which is the failure mode AC-4 names.
+ */
+/**
+ * Why a document could not be read.
+ *
+ * Each value exists because the user-facing sentence differs. `no-text-layer` is not `malformed`: the first
+ * means "this is a scan, OCR would help" and the second means "this file is broken". Collapsing them would
+ * send someone to re-export a file that needed a different pipeline.
+ */
+export const EXTRACTION_FAILURES = [
+    "unsupported-type",
+    "too-large",
+    "too-many-pages",
+    "timed-out",
+    "encrypted",
+    "no-text-layer",
+    "malformed",
+];
+/**
+ * Conservative defaults.
+ *
+ * `maxPages: 500` is a long report, not a book; `maxTextBytes: 2 MiB` is roughly 500k tokens of source text,
+ * far more than any context window, so the ceiling never binds before the read bounds do. `timeoutMs` is
+ * short on purpose — a worker occupied for a minute by one document is a worker not serving anyone else, and
+ * a document that slow is a document to reject rather than wait for.
+ */
+export const DEFAULT_EXTRACTION_LIMITS = {
+    maxBytes: 25 * 1024 * 1024,
+    maxPages: 500,
+    maxTextBytes: 2 * 1024 * 1024,
+    maxBlocks: 20_000,
+    timeoutMs: 20_000,
+};
+/** Discriminates a parser's two return shapes without a tag on the success side. */
+export const isExtractionFailure = (value) => "reason" in value;
+/**
+ * Extraction state on a file.
+ *
+ * `pending` and `failed` are both terminal-ish states a user can see, and they are separate from the file's
+ * own `state`: a file is perfectly `stored` while its extraction is `failed`, and conflating the two would
+ * make an unreadable document look like a lost upload.
+ */
+export const EXTRACTION_STATES = ["pending", "running", "extracted", "failed", "skipped"];
+export * from "./parsers/text.js";
+export * from "./parsers/pdf.js";
+export * from "./extraction.js";
+export * from "./render.js";
+export * from "./read-tool.js";
+export * from "./vision.js";
+//# sourceMappingURL=index.js.map

package/dist/documents/parsers/pdf.d.ts

@@ -0,0 +1,47 @@
+/**
+ * PDF text extraction (#131).
+ *
+ * Over the raw PDF syntax, with `node:zlib` for the compressed streams and no dependency. That is a real
+ * decision with real limits, so they are stated here rather than discovered later.
+ *
+ * **What it handles.** PDFs produced by the tools people actually use — Word, LaTeX, Chrome's print-to-PDF,
+ * Google Docs, most report generators. It walks the content streams, follows the text operators, and uses the
+ * positioning operators to reconstruct lines and paragraphs. Font size changes become heading levels, because
+ * a PDF has no headings: it has text that happens to be bigger, and inferring from that is the only signal
+ * available.
+ *
+ * **What it does not handle**, each of which is reported as a typed failure rather than as empty text:
+ *
+ * - **Encrypted documents.** Detected by `/Encrypt` in the trailer and refused. Extracting from one would
+ *   mean implementing the standard security handler, which is a decryption tool wearing a parser's clothes.
+ * - **Scans with no text layer.** A photograph of a page contains no text operators at all. Reported as
+ *   `no-text-layer`, which is the answer that sends someone to OCR instead of to a bug report.
+ * - **Custom-encoded embedded fonts.** A Type0/CID font with an embedded `ToUnicode` map needs that map
+ *   applied; without it the bytes are glyph indices, not characters. Detected as mojibake and warned about
+ *   rather than silently returned as text — a garbled answer is worse than a refusal.
+ *
+ * Tables are the honest weak spot and the comment on `groupIntoBlocks` says how far it gets: a PDF does not
+ * contain tables, it contains text at coordinates, and column inference from coordinates is a heuristic. It
+ * recovers regular grids and says so when it is unsure.
+ */
+import type { DocumentParser, ExtractedDocument, ExtractionFailure, ExtractionLimits } from "../index.js";
+/**
+ * Emitted when the inflated total is cut off.
+ *
+ * A named constant because a test asserting on it is asserting on the bound itself. The alternative — checking
+ * that *something* was truncated — passes when the block or text ceiling fires instead, which is a different
+ * bound and does nothing to stop the decompression work.
+ */
+export declare const DECOMPRESSION_LIMIT_WARNING = "Stopped decompressing: the document's streams expand past the allowed size.";
+/**
+ * `\` escapes inside a PDF literal string.
+ *
+ * Including the line continuation, which is the one people forget: a backslash at end of line means "no
+ * character here", and treating it as a literal backslash inserts one into every wrapped string.
+ */
+export declare const unescapePdfString: (raw: string) => string;
+/** A hex string, `<48656C6C6F>`. An odd final digit is padded with zero, as the spec requires. */
+export declare const decodeHexString: (raw: string) => string;
+export declare const parsePdf: (bytes: Uint8Array, limits: ExtractionLimits) => ExtractedDocument | ExtractionFailure;
+export declare const createPdfDocumentParser: () => DocumentParser;
+//# sourceMappingURL=pdf.d.ts.map