@retinue/agentkit 0.1.0

package/dist/documents/parsers/text.js

@@ -0,0 +1,325 @@
+/**
+ * Text, Markdown, CSV and JSON extraction (#131).
+ *
+ * The easy formats, and worth doing properly rather than passing straight through. A CSV handed to a model as
+ * one long string is the flattening this whole module exists to avoid — the row and column that give a number
+ * its meaning are exactly what gets lost. So a CSV becomes a `table` block with its cells intact, Markdown
+ * headings keep their level, and pipe tables survive as tables.
+ *
+ * Everything here is bounded the same way the PDF parser is, because "it's only text" is how a 200 MB CSV
+ * occupies a worker.
+ */
+/**
+ * Accumulates blocks against the limits, so every parser stops the same way.
+ *
+ * Shared rather than repeated: the two parsers in this module and the PDF one all need the same three
+ * ceilings, and three copies of "am I over budget" is three chances for one of them to be wrong.
+ */
+export const createBlockBuilder = (limits) => {
+    const blocks = [];
+    const warnings = [];
+    let textBytes = 0;
+    let truncated = false;
+    const sizeOf = (block) => {
+        switch (block.kind) {
+            case "heading":
+            case "paragraph":
+                return block.text.length;
+            case "list":
+                return block.items.reduce((n, item) => n + item.length, 0);
+            case "table":
+                return block.rows.reduce((n, row) => n + row.reduce((m, cell) => m + cell.length, 0), 0);
+        }
+    };
+    return {
+        /** Returns false once full, so a caller can stop reading rather than keep parsing into a bin. */
+        push(block) {
+            if (truncated)
+                return false;
+            if (blocks.length >= limits.maxBlocks) {
+                truncated = true;
+                warnings.push(`Stopped after ${limits.maxBlocks} blocks; the document continues.`);
+                return false;
+            }
+            const size = sizeOf(block);
+            if (textBytes + size > limits.maxTextBytes) {
+                truncated = true;
+                warnings.push(`Stopped after ${limits.maxTextBytes} bytes of text; the document continues.`);
+                return false;
+            }
+            textBytes += size;
+            blocks.push(block);
+            return true;
+        },
+        warn(message) {
+            warnings.push(message);
+        },
+        get truncated() {
+            return truncated;
+        },
+        done(pageCount) {
+            return {
+                blocks,
+                ...(pageCount === undefined ? {} : { pageCount }),
+                truncated,
+                warnings,
+            };
+        },
+    };
+};
+/** Collapse runs of whitespace. A PDF or a wrapped paragraph produces plenty, and none of it is meaning. */
+const tidy = (text) => text.replace(/\s+/g, " ").trim();
+const decode = (bytes) => 
+// `fatal: false`: a byte sequence that is not valid UTF-8 should give a replacement character, not turn a
+// readable document into an extraction failure.
+new TextDecoder("utf-8", { fatal: false }).decode(bytes);
+/**
+ * A pipe-table row split into cells.
+ *
+ * Returns null when the line is not a table row, so the caller's check and the split are one decision rather
+ * than a test followed by a parse that could disagree with it.
+ */
+const pipeCells = (line) => {
+    const trimmed = line.trim();
+    if (!trimmed.includes("|"))
+        return null;
+    // Leading and trailing pipes are optional in Markdown, so they are stripped before splitting rather than
+    // producing phantom empty cells at both ends.
+    const inner = trimmed.replace(/^\|/, "").replace(/\|$/, "");
+    return inner.split("|").map((cell) => cell.trim());
+};
+/** `|---|:--:|` — the row that marks the line above as a header. Never content. */
+const isPipeDivider = (line) => /^\s*\|?\s*:?-{2,}:?\s*(\|\s*:?-{2,}:?\s*)*\|?\s*$/.test(line);
+/**
+ * Markdown, and plain text as its degenerate case.
+ *
+ * Plain text has no headings or tables, so running it through the same parser costs nothing and means one
+ * code path instead of two that differ in how they split paragraphs.
+ */
+export const parseMarkdown = (bytes, limits) => {
+    const builder = createBlockBuilder(limits);
+    const lines = decode(bytes).split(/\r?\n/);
+    let paragraph = [];
+    const flushParagraph = () => {
+        if (paragraph.length === 0)
+            return true;
+        const text = tidy(paragraph.join(" "));
+        paragraph = [];
+        return text === "" ? true : builder.push({ kind: "paragraph", text });
+    };
+    let listItems = [];
+    let listOrdered = false;
+    const flushList = () => {
+        if (listItems.length === 0)
+            return true;
+        const items = listItems;
+        listItems = [];
+        return builder.push({ kind: "list", items, ordered: listOrdered });
+    };
+    let index = 0;
+    while (index < lines.length) {
+        const line = lines[index] ?? "";
+        const heading = /^(#{1,6})\s+(.*)$/.exec(line);
+        if (heading) {
+            if (!flushParagraph() || !flushList())
+                break;
+            const level = heading[1]?.length ?? 1;
+            if (!builder.push({ kind: "heading", level: level, text: tidy(heading[2] ?? "") }))
+                break;
+            index += 1;
+            continue;
+        }
+        // A table is recognised by its divider, not by containing a pipe: a sentence with a pipe in it is a
+        // sentence, and treating it as a one-column table would be worse than leaving it as prose.
+        const cells = pipeCells(line);
+        if (cells !== null && cells.length > 1 && isPipeDivider(lines[index + 1] ?? "")) {
+            if (!flushParagraph() || !flushList())
+                break;
+            const rows = [[...cells]];
+            let cursor = index + 2;
+            while (cursor < lines.length) {
+                const rowCells = pipeCells(lines[cursor] ?? "");
+                if (rowCells === null || rowCells.length <= 1)
+                    break;
+                rows.push([...rowCells]);
+                cursor += 1;
+            }
+            if (!builder.push({ kind: "table", rows, hasHeader: true }))
+                break;
+            index = cursor;
+            continue;
+        }
+        const bullet = /^\s*(?:[-*+]|(\d+)[.)])\s+(.*)$/.exec(line);
+        if (bullet) {
+            if (!flushParagraph())
+                break;
+            const ordered = bullet[1] !== undefined;
+            // A bullet list running into a numbered one is two lists. Keeping them as one would renumber the
+            // second, which changes what the document said.
+            if (listItems.length > 0 && ordered !== listOrdered && !flushList())
+                break;
+            listOrdered = ordered;
+            listItems.push(tidy(bullet[2] ?? ""));
+            index += 1;
+            continue;
+        }
+        if (line.trim() === "") {
+            if (!flushParagraph() || !flushList())
+                break;
+            index += 1;
+            continue;
+        }
+        if (listItems.length > 0 && !flushList())
+            break;
+        paragraph.push(line);
+        index += 1;
+    }
+    flushParagraph();
+    flushList();
+    return builder.done();
+};
+/**
+ * A CSV row splitter that understands quotes.
+ *
+ * Not `line.split(",")`. A quoted field can contain a comma, a newline and an escaped quote, and a splitter
+ * that ignores that silently shifts every subsequent column — the worst possible failure for a table, because
+ * the result is plausible.
+ */
+export const parseDelimited = (bytes, limits, delimiter = ",") => {
+    const builder = createBlockBuilder(limits);
+    const text = decode(bytes);
+    const rows = [];
+    let row = [];
+    let field = "";
+    let quoted = false;
+    let index = 0;
+    const endField = () => {
+        row.push(field);
+        field = "";
+    };
+    const endRow = () => {
+        endField();
+        // A trailing newline would otherwise add a row of one empty cell to every file.
+        if (!(row.length === 1 && row[0] === ""))
+            rows.push(row);
+        row = [];
+    };
+    while (index < text.length) {
+        const char = text[index];
+        if (quoted) {
+            if (char === '"') {
+                // `""` inside a quoted field is one literal quote, not the end of the field.
+                if (text[index + 1] === '"') {
+                    field += '"';
+                    index += 2;
+                    continue;
+                }
+                quoted = false;
+                index += 1;
+                continue;
+            }
+            field += char;
+            index += 1;
+            continue;
+        }
+        if (char === '"' && field === "") {
+            quoted = true;
+            index += 1;
+            continue;
+        }
+        if (char === delimiter) {
+            endField();
+            index += 1;
+            continue;
+        }
+        if (char === "\n") {
+            endRow();
+            index += 1;
+            continue;
+        }
+        if (char === "\r") {
+            index += 1;
+            continue;
+        }
+        field += char;
+        index += 1;
+    }
+    if (field !== "" || row.length > 0)
+        endRow();
+    if (rows.length === 0)
+        return builder.done();
+    // The first row is the header when every cell is non-empty — the usual case, and a header of blanks is a
+    // sign the file has none rather than a header worth claiming.
+    const hasHeader = (rows[0] ?? []).every((cell) => cell.trim() !== "");
+    builder.push({ kind: "table", rows, hasHeader });
+    return builder.done();
+};
+/**
+ * JSON as a document.
+ *
+ * An array of flat objects is a table and is by far the most common shape a JSON attachment has, so it is
+ * extracted as one. Anything else is rendered as indented text: still readable, and honest about the fact
+ * that there was no table to find.
+ */
+export const parseJsonDocument = (bytes, limits) => {
+    const builder = createBlockBuilder(limits);
+    let parsed;
+    try {
+        parsed = JSON.parse(decode(bytes));
+    }
+    catch {
+        return { reason: "malformed", message: "That file is not valid JSON." };
+    }
+    const isFlatRecord = (value) => typeof value === "object" &&
+        value !== null &&
+        !Array.isArray(value) &&
+        Object.values(value).every((v) => v === null || ["string", "number", "boolean"].includes(typeof v));
+    if (Array.isArray(parsed) && parsed.length > 0 && parsed.every(isFlatRecord)) {
+        // The union of keys, in first-seen order: a record missing a key gets an empty cell rather than shifting
+        // the row, which is the same silent-corruption failure quoted CSV fields avoid.
+        const columns = [];
+        for (const record of parsed)
+            for (const key of Object.keys(record))
+                if (!columns.includes(key))
+                    columns.push(key);
+        const rows = [columns, ...parsed.map((record) => columns.map((c) => String(record[c] ?? "")))];
+        builder.push({ kind: "table", rows, hasHeader: true });
+        return builder.done();
+    }
+    builder.push({ kind: "paragraph", text: tidy(JSON.stringify(parsed, null, 2)) });
+    return builder.done();
+};
+/**
+ * The parser for every text-shaped type.
+ *
+ * One parser dispatching on the type it was handed, rather than five parsers sharing a builder. The formats
+ * differ only in how a line is read; the bounds, the decoding and the block shapes are identical, and five
+ * copies of those is five chances for one to be wrong.
+ */
+export const TEXT_MEDIA_TYPES = [
+    "text/plain",
+    "text/markdown",
+    "text/x-markdown",
+    "text/csv",
+    "text/tab-separated-values",
+    "application/json",
+];
+export const createTextDocumentParser = () => ({
+    id: "text",
+    mediaTypes: TEXT_MEDIA_TYPES,
+    async parse({ bytes, mediaType, limits }) {
+        switch (mediaType) {
+            case "text/csv":
+                return parseDelimited(bytes, limits, ",");
+            case "text/tab-separated-values":
+                return parseDelimited(bytes, limits, "\t");
+            case "application/json":
+                return parseJsonDocument(bytes, limits);
+            default:
+                // Plain text goes through the Markdown parser: plain text is Markdown with no markup, so one path
+                // handles both and a `.txt` file that happens to use `#` headings is read the way its author meant.
+                return parseMarkdown(bytes, limits);
+        }
+    },
+});
+//# sourceMappingURL=text.js.map

package/dist/documents/read-tool.d.ts

@@ -0,0 +1,52 @@
+/**
+ * `read_document` — bounded reads of extracted structure (#131, AC-6).
+ *
+ * The counterpart to #130's `read_attachment`, and the reason both exist rather than one: `read_attachment`
+ * returns raw bytes, which for a PDF is binary noise. This returns the *extracted* document, rendered as
+ * Markdown so a table is still a table, in a bounded window.
+ *
+ * **The window is in blocks, not characters.** A character bound can land inside a table and hand the model
+ * half of one, which is worse than no table because the missing rows are invisible — the model answers
+ * confidently from the half it can see. A block is the smallest unit that is still true on its own.
+ *
+ * When there is nothing to read, this reports *why* (AC-4). "That PDF is a scan and needs OCR" is a sentence
+ * the assistant can pass on; an empty result is one it would paper over.
+ */
+import type { Tool } from "../tools/index.js";
+import type { ExtractionService } from "./extraction.js";
+import { summariseBlocks } from "./render.js";
+/**
+ * Blocks per call, and the character ceiling that overrides it.
+ *
+ * Both, because either alone is escapable: fifty paragraphs is a reasonable window and fifty *tables* is not,
+ * so the block count bounds the common case and the character count bounds the adversarial one.
+ */
+export declare const MAX_BLOCKS_PER_READ = 50;
+export declare const MAX_CHARS_PER_READ = 24000;
+export type ReadDocumentOutput = {
+    readonly fileId: string;
+    readonly text: string;
+    readonly fromBlock: number;
+    readonly blocksReturned: number;
+    readonly totalBlocks: number;
+    readonly truncated: boolean;
+    readonly nextBlock?: number;
+    /** Present when extraction itself truncated the document — a different fact from this window truncating. */
+    readonly documentTruncated?: boolean;
+    /**
+     * OCR/vision confidence, 0–1, and only when the extraction was probabilistic (#132).
+     *
+     * Present alongside the low-confidence warning rather than instead of it: the warning is the sentence a
+     * model should pass on, and the number is what a caller comparing two extractions needs.
+     */
+    readonly confidence?: number;
+    readonly lowConfidence?: boolean;
+    readonly warnings?: readonly string[];
+};
+export declare const createReadDocumentTool: (deps: {
+    readonly extraction: ExtractionService;
+}) => Tool;
+/** A one-line description of a document's shape, for a context section that must stay cheap. */
+export declare const describeExtraction: (blockCount: number, pageCount: number | undefined) => string;
+export { summariseBlocks };
+//# sourceMappingURL=read-tool.d.ts.map

package/dist/documents/read-tool.js

@@ -0,0 +1,109 @@
+/**
+ * `read_document` — bounded reads of extracted structure (#131, AC-6).
+ *
+ * The counterpart to #130's `read_attachment`, and the reason both exist rather than one: `read_attachment`
+ * returns raw bytes, which for a PDF is binary noise. This returns the *extracted* document, rendered as
+ * Markdown so a table is still a table, in a bounded window.
+ *
+ * **The window is in blocks, not characters.** A character bound can land inside a table and hand the model
+ * half of one, which is worse than no table because the missing rows are invisible — the model answers
+ * confidently from the half it can see. A block is the smallest unit that is still true on its own.
+ *
+ * When there is nothing to read, this reports *why* (AC-4). "That PDF is a scan and needs OCR" is a sentence
+ * the assistant can pass on; an empty result is one it would paper over.
+ */
+import { AgentPlatformError } from "../core/errors.js";
+import { asId } from "../core/ids.js";
+import { defineTool } from "../tools/define.js";
+import { renderBlocks, summariseBlocks } from "./render.js";
+import { LOW_CONFIDENCE_THRESHOLD } from "../persistence/index.js";
+/**
+ * Blocks per call, and the character ceiling that overrides it.
+ *
+ * Both, because either alone is escapable: fifty paragraphs is a reasonable window and fifty *tables* is not,
+ * so the block count bounds the common case and the character count bounds the adversarial one.
+ */
+export const MAX_BLOCKS_PER_READ = 50;
+export const MAX_CHARS_PER_READ = 24_000;
+export const createReadDocumentTool = (deps) => defineTool({
+    name: "read_document",
+    label: "Read document",
+    description: `Read a bounded portion of an attached document's extracted text, with headings and tables preserved. ` +
+        `At most ${MAX_BLOCKS_PER_READ} blocks per call; continue from the block index it returns.`,
+    category: "files",
+    effect: "read",
+    inputSchema: {
+        type: "object",
+        required: ["fileId"],
+        additionalProperties: false,
+        properties: {
+            fileId: { type: "string", description: "The id from the attachment reference, without the `file:` prefix." },
+            fromBlock: { type: "integer", minimum: 0, description: "Block index to start from. Defaults to 0." },
+            maxBlocks: {
+                type: "integer",
+                minimum: 1,
+                maximum: MAX_BLOCKS_PER_READ,
+                description: `At most ${MAX_BLOCKS_PER_READ}. A larger value is clamped, not refused.`,
+            },
+        },
+    },
+    async execute(input, context) {
+        const id = asId(input.fileId.replace(/^file:/, ""));
+        const { document, extraction } = await deps.extraction.getExtracted(context, id);
+        if (document === null) {
+            // The reason, not an empty result. This is AC-4 reaching the user: the message a parser wrote is the
+            // message the assistant repeats, so "that PDF is a scan" survives all the way out.
+            const detail = extraction === undefined
+                ? "That file has not been processed for text yet. Try again shortly."
+                : (extraction.failureMessage ??
+                    (extraction.state === "pending" || extraction.state === "running"
+                        ? "That document is still being processed. Try again shortly."
+                        : "No extracted text is available for that document."));
+            throw new AgentPlatformError({ code: "not_found", message: detail, retryable: extraction?.state !== "failed" });
+        }
+        const from = Math.max(0, Math.floor(input.fromBlock ?? 0));
+        // Clamped rather than refused: a model asking for more gets less, which it can act on, instead of an
+        // error it would answer with a retry.
+        const wanted = Math.min(Math.max(1, Math.floor(input.maxBlocks ?? MAX_BLOCKS_PER_READ)), MAX_BLOCKS_PER_READ);
+        const taken = [];
+        let chars = 0;
+        for (const block of document.blocks.slice(from, from + wanted)) {
+            const rendered = renderBlocks([block]);
+            // The character ceiling stops *before* adding an oversized block rather than after, but always takes
+            // at least one: a single table larger than the ceiling would otherwise return nothing at all, and a
+            // window that can return nothing is a window a model cannot page past.
+            if (chars + rendered.length > MAX_CHARS_PER_READ && taken.length > 0)
+                break;
+            taken.push(block);
+            chars += rendered.length;
+        }
+        const end = from + taken.length;
+        const truncated = end < document.blocks.length;
+        return {
+            fileId: id,
+            text: renderBlocks(taken),
+            fromBlock: from,
+            blocksReturned: taken.length,
+            totalBlocks: document.blocks.length,
+            truncated,
+            ...(truncated ? { nextBlock: end } : {}),
+            // Only when true. A `false` on every response is a field a model has to read and discard.
+            ...(document.truncated ? { documentTruncated: true } : {}),
+            // AC-5 reaching the model. The flag is derived here as well as warned about in the document, so a
+            // consumer that only reads structured fields still cannot mistake uncertain text for certain.
+            ...(document.confidence === undefined
+                ? {}
+                : {
+                    confidence: document.confidence,
+                    ...(document.confidence < LOW_CONFIDENCE_THRESHOLD ? { lowConfidence: true } : {}),
+                }),
+            ...(document.warnings.length > 0 ? { warnings: document.warnings } : {}),
+        };
+    },
+});
+/** A one-line description of a document's shape, for a context section that must stay cheap. */
+export const describeExtraction = (blockCount, pageCount) => pageCount !== undefined && pageCount > 0
+    ? `${pageCount} page${pageCount === 1 ? "" : "s"}, ${blockCount} block${blockCount === 1 ? "" : "s"}`
+    : `${blockCount} block${blockCount === 1 ? "" : "s"}`;
+export { summariseBlocks };
+//# sourceMappingURL=read-tool.js.map

package/dist/documents/render.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Rendering extracted structure back to text for a model (#131).
+ *
+ * The counterpart to extraction, and the reason extracting to blocks rather than a string pays off: a table
+ * is rendered as a Markdown table, so the model sees the same rows and columns the document had. Flattening
+ * happened nowhere — not at extraction, and not here.
+ *
+ * Markdown specifically, because it is the format models are most reliably trained on for tabular text and
+ * the one where a cell's column is unambiguous. A CSV rendering would be more compact and would lose the
+ * header's distinctness; an ASCII-art table would cost far more tokens for no gain.
+ */
+import type { DocumentBlock } from "./index.js";
+export declare const renderBlock: (block: DocumentBlock) => string;
+/**
+ * A window of blocks, rendered.
+ *
+ * Blocks rather than characters, because a bound that can land inside a table would hand the model half a
+ * table — which is worse than no table, since the missing rows are invisible. A block is the smallest unit
+ * that is still meaningful on its own.
+ */
+export declare const renderBlocks: (blocks: readonly DocumentBlock[]) => string;
+/**
+ * A one-line description of what a document contains.
+ *
+ * For the context section: enough for a model to decide whether reading it is worth the budget, at a cost
+ * that does not depend on the document's length.
+ */
+export declare const summariseBlocks: (blocks: readonly DocumentBlock[]) => string;
+//# sourceMappingURL=render.d.ts.map

package/dist/documents/render.js

@@ -0,0 +1,59 @@
+/**
+ * Rendering extracted structure back to text for a model (#131).
+ *
+ * The counterpart to extraction, and the reason extracting to blocks rather than a string pays off: a table
+ * is rendered as a Markdown table, so the model sees the same rows and columns the document had. Flattening
+ * happened nowhere — not at extraction, and not here.
+ *
+ * Markdown specifically, because it is the format models are most reliably trained on for tabular text and
+ * the one where a cell's column is unambiguous. A CSV rendering would be more compact and would lose the
+ * header's distinctness; an ASCII-art table would cost far more tokens for no gain.
+ */
+/** A pipe cell cannot contain a raw `|` without ending the cell, and a newline would end the row. */
+const cell = (text) => text.replace(/\|/g, "\\|").replace(/\s*\n\s*/g, " ");
+export const renderBlock = (block) => {
+    switch (block.kind) {
+        case "heading":
+            return `${"#".repeat(block.level)} ${block.text}`;
+        case "paragraph":
+            return block.text;
+        case "list":
+            return block.items.map((item, i) => (block.ordered ? `${i + 1}. ${item}` : `- ${item}`)).join("\n");
+        case "table": {
+            const rows = block.rows.map((row) => `| ${row.map(cell).join(" | ")} |`);
+            if (!block.hasHeader || rows.length === 0)
+                return rows.join("\n");
+            // The divider is what makes the first row a header rather than just the first row. Its width has to
+            // match the header's cell count or the table reads as malformed.
+            const divider = `| ${(block.rows[0] ?? []).map(() => "---").join(" | ")} |`;
+            return [rows[0], divider, ...rows.slice(1)].join("\n");
+        }
+    }
+};
+/**
+ * A window of blocks, rendered.
+ *
+ * Blocks rather than characters, because a bound that can land inside a table would hand the model half a
+ * table — which is worse than no table, since the missing rows are invisible. A block is the smallest unit
+ * that is still meaningful on its own.
+ */
+export const renderBlocks = (blocks) => blocks.map(renderBlock).join("\n\n");
+/**
+ * A one-line description of what a document contains.
+ *
+ * For the context section: enough for a model to decide whether reading it is worth the budget, at a cost
+ * that does not depend on the document's length.
+ */
+export const summariseBlocks = (blocks) => {
+    const counts = { heading: 0, paragraph: 0, table: 0, list: 0 };
+    for (const block of blocks)
+        counts[block.kind] += 1;
+    const parts = [
+        counts.heading > 0 ? `${counts.heading} heading${counts.heading === 1 ? "" : "s"}` : null,
+        counts.paragraph > 0 ? `${counts.paragraph} paragraph${counts.paragraph === 1 ? "" : "s"}` : null,
+        counts.table > 0 ? `${counts.table} table${counts.table === 1 ? "" : "s"}` : null,
+        counts.list > 0 ? `${counts.list} list${counts.list === 1 ? "" : "s"}` : null,
+    ].filter((part) => part !== null);
+    return parts.length === 0 ? "no content" : parts.join(", ");
+};
+//# sourceMappingURL=render.js.map