@theowlops/boombot 0.0.0-stage → 1.1.1

package/dist/node_modules/@boombot/core/research/parse/guards.js

@@ -0,0 +1,171 @@
+/**
+ * Security guards for document parsing (`research.parse`).
+ *
+ * Parsing attacker-influenced bytes — a URL the model or user asked
+ * `research.extract` to read — is an attack surface: zip bombs, decompression
+ * bombs hidden in PDF content streams, and absurd entry counts. This module
+ * is the choke point every parser runs through: the same role
+ * `sanitizeExtractUrl` plays for URL inputs in `orchestrator.extract()`.
+ *
+ * Two lines of defense, because declared sizes are attacker-controlled:
+ * 1. Declared sizes are checked BEFORE inflating — cheaply rejects 42.zip-style
+ *    entries whose headers claim gigabytes.
+ * 2. Actual output bytes are counted DURING streaming inflation — closes the
+ *    understated/zero-size hole (the adm-zip CVE-2026-77301 class of bug, where
+ *    a streaming path never checked accumulated output).
+ *
+ * Fail-closed: any cap breach throws `DocumentParseError`; a parser never
+ * returns a partial buffer.
+ */
+import { createInflate, createInflateRaw, } from 'node:zlib';
+export class DocumentParseError extends Error {
+    code;
+    constructor(code, message, options) {
+        super(message, options);
+        this.name = 'DocumentParseError';
+        this.code = code;
+    }
+}
+/**
+ * Hard cap on raw document bytes accepted for parsing. Matches
+ * `DEFAULT_MAX_RESPONSE_BYTES` in `../http.ts` so a document fetch can never
+ * exceed what a normal extract response may already hold.
+ */
+export const MAX_DOCUMENT_BYTES = 5_242_880; // 5 MiB
+/** Max entries in a ZIP central directory — a docx rarely has a few hundred. */
+export const MAX_ZIP_ENTRIES = 2048;
+/** Max declared uncompressed size of a single ZIP entry. */
+export const MAX_ZIP_ENTRY_BYTES = 20_971_520; // 20 MiB
+/** Max total decompressed bytes across every inflated entry/stream of one document. */
+export const MAX_TOTAL_DECOMPRESSED_BYTES = 52_428_800; // 50 MiB
+/**
+ * Declared compression ratio above this is treated as a bomb. Legitimate
+ * docx XML compresses ~5–20:1; real zip bombs start around 100,000:1, so
+ * 100:1 rejects every known bomb shape with wide margin for legit files.
+ */
+export const MAX_COMPRESSION_RATIO = 100;
+/** Max content streams inflated from a single PDF. */
+export const MAX_PDF_STREAMS = 20_000;
+/**
+ * Ceiling on extracted plain-text characters — defense-in-depth past the
+ * byte quotas, so parser output can never blow up turn context on its own.
+ */
+export const MAX_EXTRACTED_CHARS = 200_000;
+/** Reject an input buffer that exceeds {@link MAX_DOCUMENT_BYTES}. */
+export function assertDocumentSize(byteLength, label = 'document') {
+    if (byteLength > MAX_DOCUMENT_BYTES) {
+        throw new DocumentParseError('DOCUMENT_TOO_LARGE', `${label} is ${byteLength} bytes; document parsing is capped at ${MAX_DOCUMENT_BYTES} bytes.`);
+    }
+}
+/**
+ * Check a ZIP entry's DECLARED sizes before touching its data.
+ * Cheap first line of defense — rejects 42.zip-style headers outright.
+ */
+export function assertDeclaredEntrySizes(args) {
+    const { compressedSize, uncompressedSize, method, label } = args;
+    if (uncompressedSize > MAX_ZIP_ENTRY_BYTES) {
+        throw new DocumentParseError('ARCHIVE_LIMIT', `ZIP entry '${label}' declares ${uncompressedSize} uncompressed bytes (cap ${MAX_ZIP_ENTRY_BYTES}).`);
+    }
+    if (method === 8) {
+        if (compressedSize === 0) {
+            // The size-0 bypass: a deflated entry with no compressed bytes can
+            // still expand (or is simply malformed). Only a truly empty entry
+            // (0 → 0) is allowed through.
+            if (uncompressedSize !== 0) {
+                throw new DocumentParseError('ARCHIVE_LIMIT', `ZIP entry '${label}' is deflated but declares 0 compressed bytes (possible bomb).`);
+            }
+            return;
+        }
+        const ratio = uncompressedSize / compressedSize;
+        if (ratio > MAX_COMPRESSION_RATIO) {
+            throw new DocumentParseError('ARCHIVE_LIMIT', `ZIP entry '${label}' declares a ${Math.round(ratio)}:1 compression ratio ` +
+                `(cap ${MAX_COMPRESSION_RATIO}:1, possible zip bomb).`);
+        }
+    }
+    else if (method === 0) {
+        if (compressedSize !== uncompressedSize) {
+            throw new DocumentParseError('CORRUPT_DOCUMENT', `ZIP entry '${label}' is stored but compressed size ${compressedSize} ` +
+                `!= uncompressed size ${uncompressedSize}.`);
+        }
+    }
+    else {
+        throw new DocumentParseError('UNSUPPORTED_TYPE', `ZIP entry '${label}' uses compression method ${method} (only stored/deflated are supported).`);
+    }
+}
+/**
+ * Tracks the running total of decompressed bytes for one document and
+ * inflates raw-deflate blocks with both per-entry and total caps enforced
+ * DURING decompression (streaming). Declared sizes are attacker-controlled,
+ * so this second line of defense measures actual output — an entry that
+ * understates its size still trips the cap mid-inflate instead of landing
+ * a huge buffer in memory first.
+ */
+export class DecompressionQuota {
+    #usedTotal = 0;
+    /** Bytes inflated so far across all entries/streams of this document. */
+    get usedTotal() {
+        return this.#usedTotal;
+    }
+    /** Account `byteLength` raw (stored, not inflated) bytes against the total quota. */
+    trackRaw(byteLength, label) {
+        this.#usedTotal += byteLength;
+        if (this.#usedTotal > MAX_TOTAL_DECOMPRESSED_BYTES) {
+            throw new DocumentParseError('ARCHIVE_LIMIT', `Decompression quota exceeded reading '${label}' (possible zip bomb): ` +
+                `document total ${this.#usedTotal} bytes (cap ${MAX_TOTAL_DECOMPRESSED_BYTES}).`);
+        }
+    }
+    /**
+     * Inflate one deflate block through `inflater`. Rejects with `ARCHIVE_LIMIT`
+     * when the entry would exceed `maxEntryBytes` or the document would exceed
+     * {@link MAX_TOTAL_DECOMPRESSED_BYTES}.
+     */
+    #runInflate(inflater, compressed, maxEntryBytes, label) {
+        return new Promise((resolve, reject) => {
+            const chunks = [];
+            let entryBytes = 0;
+            let settled = false;
+            const fail = (err) => {
+                if (settled)
+                    return;
+                settled = true;
+                inflater.destroy();
+                reject(err);
+            };
+            inflater.on('data', (chunk) => {
+                entryBytes += chunk.length;
+                this.#usedTotal += chunk.length;
+                if (entryBytes > maxEntryBytes ||
+                    this.#usedTotal > MAX_TOTAL_DECOMPRESSED_BYTES) {
+                    fail(new DocumentParseError('ARCHIVE_LIMIT', `Decompression quota exceeded while inflating '${label}' (possible zip bomb): ` +
+                        `entry ${entryBytes} bytes (cap ${maxEntryBytes}), ` +
+                        `document total ${this.#usedTotal} bytes (cap ${MAX_TOTAL_DECOMPRESSED_BYTES}).`));
+                    return;
+                }
+                chunks.push(chunk);
+            });
+            inflater.on('end', () => {
+                if (settled)
+                    return;
+                settled = true;
+                resolve(Buffer.concat(chunks));
+            });
+            inflater.on('error', (err) => {
+                fail(new DocumentParseError('CORRUPT_DOCUMENT', `Failed to inflate '${label}': ${err.message}.`));
+            });
+            inflater.end(compressed);
+        });
+    }
+    /**
+     * Inflate one zlib-wrapped deflate block (PDF `/FlateDecode` is zlib /
+     * RFC 1950 per the PDF spec, NOT raw deflate).
+     */
+    inflate(compressed, maxEntryBytes, label) {
+        return this.#runInflate(createInflate(), compressed, maxEntryBytes, label);
+    }
+    /**
+     * Inflate one raw deflate block (ZIP entry data has no zlib wrapper).
+     */
+    inflateRaw(compressed, maxEntryBytes, label) {
+        return this.#runInflate(createInflateRaw(), compressed, maxEntryBytes, label);
+    }
+}

package/dist/node_modules/@boombot/core/research/parse/index.d.ts

@@ -0,0 +1,68 @@
+/**
+ * `research.parse` — zero-dependency document parsing for research.
+ *
+ * `parseDocument(bytes, mimeType)` extracts text from PDF and DOCX without
+ * any npm dependency (only `node:zlib` for decompression, quota-guarded in
+ * `./guards.ts`). `extractDocument()` is the fetch-level entry the
+ * orchestrator calls: it downloads the URL with a hard byte cap, sniffs the
+ * magic bytes, and parses — or hands back to the regular extract backend
+ * when the bytes are not actually a document.
+ *
+ * Security posture (parse = attack surface):
+ * - Every input passes `./guards.ts` caps (file size, entry count, declared
+ *   + actual decompression quotas) — the input-side choke point, mirroring
+ *   what `sanitizeExtractUrl` does for URLs.
+ * - Extracted text passes `sanitizeExtractedText()` before entering turn
+ *   context — the output-side analog: control characters stripped, newlines
+ *   normalized, length capped.
+ */
+import type { ExtractOpts, ExtractProvider, ExtractResult } from '../types.js';
+export { DocumentParseError } from './guards.js';
+export type { DocumentParseErrorCode } from './guards.js';
+export type { DocxExtractedText } from './docx.js';
+export type { PdfCMap, PdfExtractedText } from './pdf.js';
+export declare const PDF_MIME = "application/pdf";
+export declare const DOCX_MIME = "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
+/** Document kinds this module can parse. */
+export type SupportedDocumentKind = 'pdf' | 'docx';
+export interface ParsedDocument {
+    readonly kind: SupportedDocumentKind;
+    readonly mimeType: string;
+    readonly text: string;
+    readonly title: string;
+    /** PDF only, best-effort. */
+    readonly pageCount?: number;
+}
+/**
+ * Guess the document kind from a URL's path extension (query/fragment
+ * ignored). Used to decide whether a URL is worth fetching as a document;
+ * the magic-byte sniff always has the final say.
+ */
+export declare function detectDocumentKindFromUrl(url: string): SupportedDocumentKind | undefined;
+/** Identify a document by magic bytes: `%PDF-` or the ZIP `PK\x03\x04`. */
+export declare function sniffDocumentType(bytes: Uint8Array): SupportedDocumentKind | undefined;
+/**
+ * Parse document `bytes` of the given MIME type. Throws
+ * {@link DocumentParseError} (`UNSUPPORTED_TYPE` / `DOCUMENT_TOO_LARGE` /
+ * `ARCHIVE_LIMIT` / `CORRUPT_DOCUMENT`) — never returns partial text.
+ */
+export declare function parseDocument(bytes: Uint8Array, mimeType: string): Promise<ParsedDocument>;
+/**
+ * Sanitize extracted text before it enters turn context — the output-side
+ * analog of `sanitizeExtractUrl`: strip control characters (log/terminal
+ * injection, model-input noise), normalize newlines, collapse blank runs,
+ * and cap length as defense-in-depth past the byte quotas.
+ */
+export declare function sanitizeExtractedText(text: string): string;
+/**
+ * Fetch `url`, sniff it, and parse it as a document — the entry point
+ * `ResearchOrchestrator.extract()` uses for `.pdf`/`.docx` URLs.
+ *
+ * - Magic bytes decide the parser (extension is only the trigger); a URL
+ *   that merely LOOKS like a document but serves HTML falls back to the
+ *   regular extract backend instead of failing.
+ * - A corrupt/oversize/bomb document throws `DocumentParseError` with a
+ *   clear message (fail-closed — never silently handed to a third party).
+ * - Supports the same `offset` windowing as the other extract providers.
+ */
+export declare function extractDocument(url: string, extractProvider: ExtractProvider, signal?: AbortSignal, opts?: ExtractOpts): Promise<ExtractResult>;

package/dist/node_modules/@boombot/core/research/parse/index.js

@@ -0,0 +1,219 @@
+/**
+ * `research.parse` — zero-dependency document parsing for research.
+ *
+ * `parseDocument(bytes, mimeType)` extracts text from PDF and DOCX without
+ * any npm dependency (only `node:zlib` for decompression, quota-guarded in
+ * `./guards.ts`). `extractDocument()` is the fetch-level entry the
+ * orchestrator calls: it downloads the URL with a hard byte cap, sniffs the
+ * magic bytes, and parses — or hands back to the regular extract backend
+ * when the bytes are not actually a document.
+ *
+ * Security posture (parse = attack surface):
+ * - Every input passes `./guards.ts` caps (file size, entry count, declared
+ *   + actual decompression quotas) — the input-side choke point, mirroring
+ *   what `sanitizeExtractUrl` does for URLs.
+ * - Extracted text passes `sanitizeExtractedText()` before entering turn
+ *   context — the output-side analog: control characters stripped, newlines
+ *   normalized, length capped.
+ */
+import { Buffer } from 'node:buffer';
+import { DEFAULT_USER_AGENT, ResearchError, TRUNCATED_MARKER, } from '../http.js';
+import { extractDocxText } from './docx.js';
+import { DocumentParseError, MAX_DOCUMENT_BYTES, MAX_EXTRACTED_CHARS, assertDocumentSize, } from './guards.js';
+import { extractPdfText } from './pdf.js';
+export { DocumentParseError } from './guards.js';
+export const PDF_MIME = 'application/pdf';
+export const DOCX_MIME = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document';
+/**
+ * Guess the document kind from a URL's path extension (query/fragment
+ * ignored). Used to decide whether a URL is worth fetching as a document;
+ * the magic-byte sniff always has the final say.
+ */
+export function detectDocumentKindFromUrl(url) {
+    const path = url.split(/[?#]/, 1)[0]?.toLowerCase() ?? '';
+    if (path.endsWith('.pdf'))
+        return 'pdf';
+    if (path.endsWith('.docx'))
+        return 'docx';
+    return undefined;
+}
+/** Identify a document by magic bytes: `%PDF-` or the ZIP `PK\x03\x04`. */
+export function sniffDocumentType(bytes) {
+    if (bytes.length >= 5 &&
+        bytes[0] === 0x25 && // %
+        bytes[1] === 0x50 && // P
+        bytes[2] === 0x44 && // D
+        bytes[3] === 0x46 && // F
+        bytes[4] === 0x2d // -
+    ) {
+        return 'pdf';
+    }
+    if (bytes.length >= 4 &&
+        bytes[0] === 0x50 && // P
+        bytes[1] === 0x4b && // K
+        bytes[2] === 0x03 &&
+        bytes[3] === 0x04) {
+        return 'docx';
+    }
+    return undefined;
+}
+/**
+ * Parse document `bytes` of the given MIME type. Throws
+ * {@link DocumentParseError} (`UNSUPPORTED_TYPE` / `DOCUMENT_TOO_LARGE` /
+ * `ARCHIVE_LIMIT` / `CORRUPT_DOCUMENT`) — never returns partial text.
+ */
+export async function parseDocument(bytes, mimeType) {
+    assertDocumentSize(bytes.length);
+    const mime = mimeType.split(';', 1)[0]?.trim().toLowerCase() ?? '';
+    if (mime === PDF_MIME) {
+        const parsed = await extractPdfText(bytes);
+        return {
+            kind: 'pdf',
+            mimeType: PDF_MIME,
+            text: parsed.text,
+            title: parsed.title,
+            pageCount: parsed.pageCount,
+        };
+    }
+    if (mime === DOCX_MIME) {
+        const parsed = await extractDocxText(bytes);
+        return {
+            kind: 'docx',
+            mimeType: DOCX_MIME,
+            text: parsed.text,
+            title: parsed.title,
+        };
+    }
+    throw new DocumentParseError('UNSUPPORTED_TYPE', `Unsupported document type '${mimeType}'. Supported: ${PDF_MIME}, ${DOCX_MIME}.`);
+}
+/**
+ * Sanitize extracted text before it enters turn context — the output-side
+ * analog of `sanitizeExtractUrl`: strip control characters (log/terminal
+ * injection, model-input noise), normalize newlines, collapse blank runs,
+ * and cap length as defense-in-depth past the byte quotas.
+ */
+export function sanitizeExtractedText(text) {
+    let out = '';
+    for (const ch of text) {
+        const code = ch.codePointAt(0) ?? 0;
+        if (code === 0x0a || code === 0x09)
+            out += ch;
+        else if (code < 0x20 || code === 0x7f)
+            continue;
+        else
+            out += ch;
+    }
+    out = out
+        .replace(/\r\n?/g, '\n')
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+    if (out.length > MAX_EXTRACTED_CHARS)
+        out = out.slice(0, MAX_EXTRACTED_CHARS);
+    return out;
+}
+const FETCH_TIMEOUT_MS = 15_000;
+/** Extract window, matching the jina provider's window for paging parity. */
+const WINDOW_CHARS = 8000;
+/**
+ * Fetch a URL as bytes with a hard cap — same streaming-cap pattern as
+ * `fetchText` in `../http.ts`, but returning bytes (a document is not text
+ * until parsed). Throws typed {@link ResearchError}s like the rest of the
+ * module.
+ */
+async function fetchBytes(url, opts) {
+    const timeoutSignal = AbortSignal.timeout(opts.timeoutMs);
+    const signal = opts.signal === undefined
+        ? timeoutSignal
+        : AbortSignal.any([opts.signal, timeoutSignal]);
+    let response;
+    try {
+        response = await fetch(url, {
+            signal,
+            headers: { 'User-Agent': DEFAULT_USER_AGENT },
+        });
+    }
+    catch (err) {
+        if (err instanceof DOMException && err.name === 'TimeoutError') {
+            throw new ResearchError('TIMEOUT', `Request to ${url} timed out`);
+        }
+        if (err instanceof DOMException && err.name === 'AbortError') {
+            throw new ResearchError('ABORTED', `Request to ${url} was aborted`);
+        }
+        const detail = err instanceof Error ? err.message : String(err);
+        throw new ResearchError('NETWORK_ERROR', `Network error fetching ${url}: ${detail}`);
+    }
+    if (!response.ok) {
+        const code = response.status >= 400 && response.status < 500 ? 'HTTP_4XX' : 'HTTP_5XX';
+        throw new ResearchError(code, `HTTP ${response.status} from ${url}`);
+    }
+    const body = response.body;
+    if (body === null)
+        return new Uint8Array(0);
+    const reader = body.getReader();
+    const chunks = [];
+    let total = 0;
+    for (;;) {
+        let read;
+        try {
+            read = await reader.read();
+        }
+        catch (err) {
+            const detail = err instanceof Error ? err.message : String(err);
+            throw new ResearchError('NETWORK_ERROR', `Network error fetching ${url}: ${detail}`);
+        }
+        if (read.done)
+            break;
+        total += read.value.byteLength;
+        if (total > opts.maxBytes) {
+            await reader.cancel().catch(() => undefined);
+            throw new ResearchError('RESPONSE_TOO_LARGE', `Response from ${url} exceeds ${opts.maxBytes} bytes`);
+        }
+        chunks.push(read.value);
+    }
+    return Buffer.concat(chunks);
+}
+function firstLine(text) {
+    const line = text.split('\n', 1)[0]?.trim() ?? '';
+    return line.length > 120 ? `${line.slice(0, 120)}…` : line;
+}
+/**
+ * Fetch `url`, sniff it, and parse it as a document — the entry point
+ * `ResearchOrchestrator.extract()` uses for `.pdf`/`.docx` URLs.
+ *
+ * - Magic bytes decide the parser (extension is only the trigger); a URL
+ *   that merely LOOKS like a document but serves HTML falls back to the
+ *   regular extract backend instead of failing.
+ * - A corrupt/oversize/bomb document throws `DocumentParseError` with a
+ *   clear message (fail-closed — never silently handed to a third party).
+ * - Supports the same `offset` windowing as the other extract providers.
+ */
+export async function extractDocument(url, extractProvider, signal, opts) {
+    const bytes = await fetchBytes(url, {
+        signal,
+        timeoutMs: FETCH_TIMEOUT_MS,
+        maxBytes: MAX_DOCUMENT_BYTES,
+    });
+    // Magic bytes decide the parser (the extension is only the trigger): a URL
+    // that merely LOOKS like a document but serves something else (e.g. an
+    // HTML error page) falls back to the regular extract backend instead of
+    // failing.
+    const kind = sniffDocumentType(bytes);
+    if (kind === undefined) {
+        return extractProvider.extract(url, signal, opts);
+    }
+    const parsed = await parseDocument(bytes, kind === 'pdf' ? PDF_MIME : DOCX_MIME);
+    const text = sanitizeExtractedText(parsed.text);
+    const offset = Math.max(0, Math.floor(opts?.offset ?? 0));
+    const slice = text.slice(offset, offset + WINDOW_CHARS);
+    const totalLength = text.length;
+    const hasMore = offset + slice.length < totalLength;
+    return {
+        url,
+        title: parsed.title !== '' ? parsed.title : firstLine(slice),
+        markdown: hasMore ? slice + TRUNCATED_MARKER : slice,
+        truncated: hasMore,
+        offset,
+        totalLength,
+        nextOffset: hasMore ? offset + WINDOW_CHARS : undefined,
+    };
+}

package/dist/node_modules/@boombot/core/research/parse/pdf.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Minimal PDF text extractor — hand-written, zero npm dependencies.
+ *
+ * Only `node:zlib` (via `./guards.ts` quotas) is used, for `/FlateDecode`
+ * content streams. This is a deliberate trade-off, documented honestly:
+ *
+ * - COVERED: text-based PDFs from common generators (FlateDecode or
+ *   uncompressed content streams, `Tj`/`TJ`/`'`/`"` operators, literal and
+ *   hex strings, WinAnsi/PDFDocEncoding bytes, UTF-16BE strings, and
+ *   `/ToUnicode` CMaps — the subsetted-font encoding arXiv/LaTeX PDFs use).
+ * - NOT covered: encrypted PDFs (rejected), scanned/image-only PDFs (no OCR —
+ *   text extraction correctly yields nothing), exotic filters (ASCII85,
+ *   LZW, CCITT images are skipped), fonts with NEITHER a standard encoding
+ *   NOR a ToUnicode map (byte values are emitted best-effort).
+ *
+ * A full renderer (pdf.js, poppler) would cover more, but both violate the
+ * project's zero-new-runtime-dep / easy-self-host philosophy — poppler is an
+ * OS package that may not exist on the host, pdf.js is megabytes of npm.
+ * For `research.extract`/`research.deep`, "most text out of most PDFs" is
+ * the right weight class.
+ */
+export interface PdfExtractedText {
+    readonly text: string;
+    readonly title: string;
+    /** Best-effort page count from `/Type /Page` objects; absent when undeterminable. */
+    readonly pageCount?: number;
+}
+/** A parsed `/ToUnicode` CMap: byte-group size + code → unicode char. */
+export interface PdfCMap {
+    readonly bytesPerChar: number;
+    readonly map: ReadonlyMap<number, string>;
+}
+/**
+ * Extract text from PDF `bytes`. Throws {@link DocumentParseError} with
+ * `CORRUPT_DOCUMENT` (not a PDF), `UNSUPPORTED_TYPE` (encrypted),
+ * `DOCUMENT_TOO_LARGE`, or `ARCHIVE_LIMIT` (stream bomb).
+ */
+export declare function extractPdfText(bytes: Uint8Array): Promise<PdfExtractedText>;