@theowlops/boombot 0.0.0-stage → 1.1.1

package/dist/node_modules/@boombot/core/research/parse/pdf.js

@@ -0,0 +1,567 @@
+/**
+ * Minimal PDF text extractor — hand-written, zero npm dependencies.
+ *
+ * Only `node:zlib` (via `./guards.ts` quotas) is used, for `/FlateDecode`
+ * content streams. This is a deliberate trade-off, documented honestly:
+ *
+ * - COVERED: text-based PDFs from common generators (FlateDecode or
+ *   uncompressed content streams, `Tj`/`TJ`/`'`/`"` operators, literal and
+ *   hex strings, WinAnsi/PDFDocEncoding bytes, UTF-16BE strings, and
+ *   `/ToUnicode` CMaps — the subsetted-font encoding arXiv/LaTeX PDFs use).
+ * - NOT covered: encrypted PDFs (rejected), scanned/image-only PDFs (no OCR —
+ *   text extraction correctly yields nothing), exotic filters (ASCII85,
+ *   LZW, CCITT images are skipped), fonts with NEITHER a standard encoding
+ *   NOR a ToUnicode map (byte values are emitted best-effort).
+ *
+ * A full renderer (pdf.js, poppler) would cover more, but both violate the
+ * project's zero-new-runtime-dep / easy-self-host philosophy — poppler is an
+ * OS package that may not exist on the host, pdf.js is megabytes of npm.
+ * For `research.extract`/`research.deep`, "most text out of most PDFs" is
+ * the right weight class.
+ */
+import { Buffer } from 'node:buffer';
+import { DecompressionQuota, DocumentParseError, MAX_PDF_STREAMS, MAX_ZIP_ENTRY_BYTES, assertDocumentSize, } from './guards.js';
+/** Windows-1252-ish byte → char mapping (covers PDFDocEncoding's printable range). */
+const BYTE_TO_CHAR = (() => {
+    const win1252 = {
+        0x80: 0x20ac, 0x82: 0x201a, 0x83: 0x0192, 0x84: 0x201e, 0x85: 0x2026,
+        0x86: 0x2020, 0x87: 0x2021, 0x88: 0x02c6, 0x89: 0x2030, 0x8a: 0x0160,
+        0x8b: 0x2039, 0x8c: 0x0152, 0x8e: 0x017d, 0x91: 0x2018, 0x92: 0x2019,
+        0x93: 0x201c, 0x94: 0x201d, 0x95: 0x2022, 0x96: 0x2013, 0x97: 0x2014,
+        0x98: 0x02dc, 0x99: 0x2122, 0x9a: 0x0161, 0x9b: 0x203a, 0x9c: 0x0153,
+        0x9e: 0x017e, 0x9f: 0x0178,
+    };
+    const table = new Array(256);
+    for (let b = 0; b < 256; b += 1) {
+        if (b < 0x80)
+            table[b] = String.fromCharCode(b);
+        else if (win1252[b] !== undefined)
+            table[b] = String.fromCodePoint(win1252[b]);
+        else
+            table[b] = String.fromCharCode(b);
+    }
+    return table;
+})();
+/** Decode one PDF string's raw bytes: UTF-16BE when BOM'd, else byte-mapped. */
+function decodePdfBytes(raw) {
+    if (raw.length >= 2 && raw[0] === 0xfe && raw[1] === 0xff) {
+        let out = '';
+        for (let i = 2; i + 1 < raw.length; i += 2) {
+            const hi = raw[i] ?? 0;
+            const lo = raw[i + 1] ?? 0;
+            const code = (hi << 8) | lo;
+            // Skip a BOM re-appearing mid-string; lone surrogates are dropped
+            // rather than emitted as mojibake.
+            if (code === 0xfeff)
+                continue;
+            out += String.fromCharCode(code);
+        }
+        return out;
+    }
+    let out = '';
+    for (const b of raw)
+        out += BYTE_TO_CHAR[b] ?? '';
+    return out;
+}
+/** Decode a CMap destination hex string (UTF-16BE code units). */
+function decodeUtf16Hex(hex) {
+    let out = '';
+    for (let i = 0; i + 4 <= hex.length; i += 4) {
+        out += String.fromCharCode(parseInt(hex.slice(i, i + 4), 16));
+    }
+    return out;
+}
+/**
+ * A regex group that is present by construction (the match could not have
+ * succeeded without it). Throws rather than threading `| undefined`
+ * through every CMap call site.
+ */
+function grp(m, i) {
+    const v = m[i];
+    if (v === undefined)
+        throw new Error(`CMap match missing group ${i}`);
+    return v;
+}
+/**
+ * Parse a `begincmap ... endcmap` stream: `beginbfchar` pairs and
+ * `beginbfrange` ranges (both the `<dst>` and `[<dst> ...]` forms).
+ * Anything unrecognized is skipped — a partial map beats no map.
+ */
+function parseCMap(cmapText) {
+    let bytesPerChar = 1;
+    const rangeBlock = /begincodespacerange([\s\S]*?)endcodespacerange/.exec(cmapText);
+    if (rangeBlock !== null) {
+        let longest = 0;
+        for (const m of grp(rangeBlock, 1).matchAll(/<([0-9a-fA-F]+)>/g)) {
+            longest = Math.max(longest, grp(m, 1).length / 2);
+        }
+        if (longest > 0)
+            bytesPerChar = Math.min(longest, 2);
+    }
+    const map = new Map();
+    for (const block of cmapText.matchAll(/beginbfchar([\s\S]*?)endbfchar/g)) {
+        for (const p of grp(block, 1).matchAll(/<([0-9a-fA-F]+)>\s*<([0-9a-fA-F]+)>/g)) {
+            map.set(parseInt(grp(p, 1), 16), decodeUtf16Hex(grp(p, 2)));
+        }
+    }
+    for (const block of cmapText.matchAll(/beginbfrange([\s\S]*?)endbfrange/g)) {
+        for (const t of grp(block, 1).matchAll(/<([0-9a-fA-F]+)>\s*<([0-9a-fA-F]+)>\s*(?:<([0-9a-fA-F]+)>|\[([^\]]*)\])/g)) {
+            const lo = parseInt(grp(t, 1), 16);
+            const hi = parseInt(grp(t, 2), 16);
+            // A hostile or sloppy CMap could declare a gigantic range; cap it.
+            const end = Math.min(hi, lo + 65535);
+            if (t[3] !== undefined) {
+                const dstLo = parseInt(grp(t, 3), 16);
+                for (let c = lo; c <= end; c += 1) {
+                    map.set(c, String.fromCodePoint(dstLo + (c - lo)));
+                }
+            }
+            else if (t[4] !== undefined) {
+                const dsts = [...grp(t, 4).matchAll(/<([0-9a-fA-F]+)>/g)].map((m) => decodeUtf16Hex(grp(m, 1)));
+                for (let c = lo; c <= end && c - lo < dsts.length; c += 1) {
+                    const dst = dsts[c - lo];
+                    if (dst !== undefined)
+                        map.set(c, dst);
+                }
+            }
+        }
+    }
+    return { bytesPerChar, map };
+}
+/** Map indirect object numbers to their bodies (on stream-stripped text). */
+function buildObjectMap(stripped) {
+    const map = new Map();
+    const re = /(\d+)\s+0\s+obj([\s\S]*?)endobj/g;
+    for (;;) {
+        const m = re.exec(stripped);
+        if (m === null)
+            break;
+        map.set(parseInt(grp(m, 1), 10), grp(m, 2));
+    }
+    return map;
+}
+/**
+ * Resolve font resource names (`F1`) to their `/ToUnicode` CMaps by walking
+ * `/Resources → /Font → font object → /ToUnicode`. `cmapTexts` carries the
+ * already-inflated CMap streams keyed by object number (collected by
+ * `extractStreams`). Best-effort throughout: a PDF without this structure
+ * simply yields an empty map and falls back to byte decoding.
+ */
+function buildFontCMaps(stripped, cmapTexts) {
+    const out = new Map();
+    try {
+        const objMap = buildObjectMap(stripped);
+        const fontToUnicode = new Map();
+        for (const [num, body] of objMap) {
+            const t = /\/ToUnicode\s+(\d+)\s+0\s+R/.exec(body);
+            if (t !== null)
+                fontToUnicode.set(num, parseInt(grp(t, 1), 10));
+        }
+        if (fontToUnicode.size === 0)
+            return out;
+        // Font resource names (`F1` in `/F1 12 Tf`): for each font object, find
+        // every `/Name <fontObj> 0 R` reference sitting inside a `<<...>>` dict.
+        // This covers both inline `/Resources << /Font << /F1 9 0 R >> >>` and
+        // the common indirection where the name→object dict lives in its own
+        // object (`11 0 obj << /F1 9 0 R >>`). A stray match (e.g. `/Parent`)
+        // is harmless: the map is only consulted for names seen before `Tf`.
+        const nameToFont = new Map();
+        for (const fontNum of fontToUnicode.keys()) {
+            const refRe = new RegExp(`/([A-Za-z0-9_+\\-.]+)\\s+${fontNum}\\s+0\\s+R`, 'g');
+            for (const m of stripped.matchAll(refRe)) {
+                const open = stripped.lastIndexOf('<<', m.index);
+                if (open < 0)
+                    continue;
+                if (stripped.slice(open, m.index).includes('>>'))
+                    continue;
+                nameToFont.set(grp(m, 1), fontNum);
+            }
+        }
+        for (const [name, fontNum] of nameToFont) {
+            const uniNum = fontToUnicode.get(fontNum);
+            if (uniNum === undefined)
+                continue;
+            const cmapText = cmapTexts.get(uniNum);
+            if (cmapText === undefined)
+                continue;
+            try {
+                out.set(name, parseCMap(cmapText));
+            }
+            catch {
+                // A broken CMap must not fail the document.
+            }
+        }
+    }
+    catch {
+        // CMap resolution is best-effort; byte decoding remains.
+    }
+    return out;
+}
+/**
+ * Extract the text-showing operations of one content stream.
+ * Handles `(literal) Tj`, `<hex> Tj`, `[...] TJ`, `'...'` and `"..."`.
+ * `cmaps` maps PDF font resource names (`F1` in `/F1 12 Tf`) to their
+ * `/ToUnicode` CMaps; strings set in a mapped font are decoded through it.
+ */
+function extractContentText(content, cmaps) {
+    const parts = [];
+    let i = 0;
+    const n = content.length;
+    let currentFont;
+    const skipWs = () => {
+        while (i < n) {
+            const c = content[i];
+            if (c === ' ' || c === '\t' || c === '\n' || c === '\r' || c === '\f' || c === '\0')
+                i += 1;
+            else
+                break;
+        }
+    };
+    /** Read an operator token at the current position (after whitespace). */
+    const readOperator = () => {
+        skipWs();
+        const start = i;
+        while (i < n) {
+            const c = content[i] ?? '';
+            if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c === "'" || c === '"' || c === '*')
+                i += 1;
+            else
+                break;
+        }
+        return content.slice(start, i);
+    };
+    /** Parse a balanced literal string; `i` points at `(`. Returns raw bytes. */
+    const readLiteral = () => {
+        i += 1; // '('
+        const bytes = [];
+        let depth = 1;
+        while (i < n && depth > 0) {
+            const c = content[i] ?? '';
+            if (c === '\\') {
+                const e = content[i + 1];
+                i += 2;
+                if (e === undefined)
+                    break;
+                if (e === 'n')
+                    bytes.push(0x0a);
+                else if (e === 'r')
+                    bytes.push(0x0d);
+                else if (e === 't')
+                    bytes.push(0x09);
+                else if (e === 'b')
+                    bytes.push(0x08);
+                else if (e === 'f')
+                    bytes.push(0x0c);
+                else if (e === '(')
+                    bytes.push(0x28);
+                else if (e === ')')
+                    bytes.push(0x29);
+                else if (e === '\\')
+                    bytes.push(0x5c);
+                else if (e === '\n') { /* line continuation — nothing */ }
+                else if (e === '\r') {
+                    if (content[i] === '\n')
+                        i += 1;
+                }
+                else if (e >= '0' && e <= '7') {
+                    let oct = e;
+                    for (let k = 0; k < 2; k += 1) {
+                        const d = content[i];
+                        if (d !== undefined && d >= '0' && d <= '7') {
+                            oct += d;
+                            i += 1;
+                        }
+                        else
+                            break;
+                    }
+                    bytes.push(parseInt(oct, 8) & 0xff);
+                }
+                else {
+                    bytes.push(e.charCodeAt(0) & 0xff);
+                }
+            }
+            else if (c === '(') {
+                depth += 1;
+                bytes.push(0x28);
+                i += 1;
+            }
+            else if (c === ')') {
+                depth -= 1;
+                if (depth > 0)
+                    bytes.push(0x29);
+                i += 1;
+            }
+            else {
+                bytes.push(c.charCodeAt(0) & 0xff);
+                i += 1;
+            }
+        }
+        return Uint8Array.from(bytes);
+    };
+    /** Parse a hex string; `i` points at `<` (not `<<`). */
+    const readHex = () => {
+        i += 1; // '<'
+        let hex = '';
+        while (i < n && content[i] !== '>') {
+            const c = content[i] ?? '';
+            if (c !== ' ' && c !== '\t' && c !== '\n' && c !== '\r' && c !== '\f')
+                hex += c;
+            i += 1;
+        }
+        i += 1; // '>'
+        if (hex.length % 2 === 1)
+            hex += '0';
+        const out = new Uint8Array(hex.length / 2);
+        for (let k = 0; k < out.length; k += 1) {
+            out[k] = parseInt(hex.slice(k * 2, k * 2 + 2), 16);
+        }
+        return out;
+    };
+    /** Decode one string's bytes: the current font's ToUnicode CMap wins, else byte mapping. */
+    const decodeString = (raw) => {
+        const cmap = currentFont !== undefined ? cmaps.get(currentFont) : undefined;
+        if (cmap === undefined)
+            return decodePdfBytes(raw);
+        let out = '';
+        const step = cmap.bytesPerChar;
+        for (let k = 0; k < raw.length; k += step) {
+            let code = 0;
+            for (let b = 0; b < step && k + b < raw.length; b += 1) {
+                code = (code << 8) | (raw[k + b] ?? 0);
+            }
+            out += cmap.map.get(code) ?? decodePdfBytes(raw.slice(k, k + step));
+        }
+        return out;
+    };
+    while (i < n) {
+        skipWs();
+        if (i >= n)
+            break;
+        const c = content[i] ?? '';
+        if (c === '/') {
+            // Possible `/F1 12 Tf` — track the current font for ToUnicode decoding.
+            const nameMatch = /^\/([A-Za-z0-9_+\-.]+)/.exec(content.slice(i));
+            if (nameMatch !== null) {
+                const afterName = i + nameMatch[0].length;
+                const tfMatch = /^\s*\d+(?:\.\d+)?\s+Tf\b/.exec(content.slice(afterName));
+                if (tfMatch !== null) {
+                    currentFont = nameMatch[1];
+                    i = afterName + tfMatch[0].length;
+                    continue;
+                }
+            }
+            i += 1;
+        }
+        else if (c === '(') {
+            const raw = readLiteral();
+            const op = readOperator();
+            if (op === 'Tj')
+                parts.push(decodeString(raw));
+            else if (op === "'") {
+                // ' = T* + Tj: next line of text.
+                parts.push(decodeString(raw));
+                parts.push('\n');
+            }
+            else if (op === '"') {
+                // " = Tw Tc + ': operands (aw ac) precede the string.
+                parts.push(decodeString(raw));
+                parts.push('\n');
+            }
+        }
+        else if (c === '<' && content[i + 1] !== '<') {
+            const raw = readHex();
+            const op = readOperator();
+            if (op === 'Tj')
+                parts.push(decodeString(raw));
+        }
+        else if (c === '[') {
+            i += 1; // '['
+            const items = [];
+            for (;;) {
+                skipWs();
+                if (i >= n)
+                    break;
+                const t = content[i] ?? '';
+                if (t === ']') {
+                    i += 1;
+                    break;
+                }
+                if (t === '(')
+                    items.push(decodeString(readLiteral()));
+                else if (t === '<' && content[i + 1] !== '<')
+                    items.push(decodeString(readHex()));
+                else
+                    i += 1; // kerning numbers and anything else
+            }
+            if (readOperator() === 'TJ')
+                parts.push(items.join(' '));
+        }
+        else if (c === 'T' && content[i + 1] === '*') {
+            i += 2;
+            parts.push('\n');
+        }
+        else {
+            i += 1;
+        }
+    }
+    return parts.join(' ').replace(/ \n /g, '\n').replace(/ \n/g, '\n');
+}
+/**
+ * Inflate FlateDecode content streams (quota-guarded); pass through
+ * unfiltered streams; skip anything else (images, exotic filters).
+ * One corrupt stream never kills the document — it is skipped.
+ *
+ * `/ToUnicode` CMap streams are inflated here too (they are just streams);
+ * `extractPdfText` separates them by content (`begincmap`) afterwards.
+ */
+async function extractStreams(bytes, quota) {
+    const latin = Buffer.from(bytes).toString('latin1');
+    const out = [];
+    const re = /stream\r?\n([\s\S]*?)\r?\nendstream/g;
+    let count = 0;
+    for (;;) {
+        const m = re.exec(latin);
+        if (m === null)
+            break;
+        count += 1;
+        if (count > MAX_PDF_STREAMS) {
+            throw new DocumentParseError('ARCHIVE_LIMIT', `PDF declares more than ${MAX_PDF_STREAMS} content streams.`);
+        }
+        // The stream's owner is the last `N 0 obj` header before it
+        // (objects never nest, so the nearest header owns the stream).
+        const before = latin.slice(Math.max(0, m.index - 4000), m.index);
+        let objNum;
+        for (const om of before.matchAll(/(\d+)\s+0\s+obj/g)) {
+            objNum = parseInt(grp(om, 1), 10);
+        }
+        // Look at the stream's own dictionary only: the nearest `<<` before
+        // `stream`, so a previous object's /Filter can't mislabel this one.
+        const dictOpen = latin.lastIndexOf('<<', m.index);
+        const dictText = dictOpen >= 0 && m.index - dictOpen < 4000
+            ? latin.slice(dictOpen, m.index)
+            : '';
+        const filterMatch = /\/Filter\s*(\[[^\]]*\]|\/[A-Za-z0-9]+)/.exec(dictText);
+        const filterText = filterMatch !== null ? grp(filterMatch, 1) : '';
+        try {
+            if (filterText.includes('FlateDecode')) {
+                const raw = Buffer.from(m[1] ?? '', 'latin1');
+                // PDF FlateDecode is zlib-wrapped (RFC 1950), not raw deflate.
+                const inflated = await quota.inflate(raw, MAX_ZIP_ENTRY_BYTES, `pdf-stream#${count}`);
+                out.push({ text: Buffer.from(inflated).toString('latin1'), objNum });
+            }
+            else if (filterText === '') {
+                out.push({ text: m[1] ?? '', objNum });
+            }
+            // else: image/exotic filter — skipped, not text.
+        }
+        catch {
+            // One bad stream is skipped; the document still parses.
+        }
+    }
+    return out;
+}
+/** Best-effort `/Title (...)` lookup on stream-stripped text. */
+function findInfoString(stripped, key) {
+    const at = stripped.indexOf(`/${key}`);
+    if (at < 0)
+        return undefined;
+    let i = at + key.length + 1;
+    while (i < stripped.length && /\s/.test(stripped[i] ?? ''))
+        i += 1;
+    if (stripped[i] !== '(')
+        return undefined;
+    i += 1;
+    const bytes = [];
+    let depth = 1;
+    while (i < stripped.length && depth > 0) {
+        const c = stripped[i] ?? '';
+        if (c === '\\') {
+            const e = stripped[i + 1] ?? '';
+            bytes.push(e.charCodeAt(0) & 0xff);
+            i += 2;
+        }
+        else if (c === '(') {
+            depth += 1;
+            bytes.push(0x28);
+            i += 1;
+        }
+        else if (c === ')') {
+            depth -= 1;
+            if (depth > 0)
+                bytes.push(0x29);
+            i += 1;
+        }
+        else {
+            bytes.push(c.charCodeAt(0) & 0xff);
+            i += 1;
+        }
+    }
+    if (depth !== 0)
+        return undefined;
+    return decodePdfBytes(Uint8Array.from(bytes));
+}
+/**
+ * Extract text from PDF `bytes`. Throws {@link DocumentParseError} with
+ * `CORRUPT_DOCUMENT` (not a PDF), `UNSUPPORTED_TYPE` (encrypted),
+ * `DOCUMENT_TOO_LARGE`, or `ARCHIVE_LIMIT` (stream bomb).
+ */
+export async function extractPdfText(bytes) {
+    assertDocumentSize(bytes.length, 'PDF');
+    // Skip a UTF-8 BOM and leading whitespace before the header.
+    let p = 0;
+    while (p < bytes.length) {
+        const b = bytes[p] ?? 0;
+        if (b === 0x20 || b === 0x09 || b === 0x0a || b === 0x0d)
+            p += 1;
+        else if (b === 0xef && bytes[p + 1] === 0xbb && bytes[p + 2] === 0xbf)
+            p += 3;
+        else
+            break;
+    }
+    const magic = [0x25, 0x50, 0x44, 0x46, 0x2d]; // '%PDF-'
+    for (let k = 0; k < magic.length; k += 1) {
+        if (bytes[p + k] !== magic[k]) {
+            throw new DocumentParseError('CORRUPT_DOCUMENT', 'Not a PDF file (missing %PDF- header).');
+        }
+    }
+    const latin = Buffer.from(bytes).toString('latin1');
+    // Scan for /Encrypt outside content streams — inside a stream the token
+    // could be literal page text, which must not false-positive.
+    const stripped = latin.replace(/stream\r?\n[\s\S]*?\r?\nendstream/g, '');
+    if (/\/Encrypt\b/.test(stripped)) {
+        throw new DocumentParseError('UNSUPPORTED_TYPE', 'Encrypted PDFs are not supported.');
+    }
+    const quota = new DecompressionQuota();
+    const streams = await extractStreams(bytes, quota);
+    // Separate /ToUnicode CMap streams from content streams by content.
+    const cmapTexts = new Map();
+    const contentTexts = [];
+    for (const s of streams) {
+        if (s.objNum !== undefined &&
+            s.text.includes('begincmap') &&
+            s.text.includes('endcmap')) {
+            cmapTexts.set(s.objNum, s.text);
+        }
+        else {
+            contentTexts.push(s.text);
+        }
+    }
+    // Resolve /ToUnicode CMaps before extracting text (best-effort; a
+    // failure here only loses the mapping, never the document).
+    const cmaps = buildFontCMaps(stripped, cmapTexts);
+    if (contentTexts.length === 0) {
+        // A header alone is not a document: image-only PDFs still carry
+        // (image) streams, so zero content streams means this is not a real PDF.
+        throw new DocumentParseError('CORRUPT_DOCUMENT', 'PDF has no content streams.');
+    }
+    const texts = contentTexts
+        .map((s) => extractContentText(s, cmaps).trim())
+        .filter((t) => t !== '');
+    const title = findInfoString(stripped, 'Title') ?? '';
+    const pageMatches = stripped.match(/\/Type\s*\/Page(?![A-Za-z])/g);
+    return {
+        text: texts.join('\n\n'),
+        title,
+        pageCount: pageMatches !== null ? pageMatches.length : undefined,
+    };
+}

package/dist/node_modules/@boombot/core/research/providers/arxiv.d.ts

@@ -0,0 +1,9 @@
+/**
+ * arXiv search provider (free, no key).
+ *
+ * Uses the arXiv OAI-style query API (`export.arxiv.org/api/query`).
+ * The XML response is parsed with a small purpose-built parser — no XML
+ * dependency, no DOM. HTML entities in text nodes are decoded.
+ */
+import type { ResearchProvider } from '../types.js';
+export declare function createArxivProvider(): ResearchProvider;

package/dist/node_modules/@boombot/core/research/providers/arxiv.js

@@ -0,0 +1,72 @@
+/**
+ * arXiv search provider (free, no key).
+ *
+ * Uses the arXiv OAI-style query API (`export.arxiv.org/api/query`).
+ * The XML response is parsed with a small purpose-built parser — no XML
+ * dependency, no DOM. HTML entities in text nodes are decoded.
+ */
+import { fetchText, truncate, withRetry } from '../http.js';
+const MAX_RESULTS = 20;
+function decodeEntities(text) {
+    return text
+        .replace(/&/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'")
+        .replace(/&apos;/g, "'");
+}
+function collapseWhitespace(text) {
+    return decodeEntities(text).replace(/\s+/g, ' ').trim();
+}
+/** Extract the text of the first `<tag>…</tag>` block inside `xml`. */
+function tagText(xml, tag) {
+    const re = new RegExp(`<${tag}(?:\\s[^>]*)?>([\\s\\S]*?)<\\/${tag}>`);
+    const match = re.exec(xml);
+    return match?.[1] === undefined ? undefined : collapseWhitespace(match[1]);
+}
+function parseEntries(xml) {
+    const entries = [];
+    const entryRe = /<entry>([\s\S]*?)<\/entry>/g;
+    for (const match of xml.matchAll(entryRe)) {
+        const body = match[1] ?? '';
+        const title = tagText(body, 'title');
+        const id = tagText(body, 'id');
+        if (title === undefined || id === undefined)
+            continue;
+        entries.push({
+            title,
+            id,
+            summary: tagText(body, 'summary') ?? '',
+            published: tagText(body, 'published'),
+        });
+    }
+    return entries;
+}
+export function createArxivProvider() {
+    return {
+        id: 'arxiv',
+        kind: 'free',
+        async search(query, opts = {}) {
+            const limit = Math.min(Math.max(opts.limit ?? 8, 1), MAX_RESULTS);
+            const params = new URLSearchParams({
+                search_query: `all:${query}`,
+                start: '0',
+                max_results: String(limit),
+                sortBy: 'relevance',
+                sortOrder: 'descending',
+            });
+            const url = `https://export.arxiv.org/api/query?${params.toString()}`;
+            const xml = await withRetry(() => fetchText(url, { signal: opts.signal, timeoutMs: 15_000 }));
+            return parseEntries(xml)
+                .slice(0, limit)
+                .map((entry) => ({
+                title: entry.title,
+                url: entry.id.replace('http://', 'https://'),
+                snippet: truncate(entry.summary, 300),
+                source: 'arxiv',
+                publishedAt: entry.published,
+            }));
+        },
+    };
+}

package/dist/node_modules/@boombot/core/research/providers/brave.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Brave Search adapter (paid, API key required).
+ *
+ * Wraps the Brave Search API (`/res/v1/web/search`). The key travels in
+ * the `X-Subscription-Token` header — never in the URL, never logged.
+ */
+import type { ResearchProvider } from '../types.js';
+export interface BraveProviderOptions {
+    readonly apiKey: string;
+}
+export declare function createBraveProvider(options: BraveProviderOptions): ResearchProvider;