@theowlops/boombot 0.0.0-stage → 1.1.1

package/dist/node_modules/@boombot/core/research/crawl.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Multi-page crawl engine (`research.crawl`).
+ *
+ * The crawl is a BFS over the resolved EXTRACT backend — no separate crawl
+ * infrastructure and no async job polling: each page is one extract call,
+ * links come from the returned markdown, and the whole run shares the same
+ * `sanitizeExtractUrl` choke point as `research.extract`. Free backend is
+ * Jina Reader; a configured Firecrawl key upgrades per-page quality without
+ * changing this engine.
+ *
+ * Caps (hard max in the engine, the tool layer advertises the same numbers):
+ * depth ≤ 2, maxPages ≤ 10. Only same-site links are followed — the site
+ * boundary is the start URL's hostname with an optional leading `www.`
+ * stripped on both sides.
+ */
+import type { CrawlOpts, CrawlResult, ExtractProvider } from './types.js';
+export declare const CRAWL_HARD_MAX_PAGES = 10;
+export declare const CRAWL_HARD_MAX_DEPTH = 2;
+/**
+ * Extract absolute http(s) links from markdown, resolved against `baseUrl`.
+ * Fragment-only links (`#anchor`), non-http schemes and asset URLs are
+ * dropped. Duplicates are NOT removed here — the BFS dedupes by normalized
+ * URL across the whole run.
+ */
+export declare function extractLinks(markdown: string, baseUrl: string): string[];
+/**
+ * Run the BFS crawl over `extract`. Every URL — the start URL included —
+ * goes through `sanitizeExtractUrl` before an extract call, so a link found
+ * inside a page can never smuggle a non-http(s) scheme or embedded
+ * credentials to the backend.
+ *
+ * Levels are extracted in parallel (`Promise.allSettled`, candidate order
+ * preserved); a failed page lands in `failedUrls` and never aborts the run.
+ * Abort between fetches returns the partial result with `aborted: true`.
+ */
+export declare function crawlViaExtract(startUrl: string, extract: ExtractProvider, opts?: CrawlOpts): Promise<CrawlResult>;

package/dist/node_modules/@boombot/core/research/crawl.js

@@ -0,0 +1,179 @@
+/**
+ * Multi-page crawl engine (`research.crawl`).
+ *
+ * The crawl is a BFS over the resolved EXTRACT backend — no separate crawl
+ * infrastructure and no async job polling: each page is one extract call,
+ * links come from the returned markdown, and the whole run shares the same
+ * `sanitizeExtractUrl` choke point as `research.extract`. Free backend is
+ * Jina Reader; a configured Firecrawl key upgrades per-page quality without
+ * changing this engine.
+ *
+ * Caps (hard max in the engine, the tool layer advertises the same numbers):
+ * depth ≤ 2, maxPages ≤ 10. Only same-site links are followed — the site
+ * boundary is the start URL's hostname with an optional leading `www.`
+ * stripped on both sides.
+ */
+import { normalizeUrl } from './config.js';
+import { isHttpUrl, sanitizeExtractUrl } from './http.js';
+export const CRAWL_HARD_MAX_PAGES = 10;
+export const CRAWL_HARD_MAX_DEPTH = 2;
+const DEFAULT_MAX_PAGES = 5;
+const DEFAULT_DEPTH = 2;
+const DEFAULT_PER_PAGE_CHARS = 2000;
+/** Never follow links into binary/asset files — extract of those is noise. */
+const NON_PAGE_ASSET_RE = /\.(png|jpe?g|gif|webp|svg|ico|css|js|mjs|json|xml|rss|atom|zip|gz|tar|rar|7z|pdf|docx?|xlsx?|pptx?|mp[34]|webm|woff2?|ttf|eot)$/i;
+const MAX_FAILED_URLS = 10;
+/**
+ * Extract absolute http(s) links from markdown, resolved against `baseUrl`.
+ * Fragment-only links (`#anchor`), non-http schemes and asset URLs are
+ * dropped. Duplicates are NOT removed here — the BFS dedupes by normalized
+ * URL across the whole run.
+ */
+export function extractLinks(markdown, baseUrl) {
+    let base;
+    try {
+        base = new URL(baseUrl);
+    }
+    catch {
+        return [];
+    }
+    const out = [];
+    for (const match of markdown.matchAll(/\[[^\]]*\]\(([^)\s]+)[^)]*\)/g)) {
+        const href = match[1];
+        if (href === undefined || href === '')
+            continue;
+        if (href.startsWith('#'))
+            continue;
+        let resolved;
+        try {
+            resolved = new URL(href, base);
+        }
+        catch {
+            continue;
+        }
+        if (!isHttpUrl(resolved.toString()))
+            continue;
+        if (NON_PAGE_ASSET_RE.test(resolved.pathname))
+            continue;
+        out.push(resolved.toString());
+    }
+    return out;
+}
+/** Site identity: hostname lower-cased with a leading `www.` stripped. */
+function siteKey(url) {
+    const host = url.hostname.toLowerCase();
+    return host.startsWith('www.') ? host.slice(4) : host;
+}
+function isSameSite(a, b) {
+    try {
+        return siteKey(new URL(a)) === siteKey(new URL(b));
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Run the BFS crawl over `extract`. Every URL — the start URL included —
+ * goes through `sanitizeExtractUrl` before an extract call, so a link found
+ * inside a page can never smuggle a non-http(s) scheme or embedded
+ * credentials to the backend.
+ *
+ * Levels are extracted in parallel (`Promise.allSettled`, candidate order
+ * preserved); a failed page lands in `failedUrls` and never aborts the run.
+ * Abort between fetches returns the partial result with `aborted: true`.
+ */
+export async function crawlViaExtract(startUrl, extract, opts = {}) {
+    const maxPages = Math.min(Math.max(opts.maxPages ?? DEFAULT_MAX_PAGES, 1), CRAWL_HARD_MAX_PAGES);
+    const depthLimit = Math.min(Math.max(opts.depth ?? DEFAULT_DEPTH, 1), CRAWL_HARD_MAX_DEPTH);
+    const perPageChars = Math.min(Math.max(opts.perPageChars ?? DEFAULT_PER_PAGE_CHARS, 200), 8000);
+    const signal = opts.signal;
+    const safeStart = sanitizeExtractUrl(startUrl);
+    const pages = [];
+    const failedUrls = [];
+    const visited = new Set();
+    let depthReached = 0;
+    let aborted = false;
+    const startKey = normalizeUrl(safeStart);
+    if (startKey !== undefined)
+        visited.add(startKey);
+    const extractPage = async (url, depth) => {
+        const safeUrl = sanitizeExtractUrl(url);
+        const result = await extract.extract(safeUrl, signal);
+        const truncated = result.truncated || result.markdown.length > perPageChars;
+        return {
+            page: {
+                url: safeUrl,
+                title: result.title,
+                markdown: truncated
+                    ? result.markdown.slice(0, perPageChars)
+                    : result.markdown,
+                truncated,
+                depth,
+            },
+            // Link discovery reads the backend's own window (extract's
+            // MAX_CHARS), not the compact 2k slice the tool output shows.
+            fullMarkdown: result.markdown,
+        };
+    };
+    // Level 0: the start URL itself. A failure here is the whole crawl failing.
+    if (signal?.aborted) {
+        return { startUrl: safeStart, pages, depthReached, failedUrls, aborted: true };
+    }
+    const first = await extractPage(safeStart, 0);
+    pages.push(first.page);
+    depthReached = 0;
+    let frontier = extractLinks(first.fullMarkdown, safeStart);
+    // `depth` counts link-follow levels BEYOND the start page:
+    // depth 1 = start page only; depth 2 = follow its links once.
+    for (let depth = 1; depth < depthLimit; depth += 1) {
+        if (pages.length >= maxPages)
+            break;
+        if (signal?.aborted === true) {
+            aborted = true;
+            break;
+        }
+        // Candidates for this level: unseen, same-site, non-asset http(s) URLs.
+        const candidates = [];
+        for (const raw of frontier) {
+            if (candidates.length >= maxPages - pages.length)
+                break;
+            if (!isSameSite(safeStart, raw))
+                continue;
+            const key = normalizeUrl(raw);
+            if (key === undefined || visited.has(key))
+                continue;
+            visited.add(key);
+            candidates.push(raw);
+        }
+        if (candidates.length === 0)
+            break;
+        const settled = await Promise.allSettled(candidates.map((url) => extractPage(url, depth)));
+        const nextFrontier = [];
+        for (let i = 0; i < settled.length; i += 1) {
+            const outcome = settled[i];
+            const url = candidates[i];
+            if (outcome === undefined || url === undefined)
+                continue;
+            if (outcome.status === 'fulfilled') {
+                pages.push(outcome.value.page);
+                depthReached = Math.max(depthReached, depth);
+                if (pages.length < maxPages) {
+                    nextFrontier.push(...extractLinks(outcome.value.fullMarkdown, url));
+                }
+            }
+            else if (failedUrls.length < MAX_FAILED_URLS) {
+                failedUrls.push(url);
+            }
+        }
+        frontier = nextFrontier;
+    }
+    if (signal?.aborted === true)
+        aborted = true;
+    return {
+        startUrl: safeStart,
+        pages,
+        depthReached,
+        failedUrls,
+        aborted,
+    };
+}

package/dist/node_modules/@boombot/core/research/factory.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Composition root helpers: build provider instances from config.
+ *
+ * Paid providers are only constructed when their key is present; the
+ * registry (`resolveSearchProviders` / `resolveExtractProvider`) then
+ * orders and selects them. Keys flow in memory only.
+ */
+import type { ExtractProvider, ResearchConfig, ResearchProvider } from './types.js';
+/**
+ * Build search providers: paid instances only when their key is configured
+ * (tavily, exa, brave), plus the full free chain. Feed the result into
+ * `resolveSearchProviders(config, { paid, free })`.
+ */
+export declare function createSearchProviders(config: ResearchConfig): {
+    readonly paid: readonly ResearchProvider[];
+    readonly free: readonly ResearchProvider[];
+};
+/**
+ * Build extract providers: firecrawl only when its key is configured, jina
+ * always. Feed the result into
+ * `resolveExtractProvider(config, { firecrawl, jina })`.
+ */
+export declare function createExtractProviders(config: ResearchConfig): {
+    readonly firecrawl?: ExtractProvider;
+    readonly jina: ExtractProvider;
+};

package/dist/node_modules/@boombot/core/research/factory.js

@@ -0,0 +1,46 @@
+/**
+ * Composition root helpers: build provider instances from config.
+ *
+ * Paid providers are only constructed when their key is present; the
+ * registry (`resolveSearchProviders` / `resolveExtractProvider`) then
+ * orders and selects them. Keys flow in memory only.
+ */
+import { createArxivProvider, createBraveProvider, createDuckDuckGoProvider, createExaProvider, createFirecrawlProvider, createJinaProvider, createOpenAlexProvider, createSemanticScholarProvider, createTavilyProvider, createWikipediaProvider, } from './providers/index.js';
+/**
+ * Build search providers: paid instances only when their key is configured
+ * (tavily, exa, brave), plus the full free chain. Feed the result into
+ * `resolveSearchProviders(config, { paid, free })`.
+ */
+export function createSearchProviders(config) {
+    const paid = [];
+    if (config.tavilyApiKey !== undefined) {
+        paid.push(createTavilyProvider({ apiKey: config.tavilyApiKey }));
+    }
+    if (config.exaApiKey !== undefined) {
+        paid.push(createExaProvider({ apiKey: config.exaApiKey }));
+    }
+    if (config.braveApiKey !== undefined) {
+        paid.push(createBraveProvider({ apiKey: config.braveApiKey }));
+    }
+    const free = [
+        createWikipediaProvider(),
+        createDuckDuckGoProvider(),
+        createArxivProvider(),
+        createOpenAlexProvider(),
+        createSemanticScholarProvider(),
+    ];
+    return { paid, free };
+}
+/**
+ * Build extract providers: firecrawl only when its key is configured, jina
+ * always. Feed the result into
+ * `resolveExtractProvider(config, { firecrawl, jina })`.
+ */
+export function createExtractProviders(config) {
+    return {
+        firecrawl: config.firecrawlApiKey !== undefined
+            ? createFirecrawlProvider({ apiKey: config.firecrawlApiKey })
+            : undefined,
+        jina: createJinaProvider(),
+    };
+}

package/dist/node_modules/@boombot/core/research/http.d.ts

@@ -0,0 +1,87 @@
+/**
+ * Shared HTTP helpers for all research providers.
+ *
+ * Philosophy: hand-rolled on the global `fetch` (Node 22), zero runtime
+ * dependencies, no vendor SDKs. Every network failure surfaces as a typed
+ * {@link ResearchError} so callers (and `withRetry`) can branch on `code`.
+ */
+export declare const DEFAULT_USER_AGENT = "BoomBotResearch/1.0 (+https://github.com/TheBoomBot/BoomBot)";
+export declare const DEFAULT_TIMEOUT_MS = 10000;
+/** Machine-readable failure codes. Additive-first: new codes are added, never renamed. */
+export type ResearchErrorCode = 'TIMEOUT' | 'ABORTED' | 'HTTP_4XX' | 'HTTP_5XX' | 'NETWORK_ERROR' | 'PARSE_ERROR' | 'INVALID_URL' | 'RESPONSE_TOO_LARGE';
+export declare class ResearchError extends Error {
+    readonly code: ResearchErrorCode;
+    constructor(code: ResearchErrorCode, message: string, options?: ErrorOptions);
+}
+export interface FetchOptions {
+    readonly signal?: AbortSignal;
+    readonly headers?: Record<string, string>;
+    /** Per-request timeout in ms. Default {@link DEFAULT_TIMEOUT_MS}. */
+    readonly timeoutMs?: number;
+    /**
+     * Hard cap on response body bytes. Default {@link DEFAULT_MAX_RESPONSE_BYTES}.
+     * A malicious or misbehaving server must not exhaust gateway memory with
+     * one unbounded response.
+     */
+    readonly maxBytes?: number;
+}
+/** Default hard cap for a single response body: 5 MiB. */
+export declare const DEFAULT_MAX_RESPONSE_BYTES = 5242880;
+/** Max length accepted for a user-supplied extract URL. */
+export declare const MAX_EXTRACT_URL_LENGTH = 2048;
+export interface RetryOptions {
+    /** Number of retries after the first attempt. Default 1. */
+    readonly retries?: number;
+}
+/** Truncation marker appended by {@link truncate}. */
+export declare const TRUNCATED_MARKER = " [truncated]";
+/**
+ * Validate and sanitize a user-supplied URL before it is handed to an
+ * extract backend (`research.extract`, `research.deep`).
+ *
+ * - Must parse and use `http:`/`https:` — `file:`, `data:`, `javascript:`
+ *   and friends are rejected outright.
+ * - Embedded credentials (`https://user:pass@host/…`) are STRIPPED: the
+ *   URL is forwarded to a third-party reader (Jina/Firecrawl), and a
+ *   secret in the userinfo would leak to that third party verbatim.
+ * - Length cap against parse/log abuse.
+ *
+ * Returns the sanitized URL string. Throws {@link ResearchError} with
+ * code `INVALID_URL` when the input is not an acceptable extract target.
+ */
+export declare function sanitizeExtractUrl(rawUrl: string): string;
+/**
+ * Non-throwing check used for pre-filtering candidate URLs.
+ * Full validation (with sanitization) still happens in
+ * {@link sanitizeExtractUrl} at the extract choke point.
+ */
+export declare function isHttpUrl(value: string): boolean;
+/**
+ * Fetch `url` and return the body as text.
+ * Throws {@link ResearchError} with code `TIMEOUT` / `ABORTED` /
+ * `HTTP_4XX` / `HTTP_5XX` / `NETWORK_ERROR` / `RESPONSE_TOO_LARGE`.
+ * Never logs request details.
+ */
+export declare function fetchText(url: string, opts?: FetchOptions): Promise<string>;
+/**
+ * Fetch `url` and parse the body as JSON.
+ * Throws {@link ResearchError} with code `PARSE_ERROR` when the body is not
+ * valid JSON, plus every error {@link fetchText} can throw.
+ */
+export declare function fetchJson(url: string, opts?: FetchOptions): Promise<unknown>;
+/**
+ * POST `body` as JSON to `url` and parse the JSON response.
+ * Throws the same {@link ResearchError} codes as {@link fetchText} /
+ * {@link fetchJson}. Request bodies are never logged.
+ */
+export declare function postJson(url: string, body: unknown, opts?: FetchOptions): Promise<unknown>;
+/**
+ * Run `fn`, retrying once (default) when it fails with a transient
+ * {@link ResearchError} (`NETWORK_ERROR` — see {@link RETRYABLE}). `TIMEOUT`,
+ * `HTTP_4XX`, `ABORTED`, `PARSE_ERROR` and non-`ResearchError` failures are
+ * rethrown immediately: retrying them cannot help, and a retried timeout
+ * just doubles the caller's wait.
+ */
+export declare function withRetry<T>(fn: () => Promise<T>, opts?: RetryOptions): Promise<T>;
+/** Cut `text` to `maxChars` and append {@link TRUNCATED_MARKER}. */
+export declare function truncate(text: string, maxChars: number): string;

package/dist/node_modules/@boombot/core/research/http.js

@@ -0,0 +1,250 @@
+/**
+ * Shared HTTP helpers for all research providers.
+ *
+ * Philosophy: hand-rolled on the global `fetch` (Node 22), zero runtime
+ * dependencies, no vendor SDKs. Every network failure surfaces as a typed
+ * {@link ResearchError} so callers (and `withRetry`) can branch on `code`.
+ */
+export const DEFAULT_USER_AGENT = 'BoomBotResearch/1.0 (+https://github.com/TheBoomBot/BoomBot)';
+export const DEFAULT_TIMEOUT_MS = 10_000;
+export class ResearchError extends Error {
+    code;
+    constructor(code, message, options) {
+        super(message, options);
+        this.name = 'ResearchError';
+        this.code = code;
+    }
+}
+/** Default hard cap for a single response body: 5 MiB. */
+export const DEFAULT_MAX_RESPONSE_BYTES = 5_242_880;
+/** Max length accepted for a user-supplied extract URL. */
+export const MAX_EXTRACT_URL_LENGTH = 2048;
+/**
+ * Errors worth one more attempt.
+ *
+ * `TIMEOUT` is deliberately NOT here. Retrying a timeout doubles the wait for
+ * the caller — 10s became 20s, 15s became 30s — and the retry runs while the
+ * user is sitting on a tool call, which is where latency is felt most. A
+ * timeout says the engine is slow or unresponsive, and a second identical
+ * request rarely changes that; `NETWORK_ERROR` is the genuinely transient
+ * case a retry is for.
+ */
+const RETRYABLE = new Set(['NETWORK_ERROR']);
+/** Truncation marker appended by {@link truncate}. */
+export const TRUNCATED_MARKER = ' [truncated]';
+function toResearchError(err, url) {
+    if (err instanceof DOMException && err.name === 'TimeoutError') {
+        return new ResearchError('TIMEOUT', `Request to ${url} timed out`);
+    }
+    if (err instanceof DOMException && err.name === 'AbortError') {
+        return new ResearchError('ABORTED', `Request to ${url} was aborted`);
+    }
+    const detail = err instanceof Error ? err.message : String(err);
+    return new ResearchError('NETWORK_ERROR', `Network error fetching ${url}: ${detail}`);
+}
+/**
+ * Read the response body as text with a hard byte cap.
+ *
+ * Streams the body and aborts the moment it exceeds `maxBytes` — the
+ * unbounded `response.text()` it replaces would buffer an arbitrarily
+ * large body into memory first (memory-DoS via a hostile server).
+ */
+async function readCappedText(response, url, maxBytes) {
+    const body = response.body;
+    if (body === null)
+        return '';
+    const reader = body.getReader();
+    const chunks = [];
+    let bytes = 0;
+    for (;;) {
+        let read;
+        try {
+            read = await reader.read();
+        }
+        catch (err) {
+            throw toResearchError(err, url);
+        }
+        if (read.done)
+            break;
+        bytes += read.value.byteLength;
+        if (bytes > maxBytes) {
+            await reader.cancel().catch(() => undefined);
+            throw new ResearchError('RESPONSE_TOO_LARGE', `Response from ${url} exceeds ${maxBytes} bytes`);
+        }
+        chunks.push(read.value);
+    }
+    return Buffer.concat(chunks).toString('utf8');
+}
+/**
+ * Validate and sanitize a user-supplied URL before it is handed to an
+ * extract backend (`research.extract`, `research.deep`).
+ *
+ * - Must parse and use `http:`/`https:` — `file:`, `data:`, `javascript:`
+ *   and friends are rejected outright.
+ * - Embedded credentials (`https://user:pass@host/…`) are STRIPPED: the
+ *   URL is forwarded to a third-party reader (Jina/Firecrawl), and a
+ *   secret in the userinfo would leak to that third party verbatim.
+ * - Length cap against parse/log abuse.
+ *
+ * Returns the sanitized URL string. Throws {@link ResearchError} with
+ * code `INVALID_URL` when the input is not an acceptable extract target.
+ */
+export function sanitizeExtractUrl(rawUrl) {
+    const trimmed = rawUrl.trim();
+    if (trimmed === '') {
+        throw new ResearchError('INVALID_URL', 'Extract URL must not be empty.');
+    }
+    if (trimmed.length > MAX_EXTRACT_URL_LENGTH) {
+        throw new ResearchError('INVALID_URL', `Extract URL exceeds ${MAX_EXTRACT_URL_LENGTH} characters.`);
+    }
+    let parsed;
+    try {
+        parsed = new URL(trimmed);
+    }
+    catch {
+        throw new ResearchError('INVALID_URL', 'Extract URL is not a valid URL.');
+    }
+    if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+        throw new ResearchError('INVALID_URL', `Extract URL must use http(s), got '${parsed.protocol}'.`);
+    }
+    // Never forward embedded credentials to a third-party extract service.
+    // Only rebuild the URL when credentials were actually present, so
+    // credential-free URLs pass through byte-identical (no normalization
+    // side effects for existing callers).
+    if (parsed.username !== '' || parsed.password !== '') {
+        parsed.username = '';
+        parsed.password = '';
+        return parsed.toString();
+    }
+    return trimmed;
+}
+/**
+ * Non-throwing check used for pre-filtering candidate URLs.
+ * Full validation (with sanitization) still happens in
+ * {@link sanitizeExtractUrl} at the extract choke point.
+ */
+export function isHttpUrl(value) {
+    try {
+        const parsed = new URL(value.trim());
+        return parsed.protocol === 'http:' || parsed.protocol === 'https:';
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Fetch `url` and return the body as text.
+ * Throws {@link ResearchError} with code `TIMEOUT` / `ABORTED` /
+ * `HTTP_4XX` / `HTTP_5XX` / `NETWORK_ERROR` / `RESPONSE_TOO_LARGE`.
+ * Never logs request details.
+ */
+export async function fetchText(url, opts = {}) {
+    const timeoutSignal = AbortSignal.timeout(opts.timeoutMs ?? DEFAULT_TIMEOUT_MS);
+    const signal = opts.signal !== undefined
+        ? AbortSignal.any([opts.signal, timeoutSignal])
+        : timeoutSignal;
+    let response;
+    try {
+        response = await fetch(url, {
+            signal,
+            headers: { 'User-Agent': DEFAULT_USER_AGENT, ...opts.headers },
+        });
+    }
+    catch (err) {
+        throw toResearchError(err, url);
+    }
+    if (!response.ok) {
+        const code = response.status >= 400 && response.status < 500 ? 'HTTP_4XX' : 'HTTP_5XX';
+        throw new ResearchError(code, `HTTP ${response.status} from ${url}`);
+    }
+    // Capped streaming read — a hostile server cannot force an unbounded
+    // buffer into gateway memory. Read errors surface via toResearchError
+    // inside readCappedText; RESPONSE_TOO_LARGE propagates unchanged.
+    return readCappedText(response, url, opts.maxBytes ?? DEFAULT_MAX_RESPONSE_BYTES);
+}
+/**
+ * Fetch `url` and parse the body as JSON.
+ * Throws {@link ResearchError} with code `PARSE_ERROR` when the body is not
+ * valid JSON, plus every error {@link fetchText} can throw.
+ */
+export async function fetchJson(url, opts = {}) {
+    const text = await fetchText(url, opts);
+    try {
+        return JSON.parse(text);
+    }
+    catch (err) {
+        const detail = err instanceof Error ? err.message : String(err);
+        throw new ResearchError('PARSE_ERROR', `Invalid JSON from ${url}: ${detail}`);
+    }
+}
+/**
+ * POST `body` as JSON to `url` and parse the JSON response.
+ * Throws the same {@link ResearchError} codes as {@link fetchText} /
+ * {@link fetchJson}. Request bodies are never logged.
+ */
+export async function postJson(url, body, opts = {}) {
+    const timeoutSignal = AbortSignal.timeout(opts.timeoutMs ?? DEFAULT_TIMEOUT_MS);
+    const signal = opts.signal !== undefined
+        ? AbortSignal.any([opts.signal, timeoutSignal])
+        : timeoutSignal;
+    let response;
+    try {
+        response = await fetch(url, {
+            method: 'POST',
+            signal,
+            headers: {
+                'User-Agent': DEFAULT_USER_AGENT,
+                'Content-Type': 'application/json',
+                ...opts.headers,
+            },
+            body: JSON.stringify(body),
+        });
+    }
+    catch (err) {
+        throw toResearchError(err, url);
+    }
+    if (!response.ok) {
+        const code = response.status >= 400 && response.status < 500 ? 'HTTP_4XX' : 'HTTP_5XX';
+        throw new ResearchError(code, `HTTP ${response.status} from ${url}`);
+    }
+    const text = await readCappedText(response, url, opts.maxBytes ?? DEFAULT_MAX_RESPONSE_BYTES);
+    try {
+        return JSON.parse(text);
+    }
+    catch (err) {
+        const detail = err instanceof Error ? err.message : String(err);
+        throw new ResearchError('PARSE_ERROR', `Invalid JSON from ${url}: ${detail}`);
+    }
+}
+/**
+ * Run `fn`, retrying once (default) when it fails with a transient
+ * {@link ResearchError} (`NETWORK_ERROR` — see {@link RETRYABLE}). `TIMEOUT`,
+ * `HTTP_4XX`, `ABORTED`, `PARSE_ERROR` and non-`ResearchError` failures are
+ * rethrown immediately: retrying them cannot help, and a retried timeout
+ * just doubles the caller's wait.
+ */
+export async function withRetry(fn, opts = {}) {
+    const retries = opts.retries ?? 1;
+    let lastError;
+    for (let attempt = 0; attempt <= retries; attempt += 1) {
+        try {
+            return await fn();
+        }
+        catch (err) {
+            lastError = err;
+            if (attempt >= retries)
+                break;
+            if (!(err instanceof ResearchError) || !RETRYABLE.has(err.code)) {
+                throw err;
+            }
+            // else: fall through to the next attempt
+        }
+    }
+    throw lastError;
+}
+/** Cut `text` to `maxChars` and append {@link TRUNCATED_MARKER}. */
+export function truncate(text, maxChars) {
+    if (text.length <= maxChars)
+        return text;
+    return text.slice(0, maxChars) + TRUNCATED_MARKER;
+}

package/dist/node_modules/@boombot/core/research/index.d.ts

@@ -0,0 +1,17 @@
+/** `@boombot/core` — free-first web research for BoomBot. */
+export type { CrawlOpts, CrawlPage, CrawlResult, ExtractOpts, ExtractResult, ExtractProvider, ResearchConfig, ResearchProvider, SearchOpts, SearchResult, } from './types.js';
+export { dedupeByUrl, describeResearchConfig, isConfigured, loadResearchConfig, normalizeUrl, resolveExtractProvider, resolveSearchProviders, } from './config.js';
+export { CRAWL_HARD_MAX_DEPTH, CRAWL_HARD_MAX_PAGES, crawlViaExtract, extractLinks, } from './crawl.js';
+export { DEFAULT_MAX_RESPONSE_BYTES, DEFAULT_TIMEOUT_MS, DEFAULT_USER_AGENT, fetchJson, fetchText, isHttpUrl, MAX_EXTRACT_URL_LENGTH, postJson, ResearchError, sanitizeExtractUrl, TRUNCATED_MARKER, truncate, withRetry, } from './http.js';
+export type { FetchOptions, ResearchErrorCode, RetryOptions } from './http.js';
+export { createArxivProvider, createBraveProvider, createDuckDuckGoProvider, createExaProvider, createFirecrawlProvider, createJinaProvider, createOpenAlexProvider, createSemanticScholarProvider, createTavilyProvider, createWikipediaProvider, } from './providers/index.js';
+export type { BraveProviderOptions, ExaProviderOptions, FirecrawlProviderOptions, OpenAlexProviderOptions, TavilyProviderOptions, WikipediaProviderOptions, } from './providers/index.js';
+export { academicSearch, ResearchOrchestrator, searchChain, } from './orchestrator.js';
+export type { DeepResearchOptions, DeepResearchResult, OrchestratorDeps, SourceAttribution, } from './orchestrator.js';
+export { createExtractProviders, createSearchProviders } from './factory.js';
+export { DOCX_MIME, DocumentParseError, PDF_MIME, detectDocumentKindFromUrl, extractDocument, parseDocument, sanitizeExtractedText, sniffDocumentType, } from './parse/index.js';
+export type { DocxExtractedText, DocumentParseErrorCode, ParsedDocument, PdfCMap, PdfExtractedText, SupportedDocumentKind, } from './parse/index.js';
+export { registerResearchTools } from './wiring.js';
+export type { ResearchToolRegistry } from './wiring.js';
+export { createResearchToolExecutor, RESEARCH_ACADEMIC_TOOL, RESEARCH_CRAWL_TOOL, RESEARCH_DEEP_TOOL, RESEARCH_EXTRACT_TOOL, RESEARCH_SEARCH_TOOL, researchToolDefinitions, } from './tools.js';
+export type { ResearchToolExecutorDeps } from './tools.js';

package/dist/node_modules/@boombot/core/research/index.js

@@ -0,0 +1,10 @@
+/** `@boombot/core` — free-first web research for BoomBot. */
+export { dedupeByUrl, describeResearchConfig, isConfigured, loadResearchConfig, normalizeUrl, resolveExtractProvider, resolveSearchProviders, } from './config.js';
+export { CRAWL_HARD_MAX_DEPTH, CRAWL_HARD_MAX_PAGES, crawlViaExtract, extractLinks, } from './crawl.js';
+export { DEFAULT_MAX_RESPONSE_BYTES, DEFAULT_TIMEOUT_MS, DEFAULT_USER_AGENT, fetchJson, fetchText, isHttpUrl, MAX_EXTRACT_URL_LENGTH, postJson, ResearchError, sanitizeExtractUrl, TRUNCATED_MARKER, truncate, withRetry, } from './http.js';
+export { createArxivProvider, createBraveProvider, createDuckDuckGoProvider, createExaProvider, createFirecrawlProvider, createJinaProvider, createOpenAlexProvider, createSemanticScholarProvider, createTavilyProvider, createWikipediaProvider, } from './providers/index.js';
+export { academicSearch, ResearchOrchestrator, searchChain, } from './orchestrator.js';
+export { createExtractProviders, createSearchProviders } from './factory.js';
+export { DOCX_MIME, DocumentParseError, PDF_MIME, detectDocumentKindFromUrl, extractDocument, parseDocument, sanitizeExtractedText, sniffDocumentType, } from './parse/index.js';
+export { registerResearchTools } from './wiring.js';
+export { createResearchToolExecutor, RESEARCH_ACADEMIC_TOOL, RESEARCH_CRAWL_TOOL, RESEARCH_DEEP_TOOL, RESEARCH_EXTRACT_TOOL, RESEARCH_SEARCH_TOOL, researchToolDefinitions, } from './tools.js';