@steel-dev/atlas 0.1.0

package/dist/providers/fetch.js

@@ -0,0 +1,675 @@
+import { sleep, withTimeout } from "../async.js";
+import { readEnv } from "../env.js";
+import { errorMessage } from "../errors.js";
+import { htmlToMarkdown } from "../html-extract.js";
+import { extractPdfText } from "../pdf-extract.js";
+import { createRobotsCache } from "../robots.js";
+import { guardRedirect as guardRedirectUrl } from "../safety.js";
+import { extractionMetadataFromExa, extractionMetadataFromHtml, extractionMetadataFromPdf, extractionMetadataFromScrape, extractionMetadataFromText, } from "../source-documents.js";
+const DIRECT_PDF_MAX_BYTES = 25 * 1024 * 1024;
+const DIRECT_HTML_MAX_BYTES = 5 * 1024 * 1024;
+const DIRECT_FETCH_TIMEOUT_MS = 15_000;
+const SCRAPE_TIMEOUT_MS = 30_000;
+const DIRECT_HTML_MIN_CHARS = 100;
+const PDF_MAGIC = "%PDF";
+const FETCH_USER_AGENT = "Mozilla/5.0 (compatible; AtlasResearchBot/0.2; +https://github.com/steel-experiments/atlas)";
+const ROBOTS_AGENT_TOKEN = "atlasresearchbot";
+const STEEL_RETRY_MAX_ATTEMPTS = 5;
+const ANTI_BOT_MARKERS = [
+    "just a moment",
+    "verifying you are human",
+    "checking your browser",
+    "enable javascript and cookies",
+    "access denied",
+    "captcha",
+    "pardon our interruption",
+    "unusual traffic from your computer network",
+];
+export function looksBlocked(text) {
+    if (!text)
+        return false;
+    const lower = text.toLowerCase().slice(0, 4000);
+    return ANTI_BOT_MARKERS.some((marker) => lower.includes(marker));
+}
+const BLOCK_MARKER_EXEMPT_CHARS = 2_000;
+export function looksBlockedPage(markdown, raw) {
+    if (markdown.length >= BLOCK_MARKER_EXEMPT_CHARS)
+        return false;
+    return looksBlocked(markdown) || looksBlocked(raw);
+}
+function failed(method, note, escalate = true) {
+    return { ok: false, attempt: { method, ok: false, note }, escalate };
+}
+export function isLikelyPdfUrl(url) {
+    try {
+        const parsed = new URL(url);
+        return /\.pdf$/i.test(parsed.pathname);
+    }
+    catch {
+        return /\.pdf(?:$|[?#])/i.test(url);
+    }
+}
+function isPdfContentType(contentType) {
+    return /\bapplication\/pdf\b/i.test(contentType ?? "");
+}
+function isHtmlContentType(contentType) {
+    if (!contentType)
+        return true;
+    return /\b(?:text\/html|application\/xhtml\+xml)\b/i.test(contentType);
+}
+function isJsonContentType(contentType) {
+    return /\b(?:application|text)\/(?:[\w.+-]+\+)?json\b/i.test(contentType ?? "");
+}
+function isXmlContentType(contentType) {
+    return /\b(?:application|text)\/(?:[\w.+-]+\+)?xml\b/i.test(contentType ?? "");
+}
+function isPlainTextContentType(contentType) {
+    return /\btext\/(?:plain|csv|tab-separated-values|markdown)\b/i.test(contentType ?? "");
+}
+function isDirectTextContentType(contentType) {
+    return (isJsonContentType(contentType) ||
+        isXmlContentType(contentType) ||
+        isPlainTextContentType(contentType));
+}
+function looksLikeDirectText(data) {
+    const prefix = new TextDecoder("utf-8", { fatal: false })
+        .decode(data.slice(0, 512))
+        .trimStart();
+    return (prefix.startsWith("{") ||
+        prefix.startsWith("[") ||
+        prefix.startsWith("<?xml") ||
+        /^PMID-|\b[A-Z]{2,}-\s/.test(prefix));
+}
+function isPdfBytes(data) {
+    if (data.byteLength < PDF_MAGIC.length)
+        return false;
+    return (new TextDecoder("ascii").decode(data.slice(0, PDF_MAGIC.length)) ===
+        PDF_MAGIC);
+}
+function readContentLength(headers) {
+    const raw = headers.get("content-length");
+    if (!raw)
+        return undefined;
+    const n = Number(raw);
+    return Number.isFinite(n) && n >= 0 ? n : undefined;
+}
+function titleFromUrl(url) {
+    try {
+        const parsed = new URL(url);
+        const filename = decodeURIComponent(parsed.pathname.split("/").filter(Boolean).at(-1) ?? "");
+        return filename || parsed.hostname || url;
+    }
+    catch {
+        return url;
+    }
+}
+function normalizeDirectText(text, contentType) {
+    const trimmed = text.trim();
+    if (isJsonContentType(contentType) || /^[[{]/.test(trimmed)) {
+        try {
+            return {
+                markdown: JSON.stringify(JSON.parse(trimmed), null, 2),
+                method: "json_direct",
+            };
+        }
+        catch {
+            return { markdown: trimmed, method: "text_direct" };
+        }
+    }
+    if (isXmlContentType(contentType) || /^<\??xml\b/i.test(trimmed)) {
+        return { markdown: trimmed, method: "xml_direct" };
+    }
+    return { markdown: trimmed, method: "text_direct" };
+}
+export function basicFetch() {
+    const robots = createRobotsCache({
+        agentToken: ROBOTS_AGENT_TOKEN,
+        userAgent: FETCH_USER_AGENT,
+    });
+    const domainTails = new Map();
+    const serialize = (host, task) => {
+        const prev = domainTails.get(host) ?? Promise.resolve();
+        const next = prev.then(task, task);
+        const tail = next.then(() => { }, () => { });
+        domainTails.set(host, tail);
+        void tail.then(() => {
+            if (domainTails.get(host) === tail)
+                domainTails.delete(host);
+        });
+        return next;
+    };
+    return {
+        id: "basic",
+        fetch(req) {
+            let host;
+            try {
+                host = new URL(req.url).host.toLowerCase();
+            }
+            catch {
+                return Promise.resolve(failed("direct_http", `bad_url: not a valid URL: ${req.url}`, false));
+            }
+            return serialize(host, () => directFetch(req, robots));
+        },
+    };
+}
+class ResponseTooLargeError extends Error {
+}
+async function readCappedBytes(response, maxBytes) {
+    const body = response.body;
+    if (!body) {
+        const buf = new Uint8Array(await response.arrayBuffer());
+        if (buf.byteLength > maxBytes)
+            throw new ResponseTooLargeError();
+        return buf;
+    }
+    const reader = body.getReader();
+    const chunks = [];
+    let total = 0;
+    try {
+        for (;;) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            if (!value)
+                continue;
+            total += value.byteLength;
+            if (total > maxBytes)
+                throw new ResponseTooLargeError();
+            chunks.push(value);
+        }
+    }
+    finally {
+        await reader.cancel().catch(() => { });
+    }
+    const out = new Uint8Array(total);
+    let offset = 0;
+    for (const chunk of chunks) {
+        out.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return out;
+}
+const MAX_REDIRECT_HOPS = 5;
+async function directFetch({ url, signal, guardRedirect, dispatcher }, robots) {
+    if (!guardRedirect) {
+        const initial = await guardRedirectUrl(url, {});
+        if (!initial.ok) {
+            return failed("direct_http", `blocked_url: fetch of ${url} blocked: ${initial.reason}`, false);
+        }
+    }
+    let currentUrl = url;
+    let response;
+    for (let hop = 0;; hop++) {
+        if (!(await robots.allows(currentUrl, signal, dispatcher))) {
+            return failed("direct_http", "robots_disallowed: robots.txt disallows direct fetching of this URL");
+        }
+        try {
+            const timeout = AbortSignal.timeout(DIRECT_FETCH_TIMEOUT_MS);
+            response = await fetch(currentUrl, {
+                redirect: "manual",
+                signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
+                headers: {
+                    accept: "application/pdf,*/*;q=0.8",
+                    "user-agent": FETCH_USER_AGENT,
+                },
+                ...(dispatcher ? { dispatcher } : {}),
+            });
+        }
+        catch (err) {
+            if (signal?.aborted)
+                throw err;
+            return failed("direct_http", `network_error: direct fetch failed: ${errorMessage(err)}`);
+        }
+        const location = response.headers.get("location");
+        if (response.status >= 300 && response.status < 400 && location) {
+            await response.body?.cancel().catch(() => { });
+            if (hop >= MAX_REDIRECT_HOPS) {
+                return failed("direct_http", `too_many_redirects: gave up after ${MAX_REDIRECT_HOPS} redirects`);
+            }
+            let next;
+            try {
+                next = new URL(location, currentUrl).toString();
+            }
+            catch {
+                return failed("direct_http", `bad_redirect: invalid redirect location: ${location}`, false);
+            }
+            const verdict = guardRedirect
+                ? await guardRedirect(next)
+                : await guardRedirectUrl(next, {});
+            if (!verdict.ok) {
+                return failed("direct_http", `blocked_redirect: redirect to ${next} blocked: ${verdict.reason}`, false);
+            }
+            currentUrl = next;
+            continue;
+        }
+        break;
+    }
+    if (!response.ok) {
+        return failed("direct_http", `http_error: direct fetch returned HTTP ${response.status}`);
+    }
+    const contentType = response.headers.get("content-type") ?? undefined;
+    const contentLength = readContentLength(response.headers);
+    const maxBytes = isLikelyPdfUrl(currentUrl) || isPdfContentType(contentType)
+        ? DIRECT_PDF_MAX_BYTES
+        : DIRECT_HTML_MAX_BYTES;
+    if (contentLength !== undefined && contentLength > maxBytes) {
+        return failed("direct_http", `too_large: direct response is too large (${contentLength} bytes)`, false);
+    }
+    let data;
+    try {
+        data = await readCappedBytes(response, maxBytes);
+    }
+    catch (err) {
+        if (signal?.aborted)
+            throw err;
+        if (err instanceof ResponseTooLargeError) {
+            return failed("direct_http", `too_large: direct response exceeded ${maxBytes} bytes`, false);
+        }
+        return failed("direct_http", `network_error: reading response failed: ${errorMessage(err)}`);
+    }
+    const finalUrl = response.url || currentUrl;
+    if (isPdfBytes(data) || isPdfContentType(contentType)) {
+        return extractPdf(data, contentType, finalUrl, signal);
+    }
+    if (!contentType && looksLikeDirectText(data)) {
+        return extractText(data, contentType, finalUrl);
+    }
+    if (isHtmlContentType(contentType)) {
+        return extractHtml(data, contentType, finalUrl);
+    }
+    if (isDirectTextContentType(contentType) || looksLikeDirectText(data)) {
+        return extractText(data, contentType, finalUrl);
+    }
+    return failed("direct_http", contentType
+        ? `unsupported_content_type: direct response was ${contentType}`
+        : "unsupported_content_type: direct response was not HTML, PDF, JSON, XML, or text", false);
+}
+function extractHtml(data, contentType, finalUrl) {
+    const html = new TextDecoder("utf-8", { fatal: false }).decode(data);
+    const extracted = htmlToMarkdown(html, finalUrl);
+    if (looksBlockedPage(extracted.markdown, html)) {
+        return failed("html_direct", "blocked_or_challenge: direct HTML looked blocked");
+    }
+    if (extracted.markdown.length < DIRECT_HTML_MIN_CHARS) {
+        return failed("html_direct", `thin_content: direct HTML extracted ${extracted.markdown.length} chars`);
+    }
+    const attempt = {
+        method: "html_direct",
+        ok: true,
+        note: `html_direct: extracted ${extracted.markdown.length} text chars`,
+    };
+    return {
+        ok: true,
+        attempt,
+        page: {
+            finalUrl,
+            title: extracted.title,
+            markdown: extracted.markdown,
+            renderedWith: "html_direct",
+            metadata: extractionMetadataFromHtml({
+                markdownChars: extracted.markdown.length,
+                ...(contentType ? { contentType } : {}),
+                finalUrl,
+                attempts: [attempt],
+                discoveredLinks: extracted.links,
+                pageMetadata: extracted.metadata,
+            }),
+        },
+    };
+}
+const PDF_PARSE_TIMEOUT_MS = 30_000;
+async function extractPdf(data, contentType, finalUrl, signal) {
+    try {
+        const extracted = await withTimeout(PDF_PARSE_TIMEOUT_MS, signal, "pdf_parse", () => extractPdfText(data));
+        const markdown = extracted.text.trim();
+        if (!markdown) {
+            return failed("pdf_direct", "pdf_no_text: PDF extraction produced no text");
+        }
+        const attempt = {
+            method: "pdf_direct",
+            ok: true,
+            note: `pdf_direct: extracted ${markdown.length} text chars`,
+        };
+        return {
+            ok: true,
+            attempt,
+            page: {
+                finalUrl,
+                title: titleFromUrl(finalUrl),
+                markdown,
+                renderedWith: "pdf_direct",
+                metadata: extractionMetadataFromPdf({
+                    markdownChars: markdown.length,
+                    ...(contentType ? { contentType } : {}),
+                    finalUrl,
+                    attempts: [attempt],
+                }),
+            },
+        };
+    }
+    catch (err) {
+        return failed("pdf_direct", `pdf_parse_error: PDF extraction failed: ${errorMessage(err)}`);
+    }
+}
+function extractText(data, contentType, finalUrl) {
+    const decoded = new TextDecoder("utf-8", { fatal: false }).decode(data);
+    const parsed = normalizeDirectText(decoded, contentType);
+    const markdown = parsed.markdown.trim();
+    if (looksBlockedPage(markdown, decoded)) {
+        return failed("text_direct", "blocked_or_challenge: direct text looked blocked");
+    }
+    if (markdown.length < DIRECT_HTML_MIN_CHARS) {
+        return failed(parsed.method, `thin_content: direct text extracted ${markdown.length} chars`);
+    }
+    const attempt = {
+        method: parsed.method,
+        ok: true,
+        note: `${parsed.method}: extracted ${markdown.length} text chars`,
+    };
+    return {
+        ok: true,
+        attempt,
+        page: {
+            finalUrl,
+            title: titleFromUrl(finalUrl),
+            markdown,
+            renderedWith: parsed.method,
+            metadata: extractionMetadataFromText({
+                markdownChars: markdown.length,
+                method: parsed.method,
+                ...(contentType ? { contentType } : {}),
+                finalUrl,
+                attempts: [attempt],
+            }),
+        },
+    };
+}
+function parseRetryAfterSeconds(err) {
+    const status = err?.status;
+    const message = errorMessage(err);
+    if (status !== 429 &&
+        !/(rate limit exceeded|too many requests)/i.test(message)) {
+        return null;
+    }
+    const headers = err?.headers;
+    let headerValue;
+    if (headers && typeof headers.get === "function") {
+        headerValue =
+            headers.get("retry-after") ??
+                undefined;
+    }
+    if (headerValue) {
+        const numeric = Number(headerValue);
+        if (Number.isFinite(numeric) && numeric > 0)
+            return Math.ceil(numeric);
+    }
+    const match = /try again in\s+(\d+(?:\.\d+)?)\s*seconds?/i.exec(message);
+    if (match)
+        return Math.ceil(Number(match[1]));
+    return 15;
+}
+export function steel(opts = {}) {
+    const apiKey = opts.apiKey ?? readEnv("ATLAS_STEEL_API_KEY", "STEEL_API_KEY");
+    if (!apiKey) {
+        throw new Error("steel() requires an apiKey (or set ATLAS_STEEL_API_KEY / STEEL_API_KEY)");
+    }
+    const baseUrl = opts.baseUrl ?? readEnv("ATLAS_STEEL_BASE_URL", "STEEL_BASE_URL");
+    let clientPromise = null;
+    const client = () => {
+        clientPromise ??= createSteelClient(apiKey, baseUrl);
+        return clientPromise;
+    };
+    return {
+        id: "steel",
+        async fetch({ url, signal, onRateLimit }) {
+            let response;
+            try {
+                response = await withSteelRetry(() => client().then((steelClient) => steelClient.scrape({
+                    url,
+                    format: isLikelyPdfUrl(url) ? ["html", "markdown"] : ["html"],
+                    useProxy: opts.proxy ?? true,
+                }, { signal, timeout: SCRAPE_TIMEOUT_MS })), signal, onRateLimit);
+            }
+            catch (err) {
+                if (signal?.aborted)
+                    throw err;
+                return failed("steel_scrape", `network_error: steel scrape failed: ${errorMessage(err)}`, false);
+            }
+            return steelAttemptFromResponse(response, url);
+        },
+    };
+}
+export function steelAttemptFromResponse(response, url) {
+    const status = response.metadata?.statusCode;
+    if (status !== undefined && status >= 400) {
+        return failed("steel_scrape", `http_error: steel scrape returned HTTP ${status}`, false);
+    }
+    const finalUrl = response.metadata?.canonical || url;
+    const html = response.content?.html;
+    if (html) {
+        const extracted = htmlToMarkdown(html, finalUrl);
+        if (!looksBlockedPage(extracted.markdown, html) &&
+            extracted.markdown.length >= DIRECT_HTML_MIN_CHARS) {
+            const attempt = {
+                method: "steel_scrape",
+                ok: true,
+                note: `steel_scrape: extracted ${extracted.markdown.length} text chars`,
+            };
+            return {
+                ok: true,
+                attempt,
+                page: {
+                    finalUrl,
+                    title: extracted.title,
+                    markdown: extracted.markdown,
+                    renderedWith: "steel_scrape",
+                    metadata: extractionMetadataFromScrape({
+                        markdownChars: extracted.markdown.length,
+                        contentType: "text/html",
+                        finalUrl,
+                        attempts: [attempt],
+                        discoveredLinks: extracted.links,
+                        pageMetadata: extracted.metadata,
+                    }),
+                },
+            };
+        }
+    }
+    const markdown = response.content?.markdown?.trim();
+    if (markdown &&
+        !looksBlockedPage(markdown) &&
+        markdown.length >= DIRECT_HTML_MIN_CHARS) {
+        const attempt = {
+            method: "steel_scrape",
+            ok: true,
+            note: `steel_scrape: extracted ${markdown.length} text chars`,
+        };
+        return {
+            ok: true,
+            attempt,
+            page: {
+                finalUrl,
+                title: response.metadata?.title?.trim() || titleFromUrl(finalUrl),
+                markdown,
+                renderedWith: "steel_scrape",
+                metadata: extractionMetadataFromScrape({
+                    markdownChars: markdown.length,
+                    finalUrl,
+                    attempts: [attempt],
+                }),
+            },
+        };
+    }
+    return failed("steel_scrape", "empty_content: steel scrape returned no usable content", false);
+}
+async function createSteelClient(apiKey, baseUrl) {
+    const { default: Steel } = await import("steel-sdk");
+    return new Steel({
+        steelAPIKey: apiKey,
+        baseURL: baseUrl,
+        maxRetries: 0,
+    });
+}
+async function withSteelRetry(request, signal, onRateLimit) {
+    for (let attempt = 1;; attempt++) {
+        try {
+            return await request();
+        }
+        catch (err) {
+            const retryAfterSeconds = parseRetryAfterSeconds(err);
+            if (!retryAfterSeconds || attempt >= STEEL_RETRY_MAX_ATTEMPTS) {
+                throw err;
+            }
+            onRateLimit?.(retryAfterSeconds);
+            await sleep((retryAfterSeconds + 1) * 1000, signal);
+        }
+    }
+}
+const EXA_CONTENTS_MIN_CHARS = 200;
+export function exaContents(opts = {}) {
+    const apiKey = opts.apiKey ?? readEnv("ATLAS_EXA_API_KEY", "EXA_API_KEY");
+    const endpoint = `${(opts.baseUrl ?? "https://api.exa.ai").replace(/\/+$/, "")}/contents`;
+    return {
+        id: "exa_contents",
+        async fetch(req) {
+            if (!apiKey) {
+                return failed("exa_contents", "exa_contents: no Exa API key", true);
+            }
+            let resp;
+            try {
+                resp = await fetch(endpoint, {
+                    method: "POST",
+                    signal: req.signal ?? null,
+                    headers: { "content-type": "application/json", "x-api-key": apiKey },
+                    body: JSON.stringify({
+                        urls: [req.url],
+                        text: true,
+                        livecrawl: "fallback",
+                    }),
+                    ...(req.dispatcher ? { dispatcher: req.dispatcher } : {}),
+                });
+            }
+            catch (err) {
+                if (req.signal?.aborted)
+                    throw err;
+                return failed("exa_contents", `exa_contents: ${errorMessage(err)}`, true);
+            }
+            if (!resp.ok) {
+                return failed("exa_contents", `exa_contents: HTTP ${resp.status}`, true);
+            }
+            let data;
+            try {
+                data = (await resp.json());
+            }
+            catch (err) {
+                return failed("exa_contents", `exa_contents: bad JSON: ${errorMessage(err)}`, true);
+            }
+            const row = data.results?.[0];
+            const markdown = (row?.text ?? "").trim();
+            if (looksBlockedPage(markdown)) {
+                return failed("exa_contents", "blocked_or_challenge: exa contents looked blocked");
+            }
+            if (markdown.length < EXA_CONTENTS_MIN_CHARS) {
+                return failed("exa_contents", `thin_content: exa contents returned ${markdown.length} chars`);
+            }
+            const finalUrl = row?.url ?? req.url;
+            const attempt = {
+                method: "exa_contents",
+                ok: true,
+                note: `exa_contents: extracted ${markdown.length} text chars`,
+            };
+            return {
+                ok: true,
+                attempt,
+                page: {
+                    finalUrl,
+                    title: row?.title ?? null,
+                    markdown,
+                    renderedWith: "exa_contents",
+                    metadata: extractionMetadataFromExa({
+                        markdownChars: markdown.length,
+                        finalUrl,
+                        attempts: [attempt],
+                    }),
+                },
+            };
+        },
+    };
+}
+export function defaultFetchProviders() {
+    const providers = [basicFetch()];
+    if (readEnv("ATLAS_EXA_API_KEY", "EXA_API_KEY")) {
+        providers.push(exaContents());
+    }
+    if (readEnv("ATLAS_STEEL_API_KEY", "STEEL_API_KEY")) {
+        providers.push(steel());
+    }
+    return providers;
+}
+const FETCH_PROVIDER_TIMEOUT_MS = 120_000;
+const FETCH_CHAIN_TIMEOUT_MS = 180_000;
+const CHAIN_GOOD_CHARS = 600;
+function pageScore(page) {
+    const len = page.markdown.trim().length;
+    return looksBlockedPage(page.markdown) ? len : len + 1_000_000;
+}
+function pageIsGood(page) {
+    return (!looksBlockedPage(page.markdown) &&
+        page.markdown.trim().length >= CHAIN_GOOD_CHARS);
+}
+export async function fetchThroughChain(chain, req) {
+    const attempts = [];
+    const deadlineAt = Date.now() + FETCH_CHAIN_TIMEOUT_MS;
+    const runProvider = async (provider) => {
+        const remainingMs = deadlineAt - Date.now();
+        if (remainingMs <= 0) {
+            return failed(provider.id, "timeout: fetch chain deadline exhausted before this provider ran", false);
+        }
+        try {
+            return await withTimeout(Math.min(FETCH_PROVIDER_TIMEOUT_MS, remainingMs), req.signal, provider.id, (signal) => provider.fetch({ ...req, signal }));
+        }
+        catch (err) {
+            if (req.signal?.aborted)
+                throw err;
+            return failed(provider.id, `timeout: ${errorMessage(err)}`);
+        }
+    };
+    const finalize = (page) => {
+        const merged = [
+            ...attempts.filter((a) => !a.ok),
+            ...(page.metadata.attempts ?? []),
+        ];
+        return {
+            page: { ...page, metadata: { ...page.metadata, attempts: merged } },
+            attempts: merged,
+        };
+    };
+    if (chain.length === 0)
+        return { page: null, attempts };
+    const first = await runProvider(chain[0]);
+    attempts.push(first.attempt);
+    const firstPage = first.ok ? first.page : null;
+    if (firstPage && pageIsGood(firstPage))
+        return finalize(firstPage);
+    if (!first.ok && !first.escalate)
+        return { page: null, attempts };
+    const rest = chain.slice(1);
+    const candidates = firstPage ? [firstPage] : [];
+    if (rest.length > 0) {
+        const settled = await Promise.allSettled(rest.map((p) => runProvider(p)));
+        for (const s of settled) {
+            if (s.status === "fulfilled") {
+                attempts.push(s.value.attempt);
+                if (s.value.ok)
+                    candidates.push(s.value.page);
+            }
+            else if (req.signal?.aborted) {
+                throw s.reason;
+            }
+        }
+    }
+    if (candidates.length === 0)
+        return { page: null, attempts };
+    const best = candidates.reduce((a, b) => pageScore(b) > pageScore(a) ? b : a);
+    return finalize(best);
+}

package/dist/providers/parallel-agent.d.ts

@@ -0,0 +1,11 @@
+import type { Researcher } from "../researcher.js";
+export interface ParallelAgentOptions {
+    apiKey?: string;
+    baseUrl?: string;
+    processor?: string;
+    description?: string;
+    timeoutMs?: number;
+    pollIntervalMs?: number;
+    costPerRunUSD?: number;
+}
+export declare function parallelAgent(opts?: ParallelAgentOptions): Researcher;