@taurgis/bonsai 1.0.0

package/dist/lib/research/browser.d.ts

@@ -0,0 +1,74 @@
+export interface BrowserFetchOptions {
+    timeoutMs?: number;
+    bodyLimitBytes?: number;
+    settleMs?: number;
+}
+export interface BrowserFetchResult {
+    status: number;
+    contentType: string | null;
+    etag: string | null;
+    lastModified: string | null;
+    finalUrl: string;
+    responseSize: number;
+    content: string;
+}
+export declare class CdpClient {
+    private ws;
+    private nextId;
+    private pending;
+    private handlers;
+    constructor(url: string);
+    connect(): Promise<void>;
+    send(method: string, params?: any, sessionId?: string): Promise<any>;
+    on(event: string, handler: (params: any) => void): void;
+    close(): void;
+}
+export declare function findChromePath(): string;
+export declare function waitForLoad(client: CdpClient, sessionId: string, timeoutMs: number, settleMs?: number): Promise<void>;
+/**
+ * Polls until a content-container element holds enough RENDERED text, so SPAs that fetch their
+ * body after load are captured rendered rather than as a "Loading…" shell. Pierces open shadow
+ * roots (the container is matched across shadow boundaries) and measures `innerText`, which
+ * reflects shadow-rendered text — `document.querySelector` + `textContent` miss both. Returns
+ * without throwing on timeout; the caller captures whatever rendered and judges its quality.
+ */
+export declare function waitForContentReady(page: CdpPage, selectors: string[], minChars: number, timeoutMs: number): Promise<void>;
+export declare const BLOCKED_ASSET_URLS: string[];
+export interface CdpPage {
+    client: CdpClient;
+    sessionId: string;
+    close: () => Promise<void>;
+}
+/**
+ * Spawns a headless Chrome, attaches a flat CDP session, and enables the Page, Network,
+ * and Runtime domains. The caller drives navigation and must call close() when finished.
+ */
+export declare function openCdpPage(): Promise<CdpPage>;
+export interface ResponseMatcher {
+    key: string;
+    test: (request: {
+        url: string;
+        postData?: string;
+    }) => boolean;
+    accept?: (body: string) => boolean;
+}
+/**
+ * Buffers response bodies for requests matching one of the supplied matchers, keyed by
+ * matcher key. Matching happens on requestWillBeSent so predicates can inspect post data
+ * (e.g. to tell two same-URL XHRs apart). Works around CdpClient's single-handler-per-event
+ * model so several waitFor() calls can track different responses from one navigation.
+ * Install before navigating.
+ */
+export declare class ResponseCapture {
+    private matched;
+    private bodies;
+    private waiters;
+    private page;
+    private matchers;
+    constructor(page: CdpPage, matchers: ResponseMatcher[]);
+    private onFinished;
+    private readBody;
+    /** Resolves with the response body text once a matching response finishes, or null on timeout. */
+    waitFor(key: string, timeoutMs: number): Promise<string | null>;
+}
+export declare function fetchRenderedHtml(url: string, options?: BrowserFetchOptions): Promise<BrowserFetchResult>;

package/dist/lib/research/browser.js

@@ -0,0 +1,359 @@
+import { spawn } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { execSync } from 'node:child_process';
+import { checkDnsSafety } from './fetcher.js';
+import { normalizeUrl } from './url.js';
+export class CdpClient {
+    ws;
+    nextId = 1;
+    pending = new Map();
+    handlers = new Map();
+    constructor(url) {
+        this.ws = new WebSocket(url);
+    }
+    connect() {
+        return new Promise((resolve, reject) => {
+            this.ws.onopen = () => resolve();
+            this.ws.onerror = (err) => reject(err);
+            this.ws.onmessage = (event) => {
+                const msg = JSON.parse(event.data);
+                if (msg.id) {
+                    const promise = this.pending.get(msg.id);
+                    if (promise) {
+                        this.pending.delete(msg.id);
+                        if (msg.error) {
+                            promise.reject(new Error(msg.error.message));
+                        }
+                        else {
+                            promise.resolve(msg.result);
+                        }
+                    }
+                }
+                else if (msg.method) {
+                    const key = msg.sessionId ? `${msg.sessionId}:${msg.method}` : msg.method;
+                    const listeners = this.handlers.get(key) ?? this.handlers.get(msg.method);
+                    if (listeners) {
+                        listeners.forEach((handler) => handler(msg.params));
+                    }
+                }
+            };
+        });
+    }
+    send(method, params, sessionId) {
+        return new Promise((resolve, reject) => {
+            const id = this.nextId++;
+            this.pending.set(id, { resolve, reject });
+            const payload = { id, method, params };
+            if (sessionId) {
+                payload.sessionId = sessionId;
+            }
+            this.ws.send(JSON.stringify(payload));
+        });
+    }
+    on(event, handler) {
+        const listeners = this.handlers.get(event) ?? [];
+        listeners.push(handler);
+        this.handlers.set(event, listeners);
+    }
+    close() {
+        this.ws.close();
+    }
+}
+export function findChromePath() {
+    if (process.env.CHROME_PATH && existsSync(process.env.CHROME_PATH)) {
+        return process.env.CHROME_PATH;
+    }
+    const paths = [];
+    if (process.platform === 'darwin') {
+        paths.push('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', '/Applications/Chromium.app/Contents/MacOS/Chromium');
+    }
+    else if (process.platform === 'win32') {
+        paths.push('C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe', 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe');
+    }
+    else {
+        paths.push('/usr/bin/google-chrome', '/usr/bin/google-chrome-stable', '/usr/bin/chromium', '/usr/bin/chromium-browser');
+    }
+    for (const p of paths) {
+        if (existsSync(p))
+            return p;
+    }
+    try {
+        const cmd = process.platform === 'win32'
+            ? 'where chrome'
+            : 'which google-chrome || which chromium || which google-chrome-stable';
+        const resolved = execSync(cmd, { stdio: ['ignore', 'pipe', 'ignore'] })
+            .toString()
+            .trim()
+            .split('\n')[0];
+        if (resolved && existsSync(resolved)) {
+            return resolved;
+        }
+    }
+    catch { }
+    throw new Error('Chrome executable not found. Please set CHROME_PATH environment variable.');
+}
+async function spawnChrome(chromePath) {
+    const chromeProcess = spawn(chromePath, [
+        '--headless=new',
+        '--remote-debugging-port=0',
+        '--disable-gpu',
+        '--no-sandbox',
+        '--disable-setuid-sandbox',
+        '--disable-extensions',
+        '--disable-dev-shm-usage',
+    ]);
+    try {
+        const wsUrl = await new Promise((resolve, reject) => {
+            let output = '';
+            const timeoutId = setTimeout(() => {
+                chromeProcess.kill('SIGKILL');
+                reject(new Error('Timed out waiting for Chrome to start.'));
+            }, 10000);
+            const onData = (data) => {
+                output += data.toString();
+                const match = output.match(/ws:\/\/127\.0\.0\.1:\d+\/devtools\/browser\/[a-zA-Z0-9-]+/);
+                if (match) {
+                    clearTimeout(timeoutId);
+                    cleanup();
+                    resolve(match[0]);
+                }
+            };
+            const cleanup = () => {
+                chromeProcess.stderr.off('data', onData);
+                chromeProcess.stdout.off('data', onData);
+            };
+            chromeProcess.stderr.on('data', onData);
+            chromeProcess.stdout.on('data', onData);
+            chromeProcess.on('error', (err) => {
+                clearTimeout(timeoutId);
+                cleanup();
+                reject(err);
+            });
+            chromeProcess.on('exit', (code) => {
+                clearTimeout(timeoutId);
+                cleanup();
+                reject(new Error(`Chrome exited prematurely with code ${code}`));
+            });
+        });
+        return { chromeProcess, wsUrl };
+    }
+    catch (err) {
+        chromeProcess.kill('SIGKILL');
+        throw err;
+    }
+}
+export async function waitForLoad(client, sessionId, timeoutMs, settleMs = 1000) {
+    // Resolve on whichever fires first: DOMContentLoaded or the full load event. Heavy SPAs
+    // (e.g. Salesforce LWR) often never fire `load`, so requiring it would falsely time out.
+    let ready = false;
+    const markReady = () => {
+        ready = true;
+    };
+    client.on(`${sessionId}:Page.domContentEventFired`, markReady);
+    client.on(`${sessionId}:Page.loadEventFired`, markReady);
+    const start = Date.now();
+    while (Date.now() - start < timeoutMs) {
+        if (ready) {
+            const settle = Math.min(settleMs, timeoutMs - (Date.now() - start));
+            if (settle > 0)
+                await new Promise((r) => setTimeout(r, settle));
+            return;
+        }
+        await new Promise((r) => setTimeout(r, Math.min(100, timeoutMs - (Date.now() - start))));
+    }
+    throw new Error(`Navigation timed out after ${timeoutMs}ms`);
+}
+/**
+ * Polls until a content-container element holds enough RENDERED text, so SPAs that fetch their
+ * body after load are captured rendered rather than as a "Loading…" shell. Pierces open shadow
+ * roots (the container is matched across shadow boundaries) and measures `innerText`, which
+ * reflects shadow-rendered text — `document.querySelector` + `textContent` miss both. Returns
+ * without throwing on timeout; the caller captures whatever rendered and judges its quality.
+ */
+export async function waitForContentReady(page, selectors, minChars, timeoutMs) {
+    // Reports whether a content container has rendered (innerText pierces shadow), plus the total
+    // rendered text length so the caller can wait for the render to *settle* — heavy pages (e.g.
+    // the api-console) render the container early but keep filling it in.
+    const expression = `(() => {
+    const out = [];
+    const visit = (r) => { for (const c of Array.from(r.children || [])) { out.push(c); if (c.shadowRoot) visit(c.shadowRoot); visit(c); } };
+    visit(document);
+    const SELS = ${JSON.stringify(selectors)};
+    let has = false;
+    for (const el of out) {
+      if (!el.matches) continue;
+      if (!SELS.some((s) => { try { return el.matches(s); } catch (e) { return false; } })) continue;
+      if (((el.innerText) || '').trim().length >= ${minChars}) { has = true; break; }
+    }
+    return { has, len: document.body ? (document.body.innerText || '').length : 0 };
+  })()`;
+    const start = Date.now();
+    let previousLength = -1;
+    let stablePolls = 0;
+    while (Date.now() - start < timeoutMs) {
+        const value = (await page.client
+            .send('Runtime.evaluate', { expression, returnByValue: true }, page.sessionId)
+            .catch(() => null))?.result?.value;
+        if (value?.has) {
+            const grew = Math.abs((value.len || 0) - previousLength) > Math.max(40, previousLength * 0.02);
+            if (!grew && ++stablePolls >= 2)
+                return; // content present and rendering has settled
+            if (grew)
+                stablePolls = 0;
+            previousLength = value.len || 0;
+        }
+        await new Promise((r) => setTimeout(r, 500));
+    }
+}
+export const BLOCKED_ASSET_URLS = [
+    '*.css',
+    '*.png',
+    '*.jpg',
+    '*.jpeg',
+    '*.gif',
+    '*.svg',
+    '*.woff',
+    '*.woff2',
+    '*.ttf',
+    '*.mp4',
+    '*.mp3',
+    '*.ico',
+];
+const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+/**
+ * Spawns a headless Chrome, attaches a flat CDP session, and enables the Page, Network,
+ * and Runtime domains. The caller drives navigation and must call close() when finished.
+ */
+export async function openCdpPage() {
+    const chromePath = findChromePath();
+    const { chromeProcess, wsUrl } = await spawnChrome(chromePath);
+    const client = new CdpClient(wsUrl);
+    try {
+        await client.connect();
+        const { targetId } = await client.send('Target.createTarget', { url: 'about:blank' });
+        const { sessionId } = await client.send('Target.attachToTarget', { targetId, flatten: true });
+        await client.send('Page.enable', {}, sessionId);
+        await client.send('Network.enable', {}, sessionId);
+        await client.send('Runtime.enable', {}, sessionId);
+        await client.send('Network.setUserAgentOverride', { userAgent: DEFAULT_USER_AGENT }, sessionId);
+        const close = async () => {
+            try {
+                await client.send('Target.closeTarget', { targetId });
+            }
+            catch { }
+            client.close();
+            chromeProcess.kill('SIGKILL');
+        };
+        return { client, sessionId, close };
+    }
+    catch (err) {
+        client.close();
+        chromeProcess.kill('SIGKILL');
+        throw err;
+    }
+}
+/**
+ * Buffers response bodies for requests matching one of the supplied matchers, keyed by
+ * matcher key. Matching happens on requestWillBeSent so predicates can inspect post data
+ * (e.g. to tell two same-URL XHRs apart). Works around CdpClient's single-handler-per-event
+ * model so several waitFor() calls can track different responses from one navigation.
+ * Install before navigating.
+ */
+export class ResponseCapture {
+    matched = new Map(); // requestId -> matcher keys
+    bodies = new Map(); // matcher key -> response body text
+    waiters = new Map();
+    page;
+    matchers;
+    constructor(page, matchers) {
+        this.page = page;
+        this.matchers = matchers;
+        const { client, sessionId } = page;
+        client.on(`${sessionId}:Network.requestWillBeSent`, (params) => {
+            const request = { url: params.request?.url ?? '', postData: params.request?.postData };
+            const keys = this.matchers.filter((m) => m.test(request)).map((m) => m.key);
+            if (keys.length)
+                this.matched.set(params.requestId, keys);
+        });
+        client.on(`${sessionId}:Network.loadingFinished`, (params) => {
+            void this.onFinished(params.requestId);
+        });
+    }
+    // Body is only available after loadingFinished, per the CDP Network contract.
+    async onFinished(requestId) {
+        const keys = this.matched.get(requestId);
+        if (!keys)
+            return;
+        const body = await this.readBody(requestId);
+        if (body === null)
+            return;
+        for (const key of keys) {
+            if (this.bodies.has(key))
+                continue; // already captured for this key
+            const matcher = this.matchers.find((m) => m.key === key);
+            if (matcher?.accept && !matcher.accept(body))
+                continue; // wait for a later matching response
+            this.bodies.set(key, body);
+            const waiters = this.waiters.get(key) ?? [];
+            this.waiters.delete(key);
+            waiters.forEach((resolve) => resolve(body));
+        }
+    }
+    async readBody(requestId) {
+        try {
+            const res = await this.page.client.send('Network.getResponseBody', { requestId }, this.page.sessionId);
+            return res.base64Encoded ? Buffer.from(res.body, 'base64').toString('utf8') : res.body;
+        }
+        catch {
+            return null;
+        }
+    }
+    /** Resolves with the response body text once a matching response finishes, or null on timeout. */
+    waitFor(key, timeoutMs) {
+        const existing = this.bodies.get(key);
+        if (existing !== undefined)
+            return Promise.resolve(existing);
+        return new Promise((resolve) => {
+            const timer = setTimeout(() => resolve(null), timeoutMs);
+            const list = this.waiters.get(key) ?? [];
+            list.push((body) => {
+                clearTimeout(timer);
+                resolve(body);
+            });
+            this.waiters.set(key, list);
+        });
+    }
+}
+export async function fetchRenderedHtml(url, options = {}) {
+    const timeout = options.timeoutMs ?? 15_000;
+    const limit = options.bodyLimitBytes ?? 2 * 1024 * 1024;
+    const currentUrl = normalizeUrl(url);
+    await checkDnsSafety(new URL(currentUrl).hostname);
+    const page = await openCdpPage();
+    try {
+        await page.client.send('Network.setBlockedURLs', { urls: BLOCKED_ASSET_URLS }, page.sessionId);
+        await page.client.send('Page.navigate', { url: currentUrl }, page.sessionId);
+        await waitForLoad(page.client, page.sessionId, timeout, options.settleMs ?? 1000);
+        const evalResult = await page.client.send('Runtime.evaluate', { expression: 'document.documentElement.outerHTML', returnByValue: true }, page.sessionId);
+        const html = evalResult.result?.value;
+        if (typeof html !== 'string') {
+            throw new Error('Failed to retrieve rendered HTML from browser page.');
+        }
+        const byteLength = Buffer.byteLength(html);
+        if (byteLength > limit) {
+            throw new Error(`Response body size limit exceeded. Rendered HTML is ${byteLength} bytes.`);
+        }
+        return {
+            status: 200,
+            contentType: 'text/html',
+            etag: null,
+            lastModified: null,
+            finalUrl: currentUrl,
+            responseSize: byteLength,
+            content: html,
+        };
+    }
+    finally {
+        await page.close();
+    }
+}

package/dist/lib/research/cache-key.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Derives a deterministic hexadecimal SHA-256 cache key from a source URL.
+ * Normalizes the URL first before hashing.
+ */
+export declare function deriveCacheKey(url: string): string;

package/dist/lib/research/cache-key.js

@@ -0,0 +1,10 @@
+import { createHash } from 'node:crypto';
+import { normalizeUrl } from './url.js';
+/**
+ * Derives a deterministic hexadecimal SHA-256 cache key from a source URL.
+ * Normalizes the URL first before hashing.
+ */
+export function deriveCacheKey(url) {
+    const normalized = normalizeUrl(url);
+    return createHash('sha256').update(normalized).digest('hex');
+}

package/dist/lib/research/capture.d.ts

@@ -0,0 +1,29 @@
+import type { FetchResult } from './fetcher.js';
+import type { ExtractionResult } from './extract.js';
+import type { CaptureMethod } from './schema.js';
+import type { MachineReadableArtifact, SiteCapabilities } from './docs/capabilities.js';
+import { type TextFetcher } from './docs/machine-readable.js';
+export interface CaptureDeps {
+    fetchStatic: (url: string) => Promise<FetchResult>;
+    fetchRendered: (url: string) => Promise<FetchResult>;
+    fetchText: TextFetcher;
+}
+export interface CaptureOptions {
+    forceRendered?: boolean;
+}
+export interface CaptureOutcome {
+    fetchResult: FetchResult;
+    extraction: ExtractionResult;
+    capabilities: SiteCapabilities;
+    captureMethod: CaptureMethod;
+    sourceDocUrl: string | null;
+    machineReadable: MachineReadableArtifact[];
+    attemptedMethods: string[];
+}
+export declare function renderedFallbackNeeded(caps: SiteCapabilities, extraction: ExtractionResult | null): boolean;
+/**
+ * Captures a page's best-available content. Order: forced rendered → route/source Markdown → static
+ * HTML → rendered fallback. Returns the chosen fetch result, extraction, detected capabilities, and
+ * provenance (capture method, source URL, machine-readable artifacts, attempted methods).
+ */
+export declare function capturePage(url: string, options: CaptureOptions, deps: CaptureDeps): Promise<CaptureOutcome>;

package/dist/lib/research/capture.js

@@ -0,0 +1,181 @@
+import { extractHtmlContent } from './extract.js';
+import { detectDocsEngine } from './docs/detect.js';
+import { emptyCapabilities } from './docs/capabilities.js';
+import { probeRouteMarkdown, probeLlmsTxt } from './docs/machine-readable.js';
+import { mapUrlToSource } from './docs/source-map.js';
+import { extractFromSource } from './docs/markdown-source.js';
+import { validateTextArtifact } from './docs/validate.js';
+const MIN_USEFUL_CHARS = 600;
+// A static extraction needs rendering help when it failed outright, the detector says the page is
+// an SPA/client-rendered, or the extracted article is too thin/low-confidence to be useful.
+export function renderedFallbackNeeded(caps, extraction) {
+    if (!extraction)
+        return true;
+    if (caps.recommendedCapture === 'rendered')
+        return true;
+    return extraction.confidence === 'low' && extraction.detailedMarkdown.length < MIN_USEFUL_CHARS;
+}
+async function resolveSource(url, caps, deps) {
+    const route = await probeRouteMarkdown(url, caps, deps.fetchText);
+    if (route) {
+        return {
+            extraction: extractFromSource(route.body, route.artifact.url),
+            method: 'route_markdown',
+            url: route.artifact.url,
+            mrType: 'route-markdown',
+        };
+    }
+    const candidates = [];
+    if (caps.source?.url)
+        candidates.push(caps.source.url);
+    const mapped = mapUrlToSource(url);
+    if (mapped)
+        candidates.push(mapped.rawUrl);
+    for (const candidate of candidates) {
+        try {
+            const res = await deps.fetchText(candidate);
+            if (res.status !== 304 && res.content && validateTextArtifact(res.content).ok) {
+                return {
+                    extraction: extractFromSource(res.content, candidate),
+                    method: 'github_source',
+                    url: candidate,
+                    mrType: 'source-edit-link',
+                };
+            }
+        }
+        catch {
+            // try next candidate
+        }
+    }
+    return null;
+}
+async function discoverLlmsTxt(url, caps, deps) {
+    try {
+        const llms = await probeLlmsTxt(url, caps, deps.fetchText);
+        return llms ? [llms.artifact] : [];
+    }
+    catch {
+        return [];
+    }
+}
+async function captureRendered(url, deps) {
+    const fetchResult = await deps.fetchRendered(url);
+    return { fetchResult, extraction: extractHtmlContent(fetchResult.content, fetchResult.finalUrl) };
+}
+// Browser-rendered capture as a complete outcome. Shared by the forced-rendered path and the
+// automatic fallback; appends 'browser_fallback' to attemptedMethods (mutated in place).
+async function renderedOutcome(url, deps, attemptedMethods, machineReadable) {
+    attemptedMethods.push('browser_fallback');
+    const { fetchResult, extraction } = await captureRendered(url, deps);
+    return {
+        fetchResult,
+        extraction,
+        capabilities: detectDocsEngine(fetchResult.content, fetchResult.finalUrl),
+        captureMethod: 'browser_fallback',
+        sourceDocUrl: null,
+        machineReadable: dedupeArtifacts(machineReadable),
+        attemptedMethods,
+    };
+}
+// Static fetch + engine detection + llms.txt discovery. Never throws.
+async function runStaticPhase(url, deps) {
+    let staticResult = null;
+    let staticError = null;
+    try {
+        staticResult = await deps.fetchStatic(url);
+    }
+    catch (err) {
+        staticError = err;
+    }
+    const capabilities = staticResult
+        ? detectDocsEngine(staticResult.content, staticResult.finalUrl)
+        : emptyCapabilities();
+    const machineReadable = staticResult ? await discoverLlmsTxt(url, capabilities, deps) : [];
+    return { staticResult, capabilities, machineReadable, staticError };
+}
+// Machine-readable / public-source capture, preferred over HTML extraction. Returns null (so the
+// caller falls through to HTML/rendered) when there is no static page or no source validates.
+async function sourceOutcome(url, phase, deps, attemptedMethods) {
+    if (!phase.staticResult)
+        return null;
+    const source = await resolveSource(url, phase.capabilities, deps);
+    if (!source)
+        return null;
+    phase.machineReadable.push({ type: source.mrType, url: source.url, evidence: 'verified' });
+    return {
+        fetchResult: phase.staticResult,
+        extraction: source.extraction,
+        capabilities: phase.capabilities,
+        captureMethod: source.method,
+        sourceDocUrl: source.url,
+        machineReadable: dedupeArtifacts(phase.machineReadable),
+        attemptedMethods: [...attemptedMethods, source.method],
+    };
+}
+// Rendered fallback when static extraction is missing or low quality. Returns null when no fallback
+// is warranted; throws only when rendering fails and nothing usable was produced (preferring the
+// earlier static error).
+async function renderedFallbackOutcome(url, deps, attemptedMethods, phase, extraction) {
+    if (!renderedFallbackNeeded(phase.capabilities, extraction))
+        return null;
+    try {
+        return await renderedOutcome(url, deps, attemptedMethods, phase.machineReadable);
+    }
+    catch (renderErr) {
+        if (!extraction)
+            throw phase.staticError ?? renderErr;
+        return null;
+    }
+}
+/**
+ * Captures a page's best-available content. Order: forced rendered → route/source Markdown → static
+ * HTML → rendered fallback. Returns the chosen fetch result, extraction, detected capabilities, and
+ * provenance (capture method, source URL, machine-readable artifacts, attempted methods).
+ */
+export async function capturePage(url, options, deps) {
+    const attemptedMethods = [];
+    if (options.forceRendered) {
+        return renderedOutcome(url, deps, attemptedMethods, []);
+    }
+    attemptedMethods.push('static_fetch');
+    const phase = await runStaticPhase(url, deps);
+    // Prefer machine-readable / public source before HTML extraction.
+    const fromSource = await sourceOutcome(url, phase, deps, attemptedMethods);
+    if (fromSource)
+        return fromSource;
+    let extraction = null;
+    if (phase.staticResult) {
+        try {
+            extraction = extractHtmlContent(phase.staticResult.content, phase.staticResult.finalUrl);
+        }
+        catch (err) {
+            phase.staticError = err;
+        }
+    }
+    const fallback = await renderedFallbackOutcome(url, deps, attemptedMethods, phase, extraction);
+    if (fallback)
+        return fallback;
+    if (!extraction)
+        throw phase.staticError ?? new Error(`Could not capture ${url}`);
+    return {
+        fetchResult: phase.staticResult,
+        extraction,
+        capabilities: phase.capabilities,
+        captureMethod: 'static_fetch',
+        sourceDocUrl: null,
+        machineReadable: dedupeArtifacts(phase.machineReadable),
+        attemptedMethods,
+    };
+}
+function dedupeArtifacts(items) {
+    const seen = new Set();
+    const out = [];
+    for (const item of items) {
+        const key = `${item.type}:${item.url}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        out.push(item);
+    }
+    return out;
+}

package/dist/lib/research/compress.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Compresses Markdown text by removing images, simplifying links from [text](url) to [text],
+ * and collapsing multiple consecutive blank lines.
+ */
+export declare function compressMarkdown(markdown: string): string;

package/dist/lib/research/compress.js

@@ -0,0 +1,17 @@
+/**
+ * Compresses Markdown text by removing images, simplifying links from [text](url) to [text],
+ * and collapsing multiple consecutive blank lines.
+ */
+export function compressMarkdown(markdown) {
+    if (!markdown) {
+        return '';
+    }
+    let result = markdown;
+    // Remove markdown images: ![alt](url) -> ""
+    result = result.replace(/!\[.*?\]\(.*?\)/g, '');
+    // Simplify links: [text](url) -> [text]
+    result = result.replace(/\[(.*?)\]\(.*?\)/g, '[$1]');
+    // Collapse multiple consecutive newlines (3 or more) to maximum 2 newlines
+    result = result.replace(/\n{3,}/g, '\n\n');
+    return result.trim();
+}