@taurgis/bonsai 3.2.0 → 4.0.0

package/dist/lib/research/artifact-index.js

@@ -39,7 +39,13 @@ function getOrUpdateIndexEntry(filePath, cached, onChanged) {
         return cached;
     }
     try {
-        const entry = { sig, artifact: parseArtifactShallow(readFileSync(filePath, 'utf-8')) };
+        const artifact = parseArtifactShallow(readFileSync(filePath, 'utf-8'));
+        // Mirrors the same guard in storage.ts's scanCacheDir: parseArtifact never throws on a foreign
+        // or garbled *.md file that merely has `---`/`---` fences, it just defaults cache_key to ''. Treat
+        // that the same as a parse failure so it's skipped rather than indexed as a phantom entry.
+        if (!artifact.metadata.cache_key)
+            return undefined;
+        const entry = { sig, artifact };
         onChanged();
         return entry;
     }
@@ -52,10 +58,13 @@ function getOrUpdateIndexEntry(filePath, cached, onChanged) {
  * to skip unchanged files. Re-reads only files whose signature changed, then persists the refreshed
  * index (best-effort; a write failure never breaks callers). Corrupt/unreadable files are skipped —
  * the real read paths in storage.ts handle archiving them.
+ *
+ * Pass `{ persist: false }` under read-only/plan mode so the derived sidecar is never written.
  */
-export function loadIndexedArtifactsForDir(researchDir) {
+export function loadIndexedArtifactsForDir(researchDir, options = {}) {
     if (!existsSync(researchDir))
         return [];
+    const persist = options.persist !== false;
     const indexPath = join(researchDir, INDEX_FILE);
     const previous = loadIndex(indexPath);
     const entries = {};
@@ -78,7 +87,7 @@ export function loadIndexedArtifactsForDir(researchDir) {
     if (!changed) {
         changed = Object.keys(previous.entries).some((key) => !(key in entries));
     }
-    if (changed) {
+    if (changed && persist) {
         try {
             atomicWriteFile(indexPath, JSON.stringify({ version: INDEX_VERSION, entries }));
         }
@@ -93,11 +102,11 @@ export function loadIndexedArtifactsForDir(researchDir) {
  * key so a project entry shadows the same key in the global cache. Mirrors the dedup semantics of
  * {@link scanCacheDirs} but reads through the per-root sidecar index.
  */
-export function loadIndexedArtifacts(dataDirs) {
+export function loadIndexedArtifacts(dataDirs, options = {}) {
     const seen = new Set();
     const all = [];
     for (const dataDir of dataDirs) {
-        for (const located of loadIndexedArtifactsForDir(join(dataDir, 'research'))) {
+        for (const located of loadIndexedArtifactsForDir(join(dataDir, 'research'), options)) {
             const key = located.artifact.metadata.cache_key;
             if (seen.has(key))
                 continue;

package/dist/lib/research/artifact.js

@@ -1,12 +1,30 @@
+import { CONTROL_CHAR_PATTERN } from '../text.js';
+// Frontmatter is one field per line. An embedded newline (or tab, treated the same way as
+// legitimate Markdown whitespace elsewhere in this codebase — see sanitizeForTerminal in
+// ../text.ts and stripUnsafeControlChars in ./prompt-injection.ts) in a free-form value (e.g. an
+// agent-supplied --topic or --tags) would inject a bogus extra line — at best a dropped stray
+// line, at worst a fake `key: value` pair, or a bare `---` that closes the frontmatter fence early
+// and splices the remaining metadata fields (and even the closing fence and body sections) into
+// what parseArtifact then treats as free-text content. Collapsing that whitespace to a space
+// (rather than deleting it, which would silently merge adjacent words) keeps every value confined
+// to the single line the format requires, regardless of where the string originated. Other control
+// characters (e.g. a raw ANSI escape byte, whether from an agent-supplied flag or extracted page
+// text) are stripped outright: the cache file is plain text meant to be `cat`ed or displayed by
+// `inspect`/`list`, and an uncaught escape sequence there would reach a real terminal — unlike
+// --json output, which JSON-escapes control characters by construction.
+function sanitizeFrontmatterLine(value) {
+    return value.replace(/[\t\r\n]+/g, ' ').replace(CONTROL_CHAR_PATTERN, '');
+}
 function serializeArrayField(lines, name, items) {
     lines.push(`${name}:`);
     for (const item of items) {
-        lines.push(`  - ${item}`);
+        lines.push(`  - ${sanitizeFrontmatterLine(item)}`);
     }
 }
 // A scalar frontmatter line, blanking null/undefined. Using `??` (not `||`) preserves a numeric 0.
 function serializeScalarField(lines, name, value) {
-    lines.push(`${name}: ${value ?? ''}`);
+    const safeValue = typeof value === 'string' ? sanitizeFrontmatterLine(value) : value;
+    lines.push(`${name}: ${safeValue ?? ''}`);
 }
 /**
  * Serializes artifact metadata to standard YAML frontmatter text.

package/dist/lib/research/browser.d.ts

@@ -1,3 +1,4 @@
+import { ChildProcess } from 'node:child_process';
 export interface BrowserFetchOptions {
     timeoutMs?: number;
     bodyLimitBytes?: number;
@@ -25,12 +26,27 @@ export declare class CdpClient {
 }
 export declare function findChromePath(): string;
 export declare function buildChromeArgs(): string[];
+/**
+ * Kill Chrome and every child process it spawned (zygote, gpu-process, renderer, crashpad).
+ * `chromeProcess.kill()` alone only signals that one PID — per Node's child_process docs, a signal
+ * sent to a child is not propagated to *that child's own* children on Linux, so Chrome's
+ * multi-process tree survives a plain kill on every rendered fetch, not just an interrupted one.
+ * Spawning with `detached: true` makes Chrome the leader of its own process group, so signaling the
+ * negative PID reaches the whole tree in one call. Falls back to a direct kill if group-kill fails
+ * (already exited, or a platform without POSIX process groups).
+ */
+export declare function killChromeTree(chromeProcess: ChildProcess): void;
 export declare function waitForLoad(client: CdpClient, sessionId: string, timeoutMs: number, settleMs?: number): Promise<void>;
 export declare const BLOCKED_ASSET_URLS: string[];
 export interface CdpPage {
     client: CdpClient;
     sessionId: string;
     close: () => Promise<void>;
+    /**
+     * Returns (and clears) the reason the most recent navigation request was blocked by the
+     * per-hop safety guard installed in {@link openCdpPage}, or `null` if none was blocked.
+     */
+    takeBlockedNavigationError: () => Error | null;
 }
 /** Status line of the main-frame document response, captured from Network.responseReceived. */
 export interface MainDocumentResponse {
@@ -45,8 +61,20 @@ export interface MainDocumentResponse {
  */
 export declare function assertRenderedHttpOk(mainDoc: MainDocumentResponse | undefined): void;
 /**
- * Spawns a headless Chrome, attaches a flat CDP session, and enables the Page, Network,
- * and Runtime domains. The caller drives navigation and must call close() when finished.
+ * Validates a navigated (or redirected) Document request against the same scheme/credentials/DNS
+ * safety `normalizeUrl` and `fetchWithRedirects` enforce per hop in url.ts/fetcher.ts. Chrome
+ * follows redirects internally with no hook back into that check, so a page that starts on a safe
+ * public host could otherwise redirect a rendered-fallback capture into a private/internal address
+ * (SSRF) or a non-http(s) scheme (e.g. `file:`) that the static fetcher would have rejected outright.
+ */
+export declare function describeUnsafeNavigationTarget(url: string): Promise<Error | null>;
+/**
+ * Spawns a headless Chrome, attaches a flat CDP session, and enables the Page, Network, Runtime,
+ * and Fetch domains. Fetch domain interception guards every Document-type request (the initial
+ * navigation and each subsequent redirect) with {@link describeUnsafeNavigationTarget} before
+ * Chrome is allowed to send it; a blocked request's reason is recorded and can be read back via
+ * the returned page's `takeBlockedNavigationError()`. The caller drives navigation and must call
+ * close() when finished.
  */
 export declare function openCdpPage(): Promise<CdpPage>;
 export interface ResponseMatcher {
@@ -89,4 +117,27 @@ export declare const SANDBOX_EGRESS_ERROR_MARKER = "sandboxed execution environm
  * against the environment directly and fall back to a manual fetch if it's still unreachable.
  */
 export declare function describeNavigationFailure(errorText: string, url: string): Error;
+export declare function isTransientBrowserError(err: unknown): boolean;
+/**
+ * Navigates an already-open CDP page to `currentUrl` and captures the rendered HTML, honoring the
+ * openCdpPage Fetch-domain safety guard at every point a blocked navigation could otherwise be
+ * discarded in favor of a less specific (or entirely silent) failure:
+ *
+ * 1. Right after `Page.navigate` returns — a fast HTTP redirect chain can already be blocked and
+ *    resolved by this point, surfacing as the generic `nav.errorText` (e.g. `net::ERR_BLOCKED_BY_CLIENT`)
+ *    below if not checked first.
+ * 2. If `waitForLoad` throws — a client-side (JS-triggered) navigation into a blocked target after
+ *    the initial navigation already settled can prevent the load/domContentLoaded events
+ *    `waitForLoad` is waiting on, surfacing as a generic timeout instead of the specific reason.
+ * 3. After `waitForLoad` resolves — a client-side navigation blocked without otherwise disrupting
+ *    the load signal would otherwise sail through silently, capturing whatever the page settled on.
+ *
+ * Exported (rather than folded into `fetchRenderedHtmlOnce`) so tests can drive it against a fake
+ * `CdpPage` without spawning a real browser.
+ */
+export declare function captureNavigatedPage(page: CdpPage, currentUrl: string, options: {
+    timeout: number;
+    limit: number;
+    settleMs: number;
+}): Promise<BrowserFetchResult>;
 export declare function fetchRenderedHtml(url: string, options?: BrowserFetchOptions): Promise<BrowserFetchResult>;

package/dist/lib/research/browser.js

@@ -130,13 +130,63 @@ export function buildChromeArgs() {
 // CI runners occasionally starve headless Chrome's startup (disk/CPU contention from concurrent
 // jobs), so this is generous enough to absorb that rather than a "typical" local startup time.
 const CHROME_STARTUP_TIMEOUT_MS = 20_000;
+// Node's default SIGINT/SIGTERM disposition kills the process immediately, without waiting for the
+// async `finally { page.close() }` in fetchRenderedHtmlOnce to run — so an interrupted --rendered
+// fetch otherwise abandons a whole Chrome process tree (main + zygote/gpu/renderer children). The
+// plain 'exit' event covers ordinary completion (belt-and-braces alongside close()'s own cleanup);
+// a `once` signal listener runs the same cleanup, then re-sends the signal to this process. Because
+// `once` already removed itself, that re-delivery has no listener left and Node's own default
+// disposition takes over — the process still exits with the normal 128+signum code, with no exit-code
+// bookkeeping of our own.
+const liveChromeProcesses = new Set();
+let chromeCleanupRegistered = false;
+function killLiveChromeProcesses() {
+    for (const chromeProcess of liveChromeProcesses)
+        killChromeTree(chromeProcess);
+}
+function ensureChromeCleanupRegistered() {
+    if (chromeCleanupRegistered)
+        return;
+    chromeCleanupRegistered = true;
+    process.once('exit', killLiveChromeProcesses);
+    for (const signal of ['SIGINT', 'SIGTERM']) {
+        process.once(signal, () => {
+            killLiveChromeProcesses();
+            process.kill(process.pid, signal);
+        });
+    }
+}
+/**
+ * Kill Chrome and every child process it spawned (zygote, gpu-process, renderer, crashpad).
+ * `chromeProcess.kill()` alone only signals that one PID — per Node's child_process docs, a signal
+ * sent to a child is not propagated to *that child's own* children on Linux, so Chrome's
+ * multi-process tree survives a plain kill on every rendered fetch, not just an interrupted one.
+ * Spawning with `detached: true` makes Chrome the leader of its own process group, so signaling the
+ * negative PID reaches the whole tree in one call. Falls back to a direct kill if group-kill fails
+ * (already exited, or a platform without POSIX process groups).
+ */
+export function killChromeTree(chromeProcess) {
+    if (typeof chromeProcess.pid === 'number') {
+        try {
+            process.kill(-chromeProcess.pid, 'SIGKILL');
+            return;
+        }
+        catch {
+            // Fall through to a best-effort direct kill below.
+        }
+    }
+    chromeProcess.kill('SIGKILL');
+}
 async function spawnChrome(chromePath) {
-    const chromeProcess = spawn(chromePath, buildChromeArgs());
+    ensureChromeCleanupRegistered();
+    const chromeProcess = spawn(chromePath, buildChromeArgs(), { detached: true });
+    liveChromeProcesses.add(chromeProcess);
+    chromeProcess.once('exit', () => liveChromeProcesses.delete(chromeProcess));
     try {
         const wsUrl = await new Promise((resolve, reject) => {
             let output = '';
             const timeoutId = setTimeout(() => {
-                chromeProcess.kill('SIGKILL');
+                killChromeTree(chromeProcess);
                 reject(new Error('Timed out waiting for Chrome to start.'));
             }, CHROME_STARTUP_TIMEOUT_MS);
             const onData = (data) => {
@@ -168,7 +218,7 @@ async function spawnChrome(chromePath) {
         return { chromeProcess, wsUrl };
     }
     catch (err) {
-        chromeProcess.kill('SIGKILL');
+        killChromeTree(chromeProcess);
         throw err;
     }
 }
@@ -220,13 +270,41 @@ export function assertRenderedHttpOk(mainDoc) {
     }
 }
 /**
- * Spawns a headless Chrome, attaches a flat CDP session, and enables the Page, Network,
- * and Runtime domains. The caller drives navigation and must call close() when finished.
+ * Validates a navigated (or redirected) Document request against the same scheme/credentials/DNS
+ * safety `normalizeUrl` and `fetchWithRedirects` enforce per hop in url.ts/fetcher.ts. Chrome
+ * follows redirects internally with no hook back into that check, so a page that starts on a safe
+ * public host could otherwise redirect a rendered-fallback capture into a private/internal address
+ * (SSRF) or a non-http(s) scheme (e.g. `file:`) that the static fetcher would have rejected outright.
+ */
+export async function describeUnsafeNavigationTarget(url) {
+    let normalized;
+    try {
+        normalized = normalizeUrl(url);
+    }
+    catch (err) {
+        return err;
+    }
+    try {
+        await checkDnsSafety(new URL(normalized).hostname);
+        return null;
+    }
+    catch (err) {
+        return err;
+    }
+}
+/**
+ * Spawns a headless Chrome, attaches a flat CDP session, and enables the Page, Network, Runtime,
+ * and Fetch domains. Fetch domain interception guards every Document-type request (the initial
+ * navigation and each subsequent redirect) with {@link describeUnsafeNavigationTarget} before
+ * Chrome is allowed to send it; a blocked request's reason is recorded and can be read back via
+ * the returned page's `takeBlockedNavigationError()`. The caller drives navigation and must call
+ * close() when finished.
  */
 export async function openCdpPage() {
     const chromePath = findChromePath();
     const { chromeProcess, wsUrl } = await spawnChrome(chromePath);
     const client = new CdpClient(wsUrl);
+    let blockedNavigationError = null;
     try {
         await client.connect();
         const { targetId } = await client.send('Target.createTarget', { url: 'about:blank' });
@@ -235,19 +313,41 @@ export async function openCdpPage() {
         await client.send('Network.enable', {}, sessionId);
         await client.send('Runtime.enable', {}, sessionId);
         await client.send('Network.setUserAgentOverride', { userAgent: DEFAULT_USER_AGENT }, sessionId);
+        await client.send('Fetch.enable', { patterns: [{ urlPattern: '*', resourceType: 'Document', requestStage: 'Request' }] }, sessionId);
+        client.on(`${sessionId}:Fetch.requestPaused`, (params) => {
+            // CdpClient.on's handler type is synchronous (CDP event dispatch isn't awaited), so the
+            // actual async validate-then-continue/fail work runs in this detached IIFE instead.
+            void (async () => {
+                const { requestId, request } = params;
+                const blockReason = await describeUnsafeNavigationTarget(request.url);
+                if (blockReason) {
+                    blockedNavigationError = blockReason;
+                    await client
+                        .send('Fetch.failRequest', { requestId, errorReason: 'BlockedByClient' }, sessionId)
+                        .catch(() => { });
+                    return;
+                }
+                await client.send('Fetch.continueRequest', { requestId }, sessionId).catch(() => { });
+            })();
+        });
         const close = async () => {
             try {
                 await client.send('Target.closeTarget', { targetId });
             }
             catch { }
             client.close();
-            chromeProcess.kill('SIGKILL');
+            killChromeTree(chromeProcess);
         };
-        return { client, sessionId, close };
+        const takeBlockedNavigationError = () => {
+            const err = blockedNavigationError;
+            blockedNavigationError = null;
+            return err;
+        };
+        return { client, sessionId, close, takeBlockedNavigationError };
     }
     catch (err) {
         client.close();
-        chromeProcess.kill('SIGKILL');
+        killChromeTree(chromeProcess);
         throw err;
     }
 }
@@ -332,8 +432,8 @@ const PROXY_BLOCKED_CHROME_NET_ERRORS = [
     'ERR_PROXY_CONNECTION_FAILED',
     'ERR_SOCKS_CONNECTION_FAILED',
 ];
-// Shared substring fetch.ts's fetchFailureGuidance matches on to attach suggestions/ref to this
-// error, without duplicating the message text it needs to match against.
+// Shared substring fetch-result.ts's fetchFailureGuidance matches on to attach suggestions/ref to
+// this error, without duplicating the message text it needs to match against.
 export const SANDBOX_EGRESS_ERROR_MARKER = 'sandboxed execution environment';
 /**
  * Builds the error thrown for a failed Page.navigate. When a proxy is configured (proxy.ts's
@@ -364,8 +464,11 @@ export function describeNavigationFailure(errorText, url) {
         `or bypassing the proxy: confirm "${host}" is reachable from this environment (e.g. test the ` +
         `proxy directly), or fetch the page from a network with access and import it instead.`);
 }
-// Chrome can emit these on first navigation in headless CI when the cert verifier reloads mid-run.
-const TRANSIENT_NAV_ERROR_CODES = ['ERR_CERT_VERIFIER_CHANGED'];
+// ERR_CERT_VERIFIER_CHANGED: Chrome can emit this on first navigation in headless CI when the cert
+// verifier reloads mid-run. ERR_CONNECTION_CLOSED is Chromium's code for a plain remote TCP FIN
+// (net_error_list.h), which CI runners can hit as a one-off network blip against a real host
+// rather than a real block.
+const TRANSIENT_NAV_ERROR_CODES = ['ERR_CERT_VERIFIER_CHANGED', 'ERR_CONNECTION_CLOSED'];
 // Resource contention in CI (concurrent jobs competing for CPU/disk) can make Chrome itself fail
 // to come up in time, or die before its devtools websocket URL is printed. Neither reflects a
 // problem with the target page, so it's worth a fresh attempt rather than failing the whole fetch.
@@ -373,11 +476,93 @@ const TRANSIENT_CHROME_STARTUP_MESSAGES = [
     'Timed out waiting for Chrome to start.',
     'Chrome exited prematurely',
 ];
-function isTransientBrowserError(err) {
+// Exported so tests can assert the transient-error classification directly rather than only
+// indirectly through a live fetchRenderedHtml retry, which needs a real Chrome navigation to fail.
+export function isTransientBrowserError(err) {
     const message = err instanceof Error ? err.message : String(err);
     return (TRANSIENT_NAV_ERROR_CODES.some((code) => message.includes(code)) ||
         TRANSIENT_CHROME_STARTUP_MESSAGES.some((text) => message.includes(text)));
 }
+/**
+ * Navigates an already-open CDP page to `currentUrl` and captures the rendered HTML, honoring the
+ * openCdpPage Fetch-domain safety guard at every point a blocked navigation could otherwise be
+ * discarded in favor of a less specific (or entirely silent) failure:
+ *
+ * 1. Right after `Page.navigate` returns — a fast HTTP redirect chain can already be blocked and
+ *    resolved by this point, surfacing as the generic `nav.errorText` (e.g. `net::ERR_BLOCKED_BY_CLIENT`)
+ *    below if not checked first.
+ * 2. If `waitForLoad` throws — a client-side (JS-triggered) navigation into a blocked target after
+ *    the initial navigation already settled can prevent the load/domContentLoaded events
+ *    `waitForLoad` is waiting on, surfacing as a generic timeout instead of the specific reason.
+ * 3. After `waitForLoad` resolves — a client-side navigation blocked without otherwise disrupting
+ *    the load signal would otherwise sail through silently, capturing whatever the page settled on.
+ *
+ * Exported (rather than folded into `fetchRenderedHtmlOnce`) so tests can drive it against a fake
+ * `CdpPage` without spawning a real browser.
+ */
+export async function captureNavigatedPage(page, currentUrl, options) {
+    const { timeout, limit, settleMs } = options;
+    await page.client.send('Network.setBlockedURLs', { urls: BLOCKED_ASSET_URLS }, page.sessionId);
+    // Record the status of each frame's first Document response so we can reject HTTP errors after
+    // the page settles. Keyed by frameId; the navigated top frame is matched via the navigate result.
+    const documentResponses = new Map();
+    page.client.on(`${page.sessionId}:Network.responseReceived`, (params) => {
+        const { frameId } = params;
+        const status = params.response?.status;
+        // Only record real numeric statuses: a Document event without a usable status would
+        // otherwise store NaN, which silently slips past the `>= 400` guard and the `?? 200`
+        // fallback alike, leaking a NaN into BrowserFetchResult.status (declared `number`).
+        if (params.type !== 'Document' || !frameId || typeof status !== 'number')
+            return;
+        if (documentResponses.has(frameId))
+            return;
+        documentResponses.set(frameId, {
+            status,
+            statusText: params.response?.statusText ?? '',
+        });
+    });
+    const nav = await page.client.send('Page.navigate', { url: currentUrl }, page.sessionId);
+    // See point 1 above: prefer the safety guard's specific reason over the opaque errorText below.
+    const blockedNavigation = page.takeBlockedNavigationError();
+    if (blockedNavigation)
+        throw blockedNavigation;
+    // A network-level failure (DNS, refused/closed connection, TLS) never fires a load event, so
+    // fail fast here instead of waiting out the full navigation timeout on a dead page.
+    if (nav.errorText) {
+        throw describeNavigationFailure(nav.errorText, currentUrl);
+    }
+    try {
+        await waitForLoad(page.client, page.sessionId, timeout, settleMs);
+    }
+    catch (err) {
+        // See point 2 above: a blocked reason recorded during the wait outranks the generic timeout.
+        throw page.takeBlockedNavigationError() ?? err;
+    }
+    // See point 3 above.
+    const blockedNavigationAfterLoad = page.takeBlockedNavigationError();
+    if (blockedNavigationAfterLoad)
+        throw blockedNavigationAfterLoad;
+    const mainDoc = nav.frameId ? documentResponses.get(nav.frameId) : undefined;
+    assertRenderedHttpOk(mainDoc);
+    const evalResult = await page.client.send('Runtime.evaluate', { expression: 'document.documentElement.outerHTML', returnByValue: true }, page.sessionId);
+    const html = evalResult.result?.value;
+    if (typeof html !== 'string') {
+        throw new Error('Failed to retrieve rendered HTML from browser page.');
+    }
+    const byteLength = Buffer.byteLength(html);
+    if (byteLength > limit) {
+        throw new Error(`Response body size limit exceeded. Rendered HTML is ${byteLength} bytes.`);
+    }
+    return {
+        status: mainDoc?.status ?? 200,
+        contentType: 'text/html',
+        etag: null,
+        lastModified: null,
+        finalUrl: currentUrl,
+        responseSize: byteLength,
+        content: html,
+    };
+}
 async function fetchRenderedHtmlOnce(url, options = {}) {
     const timeout = options.timeoutMs ?? 15_000;
     const limit = options.bodyLimitBytes ?? 2 * 1024 * 1024;
@@ -385,52 +570,11 @@ async function fetchRenderedHtmlOnce(url, options = {}) {
     await checkDnsSafety(new URL(currentUrl).hostname);
     const page = await openCdpPage();
     try {
-        await page.client.send('Network.setBlockedURLs', { urls: BLOCKED_ASSET_URLS }, page.sessionId);
-        // Record the status of each frame's first Document response so we can reject HTTP errors after
-        // the page settles. Keyed by frameId; the navigated top frame is matched via the navigate result.
-        const documentResponses = new Map();
-        page.client.on(`${page.sessionId}:Network.responseReceived`, (params) => {
-            const { frameId } = params;
-            const status = params.response?.status;
-            // Only record real numeric statuses: a Document event without a usable status would
-            // otherwise store NaN, which silently slips past the `>= 400` guard and the `?? 200`
-            // fallback alike, leaking a NaN into BrowserFetchResult.status (declared `number`).
-            if (params.type !== 'Document' || !frameId || typeof status !== 'number')
-                return;
-            if (documentResponses.has(frameId))
-                return;
-            documentResponses.set(frameId, {
-                status,
-                statusText: params.response?.statusText ?? '',
-            });
+        return await captureNavigatedPage(page, currentUrl, {
+            timeout,
+            limit,
+            settleMs: options.settleMs ?? 1000,
         });
-        const nav = await page.client.send('Page.navigate', { url: currentUrl }, page.sessionId);
-        // A network-level failure (DNS, refused/closed connection, TLS) never fires a load event, so
-        // fail fast here instead of waiting out the full navigation timeout on a dead page.
-        if (nav.errorText) {
-            throw describeNavigationFailure(nav.errorText, currentUrl);
-        }
-        await waitForLoad(page.client, page.sessionId, timeout, options.settleMs ?? 1000);
-        const mainDoc = nav.frameId ? documentResponses.get(nav.frameId) : undefined;
-        assertRenderedHttpOk(mainDoc);
-        const evalResult = await page.client.send('Runtime.evaluate', { expression: 'document.documentElement.outerHTML', returnByValue: true }, page.sessionId);
-        const html = evalResult.result?.value;
-        if (typeof html !== 'string') {
-            throw new Error('Failed to retrieve rendered HTML from browser page.');
-        }
-        const byteLength = Buffer.byteLength(html);
-        if (byteLength > limit) {
-            throw new Error(`Response body size limit exceeded. Rendered HTML is ${byteLength} bytes.`);
-        }
-        return {
-            status: mainDoc?.status ?? 200,
-            contentType: 'text/html',
-            etag: null,
-            lastModified: null,
-            finalUrl: currentUrl,
-            responseSize: byteLength,
-            content: html,
-        };
     }
     finally {
         await page.close();

package/dist/lib/research/capture.d.ts

@@ -1,16 +1,19 @@
-import type { FetchResult } from './fetcher.js';
+import { type FetchResult } from './fetcher.js';
 import type { ExtractionResult } from './extract.js';
 import type { CaptureMethod } from './schema.js';
 import type { MachineReadableArtifact, SiteCapabilities } from './docs/capabilities.js';
 import { type TextFetcher } from './docs/machine-readable.js';
+/** Injected fetcher dependencies for `capturePage`. All three are required. */
 export interface CaptureDeps {
     fetchStatic: (url: string) => Promise<FetchResult>;
     fetchRendered: (url: string) => Promise<FetchResult>;
     fetchText: TextFetcher;
 }
+/** Options passed to `capturePage` by the caller. */
 export interface CaptureOptions {
     forceRendered?: boolean;
 }
+/** Full outcome of a `capturePage` call, including provenance and capability metadata. */
 export interface CaptureOutcome {
     fetchResult: FetchResult;
     extraction: ExtractionResult;
@@ -20,10 +23,29 @@ export interface CaptureOutcome {
     machineReadable: MachineReadableArtifact[];
     attemptedMethods: string[];
 }
-export declare function renderedFallbackNeeded(caps: SiteCapabilities, extraction: ExtractionResult | null): boolean;
+/**
+ * Returns true when the static extraction result is missing or too weak to serve as the artifact,
+ * meaning the caller should attempt a rendered fallback.
+ *
+ * @param caps - Detected site capabilities; `recommendedCapture === 'rendered'` forces a fallback.
+ * @param extraction - Result of static extraction, or null when static fetch failed.
+ * @param staticError - The static fetch's failure, if any. A content-type policy rejection (JSON,
+ *   PDF, binary) means the URL isn't a web page at all — a browser would just render its own
+ *   built-in viewer for it, producing a corrupted-looking "extracted" artifact — so that specific
+ *   failure is a hard stop, never a fallback trigger. `--rendered` still bypasses this entirely (it
+ *   skips the static phase), so an informed retry remains available.
+ * @returns True when a rendered (browser) fallback should be attempted.
+ */
+export declare function renderedFallbackNeeded(caps: SiteCapabilities, extraction: ExtractionResult | null, staticError?: Error | null): boolean;
 /**
  * Captures a page's best-available content. Order: forced rendered → route/source Markdown → static
  * HTML → rendered fallback. Returns the chosen fetch result, extraction, detected capabilities, and
  * provenance (capture method, source URL, machine-readable artifacts, attempted methods).
+ *
+ * @param url - The page URL to capture.
+ * @param options - Capture options (e.g. `forceRendered`).
+ * @param deps - Injected fetchers for static, rendered, and text requests.
+ * @returns Full CaptureOutcome including the best extraction and capture provenance.
+ * @throws {Error} When all capture strategies fail and no usable content is produced.
  */
 export declare function capturePage(url: string, options: CaptureOptions, deps: CaptureDeps): Promise<CaptureOutcome>;

package/dist/lib/research/capture.js

@@ -1,3 +1,4 @@
+import { isUnsupportedContentTypeError } from './fetcher.js';
 import { extractHtmlContent } from './extract.js';
 import { detectDocsEngine } from './docs/detect.js';
 import { emptyCapabilities } from './docs/capabilities.js';
@@ -6,9 +7,22 @@ import { mapUrlToSource } from './docs/source-map.js';
 import { extractFromSource } from './docs/markdown-source.js';
 import { validateTextArtifact } from './docs/validate.js';
 const MIN_USEFUL_CHARS = 600;
-// A static extraction needs rendering help when it failed outright, the detector says the page is
-// an SPA/client-rendered, or the extracted article is too thin/low-confidence to be useful.
-export function renderedFallbackNeeded(caps, extraction) {
+/**
+ * Returns true when the static extraction result is missing or too weak to serve as the artifact,
+ * meaning the caller should attempt a rendered fallback.
+ *
+ * @param caps - Detected site capabilities; `recommendedCapture === 'rendered'` forces a fallback.
+ * @param extraction - Result of static extraction, or null when static fetch failed.
+ * @param staticError - The static fetch's failure, if any. A content-type policy rejection (JSON,
+ *   PDF, binary) means the URL isn't a web page at all — a browser would just render its own
+ *   built-in viewer for it, producing a corrupted-looking "extracted" artifact — so that specific
+ *   failure is a hard stop, never a fallback trigger. `--rendered` still bypasses this entirely (it
+ *   skips the static phase), so an informed retry remains available.
+ * @returns True when a rendered (browser) fallback should be attempted.
+ */
+export function renderedFallbackNeeded(caps, extraction, staticError = null) {
+    if (staticError && isUnsupportedContentTypeError(staticError))
+        return false;
     if (!extraction)
         return true;
     if (caps.recommendedCapture === 'rendered')
@@ -116,7 +130,7 @@ async function sourceOutcome(url, phase, deps, attemptedMethods) {
 // is warranted; throws only when rendering fails and nothing usable was produced (preferring the
 // earlier static error).
 async function renderedFallbackOutcome(url, deps, attemptedMethods, phase, extraction) {
-    if (!renderedFallbackNeeded(phase.capabilities, extraction))
+    if (!renderedFallbackNeeded(phase.capabilities, extraction, phase.staticError))
         return null;
     try {
         return await renderedOutcome(url, deps, attemptedMethods, phase.machineReadable);
@@ -131,6 +145,12 @@ async function renderedFallbackOutcome(url, deps, attemptedMethods, phase, extra
  * Captures a page's best-available content. Order: forced rendered → route/source Markdown → static
  * HTML → rendered fallback. Returns the chosen fetch result, extraction, detected capabilities, and
  * provenance (capture method, source URL, machine-readable artifacts, attempted methods).
+ *
+ * @param url - The page URL to capture.
+ * @param options - Capture options (e.g. `forceRendered`).
+ * @param deps - Injected fetchers for static, rendered, and text requests.
+ * @returns Full CaptureOutcome including the best extraction and capture provenance.
+ * @throws {Error} When all capture strategies fail and no usable content is produced.
  */
 export async function capturePage(url, options, deps) {
     const attemptedMethods = [];

package/dist/lib/research/cli-io.d.ts

@@ -0,0 +1,40 @@
+/** Options accepted by {@link CliIo.error} — the subset of oclif's `this.error` options Bonsai uses. */
+export interface CliErrorOptions {
+    exit: number;
+    code?: string;
+    suggestions?: string[];
+    ref?: string;
+}
+/**
+ * Narrow command-context port passed from oclif commands into command services, so the services
+ * stay free of oclif imports and are testable without a framework harness.
+ */
+export interface CliIo {
+    bin: string;
+    configDir: string | undefined;
+    dataDir: string;
+    cwd: string;
+    json: boolean;
+    warn(msg: string): void;
+    log(msg: string): void;
+    error(msg: string, opts: CliErrorOptions): never;
+    /**
+     * Print one row of a batch failure with the same "Error: … / Code: … / Try this: …" rendering as
+     * {@link error}, but without exiting the process — a batch (`fetch url1 url2`) must keep
+     * processing the remaining rows after one fails. Using `warn` for this mislabels a real failure
+     * as non-fatal even though the row still flips the command's exit code and JSON `ok` to false.
+     */
+    errorRow(msg: string, opts: Omit<CliErrorOptions, 'exit'>): void;
+}
+/**
+ * Single exit point for a URL that failed normalization, shared by BaseCommand and the command
+ * services. A scheme-less but domain-shaped input reports MISSING_URL_SCHEME — the same code the
+ * root `bonsai <url>` shorthand uses — so an agent sees one stable code for "forgot https://"
+ * everywhere; truly unparseable input stays INVALID_URL.
+ *
+ * @param error - `this.error` (oclif) or `io.error`; must not return.
+ * @param url - Raw URL string the user supplied.
+ * @param message - Human-readable failure message.
+ * @returns Never — always exits via `error`.
+ */
+export declare function failInvalidUrl(error: (msg: string, opts: CliErrorOptions) => never, url: string, message: string): never;