@taurgis/bonsai 3.0.1 → 3.0.3

package/dist/commands/fetch.js

@@ -10,7 +10,7 @@ import { evaluateFreshness, checkMaxAgeExpired, durationFlagError, } from '../li
 import { revalidateCache, createArtifactFromFetch } from '../lib/research/revalidate.js';
 import { fetchStaticHtml, fetchText } from '../lib/research/fetcher.js';
 import { PROXY_TUNNEL_REJECTION_PATTERN } from '../lib/research/proxy.js';
-import { fetchRenderedHtml } from '../lib/research/browser.js';
+import { fetchRenderedHtml, SANDBOX_EGRESS_ERROR_MARKER } from '../lib/research/browser.js';
 import { capturePage } from '../lib/research/capture.js';
 import { persistSectionArtifacts } from '../lib/research/docs/section-artifacts.js';
 import { applyAutoTags } from '../lib/research/keywords.js';
@@ -387,6 +387,12 @@ export function fetchFailureGuidance(message, url) {
             suggestions: ['Check the hostname is spelled correctly and resolves on a public network.'],
         };
     }
+    // Chrome's own network stack hit a proxy-enforced sandbox egress block (browser.ts's
+    // describeNavigationFailure already explains the cause and the fix); only a workaround and the
+    // docs ref are added here.
+    if (message.includes(SANDBOX_EGRESS_ERROR_MARKER)) {
+        return { suggestions: [importHint], ref };
+    }
     // A configured proxy (HTTPS_PROXY/HTTP_PROXY) refused to tunnel to this host — distinct from
     // the destination's own 401/403 above, which is the site itself rejecting the request. Shared
     // with fetcher.ts's retry-fallback check so the two stay in sync by construction.

package/dist/lib/research/browser.d.ts

@@ -76,4 +76,17 @@ export declare class ResponseCapture {
     /** Resolves with the response body text once a matching response finishes, or null on timeout. */
     waitFor(key: string, timeoutMs: number): Promise<string | null>;
 }
+export declare const SANDBOX_EGRESS_ERROR_MARKER = "sandboxed execution environment";
+/**
+ * Builds the error thrown for a failed Page.navigate. When a proxy is configured (proxy.ts's
+ * signal for "this is a sandboxed execution environment") and Chrome's failure code indicates the
+ * proxy itself declined the tunnel, the destination isn't reachable because of the sandbox's
+ * network egress gateway rejecting it, not a real navigation problem — so the message says that
+ * explicitly, rather than surfacing a bare Chrome net-error code that looks like a generic
+ * connectivity failure. It deliberately doesn't claim the fix is "ask an admin to allowlist this
+ * host": the gateway that rejects the CONNECT tunnel can disagree with an org's configured domain
+ * policy (a different layer may not reach it), so the safer, always-correct advice is to verify
+ * against the environment directly and fall back to a manual fetch if it's still unreachable.
+ */
+export declare function describeNavigationFailure(errorText: string, url: string): Error;
 export declare function fetchRenderedHtml(url: string, options?: BrowserFetchOptions): Promise<BrowserFetchResult>;

package/dist/lib/research/browser.js

@@ -1,8 +1,9 @@
 import { spawn } from 'node:child_process';
 import { existsSync } from 'node:fs';
 import { execSync } from 'node:child_process';
+import { join } from 'node:path';
 import { checkDnsSafety } from './fetcher.js';
-import { getChromeProxyArgs } from './proxy.js';
+import { getChromeProxyArgs, getChromeSpkiArgs, getChromeTlsCompatibilityArgs, isProxyConfigured, } from './proxy.js';
 import { normalizeUrl } from './url.js';
 export class CdpClient {
     ws;
@@ -65,6 +66,13 @@ export function findChromePath() {
         return process.env.CHROME_PATH;
     }
     const paths = [];
+    // Claude Code's remote sandbox (and other Playwright-provisioned environments) pre-installs a
+    // Chromium build under PLAYWRIGHT_BROWSERS_PATH rather than a system package manager location,
+    // with a `chromium` symlink at its root pointing at the actual browser binary. It's headless-Chrome
+    // capable (needed for the CDP `--headless=new` flag this module uses), so it's checked first.
+    if (process.env.PLAYWRIGHT_BROWSERS_PATH) {
+        paths.push(join(process.env.PLAYWRIGHT_BROWSERS_PATH, 'chromium'));
+    }
     if (process.platform === 'darwin') {
         paths.push('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', '/Applications/Chromium.app/Contents/MacOS/Chromium');
     }
@@ -108,6 +116,15 @@ export function buildChromeArgs() {
         // CDP navigation uses Chrome's own network stack, not Node's fetch/undici, so a sandbox
         // proxy detected via *_PROXY env vars must be passed to Chrome separately (see proxy.ts).
         ...getChromeProxyArgs(),
+        // The sandbox proxy above re-terminates TLS with its own CA, which Chrome's Chrome-Root-Store
+        // verifier doesn't trust by default (unlike Node/curl/Python, which read NODE_EXTRA_CA_CERTS/
+        // SSL_CERT_FILE/CURL_CA_BUNDLE). Pin trust to that specific CA rather than disabling
+        // verification wholesale (see proxy.ts's getChromeSpkiArgs).
+        ...getChromeSpkiArgs(),
+        // Separately from cert trust: some sandbox proxies' TLS terminator never responds to Chrome's
+        // default TLS 1.3 ClientHello and the connection is eventually reset (see proxy.ts's
+        // getChromeTlsCompatibilityArgs for how this was diagnosed).
+        ...getChromeTlsCompatibilityArgs(),
     ];
 }
 async function spawnChrome(chromePath) {
@@ -303,6 +320,47 @@ export class ResponseCapture {
         });
     }
 }
+// Chrome has its own network stack (see getChromeProxyArgs), so when a sandbox's egress-enforcing
+// proxy declines to tunnel to a destination, Chrome surfaces one of these net-error codes instead
+// of the HTTP-level rejection undici sees in fetcher.ts (PROXY_TUNNEL_REJECTION_PATTERN). Matched
+// against Page.navigate's `errorText`, which arrives as e.g. "net::ERR_TUNNEL_CONNECTION_FAILED".
+const PROXY_BLOCKED_CHROME_NET_ERRORS = [
+    'ERR_TUNNEL_CONNECTION_FAILED',
+    'ERR_PROXY_CONNECTION_FAILED',
+    'ERR_SOCKS_CONNECTION_FAILED',
+];
+// Shared substring fetch.ts's fetchFailureGuidance matches on to attach suggestions/ref to this
+// error, without duplicating the message text it needs to match against.
+export const SANDBOX_EGRESS_ERROR_MARKER = 'sandboxed execution environment';
+/**
+ * Builds the error thrown for a failed Page.navigate. When a proxy is configured (proxy.ts's
+ * signal for "this is a sandboxed execution environment") and Chrome's failure code indicates the
+ * proxy itself declined the tunnel, the destination isn't reachable because of the sandbox's
+ * network egress gateway rejecting it, not a real navigation problem — so the message says that
+ * explicitly, rather than surfacing a bare Chrome net-error code that looks like a generic
+ * connectivity failure. It deliberately doesn't claim the fix is "ask an admin to allowlist this
+ * host": the gateway that rejects the CONNECT tunnel can disagree with an org's configured domain
+ * policy (a different layer may not reach it), so the safer, always-correct advice is to verify
+ * against the environment directly and fall back to a manual fetch if it's still unreachable.
+ */
+export function describeNavigationFailure(errorText, url) {
+    const isProxyBlock = isProxyConfigured() && PROXY_BLOCKED_CHROME_NET_ERRORS.some((code) => errorText.includes(code));
+    if (!isProxyBlock) {
+        return new Error(`Navigation failed: ${errorText}`);
+    }
+    let host = url;
+    try {
+        host = new URL(url).hostname;
+    }
+    catch { }
+    return new Error(`Chrome could not reach "${host}" (${errorText}). This looks like a ${SANDBOX_EGRESS_ERROR_MARKER} ` +
+        `(e.g. Claude Code's remote sandbox) whose outbound network gateway rejected the connection to ` +
+        `this host, via the configured proxy (HTTPS_PROXY/HTTP_PROXY). This can happen even when the ` +
+        `environment's domain policy looks unrestricted, since the gateway enforcing egress can be a ` +
+        `separate layer from that policy. It can't be worked around from inside the sandbox by retrying ` +
+        `or bypassing the proxy: confirm "${host}" is reachable from this environment (e.g. test the ` +
+        `proxy directly), or fetch the page from a network with access and import it instead.`);
+}
 // Chrome can emit these on first navigation in headless CI when the cert verifier reloads mid-run.
 const TRANSIENT_NAV_ERROR_CODES = ['ERR_CERT_VERIFIER_CHANGED'];
 function isTransientBrowserNavigationError(err) {
@@ -339,7 +397,7 @@ async function fetchRenderedHtmlOnce(url, options = {}) {
         // A network-level failure (DNS, refused/closed connection, TLS) never fires a load event, so
         // fail fast here instead of waiting out the full navigation timeout on a dead page.
         if (nav.errorText) {
-            throw new Error(`Navigation failed: ${nav.errorText}`);
+            throw describeNavigationFailure(nav.errorText, currentUrl);
         }
         await waitForLoad(page.client, page.sessionId, timeout, options.settleMs ?? 1000);
         const mainDoc = nav.frameId ? documentResponses.get(nav.frameId) : undefined;

package/dist/lib/research/proxy.d.ts

@@ -33,3 +33,39 @@ export declare function getProxyDispatcher(): Dispatcher | undefined;
  * be set, silently disagreeing with how fetch() routes the same traffic.
  */
 export declare function getChromeProxyArgs(): string[];
+/**
+ * Caps Chrome's TLS version at 1.2 for connections made through a detected sandbox proxy, or []
+ * when no proxy is configured. Some sandbox proxies' TLS-terminating layer never responds to
+ * Chrome's default TLS 1.3 ClientHello (BoringSSL sends ECH-GREASE by default, which most tools'
+ * TLS stacks — curl/OpenSSL's included — don't add): captured via --log-net-log, Chrome sends its
+ * ClientHello, the proxy sends nothing back, and ~6s later the socket read fails with
+ * net::ERR_CONNECTION_RESET. Capping to TLS 1.2 avoids that ClientHello shape entirely. Scoped to
+ * the proxied path only (via isProxyConfigured, the same signal every other sandbox-specific flag
+ * in this module uses) so ordinary developer machines get Chrome's normal TLS 1.3 negotiation
+ * unchanged; this doesn't affect certificate verification (see getChromeSpkiArgs), only which
+ * protocol version Chrome offers.
+ */
+export declare function getChromeTlsCompatibilityArgs(): string[];
+export declare const CA_BUNDLE_ENV_VARS: string[];
+/**
+ * Base64-encoded SHA-256 SPKI fingerprints (RFC 7469 §2.4) for every certificate in the sandbox's
+ * discoverable CA bundle, in the exact encoding Chromium's --ignore-certificate-errors-spki-list
+ * expects — equivalent to `openssl x509 -pubkey -noout | openssl pkey -pubin -outform der |
+ * openssl dgst -sha256 -binary | openssl enc -base64` for each cert. Returns [] when no bundle is
+ * discoverable or none of its entries parse, so a missing/malformed bundle degrades to Chrome's
+ * normal (unpinned) verification rather than breaking the browser launch.
+ */
+export declare function getCaBundleSpkiHashes(): string[];
+/**
+ * Chrome CLI flag that trusts exactly the sandbox's injected CA(s), pinned by SPKI hash rather
+ * than by disabling verification. Chromium's TLS stack (the Chrome Root Store) never reads
+ * NODE_EXTRA_CA_CERTS/SSL_CERT_FILE/CURL_CA_BUNDLE the way Node/curl/Python do, so a sandbox that
+ * MITMs HTTPS through its proxy (see isProxyConfigured) makes every other tool trust the proxy's
+ * cert while Chrome's handshake with it fails with ERR_CERT_AUTHORITY_INVALID. This is scoped
+ * narrowly to the specific CA(s) discovered via the CA bundle env vars — never
+ * --ignore-certificate-errors, which would disable verification for every host, including ones
+ * that have nothing to do with the sandbox's proxy. A no-op (empty array) when no proxy is
+ * configured or no CA bundle is discoverable, so default Chrome trust is untouched on ordinary
+ * developer machines.
+ */
+export declare function getChromeSpkiArgs(): string[];

package/dist/lib/research/proxy.js

@@ -1,3 +1,5 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { createHash, X509Certificate } from 'node:crypto';
 import { EnvHttpProxyAgent } from 'undici';
 // Order matters: undici's own EnvHttpProxyAgent (github.com/nodejs/undici lib/dispatcher/
 // env-http-proxy-agent.js) prefers the lowercase variant of each var over its uppercase twin
@@ -105,3 +107,82 @@ export function getChromeProxyArgs() {
         args.push(`--proxy-bypass-list=${toChromeBypassList(noProxy)}`);
     return args;
 }
+/**
+ * Caps Chrome's TLS version at 1.2 for connections made through a detected sandbox proxy, or []
+ * when no proxy is configured. Some sandbox proxies' TLS-terminating layer never responds to
+ * Chrome's default TLS 1.3 ClientHello (BoringSSL sends ECH-GREASE by default, which most tools'
+ * TLS stacks — curl/OpenSSL's included — don't add): captured via --log-net-log, Chrome sends its
+ * ClientHello, the proxy sends nothing back, and ~6s later the socket read fails with
+ * net::ERR_CONNECTION_RESET. Capping to TLS 1.2 avoids that ClientHello shape entirely. Scoped to
+ * the proxied path only (via isProxyConfigured, the same signal every other sandbox-specific flag
+ * in this module uses) so ordinary developer machines get Chrome's normal TLS 1.3 negotiation
+ * unchanged; this doesn't affect certificate verification (see getChromeSpkiArgs), only which
+ * protocol version Chrome offers.
+ */
+export function getChromeTlsCompatibilityArgs() {
+    return isProxyConfigured() ? ['--ssl-version-max=tls1.2'] : [];
+}
+// Order matters for the same reason as the *_PROXY lists above: read consistently so this module
+// and every caller agree on which file is "the" CA bundle. Node itself only honors
+// NODE_EXTRA_CA_CERTS; SSL_CERT_FILE and CURL_CA_BUNDLE are the conventions OpenSSL-linked tools
+// (curl, Python, etc.) use instead. A sandbox that MITMs TLS to route through its proxy sets all
+// three to the same bundle so every tool picks it up regardless of which convention it follows.
+export const CA_BUNDLE_ENV_VARS = ['NODE_EXTRA_CA_CERTS', 'SSL_CERT_FILE', 'CURL_CA_BUNDLE'];
+function findCaBundlePath() {
+    const path = firstEnv(CA_BUNDLE_ENV_VARS);
+    return path && existsSync(path) ? path : undefined;
+}
+function extractPemCertificates(bundle) {
+    return bundle.match(/-----BEGIN CERTIFICATE-----[\s\S]+?-----END CERTIFICATE-----/g) ?? [];
+}
+/**
+ * Base64-encoded SHA-256 SPKI fingerprints (RFC 7469 §2.4) for every certificate in the sandbox's
+ * discoverable CA bundle, in the exact encoding Chromium's --ignore-certificate-errors-spki-list
+ * expects — equivalent to `openssl x509 -pubkey -noout | openssl pkey -pubin -outform der |
+ * openssl dgst -sha256 -binary | openssl enc -base64` for each cert. Returns [] when no bundle is
+ * discoverable or none of its entries parse, so a missing/malformed bundle degrades to Chrome's
+ * normal (unpinned) verification rather than breaking the browser launch.
+ */
+export function getCaBundleSpkiHashes() {
+    const bundlePath = findCaBundlePath();
+    if (!bundlePath)
+        return [];
+    let bundle;
+    try {
+        bundle = readFileSync(bundlePath, 'utf8');
+    }
+    catch {
+        return [];
+    }
+    const hashes = new Set();
+    for (const pem of extractPemCertificates(bundle)) {
+        try {
+            const spkiDer = new X509Certificate(pem).publicKey.export({ type: 'spki', format: 'der' });
+            hashes.add(createHash('sha256').update(spkiDer).digest('base64'));
+        }
+        catch {
+            // Skip entries that don't parse as certificates rather than failing the whole bundle.
+        }
+    }
+    return Array.from(hashes);
+}
+/**
+ * Chrome CLI flag that trusts exactly the sandbox's injected CA(s), pinned by SPKI hash rather
+ * than by disabling verification. Chromium's TLS stack (the Chrome Root Store) never reads
+ * NODE_EXTRA_CA_CERTS/SSL_CERT_FILE/CURL_CA_BUNDLE the way Node/curl/Python do, so a sandbox that
+ * MITMs HTTPS through its proxy (see isProxyConfigured) makes every other tool trust the proxy's
+ * cert while Chrome's handshake with it fails with ERR_CERT_AUTHORITY_INVALID. This is scoped
+ * narrowly to the specific CA(s) discovered via the CA bundle env vars — never
+ * --ignore-certificate-errors, which would disable verification for every host, including ones
+ * that have nothing to do with the sandbox's proxy. A no-op (empty array) when no proxy is
+ * configured or no CA bundle is discoverable, so default Chrome trust is untouched on ordinary
+ * developer machines.
+ */
+export function getChromeSpkiArgs() {
+    if (!isProxyConfigured())
+        return [];
+    const hashes = getCaBundleSpkiHashes();
+    if (!hashes.length)
+        return [];
+    return [`--ignore-certificate-errors-spki-list=${hashes.join(',')}`];
+}

package/dist/sites/salesforce-doc-fetch.js

@@ -1,4 +1,4 @@
-import { openCdpPage, waitForLoad, ResponseCapture, } from '../lib/research/browser.js';
+import { openCdpPage, waitForLoad, ResponseCapture, describeNavigationFailure, } from '../lib/research/browser.js';
 import { checkDnsSafety } from '../lib/research/fetcher.js';
 import { htmlToMarkdown } from '../lib/research/markdown.js';
 import { contentMetrics, contentRicherThan } from '../lib/research/regression.js';
@@ -44,6 +44,10 @@ const BASE_REMOVE_SELECTORS = [
     'dx-scroll-manager',
     'dx-skip-nav-link',
     'dw-instrumentation',
+    // The left-hand TOC/search drawer nested inside doc-content-layout's shadow DOM. It renders the
+    // entire site's navigation tree (hundreds of <a> links) as a sibling of the real article body,
+    // so left unremoved it dwarfs and precedes the actual content in the captured Markdown.
+    'dx-sidebar-old',
     '[role="navigation"]',
     '[aria-label*="Table of Contents" i]',
     '#onetrust-consent-sdk',
@@ -279,7 +283,15 @@ async function captureSalesforceAttempt(page, url, captureScript, contentSelecto
             accept: (body) => parseDocsApiPayload(body) !== null,
         },
     ]);
-    await page.client.send('Page.navigate', { url }, page.sessionId);
+    // A network-level failure (DNS, refused/closed connection, TLS, a proxy declining to tunnel) never
+    // fires a load event and renders Chrome's own "site can't be reached" interstitial instead. Without
+    // this check that interstitial sails through pollSalesforceContentReady/assertReadableSalesforceContent
+    // as if it were real article content — it's plausible-length HTML that isn't a 404/loading shell — so
+    // the fetch would silently "succeed" with Chrome's error page cached as the documentation.
+    const nav = await page.client.send('Page.navigate', { url }, page.sessionId);
+    if (nav.errorText) {
+        throw describeNavigationFailure(nav.errorText, url);
+    }
     await waitForLoad(page.client, page.sessionId, TIMEOUT_MS, 0).catch(() => { });
     await page.client
         .send('Runtime.evaluate', { expression: ACCEPT_CONSENT_EXPRESSION }, page.sessionId)

package/oclif.manifest.json

@@ -843,5 +843,5 @@
       "stdoutIsPrimaryData": true
     }
   },
-  "version": "3.0.1"
+  "version": "3.0.3"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@taurgis/bonsai",
-  "version": "3.0.1",
+  "version": "3.0.3",
   "description": "Bonsai is a standalone local research cache CLI for AI agents",
   "repository": {
     "type": "git",