@taurgis/bonsai 3.2.0 → 3.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -4
- package/agents/agents/salesforce-docs-researcher.agent.md +14 -1
- package/agents/instructions/salesforce-research.instructions.md +2 -1
- package/bin/cli.mjs +9 -17
- package/dist/base-command.d.ts +22 -12
- package/dist/base-command.js +86 -49
- package/dist/commands/config/base.js +11 -2
- package/dist/commands/config/get.js +8 -12
- package/dist/commands/config/index.js +2 -2
- package/dist/commands/config/list.js +10 -16
- package/dist/commands/config/set.d.ts +2 -1
- package/dist/commands/config/set.js +34 -27
- package/dist/commands/config/unset.d.ts +2 -1
- package/dist/commands/config/unset.js +25 -24
- package/dist/commands/fetch.d.ts +7 -15
- package/dist/commands/fetch.js +57 -401
- package/dist/commands/import.d.ts +4 -9
- package/dist/commands/import.js +56 -284
- package/dist/commands/inspect.d.ts +7 -1
- package/dist/commands/inspect.js +103 -52
- package/dist/commands/list.d.ts +13 -1
- package/dist/commands/list.js +79 -32
- package/dist/commands/prune.d.ts +4 -1
- package/dist/commands/prune.js +55 -75
- package/dist/commands/status.d.ts +3 -2
- package/dist/commands/status.js +52 -79
- package/dist/hooks/command-not-found/suggest.d.ts +1 -0
- package/dist/hooks/command-not-found/suggest.js +66 -39
- package/dist/lib/argv.d.ts +24 -3
- package/dist/lib/argv.js +124 -53
- package/dist/lib/batch.d.ts +41 -0
- package/dist/lib/batch.js +39 -0
- package/dist/lib/cache-view.d.ts +21 -0
- package/dist/lib/cache-view.js +30 -0
- package/dist/lib/cli-emit.d.ts +28 -0
- package/dist/lib/cli-emit.js +44 -0
- package/dist/lib/cli-error-policy.d.ts +57 -0
- package/dist/lib/cli-error-policy.js +74 -0
- package/dist/lib/cli-flag-manifest.d.ts +5 -0
- package/dist/lib/cli-flag-manifest.js +8 -0
- package/dist/lib/cli-presentation.d.ts +39 -0
- package/dist/lib/cli-presentation.js +50 -0
- package/dist/lib/cli-result-types.d.ts +88 -0
- package/dist/lib/cli-result-types.js +2 -0
- package/dist/lib/config/index.d.ts +21 -3
- package/dist/lib/config/index.js +58 -3
- package/dist/lib/config-persist.d.ts +28 -0
- package/dist/lib/config-persist.js +29 -0
- package/dist/lib/envelope.d.ts +41 -3
- package/dist/lib/envelope.js +147 -4
- package/dist/lib/help-preflight.d.ts +5 -8
- package/dist/lib/help-preflight.js +13 -32
- package/dist/lib/json-meta.d.ts +0 -2
- package/dist/lib/json-meta.js +21 -25
- package/dist/lib/limit-flag.d.ts +11 -1
- package/dist/lib/limit-flag.js +19 -6
- package/dist/lib/parse-error-ux.d.ts +5 -0
- package/dist/lib/parse-error-ux.js +73 -0
- package/dist/lib/prune-flags.d.ts +28 -0
- package/dist/lib/prune-flags.js +74 -0
- package/dist/lib/research/artifact-index.d.ts +11 -2
- package/dist/lib/research/artifact-index.js +7 -4
- package/dist/lib/research/browser.d.ts +41 -2
- package/dist/lib/research/browser.js +139 -50
- package/dist/lib/research/capture.d.ts +24 -2
- package/dist/lib/research/capture.js +24 -4
- package/dist/lib/research/cli-io.d.ts +33 -0
- package/dist/lib/research/cli-io.js +27 -0
- package/dist/lib/research/compress.d.ts +7 -0
- package/dist/lib/research/compress.js +7 -3
- package/dist/lib/research/docs/capabilities.d.ts +15 -0
- package/dist/lib/research/docs/capabilities.js +5 -0
- package/dist/lib/research/docs/clean-dom.d.ts +2 -0
- package/dist/lib/research/docs/clean-dom.js +2 -0
- package/dist/lib/research/docs/code-blocks.d.ts +2 -0
- package/dist/lib/research/docs/code-blocks.js +2 -0
- package/dist/lib/research/docs/detect.d.ts +4 -0
- package/dist/lib/research/docs/detect.js +4 -0
- package/dist/lib/research/docs/machine-readable.d.ts +18 -0
- package/dist/lib/research/docs/machine-readable.js +10 -0
- package/dist/lib/research/docs/markdown-source.d.ts +20 -0
- package/dist/lib/research/docs/markdown-source.js +19 -2
- package/dist/lib/research/docs/quality-gates.d.ts +5 -0
- package/dist/lib/research/docs/quality-gates.js +4 -0
- package/dist/lib/research/docs/rsc.d.ts +9 -2
- package/dist/lib/research/docs/rsc.js +5 -2
- package/dist/lib/research/docs/section-artifacts.d.ts +12 -0
- package/dist/lib/research/docs/section-artifacts.js +12 -0
- package/dist/lib/research/docs/sections.d.ts +18 -0
- package/dist/lib/research/docs/sections.js +17 -1
- package/dist/lib/research/docs/source-map.d.ts +28 -0
- package/dist/lib/research/docs/source-map.js +27 -0
- package/dist/lib/research/docs/validate.d.ts +35 -2
- package/dist/lib/research/docs/validate.js +47 -2
- package/dist/lib/research/extract.d.ts +16 -0
- package/dist/lib/research/extract.js +12 -1
- package/dist/lib/research/fetch-command-service.d.ts +36 -0
- package/dist/lib/research/fetch-command-service.js +310 -0
- package/dist/lib/research/fetch-result.d.ts +114 -0
- package/dist/lib/research/fetch-result.js +212 -0
- package/dist/lib/research/fetcher.d.ts +46 -8
- package/dist/lib/research/fetcher.js +58 -30
- package/dist/lib/research/freshness.d.ts +54 -11
- package/dist/lib/research/freshness.js +91 -31
- package/dist/lib/research/import-command-service.d.ts +56 -0
- package/dist/lib/research/import-command-service.js +381 -0
- package/dist/lib/research/keywords.js +5 -1
- package/dist/lib/research/markdown.d.ts +12 -1
- package/dist/lib/research/markdown.js +13 -5
- package/dist/lib/research/persist-artifact.d.ts +23 -0
- package/dist/lib/research/persist-artifact.js +27 -0
- package/dist/lib/research/prompt-injection.d.ts +8 -0
- package/dist/lib/research/prompt-injection.js +42 -13
- package/dist/lib/research/proxy.d.ts +14 -8
- package/dist/lib/research/proxy.js +14 -10
- package/dist/lib/research/revalidate.d.ts +49 -20
- package/dist/lib/research/revalidate.js +80 -15
- package/dist/lib/research/schema.d.ts +7 -0
- package/dist/lib/research/schema.js +2 -0
- package/dist/lib/research/secret-scan.d.ts +6 -0
- package/dist/lib/research/secret-scan.js +8 -0
- package/dist/lib/research/secure-write.d.ts +4 -0
- package/dist/lib/research/secure-write.js +31 -2
- package/dist/lib/research/storage.d.ts +5 -1
- package/dist/lib/research/storage.js +8 -3
- package/dist/lib/research/summarize.d.ts +15 -0
- package/dist/lib/research/summarize.js +11 -0
- package/dist/lib/research/url.d.ts +9 -6
- package/dist/lib/research/url.js +33 -21
- package/dist/lib/text.d.ts +7 -2
- package/dist/lib/text.js +13 -2
- package/dist/lib/value-flags.d.ts +14 -0
- package/dist/lib/value-flags.js +28 -0
- package/dist/sites/coveo.d.ts +10 -0
- package/dist/sites/coveo.js +10 -0
- package/dist/sites/index.d.ts +16 -0
- package/dist/sites/index.js +16 -4
- package/dist/sites/markdown-twin.d.ts +31 -0
- package/dist/sites/markdown-twin.js +56 -0
- package/dist/sites/salesforce/fetch-page.d.ts +8 -2
- package/dist/sites/salesforce/fetch-page.js +14 -3
- package/dist/sites/salesforce/help-tooling-markdown.d.ts +30 -0
- package/dist/sites/salesforce/help-tooling-markdown.js +74 -0
- package/dist/sites/salesforce/index.d.ts +1 -0
- package/dist/sites/salesforce/index.js +1 -0
- package/dist/sites/salesforce-developer/fetch-page.d.ts +6 -1
- package/dist/sites/salesforce-developer/fetch-page.js +6 -1
- package/dist/sites/salesforce-developer/index.d.ts +1 -0
- package/dist/sites/salesforce-developer/index.js +1 -0
- package/dist/sites/salesforce-developer/route-markdown.d.ts +18 -6
- package/dist/sites/salesforce-developer/route-markdown.js +29 -67
- package/dist/sites/salesforce-doc-fetch.d.ts +39 -5
- package/dist/sites/salesforce-doc-fetch.js +52 -7
- package/dist/sites/salesforce-dom-probe.d.ts +20 -2
- package/dist/sites/salesforce-dom-probe.js +20 -2
- package/dist/sites/types.d.ts +14 -8
- package/dist/sites/types.js +3 -0
- package/oclif.manifest.json +104 -98
- package/package.json +19 -7
- package/agents/agents/official-docs-researcher.agent.md +0 -87
- package/agents/instructions/web-research.instructions.md +0 -44
- package/agents/skills/web-research/SKILL.md +0 -118
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import { cliErrorFields } from '../envelope.js';
|
|
2
|
+
import { SANDBOX_EGRESS_ERROR_MARKER } from './browser.js';
|
|
3
|
+
import { QUALITY_WARNING_PREFIX } from './extract.js';
|
|
4
|
+
import { PROXY_TUNNEL_REJECTION_PATTERN } from './proxy.js';
|
|
5
|
+
import { getArtifactPath } from './storage.js';
|
|
6
|
+
/**
|
|
7
|
+
* Human-prose warnings pulled out of `qualityNotes` (e.g. very-short extraction). The remaining
|
|
8
|
+
* `quality:*` machine codes stay `--json`-only; this is what stderr shows in human-readable mode.
|
|
9
|
+
*/
|
|
10
|
+
export function extractionQualityWarnings(qualityNotes) {
|
|
11
|
+
return qualityNotes
|
|
12
|
+
.filter((note) => note.startsWith(QUALITY_WARNING_PREFIX))
|
|
13
|
+
.map((note) => note.slice(QUALITY_WARNING_PREFIX.length));
|
|
14
|
+
}
|
|
15
|
+
/** Remap write-implying cache statuses when dry-run/read-only skipped persistence. */
|
|
16
|
+
export function reportCacheStatus(status, dryRun) {
|
|
17
|
+
if (!dryRun)
|
|
18
|
+
return status;
|
|
19
|
+
switch (status) {
|
|
20
|
+
case 'miss':
|
|
21
|
+
return 'would_fetch';
|
|
22
|
+
case 'refreshed':
|
|
23
|
+
return 'would_refresh';
|
|
24
|
+
case 'revalidated':
|
|
25
|
+
return 'would_revalidate';
|
|
26
|
+
default:
|
|
27
|
+
return status;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
/** Human spinner labels for reported (possibly remapped) cache statuses. */
|
|
31
|
+
export const FETCH_STATUS_LABEL = {
|
|
32
|
+
hit: 'cached',
|
|
33
|
+
miss: 'done',
|
|
34
|
+
refreshed: 'refreshed',
|
|
35
|
+
revalidated: 'revalidated',
|
|
36
|
+
stale: 'served stale',
|
|
37
|
+
would_fetch: 'previewed (not cached)',
|
|
38
|
+
would_refresh: 'previewed refresh',
|
|
39
|
+
would_revalidate: 'previewed revalidate',
|
|
40
|
+
};
|
|
41
|
+
/** Machine-readable success/preview payload for one fetch URL. */
|
|
42
|
+
export function buildFetchResultData(input) {
|
|
43
|
+
const { artifact, format, dryRun } = input;
|
|
44
|
+
const content = format === 'compressed' ? artifact.compressed : artifact.detailed;
|
|
45
|
+
return {
|
|
46
|
+
schemaVersion: 1,
|
|
47
|
+
command: 'fetch',
|
|
48
|
+
dryRun,
|
|
49
|
+
cache: {
|
|
50
|
+
key: input.cacheKey,
|
|
51
|
+
// Preview runs must not claim a write landed.
|
|
52
|
+
status: reportCacheStatus(input.cacheStatus, dryRun),
|
|
53
|
+
freshness: input.freshnessState,
|
|
54
|
+
path: getArtifactPath(input.storageDir, input.cacheKey),
|
|
55
|
+
storage: input.storageMode,
|
|
56
|
+
redirectedToGlobal: input.redirectedToGlobal,
|
|
57
|
+
},
|
|
58
|
+
source: {
|
|
59
|
+
url: input.url,
|
|
60
|
+
normalizedUrl: input.normalizedUrl,
|
|
61
|
+
captureMethod: artifact.metadata.capture_method,
|
|
62
|
+
extractionStatus: artifact.metadata.extraction_status,
|
|
63
|
+
extractionConfidence: artifact.metadata.extraction_confidence,
|
|
64
|
+
qualityNotes: artifact.metadata.quality_notes,
|
|
65
|
+
fetchedAt: artifact.metadata.fetched_at,
|
|
66
|
+
validatedAt: artifact.metadata.validated_at,
|
|
67
|
+
staleAfter: artifact.metadata.stale_after,
|
|
68
|
+
},
|
|
69
|
+
artifactType: artifact.metadata.artifact_type,
|
|
70
|
+
docsEngine: artifact.metadata.docs_engine,
|
|
71
|
+
docsFramework: artifact.metadata.docs_framework,
|
|
72
|
+
sourceDocUrl: artifact.metadata.source_doc_url,
|
|
73
|
+
searchProvider: artifact.metadata.search_provider,
|
|
74
|
+
format,
|
|
75
|
+
tokenEstimate: format === 'compressed'
|
|
76
|
+
? artifact.metadata.token_estimate.compressed
|
|
77
|
+
: artifact.metadata.token_estimate.detailed,
|
|
78
|
+
content,
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
/** Per-URL failure row for multi-URL batches — keeps prior successes in `data`. */
|
|
82
|
+
export function buildFetchFailureResult(input) {
|
|
83
|
+
const { url, dryRun, err, fallbackGuidance } = input;
|
|
84
|
+
// Prefer the throw site's suggestions/ref; otherwise attach fetch-specific recovery hints.
|
|
85
|
+
const shaped = err.suggestions?.length || err.ref ? err : { ...err, ...fallbackGuidance };
|
|
86
|
+
return {
|
|
87
|
+
schemaVersion: 1,
|
|
88
|
+
command: 'fetch',
|
|
89
|
+
dryRun,
|
|
90
|
+
error: cliErrorFields(shaped, 'FETCH_FAILED'),
|
|
91
|
+
cache: null,
|
|
92
|
+
source: { url, normalizedUrl: null },
|
|
93
|
+
content: null,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Shape a caught runtime error into a batch failure row (message + guidance).
|
|
98
|
+
* CLIError rows should call {@link buildFetchFailureResult} directly so oclif fields are preserved.
|
|
99
|
+
*/
|
|
100
|
+
export function buildFetchFailureFromCaught(bin, url, err, dryRun) {
|
|
101
|
+
const message = describeError(err);
|
|
102
|
+
return buildFetchFailureResult({
|
|
103
|
+
url,
|
|
104
|
+
dryRun,
|
|
105
|
+
err: { message, code: 'FETCH_FAILED' },
|
|
106
|
+
fallbackGuidance: fetchFailureGuidance(message, url, bin),
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
// How many `.cause` levels to walk. Real undici/Node transport errors nest at most 1-2 deep;
|
|
110
|
+
// the cap exists so a self-referential or cyclic `.cause` chain (Error.cause is an arbitrary,
|
|
111
|
+
// unvalidated property — nothing prevents `e.cause = e`, whether from a bug in this codebase or
|
|
112
|
+
// a dependency) can't hang the process, rather than being a limit expected to matter in practice.
|
|
113
|
+
const MAX_CAUSE_DEPTH = 10;
|
|
114
|
+
// Walks an Error's `.cause` chain and joins every message, deepest last. Node/undici's fetch
|
|
115
|
+
// throws a generic `TypeError: fetch failed` for transport-level failures, with the actionable
|
|
116
|
+
// detail (e.g. a proxy's tunnel rejection) nested one or two `.cause` levels down.
|
|
117
|
+
export function describeError(err) {
|
|
118
|
+
if (!(err instanceof Error))
|
|
119
|
+
return String(err);
|
|
120
|
+
const parts = [err.message];
|
|
121
|
+
let cause = err.cause;
|
|
122
|
+
for (let depth = 0; depth < MAX_CAUSE_DEPTH && cause instanceof Error; depth++) {
|
|
123
|
+
parts.push(cause.message);
|
|
124
|
+
cause = cause.cause;
|
|
125
|
+
}
|
|
126
|
+
return parts.join(': ');
|
|
127
|
+
}
|
|
128
|
+
// Maps a runtime fetch failure to recovery steps keyed off its message. Returns undefined for
|
|
129
|
+
// unrecognized failures, which then surface with their original message and no extra hint. The
|
|
130
|
+
// patterns match the thrown text in fetcher.ts/browser.ts; keep them in sync if those messages
|
|
131
|
+
// change. Resolutions mirror the published troubleshooting guide.
|
|
132
|
+
export function fetchFailureGuidance(message, url, bin = 'bonsai') {
|
|
133
|
+
const ref = 'https://bonsai.rhino-inquisitor.com/troubleshooting';
|
|
134
|
+
// Name the pipe step explicitly: bare `import --stdin` blocks waiting for input, so a reader (or
|
|
135
|
+
// an agent) running the hint verbatim would hang. Show the content being piped in from a file.
|
|
136
|
+
const importHint = `Open it in a browser, save the page, then import it: cat page.md | ${bin} import ${url} --stdin`;
|
|
137
|
+
// 401/403: an auth wall or anti-scraping WAF. Bonsai has no authenticated-fetch path in v1.
|
|
138
|
+
if (/failed with status 40[13]\b/.test(message)) {
|
|
139
|
+
return {
|
|
140
|
+
suggestions: ['The page requires authentication or blocks automated requests.', importHint],
|
|
141
|
+
ref,
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
if (/failed with status 404\b/.test(message)) {
|
|
145
|
+
return { suggestions: ['Check the URL is correct and the page still exists.'] };
|
|
146
|
+
}
|
|
147
|
+
if (/failed with status 5\d\d\b/.test(message)) {
|
|
148
|
+
return {
|
|
149
|
+
suggestions: ['The server returned an error. Retry later or verify the host is healthy.'],
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
// Server returned a non-HTML body (JSON, binary, or no Content-Type at all). Scope the opener to
|
|
153
|
+
// the real failure: --rendered does produce HTML, so "only scrapes HTML" would read as misleading.
|
|
154
|
+
if (/Rejected content type|does not look like HTML/.test(message)) {
|
|
155
|
+
return {
|
|
156
|
+
suggestions: [
|
|
157
|
+
'The server returned a non-HTML response (e.g. JSON or binary).',
|
|
158
|
+
'If the page is rendered by client-side JavaScript, retry with --rendered.',
|
|
159
|
+
importHint,
|
|
160
|
+
],
|
|
161
|
+
ref,
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
if (/DNS resolution failed/.test(message)) {
|
|
165
|
+
return {
|
|
166
|
+
suggestions: ['Check the hostname is spelled correctly and resolves on a public network.'],
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
// Chrome's own network stack hit a proxy-enforced sandbox egress block (browser.ts's
|
|
170
|
+
// describeNavigationFailure already explains the cause and the fix); only a workaround and the
|
|
171
|
+
// docs ref are added here.
|
|
172
|
+
if (message.includes(SANDBOX_EGRESS_ERROR_MARKER)) {
|
|
173
|
+
return { suggestions: [importHint], ref };
|
|
174
|
+
}
|
|
175
|
+
// A configured proxy (HTTPS_PROXY/HTTP_PROXY) refused to tunnel to this host — distinct from
|
|
176
|
+
// the destination's own 401/403 above, which is the site itself rejecting the request. Shared
|
|
177
|
+
// with fetcher.ts's retry-fallback check so the two stay in sync by construction.
|
|
178
|
+
if (PROXY_TUNNEL_REJECTION_PATTERN.test(message)) {
|
|
179
|
+
return {
|
|
180
|
+
suggestions: [
|
|
181
|
+
"The configured proxy (HTTPS_PROXY/HTTP_PROXY) won't reach this host. Check its allowlist, or add the host to NO_PROXY to bypass the proxy for it.",
|
|
182
|
+
importHint,
|
|
183
|
+
],
|
|
184
|
+
ref,
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
// The connection failed before any HTTP response was received — network-down, DNS, TLS, or a
|
|
188
|
+
// proxy issue not specifically matched above. Node/undici's fetch reports all of these as a
|
|
189
|
+
// generic "fetch failed" TypeError with no further suffix; the negative lookahead keeps this
|
|
190
|
+
// from also matching assertOk's "Fetch failed with status NNN ..." (a real HTTP response was
|
|
191
|
+
// received in that case, so it isn't a transport failure at all).
|
|
192
|
+
if (/^fetch failed\b(?!\s+with status)/i.test(message)) {
|
|
193
|
+
return {
|
|
194
|
+
suggestions: [
|
|
195
|
+
'The connection failed before a response was received. Check network connectivity and any HTTPS_PROXY/HTTP_PROXY/NO_PROXY settings.',
|
|
196
|
+
importHint,
|
|
197
|
+
],
|
|
198
|
+
ref,
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
// A hostname that only resolves to a private/local IP at request time (literal private addresses
|
|
202
|
+
// are rejected earlier with exit 2). Blocked to prevent SSRF; only public hosts can be fetched.
|
|
203
|
+
if (/blocked local or private target/.test(message)) {
|
|
204
|
+
return {
|
|
205
|
+
suggestions: [
|
|
206
|
+
'The hostname resolves to a private or local address, which is blocked to prevent SSRF. Only public http(s) hosts can be fetched.',
|
|
207
|
+
],
|
|
208
|
+
ref,
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
return undefined;
|
|
212
|
+
}
|
|
@@ -1,9 +1,15 @@
|
|
|
1
|
+
/** Optional transport limits and headers for {@link fetchStaticHtml} / {@link fetchText}. */
|
|
1
2
|
export interface FetchOptions {
|
|
3
|
+
/** Abort the request after this many milliseconds. */
|
|
2
4
|
timeoutMs?: number;
|
|
5
|
+
/** Maximum response body size in bytes. */
|
|
3
6
|
bodyLimitBytes?: number;
|
|
7
|
+
/** Maximum number of HTTP redirects to follow. */
|
|
4
8
|
maxRedirects?: number;
|
|
9
|
+
/** Extra request headers (e.g. conditional-request validators). */
|
|
5
10
|
headers?: Record<string, string>;
|
|
6
11
|
}
|
|
12
|
+
/** Successful or not-modified HTTP fetch result used by capture and revalidation. */
|
|
7
13
|
export interface FetchResult {
|
|
8
14
|
status: number;
|
|
9
15
|
contentType: string | null;
|
|
@@ -13,17 +19,49 @@ export interface FetchResult {
|
|
|
13
19
|
responseSize: number;
|
|
14
20
|
content: string;
|
|
15
21
|
}
|
|
22
|
+
/** The fetched-page fields artifact building consumes — FetchResult minus the transport status. */
|
|
23
|
+
export type FetchedContent = Omit<FetchResult, 'status'>;
|
|
24
|
+
/**
|
|
25
|
+
* Heuristic: body looks like HTML/XML when `Content-Type` is missing.
|
|
26
|
+
*
|
|
27
|
+
* @param text - Response body text.
|
|
28
|
+
* @returns `true` when the trimmed body starts with a common HTML/XML marker.
|
|
29
|
+
*/
|
|
16
30
|
export declare function looksLikeHtml(text: string): boolean;
|
|
31
|
+
/**
|
|
32
|
+
* Resolve a hostname and reject private/loopback targets.
|
|
33
|
+
* May skip local DNS when only a proxy can resolve the host.
|
|
34
|
+
*
|
|
35
|
+
* @param hostname - Host to check (IPv6 literals may be bracketed).
|
|
36
|
+
* @throws {Error} When the resolved address is not safe to fetch.
|
|
37
|
+
*/
|
|
17
38
|
export declare function checkDnsSafety(hostname: string): Promise<void>;
|
|
18
|
-
export declare function fetchStaticHtml(url: string, options?: FetchOptions): Promise<FetchResult>;
|
|
19
39
|
/**
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
40
|
+
* True when `err` is the static fetcher's own content-type policy rejection (non-HTML body or
|
|
41
|
+
* content-type), as opposed to a transient failure (network error, timeout, HTTP error status).
|
|
42
|
+
* The caller uses this to distinguish "this URL fundamentally isn't a web page" — where an
|
|
43
|
+
* automatic rendered-browser retry would just render Chrome's built-in JSON/PDF viewer and produce
|
|
44
|
+
* a corrupted-looking "extracted" artifact — from "this page may need JavaScript", where retrying
|
|
45
|
+
* rendered is the whole point of the fallback.
|
|
23
46
|
*/
|
|
24
|
-
export declare function
|
|
47
|
+
export declare function isUnsupportedContentTypeError(err: unknown): boolean;
|
|
25
48
|
/**
|
|
26
|
-
*
|
|
27
|
-
*
|
|
49
|
+
* Fetch an HTML page with DNS, redirect, timeout, and body-size safety.
|
|
50
|
+
*
|
|
51
|
+
* @param url - Absolute URL to fetch.
|
|
52
|
+
* @param options - Optional timeout, body limit, redirect cap, and headers.
|
|
53
|
+
* @returns Fetch result including body content and validator headers.
|
|
54
|
+
* @throws {Error} On unsafe DNS, non-HTML content, HTTP failure, or redirect limits.
|
|
28
55
|
*/
|
|
29
|
-
export declare function
|
|
56
|
+
export declare function fetchStaticHtml(url: string, options?: FetchOptions): Promise<FetchResult>;
|
|
57
|
+
/**
|
|
58
|
+
* Fetch a non-HTML text/data resource (llms.txt, route Markdown, search index) with the same
|
|
59
|
+
* DNS, redirect, timeout, and body-size safety as {@link fetchStaticHtml}. Does not enforce a
|
|
60
|
+
* content type; the caller must validate the returned body is the artifact kind it expected.
|
|
61
|
+
*
|
|
62
|
+
* @param url - Absolute URL to fetch.
|
|
63
|
+
* @param options - Optional timeout, body limit, redirect cap, and headers.
|
|
64
|
+
* @returns Fetch result including body text and validator headers.
|
|
65
|
+
* @throws {Error} On unsafe DNS, HTTP failure, or redirect limits.
|
|
66
|
+
*/
|
|
67
|
+
export declare function fetchText(url: string, options?: FetchOptions): Promise<FetchResult>;
|
|
@@ -3,6 +3,11 @@ import { isIP } from 'node:net';
|
|
|
3
3
|
import { fetch as undiciFetch } from 'undici';
|
|
4
4
|
import { isSafeIp, normalizeUrl } from './url.js';
|
|
5
5
|
import { getProxyDispatcher, isProxyConfigured, PROXY_TUNNEL_REJECTION_PATTERN } from './proxy.js';
|
|
6
|
+
const DEFAULT_FETCH_TIMEOUT_MS = 10_000;
|
|
7
|
+
const DEFAULT_BODY_LIMIT_BYTES = 2 * 1024 * 1024;
|
|
8
|
+
const DEFAULT_MAX_REDIRECTS = 5;
|
|
9
|
+
const HTTP_NOT_MODIFIED = 304;
|
|
10
|
+
const REDIRECT_STATUS_CODES = new Set([301, 302, 303, 307, 308]);
|
|
6
11
|
// Runs one fetch attempt with its own fresh AbortController/timeout, so that when `doFetch` makes
|
|
7
12
|
// a second attempt (the proxy fallback below) it gets the full timeout budget rather than
|
|
8
13
|
// whatever was left over from the first attempt's signal. Loosely typed at this one boundary
|
|
@@ -56,6 +61,12 @@ async function doFetch(url, init, timeoutMs) {
|
|
|
56
61
|
return fetchWithTimeout(fetch, url, init, timeoutMs);
|
|
57
62
|
}
|
|
58
63
|
}
|
|
64
|
+
/**
|
|
65
|
+
* Heuristic: body looks like HTML/XML when `Content-Type` is missing.
|
|
66
|
+
*
|
|
67
|
+
* @param text - Response body text.
|
|
68
|
+
* @returns `true` when the trimmed body starts with a common HTML/XML marker.
|
|
69
|
+
*/
|
|
59
70
|
export function looksLikeHtml(text) {
|
|
60
71
|
const trimmed = text.trimStart().toLowerCase();
|
|
61
72
|
return (trimmed.startsWith('<!doctype html') ||
|
|
@@ -63,6 +74,13 @@ export function looksLikeHtml(text) {
|
|
|
63
74
|
trimmed.startsWith('<xml') ||
|
|
64
75
|
trimmed.startsWith('<?xml'));
|
|
65
76
|
}
|
|
77
|
+
/**
|
|
78
|
+
* Resolve a hostname and reject private/loopback targets.
|
|
79
|
+
* May skip local DNS when only a proxy can resolve the host.
|
|
80
|
+
*
|
|
81
|
+
* @param hostname - Host to check (IPv6 literals may be bracketed).
|
|
82
|
+
* @throws {Error} When the resolved address is not safe to fetch.
|
|
83
|
+
*/
|
|
66
84
|
export async function checkDnsSafety(hostname) {
|
|
67
85
|
let hostToResolve = hostname;
|
|
68
86
|
if (hostToResolve.startsWith('[') && hostToResolve.endsWith(']')) {
|
|
@@ -128,15 +146,32 @@ function headerMeta(res) {
|
|
|
128
146
|
}
|
|
129
147
|
// A 304 carries no body; return an empty result that preserves the validator headers.
|
|
130
148
|
function notModifiedResult(res, currentUrl) {
|
|
131
|
-
return {
|
|
149
|
+
return {
|
|
150
|
+
status: HTTP_NOT_MODIFIED,
|
|
151
|
+
...headerMeta(res),
|
|
152
|
+
finalUrl: currentUrl,
|
|
153
|
+
responseSize: 0,
|
|
154
|
+
content: '',
|
|
155
|
+
};
|
|
132
156
|
}
|
|
133
157
|
function assertOk(res) {
|
|
134
158
|
if (!res.ok) {
|
|
135
159
|
throw new Error(`Fetch failed with status ${res.status} ${res.statusText}`);
|
|
136
160
|
}
|
|
137
161
|
}
|
|
162
|
+
/**
|
|
163
|
+
* True when `err` is the static fetcher's own content-type policy rejection (non-HTML body or
|
|
164
|
+
* content-type), as opposed to a transient failure (network error, timeout, HTTP error status).
|
|
165
|
+
* The caller uses this to distinguish "this URL fundamentally isn't a web page" — where an
|
|
166
|
+
* automatic rendered-browser retry would just render Chrome's built-in JSON/PDF viewer and produce
|
|
167
|
+
* a corrupted-looking "extracted" artifact — from "this page may need JavaScript", where retrying
|
|
168
|
+
* rendered is the whole point of the fallback.
|
|
169
|
+
*/
|
|
170
|
+
export function isUnsupportedContentTypeError(err) {
|
|
171
|
+
return err instanceof Error && /^Rejected /.test(err.message);
|
|
172
|
+
}
|
|
138
173
|
async function processFetchResponse(res, limit, currentUrl) {
|
|
139
|
-
if (res.status ===
|
|
174
|
+
if (res.status === HTTP_NOT_MODIFIED)
|
|
140
175
|
return notModifiedResult(res, currentUrl);
|
|
141
176
|
assertOk(res);
|
|
142
177
|
const contentType = res.headers.get('content-type');
|
|
@@ -163,7 +198,7 @@ async function processFetchResponse(res, limit, currentUrl) {
|
|
|
163
198
|
// processFetchResponse (304, status, size limit) but does NOT reject by content type — the caller
|
|
164
199
|
// validates that the body is the artifact kind it asked for. Treated as untrusted text.
|
|
165
200
|
async function processTextResponse(res, limit, currentUrl) {
|
|
166
|
-
if (res.status ===
|
|
201
|
+
if (res.status === HTTP_NOT_MODIFIED)
|
|
167
202
|
return notModifiedResult(res, currentUrl);
|
|
168
203
|
assertOk(res);
|
|
169
204
|
const bodyBytes = await readBodyWithLimit(res.body, limit);
|
|
@@ -178,9 +213,9 @@ async function processTextResponse(res, limit, currentUrl) {
|
|
|
178
213
|
// Shared redirect/DNS/timeout loop. The `process` callback decides how to validate and shape the
|
|
179
214
|
// final (non-redirect) response, so HTML and text/data fetches reuse identical transport safety.
|
|
180
215
|
async function fetchWithRedirects(url, options, process) {
|
|
181
|
-
const timeout = options.timeoutMs ??
|
|
182
|
-
const limit = options.bodyLimitBytes ??
|
|
183
|
-
const maxRedirects = options.maxRedirects ??
|
|
216
|
+
const timeout = options.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS;
|
|
217
|
+
const limit = options.bodyLimitBytes ?? DEFAULT_BODY_LIMIT_BYTES;
|
|
218
|
+
const maxRedirects = options.maxRedirects ?? DEFAULT_MAX_REDIRECTS;
|
|
184
219
|
const initialHeaders = options.headers ?? {};
|
|
185
220
|
let currentUrl = normalizeUrl(url);
|
|
186
221
|
let redirectCount = 0;
|
|
@@ -188,7 +223,7 @@ async function fetchWithRedirects(url, options, process) {
|
|
|
188
223
|
const parsedUrl = new URL(currentUrl);
|
|
189
224
|
await checkDnsSafety(parsedUrl.hostname);
|
|
190
225
|
const res = await doFetch(currentUrl, { method: 'GET', headers: initialHeaders, redirect: 'manual' }, timeout);
|
|
191
|
-
if (
|
|
226
|
+
if (REDIRECT_STATUS_CODES.has(res.status)) {
|
|
192
227
|
redirectCount++;
|
|
193
228
|
if (redirectCount > maxRedirects) {
|
|
194
229
|
throw new Error(`Too many redirects. Exceeded limit of ${maxRedirects}.`);
|
|
@@ -203,34 +238,27 @@ async function fetchWithRedirects(url, options, process) {
|
|
|
203
238
|
return await process(res, limit, currentUrl);
|
|
204
239
|
}
|
|
205
240
|
}
|
|
241
|
+
/**
|
|
242
|
+
* Fetch an HTML page with DNS, redirect, timeout, and body-size safety.
|
|
243
|
+
*
|
|
244
|
+
* @param url - Absolute URL to fetch.
|
|
245
|
+
* @param options - Optional timeout, body limit, redirect cap, and headers.
|
|
246
|
+
* @returns Fetch result including body content and validator headers.
|
|
247
|
+
* @throws {Error} On unsafe DNS, non-HTML content, HTTP failure, or redirect limits.
|
|
248
|
+
*/
|
|
206
249
|
export async function fetchStaticHtml(url, options = {}) {
|
|
207
250
|
return fetchWithRedirects(url, options, processFetchResponse);
|
|
208
251
|
}
|
|
209
252
|
/**
|
|
210
|
-
*
|
|
211
|
-
* DNS, redirect, timeout, and body-size safety as fetchStaticHtml. Does not enforce a
|
|
212
|
-
* type; the caller must validate the returned body is the artifact kind it expected.
|
|
253
|
+
* Fetch a non-HTML text/data resource (llms.txt, route Markdown, search index) with the same
|
|
254
|
+
* DNS, redirect, timeout, and body-size safety as {@link fetchStaticHtml}. Does not enforce a
|
|
255
|
+
* content type; the caller must validate the returned body is the artifact kind it expected.
|
|
256
|
+
*
|
|
257
|
+
* @param url - Absolute URL to fetch.
|
|
258
|
+
* @param options - Optional timeout, body limit, redirect cap, and headers.
|
|
259
|
+
* @returns Fetch result including body text and validator headers.
|
|
260
|
+
* @throws {Error} On unsafe DNS, HTTP failure, or redirect limits.
|
|
213
261
|
*/
|
|
214
262
|
export async function fetchText(url, options = {}) {
|
|
215
263
|
return fetchWithRedirects(url, options, processTextResponse);
|
|
216
264
|
}
|
|
217
|
-
/**
|
|
218
|
-
* Sends a JSON POST and returns the response body text. Used by remote search connectors (e.g.
|
|
219
|
-
* Algolia DocSearch). Enforces the same DNS safety, timeout, and body-size limits as GET fetches.
|
|
220
|
-
*/
|
|
221
|
-
export async function postJson(url, body, headers = {}, options = {}) {
|
|
222
|
-
const timeout = options.timeoutMs ?? 10_000;
|
|
223
|
-
const limit = options.bodyLimitBytes ?? 2 * 1024 * 1024;
|
|
224
|
-
const target = normalizeUrl(url);
|
|
225
|
-
await checkDnsSafety(new URL(target).hostname);
|
|
226
|
-
const res = await doFetch(target, {
|
|
227
|
-
method: 'POST',
|
|
228
|
-
headers: { 'content-type': 'application/json', ...headers },
|
|
229
|
-
body: JSON.stringify(body),
|
|
230
|
-
redirect: 'error',
|
|
231
|
-
}, timeout);
|
|
232
|
-
if (!res.ok)
|
|
233
|
-
throw new Error(`Search request failed with status ${res.status} ${res.statusText}`);
|
|
234
|
-
const bytes = await readBodyWithLimit(res.body, limit);
|
|
235
|
-
return new TextDecoder().decode(bytes);
|
|
236
|
-
}
|
|
@@ -1,29 +1,72 @@
|
|
|
1
1
|
import type { ResearchArtifactMetadata } from './schema.js';
|
|
2
|
+
/** Fresh and grace windows derived from a tier and optional TTL override. */
|
|
2
3
|
export interface FreshnessPolicy {
|
|
3
4
|
freshWindowMs: number;
|
|
4
5
|
graceWindowMs: number;
|
|
5
6
|
}
|
|
6
7
|
/**
|
|
7
|
-
* Parses
|
|
8
|
-
*
|
|
8
|
+
* Parses a TTL/duration string into milliseconds.
|
|
9
|
+
* The unit `m` means months (not minutes); the smallest unit is hours.
|
|
10
|
+
*
|
|
11
|
+
* @param ttl - Whole number plus unit (`h`/`d`/`w`/`m`/`y`).
|
|
12
|
+
* @returns Duration in milliseconds.
|
|
13
|
+
* @throws {Error} When the format is invalid or the duration is zero.
|
|
9
14
|
*/
|
|
10
15
|
export declare function parseTtlToMs(ttl: string): number;
|
|
11
16
|
/**
|
|
12
|
-
* Validate a duration-valued flag.
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
17
|
+
* Validate a duration-valued flag.
|
|
18
|
+
*
|
|
19
|
+
* @param flag - Flag token for the error message (e.g. `--ttl`).
|
|
20
|
+
* @param value - Raw flag value, or `undefined` when omitted.
|
|
21
|
+
* @returns Actionable error message, or `null` when absent/valid.
|
|
16
22
|
*/
|
|
17
23
|
export declare function durationFlagError(flag: string, value: string | undefined): string | null;
|
|
18
24
|
/**
|
|
19
|
-
* Derives the active freshness policy
|
|
25
|
+
* Derives the active freshness policy from a tier and optional TTL override.
|
|
26
|
+
* When a TTL override is set, grace scales proportionally to the tier's default ratio.
|
|
27
|
+
*
|
|
28
|
+
* @param tier - Freshness tier (`stable` / `standard` / `volatile`).
|
|
29
|
+
* @param ttlOverride - Optional TTL string replacing the tier's fresh window.
|
|
30
|
+
* @returns Fresh and grace windows in milliseconds.
|
|
20
31
|
*/
|
|
21
|
-
export declare function
|
|
32
|
+
export declare function resolveFreshnessPolicy(tier: 'stable' | 'standard' | 'volatile', ttlOverride?: string | null): FreshnessPolicy;
|
|
22
33
|
/**
|
|
23
34
|
* Evaluates the freshness status of a cached artifact.
|
|
35
|
+
* `ttlOverride` / `tierOverride` let status (and similar read-only checks) evaluate "what if"
|
|
36
|
+
* policies without mutating the stored metadata. When omitted, the artifact's own tier/TTL win.
|
|
37
|
+
*
|
|
38
|
+
* @param meta - Artifact metadata providing timestamps, tier, and stored TTL.
|
|
39
|
+
* @param currentTime - Clock used for age calculation.
|
|
40
|
+
* @param ttlOverride - Optional TTL replacing the stored TTL for this evaluation.
|
|
41
|
+
* @param tierOverride - Optional tier replacing the stored tier for this evaluation.
|
|
42
|
+
* @returns `fresh`, `stale_grace`, or `stale_expired`.
|
|
24
43
|
*/
|
|
25
|
-
export declare function evaluateFreshness(meta: ResearchArtifactMetadata, currentTime: Date, ttlOverride?: string | null): 'fresh' | 'stale_grace' | 'stale_expired';
|
|
44
|
+
export declare function evaluateFreshness(meta: ResearchArtifactMetadata, currentTime: Date, ttlOverride?: string | null, tierOverride?: 'stable' | 'standard' | 'volatile' | null): 'fresh' | 'stale_grace' | 'stale_expired';
|
|
26
45
|
/**
|
|
27
|
-
* Checks
|
|
46
|
+
* Checks whether the cached entry's age exceeds the specified max-age duration.
|
|
47
|
+
*
|
|
48
|
+
* @param cached - Artifact (or metadata bag) with fetch/validate timestamps.
|
|
49
|
+
* @param currentTime - Clock used for age calculation.
|
|
50
|
+
* @param maxAge - Duration string, or `undefined` to skip the check.
|
|
51
|
+
* @returns `true` when max-age is set and exceeded.
|
|
28
52
|
*/
|
|
29
|
-
export declare function checkMaxAgeExpired(cached:
|
|
53
|
+
export declare function checkMaxAgeExpired(cached: {
|
|
54
|
+
metadata: Pick<ResearchArtifactMetadata, 'fetched_at' | 'validated_at'>;
|
|
55
|
+
}, currentTime: Date, maxAge: string | undefined): boolean;
|
|
56
|
+
/**
|
|
57
|
+
* Freshness for a cached artifact, applying `--max-age` before the tier/TTL windows.
|
|
58
|
+
*
|
|
59
|
+
* @param cached - Artifact with full metadata.
|
|
60
|
+
* @param currentTime - Clock used for age calculation.
|
|
61
|
+
* @param options.ttl - Optional TTL override.
|
|
62
|
+
* @param options.maxAge - Optional max-age duration; when exceeded returns `stale_expired`.
|
|
63
|
+
* @param options.tier - Optional tier override.
|
|
64
|
+
* @returns `fresh`, `stale_grace`, or `stale_expired`.
|
|
65
|
+
*/
|
|
66
|
+
export declare function evaluateFreshnessWithMaxAge(cached: {
|
|
67
|
+
metadata: ResearchArtifactMetadata;
|
|
68
|
+
}, currentTime: Date, options?: {
|
|
69
|
+
ttl?: string | null;
|
|
70
|
+
maxAge?: string;
|
|
71
|
+
tier?: 'stable' | 'standard' | 'volatile' | null;
|
|
72
|
+
}): 'fresh' | 'stale_grace' | 'stale_expired';
|