@taurgis/bonsai 3.2.0 → 3.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. package/README.md +7 -4
  2. package/agents/agents/salesforce-docs-researcher.agent.md +14 -1
  3. package/agents/instructions/salesforce-research.instructions.md +2 -1
  4. package/bin/cli.mjs +9 -17
  5. package/dist/base-command.d.ts +22 -12
  6. package/dist/base-command.js +86 -49
  7. package/dist/commands/config/base.js +11 -2
  8. package/dist/commands/config/get.js +8 -12
  9. package/dist/commands/config/index.js +2 -2
  10. package/dist/commands/config/list.js +10 -16
  11. package/dist/commands/config/set.d.ts +2 -1
  12. package/dist/commands/config/set.js +34 -27
  13. package/dist/commands/config/unset.d.ts +2 -1
  14. package/dist/commands/config/unset.js +25 -24
  15. package/dist/commands/fetch.d.ts +7 -15
  16. package/dist/commands/fetch.js +57 -401
  17. package/dist/commands/import.d.ts +4 -9
  18. package/dist/commands/import.js +56 -284
  19. package/dist/commands/inspect.d.ts +7 -1
  20. package/dist/commands/inspect.js +103 -52
  21. package/dist/commands/list.d.ts +13 -1
  22. package/dist/commands/list.js +79 -32
  23. package/dist/commands/prune.d.ts +4 -1
  24. package/dist/commands/prune.js +55 -75
  25. package/dist/commands/status.d.ts +3 -2
  26. package/dist/commands/status.js +52 -79
  27. package/dist/hooks/command-not-found/suggest.d.ts +1 -0
  28. package/dist/hooks/command-not-found/suggest.js +66 -39
  29. package/dist/lib/argv.d.ts +24 -3
  30. package/dist/lib/argv.js +124 -53
  31. package/dist/lib/batch.d.ts +41 -0
  32. package/dist/lib/batch.js +39 -0
  33. package/dist/lib/cache-view.d.ts +21 -0
  34. package/dist/lib/cache-view.js +30 -0
  35. package/dist/lib/cli-emit.d.ts +28 -0
  36. package/dist/lib/cli-emit.js +44 -0
  37. package/dist/lib/cli-error-policy.d.ts +57 -0
  38. package/dist/lib/cli-error-policy.js +74 -0
  39. package/dist/lib/cli-flag-manifest.d.ts +5 -0
  40. package/dist/lib/cli-flag-manifest.js +8 -0
  41. package/dist/lib/cli-presentation.d.ts +39 -0
  42. package/dist/lib/cli-presentation.js +50 -0
  43. package/dist/lib/cli-result-types.d.ts +88 -0
  44. package/dist/lib/cli-result-types.js +2 -0
  45. package/dist/lib/config/index.d.ts +21 -3
  46. package/dist/lib/config/index.js +58 -3
  47. package/dist/lib/config-persist.d.ts +28 -0
  48. package/dist/lib/config-persist.js +29 -0
  49. package/dist/lib/envelope.d.ts +41 -3
  50. package/dist/lib/envelope.js +147 -4
  51. package/dist/lib/help-preflight.d.ts +5 -8
  52. package/dist/lib/help-preflight.js +13 -32
  53. package/dist/lib/json-meta.d.ts +0 -2
  54. package/dist/lib/json-meta.js +21 -25
  55. package/dist/lib/limit-flag.d.ts +11 -1
  56. package/dist/lib/limit-flag.js +19 -6
  57. package/dist/lib/parse-error-ux.d.ts +5 -0
  58. package/dist/lib/parse-error-ux.js +73 -0
  59. package/dist/lib/prune-flags.d.ts +28 -0
  60. package/dist/lib/prune-flags.js +74 -0
  61. package/dist/lib/research/artifact-index.d.ts +11 -2
  62. package/dist/lib/research/artifact-index.js +7 -4
  63. package/dist/lib/research/browser.d.ts +41 -2
  64. package/dist/lib/research/browser.js +139 -50
  65. package/dist/lib/research/capture.d.ts +24 -2
  66. package/dist/lib/research/capture.js +24 -4
  67. package/dist/lib/research/cli-io.d.ts +33 -0
  68. package/dist/lib/research/cli-io.js +27 -0
  69. package/dist/lib/research/compress.d.ts +7 -0
  70. package/dist/lib/research/compress.js +7 -3
  71. package/dist/lib/research/docs/capabilities.d.ts +15 -0
  72. package/dist/lib/research/docs/capabilities.js +5 -0
  73. package/dist/lib/research/docs/clean-dom.d.ts +2 -0
  74. package/dist/lib/research/docs/clean-dom.js +2 -0
  75. package/dist/lib/research/docs/code-blocks.d.ts +2 -0
  76. package/dist/lib/research/docs/code-blocks.js +2 -0
  77. package/dist/lib/research/docs/detect.d.ts +4 -0
  78. package/dist/lib/research/docs/detect.js +4 -0
  79. package/dist/lib/research/docs/machine-readable.d.ts +18 -0
  80. package/dist/lib/research/docs/machine-readable.js +10 -0
  81. package/dist/lib/research/docs/markdown-source.d.ts +20 -0
  82. package/dist/lib/research/docs/markdown-source.js +19 -2
  83. package/dist/lib/research/docs/quality-gates.d.ts +5 -0
  84. package/dist/lib/research/docs/quality-gates.js +4 -0
  85. package/dist/lib/research/docs/rsc.d.ts +9 -2
  86. package/dist/lib/research/docs/rsc.js +5 -2
  87. package/dist/lib/research/docs/section-artifacts.d.ts +12 -0
  88. package/dist/lib/research/docs/section-artifacts.js +12 -0
  89. package/dist/lib/research/docs/sections.d.ts +18 -0
  90. package/dist/lib/research/docs/sections.js +17 -1
  91. package/dist/lib/research/docs/source-map.d.ts +28 -0
  92. package/dist/lib/research/docs/source-map.js +27 -0
  93. package/dist/lib/research/docs/validate.d.ts +35 -2
  94. package/dist/lib/research/docs/validate.js +47 -2
  95. package/dist/lib/research/extract.d.ts +16 -0
  96. package/dist/lib/research/extract.js +12 -1
  97. package/dist/lib/research/fetch-command-service.d.ts +36 -0
  98. package/dist/lib/research/fetch-command-service.js +310 -0
  99. package/dist/lib/research/fetch-result.d.ts +114 -0
  100. package/dist/lib/research/fetch-result.js +212 -0
  101. package/dist/lib/research/fetcher.d.ts +46 -8
  102. package/dist/lib/research/fetcher.js +58 -30
  103. package/dist/lib/research/freshness.d.ts +54 -11
  104. package/dist/lib/research/freshness.js +91 -31
  105. package/dist/lib/research/import-command-service.d.ts +56 -0
  106. package/dist/lib/research/import-command-service.js +381 -0
  107. package/dist/lib/research/keywords.js +5 -1
  108. package/dist/lib/research/markdown.d.ts +12 -1
  109. package/dist/lib/research/markdown.js +13 -5
  110. package/dist/lib/research/persist-artifact.d.ts +23 -0
  111. package/dist/lib/research/persist-artifact.js +27 -0
  112. package/dist/lib/research/prompt-injection.d.ts +8 -0
  113. package/dist/lib/research/prompt-injection.js +42 -13
  114. package/dist/lib/research/proxy.d.ts +14 -8
  115. package/dist/lib/research/proxy.js +14 -10
  116. package/dist/lib/research/revalidate.d.ts +49 -20
  117. package/dist/lib/research/revalidate.js +80 -15
  118. package/dist/lib/research/schema.d.ts +7 -0
  119. package/dist/lib/research/schema.js +2 -0
  120. package/dist/lib/research/secret-scan.d.ts +6 -0
  121. package/dist/lib/research/secret-scan.js +8 -0
  122. package/dist/lib/research/secure-write.d.ts +4 -0
  123. package/dist/lib/research/secure-write.js +31 -2
  124. package/dist/lib/research/storage.d.ts +5 -1
  125. package/dist/lib/research/storage.js +8 -3
  126. package/dist/lib/research/summarize.d.ts +15 -0
  127. package/dist/lib/research/summarize.js +11 -0
  128. package/dist/lib/research/url.d.ts +9 -6
  129. package/dist/lib/research/url.js +33 -21
  130. package/dist/lib/text.d.ts +7 -2
  131. package/dist/lib/text.js +13 -2
  132. package/dist/lib/value-flags.d.ts +14 -0
  133. package/dist/lib/value-flags.js +28 -0
  134. package/dist/sites/coveo.d.ts +10 -0
  135. package/dist/sites/coveo.js +10 -0
  136. package/dist/sites/index.d.ts +16 -0
  137. package/dist/sites/index.js +16 -4
  138. package/dist/sites/markdown-twin.d.ts +31 -0
  139. package/dist/sites/markdown-twin.js +56 -0
  140. package/dist/sites/salesforce/fetch-page.d.ts +8 -2
  141. package/dist/sites/salesforce/fetch-page.js +14 -3
  142. package/dist/sites/salesforce/help-tooling-markdown.d.ts +30 -0
  143. package/dist/sites/salesforce/help-tooling-markdown.js +74 -0
  144. package/dist/sites/salesforce/index.d.ts +1 -0
  145. package/dist/sites/salesforce/index.js +1 -0
  146. package/dist/sites/salesforce-developer/fetch-page.d.ts +6 -1
  147. package/dist/sites/salesforce-developer/fetch-page.js +6 -1
  148. package/dist/sites/salesforce-developer/index.d.ts +1 -0
  149. package/dist/sites/salesforce-developer/index.js +1 -0
  150. package/dist/sites/salesforce-developer/route-markdown.d.ts +18 -6
  151. package/dist/sites/salesforce-developer/route-markdown.js +29 -67
  152. package/dist/sites/salesforce-doc-fetch.d.ts +39 -5
  153. package/dist/sites/salesforce-doc-fetch.js +52 -7
  154. package/dist/sites/salesforce-dom-probe.d.ts +20 -2
  155. package/dist/sites/salesforce-dom-probe.js +20 -2
  156. package/dist/sites/types.d.ts +14 -8
  157. package/dist/sites/types.js +3 -0
  158. package/oclif.manifest.json +104 -98
  159. package/package.json +19 -7
  160. package/agents/agents/official-docs-researcher.agent.md +0 -87
  161. package/agents/instructions/web-research.instructions.md +0 -44
  162. package/agents/skills/web-research/SKILL.md +0 -118
@@ -0,0 +1,212 @@
1
+ import { cliErrorFields } from '../envelope.js';
2
+ import { SANDBOX_EGRESS_ERROR_MARKER } from './browser.js';
3
+ import { QUALITY_WARNING_PREFIX } from './extract.js';
4
+ import { PROXY_TUNNEL_REJECTION_PATTERN } from './proxy.js';
5
+ import { getArtifactPath } from './storage.js';
6
+ /**
7
+ * Human-prose warnings pulled out of `qualityNotes` (e.g. very-short extraction). The remaining
8
+ * `quality:*` machine codes stay `--json`-only; this is what stderr shows in human-readable mode.
9
+ */
10
+ export function extractionQualityWarnings(qualityNotes) {
11
+ return qualityNotes
12
+ .filter((note) => note.startsWith(QUALITY_WARNING_PREFIX))
13
+ .map((note) => note.slice(QUALITY_WARNING_PREFIX.length));
14
+ }
15
+ /** Remap write-implying cache statuses when dry-run/read-only skipped persistence. */
16
+ export function reportCacheStatus(status, dryRun) {
17
+ if (!dryRun)
18
+ return status;
19
+ switch (status) {
20
+ case 'miss':
21
+ return 'would_fetch';
22
+ case 'refreshed':
23
+ return 'would_refresh';
24
+ case 'revalidated':
25
+ return 'would_revalidate';
26
+ default:
27
+ return status;
28
+ }
29
+ }
30
+ /** Human spinner labels for reported (possibly remapped) cache statuses. */
31
+ export const FETCH_STATUS_LABEL = {
32
+ hit: 'cached',
33
+ miss: 'done',
34
+ refreshed: 'refreshed',
35
+ revalidated: 'revalidated',
36
+ stale: 'served stale',
37
+ would_fetch: 'previewed (not cached)',
38
+ would_refresh: 'previewed refresh',
39
+ would_revalidate: 'previewed revalidate',
40
+ };
41
+ /** Machine-readable success/preview payload for one fetch URL. */
42
+ export function buildFetchResultData(input) {
43
+ const { artifact, format, dryRun } = input;
44
+ const content = format === 'compressed' ? artifact.compressed : artifact.detailed;
45
+ return {
46
+ schemaVersion: 1,
47
+ command: 'fetch',
48
+ dryRun,
49
+ cache: {
50
+ key: input.cacheKey,
51
+ // Preview runs must not claim a write landed.
52
+ status: reportCacheStatus(input.cacheStatus, dryRun),
53
+ freshness: input.freshnessState,
54
+ path: getArtifactPath(input.storageDir, input.cacheKey),
55
+ storage: input.storageMode,
56
+ redirectedToGlobal: input.redirectedToGlobal,
57
+ },
58
+ source: {
59
+ url: input.url,
60
+ normalizedUrl: input.normalizedUrl,
61
+ captureMethod: artifact.metadata.capture_method,
62
+ extractionStatus: artifact.metadata.extraction_status,
63
+ extractionConfidence: artifact.metadata.extraction_confidence,
64
+ qualityNotes: artifact.metadata.quality_notes,
65
+ fetchedAt: artifact.metadata.fetched_at,
66
+ validatedAt: artifact.metadata.validated_at,
67
+ staleAfter: artifact.metadata.stale_after,
68
+ },
69
+ artifactType: artifact.metadata.artifact_type,
70
+ docsEngine: artifact.metadata.docs_engine,
71
+ docsFramework: artifact.metadata.docs_framework,
72
+ sourceDocUrl: artifact.metadata.source_doc_url,
73
+ searchProvider: artifact.metadata.search_provider,
74
+ format,
75
+ tokenEstimate: format === 'compressed'
76
+ ? artifact.metadata.token_estimate.compressed
77
+ : artifact.metadata.token_estimate.detailed,
78
+ content,
79
+ };
80
+ }
81
+ /** Per-URL failure row for multi-URL batches — keeps prior successes in `data`. */
82
+ export function buildFetchFailureResult(input) {
83
+ const { url, dryRun, err, fallbackGuidance } = input;
84
+ // Prefer the throw site's suggestions/ref; otherwise attach fetch-specific recovery hints.
85
+ const shaped = err.suggestions?.length || err.ref ? err : { ...err, ...fallbackGuidance };
86
+ return {
87
+ schemaVersion: 1,
88
+ command: 'fetch',
89
+ dryRun,
90
+ error: cliErrorFields(shaped, 'FETCH_FAILED'),
91
+ cache: null,
92
+ source: { url, normalizedUrl: null },
93
+ content: null,
94
+ };
95
+ }
96
+ /**
97
+ * Shape a caught runtime error into a batch failure row (message + guidance).
98
+ * CLIError rows should call {@link buildFetchFailureResult} directly so oclif fields are preserved.
99
+ */
100
+ export function buildFetchFailureFromCaught(bin, url, err, dryRun) {
101
+ const message = describeError(err);
102
+ return buildFetchFailureResult({
103
+ url,
104
+ dryRun,
105
+ err: { message, code: 'FETCH_FAILED' },
106
+ fallbackGuidance: fetchFailureGuidance(message, url, bin),
107
+ });
108
+ }
109
+ // How many `.cause` levels to walk. Real undici/Node transport errors nest at most 1-2 deep;
110
+ // the cap exists so a self-referential or cyclic `.cause` chain (Error.cause is an arbitrary,
111
+ // unvalidated property — nothing prevents `e.cause = e`, whether from a bug in this codebase or
112
+ // a dependency) can't hang the process, rather than being a limit expected to matter in practice.
113
+ const MAX_CAUSE_DEPTH = 10;
114
+ // Walks an Error's `.cause` chain and joins every message, deepest last. Node/undici's fetch
115
+ // throws a generic `TypeError: fetch failed` for transport-level failures, with the actionable
116
+ // detail (e.g. a proxy's tunnel rejection) nested one or two `.cause` levels down.
117
+ export function describeError(err) {
118
+ if (!(err instanceof Error))
119
+ return String(err);
120
+ const parts = [err.message];
121
+ let cause = err.cause;
122
+ for (let depth = 0; depth < MAX_CAUSE_DEPTH && cause instanceof Error; depth++) {
123
+ parts.push(cause.message);
124
+ cause = cause.cause;
125
+ }
126
+ return parts.join(': ');
127
+ }
128
+ // Maps a runtime fetch failure to recovery steps keyed off its message. Returns undefined for
129
+ // unrecognized failures, which then surface with their original message and no extra hint. The
130
+ // patterns match the thrown text in fetcher.ts/browser.ts; keep them in sync if those messages
131
+ // change. Resolutions mirror the published troubleshooting guide.
132
+ export function fetchFailureGuidance(message, url, bin = 'bonsai') {
133
+ const ref = 'https://bonsai.rhino-inquisitor.com/troubleshooting';
134
+ // Name the pipe step explicitly: bare `import --stdin` blocks waiting for input, so a reader (or
135
+ // an agent) running the hint verbatim would hang. Show the content being piped in from a file.
136
+ const importHint = `Open it in a browser, save the page, then import it: cat page.md | ${bin} import ${url} --stdin`;
137
+ // 401/403: an auth wall or anti-scraping WAF. Bonsai has no authenticated-fetch path in v1.
138
+ if (/failed with status 40[13]\b/.test(message)) {
139
+ return {
140
+ suggestions: ['The page requires authentication or blocks automated requests.', importHint],
141
+ ref,
142
+ };
143
+ }
144
+ if (/failed with status 404\b/.test(message)) {
145
+ return { suggestions: ['Check the URL is correct and the page still exists.'] };
146
+ }
147
+ if (/failed with status 5\d\d\b/.test(message)) {
148
+ return {
149
+ suggestions: ['The server returned an error. Retry later or verify the host is healthy.'],
150
+ };
151
+ }
152
+ // Server returned a non-HTML body (JSON, binary, or no Content-Type at all). Scope the opener to
153
+ // the real failure: --rendered does produce HTML, so "only scrapes HTML" would read as misleading.
154
+ if (/Rejected content type|does not look like HTML/.test(message)) {
155
+ return {
156
+ suggestions: [
157
+ 'The server returned a non-HTML response (e.g. JSON or binary).',
158
+ 'If the page is rendered by client-side JavaScript, retry with --rendered.',
159
+ importHint,
160
+ ],
161
+ ref,
162
+ };
163
+ }
164
+ if (/DNS resolution failed/.test(message)) {
165
+ return {
166
+ suggestions: ['Check the hostname is spelled correctly and resolves on a public network.'],
167
+ };
168
+ }
169
+ // Chrome's own network stack hit a proxy-enforced sandbox egress block (browser.ts's
170
+ // describeNavigationFailure already explains the cause and the fix); only a workaround and the
171
+ // docs ref are added here.
172
+ if (message.includes(SANDBOX_EGRESS_ERROR_MARKER)) {
173
+ return { suggestions: [importHint], ref };
174
+ }
175
+ // A configured proxy (HTTPS_PROXY/HTTP_PROXY) refused to tunnel to this host — distinct from
176
+ // the destination's own 401/403 above, which is the site itself rejecting the request. Shared
177
+ // with fetcher.ts's retry-fallback check so the two stay in sync by construction.
178
+ if (PROXY_TUNNEL_REJECTION_PATTERN.test(message)) {
179
+ return {
180
+ suggestions: [
181
+ "The configured proxy (HTTPS_PROXY/HTTP_PROXY) won't reach this host. Check its allowlist, or add the host to NO_PROXY to bypass the proxy for it.",
182
+ importHint,
183
+ ],
184
+ ref,
185
+ };
186
+ }
187
+ // The connection failed before any HTTP response was received — network-down, DNS, TLS, or a
188
+ // proxy issue not specifically matched above. Node/undici's fetch reports all of these as a
189
+ // generic "fetch failed" TypeError with no further suffix; the negative lookahead keeps this
190
+ // from also matching assertOk's "Fetch failed with status NNN ..." (a real HTTP response was
191
+ // received in that case, so it isn't a transport failure at all).
192
+ if (/^fetch failed\b(?!\s+with status)/i.test(message)) {
193
+ return {
194
+ suggestions: [
195
+ 'The connection failed before a response was received. Check network connectivity and any HTTPS_PROXY/HTTP_PROXY/NO_PROXY settings.',
196
+ importHint,
197
+ ],
198
+ ref,
199
+ };
200
+ }
201
+ // A hostname that only resolves to a private/local IP at request time (literal private addresses
202
+ // are rejected earlier with exit 2). Blocked to prevent SSRF; only public hosts can be fetched.
203
+ if (/blocked local or private target/.test(message)) {
204
+ return {
205
+ suggestions: [
206
+ 'The hostname resolves to a private or local address, which is blocked to prevent SSRF. Only public http(s) hosts can be fetched.',
207
+ ],
208
+ ref,
209
+ };
210
+ }
211
+ return undefined;
212
+ }
@@ -1,9 +1,15 @@
1
+ /** Optional transport limits and headers for {@link fetchStaticHtml} / {@link fetchText}. */
1
2
  export interface FetchOptions {
3
+ /** Abort the request after this many milliseconds. */
2
4
  timeoutMs?: number;
5
+ /** Maximum response body size in bytes. */
3
6
  bodyLimitBytes?: number;
7
+ /** Maximum number of HTTP redirects to follow. */
4
8
  maxRedirects?: number;
9
+ /** Extra request headers (e.g. conditional-request validators). */
5
10
  headers?: Record<string, string>;
6
11
  }
12
+ /** Successful or not-modified HTTP fetch result used by capture and revalidation. */
7
13
  export interface FetchResult {
8
14
  status: number;
9
15
  contentType: string | null;
@@ -13,17 +19,49 @@ export interface FetchResult {
13
19
  responseSize: number;
14
20
  content: string;
15
21
  }
22
+ /** The fetched-page fields artifact building consumes — FetchResult minus the transport status. */
23
+ export type FetchedContent = Omit<FetchResult, 'status'>;
24
+ /**
25
+ * Heuristic: body looks like HTML/XML when `Content-Type` is missing.
26
+ *
27
+ * @param text - Response body text.
28
+ * @returns `true` when the trimmed body starts with a common HTML/XML marker.
29
+ */
16
30
  export declare function looksLikeHtml(text: string): boolean;
31
+ /**
32
+ * Resolve a hostname and reject private/loopback targets.
33
+ * May skip local DNS when only a proxy can resolve the host.
34
+ *
35
+ * @param hostname - Host to check (IPv6 literals may be bracketed).
36
+ * @throws {Error} When the resolved address is not safe to fetch.
37
+ */
17
38
  export declare function checkDnsSafety(hostname: string): Promise<void>;
18
- export declare function fetchStaticHtml(url: string, options?: FetchOptions): Promise<FetchResult>;
19
39
  /**
20
- * Fetches a non-HTML text/data resource (llms.txt, route Markdown, search index) with the same
21
- * DNS, redirect, timeout, and body-size safety as fetchStaticHtml. Does not enforce a content
22
- * type; the caller must validate the returned body is the artifact kind it expected.
40
+ * True when `err` is the static fetcher's own content-type policy rejection (non-HTML body or
41
+ * content-type), as opposed to a transient failure (network error, timeout, HTTP error status).
42
+ * The caller uses this to distinguish "this URL fundamentally isn't a web page" — where an
43
+ * automatic rendered-browser retry would just render Chrome's built-in JSON/PDF viewer and produce
44
+ * a corrupted-looking "extracted" artifact — from "this page may need JavaScript", where retrying
45
+ * rendered is the whole point of the fallback.
23
46
  */
24
- export declare function fetchText(url: string, options?: FetchOptions): Promise<FetchResult>;
47
+ export declare function isUnsupportedContentTypeError(err: unknown): boolean;
25
48
  /**
26
- * Sends a JSON POST and returns the response body text. Used by remote search connectors (e.g.
27
- * Algolia DocSearch). Enforces the same DNS safety, timeout, and body-size limits as GET fetches.
49
+ * Fetch an HTML page with DNS, redirect, timeout, and body-size safety.
50
+ *
51
+ * @param url - Absolute URL to fetch.
52
+ * @param options - Optional timeout, body limit, redirect cap, and headers.
53
+ * @returns Fetch result including body content and validator headers.
54
+ * @throws {Error} On unsafe DNS, non-HTML content, HTTP failure, or redirect limits.
28
55
  */
29
- export declare function postJson(url: string, body: unknown, headers?: Record<string, string>, options?: FetchOptions): Promise<string>;
56
+ export declare function fetchStaticHtml(url: string, options?: FetchOptions): Promise<FetchResult>;
57
+ /**
58
+ * Fetch a non-HTML text/data resource (llms.txt, route Markdown, search index) with the same
59
+ * DNS, redirect, timeout, and body-size safety as {@link fetchStaticHtml}. Does not enforce a
60
+ * content type; the caller must validate the returned body is the artifact kind it expected.
61
+ *
62
+ * @param url - Absolute URL to fetch.
63
+ * @param options - Optional timeout, body limit, redirect cap, and headers.
64
+ * @returns Fetch result including body text and validator headers.
65
+ * @throws {Error} On unsafe DNS, HTTP failure, or redirect limits.
66
+ */
67
+ export declare function fetchText(url: string, options?: FetchOptions): Promise<FetchResult>;
@@ -3,6 +3,11 @@ import { isIP } from 'node:net';
3
3
  import { fetch as undiciFetch } from 'undici';
4
4
  import { isSafeIp, normalizeUrl } from './url.js';
5
5
  import { getProxyDispatcher, isProxyConfigured, PROXY_TUNNEL_REJECTION_PATTERN } from './proxy.js';
6
+ const DEFAULT_FETCH_TIMEOUT_MS = 10_000;
7
+ const DEFAULT_BODY_LIMIT_BYTES = 2 * 1024 * 1024;
8
+ const DEFAULT_MAX_REDIRECTS = 5;
9
+ const HTTP_NOT_MODIFIED = 304;
10
+ const REDIRECT_STATUS_CODES = new Set([301, 302, 303, 307, 308]);
6
11
  // Runs one fetch attempt with its own fresh AbortController/timeout, so that when `doFetch` makes
7
12
  // a second attempt (the proxy fallback below) it gets the full timeout budget rather than
8
13
  // whatever was left over from the first attempt's signal. Loosely typed at this one boundary
@@ -56,6 +61,12 @@ async function doFetch(url, init, timeoutMs) {
56
61
  return fetchWithTimeout(fetch, url, init, timeoutMs);
57
62
  }
58
63
  }
64
+ /**
65
+ * Heuristic: body looks like HTML/XML when `Content-Type` is missing.
66
+ *
67
+ * @param text - Response body text.
68
+ * @returns `true` when the trimmed body starts with a common HTML/XML marker.
69
+ */
59
70
  export function looksLikeHtml(text) {
60
71
  const trimmed = text.trimStart().toLowerCase();
61
72
  return (trimmed.startsWith('<!doctype html') ||
@@ -63,6 +74,13 @@ export function looksLikeHtml(text) {
63
74
  trimmed.startsWith('<xml') ||
64
75
  trimmed.startsWith('<?xml'));
65
76
  }
77
+ /**
78
+ * Resolve a hostname and reject private/loopback targets.
79
+ * May skip local DNS when only a proxy can resolve the host.
80
+ *
81
+ * @param hostname - Host to check (IPv6 literals may be bracketed).
82
+ * @throws {Error} When the resolved address is not safe to fetch.
83
+ */
66
84
  export async function checkDnsSafety(hostname) {
67
85
  let hostToResolve = hostname;
68
86
  if (hostToResolve.startsWith('[') && hostToResolve.endsWith(']')) {
@@ -128,15 +146,32 @@ function headerMeta(res) {
128
146
  }
129
147
  // A 304 carries no body; return an empty result that preserves the validator headers.
130
148
  function notModifiedResult(res, currentUrl) {
131
- return { status: 304, ...headerMeta(res), finalUrl: currentUrl, responseSize: 0, content: '' };
149
+ return {
150
+ status: HTTP_NOT_MODIFIED,
151
+ ...headerMeta(res),
152
+ finalUrl: currentUrl,
153
+ responseSize: 0,
154
+ content: '',
155
+ };
132
156
  }
133
157
  function assertOk(res) {
134
158
  if (!res.ok) {
135
159
  throw new Error(`Fetch failed with status ${res.status} ${res.statusText}`);
136
160
  }
137
161
  }
162
+ /**
163
+ * True when `err` is the static fetcher's own content-type policy rejection (non-HTML body or
164
+ * content-type), as opposed to a transient failure (network error, timeout, HTTP error status).
165
+ * The caller uses this to distinguish "this URL fundamentally isn't a web page" — where an
166
+ * automatic rendered-browser retry would just render Chrome's built-in JSON/PDF viewer and produce
167
+ * a corrupted-looking "extracted" artifact — from "this page may need JavaScript", where retrying
168
+ * rendered is the whole point of the fallback.
169
+ */
170
+ export function isUnsupportedContentTypeError(err) {
171
+ return err instanceof Error && /^Rejected /.test(err.message);
172
+ }
138
173
  async function processFetchResponse(res, limit, currentUrl) {
139
- if (res.status === 304)
174
+ if (res.status === HTTP_NOT_MODIFIED)
140
175
  return notModifiedResult(res, currentUrl);
141
176
  assertOk(res);
142
177
  const contentType = res.headers.get('content-type');
@@ -163,7 +198,7 @@ async function processFetchResponse(res, limit, currentUrl) {
163
198
  // processFetchResponse (304, status, size limit) but does NOT reject by content type — the caller
164
199
  // validates that the body is the artifact kind it asked for. Treated as untrusted text.
165
200
  async function processTextResponse(res, limit, currentUrl) {
166
- if (res.status === 304)
201
+ if (res.status === HTTP_NOT_MODIFIED)
167
202
  return notModifiedResult(res, currentUrl);
168
203
  assertOk(res);
169
204
  const bodyBytes = await readBodyWithLimit(res.body, limit);
@@ -178,9 +213,9 @@ async function processTextResponse(res, limit, currentUrl) {
178
213
  // Shared redirect/DNS/timeout loop. The `process` callback decides how to validate and shape the
179
214
  // final (non-redirect) response, so HTML and text/data fetches reuse identical transport safety.
180
215
  async function fetchWithRedirects(url, options, process) {
181
- const timeout = options.timeoutMs ?? 10_000;
182
- const limit = options.bodyLimitBytes ?? 2 * 1024 * 1024;
183
- const maxRedirects = options.maxRedirects ?? 5;
216
+ const timeout = options.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS;
217
+ const limit = options.bodyLimitBytes ?? DEFAULT_BODY_LIMIT_BYTES;
218
+ const maxRedirects = options.maxRedirects ?? DEFAULT_MAX_REDIRECTS;
184
219
  const initialHeaders = options.headers ?? {};
185
220
  let currentUrl = normalizeUrl(url);
186
221
  let redirectCount = 0;
@@ -188,7 +223,7 @@ async function fetchWithRedirects(url, options, process) {
188
223
  const parsedUrl = new URL(currentUrl);
189
224
  await checkDnsSafety(parsedUrl.hostname);
190
225
  const res = await doFetch(currentUrl, { method: 'GET', headers: initialHeaders, redirect: 'manual' }, timeout);
191
- if ([301, 302, 303, 307, 308].includes(res.status)) {
226
+ if (REDIRECT_STATUS_CODES.has(res.status)) {
192
227
  redirectCount++;
193
228
  if (redirectCount > maxRedirects) {
194
229
  throw new Error(`Too many redirects. Exceeded limit of ${maxRedirects}.`);
@@ -203,34 +238,27 @@ async function fetchWithRedirects(url, options, process) {
203
238
  return await process(res, limit, currentUrl);
204
239
  }
205
240
  }
241
+ /**
242
+ * Fetch an HTML page with DNS, redirect, timeout, and body-size safety.
243
+ *
244
+ * @param url - Absolute URL to fetch.
245
+ * @param options - Optional timeout, body limit, redirect cap, and headers.
246
+ * @returns Fetch result including body content and validator headers.
247
+ * @throws {Error} On unsafe DNS, non-HTML content, HTTP failure, or redirect limits.
248
+ */
206
249
  export async function fetchStaticHtml(url, options = {}) {
207
250
  return fetchWithRedirects(url, options, processFetchResponse);
208
251
  }
209
252
  /**
210
- * Fetches a non-HTML text/data resource (llms.txt, route Markdown, search index) with the same
211
- * DNS, redirect, timeout, and body-size safety as fetchStaticHtml. Does not enforce a content
212
- * type; the caller must validate the returned body is the artifact kind it expected.
253
+ * Fetch a non-HTML text/data resource (llms.txt, route Markdown, search index) with the same
254
+ * DNS, redirect, timeout, and body-size safety as {@link fetchStaticHtml}. Does not enforce a
255
+ * content type; the caller must validate the returned body is the artifact kind it expected.
256
+ *
257
+ * @param url - Absolute URL to fetch.
258
+ * @param options - Optional timeout, body limit, redirect cap, and headers.
259
+ * @returns Fetch result including body text and validator headers.
260
+ * @throws {Error} On unsafe DNS, HTTP failure, or redirect limits.
213
261
  */
214
262
  export async function fetchText(url, options = {}) {
215
263
  return fetchWithRedirects(url, options, processTextResponse);
216
264
  }
217
- /**
218
- * Sends a JSON POST and returns the response body text. Used by remote search connectors (e.g.
219
- * Algolia DocSearch). Enforces the same DNS safety, timeout, and body-size limits as GET fetches.
220
- */
221
- export async function postJson(url, body, headers = {}, options = {}) {
222
- const timeout = options.timeoutMs ?? 10_000;
223
- const limit = options.bodyLimitBytes ?? 2 * 1024 * 1024;
224
- const target = normalizeUrl(url);
225
- await checkDnsSafety(new URL(target).hostname);
226
- const res = await doFetch(target, {
227
- method: 'POST',
228
- headers: { 'content-type': 'application/json', ...headers },
229
- body: JSON.stringify(body),
230
- redirect: 'error',
231
- }, timeout);
232
- if (!res.ok)
233
- throw new Error(`Search request failed with status ${res.status} ${res.statusText}`);
234
- const bytes = await readBodyWithLimit(res.body, limit);
235
- return new TextDecoder().decode(bytes);
236
- }
@@ -1,29 +1,72 @@
1
1
  import type { ResearchArtifactMetadata } from './schema.js';
2
+ /** Fresh and grace windows derived from a tier and optional TTL override. */
2
3
  export interface FreshnessPolicy {
3
4
  freshWindowMs: number;
4
5
  graceWindowMs: number;
5
6
  }
6
7
  /**
7
- * Parses simple TTL duration strings to milliseconds. The unit `m` is months (not minutes); the
8
- * smallest unit is hours, since sub-hour cache lifespans are not meaningful for research artifacts.
8
+ * Parses a TTL/duration string into milliseconds.
9
+ * The unit `m` means months (not minutes); the smallest unit is hours.
10
+ *
11
+ * @param ttl - Whole number plus unit (`h`/`d`/`w`/`m`/`y`).
12
+ * @returns Duration in milliseconds.
13
+ * @throws {Error} When the format is invalid or the duration is zero.
9
14
  */
10
15
  export declare function parseTtlToMs(ttl: string): number;
11
16
  /**
12
- * Validate a duration-valued flag. Returns an actionable error message that names the
13
- * offending flag when the value is malformed, or null when the flag is absent or valid.
14
- * Callers map a non-null message onto `this.error(msg, { exit: 2 })`, so every command
15
- * reports the same parse failure while naming the exact flag the user passed.
17
+ * Validate a duration-valued flag.
18
+ *
19
+ * @param flag - Flag token for the error message (e.g. `--ttl`).
20
+ * @param value - Raw flag value, or `undefined` when omitted.
21
+ * @returns Actionable error message, or `null` when absent/valid.
16
22
  */
17
23
  export declare function durationFlagError(flag: string, value: string | undefined): string | null;
18
24
  /**
19
- * Derives the active freshness policy based on the tier and optional TTL override.
25
+ * Derives the active freshness policy from a tier and optional TTL override.
26
+ * When a TTL override is set, grace scales proportionally to the tier's default ratio.
27
+ *
28
+ * @param tier - Freshness tier (`stable` / `standard` / `volatile`).
29
+ * @param ttlOverride - Optional TTL string replacing the tier's fresh window.
30
+ * @returns Fresh and grace windows in milliseconds.
20
31
  */
21
- export declare function getPolicy(tier: 'stable' | 'standard' | 'volatile', ttlOverride?: string | null): FreshnessPolicy;
32
+ export declare function resolveFreshnessPolicy(tier: 'stable' | 'standard' | 'volatile', ttlOverride?: string | null): FreshnessPolicy;
22
33
  /**
23
34
  * Evaluates the freshness status of a cached artifact.
35
+ * `ttlOverride` / `tierOverride` let status (and similar read-only checks) evaluate "what if"
36
+ * policies without mutating the stored metadata. When omitted, the artifact's own tier/TTL win.
37
+ *
38
+ * @param meta - Artifact metadata providing timestamps, tier, and stored TTL.
39
+ * @param currentTime - Clock used for age calculation.
40
+ * @param ttlOverride - Optional TTL replacing the stored TTL for this evaluation.
41
+ * @param tierOverride - Optional tier replacing the stored tier for this evaluation.
42
+ * @returns `fresh`, `stale_grace`, or `stale_expired`.
24
43
  */
25
- export declare function evaluateFreshness(meta: ResearchArtifactMetadata, currentTime: Date, ttlOverride?: string | null): 'fresh' | 'stale_grace' | 'stale_expired';
44
+ export declare function evaluateFreshness(meta: ResearchArtifactMetadata, currentTime: Date, ttlOverride?: string | null, tierOverride?: 'stable' | 'standard' | 'volatile' | null): 'fresh' | 'stale_grace' | 'stale_expired';
26
45
  /**
27
- * Checks if the cached entry's age exceeds the specified max-age duration.
46
+ * Checks whether the cached entry's age exceeds the specified max-age duration.
47
+ *
48
+ * @param cached - Artifact (or metadata bag) with fetch/validate timestamps.
49
+ * @param currentTime - Clock used for age calculation.
50
+ * @param maxAge - Duration string, or `undefined` to skip the check.
51
+ * @returns `true` when max-age is set and exceeded.
28
52
  */
29
- export declare function checkMaxAgeExpired(cached: any, currentTime: Date, maxAge: string | undefined): boolean;
53
+ export declare function checkMaxAgeExpired(cached: {
54
+ metadata: Pick<ResearchArtifactMetadata, 'fetched_at' | 'validated_at'>;
55
+ }, currentTime: Date, maxAge: string | undefined): boolean;
56
+ /**
57
+ * Freshness for a cached artifact, applying `--max-age` before the tier/TTL windows.
58
+ *
59
+ * @param cached - Artifact with full metadata.
60
+ * @param currentTime - Clock used for age calculation.
61
+ * @param options.ttl - Optional TTL override.
62
+ * @param options.maxAge - Optional max-age duration; when exceeded returns `stale_expired`.
63
+ * @param options.tier - Optional tier override.
64
+ * @returns `fresh`, `stale_grace`, or `stale_expired`.
65
+ */
66
+ export declare function evaluateFreshnessWithMaxAge(cached: {
67
+ metadata: ResearchArtifactMetadata;
68
+ }, currentTime: Date, options?: {
69
+ ttl?: string | null;
70
+ maxAge?: string;
71
+ tier?: 'stable' | 'standard' | 'volatile' | null;
72
+ }): 'fresh' | 'stale_grace' | 'stale_expired';