@taurgis/bonsai 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. package/README.md +239 -0
  2. package/agents/agents/official-docs-researcher.agent.md +98 -0
  3. package/agents/instructions/web-research.instructions.md +29 -0
  4. package/agents/skills/web-research/SKILL.md +126 -0
  5. package/bin/cli.mjs +22 -0
  6. package/dist/base-command.d.ts +14 -0
  7. package/dist/base-command.js +56 -0
  8. package/dist/commands/config/base.d.ts +23 -0
  9. package/dist/commands/config/base.js +46 -0
  10. package/dist/commands/config/get.d.ts +20 -0
  11. package/dist/commands/config/get.js +48 -0
  12. package/dist/commands/config/list.d.ts +17 -0
  13. package/dist/commands/config/list.js +48 -0
  14. package/dist/commands/config/set.d.ts +31 -0
  15. package/dist/commands/config/set.js +94 -0
  16. package/dist/commands/config/unset.d.ts +21 -0
  17. package/dist/commands/config/unset.js +65 -0
  18. package/dist/commands/fetch.d.ts +38 -0
  19. package/dist/commands/fetch.js +311 -0
  20. package/dist/commands/import.d.ts +38 -0
  21. package/dist/commands/import.js +313 -0
  22. package/dist/commands/index.d.ts +35 -0
  23. package/dist/commands/index.js +306 -0
  24. package/dist/commands/inspect.d.ts +13 -0
  25. package/dist/commands/inspect.js +82 -0
  26. package/dist/commands/list.d.ts +25 -0
  27. package/dist/commands/list.js +143 -0
  28. package/dist/commands/prune.d.ts +24 -0
  29. package/dist/commands/prune.js +141 -0
  30. package/dist/commands/research/config/base.d.ts +23 -0
  31. package/dist/commands/research/config/base.js +46 -0
  32. package/dist/commands/research/config/get.d.ts +20 -0
  33. package/dist/commands/research/config/get.js +45 -0
  34. package/dist/commands/research/config/list.d.ts +17 -0
  35. package/dist/commands/research/config/list.js +42 -0
  36. package/dist/commands/research/config/set.d.ts +31 -0
  37. package/dist/commands/research/config/set.js +94 -0
  38. package/dist/commands/research/config/unset.d.ts +21 -0
  39. package/dist/commands/research/config/unset.js +59 -0
  40. package/dist/commands/research/import.d.ts +38 -0
  41. package/dist/commands/research/import.js +313 -0
  42. package/dist/commands/research/inspect.d.ts +13 -0
  43. package/dist/commands/research/inspect.js +86 -0
  44. package/dist/commands/research/list.d.ts +25 -0
  45. package/dist/commands/research/list.js +143 -0
  46. package/dist/commands/research/prune.d.ts +24 -0
  47. package/dist/commands/research/prune.js +141 -0
  48. package/dist/commands/research/search.d.ts +32 -0
  49. package/dist/commands/research/search.js +334 -0
  50. package/dist/commands/research/status.d.ts +17 -0
  51. package/dist/commands/research/status.js +108 -0
  52. package/dist/commands/research.d.ts +36 -0
  53. package/dist/commands/research.js +304 -0
  54. package/dist/commands/search.d.ts +32 -0
  55. package/dist/commands/search.js +334 -0
  56. package/dist/commands/status.d.ts +17 -0
  57. package/dist/commands/status.js +95 -0
  58. package/dist/commands.d.ts +24 -0
  59. package/dist/commands.js +24 -0
  60. package/dist/lib/config/index.d.ts +18 -0
  61. package/dist/lib/config/index.js +33 -0
  62. package/dist/lib/config/io.d.ts +8 -0
  63. package/dist/lib/config/io.js +75 -0
  64. package/dist/lib/config/resolve.d.ts +15 -0
  65. package/dist/lib/config/resolve.js +20 -0
  66. package/dist/lib/config/schema.d.ts +28 -0
  67. package/dist/lib/config/schema.js +32 -0
  68. package/dist/lib/research/artifact.d.ts +21 -0
  69. package/dist/lib/research/artifact.js +217 -0
  70. package/dist/lib/research/browser.d.ts +74 -0
  71. package/dist/lib/research/browser.js +359 -0
  72. package/dist/lib/research/cache-key.d.ts +5 -0
  73. package/dist/lib/research/cache-key.js +10 -0
  74. package/dist/lib/research/capture.d.ts +29 -0
  75. package/dist/lib/research/capture.js +181 -0
  76. package/dist/lib/research/compress.d.ts +5 -0
  77. package/dist/lib/research/compress.js +17 -0
  78. package/dist/lib/research/docs/__fixtures__/sphinx-searchindex.d.ts +1 -0
  79. package/dist/lib/research/docs/__fixtures__/sphinx-searchindex.js +2 -0
  80. package/dist/lib/research/docs/capabilities.d.ts +43 -0
  81. package/dist/lib/research/docs/capabilities.js +9 -0
  82. package/dist/lib/research/docs/clean-dom.d.ts +4 -0
  83. package/dist/lib/research/docs/clean-dom.js +41 -0
  84. package/dist/lib/research/docs/code-blocks.d.ts +5 -0
  85. package/dist/lib/research/docs/code-blocks.js +69 -0
  86. package/dist/lib/research/docs/detect.d.ts +6 -0
  87. package/dist/lib/research/docs/detect.js +239 -0
  88. package/dist/lib/research/docs/machine-readable.d.ts +21 -0
  89. package/dist/lib/research/docs/machine-readable.js +73 -0
  90. package/dist/lib/research/docs/markdown-source.d.ts +17 -0
  91. package/dist/lib/research/docs/markdown-source.js +194 -0
  92. package/dist/lib/research/docs/quality-gates.d.ts +9 -0
  93. package/dist/lib/research/docs/quality-gates.js +65 -0
  94. package/dist/lib/research/docs/remote-search-runner.d.ts +21 -0
  95. package/dist/lib/research/docs/remote-search-runner.js +54 -0
  96. package/dist/lib/research/docs/remote-search.d.ts +14 -0
  97. package/dist/lib/research/docs/remote-search.js +101 -0
  98. package/dist/lib/research/docs/rsc.d.ts +10 -0
  99. package/dist/lib/research/docs/rsc.js +30 -0
  100. package/dist/lib/research/docs/search-index.d.ts +13 -0
  101. package/dist/lib/research/docs/search-index.js +176 -0
  102. package/dist/lib/research/docs/section-artifacts.d.ts +12 -0
  103. package/dist/lib/research/docs/section-artifacts.js +75 -0
  104. package/dist/lib/research/docs/sections.d.ts +12 -0
  105. package/dist/lib/research/docs/sections.js +67 -0
  106. package/dist/lib/research/docs/source-map.d.ts +19 -0
  107. package/dist/lib/research/docs/source-map.js +87 -0
  108. package/dist/lib/research/docs/validate.d.ts +15 -0
  109. package/dist/lib/research/docs/validate.js +63 -0
  110. package/dist/lib/research/extract.d.ts +11 -0
  111. package/dist/lib/research/extract.js +83 -0
  112. package/dist/lib/research/fetcher.d.ts +29 -0
  113. package/dist/lib/research/fetcher.js +192 -0
  114. package/dist/lib/research/freshness.d.ts +21 -0
  115. package/dist/lib/research/freshness.js +82 -0
  116. package/dist/lib/research/markdown.d.ts +6 -0
  117. package/dist/lib/research/markdown.js +66 -0
  118. package/dist/lib/research/regression.d.ts +23 -0
  119. package/dist/lib/research/regression.js +93 -0
  120. package/dist/lib/research/resolve-target.d.ts +18 -0
  121. package/dist/lib/research/resolve-target.js +20 -0
  122. package/dist/lib/research/revalidate.d.ts +32 -0
  123. package/dist/lib/research/revalidate.js +152 -0
  124. package/dist/lib/research/schema.d.ts +47 -0
  125. package/dist/lib/research/schema.js +1 -0
  126. package/dist/lib/research/secret-scan.d.ts +5 -0
  127. package/dist/lib/research/secret-scan.js +37 -0
  128. package/dist/lib/research/secure-write.d.ts +16 -0
  129. package/dist/lib/research/secure-write.js +17 -0
  130. package/dist/lib/research/storage.d.ts +52 -0
  131. package/dist/lib/research/storage.js +182 -0
  132. package/dist/lib/research/store-roots.d.ts +29 -0
  133. package/dist/lib/research/store-roots.js +28 -0
  134. package/dist/lib/research/token-estimate.d.ts +5 -0
  135. package/dist/lib/research/token-estimate.js +7 -0
  136. package/dist/lib/research/url.d.ts +24 -0
  137. package/dist/lib/research/url.js +121 -0
  138. package/dist/sites/index.d.ts +4 -0
  139. package/dist/sites/index.js +28 -0
  140. package/dist/sites/salesforce/browser-search.d.ts +7 -0
  141. package/dist/sites/salesforce/browser-search.js +82 -0
  142. package/dist/sites/salesforce/coveo.d.ts +23 -0
  143. package/dist/sites/salesforce/coveo.js +161 -0
  144. package/dist/sites/salesforce/fetch-page.d.ts +6 -0
  145. package/dist/sites/salesforce/fetch-page.js +42 -0
  146. package/dist/sites/salesforce/index.d.ts +2 -0
  147. package/dist/sites/salesforce/index.js +16 -0
  148. package/dist/sites/salesforce/token.d.ts +26 -0
  149. package/dist/sites/salesforce/token.js +154 -0
  150. package/dist/sites/salesforce-developer/fetch-page.d.ts +3 -0
  151. package/dist/sites/salesforce-developer/fetch-page.js +18 -0
  152. package/dist/sites/salesforce-developer/index.d.ts +2 -0
  153. package/dist/sites/salesforce-developer/index.js +12 -0
  154. package/dist/sites/salesforce-doc-fetch.d.ts +14 -0
  155. package/dist/sites/salesforce-doc-fetch.js +321 -0
  156. package/dist/sites/types.d.ts +27 -0
  157. package/dist/sites/types.js +1 -0
  158. package/oclif.manifest.json +845 -0
  159. package/package.json +78 -0
@@ -0,0 +1,192 @@
1
+ import { lookup } from 'node:dns/promises';
2
+ import { isIP } from 'node:net';
3
+ import { isSafeIp, normalizeUrl } from './url.js';
4
+ export function looksLikeHtml(text) {
5
+ const trimmed = text.trimStart().toLowerCase();
6
+ return (trimmed.startsWith('<!doctype html') ||
7
+ trimmed.startsWith('<html') ||
8
+ trimmed.startsWith('<xml') ||
9
+ trimmed.startsWith('<?xml'));
10
+ }
11
+ export async function checkDnsSafety(hostname) {
12
+ let hostToResolve = hostname;
13
+ if (hostToResolve.startsWith('[') && hostToResolve.endsWith(']')) {
14
+ hostToResolve = hostToResolve.slice(1, -1);
15
+ }
16
+ if (isIP(hostToResolve) !== 0) {
17
+ if (!isSafeIp(hostToResolve)) {
18
+ throw new Error(`IP address "${hostToResolve}" is a blocked local or private target.`);
19
+ }
20
+ return;
21
+ }
22
+ try {
23
+ const addresses = await lookup(hostToResolve, { all: true });
24
+ for (const addr of addresses) {
25
+ if (!isSafeIp(addr.address)) {
26
+ throw new Error(`IP address "${addr.address}" resolved for "${hostname}" is a blocked local or private target.`);
27
+ }
28
+ }
29
+ }
30
+ catch (err) {
31
+ throw new Error(`DNS resolution failed for hostname "${hostname}": ${err.message}`);
32
+ }
33
+ }
34
+ async function readBodyWithLimit(body, limit) {
35
+ if (!body) {
36
+ return new Uint8Array(0);
37
+ }
38
+ const chunks = [];
39
+ let totalBytes = 0;
40
+ for await (const chunk of body) {
41
+ const chunkBytes = typeof chunk === 'string' ? Buffer.from(chunk) : chunk;
42
+ totalBytes += chunkBytes.byteLength;
43
+ if (totalBytes > limit) {
44
+ throw new Error(`Response body size limit exceeded. Limit is ${limit} bytes.`);
45
+ }
46
+ chunks.push(chunkBytes);
47
+ }
48
+ const result = new Uint8Array(totalBytes);
49
+ let offset = 0;
50
+ for (const chunk of chunks) {
51
+ result.set(chunk, offset);
52
+ offset += chunk.byteLength;
53
+ }
54
+ return result;
55
+ }
56
+ // Shared response metadata. Both the HTML and text processors build their FetchResult from these
57
+ // same validator headers, so reading them in one place keeps the two paths from drifting.
58
+ function headerMeta(res) {
59
+ return {
60
+ contentType: res.headers.get('content-type'),
61
+ etag: res.headers.get('etag'),
62
+ lastModified: res.headers.get('last-modified'),
63
+ };
64
+ }
65
+ // A 304 carries no body; return an empty result that preserves the validator headers.
66
+ function notModifiedResult(res, currentUrl) {
67
+ return { status: 304, ...headerMeta(res), finalUrl: currentUrl, responseSize: 0, content: '' };
68
+ }
69
+ function assertOk(res) {
70
+ if (!res.ok) {
71
+ throw new Error(`Fetch failed with status ${res.status} ${res.statusText}`);
72
+ }
73
+ }
74
+ async function processFetchResponse(res, limit, currentUrl) {
75
+ if (res.status === 304)
76
+ return notModifiedResult(res, currentUrl);
77
+ assertOk(res);
78
+ const contentType = res.headers.get('content-type');
79
+ if (contentType) {
80
+ const cleanType = (contentType.split(';')[0] || '').toLowerCase().trim();
81
+ if (cleanType !== 'text/html' && cleanType !== 'application/xhtml+xml') {
82
+ throw new Error(`Rejected content type "${contentType}". Only HTML is supported.`);
83
+ }
84
+ }
85
+ const bodyBytes = await readBodyWithLimit(res.body, limit);
86
+ const content = new TextDecoder().decode(bodyBytes);
87
+ if (!contentType && !looksLikeHtml(content)) {
88
+ throw new Error('Rejected response: missing Content-Type and body does not look like HTML.');
89
+ }
90
+ return {
91
+ status: res.status,
92
+ ...headerMeta(res),
93
+ finalUrl: currentUrl,
94
+ responseSize: bodyBytes.byteLength,
95
+ content,
96
+ };
97
+ }
98
+ // Processes a non-HTML response (llms.txt, route .md, search index JSON). Mirrors the safety of
99
+ // processFetchResponse (304, status, size limit) but does NOT reject by content type — the caller
100
+ // validates that the body is the artifact kind it asked for. Treated as untrusted text.
101
+ async function processTextResponse(res, limit, currentUrl) {
102
+ if (res.status === 304)
103
+ return notModifiedResult(res, currentUrl);
104
+ assertOk(res);
105
+ const bodyBytes = await readBodyWithLimit(res.body, limit);
106
+ return {
107
+ status: res.status,
108
+ ...headerMeta(res),
109
+ finalUrl: currentUrl,
110
+ responseSize: bodyBytes.byteLength,
111
+ content: new TextDecoder().decode(bodyBytes),
112
+ };
113
+ }
114
+ // Shared redirect/DNS/timeout loop. The `process` callback decides how to validate and shape the
115
+ // final (non-redirect) response, so HTML and text/data fetches reuse identical transport safety.
116
+ async function fetchWithRedirects(url, options, process) {
117
+ const timeout = options.timeoutMs ?? 10_000;
118
+ const limit = options.bodyLimitBytes ?? 2 * 1024 * 1024;
119
+ const maxRedirects = options.maxRedirects ?? 5;
120
+ const initialHeaders = options.headers ?? {};
121
+ let currentUrl = normalizeUrl(url);
122
+ let redirectCount = 0;
123
+ while (true) {
124
+ const parsedUrl = new URL(currentUrl);
125
+ await checkDnsSafety(parsedUrl.hostname);
126
+ const controller = new AbortController();
127
+ const id = setTimeout(() => controller.abort(), timeout);
128
+ try {
129
+ const res = await fetch(currentUrl, {
130
+ method: 'GET',
131
+ headers: initialHeaders,
132
+ redirect: 'manual',
133
+ signal: controller.signal,
134
+ });
135
+ if ([301, 302, 303, 307, 308].includes(res.status)) {
136
+ redirectCount++;
137
+ if (redirectCount > maxRedirects) {
138
+ throw new Error(`Too many redirects. Exceeded limit of ${maxRedirects}.`);
139
+ }
140
+ const location = res.headers.get('location');
141
+ if (!location) {
142
+ throw new Error(`Redirect response status ${res.status} missing Location header.`);
143
+ }
144
+ currentUrl = normalizeUrl(new URL(location, currentUrl).toString());
145
+ continue;
146
+ }
147
+ return await process(res, limit, currentUrl);
148
+ }
149
+ finally {
150
+ clearTimeout(id);
151
+ }
152
+ }
153
+ }
154
+ export async function fetchStaticHtml(url, options = {}) {
155
+ return fetchWithRedirects(url, options, processFetchResponse);
156
+ }
157
+ /**
158
+ * Fetches a non-HTML text/data resource (llms.txt, route Markdown, search index) with the same
159
+ * DNS, redirect, timeout, and body-size safety as fetchStaticHtml. Does not enforce a content
160
+ * type; the caller must validate the returned body is the artifact kind it expected.
161
+ */
162
+ export async function fetchText(url, options = {}) {
163
+ return fetchWithRedirects(url, options, processTextResponse);
164
+ }
165
+ /**
166
+ * Sends a JSON POST and returns the response body text. Used by remote search connectors (e.g.
167
+ * Algolia DocSearch). Enforces the same DNS safety, timeout, and body-size limits as GET fetches.
168
+ */
169
+ export async function postJson(url, body, headers = {}, options = {}) {
170
+ const timeout = options.timeoutMs ?? 10_000;
171
+ const limit = options.bodyLimitBytes ?? 2 * 1024 * 1024;
172
+ const target = normalizeUrl(url);
173
+ await checkDnsSafety(new URL(target).hostname);
174
+ const controller = new AbortController();
175
+ const id = setTimeout(() => controller.abort(), timeout);
176
+ try {
177
+ const res = await fetch(target, {
178
+ method: 'POST',
179
+ headers: { 'content-type': 'application/json', ...headers },
180
+ body: JSON.stringify(body),
181
+ redirect: 'error',
182
+ signal: controller.signal,
183
+ });
184
+ if (!res.ok)
185
+ throw new Error(`Search request failed with status ${res.status} ${res.statusText}`);
186
+ const bytes = await readBodyWithLimit(res.body, limit);
187
+ return new TextDecoder().decode(bytes);
188
+ }
189
+ finally {
190
+ clearTimeout(id);
191
+ }
192
+ }
@@ -0,0 +1,21 @@
1
+ import type { ResearchArtifactMetadata } from './schema.js';
2
+ export interface FreshnessPolicy {
3
+ freshWindowMs: number;
4
+ graceWindowMs: number;
5
+ }
6
+ /**
7
+ * Parses simple TTL duration strings (e.g. '24h', '7d', '30d') to milliseconds.
8
+ */
9
+ export declare function parseTtlToMs(ttl: string): number;
10
+ /**
11
+ * Derives the active freshness policy based on the tier and optional TTL override.
12
+ */
13
+ export declare function getPolicy(tier: 'stable' | 'standard' | 'volatile', ttlOverride?: string | null): FreshnessPolicy;
14
+ /**
15
+ * Evaluates the freshness status of a cached artifact.
16
+ */
17
+ export declare function evaluateFreshness(meta: ResearchArtifactMetadata, currentTime: Date, ttlOverride?: string | null): 'fresh' | 'stale_grace' | 'stale_expired';
18
+ /**
19
+ * Checks if the cached entry's age exceeds the specified max-age duration.
20
+ */
21
+ export declare function checkMaxAgeExpired(cached: any, currentTime: Date, maxAge: string | undefined): boolean;
@@ -0,0 +1,82 @@
1
+ /**
2
+ * Parses simple TTL duration strings (e.g. '24h', '7d', '30d') to milliseconds.
3
+ */
4
+ export function parseTtlToMs(ttl) {
5
+ const match = ttl.match(/^(\d+)([hdwmy])$/);
6
+ if (!match) {
7
+ throw new Error(`Invalid TTL format "${ttl}". Use formats like '24h', '7d', '30d'.`);
8
+ }
9
+ const val = parseInt(match[1] || '', 10);
10
+ const unit = match[2] || '';
11
+ const hour = 3600 * 1000;
12
+ const day = 24 * hour;
13
+ switch (unit) {
14
+ case 'h':
15
+ return val * hour;
16
+ case 'd':
17
+ return val * day;
18
+ case 'w':
19
+ return val * 7 * day;
20
+ case 'm':
21
+ return val * 30 * day;
22
+ case 'y':
23
+ return val * 365 * day;
24
+ default:
25
+ return 0;
26
+ }
27
+ }
28
+ /**
29
+ * Derives the active freshness policy based on the tier and optional TTL override.
30
+ */
31
+ export function getPolicy(tier, ttlOverride) {
32
+ const day = 24 * 3600 * 1000;
33
+ let freshWindowMs = 30 * day;
34
+ let graceWindowMs = 14 * day;
35
+ if (tier === 'stable') {
36
+ freshWindowMs = 180 * day;
37
+ graceWindowMs = 60 * day;
38
+ }
39
+ else if (tier === 'volatile') {
40
+ freshWindowMs = 7 * day;
41
+ graceWindowMs = 5 * day;
42
+ }
43
+ const activeTtl = ttlOverride;
44
+ if (activeTtl) {
45
+ freshWindowMs = parseTtlToMs(activeTtl);
46
+ const defaultFresh = tier === 'stable' ? 180 * day : tier === 'volatile' ? 7 * day : 30 * day;
47
+ const defaultGrace = tier === 'stable' ? 60 * day : tier === 'volatile' ? 5 * day : 14 * day;
48
+ graceWindowMs = Math.floor(freshWindowMs * (defaultGrace / defaultFresh));
49
+ }
50
+ return { freshWindowMs, graceWindowMs };
51
+ }
52
+ /**
53
+ * Evaluates the freshness status of a cached artifact.
54
+ */
55
+ export function evaluateFreshness(meta, currentTime, ttlOverride) {
56
+ const fetched = meta.fetched_at ? new Date(meta.fetched_at).getTime() : 0;
57
+ const validated = meta.validated_at ? new Date(meta.validated_at).getTime() : 0;
58
+ const baseTime = Math.max(fetched, validated);
59
+ const { freshWindowMs, graceWindowMs } = getPolicy(meta.tier, ttlOverride || meta.ttl);
60
+ const ageMs = currentTime.getTime() - baseTime;
61
+ if (ageMs <= freshWindowMs) {
62
+ return 'fresh';
63
+ }
64
+ if (ageMs <= freshWindowMs + graceWindowMs) {
65
+ return 'stale_grace';
66
+ }
67
+ return 'stale_expired';
68
+ }
69
+ /**
70
+ * Checks if the cached entry's age exceeds the specified max-age duration.
71
+ */
72
+ export function checkMaxAgeExpired(cached, currentTime, maxAge) {
73
+ if (!maxAge)
74
+ return false;
75
+ const fetched = cached.metadata.fetched_at ? new Date(cached.metadata.fetched_at).getTime() : 0;
76
+ const validated = cached.metadata.validated_at
77
+ ? new Date(cached.metadata.validated_at).getTime()
78
+ : 0;
79
+ const baseTime = Math.max(fetched, validated);
80
+ const ageMs = currentTime.getTime() - baseTime;
81
+ return ageMs > parseTtlToMs(maxAge);
82
+ }
@@ -0,0 +1,6 @@
1
+ /**
2
+ * Converts HTML content to clean, deterministic Markdown.
3
+ * Configured to use ATX-style headings (#) and fenced code blocks (```), with GFM tables.
4
+ */
5
+ export declare function dropEmptyLinks(markdown: string): string;
6
+ export declare function htmlToMarkdown(html: string): string;
@@ -0,0 +1,66 @@
1
+ import TurndownService from 'turndown';
2
+ // Cell text must survive as a single Markdown table cell: collapse internal whitespace (newlines
3
+ // from block children would split the row) and escape pipes (an unescaped `|` adds phantom columns).
4
+ function tableCellText(node) {
5
+ return (node.textContent || '').replace(/\s+/g, ' ').trim().replace(/\|/g, '\\|');
6
+ }
7
+ function rowCells(tr) {
8
+ return Array.from(tr.children).map(tableCellText);
9
+ }
10
+ // A table's own rows are direct <tr> children or rows inside its direct <thead>/<tbody>/<tfoot>.
11
+ // Walking the structure (rather than querySelectorAll('tr')) excludes rows of tables nested inside
12
+ // a cell, which would otherwise be absorbed as phantom rows of the outer table.
13
+ function ownRows(table) {
14
+ const rows = [];
15
+ for (const child of Array.from(table.children)) {
16
+ const tag = child.tagName.toLowerCase();
17
+ if (tag === 'tr')
18
+ rows.push(child);
19
+ else if (tag === 'thead' || tag === 'tbody' || tag === 'tfoot') {
20
+ for (const row of Array.from(child.children)) {
21
+ if (row.tagName.toLowerCase() === 'tr')
22
+ rows.push(row);
23
+ }
24
+ }
25
+ }
26
+ return rows;
27
+ }
28
+ // Turndown core ships no table rule (it emits the cells as run-together text). The canonical
29
+ // turndown-plugin-gfm is unmaintained and escapes neither pipes nor newlines in cells, so we add a
30
+ // small self-contained GFM rule: first <tr> is the header, ragged rows are padded, cells sanitized.
31
+ function addGfmTableRule(service) {
32
+ service.addRule('gfmTable', {
33
+ filter: 'table',
34
+ replacement: (_content, node) => {
35
+ const matrix = ownRows(node)
36
+ .map(rowCells)
37
+ .filter((cells) => cells.length > 0);
38
+ const [header, ...body] = matrix;
39
+ if (!header)
40
+ return '';
41
+ const columns = Math.max(header.length, ...body.map((cells) => cells.length));
42
+ const pad = (cells) => [...cells, ...Array(columns - cells.length).fill('')];
43
+ const line = (cells) => `| ${pad(cells).join(' | ')} |`;
44
+ const separator = `| ${Array(columns).fill('---').join(' | ')} |`;
45
+ return ['', line(header), separator, ...body.map(line), ''].join('\n');
46
+ },
47
+ });
48
+ }
49
+ /**
50
+ * Converts HTML content to clean, deterministic Markdown.
51
+ * Configured to use ATX-style headings (#) and fenced code blocks (```), with GFM tables.
52
+ */
53
+ // Icon-only/empty anchors (e.g. a heading's "#" permalink, a ReadMe query-param chip) convert to a
54
+ // text-less link `[](url)` that carries no information and just wastes tokens. Drop them, but never
55
+ // match images `![](url)` (empty alt is still a real image).
56
+ export function dropEmptyLinks(markdown) {
57
+ return markdown.replace(/(?<!!)\[\s*\]\([^)]*\)/g, '');
58
+ }
59
+ export function htmlToMarkdown(html) {
60
+ const turndownService = new TurndownService({
61
+ headingStyle: 'atx',
62
+ codeBlockStyle: 'fenced',
63
+ });
64
+ addGfmTableRule(turndownService);
65
+ return dropEmptyLinks(turndownService.turndown(html));
66
+ }
@@ -0,0 +1,23 @@
1
+ /**
2
+ * Canonicalizes a Salesforce docs CDN asset URL by replacing its volatile version segment with a
3
+ * placeholder, so a baseline doesn't churn every time the CDN bumps an asset version. Any other
4
+ * host (e.g. react.dev, tanstack.com) is returned unchanged.
5
+ */
6
+ export declare function normalizeRegressionAssetUrl(rawUrl: string): string;
7
+ /** Normalizes volatile bits of rendered Markdown so regression diffs reflect real content drift. */
8
+ export declare function normalizeRegressionMarkdown(markdown: string): string;
9
+ export interface ContentMetrics {
10
+ chars: number;
11
+ codeBlocks: number;
12
+ tables: number;
13
+ images: number;
14
+ orderedSteps: number;
15
+ bullets: number;
16
+ headings: number;
17
+ }
18
+ /**
19
+ * Structural content signals used to catch information loss between baseline and current output:
20
+ * a sudden drop in code blocks, tables, images, or steps flags that a fetch change dropped content.
21
+ */
22
+ export declare function contentMetrics(markdown: string): ContentMetrics;
23
+ export declare function leakageSignals(markdown: string): string[];
@@ -0,0 +1,93 @@
1
+ // Helpers for the live regression suite (testing/regression). Kept here as pure, unit-tested
2
+ // functions so the suite runner stays a thin orchestration script and the comparison logic is
3
+ // covered by the normal test run.
4
+ const SALESFORCE_DOCS_IMAGE_HOST = 'sf-zdocs-cdn-prod.zoominsoftware.com';
5
+ const SALESFORCE_DOCS_IMAGE_VERSION = '__asset_version__';
6
+ const UUID_SEGMENT = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
7
+ /**
8
+ * Canonicalizes a Salesforce docs CDN asset URL by replacing its volatile version segment with a
9
+ * placeholder, so a baseline doesn't churn every time the CDN bumps an asset version. Any other
10
+ * host (e.g. react.dev, tanstack.com) is returned unchanged.
11
+ */
12
+ export function normalizeRegressionAssetUrl(rawUrl) {
13
+ let url;
14
+ try {
15
+ url = new URL(rawUrl);
16
+ }
17
+ catch {
18
+ return rawUrl;
19
+ }
20
+ if (url.hostname !== SALESFORCE_DOCS_IMAGE_HOST)
21
+ return rawUrl;
22
+ const segments = url.pathname.split('/').filter(Boolean);
23
+ // Expect /<version>/.../images/...; only rewrite when the version segment is a UUID.
24
+ if (segments.length < 3 ||
25
+ !segments.includes('images') ||
26
+ !UUID_SEGMENT.test(segments[1] ?? '')) {
27
+ return rawUrl;
28
+ }
29
+ segments[1] = SALESFORCE_DOCS_IMAGE_VERSION;
30
+ return `${url.protocol}//${url.host}/${segments.join('/')}${url.search}${url.hash}`;
31
+ }
32
+ /** Normalizes volatile bits of rendered Markdown so regression diffs reflect real content drift. */
33
+ export function normalizeRegressionMarkdown(markdown) {
34
+ return markdown
35
+ .replace(/https:\/\/sf-zdocs-cdn-prod\.zoominsoftware\.com\/[^\s)]+/g, normalizeRegressionAssetUrl)
36
+ .replace(/\r\n/g, '\n')
37
+ .replace(/[ \t]+$/gm, '')
38
+ .replace(/\n{3,}/g, '\n\n')
39
+ .trim();
40
+ }
41
+ /**
42
+ * Structural content signals used to catch information loss between baseline and current output:
43
+ * a sudden drop in code blocks, tables, images, or steps flags that a fetch change dropped content.
44
+ */
45
+ export function contentMetrics(markdown) {
46
+ return {
47
+ chars: markdown.length,
48
+ codeBlocks: Math.floor((markdown.match(/```/g) || []).length / 2),
49
+ tables: (markdown.match(/^\|(?:\s*---\s*\|)+\s*$/gm) || []).length,
50
+ images: (markdown.match(/!\[[^\]]*\]\(/g) || []).length,
51
+ orderedSteps: (markdown.match(/^\d+\.\s+/gm) || []).length,
52
+ bullets: (markdown.match(/^[-*]\s+/gm) || []).length,
53
+ headings: (markdown.match(/^#{1,6}\s+/gm) || []).length,
54
+ };
55
+ }
56
+ // Noise patterns that should NOT survive extraction into the cached Markdown. Each leaked match
57
+ // wastes agent tokens (raw chrome, nav menus, theme widgets, app-shell placeholders). Kept separate
58
+ // from contentMetrics because these are "too much", not "too little".
59
+ const LEAKAGE_PATTERNS = [
60
+ // Genuine page-chrome HTML surviving Turndown: structural chrome tags, or ANY tag carrying a
61
+ // class/id/role/aria attribute (the signature of a UI element, not a code-example tag like a bare
62
+ // <button>/<label> in JSX). Code blocks are already excluded by stripCode().
63
+ {
64
+ label: 'raw-html',
65
+ re: /<(?:nav|header|footer|aside|svg|path)\b|<[a-z][a-z0-9]*\s[^>]*\b(?:class|id|role|aria-[a-z]+)\s*=/i,
66
+ },
67
+ { label: 'base64-image', re: /data:image\/[a-z]+;base64,/i },
68
+ { label: 'skip-to-content', re: /skip to (?:main )?content/i },
69
+ { label: 'on-this-page', re: /^on this page$/im },
70
+ { label: 'theme-toggle', re: /(?:toggle|switch to) (?:dark|light|the)? ?(?:mode|theme)/i },
71
+ {
72
+ label: 'cookie-consent',
73
+ re: /(?:accept all cookies|we use cookies|cookie (?:policy|preferences|settings))/i,
74
+ },
75
+ { label: 'loading-shell', re: /^\s*loading[.…]*\s*$/im },
76
+ { label: 'search-placeholder', re: /^\s*search(?:\.{3}|…|\s+docs)?\s*$/im },
77
+ // Text-less anchor links carry no content and only waste tokens (icon/permalink leftovers).
78
+ { label: 'empty-link', re: /(?<!!)\[\s*\]\([^)]*\)/ },
79
+ // Unrendered template/KumaScript macros, e.g. {{jsxref("…")}} or {{EmbedInteractiveExample(…)}}.
80
+ // Function-call form only, so Vue/Angular `{{ count }}` interpolation is never flagged.
81
+ { label: 'template-macro', re: /\{\{\s*[A-Za-z]\w*\s*\(/ },
82
+ ];
83
+ function stripFenced(markdown) {
84
+ return markdown.replace(/```[\s\S]*?```/g, '').replace(/~~~[\s\S]*?~~~/g, '');
85
+ }
86
+ // Most leak checks must ignore code entirely (a Vue/HTML snippet is not chrome). The empty-link
87
+ // check is the exception: stripping inline code would turn a real `[`code`](url)` link into a
88
+ // bogus `[](url)`, so it scans with inline code intact (only fenced blocks removed).
89
+ export function leakageSignals(markdown) {
90
+ const fenced = stripFenced(markdown);
91
+ const noCode = fenced.replace(/`[^`]*`/g, '');
92
+ return LEAKAGE_PATTERNS.filter(({ label, re }) => re.test(label === 'empty-link' ? fenced : noCode)).map(({ label }) => label);
93
+ }
@@ -0,0 +1,18 @@
1
+ import { type LocatedArtifact } from './storage.js';
2
+ import { type StoreRoots } from './store-roots.js';
3
+ import type { StorageMode } from '../config/index.js';
4
+ export interface ResolveResearchTargetOptions {
5
+ configDir: string | undefined;
6
+ cwd: string;
7
+ dataDir: string;
8
+ flagOverride?: StorageMode;
9
+ lookup?: boolean;
10
+ url: string;
11
+ }
12
+ export interface ResolvedResearchTarget {
13
+ cacheKey: string;
14
+ located: LocatedArtifact | null;
15
+ normalizedUrl: string;
16
+ roots: StoreRoots;
17
+ }
18
+ export declare function resolveResearchTarget(opts: ResolveResearchTargetOptions): ResolvedResearchTarget;
@@ -0,0 +1,20 @@
1
+ import { deriveCacheKey } from './cache-key.js';
2
+ import { locateArtifact } from './storage.js';
3
+ import { loadStoreRoots } from './store-roots.js';
4
+ import { normalizeUrl } from './url.js';
5
+ export function resolveResearchTarget(opts) {
6
+ const normalizedUrl = normalizeUrl(opts.url);
7
+ const cacheKey = deriveCacheKey(normalizedUrl);
8
+ const roots = loadStoreRoots({
9
+ configDir: opts.configDir,
10
+ cwd: opts.cwd,
11
+ dataDir: opts.dataDir,
12
+ flagOverride: opts.flagOverride,
13
+ });
14
+ return {
15
+ cacheKey,
16
+ located: opts.lookup === false ? null : locateArtifact(roots.readRoots, cacheKey),
17
+ normalizedUrl,
18
+ roots,
19
+ };
20
+ }
@@ -0,0 +1,32 @@
1
+ import type { ResearchArtifact } from './schema.js';
2
+ export interface RevalidationResult {
3
+ status: 'revalidated' | 'refreshed' | 'stale';
4
+ artifact: ResearchArtifact;
5
+ allowed?: boolean;
6
+ error?: string;
7
+ }
8
+ /**
9
+ * Helper to construct a ResearchArtifact from a fresh HTML fetch.
10
+ */
11
+ export declare function createArtifactFromFetch(url: string, normalizedUrl: string, cacheKey: string, fetchResult: {
12
+ contentType: string | null;
13
+ etag: string | null;
14
+ lastModified: string | null;
15
+ finalUrl: string;
16
+ responseSize: number;
17
+ content: string;
18
+ }, extraction: {
19
+ title: string;
20
+ detailedMarkdown: string;
21
+ confidence: 'high' | 'medium' | 'low';
22
+ qualityNotes: string[];
23
+ }, tier: 'stable' | 'standard' | 'volatile', ttl: string | null, currentTime: Date): ResearchArtifact;
24
+ /**
25
+ * Revalidates a stale cache artifact using conditional request headers,
26
+ * falling back to stale serving within the grace period if the remote host is offline.
27
+ */
28
+ export declare function revalidateCache(dataDir: string, existing: ResearchArtifact, currentTime: Date, options: {
29
+ allowStale: boolean;
30
+ ttlOverride?: string | null;
31
+ rendered?: boolean;
32
+ }): Promise<RevalidationResult>;