@taurgis/bonsai 1.0.5 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -12
- package/agents/agents/official-docs-researcher.agent.md +15 -26
- package/agents/agents/salesforce-docs-researcher.agent.md +17 -35
- package/agents/hooks/README.md +2 -9
- package/agents/hooks/claude-code/hooks/bonsai-web-fetch.sh +1 -1
- package/agents/hooks/codex/hooks/bonsai-user-prompt-context.sh +1 -1
- package/agents/hooks/codex/hooks/bonsai-web-fetch.sh +1 -1
- package/agents/hooks/cursor/hooks/bonsai-web-fetch.sh +1 -1
- package/agents/hooks/github-copilot/bonsai-web-fetch.sh +1 -1
- package/agents/instructions/salesforce-research.instructions.md +7 -6
- package/agents/instructions/web-research.instructions.md +3 -2
- package/agents/skills/web-research/SKILL.md +24 -32
- package/bin/cli.mjs +44 -14
- package/dist/base-command.d.ts +17 -1
- package/dist/base-command.js +70 -9
- package/dist/commands/config/base.d.ts +2 -0
- package/dist/commands/config/base.js +15 -1
- package/dist/commands/config/get.d.ts +1 -1
- package/dist/commands/config/get.js +4 -5
- package/dist/commands/config/index.d.ts +12 -0
- package/dist/commands/config/index.js +34 -0
- package/dist/commands/config/list.js +2 -2
- package/dist/commands/config/set.js +18 -12
- package/dist/commands/config/unset.d.ts +1 -1
- package/dist/commands/config/unset.js +9 -6
- package/dist/commands/fetch.d.ts +2 -0
- package/dist/commands/fetch.js +69 -45
- package/dist/commands/import.d.ts +3 -0
- package/dist/commands/import.js +91 -53
- package/dist/commands/inspect.d.ts +8 -0
- package/dist/commands/inspect.js +59 -30
- package/dist/commands/list.d.ts +5 -3
- package/dist/commands/list.js +60 -35
- package/dist/commands/prune.d.ts +2 -1
- package/dist/commands/prune.js +62 -19
- package/dist/commands/status.d.ts +8 -0
- package/dist/commands/status.js +84 -26
- package/dist/commands.d.ts +2 -2
- package/dist/commands.js +2 -2
- package/dist/hooks/command-not-found/suggest.d.ts +7 -1
- package/dist/hooks/command-not-found/suggest.js +100 -46
- package/dist/lib/argv.d.ts +10 -0
- package/dist/lib/argv.js +97 -0
- package/dist/lib/color.d.ts +20 -0
- package/dist/lib/color.js +36 -0
- package/dist/lib/envelope.d.ts +36 -0
- package/dist/lib/envelope.js +82 -0
- package/dist/lib/help-preflight.d.ts +15 -0
- package/dist/lib/help-preflight.js +81 -0
- package/dist/lib/json-meta.d.ts +10 -0
- package/dist/lib/json-meta.js +98 -0
- package/dist/lib/limit-flag.d.ts +1 -0
- package/dist/lib/limit-flag.js +24 -0
- package/dist/lib/research/{search-index.d.ts → artifact-index.d.ts} +5 -5
- package/dist/lib/research/{search-index.js → artifact-index.js} +10 -10
- package/dist/lib/research/browser.d.ts +0 -8
- package/dist/lib/research/browser.js +20 -43
- package/dist/lib/research/regression.d.ts +18 -0
- package/dist/lib/research/regression.js +47 -0
- package/dist/lib/research/revalidate.js +3 -1
- package/dist/lib/research/schema.d.ts +5 -2
- package/dist/lib/research/schema.js +12 -1
- package/dist/lib/research/storage.d.ts +0 -1
- package/dist/lib/research/storage.js +15 -13
- package/dist/lib/research/url.d.ts +22 -0
- package/dist/lib/research/url.js +63 -1
- package/dist/lib/text.d.ts +5 -1
- package/dist/lib/text.js +11 -1
- package/dist/sites/coveo.d.ts +7 -0
- package/dist/sites/coveo.js +33 -0
- package/dist/sites/index.js +1 -2
- package/dist/sites/salesforce/fetch-page.js +1 -1
- package/dist/sites/salesforce/index.js +3 -8
- package/dist/sites/salesforce-developer/fetch-page.js +3 -1
- package/dist/sites/salesforce-developer/index.js +1 -2
- package/dist/sites/salesforce-doc-fetch.d.ts +11 -0
- package/dist/sites/salesforce-doc-fetch.js +117 -76
- package/dist/sites/salesforce-dom-probe.d.ts +16 -0
- package/dist/sites/salesforce-dom-probe.js +114 -0
- package/dist/sites/types.d.ts +0 -6
- package/oclif.manifest.json +153 -155
- package/package.json +38 -25
- package/dist/commands/index.d.ts +0 -35
- package/dist/commands/index.js +0 -306
- package/dist/commands/research/config/base.d.ts +0 -23
- package/dist/commands/research/config/base.js +0 -46
- package/dist/commands/research/config/get.d.ts +0 -20
- package/dist/commands/research/config/get.js +0 -45
- package/dist/commands/research/config/list.d.ts +0 -17
- package/dist/commands/research/config/list.js +0 -42
- package/dist/commands/research/config/set.d.ts +0 -31
- package/dist/commands/research/config/set.js +0 -94
- package/dist/commands/research/config/unset.d.ts +0 -21
- package/dist/commands/research/config/unset.js +0 -59
- package/dist/commands/research/import.d.ts +0 -38
- package/dist/commands/research/import.js +0 -313
- package/dist/commands/research/inspect.d.ts +0 -13
- package/dist/commands/research/inspect.js +0 -86
- package/dist/commands/research/list.d.ts +0 -25
- package/dist/commands/research/list.js +0 -143
- package/dist/commands/research/prune.d.ts +0 -24
- package/dist/commands/research/prune.js +0 -141
- package/dist/commands/research/search.d.ts +0 -32
- package/dist/commands/research/search.js +0 -334
- package/dist/commands/research/status.d.ts +0 -17
- package/dist/commands/research/status.js +0 -108
- package/dist/commands/research.d.ts +0 -36
- package/dist/commands/research.js +0 -304
- package/dist/commands/search.d.ts +0 -31
- package/dist/commands/search.js +0 -329
- package/dist/lib/research/docs/__fixtures__/sphinx-searchindex.d.ts +0 -1
- package/dist/lib/research/docs/__fixtures__/sphinx-searchindex.js +0 -2
- package/dist/lib/research/docs/discovery-index.d.ts +0 -14
- package/dist/lib/research/docs/discovery-index.js +0 -112
- package/dist/lib/research/docs/remote-search-runner.d.ts +0 -21
- package/dist/lib/research/docs/remote-search-runner.js +0 -114
- package/dist/lib/research/docs/remote-search.d.ts +0 -14
- package/dist/lib/research/docs/remote-search.js +0 -101
- package/dist/lib/research/docs/search-index.d.ts +0 -13
- package/dist/lib/research/docs/search-index.js +0 -176
- package/dist/sites/salesforce/browser-search.d.ts +0 -7
- package/dist/sites/salesforce/browser-search.js +0 -82
- package/dist/sites/salesforce/coveo.d.ts +0 -23
- package/dist/sites/salesforce/coveo.js +0 -161
- package/dist/sites/salesforce/token.d.ts +0 -26
- package/dist/sites/salesforce/token.js +0 -154
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { Config, Help, toConfiguredId } from '@oclif/core';
|
|
2
|
+
import { buildEnvelope } from './envelope.js';
|
|
3
|
+
/** Strip ANSI color codes so JSON help text stays machine-stable. */
|
|
4
|
+
function stripAnsi(text) {
|
|
5
|
+
// eslint-disable-next-line no-control-regex
|
|
6
|
+
return text.replace(/\x1b\[[0-9;]*m/g, '');
|
|
7
|
+
}
|
|
8
|
+
/** True when argv asks for help or version together with --json. */
|
|
9
|
+
export function isJsonMetaRequest(argv) {
|
|
10
|
+
if (!argv.includes('--json'))
|
|
11
|
+
return false;
|
|
12
|
+
return argv.includes('--help') || argv.includes('--version');
|
|
13
|
+
}
|
|
14
|
+
class CaptureHelp extends Help {
|
|
15
|
+
lines = [];
|
|
16
|
+
log(...args) {
|
|
17
|
+
for (const line of args) {
|
|
18
|
+
// oclif emits section separators as log(''); drop empties so data.help stays compact.
|
|
19
|
+
if (line)
|
|
20
|
+
this.lines.push(line);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
function helpSubject(argv) {
|
|
25
|
+
for (const arg of argv) {
|
|
26
|
+
if (arg === '--' || arg === '--help' || arg === 'help')
|
|
27
|
+
continue;
|
|
28
|
+
if (arg.startsWith('-'))
|
|
29
|
+
return;
|
|
30
|
+
return arg;
|
|
31
|
+
}
|
|
32
|
+
return undefined;
|
|
33
|
+
}
|
|
34
|
+
function envelopeCommandId(config, argv) {
|
|
35
|
+
const subject = helpSubject(argv);
|
|
36
|
+
if (!subject)
|
|
37
|
+
return config.bin;
|
|
38
|
+
const command = config.findCommand(subject);
|
|
39
|
+
if (command?.id)
|
|
40
|
+
return toConfiguredId(command.id, config);
|
|
41
|
+
const topic = config.findTopic(subject);
|
|
42
|
+
if (topic)
|
|
43
|
+
return topic.name;
|
|
44
|
+
return subject;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* When `--json` is combined with `--help` or `--version`, oclif would print human text to stdout.
|
|
48
|
+
* Render the same content inside the standard Bonsai JSON envelope instead.
|
|
49
|
+
*/
|
|
50
|
+
export async function tryJsonMetaOutput(argv, root) {
|
|
51
|
+
if (!isJsonMetaRequest(argv))
|
|
52
|
+
return null;
|
|
53
|
+
try {
|
|
54
|
+
const config = await Config.load({ root });
|
|
55
|
+
if (argv.includes('--version')) {
|
|
56
|
+
return {
|
|
57
|
+
exitCode: 0,
|
|
58
|
+
envelope: buildEnvelope({
|
|
59
|
+
command: config.bin,
|
|
60
|
+
ok: true,
|
|
61
|
+
exitCode: 0,
|
|
62
|
+
stderr: '',
|
|
63
|
+
data: {
|
|
64
|
+
version: config.version,
|
|
65
|
+
userAgent: config.userAgent,
|
|
66
|
+
},
|
|
67
|
+
}),
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
const help = new CaptureHelp(config, config.pjson.oclif?.helpOptions ?? {});
|
|
71
|
+
await help.showHelp([...argv]);
|
|
72
|
+
const helpText = stripAnsi(help.lines.join('\n').trimEnd());
|
|
73
|
+
return {
|
|
74
|
+
exitCode: 0,
|
|
75
|
+
envelope: buildEnvelope({
|
|
76
|
+
command: envelopeCommandId(config, argv),
|
|
77
|
+
ok: true,
|
|
78
|
+
exitCode: 0,
|
|
79
|
+
stderr: '',
|
|
80
|
+
data: { help: helpText },
|
|
81
|
+
}),
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
catch (err) {
|
|
85
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
86
|
+
return {
|
|
87
|
+
exitCode: 1,
|
|
88
|
+
envelope: buildEnvelope({
|
|
89
|
+
command: 'bonsai',
|
|
90
|
+
ok: false,
|
|
91
|
+
exitCode: 1,
|
|
92
|
+
stderr: message,
|
|
93
|
+
data: null,
|
|
94
|
+
code: 'META_RENDER_FAILED',
|
|
95
|
+
}),
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function limitFlag(max: number, defaultValue: number, description: string): import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { Errors, Flags } from '@oclif/core';
|
|
2
|
+
export function limitFlag(max, defaultValue, description) {
|
|
3
|
+
return Flags.custom({
|
|
4
|
+
parse: async (input) => {
|
|
5
|
+
if (!/^\d+$/.test(input)) {
|
|
6
|
+
throw new Errors.CLIError(`Limit must be an integer between 1 and ${max}.`, {
|
|
7
|
+
exit: 2,
|
|
8
|
+
code: 'INVALID_LIMIT',
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
const value = Number(input);
|
|
12
|
+
if (value < 1 || value > max) {
|
|
13
|
+
throw new Errors.CLIError(`Limit must be between 1 and ${max}.`, {
|
|
14
|
+
exit: 2,
|
|
15
|
+
code: 'INVALID_LIMIT',
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
return value;
|
|
19
|
+
},
|
|
20
|
+
})({
|
|
21
|
+
default: defaultValue,
|
|
22
|
+
description,
|
|
23
|
+
});
|
|
24
|
+
}
|
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
import type { ResearchArtifact } from './schema.js';
|
|
2
|
-
export interface
|
|
2
|
+
export interface IndexedArtifact {
|
|
3
3
|
artifact: ResearchArtifact;
|
|
4
4
|
filePath: string;
|
|
5
5
|
}
|
|
6
6
|
/**
|
|
7
|
-
* Returns the
|
|
7
|
+
* Returns the indexed view of every active artifact in one research dir, using the sidecar index
|
|
8
8
|
* to skip unchanged files. Re-reads only files whose signature changed, then persists the refreshed
|
|
9
|
-
* index (best-effort; a write failure never breaks
|
|
9
|
+
* index (best-effort; a write failure never breaks callers). Corrupt/unreadable files are skipped —
|
|
10
10
|
* the real read paths in storage.ts handle archiving them.
|
|
11
11
|
*/
|
|
12
|
-
export declare function
|
|
12
|
+
export declare function loadIndexedArtifactsForDir(researchDir: string): IndexedArtifact[];
|
|
13
13
|
/**
|
|
14
14
|
* Multi-root variant: scans the `research/` dir of each data dir in order and deduplicates by cache
|
|
15
15
|
* key so a project entry shadows the same key in the global cache. Mirrors the dedup semantics of
|
|
16
16
|
* {@link scanCacheDirs} but reads through the per-root sidecar index.
|
|
17
17
|
*/
|
|
18
|
-
export declare function
|
|
18
|
+
export declare function loadIndexedArtifacts(dataDirs: string[]): IndexedArtifact[];
|
|
@@ -2,11 +2,11 @@ import { join } from 'node:path';
|
|
|
2
2
|
import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
|
|
3
3
|
import { atomicWriteFile } from '../atomic-write.js';
|
|
4
4
|
import { parseArtifactShallow } from './artifact.js';
|
|
5
|
-
import { isResearchFile } from './
|
|
5
|
+
import { isResearchFile } from './url.js';
|
|
6
6
|
// Sidecar index that caches the lightweight (metadata + summary + compressed) view of each artifact
|
|
7
|
-
// so repeat
|
|
8
|
-
// or missing index never changes results, because each entry is validated against the
|
|
9
|
-
// signature and re-read on any mismatch.
|
|
7
|
+
// so repeat list/scan operations avoid re-reading and re-parsing every file. It is a derived cache
|
|
8
|
+
// only: a stale or missing index never changes results, because each entry is validated against the
|
|
9
|
+
// file's current signature and re-read on any mismatch.
|
|
10
10
|
const INDEX_FILE = '.search-index.json';
|
|
11
11
|
const INDEX_VERSION = 1;
|
|
12
12
|
// The repo writes artifacts via temp-file + rename, so every rewrite yields a new inode. Combining
|
|
@@ -48,12 +48,12 @@ function getOrUpdateIndexEntry(filePath, cached, onChanged) {
|
|
|
48
48
|
}
|
|
49
49
|
}
|
|
50
50
|
/**
|
|
51
|
-
* Returns the
|
|
51
|
+
* Returns the indexed view of every active artifact in one research dir, using the sidecar index
|
|
52
52
|
* to skip unchanged files. Re-reads only files whose signature changed, then persists the refreshed
|
|
53
|
-
* index (best-effort; a write failure never breaks
|
|
53
|
+
* index (best-effort; a write failure never breaks callers). Corrupt/unreadable files are skipped —
|
|
54
54
|
* the real read paths in storage.ts handle archiving them.
|
|
55
55
|
*/
|
|
56
|
-
export function
|
|
56
|
+
export function loadIndexedArtifactsForDir(researchDir) {
|
|
57
57
|
if (!existsSync(researchDir))
|
|
58
58
|
return [];
|
|
59
59
|
const indexPath = join(researchDir, INDEX_FILE);
|
|
@@ -83,7 +83,7 @@ export function loadSearchableArtifactsForDir(researchDir) {
|
|
|
83
83
|
atomicWriteFile(indexPath, JSON.stringify({ version: INDEX_VERSION, entries }));
|
|
84
84
|
}
|
|
85
85
|
catch {
|
|
86
|
-
// Best-effort cache write;
|
|
86
|
+
// Best-effort cache write; results are already computed without it.
|
|
87
87
|
}
|
|
88
88
|
}
|
|
89
89
|
return results;
|
|
@@ -93,11 +93,11 @@ export function loadSearchableArtifactsForDir(researchDir) {
|
|
|
93
93
|
* key so a project entry shadows the same key in the global cache. Mirrors the dedup semantics of
|
|
94
94
|
* {@link scanCacheDirs} but reads through the per-root sidecar index.
|
|
95
95
|
*/
|
|
96
|
-
export function
|
|
96
|
+
export function loadIndexedArtifacts(dataDirs) {
|
|
97
97
|
const seen = new Set();
|
|
98
98
|
const all = [];
|
|
99
99
|
for (const dataDir of dataDirs) {
|
|
100
|
-
for (const located of
|
|
100
|
+
for (const located of loadIndexedArtifactsForDir(join(dataDir, 'research'))) {
|
|
101
101
|
const key = located.artifact.metadata.cache_key;
|
|
102
102
|
if (seen.has(key))
|
|
103
103
|
continue;
|
|
@@ -25,14 +25,6 @@ export declare class CdpClient {
|
|
|
25
25
|
}
|
|
26
26
|
export declare function findChromePath(): string;
|
|
27
27
|
export declare function waitForLoad(client: CdpClient, sessionId: string, timeoutMs: number, settleMs?: number): Promise<void>;
|
|
28
|
-
/**
|
|
29
|
-
* Polls until a content-container element holds enough RENDERED text, so SPAs that fetch their
|
|
30
|
-
* body after load are captured rendered rather than as a "Loading…" shell. Pierces open shadow
|
|
31
|
-
* roots (the container is matched across shadow boundaries) and measures `innerText`, which
|
|
32
|
-
* reflects shadow-rendered text — `document.querySelector` + `textContent` miss both. Returns
|
|
33
|
-
* without throwing on timeout; the caller captures whatever rendered and judges its quality.
|
|
34
|
-
*/
|
|
35
|
-
export declare function waitForContentReady(page: CdpPage, selectors: string[], minChars: number, timeoutMs: number): Promise<void>;
|
|
36
28
|
export declare const BLOCKED_ASSET_URLS: string[];
|
|
37
29
|
export interface CdpPage {
|
|
38
30
|
client: CdpClient;
|
|
@@ -164,48 +164,6 @@ export async function waitForLoad(client, sessionId, timeoutMs, settleMs = 1000)
|
|
|
164
164
|
}
|
|
165
165
|
throw new Error(`Navigation timed out after ${timeoutMs}ms`);
|
|
166
166
|
}
|
|
167
|
-
/**
|
|
168
|
-
* Polls until a content-container element holds enough RENDERED text, so SPAs that fetch their
|
|
169
|
-
* body after load are captured rendered rather than as a "Loading…" shell. Pierces open shadow
|
|
170
|
-
* roots (the container is matched across shadow boundaries) and measures `innerText`, which
|
|
171
|
-
* reflects shadow-rendered text — `document.querySelector` + `textContent` miss both. Returns
|
|
172
|
-
* without throwing on timeout; the caller captures whatever rendered and judges its quality.
|
|
173
|
-
*/
|
|
174
|
-
export async function waitForContentReady(page, selectors, minChars, timeoutMs) {
|
|
175
|
-
// Reports whether a content container has rendered (innerText pierces shadow), plus the total
|
|
176
|
-
// rendered text length so the caller can wait for the render to *settle* — heavy pages (e.g.
|
|
177
|
-
// the api-console) render the container early but keep filling it in.
|
|
178
|
-
const expression = `(() => {
|
|
179
|
-
const out = [];
|
|
180
|
-
const visit = (r) => { for (const c of Array.from(r.children || [])) { out.push(c); if (c.shadowRoot) visit(c.shadowRoot); visit(c); } };
|
|
181
|
-
visit(document);
|
|
182
|
-
const SELS = ${JSON.stringify(selectors)};
|
|
183
|
-
let has = false;
|
|
184
|
-
for (const el of out) {
|
|
185
|
-
if (!el.matches) continue;
|
|
186
|
-
if (!SELS.some((s) => { try { return el.matches(s); } catch (e) { return false; } })) continue;
|
|
187
|
-
if (((el.innerText) || '').trim().length >= ${minChars}) { has = true; break; }
|
|
188
|
-
}
|
|
189
|
-
return { has, len: document.body ? (document.body.innerText || '').length : 0 };
|
|
190
|
-
})()`;
|
|
191
|
-
const start = Date.now();
|
|
192
|
-
let previousLength = -1;
|
|
193
|
-
let stablePolls = 0;
|
|
194
|
-
while (Date.now() - start < timeoutMs) {
|
|
195
|
-
const value = (await page.client
|
|
196
|
-
.send('Runtime.evaluate', { expression, returnByValue: true }, page.sessionId)
|
|
197
|
-
.catch(() => null))?.result?.value;
|
|
198
|
-
if (value?.has) {
|
|
199
|
-
const grew = Math.abs((value.len || 0) - previousLength) > Math.max(40, previousLength * 0.02);
|
|
200
|
-
if (!grew && ++stablePolls >= 2)
|
|
201
|
-
return; // content present and rendering has settled
|
|
202
|
-
if (grew)
|
|
203
|
-
stablePolls = 0;
|
|
204
|
-
previousLength = value.len || 0;
|
|
205
|
-
}
|
|
206
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
207
|
-
}
|
|
208
|
-
}
|
|
209
167
|
export const BLOCKED_ASSET_URLS = [
|
|
210
168
|
'*.css',
|
|
211
169
|
'*.png',
|
|
@@ -336,7 +294,13 @@ export class ResponseCapture {
|
|
|
336
294
|
});
|
|
337
295
|
}
|
|
338
296
|
}
|
|
339
|
-
|
|
297
|
+
// Chrome can emit these on first navigation in headless CI when the cert verifier reloads mid-run.
|
|
298
|
+
const TRANSIENT_NAV_ERROR_CODES = ['ERR_CERT_VERIFIER_CHANGED'];
|
|
299
|
+
function isTransientBrowserNavigationError(err) {
|
|
300
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
301
|
+
return TRANSIENT_NAV_ERROR_CODES.some((code) => message.includes(code));
|
|
302
|
+
}
|
|
303
|
+
async function fetchRenderedHtmlOnce(url, options = {}) {
|
|
340
304
|
const timeout = options.timeoutMs ?? 15_000;
|
|
341
305
|
const limit = options.bodyLimitBytes ?? 2 * 1024 * 1024;
|
|
342
306
|
const currentUrl = normalizeUrl(url);
|
|
@@ -394,3 +358,16 @@ export async function fetchRenderedHtml(url, options = {}) {
|
|
|
394
358
|
await page.close();
|
|
395
359
|
}
|
|
396
360
|
}
|
|
361
|
+
export async function fetchRenderedHtml(url, options = {}) {
|
|
362
|
+
const maxAttempts = 3;
|
|
363
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
364
|
+
try {
|
|
365
|
+
return await fetchRenderedHtmlOnce(url, options);
|
|
366
|
+
}
|
|
367
|
+
catch (err) {
|
|
368
|
+
if (attempt === maxAttempts || !isTransientBrowserNavigationError(err))
|
|
369
|
+
throw err;
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
throw new Error('Failed to fetch rendered HTML after transient browser navigation errors.');
|
|
373
|
+
}
|
|
@@ -15,6 +15,24 @@ export interface ContentMetrics {
|
|
|
15
15
|
bullets: number;
|
|
16
16
|
headings: number;
|
|
17
17
|
}
|
|
18
|
+
/** Maximum allowed capture-duration regression vs baseline (percent slower). */
|
|
19
|
+
export declare const TIMING_REGRESSION_LIMIT_PCT = 25;
|
|
20
|
+
export interface TimingRegression {
|
|
21
|
+
deltaPct: number | null;
|
|
22
|
+
regressed: boolean;
|
|
23
|
+
}
|
|
24
|
+
/** Returns whether `currentMs` exceeds baseline by more than TIMING_REGRESSION_LIMIT_PCT. */
|
|
25
|
+
export declare function timingRegression(currentMs: number, baselineMs: number | null | undefined): TimingRegression;
|
|
26
|
+
export interface TimingBaselineRecord {
|
|
27
|
+
durationMs: number;
|
|
28
|
+
capturedAt: string;
|
|
29
|
+
}
|
|
30
|
+
/** Parses a committed `baseline/<id>.timing.json` file. */
|
|
31
|
+
export declare function parseTimingBaseline(raw: string): TimingBaselineRecord | null;
|
|
32
|
+
/** Serializes a timing baseline record for `baseline/<id>.timing.json`. */
|
|
33
|
+
export declare function formatTimingBaseline(durationMs: number, capturedAt?: string): string;
|
|
34
|
+
/** True when `candidate` carries strictly more structural content than `baseline`. */
|
|
35
|
+
export declare function contentRicherThan(candidate: ContentMetrics, baseline: ContentMetrics): boolean;
|
|
18
36
|
/**
|
|
19
37
|
* Structural content signals used to catch information loss between baseline and current output:
|
|
20
38
|
* a sudden drop in code blocks, tables, images, or steps flags that a fetch change dropped content.
|
|
@@ -38,6 +38,53 @@ export function normalizeRegressionMarkdown(markdown) {
|
|
|
38
38
|
.replace(/\n{3,}/g, '\n\n')
|
|
39
39
|
.trim();
|
|
40
40
|
}
|
|
41
|
+
/** Maximum allowed capture-duration regression vs baseline (percent slower). */
|
|
42
|
+
export const TIMING_REGRESSION_LIMIT_PCT = 25;
|
|
43
|
+
/** Returns whether `currentMs` exceeds baseline by more than TIMING_REGRESSION_LIMIT_PCT. */
|
|
44
|
+
export function timingRegression(currentMs, baselineMs) {
|
|
45
|
+
if (!baselineMs || baselineMs <= 0)
|
|
46
|
+
return { deltaPct: null, regressed: false };
|
|
47
|
+
const deltaPct = Number((((currentMs - baselineMs) / baselineMs) * 100).toFixed(2));
|
|
48
|
+
return { deltaPct, regressed: deltaPct > TIMING_REGRESSION_LIMIT_PCT };
|
|
49
|
+
}
|
|
50
|
+
/** Parses a committed `baseline/<id>.timing.json` file. */
|
|
51
|
+
export function parseTimingBaseline(raw) {
|
|
52
|
+
try {
|
|
53
|
+
const parsed = JSON.parse(raw);
|
|
54
|
+
if (typeof parsed.durationMs !== 'number' || parsed.durationMs <= 0)
|
|
55
|
+
return null;
|
|
56
|
+
return {
|
|
57
|
+
durationMs: parsed.durationMs,
|
|
58
|
+
capturedAt: typeof parsed.capturedAt === 'string' ? parsed.capturedAt : '',
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
/** Serializes a timing baseline record for `baseline/<id>.timing.json`. */
|
|
66
|
+
export function formatTimingBaseline(durationMs, capturedAt = new Date().toISOString()) {
|
|
67
|
+
return `${JSON.stringify({ durationMs, capturedAt }, null, 2)}\n`;
|
|
68
|
+
}
|
|
69
|
+
/** Structural signals compared before char count when picking DOM vs API Markdown. */
|
|
70
|
+
const CONTENT_RICHNESS_KEYS = [
|
|
71
|
+
'tables',
|
|
72
|
+
'codeBlocks',
|
|
73
|
+
'headings',
|
|
74
|
+
'images',
|
|
75
|
+
'orderedSteps',
|
|
76
|
+
'chars',
|
|
77
|
+
];
|
|
78
|
+
/** True when `candidate` carries strictly more structural content than `baseline`. */
|
|
79
|
+
export function contentRicherThan(candidate, baseline) {
|
|
80
|
+
for (const key of CONTENT_RICHNESS_KEYS) {
|
|
81
|
+
if (candidate[key] > baseline[key])
|
|
82
|
+
return true;
|
|
83
|
+
if (candidate[key] < baseline[key])
|
|
84
|
+
return false;
|
|
85
|
+
}
|
|
86
|
+
return false;
|
|
87
|
+
}
|
|
41
88
|
/**
|
|
42
89
|
* Structural content signals used to catch information loss between baseline and current output:
|
|
43
90
|
* a sudden drop in code blocks, tables, images, or steps flags that a fetch change dropped content.
|
|
@@ -178,7 +178,9 @@ export async function revalidateCache(dataDir, existing, currentTime, options) {
|
|
|
178
178
|
status: 'stale',
|
|
179
179
|
artifact: existing,
|
|
180
180
|
allowed: options.allowStale,
|
|
181
|
-
|
|
181
|
+
// Carry only the underlying cause; the command layer owns the user-facing framing
|
|
182
|
+
// (so the warning reads cleanly instead of stuttering "stale … stale", "failed … failed").
|
|
183
|
+
error: err.message,
|
|
182
184
|
};
|
|
183
185
|
}
|
|
184
186
|
throw new Error(`Revalidation failed and cache has expired beyond the grace period: ${err.message}`);
|
|
@@ -2,10 +2,13 @@ export interface TokenEstimate {
|
|
|
2
2
|
compressed: number | null;
|
|
3
3
|
detailed: number | null;
|
|
4
4
|
}
|
|
5
|
-
export
|
|
5
|
+
export declare const CAPTURE_METHODS: readonly ["static_fetch", "browser_fallback", "agent_supplied", "route_markdown", "github_source"];
|
|
6
|
+
export type CaptureMethod = (typeof CAPTURE_METHODS)[number];
|
|
7
|
+
export declare const ARTIFACT_TYPES: readonly ["source", "research_note", "index", "section"];
|
|
8
|
+
export type ArtifactType = (typeof ARTIFACT_TYPES)[number];
|
|
6
9
|
export interface ResearchArtifactMetadata {
|
|
7
10
|
schema_version: number;
|
|
8
|
-
artifact_type:
|
|
11
|
+
artifact_type: ArtifactType;
|
|
9
12
|
source_url: string;
|
|
10
13
|
source_urls: string[];
|
|
11
14
|
normalized_url: string;
|
|
@@ -1 +1,12 @@
|
|
|
1
|
-
|
|
1
|
+
// Single source of truth for the capture-method enum. Command flag `options` lists derive from this
|
|
2
|
+
// so a new method can never drift out of sync with the filters that are supposed to expose it.
|
|
3
|
+
export const CAPTURE_METHODS = [
|
|
4
|
+
'static_fetch',
|
|
5
|
+
'browser_fallback',
|
|
6
|
+
'agent_supplied',
|
|
7
|
+
'route_markdown',
|
|
8
|
+
'github_source',
|
|
9
|
+
];
|
|
10
|
+
// Single source of truth for the artifact-type enum. `index` = navigation/hub or llms.txt site
|
|
11
|
+
// index; `section` = a heading-level child of a page. Command flag `options` derive from this.
|
|
12
|
+
export const ARTIFACT_TYPES = ['source', 'research_note', 'index', 'section'];
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import type { ResearchArtifact } from './schema.js';
|
|
2
|
-
export declare function isResearchFile(file: string): boolean;
|
|
3
2
|
/**
|
|
4
3
|
* Scans a cache directory and maps each valid artifact file through a callback.
|
|
5
4
|
* Return null from the callback to skip an entry; all non-null results are collected.
|
|
@@ -2,12 +2,8 @@ import { join } from 'node:path';
|
|
|
2
2
|
import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, renameSync, } from 'node:fs';
|
|
3
3
|
import { atomicWriteFile } from '../atomic-write.js';
|
|
4
4
|
import { parseArtifact, serializeArtifact } from './artifact.js';
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
!file.includes('.tmp') &&
|
|
8
|
-
!file.includes('.corrupt') &&
|
|
9
|
-
!file.includes('.superseded'));
|
|
10
|
-
}
|
|
5
|
+
import { loadIndexedArtifactsForDir } from './artifact-index.js';
|
|
6
|
+
import { isResearchFile } from './url.js';
|
|
11
7
|
/**
|
|
12
8
|
* Scans a cache directory and maps each valid artifact file through a callback.
|
|
13
9
|
* Return null from the callback to skip an entry; all non-null results are collected.
|
|
@@ -169,11 +165,17 @@ export function locateArtifact(dataDirs, key) {
|
|
|
169
165
|
*/
|
|
170
166
|
export function scanCacheDirs(dataDirs, fn) {
|
|
171
167
|
const seen = new Set();
|
|
172
|
-
return dataDirs.flatMap((dataDir) =>
|
|
173
|
-
const
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
168
|
+
return dataDirs.flatMap((dataDir) => {
|
|
169
|
+
const researchDir = join(dataDir, 'research');
|
|
170
|
+
const searchable = loadIndexedArtifactsForDir(researchDir);
|
|
171
|
+
return searchable
|
|
172
|
+
.map(({ artifact, filePath }) => {
|
|
173
|
+
const key = artifact.metadata.cache_key;
|
|
174
|
+
if (seen.has(key))
|
|
175
|
+
return null;
|
|
176
|
+
seen.add(key);
|
|
177
|
+
return fn(artifact, filePath);
|
|
178
|
+
})
|
|
179
|
+
.filter((x) => x !== null);
|
|
180
|
+
});
|
|
179
181
|
}
|
|
@@ -17,8 +17,30 @@ export declare function isSafeIp(ip: string): boolean;
|
|
|
17
17
|
* Validates a hostname to ensure it is not blocked (e.g. localhost or literal private IP).
|
|
18
18
|
*/
|
|
19
19
|
export declare function validateHostname(hostname: string): void;
|
|
20
|
+
/**
|
|
21
|
+
* Whether the input looks like a URL that simply forgot its `http(s)://` scheme
|
|
22
|
+
* (`example.com`, `docs.nestjs.com/guide`, `192.168.1.1`) rather than unparseable junk.
|
|
23
|
+
* Lets every command turn a scheme-less URL into the same actionable "did you mean https://…" hint
|
|
24
|
+
* that the root `bonsai <url>` shorthand already gives via the command_not_found hook.
|
|
25
|
+
*/
|
|
26
|
+
export declare function looksLikeSchemelessUrl(input: string): boolean;
|
|
20
27
|
/**
|
|
21
28
|
* Parses and normalizes a URL. Strips fragments, credentials, default ports,
|
|
22
29
|
* and sorts query parameters lexicographically.
|
|
23
30
|
*/
|
|
24
31
|
export declare function normalizeUrl(input: string): string;
|
|
32
|
+
/**
|
|
33
|
+
* Checks if a URL matches a wildcard pattern (case-insensitive, supports * wildcard).
|
|
34
|
+
*/
|
|
35
|
+
export declare function matchesUrlGlob(url: string, glob: string): boolean;
|
|
36
|
+
/**
|
|
37
|
+
* Checks if artifact metadata contains any URL matching the filter glob.
|
|
38
|
+
*/
|
|
39
|
+
export declare function artifactMatchesUrlFilter(meta: {
|
|
40
|
+
source_urls?: string[];
|
|
41
|
+
source_url?: string;
|
|
42
|
+
}, filterUrl: string | undefined): boolean;
|
|
43
|
+
/**
|
|
44
|
+
* Checks if a filename matches the standard research note naming pattern.
|
|
45
|
+
*/
|
|
46
|
+
export declare function isResearchFile(file: string): boolean;
|
package/dist/lib/research/url.js
CHANGED
|
@@ -83,6 +83,30 @@ export function validateHostname(hostname) {
|
|
|
83
83
|
}
|
|
84
84
|
}
|
|
85
85
|
}
|
|
86
|
+
// Dot-separated, non-empty labels: a real domain (`docs.nestjs.com`) or IPv4 (`192.168.1.1`). The
|
|
87
|
+
// URL parser also accepts a leading-dot filename (`.env`) or relative path (`./x`) as a "host", whose
|
|
88
|
+
// suggestion would be a nonsense `https://.env`; requiring well-formed labels rejects those. Anchored
|
|
89
|
+
// with no overlapping quantifiers, so it is ReDoS-safe on the (already length-bounded) host. This is
|
|
90
|
+
// the single owner of the domain-shape test; the command_not_found hook consumes it too.
|
|
91
|
+
const DOMAIN_OR_IPV4 = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$/i;
|
|
92
|
+
/**
|
|
93
|
+
* Whether the input looks like a URL that simply forgot its `http(s)://` scheme
|
|
94
|
+
* (`example.com`, `docs.nestjs.com/guide`, `192.168.1.1`) rather than unparseable junk.
|
|
95
|
+
* Lets every command turn a scheme-less URL into the same actionable "did you mean https://…" hint
|
|
96
|
+
* that the root `bonsai <url>` shorthand already gives via the command_not_found hook.
|
|
97
|
+
*/
|
|
98
|
+
export function looksLikeSchemelessUrl(input) {
|
|
99
|
+
if (input.includes('://'))
|
|
100
|
+
return false;
|
|
101
|
+
let hostname;
|
|
102
|
+
try {
|
|
103
|
+
hostname = new URL(`https://${input}`).hostname;
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
return false;
|
|
107
|
+
}
|
|
108
|
+
return DOMAIN_OR_IPV4.test(hostname);
|
|
109
|
+
}
|
|
86
110
|
/**
|
|
87
111
|
* Parses and normalizes a URL. Strips fragments, credentials, default ports,
|
|
88
112
|
* and sorts query parameters lexicographically.
|
|
@@ -95,7 +119,12 @@ export function normalizeUrl(input) {
|
|
|
95
119
|
catch {
|
|
96
120
|
// new URL() only ever throws the constant "Invalid URL" (no structured cause worth keeping),
|
|
97
121
|
// and every caller prefixes its own "Invalid URL:" — forwarding the native text would stutter.
|
|
98
|
-
//
|
|
122
|
+
// A scheme-less but domain-shaped input is the common "forgot https://" mistake, so steer the
|
|
123
|
+
// user to the corrected URL (clig.dev: show the fix, never auto-run it); otherwise emit a
|
|
124
|
+
// self-contained "couldn't parse" reason.
|
|
125
|
+
if (looksLikeSchemelessUrl(input)) {
|
|
126
|
+
throw new Error(`"${input}" is missing a URL scheme. Did you mean https://${input}? Only http:// and https:// URLs are supported.`);
|
|
127
|
+
}
|
|
99
128
|
throw new Error(`Could not parse "${input}" as a URL; provide a full http(s) URL like https://example.com/page.`);
|
|
100
129
|
}
|
|
101
130
|
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
|
@@ -122,3 +151,36 @@ export function normalizeUrl(input) {
|
|
|
122
151
|
url.search = new URLSearchParams(params).toString();
|
|
123
152
|
return url.toString();
|
|
124
153
|
}
|
|
154
|
+
/**
|
|
155
|
+
* Checks if a URL matches a wildcard pattern (case-insensitive, supports * wildcard).
|
|
156
|
+
*/
|
|
157
|
+
export function matchesUrlGlob(url, glob) {
|
|
158
|
+
// Collapse consecutive wildcards to prevent ReDoS
|
|
159
|
+
const collapsedGlob = glob.replace(/\*+/g, '*');
|
|
160
|
+
const normalizedGlob = collapsedGlob.toLowerCase();
|
|
161
|
+
const normalizedUrl = url.toLowerCase();
|
|
162
|
+
// Escape regex characters except *
|
|
163
|
+
const escapedGlob = normalizedGlob.replace(/[/\-\\^$+?.()|[\]{}]/g, '\\$&');
|
|
164
|
+
// Replace * with .*
|
|
165
|
+
const regexString = '^' + escapedGlob.replace(/\*/g, '.*') + '$';
|
|
166
|
+
const regex = new RegExp(regexString);
|
|
167
|
+
return regex.test(normalizedUrl);
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Checks if artifact metadata contains any URL matching the filter glob.
|
|
171
|
+
*/
|
|
172
|
+
export function artifactMatchesUrlFilter(meta, filterUrl) {
|
|
173
|
+
if (!filterUrl)
|
|
174
|
+
return true;
|
|
175
|
+
const urls = meta.source_urls || (meta.source_url ? [meta.source_url] : []);
|
|
176
|
+
return urls.some((u) => matchesUrlGlob(u, filterUrl));
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Checks if a filename matches the standard research note naming pattern.
|
|
180
|
+
*/
|
|
181
|
+
export function isResearchFile(file) {
|
|
182
|
+
return (file.endsWith('.md') &&
|
|
183
|
+
!file.includes('.tmp') &&
|
|
184
|
+
!file.includes('.corrupt') &&
|
|
185
|
+
!file.includes('.superseded'));
|
|
186
|
+
}
|
package/dist/lib/text.d.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
/** Display label when an artifact has no topic — not a filterable topic value. */
|
|
2
|
+
export declare const NO_TOPIC_LABEL = "(no topic)";
|
|
1
3
|
/**
|
|
2
4
|
* Pick the singular or plural noun for a count so human-readable output stays grammatical
|
|
3
5
|
* ("1 entry" / "2 entries"). English has irregular plurals, so the plural is passed explicitly
|
|
@@ -20,9 +22,11 @@ export declare function resultListHeading(total: number, shown: number, labels:
|
|
|
20
22
|
* stdout heading is suppressed and an agent would otherwise get no signal that results were capped.
|
|
21
23
|
*/
|
|
22
24
|
export declare function truncationNotice(total: number, shown: number, labels: ResultListLabels): string | null;
|
|
25
|
+
/** Max edit distance for a plausible typo; scales with input length. */
|
|
26
|
+
export declare function maxFuzzyDistance(input: string): number;
|
|
23
27
|
/**
|
|
24
28
|
* Levenshtein edit distance: the minimum number of single-character insertions, deletions, or
|
|
25
|
-
* substitutions to turn `s1` into `s2`. Shared by
|
|
29
|
+
* substitutions to turn `s1` into `s2`. Shared by command-typo suggestion.
|
|
26
30
|
*/
|
|
27
31
|
export declare function levenshtein(s1: string, s2: string): number;
|
|
28
32
|
/**
|
package/dist/lib/text.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
/** Display label when an artifact has no topic — not a filterable topic value. */
|
|
2
|
+
export const NO_TOPIC_LABEL = '(no topic)';
|
|
1
3
|
/**
|
|
2
4
|
* Pick the singular or plural noun for a count so human-readable output stays grammatical
|
|
3
5
|
* ("1 entry" / "2 entries"). English has irregular plurals, so the plural is passed explicitly
|
|
@@ -22,9 +24,17 @@ export function truncationNotice(total, shown, labels) {
|
|
|
22
24
|
return null;
|
|
23
25
|
return `${total} entries matched; returning the ${labels.order} ${shown}. Raise --limit (max ${labels.maxLimit}) to see more.`;
|
|
24
26
|
}
|
|
27
|
+
/** Max edit distance for a plausible typo; scales with input length. */
|
|
28
|
+
export function maxFuzzyDistance(input) {
|
|
29
|
+
if (input.length <= 3)
|
|
30
|
+
return 1;
|
|
31
|
+
if (input.length <= 5)
|
|
32
|
+
return 2;
|
|
33
|
+
return 3;
|
|
34
|
+
}
|
|
25
35
|
/**
|
|
26
36
|
* Levenshtein edit distance: the minimum number of single-character insertions, deletions, or
|
|
27
|
-
* substitutions to turn `s1` into `s2`. Shared by
|
|
37
|
+
* substitutions to turn `s1` into `s2`. Shared by command-typo suggestion.
|
|
28
38
|
*/
|
|
29
39
|
export function levenshtein(s1, s2) {
|
|
30
40
|
if (s1.length < s2.length)
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export declare const ALLOWED_DOC_HOSTS: Set<string>;
|
|
2
|
+
export declare function isAllowedDocHost(hostname: string): boolean;
|
|
3
|
+
/**
|
|
4
|
+
* Coveo indexes Help articles under an internal `/help_doccontent` URL; rewrite it to the
|
|
5
|
+
* canonical `/s/articleView` page a human (or our fetcher) can actually open.
|
|
6
|
+
*/
|
|
7
|
+
export declare function normalizeHelpDocContentUrl(rawUrl: string): string;
|