@taurgis/bonsai 1.0.6 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -13
- package/agents/agents/official-docs-researcher.agent.md +15 -35
- package/agents/agents/salesforce-docs-researcher.agent.md +17 -44
- package/agents/hooks/README.md +2 -9
- package/agents/hooks/claude-code/hooks/bonsai-web-fetch.sh +1 -1
- package/agents/hooks/codex/hooks/bonsai-user-prompt-context.sh +1 -1
- package/agents/hooks/codex/hooks/bonsai-web-fetch.sh +1 -1
- package/agents/hooks/cursor/hooks/bonsai-web-fetch.sh +1 -1
- package/agents/hooks/github-copilot/bonsai-web-fetch.sh +1 -1
- package/agents/instructions/salesforce-research.instructions.md +7 -8
- package/agents/instructions/web-research.instructions.md +3 -4
- package/agents/skills/web-research/SKILL.md +24 -37
- package/bin/cli.mjs +15 -2
- package/dist/base-command.d.ts +15 -2
- package/dist/base-command.js +32 -4
- package/dist/commands/fetch.d.ts +2 -0
- package/dist/commands/fetch.js +43 -29
- package/dist/commands/import.js +11 -3
- package/dist/commands/inspect.d.ts +4 -0
- package/dist/commands/inspect.js +48 -24
- package/dist/commands/list.d.ts +3 -1
- package/dist/commands/list.js +42 -25
- package/dist/commands/prune.d.ts +1 -0
- package/dist/commands/prune.js +35 -9
- package/dist/commands/status.d.ts +2 -0
- package/dist/commands/status.js +57 -25
- package/dist/commands.d.ts +0 -2
- package/dist/commands.js +0 -2
- package/dist/hooks/command-not-found/suggest.d.ts +7 -1
- package/dist/hooks/command-not-found/suggest.js +82 -57
- package/dist/lib/argv.js +51 -7
- package/dist/lib/color.d.ts +20 -0
- package/dist/lib/color.js +36 -0
- package/dist/lib/envelope.js +8 -2
- package/dist/lib/help-preflight.d.ts +15 -0
- package/dist/lib/help-preflight.js +81 -0
- package/dist/lib/limit-flag.d.ts +1 -0
- package/dist/lib/limit-flag.js +24 -0
- package/dist/lib/research/{search-index.d.ts → artifact-index.d.ts} +5 -5
- package/dist/lib/research/{search-index.js → artifact-index.js} +10 -10
- package/dist/lib/research/browser.d.ts +0 -8
- package/dist/lib/research/browser.js +20 -43
- package/dist/lib/research/regression.d.ts +18 -0
- package/dist/lib/research/regression.js +47 -0
- package/dist/lib/research/storage.d.ts +0 -1
- package/dist/lib/research/storage.js +15 -13
- package/dist/lib/research/url.d.ts +22 -0
- package/dist/lib/research/url.js +63 -1
- package/dist/lib/text.d.ts +3 -1
- package/dist/lib/text.js +9 -1
- package/dist/sites/coveo.d.ts +7 -0
- package/dist/sites/coveo.js +33 -0
- package/dist/sites/index.js +1 -18
- package/dist/sites/salesforce/fetch-page.js +1 -1
- package/dist/sites/salesforce/index.js +3 -8
- package/dist/sites/salesforce-developer/fetch-page.js +3 -1
- package/dist/sites/salesforce-developer/index.js +1 -2
- package/dist/sites/salesforce-doc-fetch.d.ts +11 -0
- package/dist/sites/salesforce-doc-fetch.js +117 -76
- package/dist/sites/salesforce-dom-probe.d.ts +16 -0
- package/dist/sites/salesforce-dom-probe.js +114 -0
- package/dist/sites/types.d.ts +0 -6
- package/oclif.manifest.json +31 -106
- package/package.json +32 -25
- package/dist/commands/index.d.ts +0 -35
- package/dist/commands/index.js +0 -306
- package/dist/commands/research/config/base.d.ts +0 -23
- package/dist/commands/research/config/base.js +0 -46
- package/dist/commands/research/config/get.d.ts +0 -20
- package/dist/commands/research/config/get.js +0 -45
- package/dist/commands/research/config/list.d.ts +0 -17
- package/dist/commands/research/config/list.js +0 -42
- package/dist/commands/research/config/set.d.ts +0 -31
- package/dist/commands/research/config/set.js +0 -94
- package/dist/commands/research/config/unset.d.ts +0 -21
- package/dist/commands/research/config/unset.js +0 -59
- package/dist/commands/research/import.d.ts +0 -38
- package/dist/commands/research/import.js +0 -313
- package/dist/commands/research/inspect.d.ts +0 -13
- package/dist/commands/research/inspect.js +0 -86
- package/dist/commands/research/list.d.ts +0 -25
- package/dist/commands/research/list.js +0 -143
- package/dist/commands/research/prune.d.ts +0 -24
- package/dist/commands/research/prune.js +0 -141
- package/dist/commands/research/search.d.ts +0 -32
- package/dist/commands/research/search.js +0 -334
- package/dist/commands/research/status.d.ts +0 -17
- package/dist/commands/research/status.js +0 -108
- package/dist/commands/research.d.ts +0 -36
- package/dist/commands/research.js +0 -304
- package/dist/commands/search.d.ts +0 -35
- package/dist/commands/search.js +0 -391
- package/dist/lib/research/docs/__fixtures__/sphinx-searchindex.d.ts +0 -1
- package/dist/lib/research/docs/__fixtures__/sphinx-searchindex.js +0 -2
- package/dist/lib/research/docs/discovery-index.d.ts +0 -14
- package/dist/lib/research/docs/discovery-index.js +0 -112
- package/dist/lib/research/docs/remote-search-runner.d.ts +0 -21
- package/dist/lib/research/docs/remote-search-runner.js +0 -114
- package/dist/lib/research/docs/remote-search.d.ts +0 -14
- package/dist/lib/research/docs/remote-search.js +0 -101
- package/dist/lib/research/docs/search-index.d.ts +0 -13
- package/dist/lib/research/docs/search-index.js +0 -176
- package/dist/sites/algolia.d.ts +0 -9
- package/dist/sites/algolia.js +0 -26
- package/dist/sites/auto.d.ts +0 -8
- package/dist/sites/auto.js +0 -30
- package/dist/sites/nextjs/index.d.ts +0 -2
- package/dist/sites/nextjs/index.js +0 -52
- package/dist/sites/react/index.d.ts +0 -2
- package/dist/sites/react/index.js +0 -17
- package/dist/sites/salesforce/browser-search.d.ts +0 -7
- package/dist/sites/salesforce/browser-search.js +0 -82
- package/dist/sites/salesforce/coveo.d.ts +0 -23
- package/dist/sites/salesforce/coveo.js +0 -161
- package/dist/sites/salesforce/token.d.ts +0 -26
- package/dist/sites/salesforce/token.js +0 -154
- package/dist/sites/tailwind/index.d.ts +0 -2
- package/dist/sites/tailwind/index.js +0 -17
- package/dist/sites/vue/index.d.ts +0 -2
- package/dist/sites/vue/index.js +0 -17
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
import type { ResearchArtifact } from './schema.js';
|
|
2
|
-
export interface
|
|
2
|
+
export interface IndexedArtifact {
|
|
3
3
|
artifact: ResearchArtifact;
|
|
4
4
|
filePath: string;
|
|
5
5
|
}
|
|
6
6
|
/**
|
|
7
|
-
* Returns the
|
|
7
|
+
* Returns the indexed view of every active artifact in one research dir, using the sidecar index
|
|
8
8
|
* to skip unchanged files. Re-reads only files whose signature changed, then persists the refreshed
|
|
9
|
-
* index (best-effort; a write failure never breaks
|
|
9
|
+
* index (best-effort; a write failure never breaks callers). Corrupt/unreadable files are skipped —
|
|
10
10
|
* the real read paths in storage.ts handle archiving them.
|
|
11
11
|
*/
|
|
12
|
-
export declare function
|
|
12
|
+
export declare function loadIndexedArtifactsForDir(researchDir: string): IndexedArtifact[];
|
|
13
13
|
/**
|
|
14
14
|
* Multi-root variant: scans the `research/` dir of each data dir in order and deduplicates by cache
|
|
15
15
|
* key so a project entry shadows the same key in the global cache. Mirrors the dedup semantics of
|
|
16
16
|
* {@link scanCacheDirs} but reads through the per-root sidecar index.
|
|
17
17
|
*/
|
|
18
|
-
export declare function
|
|
18
|
+
export declare function loadIndexedArtifacts(dataDirs: string[]): IndexedArtifact[];
|
|
@@ -2,11 +2,11 @@ import { join } from 'node:path';
|
|
|
2
2
|
import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
|
|
3
3
|
import { atomicWriteFile } from '../atomic-write.js';
|
|
4
4
|
import { parseArtifactShallow } from './artifact.js';
|
|
5
|
-
import { isResearchFile } from './
|
|
5
|
+
import { isResearchFile } from './url.js';
|
|
6
6
|
// Sidecar index that caches the lightweight (metadata + summary + compressed) view of each artifact
|
|
7
|
-
// so repeat
|
|
8
|
-
// or missing index never changes results, because each entry is validated against the
|
|
9
|
-
// signature and re-read on any mismatch.
|
|
7
|
+
// so repeat list/scan operations avoid re-reading and re-parsing every file. It is a derived cache
|
|
8
|
+
// only: a stale or missing index never changes results, because each entry is validated against the
|
|
9
|
+
// file's current signature and re-read on any mismatch.
|
|
10
10
|
const INDEX_FILE = '.search-index.json';
|
|
11
11
|
const INDEX_VERSION = 1;
|
|
12
12
|
// The repo writes artifacts via temp-file + rename, so every rewrite yields a new inode. Combining
|
|
@@ -48,12 +48,12 @@ function getOrUpdateIndexEntry(filePath, cached, onChanged) {
|
|
|
48
48
|
}
|
|
49
49
|
}
|
|
50
50
|
/**
|
|
51
|
-
* Returns the
|
|
51
|
+
* Returns the indexed view of every active artifact in one research dir, using the sidecar index
|
|
52
52
|
* to skip unchanged files. Re-reads only files whose signature changed, then persists the refreshed
|
|
53
|
-
* index (best-effort; a write failure never breaks
|
|
53
|
+
* index (best-effort; a write failure never breaks callers). Corrupt/unreadable files are skipped —
|
|
54
54
|
* the real read paths in storage.ts handle archiving them.
|
|
55
55
|
*/
|
|
56
|
-
export function
|
|
56
|
+
export function loadIndexedArtifactsForDir(researchDir) {
|
|
57
57
|
if (!existsSync(researchDir))
|
|
58
58
|
return [];
|
|
59
59
|
const indexPath = join(researchDir, INDEX_FILE);
|
|
@@ -83,7 +83,7 @@ export function loadSearchableArtifactsForDir(researchDir) {
|
|
|
83
83
|
atomicWriteFile(indexPath, JSON.stringify({ version: INDEX_VERSION, entries }));
|
|
84
84
|
}
|
|
85
85
|
catch {
|
|
86
|
-
// Best-effort cache write;
|
|
86
|
+
// Best-effort cache write; results are already computed without it.
|
|
87
87
|
}
|
|
88
88
|
}
|
|
89
89
|
return results;
|
|
@@ -93,11 +93,11 @@ export function loadSearchableArtifactsForDir(researchDir) {
|
|
|
93
93
|
* key so a project entry shadows the same key in the global cache. Mirrors the dedup semantics of
|
|
94
94
|
* {@link scanCacheDirs} but reads through the per-root sidecar index.
|
|
95
95
|
*/
|
|
96
|
-
export function
|
|
96
|
+
export function loadIndexedArtifacts(dataDirs) {
|
|
97
97
|
const seen = new Set();
|
|
98
98
|
const all = [];
|
|
99
99
|
for (const dataDir of dataDirs) {
|
|
100
|
-
for (const located of
|
|
100
|
+
for (const located of loadIndexedArtifactsForDir(join(dataDir, 'research'))) {
|
|
101
101
|
const key = located.artifact.metadata.cache_key;
|
|
102
102
|
if (seen.has(key))
|
|
103
103
|
continue;
|
|
@@ -25,14 +25,6 @@ export declare class CdpClient {
|
|
|
25
25
|
}
|
|
26
26
|
export declare function findChromePath(): string;
|
|
27
27
|
export declare function waitForLoad(client: CdpClient, sessionId: string, timeoutMs: number, settleMs?: number): Promise<void>;
|
|
28
|
-
/**
|
|
29
|
-
* Polls until a content-container element holds enough RENDERED text, so SPAs that fetch their
|
|
30
|
-
* body after load are captured rendered rather than as a "Loading…" shell. Pierces open shadow
|
|
31
|
-
* roots (the container is matched across shadow boundaries) and measures `innerText`, which
|
|
32
|
-
* reflects shadow-rendered text — `document.querySelector` + `textContent` miss both. Returns
|
|
33
|
-
* without throwing on timeout; the caller captures whatever rendered and judges its quality.
|
|
34
|
-
*/
|
|
35
|
-
export declare function waitForContentReady(page: CdpPage, selectors: string[], minChars: number, timeoutMs: number): Promise<void>;
|
|
36
28
|
export declare const BLOCKED_ASSET_URLS: string[];
|
|
37
29
|
export interface CdpPage {
|
|
38
30
|
client: CdpClient;
|
|
@@ -164,48 +164,6 @@ export async function waitForLoad(client, sessionId, timeoutMs, settleMs = 1000)
|
|
|
164
164
|
}
|
|
165
165
|
throw new Error(`Navigation timed out after ${timeoutMs}ms`);
|
|
166
166
|
}
|
|
167
|
-
/**
|
|
168
|
-
* Polls until a content-container element holds enough RENDERED text, so SPAs that fetch their
|
|
169
|
-
* body after load are captured rendered rather than as a "Loading…" shell. Pierces open shadow
|
|
170
|
-
* roots (the container is matched across shadow boundaries) and measures `innerText`, which
|
|
171
|
-
* reflects shadow-rendered text — `document.querySelector` + `textContent` miss both. Returns
|
|
172
|
-
* without throwing on timeout; the caller captures whatever rendered and judges its quality.
|
|
173
|
-
*/
|
|
174
|
-
export async function waitForContentReady(page, selectors, minChars, timeoutMs) {
|
|
175
|
-
// Reports whether a content container has rendered (innerText pierces shadow), plus the total
|
|
176
|
-
// rendered text length so the caller can wait for the render to *settle* — heavy pages (e.g.
|
|
177
|
-
// the api-console) render the container early but keep filling it in.
|
|
178
|
-
const expression = `(() => {
|
|
179
|
-
const out = [];
|
|
180
|
-
const visit = (r) => { for (const c of Array.from(r.children || [])) { out.push(c); if (c.shadowRoot) visit(c.shadowRoot); visit(c); } };
|
|
181
|
-
visit(document);
|
|
182
|
-
const SELS = ${JSON.stringify(selectors)};
|
|
183
|
-
let has = false;
|
|
184
|
-
for (const el of out) {
|
|
185
|
-
if (!el.matches) continue;
|
|
186
|
-
if (!SELS.some((s) => { try { return el.matches(s); } catch (e) { return false; } })) continue;
|
|
187
|
-
if (((el.innerText) || '').trim().length >= ${minChars}) { has = true; break; }
|
|
188
|
-
}
|
|
189
|
-
return { has, len: document.body ? (document.body.innerText || '').length : 0 };
|
|
190
|
-
})()`;
|
|
191
|
-
const start = Date.now();
|
|
192
|
-
let previousLength = -1;
|
|
193
|
-
let stablePolls = 0;
|
|
194
|
-
while (Date.now() - start < timeoutMs) {
|
|
195
|
-
const value = (await page.client
|
|
196
|
-
.send('Runtime.evaluate', { expression, returnByValue: true }, page.sessionId)
|
|
197
|
-
.catch(() => null))?.result?.value;
|
|
198
|
-
if (value?.has) {
|
|
199
|
-
const grew = Math.abs((value.len || 0) - previousLength) > Math.max(40, previousLength * 0.02);
|
|
200
|
-
if (!grew && ++stablePolls >= 2)
|
|
201
|
-
return; // content present and rendering has settled
|
|
202
|
-
if (grew)
|
|
203
|
-
stablePolls = 0;
|
|
204
|
-
previousLength = value.len || 0;
|
|
205
|
-
}
|
|
206
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
207
|
-
}
|
|
208
|
-
}
|
|
209
167
|
export const BLOCKED_ASSET_URLS = [
|
|
210
168
|
'*.css',
|
|
211
169
|
'*.png',
|
|
@@ -336,7 +294,13 @@ export class ResponseCapture {
|
|
|
336
294
|
});
|
|
337
295
|
}
|
|
338
296
|
}
|
|
339
|
-
|
|
297
|
+
// Chrome can emit these on first navigation in headless CI when the cert verifier reloads mid-run.
|
|
298
|
+
const TRANSIENT_NAV_ERROR_CODES = ['ERR_CERT_VERIFIER_CHANGED'];
|
|
299
|
+
function isTransientBrowserNavigationError(err) {
|
|
300
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
301
|
+
return TRANSIENT_NAV_ERROR_CODES.some((code) => message.includes(code));
|
|
302
|
+
}
|
|
303
|
+
async function fetchRenderedHtmlOnce(url, options = {}) {
|
|
340
304
|
const timeout = options.timeoutMs ?? 15_000;
|
|
341
305
|
const limit = options.bodyLimitBytes ?? 2 * 1024 * 1024;
|
|
342
306
|
const currentUrl = normalizeUrl(url);
|
|
@@ -394,3 +358,16 @@ export async function fetchRenderedHtml(url, options = {}) {
|
|
|
394
358
|
await page.close();
|
|
395
359
|
}
|
|
396
360
|
}
|
|
361
|
+
export async function fetchRenderedHtml(url, options = {}) {
|
|
362
|
+
const maxAttempts = 3;
|
|
363
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
364
|
+
try {
|
|
365
|
+
return await fetchRenderedHtmlOnce(url, options);
|
|
366
|
+
}
|
|
367
|
+
catch (err) {
|
|
368
|
+
if (attempt === maxAttempts || !isTransientBrowserNavigationError(err))
|
|
369
|
+
throw err;
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
throw new Error('Failed to fetch rendered HTML after transient browser navigation errors.');
|
|
373
|
+
}
|
|
@@ -15,6 +15,24 @@ export interface ContentMetrics {
|
|
|
15
15
|
bullets: number;
|
|
16
16
|
headings: number;
|
|
17
17
|
}
|
|
18
|
+
/** Maximum allowed capture-duration regression vs baseline (percent slower). */
|
|
19
|
+
export declare const TIMING_REGRESSION_LIMIT_PCT = 25;
|
|
20
|
+
export interface TimingRegression {
|
|
21
|
+
deltaPct: number | null;
|
|
22
|
+
regressed: boolean;
|
|
23
|
+
}
|
|
24
|
+
/** Returns whether `currentMs` exceeds baseline by more than TIMING_REGRESSION_LIMIT_PCT. */
|
|
25
|
+
export declare function timingRegression(currentMs: number, baselineMs: number | null | undefined): TimingRegression;
|
|
26
|
+
export interface TimingBaselineRecord {
|
|
27
|
+
durationMs: number;
|
|
28
|
+
capturedAt: string;
|
|
29
|
+
}
|
|
30
|
+
/** Parses a committed `baseline/<id>.timing.json` file. */
|
|
31
|
+
export declare function parseTimingBaseline(raw: string): TimingBaselineRecord | null;
|
|
32
|
+
/** Serializes a timing baseline record for `baseline/<id>.timing.json`. */
|
|
33
|
+
export declare function formatTimingBaseline(durationMs: number, capturedAt?: string): string;
|
|
34
|
+
/** True when `candidate` carries strictly more structural content than `baseline`. */
|
|
35
|
+
export declare function contentRicherThan(candidate: ContentMetrics, baseline: ContentMetrics): boolean;
|
|
18
36
|
/**
|
|
19
37
|
* Structural content signals used to catch information loss between baseline and current output:
|
|
20
38
|
* a sudden drop in code blocks, tables, images, or steps flags that a fetch change dropped content.
|
|
@@ -38,6 +38,53 @@ export function normalizeRegressionMarkdown(markdown) {
|
|
|
38
38
|
.replace(/\n{3,}/g, '\n\n')
|
|
39
39
|
.trim();
|
|
40
40
|
}
|
|
41
|
+
/** Maximum allowed capture-duration regression vs baseline (percent slower). */
|
|
42
|
+
export const TIMING_REGRESSION_LIMIT_PCT = 25;
|
|
43
|
+
/** Returns whether `currentMs` exceeds baseline by more than TIMING_REGRESSION_LIMIT_PCT. */
|
|
44
|
+
export function timingRegression(currentMs, baselineMs) {
|
|
45
|
+
if (!baselineMs || baselineMs <= 0)
|
|
46
|
+
return { deltaPct: null, regressed: false };
|
|
47
|
+
const deltaPct = Number((((currentMs - baselineMs) / baselineMs) * 100).toFixed(2));
|
|
48
|
+
return { deltaPct, regressed: deltaPct > TIMING_REGRESSION_LIMIT_PCT };
|
|
49
|
+
}
|
|
50
|
+
/** Parses a committed `baseline/<id>.timing.json` file. */
|
|
51
|
+
export function parseTimingBaseline(raw) {
|
|
52
|
+
try {
|
|
53
|
+
const parsed = JSON.parse(raw);
|
|
54
|
+
if (typeof parsed.durationMs !== 'number' || parsed.durationMs <= 0)
|
|
55
|
+
return null;
|
|
56
|
+
return {
|
|
57
|
+
durationMs: parsed.durationMs,
|
|
58
|
+
capturedAt: typeof parsed.capturedAt === 'string' ? parsed.capturedAt : '',
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
/** Serializes a timing baseline record for `baseline/<id>.timing.json`. */
|
|
66
|
+
export function formatTimingBaseline(durationMs, capturedAt = new Date().toISOString()) {
|
|
67
|
+
return `${JSON.stringify({ durationMs, capturedAt }, null, 2)}\n`;
|
|
68
|
+
}
|
|
69
|
+
/** Structural signals compared before char count when picking DOM vs API Markdown. */
|
|
70
|
+
const CONTENT_RICHNESS_KEYS = [
|
|
71
|
+
'tables',
|
|
72
|
+
'codeBlocks',
|
|
73
|
+
'headings',
|
|
74
|
+
'images',
|
|
75
|
+
'orderedSteps',
|
|
76
|
+
'chars',
|
|
77
|
+
];
|
|
78
|
+
/** True when `candidate` carries strictly more structural content than `baseline`. */
|
|
79
|
+
export function contentRicherThan(candidate, baseline) {
|
|
80
|
+
for (const key of CONTENT_RICHNESS_KEYS) {
|
|
81
|
+
if (candidate[key] > baseline[key])
|
|
82
|
+
return true;
|
|
83
|
+
if (candidate[key] < baseline[key])
|
|
84
|
+
return false;
|
|
85
|
+
}
|
|
86
|
+
return false;
|
|
87
|
+
}
|
|
41
88
|
/**
|
|
42
89
|
* Structural content signals used to catch information loss between baseline and current output:
|
|
43
90
|
* a sudden drop in code blocks, tables, images, or steps flags that a fetch change dropped content.
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import type { ResearchArtifact } from './schema.js';
|
|
2
|
-
export declare function isResearchFile(file: string): boolean;
|
|
3
2
|
/**
|
|
4
3
|
* Scans a cache directory and maps each valid artifact file through a callback.
|
|
5
4
|
* Return null from the callback to skip an entry; all non-null results are collected.
|
|
@@ -2,12 +2,8 @@ import { join } from 'node:path';
|
|
|
2
2
|
import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, renameSync, } from 'node:fs';
|
|
3
3
|
import { atomicWriteFile } from '../atomic-write.js';
|
|
4
4
|
import { parseArtifact, serializeArtifact } from './artifact.js';
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
!file.includes('.tmp') &&
|
|
8
|
-
!file.includes('.corrupt') &&
|
|
9
|
-
!file.includes('.superseded'));
|
|
10
|
-
}
|
|
5
|
+
import { loadIndexedArtifactsForDir } from './artifact-index.js';
|
|
6
|
+
import { isResearchFile } from './url.js';
|
|
11
7
|
/**
|
|
12
8
|
* Scans a cache directory and maps each valid artifact file through a callback.
|
|
13
9
|
* Return null from the callback to skip an entry; all non-null results are collected.
|
|
@@ -169,11 +165,17 @@ export function locateArtifact(dataDirs, key) {
|
|
|
169
165
|
*/
|
|
170
166
|
export function scanCacheDirs(dataDirs, fn) {
|
|
171
167
|
const seen = new Set();
|
|
172
|
-
return dataDirs.flatMap((dataDir) =>
|
|
173
|
-
const
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
168
|
+
return dataDirs.flatMap((dataDir) => {
|
|
169
|
+
const researchDir = join(dataDir, 'research');
|
|
170
|
+
const searchable = loadIndexedArtifactsForDir(researchDir);
|
|
171
|
+
return searchable
|
|
172
|
+
.map(({ artifact, filePath }) => {
|
|
173
|
+
const key = artifact.metadata.cache_key;
|
|
174
|
+
if (seen.has(key))
|
|
175
|
+
return null;
|
|
176
|
+
seen.add(key);
|
|
177
|
+
return fn(artifact, filePath);
|
|
178
|
+
})
|
|
179
|
+
.filter((x) => x !== null);
|
|
180
|
+
});
|
|
179
181
|
}
|
|
@@ -17,8 +17,30 @@ export declare function isSafeIp(ip: string): boolean;
|
|
|
17
17
|
* Validates a hostname to ensure it is not blocked (e.g. localhost or literal private IP).
|
|
18
18
|
*/
|
|
19
19
|
export declare function validateHostname(hostname: string): void;
|
|
20
|
+
/**
|
|
21
|
+
* Whether the input looks like a URL that simply forgot its `http(s)://` scheme
|
|
22
|
+
* (`example.com`, `docs.nestjs.com/guide`, `192.168.1.1`) rather than unparseable junk.
|
|
23
|
+
* Lets every command turn a scheme-less URL into the same actionable "did you mean https://…" hint
|
|
24
|
+
* that the root `bonsai <url>` shorthand already gives via the command_not_found hook.
|
|
25
|
+
*/
|
|
26
|
+
export declare function looksLikeSchemelessUrl(input: string): boolean;
|
|
20
27
|
/**
|
|
21
28
|
* Parses and normalizes a URL. Strips fragments, credentials, default ports,
|
|
22
29
|
* and sorts query parameters lexicographically.
|
|
23
30
|
*/
|
|
24
31
|
export declare function normalizeUrl(input: string): string;
|
|
32
|
+
/**
|
|
33
|
+
* Checks if a URL matches a wildcard pattern (case-insensitive, supports * wildcard).
|
|
34
|
+
*/
|
|
35
|
+
export declare function matchesUrlGlob(url: string, glob: string): boolean;
|
|
36
|
+
/**
|
|
37
|
+
* Checks if artifact metadata contains any URL matching the filter glob.
|
|
38
|
+
*/
|
|
39
|
+
export declare function artifactMatchesUrlFilter(meta: {
|
|
40
|
+
source_urls?: string[];
|
|
41
|
+
source_url?: string;
|
|
42
|
+
}, filterUrl: string | undefined): boolean;
|
|
43
|
+
/**
|
|
44
|
+
* Checks if a filename matches the standard research note naming pattern.
|
|
45
|
+
*/
|
|
46
|
+
export declare function isResearchFile(file: string): boolean;
|
package/dist/lib/research/url.js
CHANGED
|
@@ -83,6 +83,30 @@ export function validateHostname(hostname) {
|
|
|
83
83
|
}
|
|
84
84
|
}
|
|
85
85
|
}
|
|
86
|
+
// Dot-separated, non-empty labels: a real domain (`docs.nestjs.com`) or IPv4 (`192.168.1.1`). The
|
|
87
|
+
// URL parser also accepts a leading-dot filename (`.env`) or relative path (`./x`) as a "host", whose
|
|
88
|
+
// suggestion would be a nonsense `https://.env`; requiring well-formed labels rejects those. Anchored
|
|
89
|
+
// with no overlapping quantifiers, so it is ReDoS-safe on the (already length-bounded) host. This is
|
|
90
|
+
// the single owner of the domain-shape test; the command_not_found hook consumes it too.
|
|
91
|
+
const DOMAIN_OR_IPV4 = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$/i;
|
|
92
|
+
/**
|
|
93
|
+
* Whether the input looks like a URL that simply forgot its `http(s)://` scheme
|
|
94
|
+
* (`example.com`, `docs.nestjs.com/guide`, `192.168.1.1`) rather than unparseable junk.
|
|
95
|
+
* Lets every command turn a scheme-less URL into the same actionable "did you mean https://…" hint
|
|
96
|
+
* that the root `bonsai <url>` shorthand already gives via the command_not_found hook.
|
|
97
|
+
*/
|
|
98
|
+
export function looksLikeSchemelessUrl(input) {
|
|
99
|
+
if (input.includes('://'))
|
|
100
|
+
return false;
|
|
101
|
+
let hostname;
|
|
102
|
+
try {
|
|
103
|
+
hostname = new URL(`https://${input}`).hostname;
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
return false;
|
|
107
|
+
}
|
|
108
|
+
return DOMAIN_OR_IPV4.test(hostname);
|
|
109
|
+
}
|
|
86
110
|
/**
|
|
87
111
|
* Parses and normalizes a URL. Strips fragments, credentials, default ports,
|
|
88
112
|
* and sorts query parameters lexicographically.
|
|
@@ -95,7 +119,12 @@ export function normalizeUrl(input) {
|
|
|
95
119
|
catch {
|
|
96
120
|
// new URL() only ever throws the constant "Invalid URL" (no structured cause worth keeping),
|
|
97
121
|
// and every caller prefixes its own "Invalid URL:" — forwarding the native text would stutter.
|
|
98
|
-
//
|
|
122
|
+
// A scheme-less but domain-shaped input is the common "forgot https://" mistake, so steer the
|
|
123
|
+
// user to the corrected URL (clig.dev: show the fix, never auto-run it); otherwise emit a
|
|
124
|
+
// self-contained "couldn't parse" reason.
|
|
125
|
+
if (looksLikeSchemelessUrl(input)) {
|
|
126
|
+
throw new Error(`"${input}" is missing a URL scheme. Did you mean https://${input}? Only http:// and https:// URLs are supported.`);
|
|
127
|
+
}
|
|
99
128
|
throw new Error(`Could not parse "${input}" as a URL; provide a full http(s) URL like https://example.com/page.`);
|
|
100
129
|
}
|
|
101
130
|
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
|
@@ -122,3 +151,36 @@ export function normalizeUrl(input) {
|
|
|
122
151
|
url.search = new URLSearchParams(params).toString();
|
|
123
152
|
return url.toString();
|
|
124
153
|
}
|
|
154
|
+
/**
|
|
155
|
+
* Checks if a URL matches a wildcard pattern (case-insensitive, supports * wildcard).
|
|
156
|
+
*/
|
|
157
|
+
export function matchesUrlGlob(url, glob) {
|
|
158
|
+
// Collapse consecutive wildcards to prevent ReDoS
|
|
159
|
+
const collapsedGlob = glob.replace(/\*+/g, '*');
|
|
160
|
+
const normalizedGlob = collapsedGlob.toLowerCase();
|
|
161
|
+
const normalizedUrl = url.toLowerCase();
|
|
162
|
+
// Escape regex characters except *
|
|
163
|
+
const escapedGlob = normalizedGlob.replace(/[/\-\\^$+?.()|[\]{}]/g, '\\$&');
|
|
164
|
+
// Replace * with .*
|
|
165
|
+
const regexString = '^' + escapedGlob.replace(/\*/g, '.*') + '$';
|
|
166
|
+
const regex = new RegExp(regexString);
|
|
167
|
+
return regex.test(normalizedUrl);
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Checks if artifact metadata contains any URL matching the filter glob.
|
|
171
|
+
*/
|
|
172
|
+
export function artifactMatchesUrlFilter(meta, filterUrl) {
|
|
173
|
+
if (!filterUrl)
|
|
174
|
+
return true;
|
|
175
|
+
const urls = meta.source_urls || (meta.source_url ? [meta.source_url] : []);
|
|
176
|
+
return urls.some((u) => matchesUrlGlob(u, filterUrl));
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Checks if a filename matches the standard research note naming pattern.
|
|
180
|
+
*/
|
|
181
|
+
export function isResearchFile(file) {
|
|
182
|
+
return (file.endsWith('.md') &&
|
|
183
|
+
!file.includes('.tmp') &&
|
|
184
|
+
!file.includes('.corrupt') &&
|
|
185
|
+
!file.includes('.superseded'));
|
|
186
|
+
}
|
package/dist/lib/text.d.ts
CHANGED
|
@@ -22,9 +22,11 @@ export declare function resultListHeading(total: number, shown: number, labels:
|
|
|
22
22
|
* stdout heading is suppressed and an agent would otherwise get no signal that results were capped.
|
|
23
23
|
*/
|
|
24
24
|
export declare function truncationNotice(total: number, shown: number, labels: ResultListLabels): string | null;
|
|
25
|
+
/** Max edit distance for a plausible typo; scales with input length. */
|
|
26
|
+
export declare function maxFuzzyDistance(input: string): number;
|
|
25
27
|
/**
|
|
26
28
|
* Levenshtein edit distance: the minimum number of single-character insertions, deletions, or
|
|
27
|
-
* substitutions to turn `s1` into `s2`. Shared by
|
|
29
|
+
* substitutions to turn `s1` into `s2`. Shared by command-typo suggestion.
|
|
28
30
|
*/
|
|
29
31
|
export declare function levenshtein(s1: string, s2: string): number;
|
|
30
32
|
/**
|
package/dist/lib/text.js
CHANGED
|
@@ -24,9 +24,17 @@ export function truncationNotice(total, shown, labels) {
|
|
|
24
24
|
return null;
|
|
25
25
|
return `${total} entries matched; returning the ${labels.order} ${shown}. Raise --limit (max ${labels.maxLimit}) to see more.`;
|
|
26
26
|
}
|
|
27
|
+
/** Max edit distance for a plausible typo; scales with input length. */
|
|
28
|
+
export function maxFuzzyDistance(input) {
|
|
29
|
+
if (input.length <= 3)
|
|
30
|
+
return 1;
|
|
31
|
+
if (input.length <= 5)
|
|
32
|
+
return 2;
|
|
33
|
+
return 3;
|
|
34
|
+
}
|
|
27
35
|
/**
|
|
28
36
|
* Levenshtein edit distance: the minimum number of single-character insertions, deletions, or
|
|
29
|
-
* substitutions to turn `s1` into `s2`. Shared by
|
|
37
|
+
* substitutions to turn `s1` into `s2`. Shared by command-typo suggestion.
|
|
30
38
|
*/
|
|
31
39
|
export function levenshtein(s1, s2) {
|
|
32
40
|
if (s1.length < s2.length)
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export declare const ALLOWED_DOC_HOSTS: Set<string>;
|
|
2
|
+
export declare function isAllowedDocHost(hostname: string): boolean;
|
|
3
|
+
/**
|
|
4
|
+
* Coveo indexes Help articles under an internal `/help_doccontent` URL; rewrite it to the
|
|
5
|
+
* canonical `/s/articleView` page a human (or our fetcher) can actually open.
|
|
6
|
+
*/
|
|
7
|
+
export declare function normalizeHelpDocContentUrl(rawUrl: string): string;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export const ALLOWED_DOC_HOSTS = new Set(['help.salesforce.com', 'developer.salesforce.com']);
|
|
2
|
+
export function isAllowedDocHost(hostname) {
|
|
3
|
+
return ALLOWED_DOC_HOSTS.has(hostname.toLowerCase());
|
|
4
|
+
}
|
|
5
|
+
/**
|
|
6
|
+
* Coveo indexes Help articles under an internal `/help_doccontent` URL; rewrite it to the
|
|
7
|
+
* canonical `/s/articleView` page a human (or our fetcher) can actually open.
|
|
8
|
+
*/
|
|
9
|
+
export function normalizeHelpDocContentUrl(rawUrl) {
|
|
10
|
+
let url;
|
|
11
|
+
try {
|
|
12
|
+
url = new URL(rawUrl);
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
return rawUrl;
|
|
16
|
+
}
|
|
17
|
+
if (url.hostname.toLowerCase() !== 'help.salesforce.com')
|
|
18
|
+
return rawUrl;
|
|
19
|
+
if (!url.pathname.toLowerCase().startsWith('/help_doccontent'))
|
|
20
|
+
return rawUrl;
|
|
21
|
+
const id = url.searchParams.get('id');
|
|
22
|
+
if (!id)
|
|
23
|
+
return rawUrl;
|
|
24
|
+
const articleUrl = new URL('https://help.salesforce.com/s/articleView');
|
|
25
|
+
articleUrl.searchParams.set('id', id.endsWith('.htm') ? id : `${id}.htm`);
|
|
26
|
+
articleUrl.searchParams.set('type', '5');
|
|
27
|
+
for (const key of ['release', 'language']) {
|
|
28
|
+
const value = url.searchParams.get(key);
|
|
29
|
+
if (value)
|
|
30
|
+
articleUrl.searchParams.set(key, value);
|
|
31
|
+
}
|
|
32
|
+
return articleUrl.toString();
|
|
33
|
+
}
|
package/dist/sites/index.js
CHANGED
|
@@ -1,27 +1,10 @@
|
|
|
1
1
|
import { salesforce } from './salesforce/index.js';
|
|
2
2
|
import { salesforceDeveloper } from './salesforce-developer/index.js';
|
|
3
|
-
import { nextjs } from './nextjs/index.js';
|
|
4
|
-
import { react, vue, tailwind, jest, cypress, vitest, vitepress } from './algolia.js';
|
|
5
|
-
import { angular, redux, vite, fastify, rollup, vueuse } from './auto.js';
|
|
6
3
|
// The known documentation sites. A plain constant, not a registry: these ship
|
|
7
4
|
// with the CLI, so there is nothing to register at runtime. Sites with custom
|
|
8
|
-
// fetch
|
|
5
|
+
// fetch behavior live in their own module (e.g. ./salesforce). Salesforce Help and
|
|
9
6
|
// Salesforce Developer are separate modules — different domains, different extraction.
|
|
10
7
|
export const SITES = [
|
|
11
|
-
react,
|
|
12
|
-
vue,
|
|
13
|
-
tailwind,
|
|
14
|
-
nextjs,
|
|
15
|
-
jest,
|
|
16
|
-
cypress,
|
|
17
|
-
vitest,
|
|
18
|
-
vitepress,
|
|
19
|
-
angular,
|
|
20
|
-
redux,
|
|
21
|
-
vite,
|
|
22
|
-
fastify,
|
|
23
|
-
rollup,
|
|
24
|
-
vueuse,
|
|
25
8
|
salesforce,
|
|
26
9
|
salesforceDeveloper,
|
|
27
10
|
// TanStack docs embed a GitHub "edit this page" link, so capture resolves the page's source
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { fetchSalesforceDoc } from '../salesforce-doc-fetch.js';
|
|
2
|
-
import { normalizeHelpDocContentUrl } from '
|
|
2
|
+
import { normalizeHelpDocContentUrl } from '../coveo.js';
|
|
3
3
|
// Containers a Help article renders into (inside shadow DOM); the shared fetcher pierces
|
|
4
4
|
// shadow roots to find them, falling back to the page body when none match.
|
|
5
5
|
const CONTENT_SELECTORS = [
|
|
@@ -1,16 +1,11 @@
|
|
|
1
1
|
import { fetchSalesforcePage } from './fetch-page.js';
|
|
2
|
-
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
// API-reference docs render inside shadow DOM with tabbed/code content that needs a different
|
|
6
|
-
// extraction strategy, so it belongs in its own module when built — not bolted onto this one.
|
|
7
|
-
// (developer.salesforce.com URLs that appear in Help search results are still surfaced; see
|
|
8
|
-
// ALLOWED_DOC_HOSTS in coveo.ts.)
|
|
2
|
+
// Scoped to help.salesforce.com — an LWR Experience site with light-DOM article content.
|
|
3
|
+
// developer.salesforce.com is intentionally NOT routed here: its API-reference docs render
|
|
4
|
+
// inside shadow DOM with tabbed/code content that needs a different extraction strategy.
|
|
9
5
|
export const salesforce = {
|
|
10
6
|
id: 'salesforce',
|
|
11
7
|
name: 'Salesforce Help',
|
|
12
8
|
domains: ['help.salesforce.com'],
|
|
13
9
|
defaults: { rendered: true },
|
|
14
10
|
fetchPage: fetchSalesforcePage,
|
|
15
|
-
search: searchSalesforce,
|
|
16
11
|
};
|
|
@@ -2,8 +2,10 @@ import { fetchSalesforceDoc } from '../salesforce-doc-fetch.js';
|
|
|
2
2
|
// Containers a developer doc renders into. doc-content-layout / doc-amf-reference are the
|
|
3
3
|
// web-component hosts for guide pages and API-reference (AMF) pages respectively.
|
|
4
4
|
const CONTENT_SELECTORS = [
|
|
5
|
-
|
|
5
|
+
// API-reference pages: doc-amf-reference holds the method/type tables. doc-content-layout only
|
|
6
|
+
// carries the shared deprecation banner and must not win selector priority on those pages.
|
|
6
7
|
'doc-amf-reference',
|
|
8
|
+
'doc-content-layout',
|
|
7
9
|
'.markdown-content',
|
|
8
10
|
'main article',
|
|
9
11
|
'article',
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import { fetchDeveloperPage } from './fetch-page.js';
|
|
2
2
|
// developer.salesforce.com docs (guides + API reference) render client-side inside web
|
|
3
3
|
// components, so content lives in shadow DOM — a fundamentally different extraction strategy
|
|
4
|
-
// than help.salesforce.com (see ../salesforce).
|
|
5
|
-
// search backend, not Help's Coveo HTCommunity hub.
|
|
4
|
+
// than help.salesforce.com (see ../salesforce).
|
|
6
5
|
export const salesforceDeveloper = {
|
|
7
6
|
id: 'salesforce-developer',
|
|
8
7
|
name: 'Salesforce Developer',
|
|
@@ -2,6 +2,17 @@ import type { SiteFetchResult } from './types.js';
|
|
|
2
2
|
/** True when the extracted text is an error/not-found/loading shell rather than real content. */
|
|
3
3
|
export declare function looksLikeSalesforceError(text: string): boolean;
|
|
4
4
|
export declare function stripBoilerplate(markdown: string): string;
|
|
5
|
+
/** Parses the Salesforce private content API JSON payload. */
|
|
6
|
+
export declare function parseDocsApiPayload(raw: string): {
|
|
7
|
+
content: string;
|
|
8
|
+
title: string;
|
|
9
|
+
} | null;
|
|
10
|
+
/** Keeps whichever Markdown source carries more structural content (DOM vs content API HTML). */
|
|
11
|
+
export declare function preferRicherMarkdown(domMarkdown: string, apiHtml: string, domTitle: string, apiTitle: string): {
|
|
12
|
+
markdown: string;
|
|
13
|
+
title: string;
|
|
14
|
+
usedApi: boolean;
|
|
15
|
+
};
|
|
5
16
|
export interface SalesforceDocOptions {
|
|
6
17
|
allowedHost: string;
|
|
7
18
|
contentSelectors: string[];
|