@taurgis/bonsai 1.0.6 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -13
- package/agents/agents/official-docs-researcher.agent.md +15 -35
- package/agents/agents/salesforce-docs-researcher.agent.md +17 -44
- package/agents/hooks/README.md +2 -9
- package/agents/hooks/claude-code/hooks/bonsai-web-fetch.sh +1 -1
- package/agents/hooks/codex/hooks/bonsai-user-prompt-context.sh +1 -1
- package/agents/hooks/codex/hooks/bonsai-web-fetch.sh +1 -1
- package/agents/hooks/cursor/hooks/bonsai-web-fetch.sh +1 -1
- package/agents/hooks/github-copilot/bonsai-web-fetch.sh +1 -1
- package/agents/instructions/salesforce-research.instructions.md +7 -8
- package/agents/instructions/web-research.instructions.md +3 -4
- package/agents/skills/web-research/SKILL.md +24 -37
- package/bin/cli.mjs +15 -2
- package/dist/base-command.d.ts +15 -2
- package/dist/base-command.js +32 -4
- package/dist/commands/fetch.d.ts +2 -0
- package/dist/commands/fetch.js +43 -29
- package/dist/commands/import.js +11 -3
- package/dist/commands/inspect.d.ts +4 -0
- package/dist/commands/inspect.js +48 -24
- package/dist/commands/list.d.ts +3 -1
- package/dist/commands/list.js +42 -25
- package/dist/commands/prune.d.ts +1 -0
- package/dist/commands/prune.js +35 -9
- package/dist/commands/status.d.ts +2 -0
- package/dist/commands/status.js +57 -25
- package/dist/commands.d.ts +0 -2
- package/dist/commands.js +0 -2
- package/dist/hooks/command-not-found/suggest.d.ts +7 -1
- package/dist/hooks/command-not-found/suggest.js +82 -57
- package/dist/lib/argv.js +51 -7
- package/dist/lib/color.d.ts +20 -0
- package/dist/lib/color.js +36 -0
- package/dist/lib/envelope.js +8 -2
- package/dist/lib/help-preflight.d.ts +15 -0
- package/dist/lib/help-preflight.js +81 -0
- package/dist/lib/limit-flag.d.ts +1 -0
- package/dist/lib/limit-flag.js +24 -0
- package/dist/lib/research/{search-index.d.ts → artifact-index.d.ts} +5 -5
- package/dist/lib/research/{search-index.js → artifact-index.js} +10 -10
- package/dist/lib/research/browser.d.ts +0 -8
- package/dist/lib/research/browser.js +20 -43
- package/dist/lib/research/regression.d.ts +18 -0
- package/dist/lib/research/regression.js +47 -0
- package/dist/lib/research/storage.d.ts +0 -1
- package/dist/lib/research/storage.js +15 -13
- package/dist/lib/research/url.d.ts +22 -0
- package/dist/lib/research/url.js +63 -1
- package/dist/lib/text.d.ts +3 -1
- package/dist/lib/text.js +9 -1
- package/dist/sites/coveo.d.ts +7 -0
- package/dist/sites/coveo.js +33 -0
- package/dist/sites/index.js +1 -18
- package/dist/sites/salesforce/fetch-page.js +1 -1
- package/dist/sites/salesforce/index.js +3 -8
- package/dist/sites/salesforce-developer/fetch-page.js +3 -1
- package/dist/sites/salesforce-developer/index.js +1 -2
- package/dist/sites/salesforce-doc-fetch.d.ts +11 -0
- package/dist/sites/salesforce-doc-fetch.js +117 -76
- package/dist/sites/salesforce-dom-probe.d.ts +16 -0
- package/dist/sites/salesforce-dom-probe.js +114 -0
- package/dist/sites/types.d.ts +0 -6
- package/oclif.manifest.json +31 -106
- package/package.json +32 -25
- package/dist/commands/index.d.ts +0 -35
- package/dist/commands/index.js +0 -306
- package/dist/commands/research/config/base.d.ts +0 -23
- package/dist/commands/research/config/base.js +0 -46
- package/dist/commands/research/config/get.d.ts +0 -20
- package/dist/commands/research/config/get.js +0 -45
- package/dist/commands/research/config/list.d.ts +0 -17
- package/dist/commands/research/config/list.js +0 -42
- package/dist/commands/research/config/set.d.ts +0 -31
- package/dist/commands/research/config/set.js +0 -94
- package/dist/commands/research/config/unset.d.ts +0 -21
- package/dist/commands/research/config/unset.js +0 -59
- package/dist/commands/research/import.d.ts +0 -38
- package/dist/commands/research/import.js +0 -313
- package/dist/commands/research/inspect.d.ts +0 -13
- package/dist/commands/research/inspect.js +0 -86
- package/dist/commands/research/list.d.ts +0 -25
- package/dist/commands/research/list.js +0 -143
- package/dist/commands/research/prune.d.ts +0 -24
- package/dist/commands/research/prune.js +0 -141
- package/dist/commands/research/search.d.ts +0 -32
- package/dist/commands/research/search.js +0 -334
- package/dist/commands/research/status.d.ts +0 -17
- package/dist/commands/research/status.js +0 -108
- package/dist/commands/research.d.ts +0 -36
- package/dist/commands/research.js +0 -304
- package/dist/commands/search.d.ts +0 -35
- package/dist/commands/search.js +0 -391
- package/dist/lib/research/docs/__fixtures__/sphinx-searchindex.d.ts +0 -1
- package/dist/lib/research/docs/__fixtures__/sphinx-searchindex.js +0 -2
- package/dist/lib/research/docs/discovery-index.d.ts +0 -14
- package/dist/lib/research/docs/discovery-index.js +0 -112
- package/dist/lib/research/docs/remote-search-runner.d.ts +0 -21
- package/dist/lib/research/docs/remote-search-runner.js +0 -114
- package/dist/lib/research/docs/remote-search.d.ts +0 -14
- package/dist/lib/research/docs/remote-search.js +0 -101
- package/dist/lib/research/docs/search-index.d.ts +0 -13
- package/dist/lib/research/docs/search-index.js +0 -176
- package/dist/sites/algolia.d.ts +0 -9
- package/dist/sites/algolia.js +0 -26
- package/dist/sites/auto.d.ts +0 -8
- package/dist/sites/auto.js +0 -30
- package/dist/sites/nextjs/index.d.ts +0 -2
- package/dist/sites/nextjs/index.js +0 -52
- package/dist/sites/react/index.d.ts +0 -2
- package/dist/sites/react/index.js +0 -17
- package/dist/sites/salesforce/browser-search.d.ts +0 -7
- package/dist/sites/salesforce/browser-search.js +0 -82
- package/dist/sites/salesforce/coveo.d.ts +0 -23
- package/dist/sites/salesforce/coveo.js +0 -161
- package/dist/sites/salesforce/token.d.ts +0 -26
- package/dist/sites/salesforce/token.js +0 -154
- package/dist/sites/tailwind/index.d.ts +0 -2
- package/dist/sites/tailwind/index.js +0 -17
- package/dist/sites/vue/index.d.ts +0 -2
- package/dist/sites/vue/index.js +0 -17
|
@@ -1,17 +1,22 @@
|
|
|
1
|
-
import { openCdpPage, waitForLoad,
|
|
1
|
+
import { openCdpPage, waitForLoad, ResponseCapture, } from '../lib/research/browser.js';
|
|
2
2
|
import { checkDnsSafety } from '../lib/research/fetcher.js';
|
|
3
3
|
import { htmlToMarkdown } from '../lib/research/markdown.js';
|
|
4
|
+
import { contentMetrics, contentRicherThan } from '../lib/research/regression.js';
|
|
5
|
+
import { SALESFORCE_SHADOW_DOM_HELPERS, buildPriorityContainerPickBlock, pollSalesforceContentReady, } from './salesforce-dom-probe.js';
|
|
4
6
|
// Both help.salesforce.com and developer.salesforce.com are LWR sites that render doc content
|
|
5
7
|
// client-side inside web components (shadow DOM). document.documentElement.outerHTML captures
|
|
6
8
|
// only the "Loading…" shell, and Readability prunes the structured result to a single block —
|
|
7
9
|
// so this shared fetcher serializes the shadow DOM and converts the content container directly.
|
|
8
10
|
const TIMEOUT_MS = 45_000;
|
|
9
11
|
const SETTLE_MS = 1_500;
|
|
10
|
-
|
|
12
|
+
/** Minimum text on the priority content host before wait/capture proceed (shared threshold). */
|
|
13
|
+
const MIN_CONTAINER_CHARS = 100;
|
|
11
14
|
// Below this a capture is treated as a shell and retried (a fresh reload of the heavy page).
|
|
12
15
|
const SUBSTANTIAL_CHARS = 300;
|
|
13
16
|
const MAX_ATTEMPTS = 2;
|
|
14
17
|
const BODY_LIMIT_BYTES = 6 * 1024 * 1024;
|
|
18
|
+
/** Grace period after DOM settle for late /docs/get_document_content/ responses on guide pages. */
|
|
19
|
+
const DOCS_API_GRACE_MS = 500;
|
|
15
20
|
const ACCEPT_CONSENT_EXPRESSION = "(() => { const b = document.querySelector('#onetrust-accept-btn-handler'); if (b) b.click(); })()";
|
|
16
21
|
const ERROR_PATTERNS = [
|
|
17
22
|
/page\s*(not\s*found|doesn'?t\s*exist|can'?t\s*be\s*found)/i,
|
|
@@ -66,6 +71,33 @@ export function stripBoilerplate(markdown) {
|
|
|
66
71
|
.replace(/\n{3,}/g, '\n\n')
|
|
67
72
|
.trim();
|
|
68
73
|
}
|
|
74
|
+
const DOCS_CONTENT_API = '/docs/get_document_content/';
|
|
75
|
+
/** Parses the Salesforce private content API JSON payload. */
|
|
76
|
+
export function parseDocsApiPayload(raw) {
|
|
77
|
+
if (!raw?.trim())
|
|
78
|
+
return null;
|
|
79
|
+
try {
|
|
80
|
+
const payload = JSON.parse(raw);
|
|
81
|
+
if (!payload.content)
|
|
82
|
+
return null;
|
|
83
|
+
return { content: payload.content, title: payload.title || payload.id || '' };
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
/** Keeps whichever Markdown source carries more structural content (DOM vs content API HTML). */
|
|
90
|
+
export function preferRicherMarkdown(domMarkdown, apiHtml, domTitle, apiTitle) {
|
|
91
|
+
const apiMarkdown = stripBoilerplate(htmlToMarkdown(apiHtml));
|
|
92
|
+
if (contentRicherThan(contentMetrics(apiMarkdown), contentMetrics(domMarkdown))) {
|
|
93
|
+
return {
|
|
94
|
+
markdown: apiMarkdown,
|
|
95
|
+
title: apiTitle || domTitle,
|
|
96
|
+
usedApi: true,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
return { markdown: domMarkdown, title: domTitle, usedApi: false };
|
|
100
|
+
}
|
|
69
101
|
/**
|
|
70
102
|
* In-page capture. Expands collapsed "Show" sections, inlines dx-code-block source (its code
|
|
71
103
|
* lives in a `code-block` attribute, not text), strips chrome, then picks the richest content
|
|
@@ -73,22 +105,9 @@ export function stripBoilerplate(markdown) {
|
|
|
73
105
|
* Help pages (which lack those components) and load-bearing on Developer API-reference pages.
|
|
74
106
|
*/
|
|
75
107
|
function buildCaptureScript(contentSelectors, extraRemove) {
|
|
108
|
+
const containerPick = buildPriorityContainerPickBlock(contentSelectors, MIN_CONTAINER_CHARS);
|
|
76
109
|
return `(async () => {
|
|
77
|
-
|
|
78
|
-
// root's own shadow root (so it works both document-wide and scoped to a single component).
|
|
79
|
-
const deepElements = (root) => {
|
|
80
|
-
const out = [];
|
|
81
|
-
const visit = (node) => {
|
|
82
|
-
if (node.shadowRoot) visit(node.shadowRoot);
|
|
83
|
-
for (const child of Array.from(node.children || [])) {
|
|
84
|
-
out.push(child);
|
|
85
|
-
visit(child);
|
|
86
|
-
}
|
|
87
|
-
};
|
|
88
|
-
visit(root);
|
|
89
|
-
return out;
|
|
90
|
-
};
|
|
91
|
-
|
|
110
|
+
${SALESFORCE_SHADOW_DOM_HELPERS}
|
|
92
111
|
// Collapse controls vary by page: <anypoint-button class="toggle-button">, <dx-button>, and
|
|
93
112
|
// older <* class="complex-toggle">. They all render as a button-like element whose entire
|
|
94
113
|
// label is just "Show"/"Hide", so match on that shape rather than a fixed tag list.
|
|
@@ -229,34 +248,12 @@ function buildCaptureScript(contentSelectors, extraRemove) {
|
|
|
229
248
|
};
|
|
230
249
|
const serialize = (el) => { const root = document.createElement('div'); append(el, root, 0); return root.innerHTML; };
|
|
231
250
|
|
|
232
|
-
|
|
233
|
-
// specific→general, so the first one with substantial content is the tight article/doc wrapper.
|
|
234
|
-
// Picking the largest match instead pulls in broader page chrome (nav tree, breadcrumb, chat
|
|
235
|
-
// widget) that surrounds the article. innerText is used because it pierces shadow roots
|
|
236
|
-
// (textContent does not), so a real container is measured rather than scoring 0.
|
|
237
|
-
const SELECTORS = ${JSON.stringify(contentSelectors)};
|
|
238
|
-
const MIN_PICK_CHARS = 200;
|
|
239
|
-
let container = null;
|
|
240
|
-
for (const sel of SELECTORS) {
|
|
241
|
-
let best = null, bestLen = 0;
|
|
242
|
-
for (const el of deepElements(document)) {
|
|
243
|
-
let matched; try { matched = el.matches && el.matches(sel); } catch (e) { matched = false; }
|
|
244
|
-
if (!matched) continue;
|
|
245
|
-
const len = (el.innerText || '').trim().length;
|
|
246
|
-
if (len > bestLen) { best = el; bestLen = len; }
|
|
247
|
-
}
|
|
248
|
-
if (best && bestLen >= MIN_PICK_CHARS) { container = best; break; }
|
|
249
|
-
}
|
|
250
|
-
container = container || document.body || document.documentElement;
|
|
251
|
+
${containerPick}
|
|
251
252
|
const title = (document.querySelector('h1')?.textContent || document.title || '').trim();
|
|
252
253
|
return { html: serialize(container), title };
|
|
253
254
|
})()`;
|
|
254
255
|
}
|
|
255
|
-
|
|
256
|
-
* Renders a Salesforce LWR doc page, serializes its shadow-DOM content container, and converts
|
|
257
|
-
* it to Markdown. Validates the host before any network access.
|
|
258
|
-
*/
|
|
259
|
-
export async function fetchSalesforceDoc(url, { allowedHost, contentSelectors, removeSelectors = [] }) {
|
|
256
|
+
function parseSalesforceDocUrl(url, allowedHost) {
|
|
260
257
|
let parsed;
|
|
261
258
|
try {
|
|
262
259
|
parsed = new URL(url);
|
|
@@ -267,6 +264,74 @@ export async function fetchSalesforceDoc(url, { allowedHost, contentSelectors, r
|
|
|
267
264
|
if (parsed.hostname.toLowerCase() !== allowedHost) {
|
|
268
265
|
throw new Error(`Refusing to fetch host "${parsed.hostname}" (expected ${allowedHost}).`);
|
|
269
266
|
}
|
|
267
|
+
return parsed;
|
|
268
|
+
}
|
|
269
|
+
function assertReadableSalesforceContent(detailedMarkdown, url) {
|
|
270
|
+
if (detailedMarkdown.length < MIN_CONTAINER_CHARS || looksLikeSalesforceError(detailedMarkdown)) {
|
|
271
|
+
throw new Error(`Salesforce returned no readable content for ${url} (page may be missing, gated, or still loading).`);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
async function captureSalesforceAttempt(page, url, captureScript, contentSelectors) {
|
|
275
|
+
const docsCapture = new ResponseCapture(page, [
|
|
276
|
+
{
|
|
277
|
+
key: 'docs-content',
|
|
278
|
+
test: (request) => request.url.includes(DOCS_CONTENT_API),
|
|
279
|
+
accept: (body) => parseDocsApiPayload(body) !== null,
|
|
280
|
+
},
|
|
281
|
+
]);
|
|
282
|
+
await page.client.send('Page.navigate', { url }, page.sessionId);
|
|
283
|
+
await waitForLoad(page.client, page.sessionId, TIMEOUT_MS, 0).catch(() => { });
|
|
284
|
+
await page.client
|
|
285
|
+
.send('Runtime.evaluate', { expression: ACCEPT_CONSENT_EXPRESSION }, page.sessionId)
|
|
286
|
+
.catch(() => { });
|
|
287
|
+
await pollSalesforceContentReady(page, contentSelectors, MIN_CONTAINER_CHARS, TIMEOUT_MS);
|
|
288
|
+
await new Promise((r) => setTimeout(r, SETTLE_MS));
|
|
289
|
+
const docsPayload = parseDocsApiPayload((await docsCapture.waitFor('docs-content', DOCS_API_GRACE_MS)) ?? '');
|
|
290
|
+
const docsApiHtml = docsPayload?.content ?? '';
|
|
291
|
+
const docsApiTitle = docsPayload?.title ?? '';
|
|
292
|
+
const result = await page.client.send('Runtime.evaluate', { expression: captureScript, awaitPromise: true, returnByValue: true }, page.sessionId);
|
|
293
|
+
const captured = result?.result?.value;
|
|
294
|
+
const html = typeof captured?.html === 'string' ? captured.html : '';
|
|
295
|
+
const title = captured?.title || '';
|
|
296
|
+
if (Buffer.byteLength(html) > BODY_LIMIT_BYTES) {
|
|
297
|
+
throw new Error(`Page exceeded body size limit (${Buffer.byteLength(html)} bytes).`);
|
|
298
|
+
}
|
|
299
|
+
let detailedMarkdown = stripBoilerplate(htmlToMarkdown(html));
|
|
300
|
+
let usedDocsApi = false;
|
|
301
|
+
if (docsApiHtml) {
|
|
302
|
+
const picked = preferRicherMarkdown(detailedMarkdown, docsApiHtml, title, docsApiTitle);
|
|
303
|
+
detailedMarkdown = picked.markdown;
|
|
304
|
+
if (picked.usedApi) {
|
|
305
|
+
usedDocsApi = true;
|
|
306
|
+
}
|
|
307
|
+
return { html, title: picked.title, detailedMarkdown, usedDocsApi };
|
|
308
|
+
}
|
|
309
|
+
return { html, title, detailedMarkdown, usedDocsApi };
|
|
310
|
+
}
|
|
311
|
+
function buildSalesforceFetchResult(url, html, title, detailedMarkdown, qualityNotes) {
|
|
312
|
+
return {
|
|
313
|
+
fetchResult: {
|
|
314
|
+
contentType: 'text/html',
|
|
315
|
+
etag: null,
|
|
316
|
+
lastModified: null,
|
|
317
|
+
finalUrl: url,
|
|
318
|
+
responseSize: Buffer.byteLength(html),
|
|
319
|
+
content: html,
|
|
320
|
+
},
|
|
321
|
+
extraction: {
|
|
322
|
+
title: title || url,
|
|
323
|
+
detailedMarkdown,
|
|
324
|
+
confidence: confidenceFor(detailedMarkdown.length),
|
|
325
|
+
qualityNotes,
|
|
326
|
+
},
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
/**
|
|
330
|
+
* Renders a Salesforce LWR doc page, serializes its shadow-DOM content container, and converts
|
|
331
|
+
* it to Markdown. Validates the host before any network access.
|
|
332
|
+
*/
|
|
333
|
+
export async function fetchSalesforceDoc(url, { allowedHost, contentSelectors, removeSelectors = [] }) {
|
|
334
|
+
const parsed = parseSalesforceDocUrl(url, allowedHost);
|
|
270
335
|
await checkDnsSafety(parsed.hostname);
|
|
271
336
|
const captureScript = buildCaptureScript(contentSelectors, removeSelectors);
|
|
272
337
|
const page = await openCdpPage();
|
|
@@ -274,46 +339,22 @@ export async function fetchSalesforceDoc(url, { allowedHost, contentSelectors, r
|
|
|
274
339
|
let html = '';
|
|
275
340
|
let title = '';
|
|
276
341
|
let detailedMarkdown = '';
|
|
277
|
-
|
|
278
|
-
|
|
342
|
+
let usedDocsApi = false;
|
|
343
|
+
const qualityNotes = ['extracted from rendered shadow-DOM content container'];
|
|
279
344
|
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
|
280
|
-
await page
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
await waitForContentReady(page, contentSelectors, MIN_CONTENT_CHARS, TIMEOUT_MS);
|
|
286
|
-
await new Promise((r) => setTimeout(r, SETTLE_MS));
|
|
287
|
-
const result = await page.client.send('Runtime.evaluate', { expression: captureScript, awaitPromise: true, returnByValue: true }, page.sessionId);
|
|
288
|
-
const captured = result?.result?.value;
|
|
289
|
-
html = typeof captured?.html === 'string' ? captured.html : '';
|
|
290
|
-
title = captured?.title || '';
|
|
291
|
-
if (Buffer.byteLength(html) > BODY_LIMIT_BYTES) {
|
|
292
|
-
throw new Error(`Page exceeded body size limit (${Buffer.byteLength(html)} bytes).`);
|
|
293
|
-
}
|
|
294
|
-
detailedMarkdown = stripBoilerplate(htmlToMarkdown(html));
|
|
345
|
+
const captured = await captureSalesforceAttempt(page, url, captureScript, contentSelectors);
|
|
346
|
+
html = captured.html;
|
|
347
|
+
title = captured.title;
|
|
348
|
+
detailedMarkdown = captured.detailedMarkdown;
|
|
349
|
+
usedDocsApi = usedDocsApi || captured.usedDocsApi;
|
|
295
350
|
if (detailedMarkdown.length >= SUBSTANTIAL_CHARS)
|
|
296
351
|
break;
|
|
297
352
|
}
|
|
298
|
-
|
|
299
|
-
|
|
353
|
+
assertReadableSalesforceContent(detailedMarkdown, url);
|
|
354
|
+
if (usedDocsApi) {
|
|
355
|
+
qualityNotes.push('article body from /docs/get_document_content/ API');
|
|
300
356
|
}
|
|
301
|
-
return
|
|
302
|
-
fetchResult: {
|
|
303
|
-
contentType: 'text/html',
|
|
304
|
-
etag: null,
|
|
305
|
-
lastModified: null,
|
|
306
|
-
finalUrl: url,
|
|
307
|
-
responseSize: Buffer.byteLength(html),
|
|
308
|
-
content: html,
|
|
309
|
-
},
|
|
310
|
-
extraction: {
|
|
311
|
-
title: title || url,
|
|
312
|
-
detailedMarkdown,
|
|
313
|
-
confidence: confidenceFor(detailedMarkdown.length),
|
|
314
|
-
qualityNotes: ['extracted from rendered shadow-DOM content container'],
|
|
315
|
-
},
|
|
316
|
-
};
|
|
357
|
+
return buildSalesforceFetchResult(url, html, title, detailedMarkdown, qualityNotes);
|
|
317
358
|
}
|
|
318
359
|
finally {
|
|
319
360
|
await page.close();
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { CdpPage } from '../lib/research/browser.js';
|
|
2
|
+
/** Shared in-page helpers for traversing Salesforce LWR open shadow roots. */
|
|
3
|
+
export declare const SALESFORCE_SHADOW_DOM_HELPERS = "\nconst deepElements = (root) => {\n const out = [];\n const visit = (node) => {\n if (node.shadowRoot) visit(node.shadowRoot);\n for (const child of Array.from(node.children || [])) {\n out.push(child);\n visit(child);\n }\n };\n visit(root);\n return out;\n};\n\nconst deepText = (root) => {\n let txt = '';\n const visit = (node) => {\n if (node.nodeType === 3) {\n txt += node.textContent || '';\n } else if (node.nodeType === 1 || node.nodeType === 9 || node.nodeType === 11) {\n const tag = node.tagName ? node.tagName.toLowerCase() : '';\n if (tag !== 'style' && tag !== 'script') {\n if (node.shadowRoot) visit(node.shadowRoot);\n for (const child of Array.from(node.childNodes || [])) visit(child);\n }\n }\n };\n visit(root);\n return txt;\n};\n\nconst textLenFor = (el) =>\n Math.max((el.innerText || '').trim().length, deepText(el).trim().length);\n";
|
|
4
|
+
/**
|
|
5
|
+
* In-page block: pick the highest-priority content container that exists on the page.
|
|
6
|
+
* Does not fall through to lower-priority selectors when a higher-priority host is present
|
|
7
|
+
* (e.g. doc-content-layout banner vs doc-amf-reference table).
|
|
8
|
+
*/
|
|
9
|
+
export declare function buildPriorityContainerPickBlock(selectors: string[], minChars: number, containerVar?: string): string;
|
|
10
|
+
/** CDP expression polled until the priority content host reaches minChars and the body settles. */
|
|
11
|
+
export declare function buildContentReadyProbeExpression(selectors: string[], minChars: number): string;
|
|
12
|
+
/**
|
|
13
|
+
* Polls until the priority Salesforce content host is ready and body text has settled.
|
|
14
|
+
* Returns without throwing on timeout; the caller captures whatever rendered and judges quality.
|
|
15
|
+
*/
|
|
16
|
+
export declare function pollSalesforceContentReady(page: CdpPage, selectors: string[], minChars: number, timeoutMs: number): Promise<void>;
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/** Shared in-page helpers for traversing Salesforce LWR open shadow roots. */
|
|
2
|
+
export const SALESFORCE_SHADOW_DOM_HELPERS = `
|
|
3
|
+
const deepElements = (root) => {
|
|
4
|
+
const out = [];
|
|
5
|
+
const visit = (node) => {
|
|
6
|
+
if (node.shadowRoot) visit(node.shadowRoot);
|
|
7
|
+
for (const child of Array.from(node.children || [])) {
|
|
8
|
+
out.push(child);
|
|
9
|
+
visit(child);
|
|
10
|
+
}
|
|
11
|
+
};
|
|
12
|
+
visit(root);
|
|
13
|
+
return out;
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
const deepText = (root) => {
|
|
17
|
+
let txt = '';
|
|
18
|
+
const visit = (node) => {
|
|
19
|
+
if (node.nodeType === 3) {
|
|
20
|
+
txt += node.textContent || '';
|
|
21
|
+
} else if (node.nodeType === 1 || node.nodeType === 9 || node.nodeType === 11) {
|
|
22
|
+
const tag = node.tagName ? node.tagName.toLowerCase() : '';
|
|
23
|
+
if (tag !== 'style' && tag !== 'script') {
|
|
24
|
+
if (node.shadowRoot) visit(node.shadowRoot);
|
|
25
|
+
for (const child of Array.from(node.childNodes || [])) visit(child);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
};
|
|
29
|
+
visit(root);
|
|
30
|
+
return txt;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
const textLenFor = (el) =>
|
|
34
|
+
Math.max((el.innerText || '').trim().length, deepText(el).trim().length);
|
|
35
|
+
`;
|
|
36
|
+
/**
|
|
37
|
+
* In-page block: pick the highest-priority content container that exists on the page.
|
|
38
|
+
* Does not fall through to lower-priority selectors when a higher-priority host is present
|
|
39
|
+
* (e.g. doc-content-layout banner vs doc-amf-reference table).
|
|
40
|
+
*/
|
|
41
|
+
export function buildPriorityContainerPickBlock(selectors, minChars, containerVar = 'container') {
|
|
42
|
+
return `
|
|
43
|
+
let ${containerVar} = null;
|
|
44
|
+
const SELECTORS = ${JSON.stringify(selectors)};
|
|
45
|
+
const MIN_CHARS = ${minChars};
|
|
46
|
+
for (const sel of SELECTORS) {
|
|
47
|
+
let best = null, bestLen = 0, anyMatch = false;
|
|
48
|
+
for (const el of deepElements(document)) {
|
|
49
|
+
let matched;
|
|
50
|
+
try { matched = el.matches && el.matches(sel); } catch (e) { matched = false; }
|
|
51
|
+
if (!matched) continue;
|
|
52
|
+
anyMatch = true;
|
|
53
|
+
const len = textLenFor(el);
|
|
54
|
+
if (len > bestLen) { best = el; bestLen = len; }
|
|
55
|
+
}
|
|
56
|
+
if (anyMatch) {
|
|
57
|
+
if (best && bestLen >= MIN_CHARS) ${containerVar} = best;
|
|
58
|
+
break;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
${containerVar} = ${containerVar} || document.body || document.documentElement;
|
|
62
|
+
`;
|
|
63
|
+
}
|
|
64
|
+
/** CDP expression polled until the priority content host reaches minChars and the body settles. */
|
|
65
|
+
export function buildContentReadyProbeExpression(selectors, minChars) {
|
|
66
|
+
return `(() => {
|
|
67
|
+
${SALESFORCE_SHADOW_DOM_HELPERS}
|
|
68
|
+
const SELS = ${JSON.stringify(selectors)};
|
|
69
|
+
const MIN = ${minChars};
|
|
70
|
+
let has = false;
|
|
71
|
+
for (const sel of SELS) {
|
|
72
|
+
let bestLen = 0, anyMatch = false;
|
|
73
|
+
for (const el of deepElements(document)) {
|
|
74
|
+
let matched;
|
|
75
|
+
try { matched = el.matches && el.matches(sel); } catch (e) { matched = false; }
|
|
76
|
+
if (!matched) continue;
|
|
77
|
+
anyMatch = true;
|
|
78
|
+
const len = textLenFor(el);
|
|
79
|
+
if (len > bestLen) bestLen = len;
|
|
80
|
+
}
|
|
81
|
+
if (anyMatch) {
|
|
82
|
+
has = bestLen >= MIN;
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
const len = document.body ? ((document.body.innerText) || '').length : 0;
|
|
87
|
+
return { has, len };
|
|
88
|
+
})()`;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Polls until the priority Salesforce content host is ready and body text has settled.
|
|
92
|
+
* Returns without throwing on timeout; the caller captures whatever rendered and judges quality.
|
|
93
|
+
*/
|
|
94
|
+
export async function pollSalesforceContentReady(page, selectors, minChars, timeoutMs) {
|
|
95
|
+
const expression = buildContentReadyProbeExpression(selectors, minChars);
|
|
96
|
+
const start = Date.now();
|
|
97
|
+
let previousLength = -1;
|
|
98
|
+
let stablePolls = 0;
|
|
99
|
+
while (Date.now() - start < timeoutMs) {
|
|
100
|
+
const rawResult = await page.client
|
|
101
|
+
.send('Runtime.evaluate', { expression, returnByValue: true }, page.sessionId)
|
|
102
|
+
.catch(() => null);
|
|
103
|
+
const value = rawResult?.result?.value;
|
|
104
|
+
if (value?.has) {
|
|
105
|
+
const grew = Math.abs((value.len || 0) - previousLength) > Math.max(40, previousLength * 0.02);
|
|
106
|
+
if (!grew && ++stablePolls >= 2)
|
|
107
|
+
return;
|
|
108
|
+
if (grew)
|
|
109
|
+
stablePolls = 0;
|
|
110
|
+
previousLength = value.len || 0;
|
|
111
|
+
}
|
|
112
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
113
|
+
}
|
|
114
|
+
}
|
package/dist/sites/types.d.ts
CHANGED
|
@@ -10,11 +10,6 @@ export interface SiteFetchResult {
|
|
|
10
10
|
};
|
|
11
11
|
extraction: ExtractionResult;
|
|
12
12
|
}
|
|
13
|
-
export interface SiteSearchResult {
|
|
14
|
-
url: string;
|
|
15
|
-
title: string;
|
|
16
|
-
snippet?: string;
|
|
17
|
-
}
|
|
18
13
|
export interface SiteModule {
|
|
19
14
|
id: string;
|
|
20
15
|
name: string;
|
|
@@ -23,5 +18,4 @@ export interface SiteModule {
|
|
|
23
18
|
rendered?: boolean;
|
|
24
19
|
};
|
|
25
20
|
fetchPage?: (url: string) => Promise<SiteFetchResult>;
|
|
26
|
-
search?: (query: string) => Promise<SiteSearchResult[]>;
|
|
27
21
|
}
|
package/oclif.manifest.json
CHANGED
|
@@ -128,7 +128,7 @@
|
|
|
128
128
|
"pluginAlias": "@taurgis/bonsai",
|
|
129
129
|
"pluginName": "@taurgis/bonsai",
|
|
130
130
|
"pluginType": "core",
|
|
131
|
-
"strict":
|
|
131
|
+
"strict": false,
|
|
132
132
|
"summary": "An advanced, locally cached web research tool optimized for LLM ingestion.",
|
|
133
133
|
"enableJsonFlag": true,
|
|
134
134
|
"stdoutIsPrimaryData": true
|
|
@@ -155,6 +155,10 @@
|
|
|
155
155
|
{
|
|
156
156
|
"description": "import research from a local Markdown file",
|
|
157
157
|
"command": "<%= config.bin %> import https://example.com/docs --file path/to/notes.md"
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
"description": "import research from stdin using the standard file placeholder",
|
|
161
|
+
"command": "cat notes.md | <%= config.bin %> import https://example.com/docs --file -"
|
|
158
162
|
}
|
|
159
163
|
],
|
|
160
164
|
"flags": {
|
|
@@ -172,7 +176,7 @@
|
|
|
172
176
|
"type": "boolean"
|
|
173
177
|
},
|
|
174
178
|
"file": {
|
|
175
|
-
"description": "path to a Markdown file containing research notes to import",
|
|
179
|
+
"description": "path to a Markdown file containing research notes to import, or \"-\" for stdin",
|
|
176
180
|
"name": "file",
|
|
177
181
|
"hasDynamicHelp": false,
|
|
178
182
|
"multiple": false,
|
|
@@ -292,7 +296,7 @@
|
|
|
292
296
|
"pluginAlias": "@taurgis/bonsai",
|
|
293
297
|
"pluginName": "@taurgis/bonsai",
|
|
294
298
|
"pluginType": "core",
|
|
295
|
-
"strict":
|
|
299
|
+
"strict": false,
|
|
296
300
|
"summary": "Inspect cached research metadata for a URL.",
|
|
297
301
|
"enableJsonFlag": true,
|
|
298
302
|
"stdoutIsPrimaryData": true
|
|
@@ -313,6 +317,10 @@
|
|
|
313
317
|
{
|
|
314
318
|
"description": "list only fresh entries filtered by tags",
|
|
315
319
|
"command": "<%= config.bin %> list --freshness fresh --tags node --tags url"
|
|
320
|
+
},
|
|
321
|
+
{
|
|
322
|
+
"description": "list entries matching a specific URL glob",
|
|
323
|
+
"command": "<%= config.bin %> list --url \"https://react.dev/*\""
|
|
316
324
|
}
|
|
317
325
|
],
|
|
318
326
|
"flags": {
|
|
@@ -339,6 +347,13 @@
|
|
|
339
347
|
"multiple": true,
|
|
340
348
|
"type": "option"
|
|
341
349
|
},
|
|
350
|
+
"url": {
|
|
351
|
+
"description": "filter by source URL glob pattern (case-insensitive, supports * wildcard)",
|
|
352
|
+
"name": "url",
|
|
353
|
+
"hasDynamicHelp": false,
|
|
354
|
+
"multiple": false,
|
|
355
|
+
"type": "option"
|
|
356
|
+
},
|
|
342
357
|
"freshness": {
|
|
343
358
|
"description": "filter by freshness state",
|
|
344
359
|
"name": "freshness",
|
|
@@ -409,6 +424,10 @@
|
|
|
409
424
|
{
|
|
410
425
|
"description": "actually prune entries older than 30 days that are source scrapes",
|
|
411
426
|
"command": "<%= config.bin %> prune --older-than 30d --artifact-type source --yes"
|
|
427
|
+
},
|
|
428
|
+
{
|
|
429
|
+
"description": "prune entries matching a URL glob pattern",
|
|
430
|
+
"command": "<%= config.bin %> prune --url \"https://react.dev/*\" --yes"
|
|
412
431
|
}
|
|
413
432
|
],
|
|
414
433
|
"flags": {
|
|
@@ -433,6 +452,13 @@
|
|
|
433
452
|
"multiple": false,
|
|
434
453
|
"type": "option"
|
|
435
454
|
},
|
|
455
|
+
"url": {
|
|
456
|
+
"description": "filter pruning to source URL glob pattern (case-insensitive, supports * wildcard)",
|
|
457
|
+
"name": "url",
|
|
458
|
+
"hasDynamicHelp": false,
|
|
459
|
+
"multiple": false,
|
|
460
|
+
"type": "option"
|
|
461
|
+
},
|
|
436
462
|
"artifact-type": {
|
|
437
463
|
"description": "filter pruning to specific artifact type",
|
|
438
464
|
"name": "artifact-type",
|
|
@@ -471,107 +497,6 @@
|
|
|
471
497
|
"enableJsonFlag": true,
|
|
472
498
|
"stdoutIsPrimaryData": true
|
|
473
499
|
},
|
|
474
|
-
"search": {
|
|
475
|
-
"aliases": [],
|
|
476
|
-
"args": {
|
|
477
|
-
"query": {
|
|
478
|
-
"description": "the search query containing terms to match",
|
|
479
|
-
"name": "query",
|
|
480
|
-
"required": true
|
|
481
|
-
}
|
|
482
|
-
},
|
|
483
|
-
"description": "Scans the local research cache database and ranks matching entries based on topic, tags, source URLs, summary, and content.",
|
|
484
|
-
"examples": [
|
|
485
|
-
{
|
|
486
|
-
"description": "search for cached notes containing specific keywords",
|
|
487
|
-
"command": "<%= config.bin %> <%= command.id %> \"authentication flow\""
|
|
488
|
-
},
|
|
489
|
-
{
|
|
490
|
-
"description": "search with remote docs fallback",
|
|
491
|
-
"command": "<%= config.bin %> <%= command.id %> \"router\" --remote https://react.dev"
|
|
492
|
-
},
|
|
493
|
-
{
|
|
494
|
-
"description": "search local cache with tag filtering",
|
|
495
|
-
"command": "<%= config.bin %> <%= command.id %> \"setup\" --tags node --tags tutorial"
|
|
496
|
-
}
|
|
497
|
-
],
|
|
498
|
-
"flags": {
|
|
499
|
-
"json": {
|
|
500
|
-
"description": "Format output as json.",
|
|
501
|
-
"helpGroup": "GLOBAL",
|
|
502
|
-
"name": "json",
|
|
503
|
-
"allowNo": false,
|
|
504
|
-
"type": "boolean"
|
|
505
|
-
},
|
|
506
|
-
"topic": {
|
|
507
|
-
"char": "t",
|
|
508
|
-
"description": "filter results by exact topic (case-insensitive)",
|
|
509
|
-
"name": "topic",
|
|
510
|
-
"hasDynamicHelp": false,
|
|
511
|
-
"multiple": false,
|
|
512
|
-
"type": "option"
|
|
513
|
-
},
|
|
514
|
-
"tags": {
|
|
515
|
-
"char": "g",
|
|
516
|
-
"description": "filter results by tags (must match all tags specified)",
|
|
517
|
-
"name": "tags",
|
|
518
|
-
"hasDynamicHelp": false,
|
|
519
|
-
"multiple": true,
|
|
520
|
-
"type": "option"
|
|
521
|
-
},
|
|
522
|
-
"artifact-type": {
|
|
523
|
-
"description": "filter results by artifact type",
|
|
524
|
-
"name": "artifact-type",
|
|
525
|
-
"hasDynamicHelp": false,
|
|
526
|
-
"multiple": false,
|
|
527
|
-
"options": [
|
|
528
|
-
"source",
|
|
529
|
-
"research_note",
|
|
530
|
-
"index",
|
|
531
|
-
"section"
|
|
532
|
-
],
|
|
533
|
-
"type": "option"
|
|
534
|
-
},
|
|
535
|
-
"limit": {
|
|
536
|
-
"description": "maximum number of results to return (default 10, max 50)",
|
|
537
|
-
"name": "limit",
|
|
538
|
-
"default": 10,
|
|
539
|
-
"hasDynamicHelp": false,
|
|
540
|
-
"multiple": false,
|
|
541
|
-
"type": "option"
|
|
542
|
-
},
|
|
543
|
-
"include-stale": {
|
|
544
|
-
"description": "include stale expired cache entries in the search results",
|
|
545
|
-
"name": "include-stale",
|
|
546
|
-
"allowNo": false,
|
|
547
|
-
"type": "boolean"
|
|
548
|
-
},
|
|
549
|
-
"domain": {
|
|
550
|
-
"description": "search a documentation site directly (requires compatible site module)",
|
|
551
|
-
"name": "domain",
|
|
552
|
-
"hasDynamicHelp": false,
|
|
553
|
-
"multiple": false,
|
|
554
|
-
"type": "option"
|
|
555
|
-
},
|
|
556
|
-
"remote": {
|
|
557
|
-
"description": "discover uncached docs via public search or index (falls back to local cache)",
|
|
558
|
-
"name": "remote",
|
|
559
|
-
"hasDynamicHelp": false,
|
|
560
|
-
"multiple": false,
|
|
561
|
-
"type": "option"
|
|
562
|
-
}
|
|
563
|
-
},
|
|
564
|
-
"hasDynamicHelp": false,
|
|
565
|
-
"hiddenAliases": [],
|
|
566
|
-
"id": "search",
|
|
567
|
-
"pluginAlias": "@taurgis/bonsai",
|
|
568
|
-
"pluginName": "@taurgis/bonsai",
|
|
569
|
-
"pluginType": "core",
|
|
570
|
-
"strict": true,
|
|
571
|
-
"summary": "Search locally cached research artifacts by keywords.",
|
|
572
|
-
"enableJsonFlag": true,
|
|
573
|
-
"stdoutIsPrimaryData": true
|
|
574
|
-
},
|
|
575
500
|
"status": {
|
|
576
501
|
"aliases": [],
|
|
577
502
|
"args": {
|
|
@@ -635,7 +560,7 @@
|
|
|
635
560
|
"pluginAlias": "@taurgis/bonsai",
|
|
636
561
|
"pluginName": "@taurgis/bonsai",
|
|
637
562
|
"pluginType": "core",
|
|
638
|
-
"strict":
|
|
563
|
+
"strict": false,
|
|
639
564
|
"summary": "Check the cache status of a URL without fetching or writing.",
|
|
640
565
|
"enableJsonFlag": true,
|
|
641
566
|
"stdoutIsPrimaryData": true
|
|
@@ -918,5 +843,5 @@
|
|
|
918
843
|
"stdoutIsPrimaryData": true
|
|
919
844
|
}
|
|
920
845
|
},
|
|
921
|
-
"version": "
|
|
846
|
+
"version": "3.0.0"
|
|
922
847
|
}
|