@taurgis/bonsai 1.0.5 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -12
- package/agents/agents/official-docs-researcher.agent.md +15 -26
- package/agents/agents/salesforce-docs-researcher.agent.md +17 -35
- package/agents/hooks/README.md +2 -9
- package/agents/hooks/claude-code/hooks/bonsai-web-fetch.sh +1 -1
- package/agents/hooks/codex/hooks/bonsai-user-prompt-context.sh +1 -1
- package/agents/hooks/codex/hooks/bonsai-web-fetch.sh +1 -1
- package/agents/hooks/cursor/hooks/bonsai-web-fetch.sh +1 -1
- package/agents/hooks/github-copilot/bonsai-web-fetch.sh +1 -1
- package/agents/instructions/salesforce-research.instructions.md +7 -6
- package/agents/instructions/web-research.instructions.md +3 -2
- package/agents/skills/web-research/SKILL.md +24 -32
- package/bin/cli.mjs +44 -14
- package/dist/base-command.d.ts +17 -1
- package/dist/base-command.js +70 -9
- package/dist/commands/config/base.d.ts +2 -0
- package/dist/commands/config/base.js +15 -1
- package/dist/commands/config/get.d.ts +1 -1
- package/dist/commands/config/get.js +4 -5
- package/dist/commands/config/index.d.ts +12 -0
- package/dist/commands/config/index.js +34 -0
- package/dist/commands/config/list.js +2 -2
- package/dist/commands/config/set.js +18 -12
- package/dist/commands/config/unset.d.ts +1 -1
- package/dist/commands/config/unset.js +9 -6
- package/dist/commands/fetch.d.ts +2 -0
- package/dist/commands/fetch.js +69 -45
- package/dist/commands/import.d.ts +3 -0
- package/dist/commands/import.js +91 -53
- package/dist/commands/inspect.d.ts +8 -0
- package/dist/commands/inspect.js +59 -30
- package/dist/commands/list.d.ts +5 -3
- package/dist/commands/list.js +60 -35
- package/dist/commands/prune.d.ts +2 -1
- package/dist/commands/prune.js +62 -19
- package/dist/commands/status.d.ts +8 -0
- package/dist/commands/status.js +84 -26
- package/dist/commands.d.ts +2 -2
- package/dist/commands.js +2 -2
- package/dist/hooks/command-not-found/suggest.d.ts +7 -1
- package/dist/hooks/command-not-found/suggest.js +100 -46
- package/dist/lib/argv.d.ts +10 -0
- package/dist/lib/argv.js +97 -0
- package/dist/lib/color.d.ts +20 -0
- package/dist/lib/color.js +36 -0
- package/dist/lib/envelope.d.ts +36 -0
- package/dist/lib/envelope.js +82 -0
- package/dist/lib/help-preflight.d.ts +15 -0
- package/dist/lib/help-preflight.js +81 -0
- package/dist/lib/json-meta.d.ts +10 -0
- package/dist/lib/json-meta.js +98 -0
- package/dist/lib/limit-flag.d.ts +1 -0
- package/dist/lib/limit-flag.js +24 -0
- package/dist/lib/research/{search-index.d.ts → artifact-index.d.ts} +5 -5
- package/dist/lib/research/{search-index.js → artifact-index.js} +10 -10
- package/dist/lib/research/browser.d.ts +0 -8
- package/dist/lib/research/browser.js +20 -43
- package/dist/lib/research/regression.d.ts +18 -0
- package/dist/lib/research/regression.js +47 -0
- package/dist/lib/research/revalidate.js +3 -1
- package/dist/lib/research/schema.d.ts +5 -2
- package/dist/lib/research/schema.js +12 -1
- package/dist/lib/research/storage.d.ts +0 -1
- package/dist/lib/research/storage.js +15 -13
- package/dist/lib/research/url.d.ts +22 -0
- package/dist/lib/research/url.js +63 -1
- package/dist/lib/text.d.ts +5 -1
- package/dist/lib/text.js +11 -1
- package/dist/sites/coveo.d.ts +7 -0
- package/dist/sites/coveo.js +33 -0
- package/dist/sites/index.js +1 -2
- package/dist/sites/salesforce/fetch-page.js +1 -1
- package/dist/sites/salesforce/index.js +3 -8
- package/dist/sites/salesforce-developer/fetch-page.js +3 -1
- package/dist/sites/salesforce-developer/index.js +1 -2
- package/dist/sites/salesforce-doc-fetch.d.ts +11 -0
- package/dist/sites/salesforce-doc-fetch.js +117 -76
- package/dist/sites/salesforce-dom-probe.d.ts +16 -0
- package/dist/sites/salesforce-dom-probe.js +114 -0
- package/dist/sites/types.d.ts +0 -6
- package/oclif.manifest.json +153 -155
- package/package.json +38 -25
- package/dist/commands/index.d.ts +0 -35
- package/dist/commands/index.js +0 -306
- package/dist/commands/research/config/base.d.ts +0 -23
- package/dist/commands/research/config/base.js +0 -46
- package/dist/commands/research/config/get.d.ts +0 -20
- package/dist/commands/research/config/get.js +0 -45
- package/dist/commands/research/config/list.d.ts +0 -17
- package/dist/commands/research/config/list.js +0 -42
- package/dist/commands/research/config/set.d.ts +0 -31
- package/dist/commands/research/config/set.js +0 -94
- package/dist/commands/research/config/unset.d.ts +0 -21
- package/dist/commands/research/config/unset.js +0 -59
- package/dist/commands/research/import.d.ts +0 -38
- package/dist/commands/research/import.js +0 -313
- package/dist/commands/research/inspect.d.ts +0 -13
- package/dist/commands/research/inspect.js +0 -86
- package/dist/commands/research/list.d.ts +0 -25
- package/dist/commands/research/list.js +0 -143
- package/dist/commands/research/prune.d.ts +0 -24
- package/dist/commands/research/prune.js +0 -141
- package/dist/commands/research/search.d.ts +0 -32
- package/dist/commands/research/search.js +0 -334
- package/dist/commands/research/status.d.ts +0 -17
- package/dist/commands/research/status.js +0 -108
- package/dist/commands/research.d.ts +0 -36
- package/dist/commands/research.js +0 -304
- package/dist/commands/search.d.ts +0 -31
- package/dist/commands/search.js +0 -329
- package/dist/lib/research/docs/__fixtures__/sphinx-searchindex.d.ts +0 -1
- package/dist/lib/research/docs/__fixtures__/sphinx-searchindex.js +0 -2
- package/dist/lib/research/docs/discovery-index.d.ts +0 -14
- package/dist/lib/research/docs/discovery-index.js +0 -112
- package/dist/lib/research/docs/remote-search-runner.d.ts +0 -21
- package/dist/lib/research/docs/remote-search-runner.js +0 -114
- package/dist/lib/research/docs/remote-search.d.ts +0 -14
- package/dist/lib/research/docs/remote-search.js +0 -101
- package/dist/lib/research/docs/search-index.d.ts +0 -13
- package/dist/lib/research/docs/search-index.js +0 -176
- package/dist/sites/salesforce/browser-search.d.ts +0 -7
- package/dist/sites/salesforce/browser-search.js +0 -82
- package/dist/sites/salesforce/coveo.d.ts +0 -23
- package/dist/sites/salesforce/coveo.js +0 -161
- package/dist/sites/salesforce/token.d.ts +0 -26
- package/dist/sites/salesforce/token.js +0 -154
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import type { DocsSearchResult } from './search-index.js';
|
|
2
|
-
interface DiscoveryEntry {
|
|
3
|
-
title: string;
|
|
4
|
-
url: string;
|
|
5
|
-
body?: string;
|
|
6
|
-
}
|
|
7
|
-
export interface SitemapParseResult {
|
|
8
|
-
entries: DiscoveryEntry[];
|
|
9
|
-
sitemapUrls: string[];
|
|
10
|
-
}
|
|
11
|
-
export declare function parseSitemap(body: string): SitemapParseResult;
|
|
12
|
-
export declare function searchSitemap(body: string, indexUrl: string, query: string): DocsSearchResult[];
|
|
13
|
-
export declare function searchLlmsTxt(body: string, indexUrl: string, query: string): DocsSearchResult[];
|
|
14
|
-
export {};
|
|
@@ -1,112 +0,0 @@
|
|
|
1
|
-
import { DOMParser } from 'linkedom';
|
|
2
|
-
function queryTerms(query) {
|
|
3
|
-
return query.toLowerCase().split(/\s+/).filter(Boolean);
|
|
4
|
-
}
|
|
5
|
-
function scoreEntry(entry, terms) {
|
|
6
|
-
const haystack = `${entry.title} ${entry.url} ${entry.body ?? ''}`.toLowerCase();
|
|
7
|
-
let score = 0;
|
|
8
|
-
for (const term of terms) {
|
|
9
|
-
if (entry.title.toLowerCase().includes(term))
|
|
10
|
-
score += 10;
|
|
11
|
-
if (entry.url.toLowerCase().includes(term))
|
|
12
|
-
score += 5;
|
|
13
|
-
score += Math.min(haystack.split(term).length - 1, 5);
|
|
14
|
-
}
|
|
15
|
-
return score;
|
|
16
|
-
}
|
|
17
|
-
function snippetFrom(entry) {
|
|
18
|
-
return (entry.body || entry.url).replace(/\s+/g, ' ').trim().slice(0, 150);
|
|
19
|
-
}
|
|
20
|
-
function titleFromUrl(url) {
|
|
21
|
-
try {
|
|
22
|
-
const parsed = new URL(url);
|
|
23
|
-
const last = parsed.pathname.split('/').filter(Boolean).pop();
|
|
24
|
-
return last ? last.replace(/\.[a-z0-9]+$/i, '').replace(/[-_]+/g, ' ') : parsed.hostname;
|
|
25
|
-
}
|
|
26
|
-
catch {
|
|
27
|
-
return url;
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
function toResults(entries, indexUrl, provider, query) {
|
|
31
|
-
const terms = queryTerms(query);
|
|
32
|
-
return entries
|
|
33
|
-
.map((entry) => ({ entry, score: scoreEntry(entry, terms) }))
|
|
34
|
-
.filter(({ score }) => score > 0)
|
|
35
|
-
.map(({ entry, score }) => ({
|
|
36
|
-
title: entry.title,
|
|
37
|
-
url: entry.url,
|
|
38
|
-
snippet: snippetFrom(entry),
|
|
39
|
-
provider,
|
|
40
|
-
score,
|
|
41
|
-
indexUrl,
|
|
42
|
-
}))
|
|
43
|
-
.sort((a, b) => b.score - a.score);
|
|
44
|
-
}
|
|
45
|
-
function textContent(element) {
|
|
46
|
-
const text = element?.textContent?.trim();
|
|
47
|
-
return text || undefined;
|
|
48
|
-
}
|
|
49
|
-
export function parseSitemap(body) {
|
|
50
|
-
const trimmed = body.trim();
|
|
51
|
-
if (!trimmed)
|
|
52
|
-
return { entries: [], sitemapUrls: [] };
|
|
53
|
-
if (!trimmed.startsWith('<')) {
|
|
54
|
-
const entries = trimmed
|
|
55
|
-
.split(/\r?\n/)
|
|
56
|
-
.map((line) => line.trim())
|
|
57
|
-
.filter((line) => /^https?:\/\//i.test(line))
|
|
58
|
-
.map((url) => ({ title: titleFromUrl(url), url }));
|
|
59
|
-
return { entries, sitemapUrls: [] };
|
|
60
|
-
}
|
|
61
|
-
const doc = new DOMParser().parseFromString(trimmed, 'text/xml');
|
|
62
|
-
const sitemapUrls = Array.from(doc.querySelectorAll('sitemap > loc'))
|
|
63
|
-
.map((loc) => textContent(loc))
|
|
64
|
-
.filter((url) => !!url);
|
|
65
|
-
const entries = Array.from(doc.querySelectorAll('url'))
|
|
66
|
-
.map((urlElement) => {
|
|
67
|
-
const url = textContent(urlElement.querySelector('loc'));
|
|
68
|
-
if (!url)
|
|
69
|
-
return null;
|
|
70
|
-
const lastmod = textContent(urlElement.querySelector('lastmod'));
|
|
71
|
-
return { title: titleFromUrl(url), url, body: lastmod ? `Last modified ${lastmod}` : '' };
|
|
72
|
-
})
|
|
73
|
-
.filter((entry) => !!entry);
|
|
74
|
-
return { entries, sitemapUrls };
|
|
75
|
-
}
|
|
76
|
-
export function searchSitemap(body, indexUrl, query) {
|
|
77
|
-
return toResults(parseSitemap(body).entries, indexUrl, 'sitemap', query);
|
|
78
|
-
}
|
|
79
|
-
function referenceDefinitions(body) {
|
|
80
|
-
const refs = new Map();
|
|
81
|
-
for (const match of body.matchAll(/^\s*\[([^\]]+)\]:\s*(\S+)/gm)) {
|
|
82
|
-
refs.set(match[1].toLowerCase(), match[2]);
|
|
83
|
-
}
|
|
84
|
-
return refs;
|
|
85
|
-
}
|
|
86
|
-
export function searchLlmsTxt(body, indexUrl, query) {
|
|
87
|
-
const refs = referenceDefinitions(body);
|
|
88
|
-
const entries = [];
|
|
89
|
-
let section = '';
|
|
90
|
-
for (const line of body.split(/\r?\n/)) {
|
|
91
|
-
const heading = line.match(/^##\s+(.+?)\s*$/);
|
|
92
|
-
if (heading)
|
|
93
|
-
section = heading[1];
|
|
94
|
-
const inline = line.match(/^\s*[-*]\s+\[([^\]]+)\]\(([^)]+)\)\s*:?\s*(.*)$/);
|
|
95
|
-
const ref = line.match(/^\s*[-*]\s+\[([^\]]+)\]\s*:?\s*(.*)$/);
|
|
96
|
-
const title = inline?.[1] ?? ref?.[1];
|
|
97
|
-
const url = inline?.[2] ?? (title ? refs.get(title.toLowerCase()) : undefined);
|
|
98
|
-
if (!title || !url)
|
|
99
|
-
continue;
|
|
100
|
-
try {
|
|
101
|
-
entries.push({
|
|
102
|
-
title,
|
|
103
|
-
url: new URL(url, indexUrl).toString(),
|
|
104
|
-
body: [section, inline?.[3] ?? ref?.[2]].filter(Boolean).join(' - '),
|
|
105
|
-
});
|
|
106
|
-
}
|
|
107
|
-
catch {
|
|
108
|
-
// Ignore malformed links in untrusted machine-readable text.
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
return toResults(entries, indexUrl, 'llms.txt', query);
|
|
112
|
-
}
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import type { DocsSearchResult } from './search-index.js';
|
|
2
|
-
export interface RemoteSearchDeps {
|
|
3
|
-
fetchStatic: (url: string) => Promise<{
|
|
4
|
-
content: string;
|
|
5
|
-
finalUrl: string;
|
|
6
|
-
}>;
|
|
7
|
-
fetchText: (url: string) => Promise<{
|
|
8
|
-
content: string;
|
|
9
|
-
status: number;
|
|
10
|
-
}>;
|
|
11
|
-
postJson: (url: string, body: unknown, headers: Record<string, string>) => Promise<string>;
|
|
12
|
-
}
|
|
13
|
-
export interface RemoteSearchOutcome {
|
|
14
|
-
provider: string;
|
|
15
|
-
results: DocsSearchResult[];
|
|
16
|
-
}
|
|
17
|
-
/**
|
|
18
|
-
* Runs remote docs search for a documentation site. Detects the provider from the page, queries the
|
|
19
|
-
* matching connector, and returns normalized results. Throws when no verified connector applies.
|
|
20
|
-
*/
|
|
21
|
-
export declare function runRemoteDocsSearch(docsUrl: string, query: string, deps: RemoteSearchDeps): Promise<RemoteSearchOutcome>;
|
|
@@ -1,114 +0,0 @@
|
|
|
1
|
-
import { detectDocsEngine } from './detect.js';
|
|
2
|
-
import { probeLlmsTxt } from './machine-readable.js';
|
|
3
|
-
import { searchMkDocsIndex, searchJekyllIndex, searchSphinxIndex } from './search-index.js';
|
|
4
|
-
import { parseSitemap, searchLlmsTxt, searchSitemap } from './discovery-index.js';
|
|
5
|
-
import { extractAlgoliaConfig, algoliaQueryUrl, parseAlgoliaResponse } from './remote-search.js';
|
|
6
|
-
async function getText(url, deps) {
|
|
7
|
-
const res = await deps.fetchText(url);
|
|
8
|
-
if (res.status === 304 || !res.content)
|
|
9
|
-
throw new Error(`empty search index at ${url}`);
|
|
10
|
-
return res.content;
|
|
11
|
-
}
|
|
12
|
-
function sameHost(url, base) {
|
|
13
|
-
try {
|
|
14
|
-
return new URL(url).hostname === new URL(base).hostname;
|
|
15
|
-
}
|
|
16
|
-
catch {
|
|
17
|
-
return false;
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
// Conventional static index path + parser per framework (verified by T-25 fixtures).
|
|
21
|
-
async function staticIndexSearch(framework, origin, query, deps) {
|
|
22
|
-
if (framework === 'mkdocs' || framework === 'material-mkdocs') {
|
|
23
|
-
const indexUrl = `${origin}/search/search_index.json`;
|
|
24
|
-
return {
|
|
25
|
-
provider: 'mkdocs-local',
|
|
26
|
-
results: searchMkDocsIndex(await getText(indexUrl, deps), indexUrl, query),
|
|
27
|
-
};
|
|
28
|
-
}
|
|
29
|
-
if (framework === 'sphinx') {
|
|
30
|
-
const indexUrl = `${origin}/searchindex.js`;
|
|
31
|
-
return {
|
|
32
|
-
provider: 'sphinx-searchindex',
|
|
33
|
-
results: searchSphinxIndex(await getText(indexUrl, deps), indexUrl, query),
|
|
34
|
-
};
|
|
35
|
-
}
|
|
36
|
-
if (framework === 'just-the-docs') {
|
|
37
|
-
const indexUrl = `${origin}/assets/js/search-data.json`;
|
|
38
|
-
return {
|
|
39
|
-
provider: 'jekyll-json',
|
|
40
|
-
results: searchJekyllIndex(await getText(indexUrl, deps), indexUrl, query),
|
|
41
|
-
};
|
|
42
|
-
}
|
|
43
|
-
return null;
|
|
44
|
-
}
|
|
45
|
-
async function llmsTxtSearch(docsUrl, caps, query, deps) {
|
|
46
|
-
const llms = await probeLlmsTxt(docsUrl, caps, async (url) => {
|
|
47
|
-
const res = await deps.fetchText(url);
|
|
48
|
-
return { ...res, finalUrl: url, contentType: 'text/plain' };
|
|
49
|
-
});
|
|
50
|
-
if (!llms)
|
|
51
|
-
return null;
|
|
52
|
-
return {
|
|
53
|
-
provider: llms.artifact.type,
|
|
54
|
-
results: searchLlmsTxt(llms.body, llms.artifact.url, query),
|
|
55
|
-
};
|
|
56
|
-
}
|
|
57
|
-
async function sitemapSearch(pageUrl, query, deps) {
|
|
58
|
-
const origin = new URL(pageUrl).origin;
|
|
59
|
-
const firstSegment = new URL(pageUrl).pathname.split('/').filter(Boolean)[0];
|
|
60
|
-
const candidates = [
|
|
61
|
-
`${origin}/sitemap.xml`,
|
|
62
|
-
`${origin}/sitemap_index.xml`,
|
|
63
|
-
firstSegment ? `${origin}/${firstSegment}/sitemap.xml` : undefined,
|
|
64
|
-
].filter((url) => !!url);
|
|
65
|
-
for (const sitemapUrl of [...new Set(candidates)]) {
|
|
66
|
-
try {
|
|
67
|
-
const body = await getText(sitemapUrl, deps);
|
|
68
|
-
const results = searchSitemap(body, sitemapUrl, query);
|
|
69
|
-
if (results.length > 0)
|
|
70
|
-
return { provider: 'sitemap', results };
|
|
71
|
-
const index = parseSitemap(body);
|
|
72
|
-
const childResults = [];
|
|
73
|
-
for (const childUrl of index.sitemapUrls.slice(0, 10)) {
|
|
74
|
-
if (!sameHost(childUrl, pageUrl))
|
|
75
|
-
continue;
|
|
76
|
-
const childBody = await getText(childUrl, deps);
|
|
77
|
-
childResults.push(...searchSitemap(childBody, childUrl, query));
|
|
78
|
-
}
|
|
79
|
-
if (childResults.length > 0) {
|
|
80
|
-
return { provider: 'sitemap', results: childResults.sort((a, b) => b.score - a.score) };
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
catch {
|
|
84
|
-
// Try the next conventional sitemap location.
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
return null;
|
|
88
|
-
}
|
|
89
|
-
/**
|
|
90
|
-
* Runs remote docs search for a documentation site. Detects the provider from the page, queries the
|
|
91
|
-
* matching connector, and returns normalized results. Throws when no verified connector applies.
|
|
92
|
-
*/
|
|
93
|
-
export async function runRemoteDocsSearch(docsUrl, query, deps) {
|
|
94
|
-
const page = await deps.fetchStatic(docsUrl);
|
|
95
|
-
const origin = new URL(page.finalUrl).origin;
|
|
96
|
-
const algolia = extractAlgoliaConfig(page.content);
|
|
97
|
-
if (algolia) {
|
|
98
|
-
const url = algoliaQueryUrl(algolia);
|
|
99
|
-
const body = await deps.postJson(url, { params: `query=${encodeURIComponent(query)}` }, { 'X-Algolia-API-Key': algolia.apiKey, 'X-Algolia-Application-Id': algolia.appId });
|
|
100
|
-
return { provider: 'algolia-docsearch', results: parseAlgoliaResponse(body, url) };
|
|
101
|
-
}
|
|
102
|
-
const caps = detectDocsEngine(page.content, page.finalUrl);
|
|
103
|
-
const staticResult = await staticIndexSearch(caps.framework, origin, query, deps);
|
|
104
|
-
if (staticResult)
|
|
105
|
-
return staticResult;
|
|
106
|
-
const llmsResult = await llmsTxtSearch(page.finalUrl, caps, query, deps);
|
|
107
|
-
if (llmsResult)
|
|
108
|
-
return llmsResult;
|
|
109
|
-
const sitemapResult = await sitemapSearch(page.finalUrl, query, deps);
|
|
110
|
-
if (sitemapResult)
|
|
111
|
-
return sitemapResult;
|
|
112
|
-
throw new Error(`No verified remote search connector for this site (detected: ${caps.framework ?? caps.docsEngine ?? 'unknown'}). ` +
|
|
113
|
-
'Supported: Algolia DocSearch, MkDocs/Material, Sphinx, Just the Docs, llms.txt, sitemap.xml.');
|
|
114
|
-
}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import type { DocsSearchResult } from './search-index.js';
|
|
2
|
-
export interface AlgoliaConfig {
|
|
3
|
-
appId: string;
|
|
4
|
-
apiKey: string;
|
|
5
|
-
indexName: string;
|
|
6
|
-
}
|
|
7
|
-
export declare function extractAlgoliaConfig(html: string): AlgoliaConfig | null;
|
|
8
|
-
export declare function algoliaQueryUrl(config: AlgoliaConfig): string;
|
|
9
|
-
export declare function parseAlgoliaResponse(body: string, indexUrl: string): DocsSearchResult[];
|
|
10
|
-
/**
|
|
11
|
-
* Parses a VitePress local search index (MiniSearch documents) and ranks docs by query term hits.
|
|
12
|
-
* `base` is the docs site origin/base used to resolve the relative `id` routes.
|
|
13
|
-
*/
|
|
14
|
-
export declare function searchVitePressLocalIndex(body: string, indexUrl: string, base: string, query: string): DocsSearchResult[];
|
|
@@ -1,101 +0,0 @@
|
|
|
1
|
-
// Algolia identifiers extracted from untrusted page HTML are interpolated into the request URL, so
|
|
2
|
-
// they MUST be format-validated to prevent SSRF (an unvalidated appId like "evil.com/x" would point
|
|
3
|
-
// the request at an attacker host). App IDs are uppercase alphanumeric; public search keys are hex;
|
|
4
|
-
// the index name must contain no slash (it is path-encoded but we still reject separators).
|
|
5
|
-
const ALGOLIA_APP_ID_RE = /^[A-Z0-9]{6,24}$/;
|
|
6
|
-
const ALGOLIA_API_KEY_RE = /^[a-f0-9]{16,64}$/;
|
|
7
|
-
const ALGOLIA_INDEX_RE = /^[A-Za-z0-9._~-]{1,128}$/;
|
|
8
|
-
// Pulls a DocSearch config out of embedded site data. Returns null unless all three fields are
|
|
9
|
-
// present AND well-formed — a partial or malformed signal is not enough to query.
|
|
10
|
-
export function extractAlgoliaConfig(html) {
|
|
11
|
-
const text = html.replace(/\\"/g, '"');
|
|
12
|
-
const appId = text.match(/"appId"\s*:\s*"([^"]+)"/)?.[1];
|
|
13
|
-
const apiKey = text.match(/"apiKey"\s*:\s*"([^"]+)"/)?.[1];
|
|
14
|
-
const indexName = text.match(/"indexName"\s*:\s*"([^"]+)"/)?.[1];
|
|
15
|
-
if (!appId || !apiKey || !indexName)
|
|
16
|
-
return null;
|
|
17
|
-
if (!ALGOLIA_APP_ID_RE.test(appId) ||
|
|
18
|
-
!ALGOLIA_API_KEY_RE.test(apiKey) ||
|
|
19
|
-
!ALGOLIA_INDEX_RE.test(indexName)) {
|
|
20
|
-
return null;
|
|
21
|
-
}
|
|
22
|
-
return { appId, apiKey, indexName };
|
|
23
|
-
}
|
|
24
|
-
export function algoliaQueryUrl(config) {
|
|
25
|
-
return `https://${config.appId}-dsn.algolia.net/1/indexes/${encodeURIComponent(config.indexName)}/query`;
|
|
26
|
-
}
|
|
27
|
-
// DocSearch records nest the page hierarchy under `hierarchy` and the page URL under `url`.
|
|
28
|
-
function algoliaTitle(hit) {
|
|
29
|
-
const h = hit?.hierarchy ?? {};
|
|
30
|
-
// Prefer the deepest (most specific) hierarchy level present.
|
|
31
|
-
for (const level of ['lvl6', 'lvl5', 'lvl4', 'lvl3', 'lvl2', 'lvl1', 'lvl0']) {
|
|
32
|
-
if (h[level])
|
|
33
|
-
return String(h[level]);
|
|
34
|
-
}
|
|
35
|
-
return hit?.url ?? 'Untitled';
|
|
36
|
-
}
|
|
37
|
-
export function parseAlgoliaResponse(body, indexUrl) {
|
|
38
|
-
const data = JSON.parse(body);
|
|
39
|
-
if (!Array.isArray(data.hits))
|
|
40
|
-
throw new Error('Algolia response missing "hits"');
|
|
41
|
-
return data.hits
|
|
42
|
-
.filter((hit) => typeof hit?.url === 'string')
|
|
43
|
-
.map((hit, i) => ({
|
|
44
|
-
title: algoliaTitle(hit),
|
|
45
|
-
url: hit.url,
|
|
46
|
-
snippet: hit?.content ?? undefined,
|
|
47
|
-
provider: 'algolia-docsearch',
|
|
48
|
-
score: data.hits.length - i, // preserve Algolia's relevance order
|
|
49
|
-
indexUrl,
|
|
50
|
-
}));
|
|
51
|
-
}
|
|
52
|
-
function vitepressUrl(base, id) {
|
|
53
|
-
try {
|
|
54
|
-
return new URL(id, base).toString();
|
|
55
|
-
}
|
|
56
|
-
catch {
|
|
57
|
-
return null;
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
// Ranks one VitePress doc by query term hits: a title match weighs 10×, body occurrences 1× each
|
|
61
|
-
// (capped at 5). Kept separate so the search loop stays a flat filter → score → format.
|
|
62
|
-
function scoreVitePressDoc(doc, terms) {
|
|
63
|
-
const haystack = `${doc.title ?? ''} ${(doc.titles ?? []).join(' ')} ${doc.text ?? ''}`.toLowerCase();
|
|
64
|
-
let score = 0;
|
|
65
|
-
for (const term of terms) {
|
|
66
|
-
if ((doc.title ?? '').toLowerCase().includes(term))
|
|
67
|
-
score += 10;
|
|
68
|
-
score += Math.min(haystack.split(term).length - 1, 5);
|
|
69
|
-
}
|
|
70
|
-
return score;
|
|
71
|
-
}
|
|
72
|
-
/**
|
|
73
|
-
* Parses a VitePress local search index (MiniSearch documents) and ranks docs by query term hits.
|
|
74
|
-
* `base` is the docs site origin/base used to resolve the relative `id` routes.
|
|
75
|
-
*/
|
|
76
|
-
export function searchVitePressLocalIndex(body, indexUrl, base, query) {
|
|
77
|
-
const docs = JSON.parse(body);
|
|
78
|
-
if (!Array.isArray(docs))
|
|
79
|
-
throw new Error('VitePress local index is not an array');
|
|
80
|
-
const terms = query.toLowerCase().split(/\s+/).filter(Boolean);
|
|
81
|
-
const results = [];
|
|
82
|
-
for (const doc of docs) {
|
|
83
|
-
if (!doc.id)
|
|
84
|
-
continue;
|
|
85
|
-
const score = scoreVitePressDoc(doc, terms);
|
|
86
|
-
if (score <= 0)
|
|
87
|
-
continue;
|
|
88
|
-
const url = vitepressUrl(base, doc.id);
|
|
89
|
-
if (!url)
|
|
90
|
-
continue;
|
|
91
|
-
results.push({
|
|
92
|
-
title: doc.title ?? doc.id,
|
|
93
|
-
url,
|
|
94
|
-
snippet: (doc.text ?? '').slice(0, 150),
|
|
95
|
-
provider: 'vitepress-local',
|
|
96
|
-
score,
|
|
97
|
-
indexUrl,
|
|
98
|
-
});
|
|
99
|
-
}
|
|
100
|
-
return results.sort((a, b) => b.score - a.score);
|
|
101
|
-
}
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
import type { SearchProvider } from './capabilities.js';
|
|
2
|
-
export interface DocsSearchResult {
|
|
3
|
-
title: string;
|
|
4
|
-
url: string;
|
|
5
|
-
anchor?: string;
|
|
6
|
-
snippet?: string;
|
|
7
|
-
provider: SearchProvider;
|
|
8
|
-
score: number;
|
|
9
|
-
indexUrl: string;
|
|
10
|
-
}
|
|
11
|
-
export declare function searchMkDocsIndex(body: string, indexUrl: string, query: string): DocsSearchResult[];
|
|
12
|
-
export declare function searchJekyllIndex(body: string, indexUrl: string, query: string): DocsSearchResult[];
|
|
13
|
-
export declare function searchSphinxIndex(js: string, indexUrl: string, query: string): DocsSearchResult[];
|
|
@@ -1,176 +0,0 @@
|
|
|
1
|
-
function queryTerms(query) {
|
|
2
|
-
return query.toLowerCase().split(/\s+/).filter(Boolean);
|
|
3
|
-
}
|
|
4
|
-
// Score = title term hits (heavy) + body term hits (light). Zero means no match (dropped).
|
|
5
|
-
function scoreText(title, body, terms) {
|
|
6
|
-
const t = title.toLowerCase();
|
|
7
|
-
const b = body.toLowerCase();
|
|
8
|
-
let score = 0;
|
|
9
|
-
for (const term of terms) {
|
|
10
|
-
if (t.includes(term))
|
|
11
|
-
score += 10;
|
|
12
|
-
const bodyHits = b.split(term).length - 1;
|
|
13
|
-
score += Math.min(bodyHits, 5);
|
|
14
|
-
}
|
|
15
|
-
return score;
|
|
16
|
-
}
|
|
17
|
-
function snippetFrom(text, terms) {
|
|
18
|
-
const lower = text.toLowerCase();
|
|
19
|
-
let idx = 0;
|
|
20
|
-
for (const term of terms) {
|
|
21
|
-
const found = lower.indexOf(term);
|
|
22
|
-
if (found !== -1) {
|
|
23
|
-
idx = found;
|
|
24
|
-
break;
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
const start = Math.max(0, idx - 40);
|
|
28
|
-
return text
|
|
29
|
-
.slice(start, start + 150)
|
|
30
|
-
.replace(/\s+/g, ' ')
|
|
31
|
-
.trim();
|
|
32
|
-
}
|
|
33
|
-
function splitAnchor(location) {
|
|
34
|
-
const hashIdx = location.indexOf('#');
|
|
35
|
-
if (hashIdx === -1)
|
|
36
|
-
return { path: location };
|
|
37
|
-
return { path: location.slice(0, hashIdx), anchor: location.slice(hashIdx + 1) || undefined };
|
|
38
|
-
}
|
|
39
|
-
function resolve(base, relative) {
|
|
40
|
-
try {
|
|
41
|
-
return new URL(relative, base).toString();
|
|
42
|
-
}
|
|
43
|
-
catch {
|
|
44
|
-
return null;
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
// MkDocs / Material for MkDocs: { docs: [ { location: "page/#anchor", title, text } ] }.
|
|
48
|
-
// Locations are relative to the site root (the index sits at <root>/search/search_index.json).
|
|
49
|
-
export function searchMkDocsIndex(body, indexUrl, query) {
|
|
50
|
-
const data = JSON.parse(body);
|
|
51
|
-
if (!Array.isArray(data.docs))
|
|
52
|
-
throw new Error('MkDocs index missing "docs" array');
|
|
53
|
-
const root = resolve(indexUrl, '../') ?? indexUrl;
|
|
54
|
-
const terms = queryTerms(query);
|
|
55
|
-
const results = [];
|
|
56
|
-
for (const doc of data.docs) {
|
|
57
|
-
if (!doc.location && doc.location !== '')
|
|
58
|
-
continue;
|
|
59
|
-
const { path, anchor } = splitAnchor(doc.location ?? '');
|
|
60
|
-
const url = resolve(root, path);
|
|
61
|
-
if (!url)
|
|
62
|
-
continue;
|
|
63
|
-
const score = scoreText(doc.title ?? '', doc.text ?? '', terms);
|
|
64
|
-
if (score <= 0)
|
|
65
|
-
continue;
|
|
66
|
-
results.push({
|
|
67
|
-
title: doc.title ?? path,
|
|
68
|
-
url,
|
|
69
|
-
anchor,
|
|
70
|
-
snippet: snippetFrom(doc.text ?? '', terms),
|
|
71
|
-
provider: 'mkdocs-local',
|
|
72
|
-
score,
|
|
73
|
-
indexUrl,
|
|
74
|
-
});
|
|
75
|
-
}
|
|
76
|
-
return results.sort((a, b) => b.score - a.score);
|
|
77
|
-
}
|
|
78
|
-
// Just the Docs: { "0": { title, content, url|relUrl }, "1": {...} } keyed by stringified ints.
|
|
79
|
-
export function searchJekyllIndex(body, indexUrl, query) {
|
|
80
|
-
const data = JSON.parse(body);
|
|
81
|
-
if (typeof data !== 'object' || data === null)
|
|
82
|
-
throw new Error('Jekyll index is not an object');
|
|
83
|
-
const terms = queryTerms(query);
|
|
84
|
-
const results = [];
|
|
85
|
-
for (const entry of Object.values(data)) {
|
|
86
|
-
const rel = entry.url ?? entry.relUrl;
|
|
87
|
-
if (!rel)
|
|
88
|
-
continue;
|
|
89
|
-
const { path, anchor } = splitAnchor(rel);
|
|
90
|
-
const url = resolve(indexUrl, path);
|
|
91
|
-
if (!url)
|
|
92
|
-
continue;
|
|
93
|
-
const score = scoreText(entry.title ?? '', entry.content ?? '', terms);
|
|
94
|
-
if (score <= 0)
|
|
95
|
-
continue;
|
|
96
|
-
results.push({
|
|
97
|
-
title: entry.title ?? path,
|
|
98
|
-
url,
|
|
99
|
-
anchor,
|
|
100
|
-
snippet: snippetFrom(entry.content ?? '', terms),
|
|
101
|
-
provider: 'jekyll-json',
|
|
102
|
-
score,
|
|
103
|
-
indexUrl,
|
|
104
|
-
});
|
|
105
|
-
}
|
|
106
|
-
return results.sort((a, b) => b.score - a.score);
|
|
107
|
-
}
|
|
108
|
-
// Extracts the object literal from `Search.setIndex({...})` as inert data and JSON.parses it —
|
|
109
|
-
// never eval. Anchors on the `setIndex(` call so leading comments/whitespace can't shift the slice;
|
|
110
|
-
// falls back to first-`{`/last-`}` for index variants that assign the object differently.
|
|
111
|
-
function parseSphinxObject(js) {
|
|
112
|
-
const call = js.indexOf('setIndex(');
|
|
113
|
-
const start = call !== -1 ? js.indexOf('{', call) : js.indexOf('{');
|
|
114
|
-
const end = js.lastIndexOf('}');
|
|
115
|
-
if (start === -1 || end <= start)
|
|
116
|
-
throw new Error('Sphinx index has no object literal');
|
|
117
|
-
return JSON.parse(js.slice(start, end + 1));
|
|
118
|
-
}
|
|
119
|
-
function collectDocIndices(map, terms) {
|
|
120
|
-
const hits = new Map();
|
|
121
|
-
if (!map)
|
|
122
|
-
return hits;
|
|
123
|
-
for (const [word, posting] of Object.entries(map)) {
|
|
124
|
-
const lower = word.toLowerCase();
|
|
125
|
-
if (!terms.some((t) => lower.includes(t)))
|
|
126
|
-
continue;
|
|
127
|
-
const docs = Array.isArray(posting) ? posting : [posting];
|
|
128
|
-
for (const d of docs)
|
|
129
|
-
hits.set(d, (hits.get(d) ?? 0) + 1);
|
|
130
|
-
}
|
|
131
|
-
return hits;
|
|
132
|
-
}
|
|
133
|
-
// Scores each Sphinx doc index by term hits: title-term hits and exact title matches weigh 10×,
|
|
134
|
-
// body-term hits 1×. Kept separate so the search function stays a flat parse → score → format.
|
|
135
|
-
function scoreSphinxDocs(index, terms, titles) {
|
|
136
|
-
const scores = new Map();
|
|
137
|
-
for (const [doc, count] of collectDocIndices(index.titleterms, terms)) {
|
|
138
|
-
scores.set(doc, (scores.get(doc) ?? 0) + count * 10);
|
|
139
|
-
}
|
|
140
|
-
for (const [doc, count] of collectDocIndices(index.terms, terms)) {
|
|
141
|
-
scores.set(doc, (scores.get(doc) ?? 0) + count);
|
|
142
|
-
}
|
|
143
|
-
for (let i = 0; i < titles.length; i++) {
|
|
144
|
-
if (terms.some((t) => (titles[i] ?? '').toLowerCase().includes(t))) {
|
|
145
|
-
scores.set(i, (scores.get(i) ?? 0) + 10);
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
return scores;
|
|
149
|
-
}
|
|
150
|
-
// Sphinx searchindex.js. Page URLs are docname + ".html" relative to the index's directory.
|
|
151
|
-
export function searchSphinxIndex(js, indexUrl, query) {
|
|
152
|
-
const index = parseSphinxObject(js);
|
|
153
|
-
const docnames = index.docnames ?? [];
|
|
154
|
-
const titles = index.titles ?? [];
|
|
155
|
-
if (docnames.length === 0)
|
|
156
|
-
throw new Error('Sphinx index has no docnames');
|
|
157
|
-
const terms = queryTerms(query);
|
|
158
|
-
const scores = scoreSphinxDocs(index, terms, titles);
|
|
159
|
-
const results = [];
|
|
160
|
-
for (const [doc, score] of scores) {
|
|
161
|
-
const name = docnames[doc];
|
|
162
|
-
if (!name || score <= 0)
|
|
163
|
-
continue;
|
|
164
|
-
const url = resolve(indexUrl, `${name}.html`);
|
|
165
|
-
if (!url)
|
|
166
|
-
continue;
|
|
167
|
-
results.push({
|
|
168
|
-
title: titles[doc] ?? name,
|
|
169
|
-
url,
|
|
170
|
-
provider: 'sphinx-searchindex',
|
|
171
|
-
score,
|
|
172
|
-
indexUrl,
|
|
173
|
-
});
|
|
174
|
-
}
|
|
175
|
-
return results.sort((a, b) => b.score - a.score);
|
|
176
|
-
}
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
import type { SiteSearchResult } from '../types.js';
|
|
2
|
-
/**
|
|
3
|
-
* Searches Salesforce Help via Coveo. Reuses a cached token for a direct API call when
|
|
4
|
-
* possible; otherwise drives a headless browser, reads the page's own Coveo search response,
|
|
5
|
-
* and caches the Coveo token (intercepted from the page's Aura call) for the next run.
|
|
6
|
-
*/
|
|
7
|
-
export declare function searchSalesforce(query: string, limit?: number): Promise<SiteSearchResult[]>;
|
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
import { openCdpPage, waitForLoad, ResponseCapture } from '../../lib/research/browser.js';
|
|
2
|
-
import { checkDnsSafety } from '../../lib/research/fetcher.js';
|
|
3
|
-
import { buildSearchPageUrl, extractCoveoResults, searchCoveoDirect, COVEO_SEARCH_PATH, AURA_PATH, } from './coveo.js';
|
|
4
|
-
import { loadToken, storeToken, isTokenValid, parseAuraToken } from './token.js';
|
|
5
|
-
const DEFAULT_LIMIT = 10;
|
|
6
|
-
const PAGE_LOAD_TIMEOUT_MS = 15_000;
|
|
7
|
-
const COVEO_WAIT_MS = 30_000;
|
|
8
|
-
const TOKEN_WAIT_MS = 5_000;
|
|
9
|
-
const DEBUG = Boolean(process.env.RESEARCH_DEBUG);
|
|
10
|
-
function dbg(message) {
|
|
11
|
-
if (DEBUG)
|
|
12
|
-
console.error(`[salesforce-search] ${message}`);
|
|
13
|
-
}
|
|
14
|
-
// Salesforce gates content behind a OneTrust cookie banner; dismissing it lets the page run.
|
|
15
|
-
const ACCEPT_CONSENT_EXPRESSION = "(() => { const b = document.querySelector('#onetrust-accept-btn-handler'); if (b) b.click(); })()";
|
|
16
|
-
/**
|
|
17
|
-
* Searches Salesforce Help via Coveo. Reuses a cached token for a direct API call when
|
|
18
|
-
* possible; otherwise drives a headless browser, reads the page's own Coveo search response,
|
|
19
|
-
* and caches the Coveo token (intercepted from the page's Aura call) for the next run.
|
|
20
|
-
*/
|
|
21
|
-
export async function searchSalesforce(query, limit = DEFAULT_LIMIT) {
|
|
22
|
-
const cached = loadToken();
|
|
23
|
-
if (cached && isTokenValid(cached.expiresAtMs)) {
|
|
24
|
-
dbg('using cached token (direct Coveo call)');
|
|
25
|
-
const results = extractCoveoResults(await searchCoveoDirect(cached, query, limit), limit);
|
|
26
|
-
if (results.length)
|
|
27
|
-
return results;
|
|
28
|
-
dbg('cached token returned no results; falling back to browser');
|
|
29
|
-
}
|
|
30
|
-
return searchViaBrowser(query, limit);
|
|
31
|
-
}
|
|
32
|
-
async function searchViaBrowser(query, limit) {
|
|
33
|
-
await checkDnsSafety('help.salesforce.com');
|
|
34
|
-
const page = await openCdpPage();
|
|
35
|
-
try {
|
|
36
|
-
// No asset blocking: the search SPA must fully boot to run its query and mint its token.
|
|
37
|
-
const capture = new ResponseCapture(page, [
|
|
38
|
-
{ key: 'coveo', test: ({ url }) => url.includes(COVEO_SEARCH_PATH) },
|
|
39
|
-
// help.salesforce.com runs on LWR (no global $A to mint a token ourselves), so we
|
|
40
|
-
// intercept the page's own Aura token response — the one whose body parses to a token.
|
|
41
|
-
{
|
|
42
|
-
key: 'token',
|
|
43
|
-
test: ({ url }) => url.includes(AURA_PATH),
|
|
44
|
-
accept: (body) => parseAuraToken(body) !== null,
|
|
45
|
-
},
|
|
46
|
-
]);
|
|
47
|
-
await page.client.send('Page.navigate', { url: buildSearchPageUrl(query) }, page.sessionId);
|
|
48
|
-
await waitForLoad(page.client, page.sessionId, PAGE_LOAD_TIMEOUT_MS).catch(() => { });
|
|
49
|
-
await page.client
|
|
50
|
-
.send('Runtime.evaluate', { expression: ACCEPT_CONSENT_EXPRESSION }, page.sessionId)
|
|
51
|
-
.catch(() => { });
|
|
52
|
-
const coveoBody = await capture.waitFor('coveo', COVEO_WAIT_MS);
|
|
53
|
-
await cacheToken(capture);
|
|
54
|
-
if (!coveoBody) {
|
|
55
|
-
dbg('no Coveo search response captured');
|
|
56
|
-
return [];
|
|
57
|
-
}
|
|
58
|
-
return extractCoveoResults(parseJson(coveoBody), limit);
|
|
59
|
-
}
|
|
60
|
-
finally {
|
|
61
|
-
await page.close();
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
async function cacheToken(capture) {
|
|
65
|
-
const tokenBody = await capture.waitFor('token', TOKEN_WAIT_MS);
|
|
66
|
-
const token = tokenBody ? parseAuraToken(tokenBody) : null;
|
|
67
|
-
if (token) {
|
|
68
|
-
storeToken(token);
|
|
69
|
-
dbg('cached Coveo token for next run');
|
|
70
|
-
}
|
|
71
|
-
else {
|
|
72
|
-
dbg('no Aura token response intercepted; next run will use the browser again');
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
function parseJson(text) {
|
|
76
|
-
try {
|
|
77
|
-
return JSON.parse(text);
|
|
78
|
-
}
|
|
79
|
-
catch {
|
|
80
|
-
return null;
|
|
81
|
-
}
|
|
82
|
-
}
|