@taurgis/bonsai 1.0.6 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -13
- package/agents/agents/official-docs-researcher.agent.md +15 -35
- package/agents/agents/salesforce-docs-researcher.agent.md +17 -44
- package/agents/hooks/README.md +2 -9
- package/agents/hooks/claude-code/hooks/bonsai-web-fetch.sh +1 -1
- package/agents/hooks/codex/hooks/bonsai-user-prompt-context.sh +1 -1
- package/agents/hooks/codex/hooks/bonsai-web-fetch.sh +1 -1
- package/agents/hooks/cursor/hooks/bonsai-web-fetch.sh +1 -1
- package/agents/hooks/github-copilot/bonsai-web-fetch.sh +1 -1
- package/agents/instructions/salesforce-research.instructions.md +7 -8
- package/agents/instructions/web-research.instructions.md +3 -4
- package/agents/skills/web-research/SKILL.md +24 -37
- package/bin/cli.mjs +15 -2
- package/dist/base-command.d.ts +15 -2
- package/dist/base-command.js +32 -4
- package/dist/commands/fetch.d.ts +3 -0
- package/dist/commands/fetch.js +96 -30
- package/dist/commands/import.js +14 -5
- package/dist/commands/inspect.d.ts +4 -0
- package/dist/commands/inspect.js +48 -24
- package/dist/commands/list.d.ts +3 -1
- package/dist/commands/list.js +42 -25
- package/dist/commands/prune.d.ts +1 -0
- package/dist/commands/prune.js +35 -9
- package/dist/commands/status.d.ts +2 -0
- package/dist/commands/status.js +57 -25
- package/dist/commands.d.ts +0 -2
- package/dist/commands.js +0 -2
- package/dist/hooks/command-not-found/suggest.d.ts +7 -1
- package/dist/hooks/command-not-found/suggest.js +82 -57
- package/dist/lib/argv.js +51 -7
- package/dist/lib/color.d.ts +20 -0
- package/dist/lib/color.js +36 -0
- package/dist/lib/envelope.js +8 -2
- package/dist/lib/help-preflight.d.ts +15 -0
- package/dist/lib/help-preflight.js +81 -0
- package/dist/lib/limit-flag.d.ts +1 -0
- package/dist/lib/limit-flag.js +24 -0
- package/dist/lib/research/{search-index.d.ts → artifact-index.d.ts} +5 -5
- package/dist/lib/research/{search-index.js → artifact-index.js} +10 -10
- package/dist/lib/research/browser.d.ts +1 -8
- package/dist/lib/research/browser.js +32 -46
- package/dist/lib/research/docs/markdown-source.js +2 -1
- package/dist/lib/research/extract.js +2 -1
- package/dist/lib/research/fetcher.js +92 -48
- package/dist/lib/research/prompt-injection.d.ts +1 -0
- package/dist/lib/research/prompt-injection.js +149 -0
- package/dist/lib/research/proxy.d.ts +35 -0
- package/dist/lib/research/proxy.js +107 -0
- package/dist/lib/research/regression.d.ts +18 -0
- package/dist/lib/research/regression.js +47 -0
- package/dist/lib/research/storage.d.ts +0 -1
- package/dist/lib/research/storage.js +15 -13
- package/dist/lib/research/url.d.ts +22 -0
- package/dist/lib/research/url.js +63 -1
- package/dist/lib/text.d.ts +3 -1
- package/dist/lib/text.js +9 -1
- package/dist/sites/coveo.d.ts +7 -0
- package/dist/sites/coveo.js +33 -0
- package/dist/sites/index.js +1 -18
- package/dist/sites/salesforce/fetch-page.js +1 -1
- package/dist/sites/salesforce/index.js +3 -8
- package/dist/sites/salesforce-developer/fetch-page.js +3 -1
- package/dist/sites/salesforce-developer/index.js +1 -2
- package/dist/sites/salesforce-doc-fetch.d.ts +11 -0
- package/dist/sites/salesforce-doc-fetch.js +117 -76
- package/dist/sites/salesforce-dom-probe.d.ts +16 -0
- package/dist/sites/salesforce-dom-probe.js +114 -0
- package/dist/sites/types.d.ts +0 -6
- package/oclif.manifest.json +31 -106
- package/package.json +35 -26
- package/dist/commands/index.d.ts +0 -35
- package/dist/commands/index.js +0 -306
- package/dist/commands/research/config/base.d.ts +0 -23
- package/dist/commands/research/config/base.js +0 -46
- package/dist/commands/research/config/get.d.ts +0 -20
- package/dist/commands/research/config/get.js +0 -45
- package/dist/commands/research/config/list.d.ts +0 -17
- package/dist/commands/research/config/list.js +0 -42
- package/dist/commands/research/config/set.d.ts +0 -31
- package/dist/commands/research/config/set.js +0 -94
- package/dist/commands/research/config/unset.d.ts +0 -21
- package/dist/commands/research/config/unset.js +0 -59
- package/dist/commands/research/import.d.ts +0 -38
- package/dist/commands/research/import.js +0 -313
- package/dist/commands/research/inspect.d.ts +0 -13
- package/dist/commands/research/inspect.js +0 -86
- package/dist/commands/research/list.d.ts +0 -25
- package/dist/commands/research/list.js +0 -143
- package/dist/commands/research/prune.d.ts +0 -24
- package/dist/commands/research/prune.js +0 -141
- package/dist/commands/research/search.d.ts +0 -32
- package/dist/commands/research/search.js +0 -334
- package/dist/commands/research/status.d.ts +0 -17
- package/dist/commands/research/status.js +0 -108
- package/dist/commands/research.d.ts +0 -36
- package/dist/commands/research.js +0 -304
- package/dist/commands/search.d.ts +0 -35
- package/dist/commands/search.js +0 -391
- package/dist/lib/research/docs/__fixtures__/sphinx-searchindex.d.ts +0 -1
- package/dist/lib/research/docs/__fixtures__/sphinx-searchindex.js +0 -2
- package/dist/lib/research/docs/discovery-index.d.ts +0 -14
- package/dist/lib/research/docs/discovery-index.js +0 -112
- package/dist/lib/research/docs/remote-search-runner.d.ts +0 -21
- package/dist/lib/research/docs/remote-search-runner.js +0 -114
- package/dist/lib/research/docs/remote-search.d.ts +0 -14
- package/dist/lib/research/docs/remote-search.js +0 -101
- package/dist/lib/research/docs/search-index.d.ts +0 -13
- package/dist/lib/research/docs/search-index.js +0 -176
- package/dist/sites/algolia.d.ts +0 -9
- package/dist/sites/algolia.js +0 -26
- package/dist/sites/auto.d.ts +0 -8
- package/dist/sites/auto.js +0 -30
- package/dist/sites/nextjs/index.d.ts +0 -2
- package/dist/sites/nextjs/index.js +0 -52
- package/dist/sites/react/index.d.ts +0 -2
- package/dist/sites/react/index.js +0 -17
- package/dist/sites/salesforce/browser-search.d.ts +0 -7
- package/dist/sites/salesforce/browser-search.js +0 -82
- package/dist/sites/salesforce/coveo.d.ts +0 -23
- package/dist/sites/salesforce/coveo.js +0 -161
- package/dist/sites/salesforce/token.d.ts +0 -26
- package/dist/sites/salesforce/token.js +0 -154
- package/dist/sites/tailwind/index.d.ts +0 -2
- package/dist/sites/tailwind/index.js +0 -17
- package/dist/sites/vue/index.d.ts +0 -2
- package/dist/sites/vue/index.js +0 -17
|
@@ -1,141 +0,0 @@
|
|
|
1
|
-
import { Flags } from '@oclif/core';
|
|
2
|
-
import { unlinkSync } from 'node:fs';
|
|
3
|
-
import { join } from 'node:path';
|
|
4
|
-
import { BaseCommand } from '../../base-command.js';
|
|
5
|
-
import { scanCacheDir } from '../../lib/research/storage.js';
|
|
6
|
-
import { loadStoreRoots } from '../../lib/research/store-roots.js';
|
|
7
|
-
import { parseTtlToMs } from '../../lib/research/freshness.js';
|
|
8
|
-
export default class ResearchPrune extends BaseCommand {
|
|
9
|
-
static id = 'research prune';
|
|
10
|
-
static summary = 'Clean up old or inactive research cache entries.';
|
|
11
|
-
static description = 'Prunes cached research entries based on age, inactivity, or artifact type to free up disk space.';
|
|
12
|
-
static examples = [
|
|
13
|
-
{
|
|
14
|
-
description: 'Perform a dry run of pruning entries older than 90 days',
|
|
15
|
-
command: '<%= config.bin %> research prune --older-than 90d --dry-run',
|
|
16
|
-
},
|
|
17
|
-
{
|
|
18
|
-
description: 'Actually prune entries older than 30 days that are source scrapes',
|
|
19
|
-
command: '<%= config.bin %> research prune --older-than 30d --artifact-type source --yes',
|
|
20
|
-
},
|
|
21
|
-
];
|
|
22
|
-
static flags = {
|
|
23
|
-
'older-than': Flags.string({
|
|
24
|
-
description: 'Prune entries older than duration (e.g. "30d", "90d").',
|
|
25
|
-
}),
|
|
26
|
-
inactive: Flags.string({
|
|
27
|
-
description: 'Prune entries inactive (unvalidated/unfetched) for duration (e.g. "14d", "30d").',
|
|
28
|
-
}),
|
|
29
|
-
'artifact-type': Flags.option({
|
|
30
|
-
description: 'Filter pruning to specific artifact type.',
|
|
31
|
-
options: ['source', 'research_note'],
|
|
32
|
-
})(),
|
|
33
|
-
'dry-run': Flags.boolean({
|
|
34
|
-
description: 'List files that would be deleted without actually deleting them.',
|
|
35
|
-
default: false,
|
|
36
|
-
}),
|
|
37
|
-
yes: Flags.boolean({
|
|
38
|
-
char: 'y',
|
|
39
|
-
description: 'Skip confirmation prompt and prune files.',
|
|
40
|
-
default: false,
|
|
41
|
-
}),
|
|
42
|
-
};
|
|
43
|
-
static stdoutIsPrimaryData = true;
|
|
44
|
-
async init() {
|
|
45
|
-
await super.init();
|
|
46
|
-
const { flags } = await this.parse(ResearchPrune);
|
|
47
|
-
this.flags = flags;
|
|
48
|
-
}
|
|
49
|
-
validatePruneFlags() {
|
|
50
|
-
if (!this.flags['older-than'] && !this.flags.inactive && !this.flags['artifact-type']) {
|
|
51
|
-
this.error('Must specify at least one pruning filter: --older-than, --inactive, or --artifact-type.', { exit: 2 });
|
|
52
|
-
}
|
|
53
|
-
if (!this.flags['dry-run'] && !this.flags.yes) {
|
|
54
|
-
this.error('Safety check: Please specify --yes to confirm pruning, or --dry-run to list files that would be deleted.', { exit: 2 });
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
shouldPrune(meta, currentTime) {
|
|
58
|
-
const fetched = meta.fetched_at ? new Date(meta.fetched_at).getTime() : 0;
|
|
59
|
-
const validated = meta.validated_at ? new Date(meta.validated_at).getTime() : 0;
|
|
60
|
-
const baseTime = Math.max(fetched, validated);
|
|
61
|
-
const ageMs = currentTime.getTime() - baseTime;
|
|
62
|
-
if (this.flags['artifact-type'] && meta.artifact_type !== this.flags['artifact-type']) {
|
|
63
|
-
return false;
|
|
64
|
-
}
|
|
65
|
-
if (this.flags['older-than']) {
|
|
66
|
-
const olderThanMs = parseTtlToMs(this.flags['older-than']);
|
|
67
|
-
if (ageMs < olderThanMs) {
|
|
68
|
-
return false;
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
if (this.flags.inactive) {
|
|
72
|
-
const inactiveMs = parseTtlToMs(this.flags.inactive);
|
|
73
|
-
if (ageMs < inactiveMs) {
|
|
74
|
-
return false;
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
return true;
|
|
78
|
-
}
|
|
79
|
-
// Prune across every read root (project + global). No cross-root dedup: each file is a distinct
|
|
80
|
-
// deletion target, so a cache key present in both locations is pruned in both.
|
|
81
|
-
findPruneCandidates(readRoots, currentTime) {
|
|
82
|
-
return readRoots.flatMap((dataDir) => scanCacheDir(join(dataDir, 'research'), (artifact, filePath) => {
|
|
83
|
-
if (!this.shouldPrune(artifact.metadata, currentTime))
|
|
84
|
-
return null;
|
|
85
|
-
return {
|
|
86
|
-
cacheKey: artifact.metadata.cache_key,
|
|
87
|
-
path: filePath,
|
|
88
|
-
topic: artifact.metadata.topic,
|
|
89
|
-
url: artifact.metadata.source_url,
|
|
90
|
-
};
|
|
91
|
-
}));
|
|
92
|
-
}
|
|
93
|
-
deletePruneCandidates(filesToPrune) {
|
|
94
|
-
let prunedCount = 0;
|
|
95
|
-
for (const f of filesToPrune) {
|
|
96
|
-
try {
|
|
97
|
-
unlinkSync(f.path);
|
|
98
|
-
prunedCount++;
|
|
99
|
-
}
|
|
100
|
-
catch (err) {
|
|
101
|
-
this.warn(`Failed to delete cache file ${f.path}: ${err.message}`);
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
return prunedCount;
|
|
105
|
-
}
|
|
106
|
-
async execute() {
|
|
107
|
-
this.validatePruneFlags();
|
|
108
|
-
const roots = loadStoreRoots({
|
|
109
|
-
configDir: this.config.configDir,
|
|
110
|
-
cwd: process.cwd(),
|
|
111
|
-
dataDir: this.config.dataDir,
|
|
112
|
-
});
|
|
113
|
-
const currentTime = new Date();
|
|
114
|
-
const filesToPrune = this.findPruneCandidates(roots.readRoots, currentTime);
|
|
115
|
-
const dryRun = this.flags['dry-run'];
|
|
116
|
-
const count = filesToPrune.length;
|
|
117
|
-
if (dryRun) {
|
|
118
|
-
if (!this.requestedJson()) {
|
|
119
|
-
this.log(`[Dry Run] Found ${count} research cache entries that would be pruned:\n`);
|
|
120
|
-
filesToPrune.forEach((f) => {
|
|
121
|
-
this.log(`- [${f.topic || 'No Topic'}] Key: ${f.cacheKey} (${f.url || 'Imported note'})`);
|
|
122
|
-
});
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
|
-
else {
|
|
126
|
-
const prunedCount = this.deletePruneCandidates(filesToPrune);
|
|
127
|
-
if (!this.requestedJson()) {
|
|
128
|
-
this.log(`Successfully pruned ${prunedCount} of ${count} research cache entries.`);
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
return {
|
|
132
|
-
dryRun,
|
|
133
|
-
prunedCount: dryRun ? 0 : filesToPrune.length,
|
|
134
|
-
candidateCount: filesToPrune.length,
|
|
135
|
-
files: filesToPrune.map((f) => ({
|
|
136
|
-
cacheKey: f.cacheKey,
|
|
137
|
-
path: f.path,
|
|
138
|
-
})),
|
|
139
|
-
};
|
|
140
|
-
}
|
|
141
|
-
}
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
import { BaseCommand } from '../../base-command.js';
|
|
2
|
-
export default class ResearchSearch extends BaseCommand<typeof ResearchSearch> {
|
|
3
|
-
static id: string;
|
|
4
|
-
static summary: string;
|
|
5
|
-
static description: string;
|
|
6
|
-
static args: {
|
|
7
|
-
query: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
|
|
8
|
-
};
|
|
9
|
-
static flags: {
|
|
10
|
-
topic: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
11
|
-
tags: import("@oclif/core/interfaces").OptionFlag<string[] | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
12
|
-
'artifact-type': import("@oclif/core/interfaces").OptionFlag<"source" | "research_note" | "index" | "section" | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
13
|
-
limit: import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
|
|
14
|
-
'include-stale': import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
15
|
-
domain: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
16
|
-
remote: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
17
|
-
};
|
|
18
|
-
static stdoutIsPrimaryData: boolean;
|
|
19
|
-
init(): Promise<void>;
|
|
20
|
-
private validateSearchFlags;
|
|
21
|
-
private getSearchQueryTerms;
|
|
22
|
-
private matchesFilters;
|
|
23
|
-
private phraseMatchBonus;
|
|
24
|
-
private calculateScore;
|
|
25
|
-
private makeSnippet;
|
|
26
|
-
private scanCacheDirForResults;
|
|
27
|
-
private logSearchResults;
|
|
28
|
-
private printResults;
|
|
29
|
-
private executeSiteSearch;
|
|
30
|
-
private executeRemoteSearch;
|
|
31
|
-
execute(): Promise<unknown>;
|
|
32
|
-
}
|
|
@@ -1,334 +0,0 @@
|
|
|
1
|
-
import { Args, Flags } from '@oclif/core';
|
|
2
|
-
import { BaseCommand } from '../../base-command.js';
|
|
3
|
-
import { scanCacheDirs } from '../../lib/research/storage.js';
|
|
4
|
-
import { loadStoreRoots } from '../../lib/research/store-roots.js';
|
|
5
|
-
import { evaluateFreshness } from '../../lib/research/freshness.js';
|
|
6
|
-
import { detectSite } from '../../sites/index.js';
|
|
7
|
-
import { fetchStaticHtml, fetchText, postJson } from '../../lib/research/fetcher.js';
|
|
8
|
-
import { runRemoteDocsSearch, } from '../../lib/research/docs/remote-search-runner.js';
|
|
9
|
-
const REMOTE_SEARCH_DEPS = {
|
|
10
|
-
fetchStatic: async (url) => {
|
|
11
|
-
const res = await fetchStaticHtml(url);
|
|
12
|
-
return { content: res.content, finalUrl: res.finalUrl };
|
|
13
|
-
},
|
|
14
|
-
fetchText: async (url) => {
|
|
15
|
-
const res = await fetchText(url);
|
|
16
|
-
return { content: res.content, status: res.status };
|
|
17
|
-
},
|
|
18
|
-
postJson: (url, body, headers) => postJson(url, body, headers),
|
|
19
|
-
};
|
|
20
|
-
const FRESHNESS_BONUS = { fresh: 30, stale_grace: 10 };
|
|
21
|
-
export default class ResearchSearch extends BaseCommand {
|
|
22
|
-
static id = 'research search';
|
|
23
|
-
static summary = 'Search locally cached research artifacts by keywords.';
|
|
24
|
-
static description = 'Scans the local research cache database and ranks matching entries based on topic, tags, source URLs, summary, and content.';
|
|
25
|
-
static args = {
|
|
26
|
-
query: Args.string({
|
|
27
|
-
required: true,
|
|
28
|
-
description: 'The search query containing terms to match.',
|
|
29
|
-
}),
|
|
30
|
-
};
|
|
31
|
-
static flags = {
|
|
32
|
-
topic: Flags.string({
|
|
33
|
-
char: 't',
|
|
34
|
-
description: 'Filter results by exact topic (case-insensitive).',
|
|
35
|
-
}),
|
|
36
|
-
tags: Flags.string({
|
|
37
|
-
char: 'g',
|
|
38
|
-
description: 'Filter results by tags (must match all tags specified).',
|
|
39
|
-
multiple: true,
|
|
40
|
-
}),
|
|
41
|
-
'artifact-type': Flags.option({
|
|
42
|
-
description: 'Filter results by artifact type.',
|
|
43
|
-
options: ['source', 'research_note', 'index', 'section'],
|
|
44
|
-
})(),
|
|
45
|
-
limit: Flags.integer({
|
|
46
|
-
description: 'Maximum number of results to return (default 10, max 50).',
|
|
47
|
-
default: 10,
|
|
48
|
-
}),
|
|
49
|
-
'include-stale': Flags.boolean({
|
|
50
|
-
description: 'Include stale expired cache entries in the search results.',
|
|
51
|
-
default: false,
|
|
52
|
-
}),
|
|
53
|
-
domain: Flags.string({
|
|
54
|
-
description: 'Search a documentation site directly (e.g. help.salesforce.com) instead of the local cache. Requires a site module that implements search.',
|
|
55
|
-
}),
|
|
56
|
-
remote: Flags.string({
|
|
57
|
-
description: 'Discover uncached docs via a site’s public search (Algolia DocSearch, MkDocs/Sphinx/Just-the-Docs index). Pass a docs page URL; falls back to local cache search if no connector applies.',
|
|
58
|
-
}),
|
|
59
|
-
};
|
|
60
|
-
static stdoutIsPrimaryData = true;
|
|
61
|
-
async init() {
|
|
62
|
-
await super.init();
|
|
63
|
-
const { args, flags } = await this.parse(ResearchSearch);
|
|
64
|
-
this.args = args;
|
|
65
|
-
this.flags = flags;
|
|
66
|
-
}
|
|
67
|
-
validateSearchFlags() {
|
|
68
|
-
const limit = this.flags.limit;
|
|
69
|
-
if (limit !== undefined && (limit < 1 || limit > 50)) {
|
|
70
|
-
this.error('Limit must be between 1 and 50.', { exit: 2 });
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
getSearchQueryTerms(query) {
|
|
74
|
-
const terms = query.toLowerCase().split(/\s+/).filter(Boolean);
|
|
75
|
-
if (terms.length === 0) {
|
|
76
|
-
this.error('Query string cannot be empty.', { exit: 2 });
|
|
77
|
-
}
|
|
78
|
-
return terms;
|
|
79
|
-
}
|
|
80
|
-
matchesFilters(meta, freshness, topicFlag, tagsFlag, artTypeFlag, includeStale) {
|
|
81
|
-
if (topicFlag &&
|
|
82
|
-
(!meta.topic || meta.topic.trim().toLowerCase() !== topicFlag.trim().toLowerCase())) {
|
|
83
|
-
return false;
|
|
84
|
-
}
|
|
85
|
-
if (tagsFlag && tagsFlag.length > 0) {
|
|
86
|
-
const metaTagsLower = (meta.tags || []).map((t) => t.toLowerCase());
|
|
87
|
-
const hasAllTags = tagsFlag.every((t) => metaTagsLower.includes(t.toLowerCase()));
|
|
88
|
-
if (!hasAllTags) {
|
|
89
|
-
return false;
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
if (artTypeFlag && meta.artifact_type !== artTypeFlag) {
|
|
93
|
-
return false;
|
|
94
|
-
}
|
|
95
|
-
if (!includeStale && freshness === 'stale_expired') {
|
|
96
|
-
return false;
|
|
97
|
-
}
|
|
98
|
-
return true;
|
|
99
|
-
}
|
|
100
|
-
phraseMatchBonus(topic, tags, summ, comp, queryTerms) {
|
|
101
|
-
if (queryTerms.length <= 1)
|
|
102
|
-
return 0;
|
|
103
|
-
const fullQuery = queryTerms.join(' ');
|
|
104
|
-
let bonus = 0;
|
|
105
|
-
if (topic === fullQuery)
|
|
106
|
-
bonus += 200;
|
|
107
|
-
if (topic.includes(fullQuery))
|
|
108
|
-
bonus += 100;
|
|
109
|
-
if (tags.includes(fullQuery))
|
|
110
|
-
bonus += 150;
|
|
111
|
-
if (summ.includes(fullQuery))
|
|
112
|
-
bonus += 50 + Math.min(summ.split(fullQuery).length - 1, 5) * 10;
|
|
113
|
-
if (comp.includes(fullQuery))
|
|
114
|
-
bonus += 20 + Math.min(comp.split(fullQuery).length - 1, 10) * 5;
|
|
115
|
-
return bonus;
|
|
116
|
-
}
|
|
117
|
-
calculateScore(meta, summary, compressed, queryTerms, freshness) {
|
|
118
|
-
const topic = (meta.topic || '').toLowerCase();
|
|
119
|
-
const tags = (meta.tags || []).map((t) => t.toLowerCase());
|
|
120
|
-
const sourceUrl = (meta.source_url || '').toLowerCase();
|
|
121
|
-
const sourceUrls = (meta.source_urls || []).map((u) => u.toLowerCase());
|
|
122
|
-
const summ = summary.toLowerCase();
|
|
123
|
-
const comp = compressed.toLowerCase();
|
|
124
|
-
let score = 0;
|
|
125
|
-
let matchedAny = false;
|
|
126
|
-
for (const term of queryTerms) {
|
|
127
|
-
const termScore = scoreSingleTerm(term, topic, tags, sourceUrl, sourceUrls, summ, comp);
|
|
128
|
-
if (termScore > 0) {
|
|
129
|
-
score += termScore;
|
|
130
|
-
matchedAny = true;
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
if (!matchedAny)
|
|
134
|
-
return 0;
|
|
135
|
-
score += this.phraseMatchBonus(topic, tags, summ, comp, queryTerms);
|
|
136
|
-
score += FRESHNESS_BONUS[freshness] ?? 0;
|
|
137
|
-
return score;
|
|
138
|
-
}
|
|
139
|
-
makeSnippet(text, queryTerms) {
|
|
140
|
-
const lower = text.toLowerCase();
|
|
141
|
-
let bestIdx = 0;
|
|
142
|
-
for (const term of queryTerms) {
|
|
143
|
-
const idx = lower.indexOf(term);
|
|
144
|
-
if (idx !== -1) {
|
|
145
|
-
bestIdx = idx;
|
|
146
|
-
break;
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
const start = Math.max(0, bestIdx - 40);
|
|
150
|
-
const end = Math.min(text.length, bestIdx + 110);
|
|
151
|
-
let snippet = text.slice(start, end).replace(/\s+/g, ' ').trim();
|
|
152
|
-
if (start > 0)
|
|
153
|
-
snippet = '...' + snippet;
|
|
154
|
-
if (end < text.length)
|
|
155
|
-
snippet = snippet + '...';
|
|
156
|
-
return snippet;
|
|
157
|
-
}
|
|
158
|
-
scanCacheDirForResults(readRoots, queryTerms, currentTime) {
|
|
159
|
-
return scanCacheDirs(readRoots, (artifact, filePath) => {
|
|
160
|
-
if (artifact.metadata.status !== 'active')
|
|
161
|
-
return null;
|
|
162
|
-
const freshness = evaluateFreshness(artifact.metadata, currentTime, null);
|
|
163
|
-
if (!this.matchesFilters(artifact.metadata, freshness, this.flags.topic, this.flags.tags, this.flags['artifact-type'], Boolean(this.flags['include-stale'])))
|
|
164
|
-
return null;
|
|
165
|
-
let score = this.calculateScore(artifact.metadata, artifact.summary, artifact.compressed, queryTerms, freshness);
|
|
166
|
-
if (score <= 0)
|
|
167
|
-
return null;
|
|
168
|
-
// Section children are more precise than the whole page, so rank a section hit slightly
|
|
169
|
-
// above its parent for the same query (T-22).
|
|
170
|
-
if (artifact.metadata.artifact_type === 'section')
|
|
171
|
-
score += 15;
|
|
172
|
-
const snippetText = [artifact.summary, artifact.compressed].filter(Boolean).join('\n');
|
|
173
|
-
return {
|
|
174
|
-
cacheKey: artifact.metadata.cache_key,
|
|
175
|
-
path: filePath,
|
|
176
|
-
artifactType: artifact.metadata.artifact_type,
|
|
177
|
-
sourceUrls: artifact.metadata.source_urls,
|
|
178
|
-
topic: artifact.metadata.topic,
|
|
179
|
-
tags: artifact.metadata.tags,
|
|
180
|
-
freshness,
|
|
181
|
-
captureMethod: artifact.metadata.capture_method,
|
|
182
|
-
tokenEstimate: artifact.metadata.token_estimate,
|
|
183
|
-
snippet: this.makeSnippet(snippetText, queryTerms),
|
|
184
|
-
siteModuleId: artifact.metadata.site_module_id,
|
|
185
|
-
score,
|
|
186
|
-
};
|
|
187
|
-
});
|
|
188
|
-
}
|
|
189
|
-
logSearchResults(finalResults) {
|
|
190
|
-
if (this.requestedJson())
|
|
191
|
-
return;
|
|
192
|
-
this.log(`Found ${finalResults.length} matching cached research entries:\n`);
|
|
193
|
-
finalResults.forEach((res, index) => {
|
|
194
|
-
this.log(`${index + 1}. [${res.topic || 'No Topic'}] Score: ${res.score}`);
|
|
195
|
-
this.log(` Cache Key: ${res.cacheKey}`);
|
|
196
|
-
this.log(` Snippet: ${res.snippet}`);
|
|
197
|
-
this.log(` Source URLs: ${res.sourceUrls.join(', ')}\n`);
|
|
198
|
-
});
|
|
199
|
-
}
|
|
200
|
-
// Prints a numbered result list to stdout for human (non-JSON) output. Shared by site and remote
|
|
201
|
-
// search so the two paths can't drift in formatting.
|
|
202
|
-
printResults(heading, results) {
|
|
203
|
-
if (this.requestedJson())
|
|
204
|
-
return;
|
|
205
|
-
this.log(heading);
|
|
206
|
-
results.forEach((r, i) => {
|
|
207
|
-
this.log(`${i + 1}. ${r.title}`);
|
|
208
|
-
this.log(` URL: ${r.url}`);
|
|
209
|
-
if (r.snippet)
|
|
210
|
-
this.log(` ${r.snippet}`);
|
|
211
|
-
});
|
|
212
|
-
}
|
|
213
|
-
async executeSiteSearch(query, domain) {
|
|
214
|
-
const siteModule = detectSite(`https://${domain}`);
|
|
215
|
-
if (!siteModule) {
|
|
216
|
-
this.error(`No site module registered for domain: ${domain}`, { exit: 2 });
|
|
217
|
-
}
|
|
218
|
-
if (!siteModule.search) {
|
|
219
|
-
this.error(`Site module '${siteModule.id}' does not implement search.`, { exit: 2 });
|
|
220
|
-
}
|
|
221
|
-
const results = await siteModule.search(query);
|
|
222
|
-
this.printResults(`Found ${results.length} results from ${siteModule.name}:\n`, results);
|
|
223
|
-
return results.map((r) => ({ ...r, site_module_id: siteModule.id }));
|
|
224
|
-
}
|
|
225
|
-
// Remote docs discovery. On any connector failure, degrade to local cache search with a warning
|
|
226
|
-
// (T-20). Returns discovery results tagged remote: true so callers can tell them from cache hits.
|
|
227
|
-
async executeRemoteSearch(query, docsUrl) {
|
|
228
|
-
try {
|
|
229
|
-
const { provider, results } = await runRemoteDocsSearch(docsUrl, query, REMOTE_SEARCH_DEPS);
|
|
230
|
-
this.printResults(`Found ${results.length} remote results via ${provider}:\n`, results);
|
|
231
|
-
return results.map((r) => ({ ...r, remote: true }));
|
|
232
|
-
}
|
|
233
|
-
catch (err) {
|
|
234
|
-
this.warn(`Remote docs search unavailable, using local cache: ${err.message}`);
|
|
235
|
-
return undefined;
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
async execute() {
|
|
239
|
-
const { query } = this.args;
|
|
240
|
-
const { domain, remote } = this.flags;
|
|
241
|
-
if (domain) {
|
|
242
|
-
return this.executeSiteSearch(query, domain);
|
|
243
|
-
}
|
|
244
|
-
if (remote) {
|
|
245
|
-
const remoteResults = await this.executeRemoteSearch(query, remote);
|
|
246
|
-
if (remoteResults !== undefined)
|
|
247
|
-
return remoteResults;
|
|
248
|
-
// else: fall through to local cache search
|
|
249
|
-
}
|
|
250
|
-
this.validateSearchFlags();
|
|
251
|
-
const queryTerms = this.getSearchQueryTerms(query);
|
|
252
|
-
const roots = loadStoreRoots({
|
|
253
|
-
configDir: this.config.configDir,
|
|
254
|
-
cwd: process.cwd(),
|
|
255
|
-
dataDir: this.config.dataDir,
|
|
256
|
-
});
|
|
257
|
-
const currentTime = new Date();
|
|
258
|
-
const results = this.scanCacheDirForResults(roots.readRoots, queryTerms, currentTime);
|
|
259
|
-
results.sort((a, b) => b.score - a.score);
|
|
260
|
-
const finalResults = results.slice(0, this.flags.limit);
|
|
261
|
-
this.logSearchResults(finalResults);
|
|
262
|
-
return finalResults;
|
|
263
|
-
}
|
|
264
|
-
}
|
|
265
|
-
function scoreSingleTerm(term, topic, tags, sourceUrl, sourceUrls, summ, comp) {
|
|
266
|
-
let termScore = 0;
|
|
267
|
-
let termMatched = false;
|
|
268
|
-
// 1. Topic Match
|
|
269
|
-
if (topic === term) {
|
|
270
|
-
termScore += 100;
|
|
271
|
-
termMatched = true;
|
|
272
|
-
}
|
|
273
|
-
else if (topic.includes(term)) {
|
|
274
|
-
termScore += 60;
|
|
275
|
-
termMatched = true;
|
|
276
|
-
}
|
|
277
|
-
else if (isFuzzyMatch(term, topic)) {
|
|
278
|
-
termScore += 40;
|
|
279
|
-
termMatched = true;
|
|
280
|
-
}
|
|
281
|
-
// 2. Tags Match
|
|
282
|
-
if (tags.includes(term)) {
|
|
283
|
-
termScore += 80;
|
|
284
|
-
termMatched = true;
|
|
285
|
-
}
|
|
286
|
-
else if (tags.some((t) => isFuzzyMatch(term, t) || t.includes(term))) {
|
|
287
|
-
termScore += 30;
|
|
288
|
-
termMatched = true;
|
|
289
|
-
}
|
|
290
|
-
// 3. Source URL Match
|
|
291
|
-
if (sourceUrl.includes(term) || sourceUrls.some((u) => u.includes(term))) {
|
|
292
|
-
termScore += 50;
|
|
293
|
-
termMatched = true;
|
|
294
|
-
}
|
|
295
|
-
// 4. Summary Term Frequency Match
|
|
296
|
-
const summMatches = summ.split(term).length - 1;
|
|
297
|
-
if (summMatches > 0) {
|
|
298
|
-
termScore += 20 + Math.min(summMatches, 5) * 5;
|
|
299
|
-
termMatched = true;
|
|
300
|
-
}
|
|
301
|
-
// 5. Compressed Content Term Frequency Match
|
|
302
|
-
const compMatches = comp.split(term).length - 1;
|
|
303
|
-
if (compMatches > 0) {
|
|
304
|
-
termScore += 5 + Math.min(compMatches, 10) * 2;
|
|
305
|
-
termMatched = true;
|
|
306
|
-
}
|
|
307
|
-
if (termMatched) {
|
|
308
|
-
termScore += 10;
|
|
309
|
-
}
|
|
310
|
-
return termScore;
|
|
311
|
-
}
|
|
312
|
-
function levenshtein(s1, s2) {
|
|
313
|
-
if (s1.length < s2.length)
|
|
314
|
-
return levenshtein(s2, s1);
|
|
315
|
-
if (s2.length === 0)
|
|
316
|
-
return s1.length;
|
|
317
|
-
let prevRow = Array.from({ length: s2.length + 1 }, (_, i) => i);
|
|
318
|
-
for (let i = 0; i < s1.length; i++) {
|
|
319
|
-
const currRow = [i + 1];
|
|
320
|
-
for (let j = 0; j < s2.length; j++) {
|
|
321
|
-
const deletions = prevRow[j + 1] + 1;
|
|
322
|
-
const insertions = currRow[j] + 1;
|
|
323
|
-
const substitutions = prevRow[j] + (s1[i] === s2[j] ? 0 : 1);
|
|
324
|
-
currRow.push(Math.min(deletions, insertions, substitutions));
|
|
325
|
-
}
|
|
326
|
-
prevRow = currRow;
|
|
327
|
-
}
|
|
328
|
-
return prevRow[s2.length];
|
|
329
|
-
}
|
|
330
|
-
function isFuzzyMatch(term, target) {
|
|
331
|
-
if (term.length < 4 || target.length < 4)
|
|
332
|
-
return false;
|
|
333
|
-
return levenshtein(term, target) <= 2;
|
|
334
|
-
}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import { BaseCommand } from '../../base-command.js';
|
|
2
|
-
export default class ResearchStatus extends BaseCommand<typeof ResearchStatus> {
|
|
3
|
-
static id: string;
|
|
4
|
-
static summary: string;
|
|
5
|
-
static description: string;
|
|
6
|
-
static args: {
|
|
7
|
-
url: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
|
|
8
|
-
};
|
|
9
|
-
static flags: {
|
|
10
|
-
tier: import("@oclif/core/interfaces").OptionFlag<"stable" | "standard" | "volatile", import("@oclif/core/interfaces").CustomOptions>;
|
|
11
|
-
ttl: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
12
|
-
'max-age': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
13
|
-
};
|
|
14
|
-
static stdoutIsPrimaryData: boolean;
|
|
15
|
-
init(): Promise<void>;
|
|
16
|
-
execute(): Promise<unknown>;
|
|
17
|
-
}
|
|
@@ -1,108 +0,0 @@
|
|
|
1
|
-
import { Args, Flags } from '@oclif/core';
|
|
2
|
-
import { BaseCommand } from '../../base-command.js';
|
|
3
|
-
import { normalizeUrl } from '../../lib/research/url.js';
|
|
4
|
-
import { deriveCacheKey } from '../../lib/research/cache-key.js';
|
|
5
|
-
import { locateArtifact, getArtifactPath } from '../../lib/research/storage.js';
|
|
6
|
-
import { loadStoreRoots } from '../../lib/research/store-roots.js';
|
|
7
|
-
import { evaluateFreshness, checkMaxAgeExpired } from '../../lib/research/freshness.js';
|
|
8
|
-
export default class ResearchStatus extends BaseCommand {
|
|
9
|
-
static id = 'research status';
|
|
10
|
-
static summary = 'Check the cache status of a URL without fetching or writing.';
|
|
11
|
-
static description = 'Reports whether the URL is cached, its freshness state, and what action (fetch, revalidate, or cached return) the research command would take.';
|
|
12
|
-
static args = {
|
|
13
|
-
url: Args.string({
|
|
14
|
-
required: true,
|
|
15
|
-
description: 'The URL to check.',
|
|
16
|
-
}),
|
|
17
|
-
};
|
|
18
|
-
static flags = {
|
|
19
|
-
tier: Flags.option({
|
|
20
|
-
description: 'Freshness tier policy to evaluate against.',
|
|
21
|
-
options: ['stable', 'standard', 'volatile'],
|
|
22
|
-
default: 'standard',
|
|
23
|
-
})(),
|
|
24
|
-
ttl: Flags.string({
|
|
25
|
-
char: 'l',
|
|
26
|
-
description: 'Custom TTL duration to evaluate against (e.g. "2h", "7d").',
|
|
27
|
-
}),
|
|
28
|
-
'max-age': Flags.string({
|
|
29
|
-
description: 'Maximum age of cache to accept (e.g. "1d", "30d").',
|
|
30
|
-
}),
|
|
31
|
-
};
|
|
32
|
-
static stdoutIsPrimaryData = true;
|
|
33
|
-
async init() {
|
|
34
|
-
await super.init();
|
|
35
|
-
const { args, flags } = await this.parse(ResearchStatus);
|
|
36
|
-
this.args = args;
|
|
37
|
-
this.flags = flags;
|
|
38
|
-
}
|
|
39
|
-
async execute() {
|
|
40
|
-
const { url } = this.args;
|
|
41
|
-
const { ttl, tier, 'max-age': maxAge } = this.flags;
|
|
42
|
-
let normalizedUrl;
|
|
43
|
-
let cacheKey;
|
|
44
|
-
try {
|
|
45
|
-
normalizedUrl = normalizeUrl(url);
|
|
46
|
-
cacheKey = deriveCacheKey(normalizedUrl);
|
|
47
|
-
}
|
|
48
|
-
catch (err) {
|
|
49
|
-
this.error(`Invalid URL: ${err.message}`, { exit: 2 });
|
|
50
|
-
}
|
|
51
|
-
const roots = loadStoreRoots({
|
|
52
|
-
configDir: this.config.configDir,
|
|
53
|
-
cwd: process.cwd(),
|
|
54
|
-
dataDir: this.config.dataDir,
|
|
55
|
-
});
|
|
56
|
-
const located = locateArtifact(roots.readRoots, cacheKey);
|
|
57
|
-
const cached = located?.artifact ?? null;
|
|
58
|
-
const currentTime = new Date();
|
|
59
|
-
let status;
|
|
60
|
-
let freshness;
|
|
61
|
-
let action;
|
|
62
|
-
if (!cached) {
|
|
63
|
-
status = 'miss';
|
|
64
|
-
freshness = 'stale_expired';
|
|
65
|
-
action = 'would_fetch';
|
|
66
|
-
}
|
|
67
|
-
else {
|
|
68
|
-
let isMaxAgeExpired = false;
|
|
69
|
-
if (maxAge) {
|
|
70
|
-
try {
|
|
71
|
-
isMaxAgeExpired = checkMaxAgeExpired(cached, currentTime, maxAge);
|
|
72
|
-
}
|
|
73
|
-
catch (err) {
|
|
74
|
-
this.error(`Invalid max-age: ${err.message}`, { exit: 2 });
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
const freshnessState = isMaxAgeExpired
|
|
78
|
-
? 'stale_expired'
|
|
79
|
-
: evaluateFreshness(cached.metadata, currentTime, ttl || cached.metadata.ttl);
|
|
80
|
-
freshness = freshnessState;
|
|
81
|
-
if (freshnessState === 'fresh') {
|
|
82
|
-
status = 'hit';
|
|
83
|
-
action = 'would_return_cached';
|
|
84
|
-
}
|
|
85
|
-
else {
|
|
86
|
-
status = 'stale';
|
|
87
|
-
action = 'would_revalidate';
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
// On a hit, report where it actually lives; on a miss, where a fetch would write it.
|
|
91
|
-
const artifactPath = located?.path ?? getArtifactPath(roots.writeRoot, cacheKey);
|
|
92
|
-
if (!this.requestedJson()) {
|
|
93
|
-
this.log(`URL: ${normalizedUrl}`);
|
|
94
|
-
this.log(`Cache Key: ${cacheKey}`);
|
|
95
|
-
this.log(`Cache Path: ${artifactPath}`);
|
|
96
|
-
this.log(`Status: ${status}`);
|
|
97
|
-
this.log(`Freshness: ${freshness}`);
|
|
98
|
-
this.log(`Action: ${action}`);
|
|
99
|
-
}
|
|
100
|
-
return {
|
|
101
|
-
cacheKey,
|
|
102
|
-
cachePath: artifactPath,
|
|
103
|
-
status,
|
|
104
|
-
freshness,
|
|
105
|
-
action,
|
|
106
|
-
};
|
|
107
|
-
}
|
|
108
|
-
}
|
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
import { BaseCommand } from '../base-command.js';
|
|
2
|
-
export default class Research extends BaseCommand<typeof Research> {
|
|
3
|
-
static id: string;
|
|
4
|
-
static summary: string;
|
|
5
|
-
static description: string;
|
|
6
|
-
static examples: {
|
|
7
|
-
description: string;
|
|
8
|
-
command: string;
|
|
9
|
-
}[];
|
|
10
|
-
static args: {
|
|
11
|
-
url: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
|
|
12
|
-
};
|
|
13
|
-
static flags: {
|
|
14
|
-
topic: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
15
|
-
tags: import("@oclif/core/interfaces").OptionFlag<string[] | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
16
|
-
format: import("@oclif/core/interfaces").OptionFlag<"compressed" | "detailed", import("@oclif/core/interfaces").CustomOptions>;
|
|
17
|
-
tier: import("@oclif/core/interfaces").OptionFlag<"stable" | "standard" | "volatile", import("@oclif/core/interfaces").CustomOptions>;
|
|
18
|
-
ttl: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
19
|
-
'max-age': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
20
|
-
force: import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
21
|
-
'dry-run': import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
22
|
-
'allow-stale': import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
23
|
-
rendered: import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
24
|
-
storage: import("@oclif/core/interfaces").OptionFlag<"global" | "project" | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
25
|
-
};
|
|
26
|
-
static stdoutIsPrimaryData: boolean;
|
|
27
|
-
init(): Promise<void>;
|
|
28
|
-
private executeCacheHit;
|
|
29
|
-
private executeCacheMiss;
|
|
30
|
-
private buildResultData;
|
|
31
|
-
private validateDurationFlags;
|
|
32
|
-
private resolveArtifact;
|
|
33
|
-
private persistFreshArtifact;
|
|
34
|
-
private persistSectionsIfFresh;
|
|
35
|
-
execute(): Promise<unknown>;
|
|
36
|
-
}
|