@taurgis/bonsai 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +239 -0
- package/agents/agents/official-docs-researcher.agent.md +98 -0
- package/agents/instructions/web-research.instructions.md +29 -0
- package/agents/skills/web-research/SKILL.md +126 -0
- package/bin/cli.mjs +22 -0
- package/dist/base-command.d.ts +14 -0
- package/dist/base-command.js +56 -0
- package/dist/commands/config/base.d.ts +23 -0
- package/dist/commands/config/base.js +46 -0
- package/dist/commands/config/get.d.ts +20 -0
- package/dist/commands/config/get.js +48 -0
- package/dist/commands/config/list.d.ts +17 -0
- package/dist/commands/config/list.js +48 -0
- package/dist/commands/config/set.d.ts +31 -0
- package/dist/commands/config/set.js +94 -0
- package/dist/commands/config/unset.d.ts +21 -0
- package/dist/commands/config/unset.js +65 -0
- package/dist/commands/fetch.d.ts +38 -0
- package/dist/commands/fetch.js +311 -0
- package/dist/commands/import.d.ts +38 -0
- package/dist/commands/import.js +313 -0
- package/dist/commands/index.d.ts +35 -0
- package/dist/commands/index.js +306 -0
- package/dist/commands/inspect.d.ts +13 -0
- package/dist/commands/inspect.js +82 -0
- package/dist/commands/list.d.ts +25 -0
- package/dist/commands/list.js +143 -0
- package/dist/commands/prune.d.ts +24 -0
- package/dist/commands/prune.js +141 -0
- package/dist/commands/research/config/base.d.ts +23 -0
- package/dist/commands/research/config/base.js +46 -0
- package/dist/commands/research/config/get.d.ts +20 -0
- package/dist/commands/research/config/get.js +45 -0
- package/dist/commands/research/config/list.d.ts +17 -0
- package/dist/commands/research/config/list.js +42 -0
- package/dist/commands/research/config/set.d.ts +31 -0
- package/dist/commands/research/config/set.js +94 -0
- package/dist/commands/research/config/unset.d.ts +21 -0
- package/dist/commands/research/config/unset.js +59 -0
- package/dist/commands/research/import.d.ts +38 -0
- package/dist/commands/research/import.js +313 -0
- package/dist/commands/research/inspect.d.ts +13 -0
- package/dist/commands/research/inspect.js +86 -0
- package/dist/commands/research/list.d.ts +25 -0
- package/dist/commands/research/list.js +143 -0
- package/dist/commands/research/prune.d.ts +24 -0
- package/dist/commands/research/prune.js +141 -0
- package/dist/commands/research/search.d.ts +32 -0
- package/dist/commands/research/search.js +334 -0
- package/dist/commands/research/status.d.ts +17 -0
- package/dist/commands/research/status.js +108 -0
- package/dist/commands/research.d.ts +36 -0
- package/dist/commands/research.js +304 -0
- package/dist/commands/search.d.ts +32 -0
- package/dist/commands/search.js +334 -0
- package/dist/commands/status.d.ts +17 -0
- package/dist/commands/status.js +95 -0
- package/dist/commands.d.ts +24 -0
- package/dist/commands.js +24 -0
- package/dist/lib/config/index.d.ts +18 -0
- package/dist/lib/config/index.js +33 -0
- package/dist/lib/config/io.d.ts +8 -0
- package/dist/lib/config/io.js +75 -0
- package/dist/lib/config/resolve.d.ts +15 -0
- package/dist/lib/config/resolve.js +20 -0
- package/dist/lib/config/schema.d.ts +28 -0
- package/dist/lib/config/schema.js +32 -0
- package/dist/lib/research/artifact.d.ts +21 -0
- package/dist/lib/research/artifact.js +217 -0
- package/dist/lib/research/browser.d.ts +74 -0
- package/dist/lib/research/browser.js +359 -0
- package/dist/lib/research/cache-key.d.ts +5 -0
- package/dist/lib/research/cache-key.js +10 -0
- package/dist/lib/research/capture.d.ts +29 -0
- package/dist/lib/research/capture.js +181 -0
- package/dist/lib/research/compress.d.ts +5 -0
- package/dist/lib/research/compress.js +17 -0
- package/dist/lib/research/docs/__fixtures__/sphinx-searchindex.d.ts +1 -0
- package/dist/lib/research/docs/__fixtures__/sphinx-searchindex.js +2 -0
- package/dist/lib/research/docs/capabilities.d.ts +43 -0
- package/dist/lib/research/docs/capabilities.js +9 -0
- package/dist/lib/research/docs/clean-dom.d.ts +4 -0
- package/dist/lib/research/docs/clean-dom.js +41 -0
- package/dist/lib/research/docs/code-blocks.d.ts +5 -0
- package/dist/lib/research/docs/code-blocks.js +69 -0
- package/dist/lib/research/docs/detect.d.ts +6 -0
- package/dist/lib/research/docs/detect.js +239 -0
- package/dist/lib/research/docs/machine-readable.d.ts +21 -0
- package/dist/lib/research/docs/machine-readable.js +73 -0
- package/dist/lib/research/docs/markdown-source.d.ts +17 -0
- package/dist/lib/research/docs/markdown-source.js +194 -0
- package/dist/lib/research/docs/quality-gates.d.ts +9 -0
- package/dist/lib/research/docs/quality-gates.js +65 -0
- package/dist/lib/research/docs/remote-search-runner.d.ts +21 -0
- package/dist/lib/research/docs/remote-search-runner.js +54 -0
- package/dist/lib/research/docs/remote-search.d.ts +14 -0
- package/dist/lib/research/docs/remote-search.js +101 -0
- package/dist/lib/research/docs/rsc.d.ts +10 -0
- package/dist/lib/research/docs/rsc.js +30 -0
- package/dist/lib/research/docs/search-index.d.ts +13 -0
- package/dist/lib/research/docs/search-index.js +176 -0
- package/dist/lib/research/docs/section-artifacts.d.ts +12 -0
- package/dist/lib/research/docs/section-artifacts.js +75 -0
- package/dist/lib/research/docs/sections.d.ts +12 -0
- package/dist/lib/research/docs/sections.js +67 -0
- package/dist/lib/research/docs/source-map.d.ts +19 -0
- package/dist/lib/research/docs/source-map.js +87 -0
- package/dist/lib/research/docs/validate.d.ts +15 -0
- package/dist/lib/research/docs/validate.js +63 -0
- package/dist/lib/research/extract.d.ts +11 -0
- package/dist/lib/research/extract.js +83 -0
- package/dist/lib/research/fetcher.d.ts +29 -0
- package/dist/lib/research/fetcher.js +192 -0
- package/dist/lib/research/freshness.d.ts +21 -0
- package/dist/lib/research/freshness.js +82 -0
- package/dist/lib/research/markdown.d.ts +6 -0
- package/dist/lib/research/markdown.js +66 -0
- package/dist/lib/research/regression.d.ts +23 -0
- package/dist/lib/research/regression.js +93 -0
- package/dist/lib/research/resolve-target.d.ts +18 -0
- package/dist/lib/research/resolve-target.js +20 -0
- package/dist/lib/research/revalidate.d.ts +32 -0
- package/dist/lib/research/revalidate.js +152 -0
- package/dist/lib/research/schema.d.ts +47 -0
- package/dist/lib/research/schema.js +1 -0
- package/dist/lib/research/secret-scan.d.ts +5 -0
- package/dist/lib/research/secret-scan.js +37 -0
- package/dist/lib/research/secure-write.d.ts +16 -0
- package/dist/lib/research/secure-write.js +17 -0
- package/dist/lib/research/storage.d.ts +52 -0
- package/dist/lib/research/storage.js +182 -0
- package/dist/lib/research/store-roots.d.ts +29 -0
- package/dist/lib/research/store-roots.js +28 -0
- package/dist/lib/research/token-estimate.d.ts +5 -0
- package/dist/lib/research/token-estimate.js +7 -0
- package/dist/lib/research/url.d.ts +24 -0
- package/dist/lib/research/url.js +121 -0
- package/dist/sites/index.d.ts +4 -0
- package/dist/sites/index.js +28 -0
- package/dist/sites/salesforce/browser-search.d.ts +7 -0
- package/dist/sites/salesforce/browser-search.js +82 -0
- package/dist/sites/salesforce/coveo.d.ts +23 -0
- package/dist/sites/salesforce/coveo.js +161 -0
- package/dist/sites/salesforce/fetch-page.d.ts +6 -0
- package/dist/sites/salesforce/fetch-page.js +42 -0
- package/dist/sites/salesforce/index.d.ts +2 -0
- package/dist/sites/salesforce/index.js +16 -0
- package/dist/sites/salesforce/token.d.ts +26 -0
- package/dist/sites/salesforce/token.js +154 -0
- package/dist/sites/salesforce-developer/fetch-page.d.ts +3 -0
- package/dist/sites/salesforce-developer/fetch-page.js +18 -0
- package/dist/sites/salesforce-developer/index.d.ts +2 -0
- package/dist/sites/salesforce-developer/index.js +12 -0
- package/dist/sites/salesforce-doc-fetch.d.ts +14 -0
- package/dist/sites/salesforce-doc-fetch.js +321 -0
- package/dist/sites/types.d.ts +27 -0
- package/dist/sites/types.js +1 -0
- package/oclif.manifest.json +845 -0
- package/package.json +78 -0
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { BaseCommand } from '../base-command.js';
|
|
2
|
+
export default class Bonsai extends BaseCommand<typeof Bonsai> {
|
|
3
|
+
static summary: string;
|
|
4
|
+
static description: string;
|
|
5
|
+
static examples: {
|
|
6
|
+
description: string;
|
|
7
|
+
command: string;
|
|
8
|
+
}[];
|
|
9
|
+
static args: {
|
|
10
|
+
url: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
|
|
11
|
+
};
|
|
12
|
+
static flags: {
|
|
13
|
+
topic: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
14
|
+
tags: import("@oclif/core/interfaces").OptionFlag<string[] | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
15
|
+
format: import("@oclif/core/interfaces").OptionFlag<"compressed" | "detailed", import("@oclif/core/interfaces").CustomOptions>;
|
|
16
|
+
tier: import("@oclif/core/interfaces").OptionFlag<"stable" | "standard" | "volatile", import("@oclif/core/interfaces").CustomOptions>;
|
|
17
|
+
ttl: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
18
|
+
'max-age': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
19
|
+
force: import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
20
|
+
'dry-run': import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
21
|
+
'allow-stale': import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
22
|
+
rendered: import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
23
|
+
storage: import("@oclif/core/interfaces").OptionFlag<"global" | "project" | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
24
|
+
};
|
|
25
|
+
static stdoutIsPrimaryData: boolean;
|
|
26
|
+
init(): Promise<void>;
|
|
27
|
+
private executeCacheHit;
|
|
28
|
+
private executeCacheMiss;
|
|
29
|
+
private buildResultData;
|
|
30
|
+
private validateDurationFlags;
|
|
31
|
+
private resolveArtifact;
|
|
32
|
+
private persistFreshArtifact;
|
|
33
|
+
private persistSectionsIfFresh;
|
|
34
|
+
execute(): Promise<unknown>;
|
|
35
|
+
}
|
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
import { Args, Flags } from '@oclif/core';
|
|
2
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { BaseCommand } from '../base-command.js';
|
|
6
|
+
import { writeArtifact, getArtifactPath } from '../lib/research/storage.js';
|
|
7
|
+
import { writeArtifactSecurely } from '../lib/research/secure-write.js';
|
|
8
|
+
import { evaluateFreshness, parseTtlToMs, checkMaxAgeExpired } from '../lib/research/freshness.js';
|
|
9
|
+
import { revalidateCache, createArtifactFromFetch } from '../lib/research/revalidate.js';
|
|
10
|
+
import { fetchStaticHtml, fetchText } from '../lib/research/fetcher.js';
|
|
11
|
+
import { fetchRenderedHtml } from '../lib/research/browser.js';
|
|
12
|
+
import { capturePage } from '../lib/research/capture.js';
|
|
13
|
+
import { persistSectionArtifacts } from '../lib/research/docs/section-artifacts.js';
|
|
14
|
+
import { resolveResearchTarget } from '../lib/research/resolve-target.js';
|
|
15
|
+
import { detectSite } from '../sites/index.js';
|
|
16
|
+
const CAPTURE_DEPS = {
|
|
17
|
+
fetchStatic: (url) => fetchStaticHtml(url),
|
|
18
|
+
fetchRendered: (url) => fetchRenderedHtml(url),
|
|
19
|
+
fetchText: (url) => fetchText(url),
|
|
20
|
+
};
|
|
21
|
+
export default class Bonsai extends BaseCommand {
|
|
22
|
+
static summary = 'An advanced, locally cached web research tool optimized for LLM ingestion.';
|
|
23
|
+
static description = 'Scrapes the specified URL, strips HTML boilerplate, converts the semantic payload into clean Markdown format, and caches the result locally using dynamic TTL rules.';
|
|
24
|
+
static examples = [
|
|
25
|
+
{
|
|
26
|
+
description: 'Research a URL with detailed output, tagged with topic and tags',
|
|
27
|
+
command: '<%= config.bin %> https://docs.nestjs.com/ --topic "Backend Frameworks" --tags "Node" --tags "NestJS" --format detailed --ttl 30d',
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
description: 'Research a volatile page with compressed output and short TTL, returned as JSON',
|
|
31
|
+
command: '<%= config.bin %> https://news.ycombinator.com/ --format compressed --ttl 2h --json',
|
|
32
|
+
},
|
|
33
|
+
];
|
|
34
|
+
static args = {
|
|
35
|
+
url: Args.string({
|
|
36
|
+
required: true,
|
|
37
|
+
description: 'The full HTTP/HTTPS URL of the web page to research.',
|
|
38
|
+
}),
|
|
39
|
+
};
|
|
40
|
+
static flags = {
|
|
41
|
+
topic: Flags.string({
|
|
42
|
+
char: 't',
|
|
43
|
+
description: 'The main category/topic of the research for metadata tagging.',
|
|
44
|
+
}),
|
|
45
|
+
tags: Flags.string({
|
|
46
|
+
char: 'g',
|
|
47
|
+
description: 'Taxonomic tags for this research (can be repeated).',
|
|
48
|
+
multiple: true,
|
|
49
|
+
}),
|
|
50
|
+
format: Flags.option({
|
|
51
|
+
char: 'f',
|
|
52
|
+
description: 'Desired data density.',
|
|
53
|
+
options: ['compressed', 'detailed'],
|
|
54
|
+
default: 'compressed',
|
|
55
|
+
})(),
|
|
56
|
+
tier: Flags.option({
|
|
57
|
+
description: 'Freshness tier policy.',
|
|
58
|
+
options: ['stable', 'standard', 'volatile'],
|
|
59
|
+
default: 'standard',
|
|
60
|
+
})(),
|
|
61
|
+
ttl: Flags.string({
|
|
62
|
+
char: 'l',
|
|
63
|
+
description: 'Predicted lifespan of the data (e.g., "2h", "7d", "3m").',
|
|
64
|
+
}),
|
|
65
|
+
'max-age': Flags.string({
|
|
66
|
+
description: 'Maximum age of cache to accept (e.g., "1d", "30d").',
|
|
67
|
+
}),
|
|
68
|
+
force: Flags.boolean({
|
|
69
|
+
description: 'Force a fresh fetch, ignoring any cached entries.',
|
|
70
|
+
default: false,
|
|
71
|
+
}),
|
|
72
|
+
'dry-run': Flags.boolean({
|
|
73
|
+
description: 'Perform validation and fetch without saving to local cache.',
|
|
74
|
+
default: false,
|
|
75
|
+
}),
|
|
76
|
+
'allow-stale': Flags.boolean({
|
|
77
|
+
description: 'Allow serving stale cache if the remote server is unreachable.',
|
|
78
|
+
default: false,
|
|
79
|
+
}),
|
|
80
|
+
rendered: Flags.boolean({
|
|
81
|
+
description: 'Force using a browser-rendered scraping path for dynamic pages.',
|
|
82
|
+
default: false,
|
|
83
|
+
}),
|
|
84
|
+
storage: Flags.option({
|
|
85
|
+
description: 'Override where this result is cached (overrides configured default). Secret-bearing pages are always stored globally.',
|
|
86
|
+
options: ['global', 'project'],
|
|
87
|
+
})(),
|
|
88
|
+
};
|
|
89
|
+
static stdoutIsPrimaryData = true;
|
|
90
|
+
async init() {
|
|
91
|
+
await super.init();
|
|
92
|
+
const { args, flags } = await this.parse(Bonsai);
|
|
93
|
+
this.args = args;
|
|
94
|
+
this.flags = flags;
|
|
95
|
+
}
|
|
96
|
+
async executeCacheHit(cached, targetDir, currentTime) {
|
|
97
|
+
const { ttl, 'max-age': maxAge, 'allow-stale': allowStale, rendered } = this.flags;
|
|
98
|
+
const isExpired = checkMaxAgeExpired(cached, currentTime, maxAge);
|
|
99
|
+
const freshnessState = isExpired
|
|
100
|
+
? 'stale_expired'
|
|
101
|
+
: evaluateFreshness(cached.metadata, currentTime, ttl);
|
|
102
|
+
if (freshnessState === 'fresh') {
|
|
103
|
+
return { cacheStatus: 'hit', freshnessState, artifact: cached };
|
|
104
|
+
}
|
|
105
|
+
const revalResult = await revalidateCache(targetDir, cached, currentTime, {
|
|
106
|
+
allowStale: Boolean(allowStale),
|
|
107
|
+
ttlOverride: ttl,
|
|
108
|
+
rendered: Boolean(rendered),
|
|
109
|
+
});
|
|
110
|
+
handleStaleRevalidationResult(this, revalResult);
|
|
111
|
+
return {
|
|
112
|
+
cacheStatus: revalResult.status,
|
|
113
|
+
freshnessState,
|
|
114
|
+
artifact: revalResult.artifact,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
async executeCacheMiss(normalizedUrl, currentTime, cacheKey) {
|
|
118
|
+
const { topic, tags, tier, ttl, rendered } = this.flags;
|
|
119
|
+
const siteModule = detectSite(normalizedUrl);
|
|
120
|
+
const useRendered = rendered || Boolean(siteModule?.defaults?.rendered);
|
|
121
|
+
let fetchResult;
|
|
122
|
+
let extraction;
|
|
123
|
+
let capture = null;
|
|
124
|
+
if (siteModule?.fetchPage) {
|
|
125
|
+
// Custom site modules own their fetch/extract strategy; the generic capture path is skipped.
|
|
126
|
+
({ fetchResult, extraction } = await siteModule.fetchPage(normalizedUrl));
|
|
127
|
+
}
|
|
128
|
+
else {
|
|
129
|
+
capture = await capturePage(normalizedUrl, { forceRendered: useRendered }, CAPTURE_DEPS);
|
|
130
|
+
fetchResult = capture.fetchResult;
|
|
131
|
+
extraction = capture.extraction;
|
|
132
|
+
}
|
|
133
|
+
const artifact = createArtifactFromFetch(normalizedUrl, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl || null, currentTime);
|
|
134
|
+
artifact.metadata.topic = topic || null;
|
|
135
|
+
artifact.metadata.tags = tags || [];
|
|
136
|
+
artifact.metadata.site_module_id = siteModule?.id ?? null;
|
|
137
|
+
if (capture) {
|
|
138
|
+
applyCaptureMetadata(artifact, capture);
|
|
139
|
+
}
|
|
140
|
+
else if (useRendered) {
|
|
141
|
+
artifact.metadata.capture_method = 'browser_fallback';
|
|
142
|
+
}
|
|
143
|
+
return artifact;
|
|
144
|
+
}
|
|
145
|
+
buildResultData(url, normalizedUrl, cacheKey, storageDir, storageMode, cacheStatus, freshnessState, format, artifact, content, redirectedToGlobal) {
|
|
146
|
+
return {
|
|
147
|
+
schemaVersion: 1,
|
|
148
|
+
command: this.config.bin,
|
|
149
|
+
cache: {
|
|
150
|
+
key: cacheKey,
|
|
151
|
+
status: cacheStatus,
|
|
152
|
+
freshness: freshnessState,
|
|
153
|
+
path: getArtifactPath(storageDir, cacheKey),
|
|
154
|
+
storage: storageMode,
|
|
155
|
+
redirectedToGlobal,
|
|
156
|
+
},
|
|
157
|
+
source: {
|
|
158
|
+
url,
|
|
159
|
+
normalizedUrl,
|
|
160
|
+
captureMethod: artifact.metadata.capture_method,
|
|
161
|
+
extractionStatus: artifact.metadata.extraction_status,
|
|
162
|
+
extractionConfidence: artifact.metadata.extraction_confidence,
|
|
163
|
+
qualityNotes: artifact.metadata.quality_notes,
|
|
164
|
+
fetchedAt: artifact.metadata.fetched_at,
|
|
165
|
+
validatedAt: artifact.metadata.validated_at,
|
|
166
|
+
staleAfter: artifact.metadata.stale_after,
|
|
167
|
+
},
|
|
168
|
+
artifactType: artifact.metadata.artifact_type,
|
|
169
|
+
docsEngine: artifact.metadata.docs_engine,
|
|
170
|
+
docsFramework: artifact.metadata.docs_framework,
|
|
171
|
+
sourceDocUrl: artifact.metadata.source_doc_url,
|
|
172
|
+
searchProvider: artifact.metadata.search_provider,
|
|
173
|
+
format,
|
|
174
|
+
tokenEstimate: format === 'compressed'
|
|
175
|
+
? artifact.metadata.token_estimate.compressed
|
|
176
|
+
: artifact.metadata.token_estimate.detailed,
|
|
177
|
+
content,
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
// Validates the duration flags up front, exiting with code 2 on a malformed value.
|
|
181
|
+
validateDurationFlags(ttl, maxAge) {
|
|
182
|
+
for (const [label, value] of [
|
|
183
|
+
['TTL', ttl],
|
|
184
|
+
['max-age', maxAge],
|
|
185
|
+
]) {
|
|
186
|
+
if (!value)
|
|
187
|
+
continue;
|
|
188
|
+
try {
|
|
189
|
+
parseTtlToMs(value);
|
|
190
|
+
}
|
|
191
|
+
catch (err) {
|
|
192
|
+
this.error(`Invalid ${label}: ${err.message}`, { exit: 2 });
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
// Serves the cache when a fresh/revalidatable entry exists, otherwise fetches fresh. Reads fall
|
|
197
|
+
// back project→global; the resolved artifact is returned with the dir it lives in / landed in.
|
|
198
|
+
async resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime, located) {
|
|
199
|
+
if (located) {
|
|
200
|
+
// Revalidate where the entry already lives; on dry-run use the throwaway dir so the cache is
|
|
201
|
+
// never mutated. ponytail: revalidation rewrites in place, so a refreshed project entry that
|
|
202
|
+
// gains a secret is not re-routed — only first-time project writes are scanned.
|
|
203
|
+
const revalDir = tmpDir ?? located.dataDir;
|
|
204
|
+
const hit = await this.executeCacheHit(located.artifact, revalDir, currentTime);
|
|
205
|
+
return { ...hit, storageDir: located.dataDir, redirectedToGlobal: false };
|
|
206
|
+
}
|
|
207
|
+
const artifact = await this.executeCacheMiss(normalizedUrl, currentTime, cacheKey);
|
|
208
|
+
const { dir, redirectedToGlobal } = this.persistFreshArtifact(roots, tmpDir, cacheKey, artifact);
|
|
209
|
+
return {
|
|
210
|
+
cacheStatus: 'miss',
|
|
211
|
+
freshnessState: 'stale_expired',
|
|
212
|
+
artifact,
|
|
213
|
+
storageDir: dir,
|
|
214
|
+
redirectedToGlobal,
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
// Writes a freshly fetched artifact, honoring dry-run (throwaway dir) and the secret-safety
|
|
218
|
+
// redirect (project→global). Returns the data dir reported to the user and whether a redirect
|
|
219
|
+
// occurred (so the JSON envelope mirrors `import`).
|
|
220
|
+
persistFreshArtifact(roots, tmpDir, cacheKey, artifact) {
|
|
221
|
+
if (tmpDir) {
|
|
222
|
+
writeArtifact(tmpDir, cacheKey, artifact);
|
|
223
|
+
// Report the real would-be location, not the throwaway dir that is about to be deleted.
|
|
224
|
+
return { dir: roots.writeRoot, redirectedToGlobal: false };
|
|
225
|
+
}
|
|
226
|
+
const result = writeArtifactSecurely(roots, cacheKey, artifact);
|
|
227
|
+
if (result.redirected) {
|
|
228
|
+
this.warn(`Detected ${result.secretLabel} in the page content; stored in the global cache instead of the project to avoid committing secrets.`);
|
|
229
|
+
}
|
|
230
|
+
return { dir: result.dataDir, redirectedToGlobal: result.redirected };
|
|
231
|
+
}
|
|
232
|
+
// Long references are split into searchable/inspectable section children whenever the page artifact
|
|
233
|
+
// is freshly written (T-22). Best-effort: never let chunking break the main result.
|
|
234
|
+
persistSectionsIfFresh(targetDir, artifact, currentTime, cacheStatus) {
|
|
235
|
+
if (cacheStatus !== 'miss' && cacheStatus !== 'refreshed')
|
|
236
|
+
return;
|
|
237
|
+
try {
|
|
238
|
+
persistSectionArtifacts(targetDir, artifact, currentTime);
|
|
239
|
+
}
|
|
240
|
+
catch {
|
|
241
|
+
/* section generation is non-essential; ignore failures */
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
async execute() {
|
|
245
|
+
const { url } = this.args;
|
|
246
|
+
const { format, ttl, 'max-age': maxAge, 'dry-run': dryRun } = this.flags;
|
|
247
|
+
this.validateDurationFlags(ttl, maxAge);
|
|
248
|
+
let target;
|
|
249
|
+
try {
|
|
250
|
+
target = resolveResearchTarget({
|
|
251
|
+
configDir: this.config.configDir,
|
|
252
|
+
cwd: process.cwd(),
|
|
253
|
+
dataDir: this.config.dataDir,
|
|
254
|
+
flagOverride: this.flags.storage,
|
|
255
|
+
lookup: !this.flags.force,
|
|
256
|
+
url,
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
catch (err) {
|
|
260
|
+
this.error(`Invalid URL: ${err.message}`, { exit: 2 });
|
|
261
|
+
}
|
|
262
|
+
const { cacheKey, located, normalizedUrl, roots } = target;
|
|
263
|
+
const tmpDir = dryRun ? mkdtempSync(join(tmpdir(), 'fnr-dry-run-')) : null;
|
|
264
|
+
const currentTime = new Date();
|
|
265
|
+
try {
|
|
266
|
+
const { cacheStatus, freshnessState, artifact, storageDir, redirectedToGlobal } = await this.resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime, located);
|
|
267
|
+
this.persistSectionsIfFresh(tmpDir ?? storageDir, artifact, currentTime, cacheStatus);
|
|
268
|
+
const content = format === 'compressed' ? artifact.compressed : artifact.detailed;
|
|
269
|
+
const resultData = this.buildResultData(url, normalizedUrl, cacheKey, storageDir, roots.mode, cacheStatus, freshnessState, format, artifact, content, redirectedToGlobal);
|
|
270
|
+
if (!this.requestedJson()) {
|
|
271
|
+
this.log(content);
|
|
272
|
+
}
|
|
273
|
+
return resultData;
|
|
274
|
+
}
|
|
275
|
+
finally {
|
|
276
|
+
if (tmpDir) {
|
|
277
|
+
rmSync(tmpDir, { recursive: true, force: true });
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
// Copies Phase 2 capability provenance from a capture outcome onto the artifact metadata.
|
|
283
|
+
function applyCaptureMetadata(artifact, capture) {
|
|
284
|
+
const meta = artifact.metadata;
|
|
285
|
+
meta.capture_method = capture.captureMethod;
|
|
286
|
+
meta.docs_engine = capture.capabilities.docsEngine ?? null;
|
|
287
|
+
meta.docs_framework = capture.capabilities.framework ?? null;
|
|
288
|
+
meta.source_doc_url = capture.sourceDocUrl;
|
|
289
|
+
meta.search_provider = capture.capabilities.search?.provider ?? null;
|
|
290
|
+
if (capture.extraction.isIndexHub)
|
|
291
|
+
meta.artifact_type = 'index';
|
|
292
|
+
if (capture.sourceDocUrl && !meta.source_urls.includes(capture.sourceDocUrl)) {
|
|
293
|
+
meta.source_urls.push(capture.sourceDocUrl);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
function handleStaleRevalidationResult(command, revalResult) {
|
|
297
|
+
if (revalResult.status !== 'stale')
|
|
298
|
+
return;
|
|
299
|
+
if (revalResult.allowed) {
|
|
300
|
+
command.warn(`Serving stale content: revalidation failed: ${revalResult.error}`);
|
|
301
|
+
}
|
|
302
|
+
else {
|
|
303
|
+
command.warn(`Serving stale content (exit 5): revalidation failed: ${revalResult.error}`);
|
|
304
|
+
process.exitCode = 5;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { BaseCommand } from '../base-command.js';
|
|
2
|
+
export default class ResearchInspect extends BaseCommand<typeof ResearchInspect> {
|
|
3
|
+
static id: string;
|
|
4
|
+
static summary: string;
|
|
5
|
+
static description: string;
|
|
6
|
+
static args: {
|
|
7
|
+
url: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
|
|
8
|
+
};
|
|
9
|
+
static stdoutIsPrimaryData: boolean;
|
|
10
|
+
init(): Promise<void>;
|
|
11
|
+
execute(): Promise<unknown>;
|
|
12
|
+
private findSections;
|
|
13
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { Args } from '@oclif/core';
|
|
2
|
+
import { BaseCommand } from '../base-command.js';
|
|
3
|
+
import { scanCacheDirs } from '../lib/research/storage.js';
|
|
4
|
+
import { resolveResearchTarget } from '../lib/research/resolve-target.js';
|
|
5
|
+
export default class ResearchInspect extends BaseCommand {
|
|
6
|
+
static id = 'inspect';
|
|
7
|
+
static summary = 'Inspect cached research metadata for a URL.';
|
|
8
|
+
static description = 'Inspects local storage and displays full frontmatter metadata for the URL (without printing the content body).';
|
|
9
|
+
static args = {
|
|
10
|
+
url: Args.string({
|
|
11
|
+
required: true,
|
|
12
|
+
description: 'The URL to inspect.',
|
|
13
|
+
}),
|
|
14
|
+
};
|
|
15
|
+
static stdoutIsPrimaryData = true;
|
|
16
|
+
async init() {
|
|
17
|
+
await super.init();
|
|
18
|
+
const { args } = await this.parse(ResearchInspect);
|
|
19
|
+
this.args = args;
|
|
20
|
+
}
|
|
21
|
+
async execute() {
|
|
22
|
+
const { url } = this.args;
|
|
23
|
+
let target;
|
|
24
|
+
try {
|
|
25
|
+
target = resolveResearchTarget({
|
|
26
|
+
configDir: this.config.configDir,
|
|
27
|
+
cwd: process.cwd(),
|
|
28
|
+
dataDir: this.config.dataDir,
|
|
29
|
+
url,
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
catch (err) {
|
|
33
|
+
this.error(`Invalid URL: ${err.message}`, { exit: 2 });
|
|
34
|
+
}
|
|
35
|
+
const { cacheKey, located, normalizedUrl, roots } = target;
|
|
36
|
+
if (!located) {
|
|
37
|
+
this.error(`No cached research found for URL: ${normalizedUrl}`, { exit: 1 });
|
|
38
|
+
}
|
|
39
|
+
const cached = located.artifact;
|
|
40
|
+
const artifactPath = located.path;
|
|
41
|
+
const sections = this.findSections(roots.readRoots, cacheKey);
|
|
42
|
+
if (!this.requestedJson()) {
|
|
43
|
+
this.log(`Cache Key: ${cacheKey}`);
|
|
44
|
+
this.log(`Cache Path: ${artifactPath}`);
|
|
45
|
+
this.log(`--- Metadata ---`);
|
|
46
|
+
for (const [key, val] of Object.entries(cached.metadata)) {
|
|
47
|
+
if (typeof val === 'object' && val !== null) {
|
|
48
|
+
this.log(`${key}: ${JSON.stringify(val)}`);
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
this.log(`${key}: ${val}`);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
if (sections.length) {
|
|
55
|
+
this.log(`--- Sections (${sections.length}) ---`);
|
|
56
|
+
for (const s of sections) {
|
|
57
|
+
this.log(`${s.headingPath} [${s.anchor}] (${s.tokenEstimate.detailed} tokens) ${s.cacheKey}`);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return {
|
|
62
|
+
cacheKey,
|
|
63
|
+
cachePath: artifactPath,
|
|
64
|
+
metadata: cached.metadata,
|
|
65
|
+
sections,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
// Section children link back via parent_cache_key; list the active ones for this page (T-22).
|
|
69
|
+
findSections(readRoots, parentKey) {
|
|
70
|
+
return scanCacheDirs(readRoots, (artifact) => {
|
|
71
|
+
const meta = artifact.metadata;
|
|
72
|
+
if (meta.parent_cache_key !== parentKey || meta.status !== 'active')
|
|
73
|
+
return null;
|
|
74
|
+
return {
|
|
75
|
+
cacheKey: meta.cache_key,
|
|
76
|
+
anchor: meta.section_anchor,
|
|
77
|
+
headingPath: meta.section_heading_path,
|
|
78
|
+
tokenEstimate: meta.token_estimate,
|
|
79
|
+
};
|
|
80
|
+
}).sort((a, b) => (a.headingPath ?? '').localeCompare(b.headingPath ?? ''));
|
|
81
|
+
}
|
|
82
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { BaseCommand } from '../base-command.js';
|
|
2
|
+
export default class ResearchList extends BaseCommand<typeof ResearchList> {
|
|
3
|
+
static id: string;
|
|
4
|
+
static summary: string;
|
|
5
|
+
static description: string;
|
|
6
|
+
static examples: {
|
|
7
|
+
description: string;
|
|
8
|
+
command: string;
|
|
9
|
+
}[];
|
|
10
|
+
static flags: {
|
|
11
|
+
topic: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
12
|
+
tags: import("@oclif/core/interfaces").OptionFlag<string[] | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
13
|
+
freshness: import("@oclif/core/interfaces").OptionFlag<"fresh" | "stale_grace" | "stale_expired" | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
14
|
+
'artifact-type': import("@oclif/core/interfaces").OptionFlag<"source" | "research_note" | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
15
|
+
'capture-method': import("@oclif/core/interfaces").OptionFlag<"static_fetch" | "browser_fallback" | "agent_supplied" | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
16
|
+
limit: import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
|
|
17
|
+
};
|
|
18
|
+
static stdoutIsPrimaryData: boolean;
|
|
19
|
+
init(): Promise<void>;
|
|
20
|
+
private validateListFlags;
|
|
21
|
+
private matchesFilters;
|
|
22
|
+
private scanCacheDirForList;
|
|
23
|
+
private logListResults;
|
|
24
|
+
execute(): Promise<unknown>;
|
|
25
|
+
}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import { Flags } from '@oclif/core';
|
|
2
|
+
import { BaseCommand } from '../base-command.js';
|
|
3
|
+
import { scanCacheDirs } from '../lib/research/storage.js';
|
|
4
|
+
import { loadStoreRoots } from '../lib/research/store-roots.js';
|
|
5
|
+
import { evaluateFreshness } from '../lib/research/freshness.js';
|
|
6
|
+
export default class ResearchList extends BaseCommand {
|
|
7
|
+
static id = 'list';
|
|
8
|
+
static summary = 'List cached research artifacts by metadata filters.';
|
|
9
|
+
static description = 'Lists cached research artifacts, including metadata details like path, source count, freshness, token estimates, and quality metrics without printing full content.';
|
|
10
|
+
static examples = [
|
|
11
|
+
{
|
|
12
|
+
description: 'List all cached entries',
|
|
13
|
+
command: '<%= config.bin %> list',
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
description: 'List cached entries for a specific topic with JSON output',
|
|
17
|
+
command: '<%= config.bin %> list --topic "React Suspense" --json',
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
description: 'List only fresh entries filtered by tags',
|
|
21
|
+
command: '<%= config.bin %> list --freshness fresh --tags node --tags url',
|
|
22
|
+
},
|
|
23
|
+
];
|
|
24
|
+
static flags = {
|
|
25
|
+
topic: Flags.string({
|
|
26
|
+
char: 't',
|
|
27
|
+
description: 'Filter by exact topic (case-insensitive).',
|
|
28
|
+
}),
|
|
29
|
+
tags: Flags.string({
|
|
30
|
+
char: 'g',
|
|
31
|
+
description: 'Filter by tags (must match all tags specified).',
|
|
32
|
+
multiple: true,
|
|
33
|
+
}),
|
|
34
|
+
freshness: Flags.option({
|
|
35
|
+
description: 'Filter by freshness state.',
|
|
36
|
+
options: ['fresh', 'stale_grace', 'stale_expired'],
|
|
37
|
+
})(),
|
|
38
|
+
'artifact-type': Flags.option({
|
|
39
|
+
description: 'Filter by artifact type.',
|
|
40
|
+
options: ['source', 'research_note'],
|
|
41
|
+
})(),
|
|
42
|
+
'capture-method': Flags.option({
|
|
43
|
+
description: 'Filter by capture method.',
|
|
44
|
+
options: ['static_fetch', 'browser_fallback', 'agent_supplied'],
|
|
45
|
+
})(),
|
|
46
|
+
limit: Flags.integer({
|
|
47
|
+
description: 'Maximum number of results to return (default 50, max 100).',
|
|
48
|
+
default: 50,
|
|
49
|
+
}),
|
|
50
|
+
};
|
|
51
|
+
static stdoutIsPrimaryData = true;
|
|
52
|
+
async init() {
|
|
53
|
+
await super.init();
|
|
54
|
+
const { flags } = await this.parse(ResearchList);
|
|
55
|
+
this.flags = flags;
|
|
56
|
+
}
|
|
57
|
+
validateListFlags() {
|
|
58
|
+
const limit = this.flags.limit;
|
|
59
|
+
if (limit !== undefined && (limit < 1 || limit > 100)) {
|
|
60
|
+
this.error('Limit must be between 1 and 100.', { exit: 2 });
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
matchesFilters(meta, freshness) {
|
|
64
|
+
if (this.flags.topic &&
|
|
65
|
+
(!meta.topic || meta.topic.trim().toLowerCase() !== this.flags.topic.trim().toLowerCase())) {
|
|
66
|
+
return false;
|
|
67
|
+
}
|
|
68
|
+
if (this.flags.tags && this.flags.tags.length > 0) {
|
|
69
|
+
const metaTagsLower = (meta.tags || []).map((t) => t.toLowerCase());
|
|
70
|
+
const hasAllTags = this.flags.tags.every((t) => metaTagsLower.includes(t.toLowerCase()));
|
|
71
|
+
if (!hasAllTags) {
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
if (this.flags['artifact-type'] && meta.artifact_type !== this.flags['artifact-type']) {
|
|
76
|
+
return false;
|
|
77
|
+
}
|
|
78
|
+
if (this.flags['capture-method'] && meta.capture_method !== this.flags['capture-method']) {
|
|
79
|
+
return false;
|
|
80
|
+
}
|
|
81
|
+
if (this.flags.freshness && freshness !== this.flags.freshness) {
|
|
82
|
+
return false;
|
|
83
|
+
}
|
|
84
|
+
return true;
|
|
85
|
+
}
|
|
86
|
+
scanCacheDirForList(readRoots, currentTime) {
|
|
87
|
+
return scanCacheDirs(readRoots, (artifact, filePath) => {
|
|
88
|
+
if (artifact.metadata.status !== 'active')
|
|
89
|
+
return null;
|
|
90
|
+
const freshness = evaluateFreshness(artifact.metadata, currentTime, null);
|
|
91
|
+
if (!this.matchesFilters(artifact.metadata, freshness))
|
|
92
|
+
return null;
|
|
93
|
+
return {
|
|
94
|
+
cacheKey: artifact.metadata.cache_key,
|
|
95
|
+
path: filePath,
|
|
96
|
+
artifactType: artifact.metadata.artifact_type,
|
|
97
|
+
sourceUrls: artifact.metadata.source_urls,
|
|
98
|
+
topic: artifact.metadata.topic,
|
|
99
|
+
tags: artifact.metadata.tags,
|
|
100
|
+
freshness,
|
|
101
|
+
captureMethod: artifact.metadata.capture_method,
|
|
102
|
+
tokenEstimate: artifact.metadata.token_estimate,
|
|
103
|
+
qualityNotes: artifact.metadata.quality_notes,
|
|
104
|
+
fetchedAt: artifact.metadata.fetched_at,
|
|
105
|
+
validatedAt: artifact.metadata.validated_at,
|
|
106
|
+
};
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
logListResults(finalResults) {
|
|
110
|
+
if (this.requestedJson())
|
|
111
|
+
return;
|
|
112
|
+
if (finalResults.length === 0) {
|
|
113
|
+
this.log('No cached research entries found matching filters.');
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
this.log(`Found ${finalResults.length} cached research entries:\n`);
|
|
117
|
+
finalResults.forEach((res, index) => {
|
|
118
|
+
this.log(`${index + 1}. [${res.topic || 'No Topic'}] Key: ${res.cacheKey}`);
|
|
119
|
+
this.log(` Type: ${res.artifactType} | Freshness: ${res.freshness}`);
|
|
120
|
+
this.log(` Tokens: compressed=${res.tokenEstimate?.compressed || 0}, detailed=${res.tokenEstimate?.detailed || 0}`);
|
|
121
|
+
this.log(` Source URLs: ${res.sourceUrls.join(', ')}\n`);
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
async execute() {
|
|
125
|
+
this.validateListFlags();
|
|
126
|
+
const roots = loadStoreRoots({
|
|
127
|
+
configDir: this.config.configDir,
|
|
128
|
+
cwd: process.cwd(),
|
|
129
|
+
dataDir: this.config.dataDir,
|
|
130
|
+
});
|
|
131
|
+
const currentTime = new Date();
|
|
132
|
+
const results = this.scanCacheDirForList(roots.readRoots, currentTime);
|
|
133
|
+
// Sort by validated_at or fetched_at descending (most recent first)
|
|
134
|
+
results.sort((a, b) => {
|
|
135
|
+
const timeA = new Date(a.validatedAt || a.fetchedAt || 0).getTime();
|
|
136
|
+
const timeB = new Date(b.validatedAt || b.fetchedAt || 0).getTime();
|
|
137
|
+
return timeB - timeA;
|
|
138
|
+
});
|
|
139
|
+
const finalResults = results.slice(0, this.flags.limit);
|
|
140
|
+
this.logListResults(finalResults);
|
|
141
|
+
return finalResults;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { BaseCommand } from '../base-command.js';
|
|
2
|
+
export default class ResearchPrune extends BaseCommand<typeof ResearchPrune> {
|
|
3
|
+
static id: string;
|
|
4
|
+
static summary: string;
|
|
5
|
+
static description: string;
|
|
6
|
+
static examples: {
|
|
7
|
+
description: string;
|
|
8
|
+
command: string;
|
|
9
|
+
}[];
|
|
10
|
+
static flags: {
|
|
11
|
+
'older-than': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
12
|
+
inactive: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
13
|
+
'artifact-type': import("@oclif/core/interfaces").OptionFlag<"source" | "research_note" | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
14
|
+
'dry-run': import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
15
|
+
yes: import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
16
|
+
};
|
|
17
|
+
static stdoutIsPrimaryData: boolean;
|
|
18
|
+
init(): Promise<void>;
|
|
19
|
+
private validatePruneFlags;
|
|
20
|
+
private shouldPrune;
|
|
21
|
+
private findPruneCandidates;
|
|
22
|
+
private deletePruneCandidates;
|
|
23
|
+
execute(): Promise<unknown>;
|
|
24
|
+
}
|