@taurgis/bonsai 1.0.5 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -12
- package/agents/agents/official-docs-researcher.agent.md +15 -26
- package/agents/agents/salesforce-docs-researcher.agent.md +17 -35
- package/agents/hooks/README.md +2 -9
- package/agents/hooks/claude-code/hooks/bonsai-web-fetch.sh +1 -1
- package/agents/hooks/codex/hooks/bonsai-user-prompt-context.sh +1 -1
- package/agents/hooks/codex/hooks/bonsai-web-fetch.sh +1 -1
- package/agents/hooks/cursor/hooks/bonsai-web-fetch.sh +1 -1
- package/agents/hooks/github-copilot/bonsai-web-fetch.sh +1 -1
- package/agents/instructions/salesforce-research.instructions.md +7 -6
- package/agents/instructions/web-research.instructions.md +3 -2
- package/agents/skills/web-research/SKILL.md +24 -32
- package/bin/cli.mjs +44 -14
- package/dist/base-command.d.ts +17 -1
- package/dist/base-command.js +70 -9
- package/dist/commands/config/base.d.ts +2 -0
- package/dist/commands/config/base.js +15 -1
- package/dist/commands/config/get.d.ts +1 -1
- package/dist/commands/config/get.js +4 -5
- package/dist/commands/config/index.d.ts +12 -0
- package/dist/commands/config/index.js +34 -0
- package/dist/commands/config/list.js +2 -2
- package/dist/commands/config/set.js +18 -12
- package/dist/commands/config/unset.d.ts +1 -1
- package/dist/commands/config/unset.js +9 -6
- package/dist/commands/fetch.d.ts +2 -0
- package/dist/commands/fetch.js +69 -45
- package/dist/commands/import.d.ts +3 -0
- package/dist/commands/import.js +91 -53
- package/dist/commands/inspect.d.ts +8 -0
- package/dist/commands/inspect.js +59 -30
- package/dist/commands/list.d.ts +5 -3
- package/dist/commands/list.js +60 -35
- package/dist/commands/prune.d.ts +2 -1
- package/dist/commands/prune.js +62 -19
- package/dist/commands/status.d.ts +8 -0
- package/dist/commands/status.js +84 -26
- package/dist/commands.d.ts +2 -2
- package/dist/commands.js +2 -2
- package/dist/hooks/command-not-found/suggest.d.ts +7 -1
- package/dist/hooks/command-not-found/suggest.js +100 -46
- package/dist/lib/argv.d.ts +10 -0
- package/dist/lib/argv.js +97 -0
- package/dist/lib/color.d.ts +20 -0
- package/dist/lib/color.js +36 -0
- package/dist/lib/envelope.d.ts +36 -0
- package/dist/lib/envelope.js +82 -0
- package/dist/lib/help-preflight.d.ts +15 -0
- package/dist/lib/help-preflight.js +81 -0
- package/dist/lib/json-meta.d.ts +10 -0
- package/dist/lib/json-meta.js +98 -0
- package/dist/lib/limit-flag.d.ts +1 -0
- package/dist/lib/limit-flag.js +24 -0
- package/dist/lib/research/{search-index.d.ts → artifact-index.d.ts} +5 -5
- package/dist/lib/research/{search-index.js → artifact-index.js} +10 -10
- package/dist/lib/research/browser.d.ts +0 -8
- package/dist/lib/research/browser.js +20 -43
- package/dist/lib/research/regression.d.ts +18 -0
- package/dist/lib/research/regression.js +47 -0
- package/dist/lib/research/revalidate.js +3 -1
- package/dist/lib/research/schema.d.ts +5 -2
- package/dist/lib/research/schema.js +12 -1
- package/dist/lib/research/storage.d.ts +0 -1
- package/dist/lib/research/storage.js +15 -13
- package/dist/lib/research/url.d.ts +22 -0
- package/dist/lib/research/url.js +63 -1
- package/dist/lib/text.d.ts +5 -1
- package/dist/lib/text.js +11 -1
- package/dist/sites/coveo.d.ts +7 -0
- package/dist/sites/coveo.js +33 -0
- package/dist/sites/index.js +1 -2
- package/dist/sites/salesforce/fetch-page.js +1 -1
- package/dist/sites/salesforce/index.js +3 -8
- package/dist/sites/salesforce-developer/fetch-page.js +3 -1
- package/dist/sites/salesforce-developer/index.js +1 -2
- package/dist/sites/salesforce-doc-fetch.d.ts +11 -0
- package/dist/sites/salesforce-doc-fetch.js +117 -76
- package/dist/sites/salesforce-dom-probe.d.ts +16 -0
- package/dist/sites/salesforce-dom-probe.js +114 -0
- package/dist/sites/types.d.ts +0 -6
- package/oclif.manifest.json +153 -155
- package/package.json +38 -25
- package/dist/commands/index.d.ts +0 -35
- package/dist/commands/index.js +0 -306
- package/dist/commands/research/config/base.d.ts +0 -23
- package/dist/commands/research/config/base.js +0 -46
- package/dist/commands/research/config/get.d.ts +0 -20
- package/dist/commands/research/config/get.js +0 -45
- package/dist/commands/research/config/list.d.ts +0 -17
- package/dist/commands/research/config/list.js +0 -42
- package/dist/commands/research/config/set.d.ts +0 -31
- package/dist/commands/research/config/set.js +0 -94
- package/dist/commands/research/config/unset.d.ts +0 -21
- package/dist/commands/research/config/unset.js +0 -59
- package/dist/commands/research/import.d.ts +0 -38
- package/dist/commands/research/import.js +0 -313
- package/dist/commands/research/inspect.d.ts +0 -13
- package/dist/commands/research/inspect.js +0 -86
- package/dist/commands/research/list.d.ts +0 -25
- package/dist/commands/research/list.js +0 -143
- package/dist/commands/research/prune.d.ts +0 -24
- package/dist/commands/research/prune.js +0 -141
- package/dist/commands/research/search.d.ts +0 -32
- package/dist/commands/research/search.js +0 -334
- package/dist/commands/research/status.d.ts +0 -17
- package/dist/commands/research/status.js +0 -108
- package/dist/commands/research.d.ts +0 -36
- package/dist/commands/research.js +0 -304
- package/dist/commands/search.d.ts +0 -31
- package/dist/commands/search.js +0 -329
- package/dist/lib/research/docs/__fixtures__/sphinx-searchindex.d.ts +0 -1
- package/dist/lib/research/docs/__fixtures__/sphinx-searchindex.js +0 -2
- package/dist/lib/research/docs/discovery-index.d.ts +0 -14
- package/dist/lib/research/docs/discovery-index.js +0 -112
- package/dist/lib/research/docs/remote-search-runner.d.ts +0 -21
- package/dist/lib/research/docs/remote-search-runner.js +0 -114
- package/dist/lib/research/docs/remote-search.d.ts +0 -14
- package/dist/lib/research/docs/remote-search.js +0 -101
- package/dist/lib/research/docs/search-index.d.ts +0 -13
- package/dist/lib/research/docs/search-index.js +0 -176
- package/dist/sites/salesforce/browser-search.d.ts +0 -7
- package/dist/sites/salesforce/browser-search.js +0 -82
- package/dist/sites/salesforce/coveo.d.ts +0 -23
- package/dist/sites/salesforce/coveo.js +0 -161
- package/dist/sites/salesforce/token.d.ts +0 -26
- package/dist/sites/salesforce/token.js +0 -154
package/dist/commands/fetch.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Args, Flags } from '@oclif/core';
|
|
1
|
+
import { Args, Flags, ux, Errors } from '@oclif/core';
|
|
2
2
|
import { mkdtempSync, rmSync } from 'node:fs';
|
|
3
3
|
import { tmpdir } from 'node:os';
|
|
4
4
|
import { join } from 'node:path';
|
|
@@ -12,7 +12,6 @@ import { fetchStaticHtml, fetchText } from '../lib/research/fetcher.js';
|
|
|
12
12
|
import { fetchRenderedHtml } from '../lib/research/browser.js';
|
|
13
13
|
import { capturePage } from '../lib/research/capture.js';
|
|
14
14
|
import { persistSectionArtifacts } from '../lib/research/docs/section-artifacts.js';
|
|
15
|
-
import { resolveResearchTarget } from '../lib/research/resolve-target.js';
|
|
16
15
|
import { applyAutoTags } from '../lib/research/keywords.js';
|
|
17
16
|
import { detectSite } from '../sites/index.js';
|
|
18
17
|
const CAPTURE_DEPS = {
|
|
@@ -27,66 +26,67 @@ export default class FetchCommand extends BaseCommand {
|
|
|
27
26
|
static description = 'Scrapes the specified URL, strips HTML boilerplate, converts the semantic payload into clean Markdown format, and caches the result locally using dynamic TTL rules.\n\nUsually invoked via the shorthand `bonsai <url>` rather than `bonsai fetch <url>`.';
|
|
28
27
|
static examples = [
|
|
29
28
|
{
|
|
30
|
-
description: '
|
|
29
|
+
description: 'research a URL with detailed output, tagged with topic and tags',
|
|
31
30
|
command: '<%= config.bin %> https://docs.nestjs.com/ --topic "Backend Frameworks" --tags "Node" --tags "NestJS" --format detailed --ttl 30d',
|
|
32
31
|
},
|
|
33
32
|
{
|
|
34
|
-
description: '
|
|
33
|
+
description: 'research a volatile page with compressed output and short TTL, returned as JSON',
|
|
35
34
|
command: '<%= config.bin %> https://news.ycombinator.com/ --format compressed --ttl 2h --json',
|
|
36
35
|
},
|
|
37
36
|
];
|
|
37
|
+
static strict = false;
|
|
38
38
|
static args = {
|
|
39
39
|
url: Args.string({
|
|
40
40
|
required: true,
|
|
41
|
-
description: '
|
|
41
|
+
description: 'the full HTTP/HTTPS URL of the web page to research',
|
|
42
42
|
}),
|
|
43
43
|
};
|
|
44
44
|
static flags = {
|
|
45
45
|
topic: Flags.string({
|
|
46
46
|
char: 't',
|
|
47
|
-
description: '
|
|
47
|
+
description: 'the main category/topic of the research for metadata tagging',
|
|
48
48
|
}),
|
|
49
49
|
tags: Flags.string({
|
|
50
50
|
char: 'g',
|
|
51
|
-
description: '
|
|
51
|
+
description: 'taxonomic tags for this research (can be repeated)',
|
|
52
52
|
multiple: true,
|
|
53
53
|
}),
|
|
54
54
|
format: Flags.option({
|
|
55
55
|
char: 'f',
|
|
56
|
-
description: '
|
|
56
|
+
description: 'desired data density',
|
|
57
57
|
options: ['compressed', 'detailed'],
|
|
58
58
|
default: 'compressed',
|
|
59
59
|
})(),
|
|
60
60
|
tier: Flags.option({
|
|
61
|
-
description: '
|
|
61
|
+
description: 'freshness tier policy',
|
|
62
62
|
options: ['stable', 'standard', 'volatile'],
|
|
63
63
|
default: 'standard',
|
|
64
64
|
})(),
|
|
65
65
|
ttl: Flags.string({
|
|
66
66
|
char: 'l',
|
|
67
|
-
description: '
|
|
67
|
+
description: 'predicted lifespan: number + h/d/w/m/y (m = months), e.g. "2h", "7d", "6m"',
|
|
68
68
|
}),
|
|
69
69
|
'max-age': Flags.string({
|
|
70
|
-
description: '
|
|
70
|
+
description: 'maximum age of cache to accept (e.g., "1d", "30d")',
|
|
71
71
|
}),
|
|
72
72
|
force: Flags.boolean({
|
|
73
|
-
description: '
|
|
73
|
+
description: 'force a fresh fetch, ignoring any cached entries',
|
|
74
74
|
default: false,
|
|
75
75
|
}),
|
|
76
76
|
'dry-run': Flags.boolean({
|
|
77
|
-
description: '
|
|
77
|
+
description: 'perform validation and fetch without saving to local cache',
|
|
78
78
|
default: false,
|
|
79
79
|
}),
|
|
80
80
|
'allow-stale': Flags.boolean({
|
|
81
|
-
description: '
|
|
81
|
+
description: 'allow serving stale cache if the remote server is unreachable',
|
|
82
82
|
default: false,
|
|
83
83
|
}),
|
|
84
84
|
rendered: Flags.boolean({
|
|
85
|
-
description: '
|
|
85
|
+
description: 'force using a browser-rendered scraping path for dynamic pages',
|
|
86
86
|
default: false,
|
|
87
87
|
}),
|
|
88
88
|
storage: Flags.option({
|
|
89
|
-
description: '
|
|
89
|
+
description: 'override where this result is cached (secrets always stored globally)',
|
|
90
90
|
options: ['global', 'project'],
|
|
91
91
|
})(),
|
|
92
92
|
};
|
|
@@ -185,7 +185,7 @@ export default class FetchCommand extends BaseCommand {
|
|
|
185
185
|
validateDurationFlags(ttl, maxAge) {
|
|
186
186
|
for (const msg of [durationFlagError('--ttl', ttl), durationFlagError('--max-age', maxAge)]) {
|
|
187
187
|
if (msg)
|
|
188
|
-
this.error(msg, { exit: 2 });
|
|
188
|
+
this.error(msg, { exit: 2, code: 'INVALID_DURATION' });
|
|
189
189
|
}
|
|
190
190
|
}
|
|
191
191
|
// Re-emit a runtime fetch failure with actionable next steps. Deep fetch/extract code throws plain
|
|
@@ -197,7 +197,12 @@ export default class FetchCommand extends BaseCommand {
|
|
|
197
197
|
emitFetchError(err, url) {
|
|
198
198
|
const message = err instanceof Error ? err.message : String(err);
|
|
199
199
|
const guidance = fetchFailureGuidance(message, url);
|
|
200
|
-
this.error(message, {
|
|
200
|
+
this.error(message, {
|
|
201
|
+
exit: 1,
|
|
202
|
+
code: 'FETCH_FAILED',
|
|
203
|
+
suggestions: guidance?.suggestions,
|
|
204
|
+
ref: guidance?.ref,
|
|
205
|
+
});
|
|
201
206
|
}
|
|
202
207
|
// Serves the cache when a fresh/revalidatable entry exists, otherwise fetches fresh. Reads fall
|
|
203
208
|
// back project→global; the resolved artifact is returned with the dir it lives in / landed in.
|
|
@@ -251,43 +256,31 @@ export default class FetchCommand extends BaseCommand {
|
|
|
251
256
|
}
|
|
252
257
|
}
|
|
253
258
|
async run() {
|
|
254
|
-
const
|
|
259
|
+
const urls = this.parsedArgv;
|
|
255
260
|
const { format, ttl, 'max-age': maxAge, 'dry-run': dryRun } = this.flags;
|
|
256
261
|
this.validateDurationFlags(ttl, maxAge);
|
|
257
|
-
let target;
|
|
258
|
-
try {
|
|
259
|
-
target = resolveResearchTarget({
|
|
260
|
-
configDir: this.config.configDir,
|
|
261
|
-
cwd: process.cwd(),
|
|
262
|
-
dataDir: this.config.dataDir,
|
|
263
|
-
flagOverride: this.flags.storage,
|
|
264
|
-
lookup: !this.flags.force,
|
|
265
|
-
url,
|
|
266
|
-
});
|
|
267
|
-
}
|
|
268
|
-
catch (err) {
|
|
269
|
-
this.error(`Invalid URL: ${err.message}`, { exit: 2 });
|
|
270
|
-
}
|
|
271
|
-
const { cacheKey, located, normalizedUrl, roots } = target;
|
|
272
262
|
const summaryLevel = loadSummaryLevel(this.config.configDir, process.cwd());
|
|
273
263
|
const tmpDir = dryRun ? mkdtempSync(join(tmpdir(), 'fnr-dry-run-')) : null;
|
|
274
264
|
const currentTime = new Date();
|
|
265
|
+
const results = [];
|
|
275
266
|
try {
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
const content = format === 'compressed' ? artifact.compressed : artifact.detailed;
|
|
279
|
-
const resultData = this.buildResultData(url, normalizedUrl, cacheKey, storageDir, roots.mode, cacheStatus, freshnessState, format, artifact, content, redirectedToGlobal);
|
|
280
|
-
if (!this.jsonEnabled()) {
|
|
281
|
-
this.log(content);
|
|
267
|
+
for (const url of urls) {
|
|
268
|
+
results.push(await this.fetchSingleTarget(url, currentTime, tmpDir, summaryLevel, format, urls.length > 1));
|
|
282
269
|
}
|
|
283
|
-
return
|
|
270
|
+
return urls.length === 1 ? results[0] : results;
|
|
284
271
|
}
|
|
285
272
|
catch (err) {
|
|
273
|
+
if (!this.jsonEnabled()) {
|
|
274
|
+
ux.action.stop('failed');
|
|
275
|
+
}
|
|
276
|
+
if (err instanceof Errors.CLIError)
|
|
277
|
+
throw err;
|
|
286
278
|
// Stale-serve (exit 5) signals via process.exitCode and never throws, so it bypasses this
|
|
287
279
|
// path; only genuine fetch/extract failures land here and get classified guidance. Use the
|
|
288
280
|
// normalized URL in hints so the copy-paste command is canonical (and never echoes raw,
|
|
289
281
|
// unsanitized user input back to the terminal).
|
|
290
|
-
|
|
282
|
+
const failedUrl = urls[results.length] || urls[0] || '';
|
|
283
|
+
this.emitFetchError(err, failedUrl);
|
|
291
284
|
}
|
|
292
285
|
finally {
|
|
293
286
|
if (tmpDir) {
|
|
@@ -295,13 +288,44 @@ export default class FetchCommand extends BaseCommand {
|
|
|
295
288
|
}
|
|
296
289
|
}
|
|
297
290
|
}
|
|
291
|
+
async fetchSingleTarget(url, currentTime, tmpDir, summaryLevel, format, showSeparator) {
|
|
292
|
+
const target = this.resolveResearchTargetOrFail(url, {
|
|
293
|
+
flagOverride: this.flags.storage,
|
|
294
|
+
lookup: !this.flags.force,
|
|
295
|
+
});
|
|
296
|
+
const { cacheKey, located, normalizedUrl, roots } = target;
|
|
297
|
+
if (!this.jsonEnabled()) {
|
|
298
|
+
ux.action.start('Fetching ' + normalizedUrl);
|
|
299
|
+
}
|
|
300
|
+
const { cacheStatus, freshnessState, artifact, storageDir, redirectedToGlobal } = await this.resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime, located, summaryLevel);
|
|
301
|
+
this.persistSectionsIfFresh(tmpDir ?? storageDir, artifact, currentTime, cacheStatus, summaryLevel);
|
|
302
|
+
if (!this.jsonEnabled()) {
|
|
303
|
+
const CACHE_STATUS_LABEL = {
|
|
304
|
+
hit: 'cached',
|
|
305
|
+
miss: 'done',
|
|
306
|
+
refreshed: 'refreshed',
|
|
307
|
+
revalidated: 'revalidated',
|
|
308
|
+
stale: 'served stale',
|
|
309
|
+
};
|
|
310
|
+
ux.action.stop(CACHE_STATUS_LABEL[cacheStatus] ?? cacheStatus);
|
|
311
|
+
}
|
|
312
|
+
const content = format === 'compressed' ? artifact.compressed : artifact.detailed;
|
|
313
|
+
const resultData = this.buildResultData(url, normalizedUrl, cacheKey, storageDir, roots.mode, cacheStatus, freshnessState, format, artifact, content, redirectedToGlobal);
|
|
314
|
+
if (!this.jsonEnabled()) {
|
|
315
|
+
this.log(content);
|
|
316
|
+
if (showSeparator) {
|
|
317
|
+
this.log('\n' + '='.repeat(40) + '\n');
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
return resultData;
|
|
321
|
+
}
|
|
298
322
|
}
|
|
299
323
|
// Maps a runtime fetch failure to recovery steps keyed off its message. Returns undefined for
|
|
300
324
|
// unrecognized failures, which then surface with their original message and no extra hint. The
|
|
301
325
|
// patterns match the thrown text in fetcher.ts/browser.ts; keep them in sync if those messages
|
|
302
326
|
// change. Resolutions mirror the published troubleshooting guide.
|
|
303
327
|
export function fetchFailureGuidance(message, url) {
|
|
304
|
-
const ref = 'https://
|
|
328
|
+
const ref = 'https://bonsai.rhino-inquisitor.com/troubleshooting';
|
|
305
329
|
// Name the pipe step explicitly: bare `import --stdin` blocks waiting for input, so a reader (or
|
|
306
330
|
// an agent) running the hint verbatim would hang. Show the content being piped in from a file.
|
|
307
331
|
const importHint = `Open it in a browser, save the page, then import it: cat page.md | bonsai import ${url} --stdin`;
|
|
@@ -367,10 +391,10 @@ function handleStaleRevalidationResult(command, revalResult) {
|
|
|
367
391
|
if (revalResult.status !== 'stale')
|
|
368
392
|
return;
|
|
369
393
|
if (revalResult.allowed) {
|
|
370
|
-
command.warn(`Serving stale content: revalidation failed
|
|
394
|
+
command.warn(`Serving stale content within grace period: revalidation failed (${revalResult.error}).`);
|
|
371
395
|
}
|
|
372
396
|
else {
|
|
373
|
-
command.warn(`Serving stale content (exit 5): revalidation failed
|
|
397
|
+
command.warn(`Serving stale content within grace period (exit 5): revalidation failed (${revalResult.error}).`);
|
|
374
398
|
process.exitCode = 5;
|
|
375
399
|
}
|
|
376
400
|
}
|
|
@@ -30,6 +30,9 @@ export default class ResearchImport extends BaseCommand<typeof ResearchImport> {
|
|
|
30
30
|
protected fsExistsSync(filePath: string): boolean;
|
|
31
31
|
protected fsStatSync(filePath: string): fs.Stats;
|
|
32
32
|
protected fsReadFileSync(filePath: string): string;
|
|
33
|
+
private stdinImportSuggestions;
|
|
34
|
+
/** Non-interactive stdin that never receives data (open pipe with no writer) must not hang forever. */
|
|
35
|
+
private readStdinWithGuard;
|
|
33
36
|
private readAndValidateFile;
|
|
34
37
|
private readAndValidateInput;
|
|
35
38
|
private deriveImportCacheKey;
|
package/dist/commands/import.js
CHANGED
|
@@ -9,7 +9,7 @@ import { getArtifactPath } from '../lib/research/storage.js';
|
|
|
9
9
|
import { loadStoreRoots } from '../lib/research/store-roots.js';
|
|
10
10
|
import { writeArtifactSecurely } from '../lib/research/secure-write.js';
|
|
11
11
|
import { loadSummaryLevel } from '../lib/config/index.js';
|
|
12
|
-
import { getPolicy } from '../lib/research/freshness.js';
|
|
12
|
+
import { durationFlagError, getPolicy } from '../lib/research/freshness.js';
|
|
13
13
|
import { buildCompressed } from '../lib/research/compress.js';
|
|
14
14
|
import { applyAutoTags } from '../lib/research/keywords.js';
|
|
15
15
|
import { estimateTokens } from '../lib/research/token-estimate.js';
|
|
@@ -19,17 +19,21 @@ export default class ResearchImport extends BaseCommand {
|
|
|
19
19
|
static description = 'Accepts a synthesized Markdown note via stdin and registers it under the single target URL or multiple source URLs.';
|
|
20
20
|
static examples = [
|
|
21
21
|
{
|
|
22
|
-
description: '
|
|
22
|
+
description: 'import detailed research for a single URL via stdin',
|
|
23
23
|
command: 'echo "# My Article" | <%= config.bin %> import https://example.com/docs --stdin',
|
|
24
24
|
},
|
|
25
25
|
{
|
|
26
|
-
description: '
|
|
26
|
+
description: 'import research synthesized from multiple source URLs',
|
|
27
27
|
command: 'echo "# Synthesized" | <%= config.bin %> import --stdin --topic "React docs" --source-url https://react.dev/a --source-url https://react.dev/b',
|
|
28
28
|
},
|
|
29
29
|
{
|
|
30
|
-
description: '
|
|
30
|
+
description: 'import research from a local Markdown file',
|
|
31
31
|
command: '<%= config.bin %> import https://example.com/docs --file path/to/notes.md',
|
|
32
32
|
},
|
|
33
|
+
{
|
|
34
|
+
description: 'import research from stdin using the standard file placeholder',
|
|
35
|
+
command: 'cat notes.md | <%= config.bin %> import https://example.com/docs --file -',
|
|
36
|
+
},
|
|
33
37
|
];
|
|
34
38
|
static args = {
|
|
35
39
|
url: Args.string({
|
|
@@ -38,46 +42,46 @@ export default class ResearchImport extends BaseCommand {
|
|
|
38
42
|
// this, the Markdown piped for `--stdin` is swallowed into `url`, making multi-source import
|
|
39
43
|
// (`import --stdin --source-url ...`) wrongly look like it also got a positional URL.
|
|
40
44
|
ignoreStdin: true,
|
|
41
|
-
description: '
|
|
45
|
+
description: 'the single source URL of the page (only for single-source import)',
|
|
42
46
|
}),
|
|
43
47
|
};
|
|
44
48
|
static flags = {
|
|
45
49
|
stdin: Flags.boolean({
|
|
46
|
-
description: '
|
|
50
|
+
description: 'read Markdown from stdin',
|
|
47
51
|
default: false,
|
|
48
52
|
}),
|
|
49
53
|
file: Flags.string({
|
|
50
|
-
description: '
|
|
54
|
+
description: 'path to a Markdown file containing research notes to import, or "-" for stdin',
|
|
51
55
|
}),
|
|
52
56
|
'input-format': Flags.option({
|
|
53
|
-
description: '
|
|
57
|
+
description: 'format of the input provided',
|
|
54
58
|
options: ['compressed', 'detailed'],
|
|
55
59
|
default: 'detailed',
|
|
56
60
|
})(),
|
|
57
61
|
topic: Flags.string({
|
|
58
62
|
char: 't',
|
|
59
|
-
description: '
|
|
63
|
+
description: 'the main topic for this research note',
|
|
60
64
|
}),
|
|
61
65
|
tags: Flags.string({
|
|
62
66
|
char: 'g',
|
|
63
|
-
description: '
|
|
67
|
+
description: 'taxonomic tags (can be repeated)',
|
|
64
68
|
multiple: true,
|
|
65
69
|
}),
|
|
66
70
|
tier: Flags.option({
|
|
67
|
-
description: '
|
|
71
|
+
description: 'freshness tier policy',
|
|
68
72
|
options: ['stable', 'standard', 'volatile'],
|
|
69
73
|
default: 'standard',
|
|
70
74
|
})(),
|
|
71
75
|
'source-url': Flags.string({
|
|
72
|
-
description: '
|
|
76
|
+
description: 'repeated source URLs for multi-source import',
|
|
73
77
|
multiple: true,
|
|
74
78
|
}),
|
|
75
79
|
ttl: Flags.string({
|
|
76
80
|
char: 'l',
|
|
77
|
-
description: '
|
|
81
|
+
description: 'predicted lifespan of the data (e.g. "24h", "7d")',
|
|
78
82
|
}),
|
|
79
83
|
storage: Flags.option({
|
|
80
|
-
description: '
|
|
84
|
+
description: 'override where this note is cached (secrets always stored globally)',
|
|
81
85
|
options: ['global', 'project'],
|
|
82
86
|
})(),
|
|
83
87
|
};
|
|
@@ -121,28 +125,40 @@ export default class ResearchImport extends BaseCommand {
|
|
|
121
125
|
return normalizeUrl(u);
|
|
122
126
|
}
|
|
123
127
|
catch (err) {
|
|
124
|
-
|
|
128
|
+
// Shared exit point: a forgotten scheme reports MISSING_URL_SCHEME like every other command,
|
|
129
|
+
// while genuinely malformed input stays INVALID_URL.
|
|
130
|
+
this.failInvalidUrl(u, err.message);
|
|
125
131
|
}
|
|
126
132
|
});
|
|
127
133
|
return hasMulti ? normalized.sort() : normalized;
|
|
128
134
|
}
|
|
129
135
|
validateSourceMode(hasSingle, hasMulti, multiUrls) {
|
|
130
136
|
if (hasSingle && hasMulti) {
|
|
131
|
-
this.error('Cannot specify both positional <url> and --source-url flags.', {
|
|
137
|
+
this.error('Cannot specify both positional <url> and --source-url flags.', {
|
|
138
|
+
exit: 2,
|
|
139
|
+
code: 'CONFLICTING_FLAGS',
|
|
140
|
+
});
|
|
132
141
|
}
|
|
133
142
|
if (!hasSingle && !hasMulti) {
|
|
134
|
-
this.error('Must specify either positional <url> (for single-source) or --source-url (for multi-source) import.', { exit: 2 });
|
|
143
|
+
this.error('Must specify either positional <url> (for single-source) or --source-url (for multi-source) import.', { exit: 2, code: 'MISSING_URL' });
|
|
135
144
|
}
|
|
136
145
|
if (hasMulti && !this.flags.topic) {
|
|
137
|
-
this.error('Multi-source import requires the --topic flag.', {
|
|
146
|
+
this.error('Multi-source import requires the --topic flag.', {
|
|
147
|
+
exit: 2,
|
|
148
|
+
code: 'MISSING_TOPIC',
|
|
149
|
+
});
|
|
138
150
|
}
|
|
139
151
|
if (!this.flags.stdin && !this.flags.file) {
|
|
140
152
|
this.error('Either --stdin or --file <path> must be specified to import content.', {
|
|
141
153
|
exit: 2,
|
|
154
|
+
code: 'MISSING_INPUT',
|
|
142
155
|
});
|
|
143
156
|
}
|
|
144
157
|
if (this.flags.stdin && this.flags.file) {
|
|
145
|
-
this.error('Cannot specify both --stdin and --file <path>. Choose one.', {
|
|
158
|
+
this.error('Cannot specify both --stdin and --file <path>. Choose one.', {
|
|
159
|
+
exit: 2,
|
|
160
|
+
code: 'CONFLICTING_FLAGS',
|
|
161
|
+
});
|
|
146
162
|
}
|
|
147
163
|
}
|
|
148
164
|
fsExistsSync(filePath) {
|
|
@@ -154,51 +170,70 @@ export default class ResearchImport extends BaseCommand {
|
|
|
154
170
|
fsReadFileSync(filePath) {
|
|
155
171
|
return fs.readFileSync(filePath, 'utf8');
|
|
156
172
|
}
|
|
173
|
+
stdinImportSuggestions() {
|
|
174
|
+
const bin = this.config.bin;
|
|
175
|
+
return [
|
|
176
|
+
`Pipe Markdown content, e.g. cat notes.md | ${bin} import <url> --stdin`,
|
|
177
|
+
`Or use the standard stdin placeholder: cat notes.md | ${bin} import <url> --file -`,
|
|
178
|
+
`Or read from a file: ${bin} import <url> --file notes.md`,
|
|
179
|
+
];
|
|
180
|
+
}
|
|
181
|
+
/** Non-interactive stdin that never receives data (open pipe with no writer) must not hang forever. */
|
|
182
|
+
async readStdinWithGuard(limitBytes = 1024 * 1024) {
|
|
183
|
+
if (this.stdinIsInteractive()) {
|
|
184
|
+
this.error('No data piped to --stdin.', {
|
|
185
|
+
exit: 2,
|
|
186
|
+
code: 'MISSING_STDIN',
|
|
187
|
+
suggestions: this.stdinImportSuggestions(),
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
const timeoutMs = 1000;
|
|
191
|
+
let rawInput = '';
|
|
192
|
+
try {
|
|
193
|
+
rawInput = await Promise.race([
|
|
194
|
+
this.readStdin(limitBytes),
|
|
195
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error('STDIN_TIMEOUT')), timeoutMs)),
|
|
196
|
+
]);
|
|
197
|
+
}
|
|
198
|
+
catch (err) {
|
|
199
|
+
if (err.message === 'STDIN_TIMEOUT') {
|
|
200
|
+
this.error('No stdin data received. Pipe content or use --file.', {
|
|
201
|
+
exit: 2,
|
|
202
|
+
code: 'MISSING_STDIN',
|
|
203
|
+
suggestions: this.stdinImportSuggestions(),
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
if (err.message.includes('stdin size limit exceeded')) {
|
|
207
|
+
this.error(err.message, { exit: 1, code: 'STDIN_TOO_LARGE' });
|
|
208
|
+
}
|
|
209
|
+
// Genuine stream failure reading stdin → I/O failure (exit 1).
|
|
210
|
+
this.error(err.message, { exit: 1, code: 'IO_ERROR' });
|
|
211
|
+
}
|
|
212
|
+
return rawInput;
|
|
213
|
+
}
|
|
157
214
|
readAndValidateFile(filePath) {
|
|
158
215
|
if (!this.fsExistsSync(filePath)) {
|
|
159
|
-
this.error(`File does not exist: ${filePath}`, { exit: 2 });
|
|
216
|
+
this.error(`File does not exist: ${filePath}`, { exit: 2, code: 'FILE_NOT_FOUND' });
|
|
160
217
|
}
|
|
161
218
|
const stat = this.fsStatSync(filePath);
|
|
162
219
|
if (!stat.isFile()) {
|
|
163
|
-
this.error(`Path is not a file: ${filePath}`, { exit: 2 });
|
|
220
|
+
this.error(`Path is not a file: ${filePath}`, { exit: 2, code: 'NOT_A_FILE' });
|
|
164
221
|
}
|
|
165
222
|
if (stat.size > 1024 * 1024) {
|
|
166
|
-
this.error('File size limit exceeded (max 1 MiB).', { exit: 1 });
|
|
223
|
+
this.error('File size limit exceeded (max 1 MiB).', { exit: 1, code: 'FILE_TOO_LARGE' });
|
|
167
224
|
}
|
|
168
225
|
const content = this.fsReadFileSync(filePath);
|
|
169
226
|
if (!content.trim()) {
|
|
170
|
-
this.error('Empty file content provided.', { exit: 2 });
|
|
227
|
+
this.error('Empty file content provided.', { exit: 2, code: 'EMPTY_INPUT' });
|
|
171
228
|
}
|
|
172
229
|
return content;
|
|
173
230
|
}
|
|
174
231
|
async readAndValidateInput() {
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
// (the agent and CI path) has isTTY falsy and reads normally.
|
|
179
|
-
if (this.stdinIsInteractive()) {
|
|
180
|
-
this.error('No data piped to --stdin.', {
|
|
181
|
-
exit: 2,
|
|
182
|
-
suggestions: [
|
|
183
|
-
'Pipe Markdown content, e.g. cat notes.md | bonsai import <url> --stdin',
|
|
184
|
-
'Or read from a file: bonsai import <url> --file notes.md',
|
|
185
|
-
],
|
|
186
|
-
});
|
|
187
|
-
}
|
|
188
|
-
// Initialized so the post-catch read below never depends on this.error()'s `never` return type
|
|
189
|
-
// for definite assignment; the empty-content guard rejects '' as a usage error anyway.
|
|
190
|
-
let rawInput = '';
|
|
191
|
-
try {
|
|
192
|
-
rawInput = await this.readStdin();
|
|
193
|
-
}
|
|
194
|
-
catch (err) {
|
|
195
|
-
// Genuine stream/size failure reading stdin → I/O failure (exit 1).
|
|
196
|
-
this.error(err.message, { exit: 1 });
|
|
197
|
-
}
|
|
198
|
-
// Empty input is a usage error (exit 2), not an I/O failure — keep it out of the
|
|
199
|
-
// catch above so its exit code is not clobbered to 1.
|
|
232
|
+
const fileReadsStdin = this.flags.file === '-';
|
|
233
|
+
if (this.flags.stdin || fileReadsStdin) {
|
|
234
|
+
const rawInput = await this.readStdinWithGuard();
|
|
200
235
|
if (!rawInput.trim()) {
|
|
201
|
-
this.error('Empty stdin content provided.', { exit: 2 });
|
|
236
|
+
this.error('Empty stdin content provided.', { exit: 2, code: 'EMPTY_INPUT' });
|
|
202
237
|
}
|
|
203
238
|
return rawInput;
|
|
204
239
|
}
|
|
@@ -213,10 +248,10 @@ export default class ResearchImport extends BaseCommand {
|
|
|
213
248
|
// Only an unexpected read failure becomes a wrapped I/O failure (exit 1).
|
|
214
249
|
if (err instanceof Errors.CLIError)
|
|
215
250
|
throw err;
|
|
216
|
-
this.error(`Failed to read file: ${err.message}`, { exit: 1 });
|
|
251
|
+
this.error(`Failed to read file: ${err.message}`, { exit: 1, code: 'IO_ERROR' });
|
|
217
252
|
}
|
|
218
253
|
}
|
|
219
|
-
this.error('No input source specified.', { exit: 2 });
|
|
254
|
+
this.error('No input source specified.', { exit: 2, code: 'MISSING_INPUT' });
|
|
220
255
|
}
|
|
221
256
|
deriveImportCacheKey(hasSingle, singleNormalizedUrl, sourceUrls) {
|
|
222
257
|
if (hasSingle) {
|
|
@@ -292,6 +327,9 @@ export default class ResearchImport extends BaseCommand {
|
|
|
292
327
|
const hasSingle = Boolean(this.args.url);
|
|
293
328
|
const multiUrls = this.flags['source-url'] || [];
|
|
294
329
|
const hasMulti = multiUrls.length > 0;
|
|
330
|
+
const ttlErr = durationFlagError('--ttl', this.flags.ttl);
|
|
331
|
+
if (ttlErr)
|
|
332
|
+
this.error(ttlErr, { exit: 2, code: 'INVALID_DURATION' });
|
|
295
333
|
this.validateSourceMode(hasSingle, hasMulti, multiUrls);
|
|
296
334
|
const rawInput = await this.readAndValidateInput();
|
|
297
335
|
const sourceUrls = this.getSourceUrls(hasMulti, multiUrls, this.args.url);
|
|
@@ -311,8 +349,8 @@ export default class ResearchImport extends BaseCommand {
|
|
|
311
349
|
}
|
|
312
350
|
if (!this.jsonEnabled()) {
|
|
313
351
|
this.log(`Successfully imported research artifact.`);
|
|
314
|
-
this.log(
|
|
315
|
-
this.log(
|
|
352
|
+
this.log(`${'Cache Key:'.padEnd(25)} ${cacheKey}`);
|
|
353
|
+
this.log(`${'Storage Path:'.padEnd(25)} ${storagePath}`);
|
|
316
354
|
}
|
|
317
355
|
const inputFormat = this.flags['input-format'];
|
|
318
356
|
return {
|
|
@@ -3,10 +3,18 @@ export default class ResearchInspect extends BaseCommand<typeof ResearchInspect>
|
|
|
3
3
|
static id: string;
|
|
4
4
|
static summary: string;
|
|
5
5
|
static description: string;
|
|
6
|
+
static examples: {
|
|
7
|
+
description: string;
|
|
8
|
+
command: string;
|
|
9
|
+
}[];
|
|
10
|
+
static strict: boolean;
|
|
6
11
|
static args: {
|
|
7
12
|
url: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
|
|
8
13
|
};
|
|
9
14
|
static stdoutIsPrimaryData: boolean;
|
|
10
15
|
run(): Promise<unknown>;
|
|
16
|
+
private logMetadata;
|
|
17
|
+
private logSections;
|
|
18
|
+
private inspectSingleTarget;
|
|
11
19
|
private findSections;
|
|
12
20
|
}
|
package/dist/commands/inspect.js
CHANGED
|
@@ -1,56 +1,85 @@
|
|
|
1
1
|
import { Args } from '@oclif/core';
|
|
2
2
|
import { BaseCommand } from '../base-command.js';
|
|
3
3
|
import { scanCacheDirs } from '../lib/research/storage.js';
|
|
4
|
-
import {
|
|
4
|
+
import { colors } from '../lib/color.js';
|
|
5
5
|
export default class ResearchInspect extends BaseCommand {
|
|
6
6
|
static id = 'inspect';
|
|
7
7
|
static summary = 'Inspect cached research metadata for a URL.';
|
|
8
8
|
static description = 'Inspects local storage and displays full frontmatter metadata for the URL (without printing the content body).';
|
|
9
|
+
static examples = [
|
|
10
|
+
{
|
|
11
|
+
description: 'inspect cached research metadata for a specific URL',
|
|
12
|
+
command: '<%= config.bin %> <%= command.id %> https://example.com/docs',
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
description: 'output the full metadata as JSON for use by other tools',
|
|
16
|
+
command: '<%= config.bin %> <%= command.id %> https://example.com/docs --json',
|
|
17
|
+
},
|
|
18
|
+
];
|
|
19
|
+
static strict = false;
|
|
9
20
|
static args = {
|
|
10
21
|
url: Args.string({
|
|
11
22
|
required: true,
|
|
12
|
-
description: '
|
|
23
|
+
description: 'the URL to inspect',
|
|
13
24
|
}),
|
|
14
25
|
};
|
|
15
26
|
static stdoutIsPrimaryData = true;
|
|
16
27
|
async run() {
|
|
17
|
-
const
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
28
|
+
const urls = this.parsedArgv;
|
|
29
|
+
const results = [];
|
|
30
|
+
const missingUrls = [];
|
|
31
|
+
for (const url of urls) {
|
|
32
|
+
const target = this.resolveResearchTargetOrFail(url);
|
|
33
|
+
if (!target.located) {
|
|
34
|
+
missingUrls.push(target.normalizedUrl);
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
results.push(this.inspectSingleTarget(target, urls.length > 1));
|
|
38
|
+
}
|
|
39
|
+
if (missingUrls.length > 0) {
|
|
40
|
+
const firstMissing = missingUrls[0];
|
|
41
|
+
const suggestions = missingUrls.map((u) => `Fetch and cache it first: ${this.config.bin} ${u}`);
|
|
42
|
+
this.error(`No cached research found for URL: ${firstMissing}` +
|
|
43
|
+
(missingUrls.length > 1 ? ` and ${missingUrls.length - 1} other URLs` : ''), {
|
|
44
|
+
exit: 1,
|
|
45
|
+
code: 'CACHE_MISS',
|
|
46
|
+
suggestions,
|
|
25
47
|
});
|
|
26
48
|
}
|
|
27
|
-
|
|
28
|
-
|
|
49
|
+
return urls.length === 1 ? results[0] : results;
|
|
50
|
+
}
|
|
51
|
+
logMetadata(metadata) {
|
|
52
|
+
this.log(colors.cyan(`--- Metadata ---`));
|
|
53
|
+
for (const [key, val] of Object.entries(metadata)) {
|
|
54
|
+
const paddedKey = colors.cyan((key + ':').padEnd(Math.max(25, key.length + 2)));
|
|
55
|
+
if (typeof val === 'object' && val !== null) {
|
|
56
|
+
this.log(`${paddedKey} ${colors.bold(JSON.stringify(val))}`);
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
this.log(`${paddedKey} ${colors.bold(String(val))}`);
|
|
60
|
+
}
|
|
29
61
|
}
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
62
|
+
}
|
|
63
|
+
logSections(sections) {
|
|
64
|
+
this.log(colors.cyan(`--- Sections (${sections.length}) ---`));
|
|
65
|
+
for (const s of sections) {
|
|
66
|
+
this.log(`${colors.cyan(s.headingPath || '')} [${colors.yellow(s.anchor || '')}] (${colors.magenta(String(s.tokenEstimate.detailed || 0))} tokens) ${colors.gray(s.cacheKey)}`);
|
|
33
67
|
}
|
|
68
|
+
}
|
|
69
|
+
inspectSingleTarget(target, showSeparator) {
|
|
70
|
+
const { cacheKey, located, roots } = target;
|
|
34
71
|
const cached = located.artifact;
|
|
35
72
|
const artifactPath = located.path;
|
|
36
73
|
const sections = this.findSections(roots.readRoots, cacheKey);
|
|
37
74
|
if (!this.jsonEnabled()) {
|
|
38
|
-
this.log(
|
|
39
|
-
this.log(
|
|
40
|
-
this.
|
|
41
|
-
for (const [key, val] of Object.entries(cached.metadata)) {
|
|
42
|
-
if (typeof val === 'object' && val !== null) {
|
|
43
|
-
this.log(`${key}: ${JSON.stringify(val)}`);
|
|
44
|
-
}
|
|
45
|
-
else {
|
|
46
|
-
this.log(`${key}: ${val}`);
|
|
47
|
-
}
|
|
48
|
-
}
|
|
75
|
+
this.log(`${colors.cyan('Cache Key:'.padEnd(25))} ${colors.bold(cacheKey)}`);
|
|
76
|
+
this.log(`${colors.cyan('Cache Path:'.padEnd(25))} ${colors.gray(artifactPath)}`);
|
|
77
|
+
this.logMetadata(cached.metadata);
|
|
49
78
|
if (sections.length) {
|
|
50
|
-
this.
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
79
|
+
this.logSections(sections);
|
|
80
|
+
}
|
|
81
|
+
if (showSeparator) {
|
|
82
|
+
this.log('='.repeat(40));
|
|
54
83
|
}
|
|
55
84
|
}
|
|
56
85
|
return {
|