@taurgis/bonsai 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +81 -2
- package/agents/agents/official-docs-researcher.agent.md +10 -1
- package/agents/agents/salesforce-docs-researcher.agent.md +126 -0
- package/agents/hooks/README.md +62 -0
- package/agents/hooks/claude-code/hooks/bonsai-web-fetch.sh +22 -0
- package/agents/hooks/claude-code/settings.json +16 -0
- package/agents/hooks/codex/hooks/bonsai-user-prompt-context.sh +18 -0
- package/agents/hooks/codex/hooks/bonsai-web-fetch.sh +22 -0
- package/agents/hooks/codex/hooks.json +28 -0
- package/agents/hooks/cursor/hooks/bonsai-web-fetch.sh +21 -0
- package/agents/hooks/cursor/hooks.json +12 -0
- package/agents/hooks/github-copilot/bonsai-web-fetch.json +23 -0
- package/agents/hooks/github-copilot/bonsai-web-fetch.sh +19 -0
- package/agents/instructions/salesforce-research.instructions.md +59 -0
- package/agents/instructions/web-research.instructions.md +5 -2
- package/agents/skills/web-research/SKILL.md +8 -3
- package/bin/cli.mjs +35 -8
- package/dist/base-command.d.ts +23 -0
- package/dist/base-command.js +67 -8
- package/dist/commands/config/base.d.ts +2 -0
- package/dist/commands/config/base.js +15 -1
- package/dist/commands/config/get.d.ts +1 -1
- package/dist/commands/config/get.js +6 -7
- package/dist/commands/config/index.d.ts +12 -0
- package/dist/commands/config/index.js +34 -0
- package/dist/commands/config/list.js +4 -4
- package/dist/commands/config/set.js +20 -14
- package/dist/commands/config/unset.d.ts +1 -1
- package/dist/commands/config/unset.js +11 -8
- package/dist/commands/fetch.d.ts +5 -0
- package/dist/commands/fetch.js +122 -47
- package/dist/commands/import.d.ts +4 -0
- package/dist/commands/import.js +99 -37
- package/dist/commands/inspect.d.ts +4 -0
- package/dist/commands/inspect.js +24 -19
- package/dist/commands/list.d.ts +2 -2
- package/dist/commands/list.js +39 -16
- package/dist/commands/prune.d.ts +1 -1
- package/dist/commands/prune.js +50 -17
- package/dist/commands/search.d.ts +4 -0
- package/dist/commands/search.js +102 -41
- package/dist/commands/status.d.ts +6 -0
- package/dist/commands/status.js +58 -31
- package/dist/commands.d.ts +2 -0
- package/dist/commands.js +2 -0
- package/dist/hooks/command-not-found/suggest.d.ts +9 -0
- package/dist/hooks/command-not-found/suggest.js +122 -0
- package/dist/lib/argv.d.ts +10 -0
- package/dist/lib/argv.js +53 -0
- package/dist/lib/config/index.d.ts +1 -1
- package/dist/lib/config/index.js +1 -1
- package/dist/lib/config/resolve.d.ts +7 -0
- package/dist/lib/config/resolve.js +23 -0
- package/dist/lib/envelope.d.ts +36 -0
- package/dist/lib/envelope.js +76 -0
- package/dist/lib/json-meta.d.ts +10 -0
- package/dist/lib/json-meta.js +98 -0
- package/dist/lib/research/browser.d.ts +12 -0
- package/dist/lib/research/browser.js +40 -3
- package/dist/lib/research/docs/capabilities.d.ts +1 -1
- package/dist/lib/research/docs/discovery-index.d.ts +14 -0
- package/dist/lib/research/docs/discovery-index.js +112 -0
- package/dist/lib/research/docs/machine-readable.d.ts +2 -2
- package/dist/lib/research/docs/machine-readable.js +3 -7
- package/dist/lib/research/docs/remote-search-runner.js +61 -1
- package/dist/lib/research/docs/sections.d.ts +1 -0
- package/dist/lib/research/docs/sections.js +19 -1
- package/dist/lib/research/freshness.d.ts +9 -1
- package/dist/lib/research/freshness.js +24 -2
- package/dist/lib/research/keywords.d.ts +10 -3
- package/dist/lib/research/keywords.js +32 -5
- package/dist/lib/research/revalidate.js +3 -1
- package/dist/lib/research/schema.d.ts +5 -2
- package/dist/lib/research/schema.js +12 -1
- package/dist/lib/research/secret-scan.js +4 -2
- package/dist/lib/research/url.js +5 -2
- package/dist/lib/text.d.ts +35 -0
- package/dist/lib/text.js +65 -0
- package/dist/sites/algolia.d.ts +9 -0
- package/dist/sites/algolia.js +26 -0
- package/dist/sites/auto.d.ts +8 -0
- package/dist/sites/auto.js +30 -0
- package/dist/sites/index.js +17 -1
- package/dist/sites/nextjs/index.d.ts +2 -0
- package/dist/sites/nextjs/index.js +52 -0
- package/dist/sites/react/index.d.ts +2 -0
- package/dist/sites/react/index.js +17 -0
- package/dist/sites/tailwind/index.d.ts +2 -0
- package/dist/sites/tailwind/index.js +17 -0
- package/dist/sites/vue/index.d.ts +2 -0
- package/dist/sites/vue/index.js +17 -0
- package/oclif.manifest.json +154 -81
- package/package.json +24 -4
package/dist/commands/fetch.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Args, Flags } from '@oclif/core';
|
|
1
|
+
import { Args, Flags, ux } from '@oclif/core';
|
|
2
2
|
import { mkdtempSync, rmSync } from 'node:fs';
|
|
3
3
|
import { tmpdir } from 'node:os';
|
|
4
4
|
import { join } from 'node:path';
|
|
@@ -6,13 +6,12 @@ import { BaseCommand } from '../base-command.js';
|
|
|
6
6
|
import { writeArtifact, getArtifactPath } from '../lib/research/storage.js';
|
|
7
7
|
import { writeArtifactSecurely } from '../lib/research/secure-write.js';
|
|
8
8
|
import { loadSummaryLevel } from '../lib/config/index.js';
|
|
9
|
-
import { evaluateFreshness,
|
|
9
|
+
import { evaluateFreshness, checkMaxAgeExpired, durationFlagError, } from '../lib/research/freshness.js';
|
|
10
10
|
import { revalidateCache, createArtifactFromFetch } from '../lib/research/revalidate.js';
|
|
11
11
|
import { fetchStaticHtml, fetchText } from '../lib/research/fetcher.js';
|
|
12
12
|
import { fetchRenderedHtml } from '../lib/research/browser.js';
|
|
13
13
|
import { capturePage } from '../lib/research/capture.js';
|
|
14
14
|
import { persistSectionArtifacts } from '../lib/research/docs/section-artifacts.js';
|
|
15
|
-
import { resolveResearchTarget } from '../lib/research/resolve-target.js';
|
|
16
15
|
import { applyAutoTags } from '../lib/research/keywords.js';
|
|
17
16
|
import { detectSite } from '../sites/index.js';
|
|
18
17
|
const CAPTURE_DEPS = {
|
|
@@ -24,69 +23,69 @@ export default class FetchCommand extends BaseCommand {
|
|
|
24
23
|
static id = 'fetch';
|
|
25
24
|
static hidden = true;
|
|
26
25
|
static summary = 'An advanced, locally cached web research tool optimized for LLM ingestion.';
|
|
27
|
-
static description = 'Scrapes the specified URL, strips HTML boilerplate, converts the semantic payload into clean Markdown format, and caches the result locally using dynamic TTL rules
|
|
26
|
+
static description = 'Scrapes the specified URL, strips HTML boilerplate, converts the semantic payload into clean Markdown format, and caches the result locally using dynamic TTL rules.\n\nUsually invoked via the shorthand `bonsai <url>` rather than `bonsai fetch <url>`.';
|
|
28
27
|
static examples = [
|
|
29
28
|
{
|
|
30
|
-
description: '
|
|
29
|
+
description: 'research a URL with detailed output, tagged with topic and tags',
|
|
31
30
|
command: '<%= config.bin %> https://docs.nestjs.com/ --topic "Backend Frameworks" --tags "Node" --tags "NestJS" --format detailed --ttl 30d',
|
|
32
31
|
},
|
|
33
32
|
{
|
|
34
|
-
description: '
|
|
33
|
+
description: 'research a volatile page with compressed output and short TTL, returned as JSON',
|
|
35
34
|
command: '<%= config.bin %> https://news.ycombinator.com/ --format compressed --ttl 2h --json',
|
|
36
35
|
},
|
|
37
36
|
];
|
|
38
37
|
static args = {
|
|
39
38
|
url: Args.string({
|
|
40
39
|
required: true,
|
|
41
|
-
description: '
|
|
40
|
+
description: 'the full HTTP/HTTPS URL of the web page to research',
|
|
42
41
|
}),
|
|
43
42
|
};
|
|
44
43
|
static flags = {
|
|
45
44
|
topic: Flags.string({
|
|
46
45
|
char: 't',
|
|
47
|
-
description: '
|
|
46
|
+
description: 'the main category/topic of the research for metadata tagging',
|
|
48
47
|
}),
|
|
49
48
|
tags: Flags.string({
|
|
50
49
|
char: 'g',
|
|
51
|
-
description: '
|
|
50
|
+
description: 'taxonomic tags for this research (can be repeated)',
|
|
52
51
|
multiple: true,
|
|
53
52
|
}),
|
|
54
53
|
format: Flags.option({
|
|
55
54
|
char: 'f',
|
|
56
|
-
description: '
|
|
55
|
+
description: 'desired data density',
|
|
57
56
|
options: ['compressed', 'detailed'],
|
|
58
57
|
default: 'compressed',
|
|
59
58
|
})(),
|
|
60
59
|
tier: Flags.option({
|
|
61
|
-
description: '
|
|
60
|
+
description: 'freshness tier policy',
|
|
62
61
|
options: ['stable', 'standard', 'volatile'],
|
|
63
62
|
default: 'standard',
|
|
64
63
|
})(),
|
|
65
64
|
ttl: Flags.string({
|
|
66
65
|
char: 'l',
|
|
67
|
-
description: '
|
|
66
|
+
description: 'predicted lifespan: number + h/d/w/m/y (m = months), e.g. "2h", "7d", "6m"',
|
|
68
67
|
}),
|
|
69
68
|
'max-age': Flags.string({
|
|
70
|
-
description: '
|
|
69
|
+
description: 'maximum age of cache to accept (e.g., "1d", "30d")',
|
|
71
70
|
}),
|
|
72
71
|
force: Flags.boolean({
|
|
73
|
-
description: '
|
|
72
|
+
description: 'force a fresh fetch, ignoring any cached entries',
|
|
74
73
|
default: false,
|
|
75
74
|
}),
|
|
76
75
|
'dry-run': Flags.boolean({
|
|
77
|
-
description: '
|
|
76
|
+
description: 'perform validation and fetch without saving to local cache',
|
|
78
77
|
default: false,
|
|
79
78
|
}),
|
|
80
79
|
'allow-stale': Flags.boolean({
|
|
81
|
-
description: '
|
|
80
|
+
description: 'allow serving stale cache if the remote server is unreachable',
|
|
82
81
|
default: false,
|
|
83
82
|
}),
|
|
84
83
|
rendered: Flags.boolean({
|
|
85
|
-
description: '
|
|
84
|
+
description: 'force using a browser-rendered scraping path for dynamic pages',
|
|
86
85
|
default: false,
|
|
87
86
|
}),
|
|
88
87
|
storage: Flags.option({
|
|
89
|
-
description: '
|
|
88
|
+
description: 'override where this result is cached (secrets always stored globally)',
|
|
90
89
|
options: ['global', 'project'],
|
|
91
90
|
})(),
|
|
92
91
|
};
|
|
@@ -183,20 +182,27 @@ export default class FetchCommand extends BaseCommand {
|
|
|
183
182
|
}
|
|
184
183
|
// Validates the duration flags up front, exiting with code 2 on a malformed value.
|
|
185
184
|
validateDurationFlags(ttl, maxAge) {
|
|
186
|
-
for (const [
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
]) {
|
|
190
|
-
if (!value)
|
|
191
|
-
continue;
|
|
192
|
-
try {
|
|
193
|
-
parseTtlToMs(value);
|
|
194
|
-
}
|
|
195
|
-
catch (err) {
|
|
196
|
-
this.error(`Invalid ${label}: ${err.message}`, { exit: 2 });
|
|
197
|
-
}
|
|
185
|
+
for (const msg of [durationFlagError('--ttl', ttl), durationFlagError('--max-age', maxAge)]) {
|
|
186
|
+
if (msg)
|
|
187
|
+
this.error(msg, { exit: 2, code: 'INVALID_DURATION' });
|
|
198
188
|
}
|
|
199
189
|
}
|
|
190
|
+
// Re-emit a runtime fetch failure with actionable next steps. Deep fetch/extract code throws plain
|
|
191
|
+
// Errors that otherwise reach the user as a bare "what broke" line with no "what to do"; this
|
|
192
|
+
// attaches recovery hints (e.g. import an auth-blocked page). Everything reaching here is a runtime
|
|
193
|
+
// failure — validation errors (bad flags/URL) exit before the try — so the contract's runtime code
|
|
194
|
+
// (1) applies. Suggestions render on stderr for humans only; under --json the envelope carries just
|
|
195
|
+
// the message (toErrorJson), so machine output is unchanged.
|
|
196
|
+
emitFetchError(err, url) {
|
|
197
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
198
|
+
const guidance = fetchFailureGuidance(message, url);
|
|
199
|
+
this.error(message, {
|
|
200
|
+
exit: 1,
|
|
201
|
+
code: 'FETCH_FAILED',
|
|
202
|
+
suggestions: guidance?.suggestions,
|
|
203
|
+
ref: guidance?.ref,
|
|
204
|
+
});
|
|
205
|
+
}
|
|
200
206
|
// Serves the cache when a fresh/revalidatable entry exists, otherwise fetches fresh. Reads fall
|
|
201
207
|
// back project→global; the resolved artifact is returned with the dir it lives in / landed in.
|
|
202
208
|
async resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime, located, summaryLevel) {
|
|
@@ -210,9 +216,12 @@ export default class FetchCommand extends BaseCommand {
|
|
|
210
216
|
}
|
|
211
217
|
const artifact = await this.executeCacheMiss(normalizedUrl, currentTime, cacheKey, summaryLevel);
|
|
212
218
|
const { dir, redirectedToGlobal } = this.persistFreshArtifact(roots, tmpDir, cacheKey, artifact);
|
|
219
|
+
// No entry existed at lookup, so there is no prior freshness to report. 'none' (not
|
|
220
|
+
// 'stale_expired') keeps the field honest — nothing expired; the page was simply uncached and
|
|
221
|
+
// has now been fetched fresh. Mirrors the `status` command's miss reporting.
|
|
213
222
|
return {
|
|
214
223
|
cacheStatus: 'miss',
|
|
215
|
-
freshnessState: '
|
|
224
|
+
freshnessState: 'none',
|
|
216
225
|
artifact,
|
|
217
226
|
storageDir: dir,
|
|
218
227
|
redirectedToGlobal,
|
|
@@ -249,27 +258,30 @@ export default class FetchCommand extends BaseCommand {
|
|
|
249
258
|
const { url } = this.args;
|
|
250
259
|
const { format, ttl, 'max-age': maxAge, 'dry-run': dryRun } = this.flags;
|
|
251
260
|
this.validateDurationFlags(ttl, maxAge);
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
cwd: process.cwd(),
|
|
257
|
-
dataDir: this.config.dataDir,
|
|
258
|
-
flagOverride: this.flags.storage,
|
|
259
|
-
lookup: !this.flags.force,
|
|
260
|
-
url,
|
|
261
|
-
});
|
|
262
|
-
}
|
|
263
|
-
catch (err) {
|
|
264
|
-
this.error(`Invalid URL: ${err.message}`, { exit: 2 });
|
|
265
|
-
}
|
|
261
|
+
const target = this.resolveResearchTargetOrFail(url, {
|
|
262
|
+
flagOverride: this.flags.storage,
|
|
263
|
+
lookup: !this.flags.force,
|
|
264
|
+
});
|
|
266
265
|
const { cacheKey, located, normalizedUrl, roots } = target;
|
|
267
266
|
const summaryLevel = loadSummaryLevel(this.config.configDir, process.cwd());
|
|
268
267
|
const tmpDir = dryRun ? mkdtempSync(join(tmpdir(), 'fnr-dry-run-')) : null;
|
|
269
268
|
const currentTime = new Date();
|
|
270
269
|
try {
|
|
270
|
+
if (!this.jsonEnabled()) {
|
|
271
|
+
ux.action.start('Fetching ' + normalizedUrl);
|
|
272
|
+
}
|
|
271
273
|
const { cacheStatus, freshnessState, artifact, storageDir, redirectedToGlobal } = await this.resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime, located, summaryLevel);
|
|
272
274
|
this.persistSectionsIfFresh(tmpDir ?? storageDir, artifact, currentTime, cacheStatus, summaryLevel);
|
|
275
|
+
if (!this.jsonEnabled()) {
|
|
276
|
+
const CACHE_STATUS_LABEL = {
|
|
277
|
+
hit: 'cached',
|
|
278
|
+
miss: 'done',
|
|
279
|
+
refreshed: 'refreshed',
|
|
280
|
+
revalidated: 'revalidated',
|
|
281
|
+
stale: 'served stale',
|
|
282
|
+
};
|
|
283
|
+
ux.action.stop(CACHE_STATUS_LABEL[cacheStatus] ?? cacheStatus);
|
|
284
|
+
}
|
|
273
285
|
const content = format === 'compressed' ? artifact.compressed : artifact.detailed;
|
|
274
286
|
const resultData = this.buildResultData(url, normalizedUrl, cacheKey, storageDir, roots.mode, cacheStatus, freshnessState, format, artifact, content, redirectedToGlobal);
|
|
275
287
|
if (!this.jsonEnabled()) {
|
|
@@ -277,6 +289,16 @@ export default class FetchCommand extends BaseCommand {
|
|
|
277
289
|
}
|
|
278
290
|
return resultData;
|
|
279
291
|
}
|
|
292
|
+
catch (err) {
|
|
293
|
+
if (!this.jsonEnabled()) {
|
|
294
|
+
ux.action.stop('failed');
|
|
295
|
+
}
|
|
296
|
+
// Stale-serve (exit 5) signals via process.exitCode and never throws, so it bypasses this
|
|
297
|
+
// path; only genuine fetch/extract failures land here and get classified guidance. Use the
|
|
298
|
+
// normalized URL in hints so the copy-paste command is canonical (and never echoes raw,
|
|
299
|
+
// unsanitized user input back to the terminal).
|
|
300
|
+
this.emitFetchError(err, normalizedUrl);
|
|
301
|
+
}
|
|
280
302
|
finally {
|
|
281
303
|
if (tmpDir) {
|
|
282
304
|
rmSync(tmpDir, { recursive: true, force: true });
|
|
@@ -284,6 +306,59 @@ export default class FetchCommand extends BaseCommand {
|
|
|
284
306
|
}
|
|
285
307
|
}
|
|
286
308
|
}
|
|
309
|
+
// Maps a runtime fetch failure to recovery steps keyed off its message. Returns undefined for
|
|
310
|
+
// unrecognized failures, which then surface with their original message and no extra hint. The
|
|
311
|
+
// patterns match the thrown text in fetcher.ts/browser.ts; keep them in sync if those messages
|
|
312
|
+
// change. Resolutions mirror the published troubleshooting guide.
|
|
313
|
+
export function fetchFailureGuidance(message, url) {
|
|
314
|
+
const ref = 'https://taurgis.github.io/bonsai/troubleshooting';
|
|
315
|
+
// Name the pipe step explicitly: bare `import --stdin` blocks waiting for input, so a reader (or
|
|
316
|
+
// an agent) running the hint verbatim would hang. Show the content being piped in from a file.
|
|
317
|
+
const importHint = `Open it in a browser, save the page, then import it: cat page.md | bonsai import ${url} --stdin`;
|
|
318
|
+
// 401/403: an auth wall or anti-scraping WAF. Bonsai has no authenticated-fetch path in v1.
|
|
319
|
+
if (/failed with status 40[13]\b/.test(message)) {
|
|
320
|
+
return {
|
|
321
|
+
suggestions: ['The page requires authentication or blocks automated requests.', importHint],
|
|
322
|
+
ref,
|
|
323
|
+
};
|
|
324
|
+
}
|
|
325
|
+
if (/failed with status 404\b/.test(message)) {
|
|
326
|
+
return { suggestions: ['Check the URL is correct and the page still exists.'] };
|
|
327
|
+
}
|
|
328
|
+
if (/failed with status 5\d\d\b/.test(message)) {
|
|
329
|
+
return {
|
|
330
|
+
suggestions: ['The server returned an error. Retry later or verify the host is healthy.'],
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
// Server returned a non-HTML body (JSON, binary, or no Content-Type at all). Scope the opener to
|
|
334
|
+
// the real failure: --rendered does produce HTML, so "only scrapes HTML" would read as misleading.
|
|
335
|
+
if (/Rejected content type|does not look like HTML/.test(message)) {
|
|
336
|
+
return {
|
|
337
|
+
suggestions: [
|
|
338
|
+
'The server returned a non-HTML response (e.g. JSON or binary).',
|
|
339
|
+
'If the page is rendered by client-side JavaScript, retry with --rendered.',
|
|
340
|
+
importHint,
|
|
341
|
+
],
|
|
342
|
+
ref,
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
if (/DNS resolution failed/.test(message)) {
|
|
346
|
+
return {
|
|
347
|
+
suggestions: ['Check the hostname is spelled correctly and resolves on a public network.'],
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
// A hostname that only resolves to a private/local IP at request time (literal private addresses
|
|
351
|
+
// are rejected earlier with exit 2). Blocked to prevent SSRF; only public hosts can be fetched.
|
|
352
|
+
if (/blocked local or private target/.test(message)) {
|
|
353
|
+
return {
|
|
354
|
+
suggestions: [
|
|
355
|
+
'The hostname resolves to a private or local address, which is blocked to prevent SSRF. Only public http(s) hosts can be fetched.',
|
|
356
|
+
],
|
|
357
|
+
ref,
|
|
358
|
+
};
|
|
359
|
+
}
|
|
360
|
+
return undefined;
|
|
361
|
+
}
|
|
287
362
|
// Copies Phase 2 capability provenance from a capture outcome onto the artifact metadata.
|
|
288
363
|
function applyCaptureMetadata(artifact, capture) {
|
|
289
364
|
const meta = artifact.metadata;
|
|
@@ -302,10 +377,10 @@ function handleStaleRevalidationResult(command, revalResult) {
|
|
|
302
377
|
if (revalResult.status !== 'stale')
|
|
303
378
|
return;
|
|
304
379
|
if (revalResult.allowed) {
|
|
305
|
-
command.warn(`Serving stale content: revalidation failed
|
|
380
|
+
command.warn(`Serving stale content within grace period: revalidation failed (${revalResult.error}).`);
|
|
306
381
|
}
|
|
307
382
|
else {
|
|
308
|
-
command.warn(`Serving stale content (exit 5): revalidation failed
|
|
383
|
+
command.warn(`Serving stale content within grace period (exit 5): revalidation failed (${revalResult.error}).`);
|
|
309
384
|
process.exitCode = 5;
|
|
310
385
|
}
|
|
311
386
|
}
|
|
@@ -23,12 +23,16 @@ export default class ResearchImport extends BaseCommand<typeof ResearchImport> {
|
|
|
23
23
|
storage: import("@oclif/core/interfaces").OptionFlag<"global" | "project" | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
24
24
|
};
|
|
25
25
|
static stdoutIsPrimaryData: boolean;
|
|
26
|
+
protected stdinIsInteractive(): boolean;
|
|
26
27
|
private readStdin;
|
|
27
28
|
private getSourceUrls;
|
|
28
29
|
private validateSourceMode;
|
|
29
30
|
protected fsExistsSync(filePath: string): boolean;
|
|
30
31
|
protected fsStatSync(filePath: string): fs.Stats;
|
|
31
32
|
protected fsReadFileSync(filePath: string): string;
|
|
33
|
+
private stdinImportSuggestions;
|
|
34
|
+
/** Non-interactive stdin that never receives data (open pipe with no writer) must not hang forever. */
|
|
35
|
+
private readStdinWithGuard;
|
|
32
36
|
private readAndValidateFile;
|
|
33
37
|
private readAndValidateInput;
|
|
34
38
|
private deriveImportCacheKey;
|
package/dist/commands/import.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Args, Flags } from '@oclif/core';
|
|
1
|
+
import { Args, Errors, Flags } from '@oclif/core';
|
|
2
2
|
import { createHash } from 'node:crypto';
|
|
3
3
|
import * as fs from 'node:fs';
|
|
4
4
|
import * as path from 'node:path';
|
|
@@ -9,7 +9,7 @@ import { getArtifactPath } from '../lib/research/storage.js';
|
|
|
9
9
|
import { loadStoreRoots } from '../lib/research/store-roots.js';
|
|
10
10
|
import { writeArtifactSecurely } from '../lib/research/secure-write.js';
|
|
11
11
|
import { loadSummaryLevel } from '../lib/config/index.js';
|
|
12
|
-
import { getPolicy } from '../lib/research/freshness.js';
|
|
12
|
+
import { durationFlagError, getPolicy } from '../lib/research/freshness.js';
|
|
13
13
|
import { buildCompressed } from '../lib/research/compress.js';
|
|
14
14
|
import { applyAutoTags } from '../lib/research/keywords.js';
|
|
15
15
|
import { estimateTokens } from '../lib/research/token-estimate.js';
|
|
@@ -19,65 +19,74 @@ export default class ResearchImport extends BaseCommand {
|
|
|
19
19
|
static description = 'Accepts a synthesized Markdown note via stdin and registers it under the single target URL or multiple source URLs.';
|
|
20
20
|
static examples = [
|
|
21
21
|
{
|
|
22
|
-
description: '
|
|
22
|
+
description: 'import detailed research for a single URL via stdin',
|
|
23
23
|
command: 'echo "# My Article" | <%= config.bin %> import https://example.com/docs --stdin',
|
|
24
24
|
},
|
|
25
25
|
{
|
|
26
|
-
description: '
|
|
26
|
+
description: 'import research synthesized from multiple source URLs',
|
|
27
27
|
command: 'echo "# Synthesized" | <%= config.bin %> import --stdin --topic "React docs" --source-url https://react.dev/a --source-url https://react.dev/b',
|
|
28
28
|
},
|
|
29
29
|
{
|
|
30
|
-
description: '
|
|
30
|
+
description: 'import research from a local Markdown file',
|
|
31
31
|
command: '<%= config.bin %> import https://example.com/docs --file path/to/notes.md',
|
|
32
32
|
},
|
|
33
33
|
];
|
|
34
34
|
static args = {
|
|
35
35
|
url: Args.string({
|
|
36
36
|
required: false,
|
|
37
|
-
|
|
37
|
+
// oclif fills an omitted optional arg from piped stdin unless ignoreStdin is set. Without
|
|
38
|
+
// this, the Markdown piped for `--stdin` is swallowed into `url`, making multi-source import
|
|
39
|
+
// (`import --stdin --source-url ...`) wrongly look like it also got a positional URL.
|
|
40
|
+
ignoreStdin: true,
|
|
41
|
+
description: 'the single source URL of the page (only for single-source import)',
|
|
38
42
|
}),
|
|
39
43
|
};
|
|
40
44
|
static flags = {
|
|
41
45
|
stdin: Flags.boolean({
|
|
42
|
-
description: '
|
|
46
|
+
description: 'read Markdown from stdin',
|
|
43
47
|
default: false,
|
|
44
48
|
}),
|
|
45
49
|
file: Flags.string({
|
|
46
|
-
description: '
|
|
50
|
+
description: 'path to a Markdown file containing research notes to import',
|
|
47
51
|
}),
|
|
48
52
|
'input-format': Flags.option({
|
|
49
|
-
description: '
|
|
53
|
+
description: 'format of the input provided',
|
|
50
54
|
options: ['compressed', 'detailed'],
|
|
51
55
|
default: 'detailed',
|
|
52
56
|
})(),
|
|
53
57
|
topic: Flags.string({
|
|
54
58
|
char: 't',
|
|
55
|
-
description: '
|
|
59
|
+
description: 'the main topic for this research note',
|
|
56
60
|
}),
|
|
57
61
|
tags: Flags.string({
|
|
58
62
|
char: 'g',
|
|
59
|
-
description: '
|
|
63
|
+
description: 'taxonomic tags (can be repeated)',
|
|
60
64
|
multiple: true,
|
|
61
65
|
}),
|
|
62
66
|
tier: Flags.option({
|
|
63
|
-
description: '
|
|
67
|
+
description: 'freshness tier policy',
|
|
64
68
|
options: ['stable', 'standard', 'volatile'],
|
|
65
69
|
default: 'standard',
|
|
66
70
|
})(),
|
|
67
71
|
'source-url': Flags.string({
|
|
68
|
-
description: '
|
|
72
|
+
description: 'repeated source URLs for multi-source import',
|
|
69
73
|
multiple: true,
|
|
70
74
|
}),
|
|
71
75
|
ttl: Flags.string({
|
|
72
76
|
char: 'l',
|
|
73
|
-
description: '
|
|
77
|
+
description: 'predicted lifespan of the data (e.g. "24h", "7d")',
|
|
74
78
|
}),
|
|
75
79
|
storage: Flags.option({
|
|
76
|
-
description: '
|
|
80
|
+
description: 'override where this note is cached (secrets always stored globally)',
|
|
77
81
|
options: ['global', 'project'],
|
|
78
82
|
})(),
|
|
79
83
|
};
|
|
80
84
|
static stdoutIsPrimaryData = true;
|
|
85
|
+
// Isolated so tests can force the interactive branch; `isTTY` is `true` only on a real terminal and
|
|
86
|
+
// falsy (undefined) for pipes, files, and /dev/null. See https://nodejs.org/api/process.html#processstdin
|
|
87
|
+
stdinIsInteractive() {
|
|
88
|
+
return process.stdin.isTTY === true;
|
|
89
|
+
}
|
|
81
90
|
async readStdin(limitBytes = 1024 * 1024) {
|
|
82
91
|
return new Promise((resolve, reject) => {
|
|
83
92
|
let data = '';
|
|
@@ -112,28 +121,38 @@ export default class ResearchImport extends BaseCommand {
|
|
|
112
121
|
return normalizeUrl(u);
|
|
113
122
|
}
|
|
114
123
|
catch (err) {
|
|
115
|
-
this.error(`Invalid URL: ${err.message}`, { exit: 2 });
|
|
124
|
+
this.error(`Invalid URL: ${err.message}`, { exit: 2, code: 'INVALID_URL' });
|
|
116
125
|
}
|
|
117
126
|
});
|
|
118
127
|
return hasMulti ? normalized.sort() : normalized;
|
|
119
128
|
}
|
|
120
129
|
validateSourceMode(hasSingle, hasMulti, multiUrls) {
|
|
121
130
|
if (hasSingle && hasMulti) {
|
|
122
|
-
this.error('Cannot specify both positional <url> and --source-url flags.', {
|
|
131
|
+
this.error('Cannot specify both positional <url> and --source-url flags.', {
|
|
132
|
+
exit: 2,
|
|
133
|
+
code: 'CONFLICTING_FLAGS',
|
|
134
|
+
});
|
|
123
135
|
}
|
|
124
136
|
if (!hasSingle && !hasMulti) {
|
|
125
|
-
this.error('Must specify either positional <url> (for single-source) or --source-url (for multi-source) import.', { exit: 2 });
|
|
137
|
+
this.error('Must specify either positional <url> (for single-source) or --source-url (for multi-source) import.', { exit: 2, code: 'MISSING_URL' });
|
|
126
138
|
}
|
|
127
139
|
if (hasMulti && !this.flags.topic) {
|
|
128
|
-
this.error('Multi-source import requires the --topic flag.', {
|
|
140
|
+
this.error('Multi-source import requires the --topic flag.', {
|
|
141
|
+
exit: 2,
|
|
142
|
+
code: 'MISSING_TOPIC',
|
|
143
|
+
});
|
|
129
144
|
}
|
|
130
145
|
if (!this.flags.stdin && !this.flags.file) {
|
|
131
146
|
this.error('Either --stdin or --file <path> must be specified to import content.', {
|
|
132
147
|
exit: 2,
|
|
148
|
+
code: 'MISSING_INPUT',
|
|
133
149
|
});
|
|
134
150
|
}
|
|
135
151
|
if (this.flags.stdin && this.flags.file) {
|
|
136
|
-
this.error('Cannot specify both --stdin and --file <path>. Choose one.', {
|
|
152
|
+
this.error('Cannot specify both --stdin and --file <path>. Choose one.', {
|
|
153
|
+
exit: 2,
|
|
154
|
+
code: 'CONFLICTING_FLAGS',
|
|
155
|
+
});
|
|
137
156
|
}
|
|
138
157
|
}
|
|
139
158
|
fsExistsSync(filePath) {
|
|
@@ -145,35 +164,70 @@ export default class ResearchImport extends BaseCommand {
|
|
|
145
164
|
fsReadFileSync(filePath) {
|
|
146
165
|
return fs.readFileSync(filePath, 'utf8');
|
|
147
166
|
}
|
|
167
|
+
stdinImportSuggestions() {
|
|
168
|
+
const bin = this.config.bin;
|
|
169
|
+
return [
|
|
170
|
+
`Pipe Markdown content, e.g. cat notes.md | ${bin} import <url> --stdin`,
|
|
171
|
+
`Or read from a file: ${bin} import <url> --file notes.md`,
|
|
172
|
+
];
|
|
173
|
+
}
|
|
174
|
+
/** Non-interactive stdin that never receives data (open pipe with no writer) must not hang forever. */
|
|
175
|
+
async readStdinWithGuard(limitBytes = 1024 * 1024) {
|
|
176
|
+
if (this.stdinIsInteractive()) {
|
|
177
|
+
this.error('No data piped to --stdin.', {
|
|
178
|
+
exit: 2,
|
|
179
|
+
code: 'MISSING_STDIN',
|
|
180
|
+
suggestions: this.stdinImportSuggestions(),
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
const timeoutMs = 1000;
|
|
184
|
+
let rawInput = '';
|
|
185
|
+
try {
|
|
186
|
+
rawInput = await Promise.race([
|
|
187
|
+
this.readStdin(limitBytes),
|
|
188
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error('STDIN_TIMEOUT')), timeoutMs)),
|
|
189
|
+
]);
|
|
190
|
+
}
|
|
191
|
+
catch (err) {
|
|
192
|
+
if (err.message === 'STDIN_TIMEOUT') {
|
|
193
|
+
this.error('No stdin data received. Pipe content or use --file.', {
|
|
194
|
+
exit: 2,
|
|
195
|
+
code: 'MISSING_STDIN',
|
|
196
|
+
suggestions: this.stdinImportSuggestions(),
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
if (err.message.includes('stdin size limit exceeded')) {
|
|
200
|
+
this.error(err.message, { exit: 1, code: 'STDIN_TOO_LARGE' });
|
|
201
|
+
}
|
|
202
|
+
// Genuine stream failure reading stdin → I/O failure (exit 1).
|
|
203
|
+
this.error(err.message, { exit: 1, code: 'IO_ERROR' });
|
|
204
|
+
}
|
|
205
|
+
return rawInput;
|
|
206
|
+
}
|
|
148
207
|
readAndValidateFile(filePath) {
|
|
149
208
|
if (!this.fsExistsSync(filePath)) {
|
|
150
|
-
this.error(`File does not exist: ${filePath}`, { exit: 2 });
|
|
209
|
+
this.error(`File does not exist: ${filePath}`, { exit: 2, code: 'FILE_NOT_FOUND' });
|
|
151
210
|
}
|
|
152
211
|
const stat = this.fsStatSync(filePath);
|
|
153
212
|
if (!stat.isFile()) {
|
|
154
|
-
this.error(`Path is not a file: ${filePath}`, { exit: 2 });
|
|
213
|
+
this.error(`Path is not a file: ${filePath}`, { exit: 2, code: 'NOT_A_FILE' });
|
|
155
214
|
}
|
|
156
215
|
if (stat.size > 1024 * 1024) {
|
|
157
|
-
this.error('File size limit exceeded (max 1 MiB).', { exit: 1 });
|
|
216
|
+
this.error('File size limit exceeded (max 1 MiB).', { exit: 1, code: 'FILE_TOO_LARGE' });
|
|
158
217
|
}
|
|
159
218
|
const content = this.fsReadFileSync(filePath);
|
|
160
219
|
if (!content.trim()) {
|
|
161
|
-
this.error('Empty file content provided.', { exit: 2 });
|
|
220
|
+
this.error('Empty file content provided.', { exit: 2, code: 'EMPTY_INPUT' });
|
|
162
221
|
}
|
|
163
222
|
return content;
|
|
164
223
|
}
|
|
165
224
|
async readAndValidateInput() {
|
|
166
225
|
if (this.flags.stdin) {
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
this.error('Empty stdin content provided.', { exit: 2 });
|
|
171
|
-
}
|
|
172
|
-
return rawInput;
|
|
173
|
-
}
|
|
174
|
-
catch (err) {
|
|
175
|
-
this.error(err.message, { exit: 1 });
|
|
226
|
+
const rawInput = await this.readStdinWithGuard();
|
|
227
|
+
if (!rawInput.trim()) {
|
|
228
|
+
this.error('Empty stdin content provided.', { exit: 2, code: 'EMPTY_INPUT' });
|
|
176
229
|
}
|
|
230
|
+
return rawInput;
|
|
177
231
|
}
|
|
178
232
|
if (this.flags.file) {
|
|
179
233
|
const filePath = path.resolve(this.flags.file);
|
|
@@ -181,10 +235,15 @@ export default class ResearchImport extends BaseCommand {
|
|
|
181
235
|
return this.readAndValidateFile(filePath);
|
|
182
236
|
}
|
|
183
237
|
catch (err) {
|
|
184
|
-
|
|
238
|
+
// readAndValidateFile raises usage errors (missing/not-a-file/empty) via this.error,
|
|
239
|
+
// which throw CLIErrors carrying their own exit code and message — preserve those.
|
|
240
|
+
// Only an unexpected read failure becomes a wrapped I/O failure (exit 1).
|
|
241
|
+
if (err instanceof Errors.CLIError)
|
|
242
|
+
throw err;
|
|
243
|
+
this.error(`Failed to read file: ${err.message}`, { exit: 1, code: 'IO_ERROR' });
|
|
185
244
|
}
|
|
186
245
|
}
|
|
187
|
-
this.error('No input source specified.', { exit: 2 });
|
|
246
|
+
this.error('No input source specified.', { exit: 2, code: 'MISSING_INPUT' });
|
|
188
247
|
}
|
|
189
248
|
deriveImportCacheKey(hasSingle, singleNormalizedUrl, sourceUrls) {
|
|
190
249
|
if (hasSingle) {
|
|
@@ -260,6 +319,9 @@ export default class ResearchImport extends BaseCommand {
|
|
|
260
319
|
const hasSingle = Boolean(this.args.url);
|
|
261
320
|
const multiUrls = this.flags['source-url'] || [];
|
|
262
321
|
const hasMulti = multiUrls.length > 0;
|
|
322
|
+
const ttlErr = durationFlagError('--ttl', this.flags.ttl);
|
|
323
|
+
if (ttlErr)
|
|
324
|
+
this.error(ttlErr, { exit: 2, code: 'INVALID_DURATION' });
|
|
263
325
|
this.validateSourceMode(hasSingle, hasMulti, multiUrls);
|
|
264
326
|
const rawInput = await this.readAndValidateInput();
|
|
265
327
|
const sourceUrls = this.getSourceUrls(hasMulti, multiUrls, this.args.url);
|
|
@@ -279,8 +341,8 @@ export default class ResearchImport extends BaseCommand {
|
|
|
279
341
|
}
|
|
280
342
|
if (!this.jsonEnabled()) {
|
|
281
343
|
this.log(`Successfully imported research artifact.`);
|
|
282
|
-
this.log(
|
|
283
|
-
this.log(
|
|
344
|
+
this.log(`${'Cache Key:'.padEnd(25)} ${cacheKey}`);
|
|
345
|
+
this.log(`${'Storage Path:'.padEnd(25)} ${storagePath}`);
|
|
284
346
|
}
|
|
285
347
|
const inputFormat = this.flags['input-format'];
|
|
286
348
|
return {
|
|
@@ -3,6 +3,10 @@ export default class ResearchInspect extends BaseCommand<typeof ResearchInspect>
|
|
|
3
3
|
static id: string;
|
|
4
4
|
static summary: string;
|
|
5
5
|
static description: string;
|
|
6
|
+
static examples: {
|
|
7
|
+
description: string;
|
|
8
|
+
command: string;
|
|
9
|
+
}[];
|
|
6
10
|
static args: {
|
|
7
11
|
url: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
|
|
8
12
|
};
|