@taurgis/bonsai 1.0.3 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -6
- package/agents/agents/salesforce-docs-researcher.agent.md +117 -0
- package/agents/hooks/README.md +62 -0
- package/agents/hooks/claude-code/hooks/bonsai-web-fetch.sh +22 -0
- package/agents/hooks/claude-code/settings.json +16 -0
- package/agents/hooks/codex/hooks/bonsai-user-prompt-context.sh +18 -0
- package/agents/hooks/codex/hooks/bonsai-web-fetch.sh +22 -0
- package/agents/hooks/codex/hooks.json +28 -0
- package/agents/hooks/cursor/hooks/bonsai-web-fetch.sh +21 -0
- package/agents/hooks/cursor/hooks.json +12 -0
- package/agents/hooks/github-copilot/bonsai-web-fetch.json +23 -0
- package/agents/hooks/github-copilot/bonsai-web-fetch.sh +19 -0
- package/agents/instructions/salesforce-research.instructions.md +57 -0
- package/agents/instructions/web-research.instructions.md +2 -1
- package/bin/cli.mjs +13 -3
- package/dist/base-command.d.ts +20 -0
- package/dist/base-command.js +27 -1
- package/dist/commands/config/get.js +2 -2
- package/dist/commands/config/list.js +2 -2
- package/dist/commands/config/set.js +2 -2
- package/dist/commands/config/unset.js +2 -2
- package/dist/commands/fetch.d.ts +5 -0
- package/dist/commands/fetch.js +98 -27
- package/dist/commands/import.d.ts +1 -0
- package/dist/commands/import.js +51 -11
- package/dist/commands/list.js +18 -3
- package/dist/commands/prune.js +22 -6
- package/dist/commands/search.js +31 -30
- package/dist/commands/status.js +10 -9
- package/dist/hooks/command-not-found/suggest.d.ts +9 -0
- package/dist/hooks/command-not-found/suggest.js +93 -0
- package/dist/lib/config/index.d.ts +8 -3
- package/dist/lib/config/index.js +19 -4
- package/dist/lib/config/resolve.d.ts +21 -1
- package/dist/lib/config/resolve.js +42 -1
- package/dist/lib/config/schema.d.ts +3 -0
- package/dist/lib/config/schema.js +11 -1
- package/dist/lib/research/artifact.d.ts +13 -2
- package/dist/lib/research/artifact.js +27 -7
- package/dist/lib/research/browser.d.ts +12 -0
- package/dist/lib/research/browser.js +40 -3
- package/dist/lib/research/compress.d.ts +19 -0
- package/dist/lib/research/compress.js +31 -0
- package/dist/lib/research/docs/capabilities.d.ts +1 -1
- package/dist/lib/research/docs/code-blocks.js +24 -0
- package/dist/lib/research/docs/discovery-index.d.ts +14 -0
- package/dist/lib/research/docs/discovery-index.js +112 -0
- package/dist/lib/research/docs/machine-readable.d.ts +2 -2
- package/dist/lib/research/docs/machine-readable.js +3 -7
- package/dist/lib/research/docs/remote-search-runner.js +61 -1
- package/dist/lib/research/docs/section-artifacts.d.ts +3 -2
- package/dist/lib/research/docs/section-artifacts.js +5 -5
- package/dist/lib/research/docs/sections.d.ts +1 -0
- package/dist/lib/research/docs/sections.js +19 -1
- package/dist/lib/research/freshness.d.ts +9 -1
- package/dist/lib/research/freshness.js +24 -2
- package/dist/lib/research/keywords.d.ts +24 -0
- package/dist/lib/research/keywords.js +247 -0
- package/dist/lib/research/revalidate.d.ts +3 -1
- package/dist/lib/research/revalidate.js +8 -4
- package/dist/lib/research/search-index.d.ts +18 -0
- package/dist/lib/research/search-index.js +109 -0
- package/dist/lib/research/secret-scan.js +4 -2
- package/dist/lib/research/storage.d.ts +1 -0
- package/dist/lib/research/storage.js +1 -1
- package/dist/lib/research/summarize.d.ts +17 -0
- package/dist/lib/research/summarize.js +400 -0
- package/dist/lib/research/url.js +5 -2
- package/dist/lib/text.d.ts +33 -0
- package/dist/lib/text.js +63 -0
- package/oclif.manifest.json +8 -8
- package/package.json +17 -3
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { ConfigCommand, configScopeFlags } from './base.js';
|
|
2
|
-
import { ALL_KEYS, KEY_META, BUILT_IN_DEFAULTS, readScopedConfig } from '../../lib/config/index.js';
|
|
2
|
+
import { ALL_KEYS, KEY_META, BUILT_IN_DEFAULTS, readScopedConfig, validKeysHint, } from '../../lib/config/index.js';
|
|
3
3
|
export default class ConfigList extends ConfigCommand {
|
|
4
4
|
static id = 'config list';
|
|
5
5
|
static summary = 'List all research configuration keys and their effective values.';
|
|
6
|
-
static description =
|
|
6
|
+
static description = `Show every configuration key with its current value. Use --global/--local to show a single scope.\n\nValid keys: ${validKeysHint()}.`;
|
|
7
7
|
static examples = [
|
|
8
8
|
{
|
|
9
9
|
description: 'Show all effective config values',
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { Args, Flags } from '@oclif/core';
|
|
2
2
|
import { ConfigCommand, configScopeFlags } from './base.js';
|
|
3
|
-
import { KEY_META, writeUserConfig, writeProjectConfig } from '../../lib/config/index.js';
|
|
3
|
+
import { KEY_META, writeUserConfig, writeProjectConfig, validKeysHint, } from '../../lib/config/index.js';
|
|
4
4
|
export default class ConfigSet extends ConfigCommand {
|
|
5
5
|
static id = 'config set';
|
|
6
6
|
static summary = 'Set a research configuration key.';
|
|
7
7
|
static description = 'Persist a configuration value. Writes user-level config by default; pass --local to write the project-level config (.bonsai.json in cwd).\n\n' +
|
|
8
|
-
|
|
8
|
+
`Valid keys: ${validKeysHint()}. The inline form \`<key>=<value>\` is also accepted.`;
|
|
9
9
|
static examples = [
|
|
10
10
|
{
|
|
11
11
|
description: 'Store research cache inside the current project',
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { Args, Flags } from '@oclif/core';
|
|
2
2
|
import { ConfigCommand, configScopeFlags } from './base.js';
|
|
3
|
-
import { writeUserConfig, writeProjectConfig } from '../../lib/config/index.js';
|
|
3
|
+
import { writeUserConfig, writeProjectConfig, validKeysHint } from '../../lib/config/index.js';
|
|
4
4
|
export default class ConfigUnset extends ConfigCommand {
|
|
5
5
|
static id = 'config unset';
|
|
6
6
|
static summary = 'Remove a research configuration key.';
|
|
7
|
-
static description =
|
|
7
|
+
static description = `Delete a key from user-level config (default) or project-level config (--local), restoring the built-in default.\n\nValid keys: ${validKeysHint()}.`;
|
|
8
8
|
static examples = [
|
|
9
9
|
{
|
|
10
10
|
description: 'Remove the user-level storage setting',
|
package/dist/commands/fetch.d.ts
CHANGED
|
@@ -30,8 +30,13 @@ export default class FetchCommand extends BaseCommand<typeof FetchCommand> {
|
|
|
30
30
|
private executeCacheMiss;
|
|
31
31
|
private buildResultData;
|
|
32
32
|
private validateDurationFlags;
|
|
33
|
+
private emitFetchError;
|
|
33
34
|
private resolveArtifact;
|
|
34
35
|
private persistFreshArtifact;
|
|
35
36
|
private persistSectionsIfFresh;
|
|
36
37
|
run(): Promise<unknown>;
|
|
37
38
|
}
|
|
39
|
+
export declare function fetchFailureGuidance(message: string, url: string): {
|
|
40
|
+
suggestions: string[];
|
|
41
|
+
ref?: string;
|
|
42
|
+
} | undefined;
|
package/dist/commands/fetch.js
CHANGED
|
@@ -5,13 +5,15 @@ import { join } from 'node:path';
|
|
|
5
5
|
import { BaseCommand } from '../base-command.js';
|
|
6
6
|
import { writeArtifact, getArtifactPath } from '../lib/research/storage.js';
|
|
7
7
|
import { writeArtifactSecurely } from '../lib/research/secure-write.js';
|
|
8
|
-
import {
|
|
8
|
+
import { loadSummaryLevel } from '../lib/config/index.js';
|
|
9
|
+
import { evaluateFreshness, checkMaxAgeExpired, durationFlagError, } from '../lib/research/freshness.js';
|
|
9
10
|
import { revalidateCache, createArtifactFromFetch } from '../lib/research/revalidate.js';
|
|
10
11
|
import { fetchStaticHtml, fetchText } from '../lib/research/fetcher.js';
|
|
11
12
|
import { fetchRenderedHtml } from '../lib/research/browser.js';
|
|
12
13
|
import { capturePage } from '../lib/research/capture.js';
|
|
13
14
|
import { persistSectionArtifacts } from '../lib/research/docs/section-artifacts.js';
|
|
14
15
|
import { resolveResearchTarget } from '../lib/research/resolve-target.js';
|
|
16
|
+
import { applyAutoTags } from '../lib/research/keywords.js';
|
|
15
17
|
import { detectSite } from '../sites/index.js';
|
|
16
18
|
const CAPTURE_DEPS = {
|
|
17
19
|
fetchStatic: (url) => fetchStaticHtml(url),
|
|
@@ -22,7 +24,7 @@ export default class FetchCommand extends BaseCommand {
|
|
|
22
24
|
static id = 'fetch';
|
|
23
25
|
static hidden = true;
|
|
24
26
|
static summary = 'An advanced, locally cached web research tool optimized for LLM ingestion.';
|
|
25
|
-
static description = 'Scrapes the specified URL, strips HTML boilerplate, converts the semantic payload into clean Markdown format, and caches the result locally using dynamic TTL rules
|
|
27
|
+
static description = 'Scrapes the specified URL, strips HTML boilerplate, converts the semantic payload into clean Markdown format, and caches the result locally using dynamic TTL rules.\n\nUsually invoked via the shorthand `bonsai <url>` rather than `bonsai fetch <url>`.';
|
|
26
28
|
static examples = [
|
|
27
29
|
{
|
|
28
30
|
description: 'Research a URL with detailed output, tagged with topic and tags',
|
|
@@ -62,7 +64,7 @@ export default class FetchCommand extends BaseCommand {
|
|
|
62
64
|
})(),
|
|
63
65
|
ttl: Flags.string({
|
|
64
66
|
char: 'l',
|
|
65
|
-
description: 'Predicted lifespan
|
|
67
|
+
description: 'Predicted lifespan: number + h/d/w/m/y (m = months), e.g. "2h", "7d", "6m".',
|
|
66
68
|
}),
|
|
67
69
|
'max-age': Flags.string({
|
|
68
70
|
description: 'Maximum age of cache to accept (e.g., "1d", "30d").',
|
|
@@ -92,7 +94,7 @@ export default class FetchCommand extends BaseCommand {
|
|
|
92
94
|
envelopeCommandId() {
|
|
93
95
|
return this.config.bin;
|
|
94
96
|
}
|
|
95
|
-
async executeCacheHit(cached, targetDir, currentTime) {
|
|
97
|
+
async executeCacheHit(cached, targetDir, currentTime, summaryLevel) {
|
|
96
98
|
const { ttl, 'max-age': maxAge, 'allow-stale': allowStale, rendered } = this.flags;
|
|
97
99
|
const isExpired = checkMaxAgeExpired(cached, currentTime, maxAge);
|
|
98
100
|
const freshnessState = isExpired
|
|
@@ -105,6 +107,7 @@ export default class FetchCommand extends BaseCommand {
|
|
|
105
107
|
allowStale: Boolean(allowStale),
|
|
106
108
|
ttlOverride: ttl,
|
|
107
109
|
rendered: Boolean(rendered),
|
|
110
|
+
summaryLevel,
|
|
108
111
|
});
|
|
109
112
|
handleStaleRevalidationResult(this, revalResult);
|
|
110
113
|
return {
|
|
@@ -113,7 +116,7 @@ export default class FetchCommand extends BaseCommand {
|
|
|
113
116
|
artifact: revalResult.artifact,
|
|
114
117
|
};
|
|
115
118
|
}
|
|
116
|
-
async executeCacheMiss(normalizedUrl, currentTime, cacheKey) {
|
|
119
|
+
async executeCacheMiss(normalizedUrl, currentTime, cacheKey, summaryLevel) {
|
|
117
120
|
const { topic, tags, tier, ttl, rendered } = this.flags;
|
|
118
121
|
const siteModule = detectSite(normalizedUrl);
|
|
119
122
|
const useRendered = rendered || Boolean(siteModule?.defaults?.rendered);
|
|
@@ -129,7 +132,7 @@ export default class FetchCommand extends BaseCommand {
|
|
|
129
132
|
fetchResult = capture.fetchResult;
|
|
130
133
|
extraction = capture.extraction;
|
|
131
134
|
}
|
|
132
|
-
const artifact = createArtifactFromFetch(normalizedUrl, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl || null, currentTime);
|
|
135
|
+
const artifact = createArtifactFromFetch(normalizedUrl, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl || null, currentTime, summaryLevel);
|
|
133
136
|
artifact.metadata.topic = topic || null;
|
|
134
137
|
artifact.metadata.tags = tags || [];
|
|
135
138
|
artifact.metadata.site_module_id = siteModule?.id ?? null;
|
|
@@ -139,7 +142,9 @@ export default class FetchCommand extends BaseCommand {
|
|
|
139
142
|
else if (useRendered) {
|
|
140
143
|
artifact.metadata.capture_method = 'browser_fallback';
|
|
141
144
|
}
|
|
142
|
-
|
|
145
|
+
// Auto-tag from the fetched content when the caller supplied none, keeping cached pages
|
|
146
|
+
// searchable by keyword. Explicit --tags always win (handled in applyAutoTags).
|
|
147
|
+
return applyAutoTags(artifact);
|
|
143
148
|
}
|
|
144
149
|
buildResultData(url, normalizedUrl, cacheKey, storageDir, storageMode, cacheStatus, freshnessState, format, artifact, content, redirectedToGlobal) {
|
|
145
150
|
return {
|
|
@@ -178,36 +183,41 @@ export default class FetchCommand extends BaseCommand {
|
|
|
178
183
|
}
|
|
179
184
|
// Validates the duration flags up front, exiting with code 2 on a malformed value.
|
|
180
185
|
validateDurationFlags(ttl, maxAge) {
|
|
181
|
-
for (const [
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
]) {
|
|
185
|
-
if (!value)
|
|
186
|
-
continue;
|
|
187
|
-
try {
|
|
188
|
-
parseTtlToMs(value);
|
|
189
|
-
}
|
|
190
|
-
catch (err) {
|
|
191
|
-
this.error(`Invalid ${label}: ${err.message}`, { exit: 2 });
|
|
192
|
-
}
|
|
186
|
+
for (const msg of [durationFlagError('--ttl', ttl), durationFlagError('--max-age', maxAge)]) {
|
|
187
|
+
if (msg)
|
|
188
|
+
this.error(msg, { exit: 2 });
|
|
193
189
|
}
|
|
194
190
|
}
|
|
191
|
+
// Re-emit a runtime fetch failure with actionable next steps. Deep fetch/extract code throws plain
|
|
192
|
+
// Errors that otherwise reach the user as a bare "what broke" line with no "what to do"; this
|
|
193
|
+
// attaches recovery hints (e.g. import an auth-blocked page). Everything reaching here is a runtime
|
|
194
|
+
// failure — validation errors (bad flags/URL) exit before the try — so the contract's runtime code
|
|
195
|
+
// (1) applies. Suggestions render on stderr for humans only; under --json the envelope carries just
|
|
196
|
+
// the message (toErrorJson), so machine output is unchanged.
|
|
197
|
+
emitFetchError(err, url) {
|
|
198
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
199
|
+
const guidance = fetchFailureGuidance(message, url);
|
|
200
|
+
this.error(message, { exit: 1, suggestions: guidance?.suggestions, ref: guidance?.ref });
|
|
201
|
+
}
|
|
195
202
|
// Serves the cache when a fresh/revalidatable entry exists, otherwise fetches fresh. Reads fall
|
|
196
203
|
// back project→global; the resolved artifact is returned with the dir it lives in / landed in.
|
|
197
|
-
async resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime, located) {
|
|
204
|
+
async resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime, located, summaryLevel) {
|
|
198
205
|
if (located) {
|
|
199
206
|
// Revalidate where the entry already lives; on dry-run use the throwaway dir so the cache is
|
|
200
207
|
// never mutated. ponytail: revalidation rewrites in place, so a refreshed project entry that
|
|
201
208
|
// gains a secret is not re-routed — only first-time project writes are scanned.
|
|
202
209
|
const revalDir = tmpDir ?? located.dataDir;
|
|
203
|
-
const hit = await this.executeCacheHit(located.artifact, revalDir, currentTime);
|
|
210
|
+
const hit = await this.executeCacheHit(located.artifact, revalDir, currentTime, summaryLevel);
|
|
204
211
|
return { ...hit, storageDir: located.dataDir, redirectedToGlobal: false };
|
|
205
212
|
}
|
|
206
|
-
const artifact = await this.executeCacheMiss(normalizedUrl, currentTime, cacheKey);
|
|
213
|
+
const artifact = await this.executeCacheMiss(normalizedUrl, currentTime, cacheKey, summaryLevel);
|
|
207
214
|
const { dir, redirectedToGlobal } = this.persistFreshArtifact(roots, tmpDir, cacheKey, artifact);
|
|
215
|
+
// No entry existed at lookup, so there is no prior freshness to report. 'none' (not
|
|
216
|
+
// 'stale_expired') keeps the field honest — nothing expired; the page was simply uncached and
|
|
217
|
+
// has now been fetched fresh. Mirrors the `status` command's miss reporting.
|
|
208
218
|
return {
|
|
209
219
|
cacheStatus: 'miss',
|
|
210
|
-
freshnessState: '
|
|
220
|
+
freshnessState: 'none',
|
|
211
221
|
artifact,
|
|
212
222
|
storageDir: dir,
|
|
213
223
|
redirectedToGlobal,
|
|
@@ -230,11 +240,11 @@ export default class FetchCommand extends BaseCommand {
|
|
|
230
240
|
}
|
|
231
241
|
// Long references are split into searchable/inspectable section children whenever the page artifact
|
|
232
242
|
// is freshly written (T-22). Best-effort: never let chunking break the main result.
|
|
233
|
-
persistSectionsIfFresh(targetDir, artifact, currentTime, cacheStatus) {
|
|
243
|
+
persistSectionsIfFresh(targetDir, artifact, currentTime, cacheStatus, summaryLevel) {
|
|
234
244
|
if (cacheStatus !== 'miss' && cacheStatus !== 'refreshed')
|
|
235
245
|
return;
|
|
236
246
|
try {
|
|
237
|
-
persistSectionArtifacts(targetDir, artifact, currentTime);
|
|
247
|
+
persistSectionArtifacts(targetDir, artifact, currentTime, summaryLevel);
|
|
238
248
|
}
|
|
239
249
|
catch {
|
|
240
250
|
/* section generation is non-essential; ignore failures */
|
|
@@ -259,11 +269,12 @@ export default class FetchCommand extends BaseCommand {
|
|
|
259
269
|
this.error(`Invalid URL: ${err.message}`, { exit: 2 });
|
|
260
270
|
}
|
|
261
271
|
const { cacheKey, located, normalizedUrl, roots } = target;
|
|
272
|
+
const summaryLevel = loadSummaryLevel(this.config.configDir, process.cwd());
|
|
262
273
|
const tmpDir = dryRun ? mkdtempSync(join(tmpdir(), 'fnr-dry-run-')) : null;
|
|
263
274
|
const currentTime = new Date();
|
|
264
275
|
try {
|
|
265
|
-
const { cacheStatus, freshnessState, artifact, storageDir, redirectedToGlobal } = await this.resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime, located);
|
|
266
|
-
this.persistSectionsIfFresh(tmpDir ?? storageDir, artifact, currentTime, cacheStatus);
|
|
276
|
+
const { cacheStatus, freshnessState, artifact, storageDir, redirectedToGlobal } = await this.resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime, located, summaryLevel);
|
|
277
|
+
this.persistSectionsIfFresh(tmpDir ?? storageDir, artifact, currentTime, cacheStatus, summaryLevel);
|
|
267
278
|
const content = format === 'compressed' ? artifact.compressed : artifact.detailed;
|
|
268
279
|
const resultData = this.buildResultData(url, normalizedUrl, cacheKey, storageDir, roots.mode, cacheStatus, freshnessState, format, artifact, content, redirectedToGlobal);
|
|
269
280
|
if (!this.jsonEnabled()) {
|
|
@@ -271,6 +282,13 @@ export default class FetchCommand extends BaseCommand {
|
|
|
271
282
|
}
|
|
272
283
|
return resultData;
|
|
273
284
|
}
|
|
285
|
+
catch (err) {
|
|
286
|
+
// Stale-serve (exit 5) signals via process.exitCode and never throws, so it bypasses this
|
|
287
|
+
// path; only genuine fetch/extract failures land here and get classified guidance. Use the
|
|
288
|
+
// normalized URL in hints so the copy-paste command is canonical (and never echoes raw,
|
|
289
|
+
// unsanitized user input back to the terminal).
|
|
290
|
+
this.emitFetchError(err, normalizedUrl);
|
|
291
|
+
}
|
|
274
292
|
finally {
|
|
275
293
|
if (tmpDir) {
|
|
276
294
|
rmSync(tmpDir, { recursive: true, force: true });
|
|
@@ -278,6 +296,59 @@ export default class FetchCommand extends BaseCommand {
|
|
|
278
296
|
}
|
|
279
297
|
}
|
|
280
298
|
}
|
|
299
|
+
// Maps a runtime fetch failure to recovery steps keyed off its message. Returns undefined for
|
|
300
|
+
// unrecognized failures, which then surface with their original message and no extra hint. The
|
|
301
|
+
// patterns match the thrown text in fetcher.ts/browser.ts; keep them in sync if those messages
|
|
302
|
+
// change. Resolutions mirror the published troubleshooting guide.
|
|
303
|
+
export function fetchFailureGuidance(message, url) {
|
|
304
|
+
const ref = 'https://taurgis.github.io/bonsai/troubleshooting';
|
|
305
|
+
// Name the pipe step explicitly: bare `import --stdin` blocks waiting for input, so a reader (or
|
|
306
|
+
// an agent) running the hint verbatim would hang. Show the content being piped in from a file.
|
|
307
|
+
const importHint = `Open it in a browser, save the page, then import it: cat page.md | bonsai import ${url} --stdin`;
|
|
308
|
+
// 401/403: an auth wall or anti-scraping WAF. Bonsai has no authenticated-fetch path in v1.
|
|
309
|
+
if (/failed with status 40[13]\b/.test(message)) {
|
|
310
|
+
return {
|
|
311
|
+
suggestions: ['The page requires authentication or blocks automated requests.', importHint],
|
|
312
|
+
ref,
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
if (/failed with status 404\b/.test(message)) {
|
|
316
|
+
return { suggestions: ['Check the URL is correct and the page still exists.'] };
|
|
317
|
+
}
|
|
318
|
+
if (/failed with status 5\d\d\b/.test(message)) {
|
|
319
|
+
return {
|
|
320
|
+
suggestions: ['The server returned an error. Retry later or verify the host is healthy.'],
|
|
321
|
+
};
|
|
322
|
+
}
|
|
323
|
+
// Server returned a non-HTML body (JSON, binary, or no Content-Type at all). Scope the opener to
|
|
324
|
+
// the real failure: --rendered does produce HTML, so "only scrapes HTML" would read as misleading.
|
|
325
|
+
if (/Rejected content type|does not look like HTML/.test(message)) {
|
|
326
|
+
return {
|
|
327
|
+
suggestions: [
|
|
328
|
+
'The server returned a non-HTML response (e.g. JSON or binary).',
|
|
329
|
+
'If the page is rendered by client-side JavaScript, retry with --rendered.',
|
|
330
|
+
importHint,
|
|
331
|
+
],
|
|
332
|
+
ref,
|
|
333
|
+
};
|
|
334
|
+
}
|
|
335
|
+
if (/DNS resolution failed/.test(message)) {
|
|
336
|
+
return {
|
|
337
|
+
suggestions: ['Check the hostname is spelled correctly and resolves on a public network.'],
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
// A hostname that only resolves to a private/local IP at request time (literal private addresses
|
|
341
|
+
// are rejected earlier with exit 2). Blocked to prevent SSRF; only public hosts can be fetched.
|
|
342
|
+
if (/blocked local or private target/.test(message)) {
|
|
343
|
+
return {
|
|
344
|
+
suggestions: [
|
|
345
|
+
'The hostname resolves to a private or local address, which is blocked to prevent SSRF. Only public http(s) hosts can be fetched.',
|
|
346
|
+
],
|
|
347
|
+
ref,
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
return undefined;
|
|
351
|
+
}
|
|
281
352
|
// Copies Phase 2 capability provenance from a capture outcome onto the artifact metadata.
|
|
282
353
|
function applyCaptureMetadata(artifact, capture) {
|
|
283
354
|
const meta = artifact.metadata;
|
|
@@ -23,6 +23,7 @@ export default class ResearchImport extends BaseCommand<typeof ResearchImport> {
|
|
|
23
23
|
storage: import("@oclif/core/interfaces").OptionFlag<"global" | "project" | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
24
24
|
};
|
|
25
25
|
static stdoutIsPrimaryData: boolean;
|
|
26
|
+
protected stdinIsInteractive(): boolean;
|
|
26
27
|
private readStdin;
|
|
27
28
|
private getSourceUrls;
|
|
28
29
|
private validateSourceMode;
|
package/dist/commands/import.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Args, Flags } from '@oclif/core';
|
|
1
|
+
import { Args, Errors, Flags } from '@oclif/core';
|
|
2
2
|
import { createHash } from 'node:crypto';
|
|
3
3
|
import * as fs from 'node:fs';
|
|
4
4
|
import * as path from 'node:path';
|
|
@@ -8,8 +8,10 @@ import { deriveCacheKey } from '../lib/research/cache-key.js';
|
|
|
8
8
|
import { getArtifactPath } from '../lib/research/storage.js';
|
|
9
9
|
import { loadStoreRoots } from '../lib/research/store-roots.js';
|
|
10
10
|
import { writeArtifactSecurely } from '../lib/research/secure-write.js';
|
|
11
|
+
import { loadSummaryLevel } from '../lib/config/index.js';
|
|
11
12
|
import { getPolicy } from '../lib/research/freshness.js';
|
|
12
|
-
import {
|
|
13
|
+
import { buildCompressed } from '../lib/research/compress.js';
|
|
14
|
+
import { applyAutoTags } from '../lib/research/keywords.js';
|
|
13
15
|
import { estimateTokens } from '../lib/research/token-estimate.js';
|
|
14
16
|
export default class ResearchImport extends BaseCommand {
|
|
15
17
|
static id = 'import';
|
|
@@ -32,6 +34,10 @@ export default class ResearchImport extends BaseCommand {
|
|
|
32
34
|
static args = {
|
|
33
35
|
url: Args.string({
|
|
34
36
|
required: false,
|
|
37
|
+
// oclif fills an omitted optional arg from piped stdin unless ignoreStdin is set. Without
|
|
38
|
+
// this, the Markdown piped for `--stdin` is swallowed into `url`, making multi-source import
|
|
39
|
+
// (`import --stdin --source-url ...`) wrongly look like it also got a positional URL.
|
|
40
|
+
ignoreStdin: true,
|
|
35
41
|
description: 'The single source URL of the page (only for single-source import).',
|
|
36
42
|
}),
|
|
37
43
|
};
|
|
@@ -76,6 +82,11 @@ export default class ResearchImport extends BaseCommand {
|
|
|
76
82
|
})(),
|
|
77
83
|
};
|
|
78
84
|
static stdoutIsPrimaryData = true;
|
|
85
|
+
// Isolated so tests can force the interactive branch; `isTTY` is `true` only on a real terminal and
|
|
86
|
+
// falsy (undefined) for pipes, files, and /dev/null. See https://nodejs.org/api/process.html#processstdin
|
|
87
|
+
stdinIsInteractive() {
|
|
88
|
+
return process.stdin.isTTY === true;
|
|
89
|
+
}
|
|
79
90
|
async readStdin(limitBytes = 1024 * 1024) {
|
|
80
91
|
return new Promise((resolve, reject) => {
|
|
81
92
|
let data = '';
|
|
@@ -162,16 +173,34 @@ export default class ResearchImport extends BaseCommand {
|
|
|
162
173
|
}
|
|
163
174
|
async readAndValidateInput() {
|
|
164
175
|
if (this.flags.stdin) {
|
|
176
|
+
// On an interactive terminal with nothing piped, reading stdin blocks forever waiting for EOF —
|
|
177
|
+
// a silent hang with no feedback. Fail fast with a usage error instead; piped/redirected input
|
|
178
|
+
// (the agent and CI path) has isTTY falsy and reads normally.
|
|
179
|
+
if (this.stdinIsInteractive()) {
|
|
180
|
+
this.error('No data piped to --stdin.', {
|
|
181
|
+
exit: 2,
|
|
182
|
+
suggestions: [
|
|
183
|
+
'Pipe Markdown content, e.g. cat notes.md | bonsai import <url> --stdin',
|
|
184
|
+
'Or read from a file: bonsai import <url> --file notes.md',
|
|
185
|
+
],
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
// Initialized so the post-catch read below never depends on this.error()'s `never` return type
|
|
189
|
+
// for definite assignment; the empty-content guard rejects '' as a usage error anyway.
|
|
190
|
+
let rawInput = '';
|
|
165
191
|
try {
|
|
166
|
-
|
|
167
|
-
if (!rawInput.trim()) {
|
|
168
|
-
this.error('Empty stdin content provided.', { exit: 2 });
|
|
169
|
-
}
|
|
170
|
-
return rawInput;
|
|
192
|
+
rawInput = await this.readStdin();
|
|
171
193
|
}
|
|
172
194
|
catch (err) {
|
|
195
|
+
// Genuine stream/size failure reading stdin → I/O failure (exit 1).
|
|
173
196
|
this.error(err.message, { exit: 1 });
|
|
174
197
|
}
|
|
198
|
+
// Empty input is a usage error (exit 2), not an I/O failure — keep it out of the
|
|
199
|
+
// catch above so its exit code is not clobbered to 1.
|
|
200
|
+
if (!rawInput.trim()) {
|
|
201
|
+
this.error('Empty stdin content provided.', { exit: 2 });
|
|
202
|
+
}
|
|
203
|
+
return rawInput;
|
|
175
204
|
}
|
|
176
205
|
if (this.flags.file) {
|
|
177
206
|
const filePath = path.resolve(this.flags.file);
|
|
@@ -179,6 +208,11 @@ export default class ResearchImport extends BaseCommand {
|
|
|
179
208
|
return this.readAndValidateFile(filePath);
|
|
180
209
|
}
|
|
181
210
|
catch (err) {
|
|
211
|
+
// readAndValidateFile raises usage errors (missing/not-a-file/empty) via this.error,
|
|
212
|
+
// which throw CLIErrors carrying their own exit code and message — preserve those.
|
|
213
|
+
// Only an unexpected read failure becomes a wrapped I/O failure (exit 1).
|
|
214
|
+
if (err instanceof Errors.CLIError)
|
|
215
|
+
throw err;
|
|
182
216
|
this.error(`Failed to read file: ${err.message}`, { exit: 1 });
|
|
183
217
|
}
|
|
184
218
|
}
|
|
@@ -200,7 +234,11 @@ export default class ResearchImport extends BaseCommand {
|
|
|
200
234
|
const staleAfterTime = new Date(currentTime.getTime() + freshWindowMs);
|
|
201
235
|
const inputFormat = this.flags['input-format'];
|
|
202
236
|
const detailed = rawInput;
|
|
203
|
-
|
|
237
|
+
// A caller-supplied compressed input is trusted as-is; only a detailed import is condensed here,
|
|
238
|
+
// and it shares the same buildCompressed policy (structural pass + extractive fallback) as fetch.
|
|
239
|
+
const compressed = inputFormat === 'detailed'
|
|
240
|
+
? buildCompressed(rawInput, loadSummaryLevel(this.config.configDir, process.cwd()))
|
|
241
|
+
: rawInput;
|
|
204
242
|
const contentHash = createHash('sha256').update(detailed).digest('hex');
|
|
205
243
|
const metadata = {
|
|
206
244
|
schema_version: 1,
|
|
@@ -240,13 +278,15 @@ export default class ResearchImport extends BaseCommand {
|
|
|
240
278
|
section_anchor: null,
|
|
241
279
|
section_heading_path: null,
|
|
242
280
|
};
|
|
243
|
-
|
|
281
|
+
// Auto-tag from the content when the caller supplied none, so imported notes stay searchable by
|
|
282
|
+
// keyword without forcing manual tagging. Explicit --tags always win (handled in applyAutoTags).
|
|
283
|
+
return applyAutoTags({
|
|
244
284
|
metadata,
|
|
245
285
|
summary: `Synthesized research for ${hasSingle ? singleNormalizedUrl : this.flags.topic}`,
|
|
246
286
|
compressed,
|
|
247
287
|
detailed,
|
|
248
288
|
provenance: `Imported via agent-supplied research at ${currentTime.toISOString()}`,
|
|
249
|
-
};
|
|
289
|
+
});
|
|
250
290
|
}
|
|
251
291
|
async run() {
|
|
252
292
|
const hasSingle = Boolean(this.args.url);
|
|
@@ -292,7 +332,7 @@ export default class ResearchImport extends BaseCommand {
|
|
|
292
332
|
captureMethod: 'agent_supplied',
|
|
293
333
|
extractionStatus: 'agent_supplied',
|
|
294
334
|
extractionConfidence: 'high',
|
|
295
|
-
qualityNotes:
|
|
335
|
+
qualityNotes: artifact.metadata.quality_notes,
|
|
296
336
|
fetchedAt: null,
|
|
297
337
|
validatedAt: artifact.metadata.validated_at,
|
|
298
338
|
staleAfter: artifact.metadata.stale_after,
|
package/dist/commands/list.js
CHANGED
|
@@ -3,6 +3,9 @@ import { BaseCommand } from '../base-command.js';
|
|
|
3
3
|
import { scanCacheDirs } from '../lib/research/storage.js';
|
|
4
4
|
import { loadStoreRoots } from '../lib/research/store-roots.js';
|
|
5
5
|
import { evaluateFreshness } from '../lib/research/freshness.js';
|
|
6
|
+
import { resultListHeading, truncationNotice } from '../lib/text.js';
|
|
7
|
+
// Listings are ordered newest-first, so the truncation word is "first"; --limit caps at 100.
|
|
8
|
+
const LIST_LABELS = { noun: 'cached research', order: 'first', maxLimit: 100 };
|
|
6
9
|
export default class ResearchList extends BaseCommand {
|
|
7
10
|
static id = 'list';
|
|
8
11
|
static summary = 'List cached research artifacts by metadata filters.';
|
|
@@ -82,6 +85,12 @@ export default class ResearchList extends BaseCommand {
|
|
|
82
85
|
return scanCacheDirs(readRoots, (artifact, filePath) => {
|
|
83
86
|
if (artifact.metadata.status !== 'active')
|
|
84
87
|
return null;
|
|
88
|
+
// Section children are sub-chunks of a page, not artifacts a user "has" — they would flood the
|
|
89
|
+
// listing (one page yields dozens) and aren't in the documented source/research_note contract.
|
|
90
|
+
// They stay discoverable through `search` (which ranks them) and `inspect` (which lists a
|
|
91
|
+
// page's sections). `list` answers "what pages/notes do I have?", so keep it page-level.
|
|
92
|
+
if (artifact.metadata.artifact_type === 'section')
|
|
93
|
+
return null;
|
|
85
94
|
const freshness = evaluateFreshness(artifact.metadata, currentTime, null);
|
|
86
95
|
if (!this.matchesFilters(artifact.metadata, freshness))
|
|
87
96
|
return null;
|
|
@@ -101,14 +110,14 @@ export default class ResearchList extends BaseCommand {
|
|
|
101
110
|
};
|
|
102
111
|
});
|
|
103
112
|
}
|
|
104
|
-
logListResults(finalResults) {
|
|
113
|
+
logListResults(finalResults, totalMatched) {
|
|
105
114
|
if (this.jsonEnabled())
|
|
106
115
|
return;
|
|
107
116
|
if (finalResults.length === 0) {
|
|
108
117
|
this.log('No cached research entries found matching filters.');
|
|
109
118
|
return;
|
|
110
119
|
}
|
|
111
|
-
this.log(
|
|
120
|
+
this.log(`${resultListHeading(totalMatched, finalResults.length, LIST_LABELS)}\n`);
|
|
112
121
|
finalResults.forEach((res, index) => {
|
|
113
122
|
this.log(`${index + 1}. [${res.topic || 'No Topic'}] Key: ${res.cacheKey}`);
|
|
114
123
|
this.log(` Type: ${res.artifactType} | Freshness: ${res.freshness}`);
|
|
@@ -132,7 +141,13 @@ export default class ResearchList extends BaseCommand {
|
|
|
132
141
|
return timeB - timeA;
|
|
133
142
|
});
|
|
134
143
|
const finalResults = results.slice(0, this.flags.limit);
|
|
135
|
-
|
|
144
|
+
// Under --json the human heading is suppressed, so surface truncation on stderr instead (warn
|
|
145
|
+
// always emits, even in --json) without touching the stdout envelope. Human mode already shows
|
|
146
|
+
// it in the heading, so only warn under --json.
|
|
147
|
+
const notice = truncationNotice(results.length, finalResults.length, LIST_LABELS);
|
|
148
|
+
if (notice && this.jsonEnabled())
|
|
149
|
+
this.warn(notice);
|
|
150
|
+
this.logListResults(finalResults, results.length);
|
|
136
151
|
return finalResults;
|
|
137
152
|
}
|
|
138
153
|
}
|
package/dist/commands/prune.js
CHANGED
|
@@ -4,7 +4,8 @@ import { join } from 'node:path';
|
|
|
4
4
|
import { BaseCommand } from '../base-command.js';
|
|
5
5
|
import { scanCacheDir } from '../lib/research/storage.js';
|
|
6
6
|
import { loadStoreRoots } from '../lib/research/store-roots.js';
|
|
7
|
-
import { parseTtlToMs } from '../lib/research/freshness.js';
|
|
7
|
+
import { parseTtlToMs, durationFlagError } from '../lib/research/freshness.js';
|
|
8
|
+
import { pluralize } from '../lib/text.js';
|
|
8
9
|
export default class ResearchPrune extends BaseCommand {
|
|
9
10
|
static id = 'prune';
|
|
10
11
|
static summary = 'Clean up old or inactive research cache entries.';
|
|
@@ -48,6 +49,15 @@ export default class ResearchPrune extends BaseCommand {
|
|
|
48
49
|
if (!this.flags['dry-run'] && !this.flags.yes) {
|
|
49
50
|
this.error('Safety check: Please specify --yes to confirm pruning, or --dry-run to list files that would be deleted.', { exit: 2 });
|
|
50
51
|
}
|
|
52
|
+
// Validate durations up front: scanCacheDir swallows per-file errors, so a malformed
|
|
53
|
+
// --older-than/--inactive would otherwise be silently ignored and report "0 entries".
|
|
54
|
+
for (const msg of [
|
|
55
|
+
durationFlagError('--older-than', this.flags['older-than']),
|
|
56
|
+
durationFlagError('--inactive', this.flags.inactive),
|
|
57
|
+
]) {
|
|
58
|
+
if (msg)
|
|
59
|
+
this.error(msg, { exit: 2 });
|
|
60
|
+
}
|
|
51
61
|
}
|
|
52
62
|
shouldPrune(meta, currentTime) {
|
|
53
63
|
const fetched = meta.fetched_at ? new Date(meta.fetched_at).getTime() : 0;
|
|
@@ -109,24 +119,30 @@ export default class ResearchPrune extends BaseCommand {
|
|
|
109
119
|
const filesToPrune = this.findPruneCandidates(roots.readRoots, currentTime);
|
|
110
120
|
const dryRun = this.flags['dry-run'];
|
|
111
121
|
const count = filesToPrune.length;
|
|
122
|
+
// Track deletions actually performed so the JSON envelope and human output agree even when an
|
|
123
|
+
// unlink fails (e.g. a permission error). Reporting candidate count as `prunedCount` would
|
|
124
|
+
// overstate success and mislead an agent branching on the result.
|
|
125
|
+
let prunedCount = 0;
|
|
112
126
|
if (dryRun) {
|
|
113
127
|
if (!this.jsonEnabled()) {
|
|
114
|
-
|
|
128
|
+
const noun = pluralize(count, 'entry', 'entries');
|
|
129
|
+
this.log(`[Dry Run] Found ${count} research cache ${noun} that would be pruned:\n`);
|
|
115
130
|
filesToPrune.forEach((f) => {
|
|
116
131
|
this.log(`- [${f.topic || 'No Topic'}] Key: ${f.cacheKey} (${f.url || 'Imported note'})`);
|
|
117
132
|
});
|
|
118
133
|
}
|
|
119
134
|
}
|
|
120
135
|
else {
|
|
121
|
-
|
|
136
|
+
prunedCount = this.deletePruneCandidates(filesToPrune);
|
|
122
137
|
if (!this.jsonEnabled()) {
|
|
123
|
-
|
|
138
|
+
const noun = pluralize(count, 'entry', 'entries');
|
|
139
|
+
this.log(`Successfully pruned ${prunedCount} of ${count} research cache ${noun}.`);
|
|
124
140
|
}
|
|
125
141
|
}
|
|
126
142
|
return {
|
|
127
143
|
dryRun,
|
|
128
|
-
prunedCount
|
|
129
|
-
candidateCount:
|
|
144
|
+
prunedCount,
|
|
145
|
+
candidateCount: count,
|
|
130
146
|
files: filesToPrune.map((f) => ({
|
|
131
147
|
cacheKey: f.cacheKey,
|
|
132
148
|
path: f.path,
|