@taurgis/bonsai 1.0.3 → 1.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/README.md +38 -6
  2. package/agents/agents/salesforce-docs-researcher.agent.md +117 -0
  3. package/agents/hooks/README.md +62 -0
  4. package/agents/hooks/claude-code/hooks/bonsai-web-fetch.sh +22 -0
  5. package/agents/hooks/claude-code/settings.json +16 -0
  6. package/agents/hooks/codex/hooks/bonsai-user-prompt-context.sh +18 -0
  7. package/agents/hooks/codex/hooks/bonsai-web-fetch.sh +22 -0
  8. package/agents/hooks/codex/hooks.json +28 -0
  9. package/agents/hooks/cursor/hooks/bonsai-web-fetch.sh +21 -0
  10. package/agents/hooks/cursor/hooks.json +12 -0
  11. package/agents/hooks/github-copilot/bonsai-web-fetch.json +23 -0
  12. package/agents/hooks/github-copilot/bonsai-web-fetch.sh +19 -0
  13. package/agents/instructions/salesforce-research.instructions.md +57 -0
  14. package/agents/instructions/web-research.instructions.md +2 -1
  15. package/bin/cli.mjs +13 -3
  16. package/dist/base-command.d.ts +20 -0
  17. package/dist/base-command.js +27 -1
  18. package/dist/commands/config/get.js +2 -2
  19. package/dist/commands/config/list.js +2 -2
  20. package/dist/commands/config/set.js +2 -2
  21. package/dist/commands/config/unset.js +2 -2
  22. package/dist/commands/fetch.d.ts +5 -0
  23. package/dist/commands/fetch.js +98 -27
  24. package/dist/commands/import.d.ts +1 -0
  25. package/dist/commands/import.js +51 -11
  26. package/dist/commands/list.js +18 -3
  27. package/dist/commands/prune.js +22 -6
  28. package/dist/commands/search.js +31 -30
  29. package/dist/commands/status.js +10 -9
  30. package/dist/hooks/command-not-found/suggest.d.ts +9 -0
  31. package/dist/hooks/command-not-found/suggest.js +93 -0
  32. package/dist/lib/config/index.d.ts +8 -3
  33. package/dist/lib/config/index.js +19 -4
  34. package/dist/lib/config/resolve.d.ts +21 -1
  35. package/dist/lib/config/resolve.js +42 -1
  36. package/dist/lib/config/schema.d.ts +3 -0
  37. package/dist/lib/config/schema.js +11 -1
  38. package/dist/lib/research/artifact.d.ts +13 -2
  39. package/dist/lib/research/artifact.js +27 -7
  40. package/dist/lib/research/browser.d.ts +12 -0
  41. package/dist/lib/research/browser.js +40 -3
  42. package/dist/lib/research/compress.d.ts +19 -0
  43. package/dist/lib/research/compress.js +31 -0
  44. package/dist/lib/research/docs/capabilities.d.ts +1 -1
  45. package/dist/lib/research/docs/code-blocks.js +24 -0
  46. package/dist/lib/research/docs/discovery-index.d.ts +14 -0
  47. package/dist/lib/research/docs/discovery-index.js +112 -0
  48. package/dist/lib/research/docs/machine-readable.d.ts +2 -2
  49. package/dist/lib/research/docs/machine-readable.js +3 -7
  50. package/dist/lib/research/docs/remote-search-runner.js +61 -1
  51. package/dist/lib/research/docs/section-artifacts.d.ts +3 -2
  52. package/dist/lib/research/docs/section-artifacts.js +5 -5
  53. package/dist/lib/research/docs/sections.d.ts +1 -0
  54. package/dist/lib/research/docs/sections.js +19 -1
  55. package/dist/lib/research/freshness.d.ts +9 -1
  56. package/dist/lib/research/freshness.js +24 -2
  57. package/dist/lib/research/keywords.d.ts +24 -0
  58. package/dist/lib/research/keywords.js +247 -0
  59. package/dist/lib/research/revalidate.d.ts +3 -1
  60. package/dist/lib/research/revalidate.js +8 -4
  61. package/dist/lib/research/search-index.d.ts +18 -0
  62. package/dist/lib/research/search-index.js +109 -0
  63. package/dist/lib/research/secret-scan.js +4 -2
  64. package/dist/lib/research/storage.d.ts +1 -0
  65. package/dist/lib/research/storage.js +1 -1
  66. package/dist/lib/research/summarize.d.ts +17 -0
  67. package/dist/lib/research/summarize.js +400 -0
  68. package/dist/lib/research/url.js +5 -2
  69. package/dist/lib/text.d.ts +33 -0
  70. package/dist/lib/text.js +63 -0
  71. package/oclif.manifest.json +8 -8
  72. package/package.json +17 -3
@@ -1,9 +1,9 @@
1
1
  import { ConfigCommand, configScopeFlags } from './base.js';
2
- import { ALL_KEYS, KEY_META, BUILT_IN_DEFAULTS, readScopedConfig } from '../../lib/config/index.js';
2
+ import { ALL_KEYS, KEY_META, BUILT_IN_DEFAULTS, readScopedConfig, validKeysHint, } from '../../lib/config/index.js';
3
3
  export default class ConfigList extends ConfigCommand {
4
4
  static id = 'config list';
5
5
  static summary = 'List all research configuration keys and their effective values.';
6
- static description = 'Show every configuration key with its current value. Use --global/--local to show a single scope.\n\nValid keys: storage.';
6
+ static description = `Show every configuration key with its current value. Use --global/--local to show a single scope.\n\nValid keys: ${validKeysHint()}.`;
7
7
  static examples = [
8
8
  {
9
9
  description: 'Show all effective config values',
@@ -1,11 +1,11 @@
1
1
  import { Args, Flags } from '@oclif/core';
2
2
  import { ConfigCommand, configScopeFlags } from './base.js';
3
- import { KEY_META, writeUserConfig, writeProjectConfig } from '../../lib/config/index.js';
3
+ import { KEY_META, writeUserConfig, writeProjectConfig, validKeysHint, } from '../../lib/config/index.js';
4
4
  export default class ConfigSet extends ConfigCommand {
5
5
  static id = 'config set';
6
6
  static summary = 'Set a research configuration key.';
7
7
  static description = 'Persist a configuration value. Writes user-level config by default; pass --local to write the project-level config (.bonsai.json in cwd).\n\n' +
8
- 'Valid keys: storage. The inline form `<key>=<value>` is also accepted.';
8
+ `Valid keys: ${validKeysHint()}. The inline form \`<key>=<value>\` is also accepted.`;
9
9
  static examples = [
10
10
  {
11
11
  description: 'Store research cache inside the current project',
@@ -1,10 +1,10 @@
1
1
  import { Args, Flags } from '@oclif/core';
2
2
  import { ConfigCommand, configScopeFlags } from './base.js';
3
- import { writeUserConfig, writeProjectConfig } from '../../lib/config/index.js';
3
+ import { writeUserConfig, writeProjectConfig, validKeysHint } from '../../lib/config/index.js';
4
4
  export default class ConfigUnset extends ConfigCommand {
5
5
  static id = 'config unset';
6
6
  static summary = 'Remove a research configuration key.';
7
- static description = 'Delete a key from user-level config (default) or project-level config (--local), restoring the built-in default.\n\nValid keys: storage.';
7
+ static description = `Delete a key from user-level config (default) or project-level config (--local), restoring the built-in default.\n\nValid keys: ${validKeysHint()}.`;
8
8
  static examples = [
9
9
  {
10
10
  description: 'Remove the user-level storage setting',
@@ -30,8 +30,13 @@ export default class FetchCommand extends BaseCommand<typeof FetchCommand> {
30
30
  private executeCacheMiss;
31
31
  private buildResultData;
32
32
  private validateDurationFlags;
33
+ private emitFetchError;
33
34
  private resolveArtifact;
34
35
  private persistFreshArtifact;
35
36
  private persistSectionsIfFresh;
36
37
  run(): Promise<unknown>;
37
38
  }
39
+ export declare function fetchFailureGuidance(message: string, url: string): {
40
+ suggestions: string[];
41
+ ref?: string;
42
+ } | undefined;
@@ -5,13 +5,15 @@ import { join } from 'node:path';
5
5
  import { BaseCommand } from '../base-command.js';
6
6
  import { writeArtifact, getArtifactPath } from '../lib/research/storage.js';
7
7
  import { writeArtifactSecurely } from '../lib/research/secure-write.js';
8
- import { evaluateFreshness, parseTtlToMs, checkMaxAgeExpired } from '../lib/research/freshness.js';
8
+ import { loadSummaryLevel } from '../lib/config/index.js';
9
+ import { evaluateFreshness, checkMaxAgeExpired, durationFlagError, } from '../lib/research/freshness.js';
9
10
  import { revalidateCache, createArtifactFromFetch } from '../lib/research/revalidate.js';
10
11
  import { fetchStaticHtml, fetchText } from '../lib/research/fetcher.js';
11
12
  import { fetchRenderedHtml } from '../lib/research/browser.js';
12
13
  import { capturePage } from '../lib/research/capture.js';
13
14
  import { persistSectionArtifacts } from '../lib/research/docs/section-artifacts.js';
14
15
  import { resolveResearchTarget } from '../lib/research/resolve-target.js';
16
+ import { applyAutoTags } from '../lib/research/keywords.js';
15
17
  import { detectSite } from '../sites/index.js';
16
18
  const CAPTURE_DEPS = {
17
19
  fetchStatic: (url) => fetchStaticHtml(url),
@@ -22,7 +24,7 @@ export default class FetchCommand extends BaseCommand {
22
24
  static id = 'fetch';
23
25
  static hidden = true;
24
26
  static summary = 'An advanced, locally cached web research tool optimized for LLM ingestion.';
25
- static description = 'Scrapes the specified URL, strips HTML boilerplate, converts the semantic payload into clean Markdown format, and caches the result locally using dynamic TTL rules.';
27
+ static description = 'Scrapes the specified URL, strips HTML boilerplate, converts the semantic payload into clean Markdown format, and caches the result locally using dynamic TTL rules.\n\nUsually invoked via the shorthand `bonsai <url>` rather than `bonsai fetch <url>`.';
26
28
  static examples = [
27
29
  {
28
30
  description: 'Research a URL with detailed output, tagged with topic and tags',
@@ -62,7 +64,7 @@ export default class FetchCommand extends BaseCommand {
62
64
  })(),
63
65
  ttl: Flags.string({
64
66
  char: 'l',
65
- description: 'Predicted lifespan of the data (e.g., "2h", "7d", "3m").',
67
+ description: 'Predicted lifespan: number + h/d/w/m/y (m = months), e.g. "2h", "7d", "6m".',
66
68
  }),
67
69
  'max-age': Flags.string({
68
70
  description: 'Maximum age of cache to accept (e.g., "1d", "30d").',
@@ -92,7 +94,7 @@ export default class FetchCommand extends BaseCommand {
92
94
  envelopeCommandId() {
93
95
  return this.config.bin;
94
96
  }
95
- async executeCacheHit(cached, targetDir, currentTime) {
97
+ async executeCacheHit(cached, targetDir, currentTime, summaryLevel) {
96
98
  const { ttl, 'max-age': maxAge, 'allow-stale': allowStale, rendered } = this.flags;
97
99
  const isExpired = checkMaxAgeExpired(cached, currentTime, maxAge);
98
100
  const freshnessState = isExpired
@@ -105,6 +107,7 @@ export default class FetchCommand extends BaseCommand {
105
107
  allowStale: Boolean(allowStale),
106
108
  ttlOverride: ttl,
107
109
  rendered: Boolean(rendered),
110
+ summaryLevel,
108
111
  });
109
112
  handleStaleRevalidationResult(this, revalResult);
110
113
  return {
@@ -113,7 +116,7 @@ export default class FetchCommand extends BaseCommand {
113
116
  artifact: revalResult.artifact,
114
117
  };
115
118
  }
116
- async executeCacheMiss(normalizedUrl, currentTime, cacheKey) {
119
+ async executeCacheMiss(normalizedUrl, currentTime, cacheKey, summaryLevel) {
117
120
  const { topic, tags, tier, ttl, rendered } = this.flags;
118
121
  const siteModule = detectSite(normalizedUrl);
119
122
  const useRendered = rendered || Boolean(siteModule?.defaults?.rendered);
@@ -129,7 +132,7 @@ export default class FetchCommand extends BaseCommand {
129
132
  fetchResult = capture.fetchResult;
130
133
  extraction = capture.extraction;
131
134
  }
132
- const artifact = createArtifactFromFetch(normalizedUrl, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl || null, currentTime);
135
+ const artifact = createArtifactFromFetch(normalizedUrl, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl || null, currentTime, summaryLevel);
133
136
  artifact.metadata.topic = topic || null;
134
137
  artifact.metadata.tags = tags || [];
135
138
  artifact.metadata.site_module_id = siteModule?.id ?? null;
@@ -139,7 +142,9 @@ export default class FetchCommand extends BaseCommand {
139
142
  else if (useRendered) {
140
143
  artifact.metadata.capture_method = 'browser_fallback';
141
144
  }
142
- return artifact;
145
+ // Auto-tag from the fetched content when the caller supplied none, keeping cached pages
146
+ // searchable by keyword. Explicit --tags always win (handled in applyAutoTags).
147
+ return applyAutoTags(artifact);
143
148
  }
144
149
  buildResultData(url, normalizedUrl, cacheKey, storageDir, storageMode, cacheStatus, freshnessState, format, artifact, content, redirectedToGlobal) {
145
150
  return {
@@ -178,36 +183,41 @@ export default class FetchCommand extends BaseCommand {
178
183
  }
179
184
  // Validates the duration flags up front, exiting with code 2 on a malformed value.
180
185
  validateDurationFlags(ttl, maxAge) {
181
- for (const [label, value] of [
182
- ['TTL', ttl],
183
- ['max-age', maxAge],
184
- ]) {
185
- if (!value)
186
- continue;
187
- try {
188
- parseTtlToMs(value);
189
- }
190
- catch (err) {
191
- this.error(`Invalid ${label}: ${err.message}`, { exit: 2 });
192
- }
186
+ for (const msg of [durationFlagError('--ttl', ttl), durationFlagError('--max-age', maxAge)]) {
187
+ if (msg)
188
+ this.error(msg, { exit: 2 });
193
189
  }
194
190
  }
191
+ // Re-emit a runtime fetch failure with actionable next steps. Deep fetch/extract code throws plain
192
+ // Errors that otherwise reach the user as a bare "what broke" line with no "what to do"; this
193
+ // attaches recovery hints (e.g. import an auth-blocked page). Everything reaching here is a runtime
194
+ // failure — validation errors (bad flags/URL) exit before the try — so the contract's runtime code
195
+ // (1) applies. Suggestions render on stderr for humans only; under --json the envelope carries just
196
+ // the message (toErrorJson), so machine output is unchanged.
197
+ emitFetchError(err, url) {
198
+ const message = err instanceof Error ? err.message : String(err);
199
+ const guidance = fetchFailureGuidance(message, url);
200
+ this.error(message, { exit: 1, suggestions: guidance?.suggestions, ref: guidance?.ref });
201
+ }
195
202
  // Serves the cache when a fresh/revalidatable entry exists, otherwise fetches fresh. Reads fall
196
203
  // back project→global; the resolved artifact is returned with the dir it lives in / landed in.
197
- async resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime, located) {
204
+ async resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime, located, summaryLevel) {
198
205
  if (located) {
199
206
  // Revalidate where the entry already lives; on dry-run use the throwaway dir so the cache is
200
207
  // never mutated. ponytail: revalidation rewrites in place, so a refreshed project entry that
201
208
  // gains a secret is not re-routed — only first-time project writes are scanned.
202
209
  const revalDir = tmpDir ?? located.dataDir;
203
- const hit = await this.executeCacheHit(located.artifact, revalDir, currentTime);
210
+ const hit = await this.executeCacheHit(located.artifact, revalDir, currentTime, summaryLevel);
204
211
  return { ...hit, storageDir: located.dataDir, redirectedToGlobal: false };
205
212
  }
206
- const artifact = await this.executeCacheMiss(normalizedUrl, currentTime, cacheKey);
213
+ const artifact = await this.executeCacheMiss(normalizedUrl, currentTime, cacheKey, summaryLevel);
207
214
  const { dir, redirectedToGlobal } = this.persistFreshArtifact(roots, tmpDir, cacheKey, artifact);
215
+ // No entry existed at lookup, so there is no prior freshness to report. 'none' (not
216
+ // 'stale_expired') keeps the field honest — nothing expired; the page was simply uncached and
217
+ // has now been fetched fresh. Mirrors the `status` command's miss reporting.
208
218
  return {
209
219
  cacheStatus: 'miss',
210
- freshnessState: 'stale_expired',
220
+ freshnessState: 'none',
211
221
  artifact,
212
222
  storageDir: dir,
213
223
  redirectedToGlobal,
@@ -230,11 +240,11 @@ export default class FetchCommand extends BaseCommand {
230
240
  }
231
241
  // Long references are split into searchable/inspectable section children whenever the page artifact
232
242
  // is freshly written (T-22). Best-effort: never let chunking break the main result.
233
- persistSectionsIfFresh(targetDir, artifact, currentTime, cacheStatus) {
243
+ persistSectionsIfFresh(targetDir, artifact, currentTime, cacheStatus, summaryLevel) {
234
244
  if (cacheStatus !== 'miss' && cacheStatus !== 'refreshed')
235
245
  return;
236
246
  try {
237
- persistSectionArtifacts(targetDir, artifact, currentTime);
247
+ persistSectionArtifacts(targetDir, artifact, currentTime, summaryLevel);
238
248
  }
239
249
  catch {
240
250
  /* section generation is non-essential; ignore failures */
@@ -259,11 +269,12 @@ export default class FetchCommand extends BaseCommand {
259
269
  this.error(`Invalid URL: ${err.message}`, { exit: 2 });
260
270
  }
261
271
  const { cacheKey, located, normalizedUrl, roots } = target;
272
+ const summaryLevel = loadSummaryLevel(this.config.configDir, process.cwd());
262
273
  const tmpDir = dryRun ? mkdtempSync(join(tmpdir(), 'fnr-dry-run-')) : null;
263
274
  const currentTime = new Date();
264
275
  try {
265
- const { cacheStatus, freshnessState, artifact, storageDir, redirectedToGlobal } = await this.resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime, located);
266
- this.persistSectionsIfFresh(tmpDir ?? storageDir, artifact, currentTime, cacheStatus);
276
+ const { cacheStatus, freshnessState, artifact, storageDir, redirectedToGlobal } = await this.resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime, located, summaryLevel);
277
+ this.persistSectionsIfFresh(tmpDir ?? storageDir, artifact, currentTime, cacheStatus, summaryLevel);
267
278
  const content = format === 'compressed' ? artifact.compressed : artifact.detailed;
268
279
  const resultData = this.buildResultData(url, normalizedUrl, cacheKey, storageDir, roots.mode, cacheStatus, freshnessState, format, artifact, content, redirectedToGlobal);
269
280
  if (!this.jsonEnabled()) {
@@ -271,6 +282,13 @@ export default class FetchCommand extends BaseCommand {
271
282
  }
272
283
  return resultData;
273
284
  }
285
+ catch (err) {
286
+ // Stale-serve (exit 5) signals via process.exitCode and never throws, so it bypasses this
287
+ // path; only genuine fetch/extract failures land here and get classified guidance. Use the
288
+ // normalized URL in hints so the copy-paste command is canonical (and never echoes raw,
289
+ // unsanitized user input back to the terminal).
290
+ this.emitFetchError(err, normalizedUrl);
291
+ }
274
292
  finally {
275
293
  if (tmpDir) {
276
294
  rmSync(tmpDir, { recursive: true, force: true });
@@ -278,6 +296,59 @@ export default class FetchCommand extends BaseCommand {
278
296
  }
279
297
  }
280
298
  }
299
+ // Maps a runtime fetch failure to recovery steps keyed off its message. Returns undefined for
300
+ // unrecognized failures, which then surface with their original message and no extra hint. The
301
+ // patterns match the thrown text in fetcher.ts/browser.ts; keep them in sync if those messages
302
+ // change. Resolutions mirror the published troubleshooting guide.
303
+ export function fetchFailureGuidance(message, url) {
304
+ const ref = 'https://taurgis.github.io/bonsai/troubleshooting';
305
+ // Name the pipe step explicitly: bare `import --stdin` blocks waiting for input, so a reader (or
306
+ // an agent) running the hint verbatim would hang. Show the content being piped in from a file.
307
+ const importHint = `Open it in a browser, save the page, then import it: cat page.md | bonsai import ${url} --stdin`;
308
+ // 401/403: an auth wall or anti-scraping WAF. Bonsai has no authenticated-fetch path in v1.
309
+ if (/failed with status 40[13]\b/.test(message)) {
310
+ return {
311
+ suggestions: ['The page requires authentication or blocks automated requests.', importHint],
312
+ ref,
313
+ };
314
+ }
315
+ if (/failed with status 404\b/.test(message)) {
316
+ return { suggestions: ['Check the URL is correct and the page still exists.'] };
317
+ }
318
+ if (/failed with status 5\d\d\b/.test(message)) {
319
+ return {
320
+ suggestions: ['The server returned an error. Retry later or verify the host is healthy.'],
321
+ };
322
+ }
323
+ // Server returned a non-HTML body (JSON, binary, or no Content-Type at all). Scope the opener to
324
+ // the real failure: --rendered does produce HTML, so "only scrapes HTML" would read as misleading.
325
+ if (/Rejected content type|does not look like HTML/.test(message)) {
326
+ return {
327
+ suggestions: [
328
+ 'The server returned a non-HTML response (e.g. JSON or binary).',
329
+ 'If the page is rendered by client-side JavaScript, retry with --rendered.',
330
+ importHint,
331
+ ],
332
+ ref,
333
+ };
334
+ }
335
+ if (/DNS resolution failed/.test(message)) {
336
+ return {
337
+ suggestions: ['Check the hostname is spelled correctly and resolves on a public network.'],
338
+ };
339
+ }
340
+ // A hostname that only resolves to a private/local IP at request time (literal private addresses
341
+ // are rejected earlier with exit 2). Blocked to prevent SSRF; only public hosts can be fetched.
342
+ if (/blocked local or private target/.test(message)) {
343
+ return {
344
+ suggestions: [
345
+ 'The hostname resolves to a private or local address, which is blocked to prevent SSRF. Only public http(s) hosts can be fetched.',
346
+ ],
347
+ ref,
348
+ };
349
+ }
350
+ return undefined;
351
+ }
281
352
  // Copies Phase 2 capability provenance from a capture outcome onto the artifact metadata.
282
353
  function applyCaptureMetadata(artifact, capture) {
283
354
  const meta = artifact.metadata;
@@ -23,6 +23,7 @@ export default class ResearchImport extends BaseCommand<typeof ResearchImport> {
23
23
  storage: import("@oclif/core/interfaces").OptionFlag<"global" | "project" | undefined, import("@oclif/core/interfaces").CustomOptions>;
24
24
  };
25
25
  static stdoutIsPrimaryData: boolean;
26
+ protected stdinIsInteractive(): boolean;
26
27
  private readStdin;
27
28
  private getSourceUrls;
28
29
  private validateSourceMode;
@@ -1,4 +1,4 @@
1
- import { Args, Flags } from '@oclif/core';
1
+ import { Args, Errors, Flags } from '@oclif/core';
2
2
  import { createHash } from 'node:crypto';
3
3
  import * as fs from 'node:fs';
4
4
  import * as path from 'node:path';
@@ -8,8 +8,10 @@ import { deriveCacheKey } from '../lib/research/cache-key.js';
8
8
  import { getArtifactPath } from '../lib/research/storage.js';
9
9
  import { loadStoreRoots } from '../lib/research/store-roots.js';
10
10
  import { writeArtifactSecurely } from '../lib/research/secure-write.js';
11
+ import { loadSummaryLevel } from '../lib/config/index.js';
11
12
  import { getPolicy } from '../lib/research/freshness.js';
12
- import { compressMarkdown } from '../lib/research/compress.js';
13
+ import { buildCompressed } from '../lib/research/compress.js';
14
+ import { applyAutoTags } from '../lib/research/keywords.js';
13
15
  import { estimateTokens } from '../lib/research/token-estimate.js';
14
16
  export default class ResearchImport extends BaseCommand {
15
17
  static id = 'import';
@@ -32,6 +34,10 @@ export default class ResearchImport extends BaseCommand {
32
34
  static args = {
33
35
  url: Args.string({
34
36
  required: false,
37
+ // oclif fills an omitted optional arg from piped stdin unless ignoreStdin is set. Without
38
+ // this, the Markdown piped for `--stdin` is swallowed into `url`, making multi-source import
39
+ // (`import --stdin --source-url ...`) wrongly look like it also got a positional URL.
40
+ ignoreStdin: true,
35
41
  description: 'The single source URL of the page (only for single-source import).',
36
42
  }),
37
43
  };
@@ -76,6 +82,11 @@ export default class ResearchImport extends BaseCommand {
76
82
  })(),
77
83
  };
78
84
  static stdoutIsPrimaryData = true;
85
+ // Isolated so tests can force the interactive branch; `isTTY` is `true` only on a real terminal and
86
+ // falsy (undefined) for pipes, files, and /dev/null. See https://nodejs.org/api/process.html#processstdin
87
+ stdinIsInteractive() {
88
+ return process.stdin.isTTY === true;
89
+ }
79
90
  async readStdin(limitBytes = 1024 * 1024) {
80
91
  return new Promise((resolve, reject) => {
81
92
  let data = '';
@@ -162,16 +173,34 @@ export default class ResearchImport extends BaseCommand {
162
173
  }
163
174
  async readAndValidateInput() {
164
175
  if (this.flags.stdin) {
176
+ // On an interactive terminal with nothing piped, reading stdin blocks forever waiting for EOF —
177
+ // a silent hang with no feedback. Fail fast with a usage error instead; piped/redirected input
178
+ // (the agent and CI path) has isTTY falsy and reads normally.
179
+ if (this.stdinIsInteractive()) {
180
+ this.error('No data piped to --stdin.', {
181
+ exit: 2,
182
+ suggestions: [
183
+ 'Pipe Markdown content, e.g. cat notes.md | bonsai import <url> --stdin',
184
+ 'Or read from a file: bonsai import <url> --file notes.md',
185
+ ],
186
+ });
187
+ }
188
+ // Initialized so the post-catch read below never depends on this.error()'s `never` return type
189
+ // for definite assignment; the empty-content guard rejects '' as a usage error anyway.
190
+ let rawInput = '';
165
191
  try {
166
- const rawInput = await this.readStdin();
167
- if (!rawInput.trim()) {
168
- this.error('Empty stdin content provided.', { exit: 2 });
169
- }
170
- return rawInput;
192
+ rawInput = await this.readStdin();
171
193
  }
172
194
  catch (err) {
195
+ // Genuine stream/size failure reading stdin → I/O failure (exit 1).
173
196
  this.error(err.message, { exit: 1 });
174
197
  }
198
+ // Empty input is a usage error (exit 2), not an I/O failure — keep it out of the
199
+ // catch above so its exit code is not clobbered to 1.
200
+ if (!rawInput.trim()) {
201
+ this.error('Empty stdin content provided.', { exit: 2 });
202
+ }
203
+ return rawInput;
175
204
  }
176
205
  if (this.flags.file) {
177
206
  const filePath = path.resolve(this.flags.file);
@@ -179,6 +208,11 @@ export default class ResearchImport extends BaseCommand {
179
208
  return this.readAndValidateFile(filePath);
180
209
  }
181
210
  catch (err) {
211
+ // readAndValidateFile raises usage errors (missing/not-a-file/empty) via this.error,
212
+ // which throw CLIErrors carrying their own exit code and message — preserve those.
213
+ // Only an unexpected read failure becomes a wrapped I/O failure (exit 1).
214
+ if (err instanceof Errors.CLIError)
215
+ throw err;
182
216
  this.error(`Failed to read file: ${err.message}`, { exit: 1 });
183
217
  }
184
218
  }
@@ -200,7 +234,11 @@ export default class ResearchImport extends BaseCommand {
200
234
  const staleAfterTime = new Date(currentTime.getTime() + freshWindowMs);
201
235
  const inputFormat = this.flags['input-format'];
202
236
  const detailed = rawInput;
203
- const compressed = inputFormat === 'detailed' ? compressMarkdown(rawInput) : rawInput;
237
+ // A caller-supplied compressed input is trusted as-is; only a detailed import is condensed here,
238
+ // and it shares the same buildCompressed policy (structural pass + extractive fallback) as fetch.
239
+ const compressed = inputFormat === 'detailed'
240
+ ? buildCompressed(rawInput, loadSummaryLevel(this.config.configDir, process.cwd()))
241
+ : rawInput;
204
242
  const contentHash = createHash('sha256').update(detailed).digest('hex');
205
243
  const metadata = {
206
244
  schema_version: 1,
@@ -240,13 +278,15 @@ export default class ResearchImport extends BaseCommand {
240
278
  section_anchor: null,
241
279
  section_heading_path: null,
242
280
  };
243
- return {
281
+ // Auto-tag from the content when the caller supplied none, so imported notes stay searchable by
282
+ // keyword without forcing manual tagging. Explicit --tags always win (handled in applyAutoTags).
283
+ return applyAutoTags({
244
284
  metadata,
245
285
  summary: `Synthesized research for ${hasSingle ? singleNormalizedUrl : this.flags.topic}`,
246
286
  compressed,
247
287
  detailed,
248
288
  provenance: `Imported via agent-supplied research at ${currentTime.toISOString()}`,
249
- };
289
+ });
250
290
  }
251
291
  async run() {
252
292
  const hasSingle = Boolean(this.args.url);
@@ -292,7 +332,7 @@ export default class ResearchImport extends BaseCommand {
292
332
  captureMethod: 'agent_supplied',
293
333
  extractionStatus: 'agent_supplied',
294
334
  extractionConfidence: 'high',
295
- qualityNotes: ['agent-supplied research import'],
335
+ qualityNotes: artifact.metadata.quality_notes,
296
336
  fetchedAt: null,
297
337
  validatedAt: artifact.metadata.validated_at,
298
338
  staleAfter: artifact.metadata.stale_after,
@@ -3,6 +3,9 @@ import { BaseCommand } from '../base-command.js';
3
3
  import { scanCacheDirs } from '../lib/research/storage.js';
4
4
  import { loadStoreRoots } from '../lib/research/store-roots.js';
5
5
  import { evaluateFreshness } from '../lib/research/freshness.js';
6
+ import { resultListHeading, truncationNotice } from '../lib/text.js';
7
+ // Listings are ordered newest-first, so the truncation word is "first"; --limit caps at 100.
8
+ const LIST_LABELS = { noun: 'cached research', order: 'first', maxLimit: 100 };
6
9
  export default class ResearchList extends BaseCommand {
7
10
  static id = 'list';
8
11
  static summary = 'List cached research artifacts by metadata filters.';
@@ -82,6 +85,12 @@ export default class ResearchList extends BaseCommand {
82
85
  return scanCacheDirs(readRoots, (artifact, filePath) => {
83
86
  if (artifact.metadata.status !== 'active')
84
87
  return null;
88
+ // Section children are sub-chunks of a page, not artifacts a user "has" — they would flood the
89
+ // listing (one page yields dozens) and aren't in the documented source/research_note contract.
90
+ // They stay discoverable through `search` (which ranks them) and `inspect` (which lists a
91
+ // page's sections). `list` answers "what pages/notes do I have?", so keep it page-level.
92
+ if (artifact.metadata.artifact_type === 'section')
93
+ return null;
85
94
  const freshness = evaluateFreshness(artifact.metadata, currentTime, null);
86
95
  if (!this.matchesFilters(artifact.metadata, freshness))
87
96
  return null;
@@ -101,14 +110,14 @@ export default class ResearchList extends BaseCommand {
101
110
  };
102
111
  });
103
112
  }
104
- logListResults(finalResults) {
113
+ logListResults(finalResults, totalMatched) {
105
114
  if (this.jsonEnabled())
106
115
  return;
107
116
  if (finalResults.length === 0) {
108
117
  this.log('No cached research entries found matching filters.');
109
118
  return;
110
119
  }
111
- this.log(`Found ${finalResults.length} cached research entries:\n`);
120
+ this.log(`${resultListHeading(totalMatched, finalResults.length, LIST_LABELS)}\n`);
112
121
  finalResults.forEach((res, index) => {
113
122
  this.log(`${index + 1}. [${res.topic || 'No Topic'}] Key: ${res.cacheKey}`);
114
123
  this.log(` Type: ${res.artifactType} | Freshness: ${res.freshness}`);
@@ -132,7 +141,13 @@ export default class ResearchList extends BaseCommand {
132
141
  return timeB - timeA;
133
142
  });
134
143
  const finalResults = results.slice(0, this.flags.limit);
135
- this.logListResults(finalResults);
144
+ // Under --json the human heading is suppressed, so surface truncation on stderr instead (warn
145
+ // always emits, even in --json) without touching the stdout envelope. Human mode already shows
146
+ // it in the heading, so only warn under --json.
147
+ const notice = truncationNotice(results.length, finalResults.length, LIST_LABELS);
148
+ if (notice && this.jsonEnabled())
149
+ this.warn(notice);
150
+ this.logListResults(finalResults, results.length);
136
151
  return finalResults;
137
152
  }
138
153
  }
@@ -4,7 +4,8 @@ import { join } from 'node:path';
4
4
  import { BaseCommand } from '../base-command.js';
5
5
  import { scanCacheDir } from '../lib/research/storage.js';
6
6
  import { loadStoreRoots } from '../lib/research/store-roots.js';
7
- import { parseTtlToMs } from '../lib/research/freshness.js';
7
+ import { parseTtlToMs, durationFlagError } from '../lib/research/freshness.js';
8
+ import { pluralize } from '../lib/text.js';
8
9
  export default class ResearchPrune extends BaseCommand {
9
10
  static id = 'prune';
10
11
  static summary = 'Clean up old or inactive research cache entries.';
@@ -48,6 +49,15 @@ export default class ResearchPrune extends BaseCommand {
48
49
  if (!this.flags['dry-run'] && !this.flags.yes) {
49
50
  this.error('Safety check: Please specify --yes to confirm pruning, or --dry-run to list files that would be deleted.', { exit: 2 });
50
51
  }
52
+ // Validate durations up front: scanCacheDir swallows per-file errors, so a malformed
53
+ // --older-than/--inactive would otherwise be silently ignored and report "0 entries".
54
+ for (const msg of [
55
+ durationFlagError('--older-than', this.flags['older-than']),
56
+ durationFlagError('--inactive', this.flags.inactive),
57
+ ]) {
58
+ if (msg)
59
+ this.error(msg, { exit: 2 });
60
+ }
51
61
  }
52
62
  shouldPrune(meta, currentTime) {
53
63
  const fetched = meta.fetched_at ? new Date(meta.fetched_at).getTime() : 0;
@@ -109,24 +119,30 @@ export default class ResearchPrune extends BaseCommand {
109
119
  const filesToPrune = this.findPruneCandidates(roots.readRoots, currentTime);
110
120
  const dryRun = this.flags['dry-run'];
111
121
  const count = filesToPrune.length;
122
+ // Track deletions actually performed so the JSON envelope and human output agree even when an
123
+ // unlink fails (e.g. a permission error). Reporting candidate count as `prunedCount` would
124
+ // overstate success and mislead an agent branching on the result.
125
+ let prunedCount = 0;
112
126
  if (dryRun) {
113
127
  if (!this.jsonEnabled()) {
114
- this.log(`[Dry Run] Found ${count} research cache entries that would be pruned:\n`);
128
+ const noun = pluralize(count, 'entry', 'entries');
129
+ this.log(`[Dry Run] Found ${count} research cache ${noun} that would be pruned:\n`);
115
130
  filesToPrune.forEach((f) => {
116
131
  this.log(`- [${f.topic || 'No Topic'}] Key: ${f.cacheKey} (${f.url || 'Imported note'})`);
117
132
  });
118
133
  }
119
134
  }
120
135
  else {
121
- const prunedCount = this.deletePruneCandidates(filesToPrune);
136
+ prunedCount = this.deletePruneCandidates(filesToPrune);
122
137
  if (!this.jsonEnabled()) {
123
- this.log(`Successfully pruned ${prunedCount} of ${count} research cache entries.`);
138
+ const noun = pluralize(count, 'entry', 'entries');
139
+ this.log(`Successfully pruned ${prunedCount} of ${count} research cache ${noun}.`);
124
140
  }
125
141
  }
126
142
  return {
127
143
  dryRun,
128
- prunedCount: dryRun ? 0 : filesToPrune.length,
129
- candidateCount: filesToPrune.length,
144
+ prunedCount,
145
+ candidateCount: count,
130
146
  files: filesToPrune.map((f) => ({
131
147
  cacheKey: f.cacheKey,
132
148
  path: f.path,