@taurgis/bonsai 1.0.6 → 3.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. package/README.md +7 -13
  2. package/agents/agents/official-docs-researcher.agent.md +15 -35
  3. package/agents/agents/salesforce-docs-researcher.agent.md +17 -44
  4. package/agents/hooks/README.md +2 -9
  5. package/agents/hooks/claude-code/hooks/bonsai-web-fetch.sh +1 -1
  6. package/agents/hooks/codex/hooks/bonsai-user-prompt-context.sh +1 -1
  7. package/agents/hooks/codex/hooks/bonsai-web-fetch.sh +1 -1
  8. package/agents/hooks/cursor/hooks/bonsai-web-fetch.sh +1 -1
  9. package/agents/hooks/github-copilot/bonsai-web-fetch.sh +1 -1
  10. package/agents/instructions/salesforce-research.instructions.md +7 -8
  11. package/agents/instructions/web-research.instructions.md +3 -4
  12. package/agents/skills/web-research/SKILL.md +24 -37
  13. package/bin/cli.mjs +15 -2
  14. package/dist/base-command.d.ts +15 -2
  15. package/dist/base-command.js +32 -4
  16. package/dist/commands/fetch.d.ts +3 -0
  17. package/dist/commands/fetch.js +96 -30
  18. package/dist/commands/import.js +14 -5
  19. package/dist/commands/inspect.d.ts +4 -0
  20. package/dist/commands/inspect.js +48 -24
  21. package/dist/commands/list.d.ts +3 -1
  22. package/dist/commands/list.js +42 -25
  23. package/dist/commands/prune.d.ts +1 -0
  24. package/dist/commands/prune.js +35 -9
  25. package/dist/commands/status.d.ts +2 -0
  26. package/dist/commands/status.js +57 -25
  27. package/dist/commands.d.ts +0 -2
  28. package/dist/commands.js +0 -2
  29. package/dist/hooks/command-not-found/suggest.d.ts +7 -1
  30. package/dist/hooks/command-not-found/suggest.js +82 -57
  31. package/dist/lib/argv.js +51 -7
  32. package/dist/lib/color.d.ts +20 -0
  33. package/dist/lib/color.js +36 -0
  34. package/dist/lib/envelope.js +8 -2
  35. package/dist/lib/help-preflight.d.ts +15 -0
  36. package/dist/lib/help-preflight.js +81 -0
  37. package/dist/lib/limit-flag.d.ts +1 -0
  38. package/dist/lib/limit-flag.js +24 -0
  39. package/dist/lib/research/{search-index.d.ts → artifact-index.d.ts} +5 -5
  40. package/dist/lib/research/{search-index.js → artifact-index.js} +10 -10
  41. package/dist/lib/research/browser.d.ts +1 -8
  42. package/dist/lib/research/browser.js +32 -46
  43. package/dist/lib/research/docs/markdown-source.js +2 -1
  44. package/dist/lib/research/extract.js +2 -1
  45. package/dist/lib/research/fetcher.js +92 -48
  46. package/dist/lib/research/prompt-injection.d.ts +1 -0
  47. package/dist/lib/research/prompt-injection.js +149 -0
  48. package/dist/lib/research/proxy.d.ts +35 -0
  49. package/dist/lib/research/proxy.js +107 -0
  50. package/dist/lib/research/regression.d.ts +18 -0
  51. package/dist/lib/research/regression.js +47 -0
  52. package/dist/lib/research/storage.d.ts +0 -1
  53. package/dist/lib/research/storage.js +15 -13
  54. package/dist/lib/research/url.d.ts +22 -0
  55. package/dist/lib/research/url.js +63 -1
  56. package/dist/lib/text.d.ts +3 -1
  57. package/dist/lib/text.js +9 -1
  58. package/dist/sites/coveo.d.ts +7 -0
  59. package/dist/sites/coveo.js +33 -0
  60. package/dist/sites/index.js +1 -18
  61. package/dist/sites/salesforce/fetch-page.js +1 -1
  62. package/dist/sites/salesforce/index.js +3 -8
  63. package/dist/sites/salesforce-developer/fetch-page.js +3 -1
  64. package/dist/sites/salesforce-developer/index.js +1 -2
  65. package/dist/sites/salesforce-doc-fetch.d.ts +11 -0
  66. package/dist/sites/salesforce-doc-fetch.js +117 -76
  67. package/dist/sites/salesforce-dom-probe.d.ts +16 -0
  68. package/dist/sites/salesforce-dom-probe.js +114 -0
  69. package/dist/sites/types.d.ts +0 -6
  70. package/oclif.manifest.json +31 -106
  71. package/package.json +35 -26
  72. package/dist/commands/index.d.ts +0 -35
  73. package/dist/commands/index.js +0 -306
  74. package/dist/commands/research/config/base.d.ts +0 -23
  75. package/dist/commands/research/config/base.js +0 -46
  76. package/dist/commands/research/config/get.d.ts +0 -20
  77. package/dist/commands/research/config/get.js +0 -45
  78. package/dist/commands/research/config/list.d.ts +0 -17
  79. package/dist/commands/research/config/list.js +0 -42
  80. package/dist/commands/research/config/set.d.ts +0 -31
  81. package/dist/commands/research/config/set.js +0 -94
  82. package/dist/commands/research/config/unset.d.ts +0 -21
  83. package/dist/commands/research/config/unset.js +0 -59
  84. package/dist/commands/research/import.d.ts +0 -38
  85. package/dist/commands/research/import.js +0 -313
  86. package/dist/commands/research/inspect.d.ts +0 -13
  87. package/dist/commands/research/inspect.js +0 -86
  88. package/dist/commands/research/list.d.ts +0 -25
  89. package/dist/commands/research/list.js +0 -143
  90. package/dist/commands/research/prune.d.ts +0 -24
  91. package/dist/commands/research/prune.js +0 -141
  92. package/dist/commands/research/search.d.ts +0 -32
  93. package/dist/commands/research/search.js +0 -334
  94. package/dist/commands/research/status.d.ts +0 -17
  95. package/dist/commands/research/status.js +0 -108
  96. package/dist/commands/research.d.ts +0 -36
  97. package/dist/commands/research.js +0 -304
  98. package/dist/commands/search.d.ts +0 -35
  99. package/dist/commands/search.js +0 -391
  100. package/dist/lib/research/docs/__fixtures__/sphinx-searchindex.d.ts +0 -1
  101. package/dist/lib/research/docs/__fixtures__/sphinx-searchindex.js +0 -2
  102. package/dist/lib/research/docs/discovery-index.d.ts +0 -14
  103. package/dist/lib/research/docs/discovery-index.js +0 -112
  104. package/dist/lib/research/docs/remote-search-runner.d.ts +0 -21
  105. package/dist/lib/research/docs/remote-search-runner.js +0 -114
  106. package/dist/lib/research/docs/remote-search.d.ts +0 -14
  107. package/dist/lib/research/docs/remote-search.js +0 -101
  108. package/dist/lib/research/docs/search-index.d.ts +0 -13
  109. package/dist/lib/research/docs/search-index.js +0 -176
  110. package/dist/sites/algolia.d.ts +0 -9
  111. package/dist/sites/algolia.js +0 -26
  112. package/dist/sites/auto.d.ts +0 -8
  113. package/dist/sites/auto.js +0 -30
  114. package/dist/sites/nextjs/index.d.ts +0 -2
  115. package/dist/sites/nextjs/index.js +0 -52
  116. package/dist/sites/react/index.d.ts +0 -2
  117. package/dist/sites/react/index.js +0 -17
  118. package/dist/sites/salesforce/browser-search.d.ts +0 -7
  119. package/dist/sites/salesforce/browser-search.js +0 -82
  120. package/dist/sites/salesforce/coveo.d.ts +0 -23
  121. package/dist/sites/salesforce/coveo.js +0 -161
  122. package/dist/sites/salesforce/token.d.ts +0 -26
  123. package/dist/sites/salesforce/token.js +0 -154
  124. package/dist/sites/tailwind/index.d.ts +0 -2
  125. package/dist/sites/tailwind/index.js +0 -17
  126. package/dist/sites/vue/index.d.ts +0 -2
  127. package/dist/sites/vue/index.js +0 -17
@@ -128,7 +128,7 @@
128
128
  "pluginAlias": "@taurgis/bonsai",
129
129
  "pluginName": "@taurgis/bonsai",
130
130
  "pluginType": "core",
131
- "strict": true,
131
+ "strict": false,
132
132
  "summary": "An advanced, locally cached web research tool optimized for LLM ingestion.",
133
133
  "enableJsonFlag": true,
134
134
  "stdoutIsPrimaryData": true
@@ -155,6 +155,10 @@
155
155
  {
156
156
  "description": "import research from a local Markdown file",
157
157
  "command": "<%= config.bin %> import https://example.com/docs --file path/to/notes.md"
158
+ },
159
+ {
160
+ "description": "import research from stdin using the standard file placeholder",
161
+ "command": "cat notes.md | <%= config.bin %> import https://example.com/docs --file -"
158
162
  }
159
163
  ],
160
164
  "flags": {
@@ -172,7 +176,7 @@
172
176
  "type": "boolean"
173
177
  },
174
178
  "file": {
175
- "description": "path to a Markdown file containing research notes to import",
179
+ "description": "path to a Markdown file containing research notes to import, or \"-\" for stdin",
176
180
  "name": "file",
177
181
  "hasDynamicHelp": false,
178
182
  "multiple": false,
@@ -292,7 +296,7 @@
292
296
  "pluginAlias": "@taurgis/bonsai",
293
297
  "pluginName": "@taurgis/bonsai",
294
298
  "pluginType": "core",
295
- "strict": true,
299
+ "strict": false,
296
300
  "summary": "Inspect cached research metadata for a URL.",
297
301
  "enableJsonFlag": true,
298
302
  "stdoutIsPrimaryData": true
@@ -313,6 +317,10 @@
313
317
  {
314
318
  "description": "list only fresh entries filtered by tags",
315
319
  "command": "<%= config.bin %> list --freshness fresh --tags node --tags url"
320
+ },
321
+ {
322
+ "description": "list entries matching a specific URL glob",
323
+ "command": "<%= config.bin %> list --url \"https://react.dev/*\""
316
324
  }
317
325
  ],
318
326
  "flags": {
@@ -339,6 +347,13 @@
339
347
  "multiple": true,
340
348
  "type": "option"
341
349
  },
350
+ "url": {
351
+ "description": "filter by source URL glob pattern (case-insensitive, supports * wildcard)",
352
+ "name": "url",
353
+ "hasDynamicHelp": false,
354
+ "multiple": false,
355
+ "type": "option"
356
+ },
342
357
  "freshness": {
343
358
  "description": "filter by freshness state",
344
359
  "name": "freshness",
@@ -409,6 +424,10 @@
409
424
  {
410
425
  "description": "actually prune entries older than 30 days that are source scrapes",
411
426
  "command": "<%= config.bin %> prune --older-than 30d --artifact-type source --yes"
427
+ },
428
+ {
429
+ "description": "prune entries matching a URL glob pattern",
430
+ "command": "<%= config.bin %> prune --url \"https://react.dev/*\" --yes"
412
431
  }
413
432
  ],
414
433
  "flags": {
@@ -433,6 +452,13 @@
433
452
  "multiple": false,
434
453
  "type": "option"
435
454
  },
455
+ "url": {
456
+ "description": "filter pruning to source URL glob pattern (case-insensitive, supports * wildcard)",
457
+ "name": "url",
458
+ "hasDynamicHelp": false,
459
+ "multiple": false,
460
+ "type": "option"
461
+ },
436
462
  "artifact-type": {
437
463
  "description": "filter pruning to specific artifact type",
438
464
  "name": "artifact-type",
@@ -471,107 +497,6 @@
471
497
  "enableJsonFlag": true,
472
498
  "stdoutIsPrimaryData": true
473
499
  },
474
- "search": {
475
- "aliases": [],
476
- "args": {
477
- "query": {
478
- "description": "the search query containing terms to match",
479
- "name": "query",
480
- "required": true
481
- }
482
- },
483
- "description": "Scans the local research cache database and ranks matching entries based on topic, tags, source URLs, summary, and content.",
484
- "examples": [
485
- {
486
- "description": "search for cached notes containing specific keywords",
487
- "command": "<%= config.bin %> <%= command.id %> \"authentication flow\""
488
- },
489
- {
490
- "description": "search with remote docs fallback",
491
- "command": "<%= config.bin %> <%= command.id %> \"router\" --remote https://react.dev"
492
- },
493
- {
494
- "description": "search local cache with tag filtering",
495
- "command": "<%= config.bin %> <%= command.id %> \"setup\" --tags node --tags tutorial"
496
- }
497
- ],
498
- "flags": {
499
- "json": {
500
- "description": "Format output as json.",
501
- "helpGroup": "GLOBAL",
502
- "name": "json",
503
- "allowNo": false,
504
- "type": "boolean"
505
- },
506
- "topic": {
507
- "char": "t",
508
- "description": "filter results by exact topic (case-insensitive)",
509
- "name": "topic",
510
- "hasDynamicHelp": false,
511
- "multiple": false,
512
- "type": "option"
513
- },
514
- "tags": {
515
- "char": "g",
516
- "description": "filter results by tags (must match all tags specified)",
517
- "name": "tags",
518
- "hasDynamicHelp": false,
519
- "multiple": true,
520
- "type": "option"
521
- },
522
- "artifact-type": {
523
- "description": "filter results by artifact type",
524
- "name": "artifact-type",
525
- "hasDynamicHelp": false,
526
- "multiple": false,
527
- "options": [
528
- "source",
529
- "research_note",
530
- "index",
531
- "section"
532
- ],
533
- "type": "option"
534
- },
535
- "limit": {
536
- "description": "maximum number of results to return (default 10, max 50)",
537
- "name": "limit",
538
- "default": 10,
539
- "hasDynamicHelp": false,
540
- "multiple": false,
541
- "type": "option"
542
- },
543
- "include-stale": {
544
- "description": "include stale expired cache entries in the search results",
545
- "name": "include-stale",
546
- "allowNo": false,
547
- "type": "boolean"
548
- },
549
- "domain": {
550
- "description": "search a documentation site directly (requires compatible site module)",
551
- "name": "domain",
552
- "hasDynamicHelp": false,
553
- "multiple": false,
554
- "type": "option"
555
- },
556
- "remote": {
557
- "description": "discover uncached docs via public search or index (falls back to local cache)",
558
- "name": "remote",
559
- "hasDynamicHelp": false,
560
- "multiple": false,
561
- "type": "option"
562
- }
563
- },
564
- "hasDynamicHelp": false,
565
- "hiddenAliases": [],
566
- "id": "search",
567
- "pluginAlias": "@taurgis/bonsai",
568
- "pluginName": "@taurgis/bonsai",
569
- "pluginType": "core",
570
- "strict": true,
571
- "summary": "Search locally cached research artifacts by keywords.",
572
- "enableJsonFlag": true,
573
- "stdoutIsPrimaryData": true
574
- },
575
500
  "status": {
576
501
  "aliases": [],
577
502
  "args": {
@@ -635,7 +560,7 @@
635
560
  "pluginAlias": "@taurgis/bonsai",
636
561
  "pluginName": "@taurgis/bonsai",
637
562
  "pluginType": "core",
638
- "strict": true,
563
+ "strict": false,
639
564
  "summary": "Check the cache status of a URL without fetching or writing.",
640
565
  "enableJsonFlag": true,
641
566
  "stdoutIsPrimaryData": true
@@ -918,5 +843,5 @@
918
843
  "stdoutIsPrimaryData": true
919
844
  }
920
845
  },
921
- "version": "1.0.6"
846
+ "version": "3.0.1"
922
847
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@taurgis/bonsai",
3
- "version": "1.0.6",
3
+ "version": "3.0.1",
4
4
  "description": "Bonsai is a standalone local research cache CLI for AI agents",
5
5
  "repository": {
6
6
  "type": "git",
@@ -19,27 +19,6 @@
19
19
  "oclif.manifest.json",
20
20
  "README.md"
21
21
  ],
22
- "scripts": {
23
- "build": "tsc -b && oclif manifest",
24
- "prepack": "pnpm build",
25
- "format": "prettier --write \"src/**/*.ts\" \"tests/**/*.ts\" \"scripts/**/*.ts\"",
26
- "format:check": "prettier --check \"src/**/*.ts\" \"tests/**/*.ts\" \"scripts/**/*.ts\"",
27
- "lint": "pnpm lint:line-counts && pnpm lint:health",
28
- "lint:line-counts": "node scripts/check-line-counts.ts --all",
29
- "lint:health": "pnpm test:coverage && pnpm exec fallow health --format human",
30
- "prepare": "husky",
31
- "regression:suite": "tsc -b && node testing/regression/run-suite.mjs",
32
- "regression:promote": "tsc -b && node testing/regression/run-suite.mjs --promote",
33
- "regression:check": "tsc -b && node testing/regression/run-suite.mjs --strict",
34
- "audit:cli": "node testing/manual-audit.mjs",
35
- "nexus:sync": "npx forward-nexus update && npx forward-nexus sync",
36
- "test": "vitest",
37
- "test:coverage": "vitest run --coverage",
38
- "type-check": "tsc --noEmit",
39
- "docs:dev": "vitepress dev docs",
40
- "docs:build": "vitepress build docs",
41
- "docs:preview": "vitepress preview docs"
42
- },
43
22
  "lint-staged": {
44
23
  "src/**/*.ts": "prettier --write",
45
24
  "scripts/**/*.ts": "prettier --write",
@@ -51,9 +30,11 @@
51
30
  "@oclif/plugin-autocomplete": "^3.2.52",
52
31
  "linkedom": "^0.18.12",
53
32
  "turndown": "^7.2.0",
33
+ "undici": "^8.7.0",
54
34
  "update-notifier": "^7.3.1"
55
35
  },
56
36
  "devDependencies": {
37
+ "@changesets/cli": "^2.31.0",
57
38
  "@types/jsdom": "^21.1.7",
58
39
  "@types/mozilla-readability": "^0.2.1",
59
40
  "@types/node": "^22.10.0",
@@ -71,13 +52,16 @@
71
52
  "vue": "^3.5.39"
72
53
  },
73
54
  "engines": {
74
- "node": ">=22"
55
+ "node": ">=22.19.0"
56
+ },
57
+ "publishConfig": {
58
+ "access": "public",
59
+ "provenance": true
75
60
  },
76
- "packageManager": "pnpm@10.17.1",
77
61
  "oclif": {
78
62
  "bin": "bonsai",
79
63
  "dirname": "bonsai",
80
- "description": "Bonsai is a standalone local research cache CLI for AI agents.\nResearch and cache a web page by passing its URL directly:\n\n $ bonsai https://nodejs.org/api/url.html\n\nThe URL form is shorthand for fetching and caching the page. The commands listed below search, inspect, list, import, and prune the local research cache.",
64
+ "description": "Bonsai is a standalone local research cache CLI for AI agents.\nResearch and cache a web page by passing its URL directly:\n\n $ bonsai https://nodejs.org/api/url.html\n\nThe URL form is shorthand for fetching and caching the page; run `bonsai help fetch` for its full reference. The commands listed below inspect, list, import, and prune the local research cache.",
81
65
  "plugins": [
82
66
  "@oclif/plugin-autocomplete"
83
67
  ],
@@ -95,5 +79,30 @@
95
79
  "description": "Manage research cache storage configuration (global vs project)"
96
80
  }
97
81
  }
82
+ },
83
+ "scripts": {
84
+ "build": "tsc -b && oclif manifest",
85
+ "changeset": "changeset",
86
+ "changeset:status": "changeset status --verbose",
87
+ "format": "prettier --write \"src/**/*.ts\" \"tests/**/*.ts\" \"scripts/**/*.ts\"",
88
+ "format:check": "prettier --check \"src/**/*.ts\" \"tests/**/*.ts\" \"scripts/**/*.ts\"",
89
+ "lint": "pnpm lint:line-counts && pnpm lint:health",
90
+ "lint:line-counts": "node scripts/check-line-counts.ts --all",
91
+ "lint:health": "pnpm test:coverage && pnpm exec fallow health --format human",
92
+ "postinstall": "node -e \"if (process.env.CI || process.env.FORWARD_NEXUS_SKIP_SYNC) { console.log('Skipping forward-nexus sync.'); process.exit(0); } require('node:child_process').execFileSync('npx', ['forward-nexus', 'sync', '--pull'], { stdio: 'inherit' });\"",
93
+ "publish:snapshot": "changeset version --snapshot snapshot && changeset publish --tag snapshot",
94
+ "release": "changeset publish",
95
+ "regression:suite": "tsc -b && node testing/regression/run-suite.mjs",
96
+ "regression:promote": "tsc -b && node testing/regression/run-suite.mjs --promote",
97
+ "regression:check": "tsc -b && node testing/regression/run-suite.mjs --strict",
98
+ "audit:cli": "node testing/manual-audit.mjs",
99
+ "nexus:sync": "npx forward-nexus update && npx forward-nexus sync",
100
+ "test": "vitest",
101
+ "test:coverage": "vitest run --coverage",
102
+ "type-check": "tsc --noEmit",
103
+ "version-packages": "changeset version",
104
+ "docs:dev": "vitepress dev docs",
105
+ "docs:build": "vitepress build docs",
106
+ "docs:preview": "vitepress preview docs"
98
107
  }
99
- }
108
+ }
@@ -1,35 +0,0 @@
1
- import { BaseCommand } from '../base-command.js';
2
- export default class Bonsai extends BaseCommand<typeof Bonsai> {
3
- static summary: string;
4
- static description: string;
5
- static examples: {
6
- description: string;
7
- command: string;
8
- }[];
9
- static args: {
10
- url: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
11
- };
12
- static flags: {
13
- topic: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
14
- tags: import("@oclif/core/interfaces").OptionFlag<string[] | undefined, import("@oclif/core/interfaces").CustomOptions>;
15
- format: import("@oclif/core/interfaces").OptionFlag<"compressed" | "detailed", import("@oclif/core/interfaces").CustomOptions>;
16
- tier: import("@oclif/core/interfaces").OptionFlag<"stable" | "standard" | "volatile", import("@oclif/core/interfaces").CustomOptions>;
17
- ttl: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
18
- 'max-age': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
19
- force: import("@oclif/core/interfaces").BooleanFlag<boolean>;
20
- 'dry-run': import("@oclif/core/interfaces").BooleanFlag<boolean>;
21
- 'allow-stale': import("@oclif/core/interfaces").BooleanFlag<boolean>;
22
- rendered: import("@oclif/core/interfaces").BooleanFlag<boolean>;
23
- storage: import("@oclif/core/interfaces").OptionFlag<"global" | "project" | undefined, import("@oclif/core/interfaces").CustomOptions>;
24
- };
25
- static stdoutIsPrimaryData: boolean;
26
- init(): Promise<void>;
27
- private executeCacheHit;
28
- private executeCacheMiss;
29
- private buildResultData;
30
- private validateDurationFlags;
31
- private resolveArtifact;
32
- private persistFreshArtifact;
33
- private persistSectionsIfFresh;
34
- execute(): Promise<unknown>;
35
- }
@@ -1,306 +0,0 @@
1
- import { Args, Flags } from '@oclif/core';
2
- import { mkdtempSync, rmSync } from 'node:fs';
3
- import { tmpdir } from 'node:os';
4
- import { join } from 'node:path';
5
- import { BaseCommand } from '../base-command.js';
6
- import { writeArtifact, getArtifactPath } from '../lib/research/storage.js';
7
- import { writeArtifactSecurely } from '../lib/research/secure-write.js';
8
- import { evaluateFreshness, parseTtlToMs, checkMaxAgeExpired } from '../lib/research/freshness.js';
9
- import { revalidateCache, createArtifactFromFetch } from '../lib/research/revalidate.js';
10
- import { fetchStaticHtml, fetchText } from '../lib/research/fetcher.js';
11
- import { fetchRenderedHtml } from '../lib/research/browser.js';
12
- import { capturePage } from '../lib/research/capture.js';
13
- import { persistSectionArtifacts } from '../lib/research/docs/section-artifacts.js';
14
- import { resolveResearchTarget } from '../lib/research/resolve-target.js';
15
- import { detectSite } from '../sites/index.js';
16
- const CAPTURE_DEPS = {
17
- fetchStatic: (url) => fetchStaticHtml(url),
18
- fetchRendered: (url) => fetchRenderedHtml(url),
19
- fetchText: (url) => fetchText(url),
20
- };
21
- export default class Bonsai extends BaseCommand {
22
- static summary = 'An advanced, locally cached web research tool optimized for LLM ingestion.';
23
- static description = 'Scrapes the specified URL, strips HTML boilerplate, converts the semantic payload into clean Markdown format, and caches the result locally using dynamic TTL rules.';
24
- static examples = [
25
- {
26
- description: 'Research a URL with detailed output, tagged with topic and tags',
27
- command: '<%= config.bin %> https://docs.nestjs.com/ --topic "Backend Frameworks" --tags "Node" --tags "NestJS" --format detailed --ttl 30d',
28
- },
29
- {
30
- description: 'Research a volatile page with compressed output and short TTL, returned as JSON',
31
- command: '<%= config.bin %> https://news.ycombinator.com/ --format compressed --ttl 2h --json',
32
- },
33
- ];
34
- static args = {
35
- url: Args.string({
36
- required: true,
37
- description: 'The full HTTP/HTTPS URL of the web page to research.',
38
- }),
39
- };
40
- static flags = {
41
- topic: Flags.string({
42
- char: 't',
43
- description: 'The main category/topic of the research for metadata tagging.',
44
- }),
45
- tags: Flags.string({
46
- char: 'g',
47
- description: 'Taxonomic tags for this research (can be repeated).',
48
- multiple: true,
49
- }),
50
- format: Flags.option({
51
- char: 'f',
52
- description: 'Desired data density.',
53
- options: ['compressed', 'detailed'],
54
- default: 'compressed',
55
- })(),
56
- tier: Flags.option({
57
- description: 'Freshness tier policy.',
58
- options: ['stable', 'standard', 'volatile'],
59
- default: 'standard',
60
- })(),
61
- ttl: Flags.string({
62
- char: 'l',
63
- description: 'Predicted lifespan of the data (e.g., "2h", "7d", "3m").',
64
- }),
65
- 'max-age': Flags.string({
66
- description: 'Maximum age of cache to accept (e.g., "1d", "30d").',
67
- }),
68
- force: Flags.boolean({
69
- description: 'Force a fresh fetch, ignoring any cached entries.',
70
- default: false,
71
- }),
72
- 'dry-run': Flags.boolean({
73
- description: 'Perform validation and fetch without saving to local cache.',
74
- default: false,
75
- }),
76
- 'allow-stale': Flags.boolean({
77
- description: 'Allow serving stale cache if the remote server is unreachable.',
78
- default: false,
79
- }),
80
- rendered: Flags.boolean({
81
- description: 'Force using a browser-rendered scraping path for dynamic pages.',
82
- default: false,
83
- }),
84
- storage: Flags.option({
85
- description: 'Override where this result is cached (overrides configured default). Secret-bearing pages are always stored globally.',
86
- options: ['global', 'project'],
87
- })(),
88
- };
89
- static stdoutIsPrimaryData = true;
90
- async init() {
91
- await super.init();
92
- const { args, flags } = await this.parse(Bonsai);
93
- this.args = args;
94
- this.flags = flags;
95
- }
96
- async executeCacheHit(cached, targetDir, currentTime) {
97
- const { ttl, 'max-age': maxAge, 'allow-stale': allowStale, rendered } = this.flags;
98
- const isExpired = checkMaxAgeExpired(cached, currentTime, maxAge);
99
- const freshnessState = isExpired
100
- ? 'stale_expired'
101
- : evaluateFreshness(cached.metadata, currentTime, ttl);
102
- if (freshnessState === 'fresh') {
103
- return { cacheStatus: 'hit', freshnessState, artifact: cached };
104
- }
105
- const revalResult = await revalidateCache(targetDir, cached, currentTime, {
106
- allowStale: Boolean(allowStale),
107
- ttlOverride: ttl,
108
- rendered: Boolean(rendered),
109
- });
110
- handleStaleRevalidationResult(this, revalResult);
111
- return {
112
- cacheStatus: revalResult.status,
113
- freshnessState,
114
- artifact: revalResult.artifact,
115
- };
116
- }
117
- async executeCacheMiss(normalizedUrl, currentTime, cacheKey) {
118
- const { topic, tags, tier, ttl, rendered } = this.flags;
119
- const siteModule = detectSite(normalizedUrl);
120
- const useRendered = rendered || Boolean(siteModule?.defaults?.rendered);
121
- let fetchResult;
122
- let extraction;
123
- let capture = null;
124
- if (siteModule?.fetchPage) {
125
- // Custom site modules own their fetch/extract strategy; the generic capture path is skipped.
126
- ({ fetchResult, extraction } = await siteModule.fetchPage(normalizedUrl));
127
- }
128
- else {
129
- capture = await capturePage(normalizedUrl, { forceRendered: useRendered }, CAPTURE_DEPS);
130
- fetchResult = capture.fetchResult;
131
- extraction = capture.extraction;
132
- }
133
- const artifact = createArtifactFromFetch(normalizedUrl, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl || null, currentTime);
134
- artifact.metadata.topic = topic || null;
135
- artifact.metadata.tags = tags || [];
136
- artifact.metadata.site_module_id = siteModule?.id ?? null;
137
- if (capture) {
138
- applyCaptureMetadata(artifact, capture);
139
- }
140
- else if (useRendered) {
141
- artifact.metadata.capture_method = 'browser_fallback';
142
- }
143
- return artifact;
144
- }
145
- buildResultData(url, normalizedUrl, cacheKey, storageDir, storageMode, cacheStatus, freshnessState, format, artifact, content, redirectedToGlobal) {
146
- return {
147
- schemaVersion: 1,
148
- command: this.config.bin,
149
- cache: {
150
- key: cacheKey,
151
- status: cacheStatus,
152
- freshness: freshnessState,
153
- path: getArtifactPath(storageDir, cacheKey),
154
- storage: storageMode,
155
- redirectedToGlobal,
156
- },
157
- source: {
158
- url,
159
- normalizedUrl,
160
- captureMethod: artifact.metadata.capture_method,
161
- extractionStatus: artifact.metadata.extraction_status,
162
- extractionConfidence: artifact.metadata.extraction_confidence,
163
- qualityNotes: artifact.metadata.quality_notes,
164
- fetchedAt: artifact.metadata.fetched_at,
165
- validatedAt: artifact.metadata.validated_at,
166
- staleAfter: artifact.metadata.stale_after,
167
- },
168
- artifactType: artifact.metadata.artifact_type,
169
- docsEngine: artifact.metadata.docs_engine,
170
- docsFramework: artifact.metadata.docs_framework,
171
- sourceDocUrl: artifact.metadata.source_doc_url,
172
- searchProvider: artifact.metadata.search_provider,
173
- format,
174
- tokenEstimate: format === 'compressed'
175
- ? artifact.metadata.token_estimate.compressed
176
- : artifact.metadata.token_estimate.detailed,
177
- content,
178
- };
179
- }
180
- // Validates the duration flags up front, exiting with code 2 on a malformed value.
181
- validateDurationFlags(ttl, maxAge) {
182
- for (const [label, value] of [
183
- ['TTL', ttl],
184
- ['max-age', maxAge],
185
- ]) {
186
- if (!value)
187
- continue;
188
- try {
189
- parseTtlToMs(value);
190
- }
191
- catch (err) {
192
- this.error(`Invalid ${label}: ${err.message}`, { exit: 2 });
193
- }
194
- }
195
- }
196
- // Serves the cache when a fresh/revalidatable entry exists, otherwise fetches fresh. Reads fall
197
- // back project→global; the resolved artifact is returned with the dir it lives in / landed in.
198
- async resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime, located) {
199
- if (located) {
200
- // Revalidate where the entry already lives; on dry-run use the throwaway dir so the cache is
201
- // never mutated. ponytail: revalidation rewrites in place, so a refreshed project entry that
202
- // gains a secret is not re-routed — only first-time project writes are scanned.
203
- const revalDir = tmpDir ?? located.dataDir;
204
- const hit = await this.executeCacheHit(located.artifact, revalDir, currentTime);
205
- return { ...hit, storageDir: located.dataDir, redirectedToGlobal: false };
206
- }
207
- const artifact = await this.executeCacheMiss(normalizedUrl, currentTime, cacheKey);
208
- const { dir, redirectedToGlobal } = this.persistFreshArtifact(roots, tmpDir, cacheKey, artifact);
209
- return {
210
- cacheStatus: 'miss',
211
- freshnessState: 'stale_expired',
212
- artifact,
213
- storageDir: dir,
214
- redirectedToGlobal,
215
- };
216
- }
217
- // Writes a freshly fetched artifact, honoring dry-run (throwaway dir) and the secret-safety
218
- // redirect (project→global). Returns the data dir reported to the user and whether a redirect
219
- // occurred (so the JSON envelope mirrors `import`).
220
- persistFreshArtifact(roots, tmpDir, cacheKey, artifact) {
221
- if (tmpDir) {
222
- writeArtifact(tmpDir, cacheKey, artifact);
223
- // Report the real would-be location, not the throwaway dir that is about to be deleted.
224
- return { dir: roots.writeRoot, redirectedToGlobal: false };
225
- }
226
- const result = writeArtifactSecurely(roots, cacheKey, artifact);
227
- if (result.redirected) {
228
- this.warn(`Detected ${result.secretLabel} in the page content; stored in the global cache instead of the project to avoid committing secrets.`);
229
- }
230
- return { dir: result.dataDir, redirectedToGlobal: result.redirected };
231
- }
232
- // Long references are split into searchable/inspectable section children whenever the page artifact
233
- // is freshly written (T-22). Best-effort: never let chunking break the main result.
234
- persistSectionsIfFresh(targetDir, artifact, currentTime, cacheStatus) {
235
- if (cacheStatus !== 'miss' && cacheStatus !== 'refreshed')
236
- return;
237
- try {
238
- persistSectionArtifacts(targetDir, artifact, currentTime);
239
- }
240
- catch {
241
- /* section generation is non-essential; ignore failures */
242
- }
243
- }
244
- async execute() {
245
- const { url } = this.args;
246
- const { format, ttl, 'max-age': maxAge, 'dry-run': dryRun } = this.flags;
247
- this.validateDurationFlags(ttl, maxAge);
248
- let target;
249
- try {
250
- target = resolveResearchTarget({
251
- configDir: this.config.configDir,
252
- cwd: process.cwd(),
253
- dataDir: this.config.dataDir,
254
- flagOverride: this.flags.storage,
255
- lookup: !this.flags.force,
256
- url,
257
- });
258
- }
259
- catch (err) {
260
- this.error(`Invalid URL: ${err.message}`, { exit: 2 });
261
- }
262
- const { cacheKey, located, normalizedUrl, roots } = target;
263
- const tmpDir = dryRun ? mkdtempSync(join(tmpdir(), 'fnr-dry-run-')) : null;
264
- const currentTime = new Date();
265
- try {
266
- const { cacheStatus, freshnessState, artifact, storageDir, redirectedToGlobal } = await this.resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime, located);
267
- this.persistSectionsIfFresh(tmpDir ?? storageDir, artifact, currentTime, cacheStatus);
268
- const content = format === 'compressed' ? artifact.compressed : artifact.detailed;
269
- const resultData = this.buildResultData(url, normalizedUrl, cacheKey, storageDir, roots.mode, cacheStatus, freshnessState, format, artifact, content, redirectedToGlobal);
270
- if (!this.requestedJson()) {
271
- this.log(content);
272
- }
273
- return resultData;
274
- }
275
- finally {
276
- if (tmpDir) {
277
- rmSync(tmpDir, { recursive: true, force: true });
278
- }
279
- }
280
- }
281
- }
282
- // Copies Phase 2 capability provenance from a capture outcome onto the artifact metadata.
283
- function applyCaptureMetadata(artifact, capture) {
284
- const meta = artifact.metadata;
285
- meta.capture_method = capture.captureMethod;
286
- meta.docs_engine = capture.capabilities.docsEngine ?? null;
287
- meta.docs_framework = capture.capabilities.framework ?? null;
288
- meta.source_doc_url = capture.sourceDocUrl;
289
- meta.search_provider = capture.capabilities.search?.provider ?? null;
290
- if (capture.extraction.isIndexHub)
291
- meta.artifact_type = 'index';
292
- if (capture.sourceDocUrl && !meta.source_urls.includes(capture.sourceDocUrl)) {
293
- meta.source_urls.push(capture.sourceDocUrl);
294
- }
295
- }
296
- function handleStaleRevalidationResult(command, revalResult) {
297
- if (revalResult.status !== 'stale')
298
- return;
299
- if (revalResult.allowed) {
300
- command.warn(`Serving stale content: revalidation failed: ${revalResult.error}`);
301
- }
302
- else {
303
- command.warn(`Serving stale content (exit 5): revalidation failed: ${revalResult.error}`);
304
- process.exitCode = 5;
305
- }
306
- }
@@ -1,23 +0,0 @@
1
- import { Command } from '@oclif/core';
2
- import { BaseCommand } from '../../../base-command.js';
3
- import type { ConfigKey, ConfigScope } from '../../../lib/config/index.js';
4
- /**
5
- * Shared base for the four `research config` subcommands: the mutually-exclusive
6
- * scope-flag guard, the binary write-scope helper, and known-key validation.
7
- */
8
- export declare abstract class ConfigCommand<T extends typeof Command> extends BaseCommand<T> {
9
- protected assertScopeFlagsExclusive(global?: boolean, local?: boolean): void;
10
- /** `--local` writes/reads the project file; otherwise the user-level file. */
11
- protected writeScope(local?: boolean): 'user' | 'project';
12
- /** Read scope for get/list: an explicit `--global`/`--local`, else the merged effective view. */
13
- protected readScope(global?: boolean, local?: boolean): ConfigScope;
14
- protected assertKnownKey(keyArg: string): asserts keyArg is ConfigKey;
15
- }
16
- /** Build the `--global`/`--local` scope-flag pair with per-command descriptions. */
17
- export declare function configScopeFlags(descriptions: {
18
- global: string;
19
- local: string;
20
- }): {
21
- global: import("@oclif/core/interfaces").BooleanFlag<boolean>;
22
- local: import("@oclif/core/interfaces").BooleanFlag<boolean>;
23
- };