@taurgis/bonsai 1.0.5 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. package/README.md +59 -12
  2. package/agents/agents/official-docs-researcher.agent.md +15 -26
  3. package/agents/agents/salesforce-docs-researcher.agent.md +17 -35
  4. package/agents/hooks/README.md +2 -9
  5. package/agents/hooks/claude-code/hooks/bonsai-web-fetch.sh +1 -1
  6. package/agents/hooks/codex/hooks/bonsai-user-prompt-context.sh +1 -1
  7. package/agents/hooks/codex/hooks/bonsai-web-fetch.sh +1 -1
  8. package/agents/hooks/cursor/hooks/bonsai-web-fetch.sh +1 -1
  9. package/agents/hooks/github-copilot/bonsai-web-fetch.sh +1 -1
  10. package/agents/instructions/salesforce-research.instructions.md +7 -6
  11. package/agents/instructions/web-research.instructions.md +3 -2
  12. package/agents/skills/web-research/SKILL.md +24 -32
  13. package/bin/cli.mjs +44 -14
  14. package/dist/base-command.d.ts +17 -1
  15. package/dist/base-command.js +70 -9
  16. package/dist/commands/config/base.d.ts +2 -0
  17. package/dist/commands/config/base.js +15 -1
  18. package/dist/commands/config/get.d.ts +1 -1
  19. package/dist/commands/config/get.js +4 -5
  20. package/dist/commands/config/index.d.ts +12 -0
  21. package/dist/commands/config/index.js +34 -0
  22. package/dist/commands/config/list.js +2 -2
  23. package/dist/commands/config/set.js +18 -12
  24. package/dist/commands/config/unset.d.ts +1 -1
  25. package/dist/commands/config/unset.js +9 -6
  26. package/dist/commands/fetch.d.ts +2 -0
  27. package/dist/commands/fetch.js +69 -45
  28. package/dist/commands/import.d.ts +3 -0
  29. package/dist/commands/import.js +91 -53
  30. package/dist/commands/inspect.d.ts +8 -0
  31. package/dist/commands/inspect.js +59 -30
  32. package/dist/commands/list.d.ts +5 -3
  33. package/dist/commands/list.js +60 -35
  34. package/dist/commands/prune.d.ts +2 -1
  35. package/dist/commands/prune.js +62 -19
  36. package/dist/commands/status.d.ts +8 -0
  37. package/dist/commands/status.js +84 -26
  38. package/dist/commands.d.ts +2 -2
  39. package/dist/commands.js +2 -2
  40. package/dist/hooks/command-not-found/suggest.d.ts +7 -1
  41. package/dist/hooks/command-not-found/suggest.js +100 -46
  42. package/dist/lib/argv.d.ts +10 -0
  43. package/dist/lib/argv.js +97 -0
  44. package/dist/lib/color.d.ts +20 -0
  45. package/dist/lib/color.js +36 -0
  46. package/dist/lib/envelope.d.ts +36 -0
  47. package/dist/lib/envelope.js +82 -0
  48. package/dist/lib/help-preflight.d.ts +15 -0
  49. package/dist/lib/help-preflight.js +81 -0
  50. package/dist/lib/json-meta.d.ts +10 -0
  51. package/dist/lib/json-meta.js +98 -0
  52. package/dist/lib/limit-flag.d.ts +1 -0
  53. package/dist/lib/limit-flag.js +24 -0
  54. package/dist/lib/research/{search-index.d.ts → artifact-index.d.ts} +5 -5
  55. package/dist/lib/research/{search-index.js → artifact-index.js} +10 -10
  56. package/dist/lib/research/browser.d.ts +0 -8
  57. package/dist/lib/research/browser.js +20 -43
  58. package/dist/lib/research/regression.d.ts +18 -0
  59. package/dist/lib/research/regression.js +47 -0
  60. package/dist/lib/research/revalidate.js +3 -1
  61. package/dist/lib/research/schema.d.ts +5 -2
  62. package/dist/lib/research/schema.js +12 -1
  63. package/dist/lib/research/storage.d.ts +0 -1
  64. package/dist/lib/research/storage.js +15 -13
  65. package/dist/lib/research/url.d.ts +22 -0
  66. package/dist/lib/research/url.js +63 -1
  67. package/dist/lib/text.d.ts +5 -1
  68. package/dist/lib/text.js +11 -1
  69. package/dist/sites/coveo.d.ts +7 -0
  70. package/dist/sites/coveo.js +33 -0
  71. package/dist/sites/index.js +1 -2
  72. package/dist/sites/salesforce/fetch-page.js +1 -1
  73. package/dist/sites/salesforce/index.js +3 -8
  74. package/dist/sites/salesforce-developer/fetch-page.js +3 -1
  75. package/dist/sites/salesforce-developer/index.js +1 -2
  76. package/dist/sites/salesforce-doc-fetch.d.ts +11 -0
  77. package/dist/sites/salesforce-doc-fetch.js +117 -76
  78. package/dist/sites/salesforce-dom-probe.d.ts +16 -0
  79. package/dist/sites/salesforce-dom-probe.js +114 -0
  80. package/dist/sites/types.d.ts +0 -6
  81. package/oclif.manifest.json +153 -155
  82. package/package.json +38 -25
  83. package/dist/commands/index.d.ts +0 -35
  84. package/dist/commands/index.js +0 -306
  85. package/dist/commands/research/config/base.d.ts +0 -23
  86. package/dist/commands/research/config/base.js +0 -46
  87. package/dist/commands/research/config/get.d.ts +0 -20
  88. package/dist/commands/research/config/get.js +0 -45
  89. package/dist/commands/research/config/list.d.ts +0 -17
  90. package/dist/commands/research/config/list.js +0 -42
  91. package/dist/commands/research/config/set.d.ts +0 -31
  92. package/dist/commands/research/config/set.js +0 -94
  93. package/dist/commands/research/config/unset.d.ts +0 -21
  94. package/dist/commands/research/config/unset.js +0 -59
  95. package/dist/commands/research/import.d.ts +0 -38
  96. package/dist/commands/research/import.js +0 -313
  97. package/dist/commands/research/inspect.d.ts +0 -13
  98. package/dist/commands/research/inspect.js +0 -86
  99. package/dist/commands/research/list.d.ts +0 -25
  100. package/dist/commands/research/list.js +0 -143
  101. package/dist/commands/research/prune.d.ts +0 -24
  102. package/dist/commands/research/prune.js +0 -141
  103. package/dist/commands/research/search.d.ts +0 -32
  104. package/dist/commands/research/search.js +0 -334
  105. package/dist/commands/research/status.d.ts +0 -17
  106. package/dist/commands/research/status.js +0 -108
  107. package/dist/commands/research.d.ts +0 -36
  108. package/dist/commands/research.js +0 -304
  109. package/dist/commands/search.d.ts +0 -31
  110. package/dist/commands/search.js +0 -329
  111. package/dist/lib/research/docs/__fixtures__/sphinx-searchindex.d.ts +0 -1
  112. package/dist/lib/research/docs/__fixtures__/sphinx-searchindex.js +0 -2
  113. package/dist/lib/research/docs/discovery-index.d.ts +0 -14
  114. package/dist/lib/research/docs/discovery-index.js +0 -112
  115. package/dist/lib/research/docs/remote-search-runner.d.ts +0 -21
  116. package/dist/lib/research/docs/remote-search-runner.js +0 -114
  117. package/dist/lib/research/docs/remote-search.d.ts +0 -14
  118. package/dist/lib/research/docs/remote-search.js +0 -101
  119. package/dist/lib/research/docs/search-index.d.ts +0 -13
  120. package/dist/lib/research/docs/search-index.js +0 -176
  121. package/dist/sites/salesforce/browser-search.d.ts +0 -7
  122. package/dist/sites/salesforce/browser-search.js +0 -82
  123. package/dist/sites/salesforce/coveo.d.ts +0 -23
  124. package/dist/sites/salesforce/coveo.js +0 -161
  125. package/dist/sites/salesforce/token.d.ts +0 -26
  126. package/dist/sites/salesforce/token.js +0 -154
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@taurgis/bonsai",
3
- "version": "1.0.5",
3
+ "version": "3.0.0",
4
4
  "description": "Bonsai is a standalone local research cache CLI for AI agents",
5
5
  "repository": {
6
6
  "type": "git",
@@ -19,26 +19,6 @@
19
19
  "oclif.manifest.json",
20
20
  "README.md"
21
21
  ],
22
- "scripts": {
23
- "build": "tsc -b && oclif manifest",
24
- "prepack": "pnpm build",
25
- "format": "prettier --write \"src/**/*.ts\" \"tests/**/*.ts\" \"scripts/**/*.ts\"",
26
- "format:check": "prettier --check \"src/**/*.ts\" \"tests/**/*.ts\" \"scripts/**/*.ts\"",
27
- "lint": "pnpm lint:line-counts && pnpm lint:health",
28
- "lint:line-counts": "node scripts/check-line-counts.ts --all",
29
- "lint:health": "pnpm test:coverage && pnpm exec fallow health --format human",
30
- "prepare": "husky",
31
- "regression:suite": "tsc -b && node testing/regression/run-suite.mjs",
32
- "regression:promote": "tsc -b && node testing/regression/run-suite.mjs --promote",
33
- "regression:check": "tsc -b && node testing/regression/run-suite.mjs --strict",
34
- "nexus:sync": "npx forward-nexus update && npx forward-nexus sync",
35
- "test": "vitest",
36
- "test:coverage": "vitest run --coverage",
37
- "type-check": "tsc --noEmit",
38
- "docs:dev": "vitepress dev docs",
39
- "docs:build": "vitepress build docs",
40
- "docs:preview": "vitepress preview docs"
41
- },
42
22
  "lint-staged": {
43
23
  "src/**/*.ts": "prettier --write",
44
24
  "scripts/**/*.ts": "prettier --write",
@@ -47,10 +27,13 @@
47
27
  "dependencies": {
48
28
  "@mozilla/readability": "^0.5.0",
49
29
  "@oclif/core": "^4.11.4",
30
+ "@oclif/plugin-autocomplete": "^3.2.52",
50
31
  "linkedom": "^0.18.12",
51
- "turndown": "^7.2.0"
32
+ "turndown": "^7.2.0",
33
+ "update-notifier": "^7.3.1"
52
34
  },
53
35
  "devDependencies": {
36
+ "@changesets/cli": "^2.31.0",
54
37
  "@types/jsdom": "^21.1.7",
55
38
  "@types/mozilla-readability": "^0.2.1",
56
39
  "@types/node": "^22.10.0",
@@ -70,11 +53,17 @@
70
53
  "engines": {
71
54
  "node": ">=22"
72
55
  },
73
- "packageManager": "pnpm@10.17.1",
56
+ "publishConfig": {
57
+ "access": "public",
58
+ "provenance": true
59
+ },
74
60
  "oclif": {
75
61
  "bin": "bonsai",
76
62
  "dirname": "bonsai",
77
- "description": "Bonsai is a standalone local research cache CLI for AI agents.\nResearch and cache a web page by passing its URL directly:\n\n $ bonsai https://nodejs.org/api/url.html\n\nThe URL form is shorthand for fetching and caching the page. The commands listed below search, inspect, list, import, and prune the local research cache.",
63
+ "description": "Bonsai is a standalone local research cache CLI for AI agents.\nResearch and cache a web page by passing its URL directly:\n\n $ bonsai https://nodejs.org/api/url.html\n\nThe URL form is shorthand for fetching and caching the page; run `bonsai help fetch` for its full reference. The commands listed below inspect, list, import, and prune the local research cache.",
64
+ "plugins": [
65
+ "@oclif/plugin-autocomplete"
66
+ ],
78
67
  "commands": {
79
68
  "strategy": "explicit",
80
69
  "target": "./dist/commands.js",
@@ -89,5 +78,29 @@
89
78
  "description": "Manage research cache storage configuration (global vs project)"
90
79
  }
91
80
  }
81
+ },
82
+ "scripts": {
83
+ "build": "tsc -b && oclif manifest",
84
+ "changeset": "changeset",
85
+ "changeset:status": "changeset status --verbose",
86
+ "format": "prettier --write \"src/**/*.ts\" \"tests/**/*.ts\" \"scripts/**/*.ts\"",
87
+ "format:check": "prettier --check \"src/**/*.ts\" \"tests/**/*.ts\" \"scripts/**/*.ts\"",
88
+ "lint": "pnpm lint:line-counts && pnpm lint:health",
89
+ "lint:line-counts": "node scripts/check-line-counts.ts --all",
90
+ "lint:health": "pnpm test:coverage && pnpm exec fallow health --format human",
91
+ "publish:snapshot": "changeset version --snapshot snapshot && changeset publish --tag snapshot",
92
+ "release": "changeset publish",
93
+ "regression:suite": "tsc -b && node testing/regression/run-suite.mjs",
94
+ "regression:promote": "tsc -b && node testing/regression/run-suite.mjs --promote",
95
+ "regression:check": "tsc -b && node testing/regression/run-suite.mjs --strict",
96
+ "audit:cli": "node testing/manual-audit.mjs",
97
+ "nexus:sync": "npx forward-nexus update && npx forward-nexus sync",
98
+ "test": "vitest",
99
+ "test:coverage": "vitest run --coverage",
100
+ "type-check": "tsc --noEmit",
101
+ "version-packages": "changeset version",
102
+ "docs:dev": "vitepress dev docs",
103
+ "docs:build": "vitepress build docs",
104
+ "docs:preview": "vitepress preview docs"
92
105
  }
93
- }
106
+ }
@@ -1,35 +0,0 @@
1
- import { BaseCommand } from '../base-command.js';
2
- export default class Bonsai extends BaseCommand<typeof Bonsai> {
3
- static summary: string;
4
- static description: string;
5
- static examples: {
6
- description: string;
7
- command: string;
8
- }[];
9
- static args: {
10
- url: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
11
- };
12
- static flags: {
13
- topic: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
14
- tags: import("@oclif/core/interfaces").OptionFlag<string[] | undefined, import("@oclif/core/interfaces").CustomOptions>;
15
- format: import("@oclif/core/interfaces").OptionFlag<"compressed" | "detailed", import("@oclif/core/interfaces").CustomOptions>;
16
- tier: import("@oclif/core/interfaces").OptionFlag<"stable" | "standard" | "volatile", import("@oclif/core/interfaces").CustomOptions>;
17
- ttl: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
18
- 'max-age': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
19
- force: import("@oclif/core/interfaces").BooleanFlag<boolean>;
20
- 'dry-run': import("@oclif/core/interfaces").BooleanFlag<boolean>;
21
- 'allow-stale': import("@oclif/core/interfaces").BooleanFlag<boolean>;
22
- rendered: import("@oclif/core/interfaces").BooleanFlag<boolean>;
23
- storage: import("@oclif/core/interfaces").OptionFlag<"global" | "project" | undefined, import("@oclif/core/interfaces").CustomOptions>;
24
- };
25
- static stdoutIsPrimaryData: boolean;
26
- init(): Promise<void>;
27
- private executeCacheHit;
28
- private executeCacheMiss;
29
- private buildResultData;
30
- private validateDurationFlags;
31
- private resolveArtifact;
32
- private persistFreshArtifact;
33
- private persistSectionsIfFresh;
34
- execute(): Promise<unknown>;
35
- }
@@ -1,306 +0,0 @@
1
- import { Args, Flags } from '@oclif/core';
2
- import { mkdtempSync, rmSync } from 'node:fs';
3
- import { tmpdir } from 'node:os';
4
- import { join } from 'node:path';
5
- import { BaseCommand } from '../base-command.js';
6
- import { writeArtifact, getArtifactPath } from '../lib/research/storage.js';
7
- import { writeArtifactSecurely } from '../lib/research/secure-write.js';
8
- import { evaluateFreshness, parseTtlToMs, checkMaxAgeExpired } from '../lib/research/freshness.js';
9
- import { revalidateCache, createArtifactFromFetch } from '../lib/research/revalidate.js';
10
- import { fetchStaticHtml, fetchText } from '../lib/research/fetcher.js';
11
- import { fetchRenderedHtml } from '../lib/research/browser.js';
12
- import { capturePage } from '../lib/research/capture.js';
13
- import { persistSectionArtifacts } from '../lib/research/docs/section-artifacts.js';
14
- import { resolveResearchTarget } from '../lib/research/resolve-target.js';
15
- import { detectSite } from '../sites/index.js';
16
- const CAPTURE_DEPS = {
17
- fetchStatic: (url) => fetchStaticHtml(url),
18
- fetchRendered: (url) => fetchRenderedHtml(url),
19
- fetchText: (url) => fetchText(url),
20
- };
21
- export default class Bonsai extends BaseCommand {
22
- static summary = 'An advanced, locally cached web research tool optimized for LLM ingestion.';
23
- static description = 'Scrapes the specified URL, strips HTML boilerplate, converts the semantic payload into clean Markdown format, and caches the result locally using dynamic TTL rules.';
24
- static examples = [
25
- {
26
- description: 'Research a URL with detailed output, tagged with topic and tags',
27
- command: '<%= config.bin %> https://docs.nestjs.com/ --topic "Backend Frameworks" --tags "Node" --tags "NestJS" --format detailed --ttl 30d',
28
- },
29
- {
30
- description: 'Research a volatile page with compressed output and short TTL, returned as JSON',
31
- command: '<%= config.bin %> https://news.ycombinator.com/ --format compressed --ttl 2h --json',
32
- },
33
- ];
34
- static args = {
35
- url: Args.string({
36
- required: true,
37
- description: 'The full HTTP/HTTPS URL of the web page to research.',
38
- }),
39
- };
40
- static flags = {
41
- topic: Flags.string({
42
- char: 't',
43
- description: 'The main category/topic of the research for metadata tagging.',
44
- }),
45
- tags: Flags.string({
46
- char: 'g',
47
- description: 'Taxonomic tags for this research (can be repeated).',
48
- multiple: true,
49
- }),
50
- format: Flags.option({
51
- char: 'f',
52
- description: 'Desired data density.',
53
- options: ['compressed', 'detailed'],
54
- default: 'compressed',
55
- })(),
56
- tier: Flags.option({
57
- description: 'Freshness tier policy.',
58
- options: ['stable', 'standard', 'volatile'],
59
- default: 'standard',
60
- })(),
61
- ttl: Flags.string({
62
- char: 'l',
63
- description: 'Predicted lifespan of the data (e.g., "2h", "7d", "3m").',
64
- }),
65
- 'max-age': Flags.string({
66
- description: 'Maximum age of cache to accept (e.g., "1d", "30d").',
67
- }),
68
- force: Flags.boolean({
69
- description: 'Force a fresh fetch, ignoring any cached entries.',
70
- default: false,
71
- }),
72
- 'dry-run': Flags.boolean({
73
- description: 'Perform validation and fetch without saving to local cache.',
74
- default: false,
75
- }),
76
- 'allow-stale': Flags.boolean({
77
- description: 'Allow serving stale cache if the remote server is unreachable.',
78
- default: false,
79
- }),
80
- rendered: Flags.boolean({
81
- description: 'Force using a browser-rendered scraping path for dynamic pages.',
82
- default: false,
83
- }),
84
- storage: Flags.option({
85
- description: 'Override where this result is cached (overrides configured default). Secret-bearing pages are always stored globally.',
86
- options: ['global', 'project'],
87
- })(),
88
- };
89
- static stdoutIsPrimaryData = true;
90
- async init() {
91
- await super.init();
92
- const { args, flags } = await this.parse(Bonsai);
93
- this.args = args;
94
- this.flags = flags;
95
- }
96
- async executeCacheHit(cached, targetDir, currentTime) {
97
- const { ttl, 'max-age': maxAge, 'allow-stale': allowStale, rendered } = this.flags;
98
- const isExpired = checkMaxAgeExpired(cached, currentTime, maxAge);
99
- const freshnessState = isExpired
100
- ? 'stale_expired'
101
- : evaluateFreshness(cached.metadata, currentTime, ttl);
102
- if (freshnessState === 'fresh') {
103
- return { cacheStatus: 'hit', freshnessState, artifact: cached };
104
- }
105
- const revalResult = await revalidateCache(targetDir, cached, currentTime, {
106
- allowStale: Boolean(allowStale),
107
- ttlOverride: ttl,
108
- rendered: Boolean(rendered),
109
- });
110
- handleStaleRevalidationResult(this, revalResult);
111
- return {
112
- cacheStatus: revalResult.status,
113
- freshnessState,
114
- artifact: revalResult.artifact,
115
- };
116
- }
117
- async executeCacheMiss(normalizedUrl, currentTime, cacheKey) {
118
- const { topic, tags, tier, ttl, rendered } = this.flags;
119
- const siteModule = detectSite(normalizedUrl);
120
- const useRendered = rendered || Boolean(siteModule?.defaults?.rendered);
121
- let fetchResult;
122
- let extraction;
123
- let capture = null;
124
- if (siteModule?.fetchPage) {
125
- // Custom site modules own their fetch/extract strategy; the generic capture path is skipped.
126
- ({ fetchResult, extraction } = await siteModule.fetchPage(normalizedUrl));
127
- }
128
- else {
129
- capture = await capturePage(normalizedUrl, { forceRendered: useRendered }, CAPTURE_DEPS);
130
- fetchResult = capture.fetchResult;
131
- extraction = capture.extraction;
132
- }
133
- const artifact = createArtifactFromFetch(normalizedUrl, normalizedUrl, cacheKey, fetchResult, extraction, tier, ttl || null, currentTime);
134
- artifact.metadata.topic = topic || null;
135
- artifact.metadata.tags = tags || [];
136
- artifact.metadata.site_module_id = siteModule?.id ?? null;
137
- if (capture) {
138
- applyCaptureMetadata(artifact, capture);
139
- }
140
- else if (useRendered) {
141
- artifact.metadata.capture_method = 'browser_fallback';
142
- }
143
- return artifact;
144
- }
145
- buildResultData(url, normalizedUrl, cacheKey, storageDir, storageMode, cacheStatus, freshnessState, format, artifact, content, redirectedToGlobal) {
146
- return {
147
- schemaVersion: 1,
148
- command: this.config.bin,
149
- cache: {
150
- key: cacheKey,
151
- status: cacheStatus,
152
- freshness: freshnessState,
153
- path: getArtifactPath(storageDir, cacheKey),
154
- storage: storageMode,
155
- redirectedToGlobal,
156
- },
157
- source: {
158
- url,
159
- normalizedUrl,
160
- captureMethod: artifact.metadata.capture_method,
161
- extractionStatus: artifact.metadata.extraction_status,
162
- extractionConfidence: artifact.metadata.extraction_confidence,
163
- qualityNotes: artifact.metadata.quality_notes,
164
- fetchedAt: artifact.metadata.fetched_at,
165
- validatedAt: artifact.metadata.validated_at,
166
- staleAfter: artifact.metadata.stale_after,
167
- },
168
- artifactType: artifact.metadata.artifact_type,
169
- docsEngine: artifact.metadata.docs_engine,
170
- docsFramework: artifact.metadata.docs_framework,
171
- sourceDocUrl: artifact.metadata.source_doc_url,
172
- searchProvider: artifact.metadata.search_provider,
173
- format,
174
- tokenEstimate: format === 'compressed'
175
- ? artifact.metadata.token_estimate.compressed
176
- : artifact.metadata.token_estimate.detailed,
177
- content,
178
- };
179
- }
180
- // Validates the duration flags up front, exiting with code 2 on a malformed value.
181
- validateDurationFlags(ttl, maxAge) {
182
- for (const [label, value] of [
183
- ['TTL', ttl],
184
- ['max-age', maxAge],
185
- ]) {
186
- if (!value)
187
- continue;
188
- try {
189
- parseTtlToMs(value);
190
- }
191
- catch (err) {
192
- this.error(`Invalid ${label}: ${err.message}`, { exit: 2 });
193
- }
194
- }
195
- }
196
- // Serves the cache when a fresh/revalidatable entry exists, otherwise fetches fresh. Reads fall
197
- // back project→global; the resolved artifact is returned with the dir it lives in / landed in.
198
- async resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime, located) {
199
- if (located) {
200
- // Revalidate where the entry already lives; on dry-run use the throwaway dir so the cache is
201
- // never mutated. ponytail: revalidation rewrites in place, so a refreshed project entry that
202
- // gains a secret is not re-routed — only first-time project writes are scanned.
203
- const revalDir = tmpDir ?? located.dataDir;
204
- const hit = await this.executeCacheHit(located.artifact, revalDir, currentTime);
205
- return { ...hit, storageDir: located.dataDir, redirectedToGlobal: false };
206
- }
207
- const artifact = await this.executeCacheMiss(normalizedUrl, currentTime, cacheKey);
208
- const { dir, redirectedToGlobal } = this.persistFreshArtifact(roots, tmpDir, cacheKey, artifact);
209
- return {
210
- cacheStatus: 'miss',
211
- freshnessState: 'stale_expired',
212
- artifact,
213
- storageDir: dir,
214
- redirectedToGlobal,
215
- };
216
- }
217
- // Writes a freshly fetched artifact, honoring dry-run (throwaway dir) and the secret-safety
218
- // redirect (project→global). Returns the data dir reported to the user and whether a redirect
219
- // occurred (so the JSON envelope mirrors `import`).
220
- persistFreshArtifact(roots, tmpDir, cacheKey, artifact) {
221
- if (tmpDir) {
222
- writeArtifact(tmpDir, cacheKey, artifact);
223
- // Report the real would-be location, not the throwaway dir that is about to be deleted.
224
- return { dir: roots.writeRoot, redirectedToGlobal: false };
225
- }
226
- const result = writeArtifactSecurely(roots, cacheKey, artifact);
227
- if (result.redirected) {
228
- this.warn(`Detected ${result.secretLabel} in the page content; stored in the global cache instead of the project to avoid committing secrets.`);
229
- }
230
- return { dir: result.dataDir, redirectedToGlobal: result.redirected };
231
- }
232
- // Long references are split into searchable/inspectable section children whenever the page artifact
233
- // is freshly written (T-22). Best-effort: never let chunking break the main result.
234
- persistSectionsIfFresh(targetDir, artifact, currentTime, cacheStatus) {
235
- if (cacheStatus !== 'miss' && cacheStatus !== 'refreshed')
236
- return;
237
- try {
238
- persistSectionArtifacts(targetDir, artifact, currentTime);
239
- }
240
- catch {
241
- /* section generation is non-essential; ignore failures */
242
- }
243
- }
244
- async execute() {
245
- const { url } = this.args;
246
- const { format, ttl, 'max-age': maxAge, 'dry-run': dryRun } = this.flags;
247
- this.validateDurationFlags(ttl, maxAge);
248
- let target;
249
- try {
250
- target = resolveResearchTarget({
251
- configDir: this.config.configDir,
252
- cwd: process.cwd(),
253
- dataDir: this.config.dataDir,
254
- flagOverride: this.flags.storage,
255
- lookup: !this.flags.force,
256
- url,
257
- });
258
- }
259
- catch (err) {
260
- this.error(`Invalid URL: ${err.message}`, { exit: 2 });
261
- }
262
- const { cacheKey, located, normalizedUrl, roots } = target;
263
- const tmpDir = dryRun ? mkdtempSync(join(tmpdir(), 'fnr-dry-run-')) : null;
264
- const currentTime = new Date();
265
- try {
266
- const { cacheStatus, freshnessState, artifact, storageDir, redirectedToGlobal } = await this.resolveArtifact(normalizedUrl, cacheKey, roots, tmpDir, currentTime, located);
267
- this.persistSectionsIfFresh(tmpDir ?? storageDir, artifact, currentTime, cacheStatus);
268
- const content = format === 'compressed' ? artifact.compressed : artifact.detailed;
269
- const resultData = this.buildResultData(url, normalizedUrl, cacheKey, storageDir, roots.mode, cacheStatus, freshnessState, format, artifact, content, redirectedToGlobal);
270
- if (!this.requestedJson()) {
271
- this.log(content);
272
- }
273
- return resultData;
274
- }
275
- finally {
276
- if (tmpDir) {
277
- rmSync(tmpDir, { recursive: true, force: true });
278
- }
279
- }
280
- }
281
- }
282
- // Copies Phase 2 capability provenance from a capture outcome onto the artifact metadata.
283
- function applyCaptureMetadata(artifact, capture) {
284
- const meta = artifact.metadata;
285
- meta.capture_method = capture.captureMethod;
286
- meta.docs_engine = capture.capabilities.docsEngine ?? null;
287
- meta.docs_framework = capture.capabilities.framework ?? null;
288
- meta.source_doc_url = capture.sourceDocUrl;
289
- meta.search_provider = capture.capabilities.search?.provider ?? null;
290
- if (capture.extraction.isIndexHub)
291
- meta.artifact_type = 'index';
292
- if (capture.sourceDocUrl && !meta.source_urls.includes(capture.sourceDocUrl)) {
293
- meta.source_urls.push(capture.sourceDocUrl);
294
- }
295
- }
296
- function handleStaleRevalidationResult(command, revalResult) {
297
- if (revalResult.status !== 'stale')
298
- return;
299
- if (revalResult.allowed) {
300
- command.warn(`Serving stale content: revalidation failed: ${revalResult.error}`);
301
- }
302
- else {
303
- command.warn(`Serving stale content (exit 5): revalidation failed: ${revalResult.error}`);
304
- process.exitCode = 5;
305
- }
306
- }
@@ -1,23 +0,0 @@
1
- import { Command } from '@oclif/core';
2
- import { BaseCommand } from '../../../base-command.js';
3
- import type { ConfigKey, ConfigScope } from '../../../lib/config/index.js';
4
- /**
5
- * Shared base for the four `research config` subcommands: the mutually-exclusive
6
- * scope-flag guard, the binary write-scope helper, and known-key validation.
7
- */
8
- export declare abstract class ConfigCommand<T extends typeof Command> extends BaseCommand<T> {
9
- protected assertScopeFlagsExclusive(global?: boolean, local?: boolean): void;
10
- /** `--local` writes/reads the project file; otherwise the user-level file. */
11
- protected writeScope(local?: boolean): 'user' | 'project';
12
- /** Read scope for get/list: an explicit `--global`/`--local`, else the merged effective view. */
13
- protected readScope(global?: boolean, local?: boolean): ConfigScope;
14
- protected assertKnownKey(keyArg: string): asserts keyArg is ConfigKey;
15
- }
16
- /** Build the `--global`/`--local` scope-flag pair with per-command descriptions. */
17
- export declare function configScopeFlags(descriptions: {
18
- global: string;
19
- local: string;
20
- }): {
21
- global: import("@oclif/core/interfaces").BooleanFlag<boolean>;
22
- local: import("@oclif/core/interfaces").BooleanFlag<boolean>;
23
- };
@@ -1,46 +0,0 @@
1
- import { Flags } from '@oclif/core';
2
- import { BaseCommand } from '../../../base-command.js';
3
- import { isKnownKey, suggestKey, validKeysHint } from '../../../lib/config/index.js';
4
- /**
5
- * Shared base for the four `research config` subcommands: the mutually-exclusive
6
- * scope-flag guard, the binary write-scope helper, and known-key validation.
7
- */
8
- export class ConfigCommand extends BaseCommand {
9
- assertScopeFlagsExclusive(global, local) {
10
- if (global && local) {
11
- this.error('--global and --local are mutually exclusive.', { exit: 2 });
12
- }
13
- }
14
- /** `--local` writes/reads the project file; otherwise the user-level file. */
15
- writeScope(local) {
16
- return local ? 'project' : 'user';
17
- }
18
- /** Read scope for get/list: an explicit `--global`/`--local`, else the merged effective view. */
19
- readScope(global, local) {
20
- if (global)
21
- return 'global';
22
- if (local)
23
- return 'local';
24
- return 'effective';
25
- }
26
- assertKnownKey(keyArg) {
27
- if (!isKnownKey(keyArg)) {
28
- const suggestion = suggestKey(keyArg);
29
- const hint = suggestion ? ` Did you mean: ${suggestion}?` : '';
30
- this.error(`Unknown config key: "${keyArg}".${hint} Valid keys: ${validKeysHint()}.`, {
31
- exit: 2,
32
- });
33
- }
34
- }
35
- }
36
- /** Build the `--global`/`--local` scope-flag pair with per-command descriptions. */
37
- export function configScopeFlags(descriptions) {
38
- return {
39
- global: Flags.boolean({ char: 'g', description: descriptions.global }),
40
- local: Flags.boolean({
41
- description: descriptions.local,
42
- aliases: ['project'],
43
- charAliases: ['p'],
44
- }),
45
- };
46
- }
@@ -1,20 +0,0 @@
1
- import { ConfigCommand } from './base.js';
2
- export default class ResearchConfigGet extends ConfigCommand<typeof ResearchConfigGet> {
3
- static id: string;
4
- static summary: string;
5
- static description: string;
6
- static examples: {
7
- description: string;
8
- command: string;
9
- }[];
10
- static args: {
11
- key: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
12
- };
13
- static flags: {
14
- global: import("@oclif/core/interfaces").BooleanFlag<boolean>;
15
- local: import("@oclif/core/interfaces").BooleanFlag<boolean>;
16
- };
17
- static stdoutIsPrimaryData: boolean;
18
- init(): Promise<void>;
19
- execute(): Promise<unknown>;
20
- }
@@ -1,45 +0,0 @@
1
- import { Args } from '@oclif/core';
2
- import { ConfigCommand, configScopeFlags } from './base.js';
3
- import { KEY_META, BUILT_IN_DEFAULTS, readScopedConfig } from '../../../lib/config/index.js';
4
- export default class ResearchConfigGet extends ConfigCommand {
5
- static id = 'research config get';
6
- static summary = 'Get the effective value of a research configuration key.';
7
- static description = 'Print a config value. Without --global/--local, shows the merged effective value (flag > env > project > user > default).\n\nValid keys: storage.';
8
- static examples = [
9
- { description: 'Get the effective storage mode', command: '<%= config.bin %> research config get storage' },
10
- {
11
- description: 'Read only the project-level value',
12
- command: '<%= config.bin %> research config get storage --local',
13
- },
14
- ];
15
- static args = {
16
- key: Args.string({ required: true, description: 'The configuration key to read.' }),
17
- };
18
- static flags = {
19
- ...configScopeFlags({
20
- global: 'Read from user-level config only.',
21
- local: 'Read from project-level config only.',
22
- }),
23
- };
24
- static stdoutIsPrimaryData = true;
25
- async init() {
26
- await super.init();
27
- const { args, flags } = await this.parse(ResearchConfigGet);
28
- this.args = args;
29
- this.flags = flags;
30
- }
31
- async execute() {
32
- const key = this.args.key;
33
- this.assertKnownKey(key);
34
- this.assertScopeFlagsExclusive(this.flags.global, this.flags.local);
35
- const scope = this.readScope(this.flags.global, this.flags.local);
36
- const value = readScopedConfig(scope, this.config.configDir, process.cwd())[key];
37
- const meta = KEY_META[key];
38
- const displayValue = value !== undefined ? value : BUILT_IN_DEFAULTS[key];
39
- const formatted = meta.format(displayValue);
40
- const suffix = value === undefined ? ' (not configured)' : '';
41
- if (!this.requestedJson())
42
- this.log(formatted + suffix);
43
- return { key, value: displayValue };
44
- }
45
- }
@@ -1,17 +0,0 @@
1
- import { ConfigCommand } from './base.js';
2
- export default class ResearchConfigList extends ConfigCommand<typeof ResearchConfigList> {
3
- static id: string;
4
- static summary: string;
5
- static description: string;
6
- static examples: {
7
- description: string;
8
- command: string;
9
- }[];
10
- static flags: {
11
- global: import("@oclif/core/interfaces").BooleanFlag<boolean>;
12
- local: import("@oclif/core/interfaces").BooleanFlag<boolean>;
13
- };
14
- static stdoutIsPrimaryData: boolean;
15
- init(): Promise<void>;
16
- execute(): Promise<unknown>;
17
- }
@@ -1,42 +0,0 @@
1
- import { ConfigCommand, configScopeFlags } from './base.js';
2
- import { ALL_KEYS, KEY_META, BUILT_IN_DEFAULTS, readScopedConfig } from '../../../lib/config/index.js';
3
- export default class ResearchConfigList extends ConfigCommand {
4
- static id = 'research config list';
5
- static summary = 'List all research configuration keys and their effective values.';
6
- static description = 'Show every configuration key with its current value. Use --global/--local to show a single scope.\n\nValid keys: storage.';
7
- static examples = [
8
- { description: 'Show all effective config values', command: '<%= config.bin %> research config list' },
9
- { description: 'Show only project-level config', command: '<%= config.bin %> research config list --local' },
10
- ];
11
- static flags = {
12
- ...configScopeFlags({
13
- global: 'Show user-level config only.',
14
- local: 'Show project-level config only.',
15
- }),
16
- };
17
- static stdoutIsPrimaryData = true;
18
- async init() {
19
- await super.init();
20
- const { flags } = await this.parse(ResearchConfigList);
21
- this.flags = flags;
22
- }
23
- async execute() {
24
- this.assertScopeFlagsExclusive(this.flags.global, this.flags.local);
25
- const scope = this.readScope(this.flags.global, this.flags.local);
26
- const scoped = readScopedConfig(scope, this.config.configDir, process.cwd());
27
- if (!this.requestedJson()) {
28
- const width = Math.max(...ALL_KEYS.map((k) => k.length)) + 2;
29
- for (const key of ALL_KEYS) {
30
- const raw = scoped[key];
31
- const formatted = raw !== undefined ? KEY_META[key].format(raw) : '(not configured)';
32
- this.log(`${key.padEnd(width)}${formatted}`);
33
- }
34
- }
35
- const values = {};
36
- for (const key of ALL_KEYS) {
37
- const raw = scoped[key];
38
- values[key] = raw !== undefined ? raw : BUILT_IN_DEFAULTS[key];
39
- }
40
- return values;
41
- }
42
- }