agent-knowledge 2.0.0 → 3.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/CHANGELOG.md +59 -0
  2. package/README.md +35 -32
  3. package/dist/embeddings/local.d.ts.map +1 -1
  4. package/dist/embeddings/local.js +3 -1
  5. package/dist/embeddings/local.js.map +1 -1
  6. package/dist/events.d.ts +6 -0
  7. package/dist/events.d.ts.map +1 -1
  8. package/dist/events.js +10 -2
  9. package/dist/events.js.map +1 -1
  10. package/dist/index.js +12 -0
  11. package/dist/index.js.map +1 -1
  12. package/dist/knowledge/analyze.d.ts.map +1 -1
  13. package/dist/knowledge/analyze.js +18 -11
  14. package/dist/knowledge/analyze.js.map +1 -1
  15. package/dist/knowledge/distill.d.ts.map +1 -1
  16. package/dist/knowledge/distill.js +28 -16
  17. package/dist/knowledge/distill.js.map +1 -1
  18. package/dist/knowledge/git.d.ts +3 -4
  19. package/dist/knowledge/git.d.ts.map +1 -1
  20. package/dist/knowledge/git.js +60 -46
  21. package/dist/knowledge/git.js.map +1 -1
  22. package/dist/knowledge/graph.d.ts +2 -0
  23. package/dist/knowledge/graph.d.ts.map +1 -1
  24. package/dist/knowledge/graph.js +8 -0
  25. package/dist/knowledge/graph.js.map +1 -1
  26. package/dist/knowledge/ingest-freshness.d.ts +22 -0
  27. package/dist/knowledge/ingest-freshness.d.ts.map +1 -0
  28. package/dist/knowledge/ingest-freshness.js +93 -0
  29. package/dist/knowledge/ingest-freshness.js.map +1 -0
  30. package/dist/knowledge/remove.d.ts +11 -0
  31. package/dist/knowledge/remove.d.ts.map +1 -0
  32. package/dist/knowledge/remove.js +24 -0
  33. package/dist/knowledge/remove.js.map +1 -0
  34. package/dist/knowledge/search.d.ts +74 -8
  35. package/dist/knowledge/search.d.ts.map +1 -1
  36. package/dist/knowledge/search.js +321 -40
  37. package/dist/knowledge/search.js.map +1 -1
  38. package/dist/knowledge/similar.d.ts +7 -0
  39. package/dist/knowledge/similar.d.ts.map +1 -0
  40. package/dist/knowledge/similar.js +51 -0
  41. package/dist/knowledge/similar.js.map +1 -0
  42. package/dist/knowledge/store.d.ts +17 -0
  43. package/dist/knowledge/store.d.ts.map +1 -1
  44. package/dist/knowledge/store.js +117 -79
  45. package/dist/knowledge/store.js.map +1 -1
  46. package/dist/rest-server.d.ts +12 -2
  47. package/dist/rest-server.d.ts.map +1 -1
  48. package/dist/rest-server.js +75 -10
  49. package/dist/rest-server.js.map +1 -1
  50. package/dist/search/bm25.d.ts +8 -0
  51. package/dist/search/bm25.d.ts.map +1 -1
  52. package/dist/search/bm25.js +32 -20
  53. package/dist/search/bm25.js.map +1 -1
  54. package/dist/search/excerpt.d.ts +1 -1
  55. package/dist/search/excerpt.d.ts.map +1 -1
  56. package/dist/search/excerpt.js +19 -1
  57. package/dist/search/excerpt.js.map +1 -1
  58. package/dist/search/lexicon.d.ts +37 -0
  59. package/dist/search/lexicon.d.ts.map +1 -0
  60. package/dist/search/lexicon.js +83 -0
  61. package/dist/search/lexicon.js.map +1 -0
  62. package/dist/search/mmr.d.ts +9 -3
  63. package/dist/search/mmr.d.ts.map +1 -1
  64. package/dist/search/mmr.js +21 -7
  65. package/dist/search/mmr.js.map +1 -1
  66. package/dist/search/types.d.ts +2 -0
  67. package/dist/search/types.d.ts.map +1 -1
  68. package/dist/server.d.ts.map +1 -1
  69. package/dist/server.js +0 -5
  70. package/dist/server.js.map +1 -1
  71. package/dist/sessions/human.d.ts +12 -0
  72. package/dist/sessions/human.d.ts.map +1 -0
  73. package/dist/sessions/human.js +52 -0
  74. package/dist/sessions/human.js.map +1 -0
  75. package/dist/sessions/parser.d.ts +39 -0
  76. package/dist/sessions/parser.d.ts.map +1 -1
  77. package/dist/sessions/parser.js +142 -18
  78. package/dist/sessions/parser.js.map +1 -1
  79. package/dist/sessions/search.d.ts +2 -0
  80. package/dist/sessions/search.d.ts.map +1 -1
  81. package/dist/sessions/search.js +147 -67
  82. package/dist/sessions/search.js.map +1 -1
  83. package/dist/sessions/snapshot.d.ts +19 -0
  84. package/dist/sessions/snapshot.d.ts.map +1 -0
  85. package/dist/sessions/snapshot.js +118 -0
  86. package/dist/sessions/snapshot.js.map +1 -0
  87. package/dist/sessions/summary.d.ts +24 -5
  88. package/dist/sessions/summary.d.ts.map +1 -1
  89. package/dist/sessions/summary.js +106 -145
  90. package/dist/sessions/summary.js.map +1 -1
  91. package/dist/slice.d.ts +5 -0
  92. package/dist/slice.d.ts.map +1 -0
  93. package/dist/slice.js +28 -0
  94. package/dist/slice.js.map +1 -0
  95. package/dist/text.d.ts +9 -0
  96. package/dist/text.d.ts.map +1 -0
  97. package/dist/text.js +16 -0
  98. package/dist/text.js.map +1 -0
  99. package/dist/tool-handlers.d.ts.map +1 -1
  100. package/dist/tool-handlers.js +6 -10
  101. package/dist/tool-handlers.js.map +1 -1
  102. package/dist/transport/rest.d.ts.map +1 -1
  103. package/dist/transport/rest.js +152 -35
  104. package/dist/transport/rest.js.map +1 -1
  105. package/dist/vectorstore/store.d.ts +9 -0
  106. package/dist/vectorstore/store.d.ts.map +1 -1
  107. package/dist/vectorstore/store.js +23 -0
  108. package/dist/vectorstore/store.js.map +1 -1
  109. package/dist/wakeup.d.ts +5 -0
  110. package/dist/wakeup.d.ts.map +1 -1
  111. package/dist/wakeup.js +10 -1
  112. package/dist/wakeup.js.map +1 -1
  113. package/docs/ARCHITECTURE.md +6 -9
  114. package/docs/HOOKS.md +138 -237
  115. package/docs/SETUP.md +61 -122
  116. package/docs/USER-MANUAL.md +45 -31
  117. package/package.json +6 -6
  118. package/plugin/skills/knowledge-ingest/scripts/validate.mjs +2 -2
  119. package/scripts/plugins/opencode/agent-knowledge.ts +32 -81
  120. package/scripts/setup.js +78 -145
  121. package/scripts/source-walk.d.mts +12 -0
  122. package/scripts/source-walk.mjs +94 -0
  123. package/scripts/tree-sitter-extract.mjs +10 -76
  124. package/scripts/hooks/first-prompt-inject.mjs +0 -242
  125. package/scripts/hooks/precompact-distill.mjs +0 -105
  126. package/scripts/hooks/precompact-flush.mjs +0 -192
  127. package/scripts/hooks/session-start-ingest.mjs +0 -238
  128. package/scripts/hooks/session-start.js +0 -211
  129. package/scripts/hooks/sessionend-distill.mjs +0 -198
package/scripts/setup.js CHANGED
@@ -10,11 +10,15 @@
10
10
  //
11
11
  // What it does:
12
12
  // - Builds the project if dist/ is missing
13
- // - Registers the MCP server in the agent's config
14
- // - Adds lifecycle hooks for Claude Code (SessionStart banner, PreCompact
15
- // flush + distill, SessionEnd distill)
16
- // - Registers the opencode plugin ("agent-knowledge/opencode") in opencode.json
17
- // - Adds permission for mcp__agent-knowledge__* tools
13
+ // - Claude Code: says how to install the plugin (it runs the MCP server, the
14
+ // knowledge-ingest skill and the mod with the /knowledge pane and the session
15
+ // lifecycle), adds permission for mcp__agent-knowledge__* tools, and removes the
16
+ // command hooks agent-knowledge 2.x installed (the mod replaces them)
17
+ // - OpenCode: registers the MCP server and the opencode plugin
18
+ // ("agent-knowledge/opencode") in opencode.json
19
+ // - Any other client: prints the MCP config to add
20
+ // - Installs the knowledge-ingest skill to ~/.agents/skills (OpenCode, Cursor, Codex CLI,
21
+ // Gemini CLI)
18
22
  //
19
23
  // Usage: node scripts/setup.js [--host=claude|opencode|all|auto]
20
24
  // node scripts/setup.js [--agent claude|generic] # deprecated alias
@@ -86,22 +90,8 @@ if (!existsSync(join(PROJECT_DIR, 'dist', 'index.js'))) {
86
90
 
87
91
  const distPath = join(PROJECT_DIR, 'dist', 'index.js');
88
92
 
89
- console.log('Registering MCP server...');
90
- if (IS_CLAUDE && existsSync(CLAUDE_JSON)) {
91
- const config = JSON.parse(readFileSync(CLAUDE_JSON, 'utf-8'));
92
- if (!config.mcpServers) config.mcpServers = {};
93
-
94
- config.mcpServers['agent-knowledge'] = {
95
- type: 'stdio',
96
- command: 'node',
97
- args: [distPath],
98
- env: {},
99
- };
100
-
101
- writeFileSync(CLAUDE_JSON, JSON.stringify(config, null, 2));
102
- console.log(` Added agent-knowledge MCP server → ${distPath}`);
103
- } else if (!IS_OPENCODE) {
104
- console.log(' Add this to your MCP client config:');
93
+ if (!IS_CLAUDE && !IS_OPENCODE) {
94
+ console.log('Add this to your MCP client config:');
105
95
  console.log(' {');
106
96
  console.log(' "mcpServers": {');
107
97
  console.log(' "agent-knowledge": {');
@@ -175,6 +165,22 @@ if (IS_OPENCODE) {
175
165
  }
176
166
  }
177
167
 
168
+ // ---------------------------------------------------------------------------
169
+ // knowledge-ingest skill for every host reading ~/.agents/skills (OpenCode, Cursor,
170
+ // Codex CLI, Gemini CLI). Claude Code gets it from the plugin.
171
+ // ---------------------------------------------------------------------------
172
+
173
+ const skillSrc = join(PROJECT_DIR, 'plugin', 'skills', 'knowledge-ingest', 'SKILL.md');
174
+ const skillDest = join(HOME, '.agents', 'skills', 'knowledge-ingest', 'SKILL.md');
175
+ if (existsSync(skillSrc)) {
176
+ if (!existsSync(skillDest) || statSync(skillDest).mtimeMs < statSync(skillSrc).mtimeMs) {
177
+ mkdirSync(dirname(skillDest), { recursive: true });
178
+ cpSync(skillSrc, skillDest);
179
+ console.log(`
180
+ Installed the knowledge-ingest skill → ${skillDest}`);
181
+ }
182
+ }
183
+
178
184
  if (!IS_CLAUDE) {
179
185
  console.log(`
180
186
  Setup complete!
@@ -189,141 +195,68 @@ ${
189
195
  process.exit(0);
190
196
  }
191
197
 
192
- console.log('Configuring Claude Code hooks...');
193
- if (!existsSync(SETTINGS_JSON)) {
194
- console.log(' Warning: settings.json not found. Configure hooks manually.');
195
- process.exit(0);
196
- }
198
+ console.log('Configuring Claude Code...');
199
+ if (existsSync(SETTINGS_JSON)) {
200
+ const settings = JSON.parse(readFileSync(SETTINGS_JSON, 'utf-8'));
197
201
 
198
- const settings = JSON.parse(readFileSync(SETTINGS_JSON, 'utf-8'));
199
-
200
- if (!settings.permissions) settings.permissions = {};
201
- if (!settings.permissions.allow) settings.permissions.allow = [];
202
- if (!settings.permissions.allow.includes('mcp__agent-knowledge__*')) {
203
- settings.permissions.allow.push('mcp__agent-knowledge__*');
204
- console.log(' Added mcp__agent-knowledge__* permission');
205
- }
206
-
207
- if (!settings.hooks) settings.hooks = {};
208
-
209
- const hookDir = join(PROJECT_DIR, 'scripts', 'hooks');
210
-
211
- function addUnmatchedHook(eventName, marker, command, timeout = 10) {
212
- if (!settings.hooks[eventName]) settings.hooks[eventName] = [];
213
- const groups = settings.hooks[eventName];
214
- const existing = groups.find(
215
- (g) => g.hooks && g.hooks.some((h) => h.command && h.command.includes(marker)),
216
- );
217
- if (existing) {
218
- console.log(` ${eventName} (${marker}): already configured`);
219
- return;
202
+ if (!settings.permissions) settings.permissions = {};
203
+ if (!settings.permissions.allow) settings.permissions.allow = [];
204
+ if (!settings.permissions.allow.includes('mcp__agent-knowledge__*')) {
205
+ settings.permissions.allow.push('mcp__agent-knowledge__*');
206
+ console.log(' Added mcp__agent-knowledge__* permission');
220
207
  }
221
- if (groups.length > 0 && groups[0].hooks && !groups[0].matcher) {
222
- groups[0].hooks.push({ type: 'command', command, timeout });
223
- } else {
224
- groups.push({ hooks: [{ type: 'command', command, timeout }] });
225
- }
226
- console.log(` ${eventName}: added ${marker} hook`);
227
- }
228
-
229
- // SessionStart: /knowledge banner + L0 identity
230
- addUnmatchedHook(
231
- 'SessionStart',
232
- 'session-start.js',
233
- `node "${join(hookDir, 'session-start.js')}"`,
234
- 5,
235
- );
236
-
237
- // SessionStart: knowledge-ingest freshness check
238
- addUnmatchedHook(
239
- 'SessionStart',
240
- 'session-start-ingest.mjs',
241
- `node "${join(hookDir, 'session-start-ingest.mjs')}"`,
242
- 10,
243
- );
244
-
245
- // UserPromptSubmit: targeted knowledge injection on the session's first real prompt
246
- addUnmatchedHook(
247
- 'UserPromptSubmit',
248
- 'first-prompt-inject.mjs',
249
- `node "${join(hookDir, 'first-prompt-inject.mjs')}"`,
250
- 10,
251
- );
252
-
253
- // PreCompact: rich summary (library) + simple text distill (heuristic)
254
- addUnmatchedHook(
255
- 'PreCompact',
256
- 'precompact-flush.mjs',
257
- `node "${join(hookDir, 'precompact-flush.mjs')}"`,
258
- 10,
259
- );
260
- addUnmatchedHook(
261
- 'PreCompact',
262
- 'precompact-distill.mjs',
263
- `node "${join(hookDir, 'precompact-distill.mjs')}"`,
264
- 10,
265
- );
266
-
267
- // SessionEnd: simple text distill for cross-machine memory
268
- addUnmatchedHook(
269
- 'SessionEnd',
270
- 'sessionend-distill.mjs',
271
- `node "${join(hookDir, 'sessionend-distill.mjs')}"`,
272
- 10,
273
- );
274
-
275
- writeFileSync(SETTINGS_JSON, JSON.stringify(settings, null, 2));
276
- console.log(' Saved settings.json');
277
208
 
278
- // ---------------------------------------------------------------------------
279
- // Install knowledge-ingest skill
280
- // ---------------------------------------------------------------------------
281
-
282
- console.log('Installing knowledge-ingest skill...');
283
-
284
- const skillSrc = join(PROJECT_DIR, 'plugin', 'skills', 'knowledge-ingest', 'SKILL.md');
285
-
286
- const skillDests = [
287
- join(HOME, '.claude', 'skills', 'knowledge-ingest', 'SKILL.md'),
288
- join(HOME, '.agents', 'skills', 'knowledge-ingest', 'SKILL.md'),
289
- ];
290
-
291
- if (existsSync(skillSrc)) {
292
- const srcMtime = statSync(skillSrc).mtimeMs;
293
- for (const dest of skillDests) {
294
- mkdirSync(dirname(dest), { recursive: true });
295
- let shouldCopy = true;
296
- if (existsSync(dest)) {
297
- if (statSync(dest).mtimeMs >= srcMtime) {
298
- shouldCopy = false;
299
- }
300
- }
301
- if (shouldCopy) {
302
- cpSync(skillSrc, dest);
303
- console.log(` Installed → ${dest}`);
304
- } else {
305
- console.log(` Up to date → ${dest}`);
209
+ // The command hooks 2.x installed: the plugin's mod does their work now, and their
210
+ // scripts are gone, so each would fail at every session.
211
+ const OLD_HOOKS = [
212
+ 'session-start.js',
213
+ 'session-start-ingest.mjs',
214
+ 'first-prompt-inject.mjs',
215
+ 'precompact-flush.mjs',
216
+ 'precompact-distill.mjs',
217
+ 'sessionend-distill.mjs',
218
+ ];
219
+ const isOld = (h) =>
220
+ typeof h.command === 'string' &&
221
+ h.command.includes('agent-knowledge') &&
222
+ OLD_HOOKS.some((name) => h.command.includes(name));
223
+ for (const [event, groups] of Object.entries(settings.hooks ?? {})) {
224
+ for (const group of groups) {
225
+ const before = group.hooks?.length ?? 0;
226
+ group.hooks = (group.hooks ?? []).filter((h) => !isOld(h));
227
+ if (group.hooks.length < before)
228
+ console.log(
229
+ ` ${event}: removed ${before - group.hooks.length} agent-knowledge 2.x hook(s)`,
230
+ );
306
231
  }
232
+ settings.hooks[event] = groups.filter((g) => g.hooks.length > 0);
233
+ if (settings.hooks[event].length === 0) delete settings.hooks[event];
307
234
  }
308
- } else {
309
- console.log(' Warning: plugin/skills/knowledge-ingest/SKILL.md not found in repo');
235
+
236
+ writeFileSync(SETTINGS_JSON, JSON.stringify(settings, null, 2));
237
+ console.log(' Saved settings.json');
310
238
  }
311
239
 
240
+ const config = existsSync(CLAUDE_JSON) ? JSON.parse(readFileSync(CLAUDE_JSON, 'utf-8')) : {};
241
+ const handWritten = config.mcpServers?.['agent-knowledge'];
242
+
312
243
  console.log(`
313
244
  Setup complete!
314
245
 
315
- Restart Claude Code to load the new MCP server. Every session will now:
316
- - Name the /knowledge pane and load the L0 identity on start (session-start.js)
317
- - Report ingest freshness on start (session-start-ingest.mjs)
318
- - Inject query-targeted knowledge hits on the first real user prompt (first-prompt-inject.mjs)
319
- - Flush a rich session summary before compaction (precompact-flush.mjs)
320
- - Snapshot recent user prompts before compaction (precompact-distill.mjs)
321
- - Dump a final session summary on exit (sessionend-distill.mjs)
322
-
323
- Browse it: /knowledge in Claude Code, with the plugin installed:
246
+ Install the plugin in Claude Code:
324
247
  claude plugin marketplace add keshrath/agent-knowledge
325
248
  claude plugin install agent-knowledge@agent-knowledge
326
- Knowledge base: ~/agent-knowledge/ (configurable via AGENT_KNOWLEDGE_MEMORY_DIR)
327
249
 
328
- See docs/HOOKS.md for details and docs/SETUP.md for manual configuration.
250
+ It runs the MCP server, adds the knowledge-ingest skill and a mod that
251
+ - names the knowledge base, who you are and the project's ingest state in each conversation
252
+ - adds the matching entries to the first real prompt
253
+ - snapshots the session into the knowledge base when it is compacted and when it ends
254
+ - opens the knowledge base beside the conversation: /knowledge
255
+ ${
256
+ handWritten
257
+ ? `\nRemove the hand-written "agent-knowledge" entry from ${CLAUDE_JSON} once the plugin is in,\nso the tools do not appear twice.\n`
258
+ : ''
259
+ }
260
+ Knowledge base: ~/agent-knowledge/ (configurable via AGENT_KNOWLEDGE_MEMORY_DIR)
261
+ See docs/HOOKS.md for the lifecycle and docs/SETUP.md for other clients.
329
262
  `);
@@ -0,0 +1,12 @@
1
+ export declare const DEFAULT_EXCLUDE: string[];
2
+ export declare const DEFAULT_MAX_FILES: number;
3
+ export declare const EXTENSIONS: Map<string, string>;
4
+ export declare function walkSources(
5
+ dir: string,
6
+ options?: {
7
+ exclude?: string[];
8
+ includeExts?: string[];
9
+ maxFiles?: number;
10
+ maxEntries?: number;
11
+ },
12
+ ): Array<{ fullPath: string; ext: string; mtime: number }>;
@@ -0,0 +1,94 @@
1
+ // =============================================================================
2
+ // source-walk.mjs — the source files a knowledge ingest reads
3
+ //
4
+ // One walk for tree-sitter-extract.mjs (what is ingested) and the server's
5
+ // ingest freshness (src/knowledge/ingest-freshness.ts: what changed since), so
6
+ // both see the same files: the same folders skipped, the same extensions, the
7
+ // same cap.
8
+ // =============================================================================
9
+
10
+ import { readdirSync, statSync } from 'fs';
11
+ import { extname, join } from 'path';
12
+ import { buildExtensionMap } from './tree-sitter-lang.mjs';
13
+
14
+ export const DEFAULT_EXCLUDE = [
15
+ 'node_modules',
16
+ '.git',
17
+ 'dist',
18
+ 'build',
19
+ 'target',
20
+ 'vendor',
21
+ '__pycache__',
22
+ '.venv',
23
+ 'venv',
24
+ '.tox',
25
+ '.mypy_cache',
26
+ '.next',
27
+ '.nuxt',
28
+ 'coverage',
29
+ '.pytest_cache',
30
+ '.cargo',
31
+ 'bin',
32
+ 'obj',
33
+ '.gradle',
34
+ '.idea',
35
+ '.vscode',
36
+ ];
37
+
38
+ export const DEFAULT_MAX_FILES = 2000;
39
+
40
+ /** Extension (`.ts`) to the language that parses it. */
41
+ export const EXTENSIONS = buildExtensionMap();
42
+
43
+ /**
44
+ * The source files under `dir` in walk order, up to `maxFiles`, sorted newest first. The walk
45
+ * stops after `maxEntries` folder entries (files and folders seen), so a folder that is no
46
+ * project (a home folder) is not read through.
47
+ */
48
+ export function walkSources(
49
+ dir,
50
+ {
51
+ exclude = DEFAULT_EXCLUDE,
52
+ includeExts = [],
53
+ maxFiles = DEFAULT_MAX_FILES,
54
+ maxEntries = Infinity,
55
+ } = {},
56
+ ) {
57
+ const files = [];
58
+ let seen = 0;
59
+
60
+ function walk(currentDir) {
61
+ if (files.length >= maxFiles || seen >= maxEntries) return;
62
+ let entries;
63
+ try {
64
+ entries = readdirSync(currentDir, { withFileTypes: true });
65
+ } catch {
66
+ return;
67
+ }
68
+ for (const entry of entries) {
69
+ if (files.length >= maxFiles || ++seen > maxEntries) return;
70
+ const name = entry.name;
71
+ if (exclude.some((ex) => name === ex || name.startsWith(ex + '/'))) continue;
72
+ const fullPath = join(currentDir, name);
73
+ if (entry.isDirectory()) {
74
+ walk(fullPath);
75
+ } else if (entry.isFile()) {
76
+ const ext = extname(name).toLowerCase();
77
+ if (!EXTENSIONS.has(ext)) continue;
78
+ if (includeExts.length > 0 && !includeExts.some((ie) => name.endsWith(ie) || ext === ie))
79
+ continue;
80
+ let mtime;
81
+ try {
82
+ mtime = statSync(fullPath).mtimeMs;
83
+ } catch {
84
+ continue;
85
+ }
86
+ files.push({ fullPath, ext, mtime });
87
+ }
88
+ }
89
+ }
90
+
91
+ walk(dir);
92
+ files.sort((a, b) => b.mtime - a.mtime);
93
+ return files;
94
+ }
@@ -17,45 +17,18 @@
17
17
  // --json Pretty-print JSON output (default: compact)
18
18
  // =============================================================================
19
19
 
20
- import { readFileSync, readdirSync, statSync } from 'fs';
21
- import { join, extname, relative, resolve, dirname } from 'path';
20
+ import { readFileSync } from 'fs';
21
+ import { join, relative, resolve, dirname } from 'path';
22
22
  import { createHash } from 'crypto';
23
23
  import { fileURLToPath } from 'url';
24
24
  import { createRequire } from 'module';
25
- import { LANGUAGE_CONFIG, RATIONALE_PATTERNS, buildExtensionMap } from './tree-sitter-lang.mjs';
25
+ import { LANGUAGE_CONFIG, RATIONALE_PATTERNS } from './tree-sitter-lang.mjs';
26
+ import { DEFAULT_EXCLUDE, DEFAULT_MAX_FILES, EXTENSIONS, walkSources } from './source-walk.mjs';
26
27
 
27
28
  const __filename = fileURLToPath(import.meta.url);
28
29
  const __dirname = dirname(__filename);
29
30
  const require = createRequire(import.meta.url);
30
31
 
31
- // ---- Defaults ----
32
-
33
- const DEFAULT_EXCLUDE = [
34
- 'node_modules',
35
- '.git',
36
- 'dist',
37
- 'build',
38
- 'target',
39
- 'vendor',
40
- '__pycache__',
41
- '.venv',
42
- 'venv',
43
- '.tox',
44
- '.mypy_cache',
45
- '.next',
46
- '.nuxt',
47
- 'coverage',
48
- '.pytest_cache',
49
- '.cargo',
50
- 'bin',
51
- 'obj',
52
- '.gradle',
53
- '.idea',
54
- '.vscode',
55
- ];
56
-
57
- const DEFAULT_MAX_FILES = 2000;
58
-
59
32
  // ---- CLI argument parsing ----
60
33
 
61
34
  function parseArgs(argv) {
@@ -92,47 +65,6 @@ function parseArgs(argv) {
92
65
  return args;
93
66
  }
94
67
 
95
- // ---- File discovery ----
96
-
97
- function walkDir(dir, exclude, extMap, includeExts, maxFiles) {
98
- const files = [];
99
-
100
- function walk(currentDir) {
101
- if (files.length >= maxFiles) return;
102
- let entries;
103
- try {
104
- entries = readdirSync(currentDir, { withFileTypes: true });
105
- } catch {
106
- return;
107
- }
108
- for (const entry of entries) {
109
- if (files.length >= maxFiles) return;
110
- const name = entry.name;
111
- if (exclude.some((ex) => name === ex || name.startsWith(ex + '/'))) continue;
112
- const fullPath = join(currentDir, name);
113
- if (entry.isDirectory()) {
114
- walk(fullPath);
115
- } else if (entry.isFile()) {
116
- const ext = extname(name).toLowerCase();
117
- if (!extMap.has(ext)) continue;
118
- if (includeExts.length > 0 && !includeExts.some((ie) => name.endsWith(ie) || ext === ie))
119
- continue;
120
- let mtime;
121
- try {
122
- mtime = statSync(fullPath).mtimeMs;
123
- } catch {
124
- continue;
125
- }
126
- files.push({ fullPath, ext, mtime });
127
- }
128
- }
129
- }
130
-
131
- walk(dir);
132
- files.sort((a, b) => b.mtime - a.mtime);
133
- return files.slice(0, maxFiles);
134
- }
135
-
136
68
  // ---- Tree-sitter initialization ----
137
69
 
138
70
  let TreeSitter;
@@ -396,15 +328,17 @@ function buildDependencyGraph(files, rootPath) {
396
328
 
397
329
  async function main() {
398
330
  const args = parseArgs(process.argv);
399
- const extMap = buildExtensionMap();
400
-
401
- const discovered = walkDir(args.path, args.exclude, extMap, args.include, args.maxFiles);
331
+ const discovered = walkSources(args.path, {
332
+ exclude: args.exclude,
333
+ includeExts: args.include,
334
+ maxFiles: args.maxFiles,
335
+ });
402
336
 
403
337
  await initTreeSitter();
404
338
 
405
339
  const fileResults = [];
406
340
  for (const file of discovered) {
407
- const langName = extMap.get(file.ext);
341
+ const langName = EXTENSIONS.get(file.ext);
408
342
  let source;
409
343
  try {
410
344
  source = readFileSync(file.fullPath, 'utf-8');