agent-knowledge 2.0.0 → 3.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +59 -0
- package/README.md +35 -32
- package/dist/embeddings/local.d.ts.map +1 -1
- package/dist/embeddings/local.js +3 -1
- package/dist/embeddings/local.js.map +1 -1
- package/dist/events.d.ts +6 -0
- package/dist/events.d.ts.map +1 -1
- package/dist/events.js +10 -2
- package/dist/events.js.map +1 -1
- package/dist/index.js +12 -0
- package/dist/index.js.map +1 -1
- package/dist/knowledge/analyze.d.ts.map +1 -1
- package/dist/knowledge/analyze.js +18 -11
- package/dist/knowledge/analyze.js.map +1 -1
- package/dist/knowledge/distill.d.ts.map +1 -1
- package/dist/knowledge/distill.js +28 -16
- package/dist/knowledge/distill.js.map +1 -1
- package/dist/knowledge/git.d.ts +3 -4
- package/dist/knowledge/git.d.ts.map +1 -1
- package/dist/knowledge/git.js +60 -46
- package/dist/knowledge/git.js.map +1 -1
- package/dist/knowledge/graph.d.ts +2 -0
- package/dist/knowledge/graph.d.ts.map +1 -1
- package/dist/knowledge/graph.js +8 -0
- package/dist/knowledge/graph.js.map +1 -1
- package/dist/knowledge/ingest-freshness.d.ts +22 -0
- package/dist/knowledge/ingest-freshness.d.ts.map +1 -0
- package/dist/knowledge/ingest-freshness.js +93 -0
- package/dist/knowledge/ingest-freshness.js.map +1 -0
- package/dist/knowledge/remove.d.ts +11 -0
- package/dist/knowledge/remove.d.ts.map +1 -0
- package/dist/knowledge/remove.js +24 -0
- package/dist/knowledge/remove.js.map +1 -0
- package/dist/knowledge/search.d.ts +74 -8
- package/dist/knowledge/search.d.ts.map +1 -1
- package/dist/knowledge/search.js +321 -40
- package/dist/knowledge/search.js.map +1 -1
- package/dist/knowledge/similar.d.ts +7 -0
- package/dist/knowledge/similar.d.ts.map +1 -0
- package/dist/knowledge/similar.js +51 -0
- package/dist/knowledge/similar.js.map +1 -0
- package/dist/knowledge/store.d.ts +17 -0
- package/dist/knowledge/store.d.ts.map +1 -1
- package/dist/knowledge/store.js +117 -79
- package/dist/knowledge/store.js.map +1 -1
- package/dist/rest-server.d.ts +12 -2
- package/dist/rest-server.d.ts.map +1 -1
- package/dist/rest-server.js +75 -10
- package/dist/rest-server.js.map +1 -1
- package/dist/search/bm25.d.ts +8 -0
- package/dist/search/bm25.d.ts.map +1 -1
- package/dist/search/bm25.js +32 -20
- package/dist/search/bm25.js.map +1 -1
- package/dist/search/excerpt.d.ts +1 -1
- package/dist/search/excerpt.d.ts.map +1 -1
- package/dist/search/excerpt.js +19 -1
- package/dist/search/excerpt.js.map +1 -1
- package/dist/search/lexicon.d.ts +37 -0
- package/dist/search/lexicon.d.ts.map +1 -0
- package/dist/search/lexicon.js +83 -0
- package/dist/search/lexicon.js.map +1 -0
- package/dist/search/mmr.d.ts +9 -3
- package/dist/search/mmr.d.ts.map +1 -1
- package/dist/search/mmr.js +21 -7
- package/dist/search/mmr.js.map +1 -1
- package/dist/search/types.d.ts +2 -0
- package/dist/search/types.d.ts.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +0 -5
- package/dist/server.js.map +1 -1
- package/dist/sessions/human.d.ts +12 -0
- package/dist/sessions/human.d.ts.map +1 -0
- package/dist/sessions/human.js +52 -0
- package/dist/sessions/human.js.map +1 -0
- package/dist/sessions/parser.d.ts +39 -0
- package/dist/sessions/parser.d.ts.map +1 -1
- package/dist/sessions/parser.js +142 -18
- package/dist/sessions/parser.js.map +1 -1
- package/dist/sessions/search.d.ts +2 -0
- package/dist/sessions/search.d.ts.map +1 -1
- package/dist/sessions/search.js +147 -67
- package/dist/sessions/search.js.map +1 -1
- package/dist/sessions/snapshot.d.ts +19 -0
- package/dist/sessions/snapshot.d.ts.map +1 -0
- package/dist/sessions/snapshot.js +118 -0
- package/dist/sessions/snapshot.js.map +1 -0
- package/dist/sessions/summary.d.ts +24 -5
- package/dist/sessions/summary.d.ts.map +1 -1
- package/dist/sessions/summary.js +106 -145
- package/dist/sessions/summary.js.map +1 -1
- package/dist/slice.d.ts +5 -0
- package/dist/slice.d.ts.map +1 -0
- package/dist/slice.js +28 -0
- package/dist/slice.js.map +1 -0
- package/dist/text.d.ts +9 -0
- package/dist/text.d.ts.map +1 -0
- package/dist/text.js +16 -0
- package/dist/text.js.map +1 -0
- package/dist/tool-handlers.d.ts.map +1 -1
- package/dist/tool-handlers.js +6 -10
- package/dist/tool-handlers.js.map +1 -1
- package/dist/transport/rest.d.ts.map +1 -1
- package/dist/transport/rest.js +152 -35
- package/dist/transport/rest.js.map +1 -1
- package/dist/vectorstore/store.d.ts +9 -0
- package/dist/vectorstore/store.d.ts.map +1 -1
- package/dist/vectorstore/store.js +23 -0
- package/dist/vectorstore/store.js.map +1 -1
- package/dist/wakeup.d.ts +5 -0
- package/dist/wakeup.d.ts.map +1 -1
- package/dist/wakeup.js +10 -1
- package/dist/wakeup.js.map +1 -1
- package/docs/ARCHITECTURE.md +6 -9
- package/docs/HOOKS.md +138 -237
- package/docs/SETUP.md +61 -122
- package/docs/USER-MANUAL.md +45 -31
- package/package.json +6 -6
- package/plugin/skills/knowledge-ingest/scripts/validate.mjs +2 -2
- package/scripts/plugins/opencode/agent-knowledge.ts +32 -81
- package/scripts/setup.js +78 -145
- package/scripts/source-walk.d.mts +12 -0
- package/scripts/source-walk.mjs +94 -0
- package/scripts/tree-sitter-extract.mjs +10 -76
- package/scripts/hooks/first-prompt-inject.mjs +0 -242
- package/scripts/hooks/precompact-distill.mjs +0 -105
- package/scripts/hooks/precompact-flush.mjs +0 -192
- package/scripts/hooks/session-start-ingest.mjs +0 -238
- package/scripts/hooks/session-start.js +0 -211
- package/scripts/hooks/sessionend-distill.mjs +0 -198
package/scripts/setup.js
CHANGED
|
@@ -10,11 +10,15 @@
|
|
|
10
10
|
//
|
|
11
11
|
// What it does:
|
|
12
12
|
// - Builds the project if dist/ is missing
|
|
13
|
-
// -
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
// -
|
|
17
|
-
// -
|
|
13
|
+
// - Claude Code: says how to install the plugin (it runs the MCP server, the
|
|
14
|
+
// knowledge-ingest skill and the mod with the /knowledge pane and the session
|
|
15
|
+
// lifecycle), adds permission for mcp__agent-knowledge__* tools, and removes the
|
|
16
|
+
// command hooks agent-knowledge 2.x installed (the mod replaces them)
|
|
17
|
+
// - OpenCode: registers the MCP server and the opencode plugin
|
|
18
|
+
// ("agent-knowledge/opencode") in opencode.json
|
|
19
|
+
// - Any other client: prints the MCP config to add
|
|
20
|
+
// - Installs the knowledge-ingest skill to ~/.agents/skills (OpenCode, Cursor, Codex CLI,
|
|
21
|
+
// Gemini CLI)
|
|
18
22
|
//
|
|
19
23
|
// Usage: node scripts/setup.js [--host=claude|opencode|all|auto]
|
|
20
24
|
// node scripts/setup.js [--agent claude|generic] # deprecated alias
|
|
@@ -86,22 +90,8 @@ if (!existsSync(join(PROJECT_DIR, 'dist', 'index.js'))) {
|
|
|
86
90
|
|
|
87
91
|
const distPath = join(PROJECT_DIR, 'dist', 'index.js');
|
|
88
92
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
const config = JSON.parse(readFileSync(CLAUDE_JSON, 'utf-8'));
|
|
92
|
-
if (!config.mcpServers) config.mcpServers = {};
|
|
93
|
-
|
|
94
|
-
config.mcpServers['agent-knowledge'] = {
|
|
95
|
-
type: 'stdio',
|
|
96
|
-
command: 'node',
|
|
97
|
-
args: [distPath],
|
|
98
|
-
env: {},
|
|
99
|
-
};
|
|
100
|
-
|
|
101
|
-
writeFileSync(CLAUDE_JSON, JSON.stringify(config, null, 2));
|
|
102
|
-
console.log(` Added agent-knowledge MCP server → ${distPath}`);
|
|
103
|
-
} else if (!IS_OPENCODE) {
|
|
104
|
-
console.log(' Add this to your MCP client config:');
|
|
93
|
+
if (!IS_CLAUDE && !IS_OPENCODE) {
|
|
94
|
+
console.log('Add this to your MCP client config:');
|
|
105
95
|
console.log(' {');
|
|
106
96
|
console.log(' "mcpServers": {');
|
|
107
97
|
console.log(' "agent-knowledge": {');
|
|
@@ -175,6 +165,22 @@ if (IS_OPENCODE) {
|
|
|
175
165
|
}
|
|
176
166
|
}
|
|
177
167
|
|
|
168
|
+
// ---------------------------------------------------------------------------
|
|
169
|
+
// knowledge-ingest skill for every host reading ~/.agents/skills (OpenCode, Cursor,
|
|
170
|
+
// Codex CLI, Gemini CLI). Claude Code gets it from the plugin.
|
|
171
|
+
// ---------------------------------------------------------------------------
|
|
172
|
+
|
|
173
|
+
const skillSrc = join(PROJECT_DIR, 'plugin', 'skills', 'knowledge-ingest', 'SKILL.md');
|
|
174
|
+
const skillDest = join(HOME, '.agents', 'skills', 'knowledge-ingest', 'SKILL.md');
|
|
175
|
+
if (existsSync(skillSrc)) {
|
|
176
|
+
if (!existsSync(skillDest) || statSync(skillDest).mtimeMs < statSync(skillSrc).mtimeMs) {
|
|
177
|
+
mkdirSync(dirname(skillDest), { recursive: true });
|
|
178
|
+
cpSync(skillSrc, skillDest);
|
|
179
|
+
console.log(`
|
|
180
|
+
Installed the knowledge-ingest skill → ${skillDest}`);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
178
184
|
if (!IS_CLAUDE) {
|
|
179
185
|
console.log(`
|
|
180
186
|
Setup complete!
|
|
@@ -189,141 +195,68 @@ ${
|
|
|
189
195
|
process.exit(0);
|
|
190
196
|
}
|
|
191
197
|
|
|
192
|
-
console.log('Configuring Claude Code
|
|
193
|
-
if (
|
|
194
|
-
|
|
195
|
-
process.exit(0);
|
|
196
|
-
}
|
|
198
|
+
console.log('Configuring Claude Code...');
|
|
199
|
+
if (existsSync(SETTINGS_JSON)) {
|
|
200
|
+
const settings = JSON.parse(readFileSync(SETTINGS_JSON, 'utf-8'));
|
|
197
201
|
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
if (!settings.permissions)
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
settings.permissions.allow.push('mcp__agent-knowledge__*');
|
|
204
|
-
console.log(' Added mcp__agent-knowledge__* permission');
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
if (!settings.hooks) settings.hooks = {};
|
|
208
|
-
|
|
209
|
-
const hookDir = join(PROJECT_DIR, 'scripts', 'hooks');
|
|
210
|
-
|
|
211
|
-
function addUnmatchedHook(eventName, marker, command, timeout = 10) {
|
|
212
|
-
if (!settings.hooks[eventName]) settings.hooks[eventName] = [];
|
|
213
|
-
const groups = settings.hooks[eventName];
|
|
214
|
-
const existing = groups.find(
|
|
215
|
-
(g) => g.hooks && g.hooks.some((h) => h.command && h.command.includes(marker)),
|
|
216
|
-
);
|
|
217
|
-
if (existing) {
|
|
218
|
-
console.log(` ${eventName} (${marker}): already configured`);
|
|
219
|
-
return;
|
|
202
|
+
if (!settings.permissions) settings.permissions = {};
|
|
203
|
+
if (!settings.permissions.allow) settings.permissions.allow = [];
|
|
204
|
+
if (!settings.permissions.allow.includes('mcp__agent-knowledge__*')) {
|
|
205
|
+
settings.permissions.allow.push('mcp__agent-knowledge__*');
|
|
206
|
+
console.log(' Added mcp__agent-knowledge__* permission');
|
|
220
207
|
}
|
|
221
|
-
if (groups.length > 0 && groups[0].hooks && !groups[0].matcher) {
|
|
222
|
-
groups[0].hooks.push({ type: 'command', command, timeout });
|
|
223
|
-
} else {
|
|
224
|
-
groups.push({ hooks: [{ type: 'command', command, timeout }] });
|
|
225
|
-
}
|
|
226
|
-
console.log(` ${eventName}: added ${marker} hook`);
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
// SessionStart: /knowledge banner + L0 identity
|
|
230
|
-
addUnmatchedHook(
|
|
231
|
-
'SessionStart',
|
|
232
|
-
'session-start.js',
|
|
233
|
-
`node "${join(hookDir, 'session-start.js')}"`,
|
|
234
|
-
5,
|
|
235
|
-
);
|
|
236
|
-
|
|
237
|
-
// SessionStart: knowledge-ingest freshness check
|
|
238
|
-
addUnmatchedHook(
|
|
239
|
-
'SessionStart',
|
|
240
|
-
'session-start-ingest.mjs',
|
|
241
|
-
`node "${join(hookDir, 'session-start-ingest.mjs')}"`,
|
|
242
|
-
10,
|
|
243
|
-
);
|
|
244
|
-
|
|
245
|
-
// UserPromptSubmit: targeted knowledge injection on the session's first real prompt
|
|
246
|
-
addUnmatchedHook(
|
|
247
|
-
'UserPromptSubmit',
|
|
248
|
-
'first-prompt-inject.mjs',
|
|
249
|
-
`node "${join(hookDir, 'first-prompt-inject.mjs')}"`,
|
|
250
|
-
10,
|
|
251
|
-
);
|
|
252
|
-
|
|
253
|
-
// PreCompact: rich summary (library) + simple text distill (heuristic)
|
|
254
|
-
addUnmatchedHook(
|
|
255
|
-
'PreCompact',
|
|
256
|
-
'precompact-flush.mjs',
|
|
257
|
-
`node "${join(hookDir, 'precompact-flush.mjs')}"`,
|
|
258
|
-
10,
|
|
259
|
-
);
|
|
260
|
-
addUnmatchedHook(
|
|
261
|
-
'PreCompact',
|
|
262
|
-
'precompact-distill.mjs',
|
|
263
|
-
`node "${join(hookDir, 'precompact-distill.mjs')}"`,
|
|
264
|
-
10,
|
|
265
|
-
);
|
|
266
|
-
|
|
267
|
-
// SessionEnd: simple text distill for cross-machine memory
|
|
268
|
-
addUnmatchedHook(
|
|
269
|
-
'SessionEnd',
|
|
270
|
-
'sessionend-distill.mjs',
|
|
271
|
-
`node "${join(hookDir, 'sessionend-distill.mjs')}"`,
|
|
272
|
-
10,
|
|
273
|
-
);
|
|
274
|
-
|
|
275
|
-
writeFileSync(SETTINGS_JSON, JSON.stringify(settings, null, 2));
|
|
276
|
-
console.log(' Saved settings.json');
|
|
277
208
|
|
|
278
|
-
//
|
|
279
|
-
//
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
const
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
}
|
|
301
|
-
if (shouldCopy) {
|
|
302
|
-
cpSync(skillSrc, dest);
|
|
303
|
-
console.log(` Installed → ${dest}`);
|
|
304
|
-
} else {
|
|
305
|
-
console.log(` Up to date → ${dest}`);
|
|
209
|
+
// The command hooks 2.x installed: the plugin's mod does their work now, and their
|
|
210
|
+
// scripts are gone, so each would fail at every session.
|
|
211
|
+
const OLD_HOOKS = [
|
|
212
|
+
'session-start.js',
|
|
213
|
+
'session-start-ingest.mjs',
|
|
214
|
+
'first-prompt-inject.mjs',
|
|
215
|
+
'precompact-flush.mjs',
|
|
216
|
+
'precompact-distill.mjs',
|
|
217
|
+
'sessionend-distill.mjs',
|
|
218
|
+
];
|
|
219
|
+
const isOld = (h) =>
|
|
220
|
+
typeof h.command === 'string' &&
|
|
221
|
+
h.command.includes('agent-knowledge') &&
|
|
222
|
+
OLD_HOOKS.some((name) => h.command.includes(name));
|
|
223
|
+
for (const [event, groups] of Object.entries(settings.hooks ?? {})) {
|
|
224
|
+
for (const group of groups) {
|
|
225
|
+
const before = group.hooks?.length ?? 0;
|
|
226
|
+
group.hooks = (group.hooks ?? []).filter((h) => !isOld(h));
|
|
227
|
+
if (group.hooks.length < before)
|
|
228
|
+
console.log(
|
|
229
|
+
` ${event}: removed ${before - group.hooks.length} agent-knowledge 2.x hook(s)`,
|
|
230
|
+
);
|
|
306
231
|
}
|
|
232
|
+
settings.hooks[event] = groups.filter((g) => g.hooks.length > 0);
|
|
233
|
+
if (settings.hooks[event].length === 0) delete settings.hooks[event];
|
|
307
234
|
}
|
|
308
|
-
|
|
309
|
-
|
|
235
|
+
|
|
236
|
+
writeFileSync(SETTINGS_JSON, JSON.stringify(settings, null, 2));
|
|
237
|
+
console.log(' Saved settings.json');
|
|
310
238
|
}
|
|
311
239
|
|
|
240
|
+
const config = existsSync(CLAUDE_JSON) ? JSON.parse(readFileSync(CLAUDE_JSON, 'utf-8')) : {};
|
|
241
|
+
const handWritten = config.mcpServers?.['agent-knowledge'];
|
|
242
|
+
|
|
312
243
|
console.log(`
|
|
313
244
|
Setup complete!
|
|
314
245
|
|
|
315
|
-
|
|
316
|
-
- Name the /knowledge pane and load the L0 identity on start (session-start.js)
|
|
317
|
-
- Report ingest freshness on start (session-start-ingest.mjs)
|
|
318
|
-
- Inject query-targeted knowledge hits on the first real user prompt (first-prompt-inject.mjs)
|
|
319
|
-
- Flush a rich session summary before compaction (precompact-flush.mjs)
|
|
320
|
-
- Snapshot recent user prompts before compaction (precompact-distill.mjs)
|
|
321
|
-
- Dump a final session summary on exit (sessionend-distill.mjs)
|
|
322
|
-
|
|
323
|
-
Browse it: /knowledge in Claude Code, with the plugin installed:
|
|
246
|
+
Install the plugin in Claude Code:
|
|
324
247
|
claude plugin marketplace add keshrath/agent-knowledge
|
|
325
248
|
claude plugin install agent-knowledge@agent-knowledge
|
|
326
|
-
Knowledge base: ~/agent-knowledge/ (configurable via AGENT_KNOWLEDGE_MEMORY_DIR)
|
|
327
249
|
|
|
328
|
-
|
|
250
|
+
It runs the MCP server, adds the knowledge-ingest skill and a mod that
|
|
251
|
+
- names the knowledge base, who you are and the project's ingest state in each conversation
|
|
252
|
+
- adds the matching entries to the first real prompt
|
|
253
|
+
- snapshots the session into the knowledge base when it is compacted and when it ends
|
|
254
|
+
- opens the knowledge base beside the conversation: /knowledge
|
|
255
|
+
${
|
|
256
|
+
handWritten
|
|
257
|
+
? `\nRemove the hand-written "agent-knowledge" entry from ${CLAUDE_JSON} once the plugin is in,\nso the tools do not appear twice.\n`
|
|
258
|
+
: ''
|
|
259
|
+
}
|
|
260
|
+
Knowledge base: ~/agent-knowledge/ (configurable via AGENT_KNOWLEDGE_MEMORY_DIR)
|
|
261
|
+
See docs/HOOKS.md for the lifecycle and docs/SETUP.md for other clients.
|
|
329
262
|
`);
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export declare const DEFAULT_EXCLUDE: string[];
|
|
2
|
+
export declare const DEFAULT_MAX_FILES: number;
|
|
3
|
+
export declare const EXTENSIONS: Map<string, string>;
|
|
4
|
+
export declare function walkSources(
|
|
5
|
+
dir: string,
|
|
6
|
+
options?: {
|
|
7
|
+
exclude?: string[];
|
|
8
|
+
includeExts?: string[];
|
|
9
|
+
maxFiles?: number;
|
|
10
|
+
maxEntries?: number;
|
|
11
|
+
},
|
|
12
|
+
): Array<{ fullPath: string; ext: string; mtime: number }>;
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
// =============================================================================
|
|
2
|
+
// source-walk.mjs — the source files a knowledge ingest reads
|
|
3
|
+
//
|
|
4
|
+
// One walk for tree-sitter-extract.mjs (what is ingested) and the server's
|
|
5
|
+
// ingest freshness (src/knowledge/ingest-freshness.ts: what changed since), so
|
|
6
|
+
// both see the same files: the same folders skipped, the same extensions, the
|
|
7
|
+
// same cap.
|
|
8
|
+
// =============================================================================
|
|
9
|
+
|
|
10
|
+
import { readdirSync, statSync } from 'fs';
|
|
11
|
+
import { extname, join } from 'path';
|
|
12
|
+
import { buildExtensionMap } from './tree-sitter-lang.mjs';
|
|
13
|
+
|
|
14
|
+
export const DEFAULT_EXCLUDE = [
|
|
15
|
+
'node_modules',
|
|
16
|
+
'.git',
|
|
17
|
+
'dist',
|
|
18
|
+
'build',
|
|
19
|
+
'target',
|
|
20
|
+
'vendor',
|
|
21
|
+
'__pycache__',
|
|
22
|
+
'.venv',
|
|
23
|
+
'venv',
|
|
24
|
+
'.tox',
|
|
25
|
+
'.mypy_cache',
|
|
26
|
+
'.next',
|
|
27
|
+
'.nuxt',
|
|
28
|
+
'coverage',
|
|
29
|
+
'.pytest_cache',
|
|
30
|
+
'.cargo',
|
|
31
|
+
'bin',
|
|
32
|
+
'obj',
|
|
33
|
+
'.gradle',
|
|
34
|
+
'.idea',
|
|
35
|
+
'.vscode',
|
|
36
|
+
];
|
|
37
|
+
|
|
38
|
+
export const DEFAULT_MAX_FILES = 2000;
|
|
39
|
+
|
|
40
|
+
/** Extension (`.ts`) to the language that parses it. */
|
|
41
|
+
export const EXTENSIONS = buildExtensionMap();
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* The source files under `dir` in walk order, up to `maxFiles`, sorted newest first. The walk
|
|
45
|
+
* stops after `maxEntries` folder entries (files and folders seen), so a folder that is no
|
|
46
|
+
* project (a home folder) is not read through.
|
|
47
|
+
*/
|
|
48
|
+
export function walkSources(
|
|
49
|
+
dir,
|
|
50
|
+
{
|
|
51
|
+
exclude = DEFAULT_EXCLUDE,
|
|
52
|
+
includeExts = [],
|
|
53
|
+
maxFiles = DEFAULT_MAX_FILES,
|
|
54
|
+
maxEntries = Infinity,
|
|
55
|
+
} = {},
|
|
56
|
+
) {
|
|
57
|
+
const files = [];
|
|
58
|
+
let seen = 0;
|
|
59
|
+
|
|
60
|
+
function walk(currentDir) {
|
|
61
|
+
if (files.length >= maxFiles || seen >= maxEntries) return;
|
|
62
|
+
let entries;
|
|
63
|
+
try {
|
|
64
|
+
entries = readdirSync(currentDir, { withFileTypes: true });
|
|
65
|
+
} catch {
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
for (const entry of entries) {
|
|
69
|
+
if (files.length >= maxFiles || ++seen > maxEntries) return;
|
|
70
|
+
const name = entry.name;
|
|
71
|
+
if (exclude.some((ex) => name === ex || name.startsWith(ex + '/'))) continue;
|
|
72
|
+
const fullPath = join(currentDir, name);
|
|
73
|
+
if (entry.isDirectory()) {
|
|
74
|
+
walk(fullPath);
|
|
75
|
+
} else if (entry.isFile()) {
|
|
76
|
+
const ext = extname(name).toLowerCase();
|
|
77
|
+
if (!EXTENSIONS.has(ext)) continue;
|
|
78
|
+
if (includeExts.length > 0 && !includeExts.some((ie) => name.endsWith(ie) || ext === ie))
|
|
79
|
+
continue;
|
|
80
|
+
let mtime;
|
|
81
|
+
try {
|
|
82
|
+
mtime = statSync(fullPath).mtimeMs;
|
|
83
|
+
} catch {
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
files.push({ fullPath, ext, mtime });
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
walk(dir);
|
|
92
|
+
files.sort((a, b) => b.mtime - a.mtime);
|
|
93
|
+
return files;
|
|
94
|
+
}
|
|
@@ -17,45 +17,18 @@
|
|
|
17
17
|
// --json Pretty-print JSON output (default: compact)
|
|
18
18
|
// =============================================================================
|
|
19
19
|
|
|
20
|
-
import { readFileSync
|
|
21
|
-
import { join,
|
|
20
|
+
import { readFileSync } from 'fs';
|
|
21
|
+
import { join, relative, resolve, dirname } from 'path';
|
|
22
22
|
import { createHash } from 'crypto';
|
|
23
23
|
import { fileURLToPath } from 'url';
|
|
24
24
|
import { createRequire } from 'module';
|
|
25
|
-
import { LANGUAGE_CONFIG, RATIONALE_PATTERNS
|
|
25
|
+
import { LANGUAGE_CONFIG, RATIONALE_PATTERNS } from './tree-sitter-lang.mjs';
|
|
26
|
+
import { DEFAULT_EXCLUDE, DEFAULT_MAX_FILES, EXTENSIONS, walkSources } from './source-walk.mjs';
|
|
26
27
|
|
|
27
28
|
const __filename = fileURLToPath(import.meta.url);
|
|
28
29
|
const __dirname = dirname(__filename);
|
|
29
30
|
const require = createRequire(import.meta.url);
|
|
30
31
|
|
|
31
|
-
// ---- Defaults ----
|
|
32
|
-
|
|
33
|
-
const DEFAULT_EXCLUDE = [
|
|
34
|
-
'node_modules',
|
|
35
|
-
'.git',
|
|
36
|
-
'dist',
|
|
37
|
-
'build',
|
|
38
|
-
'target',
|
|
39
|
-
'vendor',
|
|
40
|
-
'__pycache__',
|
|
41
|
-
'.venv',
|
|
42
|
-
'venv',
|
|
43
|
-
'.tox',
|
|
44
|
-
'.mypy_cache',
|
|
45
|
-
'.next',
|
|
46
|
-
'.nuxt',
|
|
47
|
-
'coverage',
|
|
48
|
-
'.pytest_cache',
|
|
49
|
-
'.cargo',
|
|
50
|
-
'bin',
|
|
51
|
-
'obj',
|
|
52
|
-
'.gradle',
|
|
53
|
-
'.idea',
|
|
54
|
-
'.vscode',
|
|
55
|
-
];
|
|
56
|
-
|
|
57
|
-
const DEFAULT_MAX_FILES = 2000;
|
|
58
|
-
|
|
59
32
|
// ---- CLI argument parsing ----
|
|
60
33
|
|
|
61
34
|
function parseArgs(argv) {
|
|
@@ -92,47 +65,6 @@ function parseArgs(argv) {
|
|
|
92
65
|
return args;
|
|
93
66
|
}
|
|
94
67
|
|
|
95
|
-
// ---- File discovery ----
|
|
96
|
-
|
|
97
|
-
function walkDir(dir, exclude, extMap, includeExts, maxFiles) {
|
|
98
|
-
const files = [];
|
|
99
|
-
|
|
100
|
-
function walk(currentDir) {
|
|
101
|
-
if (files.length >= maxFiles) return;
|
|
102
|
-
let entries;
|
|
103
|
-
try {
|
|
104
|
-
entries = readdirSync(currentDir, { withFileTypes: true });
|
|
105
|
-
} catch {
|
|
106
|
-
return;
|
|
107
|
-
}
|
|
108
|
-
for (const entry of entries) {
|
|
109
|
-
if (files.length >= maxFiles) return;
|
|
110
|
-
const name = entry.name;
|
|
111
|
-
if (exclude.some((ex) => name === ex || name.startsWith(ex + '/'))) continue;
|
|
112
|
-
const fullPath = join(currentDir, name);
|
|
113
|
-
if (entry.isDirectory()) {
|
|
114
|
-
walk(fullPath);
|
|
115
|
-
} else if (entry.isFile()) {
|
|
116
|
-
const ext = extname(name).toLowerCase();
|
|
117
|
-
if (!extMap.has(ext)) continue;
|
|
118
|
-
if (includeExts.length > 0 && !includeExts.some((ie) => name.endsWith(ie) || ext === ie))
|
|
119
|
-
continue;
|
|
120
|
-
let mtime;
|
|
121
|
-
try {
|
|
122
|
-
mtime = statSync(fullPath).mtimeMs;
|
|
123
|
-
} catch {
|
|
124
|
-
continue;
|
|
125
|
-
}
|
|
126
|
-
files.push({ fullPath, ext, mtime });
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
walk(dir);
|
|
132
|
-
files.sort((a, b) => b.mtime - a.mtime);
|
|
133
|
-
return files.slice(0, maxFiles);
|
|
134
|
-
}
|
|
135
|
-
|
|
136
68
|
// ---- Tree-sitter initialization ----
|
|
137
69
|
|
|
138
70
|
let TreeSitter;
|
|
@@ -396,15 +328,17 @@ function buildDependencyGraph(files, rootPath) {
|
|
|
396
328
|
|
|
397
329
|
async function main() {
|
|
398
330
|
const args = parseArgs(process.argv);
|
|
399
|
-
const
|
|
400
|
-
|
|
401
|
-
|
|
331
|
+
const discovered = walkSources(args.path, {
|
|
332
|
+
exclude: args.exclude,
|
|
333
|
+
includeExts: args.include,
|
|
334
|
+
maxFiles: args.maxFiles,
|
|
335
|
+
});
|
|
402
336
|
|
|
403
337
|
await initTreeSitter();
|
|
404
338
|
|
|
405
339
|
const fileResults = [];
|
|
406
340
|
for (const file of discovered) {
|
|
407
|
-
const langName =
|
|
341
|
+
const langName = EXTENSIONS.get(file.ext);
|
|
408
342
|
let source;
|
|
409
343
|
try {
|
|
410
344
|
source = readFileSync(file.fullPath, 'utf-8');
|