agent-knowledge 1.5.2 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,366 @@
1
+ // =============================================================================
2
+ // tree-sitter-lang.mjs — per-language configurations for tree-sitter extraction
3
+ //
4
+ // Each language defines: file extensions, grammar WASM package path,
5
+ // and tree-sitter query strings for extracting symbols, imports, exports,
6
+ // calls, and rationale comments.
7
+ // =============================================================================
8
+
9
+ /**
10
+ * Rationale comment patterns — language-agnostic regex applied to raw source.
11
+ * Matches: // WHY: ..., # NOTE: ..., /* DECISION: ... *\/, etc.
12
+ */
13
+ export const RATIONALE_PATTERNS = [
14
+ /(?:\/\/|#|--|;)\s*(WHY|NOTE|DECISION|HACK|TODO|FIXME|XXX|PERF|SAFETY|WORKAROUND)\s*:?\s*(.+)/i,
15
+ /\/\*\s*(WHY|NOTE|DECISION|HACK|TODO|FIXME|XXX|PERF|SAFETY|WORKAROUND)\s*:?\s*([\s\S]*?)\*\//gi,
16
+ ];
17
+
18
+ /**
19
+ * Comment prefix per language family — used for rationale detection.
20
+ */
21
+ const COMMENT_STYLES = {
22
+ clike: ['//', '/*'],
23
+ hash: ['#'],
24
+ lua: ['--'],
25
+ };
26
+
27
+ /**
28
+ * Language configurations.
29
+ *
30
+ * `grammarPackage` is the export path from the `tree-sitter-wasms` npm package.
31
+ * The extract script resolves it via import.meta.resolve() or createRequire().
32
+ *
33
+ * Tree-sitter queries use S-expression syntax. Each query captures specific
34
+ * node types the extractor needs.
35
+ */
36
+ export const LANGUAGE_CONFIG = {
37
+ typescript: {
38
+ extensions: ['.ts', '.tsx'],
39
+ grammarPackage: 'tree-sitter-wasms/out/tree-sitter-typescript.wasm',
40
+ commentStyle: COMMENT_STYLES.clike,
41
+ queries: {
42
+ // Classes and interfaces
43
+ classes: `
44
+ (class_declaration
45
+ name: (type_identifier) @name) @class
46
+ (interface_declaration
47
+ name: (type_identifier) @name) @interface
48
+ `,
49
+ // Functions and methods
50
+ functions: `
51
+ (function_declaration
52
+ name: (identifier) @name
53
+ parameters: (formal_parameters) @params) @func
54
+ (method_definition
55
+ name: (property_identifier) @name
56
+ parameters: (formal_parameters) @params) @method
57
+ (arrow_function) @arrow
58
+ (lexical_declaration
59
+ (variable_declarator
60
+ name: (identifier) @name
61
+ value: (arrow_function
62
+ parameters: (formal_parameters) @params))) @arrow_decl
63
+ `,
64
+ imports: `
65
+ (import_statement
66
+ source: (string) @source) @import
67
+ (import_statement
68
+ (import_clause
69
+ (named_imports
70
+ (import_specifier
71
+ name: (identifier) @imported_name)))) @named_import
72
+ `,
73
+ // Export statements
74
+ exports: `
75
+ (export_statement) @export
76
+ `,
77
+ calls: `
78
+ (call_expression
79
+ function: [
80
+ (identifier) @callee
81
+ (member_expression
82
+ property: (property_identifier) @callee)
83
+ ]) @call
84
+ `,
85
+ },
86
+ },
87
+
88
+ javascript: {
89
+ extensions: ['.js', '.jsx', '.mjs', '.cjs'],
90
+ grammarPackage: 'tree-sitter-wasms/out/tree-sitter-javascript.wasm',
91
+ commentStyle: COMMENT_STYLES.clike,
92
+ queries: {
93
+ classes: `
94
+ (class_declaration
95
+ name: (identifier) @name) @class
96
+ `,
97
+ functions: `
98
+ (function_declaration
99
+ name: (identifier) @name
100
+ parameters: (formal_parameters) @params) @func
101
+ (method_definition
102
+ name: (property_identifier) @name
103
+ parameters: (formal_parameters) @params) @method
104
+ (lexical_declaration
105
+ (variable_declarator
106
+ name: (identifier) @name
107
+ value: (arrow_function
108
+ parameters: (formal_parameters) @params))) @arrow_decl
109
+ `,
110
+ imports: `
111
+ (import_statement
112
+ source: (string) @source) @import
113
+ (call_expression
114
+ function: (identifier) @require_fn
115
+ arguments: (arguments (string) @source)
116
+ (#eq? @require_fn "require")) @require_call
117
+ `,
118
+ exports: `
119
+ (export_statement) @export
120
+ `,
121
+ calls: `
122
+ (call_expression
123
+ function: [
124
+ (identifier) @callee
125
+ (member_expression
126
+ property: (property_identifier) @callee)
127
+ ]) @call
128
+ `,
129
+ },
130
+ },
131
+
132
+ python: {
133
+ extensions: ['.py'],
134
+ grammarPackage: 'tree-sitter-wasms/out/tree-sitter-python.wasm',
135
+ commentStyle: COMMENT_STYLES.hash,
136
+ queries: {
137
+ classes: `
138
+ (class_definition
139
+ name: (identifier) @name) @class
140
+ `,
141
+ functions: `
142
+ (function_definition
143
+ name: (identifier) @name
144
+ parameters: (parameters) @params) @func
145
+ `,
146
+ imports: `
147
+ (import_statement
148
+ name: (dotted_name) @source) @import
149
+ (import_from_statement
150
+ module_name: (dotted_name) @source) @from_import
151
+ (import_from_statement
152
+ name: (dotted_name) @imported_name) @from_import_name
153
+ `,
154
+ exports: `
155
+ (expression_statement
156
+ (assignment
157
+ left: (identifier) @name
158
+ (#eq? @name "__all__"))) @all_export
159
+ `,
160
+ calls: `
161
+ (call
162
+ function: [
163
+ (identifier) @callee
164
+ (attribute
165
+ attribute: (identifier) @callee)
166
+ ]) @call
167
+ `,
168
+ },
169
+ },
170
+
171
+ go: {
172
+ extensions: ['.go'],
173
+ grammarPackage: 'tree-sitter-wasms/out/tree-sitter-go.wasm',
174
+ commentStyle: COMMENT_STYLES.clike,
175
+ queries: {
176
+ classes: `
177
+ (type_declaration
178
+ (type_spec
179
+ name: (type_identifier) @name
180
+ type: (struct_type))) @struct
181
+ (type_declaration
182
+ (type_spec
183
+ name: (type_identifier) @name
184
+ type: (interface_type))) @interface
185
+ `,
186
+ functions: `
187
+ (function_declaration
188
+ name: (identifier) @name
189
+ parameters: (parameter_list) @params) @func
190
+ (method_declaration
191
+ name: (field_identifier) @name
192
+ parameters: (parameter_list) @params) @method
193
+ `,
194
+ imports: `
195
+ (import_spec
196
+ path: (interpreted_string_literal) @source) @import
197
+ `,
198
+ exports: ``,
199
+ calls: `
200
+ (call_expression
201
+ function: [
202
+ (identifier) @callee
203
+ (selector_expression
204
+ field: (field_identifier) @callee)
205
+ ]) @call
206
+ `,
207
+ },
208
+ },
209
+
210
+ rust: {
211
+ extensions: ['.rs'],
212
+ grammarPackage: 'tree-sitter-wasms/out/tree-sitter-rust.wasm',
213
+ commentStyle: COMMENT_STYLES.clike,
214
+ queries: {
215
+ classes: `
216
+ (struct_item
217
+ name: (type_identifier) @name) @struct
218
+ (enum_item
219
+ name: (type_identifier) @name) @enum
220
+ (trait_item
221
+ name: (type_identifier) @name) @trait
222
+ (impl_item
223
+ type: (type_identifier) @name) @impl
224
+ `,
225
+ functions: `
226
+ (function_item
227
+ name: (identifier) @name
228
+ parameters: (parameters) @params) @func
229
+ `,
230
+ imports: `
231
+ (use_declaration
232
+ argument: (_) @source) @use
233
+ `,
234
+ exports: `
235
+ (visibility_modifier) @pub
236
+ `,
237
+ calls: `
238
+ (call_expression
239
+ function: [
240
+ (identifier) @callee
241
+ (field_expression
242
+ field: (field_identifier) @callee)
243
+ (scoped_identifier
244
+ name: (identifier) @callee)
245
+ ]) @call
246
+ `,
247
+ },
248
+ },
249
+
250
+ java: {
251
+ extensions: ['.java'],
252
+ grammarPackage: 'tree-sitter-wasms/out/tree-sitter-java.wasm',
253
+ commentStyle: COMMENT_STYLES.clike,
254
+ queries: {
255
+ classes: `
256
+ (class_declaration
257
+ name: (identifier) @name) @class
258
+ (interface_declaration
259
+ name: (identifier) @name) @interface
260
+ (enum_declaration
261
+ name: (identifier) @name) @enum
262
+ `,
263
+ functions: `
264
+ (method_declaration
265
+ name: (identifier) @name
266
+ parameters: (formal_parameters) @params) @method
267
+ (constructor_declaration
268
+ name: (identifier) @name
269
+ parameters: (formal_parameters) @params) @constructor
270
+ `,
271
+ imports: `
272
+ (import_declaration
273
+ (scoped_identifier) @source) @import
274
+ `,
275
+ exports: ``,
276
+ calls: `
277
+ (method_invocation
278
+ name: (identifier) @callee) @call
279
+ `,
280
+ },
281
+ },
282
+
283
+ c: {
284
+ extensions: ['.c', '.h'],
285
+ grammarPackage: 'tree-sitter-wasms/out/tree-sitter-c.wasm',
286
+ commentStyle: COMMENT_STYLES.clike,
287
+ queries: {
288
+ classes: `
289
+ (struct_specifier
290
+ name: (type_identifier) @name) @struct
291
+ (enum_specifier
292
+ name: (type_identifier) @name) @enum
293
+ (type_definition
294
+ declarator: (type_identifier) @name) @typedef
295
+ `,
296
+ functions: `
297
+ (function_definition
298
+ declarator: (function_declarator
299
+ declarator: (identifier) @name
300
+ parameters: (parameter_list) @params)) @func
301
+ `,
302
+ imports: `
303
+ (preproc_include
304
+ path: (_) @source) @include
305
+ `,
306
+ exports: ``,
307
+ calls: `
308
+ (call_expression
309
+ function: (identifier) @callee) @call
310
+ `,
311
+ },
312
+ },
313
+
314
+ cpp: {
315
+ extensions: ['.cpp', '.cc', '.cxx', '.hpp', '.hh', '.hxx'],
316
+ grammarPackage: 'tree-sitter-wasms/out/tree-sitter-cpp.wasm',
317
+ commentStyle: COMMENT_STYLES.clike,
318
+ queries: {
319
+ classes: `
320
+ (class_specifier
321
+ name: (type_identifier) @name) @class
322
+ (struct_specifier
323
+ name: (type_identifier) @name) @struct
324
+ `,
325
+ functions: `
326
+ (function_definition
327
+ declarator: (function_declarator
328
+ declarator: [
329
+ (identifier) @name
330
+ (qualified_identifier
331
+ name: (identifier) @name)
332
+ (field_identifier) @name
333
+ ]
334
+ parameters: (parameter_list) @params)) @func
335
+ `,
336
+ imports: `
337
+ (preproc_include
338
+ path: (_) @source) @include
339
+ `,
340
+ exports: ``,
341
+ calls: `
342
+ (call_expression
343
+ function: [
344
+ (identifier) @callee
345
+ (field_expression
346
+ field: (field_identifier) @callee)
347
+ (qualified_identifier
348
+ name: (identifier) @callee)
349
+ ]) @call
350
+ `,
351
+ },
352
+ },
353
+ };
354
+
355
+ /**
356
+ * Build extension → language lookup map.
357
+ */
358
+ export function buildExtensionMap() {
359
+ const map = new Map();
360
+ for (const [lang, config] of Object.entries(LANGUAGE_CONFIG)) {
361
+ for (const ext of config.extensions) {
362
+ map.set(ext, lang);
363
+ }
364
+ }
365
+ return map;
366
+ }
@@ -0,0 +1,224 @@
1
+ ---
2
+ name: knowledge-ingest
3
+ description: >
4
+ Ingest or update a codebase in the agent-knowledge base. First run
5
+ bootstraps the knowledge base from scratch; subsequent runs are
6
+ incremental (only changed/new/deleted files reprocessed). Uses
7
+ tree-sitter for zero-token structural extraction. Trigger on
8
+ "/knowledge-ingest", "ingest this codebase", "load this into knowledge",
9
+ "scan this project", "index this repo", "update knowledge",
10
+ "refresh knowledge", "re-ingest".
11
+ ---
12
+
13
+ # knowledge-ingest
14
+
15
+ Populate or update agent-knowledge from a codebase. Tree-sitter extracts structure (zero LLM tokens), then the agent distills clusters into knowledge entries + graph edges via existing MCP tools.
16
+
17
+ **First run**: full ingest — scans all files, creates entries from scratch.
18
+ **Subsequent runs**: incremental — only reprocesses files whose SHA256 changed, adds entries for new files, removes entries for deleted files. The `.knowledge-ingest-cache.json` file in the target directory tracks state between runs.
19
+
20
+ ## When to use
21
+
22
+ - **Onboarding a new project** — bootstrap the knowledge base so future sessions have context
23
+ - **After a refactor** — re-run to update subsystem boundaries and relationships
24
+ - **Periodic refresh** — re-run after significant changes to keep knowledge current
25
+ - **Importing documentation** — PDFs, architecture diagrams, or external URLs
26
+
27
+ ## When NOT to use
28
+
29
+ - Single-file changes — just write a knowledge entry manually
30
+ - No code changes since last ingest — the cache will skip everything anyway (fast no-op)
31
+
32
+ ## Procedure
33
+
34
+ ### Phase 0 — Validation
35
+
36
+ 1. Confirm the target path exists and is a directory.
37
+ 2. Detect project name:
38
+ - Check `package.json` → `name` field
39
+ - Check `Cargo.toml` → `[package] name`
40
+ - Check `go.mod` → `module` line
41
+ - Check `pyproject.toml` → `[project] name`
42
+ - Fall back to directory basename
43
+ 3. Check for `.knowledge-ingest-cache.json` in the target directory. If found, load it — this is an incremental run. Report how many files changed since last ingest.
44
+
45
+ ### Phase 1 — Structural Extraction (zero tokens)
46
+
47
+ 4. Locate the tree-sitter extraction script. It ships with agent-knowledge:
48
+
49
+ ```bash
50
+ node "<agent-knowledge-repo>/scripts/tree-sitter-extract.mjs" "<target-path>" --exclude "node_modules,dist,.git,vendor,__pycache__,build,target,.venv,coverage" --json
51
+ ```
52
+
53
+ To find `<agent-knowledge-repo>`, check common locations:
54
+
55
+ - `~/.claude/mcp-servers/agent-knowledge/`
56
+ - Or locate via: `dirname $(which agent-knowledge)/../`
57
+
58
+ 5. Capture the JSON output. It contains per-file: symbols (classes, functions, methods), imports, exports, rationale comments (WHY/NOTE/DECISION/HACK/TODO/FIXME), call edges, SHA256 hashes, and a dependency graph.
59
+
60
+ 6. If the script fails (missing dependency, path error), report the error to the user and offer to fall back to manual file reading for a subset of key files.
61
+
62
+ ### Phase 2 — Clustering
63
+
64
+ 7. From the dependency graph and directory structure, group files into subsystems:
65
+ - Files that import each other heavily belong to the same cluster
66
+ - Files in the same directory with related names belong to the same cluster
67
+ - Entry points (files with no importers) get flagged as roots
68
+ - Aim for **5-30 clusters**. If more, merge the smallest.
69
+ - Name each cluster from its directory name or dominant class/module.
70
+
71
+ 8. Identify structural highlights:
72
+ - **God nodes**: symbols imported by 5+ files
73
+ - **Entry points**: files not imported by anything (CLI entry, main, index)
74
+ - **Cross-cutting concerns**: symbols used across multiple clusters
75
+
76
+ ### Phase 3 — Knowledge Entry Creation
77
+
78
+ 9. Create the **project overview** entry:
79
+
80
+ ```
81
+ knowledge({ action: "write", category: "projects", filename: "<project-name>", content: "---\ntitle: <Project Name>\ntags: [auto-ingested, <primary-language>]\nupdated: <today>\nconfidence: inferred\nconfidence_score: 0.8\n---\n\n# <Project Name>\n\n## Tech Stack\n...\n\n## Architecture\n...\n\n## Entry Points\n...\n\n## Subsystems\n- <cluster-1>: <one-line description>\n- <cluster-2>: <one-line description>\n..." })
82
+ ```
83
+
84
+ Include: project name, tech stack (languages, frameworks, key dependencies), directory structure overview, entry points, subsystem list with one-line descriptions.
85
+
86
+ 10. For each cluster/subsystem, create a **subsystem entry**:
87
+
88
+ ```
89
+ knowledge({ action: "write", category: "notes", filename: "<project>-<cluster-name>", content: "---\ntitle: <Project> — <Cluster Name>\ntags: [auto-ingested, subsystem, <language>]\nupdated: <today>\nconfidence: inferred\nconfidence_score: 0.75\n---\n\n## Purpose\n<inferred from symbol names and structure>\n\n## Key Symbols\n- `ClassName` (line N) — <from docstring or inferred>\n- `functionName(params)` (line N)\n\n## Dependencies\nImports from: <other clusters>\nImported by: <other clusters>\n\n## Rationale\n<any WHY/NOTE/DECISION comments found in this cluster>" })
90
+ ```
91
+
92
+ Keep each entry **under 300 words**. Focus on structure and relationships, not implementation details.
93
+
94
+ 11. For rationale comments (WHY/DECISION/HACK), create **decision entries**:
95
+
96
+ ```
97
+ knowledge({ action: "write", category: "decisions", filename: "<project>-<slug>", content: "---\ntitle: <Decision summary>\ntags: [auto-ingested, rationale]\nupdated: <today>\nconfidence: extracted\nconfidence_score: 1.0\n---\n\n## Decision\n<the rationale comment text>\n\n## Context\nFile: `<file-path>`, line <N>\nSymbol: `<enclosing function/class>`\n\n## Related\n<other rationale comments or subsystems this connects to>" })
98
+ ```
99
+
100
+ Only create decision entries for substantive rationale (WHY, DECISION, SAFETY). Skip generic TODOs and FIXMEs unless they contain real architectural reasoning.
101
+
102
+ 12. For CI/build configs found in the project (`.github/workflows/`, `Makefile`, `Dockerfile`, `docker-compose.yml`, `Jenkinsfile`, `.gitlab-ci.yml`), read them directly and create **workflow entries**:
103
+
104
+ ```
105
+ knowledge({ action: "write", category: "workflows", filename: "<project>-<workflow>", content: "..." })
106
+ ```
107
+
108
+ Summarize: what the workflow does, triggers, key steps, deployment targets.
109
+
110
+ ### Phase 4 — Graph Edges
111
+
112
+ 13. Create `part_of` edges from each subsystem to the project:
113
+
114
+ ```
115
+ knowledge_graph({ action: "link", source: "notes/<project>-<cluster>.md", target: "projects/<project>.md", rel_type: "part_of", strength: 0.9, origin: "ingest" })
116
+ ```
117
+
118
+ 14. Create `depends_on` edges between subsystems based on the import dependency graph:
119
+
120
+ ```
121
+ knowledge_graph({ action: "link", source: "notes/<project>-<cluster-a>.md", target: "notes/<project>-<cluster-b>.md", rel_type: "depends_on", strength: 0.8, origin: "ingest" })
122
+ ```
123
+
124
+ 15. Create `builds_on` edges from decisions to the subsystem they relate to:
125
+
126
+ ```
127
+ knowledge_graph({ action: "link", source: "decisions/<project>-<decision>.md", target: "notes/<project>-<cluster>.md", rel_type: "builds_on", strength: 0.7, origin: "ingest" })
128
+ ```
129
+
130
+ 16. Do NOT manually create `related_to` edges — auto-linking fires on every `knowledge write` and handles cross-entry similarity.
131
+
132
+ ### Phase 5 — Multi-modal (if applicable)
133
+
134
+ 17. If **PDF files** exist in the project (root, `docs/`, `papers/`):
135
+ - Read them natively (agent is multimodal)
136
+ - Write a summary entry to `notes/<project>-<pdf-name>.md`
137
+
138
+ 18. If **architecture diagram images** exist (`.png`, `.svg`, `.jpg` in `docs/`, `architecture/`, `diagrams/`):
139
+ - Read them natively
140
+ - Write a description entry to `notes/<project>-<diagram-name>.md`
141
+
142
+ 19. If **URLs** are provided by the user:
143
+ - Fetch via WebFetch
144
+ - Write summary entries to `notes/`
145
+
146
+ 20. Skip this phase entirely if no such files exist. Do not search exhaustively for media files.
147
+
148
+ ### Phase 6 — Cache
149
+
150
+ 21. Write `.knowledge-ingest-cache.json` to the target directory:
151
+
152
+ ```json
153
+ {
154
+ "version": 1,
155
+ "timestamp": "<ISO date>",
156
+ "project": "<project-name>",
157
+ "agent_knowledge_version": "<version>",
158
+ "files": {
159
+ "src/foo.ts": {
160
+ "sha256": "abc123...",
161
+ "entries": ["notes/<project>-foo-module.md"]
162
+ }
163
+ },
164
+ "entries_created": [
165
+ "projects/<project>.md",
166
+ "notes/<project>-core.md",
167
+ "decisions/<project>-auth-design.md"
168
+ ]
169
+ }
170
+ ```
171
+
172
+ 22. On **incremental re-runs**, compare SHA256 hashes:
173
+ - Unchanged files: skip entirely
174
+ - Changed files: re-extract, update corresponding entries
175
+ - Deleted files: remove their entries from knowledge base via `knowledge({ action: "delete" })`
176
+ - New files: process normally
177
+
178
+ ### Phase 7 — Validate
179
+
180
+ 23. Run the validation script to verify the ingest produced correct results:
181
+
182
+ ```bash
183
+ node "<agent-knowledge-repo>/skills/knowledge-ingest/scripts/validate.mjs" "<target-path>"
184
+ ```
185
+
186
+ 24. Parse the JSON output. If `status` is `FAIL`:
187
+ - Review each issue in the `issues` array
188
+ - Fix the issue (create missing entries, repair the cache file, etc.)
189
+ - Re-run validation until it passes
190
+ - Only proceed to the report once validation passes
191
+
192
+ ### Phase 8 — Report
193
+
194
+ 25. Print a summary:
195
+
196
+ ```
197
+ Ingested <project-name>:
198
+ Files scanned: N
199
+ Clusters identified: N
200
+ Entries created:
201
+ - projects/: 1
202
+ - notes/: N (subsystems)
203
+ - decisions/: N (rationale)
204
+ - workflows/: N (CI/build)
205
+ Graph edges: N
206
+ Skipped (cached): N files unchanged
207
+ ```
208
+
209
+ ## Anti-patterns
210
+
211
+ - **Do NOT read every file** — that defeats the purpose of tree-sitter extraction. Only read files the agent needs for summarization (CI configs, READMEs, a few key entry points).
212
+ - **Do NOT create one entry per file** — cluster into subsystems. A 200-file project should produce ~15-25 entries, not 200.
213
+ - **Do NOT skip the cache write** — it is essential for incremental updates.
214
+ - **Do NOT create `related_to` edges manually** — auto-linking handles this on every write.
215
+ - **Do NOT exceed 40 `knowledge write` calls** in a single run — batch small clusters together if needed.
216
+ - **Do NOT include raw source code** in knowledge entries — focus on structure, purpose, and relationships.
217
+
218
+ ## Examples
219
+
220
+ ```
221
+ /knowledge-ingest .
222
+ /knowledge-ingest ~/projects/my-api
223
+ /knowledge-ingest ./libs/auth --exclude "test,mock"
224
+ ```