feed-the-machine 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/generate-manifest.mjs +253 -0
- package/bin/install.mjs +134 -4
- package/docs/HOOKS.md +243 -0
- package/docs/INBOX.md +233 -0
- package/ftm/SKILL.md +34 -0
- package/ftm-audit/SKILL.md +69 -0
- package/ftm-brainstorm/SKILL.md +51 -0
- package/ftm-browse/SKILL.md +39 -0
- package/ftm-capture/SKILL.md +370 -0
- package/ftm-capture.yml +4 -0
- package/ftm-codex-gate/SKILL.md +59 -0
- package/ftm-config/SKILL.md +35 -0
- package/ftm-council/SKILL.md +56 -0
- package/ftm-dashboard/SKILL.md +163 -0
- package/ftm-debug/SKILL.md +84 -0
- package/ftm-diagram/SKILL.md +44 -0
- package/ftm-executor/SKILL.md +97 -0
- package/ftm-git/SKILL.md +60 -0
- package/ftm-inbox/backend/__init__.py +0 -0
- package/ftm-inbox/backend/__pycache__/main.cpython-314.pyc +0 -0
- package/ftm-inbox/backend/adapters/__init__.py +0 -0
- package/ftm-inbox/backend/adapters/_retry.py +64 -0
- package/ftm-inbox/backend/adapters/base.py +230 -0
- package/ftm-inbox/backend/adapters/freshservice.py +104 -0
- package/ftm-inbox/backend/adapters/gmail.py +125 -0
- package/ftm-inbox/backend/adapters/jira.py +136 -0
- package/ftm-inbox/backend/adapters/registry.py +192 -0
- package/ftm-inbox/backend/adapters/slack.py +110 -0
- package/ftm-inbox/backend/db/__init__.py +0 -0
- package/ftm-inbox/backend/db/connection.py +54 -0
- package/ftm-inbox/backend/db/schema.py +78 -0
- package/ftm-inbox/backend/executor/__init__.py +7 -0
- package/ftm-inbox/backend/executor/engine.py +149 -0
- package/ftm-inbox/backend/executor/step_runner.py +98 -0
- package/ftm-inbox/backend/main.py +103 -0
- package/ftm-inbox/backend/models/__init__.py +1 -0
- package/ftm-inbox/backend/models/unified_task.py +36 -0
- package/ftm-inbox/backend/planner/__init__.py +6 -0
- package/ftm-inbox/backend/planner/__pycache__/__init__.cpython-314.pyc +0 -0
- package/ftm-inbox/backend/planner/__pycache__/generator.cpython-314.pyc +0 -0
- package/ftm-inbox/backend/planner/__pycache__/schema.cpython-314.pyc +0 -0
- package/ftm-inbox/backend/planner/generator.py +127 -0
- package/ftm-inbox/backend/planner/schema.py +34 -0
- package/ftm-inbox/backend/requirements.txt +5 -0
- package/ftm-inbox/backend/routes/__init__.py +0 -0
- package/ftm-inbox/backend/routes/__pycache__/plan.cpython-314.pyc +0 -0
- package/ftm-inbox/backend/routes/execute.py +186 -0
- package/ftm-inbox/backend/routes/health.py +52 -0
- package/ftm-inbox/backend/routes/inbox.py +68 -0
- package/ftm-inbox/backend/routes/plan.py +271 -0
- package/ftm-inbox/bin/launchagent.mjs +91 -0
- package/ftm-inbox/bin/setup.mjs +188 -0
- package/ftm-inbox/bin/start.sh +10 -0
- package/ftm-inbox/bin/status.sh +17 -0
- package/ftm-inbox/bin/stop.sh +8 -0
- package/ftm-inbox/config.example.yml +55 -0
- package/ftm-inbox/package-lock.json +2898 -0
- package/ftm-inbox/package.json +26 -0
- package/ftm-inbox/postcss.config.js +6 -0
- package/ftm-inbox/src/app.css +199 -0
- package/ftm-inbox/src/app.html +18 -0
- package/ftm-inbox/src/lib/api.ts +166 -0
- package/ftm-inbox/src/lib/components/ExecutionLog.svelte +81 -0
- package/ftm-inbox/src/lib/components/InboxFeed.svelte +143 -0
- package/ftm-inbox/src/lib/components/PlanStep.svelte +271 -0
- package/ftm-inbox/src/lib/components/PlanView.svelte +206 -0
- package/ftm-inbox/src/lib/components/StreamPanel.svelte +99 -0
- package/ftm-inbox/src/lib/components/TaskCard.svelte +190 -0
- package/ftm-inbox/src/lib/components/ui/EmptyState.svelte +63 -0
- package/ftm-inbox/src/lib/components/ui/KawaiiCard.svelte +86 -0
- package/ftm-inbox/src/lib/components/ui/PillButton.svelte +106 -0
- package/ftm-inbox/src/lib/components/ui/StatusBadge.svelte +67 -0
- package/ftm-inbox/src/lib/components/ui/StreamDrawer.svelte +149 -0
- package/ftm-inbox/src/lib/components/ui/ThemeToggle.svelte +80 -0
- package/ftm-inbox/src/lib/theme.ts +47 -0
- package/ftm-inbox/src/routes/+layout.svelte +76 -0
- package/ftm-inbox/src/routes/+page.svelte +401 -0
- package/ftm-inbox/static/favicon.png +0 -0
- package/ftm-inbox/svelte.config.js +12 -0
- package/ftm-inbox/tailwind.config.ts +63 -0
- package/ftm-inbox/tsconfig.json +13 -0
- package/ftm-inbox/vite.config.ts +6 -0
- package/ftm-intent/SKILL.md +44 -0
- package/ftm-manifest.json +3794 -0
- package/ftm-map/SKILL.md +259 -0
- package/ftm-map/scripts/db.py +391 -0
- package/ftm-map/scripts/index.py +341 -0
- package/ftm-map/scripts/parser.py +455 -0
- package/ftm-map/scripts/queries/.gitkeep +0 -0
- package/ftm-map/scripts/queries/javascript-tags.scm +23 -0
- package/ftm-map/scripts/queries/python-tags.scm +17 -0
- package/ftm-map/scripts/queries/typescript-tags.scm +29 -0
- package/ftm-map/scripts/query.py +149 -0
- package/ftm-map/scripts/requirements.txt +2 -0
- package/ftm-map/scripts/setup-hooks.sh +27 -0
- package/ftm-map/scripts/setup.sh +45 -0
- package/ftm-map/scripts/test_db.py +124 -0
- package/ftm-map/scripts/test_parser.py +106 -0
- package/ftm-map/scripts/test_query.py +66 -0
- package/ftm-map/scripts/tests/fixtures/__init__.py +0 -0
- package/ftm-map/scripts/tests/fixtures/sample_project/api.ts +16 -0
- package/ftm-map/scripts/tests/fixtures/sample_project/auth.py +15 -0
- package/ftm-map/scripts/tests/fixtures/sample_project/utils.js +16 -0
- package/ftm-map/scripts/views.py +545 -0
- package/ftm-mind/SKILL.md +173 -66
- package/ftm-pause/SKILL.md +43 -0
- package/ftm-researcher/SKILL.md +275 -0
- package/ftm-researcher/evals/agent-diversity.yaml +17 -0
- package/ftm-researcher/evals/synthesis-quality.yaml +12 -0
- package/ftm-researcher/evals/trigger-accuracy.yaml +39 -0
- package/ftm-researcher/references/adaptive-search.md +116 -0
- package/ftm-researcher/references/agent-prompts.md +193 -0
- package/ftm-researcher/references/council-integration.md +193 -0
- package/ftm-researcher/references/output-format.md +203 -0
- package/ftm-researcher/references/synthesis-pipeline.md +165 -0
- package/ftm-researcher/scripts/score_credibility.py +234 -0
- package/ftm-researcher/scripts/validate_research.py +92 -0
- package/ftm-resume/SKILL.md +47 -0
- package/ftm-retro/SKILL.md +54 -0
- package/ftm-routine/SKILL.md +170 -0
- package/ftm-state/blackboard/capabilities.json +5 -0
- package/ftm-state/blackboard/capabilities.schema.json +27 -0
- package/ftm-upgrade/SKILL.md +41 -0
- package/ftm-upgrade/scripts/check-version.sh +1 -1
- package/ftm-upgrade/scripts/upgrade.sh +1 -1
- package/hooks/ftm-blackboard-enforcer.sh +94 -0
- package/hooks/ftm-discovery-reminder.sh +90 -0
- package/hooks/ftm-drafts-gate.sh +61 -0
- package/hooks/ftm-event-logger.mjs +107 -0
- package/hooks/ftm-map-autodetect.sh +79 -0
- package/hooks/ftm-pending-sync-check.sh +22 -0
- package/hooks/ftm-plan-gate.sh +96 -0
- package/hooks/ftm-post-commit-trigger.sh +57 -0
- package/hooks/settings-template.json +81 -0
- package/install.sh +140 -11
- package/package.json +12 -2
package/ftm-map/SKILL.md
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: ftm-map
|
|
3
|
+
description: Persistent code knowledge graph powered by tree-sitter and SQLite with FTS5 full-text search. Builds structural dependency graphs for blast radius analysis, dependency chains, and keyword search. Use when user asks "what breaks if I change X", "blast radius", "what depends on", "where do we handle", "map codebase", "index project", "what calls", "dependency chain", "ftm-map".
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# ftm-map
|
|
7
|
+
|
|
8
|
+
Persistent code knowledge graph powered by tree-sitter and SQLite with FTS5 full-text search. Parses the local codebase into a structural dependency graph stored in `.ftm-map/map.db`, then answers structural queries (blast radius, dependency chains, symbol lookup) and keyword searches without re-reading the source tree on every question.
|
|
9
|
+
|
|
10
|
+
## Events
|
|
11
|
+
|
|
12
|
+
### Emits
|
|
13
|
+
- `map_updated` — when the graph database has been updated (bootstrap or incremental)
|
|
14
|
+
- Payload: `{ project_path, symbols_count, edges_count, files_parsed, duration_ms, mode }`
|
|
15
|
+
- `task_completed` — when any ftm-map operation finishes
|
|
16
|
+
|
|
17
|
+
### Listens To
|
|
18
|
+
- `code_committed` — run incremental index on changed files, then emit `map_updated`
|
|
19
|
+
- `task_received` — begin bootstrap or query when ftm-mind routes a mapping/search request
|
|
20
|
+
|
|
21
|
+
## Config Read
|
|
22
|
+
|
|
23
|
+
Read `~/.claude/ftm-config.yml`:
|
|
24
|
+
- Check `skills.ftm-map.enabled` (default: true)
|
|
25
|
+
- Use `execution` model from active profile for indexing agents
|
|
26
|
+
|
|
27
|
+
## Blackboard Read
|
|
28
|
+
|
|
29
|
+
On startup, load context from the FTM blackboard:
|
|
30
|
+
1. Load `~/.claude/ftm-blackboard/context.json`
|
|
31
|
+
2. Filter experiences by `task_type: "map"`
|
|
32
|
+
3. Load matching experience files to inform index scope and query routing
|
|
33
|
+
4. Check for prior bootstrap records to determine if incremental mode is appropriate
|
|
34
|
+
|
|
35
|
+
## Mode Detection
|
|
36
|
+
|
|
37
|
+
Three modes, detected from request context:
|
|
38
|
+
|
|
39
|
+
```
|
|
40
|
+
Bootstrap: "map this codebase" / "index this project" / no map.db exists yet
|
|
41
|
+
Full scan of all source files. Builds graph from scratch.
|
|
42
|
+
|
|
43
|
+
Incremental: Triggered by code_committed event or PostToolUse hook
|
|
44
|
+
Parses only changed files and updates their graph entries.
|
|
45
|
+
|
|
46
|
+
Query: Structural or keyword question about existing graph
|
|
47
|
+
Detects query type and runs appropriate script.
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
If `.ftm-map/map.db` does not exist when a query arrives, fall back to offering bootstrap (see Graceful Degradation below).
|
|
51
|
+
|
|
52
|
+
## Mode 1: Bootstrap (full scan)
|
|
53
|
+
|
|
54
|
+
Trigger: user says "map this codebase" or "index this project", or `.ftm-map/map.db` does not yet exist.
|
|
55
|
+
|
|
56
|
+
1. Run `ftm-map/scripts/setup.sh` to ensure virtualenv and tree-sitter dependencies are installed
|
|
57
|
+
2. Run `ftm-map/scripts/.venv/bin/python3 ftm-map/scripts/index.py --bootstrap <project_root>`
|
|
58
|
+
3. Capture and report stats from stdout:
|
|
59
|
+
- Files parsed
|
|
60
|
+
- Symbols found
|
|
61
|
+
- Edges created
|
|
62
|
+
- Time elapsed
|
|
63
|
+
4. Emit `map_updated` with `mode: "bootstrap"`
|
|
64
|
+
|
|
65
|
+
Example invocation:
|
|
66
|
+
```
|
|
67
|
+
ftm-map/scripts/.venv/bin/python3 ftm-map/scripts/index.py --bootstrap .
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Mode 2: Incremental (post-commit)
|
|
71
|
+
|
|
72
|
+
Trigger: `code_committed` event fires, or PostToolUse hook detects a write to a source file.
|
|
73
|
+
|
|
74
|
+
1. Get changed files:
|
|
75
|
+
```
|
|
76
|
+
git diff --name-only HEAD~1
|
|
77
|
+
```
|
|
78
|
+
2. Filter to source files only (skip docs, configs, lockfiles)
|
|
79
|
+
3. Run incremental index on changed files:
|
|
80
|
+
```
|
|
81
|
+
ftm-map/scripts/.venv/bin/python3 ftm-map/scripts/index.py --incremental --files <file1> <file2> ...
|
|
82
|
+
```
|
|
83
|
+
4. Emit `map_updated` with `mode: "incremental"` and count of updated entries
|
|
84
|
+
|
|
85
|
+
## Mode 3: Query (answer structural and search questions)
|
|
86
|
+
|
|
87
|
+
Trigger: user asks a structural or keyword question about the codebase.
|
|
88
|
+
|
|
89
|
+
### Query Type Detection
|
|
90
|
+
|
|
91
|
+
| User says | Query type | Script flag |
|
|
92
|
+
|-----------|-----------|-------------|
|
|
93
|
+
| "what breaks if I change X" | blast radius | `--blast-radius X` |
|
|
94
|
+
| "blast radius of X" | blast radius | `--blast-radius X` |
|
|
95
|
+
| "what depends on X" | dependency chain | `--deps X` |
|
|
96
|
+
| "what calls X" | dependency chain (callers) | `--deps X` |
|
|
97
|
+
| "where do we handle X" | FTS5 keyword search | `--search "X"` |
|
|
98
|
+
| "find X in the codebase" | FTS5 keyword search | `--search "X"` |
|
|
99
|
+
| "tell me about function X" | symbol info | `--info X` |
|
|
100
|
+
| "show dependencies for X" | dependency chain | `--deps X` |
|
|
101
|
+
|
|
102
|
+
### Execution
|
|
103
|
+
|
|
104
|
+
Run the appropriate query script with the venv python:
|
|
105
|
+
```
|
|
106
|
+
ftm-map/scripts/.venv/bin/python3 ftm-map/scripts/query.py --blast-radius <symbol>
|
|
107
|
+
ftm-map/scripts/.venv/bin/python3 ftm-map/scripts/query.py --deps <symbol>
|
|
108
|
+
ftm-map/scripts/.venv/bin/python3 ftm-map/scripts/query.py --search "<keywords>"
|
|
109
|
+
ftm-map/scripts/.venv/bin/python3 ftm-map/scripts/query.py --info <symbol>
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### Output Formatting
|
|
113
|
+
|
|
114
|
+
Scripts return JSON. Render as readable markdown:
|
|
115
|
+
|
|
116
|
+
**Blast radius** — tree of affected symbols with file paths and line numbers:
|
|
117
|
+
```
|
|
118
|
+
Blast radius of `authenticateUser`:
|
|
119
|
+
direct callers (3):
|
|
120
|
+
• loginHandler src/handlers/auth.ts:42
|
|
121
|
+
• refreshSession src/handlers/session.ts:17
|
|
122
|
+
• testAuthFlow src/tests/auth.test.ts:88
|
|
123
|
+
transitive (5):
|
|
124
|
+
• routeMiddleware src/middleware/index.ts:12
|
|
125
|
+
...
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
**Dependency chain** — ordered list of dependencies (callee direction):
|
|
129
|
+
```
|
|
130
|
+
Dependencies of `authenticateUser`:
|
|
131
|
+
1. validateToken src/auth/tokens.ts:8
|
|
132
|
+
2. decodeJWT src/auth/jwt.ts:22
|
|
133
|
+
3. createSession src/auth/session.ts:45
|
|
134
|
+
4. storeSession src/auth/session.ts:67
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
**FTS5 search** — BM25-ranked list with file:line references:
|
|
138
|
+
```
|
|
139
|
+
Results for "rate limit" (6 matches, ranked by relevance):
|
|
140
|
+
1. applyRateLimit src/middleware/ratelimit.ts:14 score: 0.94
|
|
141
|
+
2. RateLimitConfig src/config/types.ts:88 score: 0.81
|
|
142
|
+
3. checkRateLimit src/handlers/base.ts:203 score: 0.77
|
|
143
|
+
...
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
**Symbol info** — full details card:
|
|
147
|
+
```
|
|
148
|
+
Symbol: authenticateUser
|
|
149
|
+
Kind: function
|
|
150
|
+
File: src/auth/index.ts:34
|
|
151
|
+
Signature: authenticateUser(token: string, opts?: AuthOptions) → Promise<Session>
|
|
152
|
+
Callers: 3 direct, 5 transitive
|
|
153
|
+
Callees: validateToken, decodeJWT, createSession
|
|
154
|
+
Dependents: 8 symbols total
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
## Graceful Degradation
|
|
158
|
+
|
|
159
|
+
If `.ftm-map/map.db` does not exist when a query is requested:
|
|
160
|
+
|
|
161
|
+
1. Explain that the graph has not been indexed yet
|
|
162
|
+
2. Offer to bootstrap: "Run `ftm-map bootstrap` to index this codebase?"
|
|
163
|
+
3. If user confirms, switch to Bootstrap mode immediately
|
|
164
|
+
4. Do not attempt to answer structural queries by reading source files directly — the graph is the source of truth for structural questions
|
|
165
|
+
|
|
166
|
+
## Python Script Interface
|
|
167
|
+
|
|
168
|
+
All heavy lifting is done by Python scripts in `ftm-map/scripts/`. The skill orchestrates: detects mode, runs the right script with venv python, formats the output.
|
|
169
|
+
|
|
170
|
+
| Script | Purpose |
|
|
171
|
+
|--------|---------|
|
|
172
|
+
| `setup.sh` | Creates virtualenv, installs tree-sitter and dependencies |
|
|
173
|
+
| `db.py` | SQLite schema, CRUD operations, graph traversal queries |
|
|
174
|
+
| `parser.py` | tree-sitter parsing and symbol/edge extraction |
|
|
175
|
+
| `index.py` | Full bootstrap scan and incremental file indexing |
|
|
176
|
+
| `query.py` | Blast radius, dependency chain, FTS5 keyword search, symbol info |
|
|
177
|
+
| `views.py` | INTENT.md and .mmd generation from graph data |
|
|
178
|
+
|
|
179
|
+
Always use the venv python — never the system python — to ensure tree-sitter bindings are available:
|
|
180
|
+
```
|
|
181
|
+
ftm-map/scripts/.venv/bin/python3 <script> <args>
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
## Integration Points
|
|
185
|
+
|
|
186
|
+
**ftm-intent** may call ftm-map to retrieve caller/callee relationships when writing the `Relationships` field of INTENT.md entries. ftm-map returns structured JSON that ftm-intent formats into human-readable relationship text.
|
|
187
|
+
|
|
188
|
+
**ftm-diagram** may call ftm-map to retrieve the dependency graph for a module when generating DIAGRAM.mmd files. ftm-map returns edge data that ftm-diagram renders as mermaid nodes and edges.
|
|
189
|
+
|
|
190
|
+
Both integrations use `query.py --deps` and `query.py --info` to retrieve graph data without re-parsing source.
|
|
191
|
+
|
|
192
|
+
## Blackboard Write
|
|
193
|
+
|
|
194
|
+
After `map_updated` or session end:
|
|
195
|
+
1. Update `~/.claude/ftm-blackboard/context.json` with map session summary
|
|
196
|
+
2. Write experience file: `~/.claude/ftm-blackboard/experiences/map-[timestamp].json`
|
|
197
|
+
- Fields: project_path, mode, symbols_count, edges_count, files_parsed, duration_ms
|
|
198
|
+
3. Update `~/.claude/ftm-blackboard/index.json` with new experience entry
|
|
199
|
+
4. Emit `task_completed` event
|
|
200
|
+
|
|
201
|
+
## Rules
|
|
202
|
+
|
|
203
|
+
- NEVER stop to ask for input. Make decisions and keep going.
|
|
204
|
+
- ALWAYS commit after completing with a clear message.
|
|
205
|
+
- ALWAYS review after commit: run `git diff HEAD~1`.
|
|
206
|
+
- Never reference AI/Claude in commit messages.
|
|
207
|
+
- Stay in your worktree.
|
|
208
|
+
- ALWAYS use the venv python (`ftm-map/scripts/.venv/bin/python3`), never the system python.
|
|
209
|
+
- For query mode, ALWAYS run `setup.sh` first if `.venv` does not exist.
|
|
210
|
+
|
|
211
|
+
## Requirements
|
|
212
|
+
|
|
213
|
+
- tool: `ftm-map/scripts/.venv/bin/python3` | required | Python with tree-sitter and SQLite bindings
|
|
214
|
+
- tool: `ftm-map/scripts/setup.sh` | required | virtualenv and dependency installer
|
|
215
|
+
- tool: `ftm-map/scripts/index.py` | required | bootstrap and incremental indexer
|
|
216
|
+
- tool: `ftm-map/scripts/query.py` | required | blast radius, dependency, and FTS5 search queries
|
|
217
|
+
- tool: `ftm-map/scripts/views.py` | required | INTENT.md and .mmd diagram generation from graph
|
|
218
|
+
- tool: `git` | optional | changed file detection for incremental mode
|
|
219
|
+
- config: `~/.claude/ftm-config.yml` | optional | model profile and skills.ftm-map.enabled flag
|
|
220
|
+
|
|
221
|
+
## Risk
|
|
222
|
+
|
|
223
|
+
- level: low_write
|
|
224
|
+
- scope: writes and updates .ftm-map/map.db SQLite database; does not modify any project source files; also writes blackboard experience entry
|
|
225
|
+
- rollback: delete .ftm-map/map.db to reset to unindexed state; re-run bootstrap to rebuild
|
|
226
|
+
|
|
227
|
+
## Approval Gates
|
|
228
|
+
|
|
229
|
+
- trigger: bootstrap requested on very large codebase (1000+ files) | action: report estimated file count before running, proceed unless user objects
|
|
230
|
+
- complexity_routing: micro → auto | small → auto | medium → auto | large → auto | xl → auto
|
|
231
|
+
|
|
232
|
+
## Fallbacks
|
|
233
|
+
|
|
234
|
+
- condition: .venv does not exist | action: run setup.sh first to create it before proceeding
|
|
235
|
+
- condition: tree-sitter binary missing | action: run setup.sh to install dependencies
|
|
236
|
+
- condition: .ftm-map/map.db missing when query requested | action: explain graph not indexed, offer to run bootstrap
|
|
237
|
+
- condition: git not available for incremental changed-file detection | action: fall back to indexing all modified files detected from disk timestamps
|
|
238
|
+
|
|
239
|
+
## Capabilities
|
|
240
|
+
|
|
241
|
+
- cli: `ftm-map/scripts/.venv/bin/python3` | required | tree-sitter parsing and SQLite operations
|
|
242
|
+
- cli: `git` | optional | changed file detection for incremental indexing
|
|
243
|
+
|
|
244
|
+
## Event Payloads
|
|
245
|
+
|
|
246
|
+
### map_updated
|
|
247
|
+
- skill: string — "ftm-map"
|
|
248
|
+
- project_path: string — absolute path to indexed project
|
|
249
|
+
- symbols_count: number — total symbols in the graph
|
|
250
|
+
- edges_count: number — total dependency edges
|
|
251
|
+
- files_parsed: number — files processed in this operation
|
|
252
|
+
- duration_ms: number — indexing duration
|
|
253
|
+
- mode: string — "bootstrap" | "incremental"
|
|
254
|
+
|
|
255
|
+
### task_completed
|
|
256
|
+
- skill: string — "ftm-map"
|
|
257
|
+
- operation: string — "bootstrap" | "incremental" | "query"
|
|
258
|
+
- query_type: string | null — "blast-radius" | "deps" | "search" | "info" (for query mode)
|
|
259
|
+
- duration_ms: number — total operation duration
|
|
@@ -0,0 +1,391 @@
|
|
|
1
|
+
"""
|
|
2
|
+
db.py — SQLite database module for ftm-map.
|
|
3
|
+
|
|
4
|
+
Manages the symbols/edges/FTS5 schema and provides CRUD operations for the
|
|
5
|
+
code graph. This is a library module — import it from index.py, query.py,
|
|
6
|
+
and views.py.
|
|
7
|
+
|
|
8
|
+
Schema overview:
|
|
9
|
+
symbols — indexed code symbols (functions, classes, methods, etc.)
|
|
10
|
+
edges — directed dependency relationships between symbols
|
|
11
|
+
symbols_fts — FTS5 virtual table for full-text search (BM25-ranked)
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import hashlib
|
|
15
|
+
import os
|
|
16
|
+
import sqlite3
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Optional
|
|
19
|
+
|
|
20
|
+
# ---------------------------------------------------------------------------
|
|
21
|
+
# Constants
|
|
22
|
+
# ---------------------------------------------------------------------------
|
|
23
|
+
|
|
24
|
+
DB_DIR = ".ftm-map"
|
|
25
|
+
DB_PATH = os.path.join(DB_DIR, "map.db")
|
|
26
|
+
|
|
27
|
+
# ---------------------------------------------------------------------------
|
|
28
|
+
# Schema DDL
|
|
29
|
+
# ---------------------------------------------------------------------------
|
|
30
|
+
|
|
31
|
+
_SCHEMA = """
|
|
32
|
+
CREATE TABLE IF NOT EXISTS symbols (
|
|
33
|
+
id INTEGER PRIMARY KEY,
|
|
34
|
+
name TEXT NOT NULL,
|
|
35
|
+
kind TEXT NOT NULL, -- 'function','class','method','variable','import','module'
|
|
36
|
+
file_path TEXT NOT NULL,
|
|
37
|
+
start_line INTEGER,
|
|
38
|
+
end_line INTEGER,
|
|
39
|
+
signature TEXT,
|
|
40
|
+
doc_comment TEXT,
|
|
41
|
+
content_hash TEXT -- hash of symbol body for change detection
|
|
42
|
+
);
|
|
43
|
+
|
|
44
|
+
CREATE TABLE IF NOT EXISTS edges (
|
|
45
|
+
source_id INTEGER NOT NULL REFERENCES symbols(id) ON DELETE CASCADE,
|
|
46
|
+
target_id INTEGER NOT NULL REFERENCES symbols(id) ON DELETE CASCADE,
|
|
47
|
+
kind TEXT NOT NULL, -- 'calls','imports','extends','implements','uses'
|
|
48
|
+
PRIMARY KEY (source_id, target_id, kind)
|
|
49
|
+
);
|
|
50
|
+
|
|
51
|
+
CREATE INDEX IF NOT EXISTS idx_edges_target ON edges(target_id);
|
|
52
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(file_path);
|
|
53
|
+
CREATE INDEX IF NOT EXISTS idx_symbols_name ON symbols(name);
|
|
54
|
+
|
|
55
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS symbols_fts USING fts5(
|
|
56
|
+
name, signature, doc_comment, file_path,
|
|
57
|
+
content=symbols, content_rowid=id
|
|
58
|
+
);
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
# ---------------------------------------------------------------------------
|
|
62
|
+
# Connection management
|
|
63
|
+
# ---------------------------------------------------------------------------
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def get_connection(project_root: str) -> sqlite3.Connection:
|
|
67
|
+
"""Return a connection to the project's map database.
|
|
68
|
+
|
|
69
|
+
Creates .ftm-map/ and initialises the schema if they do not exist yet.
|
|
70
|
+
WAL mode is enabled for concurrent readers; foreign-key enforcement is on.
|
|
71
|
+
"""
|
|
72
|
+
db_path = os.path.join(project_root, DB_PATH)
|
|
73
|
+
os.makedirs(os.path.dirname(db_path), exist_ok=True)
|
|
74
|
+
|
|
75
|
+
conn = sqlite3.connect(db_path)
|
|
76
|
+
conn.execute("PRAGMA journal_mode=WAL")
|
|
77
|
+
conn.execute("PRAGMA foreign_keys=ON")
|
|
78
|
+
conn.row_factory = sqlite3.Row
|
|
79
|
+
|
|
80
|
+
_init_schema(conn)
|
|
81
|
+
return conn
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _init_schema(conn: sqlite3.Connection) -> None:
|
|
85
|
+
"""Create tables, indexes, and FTS5 virtual table if they do not exist."""
|
|
86
|
+
conn.executescript(_SCHEMA)
|
|
87
|
+
conn.commit()
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
# ---------------------------------------------------------------------------
|
|
91
|
+
# Symbol CRUD
|
|
92
|
+
# ---------------------------------------------------------------------------
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def add_symbol(
|
|
96
|
+
conn: sqlite3.Connection,
|
|
97
|
+
name: str,
|
|
98
|
+
kind: str,
|
|
99
|
+
file_path: str,
|
|
100
|
+
start_line: Optional[int],
|
|
101
|
+
end_line: Optional[int],
|
|
102
|
+
signature: Optional[str] = None,
|
|
103
|
+
doc_comment: Optional[str] = None,
|
|
104
|
+
content_hash: Optional[str] = None,
|
|
105
|
+
) -> int:
|
|
106
|
+
"""Insert a symbol row and keep the FTS5 index in sync.
|
|
107
|
+
|
|
108
|
+
Returns the new symbol id.
|
|
109
|
+
"""
|
|
110
|
+
cursor = conn.execute(
|
|
111
|
+
"""
|
|
112
|
+
INSERT INTO symbols
|
|
113
|
+
(name, kind, file_path, start_line, end_line, signature, doc_comment, content_hash)
|
|
114
|
+
VALUES (?,?,?,?,?,?,?,?)
|
|
115
|
+
""",
|
|
116
|
+
(name, kind, file_path, start_line, end_line, signature, doc_comment, content_hash),
|
|
117
|
+
)
|
|
118
|
+
symbol_id = cursor.lastrowid
|
|
119
|
+
|
|
120
|
+
# FTS5 content= tables require manual insert so BM25 ranking stays accurate.
|
|
121
|
+
conn.execute(
|
|
122
|
+
"INSERT INTO symbols_fts(rowid, name, signature, doc_comment, file_path) VALUES (?,?,?,?,?)",
|
|
123
|
+
(symbol_id, name, signature or "", doc_comment or "", file_path),
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
return symbol_id
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def remove_symbols_by_file(conn: sqlite3.Connection, file_path: str) -> None:
|
|
130
|
+
"""Delete all symbols (and their edges) for a given file.
|
|
131
|
+
|
|
132
|
+
FTS5 rows are removed explicitly before the symbol rows because the
|
|
133
|
+
content= table does not handle cascaded deletes automatically.
|
|
134
|
+
ON DELETE CASCADE handles edge cleanup via the symbols foreign key.
|
|
135
|
+
"""
|
|
136
|
+
ids = [
|
|
137
|
+
row["id"]
|
|
138
|
+
for row in conn.execute("SELECT id FROM symbols WHERE file_path=?", (file_path,))
|
|
139
|
+
]
|
|
140
|
+
for sid in ids:
|
|
141
|
+
conn.execute("DELETE FROM symbols_fts WHERE rowid=?", (sid,))
|
|
142
|
+
|
|
143
|
+
conn.execute("DELETE FROM symbols WHERE file_path=?", (file_path,))
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def get_symbol_by_id(conn: sqlite3.Connection, symbol_id: int) -> Optional[dict]:
|
|
147
|
+
"""Return a symbol row as a dict, or None if not found."""
|
|
148
|
+
row = conn.execute("SELECT * FROM symbols WHERE id=?", (symbol_id,)).fetchone()
|
|
149
|
+
return dict(row) if row else None
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def get_symbol_by_name(conn: sqlite3.Connection, name: str) -> list:
|
|
153
|
+
"""Return all symbols matching *name* (name is not guaranteed unique)."""
|
|
154
|
+
rows = conn.execute("SELECT * FROM symbols WHERE name=?", (name,)).fetchall()
|
|
155
|
+
return [dict(r) for r in rows]
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
# ---------------------------------------------------------------------------
|
|
159
|
+
# Edge CRUD
|
|
160
|
+
# ---------------------------------------------------------------------------
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def add_edge(conn: sqlite3.Connection, source_id: int, target_id: int, kind: str) -> None:
|
|
164
|
+
"""Insert a directed edge. Silently ignored if the edge already exists."""
|
|
165
|
+
conn.execute(
|
|
166
|
+
"INSERT OR IGNORE INTO edges (source_id, target_id, kind) VALUES (?,?,?)",
|
|
167
|
+
(source_id, target_id, kind),
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
# ---------------------------------------------------------------------------
|
|
172
|
+
# Graph traversal — recursive CTEs
|
|
173
|
+
# ---------------------------------------------------------------------------
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def get_transitive_deps(
|
|
177
|
+
conn: sqlite3.Connection, symbol_id: int, max_depth: int = 10
|
|
178
|
+
) -> list:
|
|
179
|
+
"""Return all symbols this symbol transitively depends on (forward closure).
|
|
180
|
+
|
|
181
|
+
Cycle prevention is handled by tracking visited ids as a comma-separated
|
|
182
|
+
path string inside the CTE; a node is skipped if its id already appears in
|
|
183
|
+
the path string.
|
|
184
|
+
|
|
185
|
+
Results are ordered by traversal depth (shallowest first) and deduplicated.
|
|
186
|
+
"""
|
|
187
|
+
query = """
|
|
188
|
+
WITH RECURSIVE dep_chain(id, name, kind, file_path, depth, path) AS (
|
|
189
|
+
-- Base: direct dependencies of the seed symbol
|
|
190
|
+
SELECT s.id,
|
|
191
|
+
s.name,
|
|
192
|
+
s.kind,
|
|
193
|
+
s.file_path,
|
|
194
|
+
0,
|
|
195
|
+
CAST(s.id AS TEXT)
|
|
196
|
+
FROM edges e
|
|
197
|
+
JOIN symbols s ON s.id = e.target_id
|
|
198
|
+
WHERE e.source_id = ?
|
|
199
|
+
|
|
200
|
+
UNION ALL
|
|
201
|
+
|
|
202
|
+
-- Recursive: dependencies of already-visited nodes
|
|
203
|
+
SELECT s.id,
|
|
204
|
+
s.name,
|
|
205
|
+
s.kind,
|
|
206
|
+
s.file_path,
|
|
207
|
+
dc.depth + 1,
|
|
208
|
+
dc.path || ',' || CAST(s.id AS TEXT)
|
|
209
|
+
FROM dep_chain dc
|
|
210
|
+
JOIN edges e ON e.source_id = dc.id
|
|
211
|
+
JOIN symbols s ON s.id = e.target_id
|
|
212
|
+
WHERE dc.depth < ?
|
|
213
|
+
AND INSTR(dc.path, CAST(s.id AS TEXT)) = 0 -- cycle guard
|
|
214
|
+
)
|
|
215
|
+
SELECT DISTINCT id, name, kind, file_path, depth
|
|
216
|
+
FROM dep_chain
|
|
217
|
+
ORDER BY depth
|
|
218
|
+
"""
|
|
219
|
+
rows = conn.execute(query, (symbol_id, max_depth)).fetchall()
|
|
220
|
+
return [dict(r) for r in rows]
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def get_reverse_deps(
|
|
224
|
+
conn: sqlite3.Connection, symbol_id: int, max_depth: int = 10
|
|
225
|
+
) -> list:
|
|
226
|
+
"""Return all symbols that transitively depend on this symbol (blast radius).
|
|
227
|
+
|
|
228
|
+
Traverses edges in reverse (callers/importers of the seed symbol).
|
|
229
|
+
Same cycle-prevention strategy as get_transitive_deps.
|
|
230
|
+
"""
|
|
231
|
+
query = """
|
|
232
|
+
WITH RECURSIVE rev_chain(id, name, kind, file_path, depth, path) AS (
|
|
233
|
+
-- Base: direct dependents of the seed symbol
|
|
234
|
+
SELECT s.id,
|
|
235
|
+
s.name,
|
|
236
|
+
s.kind,
|
|
237
|
+
s.file_path,
|
|
238
|
+
0,
|
|
239
|
+
CAST(s.id AS TEXT)
|
|
240
|
+
FROM edges e
|
|
241
|
+
JOIN symbols s ON s.id = e.source_id
|
|
242
|
+
WHERE e.target_id = ?
|
|
243
|
+
|
|
244
|
+
UNION ALL
|
|
245
|
+
|
|
246
|
+
-- Recursive: dependents of already-visited nodes
|
|
247
|
+
SELECT s.id,
|
|
248
|
+
s.name,
|
|
249
|
+
s.kind,
|
|
250
|
+
s.file_path,
|
|
251
|
+
rc.depth + 1,
|
|
252
|
+
rc.path || ',' || CAST(s.id AS TEXT)
|
|
253
|
+
FROM rev_chain rc
|
|
254
|
+
JOIN edges e ON e.target_id = rc.id
|
|
255
|
+
JOIN symbols s ON s.id = e.source_id
|
|
256
|
+
WHERE rc.depth < ?
|
|
257
|
+
AND INSTR(rc.path, CAST(s.id AS TEXT)) = 0 -- cycle guard
|
|
258
|
+
)
|
|
259
|
+
SELECT DISTINCT id, name, kind, file_path, depth
|
|
260
|
+
FROM rev_chain
|
|
261
|
+
ORDER BY depth
|
|
262
|
+
"""
|
|
263
|
+
rows = conn.execute(query, (symbol_id, max_depth)).fetchall()
|
|
264
|
+
return [dict(r) for r in rows]
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
# ---------------------------------------------------------------------------
|
|
268
|
+
# Full-text search
|
|
269
|
+
# ---------------------------------------------------------------------------
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def fts_search(conn: sqlite3.Connection, query_text: str, limit: int = 10) -> list:
|
|
273
|
+
"""BM25-ranked full-text search over symbol names, signatures, and doc comments.
|
|
274
|
+
|
|
275
|
+
Returns up to *limit* symbol dicts with an additional 'rank' field.
|
|
276
|
+
Lower rank values indicate better matches (BM25 scores are negative in
|
|
277
|
+
SQLite's fts5 implementation).
|
|
278
|
+
"""
|
|
279
|
+
query = """
|
|
280
|
+
SELECT s.*, fts.rank
|
|
281
|
+
FROM symbols_fts fts
|
|
282
|
+
JOIN symbols s ON s.id = fts.rowid
|
|
283
|
+
WHERE symbols_fts MATCH ?
|
|
284
|
+
ORDER BY fts.rank
|
|
285
|
+
LIMIT ?
|
|
286
|
+
"""
|
|
287
|
+
rows = conn.execute(query, (query_text, limit)).fetchall()
|
|
288
|
+
return [dict(r) for r in rows]
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
# ---------------------------------------------------------------------------
|
|
292
|
+
# Statistics
|
|
293
|
+
# ---------------------------------------------------------------------------
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def get_stats(conn: sqlite3.Connection) -> dict:
|
|
297
|
+
"""Return high-level database statistics."""
|
|
298
|
+
symbols_count = conn.execute("SELECT COUNT(*) FROM symbols").fetchone()[0]
|
|
299
|
+
edges_count = conn.execute("SELECT COUNT(*) FROM edges").fetchone()[0]
|
|
300
|
+
files_count = conn.execute(
|
|
301
|
+
"SELECT COUNT(DISTINCT file_path) FROM symbols"
|
|
302
|
+
).fetchone()[0]
|
|
303
|
+
return {
|
|
304
|
+
"symbols": symbols_count,
|
|
305
|
+
"edges": edges_count,
|
|
306
|
+
"files": files_count,
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
# ---------------------------------------------------------------------------
|
|
311
|
+
# Utility helpers
|
|
312
|
+
# ---------------------------------------------------------------------------
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def hash_content(content: str) -> str:
|
|
316
|
+
"""Return a SHA-256 hex digest for *content*. Useful for change detection."""
|
|
317
|
+
return hashlib.sha256(content.encode("utf-8")).hexdigest()
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
# ---------------------------------------------------------------------------
|
|
321
|
+
# Smoke-test entrypoint
|
|
322
|
+
# ---------------------------------------------------------------------------
|
|
323
|
+
|
|
324
|
+
if __name__ == "__main__":
|
|
325
|
+
import tempfile
|
|
326
|
+
|
|
327
|
+
print("Running db.py smoke tests …")
|
|
328
|
+
|
|
329
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
330
|
+
conn = get_connection(tmp)
|
|
331
|
+
|
|
332
|
+
# ---- symbols ----
|
|
333
|
+
sid_a = add_symbol(
|
|
334
|
+
conn, "parse_file", "function", "src/parser.py", 10, 40,
|
|
335
|
+
signature="def parse_file(path: str) -> AST",
|
|
336
|
+
doc_comment="Parse a source file and return its AST.",
|
|
337
|
+
content_hash=hash_content("def parse_file(): pass"),
|
|
338
|
+
)
|
|
339
|
+
sid_b = add_symbol(
|
|
340
|
+
conn, "tokenize", "function", "src/lexer.py", 1, 20,
|
|
341
|
+
signature="def tokenize(src: str) -> list",
|
|
342
|
+
)
|
|
343
|
+
sid_c = add_symbol(
|
|
344
|
+
conn, "build_index", "function", "src/index.py", 5, 60,
|
|
345
|
+
signature="def build_index(root: str) -> None",
|
|
346
|
+
doc_comment="Build the code graph index for a project.",
|
|
347
|
+
)
|
|
348
|
+
|
|
349
|
+
assert get_symbol_by_id(conn, sid_a)["name"] == "parse_file", "get_symbol_by_id failed"
|
|
350
|
+
assert len(get_symbol_by_name(conn, "tokenize")) == 1, "get_symbol_by_name failed"
|
|
351
|
+
|
|
352
|
+
# ---- edges ----
|
|
353
|
+
# build_index → parse_file → tokenize
|
|
354
|
+
add_edge(conn, sid_c, sid_a, "calls")
|
|
355
|
+
add_edge(conn, sid_a, sid_b, "calls")
|
|
356
|
+
add_edge(conn, sid_c, sid_a, "calls") # duplicate — should be ignored
|
|
357
|
+
|
|
358
|
+
conn.commit()
|
|
359
|
+
|
|
360
|
+
# ---- transitive deps ----
|
|
361
|
+
deps = get_transitive_deps(conn, sid_c)
|
|
362
|
+
dep_ids = {d["id"] for d in deps}
|
|
363
|
+
assert sid_a in dep_ids and sid_b in dep_ids, f"transitive deps wrong: {dep_ids}"
|
|
364
|
+
|
|
365
|
+
# ---- reverse deps ----
|
|
366
|
+
rdeps = get_reverse_deps(conn, sid_b)
|
|
367
|
+
rdep_ids = {d["id"] for d in rdeps}
|
|
368
|
+
assert sid_a in rdep_ids and sid_c in rdep_ids, f"reverse deps wrong: {rdep_ids}"
|
|
369
|
+
|
|
370
|
+
# ---- FTS search ----
|
|
371
|
+
results = fts_search(conn, "parse")
|
|
372
|
+
assert any(r["name"] == "parse_file" for r in results), "FTS search failed"
|
|
373
|
+
|
|
374
|
+
results_doc = fts_search(conn, "index")
|
|
375
|
+
assert any(r["name"] == "build_index" for r in results_doc), "FTS doc_comment search failed"
|
|
376
|
+
|
|
377
|
+
# ---- remove by file ----
|
|
378
|
+
remove_symbols_by_file(conn, "src/lexer.py")
|
|
379
|
+
conn.commit()
|
|
380
|
+
assert get_symbol_by_id(conn, sid_b) is None, "remove_symbols_by_file failed"
|
|
381
|
+
|
|
382
|
+
# ---- stats ----
|
|
383
|
+
stats = get_stats(conn)
|
|
384
|
+
assert stats["symbols"] == 2, f"expected 2 symbols after removal, got {stats['symbols']}"
|
|
385
|
+
assert stats["files"] == 2, f"expected 2 files, got {stats['files']}"
|
|
386
|
+
# edge from parse_file → tokenize should be gone via CASCADE
|
|
387
|
+
edge_count = conn.execute("SELECT COUNT(*) FROM edges").fetchone()[0]
|
|
388
|
+
assert edge_count == 1, f"expected 1 edge after file removal, got {edge_count}"
|
|
389
|
+
|
|
390
|
+
print("All smoke tests passed.")
|
|
391
|
+
print(f"Stats: {get_stats(conn)}")
|