feed-the-machine 1.5.0 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (224) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +170 -170
  3. package/bin/generate-manifest.mjs +463 -463
  4. package/bin/install.mjs +491 -491
  5. package/docs/HOOKS.md +243 -243
  6. package/docs/INBOX.md +233 -233
  7. package/ftm/SKILL.md +122 -122
  8. package/ftm-audit/SKILL.md +623 -541
  9. package/ftm-audit/references/protocols/PROJECT-PATTERNS.md +91 -91
  10. package/ftm-audit/references/protocols/RUNTIME-WIRING.md +66 -66
  11. package/ftm-audit/references/protocols/WIRING-CONTRACTS.md +135 -135
  12. package/ftm-audit/references/strategies/AUTO-FIX-STRATEGIES.md +69 -69
  13. package/ftm-audit/references/templates/REPORT-FORMAT.md +96 -96
  14. package/ftm-audit/scripts/run-knip.sh +23 -23
  15. package/ftm-audit.yml +2 -2
  16. package/ftm-brainstorm/SKILL.md +498 -498
  17. package/ftm-brainstorm/evals/evals.json +100 -100
  18. package/ftm-brainstorm/evals/promptfoo.yaml +109 -109
  19. package/ftm-brainstorm/references/agent-prompts.md +224 -224
  20. package/ftm-brainstorm/references/plan-template.md +121 -121
  21. package/ftm-brainstorm.yml +2 -2
  22. package/ftm-browse/SKILL.md +454 -454
  23. package/ftm-browse/daemon/browser-manager.ts +206 -206
  24. package/ftm-browse/daemon/bun.lock +30 -30
  25. package/ftm-browse/daemon/cli.ts +347 -347
  26. package/ftm-browse/daemon/commands.ts +410 -410
  27. package/ftm-browse/daemon/main.ts +357 -357
  28. package/ftm-browse/daemon/package.json +17 -17
  29. package/ftm-browse/daemon/server.ts +189 -189
  30. package/ftm-browse/daemon/snapshot.ts +519 -519
  31. package/ftm-browse/daemon/tsconfig.json +22 -22
  32. package/ftm-browse.yml +4 -4
  33. package/ftm-capture/SKILL.md +370 -370
  34. package/ftm-capture.yml +4 -4
  35. package/ftm-codex-gate/SKILL.md +361 -361
  36. package/ftm-codex-gate.yml +2 -2
  37. package/ftm-config/SKILL.md +345 -345
  38. package/ftm-config.default.yml +82 -80
  39. package/ftm-config.yml +2 -2
  40. package/ftm-council/SKILL.md +416 -416
  41. package/ftm-council/references/prompts/CLAUDE-INVESTIGATION.md +60 -60
  42. package/ftm-council/references/prompts/CODEX-INVESTIGATION.md +58 -58
  43. package/ftm-council/references/prompts/GEMINI-INVESTIGATION.md +58 -58
  44. package/ftm-council/references/prompts/REBUTTAL-TEMPLATE.md +57 -57
  45. package/ftm-council/references/protocols/PREREQUISITES.md +47 -47
  46. package/ftm-council/references/protocols/STEP-0-FRAMING.md +46 -46
  47. package/ftm-council.yml +2 -2
  48. package/ftm-dashboard/SKILL.md +163 -163
  49. package/ftm-dashboard.yml +4 -4
  50. package/ftm-debug/SKILL.md +1037 -1037
  51. package/ftm-debug/references/phases/PHASE-0-INTAKE.md +58 -58
  52. package/ftm-debug/references/phases/PHASE-1-TRIAGE.md +46 -46
  53. package/ftm-debug/references/phases/PHASE-2-WAR-ROOM-AGENTS.md +279 -279
  54. package/ftm-debug/references/phases/PHASE-3-TO-6-EXECUTION.md +436 -436
  55. package/ftm-debug/references/protocols/BLACKBOARD.md +86 -86
  56. package/ftm-debug/references/protocols/EDGE-CASES.md +103 -103
  57. package/ftm-debug.yml +2 -2
  58. package/ftm-diagram/SKILL.md +277 -277
  59. package/ftm-diagram.yml +2 -2
  60. package/ftm-executor/SKILL.md +777 -767
  61. package/ftm-executor/references/STYLE-TEMPLATE.md +73 -73
  62. package/ftm-executor/references/phases/PHASE-0-VERIFICATION.md +62 -62
  63. package/ftm-executor/references/phases/PHASE-2-AGENT-ASSEMBLY.md +34 -34
  64. package/ftm-executor/references/phases/PHASE-3-WORKTREES.md +38 -38
  65. package/ftm-executor/references/phases/PHASE-4-5-AUDIT.md +72 -72
  66. package/ftm-executor/references/phases/PHASE-4-DISPATCH.md +66 -66
  67. package/ftm-executor/references/phases/PHASE-5-5-CODEX-GATE.md +73 -73
  68. package/ftm-executor/references/protocols/DOCUMENTATION-BOOTSTRAP.md +36 -36
  69. package/ftm-executor/references/protocols/MODEL-PROFILE.md +59 -44
  70. package/ftm-executor/references/protocols/PROGRESS-TRACKING.md +66 -66
  71. package/ftm-executor/runtime/ftm-runtime.mjs +252 -252
  72. package/ftm-executor/runtime/package.json +8 -8
  73. package/ftm-executor.yml +2 -2
  74. package/ftm-git/SKILL.md +441 -441
  75. package/ftm-git/evals/evals.json +26 -26
  76. package/ftm-git/evals/promptfoo.yaml +75 -75
  77. package/ftm-git/hooks/post-commit-experience.sh +92 -92
  78. package/ftm-git/references/patterns/SECRET-PATTERNS.md +104 -104
  79. package/ftm-git/references/protocols/REMEDIATION.md +139 -139
  80. package/ftm-git/scripts/pre-commit-secrets.sh +110 -110
  81. package/ftm-git.yml +2 -2
  82. package/ftm-inbox/backend/adapters/_retry.py +64 -64
  83. package/ftm-inbox/backend/adapters/base.py +230 -230
  84. package/ftm-inbox/backend/adapters/freshservice.py +104 -104
  85. package/ftm-inbox/backend/adapters/gmail.py +125 -125
  86. package/ftm-inbox/backend/adapters/jira.py +136 -136
  87. package/ftm-inbox/backend/adapters/registry.py +192 -192
  88. package/ftm-inbox/backend/adapters/slack.py +110 -110
  89. package/ftm-inbox/backend/db/connection.py +54 -54
  90. package/ftm-inbox/backend/db/schema.py +78 -78
  91. package/ftm-inbox/backend/executor/__init__.py +7 -7
  92. package/ftm-inbox/backend/executor/engine.py +149 -149
  93. package/ftm-inbox/backend/executor/step_runner.py +98 -98
  94. package/ftm-inbox/backend/main.py +103 -103
  95. package/ftm-inbox/backend/models/__init__.py +1 -1
  96. package/ftm-inbox/backend/models/unified_task.py +36 -36
  97. package/ftm-inbox/backend/planner/__init__.py +6 -6
  98. package/ftm-inbox/backend/planner/generator.py +127 -127
  99. package/ftm-inbox/backend/planner/schema.py +34 -34
  100. package/ftm-inbox/backend/requirements.txt +5 -5
  101. package/ftm-inbox/backend/routes/execute.py +186 -186
  102. package/ftm-inbox/backend/routes/health.py +52 -52
  103. package/ftm-inbox/backend/routes/inbox.py +68 -68
  104. package/ftm-inbox/backend/routes/plan.py +271 -271
  105. package/ftm-inbox/bin/launchagent.mjs +91 -91
  106. package/ftm-inbox/bin/setup.mjs +188 -188
  107. package/ftm-inbox/bin/start.sh +10 -10
  108. package/ftm-inbox/bin/status.sh +17 -17
  109. package/ftm-inbox/bin/stop.sh +8 -8
  110. package/ftm-inbox/config.example.yml +55 -55
  111. package/ftm-inbox/package-lock.json +2898 -2898
  112. package/ftm-inbox/package.json +26 -26
  113. package/ftm-inbox/postcss.config.js +6 -6
  114. package/ftm-inbox/src/app.css +199 -199
  115. package/ftm-inbox/src/app.html +18 -18
  116. package/ftm-inbox/src/lib/api.ts +166 -166
  117. package/ftm-inbox/src/lib/components/ExecutionLog.svelte +81 -81
  118. package/ftm-inbox/src/lib/components/InboxFeed.svelte +143 -143
  119. package/ftm-inbox/src/lib/components/PlanStep.svelte +271 -271
  120. package/ftm-inbox/src/lib/components/PlanView.svelte +206 -206
  121. package/ftm-inbox/src/lib/components/StreamPanel.svelte +99 -99
  122. package/ftm-inbox/src/lib/components/TaskCard.svelte +190 -190
  123. package/ftm-inbox/src/lib/components/ui/EmptyState.svelte +63 -63
  124. package/ftm-inbox/src/lib/components/ui/KawaiiCard.svelte +86 -86
  125. package/ftm-inbox/src/lib/components/ui/PillButton.svelte +106 -106
  126. package/ftm-inbox/src/lib/components/ui/StatusBadge.svelte +67 -67
  127. package/ftm-inbox/src/lib/components/ui/StreamDrawer.svelte +149 -149
  128. package/ftm-inbox/src/lib/components/ui/ThemeToggle.svelte +80 -80
  129. package/ftm-inbox/src/lib/theme.ts +47 -47
  130. package/ftm-inbox/src/routes/+layout.svelte +76 -76
  131. package/ftm-inbox/src/routes/+page.svelte +401 -401
  132. package/ftm-inbox/svelte.config.js +12 -12
  133. package/ftm-inbox/tailwind.config.ts +63 -63
  134. package/ftm-inbox/tsconfig.json +13 -13
  135. package/ftm-inbox/vite.config.ts +6 -6
  136. package/ftm-intent/SKILL.md +241 -241
  137. package/ftm-intent.yml +2 -2
  138. package/ftm-manifest.json +3794 -3794
  139. package/ftm-map/SKILL.md +291 -291
  140. package/ftm-map/scripts/db.py +712 -712
  141. package/ftm-map/scripts/index.py +415 -415
  142. package/ftm-map/scripts/parser.py +224 -224
  143. package/ftm-map/scripts/queries/go-tags.scm +20 -20
  144. package/ftm-map/scripts/queries/javascript-tags.scm +35 -35
  145. package/ftm-map/scripts/queries/python-tags.scm +31 -31
  146. package/ftm-map/scripts/queries/ruby-tags.scm +19 -19
  147. package/ftm-map/scripts/queries/rust-tags.scm +37 -37
  148. package/ftm-map/scripts/queries/typescript-tags.scm +41 -41
  149. package/ftm-map/scripts/query.py +301 -301
  150. package/ftm-map/scripts/ranker.py +377 -377
  151. package/ftm-map/scripts/requirements.txt +5 -5
  152. package/ftm-map/scripts/setup-hooks.sh +27 -27
  153. package/ftm-map/scripts/setup.sh +56 -56
  154. package/ftm-map/scripts/test_db.py +364 -364
  155. package/ftm-map/scripts/test_parser.py +174 -174
  156. package/ftm-map/scripts/test_query.py +183 -183
  157. package/ftm-map/scripts/test_ranker.py +199 -199
  158. package/ftm-map/scripts/views.py +591 -591
  159. package/ftm-map.yml +2 -2
  160. package/ftm-mind/SKILL.md +1943 -1943
  161. package/ftm-mind/evals/promptfoo.yaml +142 -142
  162. package/ftm-mind/references/blackboard-schema.md +328 -328
  163. package/ftm-mind/references/complexity-guide.md +110 -110
  164. package/ftm-mind/references/event-registry.md +319 -319
  165. package/ftm-mind/references/mcp-inventory.md +296 -296
  166. package/ftm-mind/references/protocols/COMPLEXITY-SIZING.md +72 -72
  167. package/ftm-mind/references/protocols/MCP-HEURISTICS.md +32 -32
  168. package/ftm-mind/references/protocols/PLAN-APPROVAL.md +80 -80
  169. package/ftm-mind/references/reflexion-protocol.md +249 -249
  170. package/ftm-mind/references/routing/SCENARIOS.md +22 -22
  171. package/ftm-mind/references/routing-scenarios.md +35 -35
  172. package/ftm-mind.yml +2 -2
  173. package/ftm-pause/SKILL.md +395 -395
  174. package/ftm-pause/references/protocols/SKILL-RESTORE-PROTOCOLS.md +186 -186
  175. package/ftm-pause/references/protocols/VALIDATION.md +80 -80
  176. package/ftm-pause.yml +2 -2
  177. package/ftm-researcher/SKILL.md +275 -275
  178. package/ftm-researcher/evals/agent-diversity.yaml +17 -17
  179. package/ftm-researcher/evals/synthesis-quality.yaml +12 -12
  180. package/ftm-researcher/evals/trigger-accuracy.yaml +39 -39
  181. package/ftm-researcher/references/adaptive-search.md +116 -116
  182. package/ftm-researcher/references/agent-prompts.md +193 -193
  183. package/ftm-researcher/references/council-integration.md +193 -193
  184. package/ftm-researcher/references/output-format.md +203 -203
  185. package/ftm-researcher/references/synthesis-pipeline.md +165 -165
  186. package/ftm-researcher/scripts/score_credibility.py +234 -234
  187. package/ftm-researcher/scripts/validate_research.py +92 -92
  188. package/ftm-researcher.yml +2 -2
  189. package/ftm-resume/SKILL.md +518 -518
  190. package/ftm-resume/references/protocols/VALIDATION.md +172 -172
  191. package/ftm-resume.yml +2 -2
  192. package/ftm-retro/SKILL.md +380 -380
  193. package/ftm-retro/references/protocols/SCORING-RUBRICS.md +89 -89
  194. package/ftm-retro/references/templates/REPORT-FORMAT.md +109 -109
  195. package/ftm-retro.yml +2 -2
  196. package/ftm-routine/SKILL.md +170 -170
  197. package/ftm-routine.yml +4 -4
  198. package/ftm-state/blackboard/capabilities.json +5 -5
  199. package/ftm-state/blackboard/capabilities.schema.json +27 -27
  200. package/ftm-state/blackboard/context.json +23 -23
  201. package/ftm-state/blackboard/experiences/index.json +9 -9
  202. package/ftm-state/blackboard/patterns.json +6 -6
  203. package/ftm-state/schemas/context.schema.json +130 -130
  204. package/ftm-state/schemas/experience-index.schema.json +77 -77
  205. package/ftm-state/schemas/experience.schema.json +78 -78
  206. package/ftm-state/schemas/patterns.schema.json +44 -44
  207. package/ftm-upgrade/SKILL.md +194 -194
  208. package/ftm-upgrade/scripts/check-version.sh +76 -76
  209. package/ftm-upgrade/scripts/upgrade.sh +143 -143
  210. package/ftm-upgrade.yml +2 -2
  211. package/ftm-verify.yml +2 -2
  212. package/ftm.yml +2 -2
  213. package/hooks/ftm-blackboard-enforcer.sh +93 -93
  214. package/hooks/ftm-discovery-reminder.sh +90 -90
  215. package/hooks/ftm-drafts-gate.sh +61 -61
  216. package/hooks/ftm-event-logger.mjs +107 -107
  217. package/hooks/ftm-map-autodetect.sh +79 -79
  218. package/hooks/ftm-pending-sync-check.sh +22 -22
  219. package/hooks/ftm-plan-gate.sh +92 -92
  220. package/hooks/ftm-post-commit-trigger.sh +57 -57
  221. package/hooks/settings-template.json +81 -81
  222. package/install.sh +363 -363
  223. package/package.json +84 -84
  224. package/uninstall.sh +25 -25
@@ -1,415 +1,415 @@
1
- #!/usr/bin/env python3
2
- """ftm-map indexer: builds the code knowledge graph from source files.
3
-
4
- Two-phase indexing:
5
- Phase 1 — Parse each file with tree-sitter, insert file/symbol/ref rows.
6
- Phase 2 — Materialize file_edges with Aider-style weight heuristics and
7
- symbol_edges via enclosing-scope resolution.
8
- """
9
-
10
- import argparse
11
- import json
12
- import math
13
- import os
14
- import re
15
- import subprocess
16
- import sys
17
- import time
18
- from collections import Counter
19
- from datetime import datetime, timezone
20
- from pathlib import Path
21
-
22
- # Add scripts dir to path for sibling imports
23
- sys.path.insert(0, os.path.dirname(__file__))
24
-
25
- from db import (
26
- get_connection,
27
- add_file,
28
- add_symbol,
29
- add_reference,
30
- remove_file,
31
- get_stats,
32
- rebuild_symbol_edges,
33
- )
34
- from parser import get_tags, detect_language, EXTENSION_MAP, compute_content_hash
35
-
36
- META_REGISTRY = os.path.expanduser("~/.claude/ftm-state/maps/index.json")
37
-
38
-
39
- # ---------------------------------------------------------------------------
40
- # File discovery
41
- # ---------------------------------------------------------------------------
42
-
43
-
44
- def discover_files(project_root: str) -> list[str]:
45
- """Get tracked source files using git ls-files.
46
-
47
- Filters to files whose extensions are in EXTENSION_MAP so only
48
- tree-sitter-parseable files are returned. Returns absolute paths.
49
- """
50
- result = subprocess.run(
51
- ["git", "ls-files"],
52
- capture_output=True,
53
- text=True,
54
- cwd=project_root,
55
- )
56
- if result.returncode != 0:
57
- print(
58
- f"Error: git ls-files failed: {result.stderr.strip()}",
59
- file=sys.stderr,
60
- )
61
- return []
62
-
63
- supported_exts = set(EXTENSION_MAP.keys())
64
- files = []
65
- for line in result.stdout.strip().split("\n"):
66
- if not line:
67
- continue
68
- ext = Path(line).suffix.lower()
69
- if ext in supported_exts:
70
- files.append(os.path.join(project_root, line))
71
- return files
72
-
73
-
74
- # ---------------------------------------------------------------------------
75
- # Core indexing logic
76
- # ---------------------------------------------------------------------------
77
-
78
-
79
- def index_files(conn, files: list[str], project_root: str) -> dict:
80
- """Parse and insert files, symbols, references, then materialize edges.
81
-
82
- Phase 1 — For each file: read source, compute hash, insert file row,
83
- extract def/ref tags via tree-sitter, insert symbol and ref rows.
84
- Phase 2 — Build file_edges with Aider weight heuristics (long descriptive
85
- names 10x, private 0.1x, overloaded 0.1x, sqrt-dampened counts).
86
- Then rebuild symbol_edges via enclosing-scope resolution.
87
-
88
- Returns a dict with symbols, references, file_edges, symbol_edges counts.
89
- """
90
- total_symbols = 0
91
- total_refs = 0
92
-
93
- # ------------------------------------------------------------------
94
- # Phase 1: parse each file and insert rows
95
- # ------------------------------------------------------------------
96
- for fpath in files:
97
- if not os.path.exists(fpath):
98
- print(f"Warning: file not found, skipping: {fpath}", file=sys.stderr)
99
- continue
100
-
101
- rel_path = os.path.relpath(fpath, project_root)
102
- lang = detect_language(fpath)
103
- mtime = os.path.getmtime(fpath)
104
-
105
- # Stream-friendly: read once, extract metadata, then release
106
- try:
107
- with open(fpath, "r", encoding="utf-8", errors="replace") as fh:
108
- source = fh.read()
109
- except (IOError, OSError) as exc:
110
- print(f"Warning: Cannot read {fpath}: {exc}", file=sys.stderr)
111
- continue
112
-
113
- line_count = source.count("\n") + 1
114
- content_hash = compute_content_hash(source)
115
-
116
- # Insert file record
117
- file_id = add_file(
118
- conn, rel_path, lang, mtime,
119
- hash=content_hash, line_count=line_count,
120
- )
121
-
122
- # Extract def/ref tags via tree-sitter
123
- tags = get_tags(fpath, rel_path)
124
-
125
- for tag in tags:
126
- if tag.kind == "def":
127
- add_symbol(conn, file_id, tag.name, "definition", tag.line, signature=None)
128
- total_symbols += 1
129
- elif tag.kind == "ref":
130
- add_reference(conn, file_id, tag.name, tag.line, kind="call")
131
- total_refs += 1
132
-
133
- # ------------------------------------------------------------------
134
- # Phase 2: materialize edges
135
- # ------------------------------------------------------------------
136
-
137
- # Build defines map: ident -> set of file_ids that define it
138
- defines = {}
139
- for row in conn.execute("SELECT name, file_id FROM symbols").fetchall():
140
- defines.setdefault(row["name"], set()).add(row["file_id"])
141
-
142
- # Build references map: ident -> list of file_ids that reference it
143
- references_map = {}
144
- for row in conn.execute("SELECT symbol_name, file_id FROM refs").fetchall():
145
- references_map.setdefault(row["symbol_name"], []).append(row["file_id"])
146
-
147
- # Materialize file_edges with Aider weight heuristics
148
- conn.execute("DELETE FROM file_edges")
149
-
150
- for ident, ref_file_ids in references_map.items():
151
- definers = defines.get(ident, set())
152
- if not definers:
153
- continue
154
-
155
- # Aider weight heuristics
156
- mul = 1.0
157
- # Long descriptive names (camelCase or snake_case, >= 8 chars) weighted higher
158
- if len(ident) >= 8 and re.match(r"[a-z_]+[A-Z]|[a-z]+_[a-z]", ident):
159
- mul *= 10
160
- # Private names weighted lower
161
- if ident.startswith("_"):
162
- mul *= 0.1
163
- # Overloaded names (defined in many files) weighted lower
164
- if len(definers) >= 5:
165
- mul *= 0.1
166
-
167
- # Count refs per file, then create weighted edges
168
- ref_counts = Counter(ref_file_ids)
169
-
170
- for ref_file_id, count in ref_counts.items():
171
- weight = mul * math.sqrt(count)
172
- for def_file_id in definers:
173
- if ref_file_id != def_file_id: # No self-edges
174
- conn.execute(
175
- """INSERT INTO file_edges (source_file_id, target_file_id, weight)
176
- VALUES (?, ?, ?)
177
- ON CONFLICT(source_file_id, target_file_id)
178
- DO UPDATE SET weight = MAX(weight, excluded.weight)""",
179
- (ref_file_id, def_file_id, weight),
180
- )
181
-
182
- # Materialize symbol_edges via enclosing-scope resolution
183
- rebuild_symbol_edges(conn)
184
-
185
- # Gather edge counts
186
- file_edge_count = conn.execute("SELECT COUNT(*) FROM file_edges").fetchone()[0]
187
- symbol_edge_count = conn.execute("SELECT COUNT(*) FROM symbol_edges").fetchone()[0]
188
-
189
- return {
190
- "symbols": total_symbols,
191
- "references": total_refs,
192
- "file_edges": file_edge_count,
193
- "symbol_edges": symbol_edge_count,
194
- }
195
-
196
-
197
- # ---------------------------------------------------------------------------
198
- # Bootstrap mode
199
- # ---------------------------------------------------------------------------
200
-
201
-
202
- def bootstrap(project_root: str) -> None:
203
- """Full scan: (re)build the entire code graph for *project_root*."""
204
- abs_root = os.path.abspath(project_root)
205
- start = time.time()
206
-
207
- files = discover_files(abs_root)
208
- if not files:
209
- print(
210
- json.dumps({"error": "No parseable source files found in git repository"}),
211
- file=sys.stderr,
212
- )
213
- sys.exit(1)
214
-
215
- conn = get_connection(abs_root)
216
- try:
217
- # Full rebuild — clear all tables. CASCADE handles symbols, refs, edges.
218
- # FTS5 rows must be removed before symbol rows (content= table).
219
- symbol_ids = [
220
- row[0] for row in conn.execute("SELECT id FROM symbols").fetchall()
221
- ]
222
- for sid in symbol_ids:
223
- conn.execute("DELETE FROM symbols_fts WHERE rowid=?", (sid,))
224
- conn.execute("DELETE FROM files")
225
-
226
- stats = index_files(conn, files, abs_root)
227
- conn.commit()
228
-
229
- duration = time.time() - start
230
- result = {
231
- "mode": "bootstrap",
232
- "files_parsed": len(files),
233
- "symbols": stats["symbols"],
234
- "references": stats["references"],
235
- "file_edges": stats["file_edges"],
236
- "symbol_edges": stats["symbol_edges"],
237
- "duration_s": round(duration, 2),
238
- }
239
- print(json.dumps(result))
240
- update_meta_registry(abs_root, stats["symbols"])
241
- except Exception as exc: # noqa: BLE001
242
- print(f"Error during bootstrap: {exc}", file=sys.stderr)
243
- conn.rollback()
244
- conn.close()
245
- sys.exit(1)
246
- finally:
247
- conn.close()
248
-
249
-
250
- # ---------------------------------------------------------------------------
251
- # Incremental mode
252
- # ---------------------------------------------------------------------------
253
-
254
-
255
- def incremental(project_root: str, files_str: str) -> None:
256
- """Incremental update: re-index only the specified files.
257
-
258
- *files_str* is a comma-separated list of file paths (relative or absolute).
259
- Old file/symbol/ref/edge data for each file is cascade-deleted via
260
- remove_file() before re-parsing so stale entries do not accumulate.
261
- All edges are rebuilt since changes can ripple across files.
262
- """
263
- abs_root = os.path.abspath(project_root)
264
- start = time.time()
265
-
266
- raw_files = [f.strip() for f in files_str.split(",") if f.strip()]
267
- abs_files = [
268
- f if os.path.isabs(f) else os.path.join(abs_root, f) for f in raw_files
269
- ]
270
-
271
- conn = get_connection(abs_root)
272
- try:
273
- # Remove stale data for all targeted files (cascading delete).
274
- for fpath in abs_files:
275
- rel_path = os.path.relpath(fpath, abs_root)
276
- remove_file(conn, rel_path)
277
-
278
- existing_files = [f for f in abs_files if os.path.exists(f)]
279
- if not existing_files:
280
- print(
281
- json.dumps({"error": "None of the specified files exist"}),
282
- file=sys.stderr,
283
- )
284
- conn.close()
285
- sys.exit(1)
286
-
287
- stats = index_files(conn, existing_files, abs_root)
288
- conn.commit()
289
-
290
- db_stats = get_stats(conn)
291
- duration = time.time() - start
292
- result = {
293
- "mode": "incremental",
294
- "files_parsed": len(existing_files),
295
- "symbols": stats["symbols"],
296
- "references": stats["references"],
297
- "file_edges": stats["file_edges"],
298
- "symbol_edges": stats["symbol_edges"],
299
- "duration_s": round(duration, 2),
300
- }
301
- print(json.dumps(result))
302
- update_meta_registry(abs_root, db_stats["symbol_count"])
303
- except Exception as exc: # noqa: BLE001
304
- print(f"Error during incremental update: {exc}", file=sys.stderr)
305
- conn.rollback()
306
- conn.close()
307
- sys.exit(1)
308
- finally:
309
- conn.close()
310
-
311
-
312
- # ---------------------------------------------------------------------------
313
- # Meta-registry management
314
- # ---------------------------------------------------------------------------
315
-
316
-
317
- def update_meta_registry(project_root: str, symbol_count: int) -> None:
318
- """Upsert project entry in the global meta-registry at META_REGISTRY."""
319
- registry_dir = os.path.dirname(META_REGISTRY)
320
- os.makedirs(registry_dir, exist_ok=True)
321
-
322
- registry: dict = {"projects": []}
323
- if os.path.exists(META_REGISTRY):
324
- try:
325
- with open(META_REGISTRY) as fh:
326
- registry = json.load(fh)
327
- except (json.JSONDecodeError, IOError):
328
- # Corrupt or unreadable registry — start fresh.
329
- registry = {"projects": []}
330
-
331
- abs_root = os.path.abspath(project_root)
332
- db_path = os.path.join(abs_root, ".ftm-map", "map.db")
333
- now = datetime.now(timezone.utc).isoformat()
334
-
335
- found = False
336
- for proj in registry["projects"]:
337
- if proj.get("path") == abs_root:
338
- proj["last_indexed"] = now
339
- proj["symbol_count"] = symbol_count
340
- found = True
341
- break
342
-
343
- if not found:
344
- registry["projects"].append(
345
- {
346
- "path": abs_root,
347
- "db_path": db_path,
348
- "last_indexed": now,
349
- "symbol_count": symbol_count,
350
- }
351
- )
352
-
353
- with open(META_REGISTRY, "w") as fh:
354
- json.dump(registry, fh, indent=2)
355
-
356
-
357
- # ---------------------------------------------------------------------------
358
- # CLI entry point
359
- # ---------------------------------------------------------------------------
360
-
361
-
362
- def main() -> None:
363
- parser = argparse.ArgumentParser(
364
- description="ftm-map indexer — builds the code knowledge graph from source files.",
365
- formatter_class=argparse.RawDescriptionHelpFormatter,
366
- epilog=(
367
- "Examples:\n"
368
- " python3 index.py --bootstrap /path/to/project\n"
369
- " python3 index.py --incremental --files src/foo.ts,src/bar.py\n"
370
- " python3 index.py --incremental --files src/foo.ts --project-root /path/to/project\n"
371
- ),
372
- )
373
-
374
- mode = parser.add_mutually_exclusive_group(required=True)
375
- mode.add_argument(
376
- "--bootstrap",
377
- metavar="PROJECT_ROOT",
378
- help="Full scan: index all tracked source files in PROJECT_ROOT.",
379
- )
380
- mode.add_argument(
381
- "--incremental",
382
- action="store_true",
383
- help="Incremental update: re-index only the files given by --files.",
384
- )
385
-
386
- parser.add_argument(
387
- "--files",
388
- metavar="FILE_LIST",
389
- help="Comma-separated list of files to re-index (required for --incremental).",
390
- )
391
- parser.add_argument(
392
- "--project-root",
393
- metavar="PATH",
394
- default=None,
395
- help=(
396
- "Project root used to locate the database for incremental mode. "
397
- "Defaults to the current working directory."
398
- ),
399
- )
400
-
401
- args = parser.parse_args()
402
-
403
- if args.bootstrap:
404
- bootstrap(args.bootstrap)
405
- else:
406
- # Incremental mode
407
- if not args.files:
408
- print("Error: --incremental requires --files", file=sys.stderr)
409
- sys.exit(1)
410
- project_root = args.project_root or os.getcwd()
411
- incremental(project_root, args.files)
412
-
413
-
414
- if __name__ == "__main__":
415
- main()
1
+ #!/usr/bin/env python3
2
+ """ftm-map indexer: builds the code knowledge graph from source files.
3
+
4
+ Two-phase indexing:
5
+ Phase 1 — Parse each file with tree-sitter, insert file/symbol/ref rows.
6
+ Phase 2 — Materialize file_edges with Aider-style weight heuristics and
7
+ symbol_edges via enclosing-scope resolution.
8
+ """
9
+
10
+ import argparse
11
+ import json
12
+ import math
13
+ import os
14
+ import re
15
+ import subprocess
16
+ import sys
17
+ import time
18
+ from collections import Counter
19
+ from datetime import datetime, timezone
20
+ from pathlib import Path
21
+
22
+ # Add scripts dir to path for sibling imports
23
+ sys.path.insert(0, os.path.dirname(__file__))
24
+
25
+ from db import (
26
+ get_connection,
27
+ add_file,
28
+ add_symbol,
29
+ add_reference,
30
+ remove_file,
31
+ get_stats,
32
+ rebuild_symbol_edges,
33
+ )
34
+ from parser import get_tags, detect_language, EXTENSION_MAP, compute_content_hash
35
+
36
+ META_REGISTRY = os.path.expanduser("~/.claude/ftm-state/maps/index.json")
37
+
38
+
39
+ # ---------------------------------------------------------------------------
40
+ # File discovery
41
+ # ---------------------------------------------------------------------------
42
+
43
+
44
+ def discover_files(project_root: str) -> list[str]:
45
+ """Get tracked source files using git ls-files.
46
+
47
+ Filters to files whose extensions are in EXTENSION_MAP so only
48
+ tree-sitter-parseable files are returned. Returns absolute paths.
49
+ """
50
+ result = subprocess.run(
51
+ ["git", "ls-files"],
52
+ capture_output=True,
53
+ text=True,
54
+ cwd=project_root,
55
+ )
56
+ if result.returncode != 0:
57
+ print(
58
+ f"Error: git ls-files failed: {result.stderr.strip()}",
59
+ file=sys.stderr,
60
+ )
61
+ return []
62
+
63
+ supported_exts = set(EXTENSION_MAP.keys())
64
+ files = []
65
+ for line in result.stdout.strip().split("\n"):
66
+ if not line:
67
+ continue
68
+ ext = Path(line).suffix.lower()
69
+ if ext in supported_exts:
70
+ files.append(os.path.join(project_root, line))
71
+ return files
72
+
73
+
74
+ # ---------------------------------------------------------------------------
75
+ # Core indexing logic
76
+ # ---------------------------------------------------------------------------
77
+
78
+
79
+ def index_files(conn, files: list[str], project_root: str) -> dict:
80
+ """Parse and insert files, symbols, references, then materialize edges.
81
+
82
+ Phase 1 — For each file: read source, compute hash, insert file row,
83
+ extract def/ref tags via tree-sitter, insert symbol and ref rows.
84
+ Phase 2 — Build file_edges with Aider weight heuristics (long descriptive
85
+ names 10x, private 0.1x, overloaded 0.1x, sqrt-dampened counts).
86
+ Then rebuild symbol_edges via enclosing-scope resolution.
87
+
88
+ Returns a dict with symbols, references, file_edges, symbol_edges counts.
89
+ """
90
+ total_symbols = 0
91
+ total_refs = 0
92
+
93
+ # ------------------------------------------------------------------
94
+ # Phase 1: parse each file and insert rows
95
+ # ------------------------------------------------------------------
96
+ for fpath in files:
97
+ if not os.path.exists(fpath):
98
+ print(f"Warning: file not found, skipping: {fpath}", file=sys.stderr)
99
+ continue
100
+
101
+ rel_path = os.path.relpath(fpath, project_root)
102
+ lang = detect_language(fpath)
103
+ mtime = os.path.getmtime(fpath)
104
+
105
+ # Stream-friendly: read once, extract metadata, then release
106
+ try:
107
+ with open(fpath, "r", encoding="utf-8", errors="replace") as fh:
108
+ source = fh.read()
109
+ except (IOError, OSError) as exc:
110
+ print(f"Warning: Cannot read {fpath}: {exc}", file=sys.stderr)
111
+ continue
112
+
113
+ line_count = source.count("\n") + 1
114
+ content_hash = compute_content_hash(source)
115
+
116
+ # Insert file record
117
+ file_id = add_file(
118
+ conn, rel_path, lang, mtime,
119
+ hash=content_hash, line_count=line_count,
120
+ )
121
+
122
+ # Extract def/ref tags via tree-sitter
123
+ tags = get_tags(fpath, rel_path)
124
+
125
+ for tag in tags:
126
+ if tag.kind == "def":
127
+ add_symbol(conn, file_id, tag.name, "definition", tag.line, signature=None)
128
+ total_symbols += 1
129
+ elif tag.kind == "ref":
130
+ add_reference(conn, file_id, tag.name, tag.line, kind="call")
131
+ total_refs += 1
132
+
133
+ # ------------------------------------------------------------------
134
+ # Phase 2: materialize edges
135
+ # ------------------------------------------------------------------
136
+
137
+ # Build defines map: ident -> set of file_ids that define it
138
+ defines = {}
139
+ for row in conn.execute("SELECT name, file_id FROM symbols").fetchall():
140
+ defines.setdefault(row["name"], set()).add(row["file_id"])
141
+
142
+ # Build references map: ident -> list of file_ids that reference it
143
+ references_map = {}
144
+ for row in conn.execute("SELECT symbol_name, file_id FROM refs").fetchall():
145
+ references_map.setdefault(row["symbol_name"], []).append(row["file_id"])
146
+
147
+ # Materialize file_edges with Aider weight heuristics
148
+ conn.execute("DELETE FROM file_edges")
149
+
150
+ for ident, ref_file_ids in references_map.items():
151
+ definers = defines.get(ident, set())
152
+ if not definers:
153
+ continue
154
+
155
+ # Aider weight heuristics
156
+ mul = 1.0
157
+ # Long descriptive names (camelCase or snake_case, >= 8 chars) weighted higher
158
+ if len(ident) >= 8 and re.match(r"[a-z_]+[A-Z]|[a-z]+_[a-z]", ident):
159
+ mul *= 10
160
+ # Private names weighted lower
161
+ if ident.startswith("_"):
162
+ mul *= 0.1
163
+ # Overloaded names (defined in many files) weighted lower
164
+ if len(definers) >= 5:
165
+ mul *= 0.1
166
+
167
+ # Count refs per file, then create weighted edges
168
+ ref_counts = Counter(ref_file_ids)
169
+
170
+ for ref_file_id, count in ref_counts.items():
171
+ weight = mul * math.sqrt(count)
172
+ for def_file_id in definers:
173
+ if ref_file_id != def_file_id: # No self-edges
174
+ conn.execute(
175
+ """INSERT INTO file_edges (source_file_id, target_file_id, weight)
176
+ VALUES (?, ?, ?)
177
+ ON CONFLICT(source_file_id, target_file_id)
178
+ DO UPDATE SET weight = MAX(weight, excluded.weight)""",
179
+ (ref_file_id, def_file_id, weight),
180
+ )
181
+
182
+ # Materialize symbol_edges via enclosing-scope resolution
183
+ rebuild_symbol_edges(conn)
184
+
185
+ # Gather edge counts
186
+ file_edge_count = conn.execute("SELECT COUNT(*) FROM file_edges").fetchone()[0]
187
+ symbol_edge_count = conn.execute("SELECT COUNT(*) FROM symbol_edges").fetchone()[0]
188
+
189
+ return {
190
+ "symbols": total_symbols,
191
+ "references": total_refs,
192
+ "file_edges": file_edge_count,
193
+ "symbol_edges": symbol_edge_count,
194
+ }
195
+
196
+
197
+ # ---------------------------------------------------------------------------
198
+ # Bootstrap mode
199
+ # ---------------------------------------------------------------------------
200
+
201
+
202
+ def bootstrap(project_root: str) -> None:
203
+ """Full scan: (re)build the entire code graph for *project_root*."""
204
+ abs_root = os.path.abspath(project_root)
205
+ start = time.time()
206
+
207
+ files = discover_files(abs_root)
208
+ if not files:
209
+ print(
210
+ json.dumps({"error": "No parseable source files found in git repository"}),
211
+ file=sys.stderr,
212
+ )
213
+ sys.exit(1)
214
+
215
+ conn = get_connection(abs_root)
216
+ try:
217
+ # Full rebuild — clear all tables. CASCADE handles symbols, refs, edges.
218
+ # FTS5 rows must be removed before symbol rows (content= table).
219
+ symbol_ids = [
220
+ row[0] for row in conn.execute("SELECT id FROM symbols").fetchall()
221
+ ]
222
+ for sid in symbol_ids:
223
+ conn.execute("DELETE FROM symbols_fts WHERE rowid=?", (sid,))
224
+ conn.execute("DELETE FROM files")
225
+
226
+ stats = index_files(conn, files, abs_root)
227
+ conn.commit()
228
+
229
+ duration = time.time() - start
230
+ result = {
231
+ "mode": "bootstrap",
232
+ "files_parsed": len(files),
233
+ "symbols": stats["symbols"],
234
+ "references": stats["references"],
235
+ "file_edges": stats["file_edges"],
236
+ "symbol_edges": stats["symbol_edges"],
237
+ "duration_s": round(duration, 2),
238
+ }
239
+ print(json.dumps(result))
240
+ update_meta_registry(abs_root, stats["symbols"])
241
+ except Exception as exc: # noqa: BLE001
242
+ print(f"Error during bootstrap: {exc}", file=sys.stderr)
243
+ conn.rollback()
244
+ conn.close()
245
+ sys.exit(1)
246
+ finally:
247
+ conn.close()
248
+
249
+
250
+ # ---------------------------------------------------------------------------
251
+ # Incremental mode
252
+ # ---------------------------------------------------------------------------
253
+
254
+
255
+ def incremental(project_root: str, files_str: str) -> None:
256
+ """Incremental update: re-index only the specified files.
257
+
258
+ *files_str* is a comma-separated list of file paths (relative or absolute).
259
+ Old file/symbol/ref/edge data for each file is cascade-deleted via
260
+ remove_file() before re-parsing so stale entries do not accumulate.
261
+ All edges are rebuilt since changes can ripple across files.
262
+ """
263
+ abs_root = os.path.abspath(project_root)
264
+ start = time.time()
265
+
266
+ raw_files = [f.strip() for f in files_str.split(",") if f.strip()]
267
+ abs_files = [
268
+ f if os.path.isabs(f) else os.path.join(abs_root, f) for f in raw_files
269
+ ]
270
+
271
+ conn = get_connection(abs_root)
272
+ try:
273
+ # Remove stale data for all targeted files (cascading delete).
274
+ for fpath in abs_files:
275
+ rel_path = os.path.relpath(fpath, abs_root)
276
+ remove_file(conn, rel_path)
277
+
278
+ existing_files = [f for f in abs_files if os.path.exists(f)]
279
+ if not existing_files:
280
+ print(
281
+ json.dumps({"error": "None of the specified files exist"}),
282
+ file=sys.stderr,
283
+ )
284
+ conn.close()
285
+ sys.exit(1)
286
+
287
+ stats = index_files(conn, existing_files, abs_root)
288
+ conn.commit()
289
+
290
+ db_stats = get_stats(conn)
291
+ duration = time.time() - start
292
+ result = {
293
+ "mode": "incremental",
294
+ "files_parsed": len(existing_files),
295
+ "symbols": stats["symbols"],
296
+ "references": stats["references"],
297
+ "file_edges": stats["file_edges"],
298
+ "symbol_edges": stats["symbol_edges"],
299
+ "duration_s": round(duration, 2),
300
+ }
301
+ print(json.dumps(result))
302
+ update_meta_registry(abs_root, db_stats["symbol_count"])
303
+ except Exception as exc: # noqa: BLE001
304
+ print(f"Error during incremental update: {exc}", file=sys.stderr)
305
+ conn.rollback()
306
+ conn.close()
307
+ sys.exit(1)
308
+ finally:
309
+ conn.close()
310
+
311
+
312
+ # ---------------------------------------------------------------------------
313
+ # Meta-registry management
314
+ # ---------------------------------------------------------------------------
315
+
316
+
317
+ def update_meta_registry(project_root: str, symbol_count: int) -> None:
318
+ """Upsert project entry in the global meta-registry at META_REGISTRY."""
319
+ registry_dir = os.path.dirname(META_REGISTRY)
320
+ os.makedirs(registry_dir, exist_ok=True)
321
+
322
+ registry: dict = {"projects": []}
323
+ if os.path.exists(META_REGISTRY):
324
+ try:
325
+ with open(META_REGISTRY) as fh:
326
+ registry = json.load(fh)
327
+ except (json.JSONDecodeError, IOError):
328
+ # Corrupt or unreadable registry — start fresh.
329
+ registry = {"projects": []}
330
+
331
+ abs_root = os.path.abspath(project_root)
332
+ db_path = os.path.join(abs_root, ".ftm-map", "map.db")
333
+ now = datetime.now(timezone.utc).isoformat()
334
+
335
+ found = False
336
+ for proj in registry["projects"]:
337
+ if proj.get("path") == abs_root:
338
+ proj["last_indexed"] = now
339
+ proj["symbol_count"] = symbol_count
340
+ found = True
341
+ break
342
+
343
+ if not found:
344
+ registry["projects"].append(
345
+ {
346
+ "path": abs_root,
347
+ "db_path": db_path,
348
+ "last_indexed": now,
349
+ "symbol_count": symbol_count,
350
+ }
351
+ )
352
+
353
+ with open(META_REGISTRY, "w") as fh:
354
+ json.dump(registry, fh, indent=2)
355
+
356
+
357
+ # ---------------------------------------------------------------------------
358
+ # CLI entry point
359
+ # ---------------------------------------------------------------------------
360
+
361
+
362
+ def main() -> None:
363
+ parser = argparse.ArgumentParser(
364
+ description="ftm-map indexer — builds the code knowledge graph from source files.",
365
+ formatter_class=argparse.RawDescriptionHelpFormatter,
366
+ epilog=(
367
+ "Examples:\n"
368
+ " python3 index.py --bootstrap /path/to/project\n"
369
+ " python3 index.py --incremental --files src/foo.ts,src/bar.py\n"
370
+ " python3 index.py --incremental --files src/foo.ts --project-root /path/to/project\n"
371
+ ),
372
+ )
373
+
374
+ mode = parser.add_mutually_exclusive_group(required=True)
375
+ mode.add_argument(
376
+ "--bootstrap",
377
+ metavar="PROJECT_ROOT",
378
+ help="Full scan: index all tracked source files in PROJECT_ROOT.",
379
+ )
380
+ mode.add_argument(
381
+ "--incremental",
382
+ action="store_true",
383
+ help="Incremental update: re-index only the files given by --files.",
384
+ )
385
+
386
+ parser.add_argument(
387
+ "--files",
388
+ metavar="FILE_LIST",
389
+ help="Comma-separated list of files to re-index (required for --incremental).",
390
+ )
391
+ parser.add_argument(
392
+ "--project-root",
393
+ metavar="PATH",
394
+ default=None,
395
+ help=(
396
+ "Project root used to locate the database for incremental mode. "
397
+ "Defaults to the current working directory."
398
+ ),
399
+ )
400
+
401
+ args = parser.parse_args()
402
+
403
+ if args.bootstrap:
404
+ bootstrap(args.bootstrap)
405
+ else:
406
+ # Incremental mode
407
+ if not args.files:
408
+ print("Error: --incremental requires --files", file=sys.stderr)
409
+ sys.exit(1)
410
+ project_root = args.project_root or os.getcwd()
411
+ incremental(project_root, args.files)
412
+
413
+
414
+ if __name__ == "__main__":
415
+ main()