feed-the-machine 1.6.1 → 1.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (269) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +170 -170
  3. package/bin/brain.py +1340 -0
  4. package/bin/convert_claude_skills_to_codex.py +490 -0
  5. package/bin/generate-manifest.mjs +463 -463
  6. package/bin/harden_codex_skills.py +141 -0
  7. package/bin/install.mjs +491 -491
  8. package/bin/migrate-eng-buddy-data.py +875 -0
  9. package/bin/playbook_engine/__init__.py +1 -0
  10. package/bin/playbook_engine/conftest.py +8 -0
  11. package/bin/playbook_engine/extractor.py +33 -0
  12. package/bin/playbook_engine/manager.py +102 -0
  13. package/bin/playbook_engine/models.py +84 -0
  14. package/bin/playbook_engine/registry.py +35 -0
  15. package/bin/playbook_engine/test_extractor.py +72 -0
  16. package/bin/playbook_engine/test_integration.py +129 -0
  17. package/bin/playbook_engine/test_manager.py +85 -0
  18. package/bin/playbook_engine/test_models.py +166 -0
  19. package/bin/playbook_engine/test_registry.py +67 -0
  20. package/bin/playbook_engine/test_tracer.py +86 -0
  21. package/bin/playbook_engine/tracer.py +93 -0
  22. package/bin/tasks_db.py +456 -0
  23. package/docs/HOOKS.md +243 -243
  24. package/docs/INBOX.md +233 -233
  25. package/ftm/SKILL.md +125 -122
  26. package/ftm-audit/SKILL.md +623 -623
  27. package/ftm-audit/references/protocols/PROJECT-PATTERNS.md +91 -91
  28. package/ftm-audit/references/protocols/RUNTIME-WIRING.md +66 -66
  29. package/ftm-audit/references/protocols/WIRING-CONTRACTS.md +135 -135
  30. package/ftm-audit/references/strategies/AUTO-FIX-STRATEGIES.md +69 -69
  31. package/ftm-audit/references/templates/REPORT-FORMAT.md +96 -96
  32. package/ftm-audit/scripts/run-knip.sh +23 -23
  33. package/ftm-audit.yml +2 -2
  34. package/ftm-brainstorm/SKILL.md +1003 -498
  35. package/ftm-brainstorm/evals/evals.json +180 -100
  36. package/ftm-brainstorm/evals/promptfoo.yaml +109 -109
  37. package/ftm-brainstorm/references/agent-prompts.md +552 -224
  38. package/ftm-brainstorm/references/plan-template.md +209 -121
  39. package/ftm-brainstorm.yml +2 -2
  40. package/ftm-browse/SKILL.md +454 -454
  41. package/ftm-browse/daemon/browser-manager.ts +206 -206
  42. package/ftm-browse/daemon/bun.lock +30 -30
  43. package/ftm-browse/daemon/cli.ts +347 -347
  44. package/ftm-browse/daemon/commands.ts +410 -410
  45. package/ftm-browse/daemon/main.ts +357 -357
  46. package/ftm-browse/daemon/package.json +17 -17
  47. package/ftm-browse/daemon/server.ts +189 -189
  48. package/ftm-browse/daemon/snapshot.ts +519 -519
  49. package/ftm-browse/daemon/tsconfig.json +22 -22
  50. package/ftm-browse.yml +4 -4
  51. package/ftm-capture/SKILL.md +370 -370
  52. package/ftm-capture.yml +4 -4
  53. package/ftm-codex-gate/SKILL.md +361 -361
  54. package/ftm-codex-gate.yml +2 -2
  55. package/ftm-config/SKILL.md +422 -345
  56. package/ftm-config.default.yml +125 -82
  57. package/ftm-config.yml +44 -2
  58. package/ftm-council/SKILL.md +416 -416
  59. package/ftm-council/references/prompts/CLAUDE-INVESTIGATION.md +60 -60
  60. package/ftm-council/references/prompts/CODEX-INVESTIGATION.md +58 -58
  61. package/ftm-council/references/prompts/GEMINI-INVESTIGATION.md +58 -58
  62. package/ftm-council/references/prompts/REBUTTAL-TEMPLATE.md +57 -57
  63. package/ftm-council/references/protocols/PREREQUISITES.md +47 -47
  64. package/ftm-council/references/protocols/STEP-0-FRAMING.md +46 -46
  65. package/ftm-council.yml +2 -2
  66. package/ftm-dashboard/SKILL.md +163 -163
  67. package/ftm-dashboard.yml +4 -4
  68. package/ftm-debug/SKILL.md +1037 -1037
  69. package/ftm-debug/references/phases/PHASE-0-INTAKE.md +58 -58
  70. package/ftm-debug/references/phases/PHASE-1-TRIAGE.md +46 -46
  71. package/ftm-debug/references/phases/PHASE-2-WAR-ROOM-AGENTS.md +279 -279
  72. package/ftm-debug/references/phases/PHASE-3-TO-6-EXECUTION.md +436 -436
  73. package/ftm-debug/references/protocols/BLACKBOARD.md +86 -86
  74. package/ftm-debug/references/protocols/EDGE-CASES.md +103 -103
  75. package/ftm-debug.yml +2 -2
  76. package/ftm-diagram/SKILL.md +277 -277
  77. package/ftm-diagram.yml +2 -2
  78. package/ftm-executor/SKILL.md +777 -777
  79. package/ftm-executor/references/STYLE-TEMPLATE.md +73 -73
  80. package/ftm-executor/references/phases/PHASE-0-VERIFICATION.md +62 -62
  81. package/ftm-executor/references/phases/PHASE-2-AGENT-ASSEMBLY.md +34 -34
  82. package/ftm-executor/references/phases/PHASE-3-WORKTREES.md +38 -38
  83. package/ftm-executor/references/phases/PHASE-4-5-AUDIT.md +72 -72
  84. package/ftm-executor/references/phases/PHASE-4-DISPATCH.md +66 -66
  85. package/ftm-executor/references/phases/PHASE-5-5-CODEX-GATE.md +73 -73
  86. package/ftm-executor/references/protocols/DOCUMENTATION-BOOTSTRAP.md +36 -36
  87. package/ftm-executor/references/protocols/MODEL-PROFILE.md +59 -59
  88. package/ftm-executor/references/protocols/PROGRESS-TRACKING.md +66 -66
  89. package/ftm-executor/runtime/ftm-runtime.mjs +252 -252
  90. package/ftm-executor/runtime/package.json +8 -8
  91. package/ftm-executor.yml +2 -2
  92. package/ftm-git/SKILL.md +441 -441
  93. package/ftm-git/evals/evals.json +26 -26
  94. package/ftm-git/evals/promptfoo.yaml +75 -75
  95. package/ftm-git/hooks/post-commit-experience.sh +92 -92
  96. package/ftm-git/references/patterns/SECRET-PATTERNS.md +104 -104
  97. package/ftm-git/references/protocols/REMEDIATION.md +139 -139
  98. package/ftm-git/scripts/pre-commit-secrets.sh +110 -110
  99. package/ftm-git.yml +2 -2
  100. package/ftm-inbox/backend/__pycache__/main.cpython-314.pyc +0 -0
  101. package/ftm-inbox/backend/adapters/_retry.py +64 -64
  102. package/ftm-inbox/backend/adapters/base.py +230 -230
  103. package/ftm-inbox/backend/adapters/freshservice.py +104 -104
  104. package/ftm-inbox/backend/adapters/gmail.py +125 -125
  105. package/ftm-inbox/backend/adapters/jira.py +136 -136
  106. package/ftm-inbox/backend/adapters/registry.py +192 -192
  107. package/ftm-inbox/backend/adapters/slack.py +110 -110
  108. package/ftm-inbox/backend/db/connection.py +54 -54
  109. package/ftm-inbox/backend/db/schema.py +78 -78
  110. package/ftm-inbox/backend/executor/__init__.py +7 -7
  111. package/ftm-inbox/backend/executor/engine.py +149 -149
  112. package/ftm-inbox/backend/executor/step_runner.py +98 -98
  113. package/ftm-inbox/backend/main.py +103 -103
  114. package/ftm-inbox/backend/models/__init__.py +1 -1
  115. package/ftm-inbox/backend/models/unified_task.py +36 -36
  116. package/ftm-inbox/backend/planner/__init__.py +6 -6
  117. package/ftm-inbox/backend/planner/__pycache__/__init__.cpython-314.pyc +0 -0
  118. package/ftm-inbox/backend/planner/__pycache__/generator.cpython-314.pyc +0 -0
  119. package/ftm-inbox/backend/planner/__pycache__/schema.cpython-314.pyc +0 -0
  120. package/ftm-inbox/backend/planner/generator.py +127 -127
  121. package/ftm-inbox/backend/planner/schema.py +34 -34
  122. package/ftm-inbox/backend/requirements.txt +5 -5
  123. package/ftm-inbox/backend/routes/__pycache__/plan.cpython-314.pyc +0 -0
  124. package/ftm-inbox/backend/routes/execute.py +186 -186
  125. package/ftm-inbox/backend/routes/health.py +52 -52
  126. package/ftm-inbox/backend/routes/inbox.py +68 -68
  127. package/ftm-inbox/backend/routes/plan.py +271 -271
  128. package/ftm-inbox/bin/launchagent.mjs +91 -91
  129. package/ftm-inbox/bin/setup.mjs +188 -188
  130. package/ftm-inbox/bin/start.sh +10 -10
  131. package/ftm-inbox/bin/status.sh +17 -17
  132. package/ftm-inbox/bin/stop.sh +8 -8
  133. package/ftm-inbox/config.example.yml +55 -55
  134. package/ftm-inbox/package-lock.json +2898 -2898
  135. package/ftm-inbox/package.json +26 -26
  136. package/ftm-inbox/postcss.config.js +6 -6
  137. package/ftm-inbox/src/app.css +199 -199
  138. package/ftm-inbox/src/app.html +18 -18
  139. package/ftm-inbox/src/lib/api.ts +166 -166
  140. package/ftm-inbox/src/lib/components/ExecutionLog.svelte +81 -81
  141. package/ftm-inbox/src/lib/components/InboxFeed.svelte +143 -143
  142. package/ftm-inbox/src/lib/components/PlanStep.svelte +271 -271
  143. package/ftm-inbox/src/lib/components/PlanView.svelte +206 -206
  144. package/ftm-inbox/src/lib/components/StreamPanel.svelte +99 -99
  145. package/ftm-inbox/src/lib/components/TaskCard.svelte +190 -190
  146. package/ftm-inbox/src/lib/components/ui/EmptyState.svelte +63 -63
  147. package/ftm-inbox/src/lib/components/ui/KawaiiCard.svelte +86 -86
  148. package/ftm-inbox/src/lib/components/ui/PillButton.svelte +106 -106
  149. package/ftm-inbox/src/lib/components/ui/StatusBadge.svelte +67 -67
  150. package/ftm-inbox/src/lib/components/ui/StreamDrawer.svelte +149 -149
  151. package/ftm-inbox/src/lib/components/ui/ThemeToggle.svelte +80 -80
  152. package/ftm-inbox/src/lib/theme.ts +47 -47
  153. package/ftm-inbox/src/routes/+layout.svelte +76 -76
  154. package/ftm-inbox/src/routes/+page.svelte +401 -401
  155. package/ftm-inbox/svelte.config.js +12 -12
  156. package/ftm-inbox/tailwind.config.ts +63 -63
  157. package/ftm-inbox/tsconfig.json +13 -13
  158. package/ftm-inbox/vite.config.ts +6 -6
  159. package/ftm-intent/SKILL.md +241 -241
  160. package/ftm-intent.yml +2 -2
  161. package/ftm-manifest.json +3794 -3794
  162. package/ftm-map/SKILL.md +291 -291
  163. package/ftm-map/scripts/db.py +712 -712
  164. package/ftm-map/scripts/index.py +415 -415
  165. package/ftm-map/scripts/parser.py +224 -224
  166. package/ftm-map/scripts/queries/go-tags.scm +20 -20
  167. package/ftm-map/scripts/queries/javascript-tags.scm +35 -35
  168. package/ftm-map/scripts/queries/python-tags.scm +31 -31
  169. package/ftm-map/scripts/queries/ruby-tags.scm +19 -19
  170. package/ftm-map/scripts/queries/rust-tags.scm +37 -37
  171. package/ftm-map/scripts/queries/typescript-tags.scm +41 -41
  172. package/ftm-map/scripts/query.py +301 -301
  173. package/ftm-map/scripts/ranker.py +377 -377
  174. package/ftm-map/scripts/requirements.txt +5 -5
  175. package/ftm-map/scripts/setup-hooks.sh +27 -27
  176. package/ftm-map/scripts/setup.sh +56 -56
  177. package/ftm-map/scripts/test_db.py +364 -364
  178. package/ftm-map/scripts/test_parser.py +174 -174
  179. package/ftm-map/scripts/test_query.py +183 -183
  180. package/ftm-map/scripts/test_ranker.py +199 -199
  181. package/ftm-map/scripts/views.py +591 -591
  182. package/ftm-map.yml +2 -2
  183. package/ftm-mind/SKILL.md +201 -1943
  184. package/ftm-mind/evals/promptfoo.yaml +142 -142
  185. package/ftm-mind/references/blackboard-protocol.md +110 -0
  186. package/ftm-mind/references/blackboard-schema.md +328 -328
  187. package/ftm-mind/references/complexity-guide.md +110 -110
  188. package/ftm-mind/references/complexity-sizing.md +138 -0
  189. package/ftm-mind/references/decide-act-protocol.md +172 -0
  190. package/ftm-mind/references/direct-execution.md +51 -0
  191. package/ftm-mind/references/environment-discovery.md +77 -0
  192. package/ftm-mind/references/event-registry.md +319 -319
  193. package/ftm-mind/references/mcp-inventory.md +300 -296
  194. package/ftm-mind/references/ops-routing.md +47 -0
  195. package/ftm-mind/references/orient-protocol.md +234 -0
  196. package/ftm-mind/references/personality.md +40 -0
  197. package/ftm-mind/references/protocols/COMPLEXITY-SIZING.md +72 -72
  198. package/ftm-mind/references/protocols/MCP-HEURISTICS.md +32 -32
  199. package/ftm-mind/references/protocols/PLAN-APPROVAL.md +80 -80
  200. package/ftm-mind/references/reflexion-protocol.md +249 -249
  201. package/ftm-mind/references/routing/SCENARIOS.md +22 -22
  202. package/ftm-mind/references/routing-scenarios.md +35 -35
  203. package/ftm-mind.yml +2 -2
  204. package/ftm-ops.yml +4 -0
  205. package/ftm-pause/SKILL.md +395 -395
  206. package/ftm-pause/references/protocols/SKILL-RESTORE-PROTOCOLS.md +186 -186
  207. package/ftm-pause/references/protocols/VALIDATION.md +80 -80
  208. package/ftm-pause.yml +2 -2
  209. package/ftm-researcher/SKILL.md +275 -275
  210. package/ftm-researcher/evals/agent-diversity.yaml +17 -17
  211. package/ftm-researcher/evals/synthesis-quality.yaml +12 -12
  212. package/ftm-researcher/evals/trigger-accuracy.yaml +39 -39
  213. package/ftm-researcher/references/adaptive-search.md +116 -116
  214. package/ftm-researcher/references/agent-prompts.md +193 -193
  215. package/ftm-researcher/references/council-integration.md +193 -193
  216. package/ftm-researcher/references/output-format.md +203 -203
  217. package/ftm-researcher/references/synthesis-pipeline.md +165 -165
  218. package/ftm-researcher/scripts/score_credibility.py +234 -234
  219. package/ftm-researcher/scripts/validate_research.py +92 -92
  220. package/ftm-researcher.yml +2 -2
  221. package/ftm-resume/SKILL.md +518 -518
  222. package/ftm-resume/references/protocols/VALIDATION.md +172 -172
  223. package/ftm-resume.yml +2 -2
  224. package/ftm-retro/SKILL.md +380 -380
  225. package/ftm-retro/references/protocols/SCORING-RUBRICS.md +89 -89
  226. package/ftm-retro/references/templates/REPORT-FORMAT.md +109 -109
  227. package/ftm-retro.yml +2 -2
  228. package/ftm-routine/SKILL.md +170 -170
  229. package/ftm-routine.yml +4 -4
  230. package/ftm-state/blackboard/capabilities.json +5 -5
  231. package/ftm-state/blackboard/capabilities.schema.json +27 -27
  232. package/ftm-state/blackboard/context.json +37 -23
  233. package/ftm-state/blackboard/experiences/doom-statusline-fix.json +26 -0
  234. package/ftm-state/blackboard/experiences/hackathon-pages-site.json +26 -0
  235. package/ftm-state/blackboard/experiences/hindsight-sso-kickoff.json +42 -0
  236. package/ftm-state/blackboard/experiences/index.json +58 -9
  237. package/ftm-state/blackboard/experiences/learning-ragnarok-api-access.json +23 -0
  238. package/ftm-state/blackboard/experiences/nordlayer-members-auto-assign.json +26 -0
  239. package/ftm-state/blackboard/experiences/saml2aws-stale-session-fix.json +41 -0
  240. package/ftm-state/blackboard/patterns.json +6 -6
  241. package/ftm-state/schemas/context.schema.json +130 -130
  242. package/ftm-state/schemas/experience-index.schema.json +77 -77
  243. package/ftm-state/schemas/experience.schema.json +78 -78
  244. package/ftm-state/schemas/patterns.schema.json +44 -44
  245. package/ftm-upgrade/SKILL.md +194 -194
  246. package/ftm-upgrade/scripts/check-version.sh +76 -76
  247. package/ftm-upgrade/scripts/upgrade.sh +143 -143
  248. package/ftm-upgrade.yml +2 -2
  249. package/ftm-verify.yml +2 -2
  250. package/ftm.yml +2 -2
  251. package/hooks/ftm-auto-log.sh +137 -0
  252. package/hooks/ftm-blackboard-enforcer.sh +93 -93
  253. package/hooks/ftm-discovery-reminder.sh +90 -90
  254. package/hooks/ftm-drafts-gate.sh +61 -61
  255. package/hooks/ftm-event-logger.mjs +107 -107
  256. package/hooks/ftm-install-hooks.sh +240 -0
  257. package/hooks/ftm-learning-capture.sh +117 -0
  258. package/hooks/ftm-map-autodetect.sh +79 -79
  259. package/hooks/ftm-pending-sync-check.sh +22 -22
  260. package/hooks/ftm-plan-gate.sh +92 -92
  261. package/hooks/ftm-post-commit-trigger.sh +57 -57
  262. package/hooks/ftm-post-compaction.sh +138 -0
  263. package/hooks/ftm-pre-compaction.sh +147 -0
  264. package/hooks/ftm-session-end.sh +52 -0
  265. package/hooks/ftm-session-snapshot.sh +213 -0
  266. package/hooks/settings-template.json +81 -81
  267. package/install.sh +363 -363
  268. package/package.json +84 -84
  269. package/uninstall.sh +25 -25
@@ -1,415 +1,415 @@
1
- #!/usr/bin/env python3
2
- """ftm-map indexer: builds the code knowledge graph from source files.
3
-
4
- Two-phase indexing:
5
- Phase 1 — Parse each file with tree-sitter, insert file/symbol/ref rows.
6
- Phase 2 — Materialize file_edges with Aider-style weight heuristics and
7
- symbol_edges via enclosing-scope resolution.
8
- """
9
-
10
- import argparse
11
- import json
12
- import math
13
- import os
14
- import re
15
- import subprocess
16
- import sys
17
- import time
18
- from collections import Counter
19
- from datetime import datetime, timezone
20
- from pathlib import Path
21
-
22
- # Add scripts dir to path for sibling imports
23
- sys.path.insert(0, os.path.dirname(__file__))
24
-
25
- from db import (
26
- get_connection,
27
- add_file,
28
- add_symbol,
29
- add_reference,
30
- remove_file,
31
- get_stats,
32
- rebuild_symbol_edges,
33
- )
34
- from parser import get_tags, detect_language, EXTENSION_MAP, compute_content_hash
35
-
36
- META_REGISTRY = os.path.expanduser("~/.claude/ftm-state/maps/index.json")
37
-
38
-
39
- # ---------------------------------------------------------------------------
40
- # File discovery
41
- # ---------------------------------------------------------------------------
42
-
43
-
44
- def discover_files(project_root: str) -> list[str]:
45
- """Get tracked source files using git ls-files.
46
-
47
- Filters to files whose extensions are in EXTENSION_MAP so only
48
- tree-sitter-parseable files are returned. Returns absolute paths.
49
- """
50
- result = subprocess.run(
51
- ["git", "ls-files"],
52
- capture_output=True,
53
- text=True,
54
- cwd=project_root,
55
- )
56
- if result.returncode != 0:
57
- print(
58
- f"Error: git ls-files failed: {result.stderr.strip()}",
59
- file=sys.stderr,
60
- )
61
- return []
62
-
63
- supported_exts = set(EXTENSION_MAP.keys())
64
- files = []
65
- for line in result.stdout.strip().split("\n"):
66
- if not line:
67
- continue
68
- ext = Path(line).suffix.lower()
69
- if ext in supported_exts:
70
- files.append(os.path.join(project_root, line))
71
- return files
72
-
73
-
74
- # ---------------------------------------------------------------------------
75
- # Core indexing logic
76
- # ---------------------------------------------------------------------------
77
-
78
-
79
- def index_files(conn, files: list[str], project_root: str) -> dict:
80
- """Parse and insert files, symbols, references, then materialize edges.
81
-
82
- Phase 1 — For each file: read source, compute hash, insert file row,
83
- extract def/ref tags via tree-sitter, insert symbol and ref rows.
84
- Phase 2 — Build file_edges with Aider weight heuristics (long descriptive
85
- names 10x, private 0.1x, overloaded 0.1x, sqrt-dampened counts).
86
- Then rebuild symbol_edges via enclosing-scope resolution.
87
-
88
- Returns a dict with symbols, references, file_edges, symbol_edges counts.
89
- """
90
- total_symbols = 0
91
- total_refs = 0
92
-
93
- # ------------------------------------------------------------------
94
- # Phase 1: parse each file and insert rows
95
- # ------------------------------------------------------------------
96
- for fpath in files:
97
- if not os.path.exists(fpath):
98
- print(f"Warning: file not found, skipping: {fpath}", file=sys.stderr)
99
- continue
100
-
101
- rel_path = os.path.relpath(fpath, project_root)
102
- lang = detect_language(fpath)
103
- mtime = os.path.getmtime(fpath)
104
-
105
- # Stream-friendly: read once, extract metadata, then release
106
- try:
107
- with open(fpath, "r", encoding="utf-8", errors="replace") as fh:
108
- source = fh.read()
109
- except (IOError, OSError) as exc:
110
- print(f"Warning: Cannot read {fpath}: {exc}", file=sys.stderr)
111
- continue
112
-
113
- line_count = source.count("\n") + 1
114
- content_hash = compute_content_hash(source)
115
-
116
- # Insert file record
117
- file_id = add_file(
118
- conn, rel_path, lang, mtime,
119
- hash=content_hash, line_count=line_count,
120
- )
121
-
122
- # Extract def/ref tags via tree-sitter
123
- tags = get_tags(fpath, rel_path)
124
-
125
- for tag in tags:
126
- if tag.kind == "def":
127
- add_symbol(conn, file_id, tag.name, "definition", tag.line, signature=None)
128
- total_symbols += 1
129
- elif tag.kind == "ref":
130
- add_reference(conn, file_id, tag.name, tag.line, kind="call")
131
- total_refs += 1
132
-
133
- # ------------------------------------------------------------------
134
- # Phase 2: materialize edges
135
- # ------------------------------------------------------------------
136
-
137
- # Build defines map: ident -> set of file_ids that define it
138
- defines = {}
139
- for row in conn.execute("SELECT name, file_id FROM symbols").fetchall():
140
- defines.setdefault(row["name"], set()).add(row["file_id"])
141
-
142
- # Build references map: ident -> list of file_ids that reference it
143
- references_map = {}
144
- for row in conn.execute("SELECT symbol_name, file_id FROM refs").fetchall():
145
- references_map.setdefault(row["symbol_name"], []).append(row["file_id"])
146
-
147
- # Materialize file_edges with Aider weight heuristics
148
- conn.execute("DELETE FROM file_edges")
149
-
150
- for ident, ref_file_ids in references_map.items():
151
- definers = defines.get(ident, set())
152
- if not definers:
153
- continue
154
-
155
- # Aider weight heuristics
156
- mul = 1.0
157
- # Long descriptive names (camelCase or snake_case, >= 8 chars) weighted higher
158
- if len(ident) >= 8 and re.match(r"[a-z_]+[A-Z]|[a-z]+_[a-z]", ident):
159
- mul *= 10
160
- # Private names weighted lower
161
- if ident.startswith("_"):
162
- mul *= 0.1
163
- # Overloaded names (defined in many files) weighted lower
164
- if len(definers) >= 5:
165
- mul *= 0.1
166
-
167
- # Count refs per file, then create weighted edges
168
- ref_counts = Counter(ref_file_ids)
169
-
170
- for ref_file_id, count in ref_counts.items():
171
- weight = mul * math.sqrt(count)
172
- for def_file_id in definers:
173
- if ref_file_id != def_file_id: # No self-edges
174
- conn.execute(
175
- """INSERT INTO file_edges (source_file_id, target_file_id, weight)
176
- VALUES (?, ?, ?)
177
- ON CONFLICT(source_file_id, target_file_id)
178
- DO UPDATE SET weight = MAX(weight, excluded.weight)""",
179
- (ref_file_id, def_file_id, weight),
180
- )
181
-
182
- # Materialize symbol_edges via enclosing-scope resolution
183
- rebuild_symbol_edges(conn)
184
-
185
- # Gather edge counts
186
- file_edge_count = conn.execute("SELECT COUNT(*) FROM file_edges").fetchone()[0]
187
- symbol_edge_count = conn.execute("SELECT COUNT(*) FROM symbol_edges").fetchone()[0]
188
-
189
- return {
190
- "symbols": total_symbols,
191
- "references": total_refs,
192
- "file_edges": file_edge_count,
193
- "symbol_edges": symbol_edge_count,
194
- }
195
-
196
-
197
- # ---------------------------------------------------------------------------
198
- # Bootstrap mode
199
- # ---------------------------------------------------------------------------
200
-
201
-
202
- def bootstrap(project_root: str) -> None:
203
- """Full scan: (re)build the entire code graph for *project_root*."""
204
- abs_root = os.path.abspath(project_root)
205
- start = time.time()
206
-
207
- files = discover_files(abs_root)
208
- if not files:
209
- print(
210
- json.dumps({"error": "No parseable source files found in git repository"}),
211
- file=sys.stderr,
212
- )
213
- sys.exit(1)
214
-
215
- conn = get_connection(abs_root)
216
- try:
217
- # Full rebuild — clear all tables. CASCADE handles symbols, refs, edges.
218
- # FTS5 rows must be removed before symbol rows (content= table).
219
- symbol_ids = [
220
- row[0] for row in conn.execute("SELECT id FROM symbols").fetchall()
221
- ]
222
- for sid in symbol_ids:
223
- conn.execute("DELETE FROM symbols_fts WHERE rowid=?", (sid,))
224
- conn.execute("DELETE FROM files")
225
-
226
- stats = index_files(conn, files, abs_root)
227
- conn.commit()
228
-
229
- duration = time.time() - start
230
- result = {
231
- "mode": "bootstrap",
232
- "files_parsed": len(files),
233
- "symbols": stats["symbols"],
234
- "references": stats["references"],
235
- "file_edges": stats["file_edges"],
236
- "symbol_edges": stats["symbol_edges"],
237
- "duration_s": round(duration, 2),
238
- }
239
- print(json.dumps(result))
240
- update_meta_registry(abs_root, stats["symbols"])
241
- except Exception as exc: # noqa: BLE001
242
- print(f"Error during bootstrap: {exc}", file=sys.stderr)
243
- conn.rollback()
244
- conn.close()
245
- sys.exit(1)
246
- finally:
247
- conn.close()
248
-
249
-
250
- # ---------------------------------------------------------------------------
251
- # Incremental mode
252
- # ---------------------------------------------------------------------------
253
-
254
-
255
- def incremental(project_root: str, files_str: str) -> None:
256
- """Incremental update: re-index only the specified files.
257
-
258
- *files_str* is a comma-separated list of file paths (relative or absolute).
259
- Old file/symbol/ref/edge data for each file is cascade-deleted via
260
- remove_file() before re-parsing so stale entries do not accumulate.
261
- All edges are rebuilt since changes can ripple across files.
262
- """
263
- abs_root = os.path.abspath(project_root)
264
- start = time.time()
265
-
266
- raw_files = [f.strip() for f in files_str.split(",") if f.strip()]
267
- abs_files = [
268
- f if os.path.isabs(f) else os.path.join(abs_root, f) for f in raw_files
269
- ]
270
-
271
- conn = get_connection(abs_root)
272
- try:
273
- # Remove stale data for all targeted files (cascading delete).
274
- for fpath in abs_files:
275
- rel_path = os.path.relpath(fpath, abs_root)
276
- remove_file(conn, rel_path)
277
-
278
- existing_files = [f for f in abs_files if os.path.exists(f)]
279
- if not existing_files:
280
- print(
281
- json.dumps({"error": "None of the specified files exist"}),
282
- file=sys.stderr,
283
- )
284
- conn.close()
285
- sys.exit(1)
286
-
287
- stats = index_files(conn, existing_files, abs_root)
288
- conn.commit()
289
-
290
- db_stats = get_stats(conn)
291
- duration = time.time() - start
292
- result = {
293
- "mode": "incremental",
294
- "files_parsed": len(existing_files),
295
- "symbols": stats["symbols"],
296
- "references": stats["references"],
297
- "file_edges": stats["file_edges"],
298
- "symbol_edges": stats["symbol_edges"],
299
- "duration_s": round(duration, 2),
300
- }
301
- print(json.dumps(result))
302
- update_meta_registry(abs_root, db_stats["symbol_count"])
303
- except Exception as exc: # noqa: BLE001
304
- print(f"Error during incremental update: {exc}", file=sys.stderr)
305
- conn.rollback()
306
- conn.close()
307
- sys.exit(1)
308
- finally:
309
- conn.close()
310
-
311
-
312
- # ---------------------------------------------------------------------------
313
- # Meta-registry management
314
- # ---------------------------------------------------------------------------
315
-
316
-
317
- def update_meta_registry(project_root: str, symbol_count: int) -> None:
318
- """Upsert project entry in the global meta-registry at META_REGISTRY."""
319
- registry_dir = os.path.dirname(META_REGISTRY)
320
- os.makedirs(registry_dir, exist_ok=True)
321
-
322
- registry: dict = {"projects": []}
323
- if os.path.exists(META_REGISTRY):
324
- try:
325
- with open(META_REGISTRY) as fh:
326
- registry = json.load(fh)
327
- except (json.JSONDecodeError, IOError):
328
- # Corrupt or unreadable registry — start fresh.
329
- registry = {"projects": []}
330
-
331
- abs_root = os.path.abspath(project_root)
332
- db_path = os.path.join(abs_root, ".ftm-map", "map.db")
333
- now = datetime.now(timezone.utc).isoformat()
334
-
335
- found = False
336
- for proj in registry["projects"]:
337
- if proj.get("path") == abs_root:
338
- proj["last_indexed"] = now
339
- proj["symbol_count"] = symbol_count
340
- found = True
341
- break
342
-
343
- if not found:
344
- registry["projects"].append(
345
- {
346
- "path": abs_root,
347
- "db_path": db_path,
348
- "last_indexed": now,
349
- "symbol_count": symbol_count,
350
- }
351
- )
352
-
353
- with open(META_REGISTRY, "w") as fh:
354
- json.dump(registry, fh, indent=2)
355
-
356
-
357
- # ---------------------------------------------------------------------------
358
- # CLI entry point
359
- # ---------------------------------------------------------------------------
360
-
361
-
362
- def main() -> None:
363
- parser = argparse.ArgumentParser(
364
- description="ftm-map indexer — builds the code knowledge graph from source files.",
365
- formatter_class=argparse.RawDescriptionHelpFormatter,
366
- epilog=(
367
- "Examples:\n"
368
- " python3 index.py --bootstrap /path/to/project\n"
369
- " python3 index.py --incremental --files src/foo.ts,src/bar.py\n"
370
- " python3 index.py --incremental --files src/foo.ts --project-root /path/to/project\n"
371
- ),
372
- )
373
-
374
- mode = parser.add_mutually_exclusive_group(required=True)
375
- mode.add_argument(
376
- "--bootstrap",
377
- metavar="PROJECT_ROOT",
378
- help="Full scan: index all tracked source files in PROJECT_ROOT.",
379
- )
380
- mode.add_argument(
381
- "--incremental",
382
- action="store_true",
383
- help="Incremental update: re-index only the files given by --files.",
384
- )
385
-
386
- parser.add_argument(
387
- "--files",
388
- metavar="FILE_LIST",
389
- help="Comma-separated list of files to re-index (required for --incremental).",
390
- )
391
- parser.add_argument(
392
- "--project-root",
393
- metavar="PATH",
394
- default=None,
395
- help=(
396
- "Project root used to locate the database for incremental mode. "
397
- "Defaults to the current working directory."
398
- ),
399
- )
400
-
401
- args = parser.parse_args()
402
-
403
- if args.bootstrap:
404
- bootstrap(args.bootstrap)
405
- else:
406
- # Incremental mode
407
- if not args.files:
408
- print("Error: --incremental requires --files", file=sys.stderr)
409
- sys.exit(1)
410
- project_root = args.project_root or os.getcwd()
411
- incremental(project_root, args.files)
412
-
413
-
414
- if __name__ == "__main__":
415
- main()
1
+ #!/usr/bin/env python3
2
+ """ftm-map indexer: builds the code knowledge graph from source files.
3
+
4
+ Two-phase indexing:
5
+ Phase 1 — Parse each file with tree-sitter, insert file/symbol/ref rows.
6
+ Phase 2 — Materialize file_edges with Aider-style weight heuristics and
7
+ symbol_edges via enclosing-scope resolution.
8
+ """
9
+
10
+ import argparse
11
+ import json
12
+ import math
13
+ import os
14
+ import re
15
+ import subprocess
16
+ import sys
17
+ import time
18
+ from collections import Counter
19
+ from datetime import datetime, timezone
20
+ from pathlib import Path
21
+
22
+ # Add scripts dir to path for sibling imports
23
+ sys.path.insert(0, os.path.dirname(__file__))
24
+
25
+ from db import (
26
+ get_connection,
27
+ add_file,
28
+ add_symbol,
29
+ add_reference,
30
+ remove_file,
31
+ get_stats,
32
+ rebuild_symbol_edges,
33
+ )
34
+ from parser import get_tags, detect_language, EXTENSION_MAP, compute_content_hash
35
+
36
+ META_REGISTRY = os.path.expanduser("~/.claude/ftm-state/maps/index.json")
37
+
38
+
39
+ # ---------------------------------------------------------------------------
40
+ # File discovery
41
+ # ---------------------------------------------------------------------------
42
+
43
+
44
+ def discover_files(project_root: str) -> list[str]:
45
+ """Get tracked source files using git ls-files.
46
+
47
+ Filters to files whose extensions are in EXTENSION_MAP so only
48
+ tree-sitter-parseable files are returned. Returns absolute paths.
49
+ """
50
+ result = subprocess.run(
51
+ ["git", "ls-files"],
52
+ capture_output=True,
53
+ text=True,
54
+ cwd=project_root,
55
+ )
56
+ if result.returncode != 0:
57
+ print(
58
+ f"Error: git ls-files failed: {result.stderr.strip()}",
59
+ file=sys.stderr,
60
+ )
61
+ return []
62
+
63
+ supported_exts = set(EXTENSION_MAP.keys())
64
+ files = []
65
+ for line in result.stdout.strip().split("\n"):
66
+ if not line:
67
+ continue
68
+ ext = Path(line).suffix.lower()
69
+ if ext in supported_exts:
70
+ files.append(os.path.join(project_root, line))
71
+ return files
72
+
73
+
74
+ # ---------------------------------------------------------------------------
75
+ # Core indexing logic
76
+ # ---------------------------------------------------------------------------
77
+
78
+
79
+ def index_files(conn, files: list[str], project_root: str) -> dict:
80
+ """Parse and insert files, symbols, references, then materialize edges.
81
+
82
+ Phase 1 — For each file: read source, compute hash, insert file row,
83
+ extract def/ref tags via tree-sitter, insert symbol and ref rows.
84
+ Phase 2 — Build file_edges with Aider weight heuristics (long descriptive
85
+ names 10x, private 0.1x, overloaded 0.1x, sqrt-dampened counts).
86
+ Then rebuild symbol_edges via enclosing-scope resolution.
87
+
88
+ Returns a dict with symbols, references, file_edges, symbol_edges counts.
89
+ """
90
+ total_symbols = 0
91
+ total_refs = 0
92
+
93
+ # ------------------------------------------------------------------
94
+ # Phase 1: parse each file and insert rows
95
+ # ------------------------------------------------------------------
96
+ for fpath in files:
97
+ if not os.path.exists(fpath):
98
+ print(f"Warning: file not found, skipping: {fpath}", file=sys.stderr)
99
+ continue
100
+
101
+ rel_path = os.path.relpath(fpath, project_root)
102
+ lang = detect_language(fpath)
103
+ mtime = os.path.getmtime(fpath)
104
+
105
+ # Stream-friendly: read once, extract metadata, then release
106
+ try:
107
+ with open(fpath, "r", encoding="utf-8", errors="replace") as fh:
108
+ source = fh.read()
109
+ except (IOError, OSError) as exc:
110
+ print(f"Warning: Cannot read {fpath}: {exc}", file=sys.stderr)
111
+ continue
112
+
113
+ line_count = source.count("\n") + 1
114
+ content_hash = compute_content_hash(source)
115
+
116
+ # Insert file record
117
+ file_id = add_file(
118
+ conn, rel_path, lang, mtime,
119
+ hash=content_hash, line_count=line_count,
120
+ )
121
+
122
+ # Extract def/ref tags via tree-sitter
123
+ tags = get_tags(fpath, rel_path)
124
+
125
+ for tag in tags:
126
+ if tag.kind == "def":
127
+ add_symbol(conn, file_id, tag.name, "definition", tag.line, signature=None)
128
+ total_symbols += 1
129
+ elif tag.kind == "ref":
130
+ add_reference(conn, file_id, tag.name, tag.line, kind="call")
131
+ total_refs += 1
132
+
133
+ # ------------------------------------------------------------------
134
+ # Phase 2: materialize edges
135
+ # ------------------------------------------------------------------
136
+
137
+ # Build defines map: ident -> set of file_ids that define it
138
+ defines = {}
139
+ for row in conn.execute("SELECT name, file_id FROM symbols").fetchall():
140
+ defines.setdefault(row["name"], set()).add(row["file_id"])
141
+
142
+ # Build references map: ident -> list of file_ids that reference it
143
+ references_map = {}
144
+ for row in conn.execute("SELECT symbol_name, file_id FROM refs").fetchall():
145
+ references_map.setdefault(row["symbol_name"], []).append(row["file_id"])
146
+
147
+ # Materialize file_edges with Aider weight heuristics
148
+ conn.execute("DELETE FROM file_edges")
149
+
150
+ for ident, ref_file_ids in references_map.items():
151
+ definers = defines.get(ident, set())
152
+ if not definers:
153
+ continue
154
+
155
+ # Aider weight heuristics
156
+ mul = 1.0
157
+ # Long descriptive names (camelCase or snake_case, >= 8 chars) weighted higher
158
+ if len(ident) >= 8 and re.match(r"[a-z_]+[A-Z]|[a-z]+_[a-z]", ident):
159
+ mul *= 10
160
+ # Private names weighted lower
161
+ if ident.startswith("_"):
162
+ mul *= 0.1
163
+ # Overloaded names (defined in many files) weighted lower
164
+ if len(definers) >= 5:
165
+ mul *= 0.1
166
+
167
+ # Count refs per file, then create weighted edges
168
+ ref_counts = Counter(ref_file_ids)
169
+
170
+ for ref_file_id, count in ref_counts.items():
171
+ weight = mul * math.sqrt(count)
172
+ for def_file_id in definers:
173
+ if ref_file_id != def_file_id: # No self-edges
174
+ conn.execute(
175
+ """INSERT INTO file_edges (source_file_id, target_file_id, weight)
176
+ VALUES (?, ?, ?)
177
+ ON CONFLICT(source_file_id, target_file_id)
178
+ DO UPDATE SET weight = MAX(weight, excluded.weight)""",
179
+ (ref_file_id, def_file_id, weight),
180
+ )
181
+
182
+ # Materialize symbol_edges via enclosing-scope resolution
183
+ rebuild_symbol_edges(conn)
184
+
185
+ # Gather edge counts
186
+ file_edge_count = conn.execute("SELECT COUNT(*) FROM file_edges").fetchone()[0]
187
+ symbol_edge_count = conn.execute("SELECT COUNT(*) FROM symbol_edges").fetchone()[0]
188
+
189
+ return {
190
+ "symbols": total_symbols,
191
+ "references": total_refs,
192
+ "file_edges": file_edge_count,
193
+ "symbol_edges": symbol_edge_count,
194
+ }
195
+
196
+
197
+ # ---------------------------------------------------------------------------
198
+ # Bootstrap mode
199
+ # ---------------------------------------------------------------------------
200
+
201
+
202
+ def bootstrap(project_root: str) -> None:
203
+ """Full scan: (re)build the entire code graph for *project_root*."""
204
+ abs_root = os.path.abspath(project_root)
205
+ start = time.time()
206
+
207
+ files = discover_files(abs_root)
208
+ if not files:
209
+ print(
210
+ json.dumps({"error": "No parseable source files found in git repository"}),
211
+ file=sys.stderr,
212
+ )
213
+ sys.exit(1)
214
+
215
+ conn = get_connection(abs_root)
216
+ try:
217
+ # Full rebuild — clear all tables. CASCADE handles symbols, refs, edges.
218
+ # FTS5 rows must be removed before symbol rows (content= table).
219
+ symbol_ids = [
220
+ row[0] for row in conn.execute("SELECT id FROM symbols").fetchall()
221
+ ]
222
+ for sid in symbol_ids:
223
+ conn.execute("DELETE FROM symbols_fts WHERE rowid=?", (sid,))
224
+ conn.execute("DELETE FROM files")
225
+
226
+ stats = index_files(conn, files, abs_root)
227
+ conn.commit()
228
+
229
+ duration = time.time() - start
230
+ result = {
231
+ "mode": "bootstrap",
232
+ "files_parsed": len(files),
233
+ "symbols": stats["symbols"],
234
+ "references": stats["references"],
235
+ "file_edges": stats["file_edges"],
236
+ "symbol_edges": stats["symbol_edges"],
237
+ "duration_s": round(duration, 2),
238
+ }
239
+ print(json.dumps(result))
240
+ update_meta_registry(abs_root, stats["symbols"])
241
+ except Exception as exc: # noqa: BLE001
242
+ print(f"Error during bootstrap: {exc}", file=sys.stderr)
243
+ conn.rollback()
244
+ conn.close()
245
+ sys.exit(1)
246
+ finally:
247
+ conn.close()
248
+
249
+
250
+ # ---------------------------------------------------------------------------
251
+ # Incremental mode
252
+ # ---------------------------------------------------------------------------
253
+
254
+
255
+ def incremental(project_root: str, files_str: str) -> None:
256
+ """Incremental update: re-index only the specified files.
257
+
258
+ *files_str* is a comma-separated list of file paths (relative or absolute).
259
+ Old file/symbol/ref/edge data for each file is cascade-deleted via
260
+ remove_file() before re-parsing so stale entries do not accumulate.
261
+ All edges are rebuilt since changes can ripple across files.
262
+ """
263
+ abs_root = os.path.abspath(project_root)
264
+ start = time.time()
265
+
266
+ raw_files = [f.strip() for f in files_str.split(",") if f.strip()]
267
+ abs_files = [
268
+ f if os.path.isabs(f) else os.path.join(abs_root, f) for f in raw_files
269
+ ]
270
+
271
+ conn = get_connection(abs_root)
272
+ try:
273
+ # Remove stale data for all targeted files (cascading delete).
274
+ for fpath in abs_files:
275
+ rel_path = os.path.relpath(fpath, abs_root)
276
+ remove_file(conn, rel_path)
277
+
278
+ existing_files = [f for f in abs_files if os.path.exists(f)]
279
+ if not existing_files:
280
+ print(
281
+ json.dumps({"error": "None of the specified files exist"}),
282
+ file=sys.stderr,
283
+ )
284
+ conn.close()
285
+ sys.exit(1)
286
+
287
+ stats = index_files(conn, existing_files, abs_root)
288
+ conn.commit()
289
+
290
+ db_stats = get_stats(conn)
291
+ duration = time.time() - start
292
+ result = {
293
+ "mode": "incremental",
294
+ "files_parsed": len(existing_files),
295
+ "symbols": stats["symbols"],
296
+ "references": stats["references"],
297
+ "file_edges": stats["file_edges"],
298
+ "symbol_edges": stats["symbol_edges"],
299
+ "duration_s": round(duration, 2),
300
+ }
301
+ print(json.dumps(result))
302
+ update_meta_registry(abs_root, db_stats["symbol_count"])
303
+ except Exception as exc: # noqa: BLE001
304
+ print(f"Error during incremental update: {exc}", file=sys.stderr)
305
+ conn.rollback()
306
+ conn.close()
307
+ sys.exit(1)
308
+ finally:
309
+ conn.close()
310
+
311
+
312
+ # ---------------------------------------------------------------------------
313
+ # Meta-registry management
314
+ # ---------------------------------------------------------------------------
315
+
316
+
317
+ def update_meta_registry(project_root: str, symbol_count: int) -> None:
318
+ """Upsert project entry in the global meta-registry at META_REGISTRY."""
319
+ registry_dir = os.path.dirname(META_REGISTRY)
320
+ os.makedirs(registry_dir, exist_ok=True)
321
+
322
+ registry: dict = {"projects": []}
323
+ if os.path.exists(META_REGISTRY):
324
+ try:
325
+ with open(META_REGISTRY) as fh:
326
+ registry = json.load(fh)
327
+ except (json.JSONDecodeError, IOError):
328
+ # Corrupt or unreadable registry — start fresh.
329
+ registry = {"projects": []}
330
+
331
+ abs_root = os.path.abspath(project_root)
332
+ db_path = os.path.join(abs_root, ".ftm-map", "map.db")
333
+ now = datetime.now(timezone.utc).isoformat()
334
+
335
+ found = False
336
+ for proj in registry["projects"]:
337
+ if proj.get("path") == abs_root:
338
+ proj["last_indexed"] = now
339
+ proj["symbol_count"] = symbol_count
340
+ found = True
341
+ break
342
+
343
+ if not found:
344
+ registry["projects"].append(
345
+ {
346
+ "path": abs_root,
347
+ "db_path": db_path,
348
+ "last_indexed": now,
349
+ "symbol_count": symbol_count,
350
+ }
351
+ )
352
+
353
+ with open(META_REGISTRY, "w") as fh:
354
+ json.dump(registry, fh, indent=2)
355
+
356
+
357
+ # ---------------------------------------------------------------------------
358
+ # CLI entry point
359
+ # ---------------------------------------------------------------------------
360
+
361
+
362
+ def main() -> None:
363
+ parser = argparse.ArgumentParser(
364
+ description="ftm-map indexer — builds the code knowledge graph from source files.",
365
+ formatter_class=argparse.RawDescriptionHelpFormatter,
366
+ epilog=(
367
+ "Examples:\n"
368
+ " python3 index.py --bootstrap /path/to/project\n"
369
+ " python3 index.py --incremental --files src/foo.ts,src/bar.py\n"
370
+ " python3 index.py --incremental --files src/foo.ts --project-root /path/to/project\n"
371
+ ),
372
+ )
373
+
374
+ mode = parser.add_mutually_exclusive_group(required=True)
375
+ mode.add_argument(
376
+ "--bootstrap",
377
+ metavar="PROJECT_ROOT",
378
+ help="Full scan: index all tracked source files in PROJECT_ROOT.",
379
+ )
380
+ mode.add_argument(
381
+ "--incremental",
382
+ action="store_true",
383
+ help="Incremental update: re-index only the files given by --files.",
384
+ )
385
+
386
+ parser.add_argument(
387
+ "--files",
388
+ metavar="FILE_LIST",
389
+ help="Comma-separated list of files to re-index (required for --incremental).",
390
+ )
391
+ parser.add_argument(
392
+ "--project-root",
393
+ metavar="PATH",
394
+ default=None,
395
+ help=(
396
+ "Project root used to locate the database for incremental mode. "
397
+ "Defaults to the current working directory."
398
+ ),
399
+ )
400
+
401
+ args = parser.parse_args()
402
+
403
+ if args.bootstrap:
404
+ bootstrap(args.bootstrap)
405
+ else:
406
+ # Incremental mode
407
+ if not args.files:
408
+ print("Error: --incremental requires --files", file=sys.stderr)
409
+ sys.exit(1)
410
+ project_root = args.project_root or os.getcwd()
411
+ incremental(project_root, args.files)
412
+
413
+
414
+ if __name__ == "__main__":
415
+ main()