feed-the-machine 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +170 -170
- package/bin/generate-manifest.mjs +463 -463
- package/bin/install.mjs +491 -491
- package/docs/HOOKS.md +243 -243
- package/docs/INBOX.md +233 -233
- package/ftm/SKILL.md +122 -122
- package/ftm-audit/SKILL.md +623 -541
- package/ftm-audit/references/protocols/PROJECT-PATTERNS.md +91 -91
- package/ftm-audit/references/protocols/RUNTIME-WIRING.md +66 -66
- package/ftm-audit/references/protocols/WIRING-CONTRACTS.md +135 -135
- package/ftm-audit/references/strategies/AUTO-FIX-STRATEGIES.md +69 -69
- package/ftm-audit/references/templates/REPORT-FORMAT.md +96 -96
- package/ftm-audit/scripts/run-knip.sh +23 -23
- package/ftm-audit.yml +2 -2
- package/ftm-brainstorm/SKILL.md +498 -498
- package/ftm-brainstorm/evals/evals.json +100 -100
- package/ftm-brainstorm/evals/promptfoo.yaml +109 -109
- package/ftm-brainstorm/references/agent-prompts.md +224 -224
- package/ftm-brainstorm/references/plan-template.md +121 -121
- package/ftm-brainstorm.yml +2 -2
- package/ftm-browse/SKILL.md +454 -454
- package/ftm-browse/daemon/browser-manager.ts +206 -206
- package/ftm-browse/daemon/bun.lock +30 -30
- package/ftm-browse/daemon/cli.ts +347 -347
- package/ftm-browse/daemon/commands.ts +410 -410
- package/ftm-browse/daemon/main.ts +357 -357
- package/ftm-browse/daemon/package.json +17 -17
- package/ftm-browse/daemon/server.ts +189 -189
- package/ftm-browse/daemon/snapshot.ts +519 -519
- package/ftm-browse/daemon/tsconfig.json +22 -22
- package/ftm-browse.yml +4 -4
- package/ftm-capture/SKILL.md +370 -370
- package/ftm-capture.yml +4 -4
- package/ftm-codex-gate/SKILL.md +361 -361
- package/ftm-codex-gate.yml +2 -2
- package/ftm-config/SKILL.md +345 -345
- package/ftm-config.default.yml +82 -80
- package/ftm-config.yml +2 -2
- package/ftm-council/SKILL.md +416 -416
- package/ftm-council/references/prompts/CLAUDE-INVESTIGATION.md +60 -60
- package/ftm-council/references/prompts/CODEX-INVESTIGATION.md +58 -58
- package/ftm-council/references/prompts/GEMINI-INVESTIGATION.md +58 -58
- package/ftm-council/references/prompts/REBUTTAL-TEMPLATE.md +57 -57
- package/ftm-council/references/protocols/PREREQUISITES.md +47 -47
- package/ftm-council/references/protocols/STEP-0-FRAMING.md +46 -46
- package/ftm-council.yml +2 -2
- package/ftm-dashboard/SKILL.md +163 -163
- package/ftm-dashboard.yml +4 -4
- package/ftm-debug/SKILL.md +1037 -1037
- package/ftm-debug/references/phases/PHASE-0-INTAKE.md +58 -58
- package/ftm-debug/references/phases/PHASE-1-TRIAGE.md +46 -46
- package/ftm-debug/references/phases/PHASE-2-WAR-ROOM-AGENTS.md +279 -279
- package/ftm-debug/references/phases/PHASE-3-TO-6-EXECUTION.md +436 -436
- package/ftm-debug/references/protocols/BLACKBOARD.md +86 -86
- package/ftm-debug/references/protocols/EDGE-CASES.md +103 -103
- package/ftm-debug.yml +2 -2
- package/ftm-diagram/SKILL.md +277 -277
- package/ftm-diagram.yml +2 -2
- package/ftm-executor/SKILL.md +777 -767
- package/ftm-executor/references/STYLE-TEMPLATE.md +73 -73
- package/ftm-executor/references/phases/PHASE-0-VERIFICATION.md +62 -62
- package/ftm-executor/references/phases/PHASE-2-AGENT-ASSEMBLY.md +34 -34
- package/ftm-executor/references/phases/PHASE-3-WORKTREES.md +38 -38
- package/ftm-executor/references/phases/PHASE-4-5-AUDIT.md +72 -72
- package/ftm-executor/references/phases/PHASE-4-DISPATCH.md +66 -66
- package/ftm-executor/references/phases/PHASE-5-5-CODEX-GATE.md +73 -73
- package/ftm-executor/references/protocols/DOCUMENTATION-BOOTSTRAP.md +36 -36
- package/ftm-executor/references/protocols/MODEL-PROFILE.md +59 -44
- package/ftm-executor/references/protocols/PROGRESS-TRACKING.md +66 -66
- package/ftm-executor/runtime/ftm-runtime.mjs +252 -252
- package/ftm-executor/runtime/package.json +8 -8
- package/ftm-executor.yml +2 -2
- package/ftm-git/SKILL.md +441 -441
- package/ftm-git/evals/evals.json +26 -26
- package/ftm-git/evals/promptfoo.yaml +75 -75
- package/ftm-git/hooks/post-commit-experience.sh +92 -92
- package/ftm-git/references/patterns/SECRET-PATTERNS.md +104 -104
- package/ftm-git/references/protocols/REMEDIATION.md +139 -139
- package/ftm-git/scripts/pre-commit-secrets.sh +110 -110
- package/ftm-git.yml +2 -2
- package/ftm-inbox/backend/adapters/_retry.py +64 -64
- package/ftm-inbox/backend/adapters/base.py +230 -230
- package/ftm-inbox/backend/adapters/freshservice.py +104 -104
- package/ftm-inbox/backend/adapters/gmail.py +125 -125
- package/ftm-inbox/backend/adapters/jira.py +136 -136
- package/ftm-inbox/backend/adapters/registry.py +192 -192
- package/ftm-inbox/backend/adapters/slack.py +110 -110
- package/ftm-inbox/backend/db/connection.py +54 -54
- package/ftm-inbox/backend/db/schema.py +78 -78
- package/ftm-inbox/backend/executor/__init__.py +7 -7
- package/ftm-inbox/backend/executor/engine.py +149 -149
- package/ftm-inbox/backend/executor/step_runner.py +98 -98
- package/ftm-inbox/backend/main.py +103 -103
- package/ftm-inbox/backend/models/__init__.py +1 -1
- package/ftm-inbox/backend/models/unified_task.py +36 -36
- package/ftm-inbox/backend/planner/__init__.py +6 -6
- package/ftm-inbox/backend/planner/generator.py +127 -127
- package/ftm-inbox/backend/planner/schema.py +34 -34
- package/ftm-inbox/backend/requirements.txt +5 -5
- package/ftm-inbox/backend/routes/execute.py +186 -186
- package/ftm-inbox/backend/routes/health.py +52 -52
- package/ftm-inbox/backend/routes/inbox.py +68 -68
- package/ftm-inbox/backend/routes/plan.py +271 -271
- package/ftm-inbox/bin/launchagent.mjs +91 -91
- package/ftm-inbox/bin/setup.mjs +188 -188
- package/ftm-inbox/bin/start.sh +10 -10
- package/ftm-inbox/bin/status.sh +17 -17
- package/ftm-inbox/bin/stop.sh +8 -8
- package/ftm-inbox/config.example.yml +55 -55
- package/ftm-inbox/package-lock.json +2898 -2898
- package/ftm-inbox/package.json +26 -26
- package/ftm-inbox/postcss.config.js +6 -6
- package/ftm-inbox/src/app.css +199 -199
- package/ftm-inbox/src/app.html +18 -18
- package/ftm-inbox/src/lib/api.ts +166 -166
- package/ftm-inbox/src/lib/components/ExecutionLog.svelte +81 -81
- package/ftm-inbox/src/lib/components/InboxFeed.svelte +143 -143
- package/ftm-inbox/src/lib/components/PlanStep.svelte +271 -271
- package/ftm-inbox/src/lib/components/PlanView.svelte +206 -206
- package/ftm-inbox/src/lib/components/StreamPanel.svelte +99 -99
- package/ftm-inbox/src/lib/components/TaskCard.svelte +190 -190
- package/ftm-inbox/src/lib/components/ui/EmptyState.svelte +63 -63
- package/ftm-inbox/src/lib/components/ui/KawaiiCard.svelte +86 -86
- package/ftm-inbox/src/lib/components/ui/PillButton.svelte +106 -106
- package/ftm-inbox/src/lib/components/ui/StatusBadge.svelte +67 -67
- package/ftm-inbox/src/lib/components/ui/StreamDrawer.svelte +149 -149
- package/ftm-inbox/src/lib/components/ui/ThemeToggle.svelte +80 -80
- package/ftm-inbox/src/lib/theme.ts +47 -47
- package/ftm-inbox/src/routes/+layout.svelte +76 -76
- package/ftm-inbox/src/routes/+page.svelte +401 -401
- package/ftm-inbox/svelte.config.js +12 -12
- package/ftm-inbox/tailwind.config.ts +63 -63
- package/ftm-inbox/tsconfig.json +13 -13
- package/ftm-inbox/vite.config.ts +6 -6
- package/ftm-intent/SKILL.md +241 -241
- package/ftm-intent.yml +2 -2
- package/ftm-manifest.json +3794 -3794
- package/ftm-map/SKILL.md +291 -291
- package/ftm-map/scripts/db.py +712 -712
- package/ftm-map/scripts/index.py +415 -415
- package/ftm-map/scripts/parser.py +224 -224
- package/ftm-map/scripts/queries/go-tags.scm +20 -20
- package/ftm-map/scripts/queries/javascript-tags.scm +35 -35
- package/ftm-map/scripts/queries/python-tags.scm +31 -31
- package/ftm-map/scripts/queries/ruby-tags.scm +19 -19
- package/ftm-map/scripts/queries/rust-tags.scm +37 -37
- package/ftm-map/scripts/queries/typescript-tags.scm +41 -41
- package/ftm-map/scripts/query.py +301 -301
- package/ftm-map/scripts/ranker.py +377 -377
- package/ftm-map/scripts/requirements.txt +5 -5
- package/ftm-map/scripts/setup-hooks.sh +27 -27
- package/ftm-map/scripts/setup.sh +56 -56
- package/ftm-map/scripts/test_db.py +364 -364
- package/ftm-map/scripts/test_parser.py +174 -174
- package/ftm-map/scripts/test_query.py +183 -183
- package/ftm-map/scripts/test_ranker.py +199 -199
- package/ftm-map/scripts/views.py +591 -591
- package/ftm-map.yml +2 -2
- package/ftm-mind/SKILL.md +1943 -1943
- package/ftm-mind/evals/promptfoo.yaml +142 -142
- package/ftm-mind/references/blackboard-schema.md +328 -328
- package/ftm-mind/references/complexity-guide.md +110 -110
- package/ftm-mind/references/event-registry.md +319 -319
- package/ftm-mind/references/mcp-inventory.md +296 -296
- package/ftm-mind/references/protocols/COMPLEXITY-SIZING.md +72 -72
- package/ftm-mind/references/protocols/MCP-HEURISTICS.md +32 -32
- package/ftm-mind/references/protocols/PLAN-APPROVAL.md +80 -80
- package/ftm-mind/references/reflexion-protocol.md +249 -249
- package/ftm-mind/references/routing/SCENARIOS.md +22 -22
- package/ftm-mind/references/routing-scenarios.md +35 -35
- package/ftm-mind.yml +2 -2
- package/ftm-pause/SKILL.md +395 -395
- package/ftm-pause/references/protocols/SKILL-RESTORE-PROTOCOLS.md +186 -186
- package/ftm-pause/references/protocols/VALIDATION.md +80 -80
- package/ftm-pause.yml +2 -2
- package/ftm-researcher/SKILL.md +275 -275
- package/ftm-researcher/evals/agent-diversity.yaml +17 -17
- package/ftm-researcher/evals/synthesis-quality.yaml +12 -12
- package/ftm-researcher/evals/trigger-accuracy.yaml +39 -39
- package/ftm-researcher/references/adaptive-search.md +116 -116
- package/ftm-researcher/references/agent-prompts.md +193 -193
- package/ftm-researcher/references/council-integration.md +193 -193
- package/ftm-researcher/references/output-format.md +203 -203
- package/ftm-researcher/references/synthesis-pipeline.md +165 -165
- package/ftm-researcher/scripts/score_credibility.py +234 -234
- package/ftm-researcher/scripts/validate_research.py +92 -92
- package/ftm-researcher.yml +2 -2
- package/ftm-resume/SKILL.md +518 -518
- package/ftm-resume/references/protocols/VALIDATION.md +172 -172
- package/ftm-resume.yml +2 -2
- package/ftm-retro/SKILL.md +380 -380
- package/ftm-retro/references/protocols/SCORING-RUBRICS.md +89 -89
- package/ftm-retro/references/templates/REPORT-FORMAT.md +109 -109
- package/ftm-retro.yml +2 -2
- package/ftm-routine/SKILL.md +170 -170
- package/ftm-routine.yml +4 -4
- package/ftm-state/blackboard/capabilities.json +5 -5
- package/ftm-state/blackboard/capabilities.schema.json +27 -27
- package/ftm-state/blackboard/context.json +23 -23
- package/ftm-state/blackboard/experiences/index.json +9 -9
- package/ftm-state/blackboard/patterns.json +6 -6
- package/ftm-state/schemas/context.schema.json +130 -130
- package/ftm-state/schemas/experience-index.schema.json +77 -77
- package/ftm-state/schemas/experience.schema.json +78 -78
- package/ftm-state/schemas/patterns.schema.json +44 -44
- package/ftm-upgrade/SKILL.md +194 -194
- package/ftm-upgrade/scripts/check-version.sh +76 -76
- package/ftm-upgrade/scripts/upgrade.sh +143 -143
- package/ftm-upgrade.yml +2 -2
- package/ftm-verify.yml +2 -2
- package/ftm.yml +2 -2
- package/hooks/ftm-blackboard-enforcer.sh +93 -93
- package/hooks/ftm-discovery-reminder.sh +90 -90
- package/hooks/ftm-drafts-gate.sh +61 -61
- package/hooks/ftm-event-logger.mjs +107 -107
- package/hooks/ftm-map-autodetect.sh +79 -79
- package/hooks/ftm-pending-sync-check.sh +22 -22
- package/hooks/ftm-plan-gate.sh +92 -92
- package/hooks/ftm-post-commit-trigger.sh +57 -57
- package/hooks/settings-template.json +81 -81
- package/install.sh +363 -363
- package/package.json +84 -84
- package/uninstall.sh +25 -25
package/ftm-map/scripts/index.py
CHANGED
|
@@ -1,415 +1,415 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""ftm-map indexer: builds the code knowledge graph from source files.
|
|
3
|
-
|
|
4
|
-
Two-phase indexing:
|
|
5
|
-
Phase 1 — Parse each file with tree-sitter, insert file/symbol/ref rows.
|
|
6
|
-
Phase 2 — Materialize file_edges with Aider-style weight heuristics and
|
|
7
|
-
symbol_edges via enclosing-scope resolution.
|
|
8
|
-
"""
|
|
9
|
-
|
|
10
|
-
import argparse
|
|
11
|
-
import json
|
|
12
|
-
import math
|
|
13
|
-
import os
|
|
14
|
-
import re
|
|
15
|
-
import subprocess
|
|
16
|
-
import sys
|
|
17
|
-
import time
|
|
18
|
-
from collections import Counter
|
|
19
|
-
from datetime import datetime, timezone
|
|
20
|
-
from pathlib import Path
|
|
21
|
-
|
|
22
|
-
# Add scripts dir to path for sibling imports
|
|
23
|
-
sys.path.insert(0, os.path.dirname(__file__))
|
|
24
|
-
|
|
25
|
-
from db import (
|
|
26
|
-
get_connection,
|
|
27
|
-
add_file,
|
|
28
|
-
add_symbol,
|
|
29
|
-
add_reference,
|
|
30
|
-
remove_file,
|
|
31
|
-
get_stats,
|
|
32
|
-
rebuild_symbol_edges,
|
|
33
|
-
)
|
|
34
|
-
from parser import get_tags, detect_language, EXTENSION_MAP, compute_content_hash
|
|
35
|
-
|
|
36
|
-
META_REGISTRY = os.path.expanduser("~/.claude/ftm-state/maps/index.json")
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
# ---------------------------------------------------------------------------
|
|
40
|
-
# File discovery
|
|
41
|
-
# ---------------------------------------------------------------------------
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
def discover_files(project_root: str) -> list[str]:
|
|
45
|
-
"""Get tracked source files using git ls-files.
|
|
46
|
-
|
|
47
|
-
Filters to files whose extensions are in EXTENSION_MAP so only
|
|
48
|
-
tree-sitter-parseable files are returned. Returns absolute paths.
|
|
49
|
-
"""
|
|
50
|
-
result = subprocess.run(
|
|
51
|
-
["git", "ls-files"],
|
|
52
|
-
capture_output=True,
|
|
53
|
-
text=True,
|
|
54
|
-
cwd=project_root,
|
|
55
|
-
)
|
|
56
|
-
if result.returncode != 0:
|
|
57
|
-
print(
|
|
58
|
-
f"Error: git ls-files failed: {result.stderr.strip()}",
|
|
59
|
-
file=sys.stderr,
|
|
60
|
-
)
|
|
61
|
-
return []
|
|
62
|
-
|
|
63
|
-
supported_exts = set(EXTENSION_MAP.keys())
|
|
64
|
-
files = []
|
|
65
|
-
for line in result.stdout.strip().split("\n"):
|
|
66
|
-
if not line:
|
|
67
|
-
continue
|
|
68
|
-
ext = Path(line).suffix.lower()
|
|
69
|
-
if ext in supported_exts:
|
|
70
|
-
files.append(os.path.join(project_root, line))
|
|
71
|
-
return files
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
# ---------------------------------------------------------------------------
|
|
75
|
-
# Core indexing logic
|
|
76
|
-
# ---------------------------------------------------------------------------
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
def index_files(conn, files: list[str], project_root: str) -> dict:
|
|
80
|
-
"""Parse and insert files, symbols, references, then materialize edges.
|
|
81
|
-
|
|
82
|
-
Phase 1 — For each file: read source, compute hash, insert file row,
|
|
83
|
-
extract def/ref tags via tree-sitter, insert symbol and ref rows.
|
|
84
|
-
Phase 2 — Build file_edges with Aider weight heuristics (long descriptive
|
|
85
|
-
names 10x, private 0.1x, overloaded 0.1x, sqrt-dampened counts).
|
|
86
|
-
Then rebuild symbol_edges via enclosing-scope resolution.
|
|
87
|
-
|
|
88
|
-
Returns a dict with symbols, references, file_edges, symbol_edges counts.
|
|
89
|
-
"""
|
|
90
|
-
total_symbols = 0
|
|
91
|
-
total_refs = 0
|
|
92
|
-
|
|
93
|
-
# ------------------------------------------------------------------
|
|
94
|
-
# Phase 1: parse each file and insert rows
|
|
95
|
-
# ------------------------------------------------------------------
|
|
96
|
-
for fpath in files:
|
|
97
|
-
if not os.path.exists(fpath):
|
|
98
|
-
print(f"Warning: file not found, skipping: {fpath}", file=sys.stderr)
|
|
99
|
-
continue
|
|
100
|
-
|
|
101
|
-
rel_path = os.path.relpath(fpath, project_root)
|
|
102
|
-
lang = detect_language(fpath)
|
|
103
|
-
mtime = os.path.getmtime(fpath)
|
|
104
|
-
|
|
105
|
-
# Stream-friendly: read once, extract metadata, then release
|
|
106
|
-
try:
|
|
107
|
-
with open(fpath, "r", encoding="utf-8", errors="replace") as fh:
|
|
108
|
-
source = fh.read()
|
|
109
|
-
except (IOError, OSError) as exc:
|
|
110
|
-
print(f"Warning: Cannot read {fpath}: {exc}", file=sys.stderr)
|
|
111
|
-
continue
|
|
112
|
-
|
|
113
|
-
line_count = source.count("\n") + 1
|
|
114
|
-
content_hash = compute_content_hash(source)
|
|
115
|
-
|
|
116
|
-
# Insert file record
|
|
117
|
-
file_id = add_file(
|
|
118
|
-
conn, rel_path, lang, mtime,
|
|
119
|
-
hash=content_hash, line_count=line_count,
|
|
120
|
-
)
|
|
121
|
-
|
|
122
|
-
# Extract def/ref tags via tree-sitter
|
|
123
|
-
tags = get_tags(fpath, rel_path)
|
|
124
|
-
|
|
125
|
-
for tag in tags:
|
|
126
|
-
if tag.kind == "def":
|
|
127
|
-
add_symbol(conn, file_id, tag.name, "definition", tag.line, signature=None)
|
|
128
|
-
total_symbols += 1
|
|
129
|
-
elif tag.kind == "ref":
|
|
130
|
-
add_reference(conn, file_id, tag.name, tag.line, kind="call")
|
|
131
|
-
total_refs += 1
|
|
132
|
-
|
|
133
|
-
# ------------------------------------------------------------------
|
|
134
|
-
# Phase 2: materialize edges
|
|
135
|
-
# ------------------------------------------------------------------
|
|
136
|
-
|
|
137
|
-
# Build defines map: ident -> set of file_ids that define it
|
|
138
|
-
defines = {}
|
|
139
|
-
for row in conn.execute("SELECT name, file_id FROM symbols").fetchall():
|
|
140
|
-
defines.setdefault(row["name"], set()).add(row["file_id"])
|
|
141
|
-
|
|
142
|
-
# Build references map: ident -> list of file_ids that reference it
|
|
143
|
-
references_map = {}
|
|
144
|
-
for row in conn.execute("SELECT symbol_name, file_id FROM refs").fetchall():
|
|
145
|
-
references_map.setdefault(row["symbol_name"], []).append(row["file_id"])
|
|
146
|
-
|
|
147
|
-
# Materialize file_edges with Aider weight heuristics
|
|
148
|
-
conn.execute("DELETE FROM file_edges")
|
|
149
|
-
|
|
150
|
-
for ident, ref_file_ids in references_map.items():
|
|
151
|
-
definers = defines.get(ident, set())
|
|
152
|
-
if not definers:
|
|
153
|
-
continue
|
|
154
|
-
|
|
155
|
-
# Aider weight heuristics
|
|
156
|
-
mul = 1.0
|
|
157
|
-
# Long descriptive names (camelCase or snake_case, >= 8 chars) weighted higher
|
|
158
|
-
if len(ident) >= 8 and re.match(r"[a-z_]+[A-Z]|[a-z]+_[a-z]", ident):
|
|
159
|
-
mul *= 10
|
|
160
|
-
# Private names weighted lower
|
|
161
|
-
if ident.startswith("_"):
|
|
162
|
-
mul *= 0.1
|
|
163
|
-
# Overloaded names (defined in many files) weighted lower
|
|
164
|
-
if len(definers) >= 5:
|
|
165
|
-
mul *= 0.1
|
|
166
|
-
|
|
167
|
-
# Count refs per file, then create weighted edges
|
|
168
|
-
ref_counts = Counter(ref_file_ids)
|
|
169
|
-
|
|
170
|
-
for ref_file_id, count in ref_counts.items():
|
|
171
|
-
weight = mul * math.sqrt(count)
|
|
172
|
-
for def_file_id in definers:
|
|
173
|
-
if ref_file_id != def_file_id: # No self-edges
|
|
174
|
-
conn.execute(
|
|
175
|
-
"""INSERT INTO file_edges (source_file_id, target_file_id, weight)
|
|
176
|
-
VALUES (?, ?, ?)
|
|
177
|
-
ON CONFLICT(source_file_id, target_file_id)
|
|
178
|
-
DO UPDATE SET weight = MAX(weight, excluded.weight)""",
|
|
179
|
-
(ref_file_id, def_file_id, weight),
|
|
180
|
-
)
|
|
181
|
-
|
|
182
|
-
# Materialize symbol_edges via enclosing-scope resolution
|
|
183
|
-
rebuild_symbol_edges(conn)
|
|
184
|
-
|
|
185
|
-
# Gather edge counts
|
|
186
|
-
file_edge_count = conn.execute("SELECT COUNT(*) FROM file_edges").fetchone()[0]
|
|
187
|
-
symbol_edge_count = conn.execute("SELECT COUNT(*) FROM symbol_edges").fetchone()[0]
|
|
188
|
-
|
|
189
|
-
return {
|
|
190
|
-
"symbols": total_symbols,
|
|
191
|
-
"references": total_refs,
|
|
192
|
-
"file_edges": file_edge_count,
|
|
193
|
-
"symbol_edges": symbol_edge_count,
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
# ---------------------------------------------------------------------------
|
|
198
|
-
# Bootstrap mode
|
|
199
|
-
# ---------------------------------------------------------------------------
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
def bootstrap(project_root: str) -> None:
|
|
203
|
-
"""Full scan: (re)build the entire code graph for *project_root*."""
|
|
204
|
-
abs_root = os.path.abspath(project_root)
|
|
205
|
-
start = time.time()
|
|
206
|
-
|
|
207
|
-
files = discover_files(abs_root)
|
|
208
|
-
if not files:
|
|
209
|
-
print(
|
|
210
|
-
json.dumps({"error": "No parseable source files found in git repository"}),
|
|
211
|
-
file=sys.stderr,
|
|
212
|
-
)
|
|
213
|
-
sys.exit(1)
|
|
214
|
-
|
|
215
|
-
conn = get_connection(abs_root)
|
|
216
|
-
try:
|
|
217
|
-
# Full rebuild — clear all tables. CASCADE handles symbols, refs, edges.
|
|
218
|
-
# FTS5 rows must be removed before symbol rows (content= table).
|
|
219
|
-
symbol_ids = [
|
|
220
|
-
row[0] for row in conn.execute("SELECT id FROM symbols").fetchall()
|
|
221
|
-
]
|
|
222
|
-
for sid in symbol_ids:
|
|
223
|
-
conn.execute("DELETE FROM symbols_fts WHERE rowid=?", (sid,))
|
|
224
|
-
conn.execute("DELETE FROM files")
|
|
225
|
-
|
|
226
|
-
stats = index_files(conn, files, abs_root)
|
|
227
|
-
conn.commit()
|
|
228
|
-
|
|
229
|
-
duration = time.time() - start
|
|
230
|
-
result = {
|
|
231
|
-
"mode": "bootstrap",
|
|
232
|
-
"files_parsed": len(files),
|
|
233
|
-
"symbols": stats["symbols"],
|
|
234
|
-
"references": stats["references"],
|
|
235
|
-
"file_edges": stats["file_edges"],
|
|
236
|
-
"symbol_edges": stats["symbol_edges"],
|
|
237
|
-
"duration_s": round(duration, 2),
|
|
238
|
-
}
|
|
239
|
-
print(json.dumps(result))
|
|
240
|
-
update_meta_registry(abs_root, stats["symbols"])
|
|
241
|
-
except Exception as exc: # noqa: BLE001
|
|
242
|
-
print(f"Error during bootstrap: {exc}", file=sys.stderr)
|
|
243
|
-
conn.rollback()
|
|
244
|
-
conn.close()
|
|
245
|
-
sys.exit(1)
|
|
246
|
-
finally:
|
|
247
|
-
conn.close()
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
# ---------------------------------------------------------------------------
|
|
251
|
-
# Incremental mode
|
|
252
|
-
# ---------------------------------------------------------------------------
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
def incremental(project_root: str, files_str: str) -> None:
|
|
256
|
-
"""Incremental update: re-index only the specified files.
|
|
257
|
-
|
|
258
|
-
*files_str* is a comma-separated list of file paths (relative or absolute).
|
|
259
|
-
Old file/symbol/ref/edge data for each file is cascade-deleted via
|
|
260
|
-
remove_file() before re-parsing so stale entries do not accumulate.
|
|
261
|
-
All edges are rebuilt since changes can ripple across files.
|
|
262
|
-
"""
|
|
263
|
-
abs_root = os.path.abspath(project_root)
|
|
264
|
-
start = time.time()
|
|
265
|
-
|
|
266
|
-
raw_files = [f.strip() for f in files_str.split(",") if f.strip()]
|
|
267
|
-
abs_files = [
|
|
268
|
-
f if os.path.isabs(f) else os.path.join(abs_root, f) for f in raw_files
|
|
269
|
-
]
|
|
270
|
-
|
|
271
|
-
conn = get_connection(abs_root)
|
|
272
|
-
try:
|
|
273
|
-
# Remove stale data for all targeted files (cascading delete).
|
|
274
|
-
for fpath in abs_files:
|
|
275
|
-
rel_path = os.path.relpath(fpath, abs_root)
|
|
276
|
-
remove_file(conn, rel_path)
|
|
277
|
-
|
|
278
|
-
existing_files = [f for f in abs_files if os.path.exists(f)]
|
|
279
|
-
if not existing_files:
|
|
280
|
-
print(
|
|
281
|
-
json.dumps({"error": "None of the specified files exist"}),
|
|
282
|
-
file=sys.stderr,
|
|
283
|
-
)
|
|
284
|
-
conn.close()
|
|
285
|
-
sys.exit(1)
|
|
286
|
-
|
|
287
|
-
stats = index_files(conn, existing_files, abs_root)
|
|
288
|
-
conn.commit()
|
|
289
|
-
|
|
290
|
-
db_stats = get_stats(conn)
|
|
291
|
-
duration = time.time() - start
|
|
292
|
-
result = {
|
|
293
|
-
"mode": "incremental",
|
|
294
|
-
"files_parsed": len(existing_files),
|
|
295
|
-
"symbols": stats["symbols"],
|
|
296
|
-
"references": stats["references"],
|
|
297
|
-
"file_edges": stats["file_edges"],
|
|
298
|
-
"symbol_edges": stats["symbol_edges"],
|
|
299
|
-
"duration_s": round(duration, 2),
|
|
300
|
-
}
|
|
301
|
-
print(json.dumps(result))
|
|
302
|
-
update_meta_registry(abs_root, db_stats["symbol_count"])
|
|
303
|
-
except Exception as exc: # noqa: BLE001
|
|
304
|
-
print(f"Error during incremental update: {exc}", file=sys.stderr)
|
|
305
|
-
conn.rollback()
|
|
306
|
-
conn.close()
|
|
307
|
-
sys.exit(1)
|
|
308
|
-
finally:
|
|
309
|
-
conn.close()
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
# ---------------------------------------------------------------------------
|
|
313
|
-
# Meta-registry management
|
|
314
|
-
# ---------------------------------------------------------------------------
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
def update_meta_registry(project_root: str, symbol_count: int) -> None:
|
|
318
|
-
"""Upsert project entry in the global meta-registry at META_REGISTRY."""
|
|
319
|
-
registry_dir = os.path.dirname(META_REGISTRY)
|
|
320
|
-
os.makedirs(registry_dir, exist_ok=True)
|
|
321
|
-
|
|
322
|
-
registry: dict = {"projects": []}
|
|
323
|
-
if os.path.exists(META_REGISTRY):
|
|
324
|
-
try:
|
|
325
|
-
with open(META_REGISTRY) as fh:
|
|
326
|
-
registry = json.load(fh)
|
|
327
|
-
except (json.JSONDecodeError, IOError):
|
|
328
|
-
# Corrupt or unreadable registry — start fresh.
|
|
329
|
-
registry = {"projects": []}
|
|
330
|
-
|
|
331
|
-
abs_root = os.path.abspath(project_root)
|
|
332
|
-
db_path = os.path.join(abs_root, ".ftm-map", "map.db")
|
|
333
|
-
now = datetime.now(timezone.utc).isoformat()
|
|
334
|
-
|
|
335
|
-
found = False
|
|
336
|
-
for proj in registry["projects"]:
|
|
337
|
-
if proj.get("path") == abs_root:
|
|
338
|
-
proj["last_indexed"] = now
|
|
339
|
-
proj["symbol_count"] = symbol_count
|
|
340
|
-
found = True
|
|
341
|
-
break
|
|
342
|
-
|
|
343
|
-
if not found:
|
|
344
|
-
registry["projects"].append(
|
|
345
|
-
{
|
|
346
|
-
"path": abs_root,
|
|
347
|
-
"db_path": db_path,
|
|
348
|
-
"last_indexed": now,
|
|
349
|
-
"symbol_count": symbol_count,
|
|
350
|
-
}
|
|
351
|
-
)
|
|
352
|
-
|
|
353
|
-
with open(META_REGISTRY, "w") as fh:
|
|
354
|
-
json.dump(registry, fh, indent=2)
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
# ---------------------------------------------------------------------------
|
|
358
|
-
# CLI entry point
|
|
359
|
-
# ---------------------------------------------------------------------------
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
def main() -> None:
|
|
363
|
-
parser = argparse.ArgumentParser(
|
|
364
|
-
description="ftm-map indexer — builds the code knowledge graph from source files.",
|
|
365
|
-
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
366
|
-
epilog=(
|
|
367
|
-
"Examples:\n"
|
|
368
|
-
" python3 index.py --bootstrap /path/to/project\n"
|
|
369
|
-
" python3 index.py --incremental --files src/foo.ts,src/bar.py\n"
|
|
370
|
-
" python3 index.py --incremental --files src/foo.ts --project-root /path/to/project\n"
|
|
371
|
-
),
|
|
372
|
-
)
|
|
373
|
-
|
|
374
|
-
mode = parser.add_mutually_exclusive_group(required=True)
|
|
375
|
-
mode.add_argument(
|
|
376
|
-
"--bootstrap",
|
|
377
|
-
metavar="PROJECT_ROOT",
|
|
378
|
-
help="Full scan: index all tracked source files in PROJECT_ROOT.",
|
|
379
|
-
)
|
|
380
|
-
mode.add_argument(
|
|
381
|
-
"--incremental",
|
|
382
|
-
action="store_true",
|
|
383
|
-
help="Incremental update: re-index only the files given by --files.",
|
|
384
|
-
)
|
|
385
|
-
|
|
386
|
-
parser.add_argument(
|
|
387
|
-
"--files",
|
|
388
|
-
metavar="FILE_LIST",
|
|
389
|
-
help="Comma-separated list of files to re-index (required for --incremental).",
|
|
390
|
-
)
|
|
391
|
-
parser.add_argument(
|
|
392
|
-
"--project-root",
|
|
393
|
-
metavar="PATH",
|
|
394
|
-
default=None,
|
|
395
|
-
help=(
|
|
396
|
-
"Project root used to locate the database for incremental mode. "
|
|
397
|
-
"Defaults to the current working directory."
|
|
398
|
-
),
|
|
399
|
-
)
|
|
400
|
-
|
|
401
|
-
args = parser.parse_args()
|
|
402
|
-
|
|
403
|
-
if args.bootstrap:
|
|
404
|
-
bootstrap(args.bootstrap)
|
|
405
|
-
else:
|
|
406
|
-
# Incremental mode
|
|
407
|
-
if not args.files:
|
|
408
|
-
print("Error: --incremental requires --files", file=sys.stderr)
|
|
409
|
-
sys.exit(1)
|
|
410
|
-
project_root = args.project_root or os.getcwd()
|
|
411
|
-
incremental(project_root, args.files)
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
if __name__ == "__main__":
|
|
415
|
-
main()
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""ftm-map indexer: builds the code knowledge graph from source files.
|
|
3
|
+
|
|
4
|
+
Two-phase indexing:
|
|
5
|
+
Phase 1 — Parse each file with tree-sitter, insert file/symbol/ref rows.
|
|
6
|
+
Phase 2 — Materialize file_edges with Aider-style weight heuristics and
|
|
7
|
+
symbol_edges via enclosing-scope resolution.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import json
|
|
12
|
+
import math
|
|
13
|
+
import os
|
|
14
|
+
import re
|
|
15
|
+
import subprocess
|
|
16
|
+
import sys
|
|
17
|
+
import time
|
|
18
|
+
from collections import Counter
|
|
19
|
+
from datetime import datetime, timezone
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
# Add scripts dir to path for sibling imports
|
|
23
|
+
sys.path.insert(0, os.path.dirname(__file__))
|
|
24
|
+
|
|
25
|
+
from db import (
|
|
26
|
+
get_connection,
|
|
27
|
+
add_file,
|
|
28
|
+
add_symbol,
|
|
29
|
+
add_reference,
|
|
30
|
+
remove_file,
|
|
31
|
+
get_stats,
|
|
32
|
+
rebuild_symbol_edges,
|
|
33
|
+
)
|
|
34
|
+
from parser import get_tags, detect_language, EXTENSION_MAP, compute_content_hash
|
|
35
|
+
|
|
36
|
+
META_REGISTRY = os.path.expanduser("~/.claude/ftm-state/maps/index.json")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
# ---------------------------------------------------------------------------
|
|
40
|
+
# File discovery
|
|
41
|
+
# ---------------------------------------------------------------------------
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def discover_files(project_root: str) -> list[str]:
|
|
45
|
+
"""Get tracked source files using git ls-files.
|
|
46
|
+
|
|
47
|
+
Filters to files whose extensions are in EXTENSION_MAP so only
|
|
48
|
+
tree-sitter-parseable files are returned. Returns absolute paths.
|
|
49
|
+
"""
|
|
50
|
+
result = subprocess.run(
|
|
51
|
+
["git", "ls-files"],
|
|
52
|
+
capture_output=True,
|
|
53
|
+
text=True,
|
|
54
|
+
cwd=project_root,
|
|
55
|
+
)
|
|
56
|
+
if result.returncode != 0:
|
|
57
|
+
print(
|
|
58
|
+
f"Error: git ls-files failed: {result.stderr.strip()}",
|
|
59
|
+
file=sys.stderr,
|
|
60
|
+
)
|
|
61
|
+
return []
|
|
62
|
+
|
|
63
|
+
supported_exts = set(EXTENSION_MAP.keys())
|
|
64
|
+
files = []
|
|
65
|
+
for line in result.stdout.strip().split("\n"):
|
|
66
|
+
if not line:
|
|
67
|
+
continue
|
|
68
|
+
ext = Path(line).suffix.lower()
|
|
69
|
+
if ext in supported_exts:
|
|
70
|
+
files.append(os.path.join(project_root, line))
|
|
71
|
+
return files
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
# ---------------------------------------------------------------------------
|
|
75
|
+
# Core indexing logic
|
|
76
|
+
# ---------------------------------------------------------------------------
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def index_files(conn, files: list[str], project_root: str) -> dict:
|
|
80
|
+
"""Parse and insert files, symbols, references, then materialize edges.
|
|
81
|
+
|
|
82
|
+
Phase 1 — For each file: read source, compute hash, insert file row,
|
|
83
|
+
extract def/ref tags via tree-sitter, insert symbol and ref rows.
|
|
84
|
+
Phase 2 — Build file_edges with Aider weight heuristics (long descriptive
|
|
85
|
+
names 10x, private 0.1x, overloaded 0.1x, sqrt-dampened counts).
|
|
86
|
+
Then rebuild symbol_edges via enclosing-scope resolution.
|
|
87
|
+
|
|
88
|
+
Returns a dict with symbols, references, file_edges, symbol_edges counts.
|
|
89
|
+
"""
|
|
90
|
+
total_symbols = 0
|
|
91
|
+
total_refs = 0
|
|
92
|
+
|
|
93
|
+
# ------------------------------------------------------------------
|
|
94
|
+
# Phase 1: parse each file and insert rows
|
|
95
|
+
# ------------------------------------------------------------------
|
|
96
|
+
for fpath in files:
|
|
97
|
+
if not os.path.exists(fpath):
|
|
98
|
+
print(f"Warning: file not found, skipping: {fpath}", file=sys.stderr)
|
|
99
|
+
continue
|
|
100
|
+
|
|
101
|
+
rel_path = os.path.relpath(fpath, project_root)
|
|
102
|
+
lang = detect_language(fpath)
|
|
103
|
+
mtime = os.path.getmtime(fpath)
|
|
104
|
+
|
|
105
|
+
# Stream-friendly: read once, extract metadata, then release
|
|
106
|
+
try:
|
|
107
|
+
with open(fpath, "r", encoding="utf-8", errors="replace") as fh:
|
|
108
|
+
source = fh.read()
|
|
109
|
+
except (IOError, OSError) as exc:
|
|
110
|
+
print(f"Warning: Cannot read {fpath}: {exc}", file=sys.stderr)
|
|
111
|
+
continue
|
|
112
|
+
|
|
113
|
+
line_count = source.count("\n") + 1
|
|
114
|
+
content_hash = compute_content_hash(source)
|
|
115
|
+
|
|
116
|
+
# Insert file record
|
|
117
|
+
file_id = add_file(
|
|
118
|
+
conn, rel_path, lang, mtime,
|
|
119
|
+
hash=content_hash, line_count=line_count,
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
# Extract def/ref tags via tree-sitter
|
|
123
|
+
tags = get_tags(fpath, rel_path)
|
|
124
|
+
|
|
125
|
+
for tag in tags:
|
|
126
|
+
if tag.kind == "def":
|
|
127
|
+
add_symbol(conn, file_id, tag.name, "definition", tag.line, signature=None)
|
|
128
|
+
total_symbols += 1
|
|
129
|
+
elif tag.kind == "ref":
|
|
130
|
+
add_reference(conn, file_id, tag.name, tag.line, kind="call")
|
|
131
|
+
total_refs += 1
|
|
132
|
+
|
|
133
|
+
# ------------------------------------------------------------------
|
|
134
|
+
# Phase 2: materialize edges
|
|
135
|
+
# ------------------------------------------------------------------
|
|
136
|
+
|
|
137
|
+
# Build defines map: ident -> set of file_ids that define it
|
|
138
|
+
defines = {}
|
|
139
|
+
for row in conn.execute("SELECT name, file_id FROM symbols").fetchall():
|
|
140
|
+
defines.setdefault(row["name"], set()).add(row["file_id"])
|
|
141
|
+
|
|
142
|
+
# Build references map: ident -> list of file_ids that reference it
|
|
143
|
+
references_map = {}
|
|
144
|
+
for row in conn.execute("SELECT symbol_name, file_id FROM refs").fetchall():
|
|
145
|
+
references_map.setdefault(row["symbol_name"], []).append(row["file_id"])
|
|
146
|
+
|
|
147
|
+
# Materialize file_edges with Aider weight heuristics
|
|
148
|
+
conn.execute("DELETE FROM file_edges")
|
|
149
|
+
|
|
150
|
+
for ident, ref_file_ids in references_map.items():
|
|
151
|
+
definers = defines.get(ident, set())
|
|
152
|
+
if not definers:
|
|
153
|
+
continue
|
|
154
|
+
|
|
155
|
+
# Aider weight heuristics
|
|
156
|
+
mul = 1.0
|
|
157
|
+
# Long descriptive names (camelCase or snake_case, >= 8 chars) weighted higher
|
|
158
|
+
if len(ident) >= 8 and re.match(r"[a-z_]+[A-Z]|[a-z]+_[a-z]", ident):
|
|
159
|
+
mul *= 10
|
|
160
|
+
# Private names weighted lower
|
|
161
|
+
if ident.startswith("_"):
|
|
162
|
+
mul *= 0.1
|
|
163
|
+
# Overloaded names (defined in many files) weighted lower
|
|
164
|
+
if len(definers) >= 5:
|
|
165
|
+
mul *= 0.1
|
|
166
|
+
|
|
167
|
+
# Count refs per file, then create weighted edges
|
|
168
|
+
ref_counts = Counter(ref_file_ids)
|
|
169
|
+
|
|
170
|
+
for ref_file_id, count in ref_counts.items():
|
|
171
|
+
weight = mul * math.sqrt(count)
|
|
172
|
+
for def_file_id in definers:
|
|
173
|
+
if ref_file_id != def_file_id: # No self-edges
|
|
174
|
+
conn.execute(
|
|
175
|
+
"""INSERT INTO file_edges (source_file_id, target_file_id, weight)
|
|
176
|
+
VALUES (?, ?, ?)
|
|
177
|
+
ON CONFLICT(source_file_id, target_file_id)
|
|
178
|
+
DO UPDATE SET weight = MAX(weight, excluded.weight)""",
|
|
179
|
+
(ref_file_id, def_file_id, weight),
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
# Materialize symbol_edges via enclosing-scope resolution
|
|
183
|
+
rebuild_symbol_edges(conn)
|
|
184
|
+
|
|
185
|
+
# Gather edge counts
|
|
186
|
+
file_edge_count = conn.execute("SELECT COUNT(*) FROM file_edges").fetchone()[0]
|
|
187
|
+
symbol_edge_count = conn.execute("SELECT COUNT(*) FROM symbol_edges").fetchone()[0]
|
|
188
|
+
|
|
189
|
+
return {
|
|
190
|
+
"symbols": total_symbols,
|
|
191
|
+
"references": total_refs,
|
|
192
|
+
"file_edges": file_edge_count,
|
|
193
|
+
"symbol_edges": symbol_edge_count,
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
# ---------------------------------------------------------------------------
|
|
198
|
+
# Bootstrap mode
|
|
199
|
+
# ---------------------------------------------------------------------------
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def bootstrap(project_root: str) -> None:
|
|
203
|
+
"""Full scan: (re)build the entire code graph for *project_root*."""
|
|
204
|
+
abs_root = os.path.abspath(project_root)
|
|
205
|
+
start = time.time()
|
|
206
|
+
|
|
207
|
+
files = discover_files(abs_root)
|
|
208
|
+
if not files:
|
|
209
|
+
print(
|
|
210
|
+
json.dumps({"error": "No parseable source files found in git repository"}),
|
|
211
|
+
file=sys.stderr,
|
|
212
|
+
)
|
|
213
|
+
sys.exit(1)
|
|
214
|
+
|
|
215
|
+
conn = get_connection(abs_root)
|
|
216
|
+
try:
|
|
217
|
+
# Full rebuild — clear all tables. CASCADE handles symbols, refs, edges.
|
|
218
|
+
# FTS5 rows must be removed before symbol rows (content= table).
|
|
219
|
+
symbol_ids = [
|
|
220
|
+
row[0] for row in conn.execute("SELECT id FROM symbols").fetchall()
|
|
221
|
+
]
|
|
222
|
+
for sid in symbol_ids:
|
|
223
|
+
conn.execute("DELETE FROM symbols_fts WHERE rowid=?", (sid,))
|
|
224
|
+
conn.execute("DELETE FROM files")
|
|
225
|
+
|
|
226
|
+
stats = index_files(conn, files, abs_root)
|
|
227
|
+
conn.commit()
|
|
228
|
+
|
|
229
|
+
duration = time.time() - start
|
|
230
|
+
result = {
|
|
231
|
+
"mode": "bootstrap",
|
|
232
|
+
"files_parsed": len(files),
|
|
233
|
+
"symbols": stats["symbols"],
|
|
234
|
+
"references": stats["references"],
|
|
235
|
+
"file_edges": stats["file_edges"],
|
|
236
|
+
"symbol_edges": stats["symbol_edges"],
|
|
237
|
+
"duration_s": round(duration, 2),
|
|
238
|
+
}
|
|
239
|
+
print(json.dumps(result))
|
|
240
|
+
update_meta_registry(abs_root, stats["symbols"])
|
|
241
|
+
except Exception as exc: # noqa: BLE001
|
|
242
|
+
print(f"Error during bootstrap: {exc}", file=sys.stderr)
|
|
243
|
+
conn.rollback()
|
|
244
|
+
conn.close()
|
|
245
|
+
sys.exit(1)
|
|
246
|
+
finally:
|
|
247
|
+
conn.close()
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
# ---------------------------------------------------------------------------
|
|
251
|
+
# Incremental mode
|
|
252
|
+
# ---------------------------------------------------------------------------
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def incremental(project_root: str, files_str: str) -> None:
|
|
256
|
+
"""Incremental update: re-index only the specified files.
|
|
257
|
+
|
|
258
|
+
*files_str* is a comma-separated list of file paths (relative or absolute).
|
|
259
|
+
Old file/symbol/ref/edge data for each file is cascade-deleted via
|
|
260
|
+
remove_file() before re-parsing so stale entries do not accumulate.
|
|
261
|
+
All edges are rebuilt since changes can ripple across files.
|
|
262
|
+
"""
|
|
263
|
+
abs_root = os.path.abspath(project_root)
|
|
264
|
+
start = time.time()
|
|
265
|
+
|
|
266
|
+
raw_files = [f.strip() for f in files_str.split(",") if f.strip()]
|
|
267
|
+
abs_files = [
|
|
268
|
+
f if os.path.isabs(f) else os.path.join(abs_root, f) for f in raw_files
|
|
269
|
+
]
|
|
270
|
+
|
|
271
|
+
conn = get_connection(abs_root)
|
|
272
|
+
try:
|
|
273
|
+
# Remove stale data for all targeted files (cascading delete).
|
|
274
|
+
for fpath in abs_files:
|
|
275
|
+
rel_path = os.path.relpath(fpath, abs_root)
|
|
276
|
+
remove_file(conn, rel_path)
|
|
277
|
+
|
|
278
|
+
existing_files = [f for f in abs_files if os.path.exists(f)]
|
|
279
|
+
if not existing_files:
|
|
280
|
+
print(
|
|
281
|
+
json.dumps({"error": "None of the specified files exist"}),
|
|
282
|
+
file=sys.stderr,
|
|
283
|
+
)
|
|
284
|
+
conn.close()
|
|
285
|
+
sys.exit(1)
|
|
286
|
+
|
|
287
|
+
stats = index_files(conn, existing_files, abs_root)
|
|
288
|
+
conn.commit()
|
|
289
|
+
|
|
290
|
+
db_stats = get_stats(conn)
|
|
291
|
+
duration = time.time() - start
|
|
292
|
+
result = {
|
|
293
|
+
"mode": "incremental",
|
|
294
|
+
"files_parsed": len(existing_files),
|
|
295
|
+
"symbols": stats["symbols"],
|
|
296
|
+
"references": stats["references"],
|
|
297
|
+
"file_edges": stats["file_edges"],
|
|
298
|
+
"symbol_edges": stats["symbol_edges"],
|
|
299
|
+
"duration_s": round(duration, 2),
|
|
300
|
+
}
|
|
301
|
+
print(json.dumps(result))
|
|
302
|
+
update_meta_registry(abs_root, db_stats["symbol_count"])
|
|
303
|
+
except Exception as exc: # noqa: BLE001
|
|
304
|
+
print(f"Error during incremental update: {exc}", file=sys.stderr)
|
|
305
|
+
conn.rollback()
|
|
306
|
+
conn.close()
|
|
307
|
+
sys.exit(1)
|
|
308
|
+
finally:
|
|
309
|
+
conn.close()
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
# ---------------------------------------------------------------------------
|
|
313
|
+
# Meta-registry management
|
|
314
|
+
# ---------------------------------------------------------------------------
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def update_meta_registry(project_root: str, symbol_count: int) -> None:
|
|
318
|
+
"""Upsert project entry in the global meta-registry at META_REGISTRY."""
|
|
319
|
+
registry_dir = os.path.dirname(META_REGISTRY)
|
|
320
|
+
os.makedirs(registry_dir, exist_ok=True)
|
|
321
|
+
|
|
322
|
+
registry: dict = {"projects": []}
|
|
323
|
+
if os.path.exists(META_REGISTRY):
|
|
324
|
+
try:
|
|
325
|
+
with open(META_REGISTRY) as fh:
|
|
326
|
+
registry = json.load(fh)
|
|
327
|
+
except (json.JSONDecodeError, IOError):
|
|
328
|
+
# Corrupt or unreadable registry — start fresh.
|
|
329
|
+
registry = {"projects": []}
|
|
330
|
+
|
|
331
|
+
abs_root = os.path.abspath(project_root)
|
|
332
|
+
db_path = os.path.join(abs_root, ".ftm-map", "map.db")
|
|
333
|
+
now = datetime.now(timezone.utc).isoformat()
|
|
334
|
+
|
|
335
|
+
found = False
|
|
336
|
+
for proj in registry["projects"]:
|
|
337
|
+
if proj.get("path") == abs_root:
|
|
338
|
+
proj["last_indexed"] = now
|
|
339
|
+
proj["symbol_count"] = symbol_count
|
|
340
|
+
found = True
|
|
341
|
+
break
|
|
342
|
+
|
|
343
|
+
if not found:
|
|
344
|
+
registry["projects"].append(
|
|
345
|
+
{
|
|
346
|
+
"path": abs_root,
|
|
347
|
+
"db_path": db_path,
|
|
348
|
+
"last_indexed": now,
|
|
349
|
+
"symbol_count": symbol_count,
|
|
350
|
+
}
|
|
351
|
+
)
|
|
352
|
+
|
|
353
|
+
with open(META_REGISTRY, "w") as fh:
|
|
354
|
+
json.dump(registry, fh, indent=2)
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
# ---------------------------------------------------------------------------
|
|
358
|
+
# CLI entry point
|
|
359
|
+
# ---------------------------------------------------------------------------
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def main() -> None:
|
|
363
|
+
parser = argparse.ArgumentParser(
|
|
364
|
+
description="ftm-map indexer — builds the code knowledge graph from source files.",
|
|
365
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
366
|
+
epilog=(
|
|
367
|
+
"Examples:\n"
|
|
368
|
+
" python3 index.py --bootstrap /path/to/project\n"
|
|
369
|
+
" python3 index.py --incremental --files src/foo.ts,src/bar.py\n"
|
|
370
|
+
" python3 index.py --incremental --files src/foo.ts --project-root /path/to/project\n"
|
|
371
|
+
),
|
|
372
|
+
)
|
|
373
|
+
|
|
374
|
+
mode = parser.add_mutually_exclusive_group(required=True)
|
|
375
|
+
mode.add_argument(
|
|
376
|
+
"--bootstrap",
|
|
377
|
+
metavar="PROJECT_ROOT",
|
|
378
|
+
help="Full scan: index all tracked source files in PROJECT_ROOT.",
|
|
379
|
+
)
|
|
380
|
+
mode.add_argument(
|
|
381
|
+
"--incremental",
|
|
382
|
+
action="store_true",
|
|
383
|
+
help="Incremental update: re-index only the files given by --files.",
|
|
384
|
+
)
|
|
385
|
+
|
|
386
|
+
parser.add_argument(
|
|
387
|
+
"--files",
|
|
388
|
+
metavar="FILE_LIST",
|
|
389
|
+
help="Comma-separated list of files to re-index (required for --incremental).",
|
|
390
|
+
)
|
|
391
|
+
parser.add_argument(
|
|
392
|
+
"--project-root",
|
|
393
|
+
metavar="PATH",
|
|
394
|
+
default=None,
|
|
395
|
+
help=(
|
|
396
|
+
"Project root used to locate the database for incremental mode. "
|
|
397
|
+
"Defaults to the current working directory."
|
|
398
|
+
),
|
|
399
|
+
)
|
|
400
|
+
|
|
401
|
+
args = parser.parse_args()
|
|
402
|
+
|
|
403
|
+
if args.bootstrap:
|
|
404
|
+
bootstrap(args.bootstrap)
|
|
405
|
+
else:
|
|
406
|
+
# Incremental mode
|
|
407
|
+
if not args.files:
|
|
408
|
+
print("Error: --incremental requires --files", file=sys.stderr)
|
|
409
|
+
sys.exit(1)
|
|
410
|
+
project_root = args.project_root or os.getcwd()
|
|
411
|
+
incremental(project_root, args.files)
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
if __name__ == "__main__":
|
|
415
|
+
main()
|