@guanlin4924/devflow-pi 0.1.70 → 0.1.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/base.py +5 -1
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/generic_text.py +27 -0
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/registry.py +48 -6
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/structured_scope.py +43 -5
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/code_index.py +128 -6
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/semantic.py +187 -0
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/writers/sqlite_writer.py +170 -1
- package/src/constants.ts +1 -1
- package/src/initialization-store.ts +93 -19
- package/src/project-context.ts +7 -3
- package/src/project-documents.ts +4 -0
- package/src/project-initializer.ts +37 -6
- package/src/project-migration.ts +3 -2
package/package.json
CHANGED
|
@@ -7,6 +7,11 @@ class ParseResult:
|
|
|
7
7
|
symbols: List[Dict] = field(default_factory=list)
|
|
8
8
|
relations: List[Dict] = field(default_factory=list)
|
|
9
9
|
diagnostics: List[Dict] = field(default_factory=list)
|
|
10
|
+
# Optional cross-layer facts. Adapters may populate these when they have
|
|
11
|
+
# framework/type information; the indexer also derives conservative facts
|
|
12
|
+
# in a separate linker pass.
|
|
13
|
+
entities: List[Dict] = field(default_factory=list)
|
|
14
|
+
flows: List[Dict] = field(default_factory=list)
|
|
10
15
|
|
|
11
16
|
|
|
12
17
|
class Adapter:
|
|
@@ -27,4 +32,3 @@ class Adapter:
|
|
|
27
32
|
@property
|
|
28
33
|
def parser_id(self) -> str:
|
|
29
34
|
return f"{self.name}@{self.version()}"
|
|
30
|
-
|
|
@@ -30,6 +30,29 @@ LANGUAGE_BY_EXTENSION = {
|
|
|
30
30
|
".yml": "yaml",
|
|
31
31
|
".toml": "toml",
|
|
32
32
|
".xml": "xml",
|
|
33
|
+
".h": "c",
|
|
34
|
+
".hh": "cpp",
|
|
35
|
+
".hpp": "cpp",
|
|
36
|
+
".hxx": "cpp",
|
|
37
|
+
".c": "c",
|
|
38
|
+
".cc": "cpp",
|
|
39
|
+
".cpp": "cpp",
|
|
40
|
+
".cxx": "cpp",
|
|
41
|
+
".cs": "csharp",
|
|
42
|
+
".scala": "scala",
|
|
43
|
+
".sc": "scala",
|
|
44
|
+
".lua": "lua",
|
|
45
|
+
".html": "html",
|
|
46
|
+
".htm": "html",
|
|
47
|
+
".css": "css",
|
|
48
|
+
".scss": "css",
|
|
49
|
+
".sass": "css",
|
|
50
|
+
".svelte": "svelte",
|
|
51
|
+
".graphql": "graphql",
|
|
52
|
+
".gql": "graphql",
|
|
53
|
+
".proto": "protobuf",
|
|
54
|
+
".hcl": "hcl",
|
|
55
|
+
".tf": "terraform",
|
|
33
56
|
}
|
|
34
57
|
|
|
35
58
|
|
|
@@ -166,6 +189,10 @@ class GenericTextAdapter(Adapter):
|
|
|
166
189
|
|
|
167
190
|
|
|
168
191
|
def language_for_path(path: Path) -> str:
|
|
192
|
+
if path.name.lower() == "dockerfile":
|
|
193
|
+
return "dockerfile"
|
|
194
|
+
if path.name.lower().startswith("dockerfile."):
|
|
195
|
+
return "dockerfile"
|
|
169
196
|
return LANGUAGE_BY_EXTENSION.get(path.suffix.lower(), path.suffix.lower().lstrip(".") or "text")
|
|
170
197
|
|
|
171
198
|
|
|
@@ -5,15 +5,57 @@ from .structured_scope import StructuredScopeAdapter
|
|
|
5
5
|
|
|
6
6
|
class AdapterRegistry:
|
|
7
7
|
def __init__(self):
|
|
8
|
-
self.
|
|
8
|
+
self.python = PythonAstAdapter()
|
|
9
|
+
self.structured = StructuredScopeAdapter()
|
|
10
|
+
self.generic = GenericTextAdapter()
|
|
11
|
+
self.adapters = [self.python, self.structured, self.generic]
|
|
12
|
+
|
|
13
|
+
# Keep language selection explicit. The parser implementation is
|
|
14
|
+
# conservative and shared for now, but every supported language has a
|
|
15
|
+
# deliberate registration instead of silently falling through to text
|
|
16
|
+
# regexes. This is the extension point for native AST adapters.
|
|
17
|
+
self.language_adapters = {
|
|
18
|
+
"python": self.python,
|
|
19
|
+
"typescript": self.structured,
|
|
20
|
+
"javascript": self.structured,
|
|
21
|
+
"ruby": self.structured,
|
|
22
|
+
"go": self.structured,
|
|
23
|
+
"rust": self.structured,
|
|
24
|
+
"php": self.structured,
|
|
25
|
+
"java": self.structured,
|
|
26
|
+
"kotlin": self.structured,
|
|
27
|
+
"c": self.structured,
|
|
28
|
+
"cpp": self.structured,
|
|
29
|
+
"csharp": self.structured,
|
|
30
|
+
"swift": self.structured,
|
|
31
|
+
"dart": self.structured,
|
|
32
|
+
"scala": self.structured,
|
|
33
|
+
"lua": self.structured,
|
|
34
|
+
"shell": self.structured,
|
|
35
|
+
"vue": self.structured,
|
|
36
|
+
"svelte": self.structured,
|
|
37
|
+
# Data, configuration, and template types are explicitly registered too. They
|
|
38
|
+
# currently use the safe text adapter until a native parser is
|
|
39
|
+
# available; this keeps their parser provenance explicit and the
|
|
40
|
+
# unified output contract unchanged.
|
|
41
|
+
"sql": self.generic,
|
|
42
|
+
"html": self.generic,
|
|
43
|
+
"css": self.generic,
|
|
44
|
+
"json": self.generic,
|
|
45
|
+
"yaml": self.generic,
|
|
46
|
+
"toml": self.generic,
|
|
47
|
+
"xml": self.generic,
|
|
48
|
+
"graphql": self.generic,
|
|
49
|
+
"protobuf": self.generic,
|
|
50
|
+
"hcl": self.generic,
|
|
51
|
+
"terraform": self.generic,
|
|
52
|
+
"dockerfile": self.generic,
|
|
53
|
+
"markdown": self.generic,
|
|
54
|
+
}
|
|
9
55
|
|
|
10
56
|
def adapter_for_file(self, codebase, file_record):
|
|
11
57
|
language = file_record.get("language")
|
|
12
|
-
|
|
13
|
-
return self.adapters[0]
|
|
14
|
-
if language in {"typescript", "javascript", "ruby"}:
|
|
15
|
-
return self.adapters[1]
|
|
16
|
-
return self.adapters[-1]
|
|
58
|
+
return self.language_adapters.get(language, self.generic)
|
|
17
59
|
|
|
18
60
|
def adapters_for(self, codebase, files, manifests):
|
|
19
61
|
scored = []
|
package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/structured_scope.py
CHANGED
|
@@ -6,7 +6,14 @@ from adapters.base import Adapter, ParseResult
|
|
|
6
6
|
from adapters.generic_text import GenericTextAdapter, _extract_calls, _extract_imports, _relation, _symbol
|
|
7
7
|
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
# These languages intentionally share the conservative lexical scope parser.
|
|
10
|
+
# The adapter does not claim AST precision; language-native parsers can be
|
|
11
|
+
# added later without changing the unified index contract.
|
|
12
|
+
SUPPORTED = {
|
|
13
|
+
"typescript", "javascript", "ruby", "go", "rust", "php", "java",
|
|
14
|
+
"kotlin", "c", "cpp", "csharp", "swift", "dart", "scala", "lua",
|
|
15
|
+
"shell", "vue", "svelte",
|
|
16
|
+
}
|
|
10
17
|
|
|
11
18
|
|
|
12
19
|
class StructuredScopeAdapter(Adapter):
|
|
@@ -21,7 +28,14 @@ class StructuredScopeAdapter(Adapter):
|
|
|
21
28
|
self.fallback = GenericTextAdapter()
|
|
22
29
|
|
|
23
30
|
def detect(self, codebase, files, manifests) -> int:
|
|
24
|
-
|
|
31
|
+
supported_suffixes = {
|
|
32
|
+
".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".rb", ".go",
|
|
33
|
+
".rs", ".php", ".java", ".kt", ".kts", ".c", ".h", ".cc",
|
|
34
|
+
".cpp", ".cxx", ".hh", ".hpp", ".hxx", ".cs", ".swift",
|
|
35
|
+
".dart", ".scala", ".sc", ".lua", ".sh", ".bash", ".zsh",
|
|
36
|
+
".vue", ".svelte",
|
|
37
|
+
}
|
|
38
|
+
return 80 if any(path.suffix.lower() in supported_suffixes for path in files) else 0
|
|
25
39
|
|
|
26
40
|
def version(self) -> str:
|
|
27
41
|
return "1"
|
|
@@ -30,13 +44,20 @@ class StructuredScopeAdapter(Adapter):
|
|
|
30
44
|
language = file_record["language"]
|
|
31
45
|
if language not in SUPPORTED:
|
|
32
46
|
return self.fallback.parse_file(codebase, file_record, text)
|
|
33
|
-
|
|
47
|
+
if language == "ruby":
|
|
48
|
+
return self._ruby(codebase, file_record, text)
|
|
49
|
+
if language == "shell":
|
|
50
|
+
return self._shell(codebase, file_record, text)
|
|
51
|
+
return self._brace_language(codebase, file_record, text)
|
|
34
52
|
|
|
35
53
|
def _brace_language(self, codebase, file_record, text):
|
|
36
54
|
lines = text.splitlines()
|
|
37
55
|
file_symbol = self._file(codebase, file_record, len(lines))
|
|
38
56
|
declarations: List[Tuple[int, str, str, str]] = []
|
|
39
|
-
pattern = re.compile(
|
|
57
|
+
pattern = re.compile(
|
|
58
|
+
r"\b(class|interface|struct|trait|enum|object|impl|namespace|module|function)\s+([A-Za-z_$][\w$]*)\s*([^\n{]*)"
|
|
59
|
+
r"|\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*=>"
|
|
60
|
+
)
|
|
40
61
|
for number, line in enumerate(lines, 1):
|
|
41
62
|
clean = _without_strings_and_comments(line)
|
|
42
63
|
match = pattern.search(clean)
|
|
@@ -47,11 +68,28 @@ class StructuredScopeAdapter(Adapter):
|
|
|
47
68
|
continue
|
|
48
69
|
keyword, named, tail, arrow = match.groups()
|
|
49
70
|
name = named or arrow
|
|
50
|
-
kind =
|
|
71
|
+
kind = {
|
|
72
|
+
"class": "class", "interface": "interface", "struct": "struct",
|
|
73
|
+
"trait": "trait", "enum": "class", "object": "module",
|
|
74
|
+
"namespace": "module", "module": "module", "impl": "class",
|
|
75
|
+
"function": "function",
|
|
76
|
+
}.get(keyword, "function")
|
|
51
77
|
declarations.append((number, kind, name, line.strip()))
|
|
52
78
|
symbols = self._scoped_symbols(codebase, file_record, lines, declarations, file_symbol, brace=True)
|
|
53
79
|
return self._relations(codebase, file_record, lines, file_symbol, symbols)
|
|
54
80
|
|
|
81
|
+
def _shell(self, codebase, file_record, text):
|
|
82
|
+
"""Index shell functions and command calls at conservative scope."""
|
|
83
|
+
lines = text.splitlines()
|
|
84
|
+
file_symbol = self._file(codebase, file_record, len(lines))
|
|
85
|
+
declarations = []
|
|
86
|
+
for number, line in enumerate(lines, 1):
|
|
87
|
+
match = re.match(r"^\s*(?:function\s+)?([A-Za-z_][\w-]*)\s*(?:\(\s*\))?\s*\{", line)
|
|
88
|
+
if match:
|
|
89
|
+
declarations.append((number, "function", match.group(1), line.strip()))
|
|
90
|
+
symbols = self._scoped_symbols(codebase, file_record, lines, declarations, file_symbol, brace=True)
|
|
91
|
+
return self._relations(codebase, file_record, lines, file_symbol, symbols)
|
|
92
|
+
|
|
55
93
|
def _ruby(self, codebase, file_record, text):
|
|
56
94
|
lines = text.splitlines(); file_symbol = self._file(codebase, file_record, len(lines)); declarations = []
|
|
57
95
|
for number, line in enumerate(lines, 1):
|
|
@@ -22,6 +22,7 @@ if str(SCRIPT_DIR) not in sys.path:
|
|
|
22
22
|
|
|
23
23
|
from adapters.generic_text import language_for_path
|
|
24
24
|
from adapters.registry import AdapterRegistry
|
|
25
|
+
from semantic import SEMANTIC_EXTRACTOR_VERSION, build_semantic_facts
|
|
25
26
|
from core.discovery import discover_codebases
|
|
26
27
|
from core.hashing import file_sha256
|
|
27
28
|
from core.scanner import scan_files
|
|
@@ -58,8 +59,8 @@ def _index_lock(output: Path, exclusive: bool):
|
|
|
58
59
|
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
|
59
60
|
|
|
60
61
|
|
|
61
|
-
def _publish_snapshot(staging: Path, output: Path, counts: Dict[str, int], workspace_state: Dict[str, object]) -> None:
|
|
62
|
-
generation = str(uuid.uuid4())
|
|
62
|
+
def _publish_snapshot(staging: Path, output: Path, counts: Dict[str, int], workspace_state: Dict[str, object], generation: str | None = None) -> None:
|
|
63
|
+
generation = generation or str(uuid.uuid4())
|
|
63
64
|
immutable_relative = f"snapshots/{generation}"
|
|
64
65
|
snapshot = {
|
|
65
66
|
"schemaVersion": 1,
|
|
@@ -101,13 +102,15 @@ def _workspace_state(root: Path, files: List[dict], changed: Set[str], discovere
|
|
|
101
102
|
indexed_paths = sorted(record["path"] for record in files)
|
|
102
103
|
discovered = sorted(set(discovered_paths or indexed_paths))
|
|
103
104
|
source_manifest_hash = _hash_lines(f"{record['path']}\0{record['content_hash']}" for record in files)
|
|
105
|
+
extractors = sorted(set(record.get("parser", "unknown") for record in files))
|
|
104
106
|
return {
|
|
105
107
|
"repositoryRevision": _git_output(root, ["rev-parse", "HEAD"]),
|
|
106
108
|
"contentManifestHash": source_manifest_hash,
|
|
107
109
|
"indexedPathsHash": _hash_lines(indexed_paths),
|
|
108
110
|
"discoveredPathsHash": _hash_lines(discovered),
|
|
109
111
|
"dirtyPathsAtBuild": sorted(changed),
|
|
110
|
-
"extractors":
|
|
112
|
+
"extractors": extractors,
|
|
113
|
+
"extractorSetHash": _hash_lines(extractors),
|
|
111
114
|
"capabilities": {
|
|
112
115
|
"python-ast@1": {"scope": "exact syntax tree", "targets": "lexical names; dynamic dispatch unresolved"},
|
|
113
116
|
"structured-scope@1": {"scope": "inferred lexical braces/end blocks", "targets": "name-only heuristic"},
|
|
@@ -185,6 +188,14 @@ def main(argv=None) -> int:
|
|
|
185
188
|
relations.add_argument("--snapshot")
|
|
186
189
|
relations.add_argument("--include-heuristic", action="store_true")
|
|
187
190
|
|
|
191
|
+
impact = sub.add_parser("impact", help="Traverse cross-layer semantic facts in a fixed snapshot.")
|
|
192
|
+
impact.add_argument("--index", default=".planning/code-index")
|
|
193
|
+
impact.add_argument("--entity", required=True, help="Logical entity id, feature id, API id, or symbol id.")
|
|
194
|
+
impact.add_argument("--depth", type=int, default=5)
|
|
195
|
+
impact.add_argument("--limit", type=int, default=200)
|
|
196
|
+
impact.add_argument("--snapshot")
|
|
197
|
+
impact.add_argument("--change-type", choices=["remove_field", "rename_field", "change_type", "optional_to_required", "required_to_optional", "change_nullability", "change_enum"], help="Optional proposed change classification.")
|
|
198
|
+
|
|
188
199
|
args = parser.parse_args(argv)
|
|
189
200
|
if args.command == "scan":
|
|
190
201
|
return command_scan(args)
|
|
@@ -196,6 +207,8 @@ def main(argv=None) -> int:
|
|
|
196
207
|
return command_query(args)
|
|
197
208
|
if args.command == "relations":
|
|
198
209
|
return command_relations(args)
|
|
210
|
+
if args.command == "impact":
|
|
211
|
+
return command_impact(args)
|
|
199
212
|
return 2
|
|
200
213
|
|
|
201
214
|
|
|
@@ -213,6 +226,8 @@ def command_scan(args) -> int:
|
|
|
213
226
|
files, symbols, relations, diagnostics = _existing_after_removal(existing, changed)
|
|
214
227
|
indexed_paths = {record["path"] for record in files}
|
|
215
228
|
discovered_paths: Set[str] = set()
|
|
229
|
+
adapter_entities: List[dict] = []
|
|
230
|
+
adapter_flows: List[dict] = []
|
|
216
231
|
|
|
217
232
|
for codebase in codebases:
|
|
218
233
|
if not codebase.root.exists():
|
|
@@ -249,6 +264,8 @@ def command_scan(args) -> int:
|
|
|
249
264
|
symbols.extend(result.symbols)
|
|
250
265
|
relations.extend(result.relations)
|
|
251
266
|
diagnostics.extend(result.diagnostics)
|
|
267
|
+
adapter_entities.extend(getattr(result, "entities", []) or [])
|
|
268
|
+
adapter_flows.extend(getattr(result, "flows", []) or [])
|
|
252
269
|
except Exception as exc: # keep indexing useful even when one file fails
|
|
253
270
|
diagnostics.append(_diagnostic(codebase.name, rel_path, 1, "error", "parse_failed", str(exc), adapter.parser_id))
|
|
254
271
|
|
|
@@ -263,11 +280,26 @@ def command_scan(args) -> int:
|
|
|
263
280
|
|
|
264
281
|
staging = Path(tempfile.mkdtemp(prefix=f".{output.name}.build-", dir=str(output.parent)))
|
|
265
282
|
try:
|
|
283
|
+
generation = str(uuid.uuid4())
|
|
284
|
+
workspace_state = _workspace_state(root, files, changed, discovered_paths)
|
|
285
|
+
workspace_state["generatedAt"] = datetime.now(timezone.utc).isoformat()
|
|
286
|
+
workspace_state["semanticExtractor"] = SEMANTIC_EXTRACTOR_VERSION
|
|
287
|
+
workspace_state["extractorSetHash"] = _hash_lines([*workspace_state.get("extractors", []), SEMANTIC_EXTRACTOR_VERSION])
|
|
288
|
+
semantic_entities, semantic_flows, feature_bindings = build_semantic_facts(root, files, symbols, relations)
|
|
289
|
+
semantic_entities.extend(adapter_entities)
|
|
290
|
+
semantic_flows.extend(adapter_flows)
|
|
266
291
|
write_jsonl(staging / INDEX_FILES["files"], files)
|
|
267
292
|
write_jsonl(staging / INDEX_FILES["symbols"], symbols)
|
|
268
293
|
write_jsonl(staging / INDEX_FILES["relations"], relations)
|
|
269
294
|
write_jsonl(staging / INDEX_FILES["diagnostics"], diagnostics)
|
|
270
|
-
write_sqlite(
|
|
295
|
+
write_sqlite(
|
|
296
|
+
staging / "index.sqlite", files, symbols, relations, diagnostics,
|
|
297
|
+
snapshot_id=generation,
|
|
298
|
+
workspace_state=workspace_state,
|
|
299
|
+
semantic_entities=semantic_entities,
|
|
300
|
+
semantic_flows=semantic_flows,
|
|
301
|
+
feature_bindings=feature_bindings,
|
|
302
|
+
)
|
|
271
303
|
write_markdown(staging, files, symbols, relations, diagnostics)
|
|
272
304
|
staged_errors = validate_index(*_load_tuple(staging))
|
|
273
305
|
if staged_errors:
|
|
@@ -284,7 +316,8 @@ def command_scan(args) -> int:
|
|
|
284
316
|
"relations": len(relations),
|
|
285
317
|
"diagnostics": len(diagnostics),
|
|
286
318
|
},
|
|
287
|
-
|
|
319
|
+
workspace_state,
|
|
320
|
+
generation=generation,
|
|
288
321
|
)
|
|
289
322
|
finally:
|
|
290
323
|
shutil.rmtree(staging, ignore_errors=True)
|
|
@@ -296,6 +329,46 @@ def command_scan(args) -> int:
|
|
|
296
329
|
|
|
297
330
|
def command_validate(args) -> int:
|
|
298
331
|
output = Path(args.index).resolve()
|
|
332
|
+
selected = _selected_generation(output, None)
|
|
333
|
+
errors = []
|
|
334
|
+
try:
|
|
335
|
+
snapshot = _load_snapshot(selected)
|
|
336
|
+
files, symbols, relations, diagnostics = _load_tuple(selected)
|
|
337
|
+
errors.extend(validate_index(files, symbols, relations, diagnostics))
|
|
338
|
+
conn = sqlite3.connect(str(selected / "index.sqlite"))
|
|
339
|
+
try:
|
|
340
|
+
snapshot_id = snapshot.get("generation")
|
|
341
|
+
row = conn.execute("SELECT snapshot_id FROM analysis_snapshots WHERE snapshot_id=?", (snapshot_id,)).fetchone()
|
|
342
|
+
if not row:
|
|
343
|
+
errors.append(f"semantic schema missing snapshot {snapshot_id}")
|
|
344
|
+
dangling = conn.execute("""
|
|
345
|
+
SELECT COUNT(*) FROM semantic_flows f
|
|
346
|
+
WHERE f.snapshot_id=? AND (
|
|
347
|
+
(NOT EXISTS (SELECT 1 FROM entity_versions e WHERE e.snapshot_id=f.snapshot_id AND e.logical_id=f.source_id)
|
|
348
|
+
AND NOT EXISTS (SELECT 1 FROM symbols s WHERE s.id=f.source_id))
|
|
349
|
+
OR
|
|
350
|
+
(NOT EXISTS (SELECT 1 FROM entity_versions e WHERE e.snapshot_id=f.snapshot_id AND e.logical_id=f.target_id)
|
|
351
|
+
AND NOT EXISTS (SELECT 1 FROM symbols s WHERE s.id=f.target_id))
|
|
352
|
+
)
|
|
353
|
+
""", (snapshot_id,)).fetchone()[0]
|
|
354
|
+
if dangling:
|
|
355
|
+
errors.append(f"semantic flows have {dangling} dangling source or target nodes")
|
|
356
|
+
binding_dangling = conn.execute("""
|
|
357
|
+
SELECT COUNT(*) FROM feature_bindings b
|
|
358
|
+
WHERE b.snapshot_id=? AND (
|
|
359
|
+
NOT EXISTS (SELECT 1 FROM entity_versions e WHERE e.snapshot_id=b.snapshot_id AND e.logical_id=b.feature_id)
|
|
360
|
+
OR NOT EXISTS (SELECT 1 FROM entity_versions e WHERE e.snapshot_id=b.snapshot_id AND e.logical_id=b.entity_id)
|
|
361
|
+
)
|
|
362
|
+
""", (snapshot_id,)).fetchone()[0]
|
|
363
|
+
if binding_dangling:
|
|
364
|
+
errors.append(f"feature bindings have {binding_dangling} dangling feature or entity nodes")
|
|
365
|
+
finally:
|
|
366
|
+
conn.close()
|
|
367
|
+
except (OSError, sqlite3.Error, ValueError) as exc:
|
|
368
|
+
errors.append(str(exc))
|
|
369
|
+
for error in errors:
|
|
370
|
+
print(error, file=sys.stderr)
|
|
371
|
+
return 1 if errors else 0
|
|
299
372
|
def command_query(args) -> int:
|
|
300
373
|
output = Path(args.index).resolve()
|
|
301
374
|
with _index_lock(output, exclusive=False):
|
|
@@ -366,12 +439,61 @@ def command_relations(args) -> int:
|
|
|
366
439
|
) SELECT * FROM graph ORDER BY depth,from_id,relation_type,to_id LIMIT ?
|
|
367
440
|
""", (args.object, depth, limit + 1)).fetchall()
|
|
368
441
|
truncated = len(rows) > limit
|
|
369
|
-
for row in rows[:limit]:
|
|
442
|
+
for row in rows[:limit]:
|
|
443
|
+
item = dict(row)
|
|
444
|
+
if args.change_type:
|
|
445
|
+
item["impact_kind"] = args.change_type
|
|
446
|
+
item["severity"] = _impact_severity(args.change_type, item["flow_type"])
|
|
447
|
+
print(json.dumps(item, ensure_ascii=False, sort_keys=True))
|
|
370
448
|
print(json.dumps({"_meta": {"snapshot": snapshot.get("generation"), "depthLimit": depth, "resultLimit": limit, "truncated": truncated}}, ensure_ascii=False, sort_keys=True))
|
|
371
449
|
finally: conn.close()
|
|
372
450
|
return 0
|
|
373
451
|
|
|
374
452
|
|
|
453
|
+
def _impact_severity(change_type: str, flow_type: str) -> str:
|
|
454
|
+
"""Conservative compatibility rules for an observed impact edge."""
|
|
455
|
+
if change_type in {"remove_field", "change_type", "change_nullability", "change_enum"}:
|
|
456
|
+
return "high" if flow_type in {"returns_field", "accepts_field", "state_reads", "renders_when", "depends_on"} else "medium"
|
|
457
|
+
if change_type == "optional_to_required":
|
|
458
|
+
return "high" if flow_type in {"api_declares_or_calls", "accepts_field"} else "medium"
|
|
459
|
+
if change_type == "rename_field":
|
|
460
|
+
return "high" if flow_type in {"returns_field", "accepts_field", "maps_to", "state_reads"} else "medium"
|
|
461
|
+
return "low"
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def command_impact(args) -> int:
|
|
465
|
+
output = Path(args.index).resolve()
|
|
466
|
+
depth = max(0, min(args.depth, 20)); limit = max(1, min(args.limit, 2000))
|
|
467
|
+
with _index_lock(output, exclusive=False):
|
|
468
|
+
selected = _selected_generation(output, getattr(args, "snapshot", None))
|
|
469
|
+
snapshot = _load_snapshot(selected)
|
|
470
|
+
if not getattr(args, "snapshot", None):
|
|
471
|
+
files = read_jsonl(selected / INDEX_FILES["files"])
|
|
472
|
+
if not _query_is_fresh(output.parent.parent, snapshot, files):
|
|
473
|
+
print("Stale code index: refresh required before impact query.", file=sys.stderr)
|
|
474
|
+
return 1
|
|
475
|
+
conn = sqlite3.connect(str(selected / "index.sqlite")); conn.row_factory = sqlite3.Row
|
|
476
|
+
try:
|
|
477
|
+
rows = conn.execute("""
|
|
478
|
+
WITH RECURSIVE graph(node_id, flow_type, source_path, line_start, confidence, depth, visited) AS (
|
|
479
|
+
SELECT CASE WHEN source_id=? THEN target_id ELSE source_id END, flow_type, source_path, line_start, confidence, 1, ',' || ? || ','
|
|
480
|
+
FROM semantic_flows WHERE snapshot_id=? AND (source_id=? OR target_id=?)
|
|
481
|
+
UNION ALL
|
|
482
|
+
SELECT CASE WHEN f.source_id=g.node_id THEN f.target_id ELSE f.source_id END,
|
|
483
|
+
f.flow_type, f.source_path, f.line_start, f.confidence, g.depth+1, g.visited || g.node_id || ','
|
|
484
|
+
FROM graph g JOIN semantic_flows f ON f.snapshot_id=? AND (f.source_id=g.node_id OR f.target_id=g.node_id)
|
|
485
|
+
WHERE g.depth < ? AND instr(g.visited, ',' || CASE WHEN f.source_id=g.node_id THEN f.target_id ELSE f.source_id END || ',')=0
|
|
486
|
+
) SELECT * FROM graph LIMIT ?
|
|
487
|
+
""", (args.entity, args.entity, snapshot.get("generation"), args.entity, args.entity, snapshot.get("generation"), depth, limit + 1)).fetchall()
|
|
488
|
+
for row in rows[:limit]: print(json.dumps(dict(row), ensure_ascii=False, sort_keys=True))
|
|
489
|
+
coverage_row = conn.execute("SELECT * FROM coverage_metrics WHERE snapshot_id=?", (snapshot.get("generation"),)).fetchone()
|
|
490
|
+
coverage = dict(coverage_row) if coverage_row else {"status": "unknown"}
|
|
491
|
+
print(json.dumps({"_meta": {"snapshot": snapshot.get("generation"), "coverage": coverage, "depthLimit": depth, "truncated": len(rows) > limit}}, ensure_ascii=False, sort_keys=True))
|
|
492
|
+
finally:
|
|
493
|
+
conn.close()
|
|
494
|
+
return 0
|
|
495
|
+
|
|
496
|
+
|
|
375
497
|
def _selected_generation(output: Path, generation: str | None) -> Path:
|
|
376
498
|
if not generation:
|
|
377
499
|
return output
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""Conservative cross-layer fact extraction.
|
|
2
|
+
|
|
3
|
+
This module intentionally emits candidates with evidence and confidence. It
|
|
4
|
+
does not replace framework parsers; it gives the relational index a stable
|
|
5
|
+
schema so API, database, UI and feature facts can be linked incrementally.
|
|
6
|
+
"""
|
|
7
|
+
import re
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Dict, Iterable, List, Tuple
|
|
10
|
+
|
|
11
|
+
from core.schema import short_hash, stable_key
|
|
12
|
+
from core.hashing import file_sha256
|
|
13
|
+
|
|
14
|
+
SEMANTIC_EXTRACTOR_VERSION = "semantic-linker@1"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def build_semantic_facts(root: Path, files: Iterable[dict], symbols: Iterable[dict],
|
|
18
|
+
relations: Iterable[dict]) -> Tuple[List[dict], List[dict], List[dict]]:
|
|
19
|
+
entities: List[dict] = []
|
|
20
|
+
flows: List[dict] = []
|
|
21
|
+
bindings: List[dict] = []
|
|
22
|
+
symbol_by_path = {}
|
|
23
|
+
for symbol in symbols:
|
|
24
|
+
symbol_by_path.setdefault(symbol.get("path"), []).append(symbol)
|
|
25
|
+
|
|
26
|
+
for file_record in files:
|
|
27
|
+
path = file_record["path"]
|
|
28
|
+
source = _read(root, path)
|
|
29
|
+
if source is None:
|
|
30
|
+
continue
|
|
31
|
+
try:
|
|
32
|
+
if file_sha256(root / path) != file_record.get("content_hash"):
|
|
33
|
+
# Never mix a newly-read working-tree file with an older
|
|
34
|
+
# structural record in the same immutable snapshot.
|
|
35
|
+
continue
|
|
36
|
+
except OSError:
|
|
37
|
+
continue
|
|
38
|
+
language = file_record.get("language", "")
|
|
39
|
+
feature = _feature_from_path(path)
|
|
40
|
+
if feature:
|
|
41
|
+
feature_id = f"feature:{stable_key(feature)}"
|
|
42
|
+
entities.append(_entity(feature_id, "feature", feature, file_record, 1, 1,
|
|
43
|
+
"heuristic", "path_feature_inference"))
|
|
44
|
+
for symbol in symbol_by_path.get(path, []):
|
|
45
|
+
if symbol.get("kind") in {"route", "component", "class", "function", "method"}:
|
|
46
|
+
bindings.append(_binding(feature_id, symbol["id"], "static_inferred", "heuristic", path, symbol.get("line_start", 1)))
|
|
47
|
+
|
|
48
|
+
for line_no, line in enumerate(source.splitlines(), 1):
|
|
49
|
+
_extract_api(line, path, file_record, line_no, entities, flows)
|
|
50
|
+
_extract_db(line, path, file_record, line_no, entities, flows)
|
|
51
|
+
_extract_ui(line, path, file_record, line_no, entities, flows)
|
|
52
|
+
|
|
53
|
+
# Existing structural relations remain the source of method call edges;
|
|
54
|
+
# expose them in the semantic graph without changing their meaning.
|
|
55
|
+
for relation in relations:
|
|
56
|
+
if relation.get("to_id") and relation.get("relation_type") in {"calls", "contains", "routes_to"}:
|
|
57
|
+
flows.append({
|
|
58
|
+
"source_id": relation.get("from_id"),
|
|
59
|
+
"target_id": relation.get("to_id"),
|
|
60
|
+
"flow_type": relation.get("relation_type"),
|
|
61
|
+
"mapping_kind": "structural_relation",
|
|
62
|
+
"source_path": relation.get("evidence_path"),
|
|
63
|
+
"line_start": relation.get("evidence_line"),
|
|
64
|
+
"confidence": relation.get("confidence", "heuristic"),
|
|
65
|
+
"evidence_level": "syntax",
|
|
66
|
+
"metadata": relation.get("metadata", {}),
|
|
67
|
+
})
|
|
68
|
+
return _dedupe_entities(entities), _dedupe_flows(flows), _dedupe_bindings(bindings)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _extract_api(line, path, record, line_no, entities, flows):
|
|
72
|
+
# Framework-neutral route declarations and client URLs.
|
|
73
|
+
match = re.search(r"(?:@(?:app|router|api)\.(get|post|put|patch|delete)\s*\(|(?:GET|POST|PUT|PATCH|DELETE)\s+)(['\"])(/[^'\"]+)\2", line, re.I)
|
|
74
|
+
if not match:
|
|
75
|
+
match = re.search(r"(?:fetch|axios\.(?:get|post|put|patch|delete)|client\.(?:get|post|put|patch|delete))\s*\(\s*(['\"])(/[^'\"]+)\1", line, re.I)
|
|
76
|
+
if not match:
|
|
77
|
+
return
|
|
78
|
+
route = match.group(3) if match.lastindex and match.lastindex >= 3 else match.group(2)
|
|
79
|
+
method_match = re.search(r"@(?:app|router|api)\.(get|post|put|patch|delete)\b|\b(GET|POST|PUT|PATCH|DELETE)\b|\b(?:fetch|axios\.|client\.)(get|post|put|patch|delete)\b", line, re.I)
|
|
80
|
+
method = next((value.upper() for value in (method_match.groups() if method_match else ()) if value), "UNKNOWN")
|
|
81
|
+
if method == "UNKNOWN" and re.search(r"\bfetch\s*\(", line, re.I):
|
|
82
|
+
method = "GET"
|
|
83
|
+
endpoint_id = f"api:{method}:{stable_key(route)}"
|
|
84
|
+
entity_type = "api_endpoint"
|
|
85
|
+
entities.append(_entity(endpoint_id, entity_type, f"{method} {route}", record, line_no, line_no, "heuristic", "route_or_client_regex", {"http_method": method, "path": route}))
|
|
86
|
+
flows.append(_flow(f"file:{record['codebase']}:{path}", endpoint_id, "api_declares_or_calls", path, line_no, "heuristic", "framework_neutral_route"))
|
|
87
|
+
for field in re.findall(r"(?:response|payload|data|body)\s*[.\[]\s*['\"]?([A-Za-z_][A-Za-z0-9_]*)", line):
|
|
88
|
+
field_id = f"api-field:{endpoint_id}:response:{field}"
|
|
89
|
+
entities.append(_entity(field_id, "api_field", field, record, line_no, line_no, "heuristic", "field_access_regex", {"endpoint_id": endpoint_id, "direction": "response", "field_path": field}))
|
|
90
|
+
flows.append(_flow(endpoint_id, field_id, "returns_field", path, line_no, "heuristic", "field_access_regex"))
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _extract_db(line, path, record, line_no, entities, flows):
|
|
94
|
+
# SQL and common ORM spellings. Dynamic identifiers are deliberately not
|
|
95
|
+
# resolved to a concrete column.
|
|
96
|
+
table_match = re.search(r"\b(?:FROM|JOIN|INTO|UPDATE|TABLE)\s+[`\"]?([A-Za-z_][\w$]*)", line, re.I)
|
|
97
|
+
orm_match = re.search(r"\b([A-Z][A-Za-z0-9_]*)\.(?:query|objects|select|filter|where)\b", line)
|
|
98
|
+
if not table_match and not orm_match:
|
|
99
|
+
return
|
|
100
|
+
table = table_match.group(1) if table_match else orm_match.group(1)
|
|
101
|
+
table_id = f"db-table:{stable_key(table)}"
|
|
102
|
+
entities.append(_entity(table_id, "db_table", table, record, line_no, line_no, "heuristic", "sql_or_orm_regex"))
|
|
103
|
+
operation = "writes" if re.search(r"\b(?:INSERT|UPDATE|DELETE)\b|\.update\s*\(", line, re.I) else "reads"
|
|
104
|
+
flows.append(_flow(f"file:{record['codebase']}:{path}", table_id, operation, path, line_no, "heuristic", "sql_or_orm_regex"))
|
|
105
|
+
for column in re.findall(r"\b([A-Za-z_][A-Za-z0-9_]*)\s*(?:=|,|FROM|$)", line):
|
|
106
|
+
if column.lower() in {"select", "from", "where", "set", "values", "into", "update", "join", "and", "or"}:
|
|
107
|
+
continue
|
|
108
|
+
column_id = f"db-column:{stable_key(table)}:{stable_key(column)}"
|
|
109
|
+
entities.append(_entity(column_id, "db_column", column, record, line_no, line_no, "heuristic", "sql_column_regex", {"table_id": table_id, "column": column}))
|
|
110
|
+
flows.append(_flow(table_id, column_id, f"{operation}_column", path, line_no, "heuristic", "sql_column_regex"))
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _extract_ui(line, path, record, line_no, entities, flows):
|
|
114
|
+
component = re.search(r"\b(?:function|class)\s+([A-Z][A-Za-z0-9_]*)|<([A-Z][A-Za-z0-9_.]*)\b", line)
|
|
115
|
+
state = re.search(r"\b(?:useState|ref|reactive|computed|createSignal)\s*\(.*?\)|\b(?:const|let|var)\s+([A-Za-z_][\w]*(?:State|Store|Status|Loading|Error))\s*=", line)
|
|
116
|
+
condition = re.search(r"\b(?:v-if|v-show)\s*=\s*['\"]([^'\"]+)['\"]|(?:\{\s*)?([^{}\n]+?)\s*&&\s*<", line)
|
|
117
|
+
if component:
|
|
118
|
+
name = component.group(1) or component.group(2)
|
|
119
|
+
component_id = f"ui-component:{stable_key(path)}:{stable_key(name)}"
|
|
120
|
+
entities.append(_entity(component_id, "ui_component", name, record, line_no, line_no, "inferred", "component_pattern"))
|
|
121
|
+
else:
|
|
122
|
+
component_id = f"ui-file:{stable_key(path)}"
|
|
123
|
+
entities.append(_entity(component_id, "ui_component", Path(path).stem, record, line_no, line_no, "heuristic", "ui_file_scope"))
|
|
124
|
+
if state:
|
|
125
|
+
name = state.group(1) or "anonymous_state"
|
|
126
|
+
state_id = f"ui-state:{stable_key(path)}:{stable_key(name)}"
|
|
127
|
+
entities.append(_entity(state_id, "ui_state", name, record, line_no, line_no, "heuristic", "state_pattern"))
|
|
128
|
+
flows.append(_flow(component_id, state_id, "state_reads", path, line_no, "heuristic", "state_pattern"))
|
|
129
|
+
if condition:
|
|
130
|
+
expression = condition.group(1) or condition.group(2)
|
|
131
|
+
cond_id = f"render-condition:{stable_key(path)}:{short_hash(expression)}"
|
|
132
|
+
entities.append(_entity(cond_id, "render_condition", expression, record, line_no, line_no, "heuristic", "template_condition"))
|
|
133
|
+
flows.append(_flow(component_id, cond_id, "renders_when", path, line_no, "heuristic", "template_condition"))
|
|
134
|
+
known_ids = {item["id"] for item in entities if item.get("entity_type") == "ui_state"}
|
|
135
|
+
for name in re.findall(r"\b[A-Za-z_][A-Za-z0-9_]*\b", expression):
|
|
136
|
+
state_id = f"ui-state:{stable_key(path)}:{stable_key(name)}"
|
|
137
|
+
if state_id in known_ids:
|
|
138
|
+
flows.append(_flow(cond_id, state_id, "depends_on", path, line_no, "heuristic", "condition_identifier"))
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _entity(entity_id, entity_type, name, record, start, end, confidence, extractor, metadata=None):
|
|
142
|
+
return {"id": entity_id, "entity_type": entity_type, "name": name, "codebase": record["codebase"], "path": record["path"], "line_start": start, "line_end": end, "content_hash": record["content_hash"], "confidence": confidence, "extractor": extractor, "metadata": metadata or {}}
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _flow(source_id, target_id, flow_type, path, line, confidence, extractor, metadata=None):
|
|
146
|
+
return {"source_id": source_id, "target_id": target_id, "flow_type": flow_type, "source_path": path, "line_start": line, "confidence": confidence, "evidence_level": "static_source", "extractor": extractor, "metadata": metadata or {}}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _binding(feature_id, entity_id, source, confidence, path, line):
|
|
150
|
+
return {"feature_id": feature_id, "entity_id": entity_id, "binding_type": source, "confidence": confidence, "source_path": path, "line_start": line}
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _read(root, path):
|
|
154
|
+
try:
|
|
155
|
+
return (root / path).read_text(encoding="utf-8", errors="ignore")
|
|
156
|
+
except OSError:
|
|
157
|
+
return None
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _feature_from_path(path):
|
|
161
|
+
parts = [part for part in Path(path).parts if part not in {"src", "app", "lib", "components", "services", "pages", "api", "tests"}]
|
|
162
|
+
if not parts:
|
|
163
|
+
return None
|
|
164
|
+
token = re.sub(r"[^A-Za-z0-9_-]+", "-", parts[0]).strip("-")
|
|
165
|
+
return token if token and token.lower() not in {"index", "main", "common", "shared"} else None
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _dedupe_entities(items):
|
|
169
|
+
result = {}
|
|
170
|
+
for item in items:
|
|
171
|
+
result[item["id"]] = item
|
|
172
|
+
return list(result.values())
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _dedupe_flows(items):
|
|
176
|
+
result = {}
|
|
177
|
+
for item in items:
|
|
178
|
+
key = (item.get("source_id"), item.get("target_id"), item.get("flow_type"), item.get("source_path"), item.get("line_start"))
|
|
179
|
+
result[key] = item
|
|
180
|
+
return list(result.values())
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _dedupe_bindings(items):
|
|
184
|
+
result = {}
|
|
185
|
+
for item in items:
|
|
186
|
+
result[(item["feature_id"], item["entity_id"])] = item
|
|
187
|
+
return list(result.values())
|
|
@@ -58,6 +58,119 @@ CREATE TABLE IF NOT EXISTS diagnostics (
|
|
|
58
58
|
parser TEXT
|
|
59
59
|
);
|
|
60
60
|
|
|
61
|
+
CREATE TABLE IF NOT EXISTS analysis_snapshots (
|
|
62
|
+
snapshot_id TEXT PRIMARY KEY,
|
|
63
|
+
schema_version INTEGER NOT NULL,
|
|
64
|
+
source_revision TEXT,
|
|
65
|
+
content_manifest_hash TEXT,
|
|
66
|
+
extractor_set_hash TEXT,
|
|
67
|
+
created_at TEXT NOT NULL,
|
|
68
|
+
status TEXT NOT NULL
|
|
69
|
+
);
|
|
70
|
+
|
|
71
|
+
CREATE TABLE IF NOT EXISTS logical_entities (
|
|
72
|
+
logical_id TEXT PRIMARY KEY,
|
|
73
|
+
entity_type TEXT NOT NULL,
|
|
74
|
+
stable_key TEXT NOT NULL,
|
|
75
|
+
metadata_json TEXT
|
|
76
|
+
);
|
|
77
|
+
|
|
78
|
+
CREATE TABLE IF NOT EXISTS entity_versions (
|
|
79
|
+
entity_version_id TEXT PRIMARY KEY,
|
|
80
|
+
snapshot_id TEXT NOT NULL,
|
|
81
|
+
logical_id TEXT NOT NULL,
|
|
82
|
+
entity_type TEXT NOT NULL,
|
|
83
|
+
name TEXT NOT NULL,
|
|
84
|
+
codebase TEXT,
|
|
85
|
+
path TEXT,
|
|
86
|
+
line_start INTEGER,
|
|
87
|
+
line_end INTEGER,
|
|
88
|
+
content_hash TEXT,
|
|
89
|
+
confidence TEXT NOT NULL,
|
|
90
|
+
extractor TEXT NOT NULL,
|
|
91
|
+
metadata_json TEXT
|
|
92
|
+
);
|
|
93
|
+
|
|
94
|
+
CREATE TABLE IF NOT EXISTS semantic_flows (
|
|
95
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
96
|
+
snapshot_id TEXT NOT NULL,
|
|
97
|
+
source_id TEXT NOT NULL,
|
|
98
|
+
target_id TEXT NOT NULL,
|
|
99
|
+
flow_type TEXT NOT NULL,
|
|
100
|
+
source_path TEXT,
|
|
101
|
+
line_start INTEGER,
|
|
102
|
+
confidence TEXT NOT NULL,
|
|
103
|
+
evidence_level TEXT NOT NULL,
|
|
104
|
+
extractor TEXT NOT NULL,
|
|
105
|
+
metadata_json TEXT
|
|
106
|
+
);
|
|
107
|
+
|
|
108
|
+
CREATE TABLE IF NOT EXISTS feature_bindings (
|
|
109
|
+
snapshot_id TEXT NOT NULL,
|
|
110
|
+
feature_id TEXT NOT NULL,
|
|
111
|
+
entity_id TEXT NOT NULL,
|
|
112
|
+
binding_type TEXT NOT NULL,
|
|
113
|
+
confidence TEXT NOT NULL,
|
|
114
|
+
source_path TEXT,
|
|
115
|
+
line_start INTEGER,
|
|
116
|
+
PRIMARY KEY(snapshot_id, feature_id, entity_id)
|
|
117
|
+
);
|
|
118
|
+
|
|
119
|
+
CREATE TABLE IF NOT EXISTS coverage_metrics (
|
|
120
|
+
snapshot_id TEXT PRIMARY KEY,
|
|
121
|
+
files_scanned INTEGER NOT NULL,
|
|
122
|
+
files_parsed INTEGER NOT NULL,
|
|
123
|
+
diagnostics INTEGER NOT NULL,
|
|
124
|
+
semantic_entities INTEGER NOT NULL,
|
|
125
|
+
semantic_flows INTEGER NOT NULL,
|
|
126
|
+
dynamic_limitations INTEGER NOT NULL,
|
|
127
|
+
metadata_json TEXT
|
|
128
|
+
);
|
|
129
|
+
|
|
130
|
+
CREATE TABLE IF NOT EXISTS design_entities (
|
|
131
|
+
artifact_revision TEXT NOT NULL,
|
|
132
|
+
design_entity_id TEXT NOT NULL,
|
|
133
|
+
entity_type TEXT NOT NULL,
|
|
134
|
+
name TEXT NOT NULL,
|
|
135
|
+
field_path TEXT,
|
|
136
|
+
direction TEXT,
|
|
137
|
+
status TEXT NOT NULL,
|
|
138
|
+
metadata_json TEXT,
|
|
139
|
+
PRIMARY KEY(artifact_revision, design_entity_id)
|
|
140
|
+
);
|
|
141
|
+
|
|
142
|
+
CREATE TABLE IF NOT EXISTS design_observation_links (
|
|
143
|
+
artifact_revision TEXT NOT NULL,
|
|
144
|
+
design_entity_id TEXT NOT NULL,
|
|
145
|
+
observed_entity_id TEXT NOT NULL,
|
|
146
|
+
result TEXT NOT NULL, -- matches / drift / gap / unknown
|
|
147
|
+
confidence TEXT NOT NULL,
|
|
148
|
+
evidence_json TEXT
|
|
149
|
+
);
|
|
150
|
+
|
|
151
|
+
CREATE TABLE IF NOT EXISTS proposed_changes (
|
|
152
|
+
change_id TEXT PRIMARY KEY,
|
|
153
|
+
snapshot_id TEXT NOT NULL,
|
|
154
|
+
artifact_revision TEXT,
|
|
155
|
+
change_type TEXT NOT NULL,
|
|
156
|
+
target_entity_id TEXT NOT NULL,
|
|
157
|
+
before_json TEXT,
|
|
158
|
+
after_json TEXT,
|
|
159
|
+
created_at TEXT NOT NULL
|
|
160
|
+
);
|
|
161
|
+
|
|
162
|
+
CREATE TABLE IF NOT EXISTS impact_findings (
|
|
163
|
+
impact_run_id TEXT NOT NULL,
|
|
164
|
+
snapshot_id TEXT NOT NULL,
|
|
165
|
+
change_id TEXT,
|
|
166
|
+
entity_id TEXT NOT NULL,
|
|
167
|
+
impact_kind TEXT NOT NULL,
|
|
168
|
+
severity TEXT NOT NULL,
|
|
169
|
+
confidence TEXT NOT NULL,
|
|
170
|
+
evidence_json TEXT,
|
|
171
|
+
PRIMARY KEY(impact_run_id, entity_id, impact_kind)
|
|
172
|
+
);
|
|
173
|
+
|
|
61
174
|
CREATE VIRTUAL TABLE IF NOT EXISTS symbols_fts USING fts5(
|
|
62
175
|
id UNINDEXED,
|
|
63
176
|
codebase UNINDEXED,
|
|
@@ -78,13 +191,69 @@ CREATE INDEX IF NOT EXISTS idx_relations_external ON relations(to_external);
|
|
|
78
191
|
"""
|
|
79
192
|
|
|
80
193
|
|
|
81
|
-
def write_sqlite(path: Path, files, symbols, relations, diagnostics
|
|
194
|
+
def write_sqlite(path: Path, files, symbols, relations, diagnostics, snapshot_id="pending",
|
|
195
|
+
workspace_state=None, semantic_entities=None, semantic_flows=None, feature_bindings=None) -> None:
|
|
82
196
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
83
197
|
if path.exists():
|
|
84
198
|
path.unlink()
|
|
85
199
|
conn = sqlite3.connect(str(path))
|
|
86
200
|
try:
|
|
87
201
|
conn.executescript(SCHEMA)
|
|
202
|
+
workspace_state = workspace_state or {}
|
|
203
|
+
semantic_entities = list(semantic_entities or [])
|
|
204
|
+
semantic_flows = semantic_flows or []
|
|
205
|
+
feature_bindings = feature_bindings or []
|
|
206
|
+
parse_failed_paths = {item.get("path") for item in diagnostics if item.get("kind") == "parse_failed"}
|
|
207
|
+
parsed_count = sum(1 for item in files if item.get("path") not in parse_failed_paths)
|
|
208
|
+
conn.execute(
|
|
209
|
+
"INSERT INTO analysis_snapshots(snapshot_id, schema_version, source_revision, content_manifest_hash, extractor_set_hash, created_at, status) VALUES(?,?,?,?,?,?,?)",
|
|
210
|
+
(snapshot_id, 2, workspace_state.get("repositoryRevision", ""), workspace_state.get("contentManifestHash", ""),
|
|
211
|
+
workspace_state.get("extractorSetHash", ""), workspace_state.get("generatedAt", ""), "current"),
|
|
212
|
+
)
|
|
213
|
+
# Structural symbols and files are first-class graph nodes as well.
|
|
214
|
+
# This prevents semantic flows and feature bindings from dangling when
|
|
215
|
+
# a framework linker points at an existing code symbol.
|
|
216
|
+
structural_entities = [
|
|
217
|
+
{"id": item["id"], "entity_type": item.get("kind", "symbol"), "name": item.get("name", item["id"]),
|
|
218
|
+
"codebase": item.get("codebase"), "path": item.get("path"), "line_start": item.get("line_start"),
|
|
219
|
+
"line_end": item.get("line_end"), "content_hash": item.get("content_hash"), "confidence": "exact",
|
|
220
|
+
"extractor": item.get("parser", "index"), "metadata": item.get("metadata", {})}
|
|
221
|
+
for item in [*files, *symbols]
|
|
222
|
+
]
|
|
223
|
+
by_id = {item["id"]: item for item in semantic_entities}
|
|
224
|
+
for item in structural_entities:
|
|
225
|
+
by_id.setdefault(item["id"], item)
|
|
226
|
+
semantic_entities = list(by_id.values())
|
|
227
|
+
valid_ids = set(by_id)
|
|
228
|
+
invalid_flows = [item for item in semantic_flows if item.get("source_id") not in valid_ids or item.get("target_id") not in valid_ids]
|
|
229
|
+
invalid_bindings = [item for item in feature_bindings if item.get("feature_id") not in valid_ids or item.get("entity_id") not in valid_ids]
|
|
230
|
+
if invalid_flows or invalid_bindings:
|
|
231
|
+
raise ValueError(
|
|
232
|
+
f"semantic graph contains dangling references: flows={len(invalid_flows)}, bindings={len(invalid_bindings)}"
|
|
233
|
+
)
|
|
234
|
+
for entity in semantic_entities:
|
|
235
|
+
logical_id = entity["id"]
|
|
236
|
+
entity_version_id = f"{snapshot_id}:{logical_id}"
|
|
237
|
+
conn.execute(
|
|
238
|
+
"INSERT OR IGNORE INTO logical_entities(logical_id, entity_type, stable_key, metadata_json) VALUES(?,?,?,?)",
|
|
239
|
+
(logical_id, entity["entity_type"], logical_id, json_dumps(entity.get("metadata", {}))),
|
|
240
|
+
)
|
|
241
|
+
conn.execute(
|
|
242
|
+
"INSERT INTO entity_versions(entity_version_id, snapshot_id, logical_id, entity_type, name, codebase, path, line_start, line_end, content_hash, confidence, extractor, metadata_json) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
243
|
+
(entity_version_id, snapshot_id, logical_id, entity["entity_type"], entity["name"], entity.get("codebase"), entity.get("path"), entity.get("line_start"), entity.get("line_end"), entity.get("content_hash"), entity["confidence"], entity["extractor"], json_dumps(entity.get("metadata", {}))),
|
|
244
|
+
)
|
|
245
|
+
conn.executemany(
|
|
246
|
+
"INSERT INTO semantic_flows(snapshot_id, source_id, target_id, flow_type, source_path, line_start, confidence, evidence_level, extractor, metadata_json) VALUES(?,?,?,?,?,?,?,?,?,?)",
|
|
247
|
+
[(snapshot_id, item["source_id"], item["target_id"], item["flow_type"], item.get("source_path"), item.get("line_start"), item["confidence"], item.get("evidence_level", "static_source"), item.get("extractor", "unknown"), json_dumps(item.get("metadata", {}))) for item in semantic_flows],
|
|
248
|
+
)
|
|
249
|
+
conn.executemany(
|
|
250
|
+
"INSERT INTO feature_bindings(snapshot_id, feature_id, entity_id, binding_type, confidence, source_path, line_start) VALUES(?,?,?,?,?,?,?)",
|
|
251
|
+
[(snapshot_id, item["feature_id"], item["entity_id"], item["binding_type"], item["confidence"], item.get("source_path"), item.get("line_start")) for item in feature_bindings],
|
|
252
|
+
)
|
|
253
|
+
conn.execute(
|
|
254
|
+
"INSERT INTO coverage_metrics(snapshot_id, files_scanned, files_parsed, diagnostics, semantic_entities, semantic_flows, dynamic_limitations, metadata_json) VALUES(?,?,?,?,?,?,?,?)",
|
|
255
|
+
(snapshot_id, len(files), parsed_count, len(diagnostics), len(semantic_entities), len(semantic_flows), sum(1 for item in semantic_flows if item.get("metadata", {}).get("dynamic_limitations") or item.get("confidence") in {"heuristic", "unknown"}), json_dumps({"capabilities": workspace_state.get("capabilities", {}), "semantic_extractor": workspace_state.get("semanticExtractor")})),
|
|
256
|
+
)
|
|
88
257
|
conn.executemany(
|
|
89
258
|
"""
|
|
90
259
|
INSERT INTO code_files(id, codebase, path, language, size_bytes, content_hash, indexed_at, parser)
|
package/src/constants.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
2
|
import { copyFile, cp, mkdir, readFile, realpath, stat, writeFile, rename, rm } from "node:fs/promises";
|
|
3
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
3
4
|
import path from "node:path";
|
|
4
5
|
import { Database } from "bun:sqlite";
|
|
5
6
|
|
|
@@ -263,12 +264,33 @@ export class InitializationStore {
|
|
|
263
264
|
await writeFile(migrationBackup, db.serialize(), { flag: "wx" });
|
|
264
265
|
}
|
|
265
266
|
}
|
|
266
|
-
db.transaction(() => {
|
|
267
|
+
const migrate = db.transaction(() => {
|
|
267
268
|
db.exec(SCHEMA);
|
|
269
|
+
const ensureColumn = (table: string, column: string, definition: string) => {
|
|
270
|
+
const columns = db.query(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>;
|
|
271
|
+
if (!columns.some(item => item.name === column)) db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
|
|
272
|
+
};
|
|
273
|
+
ensureColumn("context_snapshots", "index_ref_json", "TEXT");
|
|
274
|
+
ensureColumn("document_jobs", "model_role", "TEXT");
|
|
275
|
+
ensureColumn("document_jobs", "output_hash", "TEXT");
|
|
276
|
+
ensureColumn("document_jobs", "conflict_path", "TEXT");
|
|
277
|
+
ensureColumn("document_jobs", "error", "TEXT");
|
|
268
278
|
const meta = db.query("SELECT version FROM schema_meta LIMIT 1").get() as { version: number } | null;
|
|
269
|
-
if (!meta)
|
|
270
|
-
|
|
271
|
-
|
|
279
|
+
if (!meta) {
|
|
280
|
+
db.query("INSERT INTO schema_meta(version) VALUES (?)").run(INITIALIZATION_SCHEMA_VERSION);
|
|
281
|
+
} else {
|
|
282
|
+
let version = meta.version;
|
|
283
|
+
if (version < 2) {
|
|
284
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_input_snapshots_project ON input_snapshots(project_id, workspace_id, config_id)");
|
|
285
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_document_jobs_status ON document_jobs(status, updated_at)");
|
|
286
|
+
version = 2;
|
|
287
|
+
}
|
|
288
|
+
db.query("UPDATE schema_meta SET version=?").run(version);
|
|
289
|
+
}
|
|
290
|
+
const verified = db.query("SELECT version FROM schema_meta LIMIT 1").get() as { version: number };
|
|
291
|
+
if (verified.version !== INITIALIZATION_SCHEMA_VERSION) throw new Error(`Initialization schema migration stopped at ${verified.version}`);
|
|
292
|
+
});
|
|
293
|
+
try { migrate(); } catch (error) { db.close(); throw new Error(`Initialization database migration failed: ${error instanceof Error ? error.message : String(error)}`); }
|
|
272
294
|
db.query("INSERT OR IGNORE INTO projects(id,name,created_at) VALUES (?,?,?)").run(context.manifest.projectId, context.manifest.name, now());
|
|
273
295
|
db.query("INSERT OR IGNORE INTO authority_state(project_id,authority,revision,switched_at) VALUES (?,?,?,?)").run(context.manifest.projectId, "legacy", 0, now());
|
|
274
296
|
return new InitializationStore(context, databasePath, db);
|
|
@@ -311,13 +333,23 @@ export class InitializationStore {
|
|
|
311
333
|
return { id, revision: row.revision, created: true };
|
|
312
334
|
}
|
|
313
335
|
|
|
314
|
-
createInputSnapshot(workspaceId: string, configId: string, files: Array<{ repository: string; path: string; contentHash: string; size?: number }>, stable =
|
|
336
|
+
createInputSnapshot(workspaceId: string, configId: string, files: Array<{ repository: string; path: string; contentHash: string; size?: number }>, stable = false): { id: string; fingerprint: string; created: boolean } {
|
|
337
|
+
const binding = this.db.query(`SELECT w.id AS workspaceId,c.id AS configId,w.project_id AS workspaceProject,c.project_id AS configProject,c.config_json AS config
|
|
338
|
+
FROM workspaces w JOIN scan_configs c ON c.project_id=w.project_id
|
|
339
|
+
WHERE w.id=? AND c.id=? AND w.project_id=?`).get(workspaceId, configId, this.context.manifest.projectId) as { workspaceId: string; configId: string; workspaceProject: string; configProject: string; config: string } | null;
|
|
340
|
+
if (!binding) throw new Error("Input snapshot workspace and scan config do not belong to the same project boundary");
|
|
341
|
+
const configured = new Map((parse<ScanConfigInput>(binding.config).repositories ?? []).map(repository => [repository.name, repository.relativePath]));
|
|
342
|
+
for (const file of files) {
|
|
343
|
+
if (!configured.has(file.repository)) throw new Error(`Input snapshot repository is not configured: ${file.repository}`);
|
|
344
|
+
const base = configured.get(file.repository)!;
|
|
345
|
+
if (base !== "." && !path.isAbsolute(base) && file.path !== base && !file.path.startsWith(`${base.replace(/\\/g, "/").replace(/\/$/, "")}/`)) throw new Error(`Input snapshot path is outside repository: ${file.repository}/${file.path}`);
|
|
346
|
+
}
|
|
315
347
|
const normalized = files.map(file => ({ ...file, path: validateRelative(file.path) })).sort((a, b) => `${a.repository}/${a.path}`.localeCompare(`${b.repository}/${b.path}`));
|
|
316
348
|
const manifestJson = canonicalJson(normalized);
|
|
317
349
|
const fingerprint = hash(manifestJson);
|
|
318
350
|
const existing = this.db.query("SELECT id,stable FROM input_snapshots WHERE workspace_id=? AND config_id=? AND fingerprint=?").get(workspaceId, configId, fingerprint) as { id: string; stable: number } | null;
|
|
319
351
|
if (existing) {
|
|
320
|
-
if (stable && !existing.stable)
|
|
352
|
+
if (stable && !existing.stable) throw new Error("Existing input snapshot is unstable; confirm it explicitly before publishing");
|
|
321
353
|
return { id: existing.id, fingerprint, created: false };
|
|
322
354
|
}
|
|
323
355
|
const id = randomUUID();
|
|
@@ -369,27 +401,39 @@ export class InitializationStore {
|
|
|
369
401
|
recordCollection(runId: string, leaseToken: string, input: { repository?: string; kind: FactKind; status: CoverageStatus; adapter: string; capability?: Record<string, unknown>; diagnostics?: unknown[]; facts?: FactInput[] }): void {
|
|
370
402
|
this.assertWritableRun(runId, leaseToken);
|
|
371
403
|
const transaction = this.db.transaction(() => {
|
|
372
|
-
|
|
404
|
+
const repository = input.repository ?? "";
|
|
405
|
+
this.db.query("DELETE FROM evidence WHERE fact_id IN (SELECT id FROM facts WHERE run_id=? AND repository=? AND kind=?)").run(runId, repository, input.kind);
|
|
406
|
+
this.db.query("DELETE FROM facts WHERE run_id=? AND repository=? AND kind=?").run(runId, repository, input.kind);
|
|
407
|
+
this.db.query("INSERT INTO collection_results(id,run_id,repository,kind,status,adapter,capability_json,diagnostics_json,created_at) VALUES (?,?,?,?,?,?,?,?,?) ON CONFLICT(run_id,repository,kind) DO UPDATE SET status=excluded.status,adapter=excluded.adapter,capability_json=excluded.capability_json,diagnostics_json=excluded.diagnostics_json,created_at=excluded.created_at").run(randomUUID(), runId, repository, input.kind, input.status, input.adapter, canonicalJson(input.capability ?? {}), canonicalJson(input.diagnostics ?? []), now());
|
|
373
408
|
for (const fact of input.facts ?? []) {
|
|
374
409
|
if (fact.kind !== input.kind) throw new Error(`Fact kind ${fact.kind} does not match collection ${input.kind}`);
|
|
375
410
|
if (!fact.objectKey || !fact.extractor) throw new Error("Facts require objectKey and extractor");
|
|
376
411
|
if (fact.evidenceLevel === "observed" && !fact.sourcePath) throw new Error("Observed facts require source evidence");
|
|
377
412
|
validateFactShape(fact);
|
|
378
413
|
const factId = randomUUID();
|
|
379
|
-
this.db.query("INSERT INTO facts(id,run_id,kind,object_key,repository,value_json,evidence_level,extractor,confidence,created_at) VALUES (?,?,?,?,?,?,?,?,?,?)
|
|
380
|
-
const stored =
|
|
414
|
+
this.db.query("INSERT INTO facts(id,run_id,kind,object_key,repository,value_json,evidence_level,extractor,confidence,created_at) VALUES (?,?,?,?,?,?,?,?,?,?)").run(factId, runId, fact.kind, fact.objectKey, fact.repository ?? repository, canonicalJson(fact.value), fact.evidenceLevel, fact.extractor, fact.confidence ?? "inferred", now());
|
|
415
|
+
const stored = { id: factId };
|
|
381
416
|
if (fact.sourcePath) this.db.query("INSERT OR IGNORE INTO evidence(id,fact_id,source_path,source_line,source_fingerprint,method,created_at) VALUES (?,?,?,?,?,?,?)").run(randomUUID(), stored.id, validateRelative(fact.sourcePath), fact.sourceLine ?? null, fact.sourceFingerprint ?? "unavailable", fact.extractor, now());
|
|
382
417
|
}
|
|
383
418
|
});
|
|
384
419
|
transaction();
|
|
385
420
|
}
|
|
386
421
|
|
|
387
|
-
cancelRun(runId: string): boolean {
|
|
388
|
-
|
|
422
|
+
cancelRun(runId: string, leaseToken: string, expectedRevision: number): boolean {
|
|
423
|
+
const row = this.db.query("SELECT status,lease_token AS leaseToken,lease_expires_at AS expires,workspace_id AS workspaceId,config_id AS configId FROM initialization_runs WHERE id=?").get(runId) as { status: RunStatus; leaseToken: string; expires: string; workspaceId: string; configId: string } | null;
|
|
424
|
+
if (!row) throw new Error(`Unknown initialization run: ${runId}`);
|
|
425
|
+
if (row.leaseToken !== leaseToken) throw new Error("Initialization lease token mismatch");
|
|
426
|
+
if (Date.parse(row.expires) < Date.now()) throw new Error("Initialization lease expired; recover the run before cancelling");
|
|
427
|
+
const current = this.db.query("SELECT revision FROM current_context WHERE workspace_id=? AND config_id=?").get(row.workspaceId, row.configId) as { revision: number } | null;
|
|
428
|
+
if (expectedRevision !== (current?.revision ?? 0)) throw new Error("Initialization run revision conflict");
|
|
429
|
+
const timestamp = now();
|
|
430
|
+
return this.db.query("UPDATE initialization_runs SET status='cancelled',updated_at=? WHERE id=? AND lease_token=? AND lease_expires_at>? AND status IN ('preparing','collecting','validating')").run(timestamp, runId, leaseToken, timestamp).changes === 1;
|
|
389
431
|
}
|
|
390
432
|
|
|
391
433
|
failRun(runId: string, leaseToken: string, error: string): boolean {
|
|
392
|
-
|
|
434
|
+
this.assertWritableRun(runId, leaseToken);
|
|
435
|
+
const timestamp = now();
|
|
436
|
+
return this.db.query("UPDATE initialization_runs SET status='failed',error=?,updated_at=? WHERE id=? AND lease_token=? AND lease_expires_at>? AND status IN ('preparing','collecting','validating')").run(error.slice(0, 4000), timestamp, runId, leaseToken, timestamp).changes === 1;
|
|
393
437
|
}
|
|
394
438
|
|
|
395
439
|
recoverRun(runId: string, leaseMs = 15 * 60_000): { leaseToken: string } {
|
|
@@ -399,11 +443,12 @@ export class InitializationStore {
|
|
|
399
443
|
if (["failed", "finished"].includes(row.status)) throw new Error(`Run cannot be recovered from ${row.status}`);
|
|
400
444
|
if (Date.parse(row.expires) >= Date.now()) throw new Error("Run lease is still active");
|
|
401
445
|
const leaseToken = randomUUID();
|
|
402
|
-
this.db.query("UPDATE initialization_runs SET lease_token=?,lease_expires_at=?,updated_at=? WHERE id=?").run(leaseToken, new Date(Date.now() + leaseMs).toISOString(), now(), runId);
|
|
446
|
+
const changed = this.db.query("UPDATE initialization_runs SET lease_token=?,lease_expires_at=?,updated_at=? WHERE id=? AND lease_expires_at<? AND status IN ('preparing','collecting','validating')").run(leaseToken, new Date(Date.now() + leaseMs).toISOString(), now(), runId, now());
|
|
447
|
+
if (changed.changes !== 1) throw new Error("Initialization run recovery conflict");
|
|
403
448
|
return { leaseToken };
|
|
404
449
|
}
|
|
405
450
|
|
|
406
|
-
publishRun(runId: string, leaseToken: string, options: { expectedRevision
|
|
451
|
+
publishRun(runId: string, leaseToken: string, options: { expectedRevision: number; completeness?: "complete" | "partial"; indexRef?: Record<string, unknown>; updateCurrent?: boolean }): { snapshotId: string; revision: number } {
|
|
407
452
|
const run = this.assertWritableRun(runId, leaseToken);
|
|
408
453
|
const input = this.db.query("SELECT stable,fingerprint FROM input_snapshots WHERE id=?").get(run.inputSnapshotId) as { stable: number; fingerprint: string };
|
|
409
454
|
if (!input.stable) throw new Error("Cannot publish an unstable input snapshot");
|
|
@@ -422,12 +467,31 @@ export class InitializationStore {
|
|
|
422
467
|
if (completeness === "complete" && blockers.length) throw new Error(`Required collection coverage is incomplete: ${blockers.join(", ")}`);
|
|
423
468
|
const current = this.db.query("SELECT revision FROM current_context WHERE workspace_id=? AND config_id=?").get(run.workspaceId, run.configId) as { revision: number } | null;
|
|
424
469
|
const revision = (current?.revision ?? 0) + 1;
|
|
425
|
-
if (options.expectedRevision !==
|
|
470
|
+
if (options.expectedRevision !== (current?.revision ?? 0)) throw new Error(`Current context revision conflict: expected ${options.expectedRevision}, actual ${current?.revision ?? 0}`);
|
|
471
|
+
if (options.indexRef) {
|
|
472
|
+
if (options.indexRef.inputFingerprint !== input.fingerprint || options.indexRef.workspaceId !== run.workspaceId || options.indexRef.configId !== run.configId) throw new Error("Index reference does not match initialization input snapshot");
|
|
473
|
+
if (options.indexRef.status === "complete" && (typeof options.indexRef.generation !== "string" || typeof options.indexRef.snapshotHash !== "string")) throw new Error("Complete index references require generation and snapshot hash");
|
|
474
|
+
if (options.indexRef.status === "complete") {
|
|
475
|
+
if (typeof options.indexRef.contentManifestHash !== "string" || typeof options.indexRef.repositoryRevision !== "string") throw new Error("Complete index references require content manifest and repository revision");
|
|
476
|
+
const indexRoot = path.resolve(this.context.root, String(options.indexRef.stagingPath ?? options.indexRef.path ?? ".planning/code-index"));
|
|
477
|
+
const immutable = path.resolve(indexRoot, String(options.indexRef.immutablePath ?? `snapshots/${options.indexRef.generation}`));
|
|
478
|
+
const snapshotFile = path.join(immutable, "SNAPSHOT.json");
|
|
479
|
+
if (!existsSync(snapshotFile)) throw new Error("Complete index reference does not point to an immutable generation");
|
|
480
|
+
const actual = hash(readFileSync(snapshotFile));
|
|
481
|
+
if (actual !== options.indexRef.snapshotHash) throw new Error("Code index snapshot hash does not match index reference");
|
|
482
|
+
const payload = JSON.parse(readFileSync(snapshotFile, "utf8")) as { generation?: string };
|
|
483
|
+
if (payload.generation !== options.indexRef.generation) throw new Error("Code index generation does not match SNAPSHOT.json");
|
|
484
|
+
const workspace = (payload as any).workspace ?? {};
|
|
485
|
+
if (workspace.contentManifestHash !== options.indexRef.contentManifestHash) throw new Error("Code index content manifest does not match index reference");
|
|
486
|
+
if (workspace.repositoryRevision !== options.indexRef.repositoryRevision) throw new Error("Code index repository revision does not match index reference");
|
|
487
|
+
}
|
|
488
|
+
}
|
|
426
489
|
const facts = this.db.query("SELECT kind,repository,object_key,value_json,evidence_level,extractor,confidence FROM facts WHERE run_id=? ORDER BY kind,repository,object_key").all(runId);
|
|
427
490
|
const snapshotId = randomUUID();
|
|
428
491
|
const transaction = this.db.transaction(() => {
|
|
429
492
|
this.db.query("UPDATE initialization_runs SET status='validating',updated_at=? WHERE id=? AND lease_token=?").run(now(), runId, leaseToken);
|
|
430
|
-
|
|
493
|
+
const persistedIndexRef = options.indexRef ? Object.fromEntries(Object.entries(options.indexRef).filter(([key]) => key !== "stagingPath")) : null;
|
|
494
|
+
this.db.query("INSERT INTO context_snapshots(id,project_id,workspace_id,config_id,input_snapshot_id,run_id,completeness,content_hash,index_ref_json,created_at) VALUES (?,?,?,?,?,?,?,?,?,?)").run(snapshotId, this.context.manifest.projectId, run.workspaceId, run.configId, run.inputSnapshotId, runId, completeness, hash(canonicalJson(facts)), persistedIndexRef ? canonicalJson(persistedIndexRef) : null, now());
|
|
431
495
|
if (options.updateCurrent === false) {
|
|
432
496
|
// Shadow imports are immutable and queryable without becoming current.
|
|
433
497
|
} else if (current) {
|
|
@@ -484,6 +548,11 @@ export class InitializationStore {
|
|
|
484
548
|
return this.queryContext({ workspaceId, configId: current.configId, ...input });
|
|
485
549
|
}
|
|
486
550
|
|
|
551
|
+
currentRevision(workspaceId: string, configId: string): number {
|
|
552
|
+
const row = this.db.query("SELECT revision FROM current_context WHERE workspace_id=? AND config_id=?").get(workspaceId, configId) as { revision: number } | null;
|
|
553
|
+
return row?.revision ?? 0;
|
|
554
|
+
}
|
|
555
|
+
|
|
487
556
|
getInitializationConfiguration(workspaceId: string): Record<string, unknown> {
|
|
488
557
|
const row = this.db.query(`SELECT c.id,c.revision,c.content_hash AS contentHash,c.config_json AS config
|
|
489
558
|
FROM scan_configs c LEFT JOIN current_context p ON p.config_id=c.id AND p.workspace_id=?
|
|
@@ -498,11 +567,14 @@ export class InitializationStore {
|
|
|
498
567
|
|
|
499
568
|
compareSnapshots(fromSnapshotId: string, toSnapshotId: string): Record<string, unknown> {
|
|
500
569
|
if (fromSnapshotId === toSnapshotId) return { comparable: true, fromSnapshotId, toSnapshotId, added: [], changed: [], disappeared: [] };
|
|
501
|
-
const snapshots = this.db.query("SELECT id,config_id AS configId FROM context_snapshots WHERE id IN (?,?)").all(fromSnapshotId, toSnapshotId) as Array<{ id: string; configId: string }>;
|
|
570
|
+
const snapshots = this.db.query("SELECT id,project_id AS projectId,workspace_id AS workspaceId,config_id AS configId,input_snapshot_id AS inputSnapshotId FROM context_snapshots WHERE id IN (?,?)").all(fromSnapshotId, toSnapshotId) as Array<{ id: string; projectId: string; workspaceId: string; configId: string; inputSnapshotId: string }>;
|
|
502
571
|
if (snapshots.length !== 2) throw new Error("Both context snapshots must exist");
|
|
503
572
|
const fromMeta = snapshots.find(item => item.id === fromSnapshotId)!;
|
|
504
573
|
const toMeta = snapshots.find(item => item.id === toSnapshotId)!;
|
|
574
|
+
if (fromMeta.projectId !== toMeta.projectId || fromMeta.workspaceId !== toMeta.workspaceId) return { comparable: false, reason: "workspace_changed", added: [], changed: [], disappeared: [] };
|
|
505
575
|
if (fromMeta.configId !== toMeta.configId) return { comparable: false, reason: "scan_config_changed", added: [], changed: [], disappeared: [] };
|
|
576
|
+
const manifests = this.db.query("SELECT id,manifest_json AS manifest FROM input_snapshots WHERE id IN (?,?)").all(fromMeta.inputSnapshotId, toMeta.inputSnapshotId) as Array<{ id: string; manifest: string }>;
|
|
577
|
+
if (manifests.length !== 2) return { comparable: false, reason: "input_snapshot_missing", added: [], changed: [], disappeared: [] };
|
|
506
578
|
const rows = (snapshotId: string) => this.db.query(`SELECT f.kind,f.repository,f.object_key AS objectKey,f.value_json AS value
|
|
507
579
|
FROM facts f JOIN context_snapshots s ON s.run_id=f.run_id WHERE s.id=?`).all(snapshotId) as Array<{ kind: string; repository: string; objectKey: string; value: string }>;
|
|
508
580
|
const key = (row: { kind: string; repository: string; objectKey: string }) => `${row.kind}\0${row.repository}\0${row.objectKey}`;
|
|
@@ -539,7 +611,8 @@ export class InitializationStore {
|
|
|
539
611
|
const job = this.db.query("SELECT target_path AS targetPath,ownership,status FROM document_jobs WHERE id=?").get(jobId) as { targetPath: string; ownership: string; status: string } | null;
|
|
540
612
|
if (!job) throw new Error(`Unknown document job: ${jobId}`);
|
|
541
613
|
if (job.ownership === "human") throw new Error("Human-owned documents cannot be generated");
|
|
542
|
-
this.db.query("UPDATE document_jobs SET status='generating',error=NULL,updated_at=? WHERE id=? AND status IN ('queued','failed','pending_review')").run(now(), jobId);
|
|
614
|
+
const claimed = this.db.query("UPDATE document_jobs SET status='generating',error=NULL,updated_at=? WHERE id=? AND status IN ('queued','failed','pending_review')").run(now(), jobId);
|
|
615
|
+
if (claimed.changes !== 1) throw new Error(`Document job is no longer writable: ${jobId}`);
|
|
543
616
|
const target = path.resolve(this.context.root, job.targetPath);
|
|
544
617
|
if (target !== this.context.root && !target.startsWith(`${this.context.root}${path.sep}`)) throw new Error("Document path escapes project root");
|
|
545
618
|
let existing: string | null = null;
|
|
@@ -559,7 +632,8 @@ export class InitializationStore {
|
|
|
559
632
|
}
|
|
560
633
|
await mkdir(path.dirname(target), { recursive: true });
|
|
561
634
|
const temporary = `${target}.${randomUUID()}.tmp`; await writeFile(temporary, output, { encoding: "utf8", flag: "wx" }); await rename(temporary, target);
|
|
562
|
-
this.db.query("UPDATE document_jobs SET status='complete',output_hash=?,updated_at=? WHERE id=?").run(hash(output), now(), jobId);
|
|
635
|
+
const completed = this.db.query("UPDATE document_jobs SET status='complete',output_hash=?,updated_at=? WHERE id=? AND status='generating'").run(hash(output), now(), jobId);
|
|
636
|
+
if (completed.changes !== 1) throw new Error(`Document job completion conflict: ${jobId}`);
|
|
563
637
|
return { status: "complete", path: target };
|
|
564
638
|
}
|
|
565
639
|
|
package/src/project-context.ts
CHANGED
|
@@ -136,15 +136,17 @@ export async function loadProjectContext(root: string): Promise<ProjectContext>
|
|
|
136
136
|
const manifestPath = path.join(canonicalRoot, ".devflow", "project.json");
|
|
137
137
|
const manifest = validateManifest(JSON.parse(await readFile(manifestPath, "utf8")), manifestPath);
|
|
138
138
|
const planningRoot = await assertPlanningBoundary(canonicalRoot);
|
|
139
|
+
if (!(await exists(projectControlDatabasePath(manifest.projectId)))) throw new Error("Project initialization database is not registered; run explicit project initialization first");
|
|
139
140
|
return attachInitializationStore({ root: canonicalRoot, manifestPath, manifest, planningRoot });
|
|
140
141
|
}
|
|
141
142
|
|
|
142
143
|
async function attachInitializationStore(context: Omit<ProjectContext, "workspaceId" | "controlDatabasePath">): Promise<ProjectContext> {
|
|
143
144
|
const store = await InitializationStore.open(context);
|
|
144
145
|
try {
|
|
145
|
-
const
|
|
146
|
-
|
|
147
|
-
|
|
146
|
+
const location = await realpath(context.root);
|
|
147
|
+
const workspace = store.listWorkspaces().find(item => item.location === location);
|
|
148
|
+
if (!workspace) throw new Error("Project workspace is not registered; run explicit project initialization first");
|
|
149
|
+
return { ...context, workspaceId: String(workspace.id), controlDatabasePath: projectControlDatabasePath(context.manifest.projectId) };
|
|
148
150
|
} finally {
|
|
149
151
|
store.close();
|
|
150
152
|
}
|
|
@@ -174,6 +176,8 @@ export async function initializeProject(root: string): Promise<{ context: Projec
|
|
|
174
176
|
createdAt: new Date().toISOString(),
|
|
175
177
|
};
|
|
176
178
|
await atomicWriteJson(manifestPath, manifest);
|
|
179
|
+
const bootstrapStore = await InitializationStore.open({ root: canonicalRoot, manifest });
|
|
180
|
+
try { await bootstrapStore.registerWorkspace(); } finally { bootstrapStore.close(); }
|
|
177
181
|
return {
|
|
178
182
|
context: await attachInitializationStore({ root: canonicalRoot, manifestPath, manifest, planningRoot }),
|
|
179
183
|
created: true,
|
package/src/project-documents.ts
CHANGED
|
@@ -77,6 +77,10 @@ function table(headers: string[], rows: unknown[][]): string {
|
|
|
77
77
|
}
|
|
78
78
|
function yamlString(value: string): string { return JSON.stringify(value); }
|
|
79
79
|
function frontmatter(fields: Record<string, string | boolean | string[] | null | undefined>): string {
|
|
80
|
+
if (fields.source_snapshot && !fields.snapshot_id) fields.snapshot_id = fields.source_snapshot;
|
|
81
|
+
if (fields.source_snapshot) fields.generated = true;
|
|
82
|
+
if (fields.source_snapshot && !fields.generated_at) fields.generated_at = new Date().toISOString();
|
|
83
|
+
if (fields.source_snapshot && !fields.source_revision) fields.source_revision = fields.code_index_generation ?? "unknown";
|
|
80
84
|
const lines = ["---"];
|
|
81
85
|
for (const [key, value] of Object.entries(fields)) {
|
|
82
86
|
if (value === undefined || value === null) continue;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { createHash } from "node:crypto";
|
|
2
|
-
import { readFile, readdir, stat } from "node:fs/promises";
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { readFile, readdir, stat, rename, rm } from "node:fs/promises";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
|
|
5
5
|
import { FACT_KINDS, InitializationStore, type FactInput, type FactKind, type ScanConfigInput } from "./initialization-store.ts";
|
|
@@ -177,7 +177,7 @@ async function runProcess(argv: string[], cwd: string): Promise<{ stdout: string
|
|
|
177
177
|
|
|
178
178
|
async function buildCodeIndex(context: ProjectContext, config: ScanConfigInput): Promise<Record<string, unknown>> {
|
|
179
179
|
const script = path.resolve(import.meta.dir, "../plugins/devflow-intel/skills/workflow-code-index/scripts/code_index.py");
|
|
180
|
-
const output =
|
|
180
|
+
const output = `.planning/.code-index-pending-${randomUUID()}`;
|
|
181
181
|
const codebases = config.repositories.flatMap(repository => ["--codebase", `${repository.name}:${path.resolve(context.root, repository.relativePath)}:${repository.role}`]);
|
|
182
182
|
await runProcess(["python3", script, "scan", "--root", context.root, "--output", output, ...codebases], context.root);
|
|
183
183
|
await runProcess(["python3", script, "validate", "--index", output], context.root);
|
|
@@ -185,7 +185,29 @@ async function buildCodeIndex(context: ProjectContext, config: ScanConfigInput):
|
|
|
185
185
|
const snapshotPath = path.join(context.root, output, "SNAPSHOT.json");
|
|
186
186
|
const bytes = await readFile(snapshotPath); const snapshot = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
|
|
187
187
|
if (!snapshot.generation || !snapshot.immutablePath) throw new Error("Code index did not publish an immutable generation");
|
|
188
|
-
|
|
188
|
+
const workspace = (snapshot.workspace ?? {}) as Record<string, unknown>;
|
|
189
|
+
return { path: ".planning/code-index", stagingPath: output, generation: snapshot.generation, immutablePath: snapshot.immutablePath, snapshotHash: digest(bytes), contentManifestHash: workspace.contentManifestHash, repositoryRevision: workspace.repositoryRevision, status: "complete" };
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
async function promoteCodeIndex(context: ProjectContext, indexRef: Record<string, unknown>): Promise<() => Promise<void>> {
|
|
193
|
+
const stagingPath = typeof indexRef.stagingPath === "string" ? path.resolve(context.root, indexRef.stagingPath) : null;
|
|
194
|
+
const target = path.resolve(context.root, ".planning/code-index");
|
|
195
|
+
if (!stagingPath) return async () => {};
|
|
196
|
+
const previous = `${target}.previous-${randomUUID()}`;
|
|
197
|
+
const existing = await stat(target).catch(() => null);
|
|
198
|
+
if (existing) await rename(target, previous);
|
|
199
|
+
try {
|
|
200
|
+
await rename(stagingPath, target);
|
|
201
|
+
delete indexRef.stagingPath;
|
|
202
|
+
return async () => {
|
|
203
|
+
await rm(target, { recursive: true, force: true });
|
|
204
|
+
if (existing) await rename(previous, target);
|
|
205
|
+
};
|
|
206
|
+
} catch (error) {
|
|
207
|
+
await rm(target, { recursive: true, force: true }).catch(() => {});
|
|
208
|
+
if (existing) await rename(previous, target).catch(() => {});
|
|
209
|
+
throw error;
|
|
210
|
+
}
|
|
189
211
|
}
|
|
190
212
|
|
|
191
213
|
export async function runProjectInitialization(context: ProjectContext, input: { config?: ScanConfigInput; requestId?: string; documentMode?: "low-tier-model" | "template" | "none"; organizer?: import("./project-documents.ts").LowTierDocumentOrganizer; switchAuthority?: boolean } = {}): Promise<InitializationProgress> {
|
|
@@ -236,13 +258,22 @@ export async function runProjectInitialization(context: ProjectContext, input: {
|
|
|
236
258
|
facts: status === "complete" ? repositoryFacts : [],
|
|
237
259
|
});
|
|
238
260
|
}
|
|
239
|
-
const indexRef = config.mode === "full"
|
|
261
|
+
const indexRef = config.mode === "full"
|
|
262
|
+
? { ...(await buildCodeIndex(context, config)), inputFingerprint: inputSnapshot.fingerprint, workspaceId: workspace.id, configId: storedConfig.id }
|
|
263
|
+
: { status: "skipped", reason: "explicit_fast_mode", inputFingerprint: inputSnapshot.fingerprint, workspaceId: workspace.id, configId: storedConfig.id };
|
|
240
264
|
const after = await scan(context, config);
|
|
241
265
|
const afterManifest = frozenManifest(after);
|
|
242
266
|
const verified = store.createInputSnapshot(workspace.id, storedConfig.id, afterManifest, false);
|
|
243
267
|
if (verified.fingerprint !== inputSnapshot.fingerprint) throw new Error("Source changed during initialization; refusing to publish mixed input");
|
|
244
268
|
store.confirmInputSnapshot(inputSnapshot.id, inputSnapshot.fingerprint);
|
|
245
|
-
const
|
|
269
|
+
const rollbackIndex = await promoteCodeIndex(context, indexRef);
|
|
270
|
+
let published: { snapshotId: string; revision: number };
|
|
271
|
+
try {
|
|
272
|
+
published = store.publishRun(run.id, run.leaseToken, { expectedRevision: store.currentRevision(workspace.id, storedConfig.id), indexRef });
|
|
273
|
+
} catch (error) {
|
|
274
|
+
await rollbackIndex();
|
|
275
|
+
throw error;
|
|
276
|
+
}
|
|
246
277
|
if (input.switchAuthority !== false) {
|
|
247
278
|
const authority = store.getAuthority();
|
|
248
279
|
if (authority.authority === "legacy") store.switchAuthority("database", authority.revision);
|
package/src/project-migration.ts
CHANGED
|
@@ -22,7 +22,8 @@ export async function importLegacyInitialization(context: ProjectContext, source
|
|
|
22
22
|
try {
|
|
23
23
|
const workspace = await store.registerWorkspace();
|
|
24
24
|
const config = store.putScanConfig({ mode: "fast", repositories: [{ name: "legacy-materials", relativePath: ".", role: "other" }], requiredKinds: ["evidence"], adapterVersions: { "legacy-import": "1" } });
|
|
25
|
-
const input = store.createInputSnapshot(workspace.id, config.id, documents.map(item => ({ repository: "legacy-materials", path: item.relative, contentHash: item.contentHash, size: item.size })),
|
|
25
|
+
const input = store.createInputSnapshot(workspace.id, config.id, documents.map(item => ({ repository: "legacy-materials", path: item.relative, contentHash: item.contentHash, size: item.size })), false);
|
|
26
|
+
store.confirmInputSnapshot(input.id, input.fingerprint);
|
|
26
27
|
const run = store.startRun({ workspaceId: workspace.id, configId: config.id, inputSnapshotId: input.id, requestId });
|
|
27
28
|
if (!run.reused) store.recordCollection(run.id, run.leaseToken, {
|
|
28
29
|
kind: "evidence", status: "complete", adapter: "legacy-import@1",
|
|
@@ -30,7 +31,7 @@ export async function importLegacyInitialization(context: ProjectContext, source
|
|
|
30
31
|
});
|
|
31
32
|
const snapshot = run.reused && store.getRun(run.id)?.publishedSnapshotId
|
|
32
33
|
? { snapshotId: String(store.getRun(run.id)!.publishedSnapshotId) }
|
|
33
|
-
: store.publishRun(run.id, run.leaseToken, { completeness: "partial", updateCurrent: false });
|
|
34
|
+
: store.publishRun(run.id, run.leaseToken, { expectedRevision: 0, completeness: "partial", updateCurrent: false });
|
|
34
35
|
const imports = documents.map(item => store.registerHistoricalImport(workspace.id, item.relative, item.contentHash, snapshot.snapshotId));
|
|
35
36
|
const authority = store.getAuthority();
|
|
36
37
|
if (authority.authority === "legacy") store.switchAuthority("shadow", authority.revision);
|