@guanlin4924/devflow-pi 0.1.71 → 0.1.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/base.py +5 -1
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/generic_text.py +27 -0
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/registry.py +48 -6
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/structured_scope.py +43 -5
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/code_index.py +128 -6
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/semantic.py +187 -0
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/writers/sqlite_writer.py +170 -1
- package/src/constants.ts +1 -1
- package/src/initialization-store.ts +13 -3
- package/src/project-initializer.ts +25 -0
package/package.json
CHANGED
|
@@ -7,6 +7,11 @@ class ParseResult:
|
|
|
7
7
|
symbols: List[Dict] = field(default_factory=list)
|
|
8
8
|
relations: List[Dict] = field(default_factory=list)
|
|
9
9
|
diagnostics: List[Dict] = field(default_factory=list)
|
|
10
|
+
# Optional cross-layer facts. Adapters may populate these when they have
|
|
11
|
+
# framework/type information; the indexer also derives conservative facts
|
|
12
|
+
# in a separate linker pass.
|
|
13
|
+
entities: List[Dict] = field(default_factory=list)
|
|
14
|
+
flows: List[Dict] = field(default_factory=list)
|
|
10
15
|
|
|
11
16
|
|
|
12
17
|
class Adapter:
|
|
@@ -27,4 +32,3 @@ class Adapter:
|
|
|
27
32
|
@property
|
|
28
33
|
def parser_id(self) -> str:
|
|
29
34
|
return f"{self.name}@{self.version()}"
|
|
30
|
-
|
|
@@ -30,6 +30,29 @@ LANGUAGE_BY_EXTENSION = {
|
|
|
30
30
|
".yml": "yaml",
|
|
31
31
|
".toml": "toml",
|
|
32
32
|
".xml": "xml",
|
|
33
|
+
".h": "c",
|
|
34
|
+
".hh": "cpp",
|
|
35
|
+
".hpp": "cpp",
|
|
36
|
+
".hxx": "cpp",
|
|
37
|
+
".c": "c",
|
|
38
|
+
".cc": "cpp",
|
|
39
|
+
".cpp": "cpp",
|
|
40
|
+
".cxx": "cpp",
|
|
41
|
+
".cs": "csharp",
|
|
42
|
+
".scala": "scala",
|
|
43
|
+
".sc": "scala",
|
|
44
|
+
".lua": "lua",
|
|
45
|
+
".html": "html",
|
|
46
|
+
".htm": "html",
|
|
47
|
+
".css": "css",
|
|
48
|
+
".scss": "css",
|
|
49
|
+
".sass": "css",
|
|
50
|
+
".svelte": "svelte",
|
|
51
|
+
".graphql": "graphql",
|
|
52
|
+
".gql": "graphql",
|
|
53
|
+
".proto": "protobuf",
|
|
54
|
+
".hcl": "hcl",
|
|
55
|
+
".tf": "terraform",
|
|
33
56
|
}
|
|
34
57
|
|
|
35
58
|
|
|
@@ -166,6 +189,10 @@ class GenericTextAdapter(Adapter):
|
|
|
166
189
|
|
|
167
190
|
|
|
168
191
|
def language_for_path(path: Path) -> str:
|
|
192
|
+
if path.name.lower() == "dockerfile":
|
|
193
|
+
return "dockerfile"
|
|
194
|
+
if path.name.lower().startswith("dockerfile."):
|
|
195
|
+
return "dockerfile"
|
|
169
196
|
return LANGUAGE_BY_EXTENSION.get(path.suffix.lower(), path.suffix.lower().lstrip(".") or "text")
|
|
170
197
|
|
|
171
198
|
|
|
@@ -5,15 +5,57 @@ from .structured_scope import StructuredScopeAdapter
|
|
|
5
5
|
|
|
6
6
|
class AdapterRegistry:
|
|
7
7
|
def __init__(self):
|
|
8
|
-
self.
|
|
8
|
+
self.python = PythonAstAdapter()
|
|
9
|
+
self.structured = StructuredScopeAdapter()
|
|
10
|
+
self.generic = GenericTextAdapter()
|
|
11
|
+
self.adapters = [self.python, self.structured, self.generic]
|
|
12
|
+
|
|
13
|
+
# Keep language selection explicit. The parser implementation is
|
|
14
|
+
# conservative and shared for now, but every supported language has a
|
|
15
|
+
# deliberate registration instead of silently falling through to text
|
|
16
|
+
# regexes. This is the extension point for native AST adapters.
|
|
17
|
+
self.language_adapters = {
|
|
18
|
+
"python": self.python,
|
|
19
|
+
"typescript": self.structured,
|
|
20
|
+
"javascript": self.structured,
|
|
21
|
+
"ruby": self.structured,
|
|
22
|
+
"go": self.structured,
|
|
23
|
+
"rust": self.structured,
|
|
24
|
+
"php": self.structured,
|
|
25
|
+
"java": self.structured,
|
|
26
|
+
"kotlin": self.structured,
|
|
27
|
+
"c": self.structured,
|
|
28
|
+
"cpp": self.structured,
|
|
29
|
+
"csharp": self.structured,
|
|
30
|
+
"swift": self.structured,
|
|
31
|
+
"dart": self.structured,
|
|
32
|
+
"scala": self.structured,
|
|
33
|
+
"lua": self.structured,
|
|
34
|
+
"shell": self.structured,
|
|
35
|
+
"vue": self.structured,
|
|
36
|
+
"svelte": self.structured,
|
|
37
|
+
# Data, configuration, and template types are explicitly registered too. They
|
|
38
|
+
# currently use the safe text adapter until a native parser is
|
|
39
|
+
# available; this keeps their parser provenance explicit and the
|
|
40
|
+
# unified output contract unchanged.
|
|
41
|
+
"sql": self.generic,
|
|
42
|
+
"html": self.generic,
|
|
43
|
+
"css": self.generic,
|
|
44
|
+
"json": self.generic,
|
|
45
|
+
"yaml": self.generic,
|
|
46
|
+
"toml": self.generic,
|
|
47
|
+
"xml": self.generic,
|
|
48
|
+
"graphql": self.generic,
|
|
49
|
+
"protobuf": self.generic,
|
|
50
|
+
"hcl": self.generic,
|
|
51
|
+
"terraform": self.generic,
|
|
52
|
+
"dockerfile": self.generic,
|
|
53
|
+
"markdown": self.generic,
|
|
54
|
+
}
|
|
9
55
|
|
|
10
56
|
def adapter_for_file(self, codebase, file_record):
|
|
11
57
|
language = file_record.get("language")
|
|
12
|
-
|
|
13
|
-
return self.adapters[0]
|
|
14
|
-
if language in {"typescript", "javascript", "ruby"}:
|
|
15
|
-
return self.adapters[1]
|
|
16
|
-
return self.adapters[-1]
|
|
58
|
+
return self.language_adapters.get(language, self.generic)
|
|
17
59
|
|
|
18
60
|
def adapters_for(self, codebase, files, manifests):
|
|
19
61
|
scored = []
|
package/plugins/devflow-intel/skills/workflow-code-index/scripts/adapters/structured_scope.py
CHANGED
|
@@ -6,7 +6,14 @@ from adapters.base import Adapter, ParseResult
|
|
|
6
6
|
from adapters.generic_text import GenericTextAdapter, _extract_calls, _extract_imports, _relation, _symbol
|
|
7
7
|
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
# These languages intentionally share the conservative lexical scope parser.
|
|
10
|
+
# The adapter does not claim AST precision; language-native parsers can be
|
|
11
|
+
# added later without changing the unified index contract.
|
|
12
|
+
SUPPORTED = {
|
|
13
|
+
"typescript", "javascript", "ruby", "go", "rust", "php", "java",
|
|
14
|
+
"kotlin", "c", "cpp", "csharp", "swift", "dart", "scala", "lua",
|
|
15
|
+
"shell", "vue", "svelte",
|
|
16
|
+
}
|
|
10
17
|
|
|
11
18
|
|
|
12
19
|
class StructuredScopeAdapter(Adapter):
|
|
@@ -21,7 +28,14 @@ class StructuredScopeAdapter(Adapter):
|
|
|
21
28
|
self.fallback = GenericTextAdapter()
|
|
22
29
|
|
|
23
30
|
def detect(self, codebase, files, manifests) -> int:
|
|
24
|
-
|
|
31
|
+
supported_suffixes = {
|
|
32
|
+
".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".rb", ".go",
|
|
33
|
+
".rs", ".php", ".java", ".kt", ".kts", ".c", ".h", ".cc",
|
|
34
|
+
".cpp", ".cxx", ".hh", ".hpp", ".hxx", ".cs", ".swift",
|
|
35
|
+
".dart", ".scala", ".sc", ".lua", ".sh", ".bash", ".zsh",
|
|
36
|
+
".vue", ".svelte",
|
|
37
|
+
}
|
|
38
|
+
return 80 if any(path.suffix.lower() in supported_suffixes for path in files) else 0
|
|
25
39
|
|
|
26
40
|
def version(self) -> str:
|
|
27
41
|
return "1"
|
|
@@ -30,13 +44,20 @@ class StructuredScopeAdapter(Adapter):
|
|
|
30
44
|
language = file_record["language"]
|
|
31
45
|
if language not in SUPPORTED:
|
|
32
46
|
return self.fallback.parse_file(codebase, file_record, text)
|
|
33
|
-
|
|
47
|
+
if language == "ruby":
|
|
48
|
+
return self._ruby(codebase, file_record, text)
|
|
49
|
+
if language == "shell":
|
|
50
|
+
return self._shell(codebase, file_record, text)
|
|
51
|
+
return self._brace_language(codebase, file_record, text)
|
|
34
52
|
|
|
35
53
|
def _brace_language(self, codebase, file_record, text):
|
|
36
54
|
lines = text.splitlines()
|
|
37
55
|
file_symbol = self._file(codebase, file_record, len(lines))
|
|
38
56
|
declarations: List[Tuple[int, str, str, str]] = []
|
|
39
|
-
pattern = re.compile(
|
|
57
|
+
pattern = re.compile(
|
|
58
|
+
r"\b(class|interface|struct|trait|enum|object|impl|namespace|module|function)\s+([A-Za-z_$][\w$]*)\s*([^\n{]*)"
|
|
59
|
+
r"|\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*=>"
|
|
60
|
+
)
|
|
40
61
|
for number, line in enumerate(lines, 1):
|
|
41
62
|
clean = _without_strings_and_comments(line)
|
|
42
63
|
match = pattern.search(clean)
|
|
@@ -47,11 +68,28 @@ class StructuredScopeAdapter(Adapter):
|
|
|
47
68
|
continue
|
|
48
69
|
keyword, named, tail, arrow = match.groups()
|
|
49
70
|
name = named or arrow
|
|
50
|
-
kind =
|
|
71
|
+
kind = {
|
|
72
|
+
"class": "class", "interface": "interface", "struct": "struct",
|
|
73
|
+
"trait": "trait", "enum": "class", "object": "module",
|
|
74
|
+
"namespace": "module", "module": "module", "impl": "class",
|
|
75
|
+
"function": "function",
|
|
76
|
+
}.get(keyword, "function")
|
|
51
77
|
declarations.append((number, kind, name, line.strip()))
|
|
52
78
|
symbols = self._scoped_symbols(codebase, file_record, lines, declarations, file_symbol, brace=True)
|
|
53
79
|
return self._relations(codebase, file_record, lines, file_symbol, symbols)
|
|
54
80
|
|
|
81
|
+
def _shell(self, codebase, file_record, text):
|
|
82
|
+
"""Index shell functions and command calls at conservative scope."""
|
|
83
|
+
lines = text.splitlines()
|
|
84
|
+
file_symbol = self._file(codebase, file_record, len(lines))
|
|
85
|
+
declarations = []
|
|
86
|
+
for number, line in enumerate(lines, 1):
|
|
87
|
+
match = re.match(r"^\s*(?:function\s+)?([A-Za-z_][\w-]*)\s*(?:\(\s*\))?\s*\{", line)
|
|
88
|
+
if match:
|
|
89
|
+
declarations.append((number, "function", match.group(1), line.strip()))
|
|
90
|
+
symbols = self._scoped_symbols(codebase, file_record, lines, declarations, file_symbol, brace=True)
|
|
91
|
+
return self._relations(codebase, file_record, lines, file_symbol, symbols)
|
|
92
|
+
|
|
55
93
|
def _ruby(self, codebase, file_record, text):
|
|
56
94
|
lines = text.splitlines(); file_symbol = self._file(codebase, file_record, len(lines)); declarations = []
|
|
57
95
|
for number, line in enumerate(lines, 1):
|
|
@@ -22,6 +22,7 @@ if str(SCRIPT_DIR) not in sys.path:
|
|
|
22
22
|
|
|
23
23
|
from adapters.generic_text import language_for_path
|
|
24
24
|
from adapters.registry import AdapterRegistry
|
|
25
|
+
from semantic import SEMANTIC_EXTRACTOR_VERSION, build_semantic_facts
|
|
25
26
|
from core.discovery import discover_codebases
|
|
26
27
|
from core.hashing import file_sha256
|
|
27
28
|
from core.scanner import scan_files
|
|
@@ -58,8 +59,8 @@ def _index_lock(output: Path, exclusive: bool):
|
|
|
58
59
|
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
|
59
60
|
|
|
60
61
|
|
|
61
|
-
def _publish_snapshot(staging: Path, output: Path, counts: Dict[str, int], workspace_state: Dict[str, object]) -> None:
|
|
62
|
-
generation = str(uuid.uuid4())
|
|
62
|
+
def _publish_snapshot(staging: Path, output: Path, counts: Dict[str, int], workspace_state: Dict[str, object], generation: str | None = None) -> None:
|
|
63
|
+
generation = generation or str(uuid.uuid4())
|
|
63
64
|
immutable_relative = f"snapshots/{generation}"
|
|
64
65
|
snapshot = {
|
|
65
66
|
"schemaVersion": 1,
|
|
@@ -101,13 +102,15 @@ def _workspace_state(root: Path, files: List[dict], changed: Set[str], discovere
|
|
|
101
102
|
indexed_paths = sorted(record["path"] for record in files)
|
|
102
103
|
discovered = sorted(set(discovered_paths or indexed_paths))
|
|
103
104
|
source_manifest_hash = _hash_lines(f"{record['path']}\0{record['content_hash']}" for record in files)
|
|
105
|
+
extractors = sorted(set(record.get("parser", "unknown") for record in files))
|
|
104
106
|
return {
|
|
105
107
|
"repositoryRevision": _git_output(root, ["rev-parse", "HEAD"]),
|
|
106
108
|
"contentManifestHash": source_manifest_hash,
|
|
107
109
|
"indexedPathsHash": _hash_lines(indexed_paths),
|
|
108
110
|
"discoveredPathsHash": _hash_lines(discovered),
|
|
109
111
|
"dirtyPathsAtBuild": sorted(changed),
|
|
110
|
-
"extractors":
|
|
112
|
+
"extractors": extractors,
|
|
113
|
+
"extractorSetHash": _hash_lines(extractors),
|
|
111
114
|
"capabilities": {
|
|
112
115
|
"python-ast@1": {"scope": "exact syntax tree", "targets": "lexical names; dynamic dispatch unresolved"},
|
|
113
116
|
"structured-scope@1": {"scope": "inferred lexical braces/end blocks", "targets": "name-only heuristic"},
|
|
@@ -185,6 +188,14 @@ def main(argv=None) -> int:
|
|
|
185
188
|
relations.add_argument("--snapshot")
|
|
186
189
|
relations.add_argument("--include-heuristic", action="store_true")
|
|
187
190
|
|
|
191
|
+
impact = sub.add_parser("impact", help="Traverse cross-layer semantic facts in a fixed snapshot.")
|
|
192
|
+
impact.add_argument("--index", default=".planning/code-index")
|
|
193
|
+
impact.add_argument("--entity", required=True, help="Logical entity id, feature id, API id, or symbol id.")
|
|
194
|
+
impact.add_argument("--depth", type=int, default=5)
|
|
195
|
+
impact.add_argument("--limit", type=int, default=200)
|
|
196
|
+
impact.add_argument("--snapshot")
|
|
197
|
+
impact.add_argument("--change-type", choices=["remove_field", "rename_field", "change_type", "optional_to_required", "required_to_optional", "change_nullability", "change_enum"], help="Optional proposed change classification.")
|
|
198
|
+
|
|
188
199
|
args = parser.parse_args(argv)
|
|
189
200
|
if args.command == "scan":
|
|
190
201
|
return command_scan(args)
|
|
@@ -196,6 +207,8 @@ def main(argv=None) -> int:
|
|
|
196
207
|
return command_query(args)
|
|
197
208
|
if args.command == "relations":
|
|
198
209
|
return command_relations(args)
|
|
210
|
+
if args.command == "impact":
|
|
211
|
+
return command_impact(args)
|
|
199
212
|
return 2
|
|
200
213
|
|
|
201
214
|
|
|
@@ -213,6 +226,8 @@ def command_scan(args) -> int:
|
|
|
213
226
|
files, symbols, relations, diagnostics = _existing_after_removal(existing, changed)
|
|
214
227
|
indexed_paths = {record["path"] for record in files}
|
|
215
228
|
discovered_paths: Set[str] = set()
|
|
229
|
+
adapter_entities: List[dict] = []
|
|
230
|
+
adapter_flows: List[dict] = []
|
|
216
231
|
|
|
217
232
|
for codebase in codebases:
|
|
218
233
|
if not codebase.root.exists():
|
|
@@ -249,6 +264,8 @@ def command_scan(args) -> int:
|
|
|
249
264
|
symbols.extend(result.symbols)
|
|
250
265
|
relations.extend(result.relations)
|
|
251
266
|
diagnostics.extend(result.diagnostics)
|
|
267
|
+
adapter_entities.extend(getattr(result, "entities", []) or [])
|
|
268
|
+
adapter_flows.extend(getattr(result, "flows", []) or [])
|
|
252
269
|
except Exception as exc: # keep indexing useful even when one file fails
|
|
253
270
|
diagnostics.append(_diagnostic(codebase.name, rel_path, 1, "error", "parse_failed", str(exc), adapter.parser_id))
|
|
254
271
|
|
|
@@ -263,11 +280,26 @@ def command_scan(args) -> int:
|
|
|
263
280
|
|
|
264
281
|
staging = Path(tempfile.mkdtemp(prefix=f".{output.name}.build-", dir=str(output.parent)))
|
|
265
282
|
try:
|
|
283
|
+
generation = str(uuid.uuid4())
|
|
284
|
+
workspace_state = _workspace_state(root, files, changed, discovered_paths)
|
|
285
|
+
workspace_state["generatedAt"] = datetime.now(timezone.utc).isoformat()
|
|
286
|
+
workspace_state["semanticExtractor"] = SEMANTIC_EXTRACTOR_VERSION
|
|
287
|
+
workspace_state["extractorSetHash"] = _hash_lines([*workspace_state.get("extractors", []), SEMANTIC_EXTRACTOR_VERSION])
|
|
288
|
+
semantic_entities, semantic_flows, feature_bindings = build_semantic_facts(root, files, symbols, relations)
|
|
289
|
+
semantic_entities.extend(adapter_entities)
|
|
290
|
+
semantic_flows.extend(adapter_flows)
|
|
266
291
|
write_jsonl(staging / INDEX_FILES["files"], files)
|
|
267
292
|
write_jsonl(staging / INDEX_FILES["symbols"], symbols)
|
|
268
293
|
write_jsonl(staging / INDEX_FILES["relations"], relations)
|
|
269
294
|
write_jsonl(staging / INDEX_FILES["diagnostics"], diagnostics)
|
|
270
|
-
write_sqlite(
|
|
295
|
+
write_sqlite(
|
|
296
|
+
staging / "index.sqlite", files, symbols, relations, diagnostics,
|
|
297
|
+
snapshot_id=generation,
|
|
298
|
+
workspace_state=workspace_state,
|
|
299
|
+
semantic_entities=semantic_entities,
|
|
300
|
+
semantic_flows=semantic_flows,
|
|
301
|
+
feature_bindings=feature_bindings,
|
|
302
|
+
)
|
|
271
303
|
write_markdown(staging, files, symbols, relations, diagnostics)
|
|
272
304
|
staged_errors = validate_index(*_load_tuple(staging))
|
|
273
305
|
if staged_errors:
|
|
@@ -284,7 +316,8 @@ def command_scan(args) -> int:
|
|
|
284
316
|
"relations": len(relations),
|
|
285
317
|
"diagnostics": len(diagnostics),
|
|
286
318
|
},
|
|
287
|
-
|
|
319
|
+
workspace_state,
|
|
320
|
+
generation=generation,
|
|
288
321
|
)
|
|
289
322
|
finally:
|
|
290
323
|
shutil.rmtree(staging, ignore_errors=True)
|
|
@@ -296,6 +329,46 @@ def command_scan(args) -> int:
|
|
|
296
329
|
|
|
297
330
|
def command_validate(args) -> int:
|
|
298
331
|
output = Path(args.index).resolve()
|
|
332
|
+
selected = _selected_generation(output, None)
|
|
333
|
+
errors = []
|
|
334
|
+
try:
|
|
335
|
+
snapshot = _load_snapshot(selected)
|
|
336
|
+
files, symbols, relations, diagnostics = _load_tuple(selected)
|
|
337
|
+
errors.extend(validate_index(files, symbols, relations, diagnostics))
|
|
338
|
+
conn = sqlite3.connect(str(selected / "index.sqlite"))
|
|
339
|
+
try:
|
|
340
|
+
snapshot_id = snapshot.get("generation")
|
|
341
|
+
row = conn.execute("SELECT snapshot_id FROM analysis_snapshots WHERE snapshot_id=?", (snapshot_id,)).fetchone()
|
|
342
|
+
if not row:
|
|
343
|
+
errors.append(f"semantic schema missing snapshot {snapshot_id}")
|
|
344
|
+
dangling = conn.execute("""
|
|
345
|
+
SELECT COUNT(*) FROM semantic_flows f
|
|
346
|
+
WHERE f.snapshot_id=? AND (
|
|
347
|
+
(NOT EXISTS (SELECT 1 FROM entity_versions e WHERE e.snapshot_id=f.snapshot_id AND e.logical_id=f.source_id)
|
|
348
|
+
AND NOT EXISTS (SELECT 1 FROM symbols s WHERE s.id=f.source_id))
|
|
349
|
+
OR
|
|
350
|
+
(NOT EXISTS (SELECT 1 FROM entity_versions e WHERE e.snapshot_id=f.snapshot_id AND e.logical_id=f.target_id)
|
|
351
|
+
AND NOT EXISTS (SELECT 1 FROM symbols s WHERE s.id=f.target_id))
|
|
352
|
+
)
|
|
353
|
+
""", (snapshot_id,)).fetchone()[0]
|
|
354
|
+
if dangling:
|
|
355
|
+
errors.append(f"semantic flows have {dangling} dangling source or target nodes")
|
|
356
|
+
binding_dangling = conn.execute("""
|
|
357
|
+
SELECT COUNT(*) FROM feature_bindings b
|
|
358
|
+
WHERE b.snapshot_id=? AND (
|
|
359
|
+
NOT EXISTS (SELECT 1 FROM entity_versions e WHERE e.snapshot_id=b.snapshot_id AND e.logical_id=b.feature_id)
|
|
360
|
+
OR NOT EXISTS (SELECT 1 FROM entity_versions e WHERE e.snapshot_id=b.snapshot_id AND e.logical_id=b.entity_id)
|
|
361
|
+
)
|
|
362
|
+
""", (snapshot_id,)).fetchone()[0]
|
|
363
|
+
if binding_dangling:
|
|
364
|
+
errors.append(f"feature bindings have {binding_dangling} dangling feature or entity nodes")
|
|
365
|
+
finally:
|
|
366
|
+
conn.close()
|
|
367
|
+
except (OSError, sqlite3.Error, ValueError) as exc:
|
|
368
|
+
errors.append(str(exc))
|
|
369
|
+
for error in errors:
|
|
370
|
+
print(error, file=sys.stderr)
|
|
371
|
+
return 1 if errors else 0
|
|
299
372
|
def command_query(args) -> int:
|
|
300
373
|
output = Path(args.index).resolve()
|
|
301
374
|
with _index_lock(output, exclusive=False):
|
|
@@ -366,12 +439,61 @@ def command_relations(args) -> int:
|
|
|
366
439
|
) SELECT * FROM graph ORDER BY depth,from_id,relation_type,to_id LIMIT ?
|
|
367
440
|
""", (args.object, depth, limit + 1)).fetchall()
|
|
368
441
|
truncated = len(rows) > limit
|
|
369
|
-
for row in rows[:limit]:
|
|
442
|
+
for row in rows[:limit]:
|
|
443
|
+
item = dict(row)
|
|
444
|
+
if args.change_type:
|
|
445
|
+
item["impact_kind"] = args.change_type
|
|
446
|
+
item["severity"] = _impact_severity(args.change_type, item["flow_type"])
|
|
447
|
+
print(json.dumps(item, ensure_ascii=False, sort_keys=True))
|
|
370
448
|
print(json.dumps({"_meta": {"snapshot": snapshot.get("generation"), "depthLimit": depth, "resultLimit": limit, "truncated": truncated}}, ensure_ascii=False, sort_keys=True))
|
|
371
449
|
finally: conn.close()
|
|
372
450
|
return 0
|
|
373
451
|
|
|
374
452
|
|
|
453
|
+
def _impact_severity(change_type: str, flow_type: str) -> str:
|
|
454
|
+
"""Conservative compatibility rules for an observed impact edge."""
|
|
455
|
+
if change_type in {"remove_field", "change_type", "change_nullability", "change_enum"}:
|
|
456
|
+
return "high" if flow_type in {"returns_field", "accepts_field", "state_reads", "renders_when", "depends_on"} else "medium"
|
|
457
|
+
if change_type == "optional_to_required":
|
|
458
|
+
return "high" if flow_type in {"api_declares_or_calls", "accepts_field"} else "medium"
|
|
459
|
+
if change_type == "rename_field":
|
|
460
|
+
return "high" if flow_type in {"returns_field", "accepts_field", "maps_to", "state_reads"} else "medium"
|
|
461
|
+
return "low"
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def command_impact(args) -> int:
|
|
465
|
+
output = Path(args.index).resolve()
|
|
466
|
+
depth = max(0, min(args.depth, 20)); limit = max(1, min(args.limit, 2000))
|
|
467
|
+
with _index_lock(output, exclusive=False):
|
|
468
|
+
selected = _selected_generation(output, getattr(args, "snapshot", None))
|
|
469
|
+
snapshot = _load_snapshot(selected)
|
|
470
|
+
if not getattr(args, "snapshot", None):
|
|
471
|
+
files = read_jsonl(selected / INDEX_FILES["files"])
|
|
472
|
+
if not _query_is_fresh(output.parent.parent, snapshot, files):
|
|
473
|
+
print("Stale code index: refresh required before impact query.", file=sys.stderr)
|
|
474
|
+
return 1
|
|
475
|
+
conn = sqlite3.connect(str(selected / "index.sqlite")); conn.row_factory = sqlite3.Row
|
|
476
|
+
try:
|
|
477
|
+
rows = conn.execute("""
|
|
478
|
+
WITH RECURSIVE graph(node_id, flow_type, source_path, line_start, confidence, depth, visited) AS (
|
|
479
|
+
SELECT CASE WHEN source_id=? THEN target_id ELSE source_id END, flow_type, source_path, line_start, confidence, 1, ',' || ? || ','
|
|
480
|
+
FROM semantic_flows WHERE snapshot_id=? AND (source_id=? OR target_id=?)
|
|
481
|
+
UNION ALL
|
|
482
|
+
SELECT CASE WHEN f.source_id=g.node_id THEN f.target_id ELSE f.source_id END,
|
|
483
|
+
f.flow_type, f.source_path, f.line_start, f.confidence, g.depth+1, g.visited || g.node_id || ','
|
|
484
|
+
FROM graph g JOIN semantic_flows f ON f.snapshot_id=? AND (f.source_id=g.node_id OR f.target_id=g.node_id)
|
|
485
|
+
WHERE g.depth < ? AND instr(g.visited, ',' || CASE WHEN f.source_id=g.node_id THEN f.target_id ELSE f.source_id END || ',')=0
|
|
486
|
+
) SELECT * FROM graph LIMIT ?
|
|
487
|
+
""", (args.entity, args.entity, snapshot.get("generation"), args.entity, args.entity, snapshot.get("generation"), depth, limit + 1)).fetchall()
|
|
488
|
+
for row in rows[:limit]: print(json.dumps(dict(row), ensure_ascii=False, sort_keys=True))
|
|
489
|
+
coverage_row = conn.execute("SELECT * FROM coverage_metrics WHERE snapshot_id=?", (snapshot.get("generation"),)).fetchone()
|
|
490
|
+
coverage = dict(coverage_row) if coverage_row else {"status": "unknown"}
|
|
491
|
+
print(json.dumps({"_meta": {"snapshot": snapshot.get("generation"), "coverage": coverage, "depthLimit": depth, "truncated": len(rows) > limit}}, ensure_ascii=False, sort_keys=True))
|
|
492
|
+
finally:
|
|
493
|
+
conn.close()
|
|
494
|
+
return 0
|
|
495
|
+
|
|
496
|
+
|
|
375
497
|
def _selected_generation(output: Path, generation: str | None) -> Path:
|
|
376
498
|
if not generation:
|
|
377
499
|
return output
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""Conservative cross-layer fact extraction.
|
|
2
|
+
|
|
3
|
+
This module intentionally emits candidates with evidence and confidence. It
|
|
4
|
+
does not replace framework parsers; it gives the relational index a stable
|
|
5
|
+
schema so API, database, UI and feature facts can be linked incrementally.
|
|
6
|
+
"""
|
|
7
|
+
import re
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Dict, Iterable, List, Tuple
|
|
10
|
+
|
|
11
|
+
from core.schema import short_hash, stable_key
|
|
12
|
+
from core.hashing import file_sha256
|
|
13
|
+
|
|
14
|
+
SEMANTIC_EXTRACTOR_VERSION = "semantic-linker@1"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def build_semantic_facts(root: Path, files: Iterable[dict], symbols: Iterable[dict],
|
|
18
|
+
relations: Iterable[dict]) -> Tuple[List[dict], List[dict], List[dict]]:
|
|
19
|
+
entities: List[dict] = []
|
|
20
|
+
flows: List[dict] = []
|
|
21
|
+
bindings: List[dict] = []
|
|
22
|
+
symbol_by_path = {}
|
|
23
|
+
for symbol in symbols:
|
|
24
|
+
symbol_by_path.setdefault(symbol.get("path"), []).append(symbol)
|
|
25
|
+
|
|
26
|
+
for file_record in files:
|
|
27
|
+
path = file_record["path"]
|
|
28
|
+
source = _read(root, path)
|
|
29
|
+
if source is None:
|
|
30
|
+
continue
|
|
31
|
+
try:
|
|
32
|
+
if file_sha256(root / path) != file_record.get("content_hash"):
|
|
33
|
+
# Never mix a newly-read working-tree file with an older
|
|
34
|
+
# structural record in the same immutable snapshot.
|
|
35
|
+
continue
|
|
36
|
+
except OSError:
|
|
37
|
+
continue
|
|
38
|
+
language = file_record.get("language", "")
|
|
39
|
+
feature = _feature_from_path(path)
|
|
40
|
+
if feature:
|
|
41
|
+
feature_id = f"feature:{stable_key(feature)}"
|
|
42
|
+
entities.append(_entity(feature_id, "feature", feature, file_record, 1, 1,
|
|
43
|
+
"heuristic", "path_feature_inference"))
|
|
44
|
+
for symbol in symbol_by_path.get(path, []):
|
|
45
|
+
if symbol.get("kind") in {"route", "component", "class", "function", "method"}:
|
|
46
|
+
bindings.append(_binding(feature_id, symbol["id"], "static_inferred", "heuristic", path, symbol.get("line_start", 1)))
|
|
47
|
+
|
|
48
|
+
for line_no, line in enumerate(source.splitlines(), 1):
|
|
49
|
+
_extract_api(line, path, file_record, line_no, entities, flows)
|
|
50
|
+
_extract_db(line, path, file_record, line_no, entities, flows)
|
|
51
|
+
_extract_ui(line, path, file_record, line_no, entities, flows)
|
|
52
|
+
|
|
53
|
+
# Existing structural relations remain the source of method call edges;
|
|
54
|
+
# expose them in the semantic graph without changing their meaning.
|
|
55
|
+
for relation in relations:
|
|
56
|
+
if relation.get("to_id") and relation.get("relation_type") in {"calls", "contains", "routes_to"}:
|
|
57
|
+
flows.append({
|
|
58
|
+
"source_id": relation.get("from_id"),
|
|
59
|
+
"target_id": relation.get("to_id"),
|
|
60
|
+
"flow_type": relation.get("relation_type"),
|
|
61
|
+
"mapping_kind": "structural_relation",
|
|
62
|
+
"source_path": relation.get("evidence_path"),
|
|
63
|
+
"line_start": relation.get("evidence_line"),
|
|
64
|
+
"confidence": relation.get("confidence", "heuristic"),
|
|
65
|
+
"evidence_level": "syntax",
|
|
66
|
+
"metadata": relation.get("metadata", {}),
|
|
67
|
+
})
|
|
68
|
+
return _dedupe_entities(entities), _dedupe_flows(flows), _dedupe_bindings(bindings)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _extract_api(line, path, record, line_no, entities, flows):
|
|
72
|
+
# Framework-neutral route declarations and client URLs.
|
|
73
|
+
match = re.search(r"(?:@(?:app|router|api)\.(get|post|put|patch|delete)\s*\(|(?:GET|POST|PUT|PATCH|DELETE)\s+)(['\"])(/[^'\"]+)\2", line, re.I)
|
|
74
|
+
if not match:
|
|
75
|
+
match = re.search(r"(?:fetch|axios\.(?:get|post|put|patch|delete)|client\.(?:get|post|put|patch|delete))\s*\(\s*(['\"])(/[^'\"]+)\1", line, re.I)
|
|
76
|
+
if not match:
|
|
77
|
+
return
|
|
78
|
+
route = match.group(3) if match.lastindex and match.lastindex >= 3 else match.group(2)
|
|
79
|
+
method_match = re.search(r"@(?:app|router|api)\.(get|post|put|patch|delete)\b|\b(GET|POST|PUT|PATCH|DELETE)\b|\b(?:fetch|axios\.|client\.)(get|post|put|patch|delete)\b", line, re.I)
|
|
80
|
+
method = next((value.upper() for value in (method_match.groups() if method_match else ()) if value), "UNKNOWN")
|
|
81
|
+
if method == "UNKNOWN" and re.search(r"\bfetch\s*\(", line, re.I):
|
|
82
|
+
method = "GET"
|
|
83
|
+
endpoint_id = f"api:{method}:{stable_key(route)}"
|
|
84
|
+
entity_type = "api_endpoint"
|
|
85
|
+
entities.append(_entity(endpoint_id, entity_type, f"{method} {route}", record, line_no, line_no, "heuristic", "route_or_client_regex", {"http_method": method, "path": route}))
|
|
86
|
+
flows.append(_flow(f"file:{record['codebase']}:{path}", endpoint_id, "api_declares_or_calls", path, line_no, "heuristic", "framework_neutral_route"))
|
|
87
|
+
for field in re.findall(r"(?:response|payload|data|body)\s*[.\[]\s*['\"]?([A-Za-z_][A-Za-z0-9_]*)", line):
|
|
88
|
+
field_id = f"api-field:{endpoint_id}:response:{field}"
|
|
89
|
+
entities.append(_entity(field_id, "api_field", field, record, line_no, line_no, "heuristic", "field_access_regex", {"endpoint_id": endpoint_id, "direction": "response", "field_path": field}))
|
|
90
|
+
flows.append(_flow(endpoint_id, field_id, "returns_field", path, line_no, "heuristic", "field_access_regex"))
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _extract_db(line, path, record, line_no, entities, flows):
|
|
94
|
+
# SQL and common ORM spellings. Dynamic identifiers are deliberately not
|
|
95
|
+
# resolved to a concrete column.
|
|
96
|
+
table_match = re.search(r"\b(?:FROM|JOIN|INTO|UPDATE|TABLE)\s+[`\"]?([A-Za-z_][\w$]*)", line, re.I)
|
|
97
|
+
orm_match = re.search(r"\b([A-Z][A-Za-z0-9_]*)\.(?:query|objects|select|filter|where)\b", line)
|
|
98
|
+
if not table_match and not orm_match:
|
|
99
|
+
return
|
|
100
|
+
table = table_match.group(1) if table_match else orm_match.group(1)
|
|
101
|
+
table_id = f"db-table:{stable_key(table)}"
|
|
102
|
+
entities.append(_entity(table_id, "db_table", table, record, line_no, line_no, "heuristic", "sql_or_orm_regex"))
|
|
103
|
+
operation = "writes" if re.search(r"\b(?:INSERT|UPDATE|DELETE)\b|\.update\s*\(", line, re.I) else "reads"
|
|
104
|
+
flows.append(_flow(f"file:{record['codebase']}:{path}", table_id, operation, path, line_no, "heuristic", "sql_or_orm_regex"))
|
|
105
|
+
for column in re.findall(r"\b([A-Za-z_][A-Za-z0-9_]*)\s*(?:=|,|FROM|$)", line):
|
|
106
|
+
if column.lower() in {"select", "from", "where", "set", "values", "into", "update", "join", "and", "or"}:
|
|
107
|
+
continue
|
|
108
|
+
column_id = f"db-column:{stable_key(table)}:{stable_key(column)}"
|
|
109
|
+
entities.append(_entity(column_id, "db_column", column, record, line_no, line_no, "heuristic", "sql_column_regex", {"table_id": table_id, "column": column}))
|
|
110
|
+
flows.append(_flow(table_id, column_id, f"{operation}_column", path, line_no, "heuristic", "sql_column_regex"))
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _extract_ui(line, path, record, line_no, entities, flows):
|
|
114
|
+
component = re.search(r"\b(?:function|class)\s+([A-Z][A-Za-z0-9_]*)|<([A-Z][A-Za-z0-9_.]*)\b", line)
|
|
115
|
+
state = re.search(r"\b(?:useState|ref|reactive|computed|createSignal)\s*\(.*?\)|\b(?:const|let|var)\s+([A-Za-z_][\w]*(?:State|Store|Status|Loading|Error))\s*=", line)
|
|
116
|
+
condition = re.search(r"\b(?:v-if|v-show)\s*=\s*['\"]([^'\"]+)['\"]|(?:\{\s*)?([^{}\n]+?)\s*&&\s*<", line)
|
|
117
|
+
if component:
|
|
118
|
+
name = component.group(1) or component.group(2)
|
|
119
|
+
component_id = f"ui-component:{stable_key(path)}:{stable_key(name)}"
|
|
120
|
+
entities.append(_entity(component_id, "ui_component", name, record, line_no, line_no, "inferred", "component_pattern"))
|
|
121
|
+
else:
|
|
122
|
+
component_id = f"ui-file:{stable_key(path)}"
|
|
123
|
+
entities.append(_entity(component_id, "ui_component", Path(path).stem, record, line_no, line_no, "heuristic", "ui_file_scope"))
|
|
124
|
+
if state:
|
|
125
|
+
name = state.group(1) or "anonymous_state"
|
|
126
|
+
state_id = f"ui-state:{stable_key(path)}:{stable_key(name)}"
|
|
127
|
+
entities.append(_entity(state_id, "ui_state", name, record, line_no, line_no, "heuristic", "state_pattern"))
|
|
128
|
+
flows.append(_flow(component_id, state_id, "state_reads", path, line_no, "heuristic", "state_pattern"))
|
|
129
|
+
if condition:
|
|
130
|
+
expression = condition.group(1) or condition.group(2)
|
|
131
|
+
cond_id = f"render-condition:{stable_key(path)}:{short_hash(expression)}"
|
|
132
|
+
entities.append(_entity(cond_id, "render_condition", expression, record, line_no, line_no, "heuristic", "template_condition"))
|
|
133
|
+
flows.append(_flow(component_id, cond_id, "renders_when", path, line_no, "heuristic", "template_condition"))
|
|
134
|
+
known_ids = {item["id"] for item in entities if item.get("entity_type") == "ui_state"}
|
|
135
|
+
for name in re.findall(r"\b[A-Za-z_][A-Za-z0-9_]*\b", expression):
|
|
136
|
+
state_id = f"ui-state:{stable_key(path)}:{stable_key(name)}"
|
|
137
|
+
if state_id in known_ids:
|
|
138
|
+
flows.append(_flow(cond_id, state_id, "depends_on", path, line_no, "heuristic", "condition_identifier"))
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _entity(entity_id, entity_type, name, record, start, end, confidence, extractor, metadata=None):
|
|
142
|
+
return {"id": entity_id, "entity_type": entity_type, "name": name, "codebase": record["codebase"], "path": record["path"], "line_start": start, "line_end": end, "content_hash": record["content_hash"], "confidence": confidence, "extractor": extractor, "metadata": metadata or {}}
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _flow(source_id, target_id, flow_type, path, line, confidence, extractor, metadata=None):
|
|
146
|
+
return {"source_id": source_id, "target_id": target_id, "flow_type": flow_type, "source_path": path, "line_start": line, "confidence": confidence, "evidence_level": "static_source", "extractor": extractor, "metadata": metadata or {}}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _binding(feature_id, entity_id, source, confidence, path, line):
|
|
150
|
+
return {"feature_id": feature_id, "entity_id": entity_id, "binding_type": source, "confidence": confidence, "source_path": path, "line_start": line}
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _read(root, path):
|
|
154
|
+
try:
|
|
155
|
+
return (root / path).read_text(encoding="utf-8", errors="ignore")
|
|
156
|
+
except OSError:
|
|
157
|
+
return None
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _feature_from_path(path):
|
|
161
|
+
parts = [part for part in Path(path).parts if part not in {"src", "app", "lib", "components", "services", "pages", "api", "tests"}]
|
|
162
|
+
if not parts:
|
|
163
|
+
return None
|
|
164
|
+
token = re.sub(r"[^A-Za-z0-9_-]+", "-", parts[0]).strip("-")
|
|
165
|
+
return token if token and token.lower() not in {"index", "main", "common", "shared"} else None
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _dedupe_entities(items):
|
|
169
|
+
result = {}
|
|
170
|
+
for item in items:
|
|
171
|
+
result[item["id"]] = item
|
|
172
|
+
return list(result.values())
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _dedupe_flows(items):
|
|
176
|
+
result = {}
|
|
177
|
+
for item in items:
|
|
178
|
+
key = (item.get("source_id"), item.get("target_id"), item.get("flow_type"), item.get("source_path"), item.get("line_start"))
|
|
179
|
+
result[key] = item
|
|
180
|
+
return list(result.values())
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _dedupe_bindings(items):
|
|
184
|
+
result = {}
|
|
185
|
+
for item in items:
|
|
186
|
+
result[(item["feature_id"], item["entity_id"])] = item
|
|
187
|
+
return list(result.values())
|
|
@@ -58,6 +58,119 @@ CREATE TABLE IF NOT EXISTS diagnostics (
|
|
|
58
58
|
parser TEXT
|
|
59
59
|
);
|
|
60
60
|
|
|
61
|
+
CREATE TABLE IF NOT EXISTS analysis_snapshots (
|
|
62
|
+
snapshot_id TEXT PRIMARY KEY,
|
|
63
|
+
schema_version INTEGER NOT NULL,
|
|
64
|
+
source_revision TEXT,
|
|
65
|
+
content_manifest_hash TEXT,
|
|
66
|
+
extractor_set_hash TEXT,
|
|
67
|
+
created_at TEXT NOT NULL,
|
|
68
|
+
status TEXT NOT NULL
|
|
69
|
+
);
|
|
70
|
+
|
|
71
|
+
CREATE TABLE IF NOT EXISTS logical_entities (
|
|
72
|
+
logical_id TEXT PRIMARY KEY,
|
|
73
|
+
entity_type TEXT NOT NULL,
|
|
74
|
+
stable_key TEXT NOT NULL,
|
|
75
|
+
metadata_json TEXT
|
|
76
|
+
);
|
|
77
|
+
|
|
78
|
+
CREATE TABLE IF NOT EXISTS entity_versions (
|
|
79
|
+
entity_version_id TEXT PRIMARY KEY,
|
|
80
|
+
snapshot_id TEXT NOT NULL,
|
|
81
|
+
logical_id TEXT NOT NULL,
|
|
82
|
+
entity_type TEXT NOT NULL,
|
|
83
|
+
name TEXT NOT NULL,
|
|
84
|
+
codebase TEXT,
|
|
85
|
+
path TEXT,
|
|
86
|
+
line_start INTEGER,
|
|
87
|
+
line_end INTEGER,
|
|
88
|
+
content_hash TEXT,
|
|
89
|
+
confidence TEXT NOT NULL,
|
|
90
|
+
extractor TEXT NOT NULL,
|
|
91
|
+
metadata_json TEXT
|
|
92
|
+
);
|
|
93
|
+
|
|
94
|
+
CREATE TABLE IF NOT EXISTS semantic_flows (
|
|
95
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
96
|
+
snapshot_id TEXT NOT NULL,
|
|
97
|
+
source_id TEXT NOT NULL,
|
|
98
|
+
target_id TEXT NOT NULL,
|
|
99
|
+
flow_type TEXT NOT NULL,
|
|
100
|
+
source_path TEXT,
|
|
101
|
+
line_start INTEGER,
|
|
102
|
+
confidence TEXT NOT NULL,
|
|
103
|
+
evidence_level TEXT NOT NULL,
|
|
104
|
+
extractor TEXT NOT NULL,
|
|
105
|
+
metadata_json TEXT
|
|
106
|
+
);
|
|
107
|
+
|
|
108
|
+
CREATE TABLE IF NOT EXISTS feature_bindings (
|
|
109
|
+
snapshot_id TEXT NOT NULL,
|
|
110
|
+
feature_id TEXT NOT NULL,
|
|
111
|
+
entity_id TEXT NOT NULL,
|
|
112
|
+
binding_type TEXT NOT NULL,
|
|
113
|
+
confidence TEXT NOT NULL,
|
|
114
|
+
source_path TEXT,
|
|
115
|
+
line_start INTEGER,
|
|
116
|
+
PRIMARY KEY(snapshot_id, feature_id, entity_id)
|
|
117
|
+
);
|
|
118
|
+
|
|
119
|
+
CREATE TABLE IF NOT EXISTS coverage_metrics (
|
|
120
|
+
snapshot_id TEXT PRIMARY KEY,
|
|
121
|
+
files_scanned INTEGER NOT NULL,
|
|
122
|
+
files_parsed INTEGER NOT NULL,
|
|
123
|
+
diagnostics INTEGER NOT NULL,
|
|
124
|
+
semantic_entities INTEGER NOT NULL,
|
|
125
|
+
semantic_flows INTEGER NOT NULL,
|
|
126
|
+
dynamic_limitations INTEGER NOT NULL,
|
|
127
|
+
metadata_json TEXT
|
|
128
|
+
);
|
|
129
|
+
|
|
130
|
+
CREATE TABLE IF NOT EXISTS design_entities (
|
|
131
|
+
artifact_revision TEXT NOT NULL,
|
|
132
|
+
design_entity_id TEXT NOT NULL,
|
|
133
|
+
entity_type TEXT NOT NULL,
|
|
134
|
+
name TEXT NOT NULL,
|
|
135
|
+
field_path TEXT,
|
|
136
|
+
direction TEXT,
|
|
137
|
+
status TEXT NOT NULL,
|
|
138
|
+
metadata_json TEXT,
|
|
139
|
+
PRIMARY KEY(artifact_revision, design_entity_id)
|
|
140
|
+
);
|
|
141
|
+
|
|
142
|
+
CREATE TABLE IF NOT EXISTS design_observation_links (
|
|
143
|
+
artifact_revision TEXT NOT NULL,
|
|
144
|
+
design_entity_id TEXT NOT NULL,
|
|
145
|
+
observed_entity_id TEXT NOT NULL,
|
|
146
|
+
result TEXT NOT NULL, -- matches / drift / gap / unknown
|
|
147
|
+
confidence TEXT NOT NULL,
|
|
148
|
+
evidence_json TEXT
|
|
149
|
+
);
|
|
150
|
+
|
|
151
|
+
CREATE TABLE IF NOT EXISTS proposed_changes (
|
|
152
|
+
change_id TEXT PRIMARY KEY,
|
|
153
|
+
snapshot_id TEXT NOT NULL,
|
|
154
|
+
artifact_revision TEXT,
|
|
155
|
+
change_type TEXT NOT NULL,
|
|
156
|
+
target_entity_id TEXT NOT NULL,
|
|
157
|
+
before_json TEXT,
|
|
158
|
+
after_json TEXT,
|
|
159
|
+
created_at TEXT NOT NULL
|
|
160
|
+
);
|
|
161
|
+
|
|
162
|
+
CREATE TABLE IF NOT EXISTS impact_findings (
|
|
163
|
+
impact_run_id TEXT NOT NULL,
|
|
164
|
+
snapshot_id TEXT NOT NULL,
|
|
165
|
+
change_id TEXT,
|
|
166
|
+
entity_id TEXT NOT NULL,
|
|
167
|
+
impact_kind TEXT NOT NULL,
|
|
168
|
+
severity TEXT NOT NULL,
|
|
169
|
+
confidence TEXT NOT NULL,
|
|
170
|
+
evidence_json TEXT,
|
|
171
|
+
PRIMARY KEY(impact_run_id, entity_id, impact_kind)
|
|
172
|
+
);
|
|
173
|
+
|
|
61
174
|
CREATE VIRTUAL TABLE IF NOT EXISTS symbols_fts USING fts5(
|
|
62
175
|
id UNINDEXED,
|
|
63
176
|
codebase UNINDEXED,
|
|
@@ -78,13 +191,69 @@ CREATE INDEX IF NOT EXISTS idx_relations_external ON relations(to_external);
|
|
|
78
191
|
"""
|
|
79
192
|
|
|
80
193
|
|
|
81
|
-
def write_sqlite(path: Path, files, symbols, relations, diagnostics
|
|
194
|
+
def write_sqlite(path: Path, files, symbols, relations, diagnostics, snapshot_id="pending",
|
|
195
|
+
workspace_state=None, semantic_entities=None, semantic_flows=None, feature_bindings=None) -> None:
|
|
82
196
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
83
197
|
if path.exists():
|
|
84
198
|
path.unlink()
|
|
85
199
|
conn = sqlite3.connect(str(path))
|
|
86
200
|
try:
|
|
87
201
|
conn.executescript(SCHEMA)
|
|
202
|
+
workspace_state = workspace_state or {}
|
|
203
|
+
semantic_entities = list(semantic_entities or [])
|
|
204
|
+
semantic_flows = semantic_flows or []
|
|
205
|
+
feature_bindings = feature_bindings or []
|
|
206
|
+
parse_failed_paths = {item.get("path") for item in diagnostics if item.get("kind") == "parse_failed"}
|
|
207
|
+
parsed_count = sum(1 for item in files if item.get("path") not in parse_failed_paths)
|
|
208
|
+
conn.execute(
|
|
209
|
+
"INSERT INTO analysis_snapshots(snapshot_id, schema_version, source_revision, content_manifest_hash, extractor_set_hash, created_at, status) VALUES(?,?,?,?,?,?,?)",
|
|
210
|
+
(snapshot_id, 2, workspace_state.get("repositoryRevision", ""), workspace_state.get("contentManifestHash", ""),
|
|
211
|
+
workspace_state.get("extractorSetHash", ""), workspace_state.get("generatedAt", ""), "current"),
|
|
212
|
+
)
|
|
213
|
+
# Structural symbols and files are first-class graph nodes as well.
|
|
214
|
+
# This prevents semantic flows and feature bindings from dangling when
|
|
215
|
+
# a framework linker points at an existing code symbol.
|
|
216
|
+
structural_entities = [
|
|
217
|
+
{"id": item["id"], "entity_type": item.get("kind", "symbol"), "name": item.get("name", item["id"]),
|
|
218
|
+
"codebase": item.get("codebase"), "path": item.get("path"), "line_start": item.get("line_start"),
|
|
219
|
+
"line_end": item.get("line_end"), "content_hash": item.get("content_hash"), "confidence": "exact",
|
|
220
|
+
"extractor": item.get("parser", "index"), "metadata": item.get("metadata", {})}
|
|
221
|
+
for item in [*files, *symbols]
|
|
222
|
+
]
|
|
223
|
+
by_id = {item["id"]: item for item in semantic_entities}
|
|
224
|
+
for item in structural_entities:
|
|
225
|
+
by_id.setdefault(item["id"], item)
|
|
226
|
+
semantic_entities = list(by_id.values())
|
|
227
|
+
valid_ids = set(by_id)
|
|
228
|
+
invalid_flows = [item for item in semantic_flows if item.get("source_id") not in valid_ids or item.get("target_id") not in valid_ids]
|
|
229
|
+
invalid_bindings = [item for item in feature_bindings if item.get("feature_id") not in valid_ids or item.get("entity_id") not in valid_ids]
|
|
230
|
+
if invalid_flows or invalid_bindings:
|
|
231
|
+
raise ValueError(
|
|
232
|
+
f"semantic graph contains dangling references: flows={len(invalid_flows)}, bindings={len(invalid_bindings)}"
|
|
233
|
+
)
|
|
234
|
+
for entity in semantic_entities:
|
|
235
|
+
logical_id = entity["id"]
|
|
236
|
+
entity_version_id = f"{snapshot_id}:{logical_id}"
|
|
237
|
+
conn.execute(
|
|
238
|
+
"INSERT OR IGNORE INTO logical_entities(logical_id, entity_type, stable_key, metadata_json) VALUES(?,?,?,?)",
|
|
239
|
+
(logical_id, entity["entity_type"], logical_id, json_dumps(entity.get("metadata", {}))),
|
|
240
|
+
)
|
|
241
|
+
conn.execute(
|
|
242
|
+
"INSERT INTO entity_versions(entity_version_id, snapshot_id, logical_id, entity_type, name, codebase, path, line_start, line_end, content_hash, confidence, extractor, metadata_json) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
243
|
+
(entity_version_id, snapshot_id, logical_id, entity["entity_type"], entity["name"], entity.get("codebase"), entity.get("path"), entity.get("line_start"), entity.get("line_end"), entity.get("content_hash"), entity["confidence"], entity["extractor"], json_dumps(entity.get("metadata", {}))),
|
|
244
|
+
)
|
|
245
|
+
conn.executemany(
|
|
246
|
+
"INSERT INTO semantic_flows(snapshot_id, source_id, target_id, flow_type, source_path, line_start, confidence, evidence_level, extractor, metadata_json) VALUES(?,?,?,?,?,?,?,?,?,?)",
|
|
247
|
+
[(snapshot_id, item["source_id"], item["target_id"], item["flow_type"], item.get("source_path"), item.get("line_start"), item["confidence"], item.get("evidence_level", "static_source"), item.get("extractor", "unknown"), json_dumps(item.get("metadata", {}))) for item in semantic_flows],
|
|
248
|
+
)
|
|
249
|
+
conn.executemany(
|
|
250
|
+
"INSERT INTO feature_bindings(snapshot_id, feature_id, entity_id, binding_type, confidence, source_path, line_start) VALUES(?,?,?,?,?,?,?)",
|
|
251
|
+
[(snapshot_id, item["feature_id"], item["entity_id"], item["binding_type"], item["confidence"], item.get("source_path"), item.get("line_start")) for item in feature_bindings],
|
|
252
|
+
)
|
|
253
|
+
conn.execute(
|
|
254
|
+
"INSERT INTO coverage_metrics(snapshot_id, files_scanned, files_parsed, diagnostics, semantic_entities, semantic_flows, dynamic_limitations, metadata_json) VALUES(?,?,?,?,?,?,?,?)",
|
|
255
|
+
(snapshot_id, len(files), parsed_count, len(diagnostics), len(semantic_entities), len(semantic_flows), sum(1 for item in semantic_flows if item.get("metadata", {}).get("dynamic_limitations") or item.get("confidence") in {"heuristic", "unknown"}), json_dumps({"capabilities": workspace_state.get("capabilities", {}), "semantic_extractor": workspace_state.get("semanticExtractor")})),
|
|
256
|
+
)
|
|
88
257
|
conn.executemany(
|
|
89
258
|
"""
|
|
90
259
|
INSERT INTO code_files(id, codebase, path, language, size_bytes, content_hash, indexed_at, parser)
|
package/src/constants.ts
CHANGED
|
@@ -402,11 +402,21 @@ export class InitializationStore {
|
|
|
402
402
|
this.assertWritableRun(runId, leaseToken);
|
|
403
403
|
const transaction = this.db.transaction(() => {
|
|
404
404
|
const repository = input.repository ?? "";
|
|
405
|
-
|
|
406
|
-
|
|
405
|
+
if (repository) {
|
|
406
|
+
this.db.query("DELETE FROM evidence WHERE fact_id IN (SELECT id FROM facts WHERE run_id=? AND repository=? AND kind=?)").run(runId, repository, input.kind);
|
|
407
|
+
this.db.query("DELETE FROM facts WHERE run_id=? AND repository=? AND kind=?").run(runId, repository, input.kind);
|
|
408
|
+
} else {
|
|
409
|
+
this.db.query("DELETE FROM evidence WHERE fact_id IN (SELECT id FROM facts WHERE run_id=? AND kind=?)").run(runId, input.kind);
|
|
410
|
+
this.db.query("DELETE FROM facts WHERE run_id=? AND kind=?").run(runId, input.kind);
|
|
411
|
+
}
|
|
407
412
|
this.db.query("INSERT INTO collection_results(id,run_id,repository,kind,status,adapter,capability_json,diagnostics_json,created_at) VALUES (?,?,?,?,?,?,?,?,?) ON CONFLICT(run_id,repository,kind) DO UPDATE SET status=excluded.status,adapter=excluded.adapter,capability_json=excluded.capability_json,diagnostics_json=excluded.diagnostics_json,created_at=excluded.created_at").run(randomUUID(), runId, repository, input.kind, input.status, input.adapter, canonicalJson(input.capability ?? {}), canonicalJson(input.diagnostics ?? []), now());
|
|
408
|
-
|
|
413
|
+
const uniqueFacts = new Map<string, FactInput>();
|
|
414
|
+
for (const fact of input.facts ?? []) uniqueFacts.set(`${fact.repository ?? repository}\0${fact.objectKey}`, fact);
|
|
415
|
+
for (const fact of uniqueFacts.values()) {
|
|
409
416
|
if (fact.kind !== input.kind) throw new Error(`Fact kind ${fact.kind} does not match collection ${input.kind}`);
|
|
417
|
+
if (repository && fact.repository !== undefined && fact.repository !== repository) {
|
|
418
|
+
throw new Error(`Fact repository ${fact.repository} does not match collection repository ${repository}`);
|
|
419
|
+
}
|
|
410
420
|
if (!fact.objectKey || !fact.extractor) throw new Error("Facts require objectKey and extractor");
|
|
411
421
|
if (fact.evidenceLevel === "observed" && !fact.sourcePath) throw new Error("Observed facts require source evidence");
|
|
412
422
|
validateFactShape(fact);
|
|
@@ -165,6 +165,31 @@ function collect(files: SourceFile[], context: ProjectContext, config: ScanConfi
|
|
|
165
165
|
facts.get("evidence")!.push({ kind: "evidence", repository: repository.name, objectKey: "input-policy", value: { source: "local-static", discoveredCommandsExecuted: false, gitMetadataRead: true, secretsStored: false }, evidenceLevel: "explicit_rule", extractor: "initializer@1", confidence: "exact" });
|
|
166
166
|
facts.get("capability")!.push({ kind: "capability", repository: repository.name, objectKey: "collector:static@1", value: { supported: ["asset", "technology", "api", "page", "data_model", "integration", "rule", "toolchain"], limitations: ["static declarations only", "dynamic behavior unresolved", "no environment enablement inference"], project: context.manifest.name }, evidenceLevel: "explicit_rule", extractor: "initializer@1", confidence: "exact" });
|
|
167
167
|
}
|
|
168
|
+
// A repository can declare the same route/model/page through more than one
|
|
169
|
+
// scanner (or declare it twice). The control-plane key is intentionally
|
|
170
|
+
// unique per run/kind/repository/object_key, so collapse duplicates before
|
|
171
|
+
// handing facts to the store while retaining the richest value/evidence.
|
|
172
|
+
for (const [kind, entries] of facts) {
|
|
173
|
+
const deduped = new Map<string, FactInput>();
|
|
174
|
+
for (const entry of entries) {
|
|
175
|
+
const key = `${entry.repository ?? ""}\0${entry.objectKey}`;
|
|
176
|
+
const previous = deduped.get(key);
|
|
177
|
+
if (!previous) {
|
|
178
|
+
deduped.set(key, entry);
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
181
|
+
deduped.set(key, {
|
|
182
|
+
...previous,
|
|
183
|
+
...entry,
|
|
184
|
+
repository: entry.repository ?? previous.repository,
|
|
185
|
+
value: { ...(previous.value ?? {}), ...(entry.value ?? {}) },
|
|
186
|
+
sourcePath: entry.sourcePath ?? previous.sourcePath,
|
|
187
|
+
sourceLine: entry.sourceLine ?? previous.sourceLine,
|
|
188
|
+
sourceFingerprint: entry.sourceFingerprint ?? previous.sourceFingerprint,
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
facts.set(kind, [...deduped.values()]);
|
|
192
|
+
}
|
|
168
193
|
return facts;
|
|
169
194
|
}
|
|
170
195
|
|