@guanlin4924/devflow-pi 0.1.80 → 0.1.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/plugins/devflow-core/skills/workflow-orchestrator/scripts/requirement_graph.rb +4 -1
- package/plugins/devflow-intel/extensions/devflow-intel.ts +49 -3
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/code_index.py +148 -13
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/core/scanner.py +10 -2
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/domain_registry.py +168 -0
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/hybrid_query.py +367 -0
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/source_content.py +88 -0
- package/plugins/devflow-intel/skills/workflow-code-index/scripts/writers/sqlite_writer.py +5 -2
- package/src/constants.ts +1 -1
- package/src/initialization-store.ts +1 -0
- package/src/project-initializer.ts +6 -1
package/package.json
CHANGED
|
@@ -957,7 +957,10 @@ module RequirementGraph
|
|
|
957
957
|
blocking = []
|
|
958
958
|
checkers.each do |id, issues|
|
|
959
959
|
next if DEFERRED_CHECKERS.include?(id)
|
|
960
|
-
|
|
960
|
+
# The derived ID list is an observation, not a failed obligation. A
|
|
961
|
+
# confirmed high-risk node may remain high-risk forever; only its
|
|
962
|
+
# unconfirmed state (checked below) or model-authored risk keys block.
|
|
963
|
+
blocking.concat(id == "risk-derivation-check" ? issues.reject { |entry| entry.start_with?("derived high-risk nodes: ") } : issues)
|
|
961
964
|
end
|
|
962
965
|
Array(graph["nodes"]).each do |node|
|
|
963
966
|
next unless node.is_a?(Hash) && high_risk_by_structure?(node)
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { existsSync } from "node:fs";
|
|
1
|
+
import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
2
3
|
import path from "node:path";
|
|
3
4
|
import { fileURLToPath } from "node:url";
|
|
4
5
|
|
|
@@ -53,12 +54,22 @@ export default function devflowIntel(pi: ExtensionAPI) {
|
|
|
53
54
|
label: "Devflow Code Index",
|
|
54
55
|
description: "Scan, validate, audit, or query the deterministic project code index.",
|
|
55
56
|
parameters: z.object({
|
|
56
|
-
action: z.enum(["scan", "validate", "audit", "query", "relations", "impact"]),
|
|
57
|
+
action: z.enum(["scan", "validate", "audit", "query", "relations", "impact", "search-content", "resolve-domain", "query-plan", "domain-import"]),
|
|
57
58
|
term: z.string().optional(),
|
|
58
59
|
object: z.string().optional(),
|
|
59
60
|
entity: z.string().optional(),
|
|
61
|
+
projectId: z.string().default("default"),
|
|
62
|
+
plan: z.record(z.string(), z.unknown()).optional(),
|
|
63
|
+
bundle: z.record(z.string(), z.unknown()).optional(),
|
|
64
|
+
repository: z.string().optional(),
|
|
65
|
+
pathPrefix: z.string().optional(),
|
|
66
|
+
language: z.string().optional(),
|
|
67
|
+
matchKind: z.enum(["literal", "identifier", "phrase", "substring"]).default("literal"),
|
|
68
|
+
direction: z.enum(["incoming", "outgoing", "both"]).default("outgoing"),
|
|
69
|
+
expectedRevision: z.string().optional(),
|
|
60
70
|
changeType: z.enum(["remove_field", "rename_field", "change_type", "optional_to_required", "required_to_optional", "change_nullability", "change_enum"]).optional(),
|
|
61
71
|
snapshot: z.string().optional(),
|
|
72
|
+
mappingRevision: z.string().optional(),
|
|
62
73
|
depth: z.number().int().min(0).max(20).default(3),
|
|
63
74
|
includeHeuristic: z.boolean().default(false),
|
|
64
75
|
limit: z.number().int().min(1).max(100).default(20),
|
|
@@ -68,7 +79,7 @@ export default function devflowIntel(pi: ExtensionAPI) {
|
|
|
68
79
|
approval: (args) => {
|
|
69
80
|
const action = (args as { action?: string }).action;
|
|
70
81
|
// scan 写 .planning/code-index,按 exec 询问;validate/audit/query 只读,免确认。
|
|
71
|
-
return action
|
|
82
|
+
return ["scan", "domain-import"].includes(action ?? "") ? "exec" : "read";
|
|
72
83
|
},
|
|
73
84
|
async execute(_id, params, _signal, _onUpdate, ctx) {
|
|
74
85
|
const skillRoot = resolveSkillRoot(ctx.cwd);
|
|
@@ -92,6 +103,40 @@ export default function devflowIntel(pi: ExtensionAPI) {
|
|
|
92
103
|
args.push("--index", ".planning/code-index", "--term", params.term, "--limit", String(params.limit));
|
|
93
104
|
if (params.snapshot) args.push("--snapshot", params.snapshot);
|
|
94
105
|
} else {
|
|
106
|
+
if (params.action === "search-content") {
|
|
107
|
+
if (!params.term) return render({ ok: false, exitCode: 2, stdout: "", stderr: "term is required" }, {});
|
|
108
|
+
args.push("--index", ".planning/code-index", "--term", params.term, "--limit", String(params.limit));
|
|
109
|
+
args.push("--project-id", params.projectId);
|
|
110
|
+
args.push("--match-kind", params.matchKind === "phrase" ? "exact_phrase" : params.matchKind);
|
|
111
|
+
if (params.repository) args.push("--repository", params.repository);
|
|
112
|
+
if (params.pathPrefix) args.push("--path-prefix", params.pathPrefix);
|
|
113
|
+
if (params.language) args.push("--language", params.language);
|
|
114
|
+
return render(await run(args, ctx.cwd), { action: params.action });
|
|
115
|
+
} else if (params.action === "resolve-domain") {
|
|
116
|
+
if (!params.term) return render({ ok: false, exitCode: 2, stdout: "", stderr: "term is required" }, {});
|
|
117
|
+
args.push("--index", ".planning/code-index", "--term", params.term, "--project-id", params.projectId, "--limit", String(params.limit));
|
|
118
|
+
if (params.mappingRevision) args.push("--mapping-revision", params.mappingRevision);
|
|
119
|
+
return render(await run(args, ctx.cwd), { action: params.action });
|
|
120
|
+
} else if (params.action === "query-plan") {
|
|
121
|
+
if (!params.plan) return render({ ok: false, exitCode: 2, stdout: "", stderr: "plan is required" }, {});
|
|
122
|
+
const plan = { ...params.plan };
|
|
123
|
+
if (plan.limit === undefined) plan.limit = params.limit;
|
|
124
|
+
args.push("--index", ".planning/code-index", "--project-id", params.projectId, "--plan", JSON.stringify(plan));
|
|
125
|
+
if (params.snapshot) args.push("--snapshot", params.snapshot);
|
|
126
|
+
return render(await run(args, ctx.cwd), { action: params.action });
|
|
127
|
+
} else if (params.action === "domain-import") {
|
|
128
|
+
if (!params.bundle) return render({ ok: false, exitCode: 2, stdout: "", stderr: "bundle is required" }, {});
|
|
129
|
+
const bundleDir = mkdtempSync(path.join(tmpdir(), "devflow-domain-"));
|
|
130
|
+
const bundlePath = path.join(bundleDir, "bundle.json");
|
|
131
|
+
try {
|
|
132
|
+
writeFileSync(bundlePath, JSON.stringify(params.bundle), { encoding: "utf8", mode: 0o600 });
|
|
133
|
+
args.push("--index", ".planning/code-index", "--project-id", params.projectId, "--bundle", bundlePath);
|
|
134
|
+
if (params.expectedRevision) args.push("--expected-revision", params.expectedRevision);
|
|
135
|
+
return render(await run(args, ctx.cwd), { action: params.action });
|
|
136
|
+
} finally {
|
|
137
|
+
rmSync(bundleDir, { recursive: true, force: true });
|
|
138
|
+
}
|
|
139
|
+
}
|
|
95
140
|
if (params.action === "impact") {
|
|
96
141
|
if (!params.entity) return render({ ok: false, exitCode: 2, stdout: "", stderr: "entity is required" }, {});
|
|
97
142
|
args.push("--index", ".planning/code-index", "--entity", params.entity, "--depth", String(params.depth), "--limit", String(params.limit));
|
|
@@ -101,6 +146,7 @@ export default function devflowIntel(pi: ExtensionAPI) {
|
|
|
101
146
|
}
|
|
102
147
|
if (!params.object) return render({ ok: false, exitCode: 2, stdout: "", stderr: "object is required" }, {});
|
|
103
148
|
args.push("--index", ".planning/code-index", "--object", params.object, "--depth", String(params.depth), "--limit", String(params.limit));
|
|
149
|
+
args.push("--direction", params.direction);
|
|
104
150
|
if (params.snapshot) args.push("--snapshot", params.snapshot);
|
|
105
151
|
if (params.includeHeuristic) args.push("--include-heuristic");
|
|
106
152
|
}
|
|
@@ -27,7 +27,10 @@ from adapters.registry import AdapterRegistry
|
|
|
27
27
|
from semantic import SEMANTIC_EXTRACTOR_VERSION, build_semantic_facts
|
|
28
28
|
from core.discovery import discover_codebases
|
|
29
29
|
from core.hashing import file_sha256
|
|
30
|
-
from core.scanner import scan_files
|
|
30
|
+
from core.scanner import scan_files, DEFAULT_EXCLUDES
|
|
31
|
+
from hybrid_query import execute as execute_plan, read_connection, validate_plan
|
|
32
|
+
from domain_registry import import_bundle
|
|
33
|
+
from source_content import content_integrity
|
|
31
34
|
from validators.project_binding_check import audit_project_binding
|
|
32
35
|
from validators.relation_check import audit_relations
|
|
33
36
|
from validators.schema_validate import validate_index
|
|
@@ -68,13 +71,16 @@ def _publish_snapshot(staging: Path, output: Path, counts: Dict[str, int], works
|
|
|
68
71
|
# Keep the public snapshot schema stable while exposing the richer
|
|
69
72
|
# SQLite schema separately. Older consumers only know schemaVersion=1.
|
|
70
73
|
"schemaVersion": 1,
|
|
71
|
-
"indexSchemaVersion": 3,
|
|
74
|
+
"indexSchemaVersion": 4 if workspace_state.get("contentSearch") else 3,
|
|
72
75
|
"capabilities": {
|
|
73
76
|
"structural": True,
|
|
74
77
|
"semantic": True,
|
|
75
78
|
"fts": True,
|
|
76
79
|
"relations": True,
|
|
77
80
|
"provenance": True,
|
|
81
|
+
"contentSearch": workspace_state.get("contentSearch", False),
|
|
82
|
+
"domainMapping": {"version": 1, "storage": "external_registry"},
|
|
83
|
+
"queryPlan": {"version": 1},
|
|
78
84
|
},
|
|
79
85
|
"generation": generation,
|
|
80
86
|
"generatedAt": datetime.now(timezone.utc).isoformat(),
|
|
@@ -195,6 +201,8 @@ def main(argv=None) -> int:
|
|
|
195
201
|
scan.add_argument("--output", default=".planning/code-index", help="Index output directory.")
|
|
196
202
|
scan.add_argument("--codebase", action="append", default=[], help="Explicit codebase: name:path[:type]. Can repeat.")
|
|
197
203
|
scan.add_argument("--changed-files", nargs="*", default=[], help="Optional changed files for incremental refresh.")
|
|
204
|
+
scan.add_argument("--include", action="append", default=[], help="Repository-relative include glob. Can repeat.")
|
|
205
|
+
scan.add_argument("--max-file-size", type=int, default=1500000)
|
|
198
206
|
scan.add_argument("--exclude", action="append", default=[], help="Additional directory names to exclude.")
|
|
199
207
|
|
|
200
208
|
validate = sub.add_parser("validate", help="Validate an index in staged levels.")
|
|
@@ -227,6 +235,7 @@ def main(argv=None) -> int:
|
|
|
227
235
|
relations.add_argument("--limit", type=int, default=100)
|
|
228
236
|
relations.add_argument("--snapshot")
|
|
229
237
|
relations.add_argument("--include-heuristic", action="store_true")
|
|
238
|
+
relations.add_argument("--direction", choices=["incoming", "outgoing", "both"], default="outgoing")
|
|
230
239
|
|
|
231
240
|
impact = sub.add_parser("impact", help="Traverse cross-layer semantic facts in a fixed snapshot.")
|
|
232
241
|
impact.add_argument("--index", default=".planning/code-index")
|
|
@@ -236,7 +245,28 @@ def main(argv=None) -> int:
|
|
|
236
245
|
impact.add_argument("--snapshot")
|
|
237
246
|
impact.add_argument("--change-type", choices=["remove_field", "rename_field", "change_type", "optional_to_required", "required_to_optional", "change_nullability", "change_enum"], help="Optional proposed change classification.")
|
|
238
247
|
|
|
248
|
+
for action in ("search-content", "resolve-domain", "query-plan", "domain-import"):
|
|
249
|
+
operation = sub.add_parser(action)
|
|
250
|
+
operation.add_argument("--index", default=".planning/code-index")
|
|
251
|
+
operation.add_argument("--snapshot")
|
|
252
|
+
operation.add_argument("--project-id", default="")
|
|
253
|
+
operation.add_argument("--limit", type=int, default=20)
|
|
254
|
+
operation.add_argument("--repository", action="append", default=[])
|
|
255
|
+
operation.add_argument("--path-prefix", default="")
|
|
256
|
+
operation.add_argument("--mapping-revision", default="approved-current")
|
|
257
|
+
if action == "search-content":
|
|
258
|
+
operation.add_argument("--language")
|
|
259
|
+
operation.add_argument("--match-kind", choices=["literal", "substring", "identifier", "exact_phrase"], default="literal")
|
|
260
|
+
if action == "query-plan":
|
|
261
|
+
operation.add_argument("--plan", required=True, help="Structured QueryPlan JSON, never SQL")
|
|
262
|
+
elif action == "domain-import":
|
|
263
|
+
operation.add_argument("--bundle", required=True, help="JSON mapping bundle file")
|
|
264
|
+
operation.add_argument("--expected-revision", help="Expected approved current revision for CAS")
|
|
265
|
+
elif action in {"search-content", "resolve-domain"}:
|
|
266
|
+
operation.add_argument("--term", required=True)
|
|
239
267
|
args = parser.parse_args(argv)
|
|
268
|
+
if args.command in {"search-content", "resolve-domain", "query-plan", "domain-import"}:
|
|
269
|
+
return command_structured(args)
|
|
240
270
|
if args.command == "scan":
|
|
241
271
|
return command_scan(args)
|
|
242
272
|
if args.command == "validate":
|
|
@@ -255,6 +285,10 @@ def main(argv=None) -> int:
|
|
|
255
285
|
def command_scan(args) -> int:
|
|
256
286
|
root = Path(args.root).resolve()
|
|
257
287
|
output = (root / args.output).resolve() if not Path(args.output).is_absolute() else Path(args.output).resolve()
|
|
288
|
+
includes = getattr(args, "include", [])
|
|
289
|
+
max_file_size = getattr(args, "max_file_size", 1500000)
|
|
290
|
+
if not 1 <= max_file_size <= 50000000:
|
|
291
|
+
raise ValueError("max-file-size must be 1..50000000")
|
|
258
292
|
progress = _Progress(output)
|
|
259
293
|
progress.emit("start", "scan started", root=str(root), output=str(output), codebases=args.codebase)
|
|
260
294
|
with _index_lock(output, exclusive=True):
|
|
@@ -263,6 +297,10 @@ def command_scan(args) -> int:
|
|
|
263
297
|
registry = AdapterRegistry()
|
|
264
298
|
changed = _normalize_changed_files(root, args.changed_files)
|
|
265
299
|
existing = _load_existing(output)
|
|
300
|
+
previous_scope = _load_snapshot(output).get("workspace", {}).get("scanScope", {})
|
|
301
|
+
previous_roots = {(item.get("name"), item.get("path")) for item in previous_scope.get("codebases", [])}
|
|
302
|
+
if changed and (set(previous_scope.get("exclude", [])) != DEFAULT_EXCLUDES | set(args.exclude) or previous_roots != {(item.name, str(item.root)) for item in codebases} or previous_scope.get("includes", []) != sorted(set(includes)) or previous_scope.get("maxFileSize", 1500000) != max_file_size):
|
|
303
|
+
changed = set() # Scope changes require a complete rebuild, never reuse out-of-scope rows.
|
|
266
304
|
secondary_changed = _incoming_evidence_files(existing, changed)
|
|
267
305
|
changed |= secondary_changed
|
|
268
306
|
|
|
@@ -277,7 +315,7 @@ def command_scan(args) -> int:
|
|
|
277
315
|
diagnostics.append(_diagnostic(codebase.name, "", 1, "warning", "skipped_file", f"codebase root does not exist: {codebase.root}", "discovery@1"))
|
|
278
316
|
continue
|
|
279
317
|
progress.emit("discover", f"discovering {codebase.name}", codebase=codebase.name, path=str(codebase.root))
|
|
280
|
-
all_files = scan_files(codebase.root, excludes=args.exclude)
|
|
318
|
+
all_files = scan_files(codebase.root, excludes=args.exclude, includes=includes, max_bytes=max_file_size)
|
|
281
319
|
progress.emit("discover", f"discovered {len(all_files)} files in {codebase.name}", codebase=codebase.name, files=len(all_files))
|
|
282
320
|
discovered_paths.update(_workspace_rel(root, path) for path in all_files)
|
|
283
321
|
if changed:
|
|
@@ -355,11 +393,17 @@ def command_scan(args) -> int:
|
|
|
355
393
|
)
|
|
356
394
|
workspace_state["generatedAt"] = datetime.now(timezone.utc).isoformat()
|
|
357
395
|
workspace_state["scanScope"] = {
|
|
358
|
-
"exclude": sorted(set(args.exclude)),
|
|
396
|
+
"exclude": sorted(DEFAULT_EXCLUDES | set(args.exclude)),
|
|
397
|
+
"includes": sorted(set(includes)),
|
|
398
|
+
"generatedPolicy": "exclude_known_build_directories",
|
|
399
|
+
"dependencyPolicy": "exclude_known_dependency_directories",
|
|
400
|
+
"maxFileSize": max_file_size,
|
|
359
401
|
"codebases": [
|
|
360
|
-
{"name": item.name, "path": str(item.root)} for item in codebases
|
|
402
|
+
{"name": item.name, "path": str(item.root), "workspacePath": _workspace_rel(root, item.root)} for item in codebases
|
|
361
403
|
],
|
|
362
404
|
}
|
|
405
|
+
workspace_state["scopeHash"] = _hash_lines([json.dumps(workspace_state["scanScope"], sort_keys=True)])
|
|
406
|
+
workspace_state["root"] = str(root)
|
|
363
407
|
workspace_state["semanticExtractor"] = SEMANTIC_EXTRACTOR_VERSION
|
|
364
408
|
workspace_state["extractorSetHash"] = _hash_lines([*workspace_state.get("extractors", []), SEMANTIC_EXTRACTOR_VERSION])
|
|
365
409
|
progress.emit("semantic", "building semantic facts")
|
|
@@ -381,6 +425,7 @@ def command_scan(args) -> int:
|
|
|
381
425
|
semantic_entities=semantic_entities,
|
|
382
426
|
semantic_flows=semantic_flows,
|
|
383
427
|
feature_bindings=feature_bindings,
|
|
428
|
+
source_root=root,
|
|
384
429
|
)
|
|
385
430
|
progress.emit("documents", "writing index projections")
|
|
386
431
|
write_markdown(staging, files, symbols, relations, diagnostics)
|
|
@@ -430,6 +475,26 @@ def _validate_publish_gate(selected: Path, snapshot: Dict[str, object]) -> List[
|
|
|
430
475
|
quick = conn.execute("PRAGMA quick_check").fetchone()
|
|
431
476
|
if not quick or quick[0] != "ok":
|
|
432
477
|
errors.append(f"sqlite quick_check failed: {quick[0] if quick else 'no result'}")
|
|
478
|
+
if snapshot.get("capabilities", {}).get("contentSearch"):
|
|
479
|
+
content_tables = _table_names(conn)
|
|
480
|
+
for table in ("source_artifacts", "source_content_chunks", "snapshot_metadata"):
|
|
481
|
+
if table not in content_tables:
|
|
482
|
+
errors.append(f"content capability missing table {table}")
|
|
483
|
+
content_capability = snapshot["capabilities"]["contentSearch"]
|
|
484
|
+
if isinstance(content_capability, dict) and content_capability.get("tokenizer") == "trigram" and "source_content_fts" not in content_tables:
|
|
485
|
+
errors.append("trigram content capability missing FTS table")
|
|
486
|
+
if {"source_content_chunks", "snapshot_metadata"}.issubset(content_tables):
|
|
487
|
+
declared = conn.execute("SELECT value FROM snapshot_metadata WHERE key='contentSearch'").fetchone()
|
|
488
|
+
integrity = conn.execute("SELECT value FROM snapshot_metadata WHERE key='contentIntegrity'").fetchone()
|
|
489
|
+
if declared is None or json.loads(declared[0]) != content_capability:
|
|
490
|
+
errors.append("content capability metadata mismatch")
|
|
491
|
+
actual = content_integrity(conn)
|
|
492
|
+
if integrity is None or json.loads(integrity[0]) != actual or actual.get("missing_fts_rowids", 0):
|
|
493
|
+
errors.append("content/FTS rowid, count or integrity mismatch")
|
|
494
|
+
if {"source_content_chunks", "code_files", "source_artifacts"}.issubset(content_tables):
|
|
495
|
+
invalid = conn.execute("SELECT count(*) FROM source_content_chunks c LEFT JOIN code_files f ON f.id=c.code_file_id LEFT JOIN source_artifacts a ON a.content_hash=c.content_hash WHERE f.id IS NULL OR a.content_hash IS NULL OR c.content_hash<>f.content_hash OR c.code_generation<>?", (snapshot.get("generation"),)).fetchone()[0]
|
|
496
|
+
if invalid:
|
|
497
|
+
errors.append(f"content index has {invalid} invalid chunk references")
|
|
433
498
|
snapshot_id = snapshot.get("generation")
|
|
434
499
|
tables = _table_names(conn)
|
|
435
500
|
required = {"analysis_snapshots", "code_files", "symbols", "relations", "diagnostics"}
|
|
@@ -521,8 +586,69 @@ def command_validate(args) -> int:
|
|
|
521
586
|
if not errors:
|
|
522
587
|
print(f"Index validation passed ({level}).")
|
|
523
588
|
return 1 if errors else 0
|
|
589
|
+
def command_structured(args) -> int:
|
|
590
|
+
output = Path(args.index).resolve()
|
|
591
|
+
try:
|
|
592
|
+
if args.command == "query-plan":
|
|
593
|
+
if len(args.plan) > 32000:
|
|
594
|
+
raise ValueError("QueryPlan exceeds 32KB")
|
|
595
|
+
raw_plan = json.loads(args.plan)
|
|
596
|
+
if "limit" not in raw_plan:
|
|
597
|
+
raw_plan["limit"] = getattr(args, "limit", 20)
|
|
598
|
+
plan = validate_plan(raw_plan)
|
|
599
|
+
else:
|
|
600
|
+
plan = {"featureTerms": [args.term] if hasattr(args, "term") else [],
|
|
601
|
+
"phases": ["search_content", "verify_evidence"] if args.command == "search-content" else ["resolve_domain"],
|
|
602
|
+
"limit": args.limit, "repositoryFilter": getattr(args, "repository", []),
|
|
603
|
+
"pathPrefix": getattr(args, "path_prefix", ""), "mappingRevision": getattr(args, "mapping_revision", "approved-current"),
|
|
604
|
+
"language": getattr(args, "language", None), "matchKind": getattr(args, "match_kind", "literal")}
|
|
605
|
+
plan = validate_plan(plan)
|
|
606
|
+
requested = getattr(args, "snapshot", None)
|
|
607
|
+
planned = plan.get("codeGeneration")
|
|
608
|
+
if planned and planned != "current":
|
|
609
|
+
if requested and requested != planned:
|
|
610
|
+
raise ValueError("conflicting snapshot and codeGeneration")
|
|
611
|
+
requested = planned
|
|
612
|
+
with _index_lock(output, exclusive=False):
|
|
613
|
+
selected = _selected_generation(output, requested)
|
|
614
|
+
snapshot = _load_snapshot(selected)
|
|
615
|
+
errors = _validate_snapshot_bundle(selected, snapshot)
|
|
616
|
+
if errors:
|
|
617
|
+
raise ValueError(errors[0])
|
|
618
|
+
if not requested and not _query_is_fresh(Path(snapshot.get("workspace", {}).get("root", output.parent.parent)), snapshot, read_jsonl(selected / INDEX_FILES["files"])):
|
|
619
|
+
print(json.dumps({"_meta": {"status": "stale_index", "snapshot": snapshot.get("generation")}}))
|
|
620
|
+
return 1
|
|
621
|
+
conn = read_connection(selected / "index.sqlite")
|
|
622
|
+
try:
|
|
623
|
+
if snapshot.get("workspace", {}).get("scopeHash"):
|
|
624
|
+
scope_json = json.dumps(snapshot["workspace"]["scanScope"], sort_keys=True)
|
|
625
|
+
if _hash_lines([scope_json]) != snapshot["workspace"]["scopeHash"]:
|
|
626
|
+
raise ValueError("scan scope hash mismatch")
|
|
627
|
+
stored = conn.execute("SELECT value FROM snapshot_metadata WHERE key='scanScope'").fetchone()
|
|
628
|
+
if stored is None or json.loads(stored[0]) != snapshot["workspace"]["scanScope"]:
|
|
629
|
+
raise ValueError("scan scope metadata mismatch")
|
|
630
|
+
registry = output.parent / "domain-registry.sqlite"
|
|
631
|
+
if args.command == "domain-import":
|
|
632
|
+
bundle_path = Path(args.bundle)
|
|
633
|
+
if bundle_path.stat().st_size > 2_000_000:
|
|
634
|
+
raise ValueError("mapping bundle exceeds 2MB")
|
|
635
|
+
result = import_bundle(registry, args.project_id, json.loads(bundle_path.read_text()), getattr(args, "expected_revision", None), conn, snapshot["generation"])
|
|
636
|
+
else:
|
|
637
|
+
result = execute_plan(conn, snapshot, plan, registry, getattr(args, "project_id", ""))
|
|
638
|
+
print(json.dumps(result, ensure_ascii=False, sort_keys=True))
|
|
639
|
+
finally:
|
|
640
|
+
conn.close()
|
|
641
|
+
return 0
|
|
642
|
+
except (ValueError, TypeError, KeyError, OSError, sqlite3.Error) as exc:
|
|
643
|
+
status = "partial" if isinstance(exc, sqlite3.OperationalError) and "interrupt" in str(exc) else "invalid_query"
|
|
644
|
+
print(json.dumps({"_meta": {"status": status, "reason": str(exc)}}))
|
|
645
|
+
return 1
|
|
646
|
+
|
|
647
|
+
|
|
524
648
|
def command_query(args) -> int:
|
|
525
649
|
output = Path(args.index).resolve()
|
|
650
|
+
if hasattr(args, "term") and (not args.term or len(args.term) > 256 or "\x00" in args.term):
|
|
651
|
+
raise ValueError("term must contain 1..256 characters")
|
|
526
652
|
with _index_lock(output, exclusive=False):
|
|
527
653
|
selected = _selected_generation(output, getattr(args, "snapshot", None))
|
|
528
654
|
index = selected / "index.sqlite"
|
|
@@ -539,8 +665,7 @@ def command_query(args) -> int:
|
|
|
539
665
|
if not getattr(args, "snapshot", None) and not _query_is_fresh(output.parent.parent, snapshot, files):
|
|
540
666
|
print("Stale code index: refresh required before query.", file=sys.stderr)
|
|
541
667
|
return 1
|
|
542
|
-
conn =
|
|
543
|
-
conn.row_factory = sqlite3.Row
|
|
668
|
+
conn = read_connection(index)
|
|
544
669
|
try:
|
|
545
670
|
symbol_columns = {row[1] for row in conn.execute("PRAGMA table_info(symbols)")}
|
|
546
671
|
provenance = ", parser, content_hash, metadata_json" if {"parser", "content_hash", "metadata_json"}.issubset(symbol_columns) else ", NULL AS parser, NULL AS content_hash, NULL AS metadata_json"
|
|
@@ -554,16 +679,16 @@ def command_query(args) -> int:
|
|
|
554
679
|
WHERE symbols_fts MATCH ?
|
|
555
680
|
LIMIT ?
|
|
556
681
|
""",
|
|
557
|
-
(args.term, limit),
|
|
682
|
+
('"' + args.term.replace('"', '""') + '"', limit),
|
|
558
683
|
).fetchall()
|
|
559
684
|
except sqlite3.OperationalError:
|
|
560
|
-
like =
|
|
685
|
+
like = args.term
|
|
561
686
|
rows = conn.execute(
|
|
562
687
|
f"""
|
|
563
688
|
SELECT id, codebase, kind, name, qualified_name, path, line_start, line_end,
|
|
564
689
|
{provenance[2:]}
|
|
565
690
|
FROM symbols
|
|
566
|
-
WHERE name
|
|
691
|
+
WHERE instr(lower(name),lower(?))>0 OR instr(lower(qualified_name),lower(?))>0 OR instr(lower(path),lower(?))>0
|
|
567
692
|
LIMIT ?
|
|
568
693
|
""",
|
|
569
694
|
(like, like, like, limit),
|
|
@@ -580,6 +705,10 @@ def command_query(args) -> int:
|
|
|
580
705
|
|
|
581
706
|
|
|
582
707
|
def command_relations(args) -> int:
|
|
708
|
+
if getattr(args, "direction", "outgoing") != "outgoing":
|
|
709
|
+
args.plan = json.dumps({"seedEntityIds": [args.object], "phases": ["traverse_relations", "verify_evidence"], "direction": args.direction, "depth": min(args.depth, 5), "limit": min(args.limit, 100), "includeHeuristic": args.include_heuristic})
|
|
710
|
+
args.command = "query-plan"
|
|
711
|
+
return command_structured(args)
|
|
583
712
|
output = Path(args.index).resolve()
|
|
584
713
|
depth = max(0, min(args.depth, 20)); limit = _bounded_limit(args.limit, 1000)
|
|
585
714
|
with _index_lock(output, exclusive=False):
|
|
@@ -590,7 +719,7 @@ def command_relations(args) -> int:
|
|
|
590
719
|
print(bundle_errors[0], file=sys.stderr); return 1
|
|
591
720
|
if not getattr(args, "snapshot", None) and not _query_is_fresh(output.parent.parent, snapshot, files):
|
|
592
721
|
print("Stale code index: refresh required before query.", file=sys.stderr); return 1
|
|
593
|
-
conn =
|
|
722
|
+
conn = read_connection(selected / "index.sqlite")
|
|
594
723
|
try:
|
|
595
724
|
symbol_columns = {row[1] for row in conn.execute("PRAGMA table_info(symbols)")}
|
|
596
725
|
parser_expr = "s.parser" if "parser" in symbol_columns else "NULL"
|
|
@@ -621,6 +750,10 @@ def command_relations(args) -> int:
|
|
|
621
750
|
for row in rows[:limit]:
|
|
622
751
|
item = dict(row)
|
|
623
752
|
item["snapshot"] = snapshot.get("generation")
|
|
753
|
+
relation_parser = conn.execute("SELECT parser FROM relations WHERE from_id=? AND relation_type=? AND evidence_path=? AND evidence_line=? LIMIT 1", (item["from_id"], item["relation_type"], item["evidence_path"], item["evidence_line"])).fetchone()
|
|
754
|
+
item["relation_parser"] = relation_parser[0] if relation_parser else "unknown"
|
|
755
|
+
item["source_file_parser"] = item.get("from_parser")
|
|
756
|
+
item["provenance"] = {"relation_parser": item["relation_parser"], "source_file_parser": item["source_file_parser"], "snapshot": snapshot.get("generation")}
|
|
624
757
|
if getattr(args, "change_type", None):
|
|
625
758
|
item["impact_kind"] = args.change_type
|
|
626
759
|
item["severity"] = _impact_severity(args.change_type, item.get("flow_type", "relation"))
|
|
@@ -655,7 +788,7 @@ def command_impact(args) -> int:
|
|
|
655
788
|
if not _query_is_fresh(output.parent.parent, snapshot, files):
|
|
656
789
|
print("Stale code index: refresh required before impact query.", file=sys.stderr)
|
|
657
790
|
return 1
|
|
658
|
-
conn =
|
|
791
|
+
conn = read_connection(selected / "index.sqlite")
|
|
659
792
|
try:
|
|
660
793
|
tables = _table_names(conn)
|
|
661
794
|
semantic_tables = {"semantic_flows", "entity_versions", "feature_bindings", "coverage_metrics"}
|
|
@@ -792,7 +925,9 @@ def _query_is_fresh(root: Path, snapshot: Dict[str, object], files: List[dict])
|
|
|
792
925
|
if not any(path.startswith("../") for path in indexed_paths):
|
|
793
926
|
scope = workspace.get("scanScope") if isinstance(workspace.get("scanScope"), dict) else {}
|
|
794
927
|
excludes = scope.get("exclude", []) if isinstance(scope, dict) else []
|
|
795
|
-
|
|
928
|
+
repositories = scope.get("codebases", [])
|
|
929
|
+
roots = [Path(item["path"]) for item in repositories] if repositories else [root]
|
|
930
|
+
current_paths = sorted({_workspace_rel(root, item) for repository in roots for item in scan_files(repository, excludes=excludes, max_bytes=scope.get("maxFileSize", 1500000), includes=scope.get("includes", []))})
|
|
796
931
|
expected_paths_hash = workspace.get("discoveredPathsHash") or workspace.get("indexedPathsHash")
|
|
797
932
|
if _hash_lines(current_paths) != expected_paths_hash:
|
|
798
933
|
return False
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import os
|
|
2
|
+
import fnmatch
|
|
2
3
|
import subprocess
|
|
3
4
|
from pathlib import Path
|
|
4
5
|
from typing import Iterable, List, Sequence
|
|
@@ -22,6 +23,7 @@ DEFAULT_EXCLUDES = {
|
|
|
22
23
|
".dart_tool",
|
|
23
24
|
"__pycache__",
|
|
24
25
|
"Pods",
|
|
26
|
+
"server/public",
|
|
25
27
|
}
|
|
26
28
|
|
|
27
29
|
TEXT_EXTENSIONS = {
|
|
@@ -57,7 +59,7 @@ TEXT_EXTENSIONS = {
|
|
|
57
59
|
}
|
|
58
60
|
|
|
59
61
|
|
|
60
|
-
def scan_files(root: Path, excludes: Iterable[str] = (), max_bytes: int = 1_500_000) -> List[Path]:
|
|
62
|
+
def scan_files(root: Path, excludes: Iterable[str] = (), max_bytes: int = 1_500_000, includes: Iterable[str] = ()) -> List[Path]:
|
|
61
63
|
root = root.resolve()
|
|
62
64
|
excluded = DEFAULT_EXCLUDES | set(excludes)
|
|
63
65
|
paths = _scan_with_git(root, excluded) or _scan_with_rg(root, excluded) or _scan_with_walk(root, excluded)
|
|
@@ -65,6 +67,9 @@ def scan_files(root: Path, excludes: Iterable[str] = (), max_bytes: int = 1_500_
|
|
|
65
67
|
for path in sorted(paths):
|
|
66
68
|
if not path.is_file():
|
|
67
69
|
continue
|
|
70
|
+
relative = path.relative_to(root).as_posix()
|
|
71
|
+
if includes and not any(fnmatch.fnmatch(relative, pattern) or (pattern.startswith("**/") and fnmatch.fnmatch(relative, pattern[3:])) for pattern in includes):
|
|
72
|
+
continue
|
|
68
73
|
if _is_excluded(path, root, excluded):
|
|
69
74
|
continue
|
|
70
75
|
if path.suffix and path.suffix.lower() not in TEXT_EXTENSIONS:
|
|
@@ -130,5 +135,8 @@ def _is_excluded(path: Path, root: Path, excluded: Sequence[str]) -> bool:
|
|
|
130
135
|
parts = path.resolve().relative_to(root).parts
|
|
131
136
|
except ValueError:
|
|
132
137
|
return True
|
|
133
|
-
|
|
138
|
+
relative = "/".join(parts)
|
|
139
|
+
if any(fnmatch.fnmatch(relative, pattern) or fnmatch.fnmatch(relative + "/", pattern.rstrip("/") + "/") for pattern in excluded):
|
|
140
|
+
return True
|
|
141
|
+
return any(part in excluded for part in parts) or any("/" in pattern and ("/" + "/".join(parts) + "/").find("/" + pattern.strip("/") + "/") >= 0 for pattern in excluded)
|
|
134
142
|
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""Append-only project scoped mapping bundles, separate from code snapshots.
|
|
2
|
+
|
|
3
|
+
Approval import is an explicit local reviewer attestation, not an authorization service.
|
|
4
|
+
"""
|
|
5
|
+
import json
|
|
6
|
+
import sqlite3
|
|
7
|
+
from datetime import datetime, timezone
|
|
8
|
+
|
|
9
|
+
SCHEMA = '''
|
|
10
|
+
CREATE TABLE IF NOT EXISTS domain_mapping_revisions(
|
|
11
|
+
project_id TEXT NOT NULL,mapping_revision TEXT NOT NULL,code_generation TEXT NOT NULL,
|
|
12
|
+
requirement_revision TEXT,status TEXT NOT NULL,previous_revision TEXT,bundle_json TEXT NOT NULL,
|
|
13
|
+
created_at TEXT NOT NULL,PRIMARY KEY(project_id,mapping_revision));
|
|
14
|
+
CREATE TABLE IF NOT EXISTS domain_mapping_current(project_id TEXT PRIMARY KEY,current_revision TEXT NOT NULL);
|
|
15
|
+
CREATE TABLE IF NOT EXISTS domain_term_versions(
|
|
16
|
+
project_id TEXT NOT NULL,mapping_revision TEXT NOT NULL,term_id TEXT NOT NULL,canonical_name TEXT NOT NULL,
|
|
17
|
+
PRIMARY KEY(project_id,mapping_revision,term_id));
|
|
18
|
+
CREATE TABLE IF NOT EXISTS domain_term_alias_versions(
|
|
19
|
+
project_id TEXT NOT NULL,mapping_revision TEXT NOT NULL,term_id TEXT NOT NULL,alias TEXT NOT NULL,
|
|
20
|
+
PRIMARY KEY(project_id,mapping_revision,term_id,alias));
|
|
21
|
+
CREATE TABLE IF NOT EXISTS domain_feature_versions(
|
|
22
|
+
project_id TEXT NOT NULL,mapping_revision TEXT NOT NULL,feature_id TEXT NOT NULL,canonical_name TEXT NOT NULL,
|
|
23
|
+
PRIMARY KEY(project_id,mapping_revision,feature_id));
|
|
24
|
+
CREATE TABLE IF NOT EXISTS domain_feature_terms(
|
|
25
|
+
project_id TEXT NOT NULL,mapping_revision TEXT NOT NULL,feature_id TEXT NOT NULL,term_id TEXT NOT NULL,
|
|
26
|
+
PRIMARY KEY(project_id,mapping_revision,feature_id,term_id));
|
|
27
|
+
CREATE TABLE IF NOT EXISTS domain_feature_bindings(
|
|
28
|
+
project_id TEXT NOT NULL,mapping_revision TEXT NOT NULL,feature_id TEXT NOT NULL,entity_id TEXT NOT NULL,
|
|
29
|
+
entity_version_id TEXT NOT NULL,code_generation TEXT NOT NULL,
|
|
30
|
+
PRIMARY KEY(project_id,mapping_revision,feature_id,entity_version_id));
|
|
31
|
+
CREATE TABLE IF NOT EXISTS domain_mapping_decisions(
|
|
32
|
+
project_id TEXT NOT NULL,mapping_revision TEXT NOT NULL,reviewer TEXT NOT NULL,reason TEXT NOT NULL,
|
|
33
|
+
requirement_revision TEXT NOT NULL,decision TEXT NOT NULL,PRIMARY KEY(project_id,mapping_revision));
|
|
34
|
+
'''
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def import_bundle(path, project, bundle, expected, code_conn, generation):
|
|
38
|
+
if not project or len(project) > 200:
|
|
39
|
+
raise ValueError('project_id required (max 200 characters)')
|
|
40
|
+
if not isinstance(bundle, dict) or len(json.dumps(bundle)) > 2_000_000:
|
|
41
|
+
raise ValueError('invalid or oversized mapping bundle')
|
|
42
|
+
revision = bundle.get('mapping_revision')
|
|
43
|
+
if not isinstance(revision, str) or not revision or len(revision) > 200:
|
|
44
|
+
raise ValueError('mapping_revision required (max 200 characters)')
|
|
45
|
+
if bundle.get('code_generation') != generation:
|
|
46
|
+
raise ValueError('mapping code_generation mismatch')
|
|
47
|
+
status = bundle.get('status', 'candidate')
|
|
48
|
+
if status not in {'candidate', 'pending_review', 'approved', 'rejected', 'historical_unmapped'}:
|
|
49
|
+
raise ValueError('invalid mapping lifecycle status')
|
|
50
|
+
decision = bundle.get('decision', {})
|
|
51
|
+
if status in {'approved', 'rejected'} and not all(isinstance(v, str) and v.strip() for v in [
|
|
52
|
+
decision.get('reviewer'), decision.get('reason'), bundle.get('requirement_revision')]):
|
|
53
|
+
raise ValueError('review decision requires reviewer, reason and requirement_revision')
|
|
54
|
+
terms = bundle.get('terms', [])
|
|
55
|
+
features = bundle.get('features', [])
|
|
56
|
+
if not terms or not features or len(terms) + len(features) > 1000:
|
|
57
|
+
raise ValueError('bundle requires 1..1000 terms and features')
|
|
58
|
+
term_ids = {term['id'] for term in terms}
|
|
59
|
+
feature_ids = {feature['id'] for feature in features}
|
|
60
|
+
if len(term_ids) != len(terms) or len(feature_ids) != len(features):
|
|
61
|
+
raise ValueError('duplicate term or feature id')
|
|
62
|
+
for feature in features:
|
|
63
|
+
if not set(feature.get('term_ids', [])).issubset(term_ids):
|
|
64
|
+
raise ValueError('unknown term reference')
|
|
65
|
+
for binding in feature.get('bindings', []):
|
|
66
|
+
row = code_conn.execute('SELECT entity_version_id,content_hash FROM entity_versions WHERE snapshot_id=? AND logical_id=?', (generation, binding['entity_id'])).fetchone()
|
|
67
|
+
if row is None or binding.get('entity_version_id') != row[0] or binding.get('content_hash') != row[1]:
|
|
68
|
+
raise ValueError('binding entity version or content hash mismatch')
|
|
69
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
70
|
+
conn = sqlite3.connect(path)
|
|
71
|
+
try:
|
|
72
|
+
conn.executescript(SCHEMA)
|
|
73
|
+
if 'decision' not in {row[1] for row in conn.execute('PRAGMA table_info(domain_mapping_decisions)')}:
|
|
74
|
+
conn.execute("ALTER TABLE domain_mapping_decisions ADD COLUMN decision TEXT NOT NULL DEFAULT 'approved'")
|
|
75
|
+
immutable_tables = ['domain_mapping_revisions', 'domain_term_versions', 'domain_term_alias_versions', 'domain_feature_versions', 'domain_feature_terms', 'domain_feature_bindings', 'domain_mapping_decisions']
|
|
76
|
+
for table in immutable_tables:
|
|
77
|
+
for operation in ['UPDATE','DELETE']:
|
|
78
|
+
conn.execute(f"CREATE TRIGGER IF NOT EXISTS immutable_{table}_{operation} BEFORE {operation} ON {table} BEGIN SELECT RAISE(ABORT, 'mapping revisions are append-only'); END")
|
|
79
|
+
revision_children = ['domain_term_versions', 'domain_term_alias_versions', 'domain_feature_versions', 'domain_feature_terms', 'domain_feature_bindings', 'domain_mapping_decisions']
|
|
80
|
+
for table in revision_children:
|
|
81
|
+
conn.execute(f'''CREATE TRIGGER IF NOT EXISTS sealed_{table}_insert BEFORE INSERT ON {table}
|
|
82
|
+
WHEN EXISTS(SELECT 1 FROM domain_mapping_revisions r
|
|
83
|
+
WHERE r.project_id=NEW.project_id AND r.mapping_revision=NEW.mapping_revision)
|
|
84
|
+
BEGIN SELECT RAISE(ABORT, 'mapping revision is sealed'); END''')
|
|
85
|
+
for operation in ['INSERT','UPDATE']:
|
|
86
|
+
conn.execute(f'''CREATE TRIGGER IF NOT EXISTS approved_current_{operation} BEFORE {operation} ON domain_mapping_current
|
|
87
|
+
WHEN NOT EXISTS(SELECT 1 FROM domain_mapping_revisions r JOIN domain_mapping_decisions d
|
|
88
|
+
ON d.project_id=r.project_id AND d.mapping_revision=r.mapping_revision
|
|
89
|
+
WHERE r.project_id=NEW.project_id AND r.mapping_revision=NEW.current_revision
|
|
90
|
+
AND r.status='approved' AND d.decision='approved' AND length(d.reviewer)>0 AND length(d.reason)>0)
|
|
91
|
+
BEGIN SELECT RAISE(ABORT, 'current must reference an approved revision with decision'); END''')
|
|
92
|
+
conn.commit()
|
|
93
|
+
conn.execute('BEGIN IMMEDIATE')
|
|
94
|
+
current = conn.execute('SELECT current_revision FROM domain_mapping_current WHERE project_id=?', (project,)).fetchone()
|
|
95
|
+
if (current[0] if current else None) != expected:
|
|
96
|
+
raise ValueError('mapping current CAS conflict')
|
|
97
|
+
previous = bundle.get('previous_revision')
|
|
98
|
+
if previous:
|
|
99
|
+
prior = conn.execute('SELECT status,bundle_json FROM domain_mapping_revisions WHERE project_id=? AND mapping_revision=?', (project,previous)).fetchone()
|
|
100
|
+
if prior is None:
|
|
101
|
+
raise ValueError('previous mapping revision does not exist in project')
|
|
102
|
+
transitions = {'candidate': {'candidate','pending_review','rejected','historical_unmapped'}, 'pending_review': {'approved','rejected','candidate'}, 'approved': {'candidate','historical_unmapped'}, 'rejected': {'candidate'}, 'historical_unmapped': {'candidate'}}
|
|
103
|
+
if status not in transitions[prior[0]]:
|
|
104
|
+
raise ValueError('invalid mapping status transition')
|
|
105
|
+
if status == 'approved':
|
|
106
|
+
prior_bundle = json.loads(prior[1])
|
|
107
|
+
for field in ['terms','features','code_generation','requirement_revision']:
|
|
108
|
+
if bundle.get(field) != prior_bundle.get(field):
|
|
109
|
+
raise ValueError('approved mapping differs from pending review bundle')
|
|
110
|
+
elif status not in {'candidate','historical_unmapped'}:
|
|
111
|
+
raise ValueError('review states require a previous mapping revision')
|
|
112
|
+
for term in terms:
|
|
113
|
+
conn.execute('INSERT INTO domain_term_versions VALUES(?,?,?,?)', (project,revision,term['id'],term['name']))
|
|
114
|
+
for alias in set(term.get('aliases', [])):
|
|
115
|
+
conn.execute('INSERT INTO domain_term_alias_versions VALUES(?,?,?,?)', (project,revision,term['id'],alias))
|
|
116
|
+
for feature in features:
|
|
117
|
+
conn.execute('INSERT INTO domain_feature_versions VALUES(?,?,?,?)', (project,revision,feature['id'],feature['name']))
|
|
118
|
+
for term in feature.get('term_ids', []):
|
|
119
|
+
conn.execute('INSERT INTO domain_feature_terms VALUES(?,?,?,?)', (project,revision,feature['id'],term))
|
|
120
|
+
for binding in feature.get('bindings', []):
|
|
121
|
+
conn.execute('INSERT INTO domain_feature_bindings VALUES(?,?,?,?,?,?)', (project,revision,feature['id'],binding['entity_id'],binding['entity_version_id'],generation))
|
|
122
|
+
if status in {'approved','rejected'}:
|
|
123
|
+
conn.execute('INSERT INTO domain_mapping_decisions VALUES(?,?,?,?,?,?)', (project,revision,decision['reviewer'],decision['reason'],bundle['requirement_revision'],status))
|
|
124
|
+
# The revision row is the seal: child rows are staged first in this
|
|
125
|
+
# transaction and no child INSERT is allowed after the seal exists.
|
|
126
|
+
conn.execute('INSERT INTO domain_mapping_revisions VALUES(?,?,?,?,?,?,?,?)', (
|
|
127
|
+
project,revision,generation,bundle.get('requirement_revision'),status,previous,
|
|
128
|
+
json.dumps(bundle, ensure_ascii=False, sort_keys=True),datetime.now(timezone.utc).isoformat()))
|
|
129
|
+
if status == 'approved':
|
|
130
|
+
conn.execute('INSERT INTO domain_mapping_current VALUES(?,?) ON CONFLICT(project_id) DO UPDATE SET current_revision=excluded.current_revision', (project,revision))
|
|
131
|
+
conn.commit()
|
|
132
|
+
finally:
|
|
133
|
+
conn.close()
|
|
134
|
+
return {'mapping_revision': revision, 'status': status, 'project_id': project}
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def resolve(path, project, terms, generation, revision='approved-current', statuses=None, requirement=None):
|
|
138
|
+
if not path.is_file():
|
|
139
|
+
return {'status': 'capability_unavailable', 'mappings': [], 'mapping_revision': None}
|
|
140
|
+
statuses = statuses or ['approved']
|
|
141
|
+
if not set(statuses).issubset({'approved', 'candidate', 'pending_review', 'rejected', 'historical_unmapped'}):
|
|
142
|
+
raise ValueError('invalid mapping statusFilter')
|
|
143
|
+
conn = sqlite3.connect(path.as_uri() + '?mode=ro', uri=True)
|
|
144
|
+
conn.row_factory = sqlite3.Row
|
|
145
|
+
try:
|
|
146
|
+
conn.execute('BEGIN')
|
|
147
|
+
if revision == 'approved-current':
|
|
148
|
+
row = conn.execute('SELECT current_revision FROM domain_mapping_current WHERE project_id=?', (project,)).fetchone()
|
|
149
|
+
revision = row[0] if row else None
|
|
150
|
+
row = conn.execute('SELECT * FROM domain_mapping_revisions WHERE project_id=? AND mapping_revision=?', (project,revision)).fetchone()
|
|
151
|
+
result = {'status': 'no_domain_match', 'mapping_revision': revision, 'mappings': [], 'requirement_revision': row['requirement_revision'] if row else None}
|
|
152
|
+
if row is None or row['status'] not in statuses:
|
|
153
|
+
return result
|
|
154
|
+
if requirement and requirement != row['requirement_revision']:
|
|
155
|
+
raise ValueError('requirement revision mismatch')
|
|
156
|
+
if row['code_generation'] != generation:
|
|
157
|
+
result['status'] = 'mapping_snapshot_mismatch'
|
|
158
|
+
return result
|
|
159
|
+
bundle = json.loads(row['bundle_json'])
|
|
160
|
+
requested = {term.casefold() for term in terms}
|
|
161
|
+
matched = {term['id'] for term in bundle['terms'] if requested.intersection(value.casefold() for value in [term['name'], *term.get('aliases', [])])}
|
|
162
|
+
result['mappings'] = [{**feature, 'mapping_status': row['status'], 'mapping_revision': revision, 'code_generation': generation,
|
|
163
|
+
'provenance': {'source_type': 'reviewer_attestation' if row['status'] == 'approved' else 'candidate_import', 'decision': bundle.get('decision')}}
|
|
164
|
+
for feature in bundle['features'] if matched.intersection(feature.get('term_ids', []))]
|
|
165
|
+
result['status'] = 'ambiguous' if len(result['mappings']) > 1 else 'ok' if result['mappings'] else 'no_domain_match'
|
|
166
|
+
return result
|
|
167
|
+
finally:
|
|
168
|
+
conn.close()
|
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
"""Bounded structured query executor; callers never supply SQL."""
|
|
2
|
+
import json
|
|
3
|
+
import re
|
|
4
|
+
import sqlite3
|
|
5
|
+
import time
|
|
6
|
+
from pathlib import PurePosixPath
|
|
7
|
+
from adapters.capabilities import confidence_for_parser
|
|
8
|
+
from source_content import verify_span
|
|
9
|
+
from domain_registry import resolve
|
|
10
|
+
|
|
11
|
+
PHASES = ['resolve_domain', 'search_symbols', 'search_content', 'rank_seeds', 'traverse_relations', 'impact_change', 'verify_evidence', 'compose_result']
|
|
12
|
+
FIELDS = {'codeGeneration','mappingRevision','requirementRevision','featureTerms','identifiers','repositoryFilter','pathPrefix','phases','operations','seedEntityIds','changeType','coveragePolicy','direction','depth','limit','statusFilter','includeHeuristic','language','matchKind'}
|
|
13
|
+
CHANGES = {'remove_field','rename_field','change_type','optional_to_required','required_to_optional','change_nullability','change_enum'}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def validate_plan(plan):
|
|
17
|
+
if not isinstance(plan, dict) or set(plan) - FIELDS:
|
|
18
|
+
raise ValueError('QueryPlan contains unknown fields')
|
|
19
|
+
plan = dict(plan)
|
|
20
|
+
phases = plan.get('phases', plan.get('operations', ['resolve_domain', 'search_symbols', 'search_content', 'rank_seeds', 'verify_evidence', 'compose_result']))
|
|
21
|
+
if not isinstance(phases, list) or any(p not in PHASES for p in phases) or len(phases) != len(set(phases)):
|
|
22
|
+
raise ValueError('invalid query phases')
|
|
23
|
+
plan['phases'] = [p for p in PHASES if p in phases]
|
|
24
|
+
for key in ['featureTerms', 'identifiers', 'seedEntityIds', 'repositoryFilter', 'statusFilter']:
|
|
25
|
+
value = plan.get(key, ['approved'] if key == 'statusFilter' else [])
|
|
26
|
+
maximum = 16 if key != 'seedEntityIds' else 20
|
|
27
|
+
if not isinstance(value, list) or len(value) > maximum or any(not isinstance(v, str) or not v or len(v) > (1000 if key == 'seedEntityIds' else 256) or '\x00' in v for v in value):
|
|
28
|
+
raise ValueError('invalid or oversized ' + key)
|
|
29
|
+
plan[key] = value
|
|
30
|
+
for key, default, maximum in [('limit', 20, 100), ('depth', 2, 5)]:
|
|
31
|
+
value = plan.get(key, default)
|
|
32
|
+
if type(value) is not int or not 0 <= value <= maximum or key == 'limit' and value == 0:
|
|
33
|
+
raise ValueError('invalid ' + key)
|
|
34
|
+
plan[key] = value
|
|
35
|
+
prefix = plan.get('pathPrefix', '')
|
|
36
|
+
if not isinstance(prefix, str) or len(prefix) > 1000 or '\\' in prefix or '\x00' in prefix or PurePosixPath(prefix).is_absolute() or '..' in PurePosixPath(prefix).parts:
|
|
37
|
+
raise ValueError('pathPrefix must be a repository-relative path without traversal')
|
|
38
|
+
plan['pathPrefix'] = prefix.strip('/')
|
|
39
|
+
if plan.get('direction', 'outgoing') not in {'incoming','outgoing','both'}:
|
|
40
|
+
raise ValueError('invalid direction')
|
|
41
|
+
if plan.get('changeType') is not None and plan['changeType'] not in CHANGES:
|
|
42
|
+
raise ValueError('invalid changeType')
|
|
43
|
+
if 'impact_change' in phases and plan.get('changeType') not in CHANGES:
|
|
44
|
+
raise ValueError('impact_change requires changeType')
|
|
45
|
+
if plan.get('coveragePolicy', 'local_only') != 'local_only':
|
|
46
|
+
raise ValueError('only local_only coverage is supported')
|
|
47
|
+
if plan.get('matchKind', 'literal') not in {'literal','substring','identifier','exact_phrase'}:
|
|
48
|
+
raise ValueError('invalid matchKind')
|
|
49
|
+
if plan.get('language') is not None and (not isinstance(plan['language'], str) or len(plan['language']) > 80):
|
|
50
|
+
raise ValueError('invalid language')
|
|
51
|
+
if not set(plan['statusFilter']).issubset({'approved','candidate','pending_review','rejected','historical_unmapped'}):
|
|
52
|
+
raise ValueError('invalid statusFilter')
|
|
53
|
+
search = bool({'search_symbols','search_content'}.intersection(phases))
|
|
54
|
+
domain = 'resolve_domain' in phases
|
|
55
|
+
has_seed = bool(plan['seedEntityIds'])
|
|
56
|
+
if not phases:
|
|
57
|
+
raise ValueError('QueryPlan requires at least one phase')
|
|
58
|
+
if 'rank_seeds' in phases and not (search or domain or has_seed):
|
|
59
|
+
raise ValueError('rank_seeds requires search, domain resolution or explicit seeds')
|
|
60
|
+
if {'traverse_relations','impact_change'}.intersection(phases) and not (has_seed or domain or (search and 'rank_seeds' in phases)):
|
|
61
|
+
raise ValueError('traversal requires explicit seeds, resolve_domain, or search + rank_seeds')
|
|
62
|
+
if 'verify_evidence' in phases and not (search or domain or {'traverse_relations','impact_change'}.intersection(phases)):
|
|
63
|
+
raise ValueError('verify_evidence requires a result-producing phase')
|
|
64
|
+
if phases == ['compose_result']:
|
|
65
|
+
raise ValueError('compose_result requires a result-producing phase')
|
|
66
|
+
return plan
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def read_connection(path):
|
|
70
|
+
conn = sqlite3.connect(path.as_uri() + '?mode=ro&immutable=1', uri=True)
|
|
71
|
+
conn.row_factory = sqlite3.Row
|
|
72
|
+
conn.execute('PRAGMA query_only=ON')
|
|
73
|
+
deadline = time.monotonic() + 3
|
|
74
|
+
conn.set_progress_handler(lambda: int(time.monotonic() > deadline), 1000)
|
|
75
|
+
conn.execute('BEGIN')
|
|
76
|
+
return conn
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _scope(plan, snapshot):
|
|
80
|
+
scope = snapshot.get('workspace', {}).get('scanScope', {})
|
|
81
|
+
repos = scope.get('codebases', [])
|
|
82
|
+
known = {r['name']: r for r in repos}
|
|
83
|
+
if plan['repositoryFilter'] and (not known or not set(plan['repositoryFilter']).issubset(known)):
|
|
84
|
+
raise ValueError('repositoryFilter is outside snapshot scan scope')
|
|
85
|
+
prefixes = []
|
|
86
|
+
selected = plan['repositoryFilter'] or ([r['name'] for r in repos] if plan['pathPrefix'] else [])
|
|
87
|
+
for name in selected:
|
|
88
|
+
base = known[name].get('workspacePath', '')
|
|
89
|
+
prefix = '/'.join(p for p in [base.strip('/'), plan['pathPrefix']] if p and p != '.')
|
|
90
|
+
prefixes.append((name, prefix))
|
|
91
|
+
if not prefixes:
|
|
92
|
+
prefixes = [(None, plan['pathPrefix'])]
|
|
93
|
+
return prefixes
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _where(plan, prefixes, alias='s'):
|
|
97
|
+
parts, args = [], []
|
|
98
|
+
for repository, prefix in prefixes:
|
|
99
|
+
p, values = [], []
|
|
100
|
+
if repository:
|
|
101
|
+
p.append(f'{alias}.codebase=?'); values.append(repository)
|
|
102
|
+
if prefix:
|
|
103
|
+
p.append(f'({alias}.path=? OR substr({alias}.path,1,?)=?)')
|
|
104
|
+
values.extend([prefix, len(prefix) + 1, prefix + '/'])
|
|
105
|
+
parts.append('(' + (' AND '.join(p) or '1') + ')'); args.extend(values)
|
|
106
|
+
where = '(' + ' OR '.join(parts) + ')'
|
|
107
|
+
if plan.get('language'):
|
|
108
|
+
where += f' AND {alias}.language=?'; args.append(plan['language'])
|
|
109
|
+
return where, args
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _decorate(item, generation, kind):
|
|
113
|
+
item['snapshot'] = generation
|
|
114
|
+
item['entity_id'] = (item.get('id') or item.get('entity_id')) if kind == 'symbol' else item.get('entity_id')
|
|
115
|
+
item['entity_version_id'] = generation + ':' + item['entity_id'] if item.get('entity_id') else None
|
|
116
|
+
item['parser'] = item.get('parser') or 'unknown'
|
|
117
|
+
item['confidence'] = item.get('confidence') or ('exact' if kind == 'content' else confidence_for_parser(item['parser'], 'symbol'))
|
|
118
|
+
item['result_type'] = kind
|
|
119
|
+
item['provenance'] = {'relation_parser': item.get('relation_parser'), 'source_file_parser': item.get('source_file_parser', item['parser']), 'source': 'snapshot_source' if kind == 'content' else 'code_index', 'snapshot': generation, 'parser': item['parser'], 'content_hash': item.get('content_hash')}
|
|
120
|
+
return item
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def execute(conn, snapshot, plan, registry_path, project):
|
|
124
|
+
plan = validate_plan(plan)
|
|
125
|
+
generation = snapshot['generation']
|
|
126
|
+
phases = plan['phases']; limit = plan['limit']; prefixes = _scope(plan, snapshot)
|
|
127
|
+
tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
|
128
|
+
terms = list(dict.fromkeys([*plan['featureTerms'], *plan['identifiers']]))
|
|
129
|
+
conn.create_function('identifier_match', 2, lambda content, term: int(bool(re.search(r'(?<!\w)' + re.escape(term) + r'(?!\w)', content))), deterministic=True)
|
|
130
|
+
results, phase_status, seeds = {}, {}, set(plan['seedEntityIds'])
|
|
131
|
+
truncated = False
|
|
132
|
+
mappings = {'mapping_revision': None, 'requirement_revision': None, 'mappings': []}
|
|
133
|
+
if 'resolve_domain' in phases:
|
|
134
|
+
if not project:
|
|
135
|
+
raise ValueError('resolve_domain requires project_id')
|
|
136
|
+
mappings = resolve(registry_path, project, terms, generation, plan.get('mappingRevision', 'approved-current'), plan['statusFilter'], plan.get('requirementRevision'))
|
|
137
|
+
phase_status['resolve_domain'] = mappings['status']
|
|
138
|
+
if mappings['status'] == 'ok':
|
|
139
|
+
for feature in mappings['mappings']:
|
|
140
|
+
# Candidate mapping is visible but never silently becomes a traversal seed.
|
|
141
|
+
if feature['mapping_status'] == 'approved':
|
|
142
|
+
seeds.update(binding['entity_id'] for binding in feature.get('bindings', []))
|
|
143
|
+
where, scope_args = _where(plan, prefixes)
|
|
144
|
+
for seed in sorted(seeds)[:20]:
|
|
145
|
+
row = conn.execute(f'SELECT s.* FROM symbols s WHERE {where} AND s.id=?', (*scope_args,seed)).fetchone()
|
|
146
|
+
if row and any(seed == binding['entity_id'] for feature in mappings['mappings'] for binding in feature.get('bindings', [])):
|
|
147
|
+
item = _decorate(dict(row), generation, 'symbol')
|
|
148
|
+
item.update(match_kind='approved_domain_binding', score=100, matched_terms=terms, score_contributions=[{'kind':'approved_domain_binding','score':100,'reason':'approved domain feature bound to this entity version'}])
|
|
149
|
+
results[('symbol',item['id'])] = item
|
|
150
|
+
if 'search_symbols' in phases:
|
|
151
|
+
count = 0
|
|
152
|
+
for term in terms:
|
|
153
|
+
rows = conn.execute(f'''SELECT s.* FROM symbols s WHERE {where}
|
|
154
|
+
AND (instr(lower(s.name),lower(?))>0 OR instr(lower(s.qualified_name),lower(?))>0 OR instr(lower(s.path),lower(?))>0)
|
|
155
|
+
ORDER BY CASE WHEN lower(s.name)=lower(?) THEN 0
|
|
156
|
+
WHEN instr(lower(s.name),lower(?))>0 OR instr(lower(s.qualified_name),lower(?))>0 THEN 1
|
|
157
|
+
ELSE 2 END, s.path,s.line_start,s.id LIMIT ?''',
|
|
158
|
+
(*scope_args,term,term,term,term,term,term,limit + 1)).fetchall()
|
|
159
|
+
truncated |= len(rows) > limit
|
|
160
|
+
for row in rows[:limit]:
|
|
161
|
+
item = _decorate(dict(row), generation, 'symbol')
|
|
162
|
+
if item['name'].casefold() == term.casefold() and term in plan['identifiers']:
|
|
163
|
+
kind, score = 'identifier_exact', 90
|
|
164
|
+
elif item['name'].casefold() == term.casefold():
|
|
165
|
+
kind, score = 'symbol_exact', 80
|
|
166
|
+
elif term.casefold() in item['name'].casefold() or term.casefold() in (item.get('qualified_name') or '').casefold():
|
|
167
|
+
kind, score = 'symbol_substring', 60
|
|
168
|
+
else:
|
|
169
|
+
kind, score = 'path_match', 50
|
|
170
|
+
item['match_kind'] = kind
|
|
171
|
+
item['score'] = score
|
|
172
|
+
item['score_contributions'] = [{'kind':kind,'term':term,'score':score,'reason':'literal match against indexed symbol or path'}]
|
|
173
|
+
item['matched_terms'] = [term]
|
|
174
|
+
key = ('symbol', item['id'])
|
|
175
|
+
if key in results:
|
|
176
|
+
results[key]['matched_terms'].append(term)
|
|
177
|
+
results[key]['score'] = max(results[key]['score'], item['score'])
|
|
178
|
+
results[key]['score_contributions'].extend(item['score_contributions'])
|
|
179
|
+
else:
|
|
180
|
+
results[key] = item
|
|
181
|
+
count += 1
|
|
182
|
+
phase_status['search_symbols'] = 'ok' if count else 'no_symbol_match'
|
|
183
|
+
if 'search_content' in phases:
|
|
184
|
+
if 'source_content_chunks' not in tables:
|
|
185
|
+
phase_status['search_content'] = 'capability_unavailable'
|
|
186
|
+
else:
|
|
187
|
+
count = 0
|
|
188
|
+
for term in terms:
|
|
189
|
+
fts = 'source_content_fts' in tables and len(term) >= 3
|
|
190
|
+
match = ' AND c.rowid IN (SELECT rowid FROM source_content_fts WHERE source_content_fts MATCH ?)' if fts else ''
|
|
191
|
+
identifier = ' AND identifier_match(c.content,?)=1' if plan.get('matchKind') == 'identifier' else ''
|
|
192
|
+
params = [*scope_args, term]
|
|
193
|
+
if identifier:
|
|
194
|
+
params.append(term)
|
|
195
|
+
if fts:
|
|
196
|
+
params.append('"' + term.replace('"', '""') + '"')
|
|
197
|
+
rows = conn.execute(f'SELECT c.*,s.codebase,s.parser FROM source_content_chunks c JOIN code_files s ON s.id=c.code_file_id WHERE {where} AND instr(c.content,?)>0{identifier}{match} ORDER BY c.path,c.chunk_no LIMIT ?', (*params,limit + 1)).fetchall()
|
|
198
|
+
truncated |= len(rows) > limit
|
|
199
|
+
for row in rows[:limit]:
|
|
200
|
+
item = dict(row)
|
|
201
|
+
position = re.search(r'(?<!\w)' + re.escape(term) + r'(?!\w)', item['content']).start() if identifier else item['content'].find(term)
|
|
202
|
+
before = item['content'][:position]
|
|
203
|
+
item['line_start'] += before.count('\n')
|
|
204
|
+
item['line_end'] = item['line_start'] + term.count('\n')
|
|
205
|
+
item['matchedText'] = term
|
|
206
|
+
item.pop('content', None)
|
|
207
|
+
owner = conn.execute('''SELECT id FROM symbols
|
|
208
|
+
WHERE codebase=? AND path=? AND line_start<=? AND line_end>=? AND kind<>?
|
|
209
|
+
ORDER BY line_end-line_start,id LIMIT 1''',
|
|
210
|
+
(item['codebase'],item['path'],item['line_start'],item['line_start'],'file')).fetchone()
|
|
211
|
+
if owner:
|
|
212
|
+
item['entity_id'] = owner[0]
|
|
213
|
+
else:
|
|
214
|
+
# File-level content (constants/configuration outside a
|
|
215
|
+
# symbol) still needs a stable snapshot-bound identity.
|
|
216
|
+
item['entity_id'] = item['code_file_id']
|
|
217
|
+
item = _decorate(item, generation, 'content')
|
|
218
|
+
item['match_kind'] = plan.get('matchKind', 'literal')
|
|
219
|
+
item['score'] = 40
|
|
220
|
+
item['score_contributions'] = [{'kind':'content_' + item['match_kind'],'term':term,'score':40,'reason':'literal occurrence in immutable source chunk'}]
|
|
221
|
+
results[('content', item['chunk_id'], term)] = item
|
|
222
|
+
count += 1
|
|
223
|
+
phase_status['search_content'] = 'ok' if count else 'no_content_match'
|
|
224
|
+
ranked = sorted(results.values(), key=lambda x: (-x['score'], x['path'], x['line_start']))
|
|
225
|
+
if 'rank_seeds' in phases:
|
|
226
|
+
seeds.update(item['id'] for item in ranked if item['result_type'] == 'symbol' and item.get('kind') != 'file')
|
|
227
|
+
for item in ranked:
|
|
228
|
+
if item['result_type'] == 'content':
|
|
229
|
+
rows = conn.execute('SELECT id FROM symbols WHERE codebase=? AND path=? AND line_start<=? AND line_end>=? AND kind<>? ORDER BY line_end-line_start,id LIMIT 2', (item['codebase'],item['path'],item['line_start'],item['line_start'],'file')).fetchall()
|
|
230
|
+
seeds.update(row[0] for row in rows)
|
|
231
|
+
# Validate and scope every explicit, domain, and traversal seed against the pinned index.
|
|
232
|
+
valid_seeds = []
|
|
233
|
+
for seed in sorted(seeds):
|
|
234
|
+
if 'impact_change' in phases and 'entity_versions' in tables:
|
|
235
|
+
row = conn.execute(f'SELECT e.logical_id FROM entity_versions e JOIN code_files s ON s.path=e.path AND s.codebase=e.codebase WHERE {where} AND e.logical_id=? AND e.snapshot_id=?', (*scope_args, seed, generation)).fetchone()
|
|
236
|
+
else:
|
|
237
|
+
row = conn.execute(f'SELECT s.id FROM symbols s WHERE {where} AND s.id=?', (*scope_args, seed)).fetchone()
|
|
238
|
+
if row:
|
|
239
|
+
valid_seeds.append(seed)
|
|
240
|
+
truncated |= len(valid_seeds) > 20
|
|
241
|
+
valid_seeds = valid_seeds[:20]
|
|
242
|
+
edges = []
|
|
243
|
+
for phase in ['traverse_relations','impact_change']:
|
|
244
|
+
if phase not in phases:
|
|
245
|
+
continue
|
|
246
|
+
table = 'relations' if phase == 'traverse_relations' else 'semantic_flows'
|
|
247
|
+
if table not in tables:
|
|
248
|
+
phase_status[phase] = 'capability_unavailable'; continue
|
|
249
|
+
if mappings.get('status') == 'ambiguous' and not plan['seedEntityIds']:
|
|
250
|
+
phase_status[phase] = 'ambiguous'; continue
|
|
251
|
+
frontier = list(valid_seeds); visited = set(frontier); seen_edges = set()
|
|
252
|
+
before_count = len(edges)
|
|
253
|
+
for depth in range(1, plan['depth'] + 1):
|
|
254
|
+
next_frontier = []
|
|
255
|
+
for seed in frontier:
|
|
256
|
+
source, target = ('from_id','to_id') if table == 'relations' else ('source_id','target_id')
|
|
257
|
+
direction = plan.get('direction','outgoing')
|
|
258
|
+
condition = f'{source}=?' if direction == 'outgoing' else f'{target}=?' if direction == 'incoming' else f'({source}=? OR {target}=?)'
|
|
259
|
+
args = [seed,seed] if direction == 'both' else [seed]
|
|
260
|
+
if not plan.get('includeHeuristic', False):
|
|
261
|
+
condition += " AND confidence <> 'heuristic'"
|
|
262
|
+
rows = conn.execute(f'SELECT rowid AS _edge_rowid,* FROM {table} WHERE {condition} ORDER BY rowid LIMIT 102', args).fetchall()
|
|
263
|
+
truncated |= len(rows) > 100
|
|
264
|
+
for row in rows[:100]:
|
|
265
|
+
item = dict(row)
|
|
266
|
+
key = (table, item.pop('_edge_rowid'))
|
|
267
|
+
if key in seen_edges:
|
|
268
|
+
continue
|
|
269
|
+
seen_edges.add(key)
|
|
270
|
+
path = item.get('evidence_path') or item.get('source_path')
|
|
271
|
+
if table == 'relations':
|
|
272
|
+
owner = conn.execute('SELECT codebase FROM symbols WHERE id=?', (item[source],)).fetchone()
|
|
273
|
+
else:
|
|
274
|
+
owner = conn.execute('SELECT codebase FROM entity_versions WHERE snapshot_id=? AND logical_id=?', (generation,item[source])).fetchone()
|
|
275
|
+
if not owner:
|
|
276
|
+
continue
|
|
277
|
+
file = conn.execute(f'SELECT s.parser,s.content_hash FROM code_files s WHERE {where} AND s.codebase=? AND s.path=?', (*scope_args,owner[0],path)).fetchone()
|
|
278
|
+
if not file:
|
|
279
|
+
continue
|
|
280
|
+
item['relation_parser'] = item.get('parser') or item.get('extractor') or 'unknown'
|
|
281
|
+
item['source_file_parser'] = file['parser']
|
|
282
|
+
item['content_hash'] = file['content_hash']
|
|
283
|
+
item['parser'] = item['relation_parser']
|
|
284
|
+
item.update(path=path, depth=depth, line_start=item.get('evidence_line') or item.get('line_start'), result_type=phase)
|
|
285
|
+
item = _decorate(item, generation, phase)
|
|
286
|
+
if phase == 'impact_change':
|
|
287
|
+
item['impact_kind'] = plan['changeType']
|
|
288
|
+
edges.append(item)
|
|
289
|
+
node = item[target] if item[source] == seed else item[source]
|
|
290
|
+
if node and node not in visited:
|
|
291
|
+
if table == 'relations':
|
|
292
|
+
scoped_node = conn.execute(f'SELECT s.id FROM symbols s WHERE {where} AND s.id=?', (*scope_args,node)).fetchone()
|
|
293
|
+
else:
|
|
294
|
+
scoped_node = conn.execute(f'SELECT e.logical_id FROM entity_versions e JOIN code_files s ON s.path=e.path AND s.codebase=e.codebase WHERE {where} AND e.logical_id=? AND e.snapshot_id=?', (*scope_args,node,generation)).fetchone()
|
|
295
|
+
if scoped_node:
|
|
296
|
+
visited.add(node); next_frontier.append(node)
|
|
297
|
+
if len(edges) >= limit:
|
|
298
|
+
truncated = True; break
|
|
299
|
+
if len(edges) >= limit:
|
|
300
|
+
break
|
|
301
|
+
frontier = next_frontier
|
|
302
|
+
if not frontier or len(edges) >= limit:
|
|
303
|
+
break
|
|
304
|
+
phase_status[phase] = 'ok' if len(edges) > before_count else 'no_index_evidence'
|
|
305
|
+
if 'verify_evidence' in phases:
|
|
306
|
+
for item in [*ranked, *edges]:
|
|
307
|
+
if 'source_artifacts' in tables:
|
|
308
|
+
verify_span(conn, item)
|
|
309
|
+
else:
|
|
310
|
+
item['evidence_status'] = 'evidence_unavailable'
|
|
311
|
+
raw_combined = [*ranked, *edges]
|
|
312
|
+
combined, by_entity = [], {}
|
|
313
|
+
for item in raw_combined:
|
|
314
|
+
version = item.get('entity_version_id')
|
|
315
|
+
if not version:
|
|
316
|
+
combined.append(item)
|
|
317
|
+
continue
|
|
318
|
+
evidence = {key: item.get(key) for key in ('result_type','match_kind','path','line_start','line_end','matchedText','content_hash','score','evidence_status') if item.get(key) is not None}
|
|
319
|
+
if version not in by_entity:
|
|
320
|
+
item['evidence_sources'] = [evidence]
|
|
321
|
+
by_entity[version] = item
|
|
322
|
+
combined.append(item)
|
|
323
|
+
continue
|
|
324
|
+
primary = by_entity[version]
|
|
325
|
+
primary['evidence_sources'].append(evidence)
|
|
326
|
+
primary['score_contributions'] = [*primary.get('score_contributions', []), *item.get('score_contributions', [])]
|
|
327
|
+
primary['matched_terms'] = list(dict.fromkeys([*primary.get('matched_terms', []), *item.get('matched_terms', [])]))
|
|
328
|
+
if item.get('score', 0) > primary.get('score', 0):
|
|
329
|
+
position = combined.index(primary)
|
|
330
|
+
item['evidence_sources'] = primary['evidence_sources']
|
|
331
|
+
item['score_contributions'] = primary['score_contributions']
|
|
332
|
+
item['matched_terms'] = primary['matched_terms']
|
|
333
|
+
combined[position] = item
|
|
334
|
+
by_entity[version] = item
|
|
335
|
+
combined.sort(key=lambda x: (-x.get('score', 0), x.get('path') or '', x.get('line_start') or 0))
|
|
336
|
+
truncated |= len(combined) > limit
|
|
337
|
+
status = 'ok' if combined else 'no_index_evidence'
|
|
338
|
+
if ranked and all(r['result_type'] == 'content' for r in ranked) and not edges:
|
|
339
|
+
status = 'content_match_only'
|
|
340
|
+
if not combined and len(phase_status) == 1:
|
|
341
|
+
status = next(iter(phase_status.values()))
|
|
342
|
+
if any(value == 'capability_unavailable' for value in phase_status.values()):
|
|
343
|
+
status = 'partial' if combined else 'capability_unavailable'
|
|
344
|
+
if truncated:
|
|
345
|
+
status = 'partial'
|
|
346
|
+
if mappings.get('status') == 'ambiguous':
|
|
347
|
+
status = 'ambiguous'
|
|
348
|
+
if any(item.get('evidence_status') == 'evidence_unavailable' or any(source.get('evidence_status') == 'evidence_unavailable' for source in item.get('evidence_sources', [])) for item in combined[:limit]):
|
|
349
|
+
status = 'evidence_unavailable'
|
|
350
|
+
coverage = {'policy': 'local_only','status': 'not_assessed'}
|
|
351
|
+
if 'impact_change' in phases and 'coverage_metrics' in tables:
|
|
352
|
+
coverage_row = conn.execute('SELECT * FROM coverage_metrics WHERE snapshot_id=?', (generation,)).fetchone()
|
|
353
|
+
if coverage_row:
|
|
354
|
+
coverage = dict(coverage_row)
|
|
355
|
+
coverage['policy'] = 'local_only'
|
|
356
|
+
coverage['status'] = 'partial' if coverage.get('files_parsed', 0) < coverage.get('files_scanned', 0) or coverage.get('dynamic_limitations', 0) else 'complete'
|
|
357
|
+
if coverage['status'] == 'partial' and status == 'ok':
|
|
358
|
+
status = 'partial'
|
|
359
|
+
# Revision consistency outranks result/evidence availability: callers must
|
|
360
|
+
# never mistake code hits for facts from an incompatible mapping snapshot.
|
|
361
|
+
if mappings.get('status') == 'mapping_snapshot_mismatch':
|
|
362
|
+
status = 'mapping_snapshot_mismatch'
|
|
363
|
+
return {'results': combined[:limit], 'mappings': mappings['mappings'], '_meta': {
|
|
364
|
+
'status': status,'snapshot': generation,'mapping_revision': mappings.get('mapping_revision'),
|
|
365
|
+
'requirement_revision': mappings.get('requirement_revision'), 'scope_hash': snapshot.get('workspace', {}).get('scopeHash'),
|
|
366
|
+
'phases': phase_status,'seedEntityIds': valid_seeds,'truncated': truncated,'planSchemaVersion': 1,
|
|
367
|
+
'coverage': coverage, 'resolvedPlan': {**plan,'codeGeneration': generation,'mappingRevision': mappings.get('mapping_revision')}}}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Snapshot-owned source evidence. No query reads the live working tree."""
|
|
2
|
+
import hashlib
|
|
3
|
+
import json
|
|
4
|
+
import sqlite3
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def digest(data):
|
|
9
|
+
return 'sha256:' + hashlib.sha256(data).hexdigest()
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def build_content(conn, root, files, generation, scope):
|
|
13
|
+
conn.executescript('''
|
|
14
|
+
CREATE TABLE snapshot_metadata(key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
15
|
+
CREATE TABLE source_artifacts(content_hash TEXT PRIMARY KEY, content BLOB NOT NULL);
|
|
16
|
+
CREATE TABLE source_content_chunks(
|
|
17
|
+
chunk_id TEXT PRIMARY KEY, code_generation TEXT NOT NULL, code_file_id TEXT NOT NULL,
|
|
18
|
+
path TEXT NOT NULL, language TEXT NOT NULL, chunk_no INTEGER NOT NULL,
|
|
19
|
+
line_start INTEGER NOT NULL, line_end INTEGER NOT NULL, byte_start INTEGER NOT NULL,
|
|
20
|
+
byte_end INTEGER NOT NULL, content_hash TEXT NOT NULL, content TEXT NOT NULL);
|
|
21
|
+
CREATE INDEX source_chunks_file ON source_content_chunks(code_file_id, line_start);
|
|
22
|
+
''')
|
|
23
|
+
mode = 'trigram'
|
|
24
|
+
try:
|
|
25
|
+
conn.execute("CREATE VIRTUAL TABLE source_content_fts USING fts5(content, content='source_content_chunks', content_rowid='rowid', tokenize='trigram')")
|
|
26
|
+
except sqlite3.OperationalError:
|
|
27
|
+
mode = 'bounded_literal_scan'
|
|
28
|
+
for record in files:
|
|
29
|
+
data = (Path(root) / record['path']).read_bytes()
|
|
30
|
+
if digest(data) != record['content_hash']:
|
|
31
|
+
raise ValueError('source changed during snapshot build: ' + record['path'])
|
|
32
|
+
conn.execute('INSERT OR IGNORE INTO source_artifacts VALUES(?,?)', (record['content_hash'], data))
|
|
33
|
+
lines = data.splitlines(keepends=True)
|
|
34
|
+
offset = 0
|
|
35
|
+
for number, start in enumerate(range(0, len(lines), 80)):
|
|
36
|
+
chunk = b''.join(lines[start:start + 80])
|
|
37
|
+
conn.execute('INSERT INTO source_content_chunks VALUES(?,?,?,?,?,?,?,?,?,?,?,?)', (
|
|
38
|
+
generation + ':' + record['id'] + ':' + str(number), generation, record['id'],
|
|
39
|
+
record['path'], record['language'], number, start + 1, min(start + 80, len(lines)),
|
|
40
|
+
offset, offset + len(chunk), record['content_hash'], chunk.decode('utf-8', errors='replace')))
|
|
41
|
+
offset += len(chunk)
|
|
42
|
+
if mode == 'trigram':
|
|
43
|
+
conn.execute("INSERT INTO source_content_fts(source_content_fts) VALUES('rebuild')")
|
|
44
|
+
conn.execute("INSERT INTO source_content_fts(source_content_fts, rank) VALUES('integrity-check', 1)")
|
|
45
|
+
capability = {'version': 1, 'tokenizer': mode, 'shortTerms': 'bounded_literal_scan', 'historicalEvidence': True}
|
|
46
|
+
conn.executemany('INSERT INTO snapshot_metadata VALUES(?,?)', [
|
|
47
|
+
('scanScope', json.dumps(scope, sort_keys=True)), ('contentSearch', json.dumps(capability))])
|
|
48
|
+
conn.commit() # Flush FTS pending segments before fingerprinting its immutable shadow tables.
|
|
49
|
+
conn.execute('INSERT INTO snapshot_metadata VALUES(?,?)', ('contentIntegrity', json.dumps(content_integrity(conn), sort_keys=True)))
|
|
50
|
+
return capability
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def verify_span(conn, item):
|
|
54
|
+
row = conn.execute('SELECT content FROM source_artifacts WHERE content_hash=?', (item.get('content_hash'),)).fetchone()
|
|
55
|
+
if row is None or digest(row[0]) != item.get('content_hash'):
|
|
56
|
+
item['evidence_status'] = 'evidence_unavailable'
|
|
57
|
+
return item
|
|
58
|
+
lines = row[0].decode('utf-8', errors='replace').splitlines()
|
|
59
|
+
start = max(1, item.get('line_start') or item.get('evidence_line') or 1)
|
|
60
|
+
end = min(item.get('line_end') or start, start + 19)
|
|
61
|
+
item['evidence'] = '\n'.join(lines[start - 1:end])[:4000]
|
|
62
|
+
item['evidence_status'] = 'verified'
|
|
63
|
+
return item
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def content_integrity(conn):
|
|
67
|
+
"""Fingerprint the build-time integrity-checked source/FTS projection without writes."""
|
|
68
|
+
tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
|
69
|
+
checks = {}
|
|
70
|
+
for table in ['source_content_chunks', 'source_content_fts_data', 'source_content_fts_idx', 'source_content_fts_docsize', 'source_content_fts_config']:
|
|
71
|
+
if table not in tables:
|
|
72
|
+
continue
|
|
73
|
+
checksum = hashlib.sha256()
|
|
74
|
+
count = 0
|
|
75
|
+
for row in conn.execute(f'SELECT * FROM {table} ORDER BY 1,2'):
|
|
76
|
+
for value in row:
|
|
77
|
+
data = value if isinstance(value, bytes) else str(value).encode('utf-8')
|
|
78
|
+
checksum.update(len(data).to_bytes(8, 'big')); checksum.update(data)
|
|
79
|
+
count += 1
|
|
80
|
+
checks[table] = {'count': count, 'sha256': checksum.hexdigest()}
|
|
81
|
+
rowids = hashlib.sha256()
|
|
82
|
+
for row in conn.execute('SELECT rowid FROM source_content_chunks ORDER BY rowid'):
|
|
83
|
+
rowids.update(str(row[0]).encode() + b'\0')
|
|
84
|
+
checks['chunk_rowids'] = rowids.hexdigest()
|
|
85
|
+
if 'source_content_fts_docsize' in tables:
|
|
86
|
+
invalid = conn.execute('SELECT count(*) FROM source_content_chunks c LEFT JOIN source_content_fts_docsize f ON f.id=c.rowid WHERE f.id IS NULL').fetchone()[0]
|
|
87
|
+
checks['missing_fts_rowids'] = invalid
|
|
88
|
+
return checks
|
|
@@ -4,6 +4,7 @@ from pathlib import Path
|
|
|
4
4
|
|
|
5
5
|
from adapters.capabilities import confidence_for_parser
|
|
6
6
|
from core.schema import json_dumps
|
|
7
|
+
from source_content import build_content
|
|
7
8
|
|
|
8
9
|
|
|
9
10
|
SCHEMA = """
|
|
@@ -195,7 +196,7 @@ CREATE INDEX IF NOT EXISTS idx_relations_external ON relations(to_external);
|
|
|
195
196
|
|
|
196
197
|
|
|
197
198
|
def write_sqlite(path: Path, files, symbols, relations, diagnostics, snapshot_id="pending",
|
|
198
|
-
workspace_state=None, semantic_entities=None, semantic_flows=None, feature_bindings=None) -> None:
|
|
199
|
+
workspace_state=None, semantic_entities=None, semantic_flows=None, feature_bindings=None, source_root=None) -> None:
|
|
199
200
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
200
201
|
if path.exists():
|
|
201
202
|
path.unlink()
|
|
@@ -210,7 +211,7 @@ def write_sqlite(path: Path, files, symbols, relations, diagnostics, snapshot_id
|
|
|
210
211
|
parsed_count = sum(1 for item in files if item.get("path") not in parse_failed_paths)
|
|
211
212
|
conn.execute(
|
|
212
213
|
"INSERT INTO analysis_snapshots(snapshot_id, schema_version, source_revision, content_manifest_hash, extractor_set_hash, created_at, status) VALUES(?,?,?,?,?,?,?)",
|
|
213
|
-
(snapshot_id, 3, workspace_state.get("repositoryRevision", ""), workspace_state.get("contentManifestHash", ""),
|
|
214
|
+
(snapshot_id, 4 if source_root is not None else 3, workspace_state.get("repositoryRevision", ""), workspace_state.get("contentManifestHash", ""),
|
|
214
215
|
workspace_state.get("extractorSetHash", ""), workspace_state.get("generatedAt", ""), "current"),
|
|
215
216
|
)
|
|
216
217
|
# Structural symbols and files are first-class graph nodes as well.
|
|
@@ -304,6 +305,8 @@ def write_sqlite(path: Path, files, symbols, relations, diagnostics, snapshot_id
|
|
|
304
305
|
""",
|
|
305
306
|
[_symbol_row(record) for record in symbols],
|
|
306
307
|
)
|
|
308
|
+
if source_root is not None:
|
|
309
|
+
workspace_state["contentSearch"] = build_content(conn, source_root, files, snapshot_id, workspace_state.get("scanScope", {}))
|
|
307
310
|
conn.commit()
|
|
308
311
|
finally:
|
|
309
312
|
conn.close()
|
package/src/constants.ts
CHANGED
|
@@ -219,7 +219,12 @@ async function buildCodeIndex(context: ProjectContext, config: ScanConfigInput):
|
|
|
219
219
|
const script = path.resolve(import.meta.dir, "../plugins/devflow-intel/skills/workflow-code-index/scripts/code_index.py");
|
|
220
220
|
const output = `.planning/.code-index-pending-${randomUUID()}`;
|
|
221
221
|
const codebases = config.repositories.flatMap(repository => ["--codebase", `${repository.name}:${path.resolve(context.root, repository.relativePath)}:${repository.role}`]);
|
|
222
|
-
|
|
222
|
+
const scopeArgs = [
|
|
223
|
+
...(config.includes ?? ["**/*"]).flatMap(pattern => ["--include", pattern]),
|
|
224
|
+
...(config.excludes ?? [".git/**", ".planning/**", ".devflow/**"]).flatMap(pattern => ["--exclude", pattern]),
|
|
225
|
+
"--max-file-size", String(config.maxFileSize ?? 2 * 1024 * 1024),
|
|
226
|
+
];
|
|
227
|
+
await runProcess(["python3", script, "scan", "--root", context.root, "--output", output, ...scopeArgs, ...codebases], context.root, "code-index scan");
|
|
223
228
|
await runProcess(["python3", script, "validate", "--index", output, "--level", "publish"], context.root, "code-index validate");
|
|
224
229
|
await runProcess(["python3", script, "audit", "--root", context.root, "--index", output, "--skill-root", path.resolve(import.meta.dir, "../plugins/devflow-intel/skills"), "--level", "publish"], context.root, "code-index audit");
|
|
225
230
|
const snapshotPath = path.join(context.root, output, "SNAPSHOT.json");
|