@guanlin4924/devflow-pi 0.1.79 → 0.1.80

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@guanlin4924/devflow-pi",
-  "version": "0.1.79",
+  "version": "0.1.80",
   "type": "module",
   "description": "Devflow 专项研发 Pi，基于固定版本 OMP Runtime 组装研发状态机、代码智能、设计、实现与质量门禁。",
   "license": "UNLICENSED",

package/plugins/devflow-intel/extensions/devflow-intel.ts

@@ -63,6 +63,7 @@ export default function devflowIntel(pi: ExtensionAPI) {
       includeHeuristic: z.boolean().default(false),
       limit: z.number().int().min(1).max(100).default(20),
       changedFiles: z.array(z.string()).default([]),
+      validationLevel: z.enum(["fast", "publish", "full"]).default("publish"),
     }),
     approval: (args) => {
       const action = (args as { action?: string }).action;
@@ -78,9 +79,11 @@ export default function devflowIntel(pi: ExtensionAPI) {
         if (params.changedFiles.length > 0) args.push("--changed-files", ...params.changedFiles);
       } else if (params.action === "validate") {
         args.push("--index", ".planning/code-index");
+        args.push("--level", params.validationLevel);
         if (params.snapshot) args.push("--snapshot", params.snapshot);
       } else if (params.action === "audit") {
         args.push("--root", ctx.cwd, "--index", ".planning/code-index", "--skill-root", skillRoot);
+        args.push("--level", params.validationLevel === "fast" ? "publish" : params.validationLevel);
         if (params.snapshot) args.push("--snapshot", params.snapshot);
       } else if (params.action === "query") {
         if (!params.term) {

package/plugins/devflow-intel/skills/workflow-code-index/SKILL.md

@@ -47,6 +47,20 @@ devflow_code_index(action="audit")
 devflow_code_index(action="impact", entity="api:orders", changeType="remove_field")
 ```
 
+Validation is staged so large indexes do not block publication on a full
+sidecar audit:
+
+- `validationLevel="fast"` checks the snapshot bundle and generation metadata.
+- `validationLevel="publish"` (the default) adds SQLite quick-checks and
+  SQL-backed symbol/relation referential checks. It is the initialization
+  publish gate and does not load every JSONL sidecar into memory.
+- `validationLevel="full"` additionally validates every JSONL record and the
+  semantic graph. Run it as an explicit audit when complete verification is
+  required.
+
+The same split is available to `audit`: the default publish audit is bounded;
+`validationLevel="full"` performs stale-file and full relation auditing.
+
 For changed-file indexing:
 
 ```text

package/plugins/devflow-intel/skills/workflow-code-index/scripts/code_index.py

@@ -197,15 +197,22 @@ def main(argv=None) -> int:
     scan.add_argument("--changed-files", nargs="*", default=[], help="Optional changed files for incremental refresh.")
     scan.add_argument("--exclude", action="append", default=[], help="Additional directory names to exclude.")
 
-    validate = sub.add_parser("validate", help="Validate JSONL schema.")
+    validate = sub.add_parser("validate", help="Validate an index in staged levels.")
     validate.add_argument("--index", default=".planning/code-index", help="Index directory.")
     validate.add_argument("--snapshot", help="Immutable generation id.")
+    validate.add_argument(
+        "--level",
+        choices=("fast", "publish", "full"),
+        default="publish",
+        help="fast bundle checks, publish gate checks, or full JSONL/semantic audit",
+    )
 
     audit = sub.add_parser("audit", help="Audit stale files and relation integrity.")
     audit.add_argument("--root", default=".", help="Project root.")
     audit.add_argument("--index", default=".planning/code-index", help="Index directory.")
     audit.add_argument("--skill-root", default=".skills", help="Optional skill root for project-binding checks.")
     audit.add_argument("--snapshot", help="Immutable generation id.")
+    audit.add_argument("--level", choices=("publish", "full"), default="publish")
 
     query = sub.add_parser("query", help="Search symbols using SQLite FTS or LIKE fallback.")
     query.add_argument("--index", default=".planning/code-index", help="Index directory.")
@@ -405,55 +412,114 @@ def command_scan(args) -> int:
         return 0
 
 
-def command_validate(args) -> int:
-    output = Path(args.index).resolve()
-    selected = _selected_generation(output, getattr(args, "snapshot", None))
-    errors = []
+def _validate_publish_gate(selected: Path, snapshot: Dict[str, object]) -> List[str]:
+    """Run bounded checks required before publishing a generation.
+
+    This deliberately uses SQLite for referential checks.  The previous
+    implementation loaded every JSONL sidecar before it could decide whether
+    a snapshot was publishable, which made large projects spend minutes at
+    100% CPU in a validation process.
+    """
+    errors = _validate_snapshot_bundle(selected, snapshot)
+    if errors:
+        return errors
+    index = selected / "index.sqlite"
     try:
-        snapshot = _load_snapshot(selected)
-        errors.extend(_validate_snapshot_bundle(selected, snapshot))
-        files, symbols, relations, diagnostics = _load_tuple(selected)
-        errors.extend(validate_index(files, symbols, relations, diagnostics))
-        conn = sqlite3.connect(str(selected / "index.sqlite"))
+        conn = sqlite3.connect(str(index))
         try:
+            quick = conn.execute("PRAGMA quick_check").fetchone()
+            if not quick or quick[0] != "ok":
+                errors.append(f"sqlite quick_check failed: {quick[0] if quick else 'no result'}")
             snapshot_id = snapshot.get("generation")
             tables = _table_names(conn)
-            if "analysis_snapshots" in tables:
-                row = conn.execute("SELECT snapshot_id FROM analysis_snapshots WHERE snapshot_id=?", (snapshot_id,)).fetchone()
-                if not row:
-                    errors.append(f"snapshot metadata missing generation {snapshot_id}")
-            semantic_tables = {"semantic_flows", "entity_versions", "feature_bindings", "coverage_metrics"}
-            if not semantic_tables.issubset(tables):
-                # Structural-only historical snapshots remain valid.
-                snapshot.setdefault("capabilities", {})["semantic"] = False
-            else:
+            required = {"analysis_snapshots", "code_files", "symbols", "relations", "diagnostics"}
+            missing = sorted(required - tables)
+            if missing:
+                errors.append(f"index missing required tables: {', '.join(missing)}")
+            elif not conn.execute("SELECT 1 FROM analysis_snapshots WHERE snapshot_id=? LIMIT 1", (snapshot_id,)).fetchone():
+                errors.append(f"snapshot metadata missing generation {snapshot_id}")
+
+            if {"relations", "symbols"}.issubset(tables):
+                dangling = conn.execute("""
+                    SELECT COUNT(*) FROM relations r
+                    LEFT JOIN symbols s ON s.id=r.from_id
+                    WHERE s.id IS NULL
+                """).fetchone()[0]
+                if dangling:
+                    errors.append(f"relations have {dangling} missing source symbols")
                 dangling = conn.execute("""
-              SELECT COUNT(*) FROM semantic_flows f
-              WHERE f.snapshot_id=? AND (
-                (NOT EXISTS (SELECT 1 FROM entity_versions e WHERE e.snapshot_id=f.snapshot_id AND e.logical_id=f.source_id)
-                 AND NOT EXISTS (SELECT 1 FROM symbols s WHERE s.id=f.source_id))
-                OR
-                (NOT EXISTS (SELECT 1 FROM entity_versions e WHERE e.snapshot_id=f.snapshot_id AND e.logical_id=f.target_id)
-                 AND NOT EXISTS (SELECT 1 FROM symbols s WHERE s.id=f.target_id))
-              )
-                """, (snapshot_id,)).fetchone()[0]
+                    SELECT COUNT(*) FROM relations r
+                    LEFT JOIN symbols s ON s.id=r.to_id
+                    WHERE r.to_id IS NOT NULL AND s.id IS NULL AND (r.to_external IS NULL OR r.to_external='')
+                """).fetchone()[0]
                 if dangling:
-                    errors.append(f"semantic flows have {dangling} dangling source or target nodes")
-                binding_dangling = conn.execute("""
-              SELECT COUNT(*) FROM feature_bindings b
-              WHERE b.snapshot_id=? AND (
-                NOT EXISTS (SELECT 1 FROM entity_versions e WHERE e.snapshot_id=b.snapshot_id AND e.logical_id=b.feature_id)
-                OR NOT EXISTS (SELECT 1 FROM entity_versions e WHERE e.snapshot_id=b.snapshot_id AND e.logical_id=b.entity_id)
-              )
-                """, (snapshot_id,)).fetchone()[0]
-                if binding_dangling:
-                    errors.append(f"feature bindings have {binding_dangling} dangling feature or entity nodes")
+                    errors.append(f"relations have {dangling} missing target symbols")
         finally:
             conn.close()
+    except (OSError, sqlite3.Error) as exc:
+        errors.append(str(exc))
+    return errors
+
+
+def _validate_semantic_graph(conn: sqlite3.Connection, snapshot_id: str, tables: set[str]) -> List[str]:
+    errors: List[str] = []
+    semantic_tables = {"semantic_flows", "entity_versions", "feature_bindings", "coverage_metrics"}
+    if not semantic_tables.issubset(tables):
+        missing = ", ".join(sorted(semantic_tables - tables))
+        errors.append(f"full validation unavailable: missing semantic tables: {missing}")
+        return errors
+    dangling = conn.execute("""
+      SELECT COUNT(*) FROM semantic_flows f
+      WHERE f.snapshot_id=? AND (
+        (NOT EXISTS (SELECT 1 FROM entity_versions e WHERE e.snapshot_id=f.snapshot_id AND e.logical_id=f.source_id)
+         AND NOT EXISTS (SELECT 1 FROM symbols s WHERE s.id=f.source_id))
+        OR
+        (NOT EXISTS (SELECT 1 FROM entity_versions e WHERE e.snapshot_id=f.snapshot_id AND e.logical_id=f.target_id)
+         AND NOT EXISTS (SELECT 1 FROM symbols s WHERE s.id=f.target_id))
+      )
+    """, (snapshot_id,)).fetchone()[0]
+    if dangling:
+        errors.append(f"semantic flows have {dangling} dangling source or target nodes")
+    binding_dangling = conn.execute("""
+      SELECT COUNT(*) FROM feature_bindings b
+      WHERE b.snapshot_id=? AND (
+        NOT EXISTS (SELECT 1 FROM entity_versions e WHERE e.snapshot_id=b.snapshot_id AND e.logical_id=b.feature_id)
+        OR NOT EXISTS (SELECT 1 FROM entity_versions e WHERE e.snapshot_id=b.snapshot_id AND e.logical_id=b.entity_id)
+      )
+    """, (snapshot_id,)).fetchone()[0]
+    if binding_dangling:
+        errors.append(f"feature bindings have {binding_dangling} dangling feature or entity nodes")
+    return errors
+
+
+def command_validate(args) -> int:
+    output = Path(args.index).resolve()
+    selected = _selected_generation(output, getattr(args, "snapshot", None))
+    level = getattr(args, "level", "publish")
+    errors: List[str] = []
+    try:
+        snapshot = _load_snapshot(selected)
+        print(f"[code-index][validate] level={level} snapshot={snapshot.get('generation', 'unknown')}", file=sys.stderr, flush=True)
+        if level == "fast":
+            errors.extend(_validate_snapshot_bundle(selected, snapshot))
+        else:
+            errors.extend(_validate_publish_gate(selected, snapshot))
+        if level == "full" and not errors:
+            print("[code-index][validate] full JSONL schema validation", file=sys.stderr, flush=True)
+            files, symbols, relations, diagnostics = _load_tuple(selected)
+            errors.extend(validate_index(files, symbols, relations, diagnostics))
+            if not errors:
+                conn = sqlite3.connect(str(selected / "index.sqlite"))
+                try:
+                    errors.extend(_validate_semantic_graph(conn, str(snapshot.get("generation")), _table_names(conn)))
+                finally:
+                    conn.close()
     except (OSError, sqlite3.Error, ValueError) as exc:
         errors.append(str(exc))
     for error in errors:
         print(error, file=sys.stderr)
+    if not errors:
+        print(f"Index validation passed ({level}).")
     return 1 if errors else 0
 def command_query(args) -> int:
     output = Path(args.index).resolve()
@@ -625,21 +691,37 @@ def _selected_generation(output: Path, generation: str | None) -> Path:
     if not generation:
         root_snapshot = _load_snapshot(output)
         current = root_snapshot.get("generation") if root_snapshot else None
-        if current and re.fullmatch(r"[0-9a-f-]{36}", str(current), re.I):
-            selected = output / "snapshots" / str(current)
-            if selected.is_dir():
-                return selected
+        if current is not None:
+            if not re.fullmatch(r"[0-9a-f-]{36}", str(current), re.I):
+                raise ValueError(f"invalid current snapshot generation: {current}")
+            current = str(current)
+            _validate_generation_pointer(output, root_snapshot, current)
+            selected = output / "snapshots" / current
+            if not selected.is_dir():
+                raise ValueError(f"current snapshot directory is missing: {selected}")
+            selected_snapshot = _load_snapshot(selected)
+            _validate_generation_pointer(selected, selected_snapshot, current)
+            return selected
         # Legacy indexes predating immutable snapshots remain readable.
         return output
     if not re.fullmatch(r"[0-9a-f-]{36}", generation, re.I):
         raise ValueError("invalid snapshot generation")
     selected = output / "snapshots" / generation
     snapshot = _load_snapshot(selected)
-    if snapshot.get("generation") != generation:
-        raise ValueError(f"unknown snapshot generation: {generation}")
+    _validate_generation_pointer(selected, snapshot, generation)
     return selected
 
 
+def _validate_generation_pointer(selected: Path, snapshot: Dict[str, object], expected: str) -> None:
+    if snapshot.get("generation") != expected:
+        raise ValueError(f"snapshot generation mismatch: expected={expected}, metadata={snapshot.get('generation')}")
+    expected_path = f"snapshots/{expected}"
+    if snapshot.get("immutablePath") != expected_path:
+        raise ValueError(
+            f"snapshot immutablePath mismatch: expected={expected_path}, metadata={snapshot.get('immutablePath')}"
+        )
+
+
 def _load_snapshot(output: Path) -> Dict[str, object]:
     path = output / SNAPSHOT_FILE
     if not path.exists():
@@ -664,6 +746,11 @@ def _validate_snapshot_bundle(selected: Path, snapshot: Dict[str, object]) -> Li
     generation = snapshot.get("generation")
     if not generation:
         return [f"snapshot metadata missing generation in {selected}"]
+    if selected.parent.name == "snapshots":
+        try:
+            _validate_generation_pointer(selected, snapshot, selected.name)
+        except ValueError as exc:
+            errors.append(str(exc))
     required = [SNAPSHOT_FILE, "index.sqlite", *INDEX_FILES.values()]
     for name in required:
         if not (selected / name).is_file():
@@ -724,10 +811,39 @@ def command_audit(args) -> int:
             for error in bundle_errors:
                 print(error, file=sys.stderr)
             return 1
-        files, symbols, relations, diagnostics = _load_tuple(selected)
         findings = []
-        findings.extend(audit_stale(root, files))
-        findings.extend(audit_relations(symbols, relations))
+        level = getattr(args, "level", "publish")
+        if level == "full":
+            files, symbols, relations, diagnostics = _load_tuple(selected)
+            findings.extend(audit_stale(root, files))
+            findings.extend(audit_relations(symbols, relations))
+        else:
+            # The publish audit is intentionally bounded and SQL-backed.  A
+            # full stale/content audit remains available explicitly via
+            # --level full and should not block initialization publication.
+            try:
+                conn = sqlite3.connect(str(selected / "index.sqlite"))
+                try:
+                    tables = _table_names(conn)
+                    if {"relations", "symbols"}.issubset(tables):
+                        row = conn.execute("""
+                            SELECT COUNT(*) FROM relations r
+                            LEFT JOIN symbols s ON s.id=r.from_id
+                            WHERE s.id IS NULL
+                        """).fetchone()
+                        if row and row[0]:
+                            findings.append({"severity": "error", "kind": "dangling_relation", "message": f"{row[0]} relations have missing source symbols"})
+                        row = conn.execute("""
+                            SELECT COUNT(*) FROM relations r
+                            LEFT JOIN symbols s ON s.id=r.to_id
+                            WHERE r.to_id IS NOT NULL AND s.id IS NULL AND (r.to_external IS NULL OR r.to_external='')
+                        """).fetchone()
+                        if row and row[0]:
+                            findings.append({"severity": "error", "kind": "dangling_relation", "message": f"{row[0]} relations have missing target symbols"})
+                finally:
+                    conn.close()
+            except sqlite3.Error as exc:
+                findings.append({"severity": "error", "kind": "sqlite_error", "message": str(exc)})
         skill_root = Path(args.skill_root)
         if skill_root.exists():
             findings.extend(audit_project_binding([skill_root / "workflow-code-index", skill_root / "code-index-query"]))

package/src/constants.ts

@@ -1,5 +1,5 @@
 export const PRODUCT_NAME = "Devflow Pi";
-export const PRODUCT_VERSION = "0.1.79";
+export const PRODUCT_VERSION = "0.1.80";
 export const PROJECT_SCHEMA_VERSION = 1;
 export const OMP_RUNTIME_VERSION = "17.2.7";
 

package/src/project-initializer.ts

@@ -220,8 +220,8 @@ async function buildCodeIndex(context: ProjectContext, config: ScanConfigInput):
   const output = `.planning/.code-index-pending-${randomUUID()}`;
   const codebases = config.repositories.flatMap(repository => ["--codebase", `${repository.name}:${path.resolve(context.root, repository.relativePath)}:${repository.role}`]);
   await runProcess(["python3", script, "scan", "--root", context.root, "--output", output, ...codebases], context.root, "code-index scan");
-  await runProcess(["python3", script, "validate", "--index", output], context.root, "code-index validate");
-  await runProcess(["python3", script, "audit", "--root", context.root, "--index", output, "--skill-root", path.resolve(import.meta.dir, "../plugins/devflow-intel/skills")], context.root, "code-index audit");
+  await runProcess(["python3", script, "validate", "--index", output, "--level", "publish"], context.root, "code-index validate");
+  await runProcess(["python3", script, "audit", "--root", context.root, "--index", output, "--skill-root", path.resolve(import.meta.dir, "../plugins/devflow-intel/skills"), "--level", "publish"], context.root, "code-index audit");
   const snapshotPath = path.join(context.root, output, "SNAPSHOT.json");
   const bytes = await readFile(snapshotPath); const snapshot = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
   if (!snapshot.generation || !snapshot.immutablePath) throw new Error("Code index did not publish an immutable generation");