@guanlin4924/devflow-pi 0.1.72 → 0.1.74

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@guanlin4924/devflow-pi",
-  "version": "0.1.72",
+  "version": "0.1.74",
   "type": "module",
   "description": "Devflow 专项研发 Pi，基于固定版本 OMP Runtime 组装研发状态机、代码智能、设计、实现与质量门禁。",
   "license": "UNLICENSED",

package/plugins/devflow-intel/skills/workflow-code-index/scripts/code_index.py

@@ -9,6 +9,7 @@ import sqlite3
 import subprocess
 import sys
 import tempfile
+import time
 import uuid
 from datetime import datetime, timezone
 from pathlib import Path
@@ -155,6 +156,29 @@ def _hash_lines(values) -> str:
     return f"uuid5:{digest}"
 
 
+class _Progress:
+    """Emit human-readable progress immediately and retain a machine-readable trail."""
+
+    def __init__(self, output: Path):
+        self.output = output
+        self.output.mkdir(parents=True, exist_ok=True)
+        self.path = output / "scan-progress.jsonl"
+        self.started = time.monotonic()
+
+    def emit(self, phase: str, message: str, **details) -> None:
+        event = {
+            "at": datetime.now(timezone.utc).isoformat(),
+            "elapsedSeconds": round(time.monotonic() - self.started, 3),
+            "phase": phase,
+            "message": message,
+            **details,
+        }
+        with self.path.open("a", encoding="utf-8") as handle:
+            handle.write(json.dumps(event, ensure_ascii=False) + "\n")
+            handle.flush()
+        print(f"[code-index][{phase}] {message}", file=sys.stderr, flush=True)
+
+
 def main(argv=None) -> int:
     parser = argparse.ArgumentParser(description="Build and validate a generic code intelligence index.")
     sub = parser.add_subparsers(dest="command", required=True)
@@ -215,7 +239,10 @@ def main(argv=None) -> int:
 def command_scan(args) -> int:
     root = Path(args.root).resolve()
     output = (root / args.output).resolve() if not Path(args.output).is_absolute() else Path(args.output).resolve()
+    progress = _Progress(output)
+    progress.emit("start", "scan started", root=str(root), output=str(output), codebases=args.codebase)
     with _index_lock(output, exclusive=True):
+        progress.emit("lock", "exclusive index lock acquired")
         codebases = discover_codebases(root, args.codebase)
         registry = AdapterRegistry()
         changed = _normalize_changed_files(root, args.changed_files)
@@ -233,12 +260,15 @@ def command_scan(args) -> int:
             if not codebase.root.exists():
                 diagnostics.append(_diagnostic(codebase.name, "", 1, "warning", "skipped_file", f"codebase root does not exist: {codebase.root}", "discovery@1"))
                 continue
+            progress.emit("discover", f"discovering {codebase.name}", codebase=codebase.name, path=str(codebase.root))
             all_files = scan_files(codebase.root, excludes=args.exclude)
+            progress.emit("discover", f"discovered {len(all_files)} files in {codebase.name}", codebase=codebase.name, files=len(all_files))
             discovered_paths.update(_workspace_rel(root, path) for path in all_files)
             if changed:
                 all_files = [path for path in all_files if _workspace_rel(root, path) in changed]
             manifests = [path for path in all_files if path.name in {"package.json", "go.mod", "Cargo.toml", "pyproject.toml", "requirements.txt", "composer.json", "pubspec.yaml"}]
             adapters = registry.adapters_for(codebase, all_files, manifests)
+            parsed = 0
             for path in all_files:
                 rel_path = _workspace_rel(root, path)
                 if rel_path in indexed_paths:
@@ -266,11 +296,17 @@ def command_scan(args) -> int:
                     diagnostics.extend(result.diagnostics)
                     adapter_entities.extend(getattr(result, "entities", []) or [])
                     adapter_flows.extend(getattr(result, "flows", []) or [])
+                    parsed += 1
+                    if parsed == 1 or parsed % 250 == 0 or parsed == len(all_files):
+                        progress.emit("parse", f"parsed {parsed}/{len(all_files)} files in {codebase.name}", codebase=codebase.name, parsed=parsed, total=len(all_files))
                 except Exception as exc:  # keep indexing useful even when one file fails
                     diagnostics.append(_diagnostic(codebase.name, rel_path, 1, "error", "parse_failed", str(exc), adapter.parser_id))
+                    parsed += 1
 
+        progress.emit("relations", "resolving relations", files=len(files), symbols=len(symbols), relations=len(relations))
         relations = _resolve_relations(symbols, relations)
         files, symbols, relations, diagnostics = _dedupe(files, symbols, relations, diagnostics)
+        progress.emit("validate", "validating extracted facts", files=len(files), symbols=len(symbols), relations=len(relations), diagnostics=len(diagnostics))
         errors = validate_index(files, symbols, relations, diagnostics)
         if errors:
             for error in errors[:200]:
@@ -285,6 +321,7 @@ def command_scan(args) -> int:
             workspace_state["generatedAt"] = datetime.now(timezone.utc).isoformat()
             workspace_state["semanticExtractor"] = SEMANTIC_EXTRACTOR_VERSION
             workspace_state["extractorSetHash"] = _hash_lines([*workspace_state.get("extractors", []), SEMANTIC_EXTRACTOR_VERSION])
+            progress.emit("semantic", "building semantic facts")
             semantic_entities, semantic_flows, feature_bindings = build_semantic_facts(root, files, symbols, relations)
             semantic_entities.extend(adapter_entities)
             semantic_flows.extend(adapter_flows)
@@ -292,6 +329,7 @@ def command_scan(args) -> int:
             write_jsonl(staging / INDEX_FILES["symbols"], symbols)
             write_jsonl(staging / INDEX_FILES["relations"], relations)
             write_jsonl(staging / INDEX_FILES["diagnostics"], diagnostics)
+            progress.emit("sqlite", "writing sqlite index")
             write_sqlite(
                 staging / "index.sqlite", files, symbols, relations, diagnostics,
                 snapshot_id=generation,
@@ -300,6 +338,7 @@ def command_scan(args) -> int:
                 semantic_flows=semantic_flows,
                 feature_bindings=feature_bindings,
             )
+            progress.emit("documents", "writing index projections")
             write_markdown(staging, files, symbols, relations, diagnostics)
             staged_errors = validate_index(*_load_tuple(staging))
             if staged_errors:
@@ -307,6 +346,7 @@ def command_scan(args) -> int:
                     print(error, file=sys.stderr)
                 print("Staged index validation failed; the previous snapshot was preserved.", file=sys.stderr)
                 return 1
+            progress.emit("publish", "publishing immutable snapshot", generation=generation)
             _publish_snapshot(
                 staging,
                 output,
@@ -322,6 +362,7 @@ def command_scan(args) -> int:
         finally:
             shutil.rmtree(staging, ignore_errors=True)
 
+        progress.emit("complete", "scan completed", files=len(files), symbols=len(symbols), relations=len(relations), diagnostics=len(diagnostics))
         print(f"Indexed {len(files)} files, {len(symbols)} symbols, {len(relations)} relations, {len(diagnostics)} diagnostics.")
         print(f"Output: {output}")
         return 0

package/src/constants.ts

@@ -1,5 +1,5 @@
 export const PRODUCT_NAME = "Devflow Pi";
-export const PRODUCT_VERSION = "0.1.72";
+export const PRODUCT_VERSION = "0.1.74";
 export const PROJECT_SCHEMA_VERSION = 1;
 export const OMP_RUNTIME_VERSION = "17.2.7";
 

package/src/initialization-store.ts

@@ -383,12 +383,27 @@ export class InitializationStore {
         return { id: prior.id, leaseToken: prior.leaseToken, reused: true };
       }
     }
+    const active = this.db.query(`SELECT id FROM initialization_runs
+      WHERE workspace_id=? AND config_id=?
+        AND status IN ('preparing','collecting','validating')
+        AND lease_expires_at>? LIMIT 1`).get(input.workspaceId, input.configId, now()) as { id: string } | null;
+    if (active) throw new Error(`Initialization already running for workspace/config: ${active.id}`);
     const id = randomUUID(); const leaseToken = randomUUID(); const timestamp = now();
     const expires = new Date(Date.now() + (input.leaseMs ?? 15 * 60_000)).toISOString();
     this.db.query("INSERT INTO initialization_runs(id,project_id,workspace_id,config_id,input_snapshot_id,request_id,request_hash,status,lease_token,lease_expires_at,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)").run(id, this.context.manifest.projectId, input.workspaceId, input.configId, input.inputSnapshotId, input.requestId ?? null, requestHash, "collecting", leaseToken, expires, timestamp, timestamp);
     return { id, leaseToken, reused: false };
   }
 
+  renewRunLease(runId: string, leaseToken: string, leaseMs = 15 * 60_000): boolean {
+    const timestamp = now();
+    const result = this.db.query(`UPDATE initialization_runs
+      SET lease_expires_at=?,updated_at=?
+      WHERE id=? AND lease_token=? AND status IN ('preparing','collecting','validating')`).run(
+      new Date(Date.now() + leaseMs).toISOString(), timestamp, runId, leaseToken,
+    );
+    return result.changes === 1;
+  }
+
   private assertWritableRun(runId: string, leaseToken: string): { status: RunStatus; inputSnapshotId: string; workspaceId: string; configId: string } {
     const row = this.db.query("SELECT status,input_snapshot_id AS inputSnapshotId,workspace_id AS workspaceId,config_id AS configId,lease_token AS leaseToken,lease_expires_at AS expires FROM initialization_runs WHERE id=?").get(runId) as ({ status: RunStatus; inputSnapshotId: string; workspaceId: string; configId: string; leaseToken: string; expires: string }) | null;
     if (!row) throw new Error(`Unknown initialization run: ${runId}`);
@@ -402,11 +417,21 @@ export class InitializationStore {
     this.assertWritableRun(runId, leaseToken);
     const transaction = this.db.transaction(() => {
       const repository = input.repository ?? "";
-      this.db.query("DELETE FROM evidence WHERE fact_id IN (SELECT id FROM facts WHERE run_id=? AND repository=? AND kind=?)").run(runId, repository, input.kind);
-      this.db.query("DELETE FROM facts WHERE run_id=? AND repository=? AND kind=?").run(runId, repository, input.kind);
+      if (repository) {
+        this.db.query("DELETE FROM evidence WHERE fact_id IN (SELECT id FROM facts WHERE run_id=? AND repository=? AND kind=?)").run(runId, repository, input.kind);
+        this.db.query("DELETE FROM facts WHERE run_id=? AND repository=? AND kind=?").run(runId, repository, input.kind);
+      } else {
+        this.db.query("DELETE FROM evidence WHERE fact_id IN (SELECT id FROM facts WHERE run_id=? AND kind=?)").run(runId, input.kind);
+        this.db.query("DELETE FROM facts WHERE run_id=? AND kind=?").run(runId, input.kind);
+      }
       this.db.query("INSERT INTO collection_results(id,run_id,repository,kind,status,adapter,capability_json,diagnostics_json,created_at) VALUES (?,?,?,?,?,?,?,?,?) ON CONFLICT(run_id,repository,kind) DO UPDATE SET status=excluded.status,adapter=excluded.adapter,capability_json=excluded.capability_json,diagnostics_json=excluded.diagnostics_json,created_at=excluded.created_at").run(randomUUID(), runId, repository, input.kind, input.status, input.adapter, canonicalJson(input.capability ?? {}), canonicalJson(input.diagnostics ?? []), now());
-      for (const fact of input.facts ?? []) {
+      const uniqueFacts = new Map<string, FactInput>();
+      for (const fact of input.facts ?? []) uniqueFacts.set(`${fact.repository ?? repository}\0${fact.objectKey}`, fact);
+      for (const fact of uniqueFacts.values()) {
         if (fact.kind !== input.kind) throw new Error(`Fact kind ${fact.kind} does not match collection ${input.kind}`);
+        if (repository && fact.repository !== undefined && fact.repository !== repository) {
+          throw new Error(`Fact repository ${fact.repository} does not match collection repository ${repository}`);
+        }
         if (!fact.objectKey || !fact.extractor) throw new Error("Facts require objectKey and extractor");
         if (fact.evidenceLevel === "observed" && !fact.sourcePath) throw new Error("Observed facts require source evidence");
         validateFactShape(fact);

package/src/project-initializer.ts

@@ -165,12 +165,52 @@ function collect(files: SourceFile[], context: ProjectContext, config: ScanConfi
     facts.get("evidence")!.push({ kind: "evidence", repository: repository.name, objectKey: "input-policy", value: { source: "local-static", discoveredCommandsExecuted: false, gitMetadataRead: true, secretsStored: false }, evidenceLevel: "explicit_rule", extractor: "initializer@1", confidence: "exact" });
     facts.get("capability")!.push({ kind: "capability", repository: repository.name, objectKey: "collector:static@1", value: { supported: ["asset", "technology", "api", "page", "data_model", "integration", "rule", "toolchain"], limitations: ["static declarations only", "dynamic behavior unresolved", "no environment enablement inference"], project: context.manifest.name }, evidenceLevel: "explicit_rule", extractor: "initializer@1", confidence: "exact" });
   }
+  // A repository can declare the same route/model/page through more than one
+  // scanner (or declare it twice). The control-plane key is intentionally
+  // unique per run/kind/repository/object_key, so collapse duplicates before
+  // handing facts to the store while retaining the richest value/evidence.
+  for (const [kind, entries] of facts) {
+    const deduped = new Map<string, FactInput>();
+    for (const entry of entries) {
+      const key = `${entry.repository ?? ""}\0${entry.objectKey}`;
+      const previous = deduped.get(key);
+      if (!previous) {
+        deduped.set(key, entry);
+        continue;
+      }
+      deduped.set(key, {
+        ...previous,
+        ...entry,
+        repository: entry.repository ?? previous.repository,
+        value: { ...(previous.value ?? {}), ...(entry.value ?? {}) },
+        sourcePath: entry.sourcePath ?? previous.sourcePath,
+        sourceLine: entry.sourceLine ?? previous.sourceLine,
+        sourceFingerprint: entry.sourceFingerprint ?? previous.sourceFingerprint,
+      });
+    }
+    facts.set(kind, [...deduped.values()]);
+  }
   return facts;
 }
 
-async function runProcess(argv: string[], cwd: string): Promise<{ stdout: string; stderr: string }> {
+async function runProcess(argv: string[], cwd: string, label = path.basename(argv[1] ?? argv[0])): Promise<{ stdout: string; stderr: string }> {
   const child = Bun.spawn(argv, { cwd, stdout: "pipe", stderr: "pipe" });
-  const [exitCode, stdout, stderr] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]);
+  const readStream = async (stream: ReadableStream<Uint8Array>, emit?: (line: string) => void): Promise<string> => {
+    const reader = stream.getReader(); const decoder = new TextDecoder(); let text = ""; const chunks: string[] = [];
+    while (true) {
+      const chunk = await reader.read(); if (chunk.done) break;
+      const decoded = decoder.decode(chunk.value, { stream: true }); chunks.push(decoded); text += decoded;
+      const lines = text.split(/\r?\n/); text = lines.pop() ?? "";
+      for (const line of lines) if (line) emit?.(line);
+    }
+    const tail = decoder.decode(); chunks.push(tail); text += tail; if (text) emit?.(text);
+    return chunks.join("");
+  };
+  const [exitCode, stdout, stderr] = await Promise.all([
+    child.exited,
+    readStream(child.stdout),
+    readStream(child.stderr, line => console.error(`[${label}] ${line}`)),
+  ]);
   if (exitCode !== 0) throw new Error(`${path.basename(argv[1] ?? argv[0])} failed (${exitCode}): ${(stderr || stdout).trim()}`);
   return { stdout: stdout.trim(), stderr: stderr.trim() };
 }
@@ -179,9 +219,9 @@ async function buildCodeIndex(context: ProjectContext, config: ScanConfigInput):
   const script = path.resolve(import.meta.dir, "../plugins/devflow-intel/skills/workflow-code-index/scripts/code_index.py");
   const output = `.planning/.code-index-pending-${randomUUID()}`;
   const codebases = config.repositories.flatMap(repository => ["--codebase", `${repository.name}:${path.resolve(context.root, repository.relativePath)}:${repository.role}`]);
-  await runProcess(["python3", script, "scan", "--root", context.root, "--output", output, ...codebases], context.root);
-  await runProcess(["python3", script, "validate", "--index", output], context.root);
-  await runProcess(["python3", script, "audit", "--root", context.root, "--index", output, "--skill-root", path.resolve(import.meta.dir, "../plugins/devflow-intel/skills")], context.root);
+  await runProcess(["python3", script, "scan", "--root", context.root, "--output", output, ...codebases], context.root, "code-index scan");
+  await runProcess(["python3", script, "validate", "--index", output], context.root, "code-index validate");
+  await runProcess(["python3", script, "audit", "--root", context.root, "--index", output, "--skill-root", path.resolve(import.meta.dir, "../plugins/devflow-intel/skills")], context.root, "code-index audit");
   const snapshotPath = path.join(context.root, output, "SNAPSHOT.json");
   const bytes = await readFile(snapshotPath); const snapshot = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
   if (!snapshot.generation || !snapshot.immutablePath) throw new Error("Code index did not publish an immutable generation");
@@ -214,6 +254,7 @@ export async function runProjectInitialization(context: ProjectContext, input: {
   const config: ScanConfigInput = input.config ?? { mode: "full", repositories: [{ name: context.manifest.name, relativePath: ".", role: "other" }], requiredKinds: [...FACT_KINDS], adapterVersions: { static: "1" } };
   const store = await InitializationStore.open(context); const diagnostics: string[] = [];
   let activeRun: { id: string; leaseToken: string } | undefined;
+  let leaseHeartbeat: ReturnType<typeof setInterval> | undefined;
   try {
     const workspace = await store.registerWorkspace(); const storedConfig = store.putScanConfig(config);
     const initialScan = await scan(context, config); const files = initialScan.files;
@@ -230,6 +271,13 @@ export async function runProjectInitialization(context: ProjectContext, input: {
       workspaceId: context.workspaceId,
       planningRoot: context.planningRoot,
     };
+    leaseHeartbeat = setInterval(() => {
+      try {
+        if (!store.renewRunLease(run.id, run.leaseToken)) diagnostics.push("Initialization lease heartbeat could not be renewed");
+      } catch (error) {
+        diagnostics.push(`Initialization lease heartbeat failed: ${error instanceof Error ? error.message : String(error)}`);
+      }
+    }, 60_000);
     const grouped = collect(files, context, config); const required = new Set(config.requiredKinds ?? FACT_KINDS);
     for (const repository of config.repositories) for (const kind of FACT_KINDS) {
       const repositoryFacts = grouped.get(kind)!.filter(fact => fact.repository === repository.name);
@@ -306,5 +354,8 @@ export async function runProjectInitialization(context: ProjectContext, input: {
     diagnostics.push(message);
     if (activeRun) store.failRun(activeRun.id, activeRun.leaseToken, message);
     throw error;
-  } finally { store.close(); }
+  } finally {
+    if (leaseHeartbeat) clearInterval(leaseHeartbeat);
+    store.close();
+  }
 }