@guanlin4924/devflow-pi 0.1.0

package/plugins/devflow-intel/skills/workflow-code-index/scripts/code_index.py

@@ -0,0 +1,437 @@
+#!/usr/bin/env python3
+import argparse
+import contextlib
+import json
+import os
+import shutil
+import sqlite3
+import sys
+import tempfile
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Dict, Iterable, List, Set, Tuple
+
+import fcntl
+
+sys.dont_write_bytecode = True
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+if str(SCRIPT_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPT_DIR))
+
+from adapters.generic_text import language_for_path
+from adapters.registry import AdapterRegistry
+from core.discovery import discover_codebases
+from core.hashing import file_sha256
+from core.scanner import scan_files
+from validators.project_binding_check import audit_project_binding
+from validators.relation_check import audit_relations
+from validators.schema_validate import validate_index
+from validators.stale_check import audit_stale
+from writers.jsonl_writer import read_jsonl, write_jsonl
+from writers.markdown_writer import write_audit, write_markdown
+from writers.sqlite_writer import write_sqlite
+
+
+INDEX_FILES = {
+    "files": "files.jsonl",
+    "symbols": "symbols.jsonl",
+    "relations": "relations.jsonl",
+    "diagnostics": "diagnostics.jsonl",
+}
+
+MARKDOWN_FILES = ["INDEX.md", "SYMBOLS.md", "RELATIONS.md", "ENTRYPOINTS.md", "DATAFLOW.md"]
+SNAPSHOT_FILE = "SNAPSHOT.json"
+
+
+@contextlib.contextmanager
+def _index_lock(output: Path, exclusive: bool):
+    """Serialize index publication and keep readers off a partially published snapshot."""
+    output.mkdir(parents=True, exist_ok=True)
+    lock_path = output / ".index.lock"
+    with lock_path.open("a+") as handle:
+        fcntl.flock(handle.fileno(), fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
+        try:
+            yield
+        finally:
+            fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
+
+
+def _publish_snapshot(staging: Path, output: Path, counts: Dict[str, int]) -> None:
+    generation = str(uuid.uuid4())
+    snapshot = {
+        "schemaVersion": 1,
+        "generation": generation,
+        "generatedAt": datetime.now(timezone.utc).isoformat(),
+        "counts": counts,
+        "artifacts": [*INDEX_FILES.values(), "index.sqlite", *MARKDOWN_FILES],
+    }
+    snapshot_path = staging / SNAPSHOT_FILE
+    snapshot_path.write_text(json.dumps(snapshot, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    with snapshot_path.open("rb") as handle:
+        os.fsync(handle.fileno())
+
+    # Readers in this module hold the shared lock. SNAPSHOT.json is published
+    # last and is therefore the commit marker for external consumers.
+    for name in snapshot["artifacts"]:
+        os.replace(staging / name, output / name)
+    os.replace(snapshot_path, output / SNAPSHOT_FILE)
+    directory_fd = os.open(output, os.O_RDONLY)
+    try:
+        os.fsync(directory_fd)
+    finally:
+        os.close(directory_fd)
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(description="Build and validate a generic code intelligence index.")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    scan = sub.add_parser("scan", help="Build or refresh the code index.")
+    scan.add_argument("--root", default=".", help="Project root. Defaults to current directory.")
+    scan.add_argument("--output", default=".planning/code-index", help="Index output directory.")
+    scan.add_argument("--codebase", action="append", default=[], help="Explicit codebase: name:path[:type]. Can repeat.")
+    scan.add_argument("--changed-files", nargs="*", default=[], help="Optional changed files for incremental refresh.")
+    scan.add_argument("--exclude", action="append", default=[], help="Additional directory names to exclude.")
+
+    validate = sub.add_parser("validate", help="Validate JSONL schema.")
+    validate.add_argument("--index", default=".planning/code-index", help="Index directory.")
+
+    audit = sub.add_parser("audit", help="Audit stale files and relation integrity.")
+    audit.add_argument("--root", default=".", help="Project root.")
+    audit.add_argument("--index", default=".planning/code-index", help="Index directory.")
+    audit.add_argument("--skill-root", default=".skills", help="Optional skill root for project-binding checks.")
+
+    query = sub.add_parser("query", help="Search symbols using SQLite FTS or LIKE fallback.")
+    query.add_argument("--index", default=".planning/code-index", help="Index directory.")
+    query.add_argument("--term", required=True, help="Search term.")
+    query.add_argument("--limit", type=int, default=20)
+
+    args = parser.parse_args(argv)
+    if args.command == "scan":
+        return command_scan(args)
+    if args.command == "validate":
+        return command_validate(args)
+    if args.command == "audit":
+        return command_audit(args)
+    if args.command == "query":
+        return command_query(args)
+    return 2
+
+
+def command_scan(args) -> int:
+    root = Path(args.root).resolve()
+    output = (root / args.output).resolve() if not Path(args.output).is_absolute() else Path(args.output).resolve()
+    with _index_lock(output, exclusive=True):
+        codebases = discover_codebases(root, args.codebase)
+        registry = AdapterRegistry()
+        changed = _normalize_changed_files(root, args.changed_files)
+        existing = _load_existing(output)
+        secondary_changed = _incoming_evidence_files(existing, changed)
+        changed |= secondary_changed
+
+        files, symbols, relations, diagnostics = _existing_after_removal(existing, changed)
+        indexed_paths = {record["path"] for record in files}
+
+        for codebase in codebases:
+            if not codebase.root.exists():
+                diagnostics.append(_diagnostic(codebase.name, "", 1, "warning", "skipped_file", f"codebase root does not exist: {codebase.root}", "discovery@1"))
+                continue
+            all_files = scan_files(codebase.root, excludes=args.exclude)
+            if changed:
+                all_files = [path for path in all_files if _workspace_rel(root, path) in changed]
+            manifests = [path for path in all_files if path.name in {"package.json", "go.mod", "Cargo.toml", "pyproject.toml", "requirements.txt", "composer.json", "pubspec.yaml"}]
+            adapters = registry.adapters_for(codebase, all_files, manifests)
+            adapter = adapters[0]
+            for path in all_files:
+                rel_path = _workspace_rel(root, path)
+                if rel_path in indexed_paths:
+                    continue
+                try:
+                    text = path.read_text(encoding="utf-8", errors="ignore")
+                    content_hash = file_sha256(path)
+                    file_record = {
+                        "id": f"file:{codebase.name}:{rel_path}",
+                        "codebase": codebase.name,
+                        "path": rel_path,
+                        "language": language_for_path(path),
+                        "size_bytes": path.stat().st_size,
+                        "content_hash": content_hash,
+                        "indexed_at": datetime.now(timezone.utc).isoformat(),
+                        "parser": adapter.parser_id,
+                    }
+                    result = adapter.parse_file(codebase, file_record, text)
+                    files.append(file_record)
+                    symbols.extend(result.symbols)
+                    relations.extend(result.relations)
+                    diagnostics.extend(result.diagnostics)
+                except Exception as exc:  # keep indexing useful even when one file fails
+                    diagnostics.append(_diagnostic(codebase.name, rel_path, 1, "error", "parse_failed", str(exc), adapter.parser_id))
+
+        relations = _resolve_relations(symbols, relations)
+        files, symbols, relations, diagnostics = _dedupe(files, symbols, relations, diagnostics)
+        errors = validate_index(files, symbols, relations, diagnostics)
+        if errors:
+            for error in errors[:200]:
+                print(error, file=sys.stderr)
+            print("Index validation failed; the previous snapshot was preserved.", file=sys.stderr)
+            return 1
+
+        staging = Path(tempfile.mkdtemp(prefix=f".{output.name}.build-", dir=str(output.parent)))
+        try:
+            write_jsonl(staging / INDEX_FILES["files"], files)
+            write_jsonl(staging / INDEX_FILES["symbols"], symbols)
+            write_jsonl(staging / INDEX_FILES["relations"], relations)
+            write_jsonl(staging / INDEX_FILES["diagnostics"], diagnostics)
+            write_sqlite(staging / "index.sqlite", files, symbols, relations, diagnostics)
+            write_markdown(staging, files, symbols, relations, diagnostics)
+            staged_errors = validate_index(*_load_tuple(staging))
+            if staged_errors:
+                for error in staged_errors[:200]:
+                    print(error, file=sys.stderr)
+                print("Staged index validation failed; the previous snapshot was preserved.", file=sys.stderr)
+                return 1
+            _publish_snapshot(
+                staging,
+                output,
+                {
+                    "files": len(files),
+                    "symbols": len(symbols),
+                    "relations": len(relations),
+                    "diagnostics": len(diagnostics),
+                },
+            )
+        finally:
+            shutil.rmtree(staging, ignore_errors=True)
+
+        print(f"Indexed {len(files)} files, {len(symbols)} symbols, {len(relations)} relations, {len(diagnostics)} diagnostics.")
+        print(f"Output: {output}")
+        return 0
+
+
+def command_validate(args) -> int:
+    output = Path(args.index).resolve()
+    with _index_lock(output, exclusive=False):
+        files, symbols, relations, diagnostics = _load_tuple(output)
+        errors = validate_index(files, symbols, relations, diagnostics)
+    if errors:
+        for error in errors:
+            print(error)
+        return 1
+    print("Code index schema validation passed.")
+    return 0
+
+
+def command_audit(args) -> int:
+    root = Path(args.root).resolve()
+    output = Path(args.index).resolve()
+    with _index_lock(output, exclusive=True):
+        files, symbols, relations, diagnostics = _load_tuple(output)
+        findings = []
+        findings.extend(audit_stale(root, files))
+        findings.extend(audit_relations(symbols, relations))
+        skill_root = Path(args.skill_root)
+        if skill_root.exists():
+            findings.extend(audit_project_binding([skill_root / "workflow-code-index", skill_root / "code-index-query"]))
+        write_audit(output, findings)
+    if findings:
+        for finding in findings[:200]:
+            location = f" {finding.get('path', '')}:{finding.get('line', '')}".rstrip(":")
+            print(f"{finding.get('severity', 'warning')}: {finding.get('kind', 'unknown')}: {finding.get('message', '')}{location}")
+        return 1
+    print("Code index audit passed.")
+    return 0
+
+
+def command_query(args) -> int:
+    output = Path(args.index).resolve()
+    with _index_lock(output, exclusive=False):
+        index = output / "index.sqlite"
+        if not index.exists():
+            print(f"Missing index: {index}", file=sys.stderr)
+            return 1
+        conn = sqlite3.connect(str(index))
+        conn.row_factory = sqlite3.Row
+        try:
+            try:
+                rows = conn.execute(
+                    """
+                    SELECT id, codebase, kind, name, qualified_name, path, line_start, line_end
+                    FROM symbols_fts
+                    JOIN symbols USING(id)
+                    WHERE symbols_fts MATCH ?
+                    LIMIT ?
+                    """,
+                    (args.term, args.limit),
+                ).fetchall()
+            except sqlite3.OperationalError:
+                like = f"%{args.term}%"
+                rows = conn.execute(
+                    """
+                    SELECT id, codebase, kind, name, qualified_name, path, line_start, line_end
+                    FROM symbols
+                    WHERE name LIKE ? OR qualified_name LIKE ? OR path LIKE ?
+                    LIMIT ?
+                    """,
+                    (like, like, like, args.limit),
+                ).fetchall()
+            for row in rows:
+                print(json.dumps(dict(row), ensure_ascii=False, sort_keys=True))
+        finally:
+            conn.close()
+    return 0
+
+
+def _load_tuple(output: Path):
+    return (
+        read_jsonl(output / INDEX_FILES["files"]),
+        read_jsonl(output / INDEX_FILES["symbols"]),
+        read_jsonl(output / INDEX_FILES["relations"]),
+        read_jsonl(output / INDEX_FILES["diagnostics"]),
+    )
+
+
+def _load_existing(output: Path) -> Dict[str, List[dict]]:
+    return {
+        "files": read_jsonl(output / INDEX_FILES["files"]),
+        "symbols": read_jsonl(output / INDEX_FILES["symbols"]),
+        "relations": read_jsonl(output / INDEX_FILES["relations"]),
+        "diagnostics": read_jsonl(output / INDEX_FILES["diagnostics"]),
+    }
+
+
+def _normalize_changed_files(root: Path, values: Iterable[str]) -> Set[str]:
+    result: Set[str] = set()
+    for value in values:
+        path = Path(value)
+        if not path.is_absolute():
+            path = root / path
+        if path.exists():
+            result.add(_workspace_rel(root, path))
+        else:
+            try:
+                result.add(str(path.resolve().relative_to(root)).replace(os.sep, "/"))
+            except ValueError:
+                result.add(value.replace(os.sep, "/"))
+    return result
+
+
+def _incoming_evidence_files(existing: Dict[str, List[dict]], changed: Set[str]) -> Set[str]:
+    if not changed:
+        return set()
+    changed_symbol_ids = {symbol["id"] for symbol in existing["symbols"] if symbol.get("path") in changed}
+    return {
+        relation["evidence_path"]
+        for relation in existing["relations"]
+        if relation.get("to_id") in changed_symbol_ids and relation.get("evidence_path")
+    }
+
+
+def _existing_after_removal(existing: Dict[str, List[dict]], changed: Set[str]):
+    if not changed:
+        return [], [], [], []
+    changed_symbol_ids = {symbol["id"] for symbol in existing["symbols"] if symbol.get("path") in changed}
+    files = [record for record in existing["files"] if record.get("path") not in changed]
+    symbols = [record for record in existing["symbols"] if record.get("path") not in changed]
+    relations = [
+        record
+        for record in existing["relations"]
+        if record.get("evidence_path") not in changed
+        and record.get("from_id") not in changed_symbol_ids
+        and record.get("to_id") not in changed_symbol_ids
+    ]
+    diagnostics = [record for record in existing["diagnostics"] if record.get("path") not in changed]
+    return files, symbols, relations, diagnostics
+
+
+def _resolve_relations(symbols: List[dict], relations: List[dict]) -> List[dict]:
+    by_name: Dict[Tuple[str, str], List[dict]] = {}
+    by_id = {symbol["id"]: symbol for symbol in symbols}
+    for symbol in symbols:
+        by_name.setdefault((symbol["codebase"], symbol["name"]), []).append(symbol)
+
+    resolved = []
+    for relation in relations:
+        if relation.get("to_id") or not relation.get("to_external", "").startswith("call:"):
+            resolved.append(relation)
+            continue
+        source = by_id.get(relation["from_id"])
+        if not source:
+            resolved.append(relation)
+            continue
+        name = relation["to_external"][5:]
+        candidates = by_name.get((source["codebase"], name), [])
+        if len(candidates) == 1:
+            upgraded = dict(relation)
+            upgraded["to_id"] = candidates[0]["id"]
+            upgraded["confidence"] = "inferred" if upgraded["confidence"] == "heuristic" else upgraded["confidence"]
+            upgraded.setdefault("metadata", {})["resolution"] = "unique_symbol_name_in_codebase"
+            resolved.append(upgraded)
+        else:
+            resolved.append(relation)
+    return resolved
+
+
+def _dedupe(files, symbols, relations, diagnostics):
+    files = _dedupe_by(files, lambda record: record["id"])
+    symbols = _dedupe_by(symbols, lambda record: record["id"])
+    relations = _dedupe_by(
+        relations,
+        lambda record: (
+            record.get("from_id"),
+            record.get("relation_type"),
+            record.get("to_id"),
+            record.get("to_external"),
+            record.get("evidence_path"),
+            record.get("evidence_line"),
+            record.get("confidence"),
+        ),
+    )
+    diagnostics = _dedupe_by(
+        diagnostics,
+        lambda record: (
+            record.get("codebase"),
+            record.get("path"),
+            record.get("line"),
+            record.get("severity"),
+            record.get("kind"),
+            record.get("message"),
+        ),
+    )
+    return files, symbols, relations, diagnostics
+
+
+def _dedupe_by(records, key_fn):
+    seen = set()
+    result = []
+    for record in records:
+        key = key_fn(record)
+        if key in seen:
+            continue
+        seen.add(key)
+        result.append(record)
+    return result
+
+
+def _workspace_rel(root: Path, path: Path) -> str:
+    try:
+        return str(path.resolve().relative_to(root)).replace(os.sep, "/")
+    except ValueError:
+        return os.path.relpath(str(path.resolve()), str(root)).replace(os.sep, "/")
+
+
+def _diagnostic(codebase, path, line, severity, kind, message, parser):
+    return {
+        "codebase": codebase or "workspace",
+        "path": path,
+        "line": line,
+        "severity": severity,
+        "kind": kind,
+        "message": message,
+        "parser": parser,
+    }
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/plugins/devflow-intel/skills/workflow-code-index/scripts/core/__init__.py

@@ -0,0 +1 @@
+

package/plugins/devflow-intel/skills/workflow-code-index/scripts/core/discovery.py

@@ -0,0 +1,139 @@
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable, List, Optional
+
+
+@dataclass
+class Codebase:
+    name: str
+    root: Path
+    type_hint: str = ""
+    stack_hint: str = ""
+    navigation_path: str = ""
+    spec_path: str = ""
+
+
+def discover_codebases(root: Path, explicit: Iterable[str] = ()) -> List[Codebase]:
+    root = root.resolve()
+    explicit_codebases = [_parse_explicit(root, value) for value in explicit]
+    explicit_codebases = [item for item in explicit_codebases if item is not None]
+    if explicit_codebases:
+        return explicit_codebases
+
+    nav_codebases = _from_navigation(root)
+    if nav_codebases:
+        return nav_codebases
+
+    spec_codebases = _from_spec_index(root)
+    if spec_codebases:
+        return spec_codebases
+
+    return [Codebase(name=root.name or "workspace", root=root)]
+
+
+def _parse_explicit(root: Path, value: str) -> Optional[Codebase]:
+    parts = value.split(":", 2)
+    if len(parts) < 2:
+        return None
+    name, raw_path = parts[0].strip(), parts[1].strip()
+    type_hint = parts[2].strip() if len(parts) == 3 else ""
+    path = Path(raw_path)
+    if not path.is_absolute():
+        path = root / path
+    return Codebase(name=name, root=path.resolve(), type_hint=type_hint)
+
+
+def _from_navigation(root: Path) -> List[Codebase]:
+    index = root / ".planning/navigation/INDEX.md"
+    if not index.exists():
+        return []
+    rows = _read_markdown_table(index)
+    result: List[Codebase] = []
+    for row in rows:
+        name = row.get("Name") or row.get("Codebase") or row.get("name")
+        raw_path = row.get("Path") or row.get("Root") or row.get("path")
+        if not name or not raw_path:
+            continue
+        path = _clean_cell(raw_path)
+        codebase_root = Path(path)
+        if not codebase_root.is_absolute():
+            codebase_root = root / codebase_root
+        nav = _clean_cell(row.get("Navigation", ""))
+        result.append(
+            Codebase(
+                name=_clean_cell(name),
+                root=codebase_root.resolve(),
+                type_hint=_clean_cell(row.get("Type", "")),
+                navigation_path=nav,
+            )
+        )
+    return [item for item in result if item.root.exists()]
+
+
+def _from_spec_index(root: Path) -> List[Codebase]:
+    index = root / ".context/spec/index.md"
+    if not index.exists():
+        return []
+    rows = _read_markdown_table(index)
+    result: List[Codebase] = []
+    for row in rows:
+        name = row.get("Codebase") or row.get("Name")
+        nav = row.get("Navigation", "")
+        spec = row.get("Codebase Spec", "")
+        stack = row.get("Stack Spec", "")
+        if not name:
+            continue
+        nav_path = root / _clean_cell(nav)
+        codebase_root = _root_from_navigation_file(root, nav_path)
+        if codebase_root:
+            result.append(
+                Codebase(
+                    name=_clean_cell(name),
+                    root=codebase_root,
+                    stack_hint=_clean_cell(stack),
+                    navigation_path=_clean_cell(nav),
+                    spec_path=_clean_cell(spec),
+                )
+            )
+    return result
+
+
+def _root_from_navigation_file(project_root: Path, path: Path) -> Optional[Path]:
+    if not path.exists():
+        return None
+    text = path.read_text(encoding="utf-8", errors="ignore")
+    match = re.search(r"\|\s*Root\s*\|\s*`?([^`|\n]+)`?\s*\|", text)
+    if not match:
+        return None
+    raw = match.group(1).strip()
+    root = Path(raw)
+    if not root.is_absolute():
+        root = project_root / root
+    return root.resolve() if root.exists() else None
+
+
+def _read_markdown_table(path: Path) -> List[dict]:
+    rows: List[dict] = []
+    header = None
+    for line in path.read_text(encoding="utf-8", errors="ignore").splitlines():
+        stripped = line.strip()
+        if not stripped.startswith("|") or not stripped.endswith("|"):
+            continue
+        cells = [cell.strip() for cell in stripped.strip("|").split("|")]
+        if all(re.match(r"^:?-{3,}:?$", cell) for cell in cells):
+            continue
+        if header is None:
+            header = cells
+            continue
+        if len(cells) == len(header):
+            rows.append(dict(zip(header, cells)))
+    return rows
+
+
+def _clean_cell(value: str) -> str:
+    value = value.strip()
+    if value.startswith("`") and value.endswith("`"):
+        value = value[1:-1]
+    return value.strip()
+

package/plugins/devflow-intel/skills/workflow-code-index/scripts/core/hashing.py

@@ -0,0 +1,11 @@
+import hashlib
+from pathlib import Path
+
+
+def file_sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return "sha256:" + digest.hexdigest()
+