@guanlin4924/devflow-pi 0.1.0

package/plugins/devflow-intel/skills/workflow-code-index/scripts/core/scanner.py

@@ -0,0 +1,115 @@
+import os
+import subprocess
+from pathlib import Path
+from typing import Iterable, List, Sequence
+
+
+DEFAULT_EXCLUDES = {
+    ".git",
+    "node_modules",
+    "vendor",
+    "dist",
+    "build",
+    "coverage",
+    "target",
+    ".next",
+    ".nuxt",
+    ".cache",
+    ".pytest_cache",
+    ".mypy_cache",
+    ".gradle",
+    ".dart_tool",
+    "__pycache__",
+    "Pods",
+}
+
+TEXT_EXTENSIONS = {
+    ".py",
+    ".js",
+    ".jsx",
+    ".ts",
+    ".tsx",
+    ".vue",
+    ".go",
+    ".rs",
+    ".php",
+    ".java",
+    ".kt",
+    ".kts",
+    ".dart",
+    ".c",
+    ".cc",
+    ".cpp",
+    ".h",
+    ".hpp",
+    ".cs",
+    ".rb",
+    ".swift",
+    ".sql",
+    ".sh",
+    ".yaml",
+    ".yml",
+    ".json",
+    ".toml",
+    ".xml",
+    ".md",
+}
+
+
+def scan_files(root: Path, excludes: Iterable[str] = (), max_bytes: int = 1_500_000) -> List[Path]:
+    root = root.resolve()
+    excluded = DEFAULT_EXCLUDES | set(excludes)
+    paths = _scan_with_rg(root, excluded) or _scan_with_walk(root, excluded)
+    result: List[Path] = []
+    for path in sorted(paths):
+        if not path.is_file():
+            continue
+        if _is_excluded(path, root, excluded):
+            continue
+        if path.suffix and path.suffix.lower() not in TEXT_EXTENSIONS:
+            continue
+        try:
+            if path.stat().st_size > max_bytes:
+                continue
+        except OSError:
+            continue
+        result.append(path)
+    return result
+
+
+def _scan_with_rg(root: Path, excluded: Sequence[str]) -> List[Path]:
+    ignore_args = []
+    for name in excluded:
+        ignore_args.extend(["--glob", f"!**/{name}/**"])
+    try:
+        proc = subprocess.run(
+            ["rg", "--files", *ignore_args],
+            cwd=str(root),
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
+            check=False,
+        )
+    except (OSError, FileNotFoundError):
+        return []
+    if proc.returncode not in (0, 1):
+        return []
+    return [root / line.strip() for line in proc.stdout.splitlines() if line.strip()]
+
+
+def _scan_with_walk(root: Path, excluded: Sequence[str]) -> List[Path]:
+    paths: List[Path] = []
+    for current, dirs, files in os.walk(root):
+        dirs[:] = [name for name in dirs if name not in excluded]
+        for filename in files:
+            paths.append(Path(current) / filename)
+    return paths
+
+
+def _is_excluded(path: Path, root: Path, excluded: Sequence[str]) -> bool:
+    try:
+        parts = path.resolve().relative_to(root).parts
+    except ValueError:
+        return True
+    return any(part in excluded for part in parts)
+

package/plugins/devflow-intel/skills/workflow-code-index/scripts/core/schema.py

@@ -0,0 +1,123 @@
+import hashlib
+import json
+import re
+from typing import Dict, Iterable, List
+
+
+RELATION_TYPES = {
+    "contains",
+    "imports",
+    "calls",
+    "extends",
+    "implements",
+    "routes_to",
+    "reads",
+    "writes",
+    "tested_by",
+    "config_uses",
+}
+
+CONFIDENCE = {"exact", "inferred", "heuristic"}
+
+SYMBOL_KINDS = {
+    "function",
+    "method",
+    "class",
+    "struct",
+    "interface",
+    "trait",
+    "route",
+    "file",
+    "table",
+    "config",
+    "test",
+    "component",
+    "module",
+    "variable",
+}
+
+DIAGNOSTIC_SEVERITY = {"info", "warning", "error"}
+
+
+def stable_key(value: str) -> str:
+    value = value.strip().replace("\\", "/")
+    value = re.sub(r"[^A-Za-z0-9_./:$#-]+", ".", value)
+    value = re.sub(r"\.+", ".", value).strip(".")
+    return value or "unknown"
+
+
+def short_hash(value: str, length: int = 12) -> str:
+    return hashlib.sha256(value.encode("utf-8", errors="ignore")).hexdigest()[:length]
+
+
+def json_dumps(value) -> str:
+    return json.dumps(value or {}, ensure_ascii=False, sort_keys=True)
+
+
+def validate_file(record: Dict) -> List[str]:
+    required = ["id", "codebase", "path", "language", "size_bytes", "content_hash", "indexed_at", "parser"]
+    return _missing(record, required)
+
+
+def validate_symbol(record: Dict) -> List[str]:
+    required = [
+        "id",
+        "codebase",
+        "kind",
+        "name",
+        "path",
+        "line_start",
+        "line_end",
+        "language",
+        "content_hash",
+    ]
+    errors = _missing(record, required)
+    if record.get("kind") not in SYMBOL_KINDS:
+        errors.append(f"invalid symbol kind: {record.get('kind')}")
+    if int(record.get("line_start", 0) or 0) < 1:
+        errors.append("line_start must be >= 1")
+    if int(record.get("line_end", 0) or 0) < int(record.get("line_start", 0) or 0):
+        errors.append("line_end must be >= line_start")
+    return errors
+
+
+def validate_relation(record: Dict) -> List[str]:
+    required = ["from_id", "relation_type", "evidence_path", "evidence_line", "confidence", "parser"]
+    errors = _missing(record, required)
+    if not record.get("to_id") and not record.get("to_external"):
+        errors.append("relation requires to_id or to_external")
+    if record.get("relation_type") not in RELATION_TYPES:
+        errors.append(f"invalid relation_type: {record.get('relation_type')}")
+    if record.get("confidence") not in CONFIDENCE:
+        errors.append(f"invalid confidence: {record.get('confidence')}")
+    if int(record.get("evidence_line", 0) or 0) < 1:
+        errors.append("evidence_line must be >= 1")
+    return errors
+
+
+def validate_diagnostic(record: Dict) -> List[str]:
+    required = ["codebase", "severity", "kind", "message"]
+    errors = _missing(record, required)
+    if record.get("severity") not in DIAGNOSTIC_SEVERITY:
+        errors.append(f"invalid diagnostic severity: {record.get('severity')}")
+    return errors
+
+
+def validate_records(kind: str, records: Iterable[Dict]) -> List[str]:
+    validators = {
+        "files": validate_file,
+        "symbols": validate_symbol,
+        "relations": validate_relation,
+        "diagnostics": validate_diagnostic,
+    }
+    validate = validators[kind]
+    errors: List[str] = []
+    for index, record in enumerate(records, start=1):
+        for error in validate(record):
+            errors.append(f"{kind}[{index}]: {error}")
+    return errors
+
+
+def _missing(record: Dict, required: List[str]) -> List[str]:
+    return [f"missing required field: {field}" for field in required if record.get(field) in (None, "")]
+

package/plugins/devflow-intel/skills/workflow-code-index/scripts/validators/__init__.py

@@ -0,0 +1 @@
+

package/plugins/devflow-intel/skills/workflow-code-index/scripts/validators/project_binding_check.py

@@ -0,0 +1,66 @@
+import re
+from pathlib import Path
+
+
+PLACEHOLDER_OK = re.compile(r"\{[A-Za-z0-9_:-]+\}")
+
+GENERIC_TOKENS = (
+    ".planning/",
+    ".context/",
+    ".skills/",
+    "references/",
+    "scripts/",
+    "core/",
+    "adapters/",
+    "writers/",
+    "validators/",
+    "path/to/",
+    "name/path/kind",
+    "tree-sitter-",
+    "lsp-",
+    "framework-",
+)
+
+
+def audit_project_binding(paths):
+    findings = []
+    path_like = re.compile(r"(?<![`{])(?:[A-Za-z0-9_.-]+/){2,}[A-Za-z0-9_.-]+")
+    code_id_like = re.compile(r"\b(?:[a-z]+:)?[A-Za-z0-9_-]+:[A-Za-z0-9_./-]+:[A-Za-z0-9_.:$#-]+")
+    for root in paths:
+        root = Path(root)
+        if not root.exists():
+            continue
+        for path in root.rglob("*"):
+            if not path.is_file() or path.suffix.lower() not in {".md", ".py", ".yaml", ".yml"}:
+                continue
+            text = path.read_text(encoding="utf-8", errors="ignore")
+            for lineno, line in enumerate(text.splitlines(), start=1):
+                if line.startswith("#!") or any(token in line for token in GENERIC_TOKENS):
+                    continue
+                stripped = PLACEHOLDER_OK.sub("", line)
+                if (
+                    path_like.search(stripped)
+                    and ".planning/" not in stripped
+                    and ".context/" not in stripped
+                    and ".skills/" not in stripped
+                ):
+                    findings.append(
+                        {
+                            "severity": "warning",
+                            "kind": "project_binding",
+                            "message": "possible project-specific path in generic skill artifact",
+                            "path": str(path),
+                            "line": lineno,
+                        }
+                    )
+                if code_id_like.search(stripped) and "{codebase}" not in line:
+                    findings.append(
+                        {
+                            "severity": "info",
+                            "kind": "project_binding",
+                            "message": "possible concrete code-index id in generic skill artifact",
+                            "path": str(path),
+                            "line": lineno,
+                        }
+                    )
+    return findings

package/plugins/devflow-intel/skills/workflow-code-index/scripts/validators/relation_check.py

@@ -0,0 +1,36 @@
+def audit_relations(symbols, relations):
+    symbol_ids = {record["id"] for record in symbols}
+    findings = []
+    for index, relation in enumerate(relations, start=1):
+        if relation.get("to_id") and relation["to_id"] not in symbol_ids:
+            findings.append(
+                {
+                    "severity": "error",
+                    "kind": "dangling_relation",
+                    "message": f"relations[{index}] points to missing symbol {relation['to_id']}",
+                    "path": relation.get("evidence_path"),
+                    "line": relation.get("evidence_line"),
+                }
+            )
+        if not relation.get("evidence_path") or not relation.get("evidence_line"):
+            findings.append(
+                {
+                    "severity": "error",
+                    "kind": "missing_evidence",
+                    "message": f"relations[{index}] has no evidence path/line",
+                    "path": relation.get("evidence_path"),
+                    "line": relation.get("evidence_line"),
+                }
+            )
+        if relation.get("confidence") == "exact" and relation.get("to_external", "").startswith("call:"):
+            findings.append(
+                {
+                    "severity": "warning",
+                    "kind": "confidence_mismatch",
+                    "message": f"relations[{index}] marks unresolved call as exact",
+                    "path": relation.get("evidence_path"),
+                    "line": relation.get("evidence_line"),
+                }
+            )
+    return findings
+

package/plugins/devflow-intel/skills/workflow-code-index/scripts/validators/schema_validate.py

@@ -0,0 +1,11 @@
+from core.schema import validate_records
+
+
+def validate_index(files, symbols, relations, diagnostics):
+    errors = []
+    errors.extend(validate_records("files", files))
+    errors.extend(validate_records("symbols", symbols))
+    errors.extend(validate_records("relations", relations))
+    errors.extend(validate_records("diagnostics", diagnostics))
+    return errors
+

package/plugins/devflow-intel/skills/workflow-code-index/scripts/validators/stale_check.py

@@ -0,0 +1,42 @@
+from pathlib import Path
+
+from core.hashing import file_sha256
+
+
+def audit_stale(root: Path, files):
+    findings = []
+    for record in files:
+        path = root / record["path"]
+        if not path.exists():
+            findings.append(
+                {
+                    "severity": "error",
+                    "kind": "stale_index",
+                    "message": f"indexed file no longer exists: {record['path']}",
+                    "path": record["path"],
+                }
+            )
+            continue
+        try:
+            current = file_sha256(path)
+        except OSError as exc:
+            findings.append(
+                {
+                    "severity": "warning",
+                    "kind": "stale_index",
+                    "message": f"could not hash indexed file: {exc}",
+                    "path": record["path"],
+                }
+            )
+            continue
+        if current != record["content_hash"]:
+            findings.append(
+                {
+                    "severity": "error",
+                    "kind": "stale_index",
+                    "message": f"indexed hash differs from source hash: {record['path']}",
+                    "path": record["path"],
+                }
+            )
+    return findings
+

package/plugins/devflow-intel/skills/workflow-code-index/scripts/writers/__init__.py

@@ -0,0 +1 @@
+

package/plugins/devflow-intel/skills/workflow-code-index/scripts/writers/jsonl_writer.py

@@ -0,0 +1,25 @@
+import json
+import os
+from pathlib import Path
+from typing import Iterable
+
+
+def write_jsonl(path: Path, records: Iterable[dict]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as handle:
+        for record in records:
+            handle.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n")
+        handle.flush()
+        os.fsync(handle.fileno())
+
+
+def read_jsonl(path: Path):
+    if not path.exists():
+        return []
+    records = []
+    with path.open("r", encoding="utf-8") as handle:
+        for line in handle:
+            line = line.strip()
+            if line:
+                records.append(json.loads(line))
+    return records

package/plugins/devflow-intel/skills/workflow-code-index/scripts/writers/markdown_writer.py

@@ -0,0 +1,106 @@
+from collections import Counter, defaultdict
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+def write_markdown(output: Path, files, symbols, relations, diagnostics) -> None:
+    output.mkdir(parents=True, exist_ok=True)
+    now = datetime.now(timezone.utc).isoformat()
+    _write_index(output / "INDEX.md", now, files, symbols, relations, diagnostics)
+    _write_symbols(output / "SYMBOLS.md", now, symbols)
+    _write_relations(output / "RELATIONS.md", now, relations)
+    _write_entrypoints(output / "ENTRYPOINTS.md", now, symbols)
+    _write_dataflow(output / "DATAFLOW.md", now, relations)
+
+
+def write_audit(output: Path, audit) -> None:
+    output.mkdir(parents=True, exist_ok=True)
+    lines = ["# Code Index Audit", "", f"- Generated at: {datetime.now(timezone.utc).isoformat()}", ""]
+    lines.append("## Summary")
+    lines.append("")
+    lines.append(f"- Status: {'pass' if not audit else 'fail'}")
+    lines.append(f"- Findings: {len(audit)}")
+    lines.append("")
+    if audit:
+        lines.append("## Findings")
+        lines.append("")
+        for item in audit:
+            lines.append(f"- **{item.get('severity', 'warning')}** `{item.get('kind', 'unknown')}`: {item.get('message', '')}")
+            if item.get("path"):
+                lines.append(f"  - Path: `{item.get('path')}`")
+            if item.get("line"):
+                lines.append(f"  - Line: `{item.get('line')}`")
+    (output / "AUDIT.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def _write_index(path: Path, now: str, files, symbols, relations, diagnostics) -> None:
+    codebases = Counter(record["codebase"] for record in files)
+    languages = Counter(record["language"] for record in files)
+    lines = ["# Code Index", "", f"- Generated at: {now}", "- Index version: 1", ""]
+    lines.extend(["## Coverage", "", "| Codebase | Files |", "|----------|-------|"])
+    for name, count in sorted(codebases.items()):
+        lines.append(f"| `{name}` | {count} |")
+    lines.extend(["", "## Languages", "", "| Language | Files |", "|----------|-------|"])
+    for name, count in sorted(languages.items()):
+        lines.append(f"| `{name}` | {count} |")
+    lines.extend(["", "## Counts", ""])
+    lines.append(f"- Files: {len(files)}")
+    lines.append(f"- Symbols: {len(symbols)}")
+    lines.append(f"- Relations: {len(relations)}")
+    lines.append(f"- Diagnostics: {len(diagnostics)}")
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def _write_symbols(path: Path, now: str, symbols) -> None:
+    grouped = defaultdict(Counter)
+    for symbol in symbols:
+        grouped[symbol["codebase"]][symbol["kind"]] += 1
+    lines = ["# Symbols", "", f"- Generated at: {now}", ""]
+    for codebase in sorted(grouped):
+        lines.extend([f"## `{codebase}`", "", "| Kind | Count |", "|------|-------|"])
+        for kind, count in sorted(grouped[codebase].items()):
+            lines.append(f"| `{kind}` | {count} |")
+        lines.append("")
+    lines.extend(["## Sample", "", "| Symbol | Kind | Path | Lines |", "|--------|------|------|-------|"])
+    for symbol in symbols[:200]:
+        lines.append(
+            f"| `{symbol['name']}` | `{symbol['kind']}` | `{symbol['path']}` | {symbol['line_start']}-{symbol['line_end']} |"
+        )
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def _write_relations(path: Path, now: str, relations) -> None:
+    counts = Counter((record["relation_type"], record["confidence"]) for record in relations)
+    lines = ["# Relations", "", f"- Generated at: {now}", "", "| Relation | Confidence | Count |", "|----------|------------|-------|"]
+    for (relation_type, confidence), count in sorted(counts.items()):
+        lines.append(f"| `{relation_type}` | `{confidence}` | {count} |")
+    lines.extend(["", "## Sample", "", "| Type | From | To | Evidence | Confidence |", "|------|------|----|----------|------------|"])
+    for record in relations[:200]:
+        target = record.get("to_id") or record.get("to_external") or ""
+        lines.append(
+            f"| `{record['relation_type']}` | `{record['from_id']}` | `{target}` | `{record['evidence_path']}:{record['evidence_line']}` | `{record['confidence']}` |"
+        )
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def _write_entrypoints(path: Path, now: str, symbols) -> None:
+    entry_kinds = {"route", "test", "component", "module", "file"}
+    lines = ["# Entrypoints", "", f"- Generated at: {now}", "", "| Symbol | Kind | Path | Lines |", "|--------|------|------|-------|"]
+    for symbol in symbols:
+        if symbol["kind"] in entry_kinds:
+            lines.append(
+                f"| `{symbol['name']}` | `{symbol['kind']}` | `{symbol['path']}` | {symbol['line_start']}-{symbol['line_end']} |"
+            )
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def _write_dataflow(path: Path, now: str, relations) -> None:
+    lines = ["# Dataflow", "", f"- Generated at: {now}", "", "| Type | Target | Evidence | Confidence |", "|------|--------|----------|------------|"]
+    for record in relations:
+        if record["relation_type"] in {"reads", "writes", "config_uses"}:
+            target = record.get("to_id") or record.get("to_external") or ""
+            lines.append(
+                f"| `{record['relation_type']}` | `{target}` | `{record['evidence_path']}:{record['evidence_line']}` | `{record['confidence']}` |"
+            )
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+

package/plugins/devflow-intel/skills/workflow-code-index/scripts/writers/sqlite_writer.py

@@ -0,0 +1,143 @@
+import sqlite3
+import os
+from pathlib import Path
+
+from core.schema import json_dumps
+
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS code_files (
+  id TEXT PRIMARY KEY,
+  codebase TEXT NOT NULL,
+  path TEXT NOT NULL,
+  language TEXT NOT NULL,
+  size_bytes INTEGER NOT NULL,
+  content_hash TEXT NOT NULL,
+  indexed_at TEXT NOT NULL,
+  parser TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS symbols (
+  id TEXT PRIMARY KEY,
+  codebase TEXT NOT NULL,
+  kind TEXT NOT NULL,
+  name TEXT NOT NULL,
+  qualified_name TEXT,
+  container_id TEXT,
+  path TEXT NOT NULL,
+  line_start INTEGER NOT NULL,
+  line_end INTEGER NOT NULL,
+  signature TEXT,
+  language TEXT NOT NULL,
+  content_hash TEXT NOT NULL,
+  visibility TEXT,
+  metadata_json TEXT
+);
+
+CREATE TABLE IF NOT EXISTS relations (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  from_id TEXT NOT NULL,
+  relation_type TEXT NOT NULL,
+  to_id TEXT,
+  to_external TEXT,
+  evidence_path TEXT NOT NULL,
+  evidence_line INTEGER NOT NULL,
+  confidence TEXT NOT NULL,
+  parser TEXT NOT NULL,
+  metadata_json TEXT
+);
+
+CREATE TABLE IF NOT EXISTS diagnostics (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  codebase TEXT NOT NULL,
+  path TEXT,
+  line INTEGER,
+  severity TEXT NOT NULL,
+  kind TEXT NOT NULL,
+  message TEXT NOT NULL,
+  parser TEXT
+);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS symbols_fts USING fts5(
+  id UNINDEXED,
+  codebase UNINDEXED,
+  name,
+  qualified_name,
+  signature,
+  path,
+  metadata_json
+);
+
+CREATE INDEX IF NOT EXISTS idx_symbols_codebase ON symbols(codebase);
+CREATE INDEX IF NOT EXISTS idx_symbols_path ON symbols(path);
+CREATE INDEX IF NOT EXISTS idx_symbols_name ON symbols(name);
+CREATE INDEX IF NOT EXISTS idx_relations_from ON relations(from_id);
+CREATE INDEX IF NOT EXISTS idx_relations_to ON relations(to_id);
+CREATE INDEX IF NOT EXISTS idx_relations_type ON relations(relation_type);
+CREATE INDEX IF NOT EXISTS idx_relations_external ON relations(to_external);
+"""
+
+
+def write_sqlite(path: Path, files, symbols, relations, diagnostics) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    if path.exists():
+        path.unlink()
+    conn = sqlite3.connect(str(path))
+    try:
+        conn.executescript(SCHEMA)
+        conn.executemany(
+            """
+            INSERT INTO code_files(id, codebase, path, language, size_bytes, content_hash, indexed_at, parser)
+            VALUES(:id, :codebase, :path, :language, :size_bytes, :content_hash, :indexed_at, :parser)
+            """,
+            files,
+        )
+        conn.executemany(
+            """
+            INSERT INTO symbols(id, codebase, kind, name, qualified_name, container_id, path, line_start, line_end,
+              signature, language, content_hash, visibility, metadata_json)
+            VALUES(:id, :codebase, :kind, :name, :qualified_name, :container_id, :path, :line_start, :line_end,
+              :signature, :language, :content_hash, :visibility, :metadata_json)
+            """,
+            [_symbol_row(record) for record in symbols],
+        )
+        conn.executemany(
+            """
+            INSERT INTO relations(from_id, relation_type, to_id, to_external, evidence_path, evidence_line,
+              confidence, parser, metadata_json)
+            VALUES(:from_id, :relation_type, :to_id, :to_external, :evidence_path, :evidence_line,
+              :confidence, :parser, :metadata_json)
+            """,
+            [_relation_row(record) for record in relations],
+        )
+        conn.executemany(
+            """
+            INSERT INTO diagnostics(codebase, path, line, severity, kind, message, parser)
+            VALUES(:codebase, :path, :line, :severity, :kind, :message, :parser)
+            """,
+            diagnostics,
+        )
+        conn.executemany(
+            """
+            INSERT INTO symbols_fts(id, codebase, name, qualified_name, signature, path, metadata_json)
+            VALUES(:id, :codebase, :name, :qualified_name, :signature, :path, :metadata_json)
+            """,
+            [_symbol_row(record) for record in symbols],
+        )
+        conn.commit()
+    finally:
+        conn.close()
+    with path.open("rb") as handle:
+        os.fsync(handle.fileno())
+
+
+def _symbol_row(record):
+    row = dict(record)
+    row["metadata_json"] = json_dumps(row.pop("metadata", {}))
+    return row
+
+
+def _relation_row(record):
+    row = dict(record)
+    row["metadata_json"] = json_dumps(row.pop("metadata", {}))
+    return row