devcouncil 0.3.1 → 0.4.0

package/src/devcouncil/indexing/graph/cache.py

@@ -0,0 +1,329 @@
+"""Extraction cache v5 — symbols, calls, import_details, modules/specs keyed by sha256."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set, cast
+
+from devcouncil.indexing.graph.extract_python import (
+    ExtractedCall,
+    ExtractedImport,
+    ExtractedSymbol,
+    FileExtraction,
+)
+from devcouncil.indexing.graph.extract_ts import extract_file
+from devcouncil.utils.fsio import atomic_write_json
+from devcouncil.utils.json_persist import read_json
+
+logger = logging.getLogger(__name__)
+
+# v1 = Python modules; v2 = + JS/TS specs; v3 = + symbols + calls;
+# v4 = + import_details (named imports / alias maps) for warm/cold edge parity;
+# v5 = + references (non-call name/attribute loads) for callback-aware liveness;
+# v6 = generic-extractor callee-style qualname hints (owner hints poisoned
+#      same-file call resolution with self-loops).
+# v8 = declarator-chain name recovery for C-family definitions (the language
+#      pack returns name=None, dropping every C/C++/ObjC symbol) + grammar
+#      availability folded into the cache identity.
+PARSE_CACHE_VERSION = 8
+
+# Keys preserved across partial updates (RepoMapper modules/specs vs graph symbols).
+_PRESERVE_KEYS = (
+    "symbols",
+    "calls",
+    "all_exports",
+    "reexports",
+    "references",
+    "language",
+    "import_details",
+    "modules",
+    "specs",
+)
+
+
+def cache_path(root: Path) -> Path:
+    return root / ".devcouncil" / "cache" / "repo_map_parse.json"
+
+
+def load_parse_cache(root: Path) -> Dict[str, Dict[str, Any]]:
+    """Load cache. Version mismatch → empty dict (caller rebuilds in place)."""
+    try:
+        data = read_json(cache_path(root))
+        if data.get("version") == PARSE_CACHE_VERSION and isinstance(data.get("files"), dict):
+            return cast(Dict[str, Dict[str, Any]], data["files"])
+    except Exception:
+        pass
+    return {}
+
+
+def save_parse_cache(root: Path, files: Dict[str, Dict[str, Any]]) -> None:
+    try:
+        path = cache_path(root)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        atomic_write_json(path, {"version": PARSE_CACHE_VERSION, "files": files})
+    except Exception:
+        logger.debug("Failed to write graph parse cache", exc_info=True)
+
+
+def merge_parse_cache(
+    root: Path,
+    updates: Dict[str, Dict[str, Any]],
+    managed: Set[str],
+) -> None:
+    """Merge updates for ``managed`` paths; preserve sibling-language / sibling-field keys.
+
+    Paths in ``managed`` but absent from ``updates`` are pruned. When an update
+    omits graph or import fields, existing values from the prior entry are kept
+    so RepoMapper modules/specs passes and graph extract passes share one store.
+    """
+    cache = load_parse_cache(root)
+    merged = {k: v for k, v in cache.items() if k not in managed}
+    for path, entry in updates.items():
+        prev = cache.get(path)
+        if isinstance(prev, dict) and isinstance(entry, dict):
+            for key in _PRESERVE_KEYS:
+                if key not in entry and key in prev:
+                    entry = {**entry, key: prev[key]}
+        merged[path] = entry
+    if merged != cache:
+        save_parse_cache(root, merged)
+
+
+def _symbol_to_dict(s: ExtractedSymbol) -> Dict[str, Any]:
+    return {
+        "kind": s.kind,
+        "name": s.name,
+        "qualname": s.qualname,
+        "line": s.line,
+        "end_line": s.end_line,
+        "bases": list(s.bases),
+        "implements": list(getattr(s, "implements", []) or []),
+        "decorators": list(s.decorators),
+        "exported": s.exported,
+    }
+
+
+def _call_to_dict(c: ExtractedCall) -> Dict[str, Any]:
+    return {
+        "name": c.name,
+        "line": c.line,
+        "receiver": c.receiver,
+        "qualname_hint": c.qualname_hint,
+    }
+
+
+def _import_to_dict(imp: ExtractedImport) -> Dict[str, Any]:
+    return {
+        "module": imp.module,
+        "names": list(imp.names),
+        "alias_map": dict(imp.alias_map),
+    }
+
+
+def _symbol_from_dict(d: Dict[str, Any]) -> ExtractedSymbol:
+    return ExtractedSymbol(
+        kind=str(d.get("kind") or "function"),
+        name=str(d.get("name") or ""),
+        qualname=str(d.get("qualname") or d.get("name") or ""),
+        line=int(d.get("line") or 0),
+        end_line=int(d.get("end_line") or d.get("line") or 0),
+        bases=list(d.get("bases") or []),
+        implements=list(d.get("implements") or []),
+        decorators=list(d.get("decorators") or []),
+        exported=bool(d.get("exported", False)),
+    )
+
+
+def _call_from_dict(d: Dict[str, Any]) -> ExtractedCall:
+    return ExtractedCall(
+        name=str(d.get("name") or ""),
+        line=int(d.get("line") or 0),
+        receiver=str(d.get("receiver") or ""),
+        qualname_hint=str(d.get("qualname_hint") or ""),
+    )
+
+
+def _import_from_dict(d: Dict[str, Any]) -> ExtractedImport:
+    alias_raw = d.get("alias_map") or {}
+    alias_map = {
+        str(k): str(v) for k, v in alias_raw.items()
+    } if isinstance(alias_raw, dict) else {}
+    return ExtractedImport(
+        module=str(d.get("module") or ""),
+        names=[n for n in (d.get("names") or []) if isinstance(n, str)],
+        alias_map=alias_map,
+    )
+
+
+def extraction_to_cache_entry(ext: FileExtraction, digest: str) -> Dict[str, Any]:
+    entry: Dict[str, Any] = {
+        "sha256": digest,
+        "symbols": [_symbol_to_dict(s) for s in ext.symbols],
+        "calls": [_call_to_dict(c) for c in ext.calls],
+        "import_details": [_import_to_dict(i) for i in ext.import_details],
+        "all_exports": list(ext.all_exports),
+        "reexports": list(ext.reexports),
+        "references": list(ext.references),
+        "language": ext.language,
+    }
+    if ext.path.endswith(".py"):
+        entry["modules"] = list(ext.imports)
+    elif ext.path.endswith(".go"):
+        entry["specs"] = list(ext.imports)
+    else:
+        entry["specs"] = list(ext.imports)
+    return entry
+
+
+def extraction_from_cache_entry(path: str, entry: Dict[str, Any]) -> FileExtraction:
+    imports: List[str] = []
+    if isinstance(entry.get("modules"), list):
+        imports = [m for m in entry["modules"] if isinstance(m, str)]
+    elif isinstance(entry.get("specs"), list):
+        imports = [m for m in entry["specs"] if isinstance(m, str)]
+    symbols = [
+        _symbol_from_dict(s)
+        for s in (entry.get("symbols") or [])
+        if isinstance(s, dict)
+    ]
+    calls = [
+        _call_from_dict(c)
+        for c in (entry.get("calls") or [])
+        if isinstance(c, dict)
+    ]
+    import_details = [
+        _import_from_dict(i)
+        for i in (entry.get("import_details") or [])
+        if isinstance(i, dict)
+    ]
+    return FileExtraction(
+        path=path,
+        language=str(entry.get("language") or ""),
+        imports=imports,
+        import_details=import_details,
+        symbols=symbols,
+        calls=calls,
+        all_exports=[x for x in (entry.get("all_exports") or []) if isinstance(x, str)],
+        reexports=[x for x in (entry.get("reexports") or []) if isinstance(x, str)],
+        references=[x for x in (entry.get("references") or []) if isinstance(x, str)],
+    )
+
+
+def extract_cached(
+    root: Path,
+    path: str,
+    *,
+    cache: Optional[Dict[str, Dict[str, Any]]] = None,
+    force: bool = False,
+) -> tuple[FileExtraction, Dict[str, Any]]:
+    """Extract one file, using cache when sha256 matches.
+
+    Returns ``(extraction, cache_entry)``.
+    """
+    cache = cache if cache is not None else load_parse_cache(root)
+    try:
+        raw = (root / path).read_bytes()
+    except OSError:
+        empty = FileExtraction(path=path, language="")
+        return empty, {
+            "sha256": "",
+            "modules": [],
+            "symbols": [],
+            "calls": [],
+            "import_details": [],
+        }
+    digest = hashlib.sha256(raw).hexdigest()
+    grammar = _grammar_identity(path)
+    config_hash = f"parse-cache-v{PARSE_CACHE_VERSION}"
+    try:
+        from devcouncil.codeintel import get_codeintel_service
+
+        store = get_codeintel_service(root).store
+        payload = store.get_extraction(
+            content_hash=digest,
+            language=grammar[0],
+            grammar_version=grammar[1],
+            config_hash=config_hash,
+        )
+        if payload is not None and not force:
+            stored_entry = json.loads(payload.decode("utf-8"))
+            if isinstance(stored_entry, dict):
+                return extraction_from_cache_entry(path, stored_entry), stored_entry
+    except Exception:
+        logger.debug("SQLite extraction-cache read failed", exc_info=True)
+    entry = cache.get(path)
+    if (
+        not force
+        and isinstance(entry, dict)
+        and entry.get("sha256") == digest
+        and isinstance(entry.get("symbols"), list)
+        and isinstance(entry.get("calls"), list)
+        and isinstance(entry.get("import_details"), list)
+    ):
+        return extraction_from_cache_entry(path, entry), entry
+    source = raw.decode("utf-8", errors="replace")
+    try:
+        ext = extract_file(path, source)
+    except Exception:
+        # One pathological file (e.g. RecursionError on absurd nesting) must not
+        # fail every subsequent full build. Index it as opaque; sha256 stays ""
+        # so the failure is never treated as a valid warm cache entry.
+        logger.warning("extraction failed for %s; indexing as opaque file", path, exc_info=True)
+        empty = FileExtraction(path=path, language="")
+        return empty, {
+            "sha256": "",
+            "modules": [],
+            "symbols": [],
+            "calls": [],
+            "import_details": [],
+        }
+    fresh = extraction_to_cache_entry(ext, digest)
+    try:
+        from devcouncil.codeintel import get_codeintel_service
+
+        get_codeintel_service(root).store.put_extraction(
+            content_hash=digest,
+            language=grammar[0],
+            grammar_version=grammar[1],
+            config_hash=config_hash,
+            payload=json.dumps(fresh, sort_keys=True, separators=(",", ":")).encode("utf-8"),
+        )
+    except Exception:
+        logger.debug("SQLite extraction-cache write failed", exc_info=True)
+    return ext, fresh
+
+
+_GRAMMAR_PROBE_CACHE: Dict[str, bool] = {}
+
+
+def _grammar_identity(path: str) -> tuple[str, str]:
+    from devcouncil.codeintel.languages import detect_language
+
+    spec = detect_language(path)
+    language = spec.grammar if spec is not None else Path(path).suffix.lower().lstrip(".")
+    if language == "python":
+        return language, "stdlib-ast"
+    try:
+        import tree_sitter_language_pack as pack
+    except ImportError:
+        return language, "unavailable"
+    version = str(getattr(pack, "__version__", "unknown"))
+    # Per-grammar availability must be part of the identity: installing or
+    # removing the grammar companion changes extraction output without
+    # changing the pack version, and a version-only key would keep serving
+    # pre-grammar (empty/regex) extractions forever.
+    loadable = _GRAMMAR_PROBE_CACHE.get(language)
+    if loadable is None:
+        try:
+            from devcouncil.codeintel.languages.workers import _activate_companion_once
+
+            _activate_companion_once()
+            pack.get_language(language)
+            loadable = True
+        except Exception:
+            loadable = False
+        _GRAMMAR_PROBE_CACHE[language] = loadable
+    return language, f"{version}:{'native' if loadable else 'fallback'}"

package/src/devcouncil/indexing/graph/communities.py

@@ -0,0 +1,28 @@
+"""Community labeling helpers (leaf module — no intel/verifier imports)."""
+
+from __future__ import annotations
+
+from collections import Counter
+
+from devcouncil.indexing.graph.schema import CodeGraph, GraphNode, NodeKind
+
+
+def _is_file_node(n: GraphNode) -> bool:
+    kind = n.kind.value if hasattr(n.kind, "value") else str(n.kind)
+    return kind == NodeKind.FILE.value or ("::" not in n.id and not n.name)
+
+
+def community_label_for_area(graph: CodeGraph, area: str) -> str:
+    """Dominant community label among file nodes under ``area`` (generic subsystems)."""
+    counts: Counter[str] = Counter()
+    prefix = area.replace("\\", "/").rstrip("/") + "/"
+    for n in graph.nodes:
+        if not _is_file_node(n):
+            continue
+        path = (n.path or n.id).replace("\\", "/")
+        if path == area or path.startswith(prefix):
+            if n.community:
+                counts[n.community] += 1
+    if not counts:
+        return ""
+    return counts.most_common(1)[0][0]

package/src/devcouncil/indexing/graph/cypher.py

@@ -0,0 +1,107 @@
+"""Minimal openCypher-style subset over the native in-memory code graph."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+_SUPPORTED_REL = frozenset({
+    "CALLS", "IMPORTS", "CONTAINS", "REFERENCES", "IMPLEMENTS", "EXTENDS", "DECORATES",
+})
+
+_MATCH_RE = re.compile(
+    r"^\s*MATCH\s+\(a(?::(?P<alabel>[\w]+))?\)"
+    r"(?:-\[r:(?P<rel>[\w|]+)\]->\(b(?::(?P<blabel>[\w]+))?\))?"
+    r"(?:\s+WHERE\s+(?P<where>.+?))?"
+    r"\s+RETURN\s+(?P<ret>.+?)(?:\s+LIMIT\s+(?P<limit>\d+))?\s*$",
+    re.IGNORECASE | re.DOTALL,
+)
+
+
+def _parse_where(clause: str) -> Tuple[Optional[str], Optional[str]]:
+    if not clause:
+        return None, None
+    lowered = clause.strip().lower()
+    if "a.name" in lowered and "contains" in lowered:
+        m = re.search(r"contains\s*\(\s*a\.name\s*,\s*['\"]([^'\"]+)['\"]\s*\)", clause, re.I)
+        if m:
+            return m.group(1), None
+    if "b.path" in lowered and "starts with" in lowered:
+        m = re.search(r"starts\s+with\s*\(\s*b\.path\s*,\s*['\"]([^'\"]+)['\"]\s*\)", clause, re.I)
+        if m:
+            return None, m.group(1)
+    return None, None
+
+
+def run_cypher(project_root: Path, query: str, *, default_limit: int = 50) -> Dict[str, Any]:
+    """Execute a supported MATCH … RETURN subset; reject unsupported clauses."""
+    normalized = " ".join(query.split())
+    if re.search(r"\b(CREATE|DELETE|MERGE|SET|REMOVE|DETACH)\b", normalized, re.I):
+        return {"ok": False, "error": "Mutating Cypher clauses are not supported."}
+    m = _MATCH_RE.match(normalized)
+    if not m:
+        return {
+            "ok": False,
+            "error": (
+                "Unsupported Cypher. Supported: "
+                "MATCH (a)-[r:CALLS|IMPORTS|...]->(b) WHERE … RETURN a,b LIMIT n"
+            ),
+        }
+    rels = m.group("rel") or ""
+    rel_types = {r.strip().upper() for r in rels.split("|") if r.strip()}
+    for rel in rel_types:
+        if rel not in _SUPPORTED_REL:
+            return {"ok": False, "error": f"Unsupported relationship type: {rel}"}
+    limit = int(m.group("limit") or default_limit)
+    name_filter, path_prefix = _parse_where(m.group("where") or "")
+
+    from devcouncil.codeintel.query.engine import CodeIntelQueryEngine
+
+    try:
+        graph = CodeIntelQueryEngine(project_root)._graph()
+    except FileNotFoundError:
+        return {"ok": False, "error": "No committed graph generation."}
+
+    nodes_by_id = {n.id: n for n in graph.nodes}
+    rows: List[Dict[str, Any]] = []
+
+    if rel_types:
+        for edge in graph.edges:
+            if edge.kind.upper() not in rel_types:
+                continue
+            a = nodes_by_id.get(edge.source)
+            b = nodes_by_id.get(edge.target)
+            if a is None or b is None:
+                continue
+            if name_filter and name_filter.lower() not in a.name.lower():
+                continue
+            if path_prefix and not b.path.startswith(path_prefix):
+                continue
+            rows.append({
+                "a_id": a.id,
+                "a_name": a.name,
+                "a_path": a.path,
+                "a_kind": a.kind,
+                "rel": edge.kind,
+                "b_id": b.id,
+                "b_name": b.name,
+                "b_path": b.path,
+                "b_kind": b.kind,
+            })
+            if len(rows) >= limit:
+                break
+    else:
+        for node in graph.nodes:
+            if name_filter and name_filter.lower() not in node.name.lower():
+                continue
+            rows.append({
+                "a_id": node.id,
+                "a_name": node.name,
+                "a_path": node.path,
+                "a_kind": node.kind,
+            })
+            if len(rows) >= limit:
+                break
+
+    return {"ok": True, "rows": rows, "count": len(rows)}

package/src/devcouncil/indexing/graph/embeddings.py

@@ -0,0 +1,194 @@
+"""Opt-in local embeddings for semantic graph search (hash fallback)."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import math
+import re
+import sqlite3
+from pathlib import Path
+from typing import Any, Dict, List
+
+logger = logging.getLogger(__name__)
+
+_EMBEDDINGS_TABLE = "symbol_embeddings"
+
+
+def _index_db(project_root: Path) -> Path:
+    return project_root / ".devcouncil" / "codeintel" / "index.sqlite"
+
+
+def _tokenize(text: str) -> List[str]:
+    return [t for t in re.split(r"[^a-zA-Z0-9_]+", text.lower()) if len(t) > 1]
+
+
+def _hash_vector(tokens: List[str], dims: int = 64) -> List[float]:
+    vec = [0.0] * dims
+    for token in tokens:
+        digest = hashlib.sha256(token.encode("utf-8")).digest()
+        for i in range(dims):
+            vec[i] += (digest[i % len(digest)] - 128) / 128.0
+    norm = math.sqrt(sum(v * v for v in vec)) or 1.0
+    return [v / norm for v in vec]
+
+
+def _cosine(a: List[float], b: List[float]) -> float:
+    return sum(x * y for x, y in zip(a, b))
+
+
+def _current_generation(project_root: Path) -> int | None:
+    try:
+        from devcouncil.codeintel import get_codeintel_service
+
+        return get_codeintel_service(project_root).store.current_generation()
+    except Exception:
+        return None
+
+
+def ensure_embeddings_schema(project_root: Path) -> None:
+    db = _index_db(project_root)
+    db.parent.mkdir(parents=True, exist_ok=True)
+    with sqlite3.connect(db) as conn:
+        conn.execute(
+            f"""
+            CREATE TABLE IF NOT EXISTS {_EMBEDDINGS_TABLE} (
+                node_id TEXT NOT NULL,
+                path TEXT NOT NULL,
+                label TEXT NOT NULL,
+                model TEXT NOT NULL,
+                vector_json TEXT NOT NULL,
+                generation_id INTEGER NOT NULL DEFAULT 0,
+                PRIMARY KEY (generation_id, node_id)
+            )
+            """
+        )
+        cols = {
+            str(row[1])
+            for row in conn.execute(f"PRAGMA table_info({_EMBEDDINGS_TABLE})").fetchall()
+        }
+        if "generation_id" not in cols:
+            # Legacy single-generation table: rebuild on next build_embeddings.
+            conn.execute(f"ALTER TABLE {_EMBEDDINGS_TABLE} ADD COLUMN generation_id INTEGER NOT NULL DEFAULT 0")
+        conn.execute(
+            f"CREATE INDEX IF NOT EXISTS idx_{_EMBEDDINGS_TABLE}_gen "
+            f"ON {_EMBEDDINGS_TABLE}(generation_id)"
+        )
+        conn.commit()
+
+
+def embeddings_enabled(project_root: Path) -> bool:
+    try:
+        from devcouncil.app.config import load_config
+
+        return bool(load_config(project_root).indexing.embeddings.enabled)
+    except Exception:
+        return False
+
+
+def build_embeddings(project_root: Path) -> int:
+    """Rebuild symbol embeddings for the committed codeintel generation."""
+    if not embeddings_enabled(project_root):
+        return 0
+    ensure_embeddings_schema(project_root)
+    generation = _current_generation(project_root)
+    if generation is None:
+        return 0
+    from devcouncil.codeintel.query.engine import CodeIntelQueryEngine
+
+    try:
+        graph = CodeIntelQueryEngine(project_root)._graph()
+    except FileNotFoundError:
+        return 0
+    model = "hash-v1"
+    try:
+        from devcouncil.app.config import load_config
+
+        model = load_config(project_root).indexing.embeddings.model_name or model
+    except Exception:
+        pass
+    count = 0
+    try:
+        with sqlite3.connect(_index_db(project_root)) as conn:
+            conn.execute(
+                f"DELETE FROM {_EMBEDDINGS_TABLE} WHERE generation_id=?",
+                (generation,),
+            )
+            # Drop older generations; keep only current + previous.
+            conn.execute(
+                f"DELETE FROM {_EMBEDDINGS_TABLE} WHERE generation_id NOT IN (?, ?)",
+                (generation, max(0, generation - 1)),
+            )
+            for node in graph.nodes:
+                label = f"{node.path} {node.name} {node.kind}"
+                vec = _hash_vector(_tokenize(label))
+                conn.execute(
+                    f"INSERT OR REPLACE INTO {_EMBEDDINGS_TABLE} "
+                    f"(node_id, path, label, model, vector_json, generation_id) "
+                    f"VALUES (?, ?, ?, ?, ?, ?)",
+                    (node.id, node.path, node.name, model, json.dumps(vec), generation),
+                )
+                count += 1
+            conn.commit()
+    except sqlite3.Error:
+        logger.warning("embedding rebuild failed; continuing without embeddings", exc_info=True)
+        return 0
+    return count
+
+
+def semantic_search(
+    project_root: Path,
+    query: str,
+    *,
+    limit: int = 50,
+) -> Dict[str, Any]:
+    """Semantic search over stored embeddings; empty when disabled."""
+    if not embeddings_enabled(project_root):
+        return {"ok": False, "reason": "embeddings disabled", "matches": []}
+    ensure_embeddings_schema(project_root)
+    generation = _current_generation(project_root)
+    qvec = _hash_vector(_tokenize(query))
+    matches: List[Dict[str, Any]] = []
+    scanned = 0
+    max_scan = max(500, limit * 200)
+    try:
+        with sqlite3.connect(_index_db(project_root)) as conn:
+            if generation is None:
+                rows = conn.execute(
+                    f"SELECT node_id, path, label, model, vector_json FROM {_EMBEDDINGS_TABLE}"
+                )
+            else:
+                rows = conn.execute(
+                    f"SELECT node_id, path, label, model, vector_json FROM {_EMBEDDINGS_TABLE} "
+                    f"WHERE generation_id=?",
+                    (generation,),
+                )
+            for node_id, path, label, model, vector_json in rows:
+                scanned += 1
+                if scanned > max_scan:
+                    break
+                try:
+                    vec = json.loads(vector_json)
+                except json.JSONDecodeError:
+                    continue
+                score = _cosine(qvec, vec)
+                matches.append({
+                    "id": node_id,
+                    "path": path,
+                    "label": label,
+                    "score": round(score, 4),
+                    "model": model,
+                })
+    except sqlite3.Error as exc:
+        logger.warning("semantic search failed: %s", exc)
+        return {"ok": False, "reason": f"embeddings store error: {exc}", "matches": []}
+    matches.sort(key=lambda item: item["score"], reverse=True)
+    return {
+        "ok": True,
+        "matches": matches[: max(1, limit)],
+        "scanned": scanned,
+        "truncated": scanned > max_scan,
+        "generation": generation,
+        "backend": "hash-v1",
+    }