devcouncil 0.3.0 → 0.4.0

package/src/devcouncil/indexing/semantic_index.py

@@ -1,4 +1,8 @@
-"""Semantic snapshots and diff classification."""
+"""Semantic snapshots and diff classification.
+
+NOTE (Phase 4 legacy audit): task-scoped semantic diffs for verification — distinct
+from the code symbol graph in ``indexing/graph/``. Keep; not overlapping dead-code.
+"""
 
 from __future__ import annotations
 
@@ -11,8 +15,10 @@ from pathlib import Path
 
 from devcouncil.indexing.ast_matcher import AstMatcher
 from devcouncil.indexing.lsp import LspInspector
+from devcouncil.indexing.walk import iter_project_files
 from devcouncil.storage.db import get_db
 from devcouncil.storage.native import SemanticDiffRepository
+from devcouncil.utils.json_persist import read_json, write_json
 
 _CONFIG_FILES = {
     "pyproject.toml",
@@ -39,7 +45,7 @@ class SemanticIndex:
         # matching, source-file hashing, import extraction, and LSP language detection);
         # each collector now filters this single in-memory list with its own predicate, so
         # the on-disk output is unchanged while the filesystem is walked a single time.
-        all_files = [path for path in self.project_root.rglob("*") if path.is_file()]
+        all_files = list(iter_project_files(self.project_root))
         rel_files = [str(path.relative_to(self.project_root)) for path in all_files]
         symbols = self._collect_symbols(all_files)
         source_files, imports = self._collect_source_data(all_files)
@@ -56,7 +62,7 @@ class SemanticIndex:
         }
         path = self.snapshot_path(task_id, stage)
         path.parent.mkdir(parents=True, exist_ok=True)
-        path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+        write_json(path, payload)
         return path
 
     def diff(self, task_id: str) -> dict:
@@ -64,8 +70,8 @@ class SemanticIndex:
         after_path = self.snapshot_path(task_id, "after")
         if not after_path.exists():
             self.create_snapshot(task_id, "after")
-        before = json.loads(before_path.read_text(encoding="utf-8")) if before_path.exists() else {}
-        after = json.loads(after_path.read_text(encoding="utf-8"))
+        before = read_json(before_path) if before_path.exists() else {}
+        after = read_json(after_path) if after_path.exists() else {}
         classifications = self._classify(before, after)
         summary = ", ".join(item["type"] for item in classifications) or "no semantic changes"
         db = get_db(self.project_root)
@@ -107,7 +113,7 @@ class SemanticIndex:
         exactly — source entries are still sorted by path, imports stay in traversal order.
         """
         if files is None:
-            files = [p for p in self.project_root.rglob("*") if p.is_file()]
+            files = list(iter_project_files(self.project_root))
         source_entries: list[dict] = []
         imports: list[dict] = []
         for path in files:

package/src/devcouncil/indexing/subsystem_map.py

@@ -0,0 +1,163 @@
+"""Pure helpers for reasoning about subsystem areas from a loaded ``repo_map.json``.
+
+Shared by the prompt builder's impact block and the subsystem-boundary verification
+gate so both agree on how a path maps to an area, which area is which area's neighbor,
+and what a change "touches". Dependency-free and side-effect-free: everything takes an
+already-parsed repo-map dict (or its ``subsystems``/``dependents`` slices) and returns
+plain data. Never raises on malformed input — a missing/odd map degrades to "unknown".
+"""
+
+from __future__ import annotations
+
+from typing import Iterable, Mapping, Sequence
+
+
+def _norm(path: str) -> str:
+    return str(path).replace("\\", "/")
+
+
+def subsystem_areas(data: Mapping | None) -> list[str]:
+    """All subsystem ``area`` prefixes declared in the map (longest first)."""
+    subs = (data or {}).get("subsystems") or []
+    areas = [str(s.get("area")) for s in subs if isinstance(s, dict) and s.get("area")]
+    return sorted(set(areas), key=len, reverse=True)
+
+
+def area_for_path(path: str, data: Mapping | None) -> str | None:
+    """The subsystem area a path belongs to (longest matching area prefix), or ``None``.
+
+    Falls back to the per-file ``area`` recorded in the map's ``files`` list when no
+    subsystem prefix matches, so files outside a declared subsystem still resolve.
+    """
+    norm = _norm(path)
+    for area in subsystem_areas(data):
+        if norm == area or norm.startswith(area + "/"):
+            return area
+    files = (data or {}).get("files") or []
+    for entry in files:
+        if isinstance(entry, dict) and _norm(entry.get("path", "")) == norm:
+            file_area = entry.get("area")
+            return str(file_area) if file_area else None
+    return None
+
+
+def neighbors_for_area(area: str | None, data: Mapping | None) -> list[str]:
+    """Declared neighboring areas for ``area`` (empty when unknown)."""
+    if not area:
+        return []
+    for sub in (data or {}).get("subsystems") or []:
+        if isinstance(sub, dict) and str(sub.get("area")) == area:
+            return [str(n) for n in (sub.get("neighbors") or [])]
+    return []
+
+
+def are_neighbors(area_a: str | None, area_b: str | None, data: Mapping | None) -> bool:
+    """True when the two areas are the same or declared neighbors (in either direction)."""
+    if not area_a or not area_b:
+        return True  # unknown side → don't flag
+    if area_a == area_b:
+        return True
+    if area_b in neighbors_for_area(area_a, data):
+        return True
+    if area_a in neighbors_for_area(area_b, data):
+        return True
+    return False
+
+
+def dependents_of(path: str, data: Mapping | None) -> list[str]:
+    """Files that import ``path`` (reverse-import blast radius), from the map.
+
+    The list may be truncated by the mapper's ``_DEPENDENTS_MAX``; see
+    :func:`dependents_total_of` for the full importer count when truncated.
+    """
+    dependents = (data or {}).get("dependents") or {}
+    if not isinstance(dependents, dict):
+        return []
+    return [str(p) for p in (dependents.get(_norm(path)) or [])]
+
+
+def dependents_total_of(path: str, data: Mapping | None) -> int | None:
+    """Full importer count when ``dependents`` was truncated; else ``None``.
+
+    When this returns an int greater than ``len(dependents_of(...))``, the listed
+    dependents are a sample — not the complete blast radius.
+    """
+    totals = (data or {}).get("dependents_total") or {}
+    if not isinstance(totals, dict):
+        return None
+    raw = totals.get(_norm(path))
+    if raw is None:
+        return None
+    try:
+        n = int(raw)
+    except (TypeError, ValueError):
+        return None
+    return n if n > 0 else None
+
+
+def unwired_candidates_of(data: Mapping | None) -> list[str]:
+    """Files with zero inbound edges that aren't entry roots/exempt (capped list)."""
+    vals = (data or {}).get("unwired_candidates") or []
+    if not isinstance(vals, list):
+        return []
+    return [str(p) for p in vals]
+
+
+def unreachable_of(data: Mapping | None) -> list[str]:
+    """Files not reachable by BFS from any entry root (capped list)."""
+    vals = (data or {}).get("unreachable_files") or []
+    if not isinstance(vals, list):
+        return []
+    return [str(p) for p in vals]
+
+
+def dead_symbol_candidates_of(data: Mapping | None) -> list[str]:
+    """``path:line name`` entries for unused public top-level symbols (capped)."""
+    vals = (data or {}).get("dead_symbol_candidates") or []
+    if not isinstance(vals, list):
+        return []
+    return [str(p) for p in vals]
+
+
+def is_entry_root(path: str, data: Mapping | None) -> bool:
+    """True when ``path`` is listed in the map's ``entry_roots``."""
+    roots = (data or {}).get("entry_roots") or []
+    if not isinstance(roots, list):
+        return False
+    norm = _norm(path)
+    return any(_norm(str(r)) == norm for r in roots)
+
+
+def areas_touched(paths: Iterable[str], data: Mapping | None) -> list[str]:
+    """The distinct subsystem areas the given paths live in (sorted)."""
+    seen: set[str] = set()
+    for path in paths:
+        area = area_for_path(path, data)
+        if area:
+            seen.add(area)
+    return sorted(seen)
+
+
+def impact_targets(path: str, data: Mapping | None) -> tuple[list[str], list[str]]:
+    """Return ``(dependent_files, neighbor_areas)`` a change to ``path`` touches."""
+    area = area_for_path(path, data)
+    return dependents_of(path, data), neighbors_for_area(area, data)
+
+
+def cross_boundary_pairs(
+    paths: Sequence[str],
+    data: Mapping | None,
+) -> list[tuple[str, str]]:
+    """Pairs of touched areas that are neither identical nor declared neighbors.
+
+    Each pair ``(a, b)`` is emitted once (sorted) and signals an edit that spans two
+    subsystems the map does not consider adjacent — a candidate architecture-drift
+    boundary crossing.
+    """
+    areas = areas_touched(paths, data)
+    crossings: set[tuple[str, str]] = set()
+    for i, area_a in enumerate(areas):
+        for area_b in areas[i + 1:]:
+            if not are_neighbors(area_a, area_b, data):
+                crossings.add(tuple(sorted((area_a, area_b))))  # type: ignore[arg-type]
+    return sorted(crossings)

package/src/devcouncil/indexing/ts_imports.py

@@ -0,0 +1,343 @@
+"""Tree-sitter import / symbol extraction for Rust, Go, and JS/TS.
+
+``tree-sitter`` and ``tree-sitter-language-pack`` are core dependencies. Helpers
+still return ``None`` on parse failure so callers can keep regex/AST fallbacks.
+"""
+
+from __future__ import annotations
+
+import logging
+from functools import lru_cache
+from typing import Iterable, List, Optional, Sequence, Tuple
+
+logger = logging.getLogger(__name__)
+
+# language name for tree-sitter-language-pack → file suffixes that use it
+_LANG_BY_SUFFIX = {
+    ".rs": "rust",
+    ".go": "go",
+    ".ts": "typescript",
+    ".tsx": "tsx",
+    ".js": "javascript",
+    ".jsx": "javascript",
+    ".mjs": "javascript",
+    ".cjs": "javascript",
+}
+
+_SymbolHit = Tuple[str, str, int, str]  # kind, name, line, text
+
+
+def tree_sitter_available() -> bool:
+    """True when both the core binding and language pack import cleanly."""
+    try:
+        import tree_sitter  # noqa: F401
+        from tree_sitter_language_pack import get_parser  # noqa: F401
+    except Exception:
+        return False
+    return True
+
+
+@lru_cache(maxsize=16)
+def _parser_for(language: str):
+    if not tree_sitter_available():
+        return None
+    try:
+        from tree_sitter_language_pack import get_parser
+
+        return get_parser(language)
+    except Exception:
+        logger.debug("tree-sitter parser unavailable for %s", language, exc_info=True)
+        return None
+
+
+def _parse(language: str, source: bytes):
+    parser = _parser_for(language)
+    if parser is None:
+        return None
+    try:
+        return parser.parse(source)
+    except Exception:
+        logger.debug("tree-sitter parse failed for %s", language, exc_info=True)
+        return None
+
+
+def _node_text(node, source: bytes) -> str:
+    return source[node.start_byte : node.end_byte].decode("utf-8", errors="replace")
+
+
+def _walk(node) -> Iterable:
+    yield node
+    for child in node.children:
+        yield from _walk(child)
+
+
+# ---------------------------------------------------------------------------
+# Go imports
+# ---------------------------------------------------------------------------
+
+
+def extract_go_import_specs(source: str | bytes) -> Optional[List[str]]:
+    """File-level Go import path strings, or None when tree-sitter is unavailable."""
+    if not tree_sitter_available():
+        return None
+    raw = source.encode("utf-8") if isinstance(source, str) else source
+    tree = _parse("go", raw)
+    if tree is None:
+        return None
+    specs: List[str] = []
+    for node in _walk(tree.root_node):
+        if node.type != "interpreted_string_literal":
+            continue
+        # Only count string literals under import_spec.
+        parent = node.parent
+        while parent is not None and parent.type not in {"import_spec", "source_file"}:
+            parent = parent.parent
+        if parent is None or parent.type != "import_spec":
+            continue
+        text = _node_text(node, raw).strip()
+        if len(text) >= 2 and text[0] in "\"`" and text[-1] == text[0]:
+            text = text[1:-1]
+        if text:
+            specs.append(text)
+    return specs
+
+
+# ---------------------------------------------------------------------------
+# Rust mod / use
+# ---------------------------------------------------------------------------
+
+
+def _rust_path_segments(node, source: bytes) -> List[str]:
+    """Flatten a use/mod path node into identifier segments (skip crate/super/self)."""
+    segs: List[str] = []
+
+    def collect(n) -> None:
+        t = n.type
+        if t in {"identifier", "type_identifier", "field_identifier"}:
+            segs.append(_node_text(n, source))
+            return
+        if t in {"crate", "super", "self"}:
+            segs.append(t)
+            return
+        if t == "use_wildcard":
+            for c in n.children:
+                collect(c)
+            return
+        if t in {"scoped_identifier", "scoped_type_identifier", "scoped_use_list", "use_list", "use_as_clause"}:
+            for c in n.children:
+                if c.type in {",", "{", "}", "::", "as", "*"}:
+                    continue
+                collect(c)
+            return
+        for c in n.children:
+            collect(c)
+
+    collect(node)
+    return segs
+
+
+def extract_rust_import_refs(source: str | bytes) -> Optional[List[dict]]:
+    """Extract Rust ``mod`` / ``use`` references.
+
+    Returns a list of dicts:
+      ``{"kind": "mod", "name": "foo"}`` for ``mod foo;`` (file-level, no body)
+      ``{"kind": "use", "segments": ["crate", "foo", "Bar"]}`` for use paths
+
+    ``None`` when tree-sitter is unavailable.
+    """
+    if not tree_sitter_available():
+        return None
+    raw = source.encode("utf-8") if isinstance(source, str) else source
+    tree = _parse("rust", raw)
+    if tree is None:
+        return None
+    refs: List[dict] = []
+    for node in tree.root_node.children:
+        if node.type == "mod_item":
+            # File-level mod only when there is no declaration_list body.
+            has_body = any(c.type == "declaration_list" for c in node.children)
+            name = None
+            for c in node.children:
+                if c.type == "identifier":
+                    name = _node_text(c, raw)
+                    break
+            if name and not has_body:
+                refs.append({"kind": "mod", "name": name})
+        elif node.type == "use_declaration":
+            # Path is the non-keyword child after `use` / visibility.
+            path_node = None
+            for c in node.children:
+                if c.type in {"use", "visibility_modifier", ";"}:
+                    continue
+                path_node = c
+                break
+            if path_node is None:
+                continue
+            segments = _rust_path_segments(path_node, raw)
+            if segments:
+                refs.append({"kind": "use", "segments": segments})
+    return refs
+
+
+# ---------------------------------------------------------------------------
+# Symbol matching (AstMatcher)
+# ---------------------------------------------------------------------------
+
+
+def extract_symbols(language: str, source: str | bytes) -> Optional[List[_SymbolHit]]:
+    """Named top-level (and nested) definitions for ``language``.
+
+    ``language`` is an AstMatcher language id: typescript, javascript, go, rust.
+    Returns ``None`` when tree-sitter cannot parse (caller should fall back).
+    """
+    if not tree_sitter_available():
+        return None
+    lang_key = {
+        "typescript": "typescript",
+        "javascript": "javascript",
+        "go": "go",
+        "rust": "rust",
+    }.get(language)
+    if lang_key is None:
+        return None
+    raw = source.encode("utf-8") if isinstance(source, str) else source
+    # Prefer tsx grammar when JSX-looking; callers pass typescript for .tsx too.
+    tree = _parse(lang_key, raw)
+    if tree is None and language == "typescript":
+        tree = _parse("tsx", raw)
+    if tree is None:
+        return None
+
+    if language in {"typescript", "javascript"}:
+        return _symbols_js(tree.root_node, raw)
+    if language == "go":
+        return _symbols_go(tree.root_node, raw)
+    if language == "rust":
+        return _symbols_rust(tree.root_node, raw)
+    return None
+
+
+def _line_of(node) -> int:
+    return int(node.start_point[0]) + 1
+
+
+def _first_named(node, types: Sequence[str]) -> Optional[str]:
+    for c in node.children:
+        if c.type in types:
+            return str(c.text.decode("utf-8", errors="replace"))
+    return None
+
+
+def _symbols_js(root, source: bytes) -> List[_SymbolHit]:
+    hits: List[_SymbolHit] = []
+
+    def consider(node, kind: str, name_types: Sequence[str] = ("identifier", "type_identifier")) -> None:
+        name = _first_named(node, name_types)
+        if not name:
+            return
+        line = _line_of(node)
+        text = source.splitlines()[line - 1].decode("utf-8", errors="replace").strip() if line else name
+        hits.append((kind, name, line, text))
+
+    def walk(node) -> None:
+        t = node.type
+        if t == "function_declaration":
+            consider(node, "function")
+        elif t == "class_declaration":
+            consider(node, "class", ("type_identifier", "identifier"))
+        elif t == "interface_declaration":
+            consider(node, "interface", ("type_identifier", "identifier"))
+        elif t == "type_alias_declaration":
+            consider(node, "type", ("type_identifier", "identifier"))
+        elif t == "lexical_declaration":
+            for c in node.children:
+                if c.type != "variable_declarator":
+                    continue
+                name = _first_named(c, ("identifier",))
+                if not name:
+                    continue
+                line = _line_of(c)
+                text = (
+                    source.splitlines()[line - 1].decode("utf-8", errors="replace").strip()
+                    if line
+                    else name
+                )
+                hits.append(("function", name, line, text))
+        elif t == "export_statement":
+            for c in node.children:
+                walk(c)
+            return
+        for c in node.children:
+            walk(c)
+
+    walk(root)
+    return hits
+
+
+def _symbols_go(root, source: bytes) -> List[_SymbolHit]:
+    hits: List[_SymbolHit] = []
+    for node in _walk(root):
+        if node.type == "function_declaration":
+            name = _first_named(node, ("identifier",))
+            if not name:
+                continue
+            line = _line_of(node)
+            text = source.splitlines()[line - 1].decode("utf-8", errors="replace").strip()
+            hits.append(("function", name, line, text))
+        elif node.type == "method_declaration":
+            name = None
+            for c in node.children:
+                if c.type == "field_identifier":
+                    name = c.text.decode("utf-8", errors="replace")
+                    break
+            if not name:
+                continue
+            line = _line_of(node)
+            text = source.splitlines()[line - 1].decode("utf-8", errors="replace").strip()
+            hits.append(("function", name, line, text))
+        elif node.type == "type_declaration":
+            for c in node.children:
+                if c.type != "type_spec":
+                    continue
+                name = _first_named(c, ("type_identifier", "identifier"))
+                if not name:
+                    continue
+                kind = "struct"
+                for gc in c.children:
+                    if gc.type == "struct_type":
+                        kind = "struct"
+                    elif gc.type == "interface_type":
+                        kind = "interface"
+                    elif gc.type == "type_identifier":
+                        continue
+                line = _line_of(c)
+                text = source.splitlines()[line - 1].decode("utf-8", errors="replace").strip()
+                hits.append((kind, name, line, text))
+    return hits
+
+
+def _symbols_rust(root, source: bytes) -> List[_SymbolHit]:
+    hits: List[_SymbolHit] = []
+    kind_map = {
+        "function_item": "function",
+        "struct_item": "struct",
+        "enum_item": "enum",
+        "trait_item": "trait",
+        "type_item": "type",
+    }
+    for node in _walk(root):
+        kind = kind_map.get(node.type)
+        if not kind:
+            continue
+        name = _first_named(node, ("identifier", "type_identifier"))
+        if not name:
+            continue
+        line = _line_of(node)
+        text = source.splitlines()[line - 1].decode("utf-8", errors="replace").strip()
+        hits.append((kind, name, line, text))
+    return hits
+
+
+def language_for_suffix(suffix: str) -> Optional[str]:
+    """Map a file suffix to a language-pack name, or None."""
+    return _LANG_BY_SUFFIX.get(suffix.lower())