devcouncil 0.3.1 → 0.4.0

package/src/devcouncil/codeintel/languages/generic_extractor.py

@@ -0,0 +1,236 @@
+"""Generic Tree-sitter extraction used by the broad language matrix."""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from devcouncil.codeintel.languages.registry import detect_language
+from devcouncil.codeintel.languages.workers import process_tree_sitter
+from devcouncil.indexing.graph.extract_python import (
+    ExtractedCall,
+    ExtractedImport,
+    ExtractedSymbol,
+    FileExtraction,
+)
+
+_QUOTED = re.compile(r"['\"]([^'\"]+)['\"]")
+_CALL = re.compile(r"(?<![\w$])(?:(?P<receiver>[A-Za-z_$][\w$.:]*)\s*[.>:]+\s*)?(?P<name>[A-Za-z_$][\w$]*)\s*\(")
+_CALL_KEYWORDS = {"if", "for", "while", "switch", "catch", "return", "sizeof", "typeof", "match", "when"}
+
+
+def extract_generic(path: str, source: str) -> FileExtraction:
+    spec = detect_language(path)
+    if spec is None:
+        return FileExtraction(path=path, language="")
+    output = FileExtraction(path=path, language=spec.grammar)
+    result = process_tree_sitter(spec.grammar, source)
+    if result is not None:
+        _append_result(output, source, result)
+    for language, region, line_offset in _embedded_regions(spec.name, source):
+        embedded = process_tree_sitter(language, region)
+        if embedded is not None:
+            _append_result(output, region, embedded, line_offset=line_offset)
+    return output
+
+
+def _append_result(
+    output: FileExtraction,
+    source: str,
+    result: dict[str, Any],
+    *,
+    line_offset: int = 0,
+) -> None:
+    """Merge one container or embedded Tree-sitter result into a file extraction."""
+
+    exported_names = {
+        str(item.get("name", "")).strip()
+        for item in result.get("exports", [])
+        if str(item.get("name", "")).strip()
+    }
+
+    # Seen-sets instead of list scans: large generated files can carry
+    # thousands of symbols/calls, and per-append linear scans go quadratic.
+    seen_symbols = {(existing.qualname, existing.line) for existing in output.symbols}
+    seen_calls = {
+        (existing.name, existing.line, existing.receiver) for existing in output.calls
+    }
+
+    def visit(item: dict[str, Any], prefix: str = "") -> None:
+        name = str(item.get("name", "") or "").strip()
+        kind = _kind(str(item.get("kind", "") or ""))
+        if name and kind:
+            qualname = f"{prefix}.{name}" if prefix else name
+            symbol = ExtractedSymbol(
+                kind=kind,
+                name=name,
+                qualname=qualname,
+                line=int(item.get("start_line", 0)) + line_offset + 1,
+                end_line=int(item.get("end_line", 0)) + line_offset + 1,
+                decorators=[str(value) for value in item.get("decorators", [])],
+                exported=name in exported_names or not name.startswith("_"),
+            )
+            if (symbol.qualname, symbol.line) not in seen_symbols:
+                seen_symbols.add((symbol.qualname, symbol.line))
+                output.symbols.append(symbol)
+            child_prefix = qualname if kind in {"class", "interface", "struct", "trait", "enum"} else prefix
+        else:
+            child_prefix = prefix
+        for child in item.get("children", []) or []:
+            visit(child, child_prefix)
+
+    for item in result.get("structure", []) or []:
+        visit(item)
+
+    for item in result.get("imports", []) or []:
+        raw_source = str(item.get("source", "") or "")
+        match = _QUOTED.search(raw_source)
+        module = match.group(1) if match else raw_source.strip()
+        if not module:
+            continue
+        if module not in output.imports:
+            output.imports.append(module)
+        names = [str(value) for value in item.get("items", []) or []]
+        alias = str(item.get("alias", "") or "")
+        output.import_details.append(ExtractedImport(
+            module=module,
+            names=names,
+            alias_map={alias: module} if alias else {},
+        ))
+    output.all_exports = sorted(set(output.all_exports) | exported_names)
+
+    # ``qualname_hint`` is a CALLEE hint (``receiver.name`` / bare name) — the
+    # same convention as every dedicated extractor. resolve_calls resolves it
+    # against the same-file symbol index; the enclosing caller must never go
+    # here or bare in-function calls all bind back to their own caller.
+    if "calls" in result:
+        for item in result.get("calls", []) or []:
+            name = str(item.get("name", "") or "")
+            if not name or name in _CALL_KEYWORDS:
+                continue
+            line_no = int(item.get("line", 0)) + line_offset
+            receiver = str(item.get("receiver", "") or "")
+            call = ExtractedCall(
+                name=name,
+                line=line_no,
+                receiver=receiver,
+                qualname_hint=f"{receiver}.{name}" if receiver else name,
+            )
+            key = (call.name, call.line, call.receiver)
+            if key not in seen_calls:
+                seen_calls.add(key)
+                output.calls.append(call)
+                output.references.append(name)
+        return
+
+    # Compatibility for older companion packs that do not return call rows.
+    for line_no, line in enumerate(source.splitlines(), start=line_offset + 1):
+        for match in _CALL.finditer(line):
+            name = match.group("name")
+            if name in _CALL_KEYWORDS:
+                continue
+            receiver = match.group("receiver") or ""
+            output.calls.append(ExtractedCall(
+                name=name,
+                line=line_no,
+                receiver=receiver,
+                qualname_hint=f"{receiver}.{name}" if receiver else name,
+            ))
+            output.references.append(name)
+
+
+def _embedded_regions(container: str, source: str) -> list[tuple[str, str, int]]:
+    regions: list[tuple[str, str, int]] = []
+    if container in {"Svelte", "Vue", "Astro", "Liquid"}:
+        for match in re.finditer(
+            r"<script\b(?P<attrs>[^>]*)>(?P<body>.*?)</script\s*>",
+            source,
+            re.IGNORECASE | re.DOTALL,
+        ):
+            attrs = match.group("attrs")
+            language = (
+                "typescript"
+                if re.search(r"\blang\s*=\s*['\"](?:ts|typescript)['\"]", attrs, re.I)
+                else "javascript"
+            )
+            regions.append((
+                language,
+                match.group("body"),
+                source[:match.start("body")].count("\n"),
+            ))
+        for match in re.finditer(
+            r"<style\b[^>]*>(?P<body>.*?)</style\s*>",
+            source,
+            re.IGNORECASE | re.DOTALL,
+        ):
+            regions.append((
+                "css",
+                match.group("body"),
+                source[:match.start("body")].count("\n"),
+            ))
+    if container in {"Svelte", "Vue", "Astro"}:
+        for match in re.finditer(
+            r"<template\b[^>]*>(?P<body>.*?)</template\s*>",
+            source,
+            re.IGNORECASE | re.DOTALL,
+        ):
+            regions.append((
+                "html",
+                match.group("body"),
+                source[:match.start("body")].count("\n"),
+            ))
+    if container == "Astro":
+        frontmatter = re.match(r"^---\s*\n(?P<body>.*?)\n---(?:\s*\n|$)", source, re.DOTALL)
+        if frontmatter is not None:
+            regions.append((
+                "typescript",
+                frontmatter.group("body"),
+                source[:frontmatter.start("body")].count("\n"),
+            ))
+            regions.append((
+                "html",
+                source[frontmatter.end():],
+                source[:frontmatter.end()].count("\n"),
+            ))
+        else:
+            regions.append(("html", source, 0))
+    if container == "Svelte":
+        regions.append(("html", source, 0))
+    if container == "Liquid":
+        for match in re.finditer(
+            r"{%\s*javascript\s*%}(?P<body>.*?){%\s*endjavascript\s*%}",
+            source,
+            re.IGNORECASE | re.DOTALL,
+        ):
+            regions.append((
+                "javascript",
+                match.group("body"),
+                source[:match.start("body")].count("\n"),
+            ))
+        regions.append(("html", source, 0))
+    return regions
+
+
+def _kind(raw: str) -> str:
+    normalized = raw.lower().replace("_", "").replace(" ", "")
+    if "method" in normalized or "constructor" in normalized:
+        return "method"
+    if "function" in normalized or "procedure" in normalized or normalized in {"func", "subroutine"}:
+        return "function"
+    if "interface" in normalized or "protocol" in normalized:
+        return "interface"
+    if "struct" in normalized or "record" in normalized:
+        return "struct"
+    if "trait" in normalized:
+        return "trait"
+    if "enum" in normalized:
+        return "enum"
+    if "class" in normalized or "object" in normalized or "module" in normalized:
+        return "class"
+    if "type" in normalized or "alias" in normalized:
+        return "type"
+    if "property" in normalized or "field" in normalized:
+        return "property"
+    if "variable" in normalized or "constant" in normalized:
+        return "variable"
+    return ""

package/src/devcouncil/codeintel/languages/registry.py

@@ -0,0 +1,149 @@
+"""Explicit language/grammar manifest for the code-intelligence release."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(frozen=True)
+class LanguageSpec:
+    name: str
+    grammar: str
+    extensions: tuple[str, ...]
+    embedded: tuple[str, ...] = ()
+
+
+def _spec(name: str, grammar: str, extensions: str, *, embedded: tuple[str, ...] = ()) -> LanguageSpec:
+    return LanguageSpec(name, grammar, tuple(extensions.split()), embedded)
+
+
+LANGUAGE_SPECS: tuple[LanguageSpec, ...] = (
+    _spec("TypeScript", "typescript", ".ts .mts .cts"),
+    _spec("TSX", "tsx", ".tsx"),
+    _spec("JavaScript", "javascript", ".js .jsx .mjs .cjs"),
+    _spec("ArkTS", "typescript", ".ets"),
+    _spec("Python", "python", ".py .pyi"),
+    _spec("Go", "go", ".go"),
+    _spec("Rust", "rust", ".rs"),
+    _spec("Java", "java", ".java"),
+    _spec("C#", "csharp", ".cs"),
+    _spec("VB.NET", "vb", ".vb"),
+    _spec("PHP", "php", ".php .phtml"),
+    _spec("Ruby", "ruby", ".rb .rake"),
+    _spec("C", "c", ".c .h"),
+    _spec("C++", "cpp", ".cc .cpp .cxx .hh .hpp .hxx"),
+    _spec("Objective-C", "objc", ".m .mm"),
+    # Metal is a C++14-derived language; the language pack ships no separate
+    # Metal grammar, so use its maintained C++ parser plus Metal augmenters.
+    _spec("Metal", "cpp", ".metal"),
+    _spec("CUDA", "cuda", ".cu .cuh"),
+    _spec("Swift", "swift", ".swift"),
+    _spec("Kotlin", "kotlin", ".kt .kts"),
+    _spec("Scala", "scala", ".scala .sc"),
+    _spec("Dart", "dart", ".dart"),
+    _spec("Svelte", "svelte", ".svelte", embedded=("typescript", "javascript", "css", "html")),
+    _spec("Vue", "vue", ".vue", embedded=("typescript", "javascript", "css", "html")),
+    _spec("Astro", "astro", ".astro", embedded=("typescript", "javascript", "css", "html")),
+    _spec("Liquid", "liquid", ".liquid", embedded=("html", "javascript", "css")),
+    _spec("Pascal/Delphi", "pascal", ".pas .pp .dpr"),
+    _spec("Lua", "lua", ".lua"),
+    _spec("Luau", "luau", ".luau"),
+    _spec("R", "r", ".r .R"),
+    _spec("CFML", "cfml", ".cfm .cfc"),
+    _spec("COBOL", "cobol", ".cob .cbl .cpy"),
+    _spec("Erlang", "erlang", ".erl .hrl"),
+    _spec("Solidity", "solidity", ".sol"),
+    _spec("Terraform/OpenTofu", "hcl", ".tf .tfvars .hcl"),
+    _spec("Nix", "nix", ".nix"),
+)
+
+_BY_EXTENSION = {
+    extension.lower(): spec
+    for spec in LANGUAGE_SPECS
+    for extension in spec.extensions
+}
+
+
+def supported_languages() -> list[str]:
+    return [spec.name for spec in LANGUAGE_SPECS]
+
+
+def detect_language(path: str | Path) -> LanguageSpec | None:
+    return _BY_EXTENSION.get(Path(path).suffix.lower())
+
+
+def grammar_status() -> dict[str, Any]:
+    """Report local grammar availability without downloading anything."""
+
+    companion: dict[str, Any] = {"installed": False}
+    try:
+        try:
+            import devcouncil_codeintel_grammars as grammar_assets
+
+            companion = {"installed": True, **grammar_assets.activate()}
+        except ImportError:
+            pass
+        import tree_sitter_language_pack as pack
+
+        available = set(pack.available_languages())
+        pack_version = str(getattr(pack, "__version__", "unknown"))
+        cache_dir = str(pack.cache_dir())
+        error = ""
+    except Exception as exc:
+        available = set()
+        pack_version = "unavailable"
+        cache_dir = ""
+        error = f"{type(exc).__name__}: {exc}"
+    rows = []
+    for spec in LANGUAGE_SPECS:
+        required = (spec.grammar, *spec.embedded)
+        missing = sorted(set(required) - available)
+        rows.append({
+            "language": spec.name,
+            "grammar": spec.grammar,
+            "required_grammars": list(required),
+            "missing_grammars": missing,
+            "available": not missing,
+            "extensions": list(spec.extensions),
+            "embedded": list(spec.embedded),
+        })
+    missing_required = sorted({
+        grammar
+        for spec in LANGUAGE_SPECS
+        for grammar in (spec.grammar, *spec.embedded)
+        if grammar not in available
+    })
+    companion_installed = bool(companion.get("installed"))
+    companion_ok = bool(companion.get("ok"))
+    if missing_required and not companion_installed:
+        action = (
+            "Install the platform-matched devcouncil-codeintel-grammars wheel; "
+            "runtime grammar downloads are disabled."
+        )
+    elif missing_required and not companion_ok:
+        action = (
+            "Reinstall the platform-matched devcouncil-codeintel-grammars wheel; "
+            "its manifest or checksums are invalid."
+        )
+    elif missing_required:
+        action = (
+            "The installed companion wheel is incomplete for this platform; "
+            "install a wheel containing every required grammar."
+        )
+    else:
+        action = ""
+    return {
+        "ok": all(row["available"] for row in rows),
+        "pack_version": pack_version,
+        "cache_dir": cache_dir,
+        "available_count": sum(bool(row["available"]) for row in rows),
+        "required_count": len(rows),
+        "languages": rows,
+        "missing_grammars": missing_required,
+        "action": action,
+        "error": error,
+        "downloaded_at_runtime": False,
+        "companion": companion,
+    }

package/src/devcouncil/codeintel/languages/workers.py

@@ -0,0 +1,245 @@
+"""Crash-isolated native Tree-sitter worker pool."""
+
+from __future__ import annotations
+
+import atexit
+import multiprocessing
+import os
+import re
+import threading
+from concurrent.futures import ProcessPoolExecutor
+from pathlib import Path
+from typing import Any
+
+_ACTIVATION_LOCK = threading.Lock()
+_ACTIVATION_ATTEMPTED = False
+_ACTIVATION_STATUS: dict[str, Any] = {"installed": False, "activated": False}
+
+
+def _activate_companion_once() -> dict[str, Any]:
+    """Activate packaged assets once per process without fetching grammars."""
+
+    global _ACTIVATION_ATTEMPTED, _ACTIVATION_STATUS
+    with _ACTIVATION_LOCK:
+        if _ACTIVATION_ATTEMPTED:
+            return _ACTIVATION_STATUS
+        _ACTIVATION_ATTEMPTED = True
+        try:
+            import devcouncil_codeintel_grammars as grammar_assets
+        except ImportError:
+            return _ACTIVATION_STATUS
+        try:
+            _ACTIVATION_STATUS = {"installed": True, **grammar_assets.activate()}
+        except Exception as exc:
+            _ACTIVATION_STATUS = {
+                "installed": True,
+                "activated": False,
+                "ok": False,
+                "error": f"{type(exc).__name__}: {exc}",
+            }
+        return _ACTIVATION_STATUS
+
+
+def _structure_row(item: Any) -> dict[str, Any]:
+    span = getattr(item, "span", None)
+    return {
+        "name": str(getattr(item, "name", "") or ""),
+        "kind": str(getattr(item, "kind", "") or ""),
+        "start_line": int(getattr(span, "start_line", 0)) if span is not None else 0,
+        "end_line": int(getattr(span, "end_line", 0)) if span is not None else 0,
+        "decorators": [str(value) for value in getattr(item, "decorators", []) or []],
+        "children": [_structure_row(child) for child in getattr(item, "children", []) or []],
+    }
+
+
+def _declared_name(node: Any, raw: bytes) -> str:
+    """Resolve a definition's identifier through C-family declarator nesting."""
+    current = node
+    for _ in range(8):
+        candidate = None
+        for field in ("name", "declarator"):
+            try:
+                candidate = current.child_by_field_name(field)
+            except (AttributeError, TypeError):
+                candidate = None
+            if candidate is not None:
+                break
+        if candidate is None:
+            return ""
+        if str(getattr(candidate, "type", "")).endswith("identifier"):
+            return raw[candidate.start_byte:candidate.end_byte].decode("utf-8", errors="replace")
+        current = candidate
+    return ""
+
+
+def _fill_missing_structure_names(tree: Any, raw: bytes, rows: list[dict[str, Any]]) -> None:
+    """The language pack's structure pass yields ``name=None`` for C-family
+    definitions — the identifier hides inside the declarator chain — and
+    downstream extraction drops nameless rows, losing every C/C++/ObjC symbol.
+    Recover names from the parse tree, matched by start line."""
+    pending: dict[int, list[dict[str, Any]]] = {}
+
+    def collect(items: list[dict[str, Any]]) -> None:
+        for row in items:
+            if not row["name"]:
+                pending.setdefault(int(row["start_line"]), []).append(row)
+            collect(row["children"])
+
+    collect(rows)
+    if not pending:
+        return
+    stack = [tree.root_node]
+    while stack:
+        node = stack.pop()
+        stack.extend(reversed(list(getattr(node, "children", ()) or ())))
+        point = getattr(node, "start_point", None)
+        waiting = pending.get(int(point[0])) if point is not None else None
+        if not waiting:
+            continue
+        name = _declared_name(node, raw)
+        if not name:
+            continue
+        row = waiting.pop(0)
+        row["name"] = name
+
+
+def _call_rows(parser: Any, source: str, tree: Any | None = None) -> list[dict[str, Any]]:
+    raw = source.encode("utf-8")
+    if tree is None:
+        tree = parser.parse(raw)
+    rows: list[dict[str, Any]] = []
+    stack = [tree.root_node]
+    while stack:
+        node = stack.pop()
+        stack.extend(reversed(list(getattr(node, "children", ()) or ())))
+        node_type = str(getattr(node, "type", ""))
+        if "call" not in node_type or any(
+            value in node_type for value in ("callable", "calling_convention")
+        ):
+            continue
+        callee = None
+        for field in ("function", "name", "method"):
+            try:
+                callee = node.child_by_field_name(field)
+            except (AttributeError, TypeError):
+                callee = None
+            if callee is not None:
+                break
+        if callee is None:
+            named = list(getattr(node, "named_children", ()) or ())
+            callee = named[0] if named else None
+        if callee is None:
+            continue
+        text = raw[callee.start_byte:callee.end_byte].decode("utf-8", errors="replace")
+        identifiers = re.findall(r"[A-Za-z_$][\w$]*", text)
+        if not identifiers:
+            continue
+        name = identifiers[-1]
+        receiver = ".".join(identifiers[:-1])
+        rows.append({
+            "name": name,
+            "receiver": receiver,
+            "line": int(node.start_point[0]) + 1,
+        })
+    return rows
+
+
+def _native_process(language: str, source: str) -> dict[str, Any] | None:
+    """Execute inside a child so a faulty native grammar cannot kill the server."""
+    import tree_sitter_language_pack as pack
+
+    if language not in set(pack.available_languages()):
+        return None
+    result = pack.process(
+        source,
+        pack.ProcessConfig(
+            language=language,
+            structure=True,
+            imports=True,
+            exports=True,
+            symbols=True,
+            diagnostics=True,
+        ),
+    )
+    parser = pack.get_parser(language)
+    raw = source.encode("utf-8")
+    tree = parser.parse(raw)
+    structure = [_structure_row(item) for item in getattr(result, "structure", []) or []]
+    _fill_missing_structure_names(tree, raw, structure)
+    return {
+        "structure": structure,
+        "imports": [
+            {
+                "source": str(getattr(item, "source", "") or ""),
+                "items": [str(value) for value in getattr(item, "items", []) or []],
+                "alias": str(getattr(item, "alias", "") or ""),
+            }
+            for item in getattr(result, "imports", []) or []
+        ],
+        "exports": [
+            {"name": str(getattr(item, "name", "") or "")}
+            for item in getattr(result, "exports", []) or []
+        ],
+        "calls": _call_rows(parser, source, tree),
+    }
+
+
+class ParserWorkerPool:
+    """Small spawn-based pool that is discarded after a crash or timeout."""
+
+    def __init__(self, *, max_workers: int | None = None, timeout: float = 30.0):
+        cpu_bound = max(1, (os.cpu_count() or 2) // 2)
+        self.max_workers = max(1, min(4, max_workers or cpu_bound))
+        self.timeout = max(1.0, timeout)
+        self._pool: ProcessPoolExecutor | None = None
+        self._lock = threading.Lock()
+
+    def process(self, language: str, source: str) -> dict[str, Any] | None:
+        try:
+            pool = self._executor()
+            return pool.submit(_native_process, language, source).result(timeout=self.timeout)
+        except Exception:
+            self.restart()
+            return None
+
+    def _executor(self) -> ProcessPoolExecutor:
+        with self._lock:
+            if self._pool is None:
+                _activate_companion_once()
+                self._pool = ProcessPoolExecutor(
+                    max_workers=self.max_workers,
+                    mp_context=multiprocessing.get_context("spawn"),
+                    initializer=_activate_companion_once,
+                )
+            return self._pool
+
+    def restart(self) -> None:
+        with self._lock:
+            pool, self._pool = self._pool, None
+        if pool is not None:
+            pool.shutdown(wait=False, cancel_futures=True)
+
+    def close(self) -> None:
+        with self._lock:
+            pool, self._pool = self._pool, None
+        if pool is not None:
+            pool.shutdown(wait=True, cancel_futures=True)
+
+
+_POOL = ParserWorkerPool()
+atexit.register(_POOL.close)
+
+
+def process_tree_sitter(language: str, source: str) -> dict[str, Any] | None:
+    return _POOL.process(language, source)
+
+
+def parser_worker_status() -> dict[str, object]:
+    return {
+        "start_method": "spawn",
+        "max_workers": _POOL.max_workers,
+        "timeout_seconds": _POOL.timeout,
+        "pid": os.getpid(),
+        "module": str(Path(__file__).resolve()),
+        "grammar_companion": dict(_ACTIVATION_STATUS),
+    }

package/src/devcouncil/codeintel/query/__init__.py

@@ -0,0 +1,5 @@
+"""High-level code-intelligence queries."""
+
+from devcouncil.codeintel.query.engine import CodeIntelQueryEngine
+
+__all__ = ["CodeIntelQueryEngine"]