@pmaddire/gcie 0.1.2

package/parser/variable_extractor.py

@@ -0,0 +1,31 @@
+"""Variable dependency extraction helpers."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .models import FunctionEntry, ModuleParseResult
+
+
+@dataclass(frozen=True, slots=True)
+class VariableDependency:
+    """Represents variable usage by a function."""
+
+    function_name: str
+    variable_name: str
+    access_type: str  # READS | WRITES | MODIFIES
+
+
+def extract_variable_dependencies(module: ModuleParseResult) -> tuple[VariableDependency, ...]:
+    """Extract variable read/write/modifies relationships from parsed functions."""
+    out: list[VariableDependency] = []
+
+    for fn in module.functions:
+        for name in sorted(set(fn.variables_read)):
+            out.append(VariableDependency(function_name=fn.name, variable_name=name, access_type="READS"))
+
+        for name in sorted(set(fn.variables_written)):
+            out.append(VariableDependency(function_name=fn.name, variable_name=name, access_type="WRITES"))
+            out.append(VariableDependency(function_name=fn.name, variable_name=name, access_type="MODIFIES"))
+
+    return tuple(out)

package/retrieval/__init__.py

@@ -0,0 +1,17 @@
+"""Retrieval package."""
+
+from .hybrid_retriever import HybridCandidate, hybrid_retrieve
+from .query_parser import ParsedQuery, parse_query
+from .semantic_retriever import SemanticCandidate, SemanticRetriever
+from .symbolic_retriever import SymbolicCandidate, symbolic_retrieve
+
+__all__ = [
+    "HybridCandidate",
+    "ParsedQuery",
+    "SemanticCandidate",
+    "SemanticRetriever",
+    "SymbolicCandidate",
+    "hybrid_retrieve",
+    "parse_query",
+    "symbolic_retrieve",
+]

package/retrieval/cache.py

@@ -0,0 +1,22 @@
+"""In-memory retrieval cache."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(slots=True)
+class RetrievalCache:
+    _data: dict[str, tuple[str, ...]]
+
+    def __init__(self) -> None:
+        self._data = {}
+
+    def get(self, key: str) -> tuple[str, ...] | None:
+        return self._data.get(key)
+
+    def set(self, key: str, value: tuple[str, ...]) -> None:
+        self._data[key] = value
+
+    def clear(self) -> None:
+        self._data.clear()

package/retrieval/hybrid_retriever.py

@@ -0,0 +1,249 @@
+"""Hybrid symbolic + semantic retrieval pipeline."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import re
+
+import networkx as nx
+
+from .ranking import HybridScore, combine_score
+from .semantic_retriever import SemanticRetriever
+from .symbolic_retriever import symbolic_retrieve
+
+
+@dataclass(frozen=True, slots=True)
+class HybridCandidate:
+    node_id: str
+    score: float
+    rationale: str
+
+
+@dataclass(frozen=True, slots=True)
+class HybridDiagnostics:
+    query_terms: tuple[str, ...]
+    symbolic_candidates: tuple[str, ...]
+    semantic_candidates: tuple[str, ...]
+    merged_candidates: tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class _SemanticAggregate:
+    node_id: str
+    score: float
+    hit_count: int
+    best_text_score: float
+    lexical_overlap: int
+    path_relevance: float
+
+
+_STOPWORDS = {
+    "how",
+    "does",
+    "when",
+    "what",
+    "why",
+    "where",
+    "which",
+    "the",
+    "this",
+    "that",
+    "into",
+    "from",
+    "with",
+    "files",
+    "file",
+    "used",
+    "using",
+}
+
+
+def _query_terms(query: str) -> tuple[str, ...]:
+    terms = {
+        part
+        for token in re.findall(r"[a-zA-Z_][a-zA-Z0-9_]*", query.lower())
+        for part in token.split("_")
+        if len(part) >= 3 and part not in _STOPWORDS
+    }
+    return tuple(sorted(terms))
+
+
+def _node_path(node_id: str, attrs: dict) -> str:
+    path = str(attrs.get("path") or attrs.get("file") or "")
+    if path:
+        return path
+    if node_id.startswith("file:"):
+        return node_id[5:]
+    if node_id.startswith(("function:", "class:")):
+        return node_id.split(":", 1)[1].split("::", 1)[0]
+    return ""
+
+
+def _canonical_semantic_node_id(graph: nx.DiGraph, node_id: str, attrs: dict) -> str:
+    path = _node_path(node_id, attrs)
+    file_node_id = f"file:{path}" if path else ""
+    if file_node_id and graph.has_node(file_node_id):
+        return file_node_id
+    return node_id
+
+
+def _lexical_overlap(node_id: str, attrs: dict, query_terms: tuple[str, ...]) -> int:
+    if not query_terms:
+        return 0
+    label = str(attrs.get("label", ""))
+    path = _node_path(node_id, attrs)
+    haystack = f"{node_id} {label} {path}".lower()
+    return sum(1 for term in query_terms if term in haystack)
+
+
+def _path_relevance(node_id: str, attrs: dict, query_terms: tuple[str, ...]) -> float:
+    if not query_terms:
+        return 0.0
+    path = _node_path(node_id, attrs).lower()
+    if not path:
+        return 0.0
+    parts = {part for part in re.split(r"[^a-zA-Z0-9_]+", path) if part}
+    overlap = sum(1 for term in query_terms if term in path)
+    exact_parts = sum(1 for term in query_terms if term in parts)
+    return overlap * 0.08 + exact_parts * 0.05
+
+
+def _semantic_node_scores(
+    graph: nx.DiGraph,
+    query: str,
+    *,
+    top_k: int,
+) -> tuple[dict[str, _SemanticAggregate], tuple[str, ...]]:
+    query_terms = _query_terms(query)
+    entries: list[tuple[str, str]] = []
+    for node_id, attrs in sorted(graph.nodes(data=True), key=lambda item: item[0]):
+        label = str(attrs.get("label", ""))
+        path = _node_path(node_id, attrs)
+        text = f"{node_id} {path} {label}".strip()
+        entries.append((node_id, text))
+
+    if not entries:
+        return {}, ()
+
+    retriever = SemanticRetriever([text for _, text in entries])
+    semantic_top_k = min(max(top_k * 4, 24), len(entries))
+    hits = retriever.retrieve(query, top_k=semantic_top_k)
+
+    aggregates: dict[str, _SemanticAggregate] = {}
+    raw_hits: list[str] = []
+    for hit in hits:
+        source_node_id, _ = entries[hit.idx]
+        attrs = graph.nodes[source_node_id]
+        target_node_id = _canonical_semantic_node_id(graph, source_node_id, attrs)
+        target_attrs = graph.nodes[target_node_id] if graph.has_node(target_node_id) else attrs
+        lexical = _lexical_overlap(target_node_id, target_attrs, query_terms)
+        path_rel = _path_relevance(target_node_id, target_attrs, query_terms)
+        raw_hits.append(source_node_id)
+
+        existing = aggregates.get(target_node_id)
+        hit_count = 1 if existing is None else existing.hit_count + 1
+        best_text_score = hit.score if existing is None else max(existing.best_text_score, hit.score)
+        best_lexical = lexical if existing is None else max(existing.lexical_overlap, lexical)
+        best_path = path_rel if existing is None else max(existing.path_relevance, path_rel)
+        bonus = min(0.12, 0.03 * max(hit_count - 1, 0))
+        aggregate_score = min(1.0, best_text_score + bonus + min(0.08, best_lexical * 0.02) + min(0.08, best_path))
+        aggregates[target_node_id] = _SemanticAggregate(
+            node_id=target_node_id,
+            score=aggregate_score,
+            hit_count=hit_count,
+            best_text_score=best_text_score,
+            lexical_overlap=best_lexical,
+            path_relevance=best_path,
+        )
+
+    return aggregates, tuple(raw_hits)
+
+
+def collect_hybrid_diagnostics(
+    graph: nx.DiGraph,
+    query: str,
+    *,
+    git_recency_by_node: dict[str, float] | None = None,
+    coverage_risk_by_node: dict[str, float] | None = None,
+    max_hops: int = 2,
+    top_k: int = 10,
+) -> HybridDiagnostics:
+    symbolic = symbolic_retrieve(graph, query, max_hops=max_hops)
+    semantic, semantic_hits = _semantic_node_scores(graph, query, top_k=top_k)
+    git_map = git_recency_by_node or {}
+    cov_map = coverage_risk_by_node or {}
+    merged = sorted(set(c.node_id for c in symbolic) | set(semantic) | set(git_map) | set(cov_map))
+    return HybridDiagnostics(
+        query_terms=_query_terms(query),
+        symbolic_candidates=tuple(c.node_id for c in symbolic),
+        semantic_candidates=semantic_hits,
+        merged_candidates=tuple(merged),
+    )
+
+
+def hybrid_retrieve(
+    graph: nx.DiGraph,
+    query: str,
+    *,
+    git_recency_by_node: dict[str, float] | None = None,
+    coverage_risk_by_node: dict[str, float] | None = None,
+    max_hops: int = 2,
+    top_k: int = 10,
+) -> tuple[HybridCandidate, ...]:
+    """Rank retrieval candidates with hybrid weighted scoring."""
+    query_terms = _query_terms(query)
+    symbolic = symbolic_retrieve(graph, query, max_hops=max_hops)
+    sym_distance = {c.node_id: c.distance for c in symbolic}
+    semantic, _ = _semantic_node_scores(graph, query, top_k=top_k)
+
+    git_map = git_recency_by_node or {}
+    cov_map = coverage_risk_by_node or {}
+
+    all_nodes = sorted(set(sym_distance) | set(semantic) | set(git_map) | set(cov_map))
+    scored: list[HybridScore] = []
+
+    for node_id in all_nodes:
+        attrs = graph.nodes[node_id] if graph.has_node(node_id) else {}
+        semantic_item = semantic.get(node_id)
+        score = combine_score(
+            symbolic_distance=sym_distance.get(node_id),
+            semantic_score=semantic_item.score if semantic_item else None,
+            git_recency=git_map.get(node_id),
+            coverage_risk=cov_map.get(node_id),
+        )
+        scored.append(
+            HybridScore(
+                node_id=node_id,
+                score=score,
+                symbolic_distance=sym_distance.get(node_id),
+                semantic_score=semantic_item.score if semantic_item else None,
+                git_recency=git_map.get(node_id),
+                coverage_risk=cov_map.get(node_id),
+                lexical_overlap=_lexical_overlap(node_id, attrs, query_terms),
+                path_relevance=_path_relevance(node_id, attrs, query_terms),
+                semantic_hits=0 if semantic_item is None else semantic_item.hit_count,
+            )
+        )
+
+    scored.sort(
+        key=lambda s: (
+            -round(s.score, 10),
+            -s.lexical_overlap,
+            -round(s.path_relevance, 10),
+            -s.semantic_hits,
+            s.symbolic_distance if s.symbolic_distance is not None else 9999,
+            s.node_id,
+        )
+    )
+
+    out: list[HybridCandidate] = []
+    for item in scored[:top_k]:
+        rationale = (
+            f"symbolic_distance={item.symbolic_distance}, "
+            f"semantic={item.semantic_score}, semantic_hits={item.semantic_hits}, "
+            f"lexical_overlap={item.lexical_overlap}, path_relevance={item.path_relevance}, "
+            f"git={item.git_recency}, coverage={item.coverage_risk}"
+        )
+        out.append(HybridCandidate(node_id=item.node_id, score=item.score, rationale=rationale))
+
+    return tuple(out)

package/retrieval/query_parser.py

@@ -0,0 +1,38 @@
+"""Query parsing utilities for retrieval."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class ParsedQuery:
+    raw: str
+    tokens: tuple[str, ...]
+    symbols: tuple[str, ...]
+
+
+def parse_query(query: str) -> ParsedQuery:
+    """Extract likely symbol tokens from a user query."""
+    lowered = query.strip().lower()
+    words = tuple(re.findall(r"[a-zA-Z_][a-zA-Z0-9_\.]*", lowered))
+
+    stop = {
+        "why",
+        "is",
+        "the",
+        "a",
+        "an",
+        "to",
+        "in",
+        "of",
+        "and",
+        "or",
+        "for",
+        "variable",
+        "function",
+        "class",
+    }
+    symbols = tuple(w for w in words if w not in stop)
+    return ParsedQuery(raw=query, tokens=words, symbols=symbols)

package/retrieval/ranking.py

@@ -0,0 +1,43 @@
+"""Ranking utilities for hybrid retrieval."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class HybridScore:
+    node_id: str
+    score: float
+    symbolic_distance: int | None
+    semantic_score: float | None
+    git_recency: float | None
+    coverage_risk: float | None
+    lexical_overlap: int = 0
+    path_relevance: float = 0.0
+    semantic_hits: int = 0
+
+
+def combine_score(
+    *,
+    symbolic_distance: int | None,
+    semantic_score: float | None,
+    git_recency: float | None,
+    coverage_risk: float | None,
+    w_symbolic: float = 0.4,
+    w_semantic: float = 0.3,
+    w_git: float = 0.2,
+    w_coverage: float = 0.1,
+) -> float:
+    """Combine component signals into a single hybrid score."""
+    symbolic_component = 0.0 if symbolic_distance is None else (1.0 / (1.0 + float(symbolic_distance)))
+    semantic_component = 0.0 if semantic_score is None else float(semantic_score)
+    git_component = 0.0 if git_recency is None else float(git_recency)
+    coverage_component = 0.0 if coverage_risk is None else float(coverage_risk)
+
+    return (
+        w_symbolic * symbolic_component
+        + w_semantic * semantic_component
+        + w_git * git_component
+        + w_coverage * coverage_component
+    )

package/retrieval/semantic_retriever.py

@@ -0,0 +1,39 @@
+"""Semantic retriever implementation."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from embeddings.encoder import TextEncoder
+from embeddings.faiss_index import VectorIndex
+
+
+@dataclass(frozen=True, slots=True)
+class SemanticCandidate:
+    idx: int
+    text: str
+    score: float
+
+
+class SemanticRetriever:
+    """Semantic retrieval over text snippets."""
+
+    def __init__(self, snippets: list[str]) -> None:
+        self._snippets = snippets
+        self._encoder = TextEncoder()
+        self._index = VectorIndex()
+
+        vectors = self._encoder.encode(snippets)
+        self._index.add(vectors)
+
+    def retrieve(self, query: str, top_k: int = 5) -> tuple[SemanticCandidate, ...]:
+        qvec = self._encoder.encode([query])[0]
+        hits = self._index.search(qvec, top_k=top_k)
+
+        out: list[SemanticCandidate] = []
+        for hit in hits:
+            if hit.idx < 0 or hit.idx >= len(self._snippets):
+                continue
+            out.append(SemanticCandidate(idx=hit.idx, text=self._snippets[hit.idx], score=hit.score))
+
+        return tuple(out)

package/retrieval/symbolic_retriever.py

@@ -0,0 +1,80 @@
+"""Symbolic retriever over GCIE graphs."""
+
+from __future__ import annotations
+
+from collections import deque
+from dataclasses import dataclass
+
+import networkx as nx
+
+from .query_parser import ParsedQuery, parse_query
+
+
+@dataclass(frozen=True, slots=True)
+class SymbolicCandidate:
+    node_id: str
+    node_type: str
+    distance: int
+
+
+def _symbol_matches(text: str, symbol: str) -> bool:
+    t = text.lower()
+    s = symbol.lower()
+    return s == t or s in t
+
+
+def _seed_nodes(graph: nx.DiGraph, parsed: ParsedQuery) -> list[str]:
+    seeds: list[str] = []
+    for node_id, attrs in graph.nodes(data=True):
+        label = str(attrs.get("label", ""))
+        path = str(attrs.get("path", attrs.get("file", "")))
+        if any(_symbol_matches(node_id, sym) or _symbol_matches(label, sym) or _symbol_matches(path, sym) for sym in parsed.symbols):
+            seeds.append(node_id)
+    return sorted(set(seeds))
+
+
+def _bounded_traversal(graph: nx.DiGraph, seeds: list[str], max_hops: int) -> dict[str, int]:
+    distances: dict[str, int] = {}
+    queue: deque[tuple[str, int]] = deque((seed, 0) for seed in seeds)
+
+    while queue:
+        node, dist = queue.popleft()
+        if node in distances and dist >= distances[node]:
+            continue
+        distances[node] = dist
+
+        if dist >= max_hops:
+            continue
+
+        neighbors = set(graph.predecessors(node)).union(graph.successors(node))
+        for nxt in neighbors:
+            queue.append((nxt, dist + 1))
+
+    return distances
+
+
+def symbolic_retrieve(graph: nx.DiGraph, query: str, *, max_hops: int = 2) -> tuple[SymbolicCandidate, ...]:
+    """Retrieve symbolic candidates by seeded graph traversal."""
+    parsed = parse_query(query)
+    if not parsed.symbols:
+        return ()
+
+    seeds = _seed_nodes(graph, parsed)
+    if not seeds:
+        return ()
+
+    distances = _bounded_traversal(graph, seeds, max_hops=max_hops)
+
+    ranked: list[SymbolicCandidate] = []
+    for node_id, dist in distances.items():
+        attrs = graph.nodes[node_id]
+        ranked.append(
+            SymbolicCandidate(
+                node_id=node_id,
+                node_type=str(attrs.get("type", "unknown")),
+                distance=dist,
+            )
+        )
+
+    ranked.sort(key=lambda c: (c.distance, c.node_type, c.node_id))
+    return tuple(ranked)

package/scanner/__init__.py

@@ -0,0 +1,5 @@
+"""Repository scanning package for GCIE."""
+
+from .repository_scanner import scan_repository
+
+__all__ = ["scan_repository"]

package/scanner/file_filters.py

@@ -0,0 +1,37 @@
+"""Filtering and classification helpers for repository scanning."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+CONFIG_FILENAMES = {
+    "pyproject.toml",
+    "requirements.txt",
+    "requirements-dev.txt",
+    "setup.py",
+    "setup.cfg",
+    "tox.ini",
+    ".env",
+}
+
+CONFIG_EXTENSIONS = {".toml", ".ini", ".cfg", ".yaml", ".yml", ".json"}
+
+
+def classify_file(relative_path: Path) -> str:
+    """Classify file type for downstream indexing priorities."""
+    name = relative_path.name.lower()
+    suffix = relative_path.suffix.lower()
+    rel = relative_path.as_posix().lower()
+
+    if "/tests/" in f"/{rel}" or name.startswith("test_") or name.endswith("_test.py"):
+        return "test"
+    if name in CONFIG_FILENAMES or suffix in CONFIG_EXTENSIONS:
+        return "config"
+    if suffix in {".py", ".pyi"}:
+        return "source"
+    return "other"
+
+
+def should_skip_hidden_file(path: Path, include_hidden: bool) -> bool:
+    """Exclude hidden files unless explicitly enabled."""
+    return (not include_hidden) and path.name.startswith(".")

package/scanner/models.py

@@ -0,0 +1,44 @@
+"""Data models for repository scanning."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal
+
+FileKind = Literal["source", "test", "config", "other"]
+
+
+@dataclass(frozen=True, slots=True)
+class ScannedFile:
+    """A file discovered by the repository scanner."""
+
+    path: Path
+    relative_path: Path
+    size_bytes: int
+    suffix: str
+    kind: FileKind
+
+
+@dataclass(frozen=True, slots=True)
+class RepositoryManifest:
+    """Normalized scanner output for downstream indexing."""
+
+    root: Path
+    files: tuple[ScannedFile, ...]
+
+    @property
+    def total_files(self) -> int:
+        return len(self.files)
+
+    @property
+    def source_files(self) -> tuple[ScannedFile, ...]:
+        return tuple(f for f in self.files if f.kind == "source")
+
+    @property
+    def test_files(self) -> tuple[ScannedFile, ...]:
+        return tuple(f for f in self.files if f.kind == "test")
+
+    @property
+    def config_files(self) -> tuple[ScannedFile, ...]:
+        return tuple(f for f in self.files if f.kind == "config")

package/scanner/repository_scanner.py

@@ -0,0 +1,55 @@
+"""Repository scanner implementation."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from config.scanner_config import ScannerConfig
+
+from .file_filters import classify_file, should_skip_hidden_file
+from .models import RepositoryManifest, ScannedFile
+
+
+def _iter_candidate_files(root: Path, config: ScannerConfig):
+    """Yield candidate file paths in deterministic order."""
+    for current_root, dirs, files in os.walk(root, topdown=True):
+        dirs[:] = sorted(d for d in dirs if not config.is_excluded_dir(d))
+        for file_name in sorted(files):
+            candidate = Path(current_root) / file_name
+            rel = candidate.relative_to(root)
+
+            if should_skip_hidden_file(candidate, config.include_hidden):
+                continue
+            if config.matches_exclude_glob(rel):
+                continue
+            if not config.allows_extension(candidate):
+                continue
+            try:
+                size_bytes = candidate.stat().st_size
+            except OSError:
+                continue
+            if size_bytes > config.max_file_size_bytes:
+                continue
+            yield candidate, rel, size_bytes
+
+
+def scan_repository(root: str | Path, config: ScannerConfig | None = None) -> RepositoryManifest:
+    """Scan a repository and return a normalized manifest."""
+    scan_config = config or ScannerConfig()
+    root_path = Path(root).resolve()
+
+    files: list[ScannedFile] = []
+    for candidate, rel, size_bytes in _iter_candidate_files(root_path, scan_config):
+        files.append(
+            ScannedFile(
+                path=candidate,
+                relative_path=rel,
+                size_bytes=size_bytes,
+                suffix=candidate.suffix.lower(),
+                kind=classify_file(rel),
+            )
+        )
+
+    files.sort(key=lambda item: item.relative_path.as_posix())
+    return RepositoryManifest(root=root_path, files=tuple(files))