contextl 1.2.4 → 1.2.6

package/README.md

@@ -86,6 +86,43 @@ Takes the exact dependency graph built by the intelligence engine and physically
 
 ---
 
+## AI System Prompt (Query Optimizer)
+
+ContextL is a deterministic keyword and graph engine, not a semantic AI. For maximum accuracy, inject this system prompt into your AI agent's rules (e.g., `.cursorrules` or custom instructions). It forces the AI to translate your natural language questions into highly optimized, lexical keyword strings before hitting the `query_repo` tool.
+
+```text
+System Instruction: You are a translation layer designed to convert human requests into highly optimized queries for the contextl codebase search engine. The contextl engine is NOT a semantic AI; it is an advanced, dependency-aware fuzzy keyword searcher. It relies on exact token matching and import-graph ranking.
+
+Your goal is to extract the core technical keywords from the user's request and discard all conversational filler or vague concepts.
+
+Rules for generating the query string:
+
+1. Remove Conversational Filler: Drop words like "where is", "how does", "find the", "show me".
+2. Translate Concepts to Code: If the user asks for a concept, translate it into the exact keywords, classes, or library syntax a developer would type.
+Example: "routing" ➡️ createRouter Route routeTree
+Example: "entry point" ➡️ main SpringBootApplication bootstrap web.xml init
+Example: "database schema" ➡️ Entity Column Table model schema
+3. Include Known Framework Terms: If you know the language or framework of the codebase, append its standard terminology.
+4. Target File Types/Structures: If looking for configurations, use terms like config properties yaml xml json.
+5. Keep it space-separated: Output a single string of space-separated keywords without punctuation.
+
+Examples:
+
+User: "Where are the main react components and routing defined?" 
+Optimized Query: createRouter Route routeTree components layout
+
+User: "Find the application entry point and main method." 
+Optimized Query: public static void main bootstrap init ApplicationContext configuration
+
+User: "How is the user authentication handled?" 
+Optimized Query: login auth authenticate jwt token session passport
+
+Input: [Insert human request here] 
+Output: [Return ONLY the optimized keyword string]
+```
+
+---
+
 ## How the ranking works
 
 1. **Keyword match** — does the filename contain query terms?

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "contextl",
-  "version": "1.2.4",
+  "version": "1.2.6",
   "description": "contextl — finds the most relevant files in your codebase for any change request. MCP server for AI coding agents.",
   "keywords": [
     "mcp",

package/python/import_parser.py

@@ -92,19 +92,25 @@ def _extract_raw_imports(source_code: str, extension: str) -> list[str]:
     return list(set(paths))
 
 
-def _is_external(import_path: str, extension: str) -> bool:
+def _is_external(import_path: str, extension: str, alias_map: dict[str, str] = None) -> bool:
     """Return True if this is an external package rather than a local file."""
     if extension in {".py", ".java"}:
         # For Python and Java, it's hard to tell without resolving, 
         # so assume it's local and let the resolver fail if it's external.
         return False
 
-    # Node.js: Local files start with . (relative) or @ followed by / (alias like @/)
-    # but NOT @scope/package style from npm — those don't contain our alias prefix
+    # Node.js: Local files start with . (relative) or match an alias
     if import_path.startswith("./") or import_path.startswith("../"):
         return False
+    
+    if alias_map:
+        for alias in alias_map:
+            if import_path.startswith(alias):
+                return False
+
     if import_path.startswith("@/"):
         return False
+        
     return True  # Everything else is an external package
 
 
@@ -212,16 +218,18 @@ def _detect_alias_map(scan_result: ScanResult) -> dict[str, str]:
     Returns a map like {"@/": "frontend/"} or {"@/": ""}
     """
     root = Path(scan_result.root)
+    alias_map = {}
 
     # Look for tsconfig.json to find paths config
     for tsconfig in root.rglob("tsconfig.json"):
         try:
             import json
-            data = json.loads(tsconfig.read_text())
+            # naive regex to strip comments from tsconfig before parsing
+            text = tsconfig.read_text()
+            text = re.sub(r'//.*?\n|/\*.*?\*/', '', text, flags=re.S)
+            data = json.loads(text)
             paths = data.get("compilerOptions", {}).get("paths", {})
-            base_url = data.get("compilerOptions", {}).get("baseUrl", ".")
 
-            alias_map = {}
             for alias, targets in paths.items():
                 if not targets:
                     continue
@@ -232,19 +240,19 @@ def _detect_alias_map(scan_result: ScanResult) -> dict[str, str]:
                 tsconfig_dir = tsconfig.parent.relative_to(root)
                 prefix = str(tsconfig_dir / clean_target).lstrip("./")
                 alias_map[clean_alias + "/"] = prefix + "/" if prefix else ""
-
-            if alias_map:
-                return alias_map
         except Exception:
             continue
 
-    # Heuristic fallback: if all files share a common top-level dir, use that
-    top_dirs = {Path(f.path).parts[0] for f in scan_result.files if Path(f.path).parts}
-    if len(top_dirs) == 1:
-        top = list(top_dirs)[0]
-        return {"@/": top + "/"}
+    if not alias_map:
+        # Heuristic fallback: if all files share a common top-level dir, use that
+        top_dirs = {Path(f.path).parts[0] for f in scan_result.files if Path(f.path).parts}
+        if len(top_dirs) == 1:
+            top = list(top_dirs)[0]
+            alias_map["@/"] = top + "/"
+        else:
+            alias_map["@/"] = ""
 
-    return {"@/": ""}
+    return alias_map
 
 
 def parse_imports(scan_result: ScanResult) -> ParseResult:
@@ -275,7 +283,7 @@ def parse_imports(scan_result: ScanResult) -> ParseResult:
         raw_imports = _extract_raw_imports(source_code, scanned_file.extension)
 
         for raw in raw_imports:
-            if _is_external(raw, scanned_file.extension):
+            if _is_external(raw, scanned_file.extension, alias_map):
                 result.unresolved.append((scanned_file.path, raw))
                 continue
 
@@ -298,11 +306,8 @@ def parse_imports(scan_result: ScanResult) -> ParseResult:
                     candidate = raw.replace(".", "/")
             else:
                 # Node.js
-                if raw.startswith("@/"):
-                    resolved_alias = _resolve_alias(raw, alias_map)
-                    if resolved_alias is None:
-                        result.unresolved.append((scanned_file.path, raw))
-                        continue
+                resolved_alias = _resolve_alias(raw, alias_map)
+                if resolved_alias is not None:
                     candidate = resolved_alias
                 else:
                     candidate = _resolve_relative(raw, scanned_file.path)

package/python/query_engine.py

@@ -14,6 +14,7 @@ Accepts a natural-language query and ranks files by relevance using:
 No LLM. No embeddings. Pure text + graph.
 """
 
+import math
 import re
 from dataclasses import dataclass, field
 
@@ -30,6 +31,8 @@ STOP_WORDS = {
     "or", "is", "it", "this", "that", "with", "from", "how", "what",
     "where", "when", "which", "who", "make", "change", "update", "fix",
     "add", "remove", "edit", "modify", "the", "my", "our", "i", "we",
+    "string", "int", "boolean", "class", "function", "return", "import",
+    "public", "private", "logic", "handler", "data", "value"
 }
 
 
@@ -57,6 +60,38 @@ class RankedFile:
         )
 
 
+@dataclass
+class FileFeatures:
+    tokens: set[str]
+    annotations: set[str]
+    declarations: set[str]
+
+
+def _extract_file_features(content: str, extension: str) -> FileFeatures:
+    """Extract structural heuristics and lowercase tokens from a file."""
+    # 1. Declarations: class X, interface Y, enum Z
+    decl_pattern = r"\b(?:class|interface|enum)\s+([A-Za-z0-9_]+)"
+    declarations = {m.group(1).lower() for m in re.finditer(decl_pattern, content)}
+
+    # 2. Annotations: @Controller, @Injectable
+    annotations = {m.group(1).lower() for m in re.finditer(r"@([A-Za-z0-9_]+)", content)}
+
+    # 3. Base tokens
+    tokens = set(re.findall(r"[a-z0-9]+", content.lower()))
+
+    # 4. CamelCase / PascalCase Splitting for structural languages
+    if extension in {".java", ".ts", ".tsx"}:
+        for word in re.findall(r"[A-Za-z]+", content):
+            if not word.islower() and not word.isupper() and len(word) > 2:
+                # Split CamelCase
+                sub = re.sub(r"([a-z])([A-Z])", r"\1 \2", word).lower()
+                for part in re.findall(r"[a-z0-9]+", sub):
+                    if len(part) > 1:
+                        tokens.add(part)
+
+    return FileFeatures(tokens=tokens, annotations=annotations, declarations=declarations)
+
+
 def _tokenize(text: str) -> list[str]:
     """Lowercase, split on non-alphanumeric, remove stop words."""
     tokens = re.findall(r"[a-z0-9]+", text.lower())
@@ -75,7 +110,7 @@ def _path_tokens(path: str) -> list[str]:
     return [t for t in tokens if t not in STOP_WORDS and len(t) > 1]
 
 
-def _keyword_score(query_terms: list[str], file_path: str) -> tuple[float, list[str]]:
+def _keyword_score(query_terms: list[str], file_path: str, idf_weights: dict[str, float]) -> tuple[float, list[str]]:
     """
     Score how well query terms match the file path / name.
     Filename matches score higher than directory matches.
@@ -88,37 +123,47 @@ def _keyword_score(query_terms: list[str], file_path: str) -> tuple[float, list[
 
     for term in query_terms:
         if term in filename_toks:
-            score += 1.0   # Strong signal: term in filename
+            score += idf_weights.get(term, 1.0)   # Strong signal: term in filename
             matched.append(term)
         elif term in path_toks:
-            score += 0.4   # Weaker signal: term in directory path
+            score += 0.4 * idf_weights.get(term, 1.0)   # Weaker signal: term in directory path
             if term not in matched:
                 matched.append(term)
 
-    # Normalize by number of query terms
-    if query_terms:
-        score = score / len(query_terms)
+    # Normalize by max possible score (sum of all IDFs)
+    max_score = sum(idf_weights.get(t, 1.0) for t in query_terms)
+    if max_score > 0:
+        score = score / max_score
 
     return score, matched
 
 
-def _content_score(query_terms: list[str], file_path: str) -> tuple[float, list[str]]:
+def _content_score(query_terms: list[str], features: "FileFeatures", idf_weights: dict[str, float]) -> tuple[float, list[str]]:
     """
     Score how often query terms appear in the file's source code.
-    Uses term frequency, capped to avoid huge files dominating.
+    Applies massive multipliers for architectural declarations and annotations.
     """
-    try:
-        content = open(file_path, encoding="utf-8").read().lower()
-    except Exception:
+    if not features or not query_terms:
         return 0.0, []
 
-    content_tokens = set(re.findall(r"[a-z0-9]+", content))
-    matched = [t for t in query_terms if t in content_tokens]
+    matched = [t for t in query_terms if t in features.tokens]
 
-    if not query_terms:
-        return 0.0, []
+    score = 0.0
+    for t in matched:
+        base_weight = idf_weights.get(t, 1.0)
+        multiplier = 1.0
+        if t in features.declarations:
+            multiplier = 10.0
+        elif t in features.annotations:
+            multiplier = 5.0
+        score += base_weight * multiplier
+
+    max_score = sum(idf_weights.get(t, 1.0) for t in query_terms)
+    if max_score > 0:
+        score = score / max_score
+    else:
+        score = 0.0
 
-    score = len(matched) / len(query_terms)
     return score, matched
 
 
@@ -173,15 +218,42 @@ def query(
     if not query_terms:
         return []
 
+    # --- Pre-computation: Global TF-IDF ---
+    import pathlib
+    total_files = len(repo_graph.nodes)
+    document_frequency = {term: 0 for term in query_terms}
+    
+    file_features_cache = {}
+    for path, node in repo_graph.nodes.items():
+        abs_path = node.path if hasattr(node, "absolute_path") else str(__import__("pathlib").Path(repo_graph.root) / path)
+        extension = pathlib.Path(path).suffix.lower()
+        
+        try:
+            content = open(abs_path, encoding="utf-8").read()
+            features = _extract_file_features(content, extension)
+            file_features_cache[path] = features
+            
+            path_tokens = set(_path_tokens(path))
+            combined_tokens = features.tokens.union(path_tokens)
+            for term in query_terms:
+                if term in combined_tokens:
+                    document_frequency[term] += 1
+        except Exception:
+            file_features_cache[path] = None
+
+    idf_weights = {}
+    for term in query_terms:
+        df = document_frequency[term]
+        idf_weights[term] = math.log(total_files / (1 + df)) if total_files > 0 else 1.0
+
     # --- Pass 1: keyword + content scores per file ---
     keyword_scores: dict[str, float] = {}
     content_scores: dict[str, float] = {}
     matched_terms: dict[str, list[str]] = {}
 
     for path, node in repo_graph.nodes.items():
-        kscore, kterms = _keyword_score(query_terms, path)
-        cscore, cterms = _content_score(query_terms, node.path if hasattr(node, "absolute_path") else
-                                        str(__import__("pathlib").Path(repo_graph.root) / path))
+        kscore, kterms = _keyword_score(query_terms, path, idf_weights)
+        cscore, cterms = _content_score(query_terms, file_features_cache.get(path), idf_weights)
 
         keyword_scores[path] = kscore
         content_scores[path] = cscore