contextl 1.2.3 → 1.2.5

package/README.md

@@ -86,43 +86,6 @@ Takes the exact dependency graph built by the intelligence engine and physically
 
 ---
 
-## AI System Prompt (Query Optimizer)
-
-ContextL is a deterministic keyword and graph engine, not a semantic AI. For maximum accuracy, inject this system prompt into your AI agent's rules (e.g., `.cursorrules` or custom instructions). It forces the AI to translate your natural language questions into highly optimized, lexical keyword strings before hitting the `query_repo` tool.
-
-```text
-System Instruction: You are a translation layer designed to convert human requests into highly optimized queries for the contextl codebase search engine. The contextl engine is NOT a semantic AI; it is an advanced, dependency-aware fuzzy keyword searcher. It relies on exact token matching and import-graph ranking.
-
-Your goal is to extract the core technical keywords from the user's request and discard all conversational filler or vague concepts.
-
-Rules for generating the query string:
-
-1. Remove Conversational Filler: Drop words like "where is", "how does", "find the", "show me".
-2. Translate Concepts to Code: If the user asks for a concept, translate it into the exact keywords, classes, or library syntax a developer would type.
-Example: "routing" ➡️ createRouter Route routeTree
-Example: "entry point" ➡️ main SpringBootApplication bootstrap web.xml init
-Example: "database schema" ➡️ Entity Column Table model schema
-3. Include Known Framework Terms: If you know the language or framework of the codebase, append its standard terminology.
-4. Target File Types/Structures: If looking for configurations, use terms like config properties yaml xml json.
-5. Keep it space-separated: Output a single string of space-separated keywords without punctuation.
-
-Examples:
-
-User: "Where are the main react components and routing defined?" 
-Optimized Query: createRouter Route routeTree components layout
-
-User: "Find the application entry point and main method." 
-Optimized Query: public static void main bootstrap init ApplicationContext configuration
-
-User: "How is the user authentication handled?" 
-Optimized Query: login auth authenticate jwt token session passport
-
-Input: [Insert human request here] 
-Output: [Return ONLY the optimized keyword string]
-```
-
----
-
 ## How the ranking works
 
 1. **Keyword match** — does the filename contain query terms?

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "contextl",
-  "version": "1.2.3",
+  "version": "1.2.5",
   "description": "contextl — finds the most relevant files in your codebase for any change request. MCP server for AI coding agents.",
   "keywords": [
     "mcp",

package/python/import_parser.py

@@ -92,19 +92,25 @@ def _extract_raw_imports(source_code: str, extension: str) -> list[str]:
     return list(set(paths))
 
 
-def _is_external(import_path: str, extension: str) -> bool:
+def _is_external(import_path: str, extension: str, alias_map: dict[str, str] = None) -> bool:
     """Return True if this is an external package rather than a local file."""
     if extension in {".py", ".java"}:
         # For Python and Java, it's hard to tell without resolving, 
         # so assume it's local and let the resolver fail if it's external.
         return False
 
-    # Node.js: Local files start with . (relative) or @ followed by / (alias like @/)
-    # but NOT @scope/package style from npm — those don't contain our alias prefix
+    # Node.js: Local files start with . (relative) or match an alias
     if import_path.startswith("./") or import_path.startswith("../"):
         return False
+    
+    if alias_map:
+        for alias in alias_map:
+            if import_path.startswith(alias):
+                return False
+
     if import_path.startswith("@/"):
         return False
+        
     return True  # Everything else is an external package
 
 
@@ -212,16 +218,18 @@ def _detect_alias_map(scan_result: ScanResult) -> dict[str, str]:
     Returns a map like {"@/": "frontend/"} or {"@/": ""}
     """
     root = Path(scan_result.root)
+    alias_map = {}
 
     # Look for tsconfig.json to find paths config
     for tsconfig in root.rglob("tsconfig.json"):
         try:
             import json
-            data = json.loads(tsconfig.read_text())
+            # naive regex to strip comments from tsconfig before parsing
+            text = tsconfig.read_text()
+            text = re.sub(r'//.*?\n|/\*.*?\*/', '', text, flags=re.S)
+            data = json.loads(text)
             paths = data.get("compilerOptions", {}).get("paths", {})
-            base_url = data.get("compilerOptions", {}).get("baseUrl", ".")
 
-            alias_map = {}
             for alias, targets in paths.items():
                 if not targets:
                     continue
@@ -232,19 +240,19 @@ def _detect_alias_map(scan_result: ScanResult) -> dict[str, str]:
                 tsconfig_dir = tsconfig.parent.relative_to(root)
                 prefix = str(tsconfig_dir / clean_target).lstrip("./")
                 alias_map[clean_alias + "/"] = prefix + "/" if prefix else ""
-
-            if alias_map:
-                return alias_map
         except Exception:
             continue
 
-    # Heuristic fallback: if all files share a common top-level dir, use that
-    top_dirs = {Path(f.path).parts[0] for f in scan_result.files if Path(f.path).parts}
-    if len(top_dirs) == 1:
-        top = list(top_dirs)[0]
-        return {"@/": top + "/"}
+    if not alias_map:
+        # Heuristic fallback: if all files share a common top-level dir, use that
+        top_dirs = {Path(f.path).parts[0] for f in scan_result.files if Path(f.path).parts}
+        if len(top_dirs) == 1:
+            top = list(top_dirs)[0]
+            alias_map["@/"] = top + "/"
+        else:
+            alias_map["@/"] = ""
 
-    return {"@/": ""}
+    return alias_map
 
 
 def parse_imports(scan_result: ScanResult) -> ParseResult:
@@ -275,7 +283,7 @@ def parse_imports(scan_result: ScanResult) -> ParseResult:
         raw_imports = _extract_raw_imports(source_code, scanned_file.extension)
 
         for raw in raw_imports:
-            if _is_external(raw, scanned_file.extension):
+            if _is_external(raw, scanned_file.extension, alias_map):
                 result.unresolved.append((scanned_file.path, raw))
                 continue
 
@@ -298,11 +306,8 @@ def parse_imports(scan_result: ScanResult) -> ParseResult:
                     candidate = raw.replace(".", "/")
             else:
                 # Node.js
-                if raw.startswith("@/"):
-                    resolved_alias = _resolve_alias(raw, alias_map)
-                    if resolved_alias is None:
-                        result.unresolved.append((scanned_file.path, raw))
-                        continue
+                resolved_alias = _resolve_alias(raw, alias_map)
+                if resolved_alias is not None:
                     candidate = resolved_alias
                 else:
                     candidate = _resolve_relative(raw, scanned_file.path)

package/python/mcp_server.py

@@ -71,9 +71,19 @@ async def list_tools() -> list[types.Tool]:
                     "query": {
                         "type": "string",
                         "description": (
-                            "Natural-language description of the change you want to make. "
-                            "Examples: 'fix the upload error handler', "
-                            "'change the login page logo', 'add dark mode to footer'."
+                            "SYSTEM INSTRUCTION: You are a translation layer designed to convert human requests into highly optimized queries for the contextl codebase search engine. "
+                            "The contextl engine is NOT a semantic AI; it is an advanced, dependency-aware fuzzy keyword searcher. It relies on exact token matching and import-graph ranking.\n\n"
+                            "Your goal is to extract the core technical keywords from the user's request and discard all conversational filler or vague concepts.\n\n"
+                            "Rules for generating the query string:\n"
+                            "1. Remove Conversational Filler: Drop words like 'where is', 'how does', 'find the', 'show me'.\n"
+                            "2. Translate Concepts to Code: If the user asks for a concept, translate it into the exact keywords, classes, or library syntax a developer would type. (e.g., 'routing' -> 'createRouter Route routeTree', 'entry point' -> 'main bootstrap init').\n"
+                            "3. Include Known Framework Terms: Append standard terminology of the framework.\n"
+                            "4. Target File Types/Structures: Use terms like 'config', 'yaml', 'json' if looking for configurations.\n"
+                            "5. Keep it space-separated: Output a single string of space-separated keywords without punctuation.\n\n"
+                            "Examples:\n"
+                            "- 'Where are the main react components and routing defined?' -> 'createRouter Route routeTree components layout'\n"
+                            "- 'Find the application entry point and main method.' -> 'public static void main bootstrap init ApplicationContext configuration'\n"
+                            "- 'How is the user authentication handled?' -> 'login auth authenticate jwt token session passport'"
                         ),
                     },
                     "top_n": {

package/python/query_engine.py

@@ -14,6 +14,7 @@ Accepts a natural-language query and ranks files by relevance using:
 No LLM. No embeddings. Pure text + graph.
 """
 
+import math
 import re
 from dataclasses import dataclass, field
 
@@ -30,6 +31,8 @@ STOP_WORDS = {
     "or", "is", "it", "this", "that", "with", "from", "how", "what",
     "where", "when", "which", "who", "make", "change", "update", "fix",
     "add", "remove", "edit", "modify", "the", "my", "our", "i", "we",
+    "string", "int", "boolean", "class", "function", "return", "import",
+    "public", "private", "logic", "handler", "data", "value"
 }
 
 
@@ -75,7 +78,7 @@ def _path_tokens(path: str) -> list[str]:
     return [t for t in tokens if t not in STOP_WORDS and len(t) > 1]
 
 
-def _keyword_score(query_terms: list[str], file_path: str) -> tuple[float, list[str]]:
+def _keyword_score(query_terms: list[str], file_path: str, idf_weights: dict[str, float]) -> tuple[float, list[str]]:
     """
     Score how well query terms match the file path / name.
     Filename matches score higher than directory matches.
@@ -88,37 +91,39 @@ def _keyword_score(query_terms: list[str], file_path: str) -> tuple[float, list[
 
     for term in query_terms:
         if term in filename_toks:
-            score += 1.0   # Strong signal: term in filename
+            score += idf_weights.get(term, 1.0)   # Strong signal: term in filename
             matched.append(term)
         elif term in path_toks:
-            score += 0.4   # Weaker signal: term in directory path
+            score += 0.4 * idf_weights.get(term, 1.0)   # Weaker signal: term in directory path
             if term not in matched:
                 matched.append(term)
 
-    # Normalize by number of query terms
-    if query_terms:
-        score = score / len(query_terms)
+    # Normalize by max possible score (sum of all IDFs)
+    max_score = sum(idf_weights.get(t, 1.0) for t in query_terms)
+    if max_score > 0:
+        score = score / max_score
 
     return score, matched
 
 
-def _content_score(query_terms: list[str], file_path: str) -> tuple[float, list[str]]:
+def _content_score(query_terms: list[str], file_content: str, idf_weights: dict[str, float]) -> tuple[float, list[str]]:
     """
     Score how often query terms appear in the file's source code.
-    Uses term frequency, capped to avoid huge files dominating.
+    Uses TF-IDF weighting, capped to avoid huge files dominating.
     """
-    try:
-        content = open(file_path, encoding="utf-8").read().lower()
-    except Exception:
-        return 0.0, []
-
-    content_tokens = set(re.findall(r"[a-z0-9]+", content))
+    content_tokens = set(re.findall(r"[a-z0-9]+", file_content))
     matched = [t for t in query_terms if t in content_tokens]
 
     if not query_terms:
         return 0.0, []
 
-    score = len(matched) / len(query_terms)
+    score = sum(idf_weights.get(t, 1.0) for t in matched)
+    max_score = sum(idf_weights.get(t, 1.0) for t in query_terms)
+    if max_score > 0:
+        score = score / max_score
+    else:
+        score = 0.0
+
     return score, matched
 
 
@@ -173,15 +178,38 @@ def query(
     if not query_terms:
         return []
 
+    # --- Pre-computation: Global TF-IDF ---
+    total_files = len(repo_graph.nodes)
+    document_frequency = {term: 0 for term in query_terms}
+    
+    file_contents = {}
+    for path, node in repo_graph.nodes.items():
+        abs_path = node.path if hasattr(node, "absolute_path") else str(__import__("pathlib").Path(repo_graph.root) / path)
+        try:
+            content = open(abs_path, encoding="utf-8").read().lower()
+            file_contents[path] = content
+            content_tokens = set(re.findall(r"[a-z0-9]+", content))
+            path_tokens = set(_path_tokens(path))
+            combined_tokens = content_tokens.union(path_tokens)
+            for term in query_terms:
+                if term in combined_tokens:
+                    document_frequency[term] += 1
+        except Exception:
+            file_contents[path] = ""
+
+    idf_weights = {}
+    for term in query_terms:
+        df = document_frequency[term]
+        idf_weights[term] = math.log(total_files / (1 + df)) if total_files > 0 else 1.0
+
     # --- Pass 1: keyword + content scores per file ---
     keyword_scores: dict[str, float] = {}
     content_scores: dict[str, float] = {}
     matched_terms: dict[str, list[str]] = {}
 
     for path, node in repo_graph.nodes.items():
-        kscore, kterms = _keyword_score(query_terms, path)
-        cscore, cterms = _content_score(query_terms, node.path if hasattr(node, "absolute_path") else
-                                        str(__import__("pathlib").Path(repo_graph.root) / path))
+        kscore, kterms = _keyword_score(query_terms, path, idf_weights)
+        cscore, cterms = _content_score(query_terms, file_contents.get(path, ""), idf_weights)
 
         keyword_scores[path] = kscore
         content_scores[path] = cscore