contextl 1.2.6 → 1.2.8

package/README.md

@@ -84,44 +84,7 @@ Finds unused files and dead code by analyzing the dependency graph for files wit
 
 Takes the exact dependency graph built by the intelligence engine and physically writes it to disk as a directory of interconnected Markdown files. Automatically injects file metadata, JSDoc/Docstring explanations, and uses standard `[[wikilinks]]` to map out dependencies. Open the generated folder as an Obsidian vault for a stunning 3D interactive graph of your architecture!
 
----
-
-## AI System Prompt (Query Optimizer)
-
-ContextL is a deterministic keyword and graph engine, not a semantic AI. For maximum accuracy, inject this system prompt into your AI agent's rules (e.g., `.cursorrules` or custom instructions). It forces the AI to translate your natural language questions into highly optimized, lexical keyword strings before hitting the `query_repo` tool.
-
-```text
-System Instruction: You are a translation layer designed to convert human requests into highly optimized queries for the contextl codebase search engine. The contextl engine is NOT a semantic AI; it is an advanced, dependency-aware fuzzy keyword searcher. It relies on exact token matching and import-graph ranking.
-
-Your goal is to extract the core technical keywords from the user's request and discard all conversational filler or vague concepts.
-
-Rules for generating the query string:
-
-1. Remove Conversational Filler: Drop words like "where is", "how does", "find the", "show me".
-2. Translate Concepts to Code: If the user asks for a concept, translate it into the exact keywords, classes, or library syntax a developer would type.
-Example: "routing" ➡️ createRouter Route routeTree
-Example: "entry point" ➡️ main SpringBootApplication bootstrap web.xml init
-Example: "database schema" ➡️ Entity Column Table model schema
-3. Include Known Framework Terms: If you know the language or framework of the codebase, append its standard terminology.
-4. Target File Types/Structures: If looking for configurations, use terms like config properties yaml xml json.
-5. Keep it space-separated: Output a single string of space-separated keywords without punctuation.
 
-Examples:
-
-User: "Where are the main react components and routing defined?" 
-Optimized Query: createRouter Route routeTree components layout
-
-User: "Find the application entry point and main method." 
-Optimized Query: public static void main bootstrap init ApplicationContext configuration
-
-User: "How is the user authentication handled?" 
-Optimized Query: login auth authenticate jwt token session passport
-
-Input: [Insert human request here] 
-Output: [Return ONLY the optimized keyword string]
-```
-
----
 
 ## How the ranking works
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "contextl",
-  "version": "1.2.6",
+  "version": "1.2.8",
   "description": "contextl — finds the most relevant files in your codebase for any change request. MCP server for AI coding agents.",
   "keywords": [
     "mcp",

package/python/query_engine.py

@@ -32,7 +32,9 @@ STOP_WORDS = {
     "where", "when", "which", "who", "make", "change", "update", "fix",
     "add", "remove", "edit", "modify", "the", "my", "our", "i", "we",
     "string", "int", "boolean", "class", "function", "return", "import",
-    "public", "private", "logic", "handler", "data", "value"
+    "public", "private", "logic", "handler", "data", "value", "yo", "bro",
+    "bruh", "pls", "please", "bug", "issue", "error", "help", "stuff",
+    "handling", "failing", "why", "need", "can", "could", "would", "should", "fr"
 }
 
 
@@ -60,37 +62,6 @@ class RankedFile:
         )
 
 
-@dataclass
-class FileFeatures:
-    tokens: set[str]
-    annotations: set[str]
-    declarations: set[str]
-
-
-def _extract_file_features(content: str, extension: str) -> FileFeatures:
-    """Extract structural heuristics and lowercase tokens from a file."""
-    # 1. Declarations: class X, interface Y, enum Z
-    decl_pattern = r"\b(?:class|interface|enum)\s+([A-Za-z0-9_]+)"
-    declarations = {m.group(1).lower() for m in re.finditer(decl_pattern, content)}
-
-    # 2. Annotations: @Controller, @Injectable
-    annotations = {m.group(1).lower() for m in re.finditer(r"@([A-Za-z0-9_]+)", content)}
-
-    # 3. Base tokens
-    tokens = set(re.findall(r"[a-z0-9]+", content.lower()))
-
-    # 4. CamelCase / PascalCase Splitting for structural languages
-    if extension in {".java", ".ts", ".tsx"}:
-        for word in re.findall(r"[A-Za-z]+", content):
-            if not word.islower() and not word.isupper() and len(word) > 2:
-                # Split CamelCase
-                sub = re.sub(r"([a-z])([A-Z])", r"\1 \2", word).lower()
-                for part in re.findall(r"[a-z0-9]+", sub):
-                    if len(part) > 1:
-                        tokens.add(part)
-
-    return FileFeatures(tokens=tokens, annotations=annotations, declarations=declarations)
-
 
 def _tokenize(text: str) -> list[str]:
     """Lowercase, split on non-alphanumeric, remove stop words."""
@@ -122,55 +93,154 @@ def _keyword_score(query_terms: list[str], file_path: str, idf_weights: dict[str
     score = 0.0
 
     for term in query_terms:
+        # Exact match
         if term in filename_toks:
-            score += idf_weights.get(term, 1.0)   # Strong signal: term in filename
+            score += 2.0 * idf_weights.get(term, 1.0)   # Massive signal
             matched.append(term)
+        # Substring match on filename
+        elif len(term) > 3 and any(term in ft for ft in filename_toks):
+            score += 1.5 * idf_weights.get(term, 1.0)
+            matched.append(term)
+        # Exact match on path
         elif term in path_toks:
-            score += 0.4 * idf_weights.get(term, 1.0)   # Weaker signal: term in directory path
+            score += 0.8 * idf_weights.get(term, 1.0)   # Strong signal
+            if term not in matched:
+                matched.append(term)
+        # Substring match on path
+        elif len(term) > 3 and any(term in pt for pt in path_toks):
+            score += 0.4 * idf_weights.get(term, 1.0)
             if term not in matched:
                 matched.append(term)
 
     # Normalize by max possible score (sum of all IDFs)
     max_score = sum(idf_weights.get(t, 1.0) for t in query_terms)
     if max_score > 0:
-        score = score / max_score
+        score = min(1.0, score / max_score) # Cap at 1.0 since we have massive multipliers
 
     return score, matched
 
 
-def _content_score(query_terms: list[str], features: "FileFeatures", idf_weights: dict[str, float]) -> tuple[float, list[str]]:
+def _content_score(query_terms: list[str], file_path: str, file_content: str, idf_weights: dict[str, float], dl: int, avgdl: float) -> tuple[float, list[str]]:
     """
     Score how often query terms appear in the file's source code.
-    Applies massive multipliers for architectural declarations and annotations.
+    Uses BM25 Term Frequency weighting to neutralize document length bias.
+    Includes language-specific heuristics for Java and other OOP languages.
     """
-    if not features or not query_terms:
+    is_java = file_path.endswith(".java")
+    is_ts = file_path.endswith(".ts") or file_path.endswith(".tsx")
+    is_oop = is_java or is_ts
+    
+    content_tokens_list = re.findall(r"[a-z0-9]+", file_content.lower())
+    
+    # 1. CamelCase Splitting (Java only)
+    if is_java:
+        camel_tokens = []
+        for word in re.findall(r"[a-zA-Z]+", file_content):
+            if any(c.isupper() for c in word):
+                sub = re.sub(r"([A-Z])", r" \1", word).lower()
+                camel_tokens.extend(re.findall(r"[a-z0-9]+", sub))
+        content_tokens_list.extend([t for t in camel_tokens if len(t) > 1])
+
+    # 2. Extract Class Declarations, Annotations & Method Names
+    declared_classes = set()
+    ts_exports = set()
+    java_methods = set()
+    if is_oop or file_path.endswith(".py"):
+        decls = re.findall(r"\b(?:class|interface|enum)\s+([a-zA-Z0-9_]+)", file_content)
+        for d in decls:
+            declared_classes.add(d.lower())
+            
+    if is_java:
+        # Extract Java method names: public/private/protected ... methodName(
+        methods = re.findall(r"\b(?:public|private|protected)\s+(?:static\s+)?(?:[a-zA-Z0-9_<>\[\]]+)\s+([a-zA-Z0-9_]+)\s*\(", file_content)
+        for m in methods:
+            if m.lower() not in {"main", "get", "set", "is", "has"}:
+                java_methods.add(m.lower())
+                # Also add CamelCase parts of method names
+                sub = re.sub(r"([A-Z])", r" \1", m).lower()
+                for part in re.findall(r"[a-z0-9]+", sub):
+                    if len(part) > 3:
+                        java_methods.add(part)
+            
+    if is_ts:
+        exports = re.findall(r"\bexport\s+(?:const|let|var|class|interface|function|default)\s+([a-zA-Z0-9_]+)", file_content)
+        for e in exports:
+            ts_exports.add(e.lower())
+
+    annotations = set()
+    if is_oop or file_path.endswith(".py"):
+        anns = re.findall(r"@([a-zA-Z0-9_]+)", file_content)
+        for a in anns:
+            annotations.add(a.lower())
+
+    term_counts = {}
+    for t in query_terms:
+        count = 0
+        for ct in content_tokens_list:
+            if t == ct:
+                count += 1
+            elif len(t) > 3 and t in ct:
+                count += 1
+        if count > 0:
+            term_counts[t] = count
+            
+    matched = list(term_counts.keys())
+
+    if not query_terms:
         return 0.0, []
 
-    matched = [t for t in query_terms if t in features.tokens]
+    # Multiplier strength scales inversely with query length.
+    # Short queries (1-2 terms, likely vibe) need a loud structural signal.
+    # Long queries (4+ terms, likely standard LLM) rely more on BM25 math.
+    n_terms = len(query_terms)
+    if n_terms <= 2:
+        class_mult = 10.0
+        ann_mult = 5.0
+    elif n_terms <= 4:
+        class_mult = 6.0
+        ann_mult = 3.0
+    else:
+        class_mult = 3.0
+        ann_mult = 2.0
 
     score = 0.0
     for t in matched:
         base_weight = idf_weights.get(t, 1.0)
         multiplier = 1.0
-        if t in features.declarations:
-            multiplier = 10.0
-        elif t in features.annotations:
-            multiplier = 5.0
-        score += base_weight * multiplier
+        
+        # 3. Apply Multipliers — TS exports always get 8x regardless of query length
+        if t in ts_exports:
+            multiplier = 8.0
+        elif t in declared_classes:
+            multiplier = class_mult
+        elif t in java_methods:
+            multiplier = 4.0
+        elif t in annotations:
+            multiplier = ann_mult
+            
+        # 4. BM25 Term Frequency
+        count = term_counts[t]
+        k1 = 1.5
+        b = 0.75
+        tf_bm25 = (count * (k1 + 1)) / (count + k1 * (1 - b + b * (dl / avgdl)))
+            
+        score += (tf_bm25 * base_weight * multiplier)
 
     max_score = sum(idf_weights.get(t, 1.0) for t in query_terms)
     if max_score > 0:
-        score = score / max_score
+        base_normalized = min(1.0, sum(idf_weights.get(t, 1.0) for t in matched) / max_score)
+        bonus = score - sum(idf_weights.get(t, 1.0) for t in matched)
+        final_score = base_normalized + bonus
     else:
-        score = 0.0
+        final_score = 0.0
 
-    return score, matched
+    return final_score, matched
 
 
 def _apply_neighbor_bonus(
     scores: dict[str, float],
     repo_graph: RepoGraph,
-    boost: float = 0.15,
+    boost: float = 0.35,
     depth: int = 1,
 ) -> dict[str, float]:
     """
@@ -218,33 +288,59 @@ def query(
     if not query_terms:
         return []
 
-    # --- Pre-computation: Global TF-IDF ---
+    # --- Pre-computation: Global TF-IDF & BM25 ---
     import pathlib
     total_files = len(repo_graph.nodes)
     document_frequency = {term: 0 for term in query_terms}
     
-    file_features_cache = {}
+    file_contents = {}
+    doc_lengths = {}
+    total_tokens_count = 0
+    
     for path, node in repo_graph.nodes.items():
         abs_path = node.path if hasattr(node, "absolute_path") else str(__import__("pathlib").Path(repo_graph.root) / path)
-        extension = pathlib.Path(path).suffix.lower()
-        
         try:
             content = open(abs_path, encoding="utf-8").read()
-            features = _extract_file_features(content, extension)
-            file_features_cache[path] = features
+            file_contents[path] = content
             
+            raw_tokens = re.findall(r"[a-z0-9]+", content.lower())
+            dl = len(raw_tokens)
+            doc_lengths[path] = dl
+            total_tokens_count += dl
+            
+            content_tokens = set(raw_tokens)
+            
+            if path.endswith(".java"):
+                camel_tokens = []
+                for word in re.findall(r"[a-zA-Z]+", content):
+                    if any(c.isupper() for c in word):
+                        sub = re.sub(r"([A-Z])", r" \1", word).lower()
+                        camel_tokens.extend(re.findall(r"[a-z0-9]+", sub))
+                content_tokens.update([t for t in camel_tokens if len(t) > 1])
+                
             path_tokens = set(_path_tokens(path))
-            combined_tokens = features.tokens.union(path_tokens)
+            combined_tokens = content_tokens.union(path_tokens)
             for term in query_terms:
                 if term in combined_tokens:
                     document_frequency[term] += 1
+                elif len(term) > 3:
+                    for ct in combined_tokens:
+                        if term in ct:
+                            document_frequency[term] += 1
+                            break
         except Exception:
-            file_features_cache[path] = None
+            file_contents[path] = ""
+            doc_lengths[path] = 0
+
+    avgdl = total_tokens_count / total_files if total_files > 0 else 1.0
 
     idf_weights = {}
     for term in query_terms:
         df = document_frequency[term]
-        idf_weights[term] = math.log(total_files / (1 + df)) if total_files > 0 else 1.0
+        if df == 0:
+            idf_weights[term] = 0.0
+        else:
+            idf_weights[term] = math.log(total_files / (1 + df)) if total_files > 0 else 1.0
 
     # --- Pass 1: keyword + content scores per file ---
     keyword_scores: dict[str, float] = {}
@@ -253,7 +349,8 @@ def query(
 
     for path, node in repo_graph.nodes.items():
         kscore, kterms = _keyword_score(query_terms, path, idf_weights)
-        cscore, cterms = _content_score(query_terms, file_features_cache.get(path), idf_weights)
+        dl = doc_lengths.get(path, 0)
+        cscore, cterms = _content_score(query_terms, path, file_contents.get(path, ""), idf_weights, dl, avgdl)
 
         keyword_scores[path] = kscore
         content_scores[path] = cscore