@zuvia-software-solutions/code-mapper 2.2.3 → 2.3.0

package/dist/cli/analyze.js

@@ -5,7 +5,7 @@ import { execFileSync } from 'child_process';
 import v8 from 'v8';
 import cliProgress from 'cli-progress';
 import { runPipelineFromRepo } from '../core/ingestion/pipeline.js';
-import { openDb, closeDb, resetDb, getStats, insertEmbeddingsBatch, countEmbeddings } from '../core/db/adapter.js';
+import { openDb, closeDb, resetDb, getStats, insertEmbeddingsBatch, countEmbeddings, populateSearchText } from '../core/db/adapter.js';
 import { loadGraphToDb } from '../core/db/graph-loader.js';
 import { stitchRoutes } from '../core/ingestion/route-stitcher.js';
 import { toNodeId } from '../core/db/schema.js';
@@ -200,6 +200,10 @@ export const analyzeCommand = async (inputPath, options) => {
     const dbWarnings = dbResult.warnings;
     // Phase 2.5: HTTP route stitching (post-DB-load, needs content field)
     stitchRoutes(db);
+    // Phase 2.6: Populate searchText for BM25 concept matching
+    // Uses first comment + callers + module — must run after edges are loaded
+    updateBar(84, 'Building search index...');
+    populateSearchText(db);
     // Phase 3: FTS (85-90%)
     // FTS5 is auto-created by schema triggers — no manual index creation needed
     updateBar(85, 'Search indexes ready');

package/dist/core/db/adapter.d.ts

@@ -90,6 +90,12 @@ export declare function getStats(db: Database.Database): {
 export declare function insertNodesBatch(db: Database.Database, nodes: readonly NodeInsert[]): void;
 /** Batch insert edges in a single transaction. */
 export declare function insertEdgesBatch(db: Database.Database, edges: readonly EdgeInsert[]): void;
+/**
+ * Populate the searchText column for all nodes with semantic summaries.
+ * Uses first comment + callers + module to enable BM25 concept matching.
+ * Call AFTER edges are loaded (needs CALLS and MEMBER_OF edges).
+ */
+export declare function populateSearchText(db: Database.Database): void;
 /** Batch insert embeddings in a single transaction. */
 export declare function insertEmbeddingsBatch(db: Database.Database, items: readonly {
     nodeId: NodeId;

package/dist/core/db/adapter.js

@@ -145,12 +145,12 @@ const INSERT_NODE_SQL = `
     id, label, name, filePath, startLine, endLine, isExported, content, description,
     heuristicLabel, cohesion, symbolCount, keywords, enrichedBy,
     processType, stepCount, communities, entryPointId, terminalId,
-    parameterCount, returnType, nameExpanded
+    parameterCount, returnType, nameExpanded, searchText
   ) VALUES (
     @id, @label, @name, @filePath, @startLine, @endLine, @isExported, @content, @description,
     @heuristicLabel, @cohesion, @symbolCount, @keywords, @enrichedBy,
     @processType, @stepCount, @communities, @entryPointId, @terminalId,
-    @parameterCount, @returnType, @nameExpanded
+    @parameterCount, @returnType, @nameExpanded, @searchText
   )
 `;
 /** Insert or replace a node. Automatically expands name for FTS natural language matching. */
@@ -178,6 +178,7 @@ export function insertNode(db, node) {
         parameterCount: node.parameterCount ?? null,
         returnType: node.returnType ?? null,
         nameExpanded: node.nameExpanded ?? expandIdentifier(node.name ?? ''),
+        searchText: node.searchText ?? '',
     });
 }
 /** Get a node by ID. Returns undefined if not found. */
@@ -373,6 +374,7 @@ export function insertNodesBatch(db, nodes) {
                 terminalId: node.terminalId ?? null, parameterCount: node.parameterCount ?? null,
                 returnType: node.returnType ?? null,
                 nameExpanded: node.nameExpanded ?? expandIdentifier(node.name ?? ''),
+                searchText: node.searchText ?? '',
             });
         }
     });
@@ -392,6 +394,104 @@ export function insertEdgesBatch(db, edges) {
     });
     txn(edges);
 }
+/**
+ * Populate the searchText column for all nodes with semantic summaries.
+ * Uses first comment + callers + module to enable BM25 concept matching.
+ * Call AFTER edges are loaded (needs CALLS and MEMBER_OF edges).
+ */
+export function populateSearchText(db) {
+    // Extract first comment from content
+    function extractComment(content) {
+        if (!content)
+            return '';
+        const lines = content.split('\n');
+        const out = [];
+        let inBlock = false;
+        for (const l of lines) {
+            const t = l.trim();
+            if (t.startsWith('/**') || t.startsWith('/*')) {
+                inBlock = true;
+                const inner = t.replace(/^\/\*\*?\s*/, '').replace(/\*\/\s*$/, '').trim();
+                if (inner && !inner.startsWith('@'))
+                    out.push(inner);
+                if (t.includes('*/'))
+                    inBlock = false;
+                continue;
+            }
+            if (inBlock) {
+                if (t.includes('*/')) {
+                    inBlock = false;
+                    continue;
+                }
+                const inner = t.replace(/^\*\s?/, '').trim();
+                if (inner && !inner.startsWith('@'))
+                    out.push(inner);
+                if (out.length >= 3)
+                    break;
+                continue;
+            }
+            if (t.startsWith('//')) {
+                const inner = t.slice(2).trim();
+                if (inner)
+                    out.push(inner);
+                if (out.length >= 3)
+                    break;
+                continue;
+            }
+            if (out.length > 0)
+                break;
+        }
+        return out.join(' ');
+    }
+    const nodes = db.prepare("SELECT id, name, nameExpanded, content FROM nodes WHERE label IN ('Function','Class','Method','Interface','Const','TypeAlias','Enum')").all();
+    if (nodes.length === 0)
+        return;
+    // Batch fetch callers + module
+    const callerMap = new Map();
+    const moduleMap = new Map();
+    const ids = nodes.map(n => n.id);
+    for (let i = 0; i < ids.length; i += 900) {
+        const chunk = ids.slice(i, i + 900);
+        const ph = chunk.map(() => '?').join(',');
+        const callerRows = db.prepare(`SELECT e.targetId AS nid, n.name FROM edges e JOIN nodes n ON n.id = e.sourceId WHERE e.targetId IN (${ph}) AND e.type = 'CALLS' AND e.confidence >= 0.7`).all(...chunk);
+        for (const r of callerRows) {
+            if (!callerMap.has(r.nid))
+                callerMap.set(r.nid, []);
+            callerMap.get(r.nid).push(r.name);
+        }
+        const modRows = db.prepare(`SELECT e.sourceId AS nid, c.heuristicLabel AS module FROM edges e JOIN nodes c ON c.id = e.targetId WHERE e.sourceId IN (${ph}) AND e.type = 'MEMBER_OF' AND c.label = 'Community'`).all(...chunk);
+        for (const r of modRows)
+            moduleMap.set(r.nid, r.module);
+    }
+    // Build searchText and update
+    // Drop FTS triggers temporarily to avoid column-count issues during bulk update,
+    // then rebuild the FTS index in one pass (faster than per-row trigger updates)
+    db.exec("DROP TRIGGER IF EXISTS nodes_fts_au");
+    const txn = db.transaction(() => {
+        for (const node of nodes) {
+            const parts = [];
+            if (node.nameExpanded)
+                parts.push(node.nameExpanded);
+            const comment = extractComment(node.content);
+            if (comment)
+                parts.push(comment);
+            const callers = callerMap.get(node.id)?.slice(0, 5);
+            if (callers && callers.length > 0)
+                parts.push(callers.map(c => expandIdentifier(c)).join(' '));
+            const mod = moduleMap.get(node.id);
+            if (mod)
+                parts.push(mod);
+            db.prepare('UPDATE nodes SET searchText = ? WHERE id = ?').run(parts.join(' | '), node.id);
+        }
+    });
+    txn();
+    // Rebuild FTS index from scratch and recreate the trigger
+    db.exec("INSERT INTO nodes_fts(nodes_fts) VALUES('rebuild')");
+    db.exec(`CREATE TRIGGER IF NOT EXISTS nodes_fts_au AFTER UPDATE ON nodes BEGIN
+    INSERT INTO nodes_fts(nodes_fts, rowid, name, nameExpanded, searchText, filePath, content) VALUES ('delete', old.rowid, old.name, old.nameExpanded, old.searchText, old.filePath, old.content);
+    INSERT INTO nodes_fts(nodes_fts, rowid, name, nameExpanded, searchText, filePath, content) VALUES (new.rowid, new.name, new.nameExpanded, new.searchText, new.filePath, new.content);
+  END`);
+}
 /** Batch insert embeddings in a single transaction. */
 export function insertEmbeddingsBatch(db, items) {
     const stmt = db.prepare('INSERT OR REPLACE INTO embeddings (nodeId, embedding, textHash) VALUES (?, ?, ?)');

package/dist/core/db/schema.d.ts

@@ -49,6 +49,7 @@ export interface NodeRow {
     readonly parameterCount: number | null;
     readonly returnType: string | null;
     readonly nameExpanded: string;
+    readonly searchText: string;
 }
 /** An edge row as stored in the `edges` table */
 export interface EdgeRow {
@@ -91,6 +92,7 @@ export interface NodeInsert {
     readonly parameterCount?: number | null;
     readonly returnType?: string | null;
     readonly nameExpanded?: string;
+    readonly searchText?: string;
 }
 /** Fields required to insert an edge */
 export interface EdgeInsert {
@@ -105,4 +107,4 @@ export interface EdgeInsert {
 }
 /** Legacy edge table name constant (kept for compatibility) */
 export declare const REL_TABLE_NAME = "CodeRelation";
-export declare const SCHEMA_SQL = "\n-- Nodes: unified table for all code elements\nCREATE TABLE IF NOT EXISTS nodes (\n  id TEXT PRIMARY KEY,\n  label TEXT NOT NULL,\n  name TEXT NOT NULL DEFAULT '',\n  filePath TEXT NOT NULL DEFAULT '',\n  startLine INTEGER,\n  endLine INTEGER,\n  isExported INTEGER,\n  content TEXT NOT NULL DEFAULT '',\n  description TEXT NOT NULL DEFAULT '',\n  heuristicLabel TEXT,\n  cohesion REAL,\n  symbolCount INTEGER,\n  keywords TEXT,\n  enrichedBy TEXT,\n  processType TEXT,\n  stepCount INTEGER,\n  communities TEXT,\n  entryPointId TEXT,\n  terminalId TEXT,\n  parameterCount INTEGER,\n  returnType TEXT,\n  nameExpanded TEXT DEFAULT ''\n);\n\nCREATE INDEX IF NOT EXISTS idx_nodes_label ON nodes(label);\nCREATE INDEX IF NOT EXISTS idx_nodes_name ON nodes(name);\nCREATE INDEX IF NOT EXISTS idx_nodes_filePath ON nodes(filePath);\nCREATE INDEX IF NOT EXISTS idx_nodes_label_name ON nodes(label, name);\nCREATE INDEX IF NOT EXISTS idx_nodes_filePath_lines ON nodes(filePath, startLine, endLine);\n\n-- Edges: single table for all relationships\nCREATE TABLE IF NOT EXISTS edges (\n  id TEXT PRIMARY KEY,\n  sourceId TEXT NOT NULL,\n  targetId TEXT NOT NULL,\n  type TEXT NOT NULL,\n  confidence REAL NOT NULL DEFAULT 1.0,\n  reason TEXT NOT NULL DEFAULT '',\n  step INTEGER NOT NULL DEFAULT 0,\n  callLine INTEGER\n);\n\nCREATE INDEX IF NOT EXISTS idx_edges_sourceId ON edges(sourceId);\nCREATE INDEX IF NOT EXISTS idx_edges_targetId ON edges(targetId);\nCREATE INDEX IF NOT EXISTS idx_edges_type ON edges(type);\nCREATE INDEX IF NOT EXISTS idx_edges_source_type ON edges(sourceId, type);\nCREATE INDEX IF NOT EXISTS idx_edges_target_type ON edges(targetId, type);\n\n-- Embeddings: vector storage\nCREATE TABLE IF NOT EXISTS embeddings (\n  nodeId TEXT PRIMARY KEY,\n  embedding BLOB NOT NULL,\n  textHash TEXT\n);\n\n-- FTS5 virtual table (auto-updated via triggers)\nCREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5(\n  name,\n  nameExpanded,\n  filePath,\n  content,\n  content='nodes',\n  content_rowid='rowid'\n);\n\nCREATE TRIGGER IF NOT EXISTS nodes_fts_ai AFTER INSERT ON nodes BEGIN\n  INSERT INTO nodes_fts(rowid, name, nameExpanded, filePath, content) VALUES (new.rowid, new.name, new.nameExpanded, new.filePath, new.content);\nEND;\nCREATE TRIGGER IF NOT EXISTS nodes_fts_ad AFTER DELETE ON nodes BEGIN\n  INSERT INTO nodes_fts(nodes_fts, rowid, name, nameExpanded, filePath, content) VALUES ('delete', old.rowid, old.name, old.nameExpanded, old.filePath, old.content);\nEND;\nCREATE TRIGGER IF NOT EXISTS nodes_fts_au AFTER UPDATE ON nodes BEGIN\n  INSERT INTO nodes_fts(nodes_fts, rowid, name, nameExpanded, filePath, content) VALUES ('delete', old.rowid, old.name, old.nameExpanded, old.filePath, old.content);\n  INSERT INTO nodes_fts(nodes_fts, rowid, name, nameExpanded, filePath, content) VALUES (new.rowid, new.name, new.nameExpanded, new.filePath, new.content);\nEND;\n";
+export declare const SCHEMA_SQL = "\n-- Nodes: unified table for all code elements\nCREATE TABLE IF NOT EXISTS nodes (\n  id TEXT PRIMARY KEY,\n  label TEXT NOT NULL,\n  name TEXT NOT NULL DEFAULT '',\n  filePath TEXT NOT NULL DEFAULT '',\n  startLine INTEGER,\n  endLine INTEGER,\n  isExported INTEGER,\n  content TEXT NOT NULL DEFAULT '',\n  description TEXT NOT NULL DEFAULT '',\n  heuristicLabel TEXT,\n  cohesion REAL,\n  symbolCount INTEGER,\n  keywords TEXT,\n  enrichedBy TEXT,\n  processType TEXT,\n  stepCount INTEGER,\n  communities TEXT,\n  entryPointId TEXT,\n  terminalId TEXT,\n  parameterCount INTEGER,\n  returnType TEXT,\n  nameExpanded TEXT DEFAULT '',\n  searchText TEXT DEFAULT ''\n);\n\nCREATE INDEX IF NOT EXISTS idx_nodes_label ON nodes(label);\nCREATE INDEX IF NOT EXISTS idx_nodes_name ON nodes(name);\nCREATE INDEX IF NOT EXISTS idx_nodes_filePath ON nodes(filePath);\nCREATE INDEX IF NOT EXISTS idx_nodes_label_name ON nodes(label, name);\nCREATE INDEX IF NOT EXISTS idx_nodes_filePath_lines ON nodes(filePath, startLine, endLine);\n\n-- Edges: single table for all relationships\nCREATE TABLE IF NOT EXISTS edges (\n  id TEXT PRIMARY KEY,\n  sourceId TEXT NOT NULL,\n  targetId TEXT NOT NULL,\n  type TEXT NOT NULL,\n  confidence REAL NOT NULL DEFAULT 1.0,\n  reason TEXT NOT NULL DEFAULT '',\n  step INTEGER NOT NULL DEFAULT 0,\n  callLine INTEGER\n);\n\nCREATE INDEX IF NOT EXISTS idx_edges_sourceId ON edges(sourceId);\nCREATE INDEX IF NOT EXISTS idx_edges_targetId ON edges(targetId);\nCREATE INDEX IF NOT EXISTS idx_edges_type ON edges(type);\nCREATE INDEX IF NOT EXISTS idx_edges_source_type ON edges(sourceId, type);\nCREATE INDEX IF NOT EXISTS idx_edges_target_type ON edges(targetId, type);\n\n-- Embeddings: vector storage\nCREATE TABLE IF NOT EXISTS embeddings (\n  nodeId TEXT PRIMARY KEY,\n  embedding BLOB NOT NULL,\n  textHash TEXT\n);\n\n-- FTS5 virtual table (auto-updated via triggers)\nCREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5(\n  name,\n  nameExpanded,\n  searchText,\n  filePath,\n  content,\n  content='nodes',\n  content_rowid='rowid'\n);\n\nCREATE TRIGGER IF NOT EXISTS nodes_fts_ai AFTER INSERT ON nodes BEGIN\n  INSERT INTO nodes_fts(rowid, name, nameExpanded, searchText, filePath, content) VALUES (new.rowid, new.name, new.nameExpanded, new.searchText, new.filePath, new.content);\nEND;\nCREATE TRIGGER IF NOT EXISTS nodes_fts_ad AFTER DELETE ON nodes BEGIN\n  INSERT INTO nodes_fts(nodes_fts, rowid, name, nameExpanded, searchText, filePath, content) VALUES ('delete', old.rowid, old.name, old.nameExpanded, old.searchText, old.filePath, old.content);\nEND;\nCREATE TRIGGER IF NOT EXISTS nodes_fts_au AFTER UPDATE ON nodes BEGIN\n  INSERT INTO nodes_fts(nodes_fts, rowid, name, nameExpanded, searchText, filePath, content) VALUES ('delete', old.rowid, old.name, old.nameExpanded, old.searchText, old.filePath, old.content);\n  INSERT INTO nodes_fts(nodes_fts, rowid, name, nameExpanded, searchText, filePath, content) VALUES (new.rowid, new.name, new.nameExpanded, new.searchText, new.filePath, new.content);\nEND;\n";

package/dist/core/db/schema.js

@@ -79,7 +79,8 @@ CREATE TABLE IF NOT EXISTS nodes (
   terminalId TEXT,
   parameterCount INTEGER,
   returnType TEXT,
-  nameExpanded TEXT DEFAULT ''
+  nameExpanded TEXT DEFAULT '',
+  searchText TEXT DEFAULT ''
 );
 
 CREATE INDEX IF NOT EXISTS idx_nodes_label ON nodes(label);
@@ -117,6 +118,7 @@ CREATE TABLE IF NOT EXISTS embeddings (
 CREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5(
   name,
   nameExpanded,
+  searchText,
   filePath,
   content,
   content='nodes',
@@ -124,13 +126,13 @@ CREATE VIRTUAL TABLE IF NOT EXISTS nodes_fts USING fts5(
 );
 
 CREATE TRIGGER IF NOT EXISTS nodes_fts_ai AFTER INSERT ON nodes BEGIN
-  INSERT INTO nodes_fts(rowid, name, nameExpanded, filePath, content) VALUES (new.rowid, new.name, new.nameExpanded, new.filePath, new.content);
+  INSERT INTO nodes_fts(rowid, name, nameExpanded, searchText, filePath, content) VALUES (new.rowid, new.name, new.nameExpanded, new.searchText, new.filePath, new.content);
 END;
 CREATE TRIGGER IF NOT EXISTS nodes_fts_ad AFTER DELETE ON nodes BEGIN
-  INSERT INTO nodes_fts(nodes_fts, rowid, name, nameExpanded, filePath, content) VALUES ('delete', old.rowid, old.name, old.nameExpanded, old.filePath, old.content);
+  INSERT INTO nodes_fts(nodes_fts, rowid, name, nameExpanded, searchText, filePath, content) VALUES ('delete', old.rowid, old.name, old.nameExpanded, old.searchText, old.filePath, old.content);
 END;
 CREATE TRIGGER IF NOT EXISTS nodes_fts_au AFTER UPDATE ON nodes BEGIN
-  INSERT INTO nodes_fts(nodes_fts, rowid, name, nameExpanded, filePath, content) VALUES ('delete', old.rowid, old.name, old.nameExpanded, old.filePath, old.content);
-  INSERT INTO nodes_fts(nodes_fts, rowid, name, nameExpanded, filePath, content) VALUES (new.rowid, new.name, new.nameExpanded, new.filePath, new.content);
+  INSERT INTO nodes_fts(nodes_fts, rowid, name, nameExpanded, searchText, filePath, content) VALUES ('delete', old.rowid, old.name, old.nameExpanded, old.searchText, old.filePath, old.content);
+  INSERT INTO nodes_fts(nodes_fts, rowid, name, nameExpanded, searchText, filePath, content) VALUES (new.rowid, new.name, new.nameExpanded, new.searchText, new.filePath, new.content);
 END;
 `;

package/models/mlx-embedder.py

@@ -27,7 +27,7 @@ import mlx.core as mx
 import mlx.nn as nn
 from tokenizers import Tokenizer
 
-MODEL_DIR = os.path.dirname(os.path.abspath(__file__)) + "/jina-v5-small-mlx"
+MODEL_DIR = os.path.dirname(os.path.abspath(__file__)) + "/jina-code-0.5b-mlx"
 
 
 
@@ -64,7 +64,10 @@ def load_model():
     ensure_model_downloaded()
 
     sys.path.insert(0, MODEL_DIR)
-    from model import JinaEmbeddingModel
+    import importlib
+    model_module = importlib.import_module("model")
+    # Support both model class names (v5 = JinaEmbeddingModel, code-0.5b = JinaCodeEmbeddingModel)
+    JinaEmbeddingModel = getattr(model_module, "JinaEmbeddingModel", None) or getattr(model_module, "JinaCodeEmbeddingModel")
 
     with open(os.path.join(MODEL_DIR, "config.json")) as f:
         config = json.load(f)
@@ -99,8 +102,17 @@ def embed_tiered(model, tokenizer, texts, task_type="retrieval.passage", truncat
     if not texts:
         return []
 
-    # Add task prefix
-    prefix_map = {"retrieval.query": "Query: ", "retrieval.passage": "Document: "}
+    # Add task prefix — auto-detect based on model type
+    # v5 (Qwen3): "Query: " / "Document: "
+    # code-0.5b (Qwen2): "Find the most relevant code snippet...\n" / "Candidate code snippet:\n"
+    is_code_model = "jina-code" in MODEL_DIR
+    if is_code_model:
+        prefix_map = {
+            "retrieval.query": "Find the most relevant code snippet given the following query:\n",
+            "retrieval.passage": "Candidate code snippet:\n",
+        }
+    else:
+        prefix_map = {"retrieval.query": "Query: ", "retrieval.passage": "Document: "}
     prefix = prefix_map.get(task_type, "")
     prefixed = [prefix + t for t in texts] if prefix else texts
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zuvia-software-solutions/code-mapper",
-  "version": "2.2.3",
+  "version": "2.3.0",
   "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
   "author": "Abhigyan Patwari",
   "license": "PolyForm-Noncommercial-1.0.0",