@zuvia-software-solutions/code-mapper 2.4.0 → 2.5.0

package/dist/core/embeddings/nl-embedder.js

@@ -0,0 +1,431 @@
+// code-mapper/src/core/embeddings/nl-embedder.ts
+/**
+ * @file Natural language embedder using bge-small-en-v1.5.
+ *
+ * Runs entirely in Node.js via @huggingface/transformers — no Python, no GPU.
+ * Embeds human-readable descriptions extracted from code (JSDoc comments,
+ * enum values, type patterns, file headers) for conceptual search.
+ *
+ * 33M params, q8 quantized, 384-dim embeddings, ~6ms/text on CPU.
+ */
+// NL embedder — no schema imports needed
+const MODEL_ID = 'Xenova/bge-small-en-v1.5';
+// Lazy-loaded pipeline
+let extractor = null;
+let loadPromise = null;
+/** Initialize the NL embedding model (lazy, idempotent) */
+export async function initNlEmbedder() {
+    if (extractor)
+        return;
+    if (loadPromise)
+        return loadPromise;
+    loadPromise = (async () => {
+        const { pipeline, env } = await import('@huggingface/transformers');
+        // Use all available CPU threads for ONNX inference
+        if (env.backends?.onnx?.wasm) {
+            env.backends.onnx.wasm.numThreads = Math.max(1, (await import('os')).cpus().length);
+        }
+        extractor = await pipeline('feature-extraction', MODEL_ID, { quantized: true });
+    })();
+    return loadPromise;
+}
+/** Check if the NL embedder is ready */
+export function isNlEmbedderReady() {
+    return extractor !== null;
+}
+/** Embed a single text, returns Float32Array */
+export async function nlEmbed(text) {
+    if (!extractor)
+        await initNlEmbedder();
+    const result = await extractor(text, { pooling: 'cls', normalize: true });
+    return Array.from(result.data);
+}
+/** Embed a batch of texts (processes in sub-batches for memory efficiency) */
+export async function nlEmbedBatch(texts) {
+    if (!extractor)
+        await initNlEmbedder();
+    const BATCH = 32; // sub-batch size — balances throughput vs memory
+    const results = [];
+    for (let i = 0; i < texts.length; i += BATCH) {
+        const batch = texts.slice(i, i + BATCH);
+        // Process sub-batch — transformers.js handles arrays
+        const batchResults = await Promise.all(batch.map(text => extractor(text, { pooling: 'cls', normalize: true })));
+        for (const result of batchResults) {
+            results.push(Array.from(result.data));
+        }
+    }
+    return results;
+}
+/** Extract all JSDoc/block comment text (up to 10 lines) */
+function extractFullComment(content) {
+    if (!content)
+        return '';
+    const lines = content.split('\n');
+    const commentLines = [];
+    let inBlock = false;
+    for (const l of lines) {
+        const t = l.trim();
+        if (t.startsWith('/**') || t.startsWith('/*')) {
+            inBlock = true;
+            const inner = t.replace(/^\/\*\*?/, '').replace(/\*\/$/, '').trim();
+            if (inner && !inner.startsWith('@'))
+                commentLines.push(inner);
+            if (t.includes('*/'))
+                inBlock = false;
+            continue;
+        }
+        if (inBlock) {
+            if (t.includes('*/')) {
+                inBlock = false;
+                continue;
+            }
+            const inner = t.replace(/^\*\s?/, '').trim();
+            if (inner && !inner.startsWith('@'))
+                commentLines.push(inner);
+            if (commentLines.length >= 10)
+                break;
+            continue;
+        }
+        if (t.startsWith('//')) {
+            const inner = t.slice(2).trim();
+            if (inner)
+                commentLines.push(inner);
+            if (commentLines.length >= 10)
+                break;
+            continue;
+        }
+        if (t.startsWith('#') && !t.startsWith('#!')) {
+            const inner = t.slice(1).trim();
+            if (inner)
+                commentLines.push(inner);
+            if (commentLines.length >= 10)
+                break;
+            continue;
+        }
+        if (commentLines.length > 0)
+            break; // comment ended
+    }
+    return commentLines.join(' ');
+}
+/** Expand camelCase/PascalCase/snake_case to space-separated words */
+function expandIdentifier(name) {
+    return name
+        .replace(/([a-z])([A-Z])/g, '$1 $2')
+        .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2')
+        .replace(/[_\-]/g, ' ')
+        .toLowerCase();
+}
+/** Extract enum/const array values as NL text */
+function extractEnumValues(content) {
+    // Match: ['value1', 'value2', ...] as const
+    const match = content.match(/\[([^\]]+)\]\s*as\s*const/);
+    if (match?.[1]) {
+        const values = match[1].replace(/['"]/g, '').split(',').map(v => v.trim()).filter(Boolean);
+        if (values.length > 0)
+            return values.join(', ');
+    }
+    // Match: enum { Value1, Value2 }
+    const enumMatch = content.match(/enum\s+\w+\s*\{([^}]+)\}/);
+    if (enumMatch?.[1]) {
+        const values = enumMatch[1].split(',').map(v => v.trim().split('=')[0].trim()).filter(Boolean);
+        if (values.length > 0)
+            return values.map(v => expandIdentifier(v)).join(', ');
+    }
+    return '';
+}
+/** Extract parameter names from function signature */
+function extractParamNames(content) {
+    const match = content.match(/\(([^)]*)\)/);
+    if (!match?.[1])
+        return '';
+    return match[1].split(',')
+        .map(p => p.trim().split(':')[0].split('=')[0].trim())
+        .filter(p => p && p !== '')
+        .map(p => expandIdentifier(p))
+        .join(', ');
+}
+/** Build NL documents from a node */
+export function extractNlTexts(node) {
+    const docs = [];
+    const name = node.name;
+    const expandedName = expandIdentifier(name);
+    const dir = node.filePath.split('/').slice(-3, -1).join('/');
+    // 1. Comment-based NL text (primary)
+    const comment = extractFullComment(node.content);
+    if (comment) {
+        docs.push({
+            nodeId: node.id,
+            source: 'comment',
+            text: `${expandedName}: ${comment}. File: ${dir}`,
+        });
+    }
+    // 2. Name + params + return type (always available)
+    const params = extractParamNames(node.content);
+    const parts = [expandedName];
+    if (params)
+        parts.push(`Parameters: ${params}`);
+    if (dir)
+        parts.push(`in ${dir}`);
+    if (!comment) {
+        // Only add name-based doc if no comment (avoid duplication)
+        docs.push({
+            nodeId: node.id,
+            source: 'name',
+            text: parts.join('. '),
+        });
+    }
+    // 3. Enum/const values
+    if (node.label === 'Enum' || node.label === 'Const' || node.label === 'TypeAlias') {
+        const values = extractEnumValues(node.content);
+        if (values) {
+            docs.push({
+                nodeId: node.id,
+                source: 'enum',
+                text: `${expandedName}: ${values}`,
+            });
+        }
+    }
+    return docs;
+}
+// ---------------------------------------------------------------------------
+// Full NL embedding pipeline
+// ---------------------------------------------------------------------------
+/** Hash text for skip detection */
+import { createHash } from 'crypto';
+function md5(text) {
+    return createHash('md5').update(text).digest('hex');
+}
+/**
+ * Build NL embeddings for all eligible nodes in the database.
+ * Reads nodes, extracts NL text, embeds with bge-small, writes to nl_embeddings.
+ */
+export async function buildNlEmbeddings(db, onProgress) {
+    const t0 = Date.now();
+    await initNlEmbedder();
+    // Query all nodes (not just EMBEDDABLE_LABELS — we want enums, consts, types too)
+    const labels = ['Function', 'Class', 'Method', 'Interface', 'Const', 'Enum', 'TypeAlias', 'Namespace', 'Module', 'Struct'];
+    const placeholders = labels.map(() => '?').join(',');
+    const rows = db.prepare(`SELECT id, name, label, filePath, content, startLine, description FROM nodes WHERE label IN (${placeholders})`).all(...labels);
+    // NL embeddings include ALL files (including tests) — test names describe
+    // functionality in natural language which helps conceptual search.
+    // The bge-small model is fast enough (~6ms/doc) that the cost is trivial.
+    const filteredRows = rows;
+    // Extract NL documents
+    const allDocs = [];
+    for (const row of filteredRows) {
+        const docs = extractNlTexts(row);
+        for (const doc of docs)
+            allDocs.push(doc);
+    }
+    if (allDocs.length === 0) {
+        return { embedded: 0, skipped: 0, durationMs: Date.now() - t0 };
+    }
+    // Deduplicate: one embedding per nodeId — prefer 'comment' source over 'name' or 'enum'
+    const SOURCE_PRIORITY = { comment: 0, enum: 1, name: 2 };
+    const bestByNode = new Map();
+    for (const doc of allDocs) {
+        const existing = bestByNode.get(doc.nodeId);
+        if (!existing || (SOURCE_PRIORITY[doc.source] ?? 9) < (SOURCE_PRIORITY[existing.source] ?? 9)) {
+            bestByNode.set(doc.nodeId, doc);
+        }
+    }
+    const uniqueDocs = [...bestByNode.values()];
+    // Check existing hashes for skip detection
+    const existingHashes = new Map();
+    try {
+        const hashRows = db.prepare('SELECT nodeId, textHash FROM embeddings WHERE textHash IS NOT NULL').all();
+        for (const r of hashRows)
+            existingHashes.set(r.nodeId + ':' + r.textHash, '1');
+    }
+    catch { /* table might not exist yet */ }
+    // Filter to docs that need embedding
+    const toEmbed = [];
+    let skipped = 0;
+    for (const doc of uniqueDocs) {
+        const hash = md5(doc.text);
+        if (existingHashes.has(doc.nodeId + ':' + hash)) {
+            skipped++;
+            continue;
+        }
+        toEmbed.push({ ...doc, hash });
+    }
+    if (toEmbed.length === 0) {
+        return { embedded: 0, skipped, durationMs: Date.now() - t0 };
+    }
+    // Clear existing embeddings and rebuild
+    db.prepare('DELETE FROM embeddings').run();
+    try {
+        db.prepare('DELETE FROM nl_embeddings').run();
+    }
+    catch { /* table may not exist */ }
+    // Parallel multi-process embedding — same architecture as tsgo
+    // Each worker loads its own bge-small model, embeds independently.
+    const os = await import('os');
+    const { fork } = await import('child_process');
+    const { fileURLToPath } = await import('url');
+    const pathMod = await import('path');
+    const cpuCount = os.cpus().length;
+    const maxByCore = Math.max(1, Math.floor(cpuCount * 0.75));
+    const maxByWorkload = Math.max(1, Math.floor(toEmbed.length / 50));
+    const workerCount = Math.min(maxByCore, maxByWorkload, 8); // cap at 8 for memory
+    // Find worker script path
+    const thisDir = pathMod.dirname(fileURLToPath(import.meta.url));
+    const workerScript = pathMod.join(thisDir, 'nl-embed-worker.js');
+    // Split work across workers
+    const ITEMS_PER_BATCH = 50;
+    let nextIdx = 0;
+    let embedded = 0;
+    const getNextBatch = () => {
+        if (nextIdx >= toEmbed.length)
+            return null;
+        const batch = toEmbed.slice(nextIdx, nextIdx + ITEMS_PER_BATCH);
+        nextIdx += ITEMS_PER_BATCH;
+        return batch;
+    };
+    // Prepare DB statements
+    const insertStmt = db.prepare('INSERT OR REPLACE INTO embeddings (nodeId, embedding, textHash) VALUES (?, ?, ?)');
+    let nlInsertStmt = null;
+    try {
+        nlInsertStmt = db.prepare('INSERT INTO nl_embeddings (nodeId, embedding, textHash, source, text) VALUES (?, ?, ?, ?, ?)');
+    }
+    catch { /* nl_embeddings table may not exist */ }
+    // Track doc metadata for nl_embeddings text lookup
+    const docMap = new Map();
+    for (const doc of toEmbed)
+        docMap.set(doc.nodeId, { source: doc.source, text: doc.text, hash: doc.hash });
+    if (workerCount <= 1) {
+        // Single process — use in-process embedding (small workloads)
+        await initNlEmbedder();
+        db.exec('BEGIN');
+        try {
+            for (let i = 0; i < toEmbed.length; i += ITEMS_PER_BATCH) {
+                const batch = toEmbed.slice(i, i + ITEMS_PER_BATCH);
+                const vecs = await nlEmbedBatch(batch.map(d => d.text));
+                for (let j = 0; j < batch.length; j++) {
+                    const doc = batch[j];
+                    const vec = vecs[j];
+                    const blob = Buffer.from(new Float32Array(vec).buffer);
+                    insertStmt.run(doc.nodeId, blob, doc.hash);
+                    if (nlInsertStmt) {
+                        try {
+                            nlInsertStmt.run(doc.nodeId, blob, doc.hash, doc.source, doc.text);
+                        }
+                        catch { }
+                    }
+                    embedded++;
+                }
+                onProgress?.(Math.min(i + ITEMS_PER_BATCH, toEmbed.length), toEmbed.length);
+            }
+            db.exec('COMMIT');
+        }
+        catch (err) {
+            db.exec('ROLLBACK');
+            throw err;
+        }
+    }
+    else {
+        // Multi-process: spawn N workers, dynamic dispatch
+        const workers = [];
+        const workerReady = [];
+        for (let i = 0; i < workerCount; i++) {
+            const worker = fork(workerScript, [], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] });
+            workers.push(worker);
+            workerReady.push(new Promise((resolve) => {
+                const handler = (msg) => {
+                    if (msg?.type === 'ready') {
+                        worker.removeListener('message', handler);
+                        resolve();
+                    }
+                };
+                worker.on('message', handler);
+                // Timeout: if worker doesn't ready in 30s, skip it
+                setTimeout(() => resolve(), 30000);
+            }));
+        }
+        // Wait for all workers to load model
+        await Promise.all(workerReady);
+        const activeWorkers = workers.filter(w => w.connected);
+        if (activeWorkers.length === 0) {
+            // Fallback to single process
+            await initNlEmbedder();
+            db.exec('BEGIN');
+            try {
+                for (let i = 0; i < toEmbed.length; i += ITEMS_PER_BATCH) {
+                    const batch = toEmbed.slice(i, i + ITEMS_PER_BATCH);
+                    const vecs = await nlEmbedBatch(batch.map(d => d.text));
+                    for (let j = 0; j < batch.length; j++) {
+                        const doc = batch[j];
+                        const blob = Buffer.from(new Float32Array(vecs[j]).buffer);
+                        insertStmt.run(doc.nodeId, blob, doc.hash);
+                        embedded++;
+                    }
+                    onProgress?.(Math.min(i + ITEMS_PER_BATCH, toEmbed.length), toEmbed.length);
+                }
+                db.exec('COMMIT');
+            }
+            catch (err) {
+                db.exec('ROLLBACK');
+                throw err;
+            }
+        }
+        else {
+            // Dynamic dispatch: each worker requests next batch when done
+            db.exec('BEGIN');
+            let batchId = 0;
+            const runWorker = (worker) => {
+                return new Promise((resolve) => {
+                    const sendNext = () => {
+                        const batch = getNextBatch();
+                        if (!batch) {
+                            worker.send({ type: 'exit' });
+                            resolve();
+                            return;
+                        }
+                        worker.send({
+                            type: 'embed',
+                            batchId: batchId++,
+                            items: batch.map(d => ({ nodeId: d.nodeId, text: d.text })),
+                        });
+                    };
+                    worker.on('message', (msg) => {
+                        if (msg?.type === 'results') {
+                            // Write results to DB
+                            for (const r of msg.results) {
+                                const blob = Buffer.from(new Float32Array(r.vec).buffer);
+                                const meta = docMap.get(r.nodeId);
+                                insertStmt.run(r.nodeId, blob, meta?.hash ?? '');
+                                if (nlInsertStmt && meta) {
+                                    try {
+                                        nlInsertStmt.run(r.nodeId, blob, meta.hash, meta.source, meta.text);
+                                    }
+                                    catch { }
+                                }
+                                embedded++;
+                            }
+                            onProgress?.(embedded, toEmbed.length);
+                            sendNext(); // request next batch
+                        }
+                    });
+                    worker.on('exit', () => resolve());
+                    sendNext(); // start first batch
+                });
+            };
+            try {
+                await Promise.all(activeWorkers.map(w => runWorker(w)));
+                db.exec('COMMIT');
+            }
+            catch (err) {
+                db.exec('ROLLBACK');
+                throw err;
+            }
+        }
+        // Cleanup workers
+        for (const w of workers) {
+            try {
+                w.kill();
+            }
+            catch { }
+        }
+    }
+    return { embedded, skipped, durationMs: Date.now() - t0 };
+}

package/dist/core/incremental/refresh.js

@@ -492,46 +492,38 @@ export async function refreshEmbeddings(db, dirtyFiles, hasEmbeddings) {
         }
         if (newNodes.length === 0)
             return;
-        // Step 3: Enrich with graph context — same as the full analyze pipeline
-        // Lazy import to avoid circular dependency at module load time
-        const { fetchGraphContext, enrichTextWithGraphContext } = await import('../embeddings/embedding-pipeline.js');
-        const { generateEmbeddingText } = await import('../embeddings/text-generator.js');
-        const { initEmbedder, embedBatch, embeddingToArray } = await import('../embeddings/embedder.js');
-        const graphContext = fetchGraphContext(db, newNodes);
-        // Step 4: Generate enriched text + hash for skip detection
+        // Step 3: Extract NL text and embed with bge-small (same model as full analyze)
+        const { extractNlTexts, initNlEmbedder, nlEmbed } = await import('../embeddings/nl-embedder.js');
         const { createHash } = await import('crypto');
         const { getEmbeddingHashes } = await import('../db/adapter.js');
         const existingHashes = getEmbeddingHashes(db);
+        await initNlEmbedder();
         const toEmbed = [];
         for (const node of newNodes) {
-            let text = generateEmbeddingText(node);
-            const ctx = graphContext.get(node.id);
-            if (ctx) {
-                text = enrichTextWithGraphContext(text, ctx);
-            }
-            const hash = createHash('md5').update(text).digest('hex');
-            // Skip if hash unchanged (content + graph context identical)
+            const nlDocs = extractNlTexts({
+                id: node.id, name: node.name, label: node.label,
+                filePath: node.filePath, content: node.content || '',
+                startLine: node.startLine ?? null, description: node.description || '',
+            });
+            // Pick best doc (prefer comment over name)
+            const best = nlDocs.find(d => d.source === 'comment') || nlDocs[0];
+            if (!best)
+                continue;
+            const hash = createHash('md5').update(best.text).digest('hex');
             if (existingHashes.get(node.id) === hash)
                 continue;
-            toEmbed.push({ node, text, hash });
+            toEmbed.push({ nodeId: node.id, text: best.text, hash, source: best.source });
         }
         if (toEmbed.length === 0) {
             console.error(`Code Mapper: All ${newNodes.length} node(s) unchanged (hash skip)`);
             return;
         }
         console.error(`Code Mapper: Embedding ${toEmbed.length}/${newNodes.length} node(s) (${newNodes.length - toEmbed.length} unchanged)`);
-        // Step 5: Ensure embedder is ready
-        await initEmbedder();
-        // Step 6: Batch embed only changed nodes
-        const embeddings = await embedBatch(toEmbed.map(e => e.text));
-        // Step 7: Insert with hashes
+        // Step 4: Embed and insert
         const items = [];
-        for (let i = 0; i < toEmbed.length; i++) {
-            const entry = toEmbed[i];
-            const emb = embeddings[i];
-            if (entry?.node && emb) {
-                items.push({ nodeId: toNodeId(entry.node.id), embedding: embeddingToArray(emb), textHash: entry.hash });
-            }
+        for (const entry of toEmbed) {
+            const vec = await nlEmbed(entry.text);
+            items.push({ nodeId: toNodeId(entry.nodeId), embedding: vec, textHash: entry.hash });
         }
         insertEmbeddingsBatch(db, items);
         console.error(`Code Mapper: Embedded ${items.length} node(s) incrementally`);

package/dist/mcp/local/local-backend.d.ts

@@ -42,6 +42,8 @@ export declare class LocalBackend {
     private tsgoServices;
     /** Per-repo in-memory embedding cache: nodeId → Float32Array (256-dim) */
     private embeddingCaches;
+    /** Per-repo in-memory NL embedding cache: includes source text for match_reason */
+    private nlEmbeddingCaches;
     /** Get (or lazily start) a tsgo LSP service for a repo. Returns null if unavailable. */
     private getTsgo;
     /** Get (or lazily open) the SQLite database for a repo. */
@@ -50,6 +52,10 @@ export declare class LocalBackend {
     private loadEmbeddingCache;
     /** Search embeddings in memory — O(N) dot products, no disk I/O */
     private searchEmbeddingsInMemory;
+    /** Load NL embeddings into memory for fast conceptual search */
+    private loadNlEmbeddingCache;
+    /** Search NL embeddings in memory, returns match_reason text */
+    private searchNlEmbeddingsInMemory;
     /** Hard ceiling — beyond this, incremental is unreliable, warn prominently */
     private static readonly MAX_INCREMENTAL_FILES;
     /** Start file system watcher for a repo to detect source changes */
@@ -131,6 +137,11 @@ export declare class LocalBackend {
      * Semantic vector search helper
      */
     private semanticSearch;
+    /**
+     * NL semantic search: embed query with bge-small, search NL descriptions.
+     * Returns match_reason (the NL text that matched) for agent transparency.
+     */
+    private nlSemanticSearch;
     /**
      * Refs-based search: find symbols referenced in files that contain the query identifiers.
      * Bridges the gap between graph edges (incomplete) and grep (complete for exact names).