graph-indexer 1.0.3 → 1.2.0

package/storage.mjs

@@ -0,0 +1,43 @@
+/**
+ * @file storage.mjs
+ * @description Storage backend factory. Selects the in-memory engine (default,
+ *              zero-dependency) or the disk-backed SQLite store based on config,
+ *              and documents the single read contract that the MCP tools depend
+ *              on so they remain backend-agnostic.
+ * @author MaquinaTech <https://github.com/MaquinaTech>
+ * @copyright (c) 2026 MaquinaTech. All rights reserved.
+ * @license MIT
+ *
+ * ── Store contract (implemented by both MemoryGraphIndex and SqliteGraphStore) ──
+ *   load()                                   Prepare for queries (open db / parse json).
+ *   get backend()                            'memory' | 'sqlite'.
+ *   get graph()                              { dependencies, importedBy } (file-level).
+ *   searchHybrid(q, vec, topK, minScore, exactBoost) → [{ score, chunk }]
+ *   getChunk(id)                             → chunk | null
+ *   getChunksByFile(path)                    → chunk[]
+ *   resolveSymbol(name)                      → chunk[]   (exact, case-insensitive)
+ *   findCallers(funcName)                    → chunk[]
+ *   iterateChunks()                          → Iterable<chunk>  (cursor on SQLite)
+ *   getDependencies(path) / getImportedBy(path) → string[]
+ *   chunkCount() / fileCount() / symbolCount() / vectorCount() → number
+ *   stats()                                  → engine health facts
+ *   close()                                  Release fds / db handles.
+ */
+import { MemoryGraphIndex } from './core-engine.mjs';
+
+/**
+ * Construct (but do not yet load) the configured store.
+ *
+ * @param {object}  config                 Resolved config from config.mjs.
+ * @param {object}  [opts]
+ * @param {boolean} [opts.cacheEmbeddings] Eager vector cache (in-memory backend only).
+ * @returns {Promise<object>} a store implementing the contract above.
+ */
+export async function createStore(config, { cacheEmbeddings = false } = {}) {
+    if (config.storage === 'sqlite') {
+        // Imported lazily so the default path never loads node:sqlite.
+        const { SqliteGraphStore } = await import('./sqlite-store.mjs');
+        return new SqliteGraphStore(config.sqlitePath, { embeddingPath: config.embeddingPath });
+    }
+    return new MemoryGraphIndex(config.indexPath, { cacheEmbeddings });
+}

package/watch-daemon.mjs

@@ -1,7 +1,14 @@
 #!/usr/bin/env node
 /**
  * @file watch-daemon.mjs
- * @description Native FileSystem Watcher Daemon to maintain an in-memory graph index. Utilizes Tree-sitter for AST parsing and local embedding generation.
+ * @description Native FileSystem Watcher Daemon that keeps the configured index
+ *              backend fresh incrementally. Backend-agnostic: file changes are
+ *              parsed once (Tree-sitter), optionally enriched (LLM cache-first)
+ *              and embedded (Ollama), then handed to the store's applyFileUpdate
+ *              — the in-memory engine persists a debounced JSON snapshot, the
+ *              SQLite store commits a per-file WAL transaction that running MCP
+ *              servers pick up live via PRAGMA data_version. No more "SQLite
+ *              requires a full re-index on every change".
  * @author MaquinaTech <https://github.com/MaquinaTech>
  * @copyright (c) 2026 MaquinaTech. All rights reserved.
  * @license MIT
@@ -19,35 +26,119 @@
 import fs from 'fs';
 import path from 'path';
 import chokidar from 'chokidar';
-import { MemoryGraphIndex } from './core-engine.mjs';
+import { resolveConfig } from './config.mjs';
+import { createStore } from './storage.mjs';
 import {
-    MAX_FILE_SIZE_BYTES, EXTENSIONS, getParserForFile, buildIgnoreFilter,
-    extractImportsFromAST, extractSemanticChunks, resolveLocalImports, getLocalEmbeddingsBatch
+    embeddingKeyFor, computePageRank, TEST_FILE_RE, EXAMPLE_DIR_RE,
+    summaryEmbeddingText, SUMMARY_VEC_SUFFIX,
+} from './search-core.mjs';
+import {
+    MAX_FILE_SIZE_BYTES, getParserForFile, buildIgnoreFilter,
+    extractImportsFromAST, extractSemanticChunks, resolveLocalImports, getLocalEmbeddingsBatch,
+    buildEmbeddingPayload,
 } from './parser-utils.mjs';
+import {
+    loadEnrichmentCache, saveEnrichmentCache, attachEnrichment,
+    ollamaGenerate, parseEnrichResponse, buildEnrichPrompt,
+} from './enrichment.mjs';
 
-const PROJECT_ROOT = process.env.MCP_PROJECT_ROOT || process.cwd();
-const INDEX_PATH = path.join(PROJECT_ROOT, 'code-index.json');
+const config = resolveConfig();
+const PROJECT_ROOT = config.projectRoot;
 
 const ignoreFilter = buildIgnoreFilter(PROJECT_ROOT);
-const db = new MemoryGraphIndex(INDEX_PATH);
+const db = await createStore(config, { cacheEmbeddings: true });
 db.load();
 
+// ─── Enrichment (cache-first, live for core files) ──────────────────────────────
+// Cached entries re-attach for free on every change. When enrichment is enabled
+// and the changed file sits in the PageRank core, new/changed chunks are enriched
+// live (best-effort, low concurrency) so edits don't silently lose their semantic
+// metadata until the next full index run.
+const enrichCache = config.enrichment.enabled
+    ? loadEnrichmentCache(config.enrichmentCachePath)
+    : null;
+let enrichCacheDirty = false;
+let saveCacheTimer = null;
+function scheduleEnrichCacheSave() {
+    if (!enrichCacheDirty) return;
+    if (saveCacheTimer) clearTimeout(saveCacheTimer);
+    saveCacheTimer = setTimeout(() => {
+        saveCacheTimer = null;
+        try { saveEnrichmentCache(config.enrichmentCachePath, enrichCache); enrichCacheDirty = false; }
+        catch (err) { process.stderr.write(`[daemon] ⚠️ enrichment cache save failed: ${err.message}\n`); }
+    }, 3000);
+}
+
+let _coreFiles = null;
+function isEnrichableFile(filename) {
+    if (TEST_FILE_RE.test(filename) || EXAMPLE_DIR_RE.test(filename)) return false;
+    if (config.enrichment.coreRatio >= 1) return true; // all production files
+    if (!_coreFiles) {
+        const pr = computePageRank(db.graph);
+        const files = Array.from(pr.keys()).sort((a, b) => (pr.get(b) ?? 0) - (pr.get(a) ?? 0));
+        const coreCount = Math.max(1, Math.ceil(files.length * config.enrichment.coreRatio));
+        _coreFiles = new Set(files.slice(0, coreCount));
+    }
+    return _coreFiles.has(filename);
+}
+
+async function enrichChunks(filename, chunks) {
+    if (!enrichCache) return;
+    const pending = [];
+    for (const chunk of chunks) {
+        if (!chunk.content_hash) continue;
+        if (attachEnrichment(chunk, enrichCache.get(chunk.content_hash))) continue;
+        pending.push(chunk);
+    }
+    if (pending.length === 0 || !isEnrichableFile(filename)) return;
+    // Live enrichment of changed core chunks: sequential and best-effort — a file
+    // save touches a handful of chunks, so latency stays low and failures only
+    // mean "no enrichment until the next full index run".
+    for (const chunk of pending) {
+        if ((chunk.end_line - chunk.start_line) < 4) continue; // trivial stubs
+        const raw = await ollamaGenerate(buildEnrichPrompt(chunk), {
+            model: config.enrichment.model, ollamaHost: config.ollamaHost, timeoutMs: 20000,
+        });
+        const parsed = parseEnrichResponse(raw);
+        if (!parsed) break; // model unreachable — stop trying for this batch
+        chunk.summary  = parsed.summary;
+        chunk.concepts = parsed.concepts;
+        chunk.hyde     = parsed.hyde;
+        enrichCache.set(chunk.content_hash, {
+            summary: parsed.summary, concepts: parsed.concepts, model: config.enrichment.model,
+        });
+        enrichCacheDirty = true;
+    }
+    scheduleEnrichCacheSave();
+}
+
+// ─── Incremental file sync ──────────────────────────────────────────────────────
+
+// Catch-up scan: chokidar fires 'add' for every existing file at startup. Files
+// not modified since the index artifact was last written are already indexed —
+// skipping them turns daemon start from O(repo) parse work into O(changed files),
+// while still picking up edits made while the daemon was down.
+const ARTIFACT_PATH = config.storage === 'sqlite' ? config.sqlitePath : config.indexPath;
+let indexBuiltAt = 0;
+try { indexBuiltAt = fs.statSync(ARTIFACT_PATH).mtimeMs; } catch { /* no index yet → full scan */ }
+let initialScan = true;
+
 async function processFileChange(absolutePath) {
     const filename = path.relative(PROJECT_ROOT, absolutePath).replace(/\\/g, '/');
     if (ignoreFilter.ignores(filename)) return;
     if (filename.includes('.bundle.') || filename.includes('.min.')) return;
+    if (filename.startsWith('code-index.')) return; // never index our own artifacts
 
     try {
+        if (initialScan && indexBuiltAt > 0 && fs.existsSync(absolutePath)
+            && fs.statSync(absolutePath).mtimeMs <= indexBuiltAt) {
+            return; // unchanged since the last index write
+        }
+
         if (!fs.existsSync(absolutePath)) {
-            for (const [id, chunk] of db.chunks.entries()) {
-                if (chunk.file_path === filename) {
-                    db._removeLexical(id);
-                    db.chunks.delete(id);
-                    db.removeVector(id);
-                }
-            }
-            db.updateFileGraph(filename, []);
-            db.saveDebounced();
+            if (typeof db.removeFile === 'function') db.removeFile(filename);
+            else db.applyFileUpdate(filename, { chunks: [], imports: [] });
+            _coreFiles = null;
             process.stderr.write(`[daemon] 🗑️  Purged: ${filename}\n`);
             return;
         }
@@ -55,7 +146,7 @@ async function processFileChange(absolutePath) {
         const stats = fs.statSync(absolutePath);
         if (stats.size > MAX_FILE_SIZE_BYTES) return;
 
-        let content = fs.readFileSync(absolutePath, 'utf-8');
+        const content = fs.readFileSync(absolutePath, 'utf-8');
         if (!content.trim()) return;
 
         const ext = path.extname(absolutePath);
@@ -64,82 +155,84 @@ async function processFileChange(absolutePath) {
 
         const tree = parser.parse((offset) => offset < content.length ? content.slice(offset, offset + 4096) : null);
         const imports = resolveLocalImports(extractImportsFromAST(tree.rootNode, ext), filename, PROJECT_ROOT);
-        db.updateFileGraph(filename, imports);
-
         const newChunks = extractSemanticChunks(tree.rootNode, filename, content, ext);
+        _coreFiles = null; // dependency graph may change → PageRank core set is stale
 
-        for (const [id, chunk] of db.chunks.entries()) {
-            if (chunk.file_path === filename) {
-                db._removeLexical(id);
-                db.chunks.delete(id);
-                db.removeVector(id); // invalidates flat matrix so next search rebuilds it
-            }
-        }
-
-        const chunksToEmbed = newChunks.filter(c => !db.embeddingCache.has(c.content_hash));
+        // Enrichment BEFORE embedding so the summary rides the vector payload
+        // (embeddingKeyFor accounts for it, so cache lookups stay correct).
+        await enrichChunks(filename, newChunks);
 
+        // 🥇 STRICT PAYLOAD PARITY: identical embedding payloads to indexer.mjs
+        // (shared helpers) so incremental updates match the bootstrap index —
+        // including the summary-only second vector for enriched chunks.
+        const chunksToEmbed = newChunks.filter(c => !db.hasEmbedding(embeddingKeyFor(c)));
+        const embeddings = new Map();
         if (chunksToEmbed.length > 0) {
-            const textsToEmbed = chunksToEmbed.map(c => {
-                const dependencies = db.graph.dependencies[c.file_path] || [];
-                const cleanDeps = dependencies.map(d => path.basename(d, path.extname(d)));
-                const topologicalContext = cleanDeps.length > 0
-                    ? `This code architectural neighborhood connects with: ${cleanDeps.join(', ')}.`
-                    : '';
-
-                // 🥇 STRICT PAYLOAD PARITY: Eliminate real-time bleed
-                return [
-                    `File Location: ${c.file_path}`,
-                    `Symbol Name: ${c.node_type} -> ${c.name}`,
-                    c.docstring ? `Developer Documentation: ${c.docstring}` : '',
-                    topologicalContext,
-                    `--- Source Code ---`,
-                    c.code_snippet
-                ].filter(Boolean).join('\n');
+            const entries = [];
+            for (const c of chunksToEmbed) {
+                entries.push({ key: embeddingKeyFor(c), text: buildEmbeddingPayload(c, imports) });
+                const sText = summaryEmbeddingText(c);
+                if (sText) entries.push({ key: embeddingKeyFor(c) + SUMMARY_VEC_SUFFIX, text: sText });
+            }
+            const embeddingsMatrix = await getLocalEmbeddingsBatch(entries.map(e => e.text), true, {
+                ollamaHost: config.ollamaHost, model: config.embedModel,
             });
-
-            const embeddingsMatrix = await getLocalEmbeddingsBatch(textsToEmbed, true);
-            if (embeddingsMatrix) {
-                chunksToEmbed.forEach((chunk, j) => {
-                    if (embeddingsMatrix[j]) {
-                        db.embeddingCache.set(chunk.content_hash, new Float32Array(embeddingsMatrix[j]));
-                    }
+            if (embeddingsMatrix && embeddingsMatrix.length === entries.length) {
+                entries.forEach((entry, j) => {
+                    if (embeddingsMatrix[j]) embeddings.set(entry.key, new Float32Array(embeddingsMatrix[j]));
                 });
             }
         }
 
-        for (const chunk of newChunks) {
-            if (db.embeddingCache.has(chunk.content_hash)) {
-                db.addVector(chunk.id, db.embeddingCache.get(chunk.content_hash));
-            }
-
-            // Unified lexical re-indexing of neighborhood
-            const cleanDeps = imports.map(d => d.split('/').pop().split('.')[0]);
-            const enrichedContext = [
-                chunk.name,
-                chunk.docstring || '',
-                cleanDeps.join(' '),
-                (chunk.calls || []).join(' '),
-                (chunk.params || []).join(' '),
-                chunk.return_type || '',
-                chunk.class_context ? `${chunk.class_context}.${chunk.name}` : '',
-                chunk.code_snippet
-            ].join(' ');
-
-            db._indexLexical(chunk.id, enrichedContext, chunk.file_path);
-            db.chunks.set(chunk.id, chunk);
-        }
-
-        db.saveDebounced();
-        process.stderr.write(`[daemon] 🔄 Synced: ${filename} (Cache Hit: ${newChunks.length - chunksToEmbed.length}/${newChunks.length})\n`);
+        db.applyFileUpdate(filename, { chunks: newChunks, imports, embeddings });
+        process.stderr.write(
+            `[daemon] 🔄 Synced: ${filename} (${newChunks.length} chunks, `
+            + `${newChunks.length - chunksToEmbed.length} cached vectors, ${config.storage})\n`
+        );
     } catch (err) {
         process.stderr.write(`[daemon] ❌ Error in ${filename}: ${err.message}\n`);
     }
 }
 
-process.stderr.write(`🚀 Chokidar Watcher Daemon started in: ${PROJECT_ROOT}\n`);
+// Serialize updates: chokidar can fire bursts (git checkout, format-on-save across
+// files); per-file work must not interleave inside the store.
+let queue = Promise.resolve();
+function enqueue(absolutePath) {
+    queue = queue.then(() => processFileChange(absolutePath)).catch(() => {});
+}
+
+process.stderr.write(`🚀 Watcher Daemon started in: ${PROJECT_ROOT} (backend: ${config.storage})\n`);
+
+// Only watch source files we can actually index. Critically, this prevents
+// chokidar from descending into node_modules / dist / .git / .gitignored dirs —
+// watching those would exhaust the OS file-watcher limit (ENOSPC on Linux) and
+// waste CPU/memory on large projects. The ignore decision happens at the watcher
+// level, not just in processFileChange, so ignored trees are never traversed.
+function shouldIgnore(absPath) {
+    const rel = path.relative(PROJECT_ROOT, absPath).replace(/\\/g, '/');
+    if (!rel) return false;                       // the project root itself
+    if (path.basename(absPath).startsWith('.')) return true; // dotfiles / dot-dirs
+    if (rel.startsWith('code-index.')) return true;          // our own artifacts (.db/-wal/.bin/.json)
+    try { return ignoreFilter.ignores(rel); }     // node_modules, dist, .gitignore, …
+    catch { return false; }
+}
+
 const watcher = chokidar.watch(PROJECT_ROOT, {
-    ignored: /(^|[\/\\])\../, persistent: true,
+    ignored: shouldIgnore,
+    persistent: true,
     awaitWriteFinish: { stabilityThreshold: 300, pollInterval: 100 }
 });
-watcher.on('add', processFileChange).on('change', processFileChange).on('unlink', (p) => processFileChange(p));
-watcher.on('error', (err) => process.stderr.write(`[daemon] 💥 OS Watcher panic: ${err.message}\n`));
+watcher.on('add', enqueue).on('change', enqueue).on('unlink', enqueue);
+watcher.on('ready', () => { queue = queue.then(() => { initialScan = false; }); });
+watcher.on('error', (err) => process.stderr.write(`[daemon] 💥 OS Watcher panic: ${err.message}\n`));
+
+function shutdown() {
+    watcher.close().catch(() => {});
+    if (enrichCache && enrichCacheDirty) {
+        try { saveEnrichmentCache(config.enrichmentCachePath, enrichCache); } catch { /* best effort */ }
+    }
+    if (typeof db.close === 'function') db.close();
+    process.exit(0);
+}
+process.on('SIGTERM', shutdown);
+process.on('SIGINT', shutdown);