graph-indexer 1.2.0 → 2.0.0

package/git-signals.mjs

@@ -0,0 +1,248 @@
+/**
+ * @file git-signals.mjs
+ * @description Lightweight, air-gapped git-history signals for ranking and
+ *              blast-radius hints. Reads ONLY the local repository's commit log
+ *              (no network, no remote, no code sent anywhere) and derives three
+ *              cheap per-file signals competitors mostly ignore:
+ *
+ *                • churn      — how often a file changes (commit count);
+ *                • recency    — how recently it last changed (last-commit time);
+ *                • co-change  — which files historically change *together* (the
+ *                               "if you touch A you usually touch B" hint that
+ *                               sharpens the refactoring/blast-radius story).
+ *
+ *              Computed once at index time into a sidecar (code-index.git.json),
+ *              kept out of the storage contract and the ranking math: the
+ *              co-change hint is surfaced by the blast-radius tools, and the
+ *              recency/churn boost is applied opt-in in the tool layer (default
+ *              off), so the measured retrieval ranking is unchanged by default.
+ * @author MaquinaTech <https://github.com/MaquinaTech>
+ * @copyright (c) 2026 MaquinaTech. All rights reserved.
+ * @license MIT
+ */
+import { execFileSync } from 'child_process';
+import fs from 'fs';
+import path from 'path';
+
+// Record/field separators chosen to never collide with file paths.
+const REC = '\x01';
+const FS_ = '\x1f';
+
+/**
+ * Collect git signals for a repository from its local commit log.
+ *
+ * Strictly local: shells out to `git log` on `repoRoot` and parses stdout. No
+ * remote is contacted. Returns `null` (a no-op) when the directory is not a git
+ * repo, git is unavailable, or there is no history — callers degrade silently.
+ *
+ * @param {string} repoRoot
+ * @param {object} [opts]
+ * @param {number} [opts.maxCommits=500]       History depth (bounds cost + size).
+ * @param {number} [opts.topCoChange=8]        Co-changed files kept per file.
+ * @param {number} [opts.maxFilesPerCommit=50] Skip sweeping commits (mass renames,
+ *                                              formatting) whose co-change is noise.
+ * @returns {null | {generated:number, commits:number, maxChurn:number,
+ *                    churn:Object<string,number>, lastCommit:Object<string,number>,
+ *                    coChange:Object<string,Array<[string,number]>>}}
+ */
+export function collectGitSignals(repoRoot, { maxCommits = 500, topCoChange = 8, maxFilesPerCommit = 50 } = {}) {
+    // Resolve the actual repo root. If `repoRoot` is a SUBDIRECTORY of a larger
+    // repo (e.g. indexing one package, or a fixture nested in this repo), a bare
+    // `git -C <dir> log` walks UP and returns the PARENT repo's history with
+    // parent-root-relative paths that never match this index's file paths. So we
+    // find the toplevel, scope the log to this subtree, and strip the prefix so
+    // emitted paths are relative to `repoRoot` (matching chunk.file_path).
+    let prefix = '';
+    try {
+        const toplevel = execFileSync('git', ['-C', repoRoot, 'rev-parse', '--show-toplevel'],
+            { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
+        if (!toplevel) return null;
+        // Compare REAL paths: git returns the physical toplevel, while repoRoot may
+        // contain symlinks (e.g. macOS /var → /private/var), which would otherwise
+        // produce a bogus prefix.
+        const realTop = (() => { try { return fs.realpathSync(toplevel); } catch { return toplevel; } })();
+        const realRoot = (() => { try { return fs.realpathSync(path.resolve(repoRoot)); } catch { return path.resolve(repoRoot); } })();
+        const rel = path.relative(realTop, realRoot).replace(/\\/g, '/');
+        prefix = rel && rel !== '.' ? rel : '';
+    } catch {
+        return null; // not a git repo / git missing — air-gapped no-op
+    }
+
+    let raw;
+    try {
+        const args = ['-C', repoRoot, 'log', '--no-merges', `-n${maxCommits}`,
+            `--pretty=format:${REC}%H${FS_}%ct`, '--name-only'];
+        if (prefix) args.push('--', '.'); // limit to this subtree (paths stay toplevel-relative)
+        raw = execFileSync('git', args,
+            { encoding: 'utf-8', maxBuffer: 128 * 1024 * 1024, stdio: ['ignore', 'pipe', 'ignore'] });
+    } catch {
+        return null;
+    }
+    if (!raw || !raw.includes(REC)) return null;
+    const stripPrefix = prefix ? prefix + '/' : '';
+
+    const churn = Object.create(null);
+    const lastCommit = Object.create(null);
+    const pairCounts = new Map(); // file → Map<coFile, count>
+    let commits = 0;
+
+    const bump = (f, ts) => {
+        churn[f] = (churn[f] || 0) + 1;
+        if (ts > (lastCommit[f] || 0)) lastCommit[f] = ts;
+    };
+
+    for (const block of raw.split(REC)) {
+        if (!block) continue;
+        const nl = block.indexOf('\n');
+        const header = nl === -1 ? block : block.slice(0, nl);
+        const ts = parseInt(header.split(FS_)[1], 10);
+        if (!Number.isFinite(ts)) continue;
+        commits++;
+
+        const files = [];
+        if (nl !== -1) {
+            for (const line of block.slice(nl + 1).split('\n')) {
+                let f = line.trim();
+                if (!f) continue;
+                // git quotes paths with unusual chars: "src/é.ts" — unwrap.
+                if (f.startsWith('"') && f.endsWith('"')) f = f.slice(1, -1);
+                if (stripPrefix) {
+                    if (!f.startsWith(stripPrefix)) continue; // outside the indexed subtree
+                    f = f.slice(stripPrefix.length);
+                }
+                files.push(f);
+            }
+        }
+        for (const f of files) bump(f, ts);
+
+        // Co-change: only for focused commits — a 400-file sweep links nothing useful.
+        if (files.length >= 2 && files.length <= maxFilesPerCommit) {
+            for (let i = 0; i < files.length; i++) {
+                const a = files[i];
+                let row = pairCounts.get(a);
+                if (!row) { row = new Map(); pairCounts.set(a, row); }
+                for (let j = 0; j < files.length; j++) {
+                    if (i === j) continue;
+                    const b = files[j];
+                    row.set(b, (row.get(b) || 0) + 1);
+                }
+            }
+        }
+    }
+
+    if (commits === 0) return null;
+
+    // Stamp HEAD so clients can report "indexed at commit X" and detect drift
+    // (the freshness contract). Best-effort — absence just omits the commit.
+    let head = null;
+    try {
+        head = execFileSync('git', ['-C', repoRoot, 'rev-parse', '--short', 'HEAD'],
+            { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }).trim() || null;
+    } catch { /* detached/empty — no stamp */ }
+
+    const coChange = Object.create(null);
+    for (const [file, row] of pairCounts) {
+        const top = [...row.entries()]
+            .sort((x, y) => y[1] - x[1] || x[0].localeCompare(y[0]))
+            .slice(0, topCoChange);
+        if (top.length) coChange[file] = top;
+    }
+
+    let maxChurn = 1;
+    for (const k in churn) if (churn[k] > maxChurn) maxChurn = churn[k];
+
+    return { generated: Math.floor(Date.now() / 1000), head, commits, maxChurn, churn, lastCommit, coChange };
+}
+
+/** Persist signals next to the index (best-effort). */
+export function writeGitSignals(filePath, signals) {
+    try { fs.writeFileSync(filePath, JSON.stringify(signals)); return true; }
+    catch { return false; }
+}
+
+/** Load a signals sidecar, or null if absent/corrupt. */
+export function loadGitSignals(filePath) {
+    try { return JSON.parse(fs.readFileSync(filePath, 'utf-8')); }
+    catch { return null; }
+}
+
+/** Files that historically change together with `file`, strongest first. */
+export function coChangesFor(signals, file, limit = 5) {
+    if (!signals || !signals.coChange) return [];
+    const row = signals.coChange[file];
+    if (!row) return [];
+    return row.slice(0, limit).map(([f, count]) => ({ file: f, count }));
+}
+
+// ─── Index freshness (query-time drift detection) ────────────────────────────
+// The blast-radius tools are only trustworthy if the index reflects the working
+// tree. We surface a compact freshness signal on tool responses: index age, the
+// commit it was built at (stamped above), and whether the tree has drifted since.
+// The git read is cheap and cached briefly so a burst of tool calls costs one
+// `git status`; it degrades silently (null) outside a git repo.
+
+const _SRC_RE = /\.(js|jsx|mjs|cjs|ts|tsx|py|go|rs|java|kt|cs|rb|php|swift|c|h|cc|cpp|hpp|scss|css|sh|bash)$/i;
+let _gitStateCache = null; // { at, root, head, dirtyCount }
+
+/** Reset the cached working-tree state (tests). */
+export function _resetGitStateCache() { _gitStateCache = null; }
+
+/**
+ * Cheap, cached working-tree state: current short HEAD + count of changed SOURCE
+ * files (uncommitted). Cached `ttlMs` per root so a burst of tool calls shells out
+ * at most once. `exec`/`nowMs` injectable for tests. Returns {head:null,dirtyCount:null}
+ * outside a git repo.
+ */
+export function currentGitState(repoRoot, { ttlMs = 5000, nowMs = Date.now(), exec = execFileSync } = {}) {
+    if (_gitStateCache && _gitStateCache.root === repoRoot && (nowMs - _gitStateCache.at) < ttlMs) return _gitStateCache;
+    let head = null, dirtyCount = null;
+    try {
+        head = exec('git', ['-C', repoRoot, 'rev-parse', '--short', 'HEAD'],
+            { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }).trim() || null;
+        const status = exec('git', ['-C', repoRoot, 'status', '--porcelain'],
+            { encoding: 'utf-8', maxBuffer: 16 * 1024 * 1024, stdio: ['ignore', 'pipe', 'ignore'] });
+        dirtyCount = status.split('\n').filter(l => l.trim() && _SRC_RE.test(l)).length;
+    } catch { /* not a git repo / git missing → unknown */ }
+    _gitStateCache = { at: nowMs, root: repoRoot, head, dirtyCount };
+    return _gitStateCache;
+}
+
+/**
+ * Derive the freshness verdict from index age + the stamped index commit + the
+ * current working-tree state + whether the watch daemon is live. Pure (no I/O) so
+ * it is trivially testable.
+ *
+ * @returns {{ageSeconds:number|null, ageLabel:string, indexedCommit:string|null,
+ *            currentCommit:string|null, pendingChanges:number|null, commitMoved:boolean,
+ *            daemonRunning:boolean, stale:boolean, syncing:boolean}}
+ */
+export function computeFreshness({ ageSeconds = null, indexedCommit = null, current = null, daemonRunning = false } = {}) {
+    const ageLabel = ageSeconds == null ? 'unknown'
+        : ageSeconds < 60 ? `${ageSeconds}s` : ageSeconds < 3600 ? `${Math.floor(ageSeconds / 60)}m` : `${Math.floor(ageSeconds / 3600)}h`;
+    const currentCommit = current?.head ?? null;
+    const pendingChanges = current?.dirtyCount ?? null;
+    const commitMoved = Boolean(indexedCommit && currentCommit && indexedCommit !== currentCommit);
+    const hasPending = (pendingChanges != null && pendingChanges > 0) || commitMoved;
+    // A live daemon syncs changes incrementally, so pending work is "syncing", not
+    // "stale". Only when no daemon is keeping the index fresh is drift truly stale.
+    return {
+        ageSeconds, ageLabel, indexedCommit, currentCommit, pendingChanges, commitMoved,
+        daemonRunning, stale: hasPending && !daemonRunning, syncing: hasPending && daemonRunning,
+    };
+}
+
+/**
+ * A bounded 0..1 "this file is hot" score combining recency and churn, for the
+ * OPT-IN ranking boost only. Returns 0 when there is no signal for the file, so a
+ * disabled or signal-less repo never perturbs ranking.
+ */
+export function gitBoostScore(signals, file, nowSec = Date.now() / 1000) {
+    if (!signals) return 0;
+    const churn = (signals.churn && signals.churn[file]) || 0;
+    const last = (signals.lastCommit && signals.lastCommit[file]) || 0;
+    if (!churn && !last) return 0;
+    const ageDays = last ? Math.max(0, (nowSec - last) / 86400) : 3650;
+    const recency = Math.exp(-ageDays / 90);                                   // ~quarter half-life
+    const churnScore = Math.log1p(churn) / Math.log1p(signals.maxChurn || 1);  // 0..1
+    return 0.5 * recency + 0.5 * churnScore;
+}

package/indexer.mjs

@@ -12,20 +12,29 @@
  */
 import fs from 'fs';
 import path from 'path';
-import {
-    MAX_FILE_SIZE_BYTES, EXTENSIONS, getParserForFile, buildIgnoreFilter, getLocalEmbeddingsBatch,
-    extractImportsFromAST, extractSemanticChunks, resolveLocalImports, buildEmbeddingPayload,
-} from './parser-utils.mjs';
-import { readEmbeddingBinary, writeEmbeddingBinary } from './core-engine.mjs';
-import { embeddingKeyFor, summaryEmbeddingText, SUMMARY_VEC_SUFFIX } from './search-core.mjs';
-import { resolveConfig } from './config.mjs';
+import { MAX_FILE_SIZE_BYTES, buildIgnoreFilter, extractImportsFromAST, extractSemanticChunks } from './parse/extractor.mjs';
+import { EXTENSIONS, getParserForFile } from './parse/languages.mjs';
+import { extractRoutes } from './parse/routes.mjs';
+import { resolveLocalImports, buildEmbeddingPayload, fullBodyForEmbedding } from './parse/imports.mjs';
+import { readEmbeddingBinary, writeEmbeddingBinary } from './engine/binary.mjs';
+import { embeddingKeyFor, summaryEmbeddingText, SUMMARY_VEC_SUFFIX, WINDOW_VEC_SUFFIX, embeddingWindows } from './search-core.mjs';
+import { createEmbedder, describeEmbedder, readEmbedMeta, writeEmbedMeta, _resetSubprocesses } from './embeddings.mjs';
+import { resolveConfig, describeConfig, configNotices } from './config.mjs';
+import { AUTO_SQLITE_CHUNK_THRESHOLD } from './storage.mjs';
+import { ensureDataDir, migrateLegacyLayout } from './layout.mjs';
 import { enrichCoreChunks } from './enrichment.mjs';
+import { collectGitSignals, writeGitSignals } from './git-signals.mjs';
 
 const config = resolveConfig();
 const PROJECT_ROOT = config.projectRoot;
 const INDEX_PATH = config.indexPath;
 const EMBEDDINGS_PATH = config.embeddingPath;
 
+// Artifacts are written under <root>/.graph-indexer/ — make sure it exists and
+// relocate any artifacts left at the root by a pre-v1.4 install.
+ensureDataDir(PROJECT_ROOT);
+migrateLegacyLayout(PROJECT_ROOT);
+
 function walkRepo(dir, root, ig, files = []) {
     for (const entry of fs.readdirSync(dir)) {
         if (entry.startsWith('.')) continue;
@@ -43,17 +52,45 @@ function walkRepo(dir, root, ig, files = []) {
 
 async function main() {
     console.log(`\n🚀 Starting Optimized Indexer\n📂 Directory: ${PROJECT_ROOT}`);
-    console.log(`🗄  Storage: ${config.storage}${config.enrichment.enabled ? ` · 🧠 LLM enrichment: ${config.enrichment.model}` : ''}\n`);
+    console.log('⚙️  Effective configuration:');
+    for (const line of describeConfig(config)) console.log(`     ${line}`);
+    for (const notice of configNotices(config)) console.log(`⚠️  ${notice}`);
+    console.log('');
 
     const ig = buildIgnoreFilter(PROJECT_ROOT);
     const files = walkRepo(PROJECT_ROOT, PROJECT_ROOT, ig);
     console.log(`Found ${files.length} files to analyse.\n`);
 
-    const existingCache = readEmbeddingBinary(EMBEDDINGS_PATH);
+    // Resolve the embedding provider for this run (Ollama → in-process local →
+    // lexical, unless forced). Index time and query time must share the model.
+    const embedder = await createEmbedder(config);
+    console.log(`🔎 Embeddings: ${describeEmbedder(embedder)}`);
+    // Make the `auto` fallback ladder observable — never silently lexical-only.
+    if (config.embedProvider === 'auto') {
+        if (embedder.provider === 'local') {
+            console.log(`   ↪ Ollama not reachable at ${config.ollamaHost}; using the bundled in-process model (no daemon required).`);
+        } else if (embedder.provider === 'off') {
+            console.log(`   ↪ No Ollama and no in-process model installed (\`npm i @huggingface/transformers\`) — indexing lexical-only.`);
+        }
+    }
+    console.log('');
+
+    let existingCache = readEmbeddingBinary(EMBEDDINGS_PATH);
+    // A model switch invalidates the cached vectors — vectors of different models
+    // (and dims) must never be mixed in one bin. Re-embed from scratch.
+    const prevMeta = readEmbedMeta(EMBEDDINGS_PATH);
+    if (prevMeta?.model && embedder.model && prevMeta.model !== embedder.model) {
+        console.log(`⚠️  Embedding model changed (${prevMeta.model} → ${embedder.model}); re-embedding from scratch.`);
+        existingCache = new Map();
+    }
     console.log(`📦 Loaded ${existingCache.size} cached embeddings from previous runs.\n`);
 
-    const indexData = { chunks: [], graph: { dependencies: {}, importedBy: {} }, embeddingCache: {} };
+    const indexData = { chunks: [], graph: { dependencies: {}, importedBy: {}, routes: [] }, embeddingCache: {} };
     const pendingChunks = [];
+    // Full bodies of oversized chunks (snippet was truncated by the 3000-char cap),
+    // captured here while we still hold the source so the dense channel can window the
+    // WHOLE definition. Kept out of the chunk/JSON — transient, dropped after embedding.
+    const fullBodies = new Map();
     let totalCheckedFiles = 0;
 
     for (const absolutePath of files) {
@@ -79,75 +116,101 @@ async function main() {
             const imports = resolveLocalImports(rawImports, relPath, PROJECT_ROOT);
             indexData.graph.dependencies[relPath] = imports;
 
-            // Chunks are collected first; embedding/enrichment happen in batch below so
-            // we can route the high-value subset through the LLM before vectorising.
-            for (const chunk of extractSemanticChunks(tree.rootNode, relPath, content, ext)) {
+                    const fileChunks = extractSemanticChunks(tree.rootNode, relPath, content, ext);
+            for (const chunk of fileChunks) {
+                const fullBody = fullBodyForEmbedding(chunk, content);
+                if (fullBody) fullBodies.set(chunk.id, fullBody);
                 pendingChunks.push(chunk);
             }
+
+            for (const route of extractRoutes(tree.rootNode, relPath, fileChunks, ext)) {
+                indexData.graph.routes.push(route);
+            }
         } catch (err) {
             console.error(`\n💥 Error in ${relPath}: ${err.message}`);
         }
     }
 
-    // ── Optional LLM enrichment of the most central chunks (HyDE + summaries) ──────
-    // Runs before embedding so the hypothetical questions ride the same vector.
+    // Enrichment runs before embedding so hypothetical questions share the same vector space.
     if (config.enrichment.enabled) {
         await enrichCoreChunks(pendingChunks, indexData.graph, config);
     }
 
     // ── Embedding generation (cache-aware) ────────────────────────────────────────
-    // Vectors are keyed by embeddingKeyFor(chunk): content_hash for plain chunks,
-    // content_hash + enrichment digest for enriched ones. Because the enrichment
-    // cache returns the same summary for the same code, enriched chunks now HIT
-    // this cache on re-runs — previously every enriched chunk was re-embedded on
-    // every single index run.
+    // Vectors are keyed by embeddingKeyFor(chunk), which includes an enrichment
+    // digest so enriched chunks hit the cache on re-runs (previously they were
+    // re-embedded on every run).
     const toEmbed = [];
     for (const chunk of pendingChunks) {
         const vecKey = embeddingKeyFor(chunk);
         const sKey = vecKey + SUMMARY_VEC_SUFFIX;
-        // An enriched chunk needs BOTH vectors cached (code payload + summary) to
-        // skip embedding — e.g. indexes built before dual vectors only have the base.
+        // Compute the payload once here and reuse it in the worker to avoid a second build.
+        const payload = buildEmbeddingPayload(chunk, indexData.graph.dependencies[chunk.file_path] || [], fullBodies.get(chunk.id) || null);
+        const windows = embeddingWindows(payload);
+        // Indexes built before dual/window vectors only have the base key, so we must
+        // check that summary and window keys are also cached before skipping embedding.
         const summaryMissing = summaryEmbeddingText(chunk) && !existingCache.has(sKey);
-        if (existingCache.has(vecKey) && !summaryMissing) {
+        let windowsMissing = false;
+        for (let i = 1; i < windows.length; i++) if (!existingCache.has(vecKey + WINDOW_VEC_SUFFIX + i)) { windowsMissing = true; break; }
+        if (existingCache.has(vecKey) && !summaryMissing && !windowsMissing) {
             indexData.embeddingCache[vecKey] = Array.from(existingCache.get(vecKey));
             if (existingCache.has(sKey)) indexData.embeddingCache[sKey] = Array.from(existingCache.get(sKey));
+            for (let i = 1; i < windows.length; i++) {
+                const wk = vecKey + WINDOW_VEC_SUFFIX + i;
+                if (existingCache.has(wk)) indexData.embeddingCache[wk] = Array.from(existingCache.get(wk));
+            }
             indexData.chunks.push(chunk);
         } else {
-            toEmbed.push(chunk);
+            toEmbed.push({ chunk, payload, windows });
         }
     }
 
-    console.log(`\n\n🧠 Embedding Generation (Ollama)`);
+    console.log(`\n\n🧠 Embedding Generation — ${describeEmbedder(embedder)}`);
     console.log(`Chunks reused from cache: ${indexData.chunks.length}`);
     console.log(`New chunks to process: ${toEmbed.length}`);
 
     if (toEmbed.length > 0) {
         const BATCH_SIZE = 64;
-        const CONCURRENCY = 4;
+        // Embedding concurrency. A single local Ollama model serves requests
+        // serially, so a high fan-out just queues batches and risks per-request
+        // timeouts (a large model like qwen3-embedding:4b made every queued batch
+        // breach the old cap). Default 4 (fast on small models / when
+        // OLLAMA_NUM_PARALLEL is set); lower it (e.g. 1) for big models on modest
+        // hardware via INDEXER_EMBED_CONCURRENCY.
+        const CONCURRENCY = Number(process.env.INDEXER_EMBED_CONCURRENCY) > 0
+            ? Number(process.env.INDEXER_EMBED_CONCURRENCY) : 4;
         const batches = [];
         for (let i = 0; i < toEmbed.length; i += BATCH_SIZE) batches.push(toEmbed.slice(i, i + BATCH_SIZE));
 
-        let completed = 0;
+        let completed = 0, vectorBatches = 0, failedBatches = 0;
         console.time('Embedding Generation Duration');
         const worker = async (batch) => {
-            // Enriched chunks get TWO vectors: the full code payload (base key)
-            // and a compact summary-only text (`key|s`) that matches the vocabulary
-            // of natural-language queries (see search-core.summaryEmbeddingText).
+            // Oversized chunks need window vectors so semantic search can reach
+            // their tail (the embedder truncates the base text to window 0 only).
+            // Enriched chunks also get a compact summary vector whose vocabulary
+            // matches natural-language queries.
             const entries = [];
-            for (const c of batch) {
-                entries.push({ key: embeddingKeyFor(c), text: buildEmbeddingPayload(c, indexData.graph.dependencies[c.file_path] || []) });
+            for (const { chunk: c, payload, windows } of batch) {
+                const baseKey = embeddingKeyFor(c);
+                entries.push({ key: baseKey, text: payload });
+                for (let i = 1; i < windows.length; i++) entries.push({ key: baseKey + WINDOW_VEC_SUFFIX + i, text: windows[i] });
                 const sText = summaryEmbeddingText(c);
-                if (sText) entries.push({ key: embeddingKeyFor(c) + SUMMARY_VEC_SUFFIX, text: sText });
+                if (sText) entries.push({ key: baseKey + SUMMARY_VEC_SUFFIX, text: sText });
             }
-            const matrix = await getLocalEmbeddingsBatch(entries.map(e => e.text), true, {
-                ollamaHost: config.ollamaHost, model: config.embedModel,
-            });
+            // Per-batch graceful degradation: a slow/failed embedding batch (e.g. a
+            // timeout on a big model) must NOT abort the whole index. Those chunks
+            // are still indexed lexically — they just miss their vector until the
+            // next run re-tries them (the cache only stores what succeeded).
+            let matrix = null;
+            try { matrix = await embedder.embedDocuments(entries.map(e => e.text)); }
+            catch (e) { failedBatches++; process.stderr.write(`\n⚠️  embedding batch failed (${e.message}); indexing ${batch.length} chunks lexical-only this run.\n`); }
             if (matrix && matrix.length === entries.length) {
                 for (let j = 0; j < entries.length; j++) {
                     indexData.embeddingCache[entries[j].key] = matrix[j];
                 }
+                vectorBatches++;
             }
-            for (const chunk of batch) indexData.chunks.push(chunk);
+            for (const { chunk } of batch) indexData.chunks.push(chunk);
             completed += batch.length;
             process.stdout.write(`\r🤖 Embedding Progress: [${completed}/${toEmbed.length}] Chunks processed...`);
         };
@@ -155,9 +218,12 @@ async function main() {
             await Promise.all(batches.slice(i, i + CONCURRENCY).map(worker));
         }
         console.timeEnd('Embedding Generation Duration');
+        if (failedBatches > 0) {
+            console.log(`⚠️  ${failedBatches}/${batches.length} embedding batches failed (lexical-only for those chunks); re-run to fill them in. `
+                + `Tip: a large embed model on modest hardware wants INDEXER_EMBED_CONCURRENCY=1 and/or a higher INDEXER_EMBED_TIMEOUT_MS.`);
+        }
     }
 
-    // ── Reverse topology edges ────────────────────────────────────────────────────
     for (const [filePath, imports] of Object.entries(indexData.graph.dependencies)) {
         for (const dep of imports) {
             if (!indexData.graph.importedBy[dep]) indexData.graph.importedBy[dep] = [];
@@ -165,13 +231,23 @@ async function main() {
         }
     }
 
-    // ── Persist to the configured backend ─────────────────────────────────────────
-    if (config.storage === 'sqlite') {
-        const { SqliteGraphStore } = await import('./sqlite-store.mjs');
+    // Resolve 'auto' now that the true chunk count is known — the threshold is only
+    // meaningful after all chunks have been extracted.
+    const backend = config.storage === 'auto'
+        ? (indexData.chunks.length >= AUTO_SQLITE_CHUNK_THRESHOLD ? 'sqlite' : 'memory')
+        : config.storage;
+    if (config.storage === 'auto') {
+        console.log(`🗄  Storage: auto → ${backend} (${indexData.chunks.length} chunks, threshold ${AUTO_SQLITE_CHUNK_THRESHOLD}).`);
+    }
+
+    if (backend === 'sqlite') {
+        const { SqliteGraphStore } = await import('./engine/sqlite.mjs');
         const store = new SqliteGraphStore(config.sqlitePath, { embeddingPath: EMBEDDINGS_PATH });
         const res = store.buildFrom({
             chunks: indexData.chunks, graph: indexData.graph, embeddingCache: indexData.embeddingCache,
         });
+        store.close?.();
+        for (const p of [INDEX_PATH, `${INDEX_PATH}.tmp`]) { try { fs.unlinkSync(p); } catch { /* none */ } }
         console.log(`\n🎉 SQLite index built: ${res.chunks} chunks · ${res.terms} terms · dim ${res.dim}`);
         console.log(`   → ${config.sqlitePath}\n`);
     } else {
@@ -185,8 +261,41 @@ async function main() {
             fs.promises.rename(tmpPath, INDEX_PATH),
             fs.promises.rename(tmpBinPath, EMBEDDINGS_PATH),
         ]);
-        console.log(`\n🎉 Indexing completed blazingly fast. Total fragments: ${indexData.chunks.length}\n`);
+        for (const p of [config.sqlitePath, `${config.sqlitePath}-wal`, `${config.sqlitePath}-shm`]) {
+            try { fs.unlinkSync(p); } catch { /* none */ }
+        }
+        if (indexData.chunks.length === 0) {
+            console.log(`\n⚠️  Indexing finished but produced 0 chunks — no indexable definitions found under ${PROJECT_ROOT}.`);
+            console.log(`   Check the --repo path is correct and contains supported languages (see README), and note that trivial one-line definitions are intentionally skipped.\n`);
+        } else {
+            console.log(`\n🎉 Indexing complete. Total fragments: ${indexData.chunks.length}\n`);
+        }
+    }
+
+    // Stamp which model produced the shared embeddings bin, so the server queries
+    // with the same provider and the next index run detects a model switch.
+    writeEmbedMeta(EMBEDDINGS_PATH, {
+        provider: embedder.provider,
+        model: embedder.model,
+        dim: embedder.dim ?? prevMeta?.dim ?? null,
+    });
+
+    // Git signals are a sidecar kept out of the main index so they don't affect
+    // measured retrieval ranking — only opt-in tools consume them.
+    if (config.gitSignals) {
+        const signals = collectGitSignals(PROJECT_ROOT);
+        if (signals) {
+            writeGitSignals(config.gitSignalsPath, signals);
+            console.log(`🔄 Git signals: ${signals.commits} commits · ${Object.keys(signals.churn).length} files (churn/recency/co-change).\n`);
+        } else {
+            // Drop any stale sidecar so signals from a previous run don't linger.
+            try { fs.unlinkSync(config.gitSignalsPath); } catch { /* none to remove */ }
+        }
     }
+
+    // Kill any subprocess-based embedder (MLX) so the event loop can drain.
+    // Without this the readline interface on proc.stdout keeps Node alive indefinitely.
+    _resetSubprocesses();
 }
 
 main().catch(console.error);