graph-indexer 2.0.0 → 2.1.0

package/config.mjs

@@ -11,6 +11,7 @@
 import fs from 'fs';
 import path from 'path';
 import { artifactPaths, CONFIG_FILE_NAME, DATA_DIR_NAME } from './layout.mjs';
+import { assertSealCompatible } from './seal.mjs';
 
 export const DEFAULTS = Object.freeze({
     // 'auto' keeps the index in memory below AUTO_SQLITE_CHUNK_THRESHOLD chunks and
@@ -20,15 +21,20 @@ export const DEFAULTS = Object.freeze({
     embeddings: false,                 // OFF by default — lexical + stemming need zero dependencies.
                                        // Opt in with --embeddings or INDEXER_EMBEDDINGS=on.
     embedProvider: 'auto',
-    // 'auto'   → Ollama (if running w/ the model) → in-process local → lexical-only
-    // 'ollama' → force Ollama daemon   (requires: ollama serve + pulled model)
-    // 'local'  → in-process Xenova     (requires: npm i @huggingface/transformers)
-    // 'mlx'    → Apple Metal GPU       (requires: npm run embed:setup:mlx; macOS only)
-    // 'off'    → lexical-only, no vectors
+    // 'auto'      → Ollama (if running w/ the model) → in-process local → lexical-only
+    // 'ollama'    → force Ollama daemon   (requires: ollama serve + pulled model)
+    // 'local'     → in-process Xenova MiniLM (requires: npm i @huggingface/transformers)
+    // 'code-local'→ in-process code-specialized embedder (jina-code; same optional dep as
+    //               'local'; ~160 MB q8 first download, air-gapped after) — sharper code recall
+    // 'mlx'       → Apple Metal GPU       (requires: npm run embed:setup:mlx; macOS only)
+    // 'off'       → lexical-only, no vectors
     embedModel: 'nomic-embed-text',    // default Ollama embed model when embeddings are enabled.
                                        // qwen3-embedding:4b is a documented opt-in upgrade
                                        // (--embed-model / EMBED_MODEL), never a hardcoded default.
     localEmbedModel: 'Xenova/all-MiniLM-L6-v2', // in-process model (optional @huggingface/transformers)
+    codeEmbedModel: 'jinaai/jina-embeddings-v2-base-code', // in-process CODE-specialized model for
+                                       // provider 'code-local' (768-dim, MIT, 8192-token context).
+                                       // Override with --code-embed-model / INDEXER_CODE_EMBED_MODEL.
     mlxEmbedModel: 'mlx-community/all-MiniLM-L6-v2-4bit', // MLX embedder model (provider 'mlx').
                                        // Override with --mlx-embed-model / INDEXER_MLX_EMBED_MODEL;
                                        // must be an mlx_embeddings-compatible sentence model.
@@ -38,6 +44,42 @@ export const DEFAULTS = Object.freeze({
     mlxLmHost: 'http://localhost:8080',  // mlx_lm.server endpoint (default port 8080)
     gitSignals: true,                  // collect local git churn/recency/co-change at index time (air-gapped)
     gitRankBoost: 0,                   // 0..1 opt-in recency/churn weight in search_code (0 = ranking unchanged)
+    interprocedural: false,            // OPT-IN: index-time inter-procedural receiver-type fixpoint —
+                                       // propagate return types along factory call chains so multi-hop
+                                       // receivers resolve in get_call_graph / find_references. Air-gapped,
+                                       // deterministic; default index is byte-identical. Enable with
+                                       // --interprocedural / INDEXER_INTERPROCEDURAL.
+    symbolGraph: false,                // OPT-IN: persist a resolved symbol graph (chunk→chunk edges with
+                                       // kind + confidence) at index time, exposed via getEdges. findCallers/
+                                       // findReferers read it when present (identical sets, fall back to scan).
+                                       // Air-gapped, deterministic; default index byte-identical; search
+                                       // ranking unaffected. Enable with --symbol-graph / INDEXER_SYMBOL_GRAPH.
+    taint: false,                      // OPT-IN: precompute + serialize taint flows (C2) at index time so the
+                                       // trace_taint / find_tainted_sinks tools serve instantly (both backends
+                                       // serve identical flows). Default off → tools compute query-time (the
+                                       // index is byte-identical). --taint / INDEXER_TAINT.
+    ranker: 'rrf',                     // Search ranker (D3). 'rrf' (default): the RRF + boost-ladder fusion —
+                                       // BYTE-IDENTICAL to before. 'learned': a post-fusion linear re-rank over
+                                       // RRF score + A5 centrality + resolved/SCIP in-degree + git (zero-dep dot
+                                       // product; lift is on --symbol-graph indexes). --ranker / INDEXER_RANKER.
+    learnedSparse: false,              // OPT-IN: learn a corpus co-occurrence (PMI) association model (B3) at
+                                       // index time and serialize it, adding an NL-only vocabulary-EXPANSION
+                                       // channel at query time (zero new dep, fully air-gapped). Default off →
+                                       // no model → byte-identical. --learned-sparse / INDEXER_LEARNED_SPARSE.
+    sealed: 'off',                     // Sealed mode (F1): 'off' (default) | 'local' (loopback only) |
+                                       // 'strict' (zero network egress; lexical-only). Fail-closed:
+                                       // refuses to start if an enabled feature would egress beyond the
+                                       // tier, and installs a deny-by-default runtime egress guard.
+                                       // --sealed [strict|local] / INDEXER_SEALED.
+    resolver: 'heuristic',             // Resolver provider for the symbol graph (A1/A2). 'heuristic' (default):
+                                       // edges stay { high, name_only } — byte-identical. 'precise' (A1): lift
+                                       // provably-unambiguous edges (sole definition / type-pinned receiver)
+                                       // to a `resolved` tier, powering impact_of_edit precision='strict'.
+                                       // 'scip' (A2): ingest a locally-generated SCIP index (--scip-index) and
+                                       // promote/suppress `calls` edges by real cross-file bindings. Only
+                                       // meaningful with --symbol-graph. --resolver / INDEXER_RESOLVER.
+    scipIndex: null,                   // Path to a locally-generated SCIP index for --resolver scip (A2).
+                                       // --scip-index / INDEXER_SCIP_INDEX / scipIndex config; null = none.
     enrichment: Object.freeze({
         enabled: false,
         model: 'qwen2.5-coder:1.5b',   // smallest coder model; quality sufficient for summaries.
@@ -51,9 +93,16 @@ export const DEFAULTS = Object.freeze({
 
     }),
     rerank: Object.freeze({
-        enabled: false,                // opt-in: one LLM call per natural-language query
-        model: 'qwen2.5-coder:7b',     // judge quality matters: 7B measured +50% semantic rank-1, 1.5B ~nil
-        topM: 12,                      // candidates shown to the judge to reorder
+        enabled: false,                // opt-in: one rerank pass per natural-language query
+        provider: 'generative',        // 'generative' = LLM judge (default) | 'cross-encoder' = local
+                                       // air-gapped MS-MARCO cross-encoder (no Ollama/daemon) |
+                                       // 'late-interaction' = ColBERT-style multi-vector MaxSim (B4,
+                                       // reuses the embedder + stored doc vectors; needs --embeddings).
+                                       // Set with --rerank-provider / RERANK_PROVIDER.
+        model: 'qwen2.5-coder:7b',     // generative judge model: 7B measured +50% semantic rank-1, 1.5B ~nil
+        crossEncoderModel: 'Xenova/ms-marco-MiniLM-L-6-v2', // cross-encoder provider model
+                                       // (optional @huggingface/transformers; downloaded on first use)
+        topM: 12,                      // candidates the reranker reorders
         poolSize: 15,                  // over-fetch depth when reranking, so a correct-but-deep
                                        // hit can be RESCUED into top_k (then truncated to top_k)
     }),
@@ -150,10 +199,55 @@ export function resolveConfig({ argv = process.argv.slice(2), env = process.env,
         ?? (Number.isFinite(file.gitRankBoost) ? file.gitRankBoost : DEFAULTS.gitRankBoost);
     const gitRankBoost = Math.min(1, Math.max(0, Number(gitRankBoostRaw) || 0));
 
+    // Opt-in inter-procedural receiver-type fixpoint (index-time, air-gapped).
+    const interprocedural = argv.includes('--interprocedural')
+        || env.INDEXER_INTERPROCEDURAL === 'on'
+        || file.interprocedural === true;
+
+    // Opt-in persistent resolved symbol graph (index-time, air-gapped).
+    const symbolGraph = argv.includes('--symbol-graph')
+        || env.INDEXER_SYMBOL_GRAPH === 'on'
+        || file.symbolGraph === true;
+
+    // Opt-in index-time taint serialization (C2): precompute + persist the flow set.
+    const taint = argv.includes('--taint')
+        || env.INDEXER_TAINT === 'on'
+        || file.taint === true;
+
+    // Opt-in learned-sparse model (B3): learn + persist the corpus association model at index time.
+    const learnedSparse = argv.includes('--learned-sparse')
+        || env.INDEXER_LEARNED_SPARSE === 'on'
+        || file.learnedSparse === true;
+
+    // Search ranker (D3). Unknown values fall back to the default 'rrf' (byte-identical fusion).
+    const rankerRaw = flagValue(argv, '--ranker') || env.INDEXER_RANKER || file.ranker || DEFAULTS.ranker;
+    const ranker = (rankerRaw === 'learned' || rankerRaw === 'rrf') ? rankerRaw : 'rrf';
+
+    // Resolver provider for the symbol graph (A1/A2). Unknown values fall back to 'heuristic'.
+    const resolverRaw = flagValue(argv, '--resolver') || env.INDEXER_RESOLVER || file.resolver || DEFAULTS.resolver;
+    const resolver = (resolverRaw === 'precise' || resolverRaw === 'heuristic' || resolverRaw === 'scip')
+        ? resolverRaw : 'heuristic';
+    // A2: path to a locally-generated SCIP index, consumed by the `scip` resolver. flag > env > file.
+    // Resolved against projectRoot (a relative path) so a config-file value is repo-relative.
+    const scipRaw = flagValue(argv, '--scip-index') || env.INDEXER_SCIP_INDEX || file.scipIndex || null;
+    const scipIndex = scipRaw ? path.resolve(projectRoot, scipRaw) : null;
+
+    // Sealed mode (F1). flag > env > file > default. A bare `--sealed` (no value, or followed by
+    // another flag) means strict; `--sealed local` opts into the loopback tier; env 'on' → strict.
+    const sealedFlagPresent = argv.includes('--sealed');
+    const sealedFlagVal = sealedFlagPresent ? flagValue(argv, '--sealed') : undefined;
+    let sealedRaw;
+    if (sealedFlagPresent) sealedRaw = (sealedFlagVal && !sealedFlagVal.startsWith('--')) ? sealedFlagVal : 'strict';
+    else if (env.INDEXER_SEALED) sealedRaw = env.INDEXER_SEALED;
+    else sealedRaw = typeof file.sealed === 'string' ? file.sealed : DEFAULTS.sealed;
+    const sealed = sealedRaw === 'on' ? 'strict'
+        : (['strict', 'local', 'off'].includes(sealedRaw) ? sealedRaw : 'off');
+
     const paths = artifactPaths(projectRoot);
 
-    return Object.freeze({
+    const resolved = Object.freeze({
         projectRoot,
+        sealed,
         storage,
         dataDir: paths.dataDir,
         indexPath: paths.indexPath,
@@ -175,12 +269,24 @@ export function resolveConfig({ argv = process.argv.slice(2), env = process.env,
         // This is the model `auto`/`ollama` providers pull (e.g. qwen3-embedding:4b).
         embedModel: flagValue(argv, '--embed-model') || env.EMBED_MODEL || file.embedModel || DEFAULTS.embedModel,
         localEmbedModel: file.localEmbedModel || DEFAULTS.localEmbedModel,
+        // Code-specialized in-process model (provider 'code-local'): --code-embed-model flag >
+        // INDEXER_CODE_EMBED_MODEL env > config > jina-code default. Stamped into the embed-meta
+        // so query time and index time agree, and a model switch forces a clean re-embed.
+        codeEmbedModel: flagValue(argv, '--code-embed-model') || env.INDEXER_CODE_EMBED_MODEL
+            || file.codeEmbedModel || DEFAULTS.codeEmbedModel,
         // MLX embedder model: --mlx-embed-model flag > INDEXER_MLX_EMBED_MODEL env >
         // config > pinned MiniLM default. Passed to the Python server + stamped in meta.
         mlxEmbedModel: flagValue(argv, '--mlx-embed-model') || env.INDEXER_MLX_EMBED_MODEL
             || file.mlxEmbedModel || DEFAULTS.mlxEmbedModel,
         gitSignals,
         gitRankBoost,
+        interprocedural,
+        symbolGraph,
+        taint,
+        ranker,
+        resolver,
+        scipIndex,
+        learnedSparse,
 
         enrichment: Object.freeze({
             enabled: enrichmentEnabled,
@@ -194,20 +300,34 @@ export function resolveConfig({ argv = process.argv.slice(2), env = process.env,
                 : (Number(fileEnrich.concurrency) > 0 ? Number(fileEnrich.concurrency) : DEFAULTS.enrichment.concurrency),
         }),
 
-        rerank: Object.freeze({
-            // OFF unless requested: --rerank flag, RERANK_MODEL env (naming a model
-            // implies enabling it), or the project config.
-            enabled: argv.includes('--rerank') || Boolean(env.RERANK_MODEL) || Boolean((file.rerank || {}).enabled),
-            model: env.RERANK_MODEL || (file.rerank || {}).model || DEFAULTS.rerank.model,
-            topM: Number.isInteger((file.rerank || {}).topM) ? (file.rerank || {}).topM : DEFAULTS.rerank.topM,
-            poolSize: Number.isInteger((file.rerank || {}).poolSize) ? (file.rerank || {}).poolSize : DEFAULTS.rerank.poolSize,
-        }),
+        rerank: (() => {
+            // Provider: env > --rerank-provider flag > config; validated to a known provider.
+            const provRaw = env.RERANK_PROVIDER || flagValue(argv, '--rerank-provider') || (file.rerank || {}).provider;
+            const provider = ['generative', 'cross-encoder', 'late-interaction'].includes(provRaw) ? provRaw : DEFAULTS.rerank.provider;
+            return Object.freeze({
+                // OFF unless requested: --rerank flag, RERANK_MODEL env (naming a model implies
+                // enabling it), an explicit provider (env/flag), or the project config.
+                enabled: argv.includes('--rerank') || Boolean(env.RERANK_MODEL) || Boolean(provRaw)
+                    || Boolean((file.rerank || {}).enabled),
+                provider,
+                model: env.RERANK_MODEL || (file.rerank || {}).model || DEFAULTS.rerank.model,
+                crossEncoderModel: env.RERANK_CROSS_ENCODER_MODEL || (file.rerank || {}).crossEncoderModel
+                    || DEFAULTS.rerank.crossEncoderModel,
+                topM: Number.isInteger((file.rerank || {}).topM) ? (file.rerank || {}).topM : DEFAULTS.rerank.topM,
+                poolSize: Number.isInteger((file.rerank || {}).poolSize) ? (file.rerank || {}).poolSize : DEFAULTS.rerank.poolSize,
+            });
+        })(),
 
         hyde: Object.freeze({
             enabled: Boolean((file.hyde || {}).enabled),
             model: (file.hyde || {}).model || DEFAULTS.hyde.model,
         }),
     });
+
+    // F1: fail-closed. If --sealed was requested and an enabled feature would egress beyond the
+    // tier, throw here so a sealed-incompatible config can never be constructed (no-op when off).
+    assertSealCompatible(resolved);
+    return resolved;
 }
 
 // Memoised singleton for import-time consumers (e.g. parse/languages.mjs language loading).
@@ -229,10 +349,11 @@ export function getConfig() {
  * @returns {string[]}
  */
 export function describeConfig(config, { backend = config.storage } = {}) {
-    // Show the model that the chosen provider actually loads (mlx/local use their own).
+    // Show the model that the chosen provider actually loads (mlx/local/code-local use their own).
     const provModel = config.embedProvider === 'mlx' ? config.mlxEmbedModel
         : config.embedProvider === 'local' ? config.localEmbedModel
-            : config.embedModel;
+            : config.embedProvider === 'code-local' ? config.codeEmbedModel
+                : config.embedModel;
     const emb = config.embeddingsEnabled
         ? `on · provider=${config.embedProvider}, model=${provModel}`
         : 'off (lexical + stemming only)';
@@ -240,8 +361,24 @@ export function describeConfig(config, { backend = config.storage } = {}) {
         `storage     : ${backend}`,
         `embeddings  : ${emb}`,
         `enrichment  : ${config.enrichment.enabled ? `on · model=${config.enrichment.model} · provider=${config.llmProvider}` : 'off'}`,
-        `reranker    : ${config.rerank.enabled ? `on · model=${config.rerank.model} · provider=${config.llmProvider}` : 'off'}`,
+        `reranker    : ${config.rerank.enabled
+            ? (config.rerank.provider === 'cross-encoder'
+                ? `on · provider=cross-encoder · model=${config.rerank.crossEncoderModel}`
+                : config.rerank.provider === 'late-interaction'
+                    ? 'on · provider=late-interaction · ColBERT-style multi-vector MaxSim (needs --embeddings)'
+                    : `on · provider=generative · model=${config.rerank.model} · llm=${config.llmProvider}`)
+            : 'off'}`,
         `git signals : ${config.gitSignals ? 'on' : 'off'}${config.gitRankBoost ? ` · rank boost=${config.gitRankBoost}` : ''}`,
+        `interproc.  : ${config.interprocedural ? 'on · factory return-type propagation' : 'off'}`,
+        `symbol graph: ${config.symbolGraph ? 'on · persisted resolved edges (getEdges)' : 'off'}`,
+        `taint       : ${config.taint ? 'on · precomputed flows (trace_taint / find_tainted_sinks)' : 'off (query-time)'}`,
+        `ranker      : ${config.ranker === 'learned' ? 'learned · post-fusion linear re-rank (opt-in)' : 'rrf (default)'}`,
+        `sparse      : ${config.learnedSparse ? 'on · learned-sparse vocabulary expansion (NL-only channel)' : 'off'}`,
+        `resolver    : ${config.resolver === 'precise' ? 'precise · unambiguous edges → `resolved` tier'
+            : config.resolver === 'scip' ? `scip · cross-file SCIP bindings → \`resolved\` (${config.scipIndex || '⚠️ no --scip-index'})`
+            : 'heuristic (default)'}`,
+        `sealed mode : ${config.sealed === 'strict' ? '🔒 strict · zero network egress (enforced)'
+            : config.sealed === 'local' ? '🔒 local · loopback only (enforced)' : 'off'}`,
     ];
 }
 
@@ -256,11 +393,99 @@ export function describeConfig(config, { backend = config.storage } = {}) {
  */
 export function configNotices(config) {
     const out = [];
+    if (config.sealed === 'strict' || config.sealed === 'local') {
+        out.push(`Sealed mode (${config.sealed}): a deny-by-default egress guard is installed — `
+            + `${config.sealed === 'strict' ? 'NO network connections are permitted' : 'only loopback connections are permitted'}. `
+            + 'The build refused to start if any enabled feature would egress beyond the tier. '
+            + 'The in-process guard (sockets/http/fetch) is airtight; the child_process denylist is '
+            + 'best-effort. Run `idx-index --attest` for the signed-able manifest.');
+    }
+    if (config.embeddingsEnabled && config.embedProvider === 'code-local') {
+        out.push(`Code-specialized embedder (code-local): the dense channel is built in-process by `
+            + `${config.codeEmbedModel} — a CODE-trained model (identifiers, call patterns, type names) `
+            + 'that sharpens semantic recall on typed languages where the general-purpose MiniLM is '
+            + 'weakest. It needs the optional \'@huggingface/transformers\' package; the quantized '
+            + '(q8) model is ~160 MB on the FIRST run (downloaded once, then fully air-gapped — no daemon, no network). '
+            + 'It is slower per chunk than MiniLM and produces 768-dim vectors — switching to it from '
+            + 'another provider forces a clean re-embed (vectors of different models must not be mixed). '
+            + 'Pair with the persistent embed cache to pay the embedding cost only once. Measure on '
+            + 'YOUR repo (`npm run test:eval -- --embeddings --embed-provider code-local`) before adopting.');
+    }
+    if (config.interprocedural) {
+        out.push('Inter-procedural receiver types are resolved at index time (a whole-program pass). '
+            + 'The watch daemon updates one file at a time and does NOT re-run the fixpoint, so '
+            + 'cross-file factory chains refresh on the next full `idx-index`; until then they fall '
+            + 'back to single-hop resolution (never worse than the default). It only sharpens '
+            + 'get_call_graph / find_references — search ranking is unaffected.');
+    }
+    if (config.symbolGraph) {
+        out.push('The resolved symbol graph is built once at index time (a whole-program pass), adding '
+            + 'index time roughly proportional to repo size. The watch daemon does NOT rebuild it, so '
+            + 'edges refresh on the next full `idx-index`; until then findCallers/findReferers fall back '
+            + 'to the name-match scan (never worse). It powers getEdges; search ranking is unaffected.');
+    }
+    if (config.ranker === 'learned') {
+        out.push('Learned ranker (D3): the fused top-N is re-ordered by a zero-dependency linear model '
+            + 'over the RRF score + symbol centrality + resolved/SCIP in-degree + git recency. It is a '
+            + 'post-fusion re-rank (the default RRF ordering is untouched). Its discriminating features '
+            + 'are zero without --symbol-graph, so pair it with --symbol-graph for the measured lift; on a '
+            + 'plain lexical index it is ≈neutral. RRF remains the default; this is opt-in.');
+    }
+    if (config.taint) {
+        out.push('Taint flows are precomputed once at index time and serialized, so trace_taint / '
+            + 'find_tainted_sinks answer instantly (both backends serve identical flows). The watch '
+            + 'daemon does NOT rebuild them — flows refresh on the next full `idx-index`; until then the '
+            + 'tools fall back to a query-time scan (never worse). It only affects the security tools; '
+            + 'search ranking is unaffected. Pair with --symbol-graph for cross-function precision.');
+    }
+    if (config.learnedSparse) {
+        out.push('Learned-sparse (B3): a corpus co-occurrence (PMI) association model is learned + '
+            + 'serialized at index time, adding a vocabulary-EXPANSION channel that fires ONLY for '
+            + 'natural-language queries (symbolic/exact lookups stay byte-identical). Zero new dependency, '
+            + 'fully air-gapped, parity-safe. The whole-program model refreshes on the next full '
+            + '`idx-index` (the daemon drops it on a per-file edit). Measure on YOUR repo (`test:eval`) — '
+            + 'it is a recall aid that helps lexical-gap semantic queries and is ≈neutral elsewhere.');
+    }
+    if ((config.resolver === 'precise' || config.resolver === 'scip') && !config.symbolGraph) {
+        out.push(`--resolver ${config.resolver} has no effect without --symbol-graph (the resolver runs only `
+            + 'when the symbol graph is built). Add --symbol-graph to enable the `resolved` edge tier.');
+    } else if (config.resolver === 'precise') {
+        out.push('Precise resolver: edges with a provably-unambiguous binding (sole definition, or a '
+            + 'type-pinned receiver) are promoted from `high` to a `resolved` tier — used by '
+            + "impact_of_edit(precision: 'strict') for a false-positive-free blast radius. Confidence "
+            + 'strings only change; edge sets, parity, and search ranking are unaffected.');
+    } else if (config.resolver === 'scip' && !config.scipIndex) {
+        out.push('--resolver scip needs --scip-index <path.scip> (a SCIP index generated locally by your '
+            + 'own toolchain — scip-typescript, scip-python, scip-java, rust-analyzer --scip). Without it '
+            + 'the resolver is inert and edges stay heuristic { high, name_only }.');
+    } else if (config.resolver === 'scip') {
+        out.push('SCIP resolver: cross-file bindings from a locally-generated SCIP index promote CONFIRMED '
+            + '`calls` edges to a `resolved` tier AND suppress the wrong-target fan-out an ambiguous name '
+            + 'would otherwise emit (heritage/type edges stay heuristic). Suppression is sound under '
+            + 'partial coverage — a def SCIP never recorded is never dropped. Air-gapped (a local file '
+            + 'read; sealed-compatible). Files the SCIP index does not cover fall back to heuristic, never '
+            + 'worse. Coverage is printed at index time; re-generate the .scip when code changes.');
+    }
     if (config.enrichment.enabled && !config.rerank.enabled) {
         out.push('Enrichment is most effective when paired with --rerank. Running enrichment-only may '
             + 'regress semantic precision (measured: gin MRR 0.48→0.39 on qwen3 embeddings).');
     }
-    if (config.rerank.enabled) {
+    if (config.rerank.enabled && config.rerank.provider === 'cross-encoder') {
+        out.push('Cross-encoder reranker scores (query, candidate) pairs locally and air-gapped — '
+            + 'no LLM/daemon, deterministic, ~tens of ms per natural-language query. It needs the '
+            + "optional '@huggingface/transformers' package (first run downloads the model). It sees "
+            + 'only a candidate signature/summary line, not full code, so it sharpens near-ties rather '
+            + 'than reasoning about behaviour. Measure on your repo (`npm run test:eval -- --rerank '
+            + '--rerank-provider cross-encoder`) before adopting.');
+    } else if (config.rerank.enabled && config.rerank.provider === 'late-interaction') {
+        out.push('Late-interaction (ColBERT-style) reranker scores the query MULTI-vectors against '
+            + 'each candidate\'s stored document sub-vectors with MaxSim — air-gapped (the same local '
+            + 'embedder), deterministic, parity-safe, with NO storage blow-up (it reuses the vectors '
+            + '--embeddings already stores). It REQUIRES --embeddings (no doc vectors ⇒ no-op) and '
+            + 'approximates true token-level ColBERT with a sentence embedder over query sub-units. '
+            + 'Measure on your repo (`npm run test:eval -- --embeddings --rerank '
+            + '--rerank-provider late-interaction`) before adopting.');
+    } else if (config.rerank.enabled) {
         out.push('Reranker improves rank-1 for Go/Python repositories (gin: 0.20→0.40) but has shown '
             + 'regressions on JavaScript repositories (express: 0.43→0.29). Behaviour depends on '
             + 'repository language. See README for per-language benchmark data.');

package/embeddings.mjs

@@ -31,6 +31,16 @@ const PYTHON_DIR = join(__dirname, 'embedders', 'python');
 
 export const LOCAL_EMBED_MODEL_DEFAULT = 'Xenova/all-MiniLM-L6-v2';
 export const LOCAL_EMBED_DIM = 384;
+// Code-specialized in-process embedder (provider 'code-local'). Same optional
+// @huggingface/transformers feature-extraction path as the general-purpose 'local'
+// MiniLM, but a model trained on code semantics (identifiers, call patterns, type
+// names) where MiniLM is weakest. jina-embeddings-v2-base-code is 768-dim, MIT, ships
+// first-party ONNX (transformers.js reads it directly, no trust_remote_code), supports
+// an 8192-token context, and is ~160 MB (q8 quantized) on first download (air-gapped thereafter).
+// Override with --code-embed-model / INDEXER_CODE_EMBED_MODEL (e.g. the Xenova mirror
+// `Xenova/jina-embeddings-v2-base-code`, or `Xenova/codet5p-110m-embedding`).
+export const CODE_EMBED_MODEL_DEFAULT = 'jinaai/jina-embeddings-v2-base-code';
+export const CODE_EMBED_DIM = 768; // informational only — dim is derived lazily from the first vector.
 // Passed to the Python server at spawn time (argv) AND stamped into the .meta.json
 // sidecar so index time and query time always agree; the indexer's model-change
 // detection forces a clean re-embed when this differs. Override via config.mlxEmbedModel.
@@ -80,12 +90,13 @@ export async function localEmbedAvailable() {
 /**
  * Decide which provider to use for THIS index/run.
  *   • INDEXER_EMBEDDINGS=off / embedProvider 'off' → no vectors (lexical-only).
- *   • 'ollama' / 'local' / 'mlx'                    → forced (mlx is a native Python
- *     embedder on the Apple Metal GPU; macOS-only).
+ *   • 'ollama' / 'local' / 'code-local' / 'mlx'     → forced (code-local is the
+ *     in-process code-specialized embedder; mlx is a native Python embedder on the
+ *     Apple Metal GPU, macOS-only).
  *   • 'auto' (default) → running Ollama, else the bundled local model, else off.
  * Probes are injectable for deterministic tests.
  *
- * @returns {Promise<{provider:'ollama'|'local'|'mlx'|'off', model:(string|null)}>}
+ * @returns {Promise<{provider:'ollama'|'local'|'code-local'|'mlx'|'off', model:(string|null)}>}
  */
 export async function resolveEmbedProvider(config, { probe = probeOllama, hasLocal = localEmbedAvailable, hasModel = ollamaHasModel } = {}) {
     if (config.embeddingsEnabled === false) return { provider: 'off', model: null };
@@ -94,6 +105,9 @@ export async function resolveEmbedProvider(config, { probe = probeOllama, hasLoc
     if (want === 'off') return { provider: 'off', model: null };
     if (want === 'ollama') return { provider: 'ollama', model: config.embedModel };
     if (want === 'local') return { provider: 'local', model: localModel };
+    // code-local: forced, in-process, code-specialized. Never reached by `auto` (it
+    // is an explicit opt-in — `auto` stays MiniLM so the no-config path is unchanged).
+    if (want === 'code-local') return { provider: 'code-local', model: config.codeEmbedModel || CODE_EMBED_MODEL_DEFAULT };
     if (want === 'mlx') { assertProviderPlatform('mlx'); return { provider: 'mlx', model: config.mlxEmbedModel || MLX_EMBED_MODEL }; }
     // Prefer Ollama only when it actually HAS the configured model pulled — otherwise
     // the indexer would crash on the first batch. Fall back to in-process, then off.
@@ -154,7 +168,8 @@ async function _localPipeline(model) {
     catch {
         throw new Error(
             "Local embeddings need the optional '@huggingface/transformers' package. "
-            + "Install it (`npm i @huggingface/transformers`) or set embedProvider to 'ollama'/'off'."
+            + "Install it (`npm run embed:setup:local`, or `npm i @huggingface/transformers`) "
+            + "or set embedProvider to 'ollama'/'off'."
         );
     }
     _pipe = await mod.pipeline('feature-extraction', model);
@@ -173,6 +188,67 @@ async function _localEmbedMany(model, texts) {
 /** Reset the cached local pipeline (tests). */
 export function _resetLocalPipeline() { _pipe = null; _pipeModel = null; }
 
+// Code-specialized in-process embedder (provider 'code-local'). A SEPARATE lazy
+// singleton from _localPipeline so a code-local and a plain-local run in the same
+// process don't evict each other's model. The import lives inside the function, so
+// the default path (lexical / `auto` → MiniLM) NEVER loads the code model — it is
+// pulled only when 'code-local' is actually selected and a vector is requested.
+// `_codeLoads` lets tests prove that lazy-load contract without a network call.
+let _codePipe = null, _codePipeModel = null, _codeLoads = 0;
+async function _codeLocalPipeline(model) {
+    if (_codePipe && _codePipeModel === model) return _codePipe;
+    let mod;
+    try { mod = await import('@huggingface/transformers'); }
+    catch {
+        throw new Error(
+            "The code-local embedder needs the optional '@huggingface/transformers' package. "
+            + "Install it (`npm run embed:setup:local`, or `npm i @huggingface/transformers`) "
+            + "or set embedProvider to 'ollama'/'local'/'off'."
+        );
+    }
+    // q8 quantized weights: a 161M-param code model is ~4× faster on CPU at q8 with
+    // negligible recall loss (the same quantized-ONNX choice the cross-encoder makes),
+    // which is what makes an in-process code embedder practical on a laptop. transformers.js
+    // falls back to fp32 automatically if a model ships no quantized ONNX.
+    _codePipe = await mod.pipeline('feature-extraction', model, { dtype: 'q8' });
+    _codePipeModel = model;
+    _codeLoads++;
+    return _codePipe;
+}
+async function _codeLocalEmbedMany(model, texts) {
+    const pipe = await _codeLocalPipeline(model);
+    // jina-code is symmetric (no nomic-style query/document prefixes); mean-pool +
+    // normalize, identical handling to the MiniLM path, so cache keys / windowing are
+    // unchanged. The 8000-char cap matches search-core's EMBEDDING_CONTEXT_LIMIT.
+    const out = await pipe(
+        texts.map(t => (t.length > DOC_CHAR_LIMIT ? t.slice(0, DOC_CHAR_LIMIT) : t)),
+        { pooling: 'mean', normalize: true }
+    );
+    return out.tolist();
+}
+/** Reset the cached code-local pipeline (tests). */
+export function _resetCodeLocalPipeline() { _codePipe = null; _codePipeModel = null; }
+/** Number of times the code-local model has been lazy-loaded (tests: lazy-load contract). */
+export function _codeLocalPipelineLoads() { return _codeLoads; }
+
+/**
+ * Should the cached vectors be discarded before this run? True only when the previous
+ * build's embed-meta records a DIFFERENT model than the embedder about to run. Vectors
+ * of different models/dims must never be mixed in one `.bin` — a 768-dim code-local
+ * vector and a 384-dim MiniLM vector under the same cache would silently no-op the
+ * dense channel for the mismatched dim (scan/sketch skip a wrong-dim entry). Model name
+ * is the pre-embed signal (the actual dim is unknown until the first vector returns), so
+ * switching `--embed-provider local` → `code-local` forces a clean re-embed rather than a
+ * silent half-empty index.
+ *
+ * @param {object|null} prevMeta  The prior `.embeddings.bin.meta.json` ({ provider, model, dim }).
+ * @param {string|null} model     The model the current embedder will use.
+ * @returns {boolean}
+ */
+export function embedModelChanged(prevMeta, model) {
+    return Boolean(prevMeta?.model && model && prevMeta.model !== model);
+}
+
 // ─── Native Python embedder (mlx) ────────────────────────────────────────────
 // MLX (Apple Metal) is Python-first with no Node bindings, so it runs as a long-lived
 // child process speaking newline-delimited JSON:
@@ -400,6 +476,7 @@ export function _resetSubprocesses() {
 /** Canonical model id for a forced provider when no explicit model is supplied. */
 function defaultModelForProvider(provider, config) {
     if (provider === 'local') return config.localEmbedModel || LOCAL_EMBED_MODEL_DEFAULT;
+    if (provider === 'code-local') return config.codeEmbedModel || CODE_EMBED_MODEL_DEFAULT;
     if (provider === 'mlx') return config.mlxEmbedModel || MLX_EMBED_MODEL;
     return config.embedModel;
 }
@@ -411,10 +488,11 @@ function defaultModelForProvider(provider, config) {
  *
  * @param {object} config
  * @param {object} [opts]
- * @param {'ollama'|'local'|'mlx'|'off'} [opts.provider]  Force a provider (e.g.
- *        the one stamped in the index meta). When omitted, resolveEmbedProvider decides.
+ * @param {'ollama'|'local'|'code-local'|'mlx'|'off'} [opts.provider]  Force a provider
+ *        (e.g. the one stamped in the index meta). When omitted, resolveEmbedProvider decides.
  * @param {string} [opts.model]
- * @param {object} [opts.backends]  Inject { ollamaEmbedOne, ollamaEmbedMany, localEmbedMany } (tests).
+ * @param {object} [opts.backends]  Inject { ollamaEmbedOne, ollamaEmbedMany, localEmbedMany,
+ *        codeLocalEmbedMany } (tests).
  * @returns {Promise<{provider:string, model:(string|null), dim:(number|null),
  *                     embedQuery:(t:string)=>Promise<number[]|null>,
  *                     embedDocuments:(t:string[])=>Promise<number[][]|null>}>}
@@ -429,6 +507,7 @@ export async function createEmbedder(config, opts = {}) {
     const ollamaOne = b.ollamaEmbedOne || _ollamaEmbedOne;
     const ollamaMany = b.ollamaEmbedMany || _ollamaEmbedMany;
     const localMany = b.localEmbedMany || _localEmbedMany;
+    const codeLocalMany = b.codeLocalEmbedMany || _codeLocalEmbedMany;
     // Derive dim lazily from the first returned vector. Pre-setting LOCAL_EMBED_DIM=384
     // mis-stamps models whose dim isn't 384 (e.g. jina-embeddings-v2-base-code = 768)
     // into the .meta.json before the first embed call.
@@ -443,7 +522,8 @@ export async function createEmbedder(config, opts = {}) {
                 if (v && dim == null) dim = v.length;
                 return v ?? null;
             }
-            const [v] = await localMany(model, [text]);
+            const many = provider === 'code-local' ? codeLocalMany : localMany;
+            const [v] = await many(model, [text]);
             if (v && dim == null) dim = v.length;
             return v ?? null;
         } catch { return null; } // graceful: caller drops to lexical
@@ -454,6 +534,7 @@ export async function createEmbedder(config, opts = {}) {
         let vecs;
         if (provider === 'ollama') vecs = await _withRetry(() => ollamaMany(host, model, texts));
         else if (provider === 'mlx') vecs = await _subprocessEmbedMany(provider, texts, model);
+        else if (provider === 'code-local') vecs = await codeLocalMany(model, texts);
         else vecs = await localMany(model, texts);
         if (vecs && vecs[0] && dim == null) dim = vecs[0].length;
         return vecs;
@@ -484,6 +565,7 @@ export function writeEmbedMeta(embeddingPath, meta) {
 export function describeEmbedder({ provider, model } = {}) {
     if (provider === 'ollama') return `🧠 Ollama · ${model}`;
     if (provider === 'local') return `🧠 Local (in-process) · ${model}`;
+    if (provider === 'code-local') return `🧠 Code-local (in-process, code-specialized) · ${model}`;
     if (provider === 'mlx') return `🧠 MLX (Apple Metal) · ${model}`;
     return '🔤 Lexical only';
 }

package/engine/binary.mjs

@@ -65,6 +65,44 @@ export function readEmbeddingBinary(filePath) {
     return cache;
 }
 
+/**
+ * B4 — read ONLY the requested keys' vectors from an `.embeddings.bin` in ONE pass, materialising a
+ * Float32Array only for matched keys (so it never holds the whole corpus in memory — but it does read
+ * the file once and scans until every requested key is found, which for sparsely-present keys means
+ * to end-of-file). The late-interaction reranker uses this to fetch a small candidate set's stored
+ * sub-vectors IDENTICALLY for both backends (same file, same keys) → parity-by-construction. Returns
+ * the matches as Map<key, Float32Array>; missing keys are simply absent.
+ *
+ * @param {string} filePath  the `.embeddings.bin` path
+ * @param {Set<string>} keys  the exact keys to extract (base / `|s` / `|wN` embedding keys)
+ * @returns {Map<string, Float32Array>}
+ */
+export function collectVectorsByKey(filePath, keys) {
+    const out = new Map();
+    if (!keys || keys.size === 0 || !fs.existsSync(filePath)) return out;
+    const buf = fs.readFileSync(filePath);
+    if (buf.length < 4) return out;
+    // A corrupt (not merely truncated) bin can push `off` past the buffer; any decode error returns
+    // the partial Map so callers honour the "missing keys are simply absent / never throws" contract.
+    try {
+        let off = 0;
+        const count = buf.readUInt32LE(off); off += 4;
+        for (let i = 0; i < count && out.size < keys.size; i++) {
+            const hashLen = buf.readUInt32LE(off); off += 4;
+            const hash = buf.subarray(off, off + hashLen).toString('utf8'); off += hashLen;
+            const dim = buf.readUInt32LE(off); off += 4;
+            if (keys.has(hash)) {
+                const vec = new Float32Array(dim);
+                for (let d = 0; d < dim; d++) { vec[d] = buf.readFloatLE(off); off += 4; }
+                out.set(hash, vec);
+            } else {
+                off += dim * 4;   // skip the float payload
+            }
+        }
+    } catch { /* corrupt framing — return whatever was decoded cleanly */ }
+    return out;
+}
+
 /**
  * Append embedding entries to an `.embeddings.bin` file in place (creating it if
  * absent) and return the absolute float-data offset of each appended entry.