@optave/codegraph 3.10.0 → 3.11.1

package/src/domain/search/models.ts

@@ -1,14 +1,48 @@
 import { execFileSync } from 'node:child_process';
+import { createRequire } from 'node:module';
+import path from 'node:path';
 import { createInterface } from 'node:readline';
 import { info } from '../../infrastructure/logger.js';
 import { ConfigError, EngineError } from '../../shared/errors.js';
 
+const _require = createRequire(import.meta.url);
+
+/**
+ * Resolve the directory where `npm install` should run so the installed
+ * package ends up reachable by `await import(pkg)` from inside this module.
+ *
+ * Without a `cwd`, `execFileSync('npm', ['install', ...])` operates on
+ * `process.cwd()` — when the user runs codegraph against a repo that is *not*
+ * the directory where codegraph itself is installed, npm installs into the
+ * wrong `node_modules`, the dynamic import still fails, and the user gets
+ * `ENGINE_UNAVAILABLE: ... installed but failed to load`.
+ *
+ * Pin cwd to the directory that contains @optave/codegraph's `node_modules`
+ * so the install lands where Node's resolution algorithm will find it.
+ *
+ * @internal Exported for unit tests; not part of the public barrel.
+ */
+export function resolveNpmInstallCwd(): string | undefined {
+  try {
+    const pkgJsonPath = _require.resolve('@optave/codegraph/package.json');
+    // pkgJsonPath = <host>/node_modules/@optave/codegraph/package.json
+    // dirname x4: package.json → codegraph → @optave → node_modules → <host>
+    return path.dirname(path.dirname(path.dirname(path.dirname(pkgJsonPath))));
+  } catch {
+    // Source-of-truth checkout (no @optave/codegraph in node_modules) — fall back
+    // to process.cwd() so legacy behavior survives in tests.
+    return undefined;
+  }
+}
+
 export interface ModelConfig {
   name: string;
   dim: number;
   contextWindow: number;
   desc: string;
   quantized: boolean;
+  /** Pooling strategy passed to the transformers pipeline. Defaults to 'mean'. */
+  pooling?: 'mean' | 'cls';
 }
 
 // Lazy-load transformers (heavy, optional module)
@@ -59,7 +93,7 @@ export const MODELS: Record<string, ModelConfig> = {
     name: 'nomic-ai/nomic-embed-text-v1.5',
     dim: 768,
     contextWindow: 8192,
-    desc: 'Improved nomic (~137MB). Matryoshka dimensions, 8192 context.',
+    desc: 'Matryoshka MRL trained (~137MB). 8192 context. Codegraph stores full 768d (no truncation); v1 scores higher on our benchmark.',
     quantized: false,
   },
   'bge-large': {
@@ -69,11 +103,41 @@ export const MODELS: Record<string, ModelConfig> = {
     desc: 'Best general retrieval (~335MB). Top MTEB scores.',
     quantized: false,
   },
+  'mxbai-xsmall': {
+    name: 'mixedbread-ai/mxbai-embed-xsmall-v1',
+    dim: 384,
+    contextWindow: 4096,
+    desc: 'Tiny model with long context (~50MB). 4096 ctx.',
+    quantized: false,
+    pooling: 'cls',
+  },
+  'mxbai-large': {
+    name: 'mixedbread-ai/mxbai-embed-large-v1',
+    dim: 1024,
+    contextWindow: 512,
+    desc: 'Top MTEB BERT-large, Matryoshka dimensions (~400MB). 512 ctx.',
+    quantized: false,
+    pooling: 'cls',
+  },
+  'bge-m3': {
+    name: 'Xenova/bge-m3',
+    dim: 1024,
+    contextWindow: 8192,
+    desc: 'Multilingual, multi-task (~600MB). 100+ languages, 8192 context.',
+    quantized: false,
+  },
+  modernbert: {
+    name: 'nomic-ai/modernbert-embed-base',
+    dim: 768,
+    contextWindow: 8192,
+    desc: 'ModernBERT base (~150MB). Newer architecture, 8192 ctx, English.',
+    quantized: false,
+  },
 };
 
 export const EMBEDDING_STRATEGIES: readonly string[] = ['structured', 'source'];
 
-export const DEFAULT_MODEL: string = 'nomic-v1.5';
+export const DEFAULT_MODEL: string = 'nomic';
 const NPM_BIN = process.platform === 'win32' ? 'npm.cmd' : 'npm';
 const BATCH_SIZE_MAP: Record<string, number> = {
   minilm: 32,
@@ -83,6 +147,10 @@ const BATCH_SIZE_MAP: Record<string, number> = {
   nomic: 8,
   'nomic-v1.5': 8,
   'bge-large': 4,
+  'mxbai-xsmall': 32,
+  'mxbai-large': 4,
+  'bge-m3': 4,
+  modernbert: 8,
 };
 const DEFAULT_BATCH_SIZE = 32;
 
@@ -104,12 +172,14 @@ export function getModelConfig(modelKey?: string): ModelConfig {
  * @internal Not part of the public barrel.
  */
 export function promptInstall(packageName: string): Promise<boolean> {
+  const installCwd = resolveNpmInstallCwd();
   if (!process.stdin.isTTY) {
     info(`Installing ${packageName} (optional dependency for semantic search)…`);
     try {
       execFileSync(NPM_BIN, ['install', '--no-save', packageName], {
         stdio: 'inherit',
         timeout: 300_000,
+        cwd: installCwd,
       });
       return Promise.resolve(true);
     } catch (err) {
@@ -128,9 +198,10 @@ export function promptInstall(packageName: string): Promise<boolean> {
         rl.close();
         if (answer.trim().toLowerCase() !== 'y') return resolve(false);
         try {
-          execFileSync(NPM_BIN, ['install', packageName], {
+          execFileSync(NPM_BIN, ['install', '--no-save', packageName], {
             stdio: 'inherit',
             timeout: 300_000,
+            cwd: installCwd,
           });
           resolve(true);
         } catch (err) {
@@ -239,7 +310,7 @@ export async function embed(
     const batch = texts.slice(i, i + batchSize);
     const output =
       (await // biome-ignore lint/complexity/noBannedTypes: dynamically loaded extractor is untyped
-      (ext as Function)(batch, { pooling: 'mean', normalize: true })) as {
+      (ext as Function)(batch, { pooling: config.pooling ?? 'mean', normalize: true })) as {
         data: number[];
       };
 

package/src/domain/search/search/hybrid.ts

@@ -105,61 +105,72 @@ async function collectRankedLists(
   return rankedLists;
 }
 
+/** Initialise a fusion entry seeded from the first ranked item we see for a key. */
+function createFusionEntry(item: RankedItem): FusionEntry {
+  return {
+    name: item.name,
+    kind: item.kind,
+    file: item.file,
+    line: item.line,
+    endLine: (item.endLine as number | null) ?? null,
+    role: (item.role as string | null) ?? null,
+    fileHash: (item.fileHash as string | null) ?? null,
+    rrfScore: 0,
+    bm25Score: null,
+    bm25Rank: null,
+    similarity: null,
+    semanticRank: null,
+  };
+}
+
+/** Merge a single ranked item into its fusion entry: update RRF and best per-source rank. */
+function mergeRankedItem(entry: FusionEntry, item: RankedItem, k: number): void {
+  entry.rrfScore += 1 / (k + item.rank);
+  if (item.source === 'bm25') {
+    if (entry.bm25Rank === null || item.rank < entry.bm25Rank) {
+      entry.bm25Score = item.bm25Score ?? null;
+      entry.bm25Rank = item.rank;
+    }
+  } else if (entry.semanticRank === null || item.rank < entry.semanticRank) {
+    entry.similarity = item.similarity ?? null;
+    entry.semanticRank = item.rank;
+  }
+}
+
+/** Flatten a fusion entry into the public-facing hybrid result shape. */
+function toHybridResult(e: FusionEntry): HybridResult {
+  return {
+    name: e.name,
+    kind: e.kind,
+    file: e.file,
+    line: e.line,
+    endLine: e.endLine,
+    role: e.role,
+    fileHash: e.fileHash,
+    rrf: e.rrfScore,
+    bm25Score: e.bm25Score,
+    bm25Rank: e.bm25Rank,
+    similarity: e.similarity,
+    semanticRank: e.semanticRank,
+  };
+}
+
 /** Reciprocal Rank Fusion: merge ranked lists into a single scored result set. */
 function fuseResults(rankedLists: RankedItem[][], k: number, limit: number): HybridResult[] {
   const fusionMap = new Map<string, FusionEntry>();
-
   for (const list of rankedLists) {
     for (const item of list) {
       if (!fusionMap.has(item.key)) {
-        fusionMap.set(item.key, {
-          name: item.name,
-          kind: item.kind,
-          file: item.file,
-          line: item.line,
-          endLine: (item.endLine as number | null) ?? null,
-          role: (item.role as string | null) ?? null,
-          fileHash: (item.fileHash as string | null) ?? null,
-          rrfScore: 0,
-          bm25Score: null,
-          bm25Rank: null,
-          similarity: null,
-          semanticRank: null,
-        });
-      }
-      const entry = fusionMap.get(item.key)!;
-      entry.rrfScore += 1 / (k + item.rank);
-      if (item.source === 'bm25') {
-        if (entry.bm25Rank === null || item.rank < entry.bm25Rank) {
-          entry.bm25Score = (item as RankedItem & { bm25Score?: number }).bm25Score ?? null;
-          entry.bm25Rank = item.rank;
-        }
-      } else {
-        if (entry.semanticRank === null || item.rank < entry.semanticRank) {
-          entry.similarity = (item as RankedItem & { similarity?: number }).similarity ?? null;
-          entry.semanticRank = item.rank;
-        }
+        fusionMap.set(item.key, createFusionEntry(item));
       }
+      mergeRankedItem(fusionMap.get(item.key)!, item, k);
     }
   }
 
   return [...fusionMap.values()]
     .sort((a, b) => b.rrfScore - a.rrfScore)
     .slice(0, limit)
-    .map((e) => ({
-      name: e.name,
-      kind: e.kind,
-      file: e.file,
-      line: e.line,
-      endLine: e.endLine,
-      role: e.role,
-      fileHash: e.fileHash,
-      rrf: e.rrfScore,
-      bm25Score: e.bm25Score,
-      bm25Rank: e.bm25Rank,
-      similarity: e.similarity,
-      semanticRank: e.semanticRank,
-    }));
+    .map(toHybridResult);
 }
 
 export async function hybridSearchData(

package/src/domain/search/search/semantic.ts

@@ -4,7 +4,7 @@ import type { BetterSqlite3Database, CodegraphConfig } from '../../../types.js';
 import { normalizeSymbol } from '../../queries.js';
 import { embed } from '../models.js';
 import { cosineSim } from '../stores/sqlite-blob.js';
-import { prepareSearch } from './prepare.js';
+import { type PreparedSearch, prepareSearch } from './prepare.js';
 
 export interface SemanticSearchOpts {
   config?: CodegraphConfig;
@@ -30,6 +30,25 @@ export interface SearchDataResult {
   results: SemanticResult[];
 }
 
+type StoredRow = PreparedSearch['rows'][number];
+
+/** Reconstitute a stored embedding row's vector blob into a Float32Array. */
+function rowVector(row: StoredRow): Float32Array {
+  return new Float32Array(new Uint8Array(row.vector as unknown as ArrayBuffer).buffer);
+}
+
+/** Warn when stored embeddings and the query model use different dimensions. */
+function checkDimensionMismatch(storedDim: number | null, dim: number): boolean {
+  if (storedDim && dim !== storedDim) {
+    console.log(
+      `Warning: query model dimension (${dim}) doesn't match stored embeddings (${storedDim}).`,
+    );
+    console.log(`  Re-run \`codegraph embed\` with the same model, or use --model to match.`);
+    return true;
+  }
+  return false;
+}
+
 export async function searchData(
   query: string,
   customDbPath: string | undefined,
@@ -50,20 +69,12 @@ export async function searchData(
       dim,
     } = await embed([query], modelKey ?? undefined);
 
-    if (storedDim && dim !== storedDim) {
-      console.log(
-        `Warning: query model dimension (${dim}) doesn't match stored embeddings (${storedDim}).`,
-      );
-      console.log(`  Re-run \`codegraph embed\` with the same model, or use --model to match.`);
-      return null;
-    }
+    if (checkDimensionMismatch(storedDim, dim)) return null;
 
     const hc = new Map<string, string>();
     const results: SemanticResult[] = [];
     for (const row of rows) {
-      const vec = new Float32Array(new Uint8Array(row.vector as unknown as ArrayBuffer).buffer);
-      const sim = cosineSim(queryVec!, vec);
-
+      const sim = cosineSim(queryVec!, rowVector(row));
       if (sim >= minScore) {
         results.push({
           ...normalizeSymbol(row, db as BetterSqlite3Database, hc),
@@ -91,6 +102,82 @@ export interface MultiSearchResult {
   }>;
 }
 
+interface RankedHit {
+  rowIndex: number;
+  similarity: number;
+  rank: number;
+}
+
+interface FusionEntry {
+  rrfScore: number;
+  queryScores: Array<{ query: string; similarity: number; rank: number }>;
+}
+
+/**
+ * Emit a warning for any query pair whose embeddings are nearly identical,
+ * since RRF would over-weight matches shared between them.
+ */
+function warnOnSimilarQueries(
+  queries: string[],
+  queryVecs: Float32Array[],
+  threshold: number,
+): void {
+  for (let i = 0; i < queryVecs.length; i++) {
+    for (let j = i + 1; j < queryVecs.length; j++) {
+      const sim = cosineSim(queryVecs[i]!, queryVecs[j]!);
+      if (sim >= threshold) {
+        warn(
+          `Queries "${queries[i]}" and "${queries[j]}" are very similar ` +
+            `(${(sim * 100).toFixed(0)}% cosine similarity). ` +
+            `This may bias RRF results toward their shared matches. ` +
+            `Consider using more distinct queries.`,
+        );
+      }
+    }
+  }
+}
+
+/** Rank stored rows for a single query, keeping only those above minScore. */
+function rankRowsForQuery(
+  queryVec: Float32Array,
+  rowVecs: Float32Array[],
+  minScore: number,
+): RankedHit[] {
+  const scored: Array<{ rowIndex: number; similarity: number }> = [];
+  for (let ri = 0; ri < rowVecs.length; ri++) {
+    const sim = cosineSim(queryVec, rowVecs[ri]!);
+    if (sim >= minScore) {
+      scored.push({ rowIndex: ri, similarity: sim });
+    }
+  }
+  scored.sort((a, b) => b.similarity - a.similarity);
+  return scored.map((item, rank) => ({ ...item, rank: rank + 1 }));
+}
+
+/** Reciprocal Rank Fusion across each query's ranked hits. */
+function fuseRankedHits(
+  queries: string[],
+  perQueryRanked: RankedHit[][],
+  k: number,
+): Map<number, FusionEntry> {
+  const fusionMap = new Map<number, FusionEntry>();
+  for (let qi = 0; qi < queries.length; qi++) {
+    for (const item of perQueryRanked[qi]!) {
+      if (!fusionMap.has(item.rowIndex)) {
+        fusionMap.set(item.rowIndex, { rrfScore: 0, queryScores: [] });
+      }
+      const entry = fusionMap.get(item.rowIndex)!;
+      entry.rrfScore += 1 / (k + item.rank);
+      entry.queryScores.push({
+        query: queries[qi]!,
+        similarity: item.similarity,
+        rank: item.rank,
+      });
+    }
+  }
+  return fusionMap;
+}
+
 export async function multiSearchData(
   queries: string[],
   customDbPath: string | undefined,
@@ -101,6 +188,7 @@ export async function multiSearchData(
   const limit = opts.limit ?? searchCfg.topK ?? 15;
   const minScore = opts.minScore ?? searchCfg.defaultMinScore ?? 0.2;
   const k = opts.rrfK ?? searchCfg.rrfK ?? 60;
+  const similarityWarnThreshold = searchCfg.similarityWarnThreshold ?? 0.85;
 
   const prepared = prepareSearch(customDbPath, opts);
   if (!prepared) return null;
@@ -109,63 +197,15 @@ export async function multiSearchData(
   try {
     const { vectors: queryVecs, dim } = await embed(queries, modelKey ?? undefined);
 
-    const SIMILARITY_WARN_THRESHOLD = searchCfg.similarityWarnThreshold ?? 0.85;
-    for (let i = 0; i < queryVecs.length; i++) {
-      for (let j = i + 1; j < queryVecs.length; j++) {
-        const sim = cosineSim(queryVecs[i]!, queryVecs[j]!);
-        if (sim >= SIMILARITY_WARN_THRESHOLD) {
-          warn(
-            `Queries "${queries[i]}" and "${queries[j]}" are very similar ` +
-              `(${(sim * 100).toFixed(0)}% cosine similarity). ` +
-              `This may bias RRF results toward their shared matches. ` +
-              `Consider using more distinct queries.`,
-          );
-        }
-      }
-    }
+    warnOnSimilarQueries(queries, queryVecs as Float32Array[], similarityWarnThreshold);
 
-    if (storedDim && dim !== storedDim) {
-      console.log(
-        `Warning: query model dimension (${dim}) doesn't match stored embeddings (${storedDim}).`,
-      );
-      console.log(`  Re-run \`codegraph embed\` with the same model, or use --model to match.`);
-      return null;
-    }
+    if (checkDimensionMismatch(storedDim, dim)) return null;
 
-    const rowVecs = rows.map(
-      (row) => new Float32Array(new Uint8Array(row.vector as unknown as ArrayBuffer).buffer),
+    const rowVecs = rows.map(rowVector);
+    const perQueryRanked = queries.map((_q, qi) =>
+      rankRowsForQuery(queryVecs[qi]!, rowVecs, minScore),
     );
-
-    const perQueryRanked = queries.map((_query, qi) => {
-      const scored: Array<{ rowIndex: number; similarity: number }> = [];
-      for (let ri = 0; ri < rows.length; ri++) {
-        const sim = cosineSim(queryVecs[qi]!, rowVecs[ri]!);
-        if (sim >= minScore) {
-          scored.push({ rowIndex: ri, similarity: sim });
-        }
-      }
-      scored.sort((a, b) => b.similarity - a.similarity);
-      return scored.map((item, rank) => ({ ...item, rank: rank + 1 }));
-    });
-
-    const fusionMap = new Map<
-      number,
-      { rrfScore: number; queryScores: Array<{ query: string; similarity: number; rank: number }> }
-    >();
-    for (let qi = 0; qi < queries.length; qi++) {
-      for (const item of perQueryRanked[qi]!) {
-        if (!fusionMap.has(item.rowIndex)) {
-          fusionMap.set(item.rowIndex, { rrfScore: 0, queryScores: [] });
-        }
-        const entry = fusionMap.get(item.rowIndex)!;
-        entry.rrfScore += 1 / (k + item.rank);
-        entry.queryScores.push({
-          query: queries[qi]!,
-          similarity: item.similarity,
-          rank: item.rank,
-        });
-      }
-    }
+    const fusionMap = fuseRankedHits(queries, perQueryRanked, k);
 
     const hc = new Map<string, string>();
     const results: MultiSearchResult['results'] = [];