@maxgfr/codeindex 2.8.1 → 2.10.0

package/src/embed/search.ts

@@ -0,0 +1,90 @@
+import type { RepoScan } from "../scan.js";
+import { rrf } from "../util.js";
+import { byStr } from "../sort.js";
+import { searchIndex, type SearchOptions, type SearchResult } from "../bm25.js";
+import { encode, intDot } from "./encode.js";
+import type { EmbeddingIndex } from "./index.js";
+import type { StaticEmbedModel } from "./model.js";
+
+const DEFAULT_LIMIT = 20;
+const RRF_K = 60; // reuse the engine-wide RRF damping constant
+
+export interface SemanticSearchOptions extends SearchOptions {
+  // The loaded static model, needed to encode the QUERY into the same int8 space
+  // as the corpus. Absent → the search degrades to pure lexical (no throw).
+  model?: StaticEmbedModel;
+  // RRF damping (default 60), exposed for parity with the shared rrf helper.
+  rrfK?: number;
+}
+
+// A fused result. Extends the lexical SearchResult additively: `semanticSymbol`
+// is the corpus symbol whose embedding was closest to the query for this file
+// (absent when the file was contributed only by the lexical side, or when the
+// search degraded to lexical).
+export interface SemanticSearchResult extends SearchResult {
+  semanticSymbol?: string;
+}
+
+// RRF-fused semantic + lexical search. The two rankings live on incomparable
+// scales (BM25 score vs integer dot product), so we fuse by RANK via the shared
+// `rrf` helper (k=60) rather than any linear score blend. Deterministic end to
+// end: lexical is deterministic, the semantic ranking sorts by exact integer dot
+// (ties broken by path), and the fused list sorts by RRF score (ties by path).
+//
+// DEGRADATION: with no model, no index, or an empty index, this returns the pure
+// lexical ranking unchanged (as SemanticSearchResult[] with no `semanticSymbol`)
+// — the caller stays on exit 0 and simply prints a note.
+export function searchSemantic(
+  scan: RepoScan,
+  query: string,
+  index: EmbeddingIndex | undefined,
+  opts: SemanticSearchOptions = {},
+): SemanticSearchResult[] {
+  const limit = opts.limit ?? DEFAULT_LIMIT;
+  // Pull a deeper lexical list than the final limit so RRF has enough overlap to
+  // fuse meaningfully, then trim after fusion.
+  const lexical = searchIndex(scan, query, { limit: Math.max(limit, 50), fuzzy: opts.fuzzy });
+
+  if (!opts.model || !index || index.records.length === 0) {
+    return lexical.slice(0, limit); // pure-lexical degradation
+  }
+
+  const q = encode(opts.model, query);
+  // Best (highest integer dot) record per file, remembering its symbol.
+  const bestByFile = new Map<string, { score: number; symbol?: string }>();
+  for (const r of index.records) {
+    const dot = intDot(q, r.vec);
+    const prev = bestByFile.get(r.file);
+    if (!prev || dot > prev.score) bestByFile.set(r.file, { score: dot, symbol: r.symbol });
+  }
+
+  // Semantic ranked file list: positive similarity only, best-first, ties by path.
+  const semList = [...bestByFile.entries()]
+    .filter(([, v]) => v.score > 0)
+    .sort((a, b) => b[1].score - a[1].score || byStr(a[0], b[0]))
+    .map(([file]) => file);
+  const lexList = lexical.map((r) => r.file);
+
+  // Fuse the two ranked lists (identity keyOf — items are already file paths).
+  const fused = rrf<string>([lexList, semList], (f) => f, opts.rrfK ?? RRF_K);
+
+  const lexByFile = new Map(lexical.map((r) => [r.file, r] as const));
+  const results: SemanticSearchResult[] = [...fused.entries()]
+    .sort((a, b) => b[1] - a[1] || byStr(a[0], b[0]))
+    .map(([file, score]) => {
+      const lex = lexByFile.get(file);
+      const res: SemanticSearchResult = {
+        file,
+        score: Number(score.toFixed(4)),
+        matchedTerms: lex?.matchedTerms ?? [],
+        topSymbols: lex?.topSymbols ?? [],
+      };
+      const sem = bestByFile.get(file);
+      if (sem?.symbol) res.semanticSymbol = sem.symbol;
+      if (lex?.fuzzyTerms) res.fuzzyTerms = lex.fuzzyTerms;
+      return res;
+    });
+  return results.slice(0, limit);
+}
+
+export { DEFAULT_LIMIT as SEMANTIC_DEFAULT_LIMIT };

package/src/engine-cli.ts

@@ -19,6 +19,9 @@ import { symbolComplexity, riskHotspots } from "./complexity.js";
 import { renderMermaid } from "./viz.js";
 import { searchIndex } from "./bm25.js";
 import { checkRules, parseRules } from "./rules.js";
+import { EMBED_VERSION, resolveEmbedModelDir, loadEmbedModel, resolveEmbedPullUrl } from "./embed/model.js";
+import { buildEmbeddingIndex, serializeEmbeddings } from "./embed/index.js";
+import { searchSemantic } from "./embed/search.js";
 
 const HELP = `codeindex engine v${ENGINE_VERSION} — deterministic repo indexing
 
@@ -37,7 +40,14 @@ Commands:
   churn       Per-file git commit counts (JSON; --since <ref> to bound)
   grep        Search: cli.mjs grep <pattern> --repo <dir> (JSON hits)
   search      Keyless BM25 lexical search over symbol names, path segments,
-              markdown headings and summaries: cli.mjs search "<query>" --repo <dir>
+              markdown headings and summaries: cli.mjs search "<query>" --repo <dir>.
+              --semantic fuses in the deterministic static-embedding tier (RRF)
+              when a model is present; degrades to lexical (exit 0) when absent
+  embed       Deterministic static-embedding tier (opt-in by model asset):
+                embed status   Report the resolved model + EMBED_VERSION (JSON)
+                embed build    Write embeddings.bin into --out <dir> from the repo
+                embed pull     Fetch the model asset into CODEINDEX_EMBED_DIR (or
+                               <repo>/.codeindex/models/) — needs CODEINDEX_EMBED_URL
   rules       Architecture rules (forbidden edges, cycles, orphans) validated
               against the link-graph: --config <codeindex.rules.json>; exits 1
               on any error-severity violation (a CI gate)
@@ -63,6 +73,10 @@ Flags:
   --no-ast            Skip tree-sitter grammars even when present (regex tier)
   --config <file>     Rules config for \`rules\` (JSON: [{name, from, to, …}])
   --limit <n>         Max results for \`search\` (default 20)
+  --no-fuzzy          \`search\`: disable trigram fuzzy fallback for query terms
+                      with zero document frequency (default: enabled)
+  --semantic          \`search\`: RRF-fuse the deterministic static-embedding tier
+                      with lexical (needs a model asset; lexical-only otherwise)
   --recall            \`callers\`: recall-oriented binding (issue #7) — relaxes
                       the JS/TS import gate to unique repo-wide names and labels
                       each site corroborated|unique-name
@@ -84,13 +98,15 @@ interface CliFlags {
   budgetTokens?: number;
   config?: string; // rules config path
   limit?: number; // search result cap
+  fuzzy: boolean; // search: trigram fuzzy fallback for df==0 terms (default true)
+  semantic: boolean; // search: RRF-fuse the static-embedding tier (default false)
   recall?: boolean; // callers: recall-oriented binding
   projectRoot?: string; // scip: override Metadata.project_root
   positional?: string; // e.g. the grep pattern or search query
 }
 
 function parseFlags(args: string[]): CliFlags {
-  const flags: CliFlags = { repo: process.cwd(), include: [], exclude: [], gitignore: true, noAst: false };
+  const flags: CliFlags = { repo: process.cwd(), include: [], exclude: [], gitignore: true, noAst: false, fuzzy: true, semantic: false };
   for (let i = 0; i < args.length; i++) {
     const a = args[i]!;
     const next = (): string => {
@@ -122,6 +138,8 @@ function parseFlags(args: string[]): CliFlags {
     else if (a === "--since") flags.since = next();
     else if (a === "--config") flags.config = resolve(next());
     else if (a === "--limit") flags.limit = num();
+    else if (a === "--no-fuzzy") flags.fuzzy = false;
+    else if (a === "--semantic") flags.semantic = true;
     else if (a === "--recall") flags.recall = true;
     else if (!a.startsWith("--") && flags.positional === undefined) flags.positional = a;
     else throw new Error(`unknown flag: ${a}`);
@@ -199,7 +217,18 @@ export async function runCli(argv: string[]): Promise<void> {
       cachePath,
       JSON.stringify({ schemaVersion: SCHEMA_VERSION, extractorVersion: EXTRACTOR_VERSION, files }) + "\n",
     );
-    process.stderr.write(`codeindex: ${scan.files.length} files → ${outDir}/graph.json + symbols.json${scan.capped ? " (capped)" : ""}\n`);
+    // Deterministic embeddings sidecar: written next to graph.json ONLY when a
+    // model asset is present (opt-in). Silently skipped otherwise — no model, no
+    // embeddings.bin, no impact on the graph/symbols consumers.
+    let embedNote = "";
+    const modelDir = resolveEmbedModelDir(flags.repo);
+    const model = modelDir ? loadEmbedModel(modelDir) : undefined;
+    if (model) {
+      const index = buildEmbeddingIndex(scan, model);
+      writeFileSync(join(outDir, "embeddings.bin"), serializeEmbeddings(index));
+      embedNote = ` + embeddings.bin (${index.records.length} records, model ${model.modelId})`;
+    }
+    process.stderr.write(`codeindex: ${scan.files.length} files → ${outDir}/graph.json + symbols.json${embedNote}${scan.capped ? " (capped)" : ""}\n`);
   } else if (cmd === "scan") {
     const { scan } = buildIndexArtifacts(flags.repo, scanOptions(flags));
     const summary = {
@@ -234,8 +263,85 @@ export async function runCli(argv: string[]): Promise<void> {
   } else if (cmd === "search") {
     if (!flags.positional) throw new Error('search needs a query: cli.mjs search "<query>" --repo <dir>');
     const scan = scanRepo(flags.repo, scanOptions(flags));
-    const results = searchIndex(scan, flags.positional, { limit: flags.limit });
-    emit(JSON.stringify(results, null, 2) + "\n", flags.out);
+    if (flags.semantic) {
+      const modelDir = resolveEmbedModelDir(flags.repo);
+      const model = modelDir ? loadEmbedModel(modelDir) : undefined;
+      if (!model) {
+        // Degradation: --semantic without a model → lexical results + a stderr
+        // note, exit 0. The results shape is a superset of the lexical one.
+        process.stderr.write(
+          "codeindex: semantic search unavailable (no embedding model present) — returning lexical results; run `codeindex embed pull` to enable it\n",
+        );
+        const results = searchIndex(scan, flags.positional, { limit: flags.limit, fuzzy: flags.fuzzy });
+        emit(JSON.stringify(results, null, 2) + "\n", flags.out);
+      } else {
+        const index = buildEmbeddingIndex(scan, model);
+        const results = searchSemantic(scan, flags.positional, index, { model, limit: flags.limit, fuzzy: flags.fuzzy });
+        emit(JSON.stringify(results, null, 2) + "\n", flags.out);
+      }
+    } else {
+      const results = searchIndex(scan, flags.positional, { limit: flags.limit, fuzzy: flags.fuzzy });
+      emit(JSON.stringify(results, null, 2) + "\n", flags.out);
+    }
+  } else if (cmd === "embed") {
+    const sub = flags.positional;
+    const modelDir = resolveEmbedModelDir(flags.repo);
+    if (sub === "status") {
+      const model = modelDir ? loadEmbedModel(modelDir) : undefined;
+      const status = {
+        embedVersion: EMBED_VERSION,
+        model: model
+          ? { present: true, dir: modelDir, modelId: model.modelId, dim: model.dim, vocabSize: model.vocabSize }
+          : { present: false },
+        endpoint: process.env.CODEINDEX_EMBED_ENDPOINT ?? null,
+      };
+      emit(JSON.stringify(status, null, 2) + "\n", flags.out);
+    } else if (sub === "build") {
+      if (!flags.out) throw new Error("embed build needs --out <dir>");
+      if (!modelDir) {
+        process.stderr.write("codeindex: no embedding model present — run `codeindex embed pull` first (nothing written)\n");
+        process.exitCode = 1;
+        return;
+      }
+      const model = loadEmbedModel(modelDir)!;
+      mkdirSync(flags.out, { recursive: true });
+      const scan = scanRepo(flags.repo, scanOptions(flags));
+      const index = buildEmbeddingIndex(scan, model);
+      writeFileSync(join(flags.out, "embeddings.bin"), serializeEmbeddings(index));
+      process.stderr.write(`codeindex: ${index.records.length} embedding records → ${flags.out}/embeddings.bin (model ${model.modelId})\n`);
+    } else if (sub === "pull") {
+      const url = resolveEmbedPullUrl();
+      if (!url) {
+        // Clean, actionable failure — the official asset is not published yet.
+        process.stderr.write(
+          "codeindex: no model URL configured. The official static-embedding asset is not published yet.\n" +
+            "Set CODEINDEX_EMBED_URL to a model.json URL (optionally CODEINDEX_EMBED_DIR as the destination), then re-run `codeindex embed pull`.\n",
+        );
+        process.exitCode = 1;
+        return;
+      }
+      const destDir = process.env.CODEINDEX_EMBED_DIR ?? join(flags.repo, ".codeindex", "models");
+      mkdirSync(destDir, { recursive: true });
+      process.stderr.write(`codeindex: fetching model from ${url} → ${join(destDir, "model.json")}\n`);
+      const res = await fetch(url);
+      if (!res.ok) {
+        process.stderr.write(`codeindex: pull failed — HTTP ${res.status} from ${url}\n`);
+        process.exitCode = 1;
+        return;
+      }
+      const body = await res.text();
+      try {
+        JSON.parse(body);
+      } catch {
+        process.stderr.write("codeindex: pull failed — response is not a valid model.json (expected JSON)\n");
+        process.exitCode = 1;
+        return;
+      }
+      writeFileSync(join(destDir, "model.json"), body);
+      process.stderr.write(`codeindex: model written to ${join(destDir, "model.json")}\n`);
+    } else {
+      throw new Error("embed needs a subcommand: pull | build | status");
+    }
   } else if (cmd === "rules") {
     if (!flags.config) throw new Error("rules needs --config <codeindex.rules.json>");
     const rules = parseRules(JSON.parse(readFileSync(flags.config, "utf8")));

package/src/engine.ts

@@ -103,6 +103,27 @@ export type { SearchHit, GrepOptions } from "./grep.js";
 export { searchIndex, subtokens } from "./bm25.js";
 export type { SearchOptions, SearchResult } from "./bm25.js";
 
+// Deterministic static-embedding tier (v2.10.0): a keyless, byte-deterministic
+// semantic search, opt-in by model-asset presence (models NEVER ship in the
+// tarball). EMBED_VERSION is dedicated to the embeddings.bin sidecar and is
+// independent of SCHEMA_VERSION / EXTRACTOR_VERSION.
+export {
+  EMBED_VERSION,
+  resolveEmbedModelDir,
+  hasEmbedModel,
+  loadEmbedModel,
+  resolveEmbedPullUrl,
+} from "./embed/model.js";
+export type { StaticEmbedModel } from "./embed/model.js";
+export { encode, tokenize, wordpiece, basicTokenize, roundHalfToEven, intDot } from "./embed/encode.js";
+export { buildEmbeddingIndex, serializeEmbeddings, deserializeEmbeddings } from "./embed/index.js";
+export type { EmbeddingIndex, EmbeddingRecord } from "./embed/index.js";
+export { searchSemantic } from "./embed/search.js";
+export type { SemanticSearchOptions, SemanticSearchResult } from "./embed/search.js";
+// HTTP endpoint tier (v2.11 preview — NOT wired into the CLI/MCP here).
+export { embedViaEndpoint, resolveEmbedEndpoint } from "./embed/endpoint.js";
+export type { EmbedEndpointOptions } from "./embed/endpoint.js";
+
 // Architecture rules: forbidden edges + cycles/orphans builtins (issue #4).
 export { checkRules, parseRules } from "./rules.js";
 export type { ArchRule, ForbiddenEdgeRule, BuiltinRule, RuleSeverity, RuleViolation } from "./rules.js";

package/src/mcp.ts

@@ -25,6 +25,9 @@ import { replaceSymbolBody, insertAfterSymbol, insertBeforeSymbol } from "./edit
 import { writeMemory, readMemory, deleteMemory, listMemories } from "./memory.js";
 import { searchIndex } from "./bm25.js";
 import { checkRules, parseRules } from "./rules.js";
+import { EMBED_VERSION, resolveEmbedModelDir, loadEmbedModel } from "./embed/model.js";
+import { buildEmbeddingIndex } from "./embed/index.js";
+import { searchSemantic } from "./embed/search.js";
 
 interface RpcRequest {
   jsonrpc: "2.0";
@@ -266,7 +269,7 @@ const TOOLS = [
   {
     name: "search",
     description:
-      'Natural-language-ish lexical search: BM25 ranking (k1=1.2, b=0.75) over symbol names (camelCase/snake_case subtokens), file path segments, markdown headings and summary lines. NOT embeddings — deterministic, diacritic-folded, zero API keys. Answers "where is auth handled?"-style queries with ranked files, matched terms and top symbols.',
+      'Natural-language-ish lexical search: BM25 ranking (k1=1.2, b=0.75) over symbol names (camelCase/snake_case subtokens), file path segments, markdown headings and summary lines. NOT embeddings by default — deterministic, diacritic-folded, zero API keys. Answers "where is auth handled?"-style queries with ranked files, matched terms and top symbols. Query terms with zero document frequency get a deterministic trigram-fuzzy fallback (typo-tolerant) unless `fuzzy: false`. Set `semantic: true` to RRF-fuse the deterministic static-embedding tier when a model asset is present (degrades to lexical otherwise — see embed_status).',
     inputSchema: {
       type: "object",
       properties: {
@@ -274,10 +277,26 @@ const TOOLS = [
         ...scopeProps,
         query: { type: "string", description: "Natural-language or identifier query" },
         limit: { type: "number", description: "Max results (default 20)" },
+        fuzzy: {
+          type: "boolean",
+          description:
+            "Trigram fuzzy fallback for query terms with zero document frequency (default true)",
+        },
+        semantic: {
+          type: "boolean",
+          description:
+            "RRF-fuse the deterministic static-embedding tier with lexical when a model asset is present (default false; silently lexical-only when no model)",
+        },
       },
       required: ["repo", "query"],
     },
   },
+  {
+    name: "embed_status",
+    description:
+      "Report the deterministic static-embedding tier: whether a model asset is resolved (opt-in, never shipped in the package), its modelId/dim, EMBED_VERSION, and any configured HTTP endpoint. Use to check whether `search` with semantic:true will actually fuse embeddings or degrade to lexical.",
+    inputSchema: { type: "object", properties: { ...repoProp }, required: ["repo"] },
+  },
   {
     name: "check_rules",
     description:
@@ -435,10 +454,34 @@ function callTool(name: string, args: Record<string, unknown>): string {
   if (name === "search") {
     const query = str(args.query);
     if (!query) throw new Error("`query` is required");
-    const results = searchIndex(scanRepo(repo, scanOpts), query, {
-      limit: typeof args.limit === "number" ? args.limit : undefined,
-    });
-    return JSON.stringify(results, null, 2);
+    const scan = scanRepo(repo, scanOpts);
+    const limit = typeof args.limit === "number" ? args.limit : undefined;
+    const fuzzy = typeof args.fuzzy === "boolean" ? args.fuzzy : undefined;
+    if (args.semantic === true) {
+      const modelDir = resolveEmbedModelDir(repo);
+      const model = modelDir ? loadEmbedModel(modelDir) : undefined;
+      if (model) {
+        const index = buildEmbeddingIndex(scan, model);
+        return JSON.stringify(searchSemantic(scan, query, index, { model, limit, fuzzy }), null, 2);
+      }
+      // Degrade to lexical (opt-in tier not activated) — no throw.
+    }
+    return JSON.stringify(searchIndex(scan, query, { limit, fuzzy }), null, 2);
+  }
+  if (name === "embed_status") {
+    const modelDir = resolveEmbedModelDir(repo);
+    const model = modelDir ? loadEmbedModel(modelDir) : undefined;
+    return JSON.stringify(
+      {
+        embedVersion: EMBED_VERSION,
+        model: model
+          ? { present: true, dir: modelDir, modelId: model.modelId, dim: model.dim, vocabSize: model.vocabSize }
+          : { present: false },
+        endpoint: process.env.CODEINDEX_EMBED_ENDPOINT ?? null,
+      },
+      null,
+      2,
+    );
   }
   if (name === "check_rules") {
     const rules = parseRules(args.rules); // throws a descriptive error on a malformed payload

package/src/types.ts

@@ -1,7 +1,7 @@
 // Single source of truth for the engine version the bundle reports. Kept in
 // lockstep with package.json by the release pipeline. Do not edit by hand
 // outside a release.
-export const ENGINE_VERSION = "2.8.1";
+export const ENGINE_VERSION = "2.10.0";
 
 // Bumped whenever the on-disk artifact shape changes, so a consumer can reject
 // an index written by an incompatible engine instead of misreading it. The