@maxgfr/codeindex 2.9.0 → 2.10.0

package/src/engine-cli.ts

@@ -19,6 +19,9 @@ import { symbolComplexity, riskHotspots } from "./complexity.js";
 import { renderMermaid } from "./viz.js";
 import { searchIndex } from "./bm25.js";
 import { checkRules, parseRules } from "./rules.js";
+import { EMBED_VERSION, resolveEmbedModelDir, loadEmbedModel, resolveEmbedPullUrl } from "./embed/model.js";
+import { buildEmbeddingIndex, serializeEmbeddings } from "./embed/index.js";
+import { searchSemantic } from "./embed/search.js";
 
 const HELP = `codeindex engine v${ENGINE_VERSION} — deterministic repo indexing
 
@@ -37,7 +40,14 @@ Commands:
   churn       Per-file git commit counts (JSON; --since <ref> to bound)
   grep        Search: cli.mjs grep <pattern> --repo <dir> (JSON hits)
   search      Keyless BM25 lexical search over symbol names, path segments,
-              markdown headings and summaries: cli.mjs search "<query>" --repo <dir>
+              markdown headings and summaries: cli.mjs search "<query>" --repo <dir>.
+              --semantic fuses in the deterministic static-embedding tier (RRF)
+              when a model is present; degrades to lexical (exit 0) when absent
+  embed       Deterministic static-embedding tier (opt-in by model asset):
+                embed status   Report the resolved model + EMBED_VERSION (JSON)
+                embed build    Write embeddings.bin into --out <dir> from the repo
+                embed pull     Fetch the model asset into CODEINDEX_EMBED_DIR (or
+                               <repo>/.codeindex/models/) — needs CODEINDEX_EMBED_URL
   rules       Architecture rules (forbidden edges, cycles, orphans) validated
               against the link-graph: --config <codeindex.rules.json>; exits 1
               on any error-severity violation (a CI gate)
@@ -65,6 +75,8 @@ Flags:
   --limit <n>         Max results for \`search\` (default 20)
   --no-fuzzy          \`search\`: disable trigram fuzzy fallback for query terms
                       with zero document frequency (default: enabled)
+  --semantic          \`search\`: RRF-fuse the deterministic static-embedding tier
+                      with lexical (needs a model asset; lexical-only otherwise)
   --recall            \`callers\`: recall-oriented binding (issue #7) — relaxes
                       the JS/TS import gate to unique repo-wide names and labels
                       each site corroborated|unique-name
@@ -87,13 +99,14 @@ interface CliFlags {
   config?: string; // rules config path
   limit?: number; // search result cap
   fuzzy: boolean; // search: trigram fuzzy fallback for df==0 terms (default true)
+  semantic: boolean; // search: RRF-fuse the static-embedding tier (default false)
   recall?: boolean; // callers: recall-oriented binding
   projectRoot?: string; // scip: override Metadata.project_root
   positional?: string; // e.g. the grep pattern or search query
 }
 
 function parseFlags(args: string[]): CliFlags {
-  const flags: CliFlags = { repo: process.cwd(), include: [], exclude: [], gitignore: true, noAst: false, fuzzy: true };
+  const flags: CliFlags = { repo: process.cwd(), include: [], exclude: [], gitignore: true, noAst: false, fuzzy: true, semantic: false };
   for (let i = 0; i < args.length; i++) {
     const a = args[i]!;
     const next = (): string => {
@@ -126,6 +139,7 @@ function parseFlags(args: string[]): CliFlags {
     else if (a === "--config") flags.config = resolve(next());
     else if (a === "--limit") flags.limit = num();
     else if (a === "--no-fuzzy") flags.fuzzy = false;
+    else if (a === "--semantic") flags.semantic = true;
     else if (a === "--recall") flags.recall = true;
     else if (!a.startsWith("--") && flags.positional === undefined) flags.positional = a;
     else throw new Error(`unknown flag: ${a}`);
@@ -203,7 +217,18 @@ export async function runCli(argv: string[]): Promise<void> {
       cachePath,
       JSON.stringify({ schemaVersion: SCHEMA_VERSION, extractorVersion: EXTRACTOR_VERSION, files }) + "\n",
     );
-    process.stderr.write(`codeindex: ${scan.files.length} files → ${outDir}/graph.json + symbols.json${scan.capped ? " (capped)" : ""}\n`);
+    // Deterministic embeddings sidecar: written next to graph.json ONLY when a
+    // model asset is present (opt-in). Silently skipped otherwise — no model, no
+    // embeddings.bin, no impact on the graph/symbols consumers.
+    let embedNote = "";
+    const modelDir = resolveEmbedModelDir(flags.repo);
+    const model = modelDir ? loadEmbedModel(modelDir) : undefined;
+    if (model) {
+      const index = buildEmbeddingIndex(scan, model);
+      writeFileSync(join(outDir, "embeddings.bin"), serializeEmbeddings(index));
+      embedNote = ` + embeddings.bin (${index.records.length} records, model ${model.modelId})`;
+    }
+    process.stderr.write(`codeindex: ${scan.files.length} files → ${outDir}/graph.json + symbols.json${embedNote}${scan.capped ? " (capped)" : ""}\n`);
   } else if (cmd === "scan") {
     const { scan } = buildIndexArtifacts(flags.repo, scanOptions(flags));
     const summary = {
@@ -238,8 +263,85 @@ export async function runCli(argv: string[]): Promise<void> {
   } else if (cmd === "search") {
     if (!flags.positional) throw new Error('search needs a query: cli.mjs search "<query>" --repo <dir>');
     const scan = scanRepo(flags.repo, scanOptions(flags));
-    const results = searchIndex(scan, flags.positional, { limit: flags.limit, fuzzy: flags.fuzzy });
-    emit(JSON.stringify(results, null, 2) + "\n", flags.out);
+    if (flags.semantic) {
+      const modelDir = resolveEmbedModelDir(flags.repo);
+      const model = modelDir ? loadEmbedModel(modelDir) : undefined;
+      if (!model) {
+        // Degradation: --semantic without a model → lexical results + a stderr
+        // note, exit 0. The results shape is a superset of the lexical one.
+        process.stderr.write(
+          "codeindex: semantic search unavailable (no embedding model present) — returning lexical results; run `codeindex embed pull` to enable it\n",
+        );
+        const results = searchIndex(scan, flags.positional, { limit: flags.limit, fuzzy: flags.fuzzy });
+        emit(JSON.stringify(results, null, 2) + "\n", flags.out);
+      } else {
+        const index = buildEmbeddingIndex(scan, model);
+        const results = searchSemantic(scan, flags.positional, index, { model, limit: flags.limit, fuzzy: flags.fuzzy });
+        emit(JSON.stringify(results, null, 2) + "\n", flags.out);
+      }
+    } else {
+      const results = searchIndex(scan, flags.positional, { limit: flags.limit, fuzzy: flags.fuzzy });
+      emit(JSON.stringify(results, null, 2) + "\n", flags.out);
+    }
+  } else if (cmd === "embed") {
+    const sub = flags.positional;
+    const modelDir = resolveEmbedModelDir(flags.repo);
+    if (sub === "status") {
+      const model = modelDir ? loadEmbedModel(modelDir) : undefined;
+      const status = {
+        embedVersion: EMBED_VERSION,
+        model: model
+          ? { present: true, dir: modelDir, modelId: model.modelId, dim: model.dim, vocabSize: model.vocabSize }
+          : { present: false },
+        endpoint: process.env.CODEINDEX_EMBED_ENDPOINT ?? null,
+      };
+      emit(JSON.stringify(status, null, 2) + "\n", flags.out);
+    } else if (sub === "build") {
+      if (!flags.out) throw new Error("embed build needs --out <dir>");
+      if (!modelDir) {
+        process.stderr.write("codeindex: no embedding model present — run `codeindex embed pull` first (nothing written)\n");
+        process.exitCode = 1;
+        return;
+      }
+      const model = loadEmbedModel(modelDir)!;
+      mkdirSync(flags.out, { recursive: true });
+      const scan = scanRepo(flags.repo, scanOptions(flags));
+      const index = buildEmbeddingIndex(scan, model);
+      writeFileSync(join(flags.out, "embeddings.bin"), serializeEmbeddings(index));
+      process.stderr.write(`codeindex: ${index.records.length} embedding records → ${flags.out}/embeddings.bin (model ${model.modelId})\n`);
+    } else if (sub === "pull") {
+      const url = resolveEmbedPullUrl();
+      if (!url) {
+        // Clean, actionable failure — the official asset is not published yet.
+        process.stderr.write(
+          "codeindex: no model URL configured. The official static-embedding asset is not published yet.\n" +
+            "Set CODEINDEX_EMBED_URL to a model.json URL (optionally CODEINDEX_EMBED_DIR as the destination), then re-run `codeindex embed pull`.\n",
+        );
+        process.exitCode = 1;
+        return;
+      }
+      const destDir = process.env.CODEINDEX_EMBED_DIR ?? join(flags.repo, ".codeindex", "models");
+      mkdirSync(destDir, { recursive: true });
+      process.stderr.write(`codeindex: fetching model from ${url} → ${join(destDir, "model.json")}\n`);
+      const res = await fetch(url);
+      if (!res.ok) {
+        process.stderr.write(`codeindex: pull failed — HTTP ${res.status} from ${url}\n`);
+        process.exitCode = 1;
+        return;
+      }
+      const body = await res.text();
+      try {
+        JSON.parse(body);
+      } catch {
+        process.stderr.write("codeindex: pull failed — response is not a valid model.json (expected JSON)\n");
+        process.exitCode = 1;
+        return;
+      }
+      writeFileSync(join(destDir, "model.json"), body);
+      process.stderr.write(`codeindex: model written to ${join(destDir, "model.json")}\n`);
+    } else {
+      throw new Error("embed needs a subcommand: pull | build | status");
+    }
   } else if (cmd === "rules") {
     if (!flags.config) throw new Error("rules needs --config <codeindex.rules.json>");
     const rules = parseRules(JSON.parse(readFileSync(flags.config, "utf8")));

package/src/engine.ts

@@ -103,6 +103,27 @@ export type { SearchHit, GrepOptions } from "./grep.js";
 export { searchIndex, subtokens } from "./bm25.js";
 export type { SearchOptions, SearchResult } from "./bm25.js";
 
+// Deterministic static-embedding tier (v2.10.0): a keyless, byte-deterministic
+// semantic search, opt-in by model-asset presence (models NEVER ship in the
+// tarball). EMBED_VERSION is dedicated to the embeddings.bin sidecar and is
+// independent of SCHEMA_VERSION / EXTRACTOR_VERSION.
+export {
+  EMBED_VERSION,
+  resolveEmbedModelDir,
+  hasEmbedModel,
+  loadEmbedModel,
+  resolveEmbedPullUrl,
+} from "./embed/model.js";
+export type { StaticEmbedModel } from "./embed/model.js";
+export { encode, tokenize, wordpiece, basicTokenize, roundHalfToEven, intDot } from "./embed/encode.js";
+export { buildEmbeddingIndex, serializeEmbeddings, deserializeEmbeddings } from "./embed/index.js";
+export type { EmbeddingIndex, EmbeddingRecord } from "./embed/index.js";
+export { searchSemantic } from "./embed/search.js";
+export type { SemanticSearchOptions, SemanticSearchResult } from "./embed/search.js";
+// HTTP endpoint tier (v2.11 preview — NOT wired into the CLI/MCP here).
+export { embedViaEndpoint, resolveEmbedEndpoint } from "./embed/endpoint.js";
+export type { EmbedEndpointOptions } from "./embed/endpoint.js";
+
 // Architecture rules: forbidden edges + cycles/orphans builtins (issue #4).
 export { checkRules, parseRules } from "./rules.js";
 export type { ArchRule, ForbiddenEdgeRule, BuiltinRule, RuleSeverity, RuleViolation } from "./rules.js";

package/src/mcp.ts

@@ -25,6 +25,9 @@ import { replaceSymbolBody, insertAfterSymbol, insertBeforeSymbol } from "./edit
 import { writeMemory, readMemory, deleteMemory, listMemories } from "./memory.js";
 import { searchIndex } from "./bm25.js";
 import { checkRules, parseRules } from "./rules.js";
+import { EMBED_VERSION, resolveEmbedModelDir, loadEmbedModel } from "./embed/model.js";
+import { buildEmbeddingIndex } from "./embed/index.js";
+import { searchSemantic } from "./embed/search.js";
 
 interface RpcRequest {
   jsonrpc: "2.0";
@@ -266,7 +269,7 @@ const TOOLS = [
   {
     name: "search",
     description:
-      'Natural-language-ish lexical search: BM25 ranking (k1=1.2, b=0.75) over symbol names (camelCase/snake_case subtokens), file path segments, markdown headings and summary lines. NOT embeddings — deterministic, diacritic-folded, zero API keys. Answers "where is auth handled?"-style queries with ranked files, matched terms and top symbols. Query terms with zero document frequency get a deterministic trigram-fuzzy fallback (typo-tolerant) unless `fuzzy: false`.',
+      'Natural-language-ish lexical search: BM25 ranking (k1=1.2, b=0.75) over symbol names (camelCase/snake_case subtokens), file path segments, markdown headings and summary lines. NOT embeddings by default — deterministic, diacritic-folded, zero API keys. Answers "where is auth handled?"-style queries with ranked files, matched terms and top symbols. Query terms with zero document frequency get a deterministic trigram-fuzzy fallback (typo-tolerant) unless `fuzzy: false`. Set `semantic: true` to RRF-fuse the deterministic static-embedding tier when a model asset is present (degrades to lexical otherwise — see embed_status).',
     inputSchema: {
       type: "object",
       properties: {
@@ -279,10 +282,21 @@ const TOOLS = [
           description:
             "Trigram fuzzy fallback for query terms with zero document frequency (default true)",
         },
+        semantic: {
+          type: "boolean",
+          description:
+            "RRF-fuse the deterministic static-embedding tier with lexical when a model asset is present (default false; silently lexical-only when no model)",
+        },
       },
       required: ["repo", "query"],
     },
   },
+  {
+    name: "embed_status",
+    description:
+      "Report the deterministic static-embedding tier: whether a model asset is resolved (opt-in, never shipped in the package), its modelId/dim, EMBED_VERSION, and any configured HTTP endpoint. Use to check whether `search` with semantic:true will actually fuse embeddings or degrade to lexical.",
+    inputSchema: { type: "object", properties: { ...repoProp }, required: ["repo"] },
+  },
   {
     name: "check_rules",
     description:
@@ -440,11 +454,34 @@ function callTool(name: string, args: Record<string, unknown>): string {
   if (name === "search") {
     const query = str(args.query);
     if (!query) throw new Error("`query` is required");
-    const results = searchIndex(scanRepo(repo, scanOpts), query, {
-      limit: typeof args.limit === "number" ? args.limit : undefined,
-      fuzzy: typeof args.fuzzy === "boolean" ? args.fuzzy : undefined,
-    });
-    return JSON.stringify(results, null, 2);
+    const scan = scanRepo(repo, scanOpts);
+    const limit = typeof args.limit === "number" ? args.limit : undefined;
+    const fuzzy = typeof args.fuzzy === "boolean" ? args.fuzzy : undefined;
+    if (args.semantic === true) {
+      const modelDir = resolveEmbedModelDir(repo);
+      const model = modelDir ? loadEmbedModel(modelDir) : undefined;
+      if (model) {
+        const index = buildEmbeddingIndex(scan, model);
+        return JSON.stringify(searchSemantic(scan, query, index, { model, limit, fuzzy }), null, 2);
+      }
+      // Degrade to lexical (opt-in tier not activated) — no throw.
+    }
+    return JSON.stringify(searchIndex(scan, query, { limit, fuzzy }), null, 2);
+  }
+  if (name === "embed_status") {
+    const modelDir = resolveEmbedModelDir(repo);
+    const model = modelDir ? loadEmbedModel(modelDir) : undefined;
+    return JSON.stringify(
+      {
+        embedVersion: EMBED_VERSION,
+        model: model
+          ? { present: true, dir: modelDir, modelId: model.modelId, dim: model.dim, vocabSize: model.vocabSize }
+          : { present: false },
+        endpoint: process.env.CODEINDEX_EMBED_ENDPOINT ?? null,
+      },
+      null,
+      2,
+    );
   }
   if (name === "check_rules") {
     const rules = parseRules(args.rules); // throws a descriptive error on a malformed payload

package/src/types.ts

@@ -1,7 +1,7 @@
 // Single source of truth for the engine version the bundle reports. Kept in
 // lockstep with package.json by the release pipeline. Do not edit by hand
 // outside a release.
-export const ENGINE_VERSION = "2.9.0";
+export const ENGINE_VERSION = "2.10.0";
 
 // Bumped whenever the on-disk artifact shape changes, so a consumer can reject
 // an index written by an incompatible engine instead of misreading it. The