@maxgfr/codeindex 2.9.0 → 2.11.0

package/scripts/engine.mjs

@@ -14,9 +14,9 @@ var ENGINE_VERSION, SCHEMA_VERSION, EXTRACTOR_VERSION;
 var init_types = __esm({
   "src/types.ts"() {
     "use strict";
-    ENGINE_VERSION = "2.9.0";
+    ENGINE_VERSION = "2.11.0";
     SCHEMA_VERSION = 4;
-    EXTRACTOR_VERSION = 6;
+    EXTRACTOR_VERSION = 7;
   }
 });
 
@@ -6340,12 +6340,14 @@ function extractImports(ext, content) {
   }
   return [...specs].map((spec) => ({ kind: "import", spec }));
 }
-function extractReexports(rel, content) {
+function extractReexports(rel, content, localSymbols) {
   if (!JS_TS.has(rel.slice(rel.lastIndexOf(".")))) return [];
   const lang = /\.(ts|tsx|mts|cts)$/.test(rel) ? "typescript" : "javascript";
   const out2 = [];
   const seen = /* @__PURE__ */ new Set();
   const lineAt = (idx) => content.slice(0, idx).split(/\r?\n/).length;
+  const localKindOf = /* @__PURE__ */ new Map();
+  for (const s of localSymbols) if (!localKindOf.has(s.name)) localKindOf.set(s.name, s.kind);
   const named = /export\s*\{([\s\S]*?)\}\s*(?:from\s*['"]([^'"]+)['"])?\s*;?/g;
   let m;
   while ((m = named.exec(content)) && out2.length < 60) {
@@ -6353,12 +6355,14 @@ function extractReexports(rel, content) {
     for (const part of m[1].split(",")) {
       const p = part.trim().replace(/^type\s+/, "");
       const as = /^(\S+)\s+as\s+([A-Za-z_$][\w$]*)$/.exec(p);
+      const orig = as ? as[1] : p;
       const name2 = as ? as[2] : p;
       if (!/^[A-Za-z_$][\w$]*$/.test(name2) || name2 === "default" || seen.has(name2)) continue;
       seen.add(name2);
+      const mirroredKind = !from ? localKindOf.get(orig) : void 0;
       out2.push({
         name: name2,
-        kind: "reexport",
+        kind: mirroredKind ?? "reexport",
         file: rel,
         line: lineAt(m.index),
         signature: from ? `export { ${name2} } from "${from}"` : `export { ${name2} }`,
@@ -6411,7 +6415,7 @@ function extractCode(rel, ext, content) {
   const ast = extractAst(rel, ext, content);
   const symbols = (ast ? ast.symbols : extractSymbols(rel, ext, content)).slice(0, 400);
   const known = new Set(symbols.map((s) => s.name));
-  const reexports = extractReexports(rel, content).filter((s) => !known.has(s.name));
+  const reexports = extractReexports(rel, content, symbols).filter((s) => !known.has(s.name));
   return {
     symbols: [...symbols, ...reexports],
     summary: topDocComment(content),
@@ -9343,6 +9347,376 @@ var init_bm25 = __esm({
   }
 });
 
+// src/embed/model.ts
+import { existsSync as existsSync3, readFileSync as readFileSync5 } from "fs";
+import { join as join10 } from "path";
+function resolveEmbedModelDir(repo) {
+  const env = process.env.CODEINDEX_EMBED_DIR;
+  const candidates = [];
+  if (env) candidates.push(env);
+  if (repo) candidates.push(join10(repo, ".codeindex", DEFAULT_EMBED_DIRNAME));
+  candidates.push(join10(process.cwd(), ".codeindex", DEFAULT_EMBED_DIRNAME));
+  for (const c2 of candidates) {
+    if (existsSync3(join10(c2, "model.json"))) return c2;
+  }
+  return void 0;
+}
+function hasEmbedModel(repo) {
+  return resolveEmbedModelDir(repo) !== void 0;
+}
+function loadEmbedModel(dir) {
+  if (!dir) return void 0;
+  const path = join10(dir, "model.json");
+  if (!existsSync3(path)) return void 0;
+  const raw = JSON.parse(readFileSync5(path, "utf8"));
+  const { modelId, dim, vocab, weights } = raw;
+  if (typeof modelId !== "string" || !modelId) throw new Error(`embed model: missing modelId in ${path}`);
+  if (!Number.isInteger(dim) || dim <= 0) throw new Error(`embed model: bad dim ${dim} in ${path}`);
+  if (!Array.isArray(vocab) || !Array.isArray(weights) || vocab.length !== weights.length) {
+    throw new Error(`embed model: vocab/weights length mismatch in ${path}`);
+  }
+  const vocabSize = vocab.length;
+  const flat = new Float64Array(vocabSize * dim);
+  const vmap = /* @__PURE__ */ new Map();
+  for (let i2 = 0; i2 < vocabSize; i2++) {
+    const tok = vocab[i2];
+    if (typeof tok !== "string") throw new Error(`embed model: non-string vocab entry at ${i2}`);
+    if (!vmap.has(tok)) vmap.set(tok, i2);
+    const row = weights[i2];
+    if (!Array.isArray(row) || row.length !== dim) {
+      throw new Error(`embed model: row ${i2} has length ${row?.length}, expected ${dim}`);
+    }
+    for (let d = 0; d < dim; d++) flat[i2 * dim + d] = Number(row[d]);
+  }
+  const unk = typeof raw.unk === "string" ? raw.unk : "[UNK]";
+  const unkId = vmap.has(unk) ? vmap.get(unk) : -1;
+  return { modelId, dim, unk, unkId, vocabSize, vocab: vmap, weights: flat };
+}
+function resolveEmbedPullUrl() {
+  const url = process.env.CODEINDEX_EMBED_URL;
+  return url && url.trim() ? url.trim() : void 0;
+}
+var EMBED_VERSION, DEFAULT_EMBED_DIRNAME;
+var init_model = __esm({
+  "src/embed/model.ts"() {
+    "use strict";
+    EMBED_VERSION = 1;
+    DEFAULT_EMBED_DIRNAME = "models";
+  }
+});
+
+// src/embed/encode.ts
+function basicTokenize(text) {
+  const spaced = foldText(text).replace(/([a-z0-9])([A-Z])/g, "$1 $2").replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2");
+  const out2 = [];
+  for (const part of spaced.toLowerCase().split(/[^a-z0-9]+/)) {
+    if (part) out2.push(part);
+  }
+  return out2;
+}
+function wordpiece(word, model) {
+  if (!word) return [];
+  const ids = [];
+  let start2 = 0;
+  const n = word.length;
+  while (start2 < n) {
+    let end = n;
+    let match = -1;
+    while (end > start2) {
+      const piece = start2 === 0 ? word.slice(start2, end) : "##" + word.slice(start2, end);
+      const id = model.vocab.get(piece);
+      if (id !== void 0) {
+        match = id;
+        break;
+      }
+      end--;
+    }
+    if (match === -1) return model.unkId >= 0 ? [model.unkId] : [];
+    ids.push(match);
+    start2 = end;
+  }
+  return ids;
+}
+function tokenize(text, model) {
+  const ids = [];
+  for (const word of basicTokenize(text)) {
+    for (const id of wordpiece(word, model)) ids.push(id);
+  }
+  return ids;
+}
+function roundHalfToEven(x) {
+  const f = Math.floor(x);
+  const diff = x - f;
+  if (diff < 0.5) return f;
+  if (diff > 0.5) return f + 1;
+  return f % 2 === 0 ? f : f + 1;
+}
+function quantize(vec) {
+  const dim = vec.length;
+  const out2 = new Int8Array(dim);
+  let sumsq = 0;
+  for (let d = 0; d < dim; d++) sumsq += vec[d] * vec[d];
+  const norm2 = Math.sqrt(sumsq);
+  if (norm2 === 0) return out2;
+  for (let d = 0; d < dim; d++) {
+    let q = roundHalfToEven(vec[d] / norm2 * QUANT);
+    if (q > QUANT) q = QUANT;
+    else if (q < -QUANT) q = -QUANT;
+    out2[d] = q;
+  }
+  return out2;
+}
+function encode(model, text) {
+  const { dim, weights } = model;
+  const ids = tokenize(text, model);
+  if (ids.length === 0) return new Int8Array(dim);
+  const pooled = new Float64Array(dim);
+  for (const id of ids) {
+    const base = id * dim;
+    for (let d = 0; d < dim; d++) pooled[d] += weights[base + d];
+  }
+  const inv = 1 / ids.length;
+  for (let d = 0; d < dim; d++) pooled[d] *= inv;
+  return quantize(pooled);
+}
+function intDot(a, b) {
+  const n = Math.min(a.length, b.length);
+  let dot = 0;
+  for (let i2 = 0; i2 < n; i2++) dot += a[i2] * b[i2];
+  return dot;
+}
+var QUANT;
+var init_encode = __esm({
+  "src/embed/encode.ts"() {
+    "use strict";
+    init_util();
+    QUANT = 127;
+  }
+});
+
+// src/embed/index.ts
+function symbolText(rel, name2, signature, summary) {
+  return [name2, signature ?? "", summary ?? "", rel.replace(/\//g, " ")].join("\n");
+}
+function fileText(rel, title, summary, headings) {
+  return [title ?? "", summary ?? "", ...headings, rel.replace(/\//g, " ")].join("\n");
+}
+function embeddingUnits(scan2) {
+  const units = [];
+  for (const f of scan2.files) {
+    const seen = /* @__PURE__ */ new Set();
+    let hadSymbol = false;
+    for (const s of f.symbols) {
+      if (seen.has(s.name)) continue;
+      seen.add(s.name);
+      hadSymbol = true;
+      units.push({ file: f.rel, symbol: s.name, line: s.line, text: symbolText(f.rel, s.name, s.signature, f.summary) });
+    }
+    if (!hadSymbol) {
+      const text = fileText(f.rel, f.title, f.summary, f.headings);
+      if (text.replace(/\s+/g, "")) units.push({ file: f.rel, text });
+    }
+  }
+  return units;
+}
+function buildEmbeddingIndex(scan2, model) {
+  const records = embeddingUnits(scan2).map((u) => {
+    const rec = { file: u.file, vec: encode(model, u.text) };
+    if (u.symbol !== void 0) rec.symbol = u.symbol;
+    if (u.line !== void 0) rec.line = u.line;
+    return rec;
+  });
+  return { embedVersion: EMBED_VERSION, modelId: model.modelId, dim: model.dim, records };
+}
+function serializeEmbeddings(index) {
+  const header = JSON.stringify({
+    embedVersion: index.embedVersion,
+    modelId: index.modelId,
+    dim: index.dim,
+    count: index.records.length,
+    records: index.records.map((r) => ({ file: r.file, symbol: r.symbol ?? "", line: r.line ?? 0 }))
+  });
+  const headerBuf = Buffer.from(header, "utf8");
+  const body2 = Buffer.alloc(index.records.length * index.dim);
+  let off = 0;
+  for (const r of index.records) {
+    for (let d = 0; d < index.dim; d++) body2.writeInt8(r.vec[d] ?? 0, off++);
+  }
+  const out2 = Buffer.alloc(8 + headerBuf.length + body2.length);
+  out2.write(MAGIC, 0, "ascii");
+  out2.writeUInt32LE(headerBuf.length, 4);
+  headerBuf.copy(out2, 8);
+  body2.copy(out2, 8 + headerBuf.length);
+  return out2;
+}
+function deserializeEmbeddings(bytes) {
+  const buf = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  if (buf.length < 8 || buf.toString("ascii", 0, 4) !== MAGIC) {
+    throw new Error("embeddings.bin: bad magic (not a codeindex embeddings artifact)");
+  }
+  const headerLen = buf.readUInt32LE(4);
+  const header = JSON.parse(buf.toString("utf8", 8, 8 + headerLen));
+  const bodyOff = 8 + headerLen;
+  const { dim } = header;
+  const records = header.records.map((m, i2) => {
+    const vec = new Int8Array(dim);
+    for (let d = 0; d < dim; d++) vec[d] = buf.readInt8(bodyOff + i2 * dim + d);
+    const rec = { file: m.file, vec };
+    if (m.symbol) rec.symbol = m.symbol;
+    if (m.line) rec.line = m.line;
+    return rec;
+  });
+  return { embedVersion: header.embedVersion, modelId: header.modelId, dim, records };
+}
+var MAGIC;
+var init_embed = __esm({
+  "src/embed/index.ts"() {
+    "use strict";
+    init_encode();
+    init_model();
+    MAGIC = "CIE1";
+  }
+});
+
+// src/embed/search.ts
+function searchSemantic(scan2, query, index, opts = {}) {
+  const limit = opts.limit ?? DEFAULT_LIMIT2;
+  const lexical = searchIndex(scan2, query, { limit: Math.max(limit, 50), fuzzy: opts.fuzzy });
+  const q = opts.queryVec ?? (opts.model ? encode(opts.model, query) : void 0);
+  if (!q || !index || index.records.length === 0) {
+    return lexical.slice(0, limit);
+  }
+  const bestByFile = /* @__PURE__ */ new Map();
+  for (const r of index.records) {
+    const dot = intDot(q, r.vec);
+    const prev = bestByFile.get(r.file);
+    if (!prev || dot > prev.score) bestByFile.set(r.file, { score: dot, symbol: r.symbol });
+  }
+  const semList = [...bestByFile.entries()].filter(([, v]) => v.score > 0).sort((a, b) => b[1].score - a[1].score || byStr(a[0], b[0])).map(([file]) => file);
+  const lexList = lexical.map((r) => r.file);
+  const fused = rrf([lexList, semList], (f) => f, opts.rrfK ?? RRF_K);
+  const lexByFile = new Map(lexical.map((r) => [r.file, r]));
+  const results = [...fused.entries()].sort((a, b) => b[1] - a[1] || byStr(a[0], b[0])).map(([file, score]) => {
+    const lex = lexByFile.get(file);
+    const res = {
+      file,
+      score: Number(score.toFixed(4)),
+      matchedTerms: lex?.matchedTerms ?? [],
+      topSymbols: lex?.topSymbols ?? []
+    };
+    const sem = bestByFile.get(file);
+    if (sem?.symbol) res.semanticSymbol = sem.symbol;
+    if (lex?.fuzzyTerms) res.fuzzyTerms = lex.fuzzyTerms;
+    return res;
+  });
+  return results.slice(0, limit);
+}
+var DEFAULT_LIMIT2, RRF_K;
+var init_search = __esm({
+  "src/embed/search.ts"() {
+    "use strict";
+    init_util();
+    init_sort();
+    init_bm25();
+    init_encode();
+    DEFAULT_LIMIT2 = 20;
+    RRF_K = 60;
+  }
+});
+
+// src/embed/endpoint.ts
+function resolveEmbedEndpoint(opts = {}) {
+  const url = opts.url ?? process.env.CODEINDEX_EMBED_ENDPOINT;
+  return url && url.trim() ? url.trim() : void 0;
+}
+function stripTrailingSlash(url) {
+  return url.replace(/\/+$/, "");
+}
+function embedEndpointUrl(base) {
+  const b = stripTrailingSlash(base);
+  return b.endsWith("/embed") ? b : b + "/embed";
+}
+function healthzUrl(base) {
+  return stripTrailingSlash(base).replace(/\/embed$/, "") + "/healthz";
+}
+function resolveTimeout(opts) {
+  if (typeof opts.timeoutMs === "number") return opts.timeoutMs;
+  const env = Number(process.env.CODEINDEX_EMBED_TIMEOUT_MS);
+  return Number.isFinite(env) && env > 0 ? env : 3e4;
+}
+async function embedViaEndpoint(texts, opts = {}) {
+  const base = resolveEmbedEndpoint(opts);
+  if (!base) throw new Error("no embedding endpoint configured (set CODEINDEX_EMBED_ENDPOINT or pass opts.url)");
+  const url = embedEndpointUrl(base);
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), resolveTimeout(opts));
+  try {
+    const res = await fetch(url, {
+      method: "POST",
+      headers: { "content-type": "application/json", ...opts.headers ?? {} },
+      body: JSON.stringify({ texts }),
+      signal: controller.signal
+    });
+    if (!res.ok) throw new Error(`embedding endpoint ${url} returned HTTP ${res.status}`);
+    const data = await res.json();
+    const vectors = data.vectors;
+    if (!Array.isArray(vectors) || !vectors.every((v) => Array.isArray(v) && v.every((x) => typeof x === "number"))) {
+      throw new Error(`embedding endpoint ${url} returned a malformed { vectors } payload`);
+    }
+    return vectors;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+async function probeEndpoint(base, opts = {}) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), resolveTimeout(opts));
+  try {
+    const res = await fetch(healthzUrl(base), { signal: controller.signal, headers: opts.headers });
+    return res.ok;
+  } catch {
+    return false;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+async function encodeQueryViaEndpoint(query, opts = {}) {
+  const [vec] = await embedViaEndpoint([query], opts);
+  if (!vec) throw new Error("embedding endpoint returned no vector for the query");
+  return quantize(vec);
+}
+async function buildEndpointIndex(scan2, opts = {}) {
+  const units = embeddingUnits(scan2);
+  const batchSize = opts.batchSize && opts.batchSize > 0 ? opts.batchSize : 64;
+  const records = [];
+  let dim = 0;
+  for (let i2 = 0; i2 < units.length; i2 += batchSize) {
+    const batch = units.slice(i2, i2 + batchSize);
+    const vectors = await embedViaEndpoint(batch.map((u) => u.text), opts);
+    if (vectors.length !== batch.length) {
+      throw new Error(`embedding endpoint returned ${vectors.length} vectors for ${batch.length} texts`);
+    }
+    for (let j = 0; j < batch.length; j++) {
+      const u = batch[j];
+      const vec = quantize(vectors[j]);
+      if (vec.length > dim) dim = vec.length;
+      const rec = { file: u.file, vec };
+      if (u.symbol !== void 0) rec.symbol = u.symbol;
+      if (u.line !== void 0) rec.line = u.line;
+      records.push(rec);
+    }
+  }
+  return { embedVersion: EMBED_VERSION, modelId: "endpoint", dim, records };
+}
+var init_endpoint = __esm({
+  "src/embed/endpoint.ts"() {
+    "use strict";
+    init_encode();
+    init_model();
+    init_embed();
+  }
+});
+
 // src/rules.ts
 function isEntrypointLike(rel) {
   const base = rel.split("/").pop();
@@ -9647,7 +10021,7 @@ var init_deadcode = __esm({
 });
 
 // src/complexity.ts
-import { join as join10 } from "path";
+import { join as join11 } from "path";
 function complexityOfSource(source) {
   return 1 + (source.match(BRANCH_RE) ?? []).length;
 }
@@ -9657,7 +10031,7 @@ function symbolComplexity(scan2, rel, top = 50) {
     if (f.kind !== "code") continue;
     if (rel && f.rel !== rel) continue;
     if (!f.symbols.length) continue;
-    const lines = readText(join10(scan2.root, f.rel)).split("\n");
+    const lines = readText(join11(scan2.root, f.rel)).split("\n");
     for (const s of f.symbols) {
       if (s.kind === "reexport" || s.kind === "reexport-all") continue;
       const end = s.endLine ?? s.line;
@@ -9672,7 +10046,7 @@ function symbolComplexity(scan2, rel, top = 50) {
 }
 function riskHotspots(scan2, churn, top = 20) {
   const out2 = scan2.files.filter((f) => f.kind === "code").map((f) => {
-    const complexity = complexityOfSource(readText(join10(scan2.root, f.rel)));
+    const complexity = complexityOfSource(readText(join11(scan2.root, f.rel)));
     const commits = churn.get(f.rel) ?? 0;
     return { file: f.rel, complexity, commits, score: (commits + 1) * complexity };
   });
@@ -9738,7 +10112,7 @@ function str(v) {
 function strArray(v) {
   return Array.isArray(v) && v.every((x) => typeof x === "string") && v.length ? v : void 0;
 }
-function callTool(name2, args2) {
+async function callTool(name2, args2) {
   const repo = str(args2.repo);
   if (!repo) throw new Error("`repo` is required (absolute path to the repository root)");
   const scanOpts = { scope: str(args2.scope), include: strArray(args2.include), exclude: strArray(args2.exclude) };
@@ -9871,11 +10245,42 @@ function callTool(name2, args2) {
   if (name2 === "search") {
     const query = str(args2.query);
     if (!query) throw new Error("`query` is required");
-    const results = searchIndex(scanRepo(repo, scanOpts), query, {
-      limit: typeof args2.limit === "number" ? args2.limit : void 0,
-      fuzzy: typeof args2.fuzzy === "boolean" ? args2.fuzzy : void 0
-    });
-    return JSON.stringify(results, null, 2);
+    const scan2 = scanRepo(repo, scanOpts);
+    const limit = typeof args2.limit === "number" ? args2.limit : void 0;
+    const fuzzy = typeof args2.fuzzy === "boolean" ? args2.fuzzy : void 0;
+    if (args2.semantic === true) {
+      const endpoint = resolveEmbedEndpoint();
+      if (endpoint) {
+        try {
+          const index = await buildEndpointIndex(scan2);
+          const queryVec = await encodeQueryViaEndpoint(query);
+          return JSON.stringify(searchSemantic(scan2, query, index, { queryVec, limit, fuzzy }), null, 2);
+        } catch {
+          return JSON.stringify(searchIndex(scan2, query, { limit, fuzzy }), null, 2);
+        }
+      }
+      const modelDir = resolveEmbedModelDir(repo);
+      const model = modelDir ? loadEmbedModel(modelDir) : void 0;
+      if (model) {
+        const index = buildEmbeddingIndex(scan2, model);
+        return JSON.stringify(searchSemantic(scan2, query, index, { model, limit, fuzzy }), null, 2);
+      }
+    }
+    return JSON.stringify(searchIndex(scan2, query, { limit, fuzzy }), null, 2);
+  }
+  if (name2 === "embed_status") {
+    const modelDir = resolveEmbedModelDir(repo);
+    const model = modelDir ? loadEmbedModel(modelDir) : void 0;
+    const endpoint = resolveEmbedEndpoint();
+    const mode = endpoint ? "endpoint" : model ? "static" : "none";
+    const status = {
+      embedVersion: EMBED_VERSION,
+      mode,
+      model: model ? { present: true, dir: modelDir, modelId: model.modelId, dim: model.dim, vocabSize: model.vocabSize } : { present: false },
+      endpoint: endpoint ?? null
+    };
+    if (endpoint) status.endpointReachable = await probeEndpoint(endpoint);
+    return JSON.stringify(status, null, 2);
   }
   if (name2 === "check_rules") {
     const rules = parseRules(args2.rules);
@@ -9901,9 +10306,9 @@ async function runMcpServer() {
       continue;
     }
     const requests = Array.isArray(parsed) ? parsed : [parsed];
-    for (const req of requests) handle2(req);
+    for (const req of requests) await handle2(req);
   }
-  function handle2(req) {
+  async function handle2(req) {
     if (req.id === void 0 || req.id === null) return;
     try {
       if (req.method === "initialize") {
@@ -9924,7 +10329,7 @@ async function runMcpServer() {
         const name2 = str(params.name) ?? "";
         const args2 = params.arguments ?? {};
         try {
-          const text = callTool(name2, args2);
+          const text = await callTool(name2, args2);
           send({ id: req.id, result: { content: [{ type: "text", text }] } });
         } catch (e) {
           send({
@@ -9963,6 +10368,10 @@ var init_mcp = __esm({
     init_memory();
     init_bm25();
     init_rules();
+    init_model();
+    init_embed();
+    init_search();
+    init_endpoint();
     repoProp = { repo: { type: "string", description: "Absolute path to the repository root" } };
     scopeProps = {
       scope: { type: "string", description: "Restrict to one directory (repo-relative)" },
@@ -10175,7 +10584,7 @@ var init_mcp = __esm({
       },
       {
         name: "search",
-        description: 'Natural-language-ish lexical search: BM25 ranking (k1=1.2, b=0.75) over symbol names (camelCase/snake_case subtokens), file path segments, markdown headings and summary lines. NOT embeddings \u2014 deterministic, diacritic-folded, zero API keys. Answers "where is auth handled?"-style queries with ranked files, matched terms and top symbols. Query terms with zero document frequency get a deterministic trigram-fuzzy fallback (typo-tolerant) unless `fuzzy: false`.',
+        description: 'Natural-language-ish lexical search: BM25 ranking (k1=1.2, b=0.75) over symbol names (camelCase/snake_case subtokens), file path segments, markdown headings and summary lines. NOT embeddings by default \u2014 deterministic, diacritic-folded, zero API keys. Answers "where is auth handled?"-style queries with ranked files, matched terms and top symbols. Query terms with zero document frequency get a deterministic trigram-fuzzy fallback (typo-tolerant) unless `fuzzy: false`. Set `semantic: true` to RRF-fuse the deterministic static-embedding tier when a model asset is present (degrades to lexical otherwise \u2014 see embed_status).',
         inputSchema: {
           type: "object",
           properties: {
@@ -10186,11 +10595,20 @@ var init_mcp = __esm({
             fuzzy: {
               type: "boolean",
               description: "Trigram fuzzy fallback for query terms with zero document frequency (default true)"
+            },
+            semantic: {
+              type: "boolean",
+              description: "RRF-fuse an embedding tier with lexical (default false). Precedence: the HTTP endpoint (CODEINDEX_EMBED_ENDPOINT) if set, else a local static model. Degrades silently to lexical-only when neither is available/reachable \u2014 see embed_status."
             }
           },
           required: ["repo", "query"]
         }
       },
+      {
+        name: "embed_status",
+        description: "Report the embedding tier: the effective mode (none/static/endpoint; endpoint > static model), the resolved model (opt-in, never shipped in the package) with its modelId/dim, EMBED_VERSION, and the configured HTTP endpoint with its reachability. Use to check whether `search` with semantic:true will fuse embeddings or degrade to lexical.",
+        inputSchema: { type: "object", properties: { ...repoProp }, required: ["repo"] }
+      },
       {
         name: "check_rules",
         description: 'Validate dependency-cruiser-style architecture rules against the link-graph. Rules (inline JSON array): forbidden edges {name, from, to, kind?, severity?, comment?} with glob paths, plus builtins {name, builtin: "cycles"|"orphans"} (module-level import cycles; edge-less code files). Returns deterministic violations with severity error|warn \u2014 a CI gate.',
@@ -10605,6 +11023,11 @@ init_pipeline();
 init_git();
 init_grep();
 init_bm25();
+init_model();
+init_encode();
+init_embed();
+init_search();
+init_endpoint();
 init_rules();
 init_coupling();
 init_repomap();
@@ -10623,8 +11046,8 @@ init_loader();
 init_pipeline();
 init_graph_json();
 init_symbols_json();
-import { existsSync as existsSync3, mkdirSync as mkdirSync2, readFileSync as readFileSync5, writeFileSync as writeFileSync3 } from "fs";
-import { join as join11, resolve } from "path";
+import { existsSync as existsSync4, mkdirSync as mkdirSync2, readFileSync as readFileSync6, writeFileSync as writeFileSync3 } from "fs";
+import { join as join12, resolve } from "path";
 init_scan();
 init_callers();
 init_workspaces();
@@ -10637,6 +11060,11 @@ init_complexity();
 init_viz();
 init_bm25();
 init_rules();
+init_model();
+init_embed();
+init_search();
+init_endpoint();
+init_util();
 var HELP = `codeindex engine v${ENGINE_VERSION} \u2014 deterministic repo indexing
 
 Usage: engine.mjs <command> [flags]
@@ -10654,7 +11082,18 @@ Commands:
   churn       Per-file git commit counts (JSON; --since <ref> to bound)
   grep        Search: cli.mjs grep <pattern> --repo <dir> (JSON hits)
   search      Keyless BM25 lexical search over symbol names, path segments,
-              markdown headings and summaries: cli.mjs search "<query>" --repo <dir>
+              markdown headings and summaries: cli.mjs search "<query>" --repo <dir>.
+              --semantic fuses in an embedding tier (RRF) \u2014 the HTTP endpoint
+              (CODEINDEX_EMBED_ENDPOINT) if set, else a local static model;
+              degrades to lexical (exit 0) when neither is available/reachable
+  embed       Embedding tiers (opt-in). Precedence: endpoint > static model:
+                embed status   Effective mode (none/static/endpoint), model +
+                               EMBED_VERSION, and endpoint reachability (JSON)
+                embed build    Write embeddings.bin into --out <dir> (static tier)
+                embed pull     Fetch the model asset into CODEINDEX_EMBED_DIR (or
+                               <repo>/.codeindex/models/) \u2014 needs CODEINDEX_EMBED_URL
+                embed serve    Print (or --run) the docker command that starts the
+                               containerized embedding server (rich tier)
   rules       Architecture rules (forbidden edges, cycles, orphans) validated
               against the link-graph: --config <codeindex.rules.json>; exits 1
               on any error-severity violation (a CI gate)
@@ -10682,12 +11121,16 @@ Flags:
   --limit <n>         Max results for \`search\` (default 20)
   --no-fuzzy          \`search\`: disable trigram fuzzy fallback for query terms
                       with zero document frequency (default: enabled)
+  --semantic          \`search\`: RRF-fuse an embedding tier with lexical \u2014 the
+                      HTTP endpoint if CODEINDEX_EMBED_ENDPOINT is set, else a
+                      local static model (lexical-only when neither is available)
+  --run               \`embed serve\`: run the docker command instead of printing it
   --recall            \`callers\`: recall-oriented binding (issue #7) \u2014 relaxes
                       the JS/TS import gate to unique repo-wide names and labels
                       each site corroborated|unique-name
 `;
 function parseFlags(args2) {
-  const flags2 = { repo: process.cwd(), include: [], exclude: [], gitignore: true, noAst: false, fuzzy: true };
+  const flags2 = { repo: process.cwd(), include: [], exclude: [], gitignore: true, noAst: false, fuzzy: true, semantic: false };
   for (let i2 = 0; i2 < args2.length; i2++) {
     const a = args2[i2];
     const next = () => {
@@ -10720,7 +11163,9 @@ function parseFlags(args2) {
     else if (a === "--config") flags2.config = resolve(next());
     else if (a === "--limit") flags2.limit = num();
     else if (a === "--no-fuzzy") flags2.fuzzy = false;
+    else if (a === "--semantic") flags2.semantic = true;
     else if (a === "--recall") flags2.recall = true;
+    else if (a === "--run") flags2.run = true;
     else if (!a.startsWith("--") && flags2.positional === void 0) flags2.positional = a;
     else throw new Error(`unknown flag: ${a}`);
   }
@@ -10756,24 +11201,24 @@ async function runCli(argv) {
     return;
   }
   const flags2 = parseFlags(rest);
-  if (!existsSync3(flags2.repo)) throw new Error(`--repo path does not exist: ${flags2.repo}`);
+  if (!existsSync4(flags2.repo)) throw new Error(`--repo path does not exist: ${flags2.repo}`);
   if (!flags2.noAst) await ensureGrammars(allGrammarKeys());
   if (cmd === "index") {
     if (!flags2.out) throw new Error("index needs --out <dir>");
     const outDir = flags2.out;
     mkdirSync2(outDir, { recursive: true });
-    const cachePath = join11(outDir, "cache.json");
+    const cachePath = join12(outDir, "cache.json");
     let cache;
     try {
-      const parsed = JSON.parse(readFileSync5(cachePath, "utf8"));
+      const parsed = JSON.parse(readFileSync6(cachePath, "utf8"));
       if (parsed.schemaVersion === SCHEMA_VERSION && parsed.extractorVersion === EXTRACTOR_VERSION) {
         cache = new Map(Object.entries(parsed.files));
       }
     } catch {
     }
     const { scan: scan2, graph, symbols } = buildIndexArtifacts(flags2.repo, { ...scanOptions(flags2), cache, out: outDir });
-    writeFileSync3(join11(outDir, "graph.json"), renderGraphJson(graph));
-    writeFileSync3(join11(outDir, "symbols.json"), renderSymbolsJson(symbols));
+    writeFileSync3(join12(outDir, "graph.json"), renderGraphJson(graph));
+    writeFileSync3(join12(outDir, "symbols.json"), renderSymbolsJson(symbols));
     const files = {};
     for (const f of scan2.files) {
       const entry = { hash: f.hash, record: f, size: f.size };
@@ -10785,7 +11230,15 @@ async function runCli(argv) {
       cachePath,
       JSON.stringify({ schemaVersion: SCHEMA_VERSION, extractorVersion: EXTRACTOR_VERSION, files }) + "\n"
     );
-    process.stderr.write(`codeindex: ${scan2.files.length} files \u2192 ${outDir}/graph.json + symbols.json${scan2.capped ? " (capped)" : ""}
+    let embedNote = "";
+    const modelDir = resolveEmbedModelDir(flags2.repo);
+    const model = modelDir ? loadEmbedModel(modelDir) : void 0;
+    if (model) {
+      const index = buildEmbeddingIndex(scan2, model);
+      writeFileSync3(join12(outDir, "embeddings.bin"), serializeEmbeddings(index));
+      embedNote = ` + embeddings.bin (${index.records.length} records, model ${model.modelId})`;
+    }
+    process.stderr.write(`codeindex: ${scan2.files.length} files \u2192 ${outDir}/graph.json + symbols.json${embedNote}${scan2.capped ? " (capped)" : ""}
 `);
   } else if (cmd === "scan") {
     const { scan: scan2 } = buildIndexArtifacts(flags2.repo, scanOptions(flags2));
@@ -10822,11 +11275,138 @@ async function runCli(argv) {
   } else if (cmd === "search") {
     if (!flags2.positional) throw new Error('search needs a query: cli.mjs search "<query>" --repo <dir>');
     const scan2 = scanRepo(flags2.repo, scanOptions(flags2));
-    const results = searchIndex(scan2, flags2.positional, { limit: flags2.limit, fuzzy: flags2.fuzzy });
-    emit(JSON.stringify(results, null, 2) + "\n", flags2.out);
+    if (flags2.semantic) {
+      const endpoint = resolveEmbedEndpoint();
+      const lexical = () => {
+        const results = searchIndex(scan2, flags2.positional, { limit: flags2.limit, fuzzy: flags2.fuzzy });
+        emit(JSON.stringify(results, null, 2) + "\n", flags2.out);
+      };
+      if (endpoint) {
+        try {
+          const index = await buildEndpointIndex(scan2);
+          const queryVec = await encodeQueryViaEndpoint(flags2.positional);
+          const results = searchSemantic(scan2, flags2.positional, index, { queryVec, limit: flags2.limit, fuzzy: flags2.fuzzy });
+          emit(JSON.stringify(results, null, 2) + "\n", flags2.out);
+        } catch (e) {
+          process.stderr.write(
+            `codeindex: embedding endpoint ${endpoint} unavailable (${e instanceof Error ? e.message : e}) \u2014 returning lexical results
+`
+          );
+          lexical();
+        }
+      } else {
+        const modelDir = resolveEmbedModelDir(flags2.repo);
+        const model = modelDir ? loadEmbedModel(modelDir) : void 0;
+        if (!model) {
+          process.stderr.write(
+            "codeindex: semantic search unavailable (no embedding model or endpoint) \u2014 returning lexical results; run `codeindex embed pull` or set CODEINDEX_EMBED_ENDPOINT to enable it\n"
+          );
+          lexical();
+        } else {
+          const index = buildEmbeddingIndex(scan2, model);
+          const results = searchSemantic(scan2, flags2.positional, index, { model, limit: flags2.limit, fuzzy: flags2.fuzzy });
+          emit(JSON.stringify(results, null, 2) + "\n", flags2.out);
+        }
+      }
+    } else {
+      const results = searchIndex(scan2, flags2.positional, { limit: flags2.limit, fuzzy: flags2.fuzzy });
+      emit(JSON.stringify(results, null, 2) + "\n", flags2.out);
+    }
+  } else if (cmd === "embed") {
+    const sub = flags2.positional;
+    const modelDir = resolveEmbedModelDir(flags2.repo);
+    if (sub === "status") {
+      const model = modelDir ? loadEmbedModel(modelDir) : void 0;
+      const endpoint = resolveEmbedEndpoint();
+      const mode = endpoint ? "endpoint" : model ? "static" : "none";
+      const status = {
+        embedVersion: EMBED_VERSION,
+        mode,
+        model: model ? { present: true, dir: modelDir, modelId: model.modelId, dim: model.dim, vocabSize: model.vocabSize } : { present: false },
+        endpoint: endpoint ?? null
+      };
+      if (endpoint) status.endpointReachable = await probeEndpoint(endpoint);
+      emit(JSON.stringify(status, null, 2) + "\n", flags2.out);
+    } else if (sub === "serve") {
+      const dockerArgs = ["run", "-d", "-p", "8756:8756", "ghcr.io/maxgfr/codeindex-embed:latest"];
+      const oneLiner = `docker ${dockerArgs.join(" ")}`;
+      if (!have("docker")) {
+        process.stderr.write(
+          "codeindex: docker not found on PATH. Install Docker, then run:\n  " + oneLiner + "\n"
+        );
+        process.exitCode = 1;
+        return;
+      }
+      if (flags2.run) {
+        process.stderr.write(`codeindex: starting embedding server \u2192 ${oneLiner}
+`);
+        const res = sh("docker", dockerArgs);
+        if (res.stdout.trim()) process.stdout.write(res.stdout.trim() + "\n");
+        if (!res.ok) {
+          process.stderr.write(res.stderr || "codeindex: docker run failed\n");
+          process.exitCode = 1;
+          return;
+        }
+        process.stderr.write(
+          'codeindex: server starting on http://localhost:8756 \u2014 then:\n  CODEINDEX_EMBED_ENDPOINT=http://localhost:8756 codeindex search "<query>" --repo . --semantic\n'
+        );
+      } else {
+        process.stdout.write(oneLiner + "\n");
+        process.stderr.write(
+          'codeindex: run the line above to start the embedding server (or `embed serve --run`), then:\n  CODEINDEX_EMBED_ENDPOINT=http://localhost:8756 codeindex search "<query>" --repo . --semantic\n'
+        );
+      }
+    } else if (sub === "build") {
+      if (!flags2.out) throw new Error("embed build needs --out <dir>");
+      if (!modelDir) {
+        process.stderr.write("codeindex: no embedding model present \u2014 run `codeindex embed pull` first (nothing written)\n");
+        process.exitCode = 1;
+        return;
+      }
+      const model = loadEmbedModel(modelDir);
+      mkdirSync2(flags2.out, { recursive: true });
+      const scan2 = scanRepo(flags2.repo, scanOptions(flags2));
+      const index = buildEmbeddingIndex(scan2, model);
+      writeFileSync3(join12(flags2.out, "embeddings.bin"), serializeEmbeddings(index));
+      process.stderr.write(`codeindex: ${index.records.length} embedding records \u2192 ${flags2.out}/embeddings.bin (model ${model.modelId})
+`);
+    } else if (sub === "pull") {
+      const url = resolveEmbedPullUrl();
+      if (!url) {
+        process.stderr.write(
+          "codeindex: no model URL configured. The official static-embedding asset is not published yet.\nSet CODEINDEX_EMBED_URL to a model.json URL (optionally CODEINDEX_EMBED_DIR as the destination), then re-run `codeindex embed pull`.\n"
+        );
+        process.exitCode = 1;
+        return;
+      }
+      const destDir = process.env.CODEINDEX_EMBED_DIR ?? join12(flags2.repo, ".codeindex", "models");
+      mkdirSync2(destDir, { recursive: true });
+      process.stderr.write(`codeindex: fetching model from ${url} \u2192 ${join12(destDir, "model.json")}
+`);
+      const res = await fetch(url);
+      if (!res.ok) {
+        process.stderr.write(`codeindex: pull failed \u2014 HTTP ${res.status} from ${url}
+`);
+        process.exitCode = 1;
+        return;
+      }
+      const body2 = await res.text();
+      try {
+        JSON.parse(body2);
+      } catch {
+        process.stderr.write("codeindex: pull failed \u2014 response is not a valid model.json (expected JSON)\n");
+        process.exitCode = 1;
+        return;
+      }
+      writeFileSync3(join12(destDir, "model.json"), body2);
+      process.stderr.write(`codeindex: model written to ${join12(destDir, "model.json")}
+`);
+    } else {
+      throw new Error("embed needs a subcommand: status | build | pull | serve");
+    }
   } else if (cmd === "rules") {
     if (!flags2.config) throw new Error("rules needs --config <codeindex.rules.json>");
-    const rules = parseRules(JSON.parse(readFileSync5(flags2.config, "utf8")));
+    const rules = parseRules(JSON.parse(readFileSync6(flags2.config, "utf8")));
     const { graph } = buildIndexArtifacts(flags2.repo, scanOptions(flags2));
     const violations = checkRules(graph, rules);
     const errors = violations.filter((v) => v.severity === "error").length;
@@ -10887,14 +11467,18 @@ ${HELP}`);
 }
 export {
   DEFAULT_MAX_FILES,
+  EMBED_VERSION,
   ENGINE_VERSION,
   EXTRACTOR_VERSION,
   MARKDOWN_EXT,
   SCHEMA_VERSION,
   allGrammarKeys,
   applyCentrality,
+  basicTokenize,
   betweennessOf,
   buildCallerIndex,
+  buildEmbeddingIndex,
+  buildEndpointIndex,
   buildGraph,
   buildIndexArtifacts,
   buildModules,
@@ -10917,11 +11501,17 @@ export {
   computeSymbolRefs,
   computeTestMap,
   deleteMemory,
+  deserializeEmbeddings,
   detectCommunities,
   detectWorkspaces,
   diffFiles,
   diffHunks,
+  embedEndpointUrl,
+  embedViaEndpoint,
+  embeddingUnits,
   enclosingSymbol,
+  encode,
+  encodeQueryViaEndpoint,
   ensureGrammars,
   escapeRegExp,
   extToLang,
@@ -10937,10 +11527,13 @@ export {
   grammarKeyForExt,
   grammarReady,
   grepRepo,
+  hasEmbedModel,
   have,
   headCommit,
+  healthzUrl,
   insertAfterSymbol,
   insertBeforeSymbol,
+  intDot,
   isCode,
   isDoc,
   isGitWorktree,
@@ -10951,9 +11544,12 @@ export {
   keywords,
   languageOf,
   listMemories,
+  loadEmbedModel,
   pagerankOf,
   parseGitignore,
   parseRules,
+  probeEndpoint,
+  quantize,
   rankHotspots,
   rankedKeywords,
   readMemory,
@@ -10967,14 +11563,20 @@ export {
   resolveBaseRef,
   resolveCallEdges,
   resolveDocLink,
+  resolveEmbedEndpoint,
+  resolveEmbedModelDir,
+  resolveEmbedPullUrl,
   resolveImport,
   resolveUniqueSymbol,
   riskHotspots,
+  roundHalfToEven,
   rrf,
   runCli,
   runMcpServer,
   scanRepo,
   searchIndex,
+  searchSemantic,
+  serializeEmbeddings,
   sh,
   sha1,
   shortHash,
@@ -10984,10 +11586,12 @@ export {
   symbolsOverview,
   testsForModule,
   tierForPath,
+  tokenize,
   uniqueSymbolDefs,
   untestedModules,
   untrackedFiles,
   walk,
+  wordpiece,
   writeMemory
 };
 // "Copyright" and "@license" are already caught by DIRECTIVE_RE.