@maxgfr/codeindex 2.9.0 → 2.11.0

package/src/embed/encode.ts

@@ -0,0 +1,143 @@
+import { foldText } from "../util.js";
+import type { StaticEmbedModel } from "./model.js";
+
+// The pure-JS deterministic encoder — the reason this whole tier can produce
+// goldens. Every step is chosen to be byte-identical on any platform:
+//   1. tokenize  : foldText (NFKD, strip combining marks) + camelCase/ACRONYM
+//                  split + lowercase + split on non-alphanumeric runs.
+//   2. wordpiece : greedy longest-match against the model vocab, with BERT-style
+//                  "##" continuation pieces; an unsplittable word → the UNK row
+//                  (or nothing when the model has no UNK).
+//   3. gather    : look up each token's row (double precision).
+//   4. mean-pool : sum rows in FIXED token order, divide by token count. IEEE-754
+//                  double add/div is deterministic for a fixed evaluation order.
+//   5. L2-norm   : divide by sqrt(Σ v²) — sqrt is correctly-rounded per IEEE-754,
+//                  so identical across engines.
+//   6. quantize  : multiply the unit vector by 127, round-half-to-EVEN (NOT
+//                  Math.round, which is half-up), clamp to [-127, 127] → int8.
+// The fixed 1/127 scale means ranking is a PURE INTEGER dot product (no per-vector
+// float scale to store or multiply): for a fixed query every corpus vector shares
+// the same scale, so the integer dot Σ q·c is monotonic with the cosine estimate.
+
+const QUANT = 127;
+
+// Split raw text into lowercase, diacritic-folded alphanumeric words. camelCase
+// and ACRONYMWord boundaries become spaces FIRST (so "verifyAuthToken" exposes
+// "verify","auth","token" as separate words to wordpiece), then any
+// non-alphanumeric run splits. Mirrors the boundary rules bm25 `subtokens` uses,
+// keeping the two search tiers tokenizing code identifiers alike.
+export function basicTokenize(text: string): string[] {
+  const spaced = foldText(text)
+    .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+    .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2");
+  const out: string[] = [];
+  for (const part of spaced.toLowerCase().split(/[^a-z0-9]+/)) {
+    if (part) out.push(part);
+  }
+  return out;
+}
+
+// Greedy longest-match wordpiece for ONE word. The first piece is looked up as
+// written; every subsequent piece is looked up with a "##" prefix. If no piece
+// matches at some position the whole word is unresolvable → [unkId] (or [] when
+// the model has no UNK row). Deterministic: longest-match is a fixed scan.
+export function wordpiece(word: string, model: StaticEmbedModel): number[] {
+  if (!word) return [];
+  const ids: number[] = [];
+  let start = 0;
+  const n = word.length;
+  while (start < n) {
+    let end = n;
+    let match = -1;
+    while (end > start) {
+      const piece = start === 0 ? word.slice(start, end) : "##" + word.slice(start, end);
+      const id = model.vocab.get(piece);
+      if (id !== undefined) {
+        match = id;
+        break;
+      }
+      end--;
+    }
+    if (match === -1) return model.unkId >= 0 ? [model.unkId] : []; // whole word is OOV
+    ids.push(match);
+    start = end;
+  }
+  return ids;
+}
+
+// Full token-id sequence for a text: tokenize → wordpiece each word, concatenated
+// in order. The order is load-bearing for determinism, but mean-pooling is
+// order-independent anyway; the fixed order simply makes reasoning trivial.
+export function tokenize(text: string, model: StaticEmbedModel): number[] {
+  const ids: number[] = [];
+  for (const word of basicTokenize(text)) {
+    for (const id of wordpiece(word, model)) ids.push(id);
+  }
+  return ids;
+}
+
+// Round half to EVEN (banker's rounding). JS `Math.round` rounds half toward
+// +∞, which is not symmetric and would bias quantization; half-to-even is the
+// IEEE-754 default and keeps quantize(x) === -quantize(-x).
+export function roundHalfToEven(x: number): number {
+  const f = Math.floor(x);
+  const diff = x - f;
+  if (diff < 0.5) return f;
+  if (diff > 0.5) return f + 1;
+  return f % 2 === 0 ? f : f + 1; // exactly .5 → nearest even
+}
+
+// L2-normalize a float vector and quantize it to int8 at the fixed 1/127 scale
+// (round-half-to-even, clamp to [-127,127]) — the SHARED tail of both tiers. The
+// static encoder feeds its mean-pooled vector here; the endpoint tier feeds each
+// float embedding it receives. Because the quantization is byte-identical, a
+// given float vector maps to the same int8 bytes and the integer-dot ranking is
+// consistent across tiers. A zero-norm input yields an all-zero vector (never
+// NaN — it simply ranks last).
+export function quantize(vec: ArrayLike<number>): Int8Array {
+  const dim = vec.length;
+  const out = new Int8Array(dim);
+  let sumsq = 0;
+  for (let d = 0; d < dim; d++) sumsq += vec[d]! * vec[d]!;
+  const norm = Math.sqrt(sumsq);
+  if (norm === 0) return out; // zero vector — nothing to rank on
+  for (let d = 0; d < dim; d++) {
+    let q = roundHalfToEven((vec[d]! / norm) * QUANT);
+    if (q > QUANT) q = QUANT;
+    else if (q < -QUANT) q = -QUANT;
+    out[d] = q;
+  }
+  return out;
+}
+
+// Encode a text to an int8 unit-ish vector. Empty / all-OOV / zero-norm inputs
+// yield an all-zero vector (dot-products against it are 0 — it simply ranks
+// last), never NaN. Pure and deterministic: same model + text → same bytes.
+export function encode(model: StaticEmbedModel, text: string): Int8Array {
+  const { dim, weights } = model;
+  const ids = tokenize(text, model);
+  if (ids.length === 0) return new Int8Array(dim);
+
+  // mean-pool in double precision, fixed order.
+  const pooled = new Float64Array(dim);
+  for (const id of ids) {
+    const base = id * dim;
+    for (let d = 0; d < dim; d++) pooled[d]! += weights[base + d]!;
+  }
+  const inv = 1 / ids.length;
+  for (let d = 0; d < dim; d++) pooled[d]! *= inv;
+
+  // L2-normalize + int8-quantize (the shared tail, identical to the endpoint tier).
+  return quantize(pooled);
+}
+
+// Integer dot product of two int8 vectors (the ranking primitive). Widened to a
+// JS number (double), but every operand and partial sum is an exact integer
+// (max |Σ| for dim 512 ≈ 512·127² ≈ 8.3M, far inside 2⁵³) — so this is an EXACT
+// integer computation, deterministic everywhere.
+export function intDot(a: Int8Array, b: Int8Array): number {
+  const n = Math.min(a.length, b.length);
+  let dot = 0;
+  for (let i = 0; i < n; i++) dot += a[i]! * b[i]!;
+  return dot;
+}

package/src/embed/endpoint.ts

@@ -0,0 +1,144 @@
+// HTTP embedding endpoint client — the "rich" tier (v2.11.0). The engine becomes
+// a CONSUMER of a local containerized embedding server (e.g.
+// ghcr.io/maxgfr/codeindex-embed): it POSTs corpus/query texts, receives float
+// vectors, and runs them through the EXACT SAME L2-normalize + int8-quantize
+// pipeline as the static tier (encode.ts `quantize`), so ranking stays a pure
+// integer dot product. Node >=18 has global fetch, so this stays zero-dependency.
+//
+// The contract: POST { texts: string[] } → { vectors: number[][] } to
+// `${base}/embed`, plus GET `${base}/healthz`. `base` is CODEINDEX_EMBED_ENDPOINT
+// (or opts.url). The library NEVER orchestrates docker; running the server is a
+// user/CI concern (`codeindex embed serve` only prints/runs the docker command).
+//
+// DETERMINISM: this tier is deterministic PER IMAGE DIGEST (the model is baked
+// into a pinned image), NOT byte-golden — endpoint float vectors are provider-
+// dependent. That is why endpoint indexes are built at search time and never
+// serialized to embeddings.bin (which carries a determinism guarantee).
+
+import type { RepoScan } from "../scan.js";
+import { quantize } from "./encode.js";
+import { EMBED_VERSION } from "./model.js";
+import { embeddingUnits, type EmbeddingIndex, type EmbeddingRecord } from "./index.js";
+
+export interface EmbedEndpointOptions {
+  // Endpoint BASE url (e.g. http://localhost:8756). Falls back to
+  // CODEINDEX_EMBED_ENDPOINT when omitted.
+  url?: string;
+  // Abort a request after this many ms (default from CODEINDEX_EMBED_TIMEOUT_MS,
+  // else 30_000).
+  timeoutMs?: number;
+  // Extra request headers (e.g. an auth token for a private endpoint).
+  headers?: Record<string, string>;
+  // Max texts per POST /embed request when embedding a corpus (default 64).
+  batchSize?: number;
+}
+
+// Resolve the configured endpoint BASE url, or undefined when neither opts.url
+// nor CODEINDEX_EMBED_ENDPOINT is set — the "endpoint tier not requested" signal.
+export function resolveEmbedEndpoint(opts: EmbedEndpointOptions = {}): string | undefined {
+  const url = opts.url ?? process.env.CODEINDEX_EMBED_ENDPOINT;
+  return url && url.trim() ? url.trim() : undefined;
+}
+
+function stripTrailingSlash(url: string): string {
+  return url.replace(/\/+$/, "");
+}
+
+// The POST /embed URL for a base. Idempotent when the base already ends in
+// /embed (so pointing the env straight at the embed route also works).
+export function embedEndpointUrl(base: string): string {
+  const b = stripTrailingSlash(base);
+  return b.endsWith("/embed") ? b : b + "/embed";
+}
+
+// The GET /healthz URL for a base (tolerates a base that already ends in /embed).
+export function healthzUrl(base: string): string {
+  return stripTrailingSlash(base).replace(/\/embed$/, "") + "/healthz";
+}
+
+function resolveTimeout(opts: EmbedEndpointOptions): number {
+  if (typeof opts.timeoutMs === "number") return opts.timeoutMs;
+  const env = Number(process.env.CODEINDEX_EMBED_TIMEOUT_MS);
+  return Number.isFinite(env) && env > 0 ? env : 30_000;
+}
+
+// Request float embeddings for a batch of texts from the endpoint's /embed
+// route. Throws a clear error when no endpoint is configured, the request times
+// out / fails, or the response is malformed; the caller decides whether to
+// degrade. NOTE: endpoint vectors are float and are NOT byte-deterministic.
+export async function embedViaEndpoint(texts: string[], opts: EmbedEndpointOptions = {}): Promise<number[][]> {
+  const base = resolveEmbedEndpoint(opts);
+  if (!base) throw new Error("no embedding endpoint configured (set CODEINDEX_EMBED_ENDPOINT or pass opts.url)");
+  const url = embedEndpointUrl(base);
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), resolveTimeout(opts));
+  try {
+    const res = await fetch(url, {
+      method: "POST",
+      headers: { "content-type": "application/json", ...(opts.headers ?? {}) },
+      body: JSON.stringify({ texts }),
+      signal: controller.signal,
+    });
+    if (!res.ok) throw new Error(`embedding endpoint ${url} returned HTTP ${res.status}`);
+    const data = (await res.json()) as { vectors?: unknown };
+    const vectors = data.vectors;
+    if (!Array.isArray(vectors) || !vectors.every((v) => Array.isArray(v) && v.every((x) => typeof x === "number"))) {
+      throw new Error(`embedding endpoint ${url} returned a malformed { vectors } payload`);
+    }
+    return vectors as number[][];
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+// Is the endpoint reachable? GETs /healthz and returns true only on a 2xx. Never
+// throws — a down/timed-out endpoint returns false (used by `embed status`).
+export async function probeEndpoint(base: string, opts: EmbedEndpointOptions = {}): Promise<boolean> {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), resolveTimeout(opts));
+  try {
+    const res = await fetch(healthzUrl(base), { signal: controller.signal, headers: opts.headers });
+    return res.ok;
+  } catch {
+    return false;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+// Encode ONE query string via the endpoint → int8 vector, run through the shared
+// quantize pipeline (so it lives in the same int8 space as an endpoint-built
+// corpus). Throws on an unreachable/malformed endpoint — the caller degrades.
+export async function encodeQueryViaEndpoint(query: string, opts: EmbedEndpointOptions = {}): Promise<Int8Array> {
+  const [vec] = await embedViaEndpoint([query], opts);
+  if (!vec) throw new Error("embedding endpoint returned no vector for the query");
+  return quantize(vec);
+}
+
+// Build the int8 corpus index by embedding every corpus unit (the SAME units the
+// static tier uses) via the endpoint, batched, then quantizing each float vector
+// through the shared L2+int8 pipeline. The result plugs straight into
+// searchSemantic's integer-dot ranking. Built at search time (never serialized).
+export async function buildEndpointIndex(scan: RepoScan, opts: EmbedEndpointOptions = {}): Promise<EmbeddingIndex> {
+  const units = embeddingUnits(scan);
+  const batchSize = opts.batchSize && opts.batchSize > 0 ? opts.batchSize : 64;
+  const records: EmbeddingRecord[] = [];
+  let dim = 0;
+  for (let i = 0; i < units.length; i += batchSize) {
+    const batch = units.slice(i, i + batchSize);
+    const vectors = await embedViaEndpoint(batch.map((u) => u.text), opts);
+    if (vectors.length !== batch.length) {
+      throw new Error(`embedding endpoint returned ${vectors.length} vectors for ${batch.length} texts`);
+    }
+    for (let j = 0; j < batch.length; j++) {
+      const u = batch[j]!;
+      const vec = quantize(vectors[j]!);
+      if (vec.length > dim) dim = vec.length;
+      const rec: EmbeddingRecord = { file: u.file, vec };
+      if (u.symbol !== undefined) rec.symbol = u.symbol;
+      if (u.line !== undefined) rec.line = u.line;
+      records.push(rec);
+    }
+  }
+  return { embedVersion: EMBED_VERSION, modelId: "endpoint", dim, records };
+}

package/src/embed/index.ts

@@ -0,0 +1,141 @@
+import type { RepoScan } from "../scan.js";
+import { encode } from "./encode.js";
+import { EMBED_VERSION, type StaticEmbedModel } from "./model.js";
+
+// The corpus embedding artifact. Granularity is PER-SYMBOL (each symbol gets its
+// own record, so search can surface the exact symbol that matched), with a
+// PER-FILE fallback record for files that declare no symbols (docs, config) so
+// every file with content is still represented. Deterministic: records follow
+// scan order (files sorted by rel) then declaration order within a file.
+export interface EmbeddingRecord {
+  file: string; // repo-relative path
+  symbol?: string; // the declared symbol this vector represents (absent for a file-level record)
+  line?: number; // 1-based symbol line (absent for a file-level record)
+  vec: Int8Array; // length === dim
+}
+
+export interface EmbeddingIndex {
+  embedVersion: number;
+  modelId: string;
+  dim: number;
+  records: EmbeddingRecord[];
+}
+
+// The text encoded for one symbol: its name (camelCase-split by the tokenizer),
+// its signature, the owning file's one-line summary, and the file's path segments
+// — the same signal bm25 indexes, but pooled into a single vector. Newline-joined
+// so the tokenizer's non-alphanumeric split cleanly separates the parts.
+function symbolText(rel: string, name: string, signature: string | undefined, summary: string | undefined): string {
+  return [name, signature ?? "", summary ?? "", rel.replace(/\//g, " ")].join("\n");
+}
+
+// A file-level record's text (symbol-less files): title, summary, headings, path.
+function fileText(rel: string, title: string | undefined, summary: string | undefined, headings: string[]): string {
+  return [title ?? "", summary ?? "", ...headings, rel.replace(/\//g, " ")].join("\n");
+}
+
+// One corpus item to embed: its target (file, optional symbol/line) plus the
+// exact text to encode. This is the SINGLE definition of "what the corpus is",
+// shared by the static tier (buildEmbeddingIndex) and the endpoint tier
+// (buildEndpointIndex) so both embed byte-identical texts in the same order and
+// differ ONLY in the encoder. Deterministic: scan order (files by rel) then
+// declaration order within a file, deduped by symbol name (bm25 parity).
+export interface EmbeddingUnit {
+  file: string;
+  symbol?: string;
+  line?: number;
+  text: string;
+}
+
+export function embeddingUnits(scan: RepoScan): EmbeddingUnit[] {
+  const units: EmbeddingUnit[] = [];
+  for (const f of scan.files) {
+    const seen = new Set<string>();
+    let hadSymbol = false;
+    for (const s of f.symbols) {
+      if (seen.has(s.name)) continue; // dedupe by name within a file (bm25 parity)
+      seen.add(s.name);
+      hadSymbol = true;
+      units.push({ file: f.rel, symbol: s.name, line: s.line, text: symbolText(f.rel, s.name, s.signature, f.summary) });
+    }
+    if (!hadSymbol) {
+      const text = fileText(f.rel, f.title, f.summary, f.headings);
+      if (text.replace(/\s+/g, "")) units.push({ file: f.rel, text });
+    }
+  }
+  return units;
+}
+
+// Build the corpus embedding index from a scan + a loaded model. Pure and
+// deterministic (encode is byte-stable, scan order is fixed) → two builds of an
+// unchanged repo produce byte-identical serialized bytes.
+export function buildEmbeddingIndex(scan: RepoScan, model: StaticEmbedModel): EmbeddingIndex {
+  const records: EmbeddingRecord[] = embeddingUnits(scan).map((u) => {
+    const rec: EmbeddingRecord = { file: u.file, vec: encode(model, u.text) };
+    if (u.symbol !== undefined) rec.symbol = u.symbol;
+    if (u.line !== undefined) rec.line = u.line;
+    return rec;
+  });
+  return { embedVersion: EMBED_VERSION, modelId: model.modelId, dim: model.dim, records };
+}
+
+const MAGIC = "CIE1"; // codeindex embeddings, format 1
+
+// Serialize to embeddings.bin: a fixed ASCII magic, a uint32-LE header length, a
+// UTF-8 JSON header (per-record metadata in build order — carries NO absolute
+// path or timestamp), then the packed int8 body (count × dim signed bytes). The
+// header JSON key order is fixed by construction, and every body byte is written
+// explicitly, so the bytes are fully deterministic. Returns a Uint8Array (a
+// Buffer at runtime) so the public type surface stays free of the Node `Buffer`
+// global — a consumer type-checking without @types/node still resolves it.
+export function serializeEmbeddings(index: EmbeddingIndex): Uint8Array {
+  const header = JSON.stringify({
+    embedVersion: index.embedVersion,
+    modelId: index.modelId,
+    dim: index.dim,
+    count: index.records.length,
+    records: index.records.map((r) => ({ file: r.file, symbol: r.symbol ?? "", line: r.line ?? 0 })),
+  });
+  const headerBuf = Buffer.from(header, "utf8");
+  const body = Buffer.alloc(index.records.length * index.dim);
+  let off = 0;
+  for (const r of index.records) {
+    for (let d = 0; d < index.dim; d++) body.writeInt8(r.vec[d] ?? 0, off++);
+  }
+  const out = Buffer.alloc(8 + headerBuf.length + body.length);
+  out.write(MAGIC, 0, "ascii");
+  out.writeUInt32LE(headerBuf.length, 4);
+  headerBuf.copy(out, 8);
+  body.copy(out, 8 + headerBuf.length);
+  return out;
+}
+
+// Inverse of serializeEmbeddings. Accepts any Uint8Array (e.g. a fs.readFileSync
+// Buffer) and wraps it as a Buffer VIEW (no copy) for the numeric reads. Throws
+// on a bad magic (a corrupt or foreign file) so a caller fails loudly rather
+// than misreading arbitrary bytes.
+export function deserializeEmbeddings(bytes: Uint8Array): EmbeddingIndex {
+  const buf = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  if (buf.length < 8 || buf.toString("ascii", 0, 4) !== MAGIC) {
+    throw new Error("embeddings.bin: bad magic (not a codeindex embeddings artifact)");
+  }
+  const headerLen = buf.readUInt32LE(4);
+  const header = JSON.parse(buf.toString("utf8", 8, 8 + headerLen)) as {
+    embedVersion: number;
+    modelId: string;
+    dim: number;
+    count: number;
+    records: { file: string; symbol: string; line: number }[];
+  };
+  const bodyOff = 8 + headerLen;
+  const { dim } = header;
+  const records: EmbeddingRecord[] = header.records.map((m, i) => {
+    const vec = new Int8Array(dim);
+    for (let d = 0; d < dim; d++) vec[d] = buf.readInt8(bodyOff + i * dim + d);
+    const rec: EmbeddingRecord = { file: m.file, vec };
+    if (m.symbol) rec.symbol = m.symbol;
+    if (m.line) rec.line = m.line;
+    return rec;
+  });
+  return { embedVersion: header.embedVersion, modelId: header.modelId, dim, records };
+}

package/src/embed/model.ts

@@ -0,0 +1,117 @@
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+
+// The deterministic static-embedding tier (v2.10.0). A *static* embedding model
+// is a plain lookup table: token → row vector. Inference is tokenize → gather
+// rows → mean-pool → L2-normalize → int8-quantize (see encode.ts). There is NO
+// neural forward pass, no wasm, no float matmul — so the encode is pure-JS and
+// byte-identical across platforms, which is the whole reason this tier exists.
+//
+// Activation is OPT-IN by asset presence, exactly like the tree-sitter grammar
+// tier (src/ast/loader.ts): no model on disk → the engine silently stays
+// lexical. Models are NEVER shipped in the npm tarball; the asset is fetched by
+// `codeindex embed pull` into CODEINDEX_EMBED_DIR (or <repo>/.codeindex/models/).
+
+// Bumped independently of SCHEMA_VERSION / EXTRACTOR_VERSION whenever the
+// embeddings.bin artifact shape or the encode algorithm changes, so a consumer
+// rejects an embeddings.bin written by an incompatible engine (or a different
+// model) instead of misranking with it. SCHEMA_VERSION is deliberately left
+// UNTOUCHED — embeddings are a purely additive sidecar with zero impact on the
+// graph.json / symbols.json consumers.
+export const EMBED_VERSION = 1;
+
+// The default model-name a `pull` writes and a `status` reports. The real asset
+// URL is intentionally NOT hard-coded yet (see resolveEmbedPullUrl): the asset
+// is unpublished, so `pull` fails cleanly asking for CODEINDEX_EMBED_URL rather
+// than fetching a phantom.
+export const DEFAULT_EMBED_DIRNAME = "models";
+
+// A loaded static-embedding model. `vocab` maps a wordpiece token to its row
+// index; `weights` is the flat row-major (vocabSize × dim) matrix as IEEE-754
+// doubles (float rows widen to double exactly, so mean-pool stays deterministic).
+export interface StaticEmbedModel {
+  modelId: string;
+  dim: number;
+  unk: string; // the OOV token (e.g. "[UNK]"); may be absent from vocab
+  unkId: number; // row index of `unk`, or -1 when the model has no UNK row
+  vocabSize: number;
+  vocab: Map<string, number>;
+  weights: Float64Array; // length === vocabSize * dim, row-major
+}
+
+// The on-disk model.json header shape. `weights` is an array of `vocabSize`
+// rows, each of length `dim`. Kept human-authorable (plain JSON numbers) so a
+// tiny fixture model is committable and diffable; a real `pull`ed model uses the
+// identical shape at scale.
+interface ModelFile {
+  modelId: string;
+  dim: number;
+  unk?: string;
+  vocab: string[];
+  weights: number[][];
+}
+
+// Where the model asset lives. CODEINDEX_EMBED_DIR wins outright (explicit
+// override, mirrors CODEINDEX_GRAMMAR_DIR); otherwise <repo>/.codeindex/models/
+// then <cwd>/.codeindex/models/. Returns undefined when no model.json is found
+// anywhere — the silent-degradation signal. When CODEINDEX_EMBED_DIR is set but
+// empty of a model, that is still "no model" (undefined), never an error.
+export function resolveEmbedModelDir(repo?: string): string | undefined {
+  const env = process.env.CODEINDEX_EMBED_DIR;
+  const candidates: string[] = [];
+  if (env) candidates.push(env);
+  if (repo) candidates.push(join(repo, ".codeindex", DEFAULT_EMBED_DIRNAME));
+  candidates.push(join(process.cwd(), ".codeindex", DEFAULT_EMBED_DIRNAME));
+  for (const c of candidates) {
+    if (existsSync(join(c, "model.json"))) return c;
+  }
+  return undefined;
+}
+
+// True when an activatable model is present (used by `embed status` and the MCP
+// gate before attempting a semantic search).
+export function hasEmbedModel(repo?: string): boolean {
+  return resolveEmbedModelDir(repo) !== undefined;
+}
+
+// Load and validate a static model from a directory containing model.json.
+// Throws on a malformed file (bad dim, ragged weights) so a corrupt asset fails
+// loudly at load rather than silently misranking. Returns undefined only when
+// the directory has no model.json (the not-present case).
+export function loadEmbedModel(dir?: string): StaticEmbedModel | undefined {
+  if (!dir) return undefined;
+  const path = join(dir, "model.json");
+  if (!existsSync(path)) return undefined;
+  const raw = JSON.parse(readFileSync(path, "utf8")) as ModelFile;
+  const { modelId, dim, vocab, weights } = raw;
+  if (typeof modelId !== "string" || !modelId) throw new Error(`embed model: missing modelId in ${path}`);
+  if (!Number.isInteger(dim) || dim <= 0) throw new Error(`embed model: bad dim ${dim} in ${path}`);
+  if (!Array.isArray(vocab) || !Array.isArray(weights) || vocab.length !== weights.length) {
+    throw new Error(`embed model: vocab/weights length mismatch in ${path}`);
+  }
+  const vocabSize = vocab.length;
+  const flat = new Float64Array(vocabSize * dim);
+  const vmap = new Map<string, number>();
+  for (let i = 0; i < vocabSize; i++) {
+    const tok = vocab[i]!;
+    if (typeof tok !== "string") throw new Error(`embed model: non-string vocab entry at ${i}`);
+    if (!vmap.has(tok)) vmap.set(tok, i); // first occurrence wins (deterministic)
+    const row = weights[i]!;
+    if (!Array.isArray(row) || row.length !== dim) {
+      throw new Error(`embed model: row ${i} has length ${row?.length}, expected ${dim}`);
+    }
+    for (let d = 0; d < dim; d++) flat[i * dim + d] = Number(row[d]);
+  }
+  const unk = typeof raw.unk === "string" ? raw.unk : "[UNK]";
+  const unkId = vmap.has(unk) ? vmap.get(unk)! : -1;
+  return { modelId, dim, unk, unkId, vocabSize, vocab: vmap, weights: flat };
+}
+
+// Resolve the URL `embed pull` fetches from. The official asset is not published
+// yet, so there is no built-in default: the user must point CODEINDEX_EMBED_URL
+// at a model.json (or a directory serving one). Returns undefined when unset —
+// the caller turns that into a clean, actionable failure, never a crash.
+export function resolveEmbedPullUrl(): string | undefined {
+  const url = process.env.CODEINDEX_EMBED_URL;
+  return url && url.trim() ? url.trim() : undefined;
+}

package/src/embed/search.ts

@@ -0,0 +1,96 @@
+import type { RepoScan } from "../scan.js";
+import { rrf } from "../util.js";
+import { byStr } from "../sort.js";
+import { searchIndex, type SearchOptions, type SearchResult } from "../bm25.js";
+import { encode, intDot } from "./encode.js";
+import type { EmbeddingIndex } from "./index.js";
+import type { StaticEmbedModel } from "./model.js";
+
+const DEFAULT_LIMIT = 20;
+const RRF_K = 60; // reuse the engine-wide RRF damping constant
+
+export interface SemanticSearchOptions extends SearchOptions {
+  // The loaded static model, needed to encode the QUERY into the same int8 space
+  // as the corpus. Absent → the search degrades to pure lexical (no throw).
+  model?: StaticEmbedModel;
+  // A pre-encoded int8 query vector — the endpoint tier's escape hatch: it has
+  // no local model but has already quantized the endpoint's float query vector
+  // through the SAME pipeline as the corpus. Wins over `model` when both are set.
+  queryVec?: Int8Array;
+  // RRF damping (default 60), exposed for parity with the shared rrf helper.
+  rrfK?: number;
+}
+
+// A fused result. Extends the lexical SearchResult additively: `semanticSymbol`
+// is the corpus symbol whose embedding was closest to the query for this file
+// (absent when the file was contributed only by the lexical side, or when the
+// search degraded to lexical).
+export interface SemanticSearchResult extends SearchResult {
+  semanticSymbol?: string;
+}
+
+// RRF-fused semantic + lexical search. The two rankings live on incomparable
+// scales (BM25 score vs integer dot product), so we fuse by RANK via the shared
+// `rrf` helper (k=60) rather than any linear score blend. Deterministic end to
+// end: lexical is deterministic, the semantic ranking sorts by exact integer dot
+// (ties broken by path), and the fused list sorts by RRF score (ties by path).
+//
+// DEGRADATION: with no model, no index, or an empty index, this returns the pure
+// lexical ranking unchanged (as SemanticSearchResult[] with no `semanticSymbol`)
+// — the caller stays on exit 0 and simply prints a note.
+export function searchSemantic(
+  scan: RepoScan,
+  query: string,
+  index: EmbeddingIndex | undefined,
+  opts: SemanticSearchOptions = {},
+): SemanticSearchResult[] {
+  const limit = opts.limit ?? DEFAULT_LIMIT;
+  // Pull a deeper lexical list than the final limit so RRF has enough overlap to
+  // fuse meaningfully, then trim after fusion.
+  const lexical = searchIndex(scan, query, { limit: Math.max(limit, 50), fuzzy: opts.fuzzy });
+
+  // Resolve the query vector: an already-encoded one (endpoint tier) wins;
+  // otherwise encode via the local model. No vector, no index, or an empty index
+  // → pure-lexical degradation.
+  const q = opts.queryVec ?? (opts.model ? encode(opts.model, query) : undefined);
+  if (!q || !index || index.records.length === 0) {
+    return lexical.slice(0, limit); // pure-lexical degradation
+  }
+  // Best (highest integer dot) record per file, remembering its symbol.
+  const bestByFile = new Map<string, { score: number; symbol?: string }>();
+  for (const r of index.records) {
+    const dot = intDot(q, r.vec);
+    const prev = bestByFile.get(r.file);
+    if (!prev || dot > prev.score) bestByFile.set(r.file, { score: dot, symbol: r.symbol });
+  }
+
+  // Semantic ranked file list: positive similarity only, best-first, ties by path.
+  const semList = [...bestByFile.entries()]
+    .filter(([, v]) => v.score > 0)
+    .sort((a, b) => b[1].score - a[1].score || byStr(a[0], b[0]))
+    .map(([file]) => file);
+  const lexList = lexical.map((r) => r.file);
+
+  // Fuse the two ranked lists (identity keyOf — items are already file paths).
+  const fused = rrf<string>([lexList, semList], (f) => f, opts.rrfK ?? RRF_K);
+
+  const lexByFile = new Map(lexical.map((r) => [r.file, r] as const));
+  const results: SemanticSearchResult[] = [...fused.entries()]
+    .sort((a, b) => b[1] - a[1] || byStr(a[0], b[0]))
+    .map(([file, score]) => {
+      const lex = lexByFile.get(file);
+      const res: SemanticSearchResult = {
+        file,
+        score: Number(score.toFixed(4)),
+        matchedTerms: lex?.matchedTerms ?? [],
+        topSymbols: lex?.topSymbols ?? [],
+      };
+      const sem = bestByFile.get(file);
+      if (sem?.symbol) res.semanticSymbol = sem.symbol;
+      if (lex?.fuzzyTerms) res.fuzzyTerms = lex.fuzzyTerms;
+      return res;
+    });
+  return results.slice(0, limit);
+}
+
+export { DEFAULT_LIMIT as SEMANTIC_DEFAULT_LIMIT };