@maxgfr/codeindex 2.8.1 → 2.10.0

package/src/bm25.ts

@@ -8,6 +8,14 @@
 // the query reuses util keywords, so query and haystack always tokenize alike.
 // Deterministic: files are scored in scan order (sorted by rel), scores are
 // fixed to 4 decimal places, and ties break by path.
+//
+// Trigram fuzzy fallback (v2.9.0): a query term that matches NOTHING in the
+// corpus (document frequency == 0 — checked STRICTLY, so any term that
+// already matches anywhere is never touched) is expanded against the corpus
+// vocabulary via character-trigram Dice similarity (threshold 0.6, top-3
+// candidates, deterministic tie-break). This keeps every currently-matching
+// query byte-identical: the expansion only ever engages on terms that would
+// otherwise contribute nothing.
 import type { RepoScan } from "./scan.js";
 import { foldText, keywords } from "./util.js";
 import { byStr } from "./sort.js";
@@ -16,10 +24,17 @@ const K1 = 1.2;
 const B = 0.75;
 const DEFAULT_LIMIT = 20;
 const TOP_SYMBOLS = 5;
+const FUZZY_DICE_THRESHOLD = 0.6;
+const FUZZY_CAP = 3;
 
 export interface SearchOptions {
   // Maximum results returned (default 20).
   limit?: number;
+  // Trigram fuzzy fallback for query terms with zero document frequency
+  // (default true). Safe as an always-on default: the df==0 gate means it
+  // only ever engages on terms that would otherwise match nothing, so a
+  // query where every term already hits is completely unaffected.
+  fuzzy?: boolean;
 }
 
 export interface SearchResult {
@@ -27,6 +42,10 @@ export interface SearchResult {
   score: number; // BM25 score, fixed to 4 decimal places
   matchedTerms: string[]; // query tokens present in this file's document, sorted
   topSymbols: string[]; // symbols whose name matches the most query tokens (cap 5)
+  // Query terms (df==0) resolved via trigram fuzzy fallback that contributed
+  // to this result, sorted. Present only when >=1 term used the fallback —
+  // purely additive, never present for an all-exact-match result.
+  fuzzyTerms?: string[];
 }
 
 // Split an identifier/phrase into lowercase, diacritic-folded subtokens:
@@ -87,6 +106,39 @@ function buildDocs(scan: RepoScan): Doc[] {
   return docs;
 }
 
+// Character trigrams of a token, padded with two boundary sentinels on each
+// side (pg_trgm-style: "^^t…m$$") so short prefix/suffix runs still produce
+// shared grams. Deduplicated into a Set — a repeated gram doesn't inflate
+// Dice similarity.
+export function charTrigrams(term: string): Set<string> {
+  const padded = `^^${term}$$`;
+  const grams = new Set<string>();
+  for (let i = 0; i + 3 <= padded.length; i++) grams.add(padded.slice(i, i + 3));
+  return grams;
+}
+
+// Dice coefficient between two trigram sets: 2|A∩B| / (|A|+|B|). 0 when
+// either side is empty (no divide-by-zero).
+export function diceCoefficient(a: ReadonlySet<string>, b: ReadonlySet<string>): number {
+  if (!a.size || !b.size) return 0;
+  let inter = 0;
+  for (const g of a) if (b.has(g)) inter++;
+  return (2 * inter) / (a.size + b.size);
+}
+
+// Trigram index of the corpus vocabulary: every distinct doc token mapped to
+// its trigram set. Built LAZILY by searchIndex — only when >=1 query term has
+// df==0 — so a fully-matched query never pays this cost.
+function buildTrigramIndex(docs: Doc[]): Map<string, Set<string>> {
+  const index = new Map<string, Set<string>>();
+  for (const d of docs) {
+    for (const term of d.tf.keys()) {
+      if (!index.has(term)) index.set(term, charTrigrams(term));
+    }
+  }
+  return index;
+}
+
 // Rank the scanned files against a natural-language (or identifier) query.
 // Pure and deterministic: same scan + query → the same results, byte-for-byte.
 export function searchIndex(scan: RepoScan, query: string, opts: SearchOptions = {}): SearchResult[] {
@@ -118,36 +170,93 @@ export function searchIndex(scan: RepoScan, query: string, opts: SearchOptions =
     df.set(t, count);
   }
 
+  // Fuzzy fallback: STRICT df==0 gate — a term that matches anywhere, even
+  // once, is never expanded. The trigram index of the corpus vocabulary is
+  // built lazily, only when at least one term needs it, so a fully-matched
+  // query (the common case) pays zero extra cost and stays byte-identical.
+  const fuzzyEnabled = opts.fuzzy ?? true;
+  const fuzzyCandidates = new Map<string, { term: string; dice: number }[]>();
+  if (fuzzyEnabled) {
+    const unmatched = terms.filter((t) => df.get(t) === 0);
+    if (unmatched.length) {
+      const trigramIndex = buildTrigramIndex(docs);
+      for (const t of unmatched) {
+        const grams = charTrigrams(t);
+        const candidates: { term: string; dice: number }[] = [];
+        for (const [vocabTerm, vocabGrams] of trigramIndex) {
+          const dice = diceCoefficient(grams, vocabGrams);
+          if (dice >= FUZZY_DICE_THRESHOLD) candidates.push({ term: vocabTerm, dice });
+        }
+        // Deterministic: similarity desc, then vocab term asc.
+        candidates.sort((a, b) => b.dice - a.dice || byStr(a.term, b.term));
+        fuzzyCandidates.set(t, candidates.slice(0, FUZZY_CAP));
+      }
+    }
+  }
+  // df cache for expanded vocab terms (distinct from query-term df above).
+  const vocabDf = new Map<string, number>();
+  const dfOfVocabTerm = (term: string): number => {
+    const known = df.get(term) ?? vocabDf.get(term);
+    if (known !== undefined) return known;
+    let count = 0;
+    for (const d of docs) if (d.tf.has(term)) count++;
+    vocabDf.set(term, count);
+    return count;
+  };
+
   const results: SearchResult[] = [];
   for (const d of docs) {
     let score = 0;
     const matched: string[] = [];
+    const symbolTerms = new Set<string>(); // matched ∪ fuzzy-expanded vocab terms, for topSymbols ranking
+    const fuzzyHit = new Set<string>(); // original query terms resolved via fuzzy fallback, for this doc
     for (const t of terms) {
       const tf = d.tf.get(t);
-      if (!tf) continue;
-      matched.push(t);
-      const idf = Math.log(1 + (n - df.get(t)! + 0.5) / (df.get(t)! + 0.5));
-      score += (idf * (tf * (K1 + 1))) / (tf + K1 * (1 - B + (B * d.len) / avgLen));
+      if (tf) {
+        matched.push(t);
+        symbolTerms.add(t);
+        const idf = Math.log(1 + (n - df.get(t)! + 0.5) / (df.get(t)! + 0.5));
+        score += (idf * (tf * (K1 + 1))) / (tf + K1 * (1 - B + (B * d.len) / avgLen));
+        continue;
+      }
+      // Only ever reached for a term with df==0 (or absent from THIS doc but
+      // matched elsewhere — fuzzyCandidates has no entry for those, so the
+      // lookup below is a no-op and behavior is identical to before v2.9.0).
+      const candidates = fuzzyCandidates.get(t);
+      if (!candidates) continue;
+      for (const cand of candidates) {
+        const ctf = d.tf.get(cand.term);
+        if (!ctf) continue;
+        const cdf = dfOfVocabTerm(cand.term);
+        const idf = Math.log(1 + (n - cdf + 0.5) / (cdf + 0.5));
+        const contribution = (idf * (ctf * (K1 + 1))) / (ctf + K1 * (1 - B + (B * d.len) / avgLen));
+        score += contribution * cand.dice; // near-miss always scores below an exact hit (dice < 1)
+        symbolTerms.add(cand.term);
+        fuzzyHit.add(t);
+      }
     }
-    if (!matched.length) continue;
+    if (!matched.length && !fuzzyHit.size) continue;
 
-    // Symbols ranked by how many query tokens their name carries, then by name.
+    // Symbols ranked by how many query tokens (exact or fuzzy-expanded) their
+    // name carries, then by name.
     const scored = d.symbols
       .map((name) => {
         const toks = new Set(subtokens(name));
         let hits = 0;
-        for (const t of matched) if (toks.has(t)) hits++;
+        for (const t of symbolTerms) if (toks.has(t)) hits++;
         return { name, hits };
       })
       .filter((s) => s.hits > 0)
       .sort((a, b) => b.hits - a.hits || byStr(a.name, b.name));
 
-    results.push({
+    const result: SearchResult = {
       file: d.file,
       score: Number(score.toFixed(4)),
       matchedTerms: matched.sort(byStr),
       topSymbols: scored.slice(0, TOP_SYMBOLS).map((s) => s.name),
-    });
+    };
+    if (fuzzyHit.size) result.fuzzyTerms = [...fuzzyHit].sort(byStr);
+    results.push(result);
   }
 
   // Rounded score first (so 4-dp ties resolve stably), then path.

package/src/embed/encode.ts

@@ -0,0 +1,133 @@
+import { foldText } from "../util.js";
+import type { StaticEmbedModel } from "./model.js";
+
+// The pure-JS deterministic encoder — the reason this whole tier can produce
+// goldens. Every step is chosen to be byte-identical on any platform:
+//   1. tokenize  : foldText (NFKD, strip combining marks) + camelCase/ACRONYM
+//                  split + lowercase + split on non-alphanumeric runs.
+//   2. wordpiece : greedy longest-match against the model vocab, with BERT-style
+//                  "##" continuation pieces; an unsplittable word → the UNK row
+//                  (or nothing when the model has no UNK).
+//   3. gather    : look up each token's row (double precision).
+//   4. mean-pool : sum rows in FIXED token order, divide by token count. IEEE-754
+//                  double add/div is deterministic for a fixed evaluation order.
+//   5. L2-norm   : divide by sqrt(Σ v²) — sqrt is correctly-rounded per IEEE-754,
+//                  so identical across engines.
+//   6. quantize  : multiply the unit vector by 127, round-half-to-EVEN (NOT
+//                  Math.round, which is half-up), clamp to [-127, 127] → int8.
+// The fixed 1/127 scale means ranking is a PURE INTEGER dot product (no per-vector
+// float scale to store or multiply): for a fixed query every corpus vector shares
+// the same scale, so the integer dot Σ q·c is monotonic with the cosine estimate.
+
+const QUANT = 127;
+
+// Split raw text into lowercase, diacritic-folded alphanumeric words. camelCase
+// and ACRONYMWord boundaries become spaces FIRST (so "verifyAuthToken" exposes
+// "verify","auth","token" as separate words to wordpiece), then any
+// non-alphanumeric run splits. Mirrors the boundary rules bm25 `subtokens` uses,
+// keeping the two search tiers tokenizing code identifiers alike.
+export function basicTokenize(text: string): string[] {
+  const spaced = foldText(text)
+    .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+    .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2");
+  const out: string[] = [];
+  for (const part of spaced.toLowerCase().split(/[^a-z0-9]+/)) {
+    if (part) out.push(part);
+  }
+  return out;
+}
+
+// Greedy longest-match wordpiece for ONE word. The first piece is looked up as
+// written; every subsequent piece is looked up with a "##" prefix. If no piece
+// matches at some position the whole word is unresolvable → [unkId] (or [] when
+// the model has no UNK row). Deterministic: longest-match is a fixed scan.
+export function wordpiece(word: string, model: StaticEmbedModel): number[] {
+  if (!word) return [];
+  const ids: number[] = [];
+  let start = 0;
+  const n = word.length;
+  while (start < n) {
+    let end = n;
+    let match = -1;
+    while (end > start) {
+      const piece = start === 0 ? word.slice(start, end) : "##" + word.slice(start, end);
+      const id = model.vocab.get(piece);
+      if (id !== undefined) {
+        match = id;
+        break;
+      }
+      end--;
+    }
+    if (match === -1) return model.unkId >= 0 ? [model.unkId] : []; // whole word is OOV
+    ids.push(match);
+    start = end;
+  }
+  return ids;
+}
+
+// Full token-id sequence for a text: tokenize → wordpiece each word, concatenated
+// in order. The order is load-bearing for determinism, but mean-pooling is
+// order-independent anyway; the fixed order simply makes reasoning trivial.
+export function tokenize(text: string, model: StaticEmbedModel): number[] {
+  const ids: number[] = [];
+  for (const word of basicTokenize(text)) {
+    for (const id of wordpiece(word, model)) ids.push(id);
+  }
+  return ids;
+}
+
+// Round half to EVEN (banker's rounding). JS `Math.round` rounds half toward
+// +∞, which is not symmetric and would bias quantization; half-to-even is the
+// IEEE-754 default and keeps quantize(x) === -quantize(-x).
+export function roundHalfToEven(x: number): number {
+  const f = Math.floor(x);
+  const diff = x - f;
+  if (diff < 0.5) return f;
+  if (diff > 0.5) return f + 1;
+  return f % 2 === 0 ? f : f + 1; // exactly .5 → nearest even
+}
+
+// Encode a text to an int8 unit-ish vector. Empty / all-OOV / zero-norm inputs
+// yield an all-zero vector (dot-products against it are 0 — it simply ranks
+// last), never NaN. Pure and deterministic: same model + text → same bytes.
+export function encode(model: StaticEmbedModel, text: string): Int8Array {
+  const { dim, weights } = model;
+  const out = new Int8Array(dim);
+  const ids = tokenize(text, model);
+  if (ids.length === 0) return out;
+
+  // mean-pool in double precision, fixed order.
+  const pooled = new Float64Array(dim);
+  for (const id of ids) {
+    const base = id * dim;
+    for (let d = 0; d < dim; d++) pooled[d]! += weights[base + d]!;
+  }
+  const inv = 1 / ids.length;
+  for (let d = 0; d < dim; d++) pooled[d]! *= inv;
+
+  // L2-normalize.
+  let sumsq = 0;
+  for (let d = 0; d < dim; d++) sumsq += pooled[d]! * pooled[d]!;
+  const norm = Math.sqrt(sumsq);
+  if (norm === 0) return out; // zero vector — nothing to rank on
+
+  // quantize to int8 at the fixed 1/127 scale, round-half-to-even, clamp.
+  for (let d = 0; d < dim; d++) {
+    let q = roundHalfToEven((pooled[d]! / norm) * QUANT);
+    if (q > QUANT) q = QUANT;
+    else if (q < -QUANT) q = -QUANT;
+    out[d] = q;
+  }
+  return out;
+}
+
+// Integer dot product of two int8 vectors (the ranking primitive). Widened to a
+// JS number (double), but every operand and partial sum is an exact integer
+// (max |Σ| for dim 512 ≈ 512·127² ≈ 8.3M, far inside 2⁵³) — so this is an EXACT
+// integer computation, deterministic everywhere.
+export function intDot(a: Int8Array, b: Int8Array): number {
+  const n = Math.min(a.length, b.length);
+  let dot = 0;
+  for (let i = 0; i < n; i++) dot += a[i]! * b[i]!;
+  return dot;
+}

package/src/embed/endpoint.ts

@@ -0,0 +1,55 @@
+// HTTP embedding endpoint client — the "rich" tier SKELETON for v2.11.0. It is
+// intentionally NOT wired into the CLI or MCP in v2.10.0: the deterministic
+// static tier is the default, and an endpoint's vectors are float and provider-
+// dependent (NOT byte-deterministic), so this stays an explicit, opt-in escape
+// hatch a future release finalizes. Shipped now only so the module surface is
+// stable and the contract is documented.
+//
+// The contract (v2.11): POST { texts: string[] } → { vectors: number[][] } to
+// CODEINDEX_EMBED_ENDPOINT (or opts.url). Node >=18 has global fetch, so this
+// stays zero-dependency. The library never orchestrates docker; running such a
+// server (e.g. ghcr.io/maxgfr/codeindex-embed) is a user/CI concern.
+
+export interface EmbedEndpointOptions {
+  // Endpoint URL. Falls back to CODEINDEX_EMBED_ENDPOINT when omitted.
+  url?: string;
+  // Abort the request after this many ms (default 30_000).
+  timeoutMs?: number;
+  // Extra request headers (e.g. an auth token for a private endpoint).
+  headers?: Record<string, string>;
+}
+
+// Resolve the configured endpoint URL, or undefined when neither opts.url nor
+// CODEINDEX_EMBED_ENDPOINT is set — the silent "endpoint tier unavailable" signal.
+export function resolveEmbedEndpoint(opts: EmbedEndpointOptions = {}): string | undefined {
+  const url = opts.url ?? process.env.CODEINDEX_EMBED_ENDPOINT;
+  return url && url.trim() ? url.trim() : undefined;
+}
+
+// Request float embeddings for a batch of texts from the endpoint. Throws a
+// clear error when no endpoint is configured or the response is malformed; the
+// caller decides whether to degrade. NOTE: endpoint vectors are float and are
+// NOT part of any determinism guarantee — this path is for the rich tier only.
+export async function embedViaEndpoint(texts: string[], opts: EmbedEndpointOptions = {}): Promise<number[][]> {
+  const url = resolveEmbedEndpoint(opts);
+  if (!url) throw new Error("no embedding endpoint configured (set CODEINDEX_EMBED_ENDPOINT or pass opts.url)");
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), opts.timeoutMs ?? 30_000);
+  try {
+    const res = await fetch(url, {
+      method: "POST",
+      headers: { "content-type": "application/json", ...(opts.headers ?? {}) },
+      body: JSON.stringify({ texts }),
+      signal: controller.signal,
+    });
+    if (!res.ok) throw new Error(`embedding endpoint ${url} returned HTTP ${res.status}`);
+    const data = (await res.json()) as { vectors?: unknown };
+    const vectors = data.vectors;
+    if (!Array.isArray(vectors) || !vectors.every((v) => Array.isArray(v) && v.every((x) => typeof x === "number"))) {
+      throw new Error(`embedding endpoint ${url} returned a malformed { vectors } payload`);
+    }
+    return vectors as number[][];
+  } finally {
+    clearTimeout(timer);
+  }
+}

package/src/embed/index.ts

@@ -0,0 +1,125 @@
+import type { RepoScan } from "../scan.js";
+import { encode } from "./encode.js";
+import { EMBED_VERSION, type StaticEmbedModel } from "./model.js";
+
+// The corpus embedding artifact. Granularity is PER-SYMBOL (each symbol gets its
+// own record, so search can surface the exact symbol that matched), with a
+// PER-FILE fallback record for files that declare no symbols (docs, config) so
+// every file with content is still represented. Deterministic: records follow
+// scan order (files sorted by rel) then declaration order within a file.
+export interface EmbeddingRecord {
+  file: string; // repo-relative path
+  symbol?: string; // the declared symbol this vector represents (absent for a file-level record)
+  line?: number; // 1-based symbol line (absent for a file-level record)
+  vec: Int8Array; // length === dim
+}
+
+export interface EmbeddingIndex {
+  embedVersion: number;
+  modelId: string;
+  dim: number;
+  records: EmbeddingRecord[];
+}
+
+// The text encoded for one symbol: its name (camelCase-split by the tokenizer),
+// its signature, the owning file's one-line summary, and the file's path segments
+// — the same signal bm25 indexes, but pooled into a single vector. Newline-joined
+// so the tokenizer's non-alphanumeric split cleanly separates the parts.
+function symbolText(rel: string, name: string, signature: string | undefined, summary: string | undefined): string {
+  return [name, signature ?? "", summary ?? "", rel.replace(/\//g, " ")].join("\n");
+}
+
+// A file-level record's text (symbol-less files): title, summary, headings, path.
+function fileText(rel: string, title: string | undefined, summary: string | undefined, headings: string[]): string {
+  return [title ?? "", summary ?? "", ...headings, rel.replace(/\//g, " ")].join("\n");
+}
+
+// Build the corpus embedding index from a scan + a loaded model. Pure and
+// deterministic (encode is byte-stable, scan order is fixed) → two builds of an
+// unchanged repo produce byte-identical serialized bytes.
+export function buildEmbeddingIndex(scan: RepoScan, model: StaticEmbedModel): EmbeddingIndex {
+  const records: EmbeddingRecord[] = [];
+  for (const f of scan.files) {
+    const seen = new Set<string>();
+    let hadSymbol = false;
+    for (const s of f.symbols) {
+      if (seen.has(s.name)) continue; // dedupe by name within a file (bm25 parity)
+      seen.add(s.name);
+      hadSymbol = true;
+      records.push({
+        file: f.rel,
+        symbol: s.name,
+        line: s.line,
+        vec: encode(model, symbolText(f.rel, s.name, s.signature, f.summary)),
+      });
+    }
+    if (!hadSymbol) {
+      const text = fileText(f.rel, f.title, f.summary, f.headings);
+      if (text.replace(/\s+/g, "")) {
+        records.push({ file: f.rel, vec: encode(model, text) });
+      }
+    }
+  }
+  return { embedVersion: EMBED_VERSION, modelId: model.modelId, dim: model.dim, records };
+}
+
+const MAGIC = "CIE1"; // codeindex embeddings, format 1
+
+// Serialize to embeddings.bin: a fixed ASCII magic, a uint32-LE header length, a
+// UTF-8 JSON header (per-record metadata in build order — carries NO absolute
+// path or timestamp), then the packed int8 body (count × dim signed bytes). The
+// header JSON key order is fixed by construction, and every body byte is written
+// explicitly, so the bytes are fully deterministic. Returns a Uint8Array (a
+// Buffer at runtime) so the public type surface stays free of the Node `Buffer`
+// global — a consumer type-checking without @types/node still resolves it.
+export function serializeEmbeddings(index: EmbeddingIndex): Uint8Array {
+  const header = JSON.stringify({
+    embedVersion: index.embedVersion,
+    modelId: index.modelId,
+    dim: index.dim,
+    count: index.records.length,
+    records: index.records.map((r) => ({ file: r.file, symbol: r.symbol ?? "", line: r.line ?? 0 })),
+  });
+  const headerBuf = Buffer.from(header, "utf8");
+  const body = Buffer.alloc(index.records.length * index.dim);
+  let off = 0;
+  for (const r of index.records) {
+    for (let d = 0; d < index.dim; d++) body.writeInt8(r.vec[d] ?? 0, off++);
+  }
+  const out = Buffer.alloc(8 + headerBuf.length + body.length);
+  out.write(MAGIC, 0, "ascii");
+  out.writeUInt32LE(headerBuf.length, 4);
+  headerBuf.copy(out, 8);
+  body.copy(out, 8 + headerBuf.length);
+  return out;
+}
+
+// Inverse of serializeEmbeddings. Accepts any Uint8Array (e.g. a fs.readFileSync
+// Buffer) and wraps it as a Buffer VIEW (no copy) for the numeric reads. Throws
+// on a bad magic (a corrupt or foreign file) so a caller fails loudly rather
+// than misreading arbitrary bytes.
+export function deserializeEmbeddings(bytes: Uint8Array): EmbeddingIndex {
+  const buf = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  if (buf.length < 8 || buf.toString("ascii", 0, 4) !== MAGIC) {
+    throw new Error("embeddings.bin: bad magic (not a codeindex embeddings artifact)");
+  }
+  const headerLen = buf.readUInt32LE(4);
+  const header = JSON.parse(buf.toString("utf8", 8, 8 + headerLen)) as {
+    embedVersion: number;
+    modelId: string;
+    dim: number;
+    count: number;
+    records: { file: string; symbol: string; line: number }[];
+  };
+  const bodyOff = 8 + headerLen;
+  const { dim } = header;
+  const records: EmbeddingRecord[] = header.records.map((m, i) => {
+    const vec = new Int8Array(dim);
+    for (let d = 0; d < dim; d++) vec[d] = buf.readInt8(bodyOff + i * dim + d);
+    const rec: EmbeddingRecord = { file: m.file, vec };
+    if (m.symbol) rec.symbol = m.symbol;
+    if (m.line) rec.line = m.line;
+    return rec;
+  });
+  return { embedVersion: header.embedVersion, modelId: header.modelId, dim, records };
+}

package/src/embed/model.ts

@@ -0,0 +1,117 @@
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+
+// The deterministic static-embedding tier (v2.10.0). A *static* embedding model
+// is a plain lookup table: token → row vector. Inference is tokenize → gather
+// rows → mean-pool → L2-normalize → int8-quantize (see encode.ts). There is NO
+// neural forward pass, no wasm, no float matmul — so the encode is pure-JS and
+// byte-identical across platforms, which is the whole reason this tier exists.
+//
+// Activation is OPT-IN by asset presence, exactly like the tree-sitter grammar
+// tier (src/ast/loader.ts): no model on disk → the engine silently stays
+// lexical. Models are NEVER shipped in the npm tarball; the asset is fetched by
+// `codeindex embed pull` into CODEINDEX_EMBED_DIR (or <repo>/.codeindex/models/).
+
+// Bumped independently of SCHEMA_VERSION / EXTRACTOR_VERSION whenever the
+// embeddings.bin artifact shape or the encode algorithm changes, so a consumer
+// rejects an embeddings.bin written by an incompatible engine (or a different
+// model) instead of misranking with it. SCHEMA_VERSION is deliberately left
+// UNTOUCHED — embeddings are a purely additive sidecar with zero impact on the
+// graph.json / symbols.json consumers.
+export const EMBED_VERSION = 1;
+
+// The default model-name a `pull` writes and a `status` reports. The real asset
+// URL is intentionally NOT hard-coded yet (see resolveEmbedPullUrl): the asset
+// is unpublished, so `pull` fails cleanly asking for CODEINDEX_EMBED_URL rather
+// than fetching a phantom.
+export const DEFAULT_EMBED_DIRNAME = "models";
+
+// A loaded static-embedding model. `vocab` maps a wordpiece token to its row
+// index; `weights` is the flat row-major (vocabSize × dim) matrix as IEEE-754
+// doubles (float rows widen to double exactly, so mean-pool stays deterministic).
+export interface StaticEmbedModel {
+  modelId: string;
+  dim: number;
+  unk: string; // the OOV token (e.g. "[UNK]"); may be absent from vocab
+  unkId: number; // row index of `unk`, or -1 when the model has no UNK row
+  vocabSize: number;
+  vocab: Map<string, number>;
+  weights: Float64Array; // length === vocabSize * dim, row-major
+}
+
+// The on-disk model.json header shape. `weights` is an array of `vocabSize`
+// rows, each of length `dim`. Kept human-authorable (plain JSON numbers) so a
+// tiny fixture model is committable and diffable; a real `pull`ed model uses the
+// identical shape at scale.
+interface ModelFile {
+  modelId: string;
+  dim: number;
+  unk?: string;
+  vocab: string[];
+  weights: number[][];
+}
+
+// Where the model asset lives. CODEINDEX_EMBED_DIR wins outright (explicit
+// override, mirrors CODEINDEX_GRAMMAR_DIR); otherwise <repo>/.codeindex/models/
+// then <cwd>/.codeindex/models/. Returns undefined when no model.json is found
+// anywhere — the silent-degradation signal. When CODEINDEX_EMBED_DIR is set but
+// empty of a model, that is still "no model" (undefined), never an error.
+export function resolveEmbedModelDir(repo?: string): string | undefined {
+  const env = process.env.CODEINDEX_EMBED_DIR;
+  const candidates: string[] = [];
+  if (env) candidates.push(env);
+  if (repo) candidates.push(join(repo, ".codeindex", DEFAULT_EMBED_DIRNAME));
+  candidates.push(join(process.cwd(), ".codeindex", DEFAULT_EMBED_DIRNAME));
+  for (const c of candidates) {
+    if (existsSync(join(c, "model.json"))) return c;
+  }
+  return undefined;
+}
+
+// True when an activatable model is present (used by `embed status` and the MCP
+// gate before attempting a semantic search).
+export function hasEmbedModel(repo?: string): boolean {
+  return resolveEmbedModelDir(repo) !== undefined;
+}
+
+// Load and validate a static model from a directory containing model.json.
+// Throws on a malformed file (bad dim, ragged weights) so a corrupt asset fails
+// loudly at load rather than silently misranking. Returns undefined only when
+// the directory has no model.json (the not-present case).
+export function loadEmbedModel(dir?: string): StaticEmbedModel | undefined {
+  if (!dir) return undefined;
+  const path = join(dir, "model.json");
+  if (!existsSync(path)) return undefined;
+  const raw = JSON.parse(readFileSync(path, "utf8")) as ModelFile;
+  const { modelId, dim, vocab, weights } = raw;
+  if (typeof modelId !== "string" || !modelId) throw new Error(`embed model: missing modelId in ${path}`);
+  if (!Number.isInteger(dim) || dim <= 0) throw new Error(`embed model: bad dim ${dim} in ${path}`);
+  if (!Array.isArray(vocab) || !Array.isArray(weights) || vocab.length !== weights.length) {
+    throw new Error(`embed model: vocab/weights length mismatch in ${path}`);
+  }
+  const vocabSize = vocab.length;
+  const flat = new Float64Array(vocabSize * dim);
+  const vmap = new Map<string, number>();
+  for (let i = 0; i < vocabSize; i++) {
+    const tok = vocab[i]!;
+    if (typeof tok !== "string") throw new Error(`embed model: non-string vocab entry at ${i}`);
+    if (!vmap.has(tok)) vmap.set(tok, i); // first occurrence wins (deterministic)
+    const row = weights[i]!;
+    if (!Array.isArray(row) || row.length !== dim) {
+      throw new Error(`embed model: row ${i} has length ${row?.length}, expected ${dim}`);
+    }
+    for (let d = 0; d < dim; d++) flat[i * dim + d] = Number(row[d]);
+  }
+  const unk = typeof raw.unk === "string" ? raw.unk : "[UNK]";
+  const unkId = vmap.has(unk) ? vmap.get(unk)! : -1;
+  return { modelId, dim, unk, unkId, vocabSize, vocab: vmap, weights: flat };
+}
+
+// Resolve the URL `embed pull` fetches from. The official asset is not published
+// yet, so there is no built-in default: the user must point CODEINDEX_EMBED_URL
+// at a model.json (or a directory serving one). Returns undefined when unset —
+// the caller turns that into a clean, actionable failure, never a crash.
+export function resolveEmbedPullUrl(): string | undefined {
+  const url = process.env.CODEINDEX_EMBED_URL;
+  return url && url.trim() ? url.trim() : undefined;
+}