graph-indexer 3.0.1 → 3.1.1

package/src/query/textsearch.mjs

@@ -15,12 +15,14 @@ const MAX_FILE_BYTES = 1_500_000;
 // a caller's pattern gets this much matching time per search (reading files does not count): one
 // with catastrophic backtracking (`(a+)+$`) is stopped instead of blocking the server
 const REGEX_BUDGET_MS = 5000;
+const MAX_KEPT = 50_000;
 const SCAN = new vm.Script(`(() => {
     if (!re.test(text)) return null;
     const lines = text.split(/\\r?\\n/), hits = [];
     for (let i = 0; i < lines.length; i++) if (re.test(lines[i])) hits.push(i);
     return { lines, hits };
 })()`);
+const GLOB_FILTER = new vm.Script(`paths.filter(p => re.test(onName ? p.slice(p.lastIndexOf('/') + 1) : p))`);
 const SKIP_DIRS = /(^|\/)(node_modules|\.git|\.graph-indexer|dist|build|out|target|coverage|__pycache__|\.venv|venv|\.next|\.nuxt|\.cache|vendor)\//;
 const EXAMPLE_PATH = /(^|\/)(docs?_src|docs?|examples?|samples?|demos?|tutorials?|benchmarks?|playground)\//i;
 const COMMENT_LINE = /^\s*(\/\/|#(?![!\[])|\*|\/\*|<!--|--\s|;;|%)/;
@@ -54,7 +56,7 @@ function globSource(glob) {
             i++;
             if (glob[i + 1] === '/') { i++; re += '(?:.*/)?'; } else re += '.*';
         } else if (c === '?') re += '[^/]';
-        else if (c === '[' && glob.indexOf(']', i + 2) > i) { const j = glob.indexOf(']', i + 2); re += glob.slice(i, j + 1).replace(/^\[!/, '[^'); i = j; }
+        else if (c === '[' && glob.indexOf(']', i + 2) > i) { const j = glob.indexOf(']', i + 2); re += '[' + glob.slice(i + 1, j).replace(/^!/, '^').replace(/\\/g, '\\\\') + ']'; i = j; }
         else if (c === '{' && glob.indexOf('}', i) > i) { const j = glob.indexOf('}', i); re += `(?:${glob.slice(i + 1, j).split(',').map(escapeRe).join('|')})`; i = j; }
         else re += escapeRe(c);
     }
@@ -71,7 +73,16 @@ export function findFiles(root, pattern, { path: prefix = null } = {}) {
     if (/[*?[{]/.test(pattern)) {
         const onName = !pattern.includes('/');
         const re = new RegExp(onName ? `^${globSource(pattern)}$` : `(^|/)${globSource(pattern.replace(/^\.?\//, ''))}$`);
-        test = (p) => re.test(onName ? p.slice(p.lastIndexOf('/') + 1) : p);
+        // a glob of many stars (`*a*a*a*a*a*a*b`) backtracks for minutes on one long name: matched
+        // under the same budget as search_text's expressions, so it cannot block the server
+        const paths = listFiles(root).filter(p => !pre || p === pre || p.startsWith(pre + '/'));
+        const ctx = vm.createContext({ re, paths, onName });
+        try {
+            return GLOB_FILTER.runInContext(ctx, { timeout: REGEX_BUDGET_MS }).sort();
+        } catch (e) {
+            if (e.code !== 'ERR_SCRIPT_EXECUTION_TIMEOUT') throw e;
+            throw new Error(`The glob took more than ${REGEX_BUDGET_MS / 1000} s to match (too many wildcards?). Use fewer wildcards.`);
+        }
     } else {
         const t = pattern.toLowerCase();
         test = (p) => p.toLowerCase().includes(t);
@@ -133,7 +144,8 @@ export async function textSearch(intel, { pattern, path: prefix = null, literal
         const f = indexed.get(rel);
         for (const i of scan.hits) {
             total++;
-            matches.push({ path: rel, line: i + 1, text: scan.lines[i], fileId: f?.id ?? null, isTest: !!f?.is_test, code: !!f });
+            // counted, but kept (and annotated) only up to a bound: memory and time stay bounded
+            if (matches.length < MAX_KEPT) matches.push({ path: rel, line: i + 1, text: scan.lines[i], fileId: f?.id ?? null, isTest: !!f?.is_test, code: !!f });
         }
     }
     const ident = identifierOf(pattern, literal);
@@ -149,13 +161,22 @@ export async function textSearch(intel, { pattern, path: prefix = null, literal
             WHERE r.file_id = ? AND r.name = ?`, f.id, ident) : [];
         const refsAt = new Map();
         for (const r of refs) (refsAt.get(r.line) ?? refsAt.set(r.line, []).get(r.line)).push(r);
+        // innermost definition around each matched line, ignoring ones that start on it (inline
+        // classes, the def itself): one sweep over the definitions in start order, writing only the
+        // matched lines, instead of every definition for every match
+        const wanted = new Map(ms.map(m => [m.line, null]));
+        const lines = [...wanted.keys()].sort((a, b) => a - b);
+        for (const s of syms) {
+            let lo = 0, hi = lines.length;
+            while (lo < hi) { const mid = (lo + hi) >> 1; if (lines[mid] <= s.start_line) lo = mid + 1; else hi = mid; }
+            for (let k = lo; k < lines.length && lines[k] <= s.end_line; k++) wanted.set(lines[k], s);
+        }
+        const defAt = new Map();
+        if (summary) for (const s of syms) if (s.name === ident && !defAt.has(s.name_line)) defAt.set(s.name_line, s);
         for (const m of ms) {
-            // innermost definition around the line, ignoring ones that start on it (inline classes, the def itself)
-            let encl = null;
-            for (const s of syms) if (s.start_line < m.line && s.end_line >= m.line && (!encl || s.start_line >= encl.start_line)) encl = s;
-            m.encl = encl?.qname ?? null;
+            m.encl = wanted.get(m.line)?.qname ?? null;
             if (!summary) continue;
-            const def = syms.find(s => s.name === ident && s.name_line === m.line);
+            const def = defAt.get(m.line);
             const rs = refsAt.get(m.line) ?? [];
             if (def) { m.what = 'def'; m.target = def.qname; summary.defs++; continue; }
             const bound = rs.filter(r => r.dst_id != null);
@@ -174,5 +195,5 @@ export async function textSearch(intel, { pattern, path: prefix = null, literal
         }
     }
     matches.sort((a, b) => rank(a.path, a.isTest) + (a.code ? 0 : 1) - (rank(b.path, b.isTest) + (b.code ? 0 : 1)) || a.path.localeCompare(b.path) || a.line - b.line);
-    return { total, files: nfiles, shown: matches.slice(0, limit), all: matches, summary, truncated: matches.length > limit };
+    return { total, files: nfiles, shown: matches.slice(0, limit), all: matches, summary, truncated: total > limit, annotated: matches.length };
 }

package/src/search/search.mjs

@@ -4,17 +4,20 @@
  *   1. name channel     exact / case-insensitive / qualified / prefix matches on symbol names
  *   2. lexical channel  FTS5 BM25 over fields (name, qualified name, signature, doc, path, body)
  *   3. concept channel  the same index queried with thesaurus expansions (natural-language queries)
- *   4. rerank           a small linear model over interpretable features (name evidence, field
+ *   4. semantic channel (optional) cosine similarity of a static code-embedding model (vectors.mjs),
+ *                       on when its model is installed
+ *   5. rerank           a small linear model over interpretable features (name evidence, field
  *                       coverage, kind/test/export priors, graph centrality), then per-file
  *                       diversification so one file cannot flood the top results.
  */
 import { analyzeQuery, codeTokens, splitIdentifier } from './tokenize.mjs';
+import { VectorIndex } from './vectors.mjs';
 
 const FTS_WEIGHTS = [10.0, 6.0, 3.0, 2.0, 2.5, 1.0]; // name, qname, sig, doc, path, body
 const KIND_PRIOR = {
     class: 1.0, interface: 0.95, struct: 1.0, trait: 0.95, enum: 0.85, type: 0.8, object: 0.95, module: 0.7, impl: 0.7,
     function: 1.0, method: 1.0, constructor: 0.75, macro: 0.8, field: 0.55, property: 0.6, variable: 0.55, constant: 0.6,
-    selector: 0.35, keyframes: 0.3,
+    selector: 0.35, keyframes: 0.3, section: 0.5,
 };
 
 export const DEFAULT_WEIGHTS = {
@@ -39,11 +42,21 @@ export const DEFAULT_WEIGHTS = {
     nlField: -0.93,     // fields/properties/variables for natural-language (behavioural) queries
     file: 1.0,          // file-level BM25 of the symbol's file (hierarchical: file relevance first)
     fileNL: 0.5,        // extra file-level weight for natural-language (behavioural) queries
+    docs: 2.0,          // documentation channel BM25 (only for queries about docs, design or rationale)
+    // semantic channel (opt-in): cosine, min-max normalised over its top candidates. Higher weights
+    // (0.5 / 2.0) score better on symbol search and worse on localizing real commits; these are
+    // the highest that leave localization within noise (docs/BENCHMARKS.md)
+    sem: 0.25,
+    semNL: 0.5,         // extra semantic weight for natural-language queries
 };
 
 const EXAMPLE_PATH = /(^|\/)(examples?|samples?|demos?|sandbox|playground|docs?(_src|_source|_examples?)?|documentation|benchmarks?|bench|scripts?|fixtures?|testdata|tutorials?|website|\.github)\//i;
 const DECLARATION_PATH = /\.d\.[cm]?ts$/;
+export const WANTS_DOCS = /\b(docs?|documentation|documented|readme|guide|tutorial|design|rationale|architecture|adr|why|explained?|explanation|changelog|faq|how to)\b/i;
 const BM25_FLOOR = 1.0;
+const SEM_K = 200; // semantic candidates per query
+const RRF_K = 60;
+export const MAX_QUERIES = 6;
 const FILE_GATE = 0.25; // own normalized BM25 at which the file boost applies in full (tuning split)
 
 function stem(w) {
@@ -58,7 +71,8 @@ function ftsQuery(terms) {
 }
 
 export class SearchEngine {
-    constructor(indexer, { weights = DEFAULT_WEIGHTS, centrality = null, perFileCap = Number(process.env.GI_PER_FILE_CAP) || 3 } = {}) {
+    constructor(indexer, { weights = DEFAULT_WEIGHTS, centrality = null, perFileCap = Number(process.env.GI_PER_FILE_CAP) || 3, semantic = undefined } = {}) {
+        this.semantic = semantic === undefined ? VectorIndex.maybe(indexer) : semantic;
         this.perFileCap = perFileCap;
         this.ix = indexer;
         this.store = indexer.store;
@@ -78,6 +92,7 @@ export class SearchEngine {
         const kindFilter = opts.kinds ?? a.filters.kind ?? null;
         const langFilter = opts.lang ?? a.filters.lang?.[0] ?? null;
         const wantsTests = opts.includeTests ?? /\b(test|tests|spec|specs|mock|fixture)\b/i.test(query);
+        const wantsDocs = WANTS_DOCS.test(query);
         const cands = new Map(); // id -> feature bag
         const bag = (id) => { let f = cands.get(id); if (!f) cands.set(id, (f = { id })); return f; };
 
@@ -140,6 +155,23 @@ export class SearchEngine {
                 for (const r of rows) { const f = bag(r.id); if (f.bm25 == null) { f.bm25 = -r.s; if (-r.s > maxBm) maxBm = -r.s; } }
             }
         }
+        // documentation channel: Markdown sections, only when the query is about docs, design or rationale
+        let maxDoc = 0;
+        if (q1 && (wantsDocs || opts.docs)) {
+            for (const r of this.store.all('SELECT rowid AS id, bm25(doc_fts, 4.0, 1.0) AS s FROM doc_fts WHERE doc_fts MATCH ? ORDER BY s LIMIT 60', q1)) {
+                bag(r.id).doc = -r.s; if (-r.s > maxDoc) maxDoc = -r.s;
+            }
+        }
+        // semantic channel: definitions whose embedding is closest to the query's
+        let semLo = 0, semHi = 0;
+        if (this.semantic && (this.w.sem || this.w.semNL)) {
+            const hits = this.semantic.search(query, SEM_K);
+            if (hits.size) {
+                const vals = [...hits.values()];
+                semHi = Math.max(...vals); semLo = Math.min(...vals);
+                for (const [id, c] of hits) bag(id).sem = c;
+            }
+        }
         if (!cands.size) return { results: [], analysis: a };
 
         // 4. features + rerank
@@ -161,6 +193,7 @@ export class SearchEngine {
             if (pathFilter && !r.path.startsWith(pathFilter.replace(/^\.\//, ''))) continue;
             if (kindFilter && !kindFilter.includes(r.kind)) continue;
             if (langFilter && r.lang !== langFilter) continue;
+            if (r.kind === 'section' && !wantsDocs && !opts.docs) continue; // documentation answers only questions about it
             const nameWords = splitIdentifier(r.name).map(stem);
             const qnameWords = new Set(splitIdentifier(r.qname.replace(/\./g, '_')).map(stem));
             const nameCover = nameWords.length ? nameWords.filter(x => qStems.has(x)).length / nameWords.length : 0;
@@ -171,7 +204,9 @@ export class SearchEngine {
             // blown up by max-normalisation; when literal evidence is weak, the concept channel
             // (thesaurus expansions) takes over — that is exactly the vocabulary-mismatch case.
             if (f.bm25) s += w.bm25 * (f.bm25 / Math.max(maxBm, BM25_FLOOR));
+            if (f.doc) s += (w.docs ?? 0) * (f.doc / Math.max(maxDoc, BM25_FLOOR));
             if (f.concept) s += (maxBm < BM25_FLOOR ? Math.max(w.concept, 1.0) : w.concept) * (f.concept / Math.max(maxCx, BM25_FLOOR));
+            if (f.sem != null && semHi > semLo) s += ((w.sem ?? 0) + (a.natural ? (w.semNL ?? 0) : 0)) * (f.sem - semLo) / (semHi - semLo);
             if (f.exact) { s += w.exact * f.exact; reasons.push('name'); }
             if (f.exactCI) { s += w.exactCI * f.exactCI; reasons.push('name'); }
             if (f.qualified) { s += w.qualified; reasons.push('qualified name'); }
@@ -188,23 +223,56 @@ export class SearchEngine {
             if (fs && f.bm25 && r.kind !== 'selector' && r.kind !== 'keyframes') s += (w.file + (a.natural ? w.fileNL : 0)) * fs * Math.min(1, (f.bm25 / Math.max(maxBm, BM25_FLOOR)) / FILE_GATE);
             if (r.parent_kind === 'function' || r.parent_kind === 'method') s += w.nested;
             if (DECLARATION_PATH.test(r.path)) s += w.declaration;
-            if (EXAMPLE_PATH.test(r.path) && !(pathFilter && EXAMPLE_PATH.test(pathFilter))) s += w.example;
+            if (r.kind !== 'section' && EXAMPLE_PATH.test(r.path) && !(pathFilter && EXAMPLE_PATH.test(pathFilter))) s += w.example;
             if (a.natural && (r.kind === 'field' || r.kind === 'property' || r.kind === 'variable' || r.kind === 'constant')) s += w.nlField;
             scored.push({ id: r.id, score: s, reasons, path: r.path, ...(opts.debug ? { features: { ...f, nameCover, queryCover, kind: r.kind } } : {}) });
         }
         scored.sort((x, y) => y.score - x.score || x.id - y.id);
-        // per-file diversification: 3rd+ result from the same file is pushed down
-        const perFile = new Map();
-        const out = [];
-        const deferred = [];
-        for (const r of scored) {
-            const n = perFile.get(r.path) ?? 0;
-            if (n >= this.perFileCap) { deferred.push(r); continue; }
-            perFile.set(r.path, n + 1);
-            out.push(r);
-            if (out.length >= limit) break;
-        }
-        for (const r of deferred) { if (out.length >= limit) break; out.push(r); }
-        return { results: out, total: scored.length, analysis: a };
+        return { results: diversify(scored, limit, this.perFileCap), total: scored.length, analysis: a };
+    }
+
+    /**
+     * Several phrasings of one need ("where are tokens refreshed", "refreshToken", "session renewal")
+     * in one call: each is searched on its own and the rankings are fused by reciprocal rank
+     * (k = 60), so a definition several phrasings agree on rises and one phrasing's noise does not
+     * dominate. One query is the same as search().
+     */
+    searchMany(queries, opts = {}) {
+        const qs = [...new Set((queries ?? []).map(q => String(q ?? '').trim()).filter(Boolean))].slice(0, MAX_QUERIES);
+        if (qs.length <= 1) return this.search(qs[0] ?? '', opts);
+        const limit = Math.max(1, Math.min(opts.limit ?? 10, 100));
+        const per = Math.max(2 * limit, 20);
+        const fused = new Map();
+        let total = 0, analysis = null;
+        qs.forEach((q, qi) => {
+            const r = this.search(q, { ...opts, limit: per });
+            analysis ??= r.analysis;
+            total = Math.max(total, r.total ?? r.results.length);
+            r.results.forEach((x, rank) => {
+                let e = fused.get(x.id);
+                if (!e) fused.set(x.id, (e = { id: x.id, score: 0, reasons: [], path: x.path, queries: [] }));
+                e.score += 1 / (RRF_K + rank + 1);
+                e.queries.push(qi);
+                for (const why of x.reasons) if (!e.reasons.includes(why)) e.reasons.push(why);
+            });
+        });
+        const scored = [...fused.values()].sort((x, y) => y.score - x.score || x.id - y.id);
+        return { results: diversify(scored, limit, this.perFileCap), total: Math.max(total, scored.length), analysis, queries: qs };
+    }
+}
+
+/** Per-file diversification: results past the cap from one file are pushed below the others. */
+function diversify(scored, limit, cap) {
+    const perFile = new Map();
+    const out = [];
+    const deferred = [];
+    for (const r of scored) {
+        const n = perFile.get(r.path) ?? 0;
+        if (n >= cap) { deferred.push(r); continue; }
+        perFile.set(r.path, n + 1);
+        out.push(r);
+        if (out.length >= limit) break;
     }
+    for (const r of deferred) { if (out.length >= limit) break; out.push(r); }
+    return out;
 }

package/src/search/semantic.mjs

@@ -0,0 +1,201 @@
+/**
+ * Optional semantic channel: a static (Model2Vec) code embedding model, run in plain JavaScript.
+ *
+ * A static model is a table of one vector per WordPiece token; a text's embedding is the mean of
+ * its tokens' vectors, normalised. There is no neural network to run, so it needs no native code,
+ * no ONNX runtime and no GPU: embedding a whole repository takes seconds on one core.
+ *
+ * The model is not part of the package. `graph-indexer semantic install` downloads it once, at a
+ * pinned revision with checked SHA-256 digests, into ~/.cache/graph-indexer/models (or
+ * GRAPH_INDEXER_MODEL_DIR). The channel is opt-in: it runs only with GRAPH_INDEXER_SEMANTIC=1 and
+ * the model installed; otherwise search is purely lexical + graph. On the benchmarks it helps
+ * symbol search for queries in words and does not help localizing real commits (docs/BENCHMARKS.md),
+ * so it is not the default.
+ *
+ * Vectors are computed lazily, file by file, the first time a search needs them, and kept in the
+ * index (`vectors`, `vec_files`), keyed by the file's content hash so edits re-embed only that file.
+ */
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import crypto from 'node:crypto';
+
+export const MODEL = {
+    id: 'potion-code-16M-v2',
+    repo: 'minishlab/potion-code-16M-v2',
+    revision: 'e9d2a44ca6a05ac6685f3b23709ea57eb7352d5b',
+    license: 'MIT',
+    files: {
+        'model.safetensors': '75cf7a6c2171b230ad19b1e7d8e0b1aee86da5a02af8e7cacedd9921d227623c',
+        'tokenizer.json': '107bbdcbad4bff1d299b7a4c3a2fb17c52890688b7dd0e4c9deab79d3c4f3d45',
+        'config.json': '148e5691a6fcc553437156859701fba017a1ba5d340b170f17e0f3668fb861a7',
+    },
+};
+
+const MAX_TOKENS = 512;       // Model2Vec's default truncation
+const MAX_WORD_CHARS = 100;   // WordPiece max_input_chars_per_word
+
+export function modelDir() {
+    const base = process.env.GRAPH_INDEXER_MODEL_DIR || path.join(os.homedir(), '.cache', 'graph-indexer', 'models');
+    return path.join(base, MODEL.id);
+}
+
+export function semanticEnabled() {
+    if (process.env.GRAPH_INDEXER_SEMANTIC !== '1') return false;
+    return fs.existsSync(path.join(modelDir(), 'model.safetensors')) && fs.existsSync(path.join(modelDir(), 'tokenizer.json'));
+}
+
+// ── tokenizer: BertNormalizer(lowercase, strip accents) + BertPreTokenizer + WordPiece ──────────
+
+function isControl(c) {
+    if (c === 9 || c === 10 || c === 13) return false;
+    return (c < 32) || (c >= 127 && c < 160);
+}
+function isWhitespace(c) { return c === 32 || c === 9 || c === 10 || c === 13 || c === 0xa0 || c === 0x3000 || (c >= 0x2000 && c <= 0x200a) || c === 0x2028 || c === 0x2029 || c === 0x202f || c === 0x205f || c === 0x1680; }
+function isCJK(c) {
+    return (c >= 0x4e00 && c <= 0x9fff) || (c >= 0x3400 && c <= 0x4dbf) || (c >= 0x20000 && c <= 0x2a6df) || (c >= 0x2a700 && c <= 0x2b73f) ||
+        (c >= 0x2b740 && c <= 0x2b81f) || (c >= 0x2b820 && c <= 0x2ceaf) || (c >= 0xf900 && c <= 0xfaff) || (c >= 0x2f800 && c <= 0x2fa1f);
+}
+const PUNCT_RE = /\p{P}/u;
+function isPunct(ch, c) {
+    return (c >= 33 && c <= 47) || (c >= 58 && c <= 64) || (c >= 91 && c <= 96) || (c >= 123 && c <= 126) || (c > 127 && PUNCT_RE.test(ch));
+}
+
+/** Text → normalised pre-tokens (words and single punctuation characters). */
+export function preTokenize(text) {
+    let s = '';
+    for (const ch of text) {
+        const c = ch.codePointAt(0);
+        if (c === 0 || c === 0xfffd || isControl(c)) continue;
+        if (isWhitespace(c)) { s += ' '; continue; }
+        if (isCJK(c)) { s += ' ' + ch + ' '; continue; }
+        s += ch;
+    }
+    s = s.toLowerCase().normalize('NFD').replace(/\p{Mn}/gu, '');
+    const out = [];
+    let cur = '';
+    for (const ch of s) {
+        const c = ch.codePointAt(0);
+        if (c === 32) { if (cur) { out.push(cur); cur = ''; } continue; }
+        if (isPunct(ch, c)) { if (cur) { out.push(cur); cur = ''; } out.push(ch); continue; }
+        cur += ch;
+    }
+    if (cur) out.push(cur);
+    return out;
+}
+
+export class StaticModel {
+    constructor(dir = modelDir()) {
+        const tok = JSON.parse(fs.readFileSync(path.join(dir, 'tokenizer.json'), 'utf8'));
+        const m = tok.model;
+        if (m?.type !== 'WordPiece') throw new Error(`unsupported tokenizer model ${m?.type}`);
+        this.vocab = new Map(Object.entries(m.vocab));
+        this.prefix = m.continuing_subword_prefix ?? '##';
+        this.unk = this.vocab.get(m.unk_token ?? '[UNK]') ?? -1;
+        const buf = fs.readFileSync(path.join(dir, 'model.safetensors'));
+        const hlen = Number(buf.readBigUInt64LE(0));
+        const header = JSON.parse(buf.subarray(8, 8 + hlen).toString('utf8'));
+        const t = header.embeddings ?? Object.values(header).find(v => v && v.shape);
+        const [rows, dim] = t.shape;
+        const [a, b] = t.data_offsets;
+        const raw = buf.subarray(8 + hlen + a, 8 + hlen + b);
+        this.dim = dim;
+        this.rows = rows;
+        this.table = new Float32Array(rows * dim);
+        if (t.dtype === 'F32') for (let i = 0; i < rows * dim; i++) this.table[i] = raw.readFloatLE(i * 4);
+        else if (t.dtype === 'F16') for (let i = 0; i < rows * dim; i++) this.table[i] = f16(raw.readUInt16LE(i * 2));
+        else throw new Error(`unsupported embedding dtype ${t.dtype}`);
+        this.cache = new Map(); // word -> token ids
+    }
+
+    /** WordPiece ids of one pre-token (greedy longest match), [] when unknown. */
+    #word(w) {
+        let ids = this.cache.get(w);
+        if (ids) return ids;
+        ids = [];
+        if ([...w].length > MAX_WORD_CHARS) ids = [this.unk];
+        else {
+            let start = 0;
+            while (start < w.length) {
+                let end = w.length, found = -1;
+                while (start < end) {
+                    const sub = (start > 0 ? this.prefix : '') + w.slice(start, end);
+                    const id = this.vocab.get(sub);
+                    if (id !== undefined) { found = id; break; }
+                    end--;
+                }
+                if (found < 0) { ids = [this.unk]; break; }
+                ids.push(found);
+                start = end;
+            }
+        }
+        if (this.cache.size > 200000) this.cache.clear();
+        this.cache.set(w, ids);
+        return ids;
+    }
+
+    tokenize(text) {
+        const out = [];
+        for (const w of preTokenize(text)) {
+            for (const id of this.#word(w)) { if (id !== this.unk) out.push(id); }
+            if (out.length >= MAX_TOKENS) break;
+        }
+        return out.length > MAX_TOKENS ? out.slice(0, MAX_TOKENS) : out;
+    }
+
+    /** Mean of token vectors, L2-normalised; null for text without known tokens. */
+    embed(text) {
+        const ids = this.tokenize(text);
+        if (!ids.length) return null;
+        const d = this.dim, v = new Float32Array(d), T = this.table;
+        for (const id of ids) { const o = id * d; for (let j = 0; j < d; j++) v[j] += T[o + j]; }
+        let n = 0;
+        for (let j = 0; j < d; j++) n += v[j] * v[j];
+        n = Math.sqrt(n);
+        if (!n) return null;
+        for (let j = 0; j < d; j++) v[j] /= n;
+        return v;
+    }
+}
+
+function f16(h) {
+    const s = (h & 0x8000) ? -1 : 1, e = (h >> 10) & 0x1f, f = h & 0x3ff;
+    if (e === 0) return s * Math.pow(2, -14) * (f / 1024);
+    if (e === 31) return f ? NaN : s * Infinity;
+    return s * Math.pow(2, e - 15) * (1 + f / 1024);
+}
+
+/** int8 with one scale per vector: v ≈ q * scale. */
+export function quantize(v) {
+    let m = 0;
+    for (let j = 0; j < v.length; j++) { const a = Math.abs(v[j]); if (a > m) m = a; }
+    const q = new Int8Array(v.length);
+    const scale = m / 127 || 1;
+    for (let j = 0; j < v.length; j++) q[j] = Math.round(v[j] / scale);
+    return { q, scale };
+}
+
+// ── model download (explicit, user-initiated) ──────────────────────────────────────────────────
+
+export async function installModel({ log = console.error } = {}) {
+    const dir = modelDir();
+    fs.mkdirSync(dir, { recursive: true });
+    for (const [name, digest] of Object.entries(MODEL.files)) {
+        const dest = path.join(dir, name);
+        if (fs.existsSync(dest) && sha256File(dest) === digest) { log(`${name}: present`); continue; }
+        const url = `https://huggingface.co/${MODEL.repo}/resolve/${MODEL.revision}/${name}`;
+        log(`downloading ${url}`);
+        const res = await fetch(url, { redirect: 'follow' });
+        if (!res.ok) throw new Error(`${name}: HTTP ${res.status}`);
+        const data = Buffer.from(await res.arrayBuffer());
+        const got = crypto.createHash('sha256').update(data).digest('hex');
+        if (got !== digest) throw new Error(`${name}: SHA-256 mismatch (expected ${digest}, got ${got}); nothing was installed`);
+        const tmp = dest + '.part';
+        fs.writeFileSync(tmp, data);
+        fs.renameSync(tmp, dest);
+        log(`${name}: ${(data.length / 1048576).toFixed(1)} MB, verified`);
+    }
+    return dir;
+}
+
+function sha256File(p) { return crypto.createHash('sha256').update(fs.readFileSync(p)).digest('hex'); }

package/src/search/vectors.mjs

@@ -0,0 +1,122 @@
+/**
+ * Vector index over definitions for the semantic channel (see semantic.mjs).
+ *
+ * One int8 vector per definition, embedded from its qualified name, signature, doc comment and
+ * the head of its body. Vectors are filled lazily, file by file, keyed by the file's content
+ * hash, so an edit re-embeds only the files it touched. Search is an exact scan over the matrix
+ * held in memory (no ANN structure to go stale): ~1 ms per 10k definitions.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { StaticModel, quantize, semanticEnabled } from './semantic.mjs';
+import { splitIdentifier } from './tokenize.mjs';
+
+const EMBED_KINDS = new Set(['class', 'interface', 'struct', 'enum', 'trait', 'type', 'object', 'module', 'impl',
+    'function', 'method', 'constructor', 'macro', 'section']);
+const BODY_LINES = 60;
+
+let sharedModel = null;
+function model() { return (sharedModel ??= new StaticModel()); }
+
+/** The text a definition is embedded from. */
+export function symbolText(sym, path_, lines) {
+    const words = splitIdentifier(sym.name).join(' ');
+    const head = `${path_}\n${sym.qname} (${words})\n${sym.sig ?? ''}\n${sym.doc ?? ''}\n`;
+    const body = lines.slice(sym.start_line - 1, Math.min(sym.end_line, sym.start_line - 1 + BODY_LINES)).join('\n');
+    return head + body;
+}
+
+export class VectorIndex {
+    constructor(indexer, { m = null } = {}) {
+        this.ix = indexer;
+        this.store = indexer.store;
+        this.m = m;
+        this.version = -1;
+        this.ids = null;      // Int32Array of symbol ids
+        this.mat = null;      // Int8Array rows × dim
+        this.scales = null;   // Float32Array
+    }
+
+    static maybe(indexer) {
+        try { return semanticEnabled() ? new VectorIndex(indexer) : null; } catch { return null; }
+    }
+
+    get model() { return (this.m ??= model()); }
+
+    /** Embed the files whose vectors are missing or stale; returns how many files were embedded. */
+    fill() {
+        const s = this.store;
+        const stale = s.all(`SELECT f.id, f.path, f.hash FROM files f LEFT JOIN vec_files v ON v.file_id = f.id
+            WHERE v.hash IS NULL OR v.hash <> f.hash`);
+        const gone = s.all('SELECT v.file_id AS id FROM vec_files v LEFT JOIN files f ON f.id = v.file_id WHERE f.id IS NULL');
+        if (!stale.length && !gone.length) return 0;
+        const dim = this.model.dim;
+        let n = 0;
+        const BATCH = 200;
+        for (let i = 0; i < stale.length; i += BATCH) {
+            s.tx(() => {
+                for (const f of stale.slice(i, i + BATCH)) {
+                    s.run('DELETE FROM vectors WHERE file_id = ?', f.id);
+                    let src = null;
+                    try { src = fs.readFileSync(path.join(this.ix.root, f.path), 'utf8'); } catch { /* deleted since */ }
+                    if (src != null) {
+                        const lines = src.split('\n');
+                        const syms = s.all('SELECT id, name, qname, kind, sig, doc, start_line, end_line FROM symbols WHERE file_id = ?', f.id);
+                        for (const sym of syms) {
+                            if (!EMBED_KINDS.has(sym.kind)) continue;
+                            const v = this.model.embed(symbolText(sym, f.path, lines));
+                            if (!v || v.length !== dim) continue;
+                            const { q, scale } = quantize(v);
+                            s.run('INSERT INTO vectors (symbol_id, file_id, scale, v) VALUES (?, ?, ?, ?)', sym.id, f.id, scale, Buffer.from(q.buffer));
+                        }
+                    }
+                    s.run('INSERT INTO vec_files (file_id, hash) VALUES (?, ?) ON CONFLICT(file_id) DO UPDATE SET hash = excluded.hash', f.id, f.hash);
+                    n++;
+                }
+            });
+        }
+        if (gone.length) s.tx(() => { for (const g of gone) { s.run('DELETE FROM vectors WHERE file_id = ?', g.id); s.run('DELETE FROM vec_files WHERE file_id = ?', g.id); } });
+        return n;
+    }
+
+    #load() {
+        if (this.version === this.ix.version && this.mat) return;
+        this.fill();
+        const rows = this.store.all('SELECT symbol_id, scale, v FROM vectors');
+        const dim = this.model.dim;
+        this.ids = new Int32Array(rows.length);
+        this.scales = new Float32Array(rows.length);
+        this.mat = new Int8Array(rows.length * dim);
+        rows.forEach((r, i) => {
+            this.ids[i] = r.symbol_id;
+            this.scales[i] = r.scale;
+            this.mat.set(new Int8Array(r.v.buffer, r.v.byteOffset, dim), i * dim);
+        });
+        this.version = this.ix.version;
+    }
+
+    /** Top-k definitions by cosine similarity to the query: Map(symbol id → cosine). */
+    search(query, k = 200) {
+        const qv = this.model.embed(query);
+        if (!qv) return new Map();
+        this.#load();
+        const dim = this.model.dim, M = this.mat, n = this.ids.length;
+        const top = [];  // small min-heap would be faster; k ≪ n and n is modest, a threshold scan is enough
+        let floor = -Infinity;
+        for (let i = 0; i < n; i++) {
+            let d = 0;
+            const o = i * dim;
+            for (let j = 0; j < dim; j++) d += M[o + j] * qv[j];
+            d *= this.scales[i];
+            if (top.length < k) { top.push([i, d]); if (top.length === k) { top.sort((a, b) => a[1] - b[1]); floor = top[0][1]; } }
+            else if (d > floor) {
+                top[0] = [i, d];
+                // re-establish the minimum at index 0 (insertion into sorted position)
+                let p = 0;
+                while (p + 1 < top.length && top[p][1] > top[p + 1][1]) { const t = top[p]; top[p] = top[p + 1]; top[p + 1] = t; p++; }
+                floor = top[0][1];
+            }
+        }
+        return new Map(top.map(([i, d]) => [this.ids[i], d]));
+    }
+}