graph-indexer 3.0.1 → 3.1.0

package/src/search/search.mjs

@@ -4,17 +4,20 @@
  *   1. name channel     exact / case-insensitive / qualified / prefix matches on symbol names
  *   2. lexical channel  FTS5 BM25 over fields (name, qualified name, signature, doc, path, body)
  *   3. concept channel  the same index queried with thesaurus expansions (natural-language queries)
- *   4. rerank           a small linear model over interpretable features (name evidence, field
+ *   4. semantic channel (optional) cosine similarity of a static code-embedding model (vectors.mjs),
+ *                       on when its model is installed
+ *   5. rerank           a small linear model over interpretable features (name evidence, field
  *                       coverage, kind/test/export priors, graph centrality), then per-file
  *                       diversification so one file cannot flood the top results.
  */
 import { analyzeQuery, codeTokens, splitIdentifier } from './tokenize.mjs';
+import { VectorIndex } from './vectors.mjs';
 
 const FTS_WEIGHTS = [10.0, 6.0, 3.0, 2.0, 2.5, 1.0]; // name, qname, sig, doc, path, body
 const KIND_PRIOR = {
     class: 1.0, interface: 0.95, struct: 1.0, trait: 0.95, enum: 0.85, type: 0.8, object: 0.95, module: 0.7, impl: 0.7,
     function: 1.0, method: 1.0, constructor: 0.75, macro: 0.8, field: 0.55, property: 0.6, variable: 0.55, constant: 0.6,
-    selector: 0.35, keyframes: 0.3,
+    selector: 0.35, keyframes: 0.3, section: 0.5,
 };
 
 export const DEFAULT_WEIGHTS = {
@@ -39,11 +42,21 @@ export const DEFAULT_WEIGHTS = {
     nlField: -0.93,     // fields/properties/variables for natural-language (behavioural) queries
     file: 1.0,          // file-level BM25 of the symbol's file (hierarchical: file relevance first)
     fileNL: 0.5,        // extra file-level weight for natural-language (behavioural) queries
+    docs: 2.0,          // documentation channel BM25 (only for queries about docs, design or rationale)
+    // semantic channel (opt-in): cosine, min-max normalised over its top candidates. Higher weights
+    // (0.5 / 2.0) score better on symbol search and worse on localizing real commits; these are
+    // the highest that leave localization within noise (docs/BENCHMARKS.md)
+    sem: 0.25,
+    semNL: 0.5,         // extra semantic weight for natural-language queries
 };
 
 const EXAMPLE_PATH = /(^|\/)(examples?|samples?|demos?|sandbox|playground|docs?(_src|_source|_examples?)?|documentation|benchmarks?|bench|scripts?|fixtures?|testdata|tutorials?|website|\.github)\//i;
 const DECLARATION_PATH = /\.d\.[cm]?ts$/;
+export const WANTS_DOCS = /\b(docs?|documentation|documented|readme|guide|tutorial|design|rationale|architecture|adr|why|explained?|explanation|changelog|faq|how to)\b/i;
 const BM25_FLOOR = 1.0;
+const SEM_K = 200; // semantic candidates per query
+const RRF_K = 60;
+export const MAX_QUERIES = 6;
 const FILE_GATE = 0.25; // own normalized BM25 at which the file boost applies in full (tuning split)
 
 function stem(w) {
@@ -58,7 +71,8 @@ function ftsQuery(terms) {
 }
 
 export class SearchEngine {
-    constructor(indexer, { weights = DEFAULT_WEIGHTS, centrality = null, perFileCap = Number(process.env.GI_PER_FILE_CAP) || 3 } = {}) {
+    constructor(indexer, { weights = DEFAULT_WEIGHTS, centrality = null, perFileCap = Number(process.env.GI_PER_FILE_CAP) || 3, semantic = undefined } = {}) {
+        this.semantic = semantic === undefined ? VectorIndex.maybe(indexer) : semantic;
         this.perFileCap = perFileCap;
         this.ix = indexer;
         this.store = indexer.store;
@@ -78,6 +92,7 @@ export class SearchEngine {
         const kindFilter = opts.kinds ?? a.filters.kind ?? null;
         const langFilter = opts.lang ?? a.filters.lang?.[0] ?? null;
         const wantsTests = opts.includeTests ?? /\b(test|tests|spec|specs|mock|fixture)\b/i.test(query);
+        const wantsDocs = WANTS_DOCS.test(query);
         const cands = new Map(); // id -> feature bag
         const bag = (id) => { let f = cands.get(id); if (!f) cands.set(id, (f = { id })); return f; };
 
@@ -140,6 +155,23 @@ export class SearchEngine {
                 for (const r of rows) { const f = bag(r.id); if (f.bm25 == null) { f.bm25 = -r.s; if (-r.s > maxBm) maxBm = -r.s; } }
             }
         }
+        // documentation channel: Markdown sections, only when the query is about docs, design or rationale
+        let maxDoc = 0;
+        if (q1 && (wantsDocs || opts.docs)) {
+            for (const r of this.store.all('SELECT rowid AS id, bm25(doc_fts, 4.0, 1.0) AS s FROM doc_fts WHERE doc_fts MATCH ? ORDER BY s LIMIT 60', q1)) {
+                bag(r.id).doc = -r.s; if (-r.s > maxDoc) maxDoc = -r.s;
+            }
+        }
+        // semantic channel: definitions whose embedding is closest to the query's
+        let semLo = 0, semHi = 0;
+        if (this.semantic && (this.w.sem || this.w.semNL)) {
+            const hits = this.semantic.search(query, SEM_K);
+            if (hits.size) {
+                const vals = [...hits.values()];
+                semHi = Math.max(...vals); semLo = Math.min(...vals);
+                for (const [id, c] of hits) bag(id).sem = c;
+            }
+        }
         if (!cands.size) return { results: [], analysis: a };
 
         // 4. features + rerank
@@ -161,6 +193,7 @@ export class SearchEngine {
             if (pathFilter && !r.path.startsWith(pathFilter.replace(/^\.\//, ''))) continue;
             if (kindFilter && !kindFilter.includes(r.kind)) continue;
             if (langFilter && r.lang !== langFilter) continue;
+            if (r.kind === 'section' && !wantsDocs && !opts.docs) continue; // documentation answers only questions about it
             const nameWords = splitIdentifier(r.name).map(stem);
             const qnameWords = new Set(splitIdentifier(r.qname.replace(/\./g, '_')).map(stem));
             const nameCover = nameWords.length ? nameWords.filter(x => qStems.has(x)).length / nameWords.length : 0;
@@ -171,7 +204,9 @@ export class SearchEngine {
             // blown up by max-normalisation; when literal evidence is weak, the concept channel
             // (thesaurus expansions) takes over — that is exactly the vocabulary-mismatch case.
             if (f.bm25) s += w.bm25 * (f.bm25 / Math.max(maxBm, BM25_FLOOR));
+            if (f.doc) s += (w.docs ?? 0) * (f.doc / Math.max(maxDoc, BM25_FLOOR));
             if (f.concept) s += (maxBm < BM25_FLOOR ? Math.max(w.concept, 1.0) : w.concept) * (f.concept / Math.max(maxCx, BM25_FLOOR));
+            if (f.sem != null && semHi > semLo) s += ((w.sem ?? 0) + (a.natural ? (w.semNL ?? 0) : 0)) * (f.sem - semLo) / (semHi - semLo);
             if (f.exact) { s += w.exact * f.exact; reasons.push('name'); }
             if (f.exactCI) { s += w.exactCI * f.exactCI; reasons.push('name'); }
             if (f.qualified) { s += w.qualified; reasons.push('qualified name'); }
@@ -188,23 +223,56 @@ export class SearchEngine {
             if (fs && f.bm25 && r.kind !== 'selector' && r.kind !== 'keyframes') s += (w.file + (a.natural ? w.fileNL : 0)) * fs * Math.min(1, (f.bm25 / Math.max(maxBm, BM25_FLOOR)) / FILE_GATE);
             if (r.parent_kind === 'function' || r.parent_kind === 'method') s += w.nested;
             if (DECLARATION_PATH.test(r.path)) s += w.declaration;
-            if (EXAMPLE_PATH.test(r.path) && !(pathFilter && EXAMPLE_PATH.test(pathFilter))) s += w.example;
+            if (r.kind !== 'section' && EXAMPLE_PATH.test(r.path) && !(pathFilter && EXAMPLE_PATH.test(pathFilter))) s += w.example;
             if (a.natural && (r.kind === 'field' || r.kind === 'property' || r.kind === 'variable' || r.kind === 'constant')) s += w.nlField;
             scored.push({ id: r.id, score: s, reasons, path: r.path, ...(opts.debug ? { features: { ...f, nameCover, queryCover, kind: r.kind } } : {}) });
         }
         scored.sort((x, y) => y.score - x.score || x.id - y.id);
-        // per-file diversification: 3rd+ result from the same file is pushed down
-        const perFile = new Map();
-        const out = [];
-        const deferred = [];
-        for (const r of scored) {
-            const n = perFile.get(r.path) ?? 0;
-            if (n >= this.perFileCap) { deferred.push(r); continue; }
-            perFile.set(r.path, n + 1);
-            out.push(r);
-            if (out.length >= limit) break;
-        }
-        for (const r of deferred) { if (out.length >= limit) break; out.push(r); }
-        return { results: out, total: scored.length, analysis: a };
+        return { results: diversify(scored, limit, this.perFileCap), total: scored.length, analysis: a };
+    }
+
+    /**
+     * Several phrasings of one need ("where are tokens refreshed", "refreshToken", "session renewal")
+     * in one call: each is searched on its own and the rankings are fused by reciprocal rank
+     * (k = 60), so a definition several phrasings agree on rises and one phrasing's noise does not
+     * dominate. One query is the same as search().
+     */
+    searchMany(queries, opts = {}) {
+        const qs = [...new Set((queries ?? []).map(q => String(q ?? '').trim()).filter(Boolean))].slice(0, MAX_QUERIES);
+        if (qs.length <= 1) return this.search(qs[0] ?? '', opts);
+        const limit = Math.max(1, Math.min(opts.limit ?? 10, 100));
+        const per = Math.max(2 * limit, 20);
+        const fused = new Map();
+        let total = 0, analysis = null;
+        qs.forEach((q, qi) => {
+            const r = this.search(q, { ...opts, limit: per });
+            analysis ??= r.analysis;
+            total = Math.max(total, r.total ?? r.results.length);
+            r.results.forEach((x, rank) => {
+                let e = fused.get(x.id);
+                if (!e) fused.set(x.id, (e = { id: x.id, score: 0, reasons: [], path: x.path, queries: [] }));
+                e.score += 1 / (RRF_K + rank + 1);
+                e.queries.push(qi);
+                for (const why of x.reasons) if (!e.reasons.includes(why)) e.reasons.push(why);
+            });
+        });
+        const scored = [...fused.values()].sort((x, y) => y.score - x.score || x.id - y.id);
+        return { results: diversify(scored, limit, this.perFileCap), total: Math.max(total, scored.length), analysis, queries: qs };
+    }
+}
+
+/** Per-file diversification: results past the cap from one file are pushed below the others. */
+function diversify(scored, limit, cap) {
+    const perFile = new Map();
+    const out = [];
+    const deferred = [];
+    for (const r of scored) {
+        const n = perFile.get(r.path) ?? 0;
+        if (n >= cap) { deferred.push(r); continue; }
+        perFile.set(r.path, n + 1);
+        out.push(r);
+        if (out.length >= limit) break;
     }
+    for (const r of deferred) { if (out.length >= limit) break; out.push(r); }
+    return out;
 }

package/src/search/semantic.mjs

@@ -0,0 +1,201 @@
+/**
+ * Optional semantic channel: a static (Model2Vec) code embedding model, run in plain JavaScript.
+ *
+ * A static model is a table of one vector per WordPiece token; a text's embedding is the mean of
+ * its tokens' vectors, normalised. There is no neural network to run, so it needs no native code,
+ * no ONNX runtime and no GPU: embedding a whole repository takes seconds on one core.
+ *
+ * The model is not part of the package. `graph-indexer semantic install` downloads it once, at a
+ * pinned revision with checked SHA-256 digests, into ~/.cache/graph-indexer/models (or
+ * GRAPH_INDEXER_MODEL_DIR). The channel is opt-in: it runs only with GRAPH_INDEXER_SEMANTIC=1 and
+ * the model installed; otherwise search is purely lexical + graph. On the benchmarks it helps
+ * symbol search for queries in words and does not help localizing real commits (docs/BENCHMARKS.md),
+ * so it is not the default.
+ *
+ * Vectors are computed lazily, file by file, the first time a search needs them, and kept in the
+ * index (`vectors`, `vec_files`), keyed by the file's content hash so edits re-embed only that file.
+ */
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import crypto from 'node:crypto';
+
+export const MODEL = {
+    id: 'potion-code-16M-v2',
+    repo: 'minishlab/potion-code-16M-v2',
+    revision: 'e9d2a44ca6a05ac6685f3b23709ea57eb7352d5b',
+    license: 'MIT',
+    files: {
+        'model.safetensors': '75cf7a6c2171b230ad19b1e7d8e0b1aee86da5a02af8e7cacedd9921d227623c',
+        'tokenizer.json': '107bbdcbad4bff1d299b7a4c3a2fb17c52890688b7dd0e4c9deab79d3c4f3d45',
+        'config.json': '148e5691a6fcc553437156859701fba017a1ba5d340b170f17e0f3668fb861a7',
+    },
+};
+
+const MAX_TOKENS = 512;       // Model2Vec's default truncation
+const MAX_WORD_CHARS = 100;   // WordPiece max_input_chars_per_word
+
+export function modelDir() {
+    const base = process.env.GRAPH_INDEXER_MODEL_DIR || path.join(os.homedir(), '.cache', 'graph-indexer', 'models');
+    return path.join(base, MODEL.id);
+}
+
+export function semanticEnabled() {
+    if (process.env.GRAPH_INDEXER_SEMANTIC !== '1') return false;
+    return fs.existsSync(path.join(modelDir(), 'model.safetensors')) && fs.existsSync(path.join(modelDir(), 'tokenizer.json'));
+}
+
+// ── tokenizer: BertNormalizer(lowercase, strip accents) + BertPreTokenizer + WordPiece ──────────
+
+function isControl(c) {
+    if (c === 9 || c === 10 || c === 13) return false;
+    return (c < 32) || (c >= 127 && c < 160);
+}
+function isWhitespace(c) { return c === 32 || c === 9 || c === 10 || c === 13 || c === 0xa0 || c === 0x3000 || (c >= 0x2000 && c <= 0x200a) || c === 0x2028 || c === 0x2029 || c === 0x202f || c === 0x205f || c === 0x1680; }
+function isCJK(c) {
+    return (c >= 0x4e00 && c <= 0x9fff) || (c >= 0x3400 && c <= 0x4dbf) || (c >= 0x20000 && c <= 0x2a6df) || (c >= 0x2a700 && c <= 0x2b73f) ||
+        (c >= 0x2b740 && c <= 0x2b81f) || (c >= 0x2b820 && c <= 0x2ceaf) || (c >= 0xf900 && c <= 0xfaff) || (c >= 0x2f800 && c <= 0x2fa1f);
+}
+const PUNCT_RE = /\p{P}/u;
+function isPunct(ch, c) {
+    return (c >= 33 && c <= 47) || (c >= 58 && c <= 64) || (c >= 91 && c <= 96) || (c >= 123 && c <= 126) || (c > 127 && PUNCT_RE.test(ch));
+}
+
+/** Text → normalised pre-tokens (words and single punctuation characters). */
+export function preTokenize(text) {
+    let s = '';
+    for (const ch of text) {
+        const c = ch.codePointAt(0);
+        if (c === 0 || c === 0xfffd || isControl(c)) continue;
+        if (isWhitespace(c)) { s += ' '; continue; }
+        if (isCJK(c)) { s += ' ' + ch + ' '; continue; }
+        s += ch;
+    }
+    s = s.toLowerCase().normalize('NFD').replace(/\p{Mn}/gu, '');
+    const out = [];
+    let cur = '';
+    for (const ch of s) {
+        const c = ch.codePointAt(0);
+        if (c === 32) { if (cur) { out.push(cur); cur = ''; } continue; }
+        if (isPunct(ch, c)) { if (cur) { out.push(cur); cur = ''; } out.push(ch); continue; }
+        cur += ch;
+    }
+    if (cur) out.push(cur);
+    return out;
+}
+
+export class StaticModel {
+    constructor(dir = modelDir()) {
+        const tok = JSON.parse(fs.readFileSync(path.join(dir, 'tokenizer.json'), 'utf8'));
+        const m = tok.model;
+        if (m?.type !== 'WordPiece') throw new Error(`unsupported tokenizer model ${m?.type}`);
+        this.vocab = new Map(Object.entries(m.vocab));
+        this.prefix = m.continuing_subword_prefix ?? '##';
+        this.unk = this.vocab.get(m.unk_token ?? '[UNK]') ?? -1;
+        const buf = fs.readFileSync(path.join(dir, 'model.safetensors'));
+        const hlen = Number(buf.readBigUInt64LE(0));
+        const header = JSON.parse(buf.subarray(8, 8 + hlen).toString('utf8'));
+        const t = header.embeddings ?? Object.values(header).find(v => v && v.shape);
+        const [rows, dim] = t.shape;
+        const [a, b] = t.data_offsets;
+        const raw = buf.subarray(8 + hlen + a, 8 + hlen + b);
+        this.dim = dim;
+        this.rows = rows;
+        this.table = new Float32Array(rows * dim);
+        if (t.dtype === 'F32') for (let i = 0; i < rows * dim; i++) this.table[i] = raw.readFloatLE(i * 4);
+        else if (t.dtype === 'F16') for (let i = 0; i < rows * dim; i++) this.table[i] = f16(raw.readUInt16LE(i * 2));
+        else throw new Error(`unsupported embedding dtype ${t.dtype}`);
+        this.cache = new Map(); // word -> token ids
+    }
+
+    /** WordPiece ids of one pre-token (greedy longest match), [] when unknown. */
+    #word(w) {
+        let ids = this.cache.get(w);
+        if (ids) return ids;
+        ids = [];
+        if ([...w].length > MAX_WORD_CHARS) ids = [this.unk];
+        else {
+            let start = 0;
+            while (start < w.length) {
+                let end = w.length, found = -1;
+                while (start < end) {
+                    const sub = (start > 0 ? this.prefix : '') + w.slice(start, end);
+                    const id = this.vocab.get(sub);
+                    if (id !== undefined) { found = id; break; }
+                    end--;
+                }
+                if (found < 0) { ids = [this.unk]; break; }
+                ids.push(found);
+                start = end;
+            }
+        }
+        if (this.cache.size > 200000) this.cache.clear();
+        this.cache.set(w, ids);
+        return ids;
+    }
+
+    tokenize(text) {
+        const out = [];
+        for (const w of preTokenize(text)) {
+            for (const id of this.#word(w)) { if (id !== this.unk) out.push(id); }
+            if (out.length >= MAX_TOKENS) break;
+        }
+        return out.length > MAX_TOKENS ? out.slice(0, MAX_TOKENS) : out;
+    }
+
+    /** Mean of token vectors, L2-normalised; null for text without known tokens. */
+    embed(text) {
+        const ids = this.tokenize(text);
+        if (!ids.length) return null;
+        const d = this.dim, v = new Float32Array(d), T = this.table;
+        for (const id of ids) { const o = id * d; for (let j = 0; j < d; j++) v[j] += T[o + j]; }
+        let n = 0;
+        for (let j = 0; j < d; j++) n += v[j] * v[j];
+        n = Math.sqrt(n);
+        if (!n) return null;
+        for (let j = 0; j < d; j++) v[j] /= n;
+        return v;
+    }
+}
+
+function f16(h) {
+    const s = (h & 0x8000) ? -1 : 1, e = (h >> 10) & 0x1f, f = h & 0x3ff;
+    if (e === 0) return s * Math.pow(2, -14) * (f / 1024);
+    if (e === 31) return f ? NaN : s * Infinity;
+    return s * Math.pow(2, e - 15) * (1 + f / 1024);
+}
+
+/** int8 with one scale per vector: v ≈ q * scale. */
+export function quantize(v) {
+    let m = 0;
+    for (let j = 0; j < v.length; j++) { const a = Math.abs(v[j]); if (a > m) m = a; }
+    const q = new Int8Array(v.length);
+    const scale = m / 127 || 1;
+    for (let j = 0; j < v.length; j++) q[j] = Math.round(v[j] / scale);
+    return { q, scale };
+}
+
+// ── model download (explicit, user-initiated) ──────────────────────────────────────────────────
+
+export async function installModel({ log = console.error } = {}) {
+    const dir = modelDir();
+    fs.mkdirSync(dir, { recursive: true });
+    for (const [name, digest] of Object.entries(MODEL.files)) {
+        const dest = path.join(dir, name);
+        if (fs.existsSync(dest) && sha256File(dest) === digest) { log(`${name}: present`); continue; }
+        const url = `https://huggingface.co/${MODEL.repo}/resolve/${MODEL.revision}/${name}`;
+        log(`downloading ${url}`);
+        const res = await fetch(url, { redirect: 'follow' });
+        if (!res.ok) throw new Error(`${name}: HTTP ${res.status}`);
+        const data = Buffer.from(await res.arrayBuffer());
+        const got = crypto.createHash('sha256').update(data).digest('hex');
+        if (got !== digest) throw new Error(`${name}: SHA-256 mismatch (expected ${digest}, got ${got}); nothing was installed`);
+        const tmp = dest + '.part';
+        fs.writeFileSync(tmp, data);
+        fs.renameSync(tmp, dest);
+        log(`${name}: ${(data.length / 1048576).toFixed(1)} MB, verified`);
+    }
+    return dir;
+}
+
+function sha256File(p) { return crypto.createHash('sha256').update(fs.readFileSync(p)).digest('hex'); }

package/src/search/vectors.mjs

@@ -0,0 +1,122 @@
+/**
+ * Vector index over definitions for the semantic channel (see semantic.mjs).
+ *
+ * One int8 vector per definition, embedded from its qualified name, signature, doc comment and
+ * the head of its body. Vectors are filled lazily, file by file, keyed by the file's content
+ * hash, so an edit re-embeds only the files it touched. Search is an exact scan over the matrix
+ * held in memory (no ANN structure to go stale): ~1 ms per 10k definitions.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { StaticModel, quantize, semanticEnabled } from './semantic.mjs';
+import { splitIdentifier } from './tokenize.mjs';
+
+const EMBED_KINDS = new Set(['class', 'interface', 'struct', 'enum', 'trait', 'type', 'object', 'module', 'impl',
+    'function', 'method', 'constructor', 'macro', 'section']);
+const BODY_LINES = 60;
+
+let sharedModel = null;
+function model() { return (sharedModel ??= new StaticModel()); }
+
+/** The text a definition is embedded from. */
+export function symbolText(sym, path_, lines) {
+    const words = splitIdentifier(sym.name).join(' ');
+    const head = `${path_}\n${sym.qname} (${words})\n${sym.sig ?? ''}\n${sym.doc ?? ''}\n`;
+    const body = lines.slice(sym.start_line - 1, Math.min(sym.end_line, sym.start_line - 1 + BODY_LINES)).join('\n');
+    return head + body;
+}
+
+export class VectorIndex {
+    constructor(indexer, { m = null } = {}) {
+        this.ix = indexer;
+        this.store = indexer.store;
+        this.m = m;
+        this.version = -1;
+        this.ids = null;      // Int32Array of symbol ids
+        this.mat = null;      // Int8Array rows × dim
+        this.scales = null;   // Float32Array
+    }
+
+    static maybe(indexer) {
+        try { return semanticEnabled() ? new VectorIndex(indexer) : null; } catch { return null; }
+    }
+
+    get model() { return (this.m ??= model()); }
+
+    /** Embed the files whose vectors are missing or stale; returns how many files were embedded. */
+    fill() {
+        const s = this.store;
+        const stale = s.all(`SELECT f.id, f.path, f.hash FROM files f LEFT JOIN vec_files v ON v.file_id = f.id
+            WHERE v.hash IS NULL OR v.hash <> f.hash`);
+        const gone = s.all('SELECT v.file_id AS id FROM vec_files v LEFT JOIN files f ON f.id = v.file_id WHERE f.id IS NULL');
+        if (!stale.length && !gone.length) return 0;
+        const dim = this.model.dim;
+        let n = 0;
+        const BATCH = 200;
+        for (let i = 0; i < stale.length; i += BATCH) {
+            s.tx(() => {
+                for (const f of stale.slice(i, i + BATCH)) {
+                    s.run('DELETE FROM vectors WHERE file_id = ?', f.id);
+                    let src = null;
+                    try { src = fs.readFileSync(path.join(this.ix.root, f.path), 'utf8'); } catch { /* deleted since */ }
+                    if (src != null) {
+                        const lines = src.split('\n');
+                        const syms = s.all('SELECT id, name, qname, kind, sig, doc, start_line, end_line FROM symbols WHERE file_id = ?', f.id);
+                        for (const sym of syms) {
+                            if (!EMBED_KINDS.has(sym.kind)) continue;
+                            const v = this.model.embed(symbolText(sym, f.path, lines));
+                            if (!v || v.length !== dim) continue;
+                            const { q, scale } = quantize(v);
+                            s.run('INSERT INTO vectors (symbol_id, file_id, scale, v) VALUES (?, ?, ?, ?)', sym.id, f.id, scale, Buffer.from(q.buffer));
+                        }
+                    }
+                    s.run('INSERT INTO vec_files (file_id, hash) VALUES (?, ?) ON CONFLICT(file_id) DO UPDATE SET hash = excluded.hash', f.id, f.hash);
+                    n++;
+                }
+            });
+        }
+        if (gone.length) s.tx(() => { for (const g of gone) { s.run('DELETE FROM vectors WHERE file_id = ?', g.id); s.run('DELETE FROM vec_files WHERE file_id = ?', g.id); } });
+        return n;
+    }
+
+    #load() {
+        if (this.version === this.ix.version && this.mat) return;
+        this.fill();
+        const rows = this.store.all('SELECT symbol_id, scale, v FROM vectors');
+        const dim = this.model.dim;
+        this.ids = new Int32Array(rows.length);
+        this.scales = new Float32Array(rows.length);
+        this.mat = new Int8Array(rows.length * dim);
+        rows.forEach((r, i) => {
+            this.ids[i] = r.symbol_id;
+            this.scales[i] = r.scale;
+            this.mat.set(new Int8Array(r.v.buffer, r.v.byteOffset, dim), i * dim);
+        });
+        this.version = this.ix.version;
+    }
+
+    /** Top-k definitions by cosine similarity to the query: Map(symbol id → cosine). */
+    search(query, k = 200) {
+        const qv = this.model.embed(query);
+        if (!qv) return new Map();
+        this.#load();
+        const dim = this.model.dim, M = this.mat, n = this.ids.length;
+        const top = [];  // small min-heap would be faster; k ≪ n and n is modest, a threshold scan is enough
+        let floor = -Infinity;
+        for (let i = 0; i < n; i++) {
+            let d = 0;
+            const o = i * dim;
+            for (let j = 0; j < dim; j++) d += M[o + j] * qv[j];
+            d *= this.scales[i];
+            if (top.length < k) { top.push([i, d]); if (top.length === k) { top.sort((a, b) => a[1] - b[1]); floor = top[0][1]; } }
+            else if (d > floor) {
+                top[0] = [i, d];
+                // re-establish the minimum at index 0 (insertion into sorted position)
+                let p = 0;
+                while (p + 1 < top.length && top[p][1] > top[p + 1][1]) { const t = top[p]; top[p] = top[p + 1]; top[p + 1] = t; p++; }
+                floor = top[0][1];
+            }
+        }
+        return new Map(top.map(([i, d]) => [this.ids[i], d]));
+    }
+}

package/src/store/db.mjs

@@ -15,7 +15,7 @@ process.emitWarning = function (warning, ...rest) {
 };
 const { DatabaseSync } = await import('node:sqlite');
 
-export const SCHEMA_VERSION = 9;
+export const SCHEMA_VERSION = 11;
 
 const SCHEMA = `
 CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT);
@@ -29,7 +29,8 @@ CREATE TABLE IF NOT EXISTS files (
   lines INTEGER NOT NULL,
   package TEXT,
   is_test INTEGER NOT NULL DEFAULT 0,
-  parse_errors INTEGER NOT NULL DEFAULT 0,
+  parse_errors INTEGER NOT NULL DEFAULT 0, -- regions the parser could not read (ERROR nodes)
+  parse_error_lines TEXT,                   -- their [start, end] lines as JSON, the first 100
   indexed_at INTEGER NOT NULL
 );
 CREATE TABLE IF NOT EXISTS symbols (
@@ -69,7 +70,8 @@ CREATE TABLE IF NOT EXISTS refs (
   dst_id INTEGER,
   conf REAL NOT NULL DEFAULT 0,
   ncand INTEGER NOT NULL DEFAULT 0,
-  argc INTEGER -- arguments of a call or construction (-1: a spread makes it a lower bound), else NULL
+  argc INTEGER, -- arguments of a call or construction (-1: a spread makes it a lower bound), else NULL
+  arg_types TEXT -- their static types where the language overloads by type (JSON: "int", "String", "?" unknown), else NULL
 );
 CREATE INDEX IF NOT EXISTS refs_file ON refs(file_id);
 CREATE INDEX IF NOT EXISTS refs_dst ON refs(dst_id);
@@ -102,6 +104,21 @@ CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(
   tokenize = 'porter unicode61 remove_diacritics 2',
   content = '', contentless_delete = 1
 );
+-- semantic channel (optional, src/search/vectors.mjs): one int8 vector per definition, filled lazily
+CREATE TABLE IF NOT EXISTS vectors (
+  symbol_id INTEGER PRIMARY KEY,
+  file_id INTEGER NOT NULL,
+  scale REAL NOT NULL,
+  v BLOB NOT NULL
+);
+CREATE INDEX IF NOT EXISTS vectors_file ON vectors(file_id);
+CREATE TABLE IF NOT EXISTS vec_files (file_id INTEGER PRIMARY KEY, hash TEXT NOT NULL);
+-- documentation sections (Markdown headings), apart from code so prose never shifts code's BM25 statistics
+CREATE VIRTUAL TABLE IF NOT EXISTS doc_fts USING fts5(
+  name, body,
+  tokenize = 'porter unicode61 remove_diacritics 2',
+  content = '', contentless_delete = 1
+);
 -- one document per file (rowid = files.id): file-level relevance for hierarchical ranking
 CREATE VIRTUAL TABLE IF NOT EXISTS file_fts USING fts5(
   path, body,
@@ -156,9 +173,10 @@ export class Store {
     /** Drop every indexed fact but keep the database (the index is a cache of the source tree). */
     reset() {
         this.tx(() => {
-            for (const t of ['files', 'symbols', 'refs', 'imports', 'fields']) this.db.exec(`DELETE FROM ${t}`);
+            for (const t of ['files', 'symbols', 'refs', 'imports', 'fields', 'vectors', 'vec_files']) this.db.exec(`DELETE FROM ${t}`);
             this.db.exec("INSERT INTO fts(fts) VALUES ('delete-all')");
             this.db.exec("INSERT INTO file_fts(file_fts) VALUES ('delete-all')");
+            this.db.exec("INSERT INTO doc_fts(doc_fts) VALUES ('delete-all')");
         });
     }
 

package/vendor/grammars/README.md

@@ -0,0 +1,47 @@
+# Vendored grammars
+
+Each `NAME.wasm.gz` is a tree-sitter grammar compiled to WebAssembly. `manifest.json` records where it
+comes from and the SHA-256 and size of the uncompressed `.wasm`; `test/vendor.test.mjs` checks every
+file against it, and the index fingerprint covers the manifest, so a changed grammar rebuilds indexes.
+
+- **`tag` and `asset`**: the `.wasm` published with that release of the grammar's repository,
+  downloaded as is.
+- **`commit` and `built`**: the grammar has no release with what graph-indexer needs (Kotlin: the
+  latest release, 0.3.8, predates `fun interface` and Kotlin 2.x syntax), so the `.wasm` is built
+  from that commit. The build is reproducible: anyone can check the vendored file against its
+  source.
+
+## Building a grammar from a commit
+
+This is `tree-sitter build --wasm` with tree-sitter-cli 0.25.10, the version of the vendored
+`web-tree-sitter`, run as the `emcc` command that CLI issues (`compile_parser_to_wasm` in
+`cli/loader/src/lib.rs`), in the Emscripten image it pins. The checkout must be under a directory
+the container can mount (with Colima, your home directory).
+
+```sh
+git clone https://github.com/fwcd/tree-sitter-kotlin && cd tree-sitter-kotlin
+git checkout d6af96d2e23ef0a8ff5ab512864378cf08fba311
+docker run --rm --platform linux/amd64 --workdir /src/src -v "$PWD":/src --user "$(id -u)" \
+  emscripten/emsdk:4.0.4 emcc -o output.wasm -Os -s WASM=1 -s SIDE_MODULE=2 -s TOTAL_MEMORY=33554432 \
+  -s NODEJS_CATCH_EXIT=0 -s 'EXPORTED_FUNCTIONS=["_tree_sitter_kotlin"]' -fno-exceptions -fvisibility=hidden \
+  -I . scanner.c parser.c
+shasum -a 256 src/output.wasm          # must equal manifest.json's sha256
+gzip -9 -n -c src/output.wasm > kotlin.wasm.gz
+```
+
+For another grammar, replace the repository, the commit and the exported symbol
+(`_tree_sitter_<name>`). The grammar's `src/parser.c` must declare `LANGUAGE_VERSION` 14 or 15,
+the ABI versions web-tree-sitter 0.25 loads.
+
+## Checking a new grammar before vendoring it
+
+A grammar change can move or rename nodes the extraction queries in `src/parse/lang/` match. Before
+replacing one:
+
+1. `npm test`, which compiles every query against its grammar.
+2. Parse a corpus of real code in that language with the old and the new grammar and compare the
+   files with ERROR regions, the symbols (by kind and qualified name) and the references each
+   extracts. A symbol that disappears needs a reason; one that changes its qualified name usually
+   moved back into its class.
+3. For a language with a compiler oracle (`bench/eval-graph*.mjs`), the references must keep their
+   precision and recall.

package/vendor/grammars/manifest.json

@@ -82,10 +82,10 @@
     },
     "kotlin": {
       "repo": "https://github.com/fwcd/tree-sitter-kotlin",
-      "tag": "0.3.8",
-      "asset": "tree-sitter-kotlin.wasm",
-      "sha256": "c624e7443b371c28adc5d81674e73067564c12555ebe3ed96a6c8db814b7602d",
-      "bytes": 4052625,
+      "commit": "d6af96d2e23ef0a8ff5ab512864378cf08fba311",
+      "built": "tree-sitter build --wasm (tree-sitter-cli 0.25.10, emscripten/emsdk:4.0.4): Kotlin 2.x syntax (guard conditions, `..<`, `$$` strings, context parameters) and `fun interface` are in no release yet, the latest being 0.3.8",
+      "sha256": "90689092295c4e11d21f2c4f1b9ffce1a3c68f7ce6d4de57c365cd0fc888c7ee",
+      "bytes": 6078823,
       "license": "MIT"
     },
     "php": {