graph-indexer 3.0.0 → 3.1.0

package/src/query/testcmd.mjs

@@ -6,16 +6,24 @@
  */
 import fs from 'node:fs';
 import path from 'node:path';
+import { readRepoFile } from '../util/paths.mjs';
 
-function readJson(p) { try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch { return null; } }
+function readJson(root, rel) { try { return JSON.parse(readRepoFile(root, rel)); } catch { return null; } }
 const exists = (root, rel) => fs.existsSync(path.join(root, rel));
+// commands are pasted into a shell as they are: a file named `t$(cmd).test.ts` (repository content)
+// must reach the test runner as a name, never run as a command substitution
+const plain = (s) => /^[\w./@+=:,-]+$/.test(s);
+/** One shell word: as is when plain, else single-quoted. */
+const q = (s) => (plain(s) ? s : `'${s.replace(/'/g, `'\\''`)}'`);
+// a name that would read as an option, or a test identifier that is not a plain word, gets no command
+const shellSafe = (s) => !s.startsWith('-') && !/[\0\r\n]/.test(s);
 
 /** Nearest package.json above a file (monorepos have one per package). */
 function nearestPackage(root, rel) {
     let dir = path.posix.dirname(rel);
     for (;;) {
-        const p = path.join(root, dir === '.' ? '' : dir, 'package.json');
-        if (fs.existsSync(p)) return { dir: dir === '.' ? '' : dir, pkg: readJson(p) ?? {} };
+        const p = path.posix.join(dir === '.' ? '' : dir, 'package.json');
+        if (exists(root, p)) return { dir: dir === '.' ? '' : dir, pkg: readJson(root, p) ?? {} };
         if (dir === '.' || dir === '' || dir === '/') return null;
         dir = path.posix.dirname(dir);
     }
@@ -23,7 +31,7 @@ function nearestPackage(root, rel) {
 
 function jsRunner(root, rel) {
     const near = nearestPackage(root, rel);
-    const top = readJson(path.join(root, 'package.json')) ?? {};
+    const top = readJson(root, 'package.json') ?? {};
     const pkgs = [near?.pkg, top].filter(Boolean);
     const deps = Object.assign({}, ...pkgs.map(p => ({ ...p.dependencies, ...p.devDependencies })));
     const scripts = pkgs.map(p => p.scripts?.test ?? '').join(' ');
@@ -38,7 +46,7 @@ function jsRunner(root, rel) {
 
 function pyRunner(root) {
     if (exists(root, 'tests/runtests.py')) return { kind: 'django' };
-    const pyproject = (() => { try { return fs.readFileSync(path.join(root, 'pyproject.toml'), 'utf8'); } catch { return ''; } })();
+    const pyproject = readRepoFile(root, 'pyproject.toml') ?? '';
     if (exists(root, 'pytest.ini') || exists(root, 'conftest.py') || /\[tool\.pytest/.test(pyproject) || exists(root, 'tox.ini')) return { kind: 'pytest' };
     return { kind: 'unittest' };
 }
@@ -51,7 +59,7 @@ function pyRunner(root) {
 export function testCommands(root, files) {
     const groups = new Map();
     const add = (key, f) => (groups.get(key) ?? groups.set(key, []).get(key)).push(f);
-    for (const f of files) {
+    for (const f of files.filter(shellSafe)) {
         const ext = path.posix.extname(f);
         if (/\.[cm]?[jt]sx?$/.test(ext)) add(`js:${jsRunner(root, f)}`, f);
         else if (ext === '.py') add('py', f);
@@ -64,19 +72,20 @@ export function testCommands(root, files) {
     }
     const out = [];
     for (const [key, fs_] of groups) {
-        if (key.startsWith('js:')) out.push(`${key.slice(3)} ${fs_.join(' ')}`);
+        if (key.startsWith('js:')) out.push(`${key.slice(3)} ${fs_.map(q).join(' ')}`);
         else if (key === 'py') {
             const r = pyRunner(root);
-            if (r.kind === 'django') out.push(`python tests/runtests.py ${[...new Set(fs_.map(f => f.replace(/^tests\//, '').replace(/\/[^/]*$/, '').replace(/\//g, '.')))].join(' ')}`);
-            else if (r.kind === 'pytest') out.push(`python -m pytest -q ${fs_.join(' ')}`);
-            else out.push(`python -m unittest ${fs_.map(f => f.replace(/\.py$/, '').replace(/\//g, '.')).join(' ')}`);
-        } else if (key === 'go') out.push(`go test ${[...new Set(fs_.map(f => './' + path.posix.dirname(f)))].join(' ')}`);
+            if (r.kind === 'django') out.push(`python tests/runtests.py ${[...new Set(fs_.map(f => f.replace(/^tests\//, '').replace(/\/[^/]*$/, '').replace(/\//g, '.')))].map(q).join(' ')}`);
+            else if (r.kind === 'pytest') out.push(`python -m pytest -q ${fs_.map(q).join(' ')}`);
+            else out.push(`python -m unittest ${fs_.map(f => q(f.replace(/\.py$/, '').replace(/\//g, '.'))).join(' ')}`);
+        } else if (key === 'go') out.push(`go test ${[...new Set(fs_.map(f => './' + path.posix.dirname(f)))].map(q).join(' ')}`);
         else if (key === 'rs') out.push('cargo test');
         else if (key === 'jvm') {
-            const classes = [...new Set(fs_.map(f => path.posix.basename(f).replace(/\.\w+$/, '')))].join(',');
+            const classes = [...new Set(fs_.map(f => path.posix.basename(f).replace(/\.\w+$/, '')).filter(c => /^\w+$/.test(c)))].join(',');
+            if (!classes) continue;
             out.push(exists(root, 'pom.xml') ? `mvn -q test -Dtest=${classes}` : `./gradlew test ${classes.split(',').map(c => `--tests '*${c}'`).join(' ')}`);
-        } else if (key === 'rb') out.push(exists(root, '.rspec') || fs_.some(f => f.endsWith('_spec.rb')) ? `bundle exec rspec ${fs_.join(' ')}` : `bundle exec ruby -Itest ${fs_[0]}`);
-        else if (key === 'php') out.push(`vendor/bin/phpunit ${fs_.join(' ')}`);
+        } else if (key === 'rb') out.push(exists(root, '.rspec') || fs_.some(f => f.endsWith('_spec.rb')) ? `bundle exec rspec ${fs_.map(q).join(' ')}` : `bundle exec ruby -Itest ${q(fs_[0])}`);
+        else if (key === 'php') out.push(`vendor/bin/phpunit ${fs_.map(q).join(' ')}`);
         else if (key === 'cs') out.push('dotnet test');
     }
     return out;
@@ -90,6 +99,8 @@ export function testCommands(root, files) {
  */
 export function testCaseCommands(root, cases) {
     const out = [];
+    const word = (s) => /^\w+$/.test(s);
+    cases = cases.filter(c => shellSafe(c.path) && plain(c.path) && word(c.name) && (!c.cls || word(c.cls)));
     const py = cases.filter(c => c.path.endsWith('.py'));
     if (py.length) {
         const r = pyRunner(root);

package/src/query/textsearch.mjs

@@ -7,15 +7,26 @@
  */
 import fs from 'node:fs';
 import path from 'node:path';
-import { spawnSync } from 'node:child_process';
+import vm from 'node:vm';
+import { withinRoot } from '../util/paths.mjs';
+import { gitSync } from '../util/git.mjs';
 
 const MAX_FILE_BYTES = 1_500_000;
+// a caller's pattern gets this much matching time per search (reading files does not count): one
+// with catastrophic backtracking (`(a+)+$`) is stopped instead of blocking the server
+const REGEX_BUDGET_MS = 5000;
+const SCAN = new vm.Script(`(() => {
+    if (!re.test(text)) return null;
+    const lines = text.split(/\\r?\\n/), hits = [];
+    for (let i = 0; i < lines.length; i++) if (re.test(lines[i])) hits.push(i);
+    return { lines, hits };
+})()`);
 const SKIP_DIRS = /(^|\/)(node_modules|\.git|\.graph-indexer|dist|build|out|target|coverage|__pycache__|\.venv|venv|\.next|\.nuxt|\.cache|vendor)\//;
 const EXAMPLE_PATH = /(^|\/)(docs?_src|docs?|examples?|samples?|demos?|tutorials?|benchmarks?|playground)\//i;
 const COMMENT_LINE = /^\s*(\/\/|#(?![!\[])|\*|\/\*|<!--|--\s|;;|%)/;
 
 function listFiles(root) {
-    const r = spawnSync('git', ['ls-files', '-z', '--cached', '--others', '--exclude-standard'], { cwd: root, encoding: 'utf8', maxBuffer: 1 << 30 });
+    const r = gitSync(root, ['ls-files', '-z', '--cached', '--others', '--exclude-standard'], { maxBuffer: 1 << 30 });
     if (r.status === 0) return r.stdout.split('\0').filter(Boolean).filter(p => !SKIP_DIRS.test(p));
     const out = [];
     const stack = [''];
@@ -98,25 +109,32 @@ export async function textSearch(intel, { pattern, path: prefix = null, literal
     const indexed = new Map(intel.store.all('SELECT id, path, is_test FROM files').map(f => [f.path, f]));
     const matches = [];
     let total = 0, nfiles = 0;
+    const ctx = vm.createContext({ re, text: '' });
+    let budget = REGEX_BUDGET_MS;
     for (const rel of files) {
         let text;
         try {
+            if (!withinRoot(root, rel)) continue;
             const st = fs.statSync(path.join(root, rel));
             if (!st.isFile() || st.size > MAX_FILE_BYTES) continue;
             text = fs.readFileSync(path.join(root, rel), 'utf8');
         } catch { continue; }
-        if (!re.test(text)) continue;
         if (text.indexOf('\0') >= 0 && text.indexOf('\0') < 8000) continue; // binary
-        const lines = text.split(/\r?\n/);
-        let any = false;
-        for (let i = 0; i < lines.length; i++) {
-            if (!re.test(lines[i])) continue;
-            any = true;
+        ctx.text = text;
+        let scan;
+        const t0 = performance.now();
+        try { scan = SCAN.runInContext(ctx, { timeout: Math.max(1, Math.ceil(budget)) }); } catch (e) {
+            if (e.code !== 'ERR_SCRIPT_EXECUTION_TIMEOUT') throw e;
+            return { error: `The regular expression took more than ${REGEX_BUDGET_MS / 1000} s (catastrophic backtracking?). Simplify it, or pass literal: true to search for the text as is.` };
+        }
+        budget -= performance.now() - t0;
+        if (!scan?.hits.length) continue;
+        nfiles++;
+        const f = indexed.get(rel);
+        for (const i of scan.hits) {
             total++;
-            const f = indexed.get(rel);
-            matches.push({ path: rel, line: i + 1, text: lines[i], fileId: f?.id ?? null, isTest: !!f?.is_test, code: !!f });
+            matches.push({ path: rel, line: i + 1, text: scan.lines[i], fileId: f?.id ?? null, isTest: !!f?.is_test, code: !!f });
         }
-        if (any) nfiles++;
     }
     const ident = identifierOf(pattern, literal);
     // annotate: enclosing definition, and what an identifier match refers to

package/src/search/search.mjs

@@ -4,17 +4,20 @@
  *   1. name channel     exact / case-insensitive / qualified / prefix matches on symbol names
  *   2. lexical channel  FTS5 BM25 over fields (name, qualified name, signature, doc, path, body)
  *   3. concept channel  the same index queried with thesaurus expansions (natural-language queries)
- *   4. rerank           a small linear model over interpretable features (name evidence, field
+ *   4. semantic channel (optional) cosine similarity of a static code-embedding model (vectors.mjs),
+ *                       on when its model is installed
+ *   5. rerank           a small linear model over interpretable features (name evidence, field
  *                       coverage, kind/test/export priors, graph centrality), then per-file
  *                       diversification so one file cannot flood the top results.
  */
 import { analyzeQuery, codeTokens, splitIdentifier } from './tokenize.mjs';
+import { VectorIndex } from './vectors.mjs';
 
 const FTS_WEIGHTS = [10.0, 6.0, 3.0, 2.0, 2.5, 1.0]; // name, qname, sig, doc, path, body
 const KIND_PRIOR = {
     class: 1.0, interface: 0.95, struct: 1.0, trait: 0.95, enum: 0.85, type: 0.8, object: 0.95, module: 0.7, impl: 0.7,
     function: 1.0, method: 1.0, constructor: 0.75, macro: 0.8, field: 0.55, property: 0.6, variable: 0.55, constant: 0.6,
-    selector: 0.35, keyframes: 0.3,
+    selector: 0.35, keyframes: 0.3, section: 0.5,
 };
 
 export const DEFAULT_WEIGHTS = {
@@ -39,11 +42,21 @@ export const DEFAULT_WEIGHTS = {
     nlField: -0.93,     // fields/properties/variables for natural-language (behavioural) queries
     file: 1.0,          // file-level BM25 of the symbol's file (hierarchical: file relevance first)
     fileNL: 0.5,        // extra file-level weight for natural-language (behavioural) queries
+    docs: 2.0,          // documentation channel BM25 (only for queries about docs, design or rationale)
+    // semantic channel (opt-in): cosine, min-max normalised over its top candidates. Higher weights
+    // (0.5 / 2.0) score better on symbol search and worse on localizing real commits; these are
+    // the highest that leave localization within noise (docs/BENCHMARKS.md)
+    sem: 0.25,
+    semNL: 0.5,         // extra semantic weight for natural-language queries
 };
 
 const EXAMPLE_PATH = /(^|\/)(examples?|samples?|demos?|sandbox|playground|docs?(_src|_source|_examples?)?|documentation|benchmarks?|bench|scripts?|fixtures?|testdata|tutorials?|website|\.github)\//i;
 const DECLARATION_PATH = /\.d\.[cm]?ts$/;
+export const WANTS_DOCS = /\b(docs?|documentation|documented|readme|guide|tutorial|design|rationale|architecture|adr|why|explained?|explanation|changelog|faq|how to)\b/i;
 const BM25_FLOOR = 1.0;
+const SEM_K = 200; // semantic candidates per query
+const RRF_K = 60;
+export const MAX_QUERIES = 6;
 const FILE_GATE = 0.25; // own normalized BM25 at which the file boost applies in full (tuning split)
 
 function stem(w) {
@@ -58,7 +71,8 @@ function ftsQuery(terms) {
 }
 
 export class SearchEngine {
-    constructor(indexer, { weights = DEFAULT_WEIGHTS, centrality = null, perFileCap = Number(process.env.GI_PER_FILE_CAP) || 3 } = {}) {
+    constructor(indexer, { weights = DEFAULT_WEIGHTS, centrality = null, perFileCap = Number(process.env.GI_PER_FILE_CAP) || 3, semantic = undefined } = {}) {
+        this.semantic = semantic === undefined ? VectorIndex.maybe(indexer) : semantic;
         this.perFileCap = perFileCap;
         this.ix = indexer;
         this.store = indexer.store;
@@ -78,6 +92,7 @@ export class SearchEngine {
         const kindFilter = opts.kinds ?? a.filters.kind ?? null;
         const langFilter = opts.lang ?? a.filters.lang?.[0] ?? null;
         const wantsTests = opts.includeTests ?? /\b(test|tests|spec|specs|mock|fixture)\b/i.test(query);
+        const wantsDocs = WANTS_DOCS.test(query);
         const cands = new Map(); // id -> feature bag
         const bag = (id) => { let f = cands.get(id); if (!f) cands.set(id, (f = { id })); return f; };
 
@@ -140,6 +155,23 @@ export class SearchEngine {
                 for (const r of rows) { const f = bag(r.id); if (f.bm25 == null) { f.bm25 = -r.s; if (-r.s > maxBm) maxBm = -r.s; } }
             }
         }
+        // documentation channel: Markdown sections, only when the query is about docs, design or rationale
+        let maxDoc = 0;
+        if (q1 && (wantsDocs || opts.docs)) {
+            for (const r of this.store.all('SELECT rowid AS id, bm25(doc_fts, 4.0, 1.0) AS s FROM doc_fts WHERE doc_fts MATCH ? ORDER BY s LIMIT 60', q1)) {
+                bag(r.id).doc = -r.s; if (-r.s > maxDoc) maxDoc = -r.s;
+            }
+        }
+        // semantic channel: definitions whose embedding is closest to the query's
+        let semLo = 0, semHi = 0;
+        if (this.semantic && (this.w.sem || this.w.semNL)) {
+            const hits = this.semantic.search(query, SEM_K);
+            if (hits.size) {
+                const vals = [...hits.values()];
+                semHi = Math.max(...vals); semLo = Math.min(...vals);
+                for (const [id, c] of hits) bag(id).sem = c;
+            }
+        }
         if (!cands.size) return { results: [], analysis: a };
 
         // 4. features + rerank
@@ -161,6 +193,7 @@ export class SearchEngine {
             if (pathFilter && !r.path.startsWith(pathFilter.replace(/^\.\//, ''))) continue;
             if (kindFilter && !kindFilter.includes(r.kind)) continue;
             if (langFilter && r.lang !== langFilter) continue;
+            if (r.kind === 'section' && !wantsDocs && !opts.docs) continue; // documentation answers only questions about it
             const nameWords = splitIdentifier(r.name).map(stem);
             const qnameWords = new Set(splitIdentifier(r.qname.replace(/\./g, '_')).map(stem));
             const nameCover = nameWords.length ? nameWords.filter(x => qStems.has(x)).length / nameWords.length : 0;
@@ -171,7 +204,9 @@ export class SearchEngine {
             // blown up by max-normalisation; when literal evidence is weak, the concept channel
             // (thesaurus expansions) takes over — that is exactly the vocabulary-mismatch case.
             if (f.bm25) s += w.bm25 * (f.bm25 / Math.max(maxBm, BM25_FLOOR));
+            if (f.doc) s += (w.docs ?? 0) * (f.doc / Math.max(maxDoc, BM25_FLOOR));
             if (f.concept) s += (maxBm < BM25_FLOOR ? Math.max(w.concept, 1.0) : w.concept) * (f.concept / Math.max(maxCx, BM25_FLOOR));
+            if (f.sem != null && semHi > semLo) s += ((w.sem ?? 0) + (a.natural ? (w.semNL ?? 0) : 0)) * (f.sem - semLo) / (semHi - semLo);
             if (f.exact) { s += w.exact * f.exact; reasons.push('name'); }
             if (f.exactCI) { s += w.exactCI * f.exactCI; reasons.push('name'); }
             if (f.qualified) { s += w.qualified; reasons.push('qualified name'); }
@@ -188,23 +223,56 @@ export class SearchEngine {
             if (fs && f.bm25 && r.kind !== 'selector' && r.kind !== 'keyframes') s += (w.file + (a.natural ? w.fileNL : 0)) * fs * Math.min(1, (f.bm25 / Math.max(maxBm, BM25_FLOOR)) / FILE_GATE);
             if (r.parent_kind === 'function' || r.parent_kind === 'method') s += w.nested;
             if (DECLARATION_PATH.test(r.path)) s += w.declaration;
-            if (EXAMPLE_PATH.test(r.path) && !(pathFilter && EXAMPLE_PATH.test(pathFilter))) s += w.example;
+            if (r.kind !== 'section' && EXAMPLE_PATH.test(r.path) && !(pathFilter && EXAMPLE_PATH.test(pathFilter))) s += w.example;
             if (a.natural && (r.kind === 'field' || r.kind === 'property' || r.kind === 'variable' || r.kind === 'constant')) s += w.nlField;
             scored.push({ id: r.id, score: s, reasons, path: r.path, ...(opts.debug ? { features: { ...f, nameCover, queryCover, kind: r.kind } } : {}) });
         }
         scored.sort((x, y) => y.score - x.score || x.id - y.id);
-        // per-file diversification: 3rd+ result from the same file is pushed down
-        const perFile = new Map();
-        const out = [];
-        const deferred = [];
-        for (const r of scored) {
-            const n = perFile.get(r.path) ?? 0;
-            if (n >= this.perFileCap) { deferred.push(r); continue; }
-            perFile.set(r.path, n + 1);
-            out.push(r);
-            if (out.length >= limit) break;
-        }
-        for (const r of deferred) { if (out.length >= limit) break; out.push(r); }
-        return { results: out, total: scored.length, analysis: a };
+        return { results: diversify(scored, limit, this.perFileCap), total: scored.length, analysis: a };
+    }
+
+    /**
+     * Several phrasings of one need ("where are tokens refreshed", "refreshToken", "session renewal")
+     * in one call: each is searched on its own and the rankings are fused by reciprocal rank
+     * (k = 60), so a definition several phrasings agree on rises and one phrasing's noise does not
+     * dominate. One query is the same as search().
+     */
+    searchMany(queries, opts = {}) {
+        const qs = [...new Set((queries ?? []).map(q => String(q ?? '').trim()).filter(Boolean))].slice(0, MAX_QUERIES);
+        if (qs.length <= 1) return this.search(qs[0] ?? '', opts);
+        const limit = Math.max(1, Math.min(opts.limit ?? 10, 100));
+        const per = Math.max(2 * limit, 20);
+        const fused = new Map();
+        let total = 0, analysis = null;
+        qs.forEach((q, qi) => {
+            const r = this.search(q, { ...opts, limit: per });
+            analysis ??= r.analysis;
+            total = Math.max(total, r.total ?? r.results.length);
+            r.results.forEach((x, rank) => {
+                let e = fused.get(x.id);
+                if (!e) fused.set(x.id, (e = { id: x.id, score: 0, reasons: [], path: x.path, queries: [] }));
+                e.score += 1 / (RRF_K + rank + 1);
+                e.queries.push(qi);
+                for (const why of x.reasons) if (!e.reasons.includes(why)) e.reasons.push(why);
+            });
+        });
+        const scored = [...fused.values()].sort((x, y) => y.score - x.score || x.id - y.id);
+        return { results: diversify(scored, limit, this.perFileCap), total: Math.max(total, scored.length), analysis, queries: qs };
+    }
+}
+
+/** Per-file diversification: results past the cap from one file are pushed below the others. */
+function diversify(scored, limit, cap) {
+    const perFile = new Map();
+    const out = [];
+    const deferred = [];
+    for (const r of scored) {
+        const n = perFile.get(r.path) ?? 0;
+        if (n >= cap) { deferred.push(r); continue; }
+        perFile.set(r.path, n + 1);
+        out.push(r);
+        if (out.length >= limit) break;
     }
+    for (const r of deferred) { if (out.length >= limit) break; out.push(r); }
+    return out;
 }

package/src/search/semantic.mjs

@@ -0,0 +1,201 @@
+/**
+ * Optional semantic channel: a static (Model2Vec) code embedding model, run in plain JavaScript.
+ *
+ * A static model is a table of one vector per WordPiece token; a text's embedding is the mean of
+ * its tokens' vectors, normalised. There is no neural network to run, so it needs no native code,
+ * no ONNX runtime and no GPU: embedding a whole repository takes seconds on one core.
+ *
+ * The model is not part of the package. `graph-indexer semantic install` downloads it once, at a
+ * pinned revision with checked SHA-256 digests, into ~/.cache/graph-indexer/models (or
+ * GRAPH_INDEXER_MODEL_DIR). The channel is opt-in: it runs only with GRAPH_INDEXER_SEMANTIC=1 and
+ * the model installed; otherwise search is purely lexical + graph. On the benchmarks it helps
+ * symbol search for queries in words and does not help localizing real commits (docs/BENCHMARKS.md),
+ * so it is not the default.
+ *
+ * Vectors are computed lazily, file by file, the first time a search needs them, and kept in the
+ * index (`vectors`, `vec_files`), keyed by the file's content hash so edits re-embed only that file.
+ */
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import crypto from 'node:crypto';
+
+export const MODEL = {
+    id: 'potion-code-16M-v2',
+    repo: 'minishlab/potion-code-16M-v2',
+    revision: 'e9d2a44ca6a05ac6685f3b23709ea57eb7352d5b',
+    license: 'MIT',
+    files: {
+        'model.safetensors': '75cf7a6c2171b230ad19b1e7d8e0b1aee86da5a02af8e7cacedd9921d227623c',
+        'tokenizer.json': '107bbdcbad4bff1d299b7a4c3a2fb17c52890688b7dd0e4c9deab79d3c4f3d45',
+        'config.json': '148e5691a6fcc553437156859701fba017a1ba5d340b170f17e0f3668fb861a7',
+    },
+};
+
+const MAX_TOKENS = 512;       // Model2Vec's default truncation
+const MAX_WORD_CHARS = 100;   // WordPiece max_input_chars_per_word
+
+export function modelDir() {
+    const base = process.env.GRAPH_INDEXER_MODEL_DIR || path.join(os.homedir(), '.cache', 'graph-indexer', 'models');
+    return path.join(base, MODEL.id);
+}
+
+export function semanticEnabled() {
+    if (process.env.GRAPH_INDEXER_SEMANTIC !== '1') return false;
+    return fs.existsSync(path.join(modelDir(), 'model.safetensors')) && fs.existsSync(path.join(modelDir(), 'tokenizer.json'));
+}
+
+// ── tokenizer: BertNormalizer(lowercase, strip accents) + BertPreTokenizer + WordPiece ──────────
+
+function isControl(c) {
+    if (c === 9 || c === 10 || c === 13) return false;
+    return (c < 32) || (c >= 127 && c < 160);
+}
+function isWhitespace(c) { return c === 32 || c === 9 || c === 10 || c === 13 || c === 0xa0 || c === 0x3000 || (c >= 0x2000 && c <= 0x200a) || c === 0x2028 || c === 0x2029 || c === 0x202f || c === 0x205f || c === 0x1680; }
+function isCJK(c) {
+    return (c >= 0x4e00 && c <= 0x9fff) || (c >= 0x3400 && c <= 0x4dbf) || (c >= 0x20000 && c <= 0x2a6df) || (c >= 0x2a700 && c <= 0x2b73f) ||
+        (c >= 0x2b740 && c <= 0x2b81f) || (c >= 0x2b820 && c <= 0x2ceaf) || (c >= 0xf900 && c <= 0xfaff) || (c >= 0x2f800 && c <= 0x2fa1f);
+}
+const PUNCT_RE = /\p{P}/u;
+function isPunct(ch, c) {
+    return (c >= 33 && c <= 47) || (c >= 58 && c <= 64) || (c >= 91 && c <= 96) || (c >= 123 && c <= 126) || (c > 127 && PUNCT_RE.test(ch));
+}
+
+/** Text → normalised pre-tokens (words and single punctuation characters). */
+export function preTokenize(text) {
+    let s = '';
+    for (const ch of text) {
+        const c = ch.codePointAt(0);
+        if (c === 0 || c === 0xfffd || isControl(c)) continue;
+        if (isWhitespace(c)) { s += ' '; continue; }
+        if (isCJK(c)) { s += ' ' + ch + ' '; continue; }
+        s += ch;
+    }
+    s = s.toLowerCase().normalize('NFD').replace(/\p{Mn}/gu, '');
+    const out = [];
+    let cur = '';
+    for (const ch of s) {
+        const c = ch.codePointAt(0);
+        if (c === 32) { if (cur) { out.push(cur); cur = ''; } continue; }
+        if (isPunct(ch, c)) { if (cur) { out.push(cur); cur = ''; } out.push(ch); continue; }
+        cur += ch;
+    }
+    if (cur) out.push(cur);
+    return out;
+}
+
+export class StaticModel {
+    constructor(dir = modelDir()) {
+        const tok = JSON.parse(fs.readFileSync(path.join(dir, 'tokenizer.json'), 'utf8'));
+        const m = tok.model;
+        if (m?.type !== 'WordPiece') throw new Error(`unsupported tokenizer model ${m?.type}`);
+        this.vocab = new Map(Object.entries(m.vocab));
+        this.prefix = m.continuing_subword_prefix ?? '##';
+        this.unk = this.vocab.get(m.unk_token ?? '[UNK]') ?? -1;
+        const buf = fs.readFileSync(path.join(dir, 'model.safetensors'));
+        const hlen = Number(buf.readBigUInt64LE(0));
+        const header = JSON.parse(buf.subarray(8, 8 + hlen).toString('utf8'));
+        const t = header.embeddings ?? Object.values(header).find(v => v && v.shape);
+        const [rows, dim] = t.shape;
+        const [a, b] = t.data_offsets;
+        const raw = buf.subarray(8 + hlen + a, 8 + hlen + b);
+        this.dim = dim;
+        this.rows = rows;
+        this.table = new Float32Array(rows * dim);
+        if (t.dtype === 'F32') for (let i = 0; i < rows * dim; i++) this.table[i] = raw.readFloatLE(i * 4);
+        else if (t.dtype === 'F16') for (let i = 0; i < rows * dim; i++) this.table[i] = f16(raw.readUInt16LE(i * 2));
+        else throw new Error(`unsupported embedding dtype ${t.dtype}`);
+        this.cache = new Map(); // word -> token ids
+    }
+
+    /** WordPiece ids of one pre-token (greedy longest match), [] when unknown. */
+    #word(w) {
+        let ids = this.cache.get(w);
+        if (ids) return ids;
+        ids = [];
+        if ([...w].length > MAX_WORD_CHARS) ids = [this.unk];
+        else {
+            let start = 0;
+            while (start < w.length) {
+                let end = w.length, found = -1;
+                while (start < end) {
+                    const sub = (start > 0 ? this.prefix : '') + w.slice(start, end);
+                    const id = this.vocab.get(sub);
+                    if (id !== undefined) { found = id; break; }
+                    end--;
+                }
+                if (found < 0) { ids = [this.unk]; break; }
+                ids.push(found);
+                start = end;
+            }
+        }
+        if (this.cache.size > 200000) this.cache.clear();
+        this.cache.set(w, ids);
+        return ids;
+    }
+
+    tokenize(text) {
+        const out = [];
+        for (const w of preTokenize(text)) {
+            for (const id of this.#word(w)) { if (id !== this.unk) out.push(id); }
+            if (out.length >= MAX_TOKENS) break;
+        }
+        return out.length > MAX_TOKENS ? out.slice(0, MAX_TOKENS) : out;
+    }
+
+    /** Mean of token vectors, L2-normalised; null for text without known tokens. */
+    embed(text) {
+        const ids = this.tokenize(text);
+        if (!ids.length) return null;
+        const d = this.dim, v = new Float32Array(d), T = this.table;
+        for (const id of ids) { const o = id * d; for (let j = 0; j < d; j++) v[j] += T[o + j]; }
+        let n = 0;
+        for (let j = 0; j < d; j++) n += v[j] * v[j];
+        n = Math.sqrt(n);
+        if (!n) return null;
+        for (let j = 0; j < d; j++) v[j] /= n;
+        return v;
+    }
+}
+
+function f16(h) {
+    const s = (h & 0x8000) ? -1 : 1, e = (h >> 10) & 0x1f, f = h & 0x3ff;
+    if (e === 0) return s * Math.pow(2, -14) * (f / 1024);
+    if (e === 31) return f ? NaN : s * Infinity;
+    return s * Math.pow(2, e - 15) * (1 + f / 1024);
+}
+
+/** int8 with one scale per vector: v ≈ q * scale. */
+export function quantize(v) {
+    let m = 0;
+    for (let j = 0; j < v.length; j++) { const a = Math.abs(v[j]); if (a > m) m = a; }
+    const q = new Int8Array(v.length);
+    const scale = m / 127 || 1;
+    for (let j = 0; j < v.length; j++) q[j] = Math.round(v[j] / scale);
+    return { q, scale };
+}
+
+// ── model download (explicit, user-initiated) ──────────────────────────────────────────────────
+
+export async function installModel({ log = console.error } = {}) {
+    const dir = modelDir();
+    fs.mkdirSync(dir, { recursive: true });
+    for (const [name, digest] of Object.entries(MODEL.files)) {
+        const dest = path.join(dir, name);
+        if (fs.existsSync(dest) && sha256File(dest) === digest) { log(`${name}: present`); continue; }
+        const url = `https://huggingface.co/${MODEL.repo}/resolve/${MODEL.revision}/${name}`;
+        log(`downloading ${url}`);
+        const res = await fetch(url, { redirect: 'follow' });
+        if (!res.ok) throw new Error(`${name}: HTTP ${res.status}`);
+        const data = Buffer.from(await res.arrayBuffer());
+        const got = crypto.createHash('sha256').update(data).digest('hex');
+        if (got !== digest) throw new Error(`${name}: SHA-256 mismatch (expected ${digest}, got ${got}); nothing was installed`);
+        const tmp = dest + '.part';
+        fs.writeFileSync(tmp, data);
+        fs.renameSync(tmp, dest);
+        log(`${name}: ${(data.length / 1048576).toFixed(1)} MB, verified`);
+    }
+    return dir;
+}
+
+function sha256File(p) { return crypto.createHash('sha256').update(fs.readFileSync(p)).digest('hex'); }