graph-indexer 3.0.0 → 3.1.0

package/src/search/vectors.mjs

@@ -0,0 +1,122 @@
+/**
+ * Vector index over definitions for the semantic channel (see semantic.mjs).
+ *
+ * One int8 vector per definition, embedded from its qualified name, signature, doc comment and
+ * the head of its body. Vectors are filled lazily, file by file, keyed by the file's content
+ * hash, so an edit re-embeds only the files it touched. Search is an exact scan over the matrix
+ * held in memory (no ANN structure to go stale): ~1 ms per 10k definitions.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { StaticModel, quantize, semanticEnabled } from './semantic.mjs';
+import { splitIdentifier } from './tokenize.mjs';
+
+const EMBED_KINDS = new Set(['class', 'interface', 'struct', 'enum', 'trait', 'type', 'object', 'module', 'impl',
+    'function', 'method', 'constructor', 'macro', 'section']);
+const BODY_LINES = 60;
+
+let sharedModel = null;
+function model() { return (sharedModel ??= new StaticModel()); }
+
+/** The text a definition is embedded from. */
+export function symbolText(sym, path_, lines) {
+    const words = splitIdentifier(sym.name).join(' ');
+    const head = `${path_}\n${sym.qname} (${words})\n${sym.sig ?? ''}\n${sym.doc ?? ''}\n`;
+    const body = lines.slice(sym.start_line - 1, Math.min(sym.end_line, sym.start_line - 1 + BODY_LINES)).join('\n');
+    return head + body;
+}
+
+export class VectorIndex {
+    constructor(indexer, { m = null } = {}) {
+        this.ix = indexer;
+        this.store = indexer.store;
+        this.m = m;
+        this.version = -1;
+        this.ids = null;      // Int32Array of symbol ids
+        this.mat = null;      // Int8Array rows × dim
+        this.scales = null;   // Float32Array
+    }
+
+    static maybe(indexer) {
+        try { return semanticEnabled() ? new VectorIndex(indexer) : null; } catch { return null; }
+    }
+
+    get model() { return (this.m ??= model()); }
+
+    /** Embed the files whose vectors are missing or stale; returns how many files were embedded. */
+    fill() {
+        const s = this.store;
+        const stale = s.all(`SELECT f.id, f.path, f.hash FROM files f LEFT JOIN vec_files v ON v.file_id = f.id
+            WHERE v.hash IS NULL OR v.hash <> f.hash`);
+        const gone = s.all('SELECT v.file_id AS id FROM vec_files v LEFT JOIN files f ON f.id = v.file_id WHERE f.id IS NULL');
+        if (!stale.length && !gone.length) return 0;
+        const dim = this.model.dim;
+        let n = 0;
+        const BATCH = 200;
+        for (let i = 0; i < stale.length; i += BATCH) {
+            s.tx(() => {
+                for (const f of stale.slice(i, i + BATCH)) {
+                    s.run('DELETE FROM vectors WHERE file_id = ?', f.id);
+                    let src = null;
+                    try { src = fs.readFileSync(path.join(this.ix.root, f.path), 'utf8'); } catch { /* deleted since */ }
+                    if (src != null) {
+                        const lines = src.split('\n');
+                        const syms = s.all('SELECT id, name, qname, kind, sig, doc, start_line, end_line FROM symbols WHERE file_id = ?', f.id);
+                        for (const sym of syms) {
+                            if (!EMBED_KINDS.has(sym.kind)) continue;
+                            const v = this.model.embed(symbolText(sym, f.path, lines));
+                            if (!v || v.length !== dim) continue;
+                            const { q, scale } = quantize(v);
+                            s.run('INSERT INTO vectors (symbol_id, file_id, scale, v) VALUES (?, ?, ?, ?)', sym.id, f.id, scale, Buffer.from(q.buffer));
+                        }
+                    }
+                    s.run('INSERT INTO vec_files (file_id, hash) VALUES (?, ?) ON CONFLICT(file_id) DO UPDATE SET hash = excluded.hash', f.id, f.hash);
+                    n++;
+                }
+            });
+        }
+        if (gone.length) s.tx(() => { for (const g of gone) { s.run('DELETE FROM vectors WHERE file_id = ?', g.id); s.run('DELETE FROM vec_files WHERE file_id = ?', g.id); } });
+        return n;
+    }
+
+    #load() {
+        if (this.version === this.ix.version && this.mat) return;
+        this.fill();
+        const rows = this.store.all('SELECT symbol_id, scale, v FROM vectors');
+        const dim = this.model.dim;
+        this.ids = new Int32Array(rows.length);
+        this.scales = new Float32Array(rows.length);
+        this.mat = new Int8Array(rows.length * dim);
+        rows.forEach((r, i) => {
+            this.ids[i] = r.symbol_id;
+            this.scales[i] = r.scale;
+            this.mat.set(new Int8Array(r.v.buffer, r.v.byteOffset, dim), i * dim);
+        });
+        this.version = this.ix.version;
+    }
+
+    /** Top-k definitions by cosine similarity to the query: Map(symbol id → cosine). */
+    search(query, k = 200) {
+        const qv = this.model.embed(query);
+        if (!qv) return new Map();
+        this.#load();
+        const dim = this.model.dim, M = this.mat, n = this.ids.length;
+        const top = [];  // small min-heap would be faster; k ≪ n and n is modest, a threshold scan is enough
+        let floor = -Infinity;
+        for (let i = 0; i < n; i++) {
+            let d = 0;
+            const o = i * dim;
+            for (let j = 0; j < dim; j++) d += M[o + j] * qv[j];
+            d *= this.scales[i];
+            if (top.length < k) { top.push([i, d]); if (top.length === k) { top.sort((a, b) => a[1] - b[1]); floor = top[0][1]; } }
+            else if (d > floor) {
+                top[0] = [i, d];
+                // re-establish the minimum at index 0 (insertion into sorted position)
+                let p = 0;
+                while (p + 1 < top.length && top[p][1] > top[p + 1][1]) { const t = top[p]; top[p] = top[p + 1]; top[p + 1] = t; p++; }
+                floor = top[0][1];
+            }
+        }
+        return new Map(top.map(([i, d]) => [this.ids[i], d]));
+    }
+}

package/src/store/db.mjs

@@ -15,7 +15,7 @@ process.emitWarning = function (warning, ...rest) {
 };
 const { DatabaseSync } = await import('node:sqlite');
 
-export const SCHEMA_VERSION = 9;
+export const SCHEMA_VERSION = 11;
 
 const SCHEMA = `
 CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT);
@@ -29,7 +29,8 @@ CREATE TABLE IF NOT EXISTS files (
   lines INTEGER NOT NULL,
   package TEXT,
   is_test INTEGER NOT NULL DEFAULT 0,
-  parse_errors INTEGER NOT NULL DEFAULT 0,
+  parse_errors INTEGER NOT NULL DEFAULT 0, -- regions the parser could not read (ERROR nodes)
+  parse_error_lines TEXT,                   -- their [start, end] lines as JSON, the first 100
   indexed_at INTEGER NOT NULL
 );
 CREATE TABLE IF NOT EXISTS symbols (
@@ -69,7 +70,8 @@ CREATE TABLE IF NOT EXISTS refs (
   dst_id INTEGER,
   conf REAL NOT NULL DEFAULT 0,
   ncand INTEGER NOT NULL DEFAULT 0,
-  argc INTEGER -- arguments of a call or construction (-1: a spread makes it a lower bound), else NULL
+  argc INTEGER, -- arguments of a call or construction (-1: a spread makes it a lower bound), else NULL
+  arg_types TEXT -- their static types where the language overloads by type (JSON: "int", "String", "?" unknown), else NULL
 );
 CREATE INDEX IF NOT EXISTS refs_file ON refs(file_id);
 CREATE INDEX IF NOT EXISTS refs_dst ON refs(dst_id);
@@ -102,6 +104,21 @@ CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(
   tokenize = 'porter unicode61 remove_diacritics 2',
   content = '', contentless_delete = 1
 );
+-- semantic channel (optional, src/search/vectors.mjs): one int8 vector per definition, filled lazily
+CREATE TABLE IF NOT EXISTS vectors (
+  symbol_id INTEGER PRIMARY KEY,
+  file_id INTEGER NOT NULL,
+  scale REAL NOT NULL,
+  v BLOB NOT NULL
+);
+CREATE INDEX IF NOT EXISTS vectors_file ON vectors(file_id);
+CREATE TABLE IF NOT EXISTS vec_files (file_id INTEGER PRIMARY KEY, hash TEXT NOT NULL);
+-- documentation sections (Markdown headings), apart from code so prose never shifts code's BM25 statistics
+CREATE VIRTUAL TABLE IF NOT EXISTS doc_fts USING fts5(
+  name, body,
+  tokenize = 'porter unicode61 remove_diacritics 2',
+  content = '', contentless_delete = 1
+);
 -- one document per file (rowid = files.id): file-level relevance for hierarchical ranking
 CREATE VIRTUAL TABLE IF NOT EXISTS file_fts USING fts5(
   path, body,
@@ -156,9 +173,10 @@ export class Store {
     /** Drop every indexed fact but keep the database (the index is a cache of the source tree). */
     reset() {
         this.tx(() => {
-            for (const t of ['files', 'symbols', 'refs', 'imports', 'fields']) this.db.exec(`DELETE FROM ${t}`);
+            for (const t of ['files', 'symbols', 'refs', 'imports', 'fields', 'vectors', 'vec_files']) this.db.exec(`DELETE FROM ${t}`);
             this.db.exec("INSERT INTO fts(fts) VALUES ('delete-all')");
             this.db.exec("INSERT INTO file_fts(file_fts) VALUES ('delete-all')");
+            this.db.exec("INSERT INTO doc_fts(doc_fts) VALUES ('delete-all')");
         });
     }
 

package/src/util/git.mjs

@@ -0,0 +1,30 @@
+/**
+ * git, run so that an indexed repository's own config cannot make it execute anything: no
+ * fsmonitor, no hooks, no signature verification, no external diff or textconv, and every
+ * clean/smudge/process filter switched off. A repository copied with its .git directory (an
+ * archive, a shared folder) is not trusted.
+ */
+import { spawnSync } from 'node:child_process';
+
+const COMMAND_KEY = /^(filter\.[^\n]+)\.(clean|smudge|process)$|^diff\.[^\n]+\.(textconv|command)$/;
+
+function hardening(root) {
+    const args = ['-c', 'core.fsmonitor=false', '-c', 'core.hooksPath=/dev/null', '-c', 'diff.external=', '-c', 'log.showSignature=false'];
+    const r = spawnSync('git', ['config', '--name-only', '--get-regexp', '^(filter|diff)\\.'], { cwd: root, encoding: 'utf8' });
+    for (const key of (r.status === 0 ? r.stdout : '').split('\n')) {
+        const m = COMMAND_KEY.exec(key);
+        if (!m) continue;
+        args.push('-c', `${key}=`);
+        // a required filter (Git LFS sets one) would otherwise fail the diff once its command is blank
+        if (m[1]) args.push('-c', `${m[1]}.required=false`);
+    }
+    return args;
+}
+
+const NO_TOOLS = { diff: ['--no-ext-diff', '--no-textconv'], log: ['--no-ext-diff', '--no-textconv', '--no-show-signature'] };
+
+/** spawnSync('git', args) in `root`, hardened as above; diff and log never run external tools. */
+export function gitSync(root, args, opts = {}) {
+    const sub = NO_TOOLS[args[0]] ? [args[0], ...NO_TOOLS[args[0]], ...args.slice(1)] : args;
+    return spawnSync('git', [...hardening(root), ...sub], { cwd: root, encoding: 'utf8', ...opts });
+}

package/src/util/paths.mjs

@@ -6,7 +6,12 @@ export const DATA_DIR_NAME = '.graph-indexer';
 
 export function dataDir(root) {
     const dir = path.join(root, DATA_DIR_NAME);
-    fs.mkdirSync(dir, { recursive: true });
+    fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
+    // a .graph-indexer the repository committed as a link would put the index (excerpts of the
+    // source) wherever it points
+    const st = fs.lstatSync(dir);
+    if (!st.isDirectory()) throw new Error(`${dir} is not a directory (a symbolic link?); remove it and run again`);
+    if ((st.mode & 0o077) !== 0) { try { fs.chmodSync(dir, 0o700); } catch { /* read-only */ } }
     const gi = path.join(dir, '.gitignore');
     if (!fs.existsSync(gi)) { try { fs.writeFileSync(gi, '*\n'); } catch { /* read-only */ } }
     return dir;
@@ -22,3 +27,37 @@ export function findRepoRoot(start) {
         dir = parent;
     }
 }
+
+const realRoots = new Map();
+
+/**
+ * `rel` names a file inside the repository once `..` and symlinks are resolved, so a tool target
+ * like `../../etc/hosts` or a tracked link to ~/.ssh/… is never read. The index's own data and git's
+ * internals are not served either.
+ */
+export function withinRoot(root, rel) {
+    let realRoot = realRoots.get(root);
+    if (!realRoot) { try { realRoot = fs.realpathSync(root); } catch { realRoot = path.resolve(root); } realRoots.set(root, realRoot); }
+    let real;
+    try { real = fs.realpathSync(path.join(root, rel)); } catch { return false; }
+    if (real !== realRoot && !real.startsWith(realRoot + path.sep)) return false;
+    const top = path.relative(realRoot, real).split(path.sep)[0];
+    return top !== '.git' && top !== DATA_DIR_NAME;
+}
+
+/** A git revision given by a caller: never an option (`--output=…`), whitespace or NUL. */
+export const isRevision = (s) => typeof s === 'string' && s.length > 0 && s.length <= 256 && !s.startsWith('-') && !/[\s\0]/.test(s);
+
+/**
+ * A small regular file of the repository (a manifest, a config), or null: never a device, a FIFO,
+ * a file larger than `maxBytes` or a link out of the root — a `tsconfig.json` linked to /dev/zero
+ * would otherwise hang the indexer.
+ */
+export function readRepoFile(root, rel, maxBytes = 2_000_000) {
+    if (!withinRoot(root, rel)) return null;
+    try {
+        const abs = path.join(root, rel);
+        const st = fs.statSync(abs);
+        return st.isFile() && st.size <= maxBytes ? fs.readFileSync(abs, 'utf8') : null;
+    } catch { return null; }
+}

package/src/util/regex.mjs

@@ -0,0 +1,2 @@
+/** A string as a literal inside a regular expression: every metacharacter escaped. */
+export const escapeRegExp = (s) => String(s).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');

package/vendor/grammars/README.md

@@ -0,0 +1,47 @@
+# Vendored grammars
+
+Each `NAME.wasm.gz` is a tree-sitter grammar compiled to WebAssembly. `manifest.json` records where it
+comes from and the SHA-256 and size of the uncompressed `.wasm`; `test/vendor.test.mjs` checks every
+file against it, and the index fingerprint covers the manifest, so a changed grammar rebuilds indexes.
+
+- **`tag` and `asset`**: the `.wasm` published with that release of the grammar's repository,
+  downloaded as is.
+- **`commit` and `built`**: the grammar has no release with what graph-indexer needs (Kotlin: the
+  latest release, 0.3.8, predates `fun interface` and Kotlin 2.x syntax), so the `.wasm` is built
+  from that commit. The build is reproducible: anyone can check the vendored file against its
+  source.
+
+## Building a grammar from a commit
+
+This is `tree-sitter build --wasm` with tree-sitter-cli 0.25.10, the version of the vendored
+`web-tree-sitter`, run as the `emcc` command that CLI issues (`compile_parser_to_wasm` in
+`cli/loader/src/lib.rs`), in the Emscripten image it pins. The checkout must be under a directory
+the container can mount (with Colima, your home directory).
+
+```sh
+git clone https://github.com/fwcd/tree-sitter-kotlin && cd tree-sitter-kotlin
+git checkout d6af96d2e23ef0a8ff5ab512864378cf08fba311
+docker run --rm --platform linux/amd64 --workdir /src/src -v "$PWD":/src --user "$(id -u)" \
+  emscripten/emsdk:4.0.4 emcc -o output.wasm -Os -s WASM=1 -s SIDE_MODULE=2 -s TOTAL_MEMORY=33554432 \
+  -s NODEJS_CATCH_EXIT=0 -s 'EXPORTED_FUNCTIONS=["_tree_sitter_kotlin"]' -fno-exceptions -fvisibility=hidden \
+  -I . scanner.c parser.c
+shasum -a 256 src/output.wasm          # must equal manifest.json's sha256
+gzip -9 -n -c src/output.wasm > kotlin.wasm.gz
+```
+
+For another grammar, replace the repository, the commit and the exported symbol
+(`_tree_sitter_<name>`). The grammar's `src/parser.c` must declare `LANGUAGE_VERSION` 14 or 15,
+the ABI versions web-tree-sitter 0.25 loads.
+
+## Checking a new grammar before vendoring it
+
+A grammar change can move or rename nodes the extraction queries in `src/parse/lang/` match. Before
+replacing one:
+
+1. `npm test`, which compiles every query against its grammar.
+2. Parse a corpus of real code in that language with the old and the new grammar and compare the
+   files with ERROR regions, the symbols (by kind and qualified name) and the references each
+   extracts. A symbol that disappears needs a reason; one that changes its qualified name usually
+   moved back into its class.
+3. For a language with a compiler oracle (`bench/eval-graph*.mjs`), the references must keep their
+   precision and recall.

package/vendor/grammars/manifest.json

@@ -82,10 +82,10 @@
     },
     "kotlin": {
       "repo": "https://github.com/fwcd/tree-sitter-kotlin",
-      "tag": "0.3.8",
-      "asset": "tree-sitter-kotlin.wasm",
-      "sha256": "c624e7443b371c28adc5d81674e73067564c12555ebe3ed96a6c8db814b7602d",
-      "bytes": 4052625,
+      "commit": "d6af96d2e23ef0a8ff5ab512864378cf08fba311",
+      "built": "tree-sitter build --wasm (tree-sitter-cli 0.25.10, emscripten/emsdk:4.0.4): Kotlin 2.x syntax (guard conditions, `..<`, `$$` strings, context parameters) and `fun interface` are in no release yet, the latest being 0.3.8",
+      "sha256": "90689092295c4e11d21f2c4f1b9ffce1a3c68f7ce6d4de57c365cd0fc888c7ee",
+      "bytes": 6078823,
       "license": "MIT"
     },
     "php": {