graph-indexer 3.0.1 → 3.1.1

package/src/store/db.mjs

@@ -15,7 +15,7 @@ process.emitWarning = function (warning, ...rest) {
 };
 const { DatabaseSync } = await import('node:sqlite');
 
-export const SCHEMA_VERSION = 9;
+export const SCHEMA_VERSION = 11;
 
 const SCHEMA = `
 CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT);
@@ -29,7 +29,8 @@ CREATE TABLE IF NOT EXISTS files (
   lines INTEGER NOT NULL,
   package TEXT,
   is_test INTEGER NOT NULL DEFAULT 0,
-  parse_errors INTEGER NOT NULL DEFAULT 0,
+  parse_errors INTEGER NOT NULL DEFAULT 0, -- regions the parser could not read (ERROR nodes)
+  parse_error_lines TEXT,                   -- their [start, end] lines as JSON, the first 100
   indexed_at INTEGER NOT NULL
 );
 CREATE TABLE IF NOT EXISTS symbols (
@@ -69,7 +70,8 @@ CREATE TABLE IF NOT EXISTS refs (
   dst_id INTEGER,
   conf REAL NOT NULL DEFAULT 0,
   ncand INTEGER NOT NULL DEFAULT 0,
-  argc INTEGER -- arguments of a call or construction (-1: a spread makes it a lower bound), else NULL
+  argc INTEGER, -- arguments of a call or construction (-1: a spread makes it a lower bound), else NULL
+  arg_types TEXT -- their static types where the language overloads by type (JSON: "int", "String", "?" unknown), else NULL
 );
 CREATE INDEX IF NOT EXISTS refs_file ON refs(file_id);
 CREATE INDEX IF NOT EXISTS refs_dst ON refs(dst_id);
@@ -102,6 +104,21 @@ CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(
   tokenize = 'porter unicode61 remove_diacritics 2',
   content = '', contentless_delete = 1
 );
+-- semantic channel (optional, src/search/vectors.mjs): one int8 vector per definition, filled lazily
+CREATE TABLE IF NOT EXISTS vectors (
+  symbol_id INTEGER PRIMARY KEY,
+  file_id INTEGER NOT NULL,
+  scale REAL NOT NULL,
+  v BLOB NOT NULL
+);
+CREATE INDEX IF NOT EXISTS vectors_file ON vectors(file_id);
+CREATE TABLE IF NOT EXISTS vec_files (file_id INTEGER PRIMARY KEY, hash TEXT NOT NULL);
+-- documentation sections (Markdown headings), apart from code so prose never shifts code's BM25 statistics
+CREATE VIRTUAL TABLE IF NOT EXISTS doc_fts USING fts5(
+  name, body,
+  tokenize = 'porter unicode61 remove_diacritics 2',
+  content = '', contentless_delete = 1
+);
 -- one document per file (rowid = files.id): file-level relevance for hierarchical ranking
 CREATE VIRTUAL TABLE IF NOT EXISTS file_fts USING fts5(
   path, body,
@@ -111,11 +128,21 @@ CREATE VIRTUAL TABLE IF NOT EXISTS file_fts USING fts5(
 `;
 
 export class Store {
+    /**
+     * An index.db the repository shipped is not trusted: defensive mode refuses the statements that
+     * can corrupt the file (writable_schema, …) and an untrusted schema may not call functions.
+     */
+    #harden() {
+        try { this.db.enableDefensive?.(true); } catch { /* older Node */ }
+        this.db.exec('PRAGMA trusted_schema = OFF;');
+    }
+
     constructor(dbPath) {
         this.path = dbPath;
         this.stmts = new Map();
         fs.mkdirSync(path.dirname(dbPath), { recursive: true });
         this.db = new DatabaseSync(dbPath);
+        this.#harden();
         this.db.exec('PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL; PRAGMA temp_store = MEMORY; PRAGMA cache_size = -65536; PRAGMA busy_timeout = 5000;');
         const v = this.#schemaVersion();
         if (v !== null && v !== SCHEMA_VERSION) {
@@ -123,6 +150,7 @@ export class Store {
             this.db.close();
             for (const suffix of ['', '-wal', '-shm']) { try { fs.unlinkSync(dbPath + suffix); } catch { /* none */ } }
             this.db = new DatabaseSync(dbPath);
+            this.#harden();
             this.db.exec('PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL; PRAGMA temp_store = MEMORY; PRAGMA cache_size = -65536; PRAGMA busy_timeout = 5000;');
         }
         this.db.exec(SCHEMA);
@@ -156,9 +184,10 @@ export class Store {
     /** Drop every indexed fact but keep the database (the index is a cache of the source tree). */
     reset() {
         this.tx(() => {
-            for (const t of ['files', 'symbols', 'refs', 'imports', 'fields']) this.db.exec(`DELETE FROM ${t}`);
+            for (const t of ['files', 'symbols', 'refs', 'imports', 'fields', 'vectors', 'vec_files']) this.db.exec(`DELETE FROM ${t}`);
             this.db.exec("INSERT INTO fts(fts) VALUES ('delete-all')");
             this.db.exec("INSERT INTO file_fts(file_fts) VALUES ('delete-all')");
+            this.db.exec("INSERT INTO doc_fts(doc_fts) VALUES ('delete-all')");
         });
     }
 

package/src/util/git.mjs

@@ -1,30 +1,79 @@
 /**
  * git, run so that an indexed repository's own config cannot make it execute anything: no
- * fsmonitor, no hooks, no signature verification, no external diff or textconv, and every
- * clean/smudge/process filter switched off. A repository copied with its .git directory (an
- * archive, a shared folder) is not trusted.
+ * fsmonitor, no hooks, no signature verification, no external diff or textconv, every
+ * clean/smudge/process filter switched off, and no descent into submodules (whose own config the
+ * overrides below do not cover). A repository copied with its .git directory (an archive, a shared
+ * folder) is not trusted. Every git call graph-indexer makes goes through gitSync.
  */
 import { spawnSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
 
 const COMMAND_KEY = /^(filter\.[^\n]+)\.(clean|smudge|process)$|^diff\.[^\n]+\.(textconv|command)$/;
 
+let gitBin = null;
+/**
+ * The git executable. On Windows a bare `git` is looked up in the working directory before PATH,
+ * so a `git.exe` committed at the repository root would run instead: look it up on PATH only.
+ */
+export function gitExecutable() {
+    if (gitBin) return gitBin;
+    gitBin = 'git';
+    if (process.platform === 'win32') {
+        const exts = (process.env.PATHEXT || '.EXE;.CMD;.BAT').split(';').filter(Boolean);
+        for (const dir of (process.env.PATH || process.env.Path || '').split(path.delimiter)) {
+            if (!dir || !path.isAbsolute(dir)) continue;
+            const hit = exts.map(e => path.join(dir, 'git' + e.toLowerCase())).find(f => { try { return fs.statSync(f).isFile(); } catch { return false; } });
+            if (hit) { gitBin = hit; break; }
+        }
+    }
+    return gitBin;
+}
+
+let envConfig = null;
+/** git ≥ 2.31 takes config from GIT_CONFIG_COUNT/KEY/VALUE, which carry a key verbatim. */
+function supportsEnvConfig() {
+    if (envConfig != null) return envConfig;
+    const m = /(\d+)\.(\d+)/.exec(spawnSync(gitExecutable(), ['--version'], { encoding: 'utf8' }).stdout ?? '');
+    return (envConfig = !!m && (+m[1] > 2 || (+m[1] === 2 && +m[2] >= 31)));
+}
+
+/**
+ * The overrides for `root`: fixed ones, plus one blanking each command its config names. Those go
+ * in the environment, not as `-c key=`: git splits a `-c` at the first `=`, and a driver may be
+ * named `x=y`, so `-c filter.x=y.clean=` would set `filter.x` and leave the real command on.
+ * Returns null when the overrides cannot be expressed (an old git and such a name): git is not run.
+ */
 function hardening(root) {
-    const args = ['-c', 'core.fsmonitor=false', '-c', 'core.hooksPath=/dev/null', '-c', 'diff.external=', '-c', 'log.showSignature=false'];
-    const r = spawnSync('git', ['config', '--name-only', '--get-regexp', '^(filter|diff)\\.'], { cwd: root, encoding: 'utf8' });
-    for (const key of (r.status === 0 ? r.stdout : '').split('\n')) {
+    const pairs = [['core.fsmonitor', 'false'], ['core.hooksPath', '/dev/null'], ['diff.external', ''], ['log.showSignature', 'false']];
+    const r = spawnSync(gitExecutable(), ['config', '-z', '--name-only', '--get-regexp', '^(filter|diff)\\.'], { cwd: root, encoding: 'utf8' });
+    for (const key of (r.status === 0 ? r.stdout : '').split('\0')) {
         const m = COMMAND_KEY.exec(key);
         if (!m) continue;
-        args.push('-c', `${key}=`);
+        pairs.push([key, '']);
         // a required filter (Git LFS sets one) would otherwise fail the diff once its command is blank
-        if (m[1]) args.push('-c', `${m[1]}.required=false`);
+        if (m[1]) pairs.push([`${m[1]}.required`, 'false']);
+    }
+    if (supportsEnvConfig()) {
+        const base = Number(process.env.GIT_CONFIG_COUNT) || 0;
+        const env = { GIT_CONFIG_COUNT: String(base + pairs.length) };
+        pairs.forEach(([k, v], i) => { env[`GIT_CONFIG_KEY_${base + i}`] = k; env[`GIT_CONFIG_VALUE_${base + i}`] = v; });
+        return { args: [], env };
     }
-    return args;
+    if (pairs.some(([k]) => k.includes('='))) return null;
+    return { args: pairs.flatMap(([k, v]) => ['-c', `${k}=${v}`]), env: {} };
 }
 
-const NO_TOOLS = { diff: ['--no-ext-diff', '--no-textconv'], log: ['--no-ext-diff', '--no-textconv', '--no-show-signature'] };
+const NO_TOOLS = {
+    diff: ['--no-ext-diff', '--no-textconv', '--ignore-submodules=all'],
+    log: ['--no-ext-diff', '--no-textconv', '--no-show-signature'],
+    status: ['--ignore-submodules=all'],
+};
 
 /** spawnSync('git', args) in `root`, hardened as above; diff and log never run external tools. */
 export function gitSync(root, args, opts = {}) {
     const sub = NO_TOOLS[args[0]] ? [args[0], ...NO_TOOLS[args[0]], ...args.slice(1)] : args;
-    return spawnSync('git', [...hardening(root), ...sub], { cwd: root, encoding: 'utf8', ...opts });
+    const h = hardening(root);
+    if (!h) return { status: 1, stdout: '', stderr: 'git not run: the repository config names a filter or diff driver this git cannot switch off', error: null };
+    return spawnSync(gitExecutable(), [...h.args, ...sub], { cwd: root, encoding: 'utf8', ...opts, env: { ...process.env, ...opts.env, ...h.env } });
 }

package/src/util/paths.mjs

@@ -12,6 +12,11 @@ export function dataDir(root) {
     const st = fs.lstatSync(dir);
     if (!st.isDirectory()) throw new Error(`${dir} is not a directory (a symbolic link?); remove it and run again`);
     if ((st.mode & 0o077) !== 0) { try { fs.chmodSync(dir, 0o700); } catch { /* read-only */ } }
+    // nor may a link inside it (`index.db-wal` → ~/.zshrc, `resident.started` → ~/.ssh/…) send the
+    // index's writes elsewhere: everything in here is ours, so a link is never legitimate
+    try {
+        for (const e of fs.readdirSync(dir, { withFileTypes: true })) if (e.isSymbolicLink()) fs.unlinkSync(path.join(dir, e.name));
+    } catch { /* read-only */ }
     const gi = path.join(dir, '.gitignore');
     if (!fs.existsSync(gi)) { try { fs.writeFileSync(gi, '*\n'); } catch { /* read-only */ } }
     return dir;
@@ -37,11 +42,13 @@ const realRoots = new Map();
  */
 export function withinRoot(root, rel) {
     let realRoot = realRoots.get(root);
-    if (!realRoot) { try { realRoot = fs.realpathSync(root); } catch { realRoot = path.resolve(root); } realRoots.set(root, realRoot); }
+    if (!realRoot) { try { realRoot = fs.realpathSync.native(root); } catch { realRoot = path.resolve(root); } realRoots.set(root, realRoot); }
     let real;
-    try { real = fs.realpathSync(path.join(root, rel)); } catch { return false; }
+    // the native realpath gives the name as stored on disk, so on a case-insensitive file system
+    // (macOS, Windows) `.GIT/config` resolves to `.git/config` and is refused like it
+    try { real = fs.realpathSync.native(path.join(root, rel)); } catch { return false; }
     if (real !== realRoot && !real.startsWith(realRoot + path.sep)) return false;
-    const top = path.relative(realRoot, real).split(path.sep)[0];
+    const top = path.relative(realRoot, real).split(path.sep)[0].toLowerCase();
     return top !== '.git' && top !== DATA_DIR_NAME;
 }
 

package/vendor/grammars/README.md

@@ -0,0 +1,47 @@
+# Vendored grammars
+
+Each `NAME.wasm.gz` is a tree-sitter grammar compiled to WebAssembly. `manifest.json` records where it
+comes from and the SHA-256 and size of the uncompressed `.wasm`; `test/vendor.test.mjs` checks every
+file against it, and the index fingerprint covers the manifest, so a changed grammar rebuilds indexes.
+
+- **`tag` and `asset`**: the `.wasm` published with that release of the grammar's repository,
+  downloaded as is.
+- **`commit` and `built`**: the grammar has no release with what graph-indexer needs (Kotlin: the
+  latest release, 0.3.8, predates `fun interface` and Kotlin 2.x syntax), so the `.wasm` is built
+  from that commit. The build is reproducible: anyone can check the vendored file against its
+  source.
+
+## Building a grammar from a commit
+
+This is `tree-sitter build --wasm` with tree-sitter-cli 0.25.10, the version of the vendored
+`web-tree-sitter`, run as the `emcc` command that CLI issues (`compile_parser_to_wasm` in
+`cli/loader/src/lib.rs`), in the Emscripten image it pins. The checkout must be under a directory
+the container can mount (with Colima, your home directory).
+
+```sh
+git clone https://github.com/fwcd/tree-sitter-kotlin && cd tree-sitter-kotlin
+git checkout d6af96d2e23ef0a8ff5ab512864378cf08fba311
+docker run --rm --platform linux/amd64 --workdir /src/src -v "$PWD":/src --user "$(id -u)" \
+  emscripten/emsdk:4.0.4 emcc -o output.wasm -Os -s WASM=1 -s SIDE_MODULE=2 -s TOTAL_MEMORY=33554432 \
+  -s NODEJS_CATCH_EXIT=0 -s 'EXPORTED_FUNCTIONS=["_tree_sitter_kotlin"]' -fno-exceptions -fvisibility=hidden \
+  -I . scanner.c parser.c
+shasum -a 256 src/output.wasm          # must equal manifest.json's sha256
+gzip -9 -n -c src/output.wasm > kotlin.wasm.gz
+```
+
+For another grammar, replace the repository, the commit and the exported symbol
+(`_tree_sitter_<name>`). The grammar's `src/parser.c` must declare `LANGUAGE_VERSION` 14 or 15,
+the ABI versions web-tree-sitter 0.25 loads.
+
+## Checking a new grammar before vendoring it
+
+A grammar change can move or rename nodes the extraction queries in `src/parse/lang/` match. Before
+replacing one:
+
+1. `npm test`, which compiles every query against its grammar.
+2. Parse a corpus of real code in that language with the old and the new grammar and compare the
+   files with ERROR regions, the symbols (by kind and qualified name) and the references each
+   extracts. A symbol that disappears needs a reason; one that changes its qualified name usually
+   moved back into its class.
+3. For a language with a compiler oracle (`bench/eval-graph*.mjs`), the references must keep their
+   precision and recall.

package/vendor/grammars/manifest.json

@@ -82,10 +82,10 @@
     },
     "kotlin": {
       "repo": "https://github.com/fwcd/tree-sitter-kotlin",
-      "tag": "0.3.8",
-      "asset": "tree-sitter-kotlin.wasm",
-      "sha256": "c624e7443b371c28adc5d81674e73067564c12555ebe3ed96a6c8db814b7602d",
-      "bytes": 4052625,
+      "commit": "d6af96d2e23ef0a8ff5ab512864378cf08fba311",
+      "built": "tree-sitter build --wasm (tree-sitter-cli 0.25.10, emscripten/emsdk:4.0.4): Kotlin 2.x syntax (guard conditions, `..<`, `$$` strings, context parameters) and `fun interface` are in no release yet, the latest being 0.3.8",
+      "sha256": "90689092295c4e11d21f2c4f1b9ffce1a3c68f7ce6d4de57c365cd0fc888c7ee",
+      "bytes": 6078823,
       "license": "MIT"
     },
     "php": {