graph-indexer 2.1.1 → 3.0.1

package/src/search/tokenize.mjs

@@ -0,0 +1,188 @@
+/**
+ * Code-aware tokenization shared by indexing and querying.
+ *
+ * An identifier contributes its full lowercase form AND its sub-words, so `getUserById`
+ * matches the query "get user by id", the query "getUserById", and the query "user".
+ * Acronyms split correctly (`parseHTTPHeader` → parse, http, header). Stemming happens in
+ * SQLite's FTS5 porter tokenizer on top of these tokens.
+ */
+
+const IDENT_RE = /[A-Za-z_$][A-Za-z0-9_$]*|[0-9]+/g;
+
+/** Split one identifier into lowercase sub-words (camel/Pascal/snake/kebab/acronyms/digits). */
+export function splitIdentifier(id) {
+    const parts = id
+        .replace(/[$_]+/g, ' ')
+        .replace(/([a-z0-9])([A-Z])/g, '$1 $2')
+        .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2')
+        .replace(/([A-Za-z])([0-9])/g, '$1 $2')
+        .replace(/([0-9])([A-Za-z])/g, '$1 $2')
+        .toLowerCase()
+        .split(/\s+/)
+        .filter(Boolean);
+    return parts;
+}
+
+// Identifiers repeat heavily across a code base: cache each one's token expansion.
+const TOKEN_CACHE = new Map();
+const TOKEN_CACHE_MAX = 200_000;
+
+function identifierTokens(id) {
+    let t = TOKEN_CACHE.get(id);
+    if (t) return t;
+    if (/^[0-9]+$/.test(id)) t = id.length <= 6 ? [id] : [];
+    else {
+        const lower = id.toLowerCase().replace(/^[$_]+|[$_]+$/g, '');
+        const parts = splitIdentifier(id);
+        t = [];
+        if (lower.length >= 2 && (parts.length !== 1 || parts[0] !== lower)) t.push(lower.replace(/[$]/g, ''));
+        for (const p of parts) if (p.length >= 2 || parts.length === 1) t.push(p);
+    }
+    if (TOKEN_CACHE.size >= TOKEN_CACHE_MAX) TOKEN_CACHE.clear();
+    TOKEN_CACHE.set(id, t);
+    return t;
+}
+
+/** Tokens for indexing a piece of code/text (identifier forms + sub-words). */
+export function codeTokens(text, { maxTokens = Infinity } = {}) {
+    const out = [];
+    if (!text) return out;
+    IDENT_RE.lastIndex = 0;
+    let m;
+    while ((m = IDENT_RE.exec(text)) && out.length < maxTokens) {
+        for (const t of identifierTokens(m[0])) out.push(t);
+    }
+    return out;
+}
+
+export function tokenString(text, opts) { return codeTokens(text, opts).join(' '); }
+
+// Words that carry no retrieval signal in a question about code.
+export const STOPWORDS = new Set([
+    'a', 'an', 'the', 'and', 'or', 'but', 'of', 'to', 'in', 'on', 'at', 'by', 'for', 'with', 'from', 'into', 'onto',
+    'is', 'are', 'was', 'were', 'be', 'been', 'being', 'am', 'do', 'does', 'did', 'done', 'doing', 'have', 'has', 'had',
+    'it', 'its', 'this', 'that', 'these', 'those', 'there', 'here', 'what', 'which', 'who', 'whom', 'whose', 'where',
+    'when', 'why', 'how', 'can', 'could', 'should', 'would', 'will', 'shall', 'may', 'might', 'must', 'i', 'we', 'you',
+    'they', 'he', 'she', 'me', 'us', 'them', 'my', 'our', 'your', 'their', 'if', 'then', 'else', 'than', 'so', 'such',
+    'as', 'not', 'no', 'yes', 'all', 'any', 'each', 'every', 'some', 'most', 'more', 'other', 'about', 'over', 'under',
+    'up', 'down', 'out', 'off', 'again', 'further', 'once', 'also', 'just', 'only', 'very', 'too', 'via', 'per', 'like',
+    'code', 'function', 'functions', 'method', 'methods', 'class', 'classes', 'implementation', 'implemented', 'implement',
+    'implements', 'logic', 'where', 'find', 'show', 'defined', 'handled', 'handle', 'happens', 'used', 'using', 'use',
+    'get', 'gets', 'responsible', 'thing', 'things', 'part', 'called', 'call', 'calls', 'file', 'files', 'module',
+]);
+// NOTE: 'get', 'handle', 'find', 'use', 'call' are stopwords only for natural-language queries;
+// identifier-shaped queries keep them (see analyzeQuery).
+
+/**
+ * Small, curated concept thesaurus for natural-language → code vocabulary gaps.
+ * Each group is symmetric; expansions are scored below literal terms.
+ */
+const CONCEPT_GROUPS = [
+    ['auth', 'authentication', 'authenticate', 'authorization', 'authorize', 'login', 'credential', 'credentials'],
+    ['db', 'database', 'sql', 'query', 'repository', 'repo', 'dao', 'store', 'persistence'],
+    ['config', 'configuration', 'settings', 'options', 'opts', 'conf', 'cfg'],
+    ['init', 'initialize', 'initialise', 'setup', 'bootstrap', 'boot', 'startup'],
+    ['err', 'error', 'errors', 'exception', 'failure', 'fault'],
+    ['req', 'request'], ['res', 'resp', 'response', 'reply'],
+    ['msg', 'message'], ['ctx', 'context'], ['env', 'environment'], ['impl', 'implementation'],
+    ['util', 'utils', 'utility', 'helper', 'helpers'], ['str', 'string', 'text'], ['num', 'number', 'count'],
+    ['param', 'params', 'parameter', 'arg', 'args', 'argument'], ['fn', 'func', 'callback', 'cb', 'handler'],
+    ['del', 'delete', 'remove', 'destroy', 'drop', 'erase'], ['create', 'new', 'make', 'build', 'construct', 'add', 'insert'],
+    ['fetch', 'get', 'retrieve', 'load', 'read', 'lookup'], ['save', 'store', 'persist', 'write', 'put', 'commit'],
+    ['update', 'modify', 'patch', 'edit', 'change', 'set'],
+    ['search', 'find', 'query', 'lookup', 'filter', 'match'], ['list', 'all', 'collection', 'index'],
+    ['parse', 'decode', 'deserialize', 'unmarshal', 'unmarshall', 'read'],
+    ['serialize', 'encode', 'marshal', 'stringify', 'dump', 'render'],
+    ['validate', 'validation', 'validator', 'check', 'verify', 'assert', 'ensure', 'sanitize'],
+    ['send', 'emit', 'dispatch', 'publish', 'post', 'notify', 'broadcast'],
+    ['receive', 'consume', 'listen', 'subscribe', 'on', 'handle'],
+    ['route', 'routes', 'router', 'routing', 'endpoint', 'path', 'url', 'controller'],
+    ['middleware', 'interceptor', 'hook', 'filter', 'guard', 'pipe', 'plugin'],
+    ['cache', 'cached', 'memo', 'memoize', 'lru'], ['retry', 'retries', 'backoff', 'attempt'],
+    ['timeout', 'deadline', 'expire', 'expiry', 'ttl'], ['log', 'logger', 'logging', 'trace', 'debug'],
+    ['user', 'account', 'member', 'profile', 'owner'], ['permission', 'role', 'access', 'acl', 'policy', 'privilege'],
+    ['token', 'jwt', 'session', 'cookie', 'bearer'], ['password', 'secret', 'hash', 'crypt', 'encrypt', 'cipher'],
+    ['upload', 'file', 'attachment', 'blob', 'multipart'], ['schedule', 'cron', 'job', 'task', 'worker', 'queue'],
+    ['event', 'listener', 'subscriber', 'observer', 'emitter', 'signal'], ['test', 'spec', 'mock', 'stub', 'fixture'],
+    ['bind', 'binding', 'bound'], ['convert', 'transform', 'map', 'cast', 'coerce', 'normalize'],
+    ['compare', 'equal', 'equals', 'diff', 'eq'], ['sort', 'order', 'rank', 'priority'],
+    ['merge', 'combine', 'join', 'concat', 'extend', 'assign'], ['split', 'chunk', 'partition', 'segment'],
+    ['wrap', 'wrapper', 'decorate', 'decorator', 'proxy', 'adapter'], ['escape', 'sanitize', 'encode', 'quote'],
+    ['start', 'begin', 'launch', 'run', 'serve', 'listen'], ['stop', 'end', 'shutdown', 'close', 'terminate', 'dispose', 'teardown'],
+    ['http', 'rest', 'api', 'client', 'fetch', 'axios'], ['header', 'headers'], ['body', 'payload', 'content', 'data'],
+    ['redirect', 'forward', 'location'], ['status', 'code', 'state'], ['json', 'jsonify'], ['xml', 'html', 'template', 'view'],
+    ['stream', 'pipe', 'buffer', 'reader', 'writer'], ['async', 'await', 'promise', 'future', 'deferred'],
+    ['thread', 'lock', 'mutex', 'concurrent', 'sync'], ['socket', 'websocket', 'ws', 'connection', 'conn'],
+    ['cors', 'origin', 'crossorigin'], ['cookie', 'cookies'], ['static', 'asset', 'assets', 'public'],
+    ['dependency', 'dependencies', 'inject', 'injection', 'injectable', 'provider', 'container', 'di'],
+    ['schema', 'model', 'entity', 'dto', 'type'], ['migrate', 'migration', 'schema'], ['email', 'mail', 'smtp'],
+    ['payment', 'charge', 'billing', 'invoice', 'stripe'], ['image', 'img', 'picture', 'photo', 'thumbnail'],
+    ['date', 'time', 'timestamp', 'datetime', 'clock'], ['random', 'rand', 'uuid', 'id', 'identifier'],
+    ['env', 'environment', 'variable', 'var'], ['command', 'cmd', 'cli', 'arg', 'flag'],
+    ['proxy', 'forward', 'tunnel'], ['compress', 'gzip', 'zip', 'deflate'], ['version', 'semver'],
+];
+
+const CONCEPTS = new Map();
+for (const group of CONCEPT_GROUPS) {
+    for (const w of group) {
+        const set = CONCEPTS.get(w) ?? new Set();
+        for (const o of group) if (o !== w) set.add(o);
+        CONCEPTS.set(w, set);
+    }
+}
+
+export function expandConcept(word) { return CONCEPTS.get(word) ?? null; }
+
+const IDENTIFIER_SHAPE = /^(?:[a-z]+(?:[A-Z][a-z0-9]*)+|[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]*)+|[A-Za-z0-9]+(?:_[A-Za-z0-9]+)+|[A-Z][A-Z0-9_]{2,}|[A-Za-z_$][\w$]*(?:\.|::|#|->)[A-Za-z_$][\w$.:#>-]*|[A-Za-z_$][\w$]*\(\))$/;
+
+/** Mixed-case (APIRoute, HTTPBearer, iOS), snake/kebab-free words with inner capitals, or a
+ *  capitalized word after the first position ("... using Promise") read as code identifiers. */
+function looksLikeIdentifier(t, position) {
+    if (!/^[A-Za-z_$][\w$]*$/.test(t)) return false;
+    if (/[A-Z]/.test(t.slice(1)) && /[a-z]/.test(t)) return true;
+    if (/[_$]/.test(t) || /[a-z][0-9]|[0-9][a-z]/i.test(t) && /[a-z]/i.test(t) && t.length > 2) return true;
+    if (position > 0 && /^[A-Z][a-z0-9]+$/.test(t)) return true;
+    return false;
+}
+
+/**
+ * Analyse a search query.
+ * @returns {{ raw, identifiers: string[], words: string[], terms: string[], expansions: string[], natural: boolean, filters: object }}
+ */
+export function analyzeQuery(query) {
+    const filters = {};
+    let q = String(query).replace(/\b(path|file|in|kind|lang|language):(\S+)/gi, (_, k, v) => {
+        const key = k.toLowerCase() === 'file' || k.toLowerCase() === 'in' ? 'path' : k.toLowerCase() === 'language' ? 'lang' : k.toLowerCase();
+        (filters[key] ??= []).push(v);
+        return ' ';
+    });
+    q = q.trim();
+    const rawTokens = q.split(/[\s,;]+/).filter(Boolean).map(t => t.replace(/^[`'"(]+|[`'"),.?!:]+$/g, '')).filter(Boolean);
+    const identifiers = [];
+    const words = [];
+    rawTokens.forEach((t, i) => {
+        if (IDENTIFIER_SHAPE.test(t) || looksLikeIdentifier(t, i)) identifiers.push(t.replace(/\(\)$/, ''));
+        else words.push(t);
+    });
+    const lowerWords = words.map(w => w.toLowerCase());
+    const stopCount = lowerWords.filter(w => STOPWORDS.has(w)).length;
+    // Natural language = several words, mostly not identifier-shaped, with function words.
+    const natural = rawTokens.length >= 3 && identifiers.length <= rawTokens.length / 2 && (stopCount >= 1 || rawTokens.length >= 5);
+    const termSet = new Set();
+    for (const id of identifiers) for (const t of codeTokens(id)) termSet.add(t);
+    for (const w of lowerWords) {
+        if (natural && STOPWORDS.has(w)) continue;
+        for (const t of codeTokens(w)) termSet.add(t);
+    }
+    // a single bare word query ("router") is an identifier lookup as much as a word
+    if (!identifiers.length && rawTokens.length <= 2) for (const w of rawTokens) if (/^[A-Za-z_$][\w$]*$/.test(w)) identifiers.push(w);
+    const terms = [...termSet].filter(t => t.length >= 2 || /^[a-z]$/.test(t) === false);
+    const expansions = new Set();
+    if (natural || identifiers.length === 0) {
+        for (const t of terms) {
+            const e = expandConcept(t);
+            if (e) for (const x of e) if (!termSet.has(x)) expansions.add(x);
+        }
+    }
+    return { raw: q, identifiers, words: lowerWords, terms, expansions: [...expansions], natural, filters };
+}

package/src/store/db.mjs

@@ -0,0 +1,169 @@
+/**
+ * Persistent index store (node:sqlite, WAL). Every fact is owned by exactly one file, so
+ * re-indexing a file is "delete its rows, insert the new ones" — the Glean model, which keeps
+ * incremental updates simple and correct.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+
+// node:sqlite prints an ExperimentalWarning on Node 22; it is noise on an MCP server's stderr.
+const origEmit = process.emitWarning;
+process.emitWarning = function (warning, ...rest) {
+    const msg = typeof warning === 'string' ? warning : warning?.message;
+    if (msg && msg.includes('SQLite is an experimental feature')) return;
+    return origEmit.call(this, warning, ...rest);
+};
+const { DatabaseSync } = await import('node:sqlite');
+
+export const SCHEMA_VERSION = 9;
+
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT);
+CREATE TABLE IF NOT EXISTS files (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  path TEXT NOT NULL UNIQUE,
+  lang TEXT NOT NULL,
+  size INTEGER NOT NULL,
+  mtime_ms INTEGER NOT NULL,
+  hash TEXT NOT NULL,
+  lines INTEGER NOT NULL,
+  package TEXT,
+  is_test INTEGER NOT NULL DEFAULT 0,
+  parse_errors INTEGER NOT NULL DEFAULT 0,
+  indexed_at INTEGER NOT NULL
+);
+CREATE TABLE IF NOT EXISTS symbols (
+  id INTEGER PRIMARY KEY AUTOINCREMENT, -- ids are never reused: stable ids of re-indexed symbols are re-inserted explicitly
+  file_id INTEGER NOT NULL,
+  name TEXT NOT NULL,
+  name_lc TEXT NOT NULL,
+  qname TEXT NOT NULL,
+  kind TEXT NOT NULL,
+  parent_id INTEGER,
+  owner TEXT,
+  start_line INTEGER NOT NULL, start_col INTEGER NOT NULL,
+  end_line INTEGER NOT NULL, end_col INTEGER NOT NULL,
+  name_line INTEGER NOT NULL, name_col INTEGER NOT NULL,
+  sig TEXT, doc TEXT, type TEXT,
+  exported INTEGER NOT NULL DEFAULT 0,
+  visibility TEXT,
+  decorators TEXT,
+  bases TEXT,
+  ordinal INTEGER NOT NULL DEFAULT 0,
+  is_static INTEGER NOT NULL DEFAULT 0
+);
+CREATE INDEX IF NOT EXISTS symbols_file ON symbols(file_id);
+CREATE INDEX IF NOT EXISTS symbols_name ON symbols(name_lc);
+CREATE INDEX IF NOT EXISTS symbols_qname ON symbols(qname);
+CREATE INDEX IF NOT EXISTS symbols_parent ON symbols(parent_id);
+CREATE TABLE IF NOT EXISTS refs (
+  id INTEGER PRIMARY KEY,
+  file_id INTEGER NOT NULL,
+  src_id INTEGER,
+  name TEXT NOT NULL,
+  kind TEXT NOT NULL,
+  line INTEGER NOT NULL,
+  col INTEGER NOT NULL,
+  recv TEXT,
+  recv_type TEXT,
+  dst_id INTEGER,
+  conf REAL NOT NULL DEFAULT 0,
+  ncand INTEGER NOT NULL DEFAULT 0,
+  argc INTEGER -- arguments of a call or construction (-1: a spread makes it a lower bound), else NULL
+);
+CREATE INDEX IF NOT EXISTS refs_file ON refs(file_id);
+CREATE INDEX IF NOT EXISTS refs_dst ON refs(dst_id);
+CREATE INDEX IF NOT EXISTS refs_src ON refs(src_id);
+CREATE INDEX IF NOT EXISTS refs_name ON refs(name);
+CREATE TABLE IF NOT EXISTS imports (
+  file_id INTEGER NOT NULL,
+  source TEXT,
+  imported TEXT,
+  local TEXT,
+  reexport TEXT,
+  wildcard INTEGER NOT NULL DEFAULT 0,
+  line INTEGER,
+  target_file_id INTEGER,
+  target_dir TEXT,
+  target_path TEXT
+);
+CREATE INDEX IF NOT EXISTS imports_file ON imports(file_id);
+CREATE INDEX IF NOT EXISTS imports_target ON imports(target_file_id);
+CREATE TABLE IF NOT EXISTS fields (
+  file_id INTEGER NOT NULL,
+  owner_qname TEXT NOT NULL,
+  name TEXT NOT NULL,
+  type TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS fields_file ON fields(file_id);
+CREATE INDEX IF NOT EXISTS fields_owner ON fields(owner_qname);
+CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(
+  name, qname, sig, doc, path, body,
+  tokenize = 'porter unicode61 remove_diacritics 2',
+  content = '', contentless_delete = 1
+);
+-- one document per file (rowid = files.id): file-level relevance for hierarchical ranking
+CREATE VIRTUAL TABLE IF NOT EXISTS file_fts USING fts5(
+  path, body,
+  tokenize = 'porter unicode61 remove_diacritics 2',
+  content = '', contentless_delete = 1
+);
+`;
+
+export class Store {
+    constructor(dbPath) {
+        this.path = dbPath;
+        this.stmts = new Map();
+        fs.mkdirSync(path.dirname(dbPath), { recursive: true });
+        this.db = new DatabaseSync(dbPath);
+        this.db.exec('PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL; PRAGMA temp_store = MEMORY; PRAGMA cache_size = -65536; PRAGMA busy_timeout = 5000;');
+        const v = this.#schemaVersion();
+        if (v !== null && v !== SCHEMA_VERSION) {
+            // schema changed: rebuild from scratch (the index is a cache of the source tree)
+            this.db.close();
+            for (const suffix of ['', '-wal', '-shm']) { try { fs.unlinkSync(dbPath + suffix); } catch { /* none */ } }
+            this.db = new DatabaseSync(dbPath);
+            this.db.exec('PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL; PRAGMA temp_store = MEMORY; PRAGMA cache_size = -65536; PRAGMA busy_timeout = 5000;');
+        }
+        this.db.exec(SCHEMA);
+        this.setMeta('schema_version', String(SCHEMA_VERSION));
+    }
+
+    #schemaVersion() {
+        try {
+            const row = this.db.prepare("SELECT value FROM meta WHERE key = 'schema_version'").get();
+            return row ? Number(row.value) : null;
+        } catch { return null; }
+    }
+
+    /** Cached prepared statement. */
+    q(sql) {
+        let s = this.stmts.get(sql);
+        if (!s) { s = this.db.prepare(sql); this.stmts.set(sql, s); }
+        return s;
+    }
+
+    all(sql, ...params) { return this.q(sql).all(...params); }
+    get(sql, ...params) { return this.q(sql).get(...params); }
+    run(sql, ...params) { return this.q(sql).run(...params); }
+
+    tx(fn) {
+        this.db.exec('BEGIN');
+        try { const r = fn(); this.db.exec('COMMIT'); return r; }
+        catch (e) { try { this.db.exec('ROLLBACK'); } catch { /* already rolled back */ } throw e; }
+    }
+
+    /** Drop every indexed fact but keep the database (the index is a cache of the source tree). */
+    reset() {
+        this.tx(() => {
+            for (const t of ['files', 'symbols', 'refs', 'imports', 'fields']) this.db.exec(`DELETE FROM ${t}`);
+            this.db.exec("INSERT INTO fts(fts) VALUES ('delete-all')");
+            this.db.exec("INSERT INTO file_fts(file_fts) VALUES ('delete-all')");
+        });
+    }
+
+    getMeta(key) { return this.get('SELECT value FROM meta WHERE key = ?', key)?.value ?? null; }
+    setMeta(key, value) { this.run('INSERT INTO meta(key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value', key, value); }
+
+    close() { try { this.db.close(); } catch { /* closed */ } }
+}

package/src/util/git.mjs

@@ -0,0 +1,30 @@
+/**
+ * git, run so that an indexed repository's own config cannot make it execute anything: no
+ * fsmonitor, no hooks, no signature verification, no external diff or textconv, and every
+ * clean/smudge/process filter switched off. A repository copied with its .git directory (an
+ * archive, a shared folder) is not trusted.
+ */
+import { spawnSync } from 'node:child_process';
+
+const COMMAND_KEY = /^(filter\.[^\n]+)\.(clean|smudge|process)$|^diff\.[^\n]+\.(textconv|command)$/;
+
+function hardening(root) {
+    const args = ['-c', 'core.fsmonitor=false', '-c', 'core.hooksPath=/dev/null', '-c', 'diff.external=', '-c', 'log.showSignature=false'];
+    const r = spawnSync('git', ['config', '--name-only', '--get-regexp', '^(filter|diff)\\.'], { cwd: root, encoding: 'utf8' });
+    for (const key of (r.status === 0 ? r.stdout : '').split('\n')) {
+        const m = COMMAND_KEY.exec(key);
+        if (!m) continue;
+        args.push('-c', `${key}=`);
+        // a required filter (Git LFS sets one) would otherwise fail the diff once its command is blank
+        if (m[1]) args.push('-c', `${m[1]}.required=false`);
+    }
+    return args;
+}
+
+const NO_TOOLS = { diff: ['--no-ext-diff', '--no-textconv'], log: ['--no-ext-diff', '--no-textconv', '--no-show-signature'] };
+
+/** spawnSync('git', args) in `root`, hardened as above; diff and log never run external tools. */
+export function gitSync(root, args, opts = {}) {
+    const sub = NO_TOOLS[args[0]] ? [args[0], ...NO_TOOLS[args[0]], ...args.slice(1)] : args;
+    return spawnSync('git', [...hardening(root), ...sub], { cwd: root, encoding: 'utf8', ...opts });
+}

package/src/util/paths.mjs

@@ -0,0 +1,63 @@
+/** Data layout: everything graph-indexer writes lives in <repo>/.graph-indexer/ (git-ignored). */
+import fs from 'node:fs';
+import path from 'node:path';
+
+export const DATA_DIR_NAME = '.graph-indexer';
+
+export function dataDir(root) {
+    const dir = path.join(root, DATA_DIR_NAME);
+    fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
+    // a .graph-indexer the repository committed as a link would put the index (excerpts of the
+    // source) wherever it points
+    const st = fs.lstatSync(dir);
+    if (!st.isDirectory()) throw new Error(`${dir} is not a directory (a symbolic link?); remove it and run again`);
+    if ((st.mode & 0o077) !== 0) { try { fs.chmodSync(dir, 0o700); } catch { /* read-only */ } }
+    const gi = path.join(dir, '.gitignore');
+    if (!fs.existsSync(gi)) { try { fs.writeFileSync(gi, '*\n'); } catch { /* read-only */ } }
+    return dir;
+}
+
+/** Walk up from `start` to the nearest directory containing .git (or return start). */
+export function findRepoRoot(start) {
+    let dir = path.resolve(start);
+    for (;;) {
+        if (fs.existsSync(path.join(dir, '.git'))) return dir;
+        const parent = path.dirname(dir);
+        if (parent === dir) return path.resolve(start);
+        dir = parent;
+    }
+}
+
+const realRoots = new Map();
+
+/**
+ * `rel` names a file inside the repository once `..` and symlinks are resolved, so a tool target
+ * like `../../etc/hosts` or a tracked link to ~/.ssh/… is never read. The index's own data and git's
+ * internals are not served either.
+ */
+export function withinRoot(root, rel) {
+    let realRoot = realRoots.get(root);
+    if (!realRoot) { try { realRoot = fs.realpathSync(root); } catch { realRoot = path.resolve(root); } realRoots.set(root, realRoot); }
+    let real;
+    try { real = fs.realpathSync(path.join(root, rel)); } catch { return false; }
+    if (real !== realRoot && !real.startsWith(realRoot + path.sep)) return false;
+    const top = path.relative(realRoot, real).split(path.sep)[0];
+    return top !== '.git' && top !== DATA_DIR_NAME;
+}
+
+/** A git revision given by a caller: never an option (`--output=…`), whitespace or NUL. */
+export const isRevision = (s) => typeof s === 'string' && s.length > 0 && s.length <= 256 && !s.startsWith('-') && !/[\s\0]/.test(s);
+
+/**
+ * A small regular file of the repository (a manifest, a config), or null: never a device, a FIFO,
+ * a file larger than `maxBytes` or a link out of the root — a `tsconfig.json` linked to /dev/zero
+ * would otherwise hang the indexer.
+ */
+export function readRepoFile(root, rel, maxBytes = 2_000_000) {
+    if (!withinRoot(root, rel)) return null;
+    try {
+        const abs = path.join(root, rel);
+        const st = fs.statSync(abs);
+        return st.isFile() && st.size <= maxBytes ? fs.readFileSync(abs, 'utf8') : null;
+    } catch { return null; }
+}

package/src/util/regex.mjs

@@ -0,0 +1,2 @@
+/** A string as a literal inside a regular expression: every metacharacter escaped. */
+export const escapeRegExp = (s) => String(s).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');

package/vendor/grammars/manifest.json

@@ -0,0 +1,148 @@
+{
+  "runtime": {
+    "package": "web-tree-sitter",
+    "version": "0.25.10",
+    "source": "https://github.com/tree-sitter/tree-sitter/releases/tag/v0.25.10",
+    "license": "MIT",
+    "files": {
+      "web-tree-sitter.js": {
+        "sha256": "d4fc466df6358055253bc4cdaab23cdf84fe00741fa5a4ff56506c06c533da42",
+        "bytes": 150999
+      },
+      "web-tree-sitter.wasm": {
+        "sha256": "f38dcc4b43b818f9a0785bc1c6d5611a75ac4cdd428ff3f02757c34ca4e46d7f",
+        "bytes": 205488
+      }
+    }
+  },
+  "grammars": {
+    "bash": {
+      "repo": "https://github.com/tree-sitter/tree-sitter-bash",
+      "tag": "v0.25.1",
+      "asset": "tree-sitter-bash.wasm",
+      "sha256": "8292919c88a0f7d3fb31d0cd0253ca5a9531bc1ede82b0537f2c63dd8abe6a7a",
+      "bytes": 1358224,
+      "license": "MIT"
+    },
+    "c": {
+      "repo": "https://github.com/tree-sitter/tree-sitter-c",
+      "tag": "v0.24.2",
+      "asset": "tree-sitter-c.wasm",
+      "sha256": "83e8d7902b9d7f8c7c5cd4bd9acb5c7eb5faf42c09f85546b183964d3b5f48f9",
+      "bytes": 645157,
+      "license": "MIT"
+    },
+    "c_sharp": {
+      "repo": "https://github.com/tree-sitter/tree-sitter-c-sharp",
+      "tag": "v0.23.5",
+      "asset": "tree-sitter-c_sharp.wasm",
+      "sha256": "6f69e1cae44e1c32c1eccc170dc5a9778fb94ff716f71113fe1f8c4299aa2f40",
+      "bytes": 5350581,
+      "license": "MIT"
+    },
+    "cpp": {
+      "repo": "https://github.com/tree-sitter/tree-sitter-cpp",
+      "tag": "v0.23.4",
+      "asset": "tree-sitter-cpp.wasm",
+      "sha256": "174eb0deb75b2ec7881bcacda9f995648d8e683956e5c2267e69ab6dc503fcbf",
+      "bytes": 3434931,
+      "license": "MIT"
+    },
+    "css": {
+      "repo": "https://github.com/tree-sitter/tree-sitter-css",
+      "tag": "v0.25.0",
+      "asset": "tree-sitter-css.wasm",
+      "sha256": "8a23977fe271357cce6f254ef88c9bebf3854602d8046605aef6a45c02135c59",
+      "bytes": 128668,
+      "license": "MIT"
+    },
+    "go": {
+      "repo": "https://github.com/tree-sitter/tree-sitter-go",
+      "tag": "v0.25.0",
+      "asset": "tree-sitter-go.wasm",
+      "sha256": "9504573f352b20be7f2f1911754d710622aedc15afff16d5ed8fb5645681aee7",
+      "bytes": 217182,
+      "license": "MIT"
+    },
+    "java": {
+      "repo": "https://github.com/tree-sitter/tree-sitter-java",
+      "tag": "v0.23.5",
+      "asset": "tree-sitter-java.wasm",
+      "sha256": "4fdeac4ca6ca089f06c6f7e562abcac1733cd465728cc7031ebb73c2019122c4",
+      "bytes": 414641,
+      "license": "MIT"
+    },
+    "javascript": {
+      "repo": "https://github.com/tree-sitter/tree-sitter-javascript",
+      "tag": "v0.25.0",
+      "asset": "tree-sitter-javascript.wasm",
+      "sha256": "5fb488d0cabb4775a594bab85682de5ad6ce83c0d6ac997a9f82dd084d571240",
+      "bytes": 411770,
+      "license": "MIT"
+    },
+    "kotlin": {
+      "repo": "https://github.com/fwcd/tree-sitter-kotlin",
+      "tag": "0.3.8",
+      "asset": "tree-sitter-kotlin.wasm",
+      "sha256": "c624e7443b371c28adc5d81674e73067564c12555ebe3ed96a6c8db814b7602d",
+      "bytes": 4052625,
+      "license": "MIT"
+    },
+    "php": {
+      "repo": "https://github.com/tree-sitter/tree-sitter-php",
+      "tag": "v0.24.2",
+      "asset": "tree-sitter-php.wasm",
+      "sha256": "d4df6a6ff08c87c3ec4f9cbb785fe09998a0cb570e03f57d7b19b3acfb146aa7",
+      "bytes": 1058041,
+      "license": "MIT"
+    },
+    "python": {
+      "repo": "https://github.com/tree-sitter/tree-sitter-python",
+      "tag": "v0.25.0",
+      "asset": "tree-sitter-python.wasm",
+      "sha256": "16108b50df4ee9a30168794252ab55e7c93bfc5765d7fa0aa3e335752c515f47",
+      "bytes": 457883,
+      "license": "MIT"
+    },
+    "ruby": {
+      "repo": "https://github.com/tree-sitter/tree-sitter-ruby",
+      "tag": "v0.23.1",
+      "asset": "tree-sitter-ruby.wasm",
+      "sha256": "09a96427d7c72f0613ed470cd9812223fc4a91d6a9c025c0235cc6bd59ff96f4",
+      "bytes": 2106352,
+      "license": "MIT"
+    },
+    "rust": {
+      "repo": "https://github.com/tree-sitter/tree-sitter-rust",
+      "tag": "v0.24.2",
+      "asset": "tree-sitter-rust.wasm",
+      "sha256": "24c89bd9252255e4aebbcbd7d2d308bd92c86dd95a130fdc80efa49577b8d738",
+      "bytes": 1118099,
+      "license": "MIT"
+    },
+    "scala": {
+      "repo": "https://github.com/tree-sitter/tree-sitter-scala",
+      "tag": "v0.26.2",
+      "asset": "tree-sitter-scala.wasm",
+      "sha256": "37d7fe5a91ca98941dc05493b0c05a0df0f36df5035890fa00b02497c68aaac3",
+      "bytes": 3973559,
+      "license": "MIT"
+    },
+    "tsx": {
+      "repo": "https://github.com/tree-sitter/tree-sitter-typescript",
+      "tag": "v0.23.2",
+      "asset": "tree-sitter-tsx.wasm",
+      "sha256": "79e5da75ea62855a0cd67177685f0164eac87d5f630b3cbe1e0a099751ad30f8",
+      "bytes": 1445638,
+      "license": "MIT"
+    },
+    "typescript": {
+      "repo": "https://github.com/tree-sitter/tree-sitter-typescript",
+      "tag": "v0.23.2",
+      "asset": "tree-sitter-typescript.wasm",
+      "sha256": "778025db5a8be0e70f8ccc3671e486dfeddd048c25d9e8a70c26de2e1bf6f97d",
+      "bytes": 1413849,
+      "license": "MIT"
+    }
+  }
+}