graph-indexer 2.1.0 → 3.0.0

package/src/cli/resident.mjs

@@ -0,0 +1,175 @@
+/**
+ * The resident process: a long-lived graph-indexer that keeps one repository's index open and in
+ * sync, and answers hooks and CLI queries over a local socket.
+ *
+ * Opening the index and reconciling it with the working tree costs 200–500 ms on a repository of
+ * 1,600 files, on every hook call and every CLI query. Hooks run on each read, search and edit, so
+ * that cost lands on every step of the agent. The MCP server (`serve`) is already long-lived and
+ * serves as the resident when the agent runs it; `graph-indexer daemon` does the same for agents
+ * that only use hooks or the CLI, and exits after a while without requests.
+ *
+ * Clients ask first and fall back to opening the index themselves when nothing answers, so a
+ * missing, dead or different-version resident only costs the old latency.
+ *
+ * Protocol: one JSON request per connection, one JSON reply, each ended by a newline.
+ *   { v, version, op: 'ping' }                  → { ok, root, pid }
+ *   { v, version, op: 'hook', event, input }     → { ok, out }   out: what the hook prints, or null
+ *   { v, version, op: 'tool', calls }            → { ok, out }   out: the CLI's output text
+ * A request from another graph-indexer version gets { ok: false } and the client falls back, except
+ * from thin clients that only relay hooks (`any: true`, integrations/claude-code/hooks/gi-hook.mjs).
+ *
+ * The socket is only reachable by its owner: inside the repository's .graph-indexer/ directory,
+ * or, when that path is too long for a Unix socket, in a per-user directory with mode 0700 that
+ * both sides check before using it. On Windows it is a named pipe.
+ */
+import net from 'node:net';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { DATA_DIR_NAME } from '../util/paths.mjs';
+
+const PROTOCOL = 1;
+const MAX_REQUEST = 8 * 1024 * 1024;
+const WIN = process.platform === 'win32';
+
+/** Where the resident for `root` listens. */
+export function socketPath(root) {
+    const abs = path.resolve(root);
+    const hash = crypto.createHash('sha256').update(abs).digest('hex').slice(0, 16);
+    if (WIN) return `\\\\.\\pipe\\graph-indexer-${hash}`;
+    const inRepo = path.join(abs, DATA_DIR_NAME, 'resident.sock');
+    // Unix socket paths are limited to 104–108 bytes
+    if (Buffer.byteLength(inRepo) <= 100) return inRepo;
+    return path.join(os.tmpdir(), `graph-indexer-${process.getuid?.() ?? 'user'}`, `${hash}.sock`);
+}
+
+/** A per-user directory in the temp dir must belong to us and be closed to everyone else. */
+function privateDir(dir, root) {
+    if (WIN || dir === path.join(path.resolve(root), DATA_DIR_NAME)) return true;
+    try {
+        const st = fs.lstatSync(dir);
+        return st.isDirectory() && (process.getuid ? st.uid === process.getuid() : true) && (st.mode & 0o077) === 0;
+    } catch { return false; }
+}
+
+const disabled = () => process.env.GRAPH_INDEXER_RESIDENT === '0';
+/** GRAPH_INDEXER_RESIDENT=require: answer only through a resident, never by opening the index (tests, debugging). */
+export const residentRequired = () => process.env.GRAPH_INDEXER_RESIDENT === 'require';
+
+/**
+ * Ask the resident serving `root`. Resolves to its reply ({ ok: true, … }) or null when there is
+ * none, it is another version, it fails, or it is slower than `totalMs`.
+ */
+export function askResident(root, request, { version, connectMs = 250, totalMs = 5000 } = {}) {
+    if (disabled()) return Promise.resolve(null);
+    const file = socketPath(root);
+    if (!WIN && (!fs.existsSync(file) || !privateDir(path.dirname(file), root))) return Promise.resolve(null);
+    return new Promise((resolve) => {
+        let done = false, connected = false, buf = '';
+        const sock = net.createConnection(file);
+        const finish = (v) => {
+            if (done) return;
+            done = true;
+            clearTimeout(connectTimer);
+            clearTimeout(totalTimer);
+            sock.destroy();
+            resolve(v);
+        };
+        const connectTimer = setTimeout(() => { if (!connected) finish(null); }, connectMs);
+        const totalTimer = setTimeout(() => finish(null), totalMs);
+        sock.setEncoding('utf8');
+        sock.on('connect', () => {
+            connected = true;
+            sock.write(JSON.stringify({ v: PROTOCOL, version, ...request }) + '\n');
+        });
+        sock.on('data', (d) => {
+            buf += d;
+            const i = buf.indexOf('\n');
+            if (i < 0) return;
+            let reply = null;
+            try { reply = JSON.parse(buf.slice(0, i)); } catch { /* garbled */ }
+            finish(reply?.ok ? reply : null);
+        });
+        sock.on('error', () => finish(null));
+        sock.on('close', () => finish(null));
+    });
+}
+
+/**
+ * Serve `root` on its socket. `handle(request)` answers hook and tool requests. Returns null when
+ * another resident already serves the repository (or the socket cannot be made private), else
+ * { file, close }. `idleMs` > 0 calls `onIdle` after that long without a request, or as soon as
+ * `wanted()` turns false.
+ */
+export async function startResident({ root, version, handle, log = () => {}, idleMs = 0, onIdle = null, wanted = () => true }) {
+    if (disabled()) return null;
+    const file = socketPath(root);
+    if (!WIN) {
+        const dir = path.dirname(file);
+        try { fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); } catch { /* checked below */ }
+        if (!privateDir(dir, root)) { log(`resident: ${dir} is not private; not listening`); return null; }
+    }
+    // another resident may serve this repository already; a dead one leaves its socket file behind
+    if (await askResident(root, { op: 'ping' }, { version, connectMs: 300, totalMs: 1500 })) return null;
+    if (!WIN) { try { fs.unlinkSync(file); } catch { /* none */ } }
+
+    let last = Date.now();
+    const server = net.createServer((sock) => {
+        let buf = '';
+        sock.setEncoding('utf8');
+        sock.on('error', () => { /* client went away */ });
+        sock.on('data', async (d) => {
+            buf += d;
+            if (buf.length > MAX_REQUEST) { sock.destroy(); return; }
+            const i = buf.indexOf('\n');
+            if (i < 0) return;
+            const line = buf.slice(0, i);
+            buf = '';
+            last = Date.now();
+            let reply;
+            try {
+                const req = JSON.parse(line);
+                // `any`: a thin client with no logic of its own (the Claude Code plugin's hook command)
+                if (req.v !== PROTOCOL || (req.version !== version && req.any !== true)) reply = { ok: false, reason: 'version' };
+                else if (req.op === 'ping') reply = { ok: true, root, pid: process.pid };
+                else reply = { ok: true, out: (await handle(req)) ?? null };
+            } catch (e) { reply = { ok: false, reason: String(e?.message ?? e).slice(0, 200) }; }
+            last = Date.now();
+            if (!sock.destroyed) sock.end(JSON.stringify(reply) + '\n');
+        });
+    });
+    try {
+        await new Promise((resolve, reject) => { server.once('error', reject); server.listen(file, resolve); });
+    } catch (e) {
+        log(`resident: cannot listen on ${file} (${e.code ?? e.message})`);
+        return null;
+    }
+    let ino = null;
+    if (!WIN) {
+        try { fs.chmodSync(file, 0o600); } catch { /* best effort */ }
+        try { ino = fs.statSync(file).ino; } catch { /* checked on close */ }
+    }
+    server.unref(); // the caller's own work (stdio, watcher, idle timer) decides the process lifetime
+
+    let closed = false;
+    const close = () => {
+        if (closed) return;
+        closed = true;
+        try { server.close(); } catch { /* closed */ }
+        // only our own socket file: a resident started after us may have replaced it
+        if (!WIN) { try { if (fs.statSync(file).ino === ino) fs.unlinkSync(file); } catch { /* gone */ } }
+    };
+    process.once('exit', close);
+    if (idleMs > 0) {
+        // also stop when the repository or its index goes away
+        const timer = setInterval(() => {
+            if (Date.now() - last < idleMs && wanted()) return;
+            clearInterval(timer);
+            close();
+            onIdle?.();
+        }, Math.min(idleMs, 30_000));
+    }
+    log(`resident: answering hooks and CLI queries on ${file}`);
+    return { file, close };
+}

package/src/index/discover.mjs

@@ -0,0 +1,130 @@
+/**
+ * File discovery. Inside a git work tree we ask git (tracked + untracked-but-not-ignored),
+ * which honours every .gitignore/.git/info/exclude rule exactly; elsewhere we walk the tree
+ * with a small .gitignore matcher. On top of that we skip what an agent never wants in its
+ * index: dependencies, build output, vendored bundles, minified and generated files.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { specForPath } from '../parse/languages.mjs';
+
+export const MAX_FILE_BYTES = 1_500_000;
+
+const SKIP_DIRS = new Set([
+    'node_modules', 'bower_components', 'jspm_packages', '.git', '.hg', '.svn', '.graph-indexer',
+    'dist', 'build', 'out', 'target', '.next', '.nuxt', '.svelte-kit', '.output', '.turbo', '.cache', '.parcel-cache',
+    'coverage', '.nyc_output', '__pycache__', '.venv', 'venv', 'env', '.env', '.tox', '.mypy_cache', '.pytest_cache',
+    'site-packages', '.gradle', '.idea', '.vscode', 'Pods', 'DerivedData', 'vendor', 'third_party', 'thirdparty',
+    'external', 'deps', '_deps', 'obj', 'bin', 'packages-cache', '.yarn', '.pnpm-store', 'elm-stuff', '.terraform',
+]);
+// `vendor` / `bin` / `obj` are common real source dirs in some ecosystems only when they contain
+// sources the user wrote; the conservative default is to skip them (override via config.include).
+
+const GENERATED_FILE = /(\.min\.(js|css|mjs)$|\.bundle\.js$|[.-]bundle\.[cm]?js$|\.chunk\.js$|\.map$|\.pb\.go$|_pb2(_grpc)?\.py$|\.pb\.(cc|h)$|\.generated\.\w+$|\.g\.dart$|\.designer\.cs$|(^|\/)generated\/|\.d\.ts\.map$)/i;
+
+function isGitRepo(root) {
+    const r = spawnSync('git', ['rev-parse', '--is-inside-work-tree'], { cwd: root, encoding: 'utf8' });
+    return r.status === 0 && r.stdout.trim() === 'true';
+}
+
+function gitListFiles(root) {
+    const r = spawnSync('git', ['ls-files', '-z', '--cached', '--others', '--exclude-standard'], {
+        cwd: root, encoding: 'utf8', maxBuffer: 1 << 30,
+    });
+    if (r.status !== 0) return null;
+    return r.stdout.split('\0').filter(Boolean);
+}
+
+/** Minimal .gitignore → predicate (used only outside git repos). */
+function loadGitignore(root) {
+    const rules = [];
+    try {
+        for (let line of fs.readFileSync(path.join(root, '.gitignore'), 'utf8').split(/\r?\n/)) {
+            line = line.trim();
+            if (!line || line.startsWith('#')) continue;
+            const neg = line.startsWith('!');
+            if (neg) line = line.slice(1);
+            const dirOnly = line.endsWith('/');
+            if (dirOnly) line = line.slice(0, -1);
+            const anchored = line.startsWith('/') || line.includes('/');
+            line = line.replace(/^\//, '');
+            const re = line.split('**').map(seg => seg.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '[^/]*').replace(/\?/g, '[^/]')).join('.*');
+            rules.push({ neg, re: new RegExp(anchored ? `^${re}(/|$)` : `(^|/)${re}(/|$)`) });
+        }
+    } catch { /* no .gitignore */ }
+    return (rel) => {
+        let ignored = false;
+        for (const r of rules) if (r.re.test(rel)) ignored = !r.neg;
+        return ignored;
+    };
+}
+
+function walk(root, ignored) {
+    const out = [];
+    const stack = [''];
+    while (stack.length) {
+        const rel = stack.pop();
+        let entries;
+        try { entries = fs.readdirSync(path.join(root, rel), { withFileTypes: true }); } catch { continue; }
+        for (const e of entries) {
+            const r = rel ? rel + '/' + e.name : e.name;
+            if (e.isDirectory()) {
+                if (SKIP_DIRS.has(e.name) || e.name.startsWith('.') || ignored(r)) continue;
+                stack.push(r);
+            } else if (e.isFile() && !ignored(r)) out.push(r);
+        }
+    }
+    return out;
+}
+
+export function isIndexablePath(rel, { include = null } = {}) {
+    if (!specForPath(rel)) return false;
+    const parts = rel.split('/');
+    if (include && include.some(p => rel.startsWith(p))) return !GENERATED_FILE.test(rel);
+    for (let i = 0; i < parts.length - 1; i++) if (SKIP_DIRS.has(parts[i])) return false;
+    return !GENERATED_FILE.test(rel);
+}
+
+/**
+ * List indexable files (repo-relative, forward slashes), sorted for determinism.
+ * @returns {{ files: string[], viaGit: boolean }}
+ */
+export function discoverFiles(root, opts = {}) {
+    let rels = null, viaGit = false;
+    if (!opts.noGit && isGitRepo(root)) {
+        rels = gitListFiles(root);
+        viaGit = rels !== null;
+    }
+    if (!rels) rels = walk(root, loadGitignore(root));
+    const files = rels.map(r => r.replace(/\\/g, '/')).filter(r => isIndexablePath(r, opts) && (!viaGit || insideRoot(root, r)));
+    files.sort();
+    return { files, viaGit };
+}
+
+const realRoots = new Map();
+
+/**
+ * A tracked file must resolve inside the repository: symlinks are followed only when their target
+ * stays under the root, so a link to /etc/… or ~/.ssh/… is never read, indexed or served.
+ */
+export function insideRoot(root, rel) {
+    const abs = path.join(root, rel);
+    let st;
+    try { st = fs.lstatSync(abs); } catch { return true; } // vanished: the caller handles deletion
+    if (!st.isSymbolicLink()) return true;
+    let realRoot = realRoots.get(root);
+    if (!realRoot) { try { realRoot = fs.realpathSync(root); } catch { realRoot = root; } realRoots.set(root, realRoot); }
+    try {
+        const target = fs.realpathSync(abs);
+        return target === realRoot || target.startsWith(realRoot + path.sep);
+    } catch { return false; } // dangling link
+}
+
+/** Heuristic: minified / generated content that slipped past the name filters. */
+export function looksMinified(source) {
+    if (source.length < 2000) return false;
+    const lines = source.split('\n', 50);
+    const long = lines.filter(l => l.length > 500).length;
+    return long >= 3 || source.length / Math.max(1, source.split('\n').length) > 300;
+}

package/src/index/fingerprint.mjs

@@ -0,0 +1,30 @@
+/**
+ * Fingerprint of everything that determines the stored facts: extraction queries and code,
+ * module and reference resolution, FTS tokenization, the schema and the grammar builds. An index
+ * written under a different fingerprint is rebuilt, so upgrading graph-indexer never keeps serving
+ * facts that an older extractor produced for files that have not changed since.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { fileURLToPath } from 'node:url';
+
+const SRC = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const INPUTS = ['parse', 'index', 'search/tokenize.mjs', 'store/db.mjs', '../vendor/grammars/manifest.json'];
+const QUERY_TIME_ONLY = new Set(['graph.mjs']);
+
+let cached = null;
+
+export function indexFingerprint() {
+    if (cached) return cached;
+    const h = crypto.createHash('sha1');
+    const add = (p) => {
+        const st = fs.statSync(p, { throwIfNoEntry: false });
+        if (!st) return;
+        if (st.isDirectory()) { for (const e of fs.readdirSync(p).sort()) add(path.join(p, e)); return; }
+        if (!/\.(mjs|json)$/.test(p) || QUERY_TIME_ONLY.has(path.basename(p))) return;
+        h.update(path.relative(SRC, p).split(path.sep).join('/')).update('\0').update(fs.readFileSync(p)).update('\0');
+    };
+    for (const i of INPUTS) add(path.join(SRC, i));
+    return (cached = h.digest('hex').slice(0, 16));
+}

package/src/index/graph.mjs

@@ -0,0 +1,61 @@
+/**
+ * Graph analytics over the resolved reference graph: symbol centrality (PageRank, optionally
+ * personalized to a focus set — the Aider repo-map idea applied to symbols) and helpers for
+ * traversals used by the impact / call-graph tools.
+ */
+
+/** Load the weighted symbol graph: edges src → dst (references), plus member → container. */
+export function loadGraph(store) {
+    const edges = store.all(`SELECT src_id AS s, dst_id AS d, SUM(conf) AS w, COUNT(*) AS n FROM refs
+        WHERE dst_id IS NOT NULL AND src_id IS NOT NULL AND src_id != dst_id GROUP BY src_id, dst_id`);
+    const contain = store.all('SELECT id AS s, parent_id AS d FROM symbols WHERE parent_id IS NOT NULL');
+    const ids = store.all('SELECT id FROM symbols').map(r => r.id);
+    return { ids, edges, contain };
+}
+
+/**
+ * PageRank over symbols. Edge weight = Σconf (√ damped so one hot call site does not dominate).
+ * @param {object} store
+ * @param {{ personalization?: Map<number, number>, damping?: number, iters?: number }} opts
+ * @returns {Map<number, number>} id → score normalized to a [0,1] percentile
+ */
+export function computeCentrality(store, { personalization = null, damping = 0.85, iters = 40, graph = null } = {}) {
+    const g = graph ?? loadGraph(store);
+    const n = g.ids.length;
+    if (!n) return new Map();
+    const index = new Map(g.ids.map((id, i) => [id, i]));
+    const out = new Float64Array(n);
+    const src = [], dst = [], wt = [];
+    const add = (s, d, w) => {
+        const a = index.get(s), b = index.get(d);
+        if (a === undefined || b === undefined) return;
+        src.push(a); dst.push(b); wt.push(w); out[a] += w;
+    };
+    for (const e of g.edges) add(e.s, e.d, Math.sqrt(e.w));
+    for (const e of g.contain) add(e.s, e.d, 0.3);
+    let pers = null;
+    if (personalization?.size) {
+        pers = new Float64Array(n);
+        let tot = 0;
+        for (const [id, v] of personalization) { const i = index.get(id); if (i !== undefined) { pers[i] += v; tot += v; } }
+        if (tot > 0) for (let i = 0; i < n; i++) pers[i] /= tot; else pers = null;
+    }
+    let rank = new Float64Array(n).fill(1 / n);
+    for (let it = 0; it < iters; it++) {
+        const next = new Float64Array(n);
+        let dangling = 0;
+        for (let i = 0; i < n; i++) if (out[i] === 0) dangling += rank[i];
+        for (let k = 0; k < src.length; k++) next[dst[k]] += damping * rank[src[k]] * wt[k] / out[src[k]];
+        for (let i = 0; i < n; i++) {
+            const teleport = pers ? pers[i] : 1 / n;
+            next[i] += (1 - damping + damping * dangling) * teleport;
+        }
+        rank = next;
+    }
+    if (personalization) return new Map(g.ids.map((id, i) => [id, rank[i]]));
+    // percentile normalization: robust to the heavy tail
+    const order = [...rank.keys()].sort((a, b) => rank[a] - rank[b]);
+    const pct = new Map();
+    for (let r = 0; r < order.length; r++) pct.set(g.ids[order[r]], n > 1 ? r / (n - 1) : 1);
+    return pct;
+}