graph-indexer 2.1.1 → 3.0.1

package/src/query/read.mjs

@@ -0,0 +1,81 @@
+/**
+ * What a piece of code depends on, for reading it without searching: the definitions its lines use
+ * that live outside those lines, with where each one is and its signature.
+ *
+ * An agent exploring code reads a function, meets a name, searches for its definition, reads that,
+ * meets another name… one hop per call. The index already resolved those names (scope, imports,
+ * receiver types), so a read can carry the answers: this module ranks them and the tools render them
+ * next to the code (a "defined elsewhere" card) and in hooks after the agent's own reads.
+ */
+
+// how much a use says about what the reader needs to know
+const KIND_WEIGHT = { call: 1, new: 1, inherit: 1, type: 0.8, decorator: 0.6, value: 0.6, read: 0.5 };
+const NAMED_KINDS = ['call', 'new', 'inherit', 'type', 'decorator'];
+// what the reader most often needs next: code it calls and types it uses; data (fields, constants,
+// enum members) comes after, its one-line definition is usually all there is to it
+const TARGET_WEIGHT = (kind) => (['field', 'property', 'variable', 'constant'].includes(kind) ? 0.35 : 1);
+const DEFINITION_KINDS = ['function', 'method', 'constructor', 'class', 'interface', 'struct', 'trait', 'enum', 'type', 'constant', 'macro', 'object', 'module'];
+
+/**
+ * Definitions used by lines [from, to] of `file` that are defined outside that range.
+ *
+ * Bound references come from the resolver; a name that stayed unbound is included when exactly one
+ * definition in the repository's non-test code carries it (marked `byName`). Ranked by how the code
+ * uses them (calls, instantiations and base types first; more uses first), then by first use.
+ *
+ * @param {object} opts
+ * @param {number} [opts.max] rows returned in `rows`; the rest go to `more`
+ * @param {Set<number>} [opts.skipIds] definitions not to list (already shown to the reader)
+ * @param {Set<number>} [opts.skipAncestorsOf] symbol id whose enclosing definitions are not listed
+ * @returns {{ rows: object[], more: object[] }}
+ */
+export function definitionsUsed(intel, file, from, to, { max = 10, skipIds = new Set(), skipAncestorsOf = null } = {}) {
+    const f = intel.store.get('SELECT id FROM files WHERE path = ?', file);
+    if (!f) return { rows: [], more: [] };
+    const skip = new Set(skipIds);
+    for (let p = skipAncestorsOf != null ? intel.store.get('SELECT parent_id FROM symbols WHERE id = ?', skipAncestorsOf)?.parent_id : null; p != null;
+        p = intel.store.get('SELECT parent_id FROM symbols WHERE id = ?', p)?.parent_id) skip.add(p);
+    const refs = intel.store.all('SELECT name, kind, line, dst_id, conf FROM refs WHERE file_id = ? AND line BETWEEN ? AND ? ORDER BY line', f.id, from, to);
+    const bound = new Map(), unbound = new Map();
+    for (const r of refs) {
+        const w = KIND_WEIGHT[r.kind] ?? 0.5;
+        if (r.dst_id != null) {
+            // weak name-only bindings of plain values are mostly locals that share a global's name
+            if ((r.kind === 'value' || r.kind === 'read') && (r.conf ?? 1) < 0.5) continue;
+            const e = bound.get(r.dst_id) ?? bound.set(r.dst_id, { n: 0, score: 0, first: r.line }).get(r.dst_id);
+            e.n++; e.score += w * Math.max(0.3, r.conf ?? 1);
+        } else if (NAMED_KINDS.includes(r.kind)) {
+            const e = unbound.get(r.name) ?? unbound.set(r.name, { n: 0, score: 0, first: r.line }).get(r.name);
+            e.n++; e.score += w * 0.5;
+        }
+    }
+    const inView = (s) => s.path === file && s.start_line >= from && s.end_line <= to;
+    const rows = [];
+    const get = intel.store.q(`SELECT s.id, s.name, s.qname, s.kind, s.start_line, s.end_line, s.sig, s.doc, s.type, f.path, f.is_test
+        FROM symbols s JOIN files f ON f.id = s.file_id WHERE s.id = ?`);
+    for (const [id, e] of bound) {
+        if (skip.has(id)) continue;
+        const s = get.get(id);
+        if (!s || inView(s)) continue;
+        rows.push({ ...s, ...e, score: e.score * TARGET_WEIGHT(s.kind), byName: false });
+    }
+    const seen = new Set(rows.map(r => r.id));
+    for (const [name, e] of unbound) {
+        const cands = intel.store.all(`SELECT s.id FROM symbols s JOIN files f ON f.id = s.file_id
+            WHERE s.name = ? AND f.is_test = 0 AND s.kind IN (${DEFINITION_KINDS.map(() => '?').join(',')}) LIMIT 2`, name, ...DEFINITION_KINDS);
+        if (cands.length !== 1 || seen.has(cands[0].id) || skip.has(cands[0].id)) continue;
+        const s = get.get(cands[0].id);
+        if (!s || inView(s)) continue;
+        seen.add(s.id);
+        rows.push({ ...s, ...e, score: e.score * TARGET_WEIGHT(s.kind), byName: true });
+    }
+    rows.sort((a, b) => b.score - a.score || a.first - b.first);
+    return { rows: rows.slice(0, max), more: rows.slice(max) };
+}
+
+/** Test functions that reference a definition (directly), for "which tests exercise this". */
+export function testsUsing(intel, id, { max = 3 } = {}) {
+    return intel.store.all(`SELECT f.path, COALESCE(s.qname, '(module)') AS qname, MIN(r.line) AS line FROM refs r
+        JOIN files f ON f.id = r.file_id LEFT JOIN symbols s ON s.id = r.src_id
+        WHERE r.dst_id = ? AND f.is_test = 1 GROUP BY f.path, s.qname ORDER BY f.path, line LIMIT ?`, id, max + 1);
+}

package/src/query/testcmd.mjs

@@ -0,0 +1,130 @@
+/**
+ * How to run a set of test files in this repository: one command per test ecosystem, detected from
+ * the project's own configuration (package.json scripts and dependencies, pytest/unittest layout,
+ * go.mod, Cargo.toml, Maven/Gradle builds). Commands target only the given files, so an agent can
+ * verify a change in seconds instead of running the whole suite.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { readRepoFile } from '../util/paths.mjs';
+
+function readJson(root, rel) { try { return JSON.parse(readRepoFile(root, rel)); } catch { return null; } }
+const exists = (root, rel) => fs.existsSync(path.join(root, rel));
+// commands are pasted into a shell as they are: a file named `t$(cmd).test.ts` (repository content)
+// must reach the test runner as a name, never run as a command substitution
+const plain = (s) => /^[\w./@+=:,-]+$/.test(s);
+/** One shell word: as is when plain, else single-quoted. */
+const q = (s) => (plain(s) ? s : `'${s.replace(/'/g, `'\\''`)}'`);
+// a name that would read as an option, or a test identifier that is not a plain word, gets no command
+const shellSafe = (s) => !s.startsWith('-') && !/[\0\r\n]/.test(s);
+
+/** Nearest package.json above a file (monorepos have one per package). */
+function nearestPackage(root, rel) {
+    let dir = path.posix.dirname(rel);
+    for (;;) {
+        const p = path.posix.join(dir === '.' ? '' : dir, 'package.json');
+        if (exists(root, p)) return { dir: dir === '.' ? '' : dir, pkg: readJson(root, p) ?? {} };
+        if (dir === '.' || dir === '' || dir === '/') return null;
+        dir = path.posix.dirname(dir);
+    }
+}
+
+function jsRunner(root, rel) {
+    const near = nearestPackage(root, rel);
+    const top = readJson(root, 'package.json') ?? {};
+    const pkgs = [near?.pkg, top].filter(Boolean);
+    const deps = Object.assign({}, ...pkgs.map(p => ({ ...p.dependencies, ...p.devDependencies })));
+    const scripts = pkgs.map(p => p.scripts?.test ?? '').join(' ');
+    if (deps.vitest || /\bvitest\b/.test(scripts)) return 'npx vitest run';
+    if (deps.jest || /\bjest\b/.test(scripts)) return 'npx jest';
+    if (deps.mocha || /\bmocha\b/.test(scripts) || exists(root, '.mocharc.js') || exists(root, '.mocharc.json') || exists(root, '.mocharc.yml')) return 'npx mocha';
+    if (deps.ava || /\bava\b/.test(scripts)) return 'npx ava';
+    if (deps.tap || /\btap\b/.test(scripts)) return 'npx tap';
+    if (/node --test/.test(scripts)) return 'node --test';
+    return 'npm test --';
+}
+
+function pyRunner(root) {
+    if (exists(root, 'tests/runtests.py')) return { kind: 'django' };
+    const pyproject = readRepoFile(root, 'pyproject.toml') ?? '';
+    if (exists(root, 'pytest.ini') || exists(root, 'conftest.py') || /\[tool\.pytest/.test(pyproject) || exists(root, 'tox.ini')) return { kind: 'pytest' };
+    return { kind: 'unittest' };
+}
+
+/**
+ * @param {string} root repository root
+ * @param {string[]} files test files (repo-relative)
+ * @returns {string[]} shell commands
+ */
+export function testCommands(root, files) {
+    const groups = new Map();
+    const add = (key, f) => (groups.get(key) ?? groups.set(key, []).get(key)).push(f);
+    for (const f of files.filter(shellSafe)) {
+        const ext = path.posix.extname(f);
+        if (/\.[cm]?[jt]sx?$/.test(ext)) add(`js:${jsRunner(root, f)}`, f);
+        else if (ext === '.py') add('py', f);
+        else if (ext === '.go') add('go', f);
+        else if (ext === '.rs') add('rs', f);
+        else if (ext === '.java' || ext === '.kt' || ext === '.scala') add('jvm', f);
+        else if (ext === '.rb') add('rb', f);
+        else if (ext === '.php') add('php', f);
+        else if (ext === '.cs') add('cs', f);
+    }
+    const out = [];
+    for (const [key, fs_] of groups) {
+        if (key.startsWith('js:')) out.push(`${key.slice(3)} ${fs_.map(q).join(' ')}`);
+        else if (key === 'py') {
+            const r = pyRunner(root);
+            if (r.kind === 'django') out.push(`python tests/runtests.py ${[...new Set(fs_.map(f => f.replace(/^tests\//, '').replace(/\/[^/]*$/, '').replace(/\//g, '.')))].map(q).join(' ')}`);
+            else if (r.kind === 'pytest') out.push(`python -m pytest -q ${fs_.map(q).join(' ')}`);
+            else out.push(`python -m unittest ${fs_.map(f => q(f.replace(/\.py$/, '').replace(/\//g, '.'))).join(' ')}`);
+        } else if (key === 'go') out.push(`go test ${[...new Set(fs_.map(f => './' + path.posix.dirname(f)))].map(q).join(' ')}`);
+        else if (key === 'rs') out.push('cargo test');
+        else if (key === 'jvm') {
+            const classes = [...new Set(fs_.map(f => path.posix.basename(f).replace(/\.\w+$/, '')).filter(c => /^\w+$/.test(c)))].join(',');
+            if (!classes) continue;
+            out.push(exists(root, 'pom.xml') ? `mvn -q test -Dtest=${classes}` : `./gradlew test ${classes.split(',').map(c => `--tests '*${c}'`).join(' ')}`);
+        } else if (key === 'rb') out.push(exists(root, '.rspec') || fs_.some(f => f.endsWith('_spec.rb')) ? `bundle exec rspec ${fs_.map(q).join(' ')}` : `bundle exec ruby -Itest ${q(fs_[0])}`);
+        else if (key === 'php') out.push(`vendor/bin/phpunit ${fs_.map(q).join(' ')}`);
+        else if (key === 'cs') out.push('dotnet test');
+    }
+    return out;
+}
+
+/**
+ * Commands that run only the given test functions: [{ path, name, cls }] (cls: the enclosing test
+ * class, if any). pytest, unittest and Django (Python), `go test -run`, Maven and Gradle; other
+ * ecosystems name tests by strings the index does not see, so they get no command here.
+ * @returns {string[]} shell commands
+ */
+export function testCaseCommands(root, cases) {
+    const out = [];
+    const word = (s) => /^\w+$/.test(s);
+    cases = cases.filter(c => shellSafe(c.path) && plain(c.path) && word(c.name) && (!c.cls || word(c.cls)));
+    const py = cases.filter(c => c.path.endsWith('.py'));
+    if (py.length) {
+        const r = pyRunner(root);
+        const mod = (p) => p.replace(/\.py$/, '').replace(/\//g, '.');
+        if (r.kind === 'pytest') out.push(`python -m pytest -q ${py.map(c => `${c.path}::${c.cls ? `${c.cls}::` : ''}${c.name}`).join(' ')}`);
+        else if (r.kind === 'django') {
+            const withCls = py.filter(c => c.cls);
+            if (withCls.length) out.push(`python tests/runtests.py ${withCls.map(c => `${mod(c.path.replace(/^tests\//, ''))}.${c.cls}.${c.name}`).join(' ')}`);
+        } else {
+            const withCls = py.filter(c => c.cls);
+            if (withCls.length) out.push(`python -m unittest ${withCls.map(c => `${mod(c.path)}.${c.cls}.${c.name}`).join(' ')}`);
+        }
+    }
+    const byDir = new Map();
+    for (const c of cases.filter(c => c.path.endsWith('.go'))) (byDir.get(path.posix.dirname(c.path)) ?? byDir.set(path.posix.dirname(c.path), []).get(path.posix.dirname(c.path))).push(c.name);
+    for (const [dir, names] of byDir) out.push(`go test ./${dir} -run '^(${[...new Set(names)].join('|')})$'`);
+    const jvm = cases.filter(c => /\.(java|kt|scala)$/.test(c.path));
+    if (jvm.length) {
+        const cls = (c) => c.cls ?? path.posix.basename(c.path).replace(/\.\w+$/, '');
+        if (exists(root, 'pom.xml')) {
+            const by = new Map();
+            for (const c of jvm) (by.get(cls(c)) ?? by.set(cls(c), []).get(cls(c))).push(c.name);
+            out.push(`mvn -q test -Dtest=${[...by].map(([k, v]) => `${k}#${[...new Set(v)].join('+')}`).join(',')}`);
+        } else out.push(`./gradlew test ${jvm.map(c => `--tests '*${cls(c)}.${c.name}'`).join(' ')}`);
+    }
+    return out;
+}

package/src/query/textsearch.mjs

@@ -0,0 +1,178 @@
+/**
+ * Text search over every file in the repository (code, config, docs), with the structure grep
+ * cannot give: each code match is attributed to its enclosing definition, and when the pattern is
+ * an identifier each match says what it is — the definition itself, a reference bound to a
+ * specific symbol (so same-named methods of different classes are told apart), a reference the
+ * index could not bind, or text in a comment or string.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import vm from 'node:vm';
+import { withinRoot } from '../util/paths.mjs';
+import { gitSync } from '../util/git.mjs';
+
+const MAX_FILE_BYTES = 1_500_000;
+// a caller's pattern gets this much matching time per search (reading files does not count): one
+// with catastrophic backtracking (`(a+)+$`) is stopped instead of blocking the server
+const REGEX_BUDGET_MS = 5000;
+const SCAN = new vm.Script(`(() => {
+    if (!re.test(text)) return null;
+    const lines = text.split(/\\r?\\n/), hits = [];
+    for (let i = 0; i < lines.length; i++) if (re.test(lines[i])) hits.push(i);
+    return { lines, hits };
+})()`);
+const SKIP_DIRS = /(^|\/)(node_modules|\.git|\.graph-indexer|dist|build|out|target|coverage|__pycache__|\.venv|venv|\.next|\.nuxt|\.cache|vendor)\//;
+const EXAMPLE_PATH = /(^|\/)(docs?_src|docs?|examples?|samples?|demos?|tutorials?|benchmarks?|playground)\//i;
+const COMMENT_LINE = /^\s*(\/\/|#(?![!\[])|\*|\/\*|<!--|--\s|;;|%)/;
+
+function listFiles(root) {
+    const r = gitSync(root, ['ls-files', '-z', '--cached', '--others', '--exclude-standard'], { maxBuffer: 1 << 30 });
+    if (r.status === 0) return r.stdout.split('\0').filter(Boolean).filter(p => !SKIP_DIRS.test(p));
+    const out = [];
+    const stack = [''];
+    while (stack.length) {
+        const rel = stack.pop();
+        let entries;
+        try { entries = fs.readdirSync(path.join(root, rel), { withFileTypes: true }); } catch { continue; }
+        for (const e of entries) {
+            const p = rel ? `${rel}/${e.name}` : e.name;
+            if (e.isDirectory()) { if (!e.name.startsWith('.') && !SKIP_DIRS.test(p + '/')) stack.push(p); } else if (e.isFile()) out.push(p);
+        }
+    }
+    return out;
+}
+
+const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\/]/g, '\\$&');
+
+/** Regular expression source for a glob: `*` within a directory, `**` across directories, `?`, `[…]`, `{a,b}`. */
+function globSource(glob) {
+    let re = '';
+    for (let i = 0; i < glob.length; i++) {
+        const c = glob[i];
+        if (c === '*') {
+            if (glob[i + 1] !== '*') { re += '[^/]*'; continue; }
+            i++;
+            if (glob[i + 1] === '/') { i++; re += '(?:.*/)?'; } else re += '.*';
+        } else if (c === '?') re += '[^/]';
+        else if (c === '[' && glob.indexOf(']', i + 2) > i) { const j = glob.indexOf(']', i + 2); re += glob.slice(i, j + 1).replace(/^\[!/, '[^'); i = j; }
+        else if (c === '{' && glob.indexOf('}', i) > i) { const j = glob.indexOf('}', i); re += `(?:${glob.slice(i + 1, j).split(',').map(escapeRe).join('|')})`; i = j; }
+        else re += escapeRe(c);
+    }
+    return re;
+}
+
+/**
+ * Files whose path matches a glob (`*.toml`, `tests/test_*.py`, `**` for any depth; without a slash
+ * it is matched against file names) or, without wildcards, contains the text (case-insensitive).
+ */
+export function findFiles(root, pattern, { path: prefix = null } = {}) {
+    const pre = prefix ? String(prefix).replace(/^\.\//, '').replace(/\/+$/, '') : null;
+    let test;
+    if (/[*?[{]/.test(pattern)) {
+        const onName = !pattern.includes('/');
+        const re = new RegExp(onName ? `^${globSource(pattern)}$` : `(^|/)${globSource(pattern.replace(/^\.?\//, ''))}$`);
+        test = (p) => re.test(onName ? p.slice(p.lastIndexOf('/') + 1) : p);
+    } else {
+        const t = pattern.toLowerCase();
+        test = (p) => p.toLowerCase().includes(t);
+    }
+    return listFiles(root).filter(p => (!pre || p === pre || p.startsWith(pre + '/')) && test(p)).sort();
+}
+
+/** The identifier a pattern is about (`\bget\(` → get, `Reflector.get` → get), if any. */
+export function identifierOf(pattern, literal) {
+    const src = literal ? pattern : pattern.replace(/\\[bBsSdDwW]/g, ' ').replace(/\\(.)/g, '$1').replace(/[()[\]{}^$*+?|]/g, ' ');
+    const words = src.split(/[^A-Za-z0-9_$]+/).filter(w => /^[A-Za-z_$][\w$]*$/.test(w));
+    return words.length ? words[words.length - 1] : null;
+}
+
+function rank(p, isTest) {
+    if (isTest) return 2;
+    if (EXAMPLE_PATH.test(p)) return 3;
+    return 0;
+}
+
+/**
+ * @returns {{ error?: string, total: number, files: number, shown: object[], summary: object|null, truncated: boolean }}
+ */
+export async function textSearch(intel, { pattern, path: prefix = null, literal = false, ignoreCase = false, limit = 60 }) {
+    let re;
+    try {
+        const src = literal ? pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') : pattern;
+        re = new RegExp(src, ignoreCase ? 'i' : '');
+    } catch (e) {
+        return { error: `Invalid regular expression: ${e.message}. Pass literal: true to search for the text as is.` };
+    }
+    const root = intel.root;
+    const pre = prefix ? String(prefix).replace(/^\.\//, '').replace(/^\/+/, '') : null;
+    const files = listFiles(root).filter(p => !pre || p === pre || p.startsWith(pre.endsWith('/') ? pre : pre + '/') || p.startsWith(pre));
+    const indexed = new Map(intel.store.all('SELECT id, path, is_test FROM files').map(f => [f.path, f]));
+    const matches = [];
+    let total = 0, nfiles = 0;
+    const ctx = vm.createContext({ re, text: '' });
+    let budget = REGEX_BUDGET_MS;
+    for (const rel of files) {
+        let text;
+        try {
+            if (!withinRoot(root, rel)) continue;
+            const st = fs.statSync(path.join(root, rel));
+            if (!st.isFile() || st.size > MAX_FILE_BYTES) continue;
+            text = fs.readFileSync(path.join(root, rel), 'utf8');
+        } catch { continue; }
+        if (text.indexOf('\0') >= 0 && text.indexOf('\0') < 8000) continue; // binary
+        ctx.text = text;
+        let scan;
+        const t0 = performance.now();
+        try { scan = SCAN.runInContext(ctx, { timeout: Math.max(1, Math.ceil(budget)) }); } catch (e) {
+            if (e.code !== 'ERR_SCRIPT_EXECUTION_TIMEOUT') throw e;
+            return { error: `The regular expression took more than ${REGEX_BUDGET_MS / 1000} s (catastrophic backtracking?). Simplify it, or pass literal: true to search for the text as is.` };
+        }
+        budget -= performance.now() - t0;
+        if (!scan?.hits.length) continue;
+        nfiles++;
+        const f = indexed.get(rel);
+        for (const i of scan.hits) {
+            total++;
+            matches.push({ path: rel, line: i + 1, text: scan.lines[i], fileId: f?.id ?? null, isTest: !!f?.is_test, code: !!f });
+        }
+    }
+    const ident = identifierOf(pattern, literal);
+    // annotate: enclosing definition, and what an identifier match refers to
+    const byFile = new Map();
+    for (const m of matches) (byFile.get(m.path) ?? byFile.set(m.path, []).get(m.path)).push(m);
+    const summary = ident ? { ident, targets: new Map(), defs: 0, unbound: 0, text: 0, nonCode: 0 } : null;
+    for (const [rel, ms] of byFile) {
+        const f = indexed.get(rel);
+        if (!f) { for (const m of ms) { m.what = 'file'; if (summary) summary.nonCode++; } continue; }
+        const syms = intel.store.all('SELECT id, qname, kind, name, start_line, end_line, name_line FROM symbols WHERE file_id = ? ORDER BY start_line', f.id);
+        const refs = ident ? intel.store.all(`SELECT r.line, r.kind, r.dst_id, d.qname AS dst_qname, df.path AS dst_path FROM refs r LEFT JOIN symbols d ON d.id = r.dst_id LEFT JOIN files df ON df.id = d.file_id
+            WHERE r.file_id = ? AND r.name = ?`, f.id, ident) : [];
+        const refsAt = new Map();
+        for (const r of refs) (refsAt.get(r.line) ?? refsAt.set(r.line, []).get(r.line)).push(r);
+        for (const m of ms) {
+            // innermost definition around the line, ignoring ones that start on it (inline classes, the def itself)
+            let encl = null;
+            for (const s of syms) if (s.start_line < m.line && s.end_line >= m.line && (!encl || s.start_line >= encl.start_line)) encl = s;
+            m.encl = encl?.qname ?? null;
+            if (!summary) continue;
+            const def = syms.find(s => s.name === ident && s.name_line === m.line);
+            const rs = refsAt.get(m.line) ?? [];
+            if (def) { m.what = 'def'; m.target = def.qname; summary.defs++; continue; }
+            const bound = rs.filter(r => r.dst_id != null);
+            if (bound.length) {
+                m.what = 'ref';
+                m.target = [...new Set(bound.map(r => r.dst_qname))].join(', ');
+                for (const r of bound) {
+                    const k = `${r.dst_qname}\u0000${r.dst_path}`;
+                    summary.targets.set(k, (summary.targets.get(k) ?? 0) + 1);
+                }
+                continue;
+            }
+            if (rs.length) { m.what = 'unbound'; summary.unbound++; continue; }
+            m.what = COMMENT_LINE.test(m.text) ? 'comment' : 'text';
+            summary.text++;
+        }
+    }
+    matches.sort((a, b) => rank(a.path, a.isTest) + (a.code ? 0 : 1) - (rank(b.path, b.isTest) + (b.code ? 0 : 1)) || a.path.localeCompare(b.path) || a.line - b.line);
+    return { total, files: nfiles, shown: matches.slice(0, limit), all: matches, summary, truncated: matches.length > limit };
+}

package/src/search/search.mjs

@@ -0,0 +1,210 @@
+/**
+ * Hybrid symbol search.
+ *
+ *   1. name channel     exact / case-insensitive / qualified / prefix matches on symbol names
+ *   2. lexical channel  FTS5 BM25 over fields (name, qualified name, signature, doc, path, body)
+ *   3. concept channel  the same index queried with thesaurus expansions (natural-language queries)
+ *   4. rerank           a small linear model over interpretable features (name evidence, field
+ *                       coverage, kind/test/export priors, graph centrality), then per-file
+ *                       diversification so one file cannot flood the top results.
+ */
+import { analyzeQuery, codeTokens, splitIdentifier } from './tokenize.mjs';
+
+const FTS_WEIGHTS = [10.0, 6.0, 3.0, 2.0, 2.5, 1.0]; // name, qname, sig, doc, path, body
+const KIND_PRIOR = {
+    class: 1.0, interface: 0.95, struct: 1.0, trait: 0.95, enum: 0.85, type: 0.8, object: 0.95, module: 0.7, impl: 0.7,
+    function: 1.0, method: 1.0, constructor: 0.75, macro: 0.8, field: 0.55, property: 0.6, variable: 0.55, constant: 0.6,
+    selector: 0.35, keyframes: 0.3,
+};
+
+export const DEFAULT_WEIGHTS = {
+    // Tuned by coordinate ascent on the TUNING split of bench/suites (bench/tune-weights.mjs);
+    // the held-out split is reported, never used for selection.
+    bm25: 2.66,         // normalized lexical score
+    concept: 0.056,     // normalized concept-expansion score
+    exact: 1.6,         // query identifier == symbol name (case-sensitive)
+    exactCI: 0.8,       // case-insensitive (×0.75 for singular/plural)
+    qualified: 1.8,     // query `A.b` matches qualified name suffix
+    prefix: 0.27,       // symbol name starts with the query identifier
+    nameCover: 0.45,    // fraction of symbol-name words present in the query
+    queryCover: 0.8,    // fraction of query terms present in name/qname
+    kind: 0.35,         // kind prior
+    test: -1.6,         // test file (unless the query is about tests)
+    exported: 0.06,
+    doc: 0.033,
+    central: 0.25,      // graph centrality (PageRank percentile)
+    nested: -0.3,       // local/nested function inside another function
+    declaration: -1.4,  // .d.ts declarations (implementations are what agents want)
+    example: -1.2,      // examples/, samples/, sandbox/, docs/, benchmarks/ trees
+    nlField: -0.93,     // fields/properties/variables for natural-language (behavioural) queries
+    file: 1.0,          // file-level BM25 of the symbol's file (hierarchical: file relevance first)
+    fileNL: 0.5,        // extra file-level weight for natural-language (behavioural) queries
+};
+
+const EXAMPLE_PATH = /(^|\/)(examples?|samples?|demos?|sandbox|playground|docs?(_src|_source|_examples?)?|documentation|benchmarks?|bench|scripts?|fixtures?|testdata|tutorials?|website|\.github)\//i;
+const DECLARATION_PATH = /\.d\.[cm]?ts$/;
+const BM25_FLOOR = 1.0;
+const FILE_GATE = 0.25; // own normalized BM25 at which the file boost applies in full (tuning split)
+
+function stem(w) {
+    // light stemmer for coverage features (FTS5 does the real stemming for retrieval)
+    return w.replace(/(ies)$/, 'y').replace(/(sses|ches|shes|xes)$/, (m) => m.slice(0, -2)).replace(/([^s])s$/, '$1')
+        .replace(/(ing|ed|er|or|ion|ions|ation|ations)$/, '').replace(/e$/, '');
+}
+
+function ftsQuery(terms) {
+    const uniq = [...new Set(terms.filter(t => /^[a-z0-9]+$/i.test(t)))].slice(0, 32);
+    return uniq.length ? uniq.map(t => `"${t}"`).join(' OR ') : null;
+}
+
+export class SearchEngine {
+    constructor(indexer, { weights = DEFAULT_WEIGHTS, centrality = null, perFileCap = Number(process.env.GI_PER_FILE_CAP) || 3 } = {}) {
+        this.perFileCap = perFileCap;
+        this.ix = indexer;
+        this.store = indexer.store;
+        this.w = weights;
+        this.centrality = centrality; // () => Map(id -> percentile 0..1)
+    }
+
+    /**
+     * @param {string} query
+     * @param {{ limit?: number, path?: string, kinds?: string[], includeTests?: boolean, lang?: string }} opts
+     * @returns {{ results: Array<{ id:number, score:number, reasons:string[] }>, analysis: object }}
+     */
+    search(query, opts = {}) {
+        const limit = Math.max(1, Math.min(opts.limit ?? 10, 100));
+        const a = analyzeQuery(query);
+        const pathFilter = opts.path ?? a.filters.path?.[0] ?? null;
+        const kindFilter = opts.kinds ?? a.filters.kind ?? null;
+        const langFilter = opts.lang ?? a.filters.lang?.[0] ?? null;
+        const wantsTests = opts.includeTests ?? /\b(test|tests|spec|specs|mock|fixture)\b/i.test(query);
+        const cands = new Map(); // id -> feature bag
+        const bag = (id) => { let f = cands.get(id); if (!f) cands.set(id, (f = { id })); return f; };
+
+        // 1. name channel
+        const idents = new Set(a.identifiers);
+        for (const id of idents) {
+            const qualified = /[.#:]|->/.test(id);
+            if (qualified) {
+                const parts = id.split(/::|\.|#|->/).filter(Boolean);
+                const name = parts[parts.length - 1];
+                const suffix = parts.join('.');
+                for (const r of this.store.all('SELECT id, qname FROM symbols WHERE name = ? LIMIT 200', name)) {
+                    if (r.qname === suffix || r.qname.endsWith('.' + suffix)) bag(r.id).qualified = 1;
+                    else bag(r.id).exact = Math.max(bag(r.id).exact ?? 0, 0.5);
+                }
+                continue;
+            }
+            const lc = id.toLowerCase();
+            const variants = [lc, lc.endsWith('s') ? lc.slice(0, -1) : lc + 's'];
+            for (const r of this.store.all('SELECT id, name, name_lc FROM symbols WHERE name_lc IN (?, ?) LIMIT 300', variants[0], variants[1])) {
+                const f = bag(r.id);
+                if (r.name === id) f.exact = 1;
+                else if (r.name_lc === lc) f.exactCI = 1;
+                else f.exactCI = Math.max(f.exactCI ?? 0, 0.75); // singular/plural (BackgroundTask ↔ BackgroundTasks)
+            }
+            if (id.length >= 4) {
+                for (const r of this.store.all("SELECT id FROM symbols WHERE name_lc >= ? AND name_lc < ? LIMIT 60", id.toLowerCase(), id.toLowerCase() + '￿')) {
+                    const f = bag(r.id);
+                    if (!f.exact && !f.exactCI) f.prefix = 1;
+                }
+            }
+        }
+
+        // 2. lexical channel
+        const lexTerms = a.terms.length ? a.terms : codeTokens(query);
+        const q1 = ftsQuery(lexTerms);
+        let maxBm = 0;
+        if (q1) {
+            const rows = this.store.all(`SELECT rowid AS id, bm25(fts, ${FTS_WEIGHTS.join(', ')}) AS s FROM fts WHERE fts MATCH ? ORDER BY s LIMIT 400`, q1);
+            for (const r of rows) { const v = -r.s; bag(r.id).bm25 = v; if (v > maxBm) maxBm = v; }
+        }
+        // 3. concept channel
+        let maxCx = 0;
+        const q2 = a.expansions.length ? ftsQuery(a.expansions) : null;
+        if (q2) {
+            const rows = this.store.all(`SELECT rowid AS id, bm25(fts, ${FTS_WEIGHTS.join(', ')}) AS s FROM fts WHERE fts MATCH ? ORDER BY s LIMIT 150`, q2);
+            for (const r of rows) { const v = -r.s; bag(r.id).concept = v; if (v > maxCx) maxCx = v; }
+        }
+        // 4. file channel: files whose whole text matches the query best; their best-matching
+        //    symbols become candidates even when long-body length normalisation buried them
+        const fileScore = new Map();
+        if (q1 && (this.w.file || this.w.fileNL)) {
+            const frows = this.store.all('SELECT rowid AS fid, bm25(file_fts, 2.0, 1.0) AS s FROM file_fts WHERE file_fts MATCH ? ORDER BY s LIMIT 40', q1);
+            const maxF = frows.length ? -frows[0].s : 0;
+            for (const r of frows) fileScore.set(r.fid, -r.s / Math.max(maxF, BM25_FLOOR));
+            const top = frows.slice(0, 8).map(r => r.fid);
+            if (top.length) {
+                const rows = this.store.all(`SELECT rowid AS id, bm25(fts, ${FTS_WEIGHTS.join(', ')}) AS s FROM fts WHERE fts MATCH ?
+                    AND rowid IN (SELECT id FROM symbols WHERE file_id IN (${top.map(() => '?').join(',')})) ORDER BY s LIMIT 200`, q1, ...top);
+                for (const r of rows) { const f = bag(r.id); if (f.bm25 == null) { f.bm25 = -r.s; if (-r.s > maxBm) maxBm = -r.s; } }
+            }
+        }
+        if (!cands.size) return { results: [], analysis: a };
+
+        // 4. features + rerank
+        const ids = [...cands.keys()];
+        const rows = new Map();
+        for (let i = 0; i < ids.length; i += 500) {
+            const chunk = ids.slice(i, i + 500);
+            for (const r of this.store.all(`SELECT s.id, s.name, s.qname, s.kind, s.exported, s.doc IS NOT NULL AS has_doc, s.parent_id, s.file_id, f.path, f.lang, f.is_test,
+                (SELECT kind FROM symbols p WHERE p.id = s.parent_id) AS parent_kind
+                FROM symbols s JOIN files f ON f.id = s.file_id WHERE s.id IN (${chunk.map(() => '?').join(',')})`, ...chunk)) rows.set(r.id, r);
+        }
+        const qStems = new Set(lexTerms.map(stem));
+        const central = this.centrality?.() ?? null;
+        const w = this.w;
+        const scored = [];
+        for (const f of cands.values()) {
+            const r = rows.get(f.id);
+            if (!r) continue;
+            if (pathFilter && !r.path.startsWith(pathFilter.replace(/^\.\//, ''))) continue;
+            if (kindFilter && !kindFilter.includes(r.kind)) continue;
+            if (langFilter && r.lang !== langFilter) continue;
+            const nameWords = splitIdentifier(r.name).map(stem);
+            const qnameWords = new Set(splitIdentifier(r.qname.replace(/\./g, '_')).map(stem));
+            const nameCover = nameWords.length ? nameWords.filter(x => qStems.has(x)).length / nameWords.length : 0;
+            const queryCover = qStems.size ? [...qStems].filter(x => qnameWords.has(x)).length / qStems.size : 0;
+            let s = 0;
+            const reasons = [];
+            // floors keep near-zero-IDF matches (a word present in most documents) from being
+            // blown up by max-normalisation; when literal evidence is weak, the concept channel
+            // (thesaurus expansions) takes over — that is exactly the vocabulary-mismatch case.
+            if (f.bm25) s += w.bm25 * (f.bm25 / Math.max(maxBm, BM25_FLOOR));
+            if (f.concept) s += (maxBm < BM25_FLOOR ? Math.max(w.concept, 1.0) : w.concept) * (f.concept / Math.max(maxCx, BM25_FLOOR));
+            if (f.exact) { s += w.exact * f.exact; reasons.push('name'); }
+            if (f.exactCI) { s += w.exactCI * f.exactCI; reasons.push('name'); }
+            if (f.qualified) { s += w.qualified; reasons.push('qualified name'); }
+            if (f.prefix) s += w.prefix;
+            s += w.nameCover * nameCover + w.queryCover * queryCover;
+            s += w.kind * (KIND_PRIOR[r.kind] ?? 0.5);
+            if (r.is_test && !wantsTests) s += w.test;
+            if (r.exported) s += w.exported;
+            if (r.has_doc) s += w.doc;
+            if (central) s += w.central * (central.get(r.id) ?? 0);
+            // hierarchical evidence: a relevant file lifts its symbols in proportion to their own
+            // lexical match (P(file) × P(symbol | file)), never symbols that merely live there
+            const fs = fileScore.get(r.file_id);
+            if (fs && f.bm25 && r.kind !== 'selector' && r.kind !== 'keyframes') s += (w.file + (a.natural ? w.fileNL : 0)) * fs * Math.min(1, (f.bm25 / Math.max(maxBm, BM25_FLOOR)) / FILE_GATE);
+            if (r.parent_kind === 'function' || r.parent_kind === 'method') s += w.nested;
+            if (DECLARATION_PATH.test(r.path)) s += w.declaration;
+            if (EXAMPLE_PATH.test(r.path) && !(pathFilter && EXAMPLE_PATH.test(pathFilter))) s += w.example;
+            if (a.natural && (r.kind === 'field' || r.kind === 'property' || r.kind === 'variable' || r.kind === 'constant')) s += w.nlField;
+            scored.push({ id: r.id, score: s, reasons, path: r.path, ...(opts.debug ? { features: { ...f, nameCover, queryCover, kind: r.kind } } : {}) });
+        }
+        scored.sort((x, y) => y.score - x.score || x.id - y.id);
+        // per-file diversification: 3rd+ result from the same file is pushed down
+        const perFile = new Map();
+        const out = [];
+        const deferred = [];
+        for (const r of scored) {
+            const n = perFile.get(r.path) ?? 0;
+            if (n >= this.perFileCap) { deferred.push(r); continue; }
+            perFile.set(r.path, n + 1);
+            out.push(r);
+            if (out.length >= limit) break;
+        }
+        for (const r of deferred) { if (out.length >= limit) break; out.push(r); }
+        return { results: out, total: scored.length, analysis: a };
+    }
+}