graph-indexer 2.1.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +260 -1
- package/README.md +232 -312
- package/SECURITY.md +37 -59
- package/bin/graph-indexer.mjs +251 -0
- package/integrations/claude-code/.claude-plugin/plugin.json +8 -0
- package/integrations/claude-code/.mcp.json +8 -0
- package/integrations/claude-code/agents/code-structure.md +22 -0
- package/integrations/claude-code/hooks/gi-hook.mjs +99 -0
- package/integrations/claude-code/hooks/hooks.json +16 -0
- package/integrations/opencode/graph-indexer.js +124 -0
- package/package.json +32 -109
- package/src/cli/helper.mjs +67 -0
- package/src/cli/hook.mjs +356 -0
- package/src/cli/init.mjs +255 -0
- package/src/cli/resident.mjs +175 -0
- package/src/index/discover.mjs +130 -0
- package/src/index/fingerprint.mjs +30 -0
- package/src/index/graph.mjs +61 -0
- package/src/index/indexer.mjs +476 -0
- package/src/index/modules.mjs +331 -0
- package/src/index/resolver.mjs +954 -0
- package/src/mcp/render.mjs +60 -0
- package/src/mcp/server.mjs +147 -0
- package/src/mcp/tools.mjs +973 -0
- package/src/parse/arity.mjs +152 -0
- package/src/parse/extract.mjs +1031 -0
- package/src/parse/lang/csharp.mjs +106 -0
- package/src/parse/lang/ecma.mjs +438 -0
- package/src/parse/lang/go.mjs +194 -0
- package/src/parse/lang/jvm.mjs +118 -0
- package/src/parse/lang/kotlin.mjs +97 -0
- package/src/parse/lang/misc.mjs +115 -0
- package/src/parse/lang/native.mjs +126 -0
- package/src/parse/lang/python.mjs +153 -0
- package/src/parse/lang/rust.mjs +157 -0
- package/src/parse/lang/scripting.mjs +219 -0
- package/src/parse/languages.mjs +48 -0
- package/src/parse/runtime.mjs +56 -0
- package/src/query/check.mjs +343 -0
- package/src/query/facts.mjs +323 -0
- package/src/query/intel.mjs +680 -0
- package/src/query/read.mjs +81 -0
- package/src/query/testcmd.mjs +119 -0
- package/src/query/textsearch.mjs +160 -0
- package/src/search/search.mjs +210 -0
- package/src/search/tokenize.mjs +188 -0
- package/src/store/db.mjs +169 -0
- package/src/util/paths.mjs +24 -0
- package/vendor/grammars/bash.wasm.gz +0 -0
- package/vendor/grammars/c.wasm.gz +0 -0
- package/vendor/grammars/c_sharp.wasm.gz +0 -0
- package/vendor/grammars/cpp.wasm.gz +0 -0
- package/vendor/grammars/css.wasm.gz +0 -0
- package/vendor/grammars/go.wasm.gz +0 -0
- package/vendor/grammars/java.wasm.gz +0 -0
- package/vendor/grammars/javascript.wasm.gz +0 -0
- package/vendor/grammars/kotlin.wasm.gz +0 -0
- package/vendor/grammars/manifest.json +148 -0
- package/vendor/grammars/php.wasm.gz +0 -0
- package/vendor/grammars/python.wasm.gz +0 -0
- package/vendor/grammars/ruby.wasm.gz +0 -0
- package/vendor/grammars/rust.wasm.gz +0 -0
- package/vendor/grammars/scala.wasm.gz +0 -0
- package/vendor/grammars/tsx.wasm.gz +0 -0
- package/vendor/grammars/typescript.wasm.gz +0 -0
- package/vendor/web-tree-sitter/web-tree-sitter.js +3980 -0
- package/vendor/web-tree-sitter/web-tree-sitter.wasm +0 -0
- package/cli-ui.mjs +0 -78
- package/colbert.mjs +0 -168
- package/config.mjs +0 -494
- package/daemon-ctl.mjs +0 -175
- package/daemon-lock.mjs +0 -69
- package/embedders/python/mlx_embed_server.py +0 -61
- package/embedders/python/requirements-mlx.txt +0 -5
- package/embedders/setup-mlx.mjs +0 -110
- package/embeddings.mjs +0 -571
- package/engine/binary.mjs +0 -437
- package/engine/memory.mjs +0 -1003
- package/engine/sqlite.mjs +0 -1136
- package/enrichment.mjs +0 -521
- package/git-signals.mjs +0 -248
- package/indexer.mjs +0 -546
- package/init.mjs +0 -1809
- package/layout.mjs +0 -123
- package/mcp/centrality.mjs +0 -117
- package/mcp/format.mjs +0 -61
- package/mcp/resolver.mjs +0 -116
- package/mcp/symbolgraph.mjs +0 -137
- package/mcp/taint.mjs +0 -225
- package/mcp/tools.mjs +0 -1582
- package/mcp/topology.mjs +0 -676
- package/mcp-server.mjs +0 -197
- package/parse/extractor.mjs +0 -628
- package/parse/imports.mjs +0 -365
- package/parse/interprocedural.mjs +0 -134
- package/parse/languages.mjs +0 -253
- package/parse/metadata.mjs +0 -701
- package/parse/routes.mjs +0 -327
- package/parse/scip.mjs +0 -391
- package/parse/taint-patterns.mjs +0 -119
- package/prompts/CORE.md +0 -91
- package/prompts/DOMAIN_TEMPLATE.md +0 -46
- package/prompts/INTEGRATION.md +0 -123
- package/prompts/frameworks/ANDROID.md +0 -17
- package/prompts/frameworks/ASPNET_CORE.md +0 -19
- package/prompts/frameworks/FASTAPI_DJANGO.md +0 -20
- package/prompts/frameworks/LARAVEL_SYMFONY.md +0 -22
- package/prompts/frameworks/NODE_EXPRESS_NESTJS.md +0 -19
- package/prompts/frameworks/RAILS.md +0 -20
- package/prompts/frameworks/REACT.md +0 -20
- package/prompts/frameworks/SPRING_BOOT.md +0 -20
- package/prompts/languages/BASH.md +0 -22
- package/prompts/languages/C.md +0 -21
- package/prompts/languages/CSHARP.md +0 -24
- package/prompts/languages/CSS_SCSS.md +0 -22
- package/prompts/languages/GO.md +0 -24
- package/prompts/languages/JAVA.md +0 -23
- package/prompts/languages/JAVASCRIPT_TYPESCRIPT.md +0 -24
- package/prompts/languages/KOTLIN.md +0 -24
- package/prompts/languages/PHP.md +0 -23
- package/prompts/languages/PYTHON.md +0 -23
- package/prompts/languages/RUBY.md +0 -23
- package/prompts/languages/RUST.md +0 -21
- package/prompts/languages/SWIFT.md +0 -21
- package/seal.mjs +0 -400
- package/search-core.mjs +0 -777
- package/search-sparse.mjs +0 -183
- package/storage.mjs +0 -73
- package/watch-daemon.mjs +0 -289
|
@@ -0,0 +1,476 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Indexer: keeps the SQLite store in sync with the working tree.
|
|
3
|
+
*
|
|
4
|
+
* sync() full reconciliation (discover → stat → hash → parse changed → resolve)
|
|
5
|
+
* syncPaths(ps) targeted refresh for a handful of files (watcher events, query-time checks)
|
|
6
|
+
*
|
|
7
|
+
* Change detection follows git's model: (size, mtime) is the cheap signature, the content hash
|
|
8
|
+
* decides. Symbol ids are stable across re-indexing (matched by qualified name + kind + ordinal),
|
|
9
|
+
* so references from untouched files stay valid; only references that pointed at deleted symbols,
|
|
10
|
+
* or that could now bind to newly added names, are re-resolved.
|
|
11
|
+
*/
|
|
12
|
+
import fs from 'node:fs';
|
|
13
|
+
import path from 'node:path';
|
|
14
|
+
import crypto from 'node:crypto';
|
|
15
|
+
import { discoverFiles, isIndexablePath, insideRoot, looksMinified, MAX_FILE_BYTES } from './discover.mjs';
|
|
16
|
+
import { ModuleResolver } from './modules.mjs';
|
|
17
|
+
import { SymbolTable, Resolver } from './resolver.mjs';
|
|
18
|
+
import { extractFile } from '../parse/extract.mjs';
|
|
19
|
+
import { specForPath, LANGUAGES } from '../parse/languages.mjs';
|
|
20
|
+
import { tokenString, codeTokens } from '../search/tokenize.mjs';
|
|
21
|
+
import { indexFingerprint } from './fingerprint.mjs';
|
|
22
|
+
|
|
23
|
+
const SPECS = Object.fromEntries(LANGUAGES.map(l => [l.id, l]));
|
|
24
|
+
const TYPE_KINDS = new Set(['class', 'interface', 'struct', 'enum', 'trait', 'type', 'object', 'module', 'impl']);
|
|
25
|
+
const BODY_TOKEN_CAP = 1200;
|
|
26
|
+
const FILE_TOKEN_CAP = 30000;
|
|
27
|
+
|
|
28
|
+
function sha1(s) { return crypto.createHash('sha1').update(s).digest('hex'); }
|
|
29
|
+
|
|
30
|
+
function lineStarts(src) {
|
|
31
|
+
const starts = [0];
|
|
32
|
+
for (let i = 0; i < src.length; i++) if (src.charCodeAt(i) === 10) starts.push(i + 1);
|
|
33
|
+
return starts;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export class Indexer {
|
|
37
|
+
constructor({ root, store, log = () => {}, include = null }) {
|
|
38
|
+
this.root = path.resolve(root);
|
|
39
|
+
this.store = store;
|
|
40
|
+
this.log = log;
|
|
41
|
+
this.include = include;
|
|
42
|
+
this.table = new SymbolTable();
|
|
43
|
+
this.resolver = new Resolver(this.table, SPECS, { sigOf: (id) => this.store.get('SELECT sig FROM symbols WHERE id = ?', id)?.sig ?? null });
|
|
44
|
+
this.modules = null;
|
|
45
|
+
this.loaded = false;
|
|
46
|
+
this.resolveAll = false;
|
|
47
|
+
this.version = 0; // bumps on every change; caches key off it
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// ── bootstrap from the store ─────────────────────────────────────────────────
|
|
51
|
+
load() {
|
|
52
|
+
const s = this.store;
|
|
53
|
+
const fp = indexFingerprint();
|
|
54
|
+
if (s.getMeta('fingerprint') !== fp) {
|
|
55
|
+
if (s.get('SELECT 1 AS x FROM files LIMIT 1')) { this.log('index was written by another graph-indexer version: rebuilding'); s.reset(); }
|
|
56
|
+
s.setMeta('fingerprint', fp);
|
|
57
|
+
}
|
|
58
|
+
// an interrupted run may have written files whose references were never resolved
|
|
59
|
+
this.resolveAll = s.getMeta('resolve_pending') === '1';
|
|
60
|
+
const files = s.all('SELECT id, path, lang, package, is_test FROM files');
|
|
61
|
+
for (const f of files) this.table.addFile({ id: f.id, path: f.path, lang: f.lang, package: f.package, isTest: !!f.is_test });
|
|
62
|
+
for (const r of s.all('SELECT id, file_id, name, qname, kind, parent_id, owner, type, bases, exported, is_static, start_line, end_line FROM symbols')) {
|
|
63
|
+
this.table.addSym(rowToSym(r));
|
|
64
|
+
}
|
|
65
|
+
// the interfaces a Go type satisfies implicitly are stored only as `~structural` edges
|
|
66
|
+
for (const r of s.all("SELECT DISTINCT src_id, name FROM refs WHERE kind = 'inherit' AND recv_type = '~structural'")) {
|
|
67
|
+
const ty = this.table.sym(r.src_id);
|
|
68
|
+
if (!ty) continue;
|
|
69
|
+
ty.declaredBases ??= [...ty.bases];
|
|
70
|
+
ty.structural ??= new Set();
|
|
71
|
+
if (!ty.bases.includes(r.name)) { ty.bases.push(r.name); ty.structural.add(r.name); }
|
|
72
|
+
}
|
|
73
|
+
const imps = new Map();
|
|
74
|
+
for (const r of s.all('SELECT file_id, source, imported, local, reexport, wildcard, target_file_id, target_dir, target_path FROM imports')) {
|
|
75
|
+
const list = imps.get(r.file_id) ?? imps.set(r.file_id, []).get(r.file_id);
|
|
76
|
+
list.push(rowToImport(r, r.target_dir, r.target_path));
|
|
77
|
+
}
|
|
78
|
+
for (const [fid, list] of imps) this.table.setImports(fid, list);
|
|
79
|
+
const fields = new Map();
|
|
80
|
+
for (const r of s.all('SELECT file_id, owner_qname, name, type FROM fields')) (fields.get(r.file_id) ?? fields.set(r.file_id, []).get(r.file_id)).push(r);
|
|
81
|
+
for (const [fid, rows] of fields) this.table.setFieldTypes(fid, rows);
|
|
82
|
+
this.modules = new ModuleResolver(this.root, files.map(f => f.path));
|
|
83
|
+
this.loaded = true;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// ── public API ──────────────────────────────────────────────────────────────
|
|
87
|
+
/** Full reconciliation with the working tree. */
|
|
88
|
+
async sync({ onProgress = null } = {}) {
|
|
89
|
+
if (!this.loaded) this.load();
|
|
90
|
+
const t0 = Date.now();
|
|
91
|
+
const { files } = discoverFiles(this.root, { include: this.include });
|
|
92
|
+
const known = new Map(this.store.all('SELECT id, path, size, mtime_ms, hash FROM files').map(r => [r.path, r]));
|
|
93
|
+
const present = new Set(files);
|
|
94
|
+
const removed = [...known.keys()].filter(p => !present.has(p));
|
|
95
|
+
this.modules.setFiles(files);
|
|
96
|
+
const toParse = [];
|
|
97
|
+
for (const rel of files) {
|
|
98
|
+
const st = safeStat(path.join(this.root, rel));
|
|
99
|
+
if (!st || st.size > MAX_FILE_BYTES) { if (known.has(rel)) removed.push(rel); continue; }
|
|
100
|
+
const k = known.get(rel);
|
|
101
|
+
if (k && k.size === st.size && k.mtime_ms === Math.floor(st.mtimeMs)) continue;
|
|
102
|
+
toParse.push({ rel, st, prev: k ?? null });
|
|
103
|
+
}
|
|
104
|
+
const res = await this.#apply(toParse, removed, { onProgress });
|
|
105
|
+
this.store.setMeta('last_sync', String(Date.now()));
|
|
106
|
+
this.store.setMeta('root', this.root);
|
|
107
|
+
return { ...res, total: files.length, ms: Date.now() - t0 };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** Re-check specific paths (created/modified/deleted). Cheap; used before answering queries. */
|
|
111
|
+
async syncPaths(paths) {
|
|
112
|
+
if (!this.loaded) this.load();
|
|
113
|
+
const toParse = [];
|
|
114
|
+
const removed = [];
|
|
115
|
+
for (const p of new Set(paths)) {
|
|
116
|
+
const rel = path.isAbsolute(p) ? path.relative(this.root, p).split(path.sep).join('/') : p;
|
|
117
|
+
if (rel.startsWith('..') || !isIndexablePath(rel, { include: this.include })) continue;
|
|
118
|
+
const st = safeStat(path.join(this.root, rel));
|
|
119
|
+
const k = this.store.get('SELECT id, path, size, mtime_ms, hash FROM files WHERE path = ?', rel);
|
|
120
|
+
if (!st || !st.isFile() || st.size > MAX_FILE_BYTES || !insideRoot(this.root, rel)) { if (k) removed.push(rel); continue; }
|
|
121
|
+
if (k && k.size === st.size && k.mtime_ms === Math.floor(st.mtimeMs)) continue;
|
|
122
|
+
toParse.push({ rel, st, prev: k ?? null });
|
|
123
|
+
this.modules.addFile(rel);
|
|
124
|
+
}
|
|
125
|
+
for (const r of removed) this.modules.removeFile(r);
|
|
126
|
+
if (!toParse.length && !removed.length) return { changed: 0, removed: 0 };
|
|
127
|
+
return this.#apply(toParse, removed, {});
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Which of these indexed paths differ from disk right now (size/mtime)? */
|
|
131
|
+
staleAmong(paths) {
|
|
132
|
+
const out = [];
|
|
133
|
+
for (const rel of new Set(paths)) {
|
|
134
|
+
const k = this.store.get('SELECT size, mtime_ms FROM files WHERE path = ?', rel);
|
|
135
|
+
const st = safeStat(path.join(this.root, rel));
|
|
136
|
+
if (!k || !st || k.size !== st.size || k.mtime_ms !== Math.floor(st.mtimeMs)) out.push(rel);
|
|
137
|
+
}
|
|
138
|
+
return out;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
// ── internals ────────────────────────────────────────────────────────────────
|
|
142
|
+
async #apply(toParse, removed, { onProgress }) {
|
|
143
|
+
const s = this.store;
|
|
144
|
+
let changed = 0, unchangedContent = 0, added = 0;
|
|
145
|
+
const touchedFileIds = [];
|
|
146
|
+
const removedSymIds = [];
|
|
147
|
+
const addedNames = new Set();
|
|
148
|
+
|
|
149
|
+
// remove deleted files
|
|
150
|
+
if (removed.length) {
|
|
151
|
+
s.tx(() => {
|
|
152
|
+
for (const rel of removed) {
|
|
153
|
+
const f = s.get('SELECT id FROM files WHERE path = ?', rel);
|
|
154
|
+
if (!f) continue;
|
|
155
|
+
for (const r of s.all('SELECT id FROM symbols WHERE file_id = ?', f.id)) { removedSymIds.push(r.id); s.run('DELETE FROM fts WHERE rowid = ?', r.id); }
|
|
156
|
+
s.run('DELETE FROM file_fts WHERE rowid = ?', f.id);
|
|
157
|
+
for (const t of ['symbols', 'refs', 'imports', 'fields']) s.run(`DELETE FROM ${t} WHERE file_id = ?`, f.id);
|
|
158
|
+
s.run('DELETE FROM files WHERE id = ?', f.id);
|
|
159
|
+
this.table.removeFile(f.id);
|
|
160
|
+
this.modules?.removeFile(rel);
|
|
161
|
+
}
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// parse changed/new files (async extraction), write in batches
|
|
166
|
+
if (toParse.length || removed.length) s.setMeta('resolve_pending', '1');
|
|
167
|
+
const BATCH = 200;
|
|
168
|
+
let done = 0;
|
|
169
|
+
for (let i = 0; i < toParse.length; i += BATCH) {
|
|
170
|
+
const batch = toParse.slice(i, i + BATCH);
|
|
171
|
+
const parsed = [];
|
|
172
|
+
for (const item of batch) {
|
|
173
|
+
const abs = path.join(this.root, item.rel);
|
|
174
|
+
let source;
|
|
175
|
+
try { source = fs.readFileSync(abs, 'utf8'); } catch { continue; }
|
|
176
|
+
const hash = sha1(source);
|
|
177
|
+
if (item.prev && item.prev.hash === hash) {
|
|
178
|
+
s.run('UPDATE files SET size = ?, mtime_ms = ? WHERE id = ?', item.st.size, Math.floor(item.st.mtimeMs), item.prev.id);
|
|
179
|
+
unchangedContent++;
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
const spec = specForPath(item.rel);
|
|
183
|
+
if (!spec) continue;
|
|
184
|
+
let extraction;
|
|
185
|
+
if (looksMinified(source)) extraction = { symbols: [], refs: [], imports: [], fields: [], fileInfo: null, errors: 0 };
|
|
186
|
+
else {
|
|
187
|
+
try { extraction = await extractFile(spec, source, item.rel); }
|
|
188
|
+
catch (e) { this.log(`parse failed for ${item.rel}: ${e.message}`); extraction = { symbols: [], refs: [], imports: [], fields: [], fileInfo: null, errors: 1 }; }
|
|
189
|
+
}
|
|
190
|
+
parsed.push({ ...item, source, hash, spec, extraction });
|
|
191
|
+
}
|
|
192
|
+
s.tx(() => {
|
|
193
|
+
for (const p of parsed) {
|
|
194
|
+
const r = this.#writeFile(p);
|
|
195
|
+
touchedFileIds.push(r.fileId);
|
|
196
|
+
removedSymIds.push(...r.removedIds);
|
|
197
|
+
for (const n of r.addedNames) addedNames.add(n);
|
|
198
|
+
if (p.prev) changed++; else added++;
|
|
199
|
+
}
|
|
200
|
+
});
|
|
201
|
+
done += batch.length;
|
|
202
|
+
onProgress?.(done, toParse.length);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// resolve: refs of touched files + refs that pointed at removed symbols + refs that may bind to new names
|
|
206
|
+
this.resolver.resetMemo();
|
|
207
|
+
const refIds = new Set();
|
|
208
|
+
if (touchedFileIds.length) {
|
|
209
|
+
for (const fid of touchedFileIds) for (const r of s.all('SELECT id FROM refs WHERE file_id = ?', fid)) refIds.add(r.id);
|
|
210
|
+
}
|
|
211
|
+
const dangling = [];
|
|
212
|
+
if (removedSymIds.length) {
|
|
213
|
+
for (let i = 0; i < removedSymIds.length; i += 500) {
|
|
214
|
+
const chunk = removedSymIds.slice(i, i + 500);
|
|
215
|
+
for (const r of s.all(`SELECT id FROM refs WHERE dst_id IN (${chunk.map(() => '?').join(',')})`, ...chunk)) { refIds.add(r.id); dangling.push(r.id); }
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
const isIncremental = touchedFileIds.length < this.table.files.size;
|
|
219
|
+
if (addedNames.size && isIncremental) {
|
|
220
|
+
const names = [...addedNames];
|
|
221
|
+
for (let i = 0; i < names.length; i += 500) {
|
|
222
|
+
const chunk = names.slice(i, i + 500);
|
|
223
|
+
for (const r of s.all(`SELECT id FROM refs WHERE name IN (${chunk.map(() => '?').join(',')}) AND (dst_id IS NULL OR conf < 0.9)`, ...chunk)) refIds.add(r.id);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
const goTouched = this.resolveAll || removed.some(r => r.endsWith('.go')) || toParse.some(p => p.rel.endsWith('.go'));
|
|
227
|
+
if (this.resolveAll) for (const r of s.all('SELECT id FROM refs')) refIds.add(r.id);
|
|
228
|
+
const resolved = this.#resolveRefs([...refIds]);
|
|
229
|
+
if (goTouched) this.#goImplements();
|
|
230
|
+
this.resolveAll = false;
|
|
231
|
+
if (toParse.length || removed.length) s.setMeta('resolve_pending', '0');
|
|
232
|
+
if (touchedFileIds.length || removed.length) this.version++;
|
|
233
|
+
return { added, changed, removed: removed.length, unchangedContent, resolved };
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
#writeFile({ rel, st, prev, source, hash, spec, extraction }) {
|
|
237
|
+
const s = this.store;
|
|
238
|
+
const { symbols, refs, imports, fields, fileInfo, errors } = extraction;
|
|
239
|
+
const lines = lineStarts(source);
|
|
240
|
+
const isTest = spec.testFile?.test(rel) ? 1 : 0;
|
|
241
|
+
const pkg = fileInfo?.package ?? null;
|
|
242
|
+
let fileId;
|
|
243
|
+
let oldSyms = [];
|
|
244
|
+
if (prev) {
|
|
245
|
+
fileId = prev.id;
|
|
246
|
+
oldSyms = s.all('SELECT id, qname, kind, ordinal FROM symbols WHERE file_id = ?', fileId);
|
|
247
|
+
for (const o of oldSyms) s.run('DELETE FROM fts WHERE rowid = ?', o.id);
|
|
248
|
+
s.run('DELETE FROM file_fts WHERE rowid = ?', fileId);
|
|
249
|
+
for (const t of ['symbols', 'refs', 'imports', 'fields']) s.run(`DELETE FROM ${t} WHERE file_id = ?`, fileId);
|
|
250
|
+
s.run('UPDATE files SET lang = ?, size = ?, mtime_ms = ?, hash = ?, lines = ?, package = ?, is_test = ?, parse_errors = ?, indexed_at = ? WHERE id = ?',
|
|
251
|
+
spec.id, st.size, Math.floor(st.mtimeMs), hash, lines.length, pkg, isTest, errors, Date.now(), fileId);
|
|
252
|
+
this.table.removeFile(fileId);
|
|
253
|
+
} else {
|
|
254
|
+
const r = s.run('INSERT INTO files (path, lang, size, mtime_ms, hash, lines, package, is_test, parse_errors, indexed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
|
|
255
|
+
rel, spec.id, st.size, Math.floor(st.mtimeMs), hash, lines.length, pkg, isTest, errors, Date.now());
|
|
256
|
+
fileId = Number(r.lastInsertRowid);
|
|
257
|
+
}
|
|
258
|
+
this.table.addFile({ id: fileId, path: rel, lang: spec.id, package: pkg, isTest: !!isTest });
|
|
259
|
+
|
|
260
|
+
// stable ids: reuse by (qname, kind, ordinal)
|
|
261
|
+
const oldByKey = new Map(oldSyms.map(o => [o.qname + '\0' + o.kind + '\0' + o.ordinal, o.id]));
|
|
262
|
+
const ordinals = new Map();
|
|
263
|
+
const ids = new Array(symbols.length);
|
|
264
|
+
const reused = new Set();
|
|
265
|
+
const addedNames = [];
|
|
266
|
+
const pathTokens = tokenString(rel.replace(/\.[^.]+$/, ''));
|
|
267
|
+
for (let i = 0; i < symbols.length; i++) {
|
|
268
|
+
const sym = symbols[i];
|
|
269
|
+
const k0 = sym.qname + '\0' + sym.kind;
|
|
270
|
+
const ord = ordinals.get(k0) ?? 0;
|
|
271
|
+
ordinals.set(k0, ord + 1);
|
|
272
|
+
const key = k0 + '\0' + ord;
|
|
273
|
+
const reuse = oldByKey.get(key);
|
|
274
|
+
const parentId = sym.parentIdx >= 0 ? ids[sym.parentIdx] : null;
|
|
275
|
+
const decorators = sym.decorators?.length ? JSON.stringify(sym.decorators) : null;
|
|
276
|
+
const bases = sym.bases?.length ? JSON.stringify(sym.bases) : null;
|
|
277
|
+
const r = s.run(`INSERT INTO symbols (id, file_id, name, name_lc, qname, kind, parent_id, owner, start_line, start_col, end_line, end_col, name_line, name_col, sig, doc, type, exported, visibility, decorators, bases, ordinal, is_static)
|
|
278
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
279
|
+
reuse ?? null, fileId, sym.name, sym.name.toLowerCase(), sym.qname, sym.kind, parentId, sym.owner, sym.startLine, sym.startCol, sym.endLine, sym.endCol,
|
|
280
|
+
sym.nameLine, sym.nameCol, sym.sig || null, sym.doc || null, sym.type, sym.exported ? 1 : 0, sym.visibility, decorators, bases, ord, sym.isStatic ? 1 : 0);
|
|
281
|
+
const id = reuse ?? Number(r.lastInsertRowid);
|
|
282
|
+
ids[i] = id;
|
|
283
|
+
if (reuse != null) reused.add(reuse); else addedNames.push(sym.name);
|
|
284
|
+
this.table.addSym({ id, fileId, name: sym.name, qname: sym.qname, kind: sym.kind, parentId, owner: sym.owner, type: sym.type, bases: sym.bases ?? [], exported: !!sym.exported, isStatic: !!sym.isStatic, startLine: sym.startLine, endLine: sym.endLine });
|
|
285
|
+
}
|
|
286
|
+
// FTS documents
|
|
287
|
+
for (let i = 0; i < symbols.length; i++) {
|
|
288
|
+
const sym = symbols[i];
|
|
289
|
+
let body;
|
|
290
|
+
if (TYPE_KINDS.has(sym.kind)) {
|
|
291
|
+
const kids = [];
|
|
292
|
+
for (let j = i + 1; j < symbols.length && kids.length < 200; j++) if (symbols[j].parentIdx === i) kids.push(symbols[j].name, symbols[j].sig ?? '');
|
|
293
|
+
body = tokenString(kids.join(' '), { maxTokens: BODY_TOKEN_CAP });
|
|
294
|
+
} else if (sym.kind === 'macro' || sym.kind === 'selector' || sym.kind === 'variable' && sym.endLine - sym.startLine > 30) {
|
|
295
|
+
// macro_rules bodies, CSS blocks and big literal tables are token soup: index their head only
|
|
296
|
+
const a = lines[sym.startLine - 1] ?? 0;
|
|
297
|
+
const b = lines[Math.min(sym.startLine + 2, lines.length - 1)] ?? source.length;
|
|
298
|
+
body = codeTokens(source.slice(a, b), { maxTokens: 60 }).join(' ');
|
|
299
|
+
} else {
|
|
300
|
+
const a = lines[sym.startLine - 1] ?? 0;
|
|
301
|
+
const b = lines[Math.min(sym.endLine, lines.length - 1)] ?? source.length;
|
|
302
|
+
body = codeTokens(source.slice(a, sym.endLine >= lines.length ? source.length : b), { maxTokens: BODY_TOKEN_CAP }).join(' ');
|
|
303
|
+
}
|
|
304
|
+
s.run('INSERT INTO fts (rowid, name, qname, sig, doc, path, body) VALUES (?, ?, ?, ?, ?, ?, ?)',
|
|
305
|
+
ids[i], tokenString(sym.name), tokenString(sym.qname), tokenString(sym.sig ?? ''), tokenString(sym.doc ?? ''), pathTokens, body);
|
|
306
|
+
}
|
|
307
|
+
// file document (hierarchical ranking: which files talk about the query at all)
|
|
308
|
+
s.run('INSERT INTO file_fts (rowid, path, body) VALUES (?, ?, ?)', fileId, pathTokens, codeTokens(source, { maxTokens: FILE_TOKEN_CAP }).join(' '));
|
|
309
|
+
// refs (unresolved for now)
|
|
310
|
+
for (const r of refs) {
|
|
311
|
+
s.run('INSERT INTO refs (file_id, src_id, name, kind, line, col, recv, recv_type, argc) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
|
|
312
|
+
fileId, r.symIdx >= 0 ? ids[r.symIdx] : null, r.name, r.kind, r.line, r.col, r.recv || null, r.recvType || null, r.argc ?? null);
|
|
313
|
+
}
|
|
314
|
+
// imports (module resolution needs only the file list)
|
|
315
|
+
const impRows = [];
|
|
316
|
+
for (const imp of imports) {
|
|
317
|
+
const target = this.modules.resolve(spec.id, rel, imp.source);
|
|
318
|
+
let targetFileId = null;
|
|
319
|
+
if (target?.file) targetFileId = s.get('SELECT id FROM files WHERE path = ?', target.file)?.id ?? null;
|
|
320
|
+
const row = { source: imp.source ?? null, imported: imp.imported ?? null, local: imp.local ?? null, reexport: imp.reexport ?? null, wildcard: imp.wildcard ? 1 : 0, line: imp.line ?? null, target_file_id: targetFileId, target_dir: target?.dir ?? null };
|
|
321
|
+
s.run('INSERT INTO imports (file_id, source, imported, local, reexport, wildcard, line, target_file_id, target_dir, target_path) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
|
|
322
|
+
fileId, row.source, row.imported, row.local, row.reexport, row.wildcard, row.line, row.target_file_id, row.target_dir, target?.file ?? null);
|
|
323
|
+
impRows.push(rowToImport(row, target?.dir ?? null, target?.file ?? null));
|
|
324
|
+
}
|
|
325
|
+
this.table.setImports(fileId, impRows);
|
|
326
|
+
for (const f of fields ?? []) s.run('INSERT INTO fields (file_id, owner_qname, name, type) VALUES (?, ?, ?, ?)', fileId, f.owner, f.name, f.type);
|
|
327
|
+
this.table.setFieldTypes(fileId, (fields ?? []).map(f => ({ owner_qname: f.owner, name: f.name, type: f.type })));
|
|
328
|
+
const removedIds = oldSyms.filter(o => !reused.has(o.id)).map(o => o.id);
|
|
329
|
+
return { fileId, removedIds, addedNames };
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/** Import targets whose file was not indexed yet at write time are fixed up lazily here. */
|
|
333
|
+
#fixImportTargets() {
|
|
334
|
+
const s = this.store;
|
|
335
|
+
for (const [fid, imps] of this.table.imports) {
|
|
336
|
+
const f = this.table.file(fid);
|
|
337
|
+
if (!f) continue;
|
|
338
|
+
let changed = false;
|
|
339
|
+
for (const imp of imps) {
|
|
340
|
+
// a package import can have both a directory and an entry file (Python `pkg/__init__.py`)
|
|
341
|
+
if (imp.targetFileId != null || !imp.targetPath) continue;
|
|
342
|
+
const t = this.table.fileByPath.get(imp.targetPath);
|
|
343
|
+
if (t) { imp.targetFileId = t.id; changed = true; }
|
|
344
|
+
}
|
|
345
|
+
if (changed) {
|
|
346
|
+
for (const imp of imps) if (imp.targetFileId != null && imp.targetPath) s.run('UPDATE imports SET target_file_id = ? WHERE file_id = ? AND target_path = ? AND target_file_id IS NULL', imp.targetFileId, fid, imp.targetPath);
|
|
347
|
+
this.table.setImports(fid, imps);
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* Go interfaces are satisfied implicitly: a named type implements an interface when its method
|
|
354
|
+
* set (methods declared on the type or its pointer, in its package) contains every method the
|
|
355
|
+
* interface declares, embedded interfaces included. Recorded as `inherit` edges marked
|
|
356
|
+
* `~structural`, recomputed whenever a Go file changes, and added to the type's bases so calls
|
|
357
|
+
* through the interface count as calls to the implementations and vice versa.
|
|
358
|
+
*/
|
|
359
|
+
#goImplements() {
|
|
360
|
+
const s = this.store, t = this.table;
|
|
361
|
+
s.tx(() => {
|
|
362
|
+
const old = s.all("SELECT src_id, name FROM refs WHERE kind = 'inherit' AND recv_type = '~structural'");
|
|
363
|
+
s.run("DELETE FROM refs WHERE kind = 'inherit' AND recv_type = '~structural'");
|
|
364
|
+
for (const o of old) {
|
|
365
|
+
const sym = t.sym(o.src_id);
|
|
366
|
+
if (sym && sym.structural?.has(o.name)) { sym.bases = sym.bases.filter(b => b !== o.name || sym.declaredBases?.includes(b)); sym.structural.delete(o.name); }
|
|
367
|
+
}
|
|
368
|
+
const goFile = (fid) => t.file(fid)?.lang === 'go';
|
|
369
|
+
const dirOf = (fid) => { const f = t.file(fid); return f ? path.posix.dirname(f.path) : null; };
|
|
370
|
+
const ifaces = [], types = [];
|
|
371
|
+
for (const sym of t.syms.values()) {
|
|
372
|
+
if (!goFile(sym.fileId)) continue;
|
|
373
|
+
if (sym.kind === 'interface') ifaces.push(sym);
|
|
374
|
+
else if (sym.kind === 'struct' || sym.kind === 'type' || sym.kind === 'class') types.push(sym);
|
|
375
|
+
}
|
|
376
|
+
if (!ifaces.length || !types.length) return;
|
|
377
|
+
const ifaceByName = new Map();
|
|
378
|
+
for (const i of ifaces) (ifaceByName.get(i.name) ?? ifaceByName.set(i.name, []).get(i.name)).push(i);
|
|
379
|
+
// a method's signature shape (see go.mjs methodShape); null when unreadable, which matches any
|
|
380
|
+
const shapeOf = (m) => this.resolver.specs.go?.methodShape?.(s.get('SELECT sig FROM symbols WHERE id = ?', m.id)?.sig, m.name) ?? null;
|
|
381
|
+
// methods of an interface (name → shape), following embedded interfaces declared in the same package
|
|
382
|
+
const methodSet = (i, seen = new Set()) => {
|
|
383
|
+
const out = new Map();
|
|
384
|
+
if (seen.has(i.id)) return out;
|
|
385
|
+
seen.add(i.id);
|
|
386
|
+
for (const [n, ids] of t.byParent.get(i.id) ?? []) {
|
|
387
|
+
const m = ids.map(id => t.sym(id)).find(x => x?.kind === 'method');
|
|
388
|
+
if (m) out.set(n, shapeOf(m));
|
|
389
|
+
}
|
|
390
|
+
for (const b of i.declaredBases ?? i.bases ?? []) for (const e of ifaceByName.get(b) ?? []) if (dirOf(e.fileId) === dirOf(i.fileId)) for (const [n, sh] of methodSet(e, seen)) if (!out.has(n)) out.set(n, sh);
|
|
391
|
+
return out;
|
|
392
|
+
};
|
|
393
|
+
// methods per type (methods live outside the type body in Go: owner = type name), plus
|
|
394
|
+
// those promoted from embedded types of the repository (`type Engine struct { RouterGroup }`)
|
|
395
|
+
const own = (ty) => {
|
|
396
|
+
if (ty.kind === 'interface') return methodSet(ty);
|
|
397
|
+
const dir = dirOf(ty.fileId), out = new Map();
|
|
398
|
+
for (const id of t.byOwner.get(ty.name) ?? []) {
|
|
399
|
+
const m = t.sym(id);
|
|
400
|
+
if (m && m.kind === 'method' && dirOf(m.fileId) === dir && !out.has(m.name)) out.set(m.name, shapeOf(m));
|
|
401
|
+
}
|
|
402
|
+
return out;
|
|
403
|
+
};
|
|
404
|
+
const promoted = (ty, depth = 0, seen = new Set()) => {
|
|
405
|
+
const out = own(ty);
|
|
406
|
+
if (depth > 4 || seen.has(ty.id)) return out;
|
|
407
|
+
seen.add(ty.id);
|
|
408
|
+
// an alias (`type DummyLogger = testhelper.DummyLogger`) has its target's methods
|
|
409
|
+
const alias = ty.kind === 'type' && !out.size && ty.type && !/(\[\]|\{\})$/.test(ty.type) ? this.resolver.resolveTypeName(ty.type, ty.fileId) : null;
|
|
410
|
+
if (alias && alias.id !== ty.id) for (const [n, sh] of promoted(t.sym(alias.id) ?? alias, depth + 1, seen)) if (!out.has(n)) out.set(n, sh);
|
|
411
|
+
for (const b of ty.declaredBases ?? ty.bases ?? []) {
|
|
412
|
+
const e = this.resolver.resolveTypeName(b, ty.fileId);
|
|
413
|
+
if (e && e.id !== ty.id) for (const [n, sh] of promoted(t.sym(e.id) ?? e, depth + 1, seen)) if (!out.has(n)) out.set(n, sh);
|
|
414
|
+
}
|
|
415
|
+
return out;
|
|
416
|
+
};
|
|
417
|
+
const byMethod = new Map(); // name → Map(type id → shape)
|
|
418
|
+
for (const ty of types) {
|
|
419
|
+
for (const [n, sh] of promoted(ty)) (byMethod.get(n) ?? byMethod.set(n, new Map()).get(n)).set(ty.id, sh);
|
|
420
|
+
}
|
|
421
|
+
const fits = (a, b) => a == null || b == null || a === b;
|
|
422
|
+
for (const i of ifaces) {
|
|
423
|
+
const need = [...methodSet(i)];
|
|
424
|
+
if (!need.length) continue;
|
|
425
|
+
need.sort((a, b) => (byMethod.get(a[0])?.size ?? 0) - (byMethod.get(b[0])?.size ?? 0));
|
|
426
|
+
const cands = byMethod.get(need[0][0]);
|
|
427
|
+
if (!cands) continue;
|
|
428
|
+
for (const tyId of cands.keys()) {
|
|
429
|
+
if (!need.every(([n, sh]) => byMethod.get(n)?.has(tyId) && fits(sh, byMethod.get(n).get(tyId)))) continue;
|
|
430
|
+
const ty = t.sym(tyId);
|
|
431
|
+
s.run("INSERT INTO refs (file_id, src_id, name, kind, line, col, recv, recv_type, dst_id, conf, ncand) VALUES (?, ?, ?, 'inherit', ?, 0, NULL, '~structural', ?, 0.8, 1)",
|
|
432
|
+
ty.fileId, ty.id, i.name, ty.startLine, i.id);
|
|
433
|
+
ty.declaredBases ??= [...ty.bases];
|
|
434
|
+
ty.structural ??= new Set();
|
|
435
|
+
if (!ty.bases.includes(i.name)) { ty.bases.push(i.name); ty.structural.add(i.name); }
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
});
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
#resolveRefs(refIds) {
|
|
442
|
+
if (!refIds.length) return 0;
|
|
443
|
+
const s = this.store;
|
|
444
|
+
this.#fixImportTargets();
|
|
445
|
+
let n = 0;
|
|
446
|
+
s.tx(() => {
|
|
447
|
+
for (let i = 0; i < refIds.length; i += 900) {
|
|
448
|
+
const chunk = refIds.slice(i, i + 900);
|
|
449
|
+
const rows = s.all(`SELECT id, file_id, src_id, name, kind, recv, recv_type, argc FROM refs WHERE id IN (${chunk.map(() => '?').join(',')})`, ...chunk);
|
|
450
|
+
for (const r of rows) {
|
|
451
|
+
const res = this.resolver.resolve(r);
|
|
452
|
+
if (res.id == null && (r.kind === 'value' || r.kind === 'read' || r.kind === 'type' || res.external)) { s.run('DELETE FROM refs WHERE id = ?', r.id); continue; }
|
|
453
|
+
s.run('UPDATE refs SET dst_id = ?, conf = ?, ncand = ? WHERE id = ?', res.id, res.conf, res.ncand ?? 0, r.id);
|
|
454
|
+
if (res.id != null) n++;
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
});
|
|
458
|
+
return n;
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
function safeStat(p) { try { return fs.statSync(p); } catch { return null; } }
|
|
463
|
+
|
|
464
|
+
function rowToSym(r) {
|
|
465
|
+
return {
|
|
466
|
+
id: r.id, fileId: r.file_id, name: r.name, qname: r.qname, kind: r.kind, parentId: r.parent_id ?? null, owner: r.owner ?? null,
|
|
467
|
+
type: r.type ?? null, bases: r.bases ? JSON.parse(r.bases) : [], exported: !!r.exported, isStatic: !!r.is_static, startLine: r.start_line, endLine: r.end_line,
|
|
468
|
+
};
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
function rowToImport(r, dir = null, targetPath = null) {
|
|
472
|
+
return {
|
|
473
|
+
source: r.source, imported: r.imported, local: r.local, reexport: r.reexport, wildcard: !!r.wildcard,
|
|
474
|
+
targetFileId: r.target_file_id ?? null, targetDir: dir ?? r.target_dir ?? null, targetPath,
|
|
475
|
+
};
|
|
476
|
+
}
|