graph-indexer 2.1.1 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +312 -1
- package/README.md +236 -312
- package/SECURITY.md +53 -58
- package/bin/graph-indexer.mjs +251 -0
- package/integrations/claude-code/.claude-plugin/plugin.json +9 -0
- package/integrations/claude-code/.mcp.json +8 -0
- package/integrations/claude-code/agents/code-structure.md +22 -0
- package/integrations/claude-code/hooks/gi-hook.mjs +103 -0
- package/integrations/claude-code/hooks/hooks.json +16 -0
- package/integrations/opencode/graph-indexer.js +129 -0
- package/package.json +32 -109
- package/src/cli/helper.mjs +67 -0
- package/src/cli/hook.mjs +366 -0
- package/src/cli/init.mjs +272 -0
- package/src/cli/resident.mjs +185 -0
- package/src/index/discover.mjs +128 -0
- package/src/index/fingerprint.mjs +30 -0
- package/src/index/graph.mjs +61 -0
- package/src/index/indexer.mjs +476 -0
- package/src/index/modules.mjs +332 -0
- package/src/index/resolver.mjs +961 -0
- package/src/mcp/render.mjs +60 -0
- package/src/mcp/server.mjs +147 -0
- package/src/mcp/tools.mjs +974 -0
- package/src/parse/arity.mjs +153 -0
- package/src/parse/extract.mjs +1038 -0
- package/src/parse/lang/csharp.mjs +106 -0
- package/src/parse/lang/ecma.mjs +438 -0
- package/src/parse/lang/go.mjs +194 -0
- package/src/parse/lang/jvm.mjs +118 -0
- package/src/parse/lang/kotlin.mjs +97 -0
- package/src/parse/lang/misc.mjs +115 -0
- package/src/parse/lang/native.mjs +126 -0
- package/src/parse/lang/python.mjs +153 -0
- package/src/parse/lang/rust.mjs +157 -0
- package/src/parse/lang/scripting.mjs +219 -0
- package/src/parse/languages.mjs +48 -0
- package/src/parse/runtime.mjs +56 -0
- package/src/query/check.mjs +345 -0
- package/src/query/facts.mjs +323 -0
- package/src/query/intel.mjs +683 -0
- package/src/query/read.mjs +81 -0
- package/src/query/testcmd.mjs +130 -0
- package/src/query/textsearch.mjs +178 -0
- package/src/search/search.mjs +210 -0
- package/src/search/tokenize.mjs +188 -0
- package/src/store/db.mjs +169 -0
- package/src/util/git.mjs +30 -0
- package/src/util/paths.mjs +63 -0
- package/src/util/regex.mjs +2 -0
- package/vendor/grammars/bash.wasm.gz +0 -0
- package/vendor/grammars/c.wasm.gz +0 -0
- package/vendor/grammars/c_sharp.wasm.gz +0 -0
- package/vendor/grammars/cpp.wasm.gz +0 -0
- package/vendor/grammars/css.wasm.gz +0 -0
- package/vendor/grammars/go.wasm.gz +0 -0
- package/vendor/grammars/java.wasm.gz +0 -0
- package/vendor/grammars/javascript.wasm.gz +0 -0
- package/vendor/grammars/kotlin.wasm.gz +0 -0
- package/vendor/grammars/manifest.json +148 -0
- package/vendor/grammars/php.wasm.gz +0 -0
- package/vendor/grammars/python.wasm.gz +0 -0
- package/vendor/grammars/ruby.wasm.gz +0 -0
- package/vendor/grammars/rust.wasm.gz +0 -0
- package/vendor/grammars/scala.wasm.gz +0 -0
- package/vendor/grammars/tsx.wasm.gz +0 -0
- package/vendor/grammars/typescript.wasm.gz +0 -0
- package/vendor/web-tree-sitter/web-tree-sitter.js +3980 -0
- package/vendor/web-tree-sitter/web-tree-sitter.wasm +0 -0
- package/cli-ui.mjs +0 -78
- package/colbert.mjs +0 -168
- package/config.mjs +0 -494
- package/daemon-ctl.mjs +0 -175
- package/daemon-lock.mjs +0 -69
- package/embedders/python/mlx_embed_server.py +0 -61
- package/embedders/python/requirements-mlx.txt +0 -5
- package/embedders/setup-mlx.mjs +0 -110
- package/embeddings.mjs +0 -571
- package/engine/binary.mjs +0 -437
- package/engine/memory.mjs +0 -1003
- package/engine/sqlite.mjs +0 -1136
- package/enrichment.mjs +0 -521
- package/git-signals.mjs +0 -248
- package/indexer.mjs +0 -546
- package/init.mjs +0 -1809
- package/layout.mjs +0 -123
- package/mcp/centrality.mjs +0 -117
- package/mcp/format.mjs +0 -61
- package/mcp/resolver.mjs +0 -116
- package/mcp/symbolgraph.mjs +0 -137
- package/mcp/taint.mjs +0 -225
- package/mcp/tools.mjs +0 -1582
- package/mcp/topology.mjs +0 -676
- package/mcp-server.mjs +0 -197
- package/parse/extractor.mjs +0 -628
- package/parse/imports.mjs +0 -365
- package/parse/interprocedural.mjs +0 -134
- package/parse/languages.mjs +0 -253
- package/parse/metadata.mjs +0 -701
- package/parse/routes.mjs +0 -327
- package/parse/scip.mjs +0 -391
- package/parse/taint-patterns.mjs +0 -119
- package/prompts/CORE.md +0 -91
- package/prompts/DOMAIN_TEMPLATE.md +0 -46
- package/prompts/INTEGRATION.md +0 -123
- package/prompts/frameworks/ANDROID.md +0 -17
- package/prompts/frameworks/ASPNET_CORE.md +0 -19
- package/prompts/frameworks/FASTAPI_DJANGO.md +0 -20
- package/prompts/frameworks/LARAVEL_SYMFONY.md +0 -22
- package/prompts/frameworks/NODE_EXPRESS_NESTJS.md +0 -19
- package/prompts/frameworks/RAILS.md +0 -20
- package/prompts/frameworks/REACT.md +0 -20
- package/prompts/frameworks/SPRING_BOOT.md +0 -20
- package/prompts/languages/BASH.md +0 -22
- package/prompts/languages/C.md +0 -21
- package/prompts/languages/CSHARP.md +0 -24
- package/prompts/languages/CSS_SCSS.md +0 -22
- package/prompts/languages/GO.md +0 -24
- package/prompts/languages/JAVA.md +0 -23
- package/prompts/languages/JAVASCRIPT_TYPESCRIPT.md +0 -24
- package/prompts/languages/KOTLIN.md +0 -24
- package/prompts/languages/PHP.md +0 -23
- package/prompts/languages/PYTHON.md +0 -23
- package/prompts/languages/RUBY.md +0 -23
- package/prompts/languages/RUST.md +0 -21
- package/prompts/languages/SWIFT.md +0 -21
- package/seal.mjs +0 -400
- package/search-core.mjs +0 -777
- package/search-sparse.mjs +0 -183
- package/storage.mjs +0 -73
- package/watch-daemon.mjs +0 -289
package/engine/sqlite.mjs
DELETED
|
@@ -1,1136 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file engine/sqlite.mjs
|
|
3
|
-
* @description Disk-backed graph store for enterprise-scale monorepos. Implements
|
|
4
|
-
* the same read contract as MemoryGraphIndex (see storage.mjs) over
|
|
5
|
-
* `node:sqlite` — built into Node, so this adds ZERO dependencies.
|
|
6
|
-
*
|
|
7
|
-
* The whole point is bounded RAM: chunk payloads, the BM25 posting
|
|
8
|
-
* lists and the symbol/call indexes live in SQLite tables and are
|
|
9
|
-
* touched only for the candidates a query actually needs. Vectors
|
|
10
|
-
* live in the shared `.embeddings.bin`: point reads are pread on
|
|
11
|
-
* demand, and the semantic channel streams the whole bin in bounded
|
|
12
|
-
* buffers. Nothing scales with corpus size in the resident set
|
|
13
|
-
* except the small file-level dependency graph (needed wholesale by
|
|
14
|
-
* PageRank).
|
|
15
|
-
*
|
|
16
|
-
* The store is LIVE: the watch daemon applies per-file incremental
|
|
17
|
-
* updates through applyFileUpdate() (WAL mode, single transaction
|
|
18
|
-
* per file), and reader processes detect those commits via
|
|
19
|
-
* `PRAGMA data_version` and transparently refresh their cached
|
|
20
|
-
* meta/graph state — no re-indexing, no server restarts.
|
|
21
|
-
*
|
|
22
|
-
* Ranking is delegated to search-core.fuseAndRank, so results are
|
|
23
|
-
* consistent with the in-memory engine.
|
|
24
|
-
* @author MaquinaTech <https://github.com/MaquinaTech>
|
|
25
|
-
* @copyright (c) 2026 MaquinaTech. All rights reserved.
|
|
26
|
-
* @license MIT
|
|
27
|
-
*/
|
|
28
|
-
import fs from 'fs';
|
|
29
|
-
import {
|
|
30
|
-
tokenize, okapiIdf, bm25Score, fuseAndRank, buildLexicalDocument, embeddingKeyFor,
|
|
31
|
-
LEXICAL_FUSION_CAP, VECTOR_SCAN_RAW_N, finalizeVectorCandidates,
|
|
32
|
-
isNaturalLanguageQuery, baseEmbeddingKey, scoreSparseExpanded,
|
|
33
|
-
} from '../search-core.mjs';
|
|
34
|
-
import { expandSparseQuery } from '../search-sparse.mjs';
|
|
35
|
-
import {
|
|
36
|
-
writeEmbeddingBinary, appendEmbeddingBinary, scanEmbeddingBinary,
|
|
37
|
-
updateVectorSketch, searchVectorSketch, SKETCH_THRESHOLD,
|
|
38
|
-
} from './binary.mjs';
|
|
39
|
-
|
|
40
|
-
// node:sqlite is loaded lazily so the default in-memory path never requires it.
|
|
41
|
-
let DatabaseSync = null;
|
|
42
|
-
try { ({ DatabaseSync } = await import('node:sqlite')); } catch { /* reported in ctor */ }
|
|
43
|
-
|
|
44
|
-
// SKETCH_THRESHOLD (shared with the memory store via binary.mjs) — build the binary
|
|
45
|
-
// sketch above this many vectors. Below it the exact streaming scan stays under ~20 ms
|
|
46
|
-
// AND both backends rank identically (the sketch is
|
|
47
|
-
// approximate in the candidate tail); above it the
|
|
48
|
-
// sketch caps latency at ~5–15 ms regardless of size.
|
|
49
|
-
|
|
50
|
-
const SCHEMA_TABLES = `
|
|
51
|
-
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT);
|
|
52
|
-
CREATE TABLE IF NOT EXISTS chunks (
|
|
53
|
-
id TEXT PRIMARY KEY, file_path TEXT NOT NULL, node_type TEXT,
|
|
54
|
-
name TEXT, name_lower TEXT, docstring TEXT, code_snippet TEXT, content_hash TEXT,
|
|
55
|
-
start_line INTEGER, end_line INTEGER,
|
|
56
|
-
calls TEXT, params TEXT, return_type TEXT, class_context TEXT,
|
|
57
|
-
type_refs TEXT, decorators TEXT, extends_ TEXT, hyde TEXT, summary TEXT, concepts TEXT,
|
|
58
|
-
doc_len INTEGER, path_tokens TEXT, vec_key TEXT, vec_offset INTEGER, vec_dim INTEGER,
|
|
59
|
-
call_sites TEXT
|
|
60
|
-
);
|
|
61
|
-
CREATE TABLE IF NOT EXISTS postings (term TEXT, chunk_id TEXT, tf INTEGER);
|
|
62
|
-
CREATE TABLE IF NOT EXISTS terms (term TEXT PRIMARY KEY, df INTEGER);
|
|
63
|
-
CREATE TABLE IF NOT EXISTS call_edges (callee TEXT, chunk_id TEXT);
|
|
64
|
-
CREATE TABLE IF NOT EXISTS edges (from_chunk_id TEXT, to_chunk_id TEXT, kind TEXT, confidence TEXT);
|
|
65
|
-
CREATE TABLE IF NOT EXISTS centrality (chunk_id TEXT, score REAL, rank INTEGER);
|
|
66
|
-
CREATE TABLE IF NOT EXISTS taint (id INTEGER PRIMARY KEY, payload TEXT);
|
|
67
|
-
CREATE TABLE IF NOT EXISTS scip_refs (id INTEGER PRIMARY KEY, payload TEXT);
|
|
68
|
-
CREATE TABLE IF NOT EXISTS sparse_model (id INTEGER PRIMARY KEY, payload TEXT);
|
|
69
|
-
CREATE TABLE IF NOT EXISTS deps (file TEXT, dep TEXT);
|
|
70
|
-
CREATE TABLE IF NOT EXISTS routes (
|
|
71
|
-
id INTEGER PRIMARY KEY,
|
|
72
|
-
method TEXT NOT NULL,
|
|
73
|
-
path TEXT NOT NULL,
|
|
74
|
-
handler_name TEXT,
|
|
75
|
-
handler_chunk_id TEXT,
|
|
76
|
-
file_path TEXT NOT NULL,
|
|
77
|
-
line INTEGER,
|
|
78
|
-
framework TEXT
|
|
79
|
-
);
|
|
80
|
-
`;
|
|
81
|
-
|
|
82
|
-
const SCHEMA_INDEXES = `
|
|
83
|
-
CREATE INDEX IF NOT EXISTS idx_chunks_file ON chunks(file_path);
|
|
84
|
-
CREATE INDEX IF NOT EXISTS idx_chunks_name ON chunks(name_lower);
|
|
85
|
-
CREATE INDEX IF NOT EXISTS idx_chunks_veckey ON chunks(vec_key);
|
|
86
|
-
CREATE INDEX IF NOT EXISTS idx_chunks_hash ON chunks(content_hash);
|
|
87
|
-
CREATE INDEX IF NOT EXISTS idx_postings_term ON postings(term);
|
|
88
|
-
CREATE INDEX IF NOT EXISTS idx_postings_chunk ON postings(chunk_id);
|
|
89
|
-
CREATE INDEX IF NOT EXISTS idx_calledges_callee ON call_edges(callee);
|
|
90
|
-
CREATE INDEX IF NOT EXISTS idx_edges_to ON edges(to_chunk_id);
|
|
91
|
-
CREATE INDEX IF NOT EXISTS idx_edges_from ON edges(from_chunk_id);
|
|
92
|
-
CREATE INDEX IF NOT EXISTS idx_centrality_chunk ON centrality(chunk_id);
|
|
93
|
-
CREATE INDEX IF NOT EXISTS idx_centrality_rank ON centrality(rank);
|
|
94
|
-
CREATE INDEX IF NOT EXISTS idx_deps_file ON deps(file);
|
|
95
|
-
CREATE INDEX IF NOT EXISTS idx_deps_dep ON deps(dep);
|
|
96
|
-
CREATE INDEX IF NOT EXISTS idx_routes_method ON routes(method);
|
|
97
|
-
CREATE INDEX IF NOT EXISTS idx_routes_path ON routes(path);
|
|
98
|
-
CREATE INDEX IF NOT EXISTS idx_routes_handler ON routes(handler_chunk_id);
|
|
99
|
-
`;
|
|
100
|
-
|
|
101
|
-
const CHUNK_COLS = [
|
|
102
|
-
'id', 'file_path', 'node_type', 'name', 'name_lower', 'docstring', 'code_snippet',
|
|
103
|
-
'content_hash', 'start_line', 'end_line', 'calls', 'params', 'return_type',
|
|
104
|
-
'class_context', 'type_refs', 'decorators', 'extends_', 'hyde', 'summary', 'concepts',
|
|
105
|
-
'doc_len', 'path_tokens', 'vec_key', 'vec_offset', 'vec_dim', 'call_sites',
|
|
106
|
-
];
|
|
107
|
-
|
|
108
|
-
function jsonArr(v) { return JSON.stringify(Array.isArray(v) ? v : []); }
|
|
109
|
-
function parseArr(s) {
|
|
110
|
-
if (!s) return [];
|
|
111
|
-
try { const v = JSON.parse(s); return Array.isArray(v) ? v : []; } catch { return []; }
|
|
112
|
-
}
|
|
113
|
-
function nameLowerOf(name) {
|
|
114
|
-
return (name && name !== 'anonymous') ? String(name).toLowerCase() : null;
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
export class SqliteGraphStore {
|
|
118
|
-
constructor(dbPath, { embeddingPath, rrfK = 60 } = {}) {
|
|
119
|
-
if (!DatabaseSync) {
|
|
120
|
-
throw new Error(
|
|
121
|
-
'node:sqlite is unavailable — the --use-sqlite backend requires Node >= 22.5. '
|
|
122
|
-
+ 'Use the default in-memory engine, or upgrade Node.'
|
|
123
|
-
);
|
|
124
|
-
}
|
|
125
|
-
this.dbPath = dbPath;
|
|
126
|
-
this._embeddingPath = embeddingPath || dbPath.replace(/\.db$/, '.embeddings.bin');
|
|
127
|
-
this.db = null;
|
|
128
|
-
this._vecFd = -1;
|
|
129
|
-
this.rrfK = rrfK;
|
|
130
|
-
|
|
131
|
-
// File-level dependency graph — small (per file, not per chunk); held in RAM
|
|
132
|
-
// because PageRank needs the whole thing. This is the only corpus-scaling
|
|
133
|
-
// structure kept resident, and it is orders of magnitude smaller than chunks.
|
|
134
|
-
this._graph = { dependencies: {}, importedBy: {} };
|
|
135
|
-
this._docCount = 0;
|
|
136
|
-
this._totalDocLen = 0;
|
|
137
|
-
this._avgdl = 1;
|
|
138
|
-
this._dim = 0;
|
|
139
|
-
this._dataVersion = -1;
|
|
140
|
-
this._writeReady = false;
|
|
141
|
-
this._sketch = null; // binary-quantized vector sketch (built above SKETCH_THRESHOLD)
|
|
142
|
-
this._hasEdges = false; // A4: resolved symbol graph present (opt-in)
|
|
143
|
-
this._hasCentrality = false; // A5: symbol centrality present (opt-in, built with the graph)
|
|
144
|
-
this._centralityTotal = 0;
|
|
145
|
-
this._hasTaint = false; // C2: serialized taint flows present (opt-in --taint)
|
|
146
|
-
this._hasScipRefs = false; // A2 v2: SCIP precise references present (opt-in --resolver scip)
|
|
147
|
-
this._scipRefsObj = null; // lazily-parsed `{ defId: [refId,…] }` payload (memoized)
|
|
148
|
-
this._hasSparseModel = false; // B3: learned-sparse model present (opt-in --learned-sparse)
|
|
149
|
-
this._sparseModelObj = null; // lazily-parsed `{ assoc: {…} }` payload (memoized)
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
get backend() { return 'sqlite'; }
|
|
153
|
-
|
|
154
|
-
/** Live view of the file dependency graph (refreshed when another process commits). */
|
|
155
|
-
get graph() {
|
|
156
|
-
this._maybeRefresh();
|
|
157
|
-
return this._graph;
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
// ─── Open / load ───────────────────────────────────────────────────────────
|
|
161
|
-
|
|
162
|
-
_open() {
|
|
163
|
-
this.db = new DatabaseSync(this.dbPath);
|
|
164
|
-
// WAL: one writer (daemon) + many readers (MCP server) across processes.
|
|
165
|
-
// busy_timeout absorbs brief writer/checkpoint locks instead of throwing.
|
|
166
|
-
this.db.exec('PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL; PRAGMA busy_timeout = 3000;');
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
load() {
|
|
170
|
-
const existed = fs.existsSync(this.dbPath);
|
|
171
|
-
this._open();
|
|
172
|
-
this.db.exec(SCHEMA_TABLES);
|
|
173
|
-
if (existed) {
|
|
174
|
-
// Forward migrations for indexes built by older versions.
|
|
175
|
-
try { this.db.exec('ALTER TABLE chunks ADD COLUMN concepts TEXT DEFAULT NULL'); } catch { /* exists */ }
|
|
176
|
-
try {
|
|
177
|
-
this.db.exec('ALTER TABLE chunks ADD COLUMN vec_key TEXT DEFAULT NULL');
|
|
178
|
-
// Pre-enrichment-key indexes stored vectors under the plain content hash.
|
|
179
|
-
this.db.exec('UPDATE chunks SET vec_key = content_hash WHERE vec_key IS NULL');
|
|
180
|
-
} catch { /* exists */ }
|
|
181
|
-
// Receiver-aware call graph (precise get_call_graph) landed later; older
|
|
182
|
-
// indexes lack the column. NULL → callers fall back to name-only grouping.
|
|
183
|
-
try { this.db.exec('ALTER TABLE chunks ADD COLUMN call_sites TEXT DEFAULT NULL'); } catch { /* exists */ }
|
|
184
|
-
}
|
|
185
|
-
this.db.exec(SCHEMA_INDEXES);
|
|
186
|
-
this._reloadMeta();
|
|
187
|
-
// Does this corpus carry LLM enrichment? One cheap probe → drives the NL
|
|
188
|
-
// vector-channel weight in fuseAndRank (parity with the in-memory engine).
|
|
189
|
-
try {
|
|
190
|
-
this._corpusEnriched = Boolean(
|
|
191
|
-
this.db.prepare("SELECT 1 FROM chunks WHERE summary IS NOT NULL AND summary != '' LIMIT 1").get()
|
|
192
|
-
);
|
|
193
|
-
} catch { this._corpusEnriched = false; }
|
|
194
|
-
this._loadGraph();
|
|
195
|
-
this._openVecFd();
|
|
196
|
-
this._refreshSketch();
|
|
197
|
-
this._prepare();
|
|
198
|
-
this._refreshHasEdges();
|
|
199
|
-
this._refreshHasCentrality();
|
|
200
|
-
this._refreshHasTaint();
|
|
201
|
-
this._refreshHasScipRefs();
|
|
202
|
-
this._refreshHasSparseModel();
|
|
203
|
-
this._dataVersion = this._readDataVersion();
|
|
204
|
-
return this;
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
/** Whether the opt-in resolved symbol graph (A4) has any edges. */
|
|
208
|
-
_refreshHasEdges() {
|
|
209
|
-
try { this._hasEdges = Boolean(this.db.prepare('SELECT 1 FROM edges LIMIT 1').get()); }
|
|
210
|
-
catch { this._hasEdges = false; }
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
/** Whether opt-in symbol centrality (A5) is present, and how many chunks are ranked. */
|
|
214
|
-
_refreshHasCentrality() {
|
|
215
|
-
try {
|
|
216
|
-
const row = this.db.prepare('SELECT COUNT(*) AS n FROM centrality').get();
|
|
217
|
-
this._centralityTotal = row ? row.n : 0;
|
|
218
|
-
this._hasCentrality = this._centralityTotal > 0;
|
|
219
|
-
} catch { this._hasCentrality = false; this._centralityTotal = 0; }
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
/** Whether opt-in serialized taint flows (C2 --taint) are present. */
|
|
223
|
-
_refreshHasTaint() {
|
|
224
|
-
try {
|
|
225
|
-
const row = this.db.prepare('SELECT COUNT(*) AS n FROM taint').get();
|
|
226
|
-
this._hasTaint = Boolean(row && row.n > 0);
|
|
227
|
-
} catch { this._hasTaint = false; }
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
/** Whether opt-in SCIP precise references (A2 v2 --resolver scip) are present. */
|
|
231
|
-
_refreshHasScipRefs() {
|
|
232
|
-
try {
|
|
233
|
-
const row = this.db.prepare('SELECT COUNT(*) AS n FROM scip_refs').get();
|
|
234
|
-
this._hasScipRefs = Boolean(row && row.n > 0);
|
|
235
|
-
} catch { this._hasScipRefs = false; }
|
|
236
|
-
this._scipRefsObj = null; // drop any memoized payload — the row may have changed
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
/** Whether opt-in learned-sparse associations (B3 --learned-sparse) are present. */
|
|
240
|
-
_refreshHasSparseModel() {
|
|
241
|
-
try {
|
|
242
|
-
const row = this.db.prepare('SELECT COUNT(*) AS n FROM sparse_model').get();
|
|
243
|
-
this._hasSparseModel = Boolean(row && row.n > 0);
|
|
244
|
-
} catch { this._hasSparseModel = false; }
|
|
245
|
-
this._sparseModelObj = null; // drop any memoized payload — the row may have changed
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
_openVecFd() {
|
|
249
|
-
if (this._vecFd >= 0) { try { fs.closeSync(this._vecFd); } catch {} this._vecFd = -1; }
|
|
250
|
-
if (fs.existsSync(this._embeddingPath)) {
|
|
251
|
-
try { this._vecFd = fs.openSync(this._embeddingPath, 'r'); } catch { this._vecFd = -1; }
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
/** Read the bin's entry count + first entry key (cheap, two small preads). */
|
|
256
|
-
_binFingerprint() {
|
|
257
|
-
if (this._vecFd < 0) return null;
|
|
258
|
-
try {
|
|
259
|
-
const hdr = Buffer.allocUnsafe(8);
|
|
260
|
-
if (fs.readSync(this._vecFd, hdr, 0, 8, 0) < 8) return null;
|
|
261
|
-
const count = hdr.readUInt32LE(0);
|
|
262
|
-
const keyLen = hdr.readUInt32LE(4);
|
|
263
|
-
if (keyLen <= 0 || keyLen > 4096) return { count, firstKey: null };
|
|
264
|
-
const kb = Buffer.allocUnsafe(keyLen);
|
|
265
|
-
if (fs.readSync(this._vecFd, kb, 0, keyLen, 8) < keyLen) return { count, firstKey: null };
|
|
266
|
-
return { count, firstKey: kb.toString('utf8') };
|
|
267
|
-
} catch { return null; }
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
/**
|
|
271
|
-
* Build / extend / rebuild the binary vector sketch to match the bin on disk.
|
|
272
|
-
* Appends (the daemon's write path) extend the sketch by scanning only the
|
|
273
|
-
* unseen tail; a replaced bin (full re-index) is detected via the entry count
|
|
274
|
-
* + first-key fingerprint and triggers a rebuild.
|
|
275
|
-
*/
|
|
276
|
-
_refreshSketch() {
|
|
277
|
-
if (this._vecFd < 0) { this._sketch = null; return; }
|
|
278
|
-
const fp = this._binFingerprint();
|
|
279
|
-
if (!fp || fp.count < SKETCH_THRESHOLD) { this._sketch = null; return; }
|
|
280
|
-
try {
|
|
281
|
-
const size = fs.fstatSync(this._vecFd).size;
|
|
282
|
-
if (this._sketch) {
|
|
283
|
-
const replaced = fp.count < this._sketch.n
|
|
284
|
-
|| size < this._sketch.consumed
|
|
285
|
-
|| (this._sketch.firstKey !== null && fp.firstKey !== this._sketch.firstKey);
|
|
286
|
-
if (replaced) this._sketch = null;
|
|
287
|
-
}
|
|
288
|
-
this._sketch = updateVectorSketch(this._sketch, { fd: this._vecFd });
|
|
289
|
-
} catch { this._sketch = null; }
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
_reloadMeta() {
|
|
293
|
-
const meta = {};
|
|
294
|
-
for (const r of this.db.prepare('SELECT key, value FROM meta').all()) meta[r.key] = r.value;
|
|
295
|
-
this._docCount = Number(meta.doc_count || 0);
|
|
296
|
-
this._totalDocLen = Number(meta.total_doc_len || 0);
|
|
297
|
-
this._avgdl = this._docCount > 0 ? this._totalDocLen / this._docCount : 1;
|
|
298
|
-
this._dim = Number(meta.dim || 0);
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
_loadGraph() {
|
|
302
|
-
const dependencies = {};
|
|
303
|
-
const importedBy = {};
|
|
304
|
-
for (const { file, dep } of this.db.prepare('SELECT file, dep FROM deps').all()) {
|
|
305
|
-
(dependencies[file] ||= []);
|
|
306
|
-
if (dep != null) {
|
|
307
|
-
dependencies[file].push(dep);
|
|
308
|
-
(importedBy[dep] ||= []).push(file);
|
|
309
|
-
}
|
|
310
|
-
}
|
|
311
|
-
// Ensure every file that has chunks is a graph node (PageRank node set parity).
|
|
312
|
-
for (const { f } of this.db.prepare('SELECT DISTINCT file_path AS f FROM chunks').all()) {
|
|
313
|
-
dependencies[f] ||= [];
|
|
314
|
-
}
|
|
315
|
-
this._graph = { dependencies, importedBy };
|
|
316
|
-
}
|
|
317
|
-
|
|
318
|
-
_readDataVersion() {
|
|
319
|
-
try { return this.db.prepare('PRAGMA data_version').get()?.data_version ?? -1; }
|
|
320
|
-
catch { return -1; }
|
|
321
|
-
}
|
|
322
|
-
|
|
323
|
-
/**
|
|
324
|
-
* Detect commits made by OTHER processes (the watch daemon, a re-run of the
|
|
325
|
-
* indexer) via PRAGMA data_version — a single cheap pragma per call — and
|
|
326
|
-
* refresh the cached meta, dependency graph and embeddings fd when the
|
|
327
|
-
* underlying database changed. This is what keeps a long-running MCP server
|
|
328
|
-
* consistent with the daemon without restarts or full re-indexes.
|
|
329
|
-
*/
|
|
330
|
-
_maybeRefresh() {
|
|
331
|
-
if (!this.db) return;
|
|
332
|
-
try {
|
|
333
|
-
const v = this._readDataVersion();
|
|
334
|
-
if (v === this._dataVersion) return;
|
|
335
|
-
this._dataVersion = v;
|
|
336
|
-
this._reloadMeta();
|
|
337
|
-
this._loadGraph();
|
|
338
|
-
this._openVecFd();
|
|
339
|
-
this._refreshSketch();
|
|
340
|
-
} catch { /* keep previous snapshot on transient errors */ }
|
|
341
|
-
}
|
|
342
|
-
|
|
343
|
-
_prepare() {
|
|
344
|
-
this._stmtChunk = this.db.prepare('SELECT * FROM chunks WHERE id = ?');
|
|
345
|
-
this._stmtByFile = this.db.prepare('SELECT * FROM chunks WHERE file_path = ?');
|
|
346
|
-
this._stmtByName = this.db.prepare('SELECT * FROM chunks WHERE name_lower = ?');
|
|
347
|
-
this._stmtIdByName = this.db.prepare('SELECT id FROM chunks WHERE name_lower = ?');
|
|
348
|
-
this._stmtCallers = this.db.prepare(
|
|
349
|
-
'SELECT c.* FROM chunks c JOIN call_edges e ON e.chunk_id = c.id WHERE e.callee = ?'
|
|
350
|
-
);
|
|
351
|
-
// type_refs / extends_ are JSON arrays of names. LIKE is a cheap prefilter
|
|
352
|
-
// (ASCII case-insensitive in SQLite); findReferers() verifies exact, quoted
|
|
353
|
-
// membership in JS so e.g. "User" never matches "UserProfile".
|
|
354
|
-
this._stmtReferers = this.db.prepare(
|
|
355
|
-
'SELECT * FROM chunks WHERE type_refs LIKE ? ESCAPE \'\\\' OR extends_ LIKE ? ESCAPE \'\\\''
|
|
356
|
-
);
|
|
357
|
-
this._stmtPosting = this.db.prepare(
|
|
358
|
-
'SELECT p.chunk_id AS id, p.tf AS tf, c.doc_len AS doc_len '
|
|
359
|
-
+ 'FROM postings p JOIN chunks c ON c.id = p.chunk_id WHERE p.term = ?'
|
|
360
|
-
);
|
|
361
|
-
this._stmtDf = this.db.prepare('SELECT df FROM terms WHERE term = ?');
|
|
362
|
-
this._stmtVec = this.db.prepare('SELECT vec_offset, vec_dim FROM chunks WHERE id = ?');
|
|
363
|
-
// ORDER BY id: duplicate-content chunks share one vector key; emitting their
|
|
364
|
-
// ids in a deterministic order keeps both backends rank-identical on ties.
|
|
365
|
-
this._stmtIdsByKey = this.db.prepare('SELECT id FROM chunks WHERE vec_key = ? ORDER BY id');
|
|
366
|
-
this._stmtIdsByHash = this.db.prepare('SELECT id FROM chunks WHERE content_hash = ? ORDER BY id');
|
|
367
|
-
// ORDER BY id (rowid = insertion order) keeps routes in index-build order,
|
|
368
|
-
// matching the in-memory graph.routes array order for cross-backend parity.
|
|
369
|
-
this._stmtRoutes = this.db.prepare(
|
|
370
|
-
'SELECT method, path, handler_name, handler_chunk_id, file_path, line, framework '
|
|
371
|
-
+ 'FROM routes ORDER BY id'
|
|
372
|
-
);
|
|
373
|
-
// A4 symbol graph. ORDER mirrors the in-memory engine's per-endpoint slice of the
|
|
374
|
-
// globally-sorted edge list (fixed `to` → sort by from,kind,confidence; fixed `from`
|
|
375
|
-
// → sort by to,kind,confidence) so getEdges is byte-identical cross-backend.
|
|
376
|
-
this._stmtEdgesIn = this.db.prepare(
|
|
377
|
-
'SELECT from_chunk_id, to_chunk_id, kind, confidence FROM edges WHERE to_chunk_id = ? '
|
|
378
|
-
+ 'ORDER BY from_chunk_id, kind, confidence'
|
|
379
|
-
);
|
|
380
|
-
this._stmtEdgesOut = this.db.prepare(
|
|
381
|
-
'SELECT from_chunk_id, to_chunk_id, kind, confidence FROM edges WHERE from_chunk_id = ? '
|
|
382
|
-
+ 'ORDER BY to_chunk_id, kind, confidence'
|
|
383
|
-
);
|
|
384
|
-
// A5 symbol centrality. rank/score are precomputed at index time; topCentral orders by
|
|
385
|
-
// the stored rank (chunk_id tie-break mirrors the in-memory ranked cache).
|
|
386
|
-
this._stmtCentrality = this.db.prepare('SELECT score, rank FROM centrality WHERE chunk_id = ?');
|
|
387
|
-
this._stmtTopCentral = this.db.prepare(
|
|
388
|
-
'SELECT chunk_id, score, rank FROM centrality ORDER BY rank ASC, chunk_id ASC LIMIT ?'
|
|
389
|
-
);
|
|
390
|
-
// C2 serialized taint flows — a single JSON payload row (id = 0).
|
|
391
|
-
this._stmtTaint = this.db.prepare('SELECT payload FROM taint WHERE id = 0');
|
|
392
|
-
this._stmtScipRefs = this.db.prepare('SELECT payload FROM scip_refs WHERE id = 0');
|
|
393
|
-
this._stmtSparseModel = this.db.prepare('SELECT payload FROM sparse_model WHERE id = 0');
|
|
394
|
-
}
|
|
395
|
-
|
|
396
|
-
_prepareWrite() {
|
|
397
|
-
if (this._writeReady) return;
|
|
398
|
-
this._stmtInsChunk = this.db.prepare(
|
|
399
|
-
`INSERT OR REPLACE INTO chunks (${CHUNK_COLS.join(', ')}) `
|
|
400
|
-
+ `VALUES (${CHUNK_COLS.map(() => '?').join(', ')})`
|
|
401
|
-
);
|
|
402
|
-
this._stmtInsPost = this.db.prepare('INSERT INTO postings (term, chunk_id, tf) VALUES (?, ?, ?)');
|
|
403
|
-
this._stmtInsEdge = this.db.prepare('INSERT INTO call_edges (callee, chunk_id) VALUES (?, ?)');
|
|
404
|
-
this._stmtInsDep = this.db.prepare('INSERT INTO deps (file, dep) VALUES (?, ?)');
|
|
405
|
-
this._stmtDelDeps = this.db.prepare('DELETE FROM deps WHERE file = ?');
|
|
406
|
-
this._stmtUpsertTerm = this.db.prepare(
|
|
407
|
-
'INSERT INTO terms (term, df) VALUES (?, 1) ON CONFLICT(term) DO UPDATE SET df = df + 1'
|
|
408
|
-
);
|
|
409
|
-
this._stmtDecTerm = this.db.prepare('UPDATE terms SET df = df - 1 WHERE term = ?');
|
|
410
|
-
this._stmtPruneTerms = this.db.prepare('DELETE FROM terms WHERE df <= 0');
|
|
411
|
-
this._stmtPostByChunk = this.db.prepare('SELECT term FROM postings WHERE chunk_id = ?');
|
|
412
|
-
this._stmtDelPostings = this.db.prepare('DELETE FROM postings WHERE chunk_id = ?');
|
|
413
|
-
this._stmtDelEdges = this.db.prepare('DELETE FROM call_edges WHERE chunk_id = ?');
|
|
414
|
-
this._stmtDelChunk = this.db.prepare('DELETE FROM chunks WHERE id = ?');
|
|
415
|
-
this._stmtInsMeta = this.db.prepare('INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)');
|
|
416
|
-
this._stmtVecByKey = this.db.prepare(
|
|
417
|
-
'SELECT vec_offset, vec_dim FROM chunks WHERE vec_key = ? AND vec_offset >= 0 LIMIT 1'
|
|
418
|
-
);
|
|
419
|
-
this._stmtVecByHash = this.db.prepare(
|
|
420
|
-
'SELECT vec_offset, vec_dim FROM chunks WHERE content_hash = ? AND vec_offset >= 0 LIMIT 1'
|
|
421
|
-
);
|
|
422
|
-
this._writeReady = true;
|
|
423
|
-
}
|
|
424
|
-
|
|
425
|
-
// ─── Row → chunk ───────────────────────────────────────────────────────────
|
|
426
|
-
|
|
427
|
-
_rowToChunk(row) {
|
|
428
|
-
if (!row) return null;
|
|
429
|
-
return {
|
|
430
|
-
id: row.id, file_path: row.file_path, node_type: row.node_type, name: row.name,
|
|
431
|
-
docstring: row.docstring || '', code_snippet: row.code_snippet || '',
|
|
432
|
-
content_hash: row.content_hash, start_line: row.start_line, end_line: row.end_line,
|
|
433
|
-
calls: parseArr(row.calls), params: parseArr(row.params),
|
|
434
|
-
return_type: row.return_type || '', class_context: row.class_context || '',
|
|
435
|
-
type_refs: parseArr(row.type_refs), decorators: parseArr(row.decorators),
|
|
436
|
-
extends: parseArr(row.extends_), hyde: row.hyde || '', summary: row.summary || '',
|
|
437
|
-
concepts: parseArr(row.concepts), call_sites: parseArr(row.call_sites),
|
|
438
|
-
};
|
|
439
|
-
}
|
|
440
|
-
|
|
441
|
-
// ─── Contract: point reads ─────────────────────────────────────────────────
|
|
442
|
-
|
|
443
|
-
getChunk(id) { return this._rowToChunk(this._stmtChunk.get(id)); }
|
|
444
|
-
|
|
445
|
-
getChunksByFile(filePath) { return this._stmtByFile.all(filePath).map(r => this._rowToChunk(r)); }
|
|
446
|
-
|
|
447
|
-
resolveSymbol(name) {
|
|
448
|
-
const key = nameLowerOf(name);
|
|
449
|
-
if (!key) return [];
|
|
450
|
-
return this._stmtByName.all(key).map(r => this._rowToChunk(r));
|
|
451
|
-
}
|
|
452
|
-
|
|
453
|
-
/**
|
|
454
|
-
* Resolved symbol-graph edges incident to a chunk (A4). Mirrors MemoryGraphIndex.getEdges,
|
|
455
|
-
* including deterministic order. Returns [] unless the opt-in graph was built.
|
|
456
|
-
*/
|
|
457
|
-
/** Whether the opt-in resolved symbol graph (A4) is loaded. */
|
|
458
|
-
hasSymbolGraph() { return this._hasEdges; }
|
|
459
|
-
|
|
460
|
-
getEdges(chunkId, { kind = null, direction = 'in' } = {}) {
|
|
461
|
-
if (!this._hasEdges) return [];
|
|
462
|
-
const rows = direction === 'out' ? this._stmtEdgesOut.all(chunkId) : this._stmtEdgesIn.all(chunkId);
|
|
463
|
-
const out = [];
|
|
464
|
-
for (const e of rows) {
|
|
465
|
-
if (kind && e.kind !== kind) continue;
|
|
466
|
-
const otherId = direction === 'out' ? e.to_chunk_id : e.from_chunk_id;
|
|
467
|
-
out.push({ from_chunk_id: e.from_chunk_id, to_chunk_id: e.to_chunk_id, kind: e.kind, confidence: e.confidence, chunk: this.getChunk(otherId) });
|
|
468
|
-
}
|
|
469
|
-
return out;
|
|
470
|
-
}
|
|
471
|
-
|
|
472
|
-
/** Whether opt-in symbol centrality (A5) is loaded. */
|
|
473
|
-
hasCentrality() { return this._hasCentrality; }
|
|
474
|
-
|
|
475
|
-
/** Whether opt-in serialized taint flows (C2 --taint) are loaded. */
|
|
476
|
-
hasTaint() { return this._hasTaint; }
|
|
477
|
-
|
|
478
|
-
/** The serialized taint payload `{ flows, meta }` (C2), or null. Mirrors MemoryGraphIndex. */
|
|
479
|
-
getTaintFlows() {
|
|
480
|
-
if (!this._hasTaint) return null;
|
|
481
|
-
const row = this._stmtTaint.get();
|
|
482
|
-
if (!row || !row.payload) return null;
|
|
483
|
-
try { return JSON.parse(row.payload); } catch { return null; }
|
|
484
|
-
}
|
|
485
|
-
|
|
486
|
-
/** Whether opt-in SCIP precise references (A2 v2 --resolver scip) are loaded. */
|
|
487
|
-
hasScipRefs() { return this._hasScipRefs; }
|
|
488
|
-
|
|
489
|
-
/** Parse the single JSON payload `{ defId: [refId,…] }` once, then memoize. */
|
|
490
|
-
_loadScipRefs() {
|
|
491
|
-
if (!this._hasScipRefs) return {};
|
|
492
|
-
if (!this._scipRefsObj) {
|
|
493
|
-
const row = this._stmtScipRefs.get();
|
|
494
|
-
try { this._scipRefsObj = row && row.payload ? JSON.parse(row.payload) : {}; }
|
|
495
|
-
catch { this._scipRefsObj = {}; }
|
|
496
|
-
}
|
|
497
|
-
return this._scipRefsObj;
|
|
498
|
-
}
|
|
499
|
-
|
|
500
|
-
/** SCIP-confirmed referer chunk ids for a definition chunk (A2 v2), or []. Mirrors MemoryGraphIndex. */
|
|
501
|
-
getScipReferers(defChunkId) {
|
|
502
|
-
const refs = this._loadScipRefs()[defChunkId];
|
|
503
|
-
return Array.isArray(refs) ? refs : [];
|
|
504
|
-
}
|
|
505
|
-
|
|
506
|
-
/** How many definitions carry SCIP precise references (A2 v2). 0 when not loaded. */
|
|
507
|
-
scipRefCount() { return this._hasScipRefs ? Object.keys(this._loadScipRefs()).length : 0; }
|
|
508
|
-
|
|
509
|
-
/** Whether opt-in learned-sparse associations (B3 --learned-sparse) are loaded. */
|
|
510
|
-
hasSparseModel() { return this._hasSparseModel; }
|
|
511
|
-
|
|
512
|
-
/** Path to the shared `.embeddings.bin` (B4 late-interaction reranker reads candidate vectors from it). */
|
|
513
|
-
embeddingBinPath() { return this._embeddingPath; }
|
|
514
|
-
|
|
515
|
-
/** Parse the single learned-sparse payload once, then memoize. Mirrors MemoryGraphIndex._sparseModel. */
|
|
516
|
-
_loadSparseModel() {
|
|
517
|
-
if (!this._hasSparseModel) return null;
|
|
518
|
-
if (!this._sparseModelObj) {
|
|
519
|
-
const row = this._stmtSparseModel.get();
|
|
520
|
-
try { this._sparseModelObj = row && row.payload ? JSON.parse(row.payload) : null; }
|
|
521
|
-
catch { this._sparseModelObj = null; }
|
|
522
|
-
}
|
|
523
|
-
return this._sparseModelObj;
|
|
524
|
-
}
|
|
525
|
-
|
|
526
|
-
/**
|
|
527
|
-
* Symbol centrality (A5) for one chunk. Mirrors MemoryGraphIndex.getCentrality — same
|
|
528
|
-
* { score, rank, total } shape; null when unranked or centrality was not built.
|
|
529
|
-
*/
|
|
530
|
-
getCentrality(chunkId) {
|
|
531
|
-
if (!this._hasCentrality) return null;
|
|
532
|
-
const row = this._stmtCentrality.get(chunkId);
|
|
533
|
-
return row ? { score: row.score, rank: row.rank, total: this._centralityTotal } : null;
|
|
534
|
-
}
|
|
535
|
-
|
|
536
|
-
/** The most central symbols (A5), rank-ascending. @returns {Array<{chunk,score,rank}>} */
|
|
537
|
-
topCentral(limit = 20) {
|
|
538
|
-
if (!this._hasCentrality) return [];
|
|
539
|
-
const out = [];
|
|
540
|
-
for (const row of this._stmtTopCentral.all(limit)) {
|
|
541
|
-
const chunk = this.getChunk(row.chunk_id);
|
|
542
|
-
if (chunk) out.push({ chunk, score: row.score, rank: row.rank });
|
|
543
|
-
}
|
|
544
|
-
return out;
|
|
545
|
-
}
|
|
546
|
-
|
|
547
|
-
/** From-chunks of edges of a kept kind pointing at any of `defs`, id-sorted. */
|
|
548
|
-
_referrersByEdge(defs, keepKind) {
|
|
549
|
-
const ids = new Set();
|
|
550
|
-
for (const def of defs) for (const e of this._stmtEdgesIn.all(def.id)) if (keepKind(e.kind)) ids.add(e.from_chunk_id);
|
|
551
|
-
const out = [];
|
|
552
|
-
for (const id of [...ids].sort()) { const c = this.getChunk(id); if (c) out.push(c); }
|
|
553
|
-
return out;
|
|
554
|
-
}
|
|
555
|
-
|
|
556
|
-
findCallers(funcName) {
|
|
557
|
-
if (this._hasEdges) {
|
|
558
|
-
const defs = this.resolveSymbol(funcName);
|
|
559
|
-
if (defs.length > 0) return this._referrersByEdge(defs, k => k === 'calls');
|
|
560
|
-
}
|
|
561
|
-
return this._stmtCallers.all(funcName).map(r => this._rowToChunk(r));
|
|
562
|
-
}
|
|
563
|
-
|
|
564
|
-
findReferers(symbol) {
|
|
565
|
-
const key = String(symbol).toLowerCase().trim();
|
|
566
|
-
if (!key) return [];
|
|
567
|
-
if (this._hasEdges) {
|
|
568
|
-
const defs = this.resolveSymbol(symbol);
|
|
569
|
-
if (defs.length > 0) return this._referrersByEdge(defs, k => k === 'extends' || k === 'type');
|
|
570
|
-
}
|
|
571
|
-
// Escape LIKE metacharacters in the symbol, then bracket with JSON quotes so
|
|
572
|
-
// the prefilter only fires on a whole array element.
|
|
573
|
-
const esc = key.replace(/[\\%_]/g, m => '\\' + m);
|
|
574
|
-
const pat = `%"${esc}"%`;
|
|
575
|
-
const out = [];
|
|
576
|
-
for (const row of this._stmtReferers.all(pat, pat)) {
|
|
577
|
-
const c = this._rowToChunk(row);
|
|
578
|
-
// A class/type lists its own name in type_refs — that self-mention is a
|
|
579
|
-
// definition, not a reference; skip it.
|
|
580
|
-
if (c.name && c.name.toLowerCase() === key) continue;
|
|
581
|
-
const hit = c.type_refs.some(t => t.toLowerCase() === key)
|
|
582
|
-
|| c.extends.some(t => t.toLowerCase() === key);
|
|
583
|
-
if (hit) out.push(c);
|
|
584
|
-
}
|
|
585
|
-
return out;
|
|
586
|
-
}
|
|
587
|
-
|
|
588
|
-
/**
|
|
589
|
-
* HTTP routes filtered by method and/or path prefix, each augmented with its
|
|
590
|
-
* resolved handler chunk (`chunk`, or null). Filtering logic is byte-identical
|
|
591
|
-
* to MemoryGraphIndex.findRoutes; rows arrive in index-build order (ORDER BY id)
|
|
592
|
-
* so both backends return identical records. An index built before this feature
|
|
593
|
-
* (empty routes table) yields [].
|
|
594
|
-
* @returns {object[]}
|
|
595
|
-
*/
|
|
596
|
-
findRoutes({ method = null, pathPrefix = null } = {}) {
|
|
597
|
-
this._maybeRefresh();
|
|
598
|
-
const m = method ? String(method).toUpperCase().trim() : null;
|
|
599
|
-
const p = pathPrefix ? String(pathPrefix).toLowerCase() : null;
|
|
600
|
-
const out = [];
|
|
601
|
-
for (const r of this._stmtRoutes.all()) {
|
|
602
|
-
if (m && String(r.method || '').toUpperCase() !== m) continue;
|
|
603
|
-
if (p && !String(r.path || '').toLowerCase().startsWith(p)) continue;
|
|
604
|
-
const chunk = r.handler_chunk_id ? this.getChunk(r.handler_chunk_id) : null;
|
|
605
|
-
out.push({
|
|
606
|
-
method: r.method, path: r.path, handler_name: r.handler_name,
|
|
607
|
-
handler_chunk_id: r.handler_chunk_id, file_path: r.file_path,
|
|
608
|
-
line: r.line, framework: r.framework, chunk,
|
|
609
|
-
});
|
|
610
|
-
}
|
|
611
|
-
return out;
|
|
612
|
-
}
|
|
613
|
-
|
|
614
|
-
*iterateChunks() {
|
|
615
|
-
// Stream rows via a cursor so repo-map / call-graph passes never materialise
|
|
616
|
-
// the whole table. Falls back to paging if .iterate() is unavailable.
|
|
617
|
-
const stmt = this.db.prepare('SELECT * FROM chunks');
|
|
618
|
-
if (typeof stmt.iterate === 'function') {
|
|
619
|
-
for (const row of stmt.iterate()) yield this._rowToChunk(row);
|
|
620
|
-
return;
|
|
621
|
-
}
|
|
622
|
-
const page = this.db.prepare('SELECT * FROM chunks LIMIT ? OFFSET ?');
|
|
623
|
-
let offset = 0;
|
|
624
|
-
for (;;) {
|
|
625
|
-
const rows = page.all(1000, offset);
|
|
626
|
-
if (rows.length === 0) break;
|
|
627
|
-
for (const row of rows) yield this._rowToChunk(row);
|
|
628
|
-
offset += rows.length;
|
|
629
|
-
}
|
|
630
|
-
}
|
|
631
|
-
|
|
632
|
-
getDependencies(filePath) { return this.graph.dependencies[filePath] || []; }
|
|
633
|
-
getImportedBy(filePath) { return this.graph.importedBy[filePath] || []; }
|
|
634
|
-
|
|
635
|
-
chunkCount() { return this.db.prepare('SELECT COUNT(*) AS n FROM chunks').get().n; }
|
|
636
|
-
symbolCount() { return this.db.prepare('SELECT COUNT(DISTINCT name_lower) AS n FROM chunks WHERE name_lower IS NOT NULL').get().n; }
|
|
637
|
-
fileCount() { return this.db.prepare('SELECT COUNT(DISTINCT file_path) AS n FROM chunks').get().n; }
|
|
638
|
-
vectorCount() { return this.db.prepare('SELECT COUNT(*) AS n FROM chunks WHERE vec_offset >= 0').get().n; }
|
|
639
|
-
|
|
640
|
-
stats() {
|
|
641
|
-
this._maybeRefresh();
|
|
642
|
-
// Extension tally, streamed via a cursor so it never materialises all rows.
|
|
643
|
-
const extCounts = new Map();
|
|
644
|
-
const stmt = this.db.prepare('SELECT file_path FROM chunks');
|
|
645
|
-
const rows = typeof stmt.iterate === 'function' ? stmt.iterate() : stmt.all();
|
|
646
|
-
for (const r of rows) {
|
|
647
|
-
const ext = r.file_path.split('.').pop() || 'unknown';
|
|
648
|
-
extCounts.set(ext, (extCounts.get(ext) || 0) + 1);
|
|
649
|
-
}
|
|
650
|
-
const vectors = this.vectorCount();
|
|
651
|
-
return {
|
|
652
|
-
backend: 'sqlite',
|
|
653
|
-
chunks: this.chunkCount(),
|
|
654
|
-
files: this.fileCount(),
|
|
655
|
-
symbols: this.symbolCount(),
|
|
656
|
-
vectors,
|
|
657
|
-
hasVectors: vectors > 0,
|
|
658
|
-
lazyMode: true,
|
|
659
|
-
vectorSource: this._vecFd >= 0 ? 'sqlite + disk-backed .bin' : 'sqlite (no vectors)',
|
|
660
|
-
extCounts,
|
|
661
|
-
};
|
|
662
|
-
}
|
|
663
|
-
|
|
664
|
-
// ─── Vector access (pread from shared .embeddings.bin) ─────────────────────
|
|
665
|
-
|
|
666
|
-
_readVector(id) {
|
|
667
|
-
if (this._vecFd < 0) return null;
|
|
668
|
-
const row = this._stmtVec.get(id);
|
|
669
|
-
if (!row || row.vec_offset == null || row.vec_offset < 0 || !row.vec_dim) return null;
|
|
670
|
-
const byteLen = row.vec_dim * 4;
|
|
671
|
-
const raw = Buffer.allocUnsafe(byteLen);
|
|
672
|
-
try {
|
|
673
|
-
const read = fs.readSync(this._vecFd, raw, 0, byteLen, row.vec_offset);
|
|
674
|
-
if (read < byteLen) return null;
|
|
675
|
-
const vec = new Float32Array(row.vec_dim);
|
|
676
|
-
for (let d = 0; d < row.vec_dim; d++) vec[d] = raw.readFloatLE(d * 4);
|
|
677
|
-
return vec;
|
|
678
|
-
} catch { return null; }
|
|
679
|
-
}
|
|
680
|
-
|
|
681
|
-
/** Whether a vector for this embedding key is already stored (skip re-embedding). */
|
|
682
|
-
hasEmbedding(key) {
|
|
683
|
-
this._prepareWrite();
|
|
684
|
-
return Boolean(this._stmtVecByKey.get(key));
|
|
685
|
-
}
|
|
686
|
-
|
|
687
|
-
// ─── Hybrid search ─────────────────────────────────────────────────────────
|
|
688
|
-
|
|
689
|
-
searchHybrid(queryText, queryVector, topK = 5, minScore = 0.3, exactBoostName = null) {
|
|
690
|
-
this._maybeRefresh();
|
|
691
|
-
|
|
692
|
-
// Asymmetric stemming: stem only for natural-language queries so symbolic
|
|
693
|
-
// lookups stay exact — mirrors the in-memory engine for parity.
|
|
694
|
-
const qTokens = tokenize(queryText, isNaturalLanguageQuery(queryText));
|
|
695
|
-
const occ = new Map();
|
|
696
|
-
for (const t of qTokens) occ.set(t, (occ.get(t) || 0) + 1);
|
|
697
|
-
|
|
698
|
-
const lexScores = new Map();
|
|
699
|
-
for (const [token, n] of occ) {
|
|
700
|
-
const dfRow = this._stmtDf.get(token);
|
|
701
|
-
if (!dfRow || dfRow.df <= 0) continue;
|
|
702
|
-
const idf = okapiIdf(this._docCount, dfRow.df);
|
|
703
|
-
for (const row of this._stmtPosting.all(token)) {
|
|
704
|
-
const dl = row.doc_len ?? this._avgdl;
|
|
705
|
-
const s = bm25Score(idf, row.tf, dl, this._avgdl) * n;
|
|
706
|
-
lexScores.set(row.id, (lexScores.get(row.id) || 0) + s);
|
|
707
|
-
}
|
|
708
|
-
}
|
|
709
|
-
const lexicalResults = Array.from(lexScores.entries())
|
|
710
|
-
.sort((a, b) => (b[1] - a[1]) || (a[0] < b[0] ? -1 : 1)) // id tie-break: backend parity
|
|
711
|
-
.slice(0, LEXICAL_FUSION_CAP)
|
|
712
|
-
.map(([id, score], i) => ({ id, score, rank: i + 1 }));
|
|
713
|
-
|
|
714
|
-
// B3 learned-sparse channel: opt-in (model present) + NL-asymmetric. Shares scoreSparseExpanded
|
|
715
|
-
// with the in-memory engine over identically-ordered terms → byte-identical to memory. [] when
|
|
716
|
-
// inert (no model / symbolic query / no association) → fusion byte-identical to the default path.
|
|
717
|
-
const sparseResults = this._searchSparseExpanded(queryText);
|
|
718
|
-
|
|
719
|
-
// Stream the FULL embeddings bin (bounded buffers) so conceptual queries
|
|
720
|
-
// that share no tokens with the code can still match — scoring only lexical
|
|
721
|
-
// candidates silently disabled semantic search where it mattered most.
|
|
722
|
-
let vectorResults = [];
|
|
723
|
-
if (queryVector && this._vecFd >= 0) {
|
|
724
|
-
// Binary sketch above SKETCH_THRESHOLD (Hamming prefilter + exact
|
|
725
|
-
// rescore, O(candidates) disk reads); exact streaming scan below it.
|
|
726
|
-
const hits = this._sketch
|
|
727
|
-
? searchVectorSketch(this._sketch, { fd: this._vecFd }, queryVector, {
|
|
728
|
-
topN: VECTOR_SCAN_RAW_N, minScore,
|
|
729
|
-
})
|
|
730
|
-
: scanEmbeddingBinary({ fd: this._vecFd }, queryVector, {
|
|
731
|
-
topN: VECTOR_SCAN_RAW_N, minScore,
|
|
732
|
-
});
|
|
733
|
-
const entries = [];
|
|
734
|
-
for (const { key, score } of hits) {
|
|
735
|
-
// Summary-only (`<key>|s`) and per-window (`<key>|wN`) vectors fold onto
|
|
736
|
-
// the chunk; finalizeVectorCandidates keeps the max-sim score per chunk.
|
|
737
|
-
const baseKey = baseEmbeddingKey(key);
|
|
738
|
-
let rows = this._stmtIdsByKey.all(baseKey);
|
|
739
|
-
if (rows.length === 0) rows = this._stmtIdsByHash.all(baseKey); // legacy bins
|
|
740
|
-
for (const { id } of rows) entries.push({ id, score });
|
|
741
|
-
}
|
|
742
|
-
vectorResults = finalizeVectorCandidates(entries);
|
|
743
|
-
}
|
|
744
|
-
|
|
745
|
-
const rowCache = new Map();
|
|
746
|
-
const fetchRow = (id) => {
|
|
747
|
-
if (rowCache.has(id)) return rowCache.get(id);
|
|
748
|
-
const row = this._stmtChunk.get(id);
|
|
749
|
-
rowCache.set(id, row);
|
|
750
|
-
return row;
|
|
751
|
-
};
|
|
752
|
-
for (const r of lexicalResults) fetchRow(r.id);
|
|
753
|
-
for (const r of vectorResults) fetchRow(r.id);
|
|
754
|
-
|
|
755
|
-
const chunkCache = new Map();
|
|
756
|
-
const getChunk = (id) => {
|
|
757
|
-
if (chunkCache.has(id)) return chunkCache.get(id);
|
|
758
|
-
const c = this._rowToChunk(fetchRow(id));
|
|
759
|
-
chunkCache.set(id, c);
|
|
760
|
-
return c;
|
|
761
|
-
};
|
|
762
|
-
const pathCache = new Map();
|
|
763
|
-
const getPathTokens = (id) => {
|
|
764
|
-
if (pathCache.has(id)) return pathCache.get(id);
|
|
765
|
-
const row = fetchRow(id);
|
|
766
|
-
const set = row ? new Set(parseArr(row.path_tokens)) : undefined;
|
|
767
|
-
pathCache.set(id, set);
|
|
768
|
-
return set;
|
|
769
|
-
};
|
|
770
|
-
const dfCache = new Map();
|
|
771
|
-
const getDf = (t) => {
|
|
772
|
-
if (dfCache.has(t)) return dfCache.get(t);
|
|
773
|
-
const row = this._stmtDf.get(t);
|
|
774
|
-
const df = row ? row.df : 0;
|
|
775
|
-
dfCache.set(t, df);
|
|
776
|
-
return df;
|
|
777
|
-
};
|
|
778
|
-
|
|
779
|
-
return fuseAndRank({
|
|
780
|
-
lexicalResults, vectorResults, sparseResults, getChunk, getPathTokens, getDf,
|
|
781
|
-
docCount: this._docCount, rrfK: this.rrfK, topK, queryText, exactBoostName,
|
|
782
|
-
corpusEnriched: this._corpusEnriched,
|
|
783
|
-
resolveExact: (term) => this._stmtIdByName.all(term).map(r => r.id),
|
|
784
|
-
});
|
|
785
|
-
}
|
|
786
|
-
|
|
787
|
-
/**
|
|
788
|
-
* B3 — the learned-sparse expansion channel (NL queries, model present). Mirrors
|
|
789
|
-
* MemoryGraphIndex._searchSparseExpanded: identical expansion + the shared scoreSparseExpanded
|
|
790
|
-
* scorer over identically-ordered terms → byte-identical to memory. Returns [] when inert.
|
|
791
|
-
*/
|
|
792
|
-
_searchSparseExpanded(queryText) {
|
|
793
|
-
if (!this._hasSparseModel || !isNaturalLanguageQuery(queryText)) return [];
|
|
794
|
-
const model = this._loadSparseModel();
|
|
795
|
-
const termWeights = model && expandSparseQuery(queryText, model);
|
|
796
|
-
if (!termWeights) return [];
|
|
797
|
-
const avgdl = this._avgdl;
|
|
798
|
-
const docLenCache = new Map();
|
|
799
|
-
return scoreSparseExpanded(termWeights, {
|
|
800
|
-
docCount: this._docCount,
|
|
801
|
-
avgdl,
|
|
802
|
-
getDf: (t) => { const r = this._stmtDf.get(t); return r ? r.df : 0; },
|
|
803
|
-
// .map() materialises the postings AND populates the doc-len cache before scoring iterates,
|
|
804
|
-
// so getDocLen always resolves without a second query.
|
|
805
|
-
getPostings: (t) => this._stmtPosting.all(t).map(r => { docLenCache.set(r.id, r.doc_len); return [r.id, r.tf]; }),
|
|
806
|
-
getDocLen: (id) => docLenCache.get(id) ?? avgdl,
|
|
807
|
-
});
|
|
808
|
-
}
|
|
809
|
-
|
|
810
|
-
close() {
|
|
811
|
-
if (this._vecFd >= 0) { try { fs.closeSync(this._vecFd); } catch {} this._vecFd = -1; }
|
|
812
|
-
if (this.db) { try { this.db.close(); } catch {} this.db = null; }
|
|
813
|
-
}
|
|
814
|
-
|
|
815
|
-
// ─── Incremental writes (used by the watch daemon) ─────────────────────────
|
|
816
|
-
|
|
817
|
-
/** Internal: index one chunk's lexical document + row + call edges. */
|
|
818
|
-
_insertChunk(chunk, imports, vecEntry) {
|
|
819
|
-
const doc = buildLexicalDocument(chunk, imports);
|
|
820
|
-
const tokens = tokenize(doc); // raw + additive stems
|
|
821
|
-
const docLen = tokenize(doc, false).length; // raw-only length for BM25 avgdl (parity w/ memory engine)
|
|
822
|
-
|
|
823
|
-
const termCounts = new Map();
|
|
824
|
-
for (const t of tokens) termCounts.set(t, (termCounts.get(t) || 0) + 1);
|
|
825
|
-
for (const [term, cnt] of termCounts) {
|
|
826
|
-
this._stmtUpsertTerm.run(term);
|
|
827
|
-
this._stmtInsPost.run(term, chunk.id, cnt);
|
|
828
|
-
}
|
|
829
|
-
|
|
830
|
-
const pathTokens = Array.from(new Set(tokenize(chunk.file_path.replace(/[/\-_.]/g, ' '), false)));
|
|
831
|
-
this._stmtInsChunk.run(
|
|
832
|
-
chunk.id, chunk.file_path, chunk.node_type ?? null, chunk.name ?? null,
|
|
833
|
-
nameLowerOf(chunk.name), chunk.docstring ?? '', chunk.code_snippet ?? '',
|
|
834
|
-
chunk.content_hash ?? null, chunk.start_line ?? null, chunk.end_line ?? null,
|
|
835
|
-
jsonArr(chunk.calls), jsonArr(chunk.params), chunk.return_type ?? '',
|
|
836
|
-
chunk.class_context ?? '', jsonArr(chunk.type_refs), jsonArr(chunk.decorators),
|
|
837
|
-
jsonArr(chunk.extends), chunk.hyde ?? null, chunk.summary ?? null, jsonArr(chunk.concepts),
|
|
838
|
-
docLen, JSON.stringify(pathTokens),
|
|
839
|
-
chunk.content_hash ? embeddingKeyFor(chunk) : null,
|
|
840
|
-
vecEntry ? vecEntry.offset : -1, vecEntry ? vecEntry.dim : 0,
|
|
841
|
-
jsonArr(chunk.call_sites)
|
|
842
|
-
);
|
|
843
|
-
for (const callee of (chunk.calls || [])) this._stmtInsEdge.run(callee, chunk.id);
|
|
844
|
-
return docLen;
|
|
845
|
-
}
|
|
846
|
-
|
|
847
|
-
/**
|
|
848
|
-
* Atomically replace every chunk of one file — the incremental-update entry
|
|
849
|
-
* point used by the watch daemon. Mirrors MemoryGraphIndex.applyFileUpdate.
|
|
850
|
-
*
|
|
851
|
-
* New vectors are APPENDED to the shared .embeddings.bin (O(changed chunks),
|
|
852
|
-
* never a full rewrite); unchanged chunks reuse their existing bin offsets.
|
|
853
|
-
* All SQL runs in one WAL transaction, so concurrent readers see either the
|
|
854
|
-
* old or the new file state, never a partial one. BM25 document-frequency
|
|
855
|
-
* and length accounting is decremented from the chunk's actual posting rows,
|
|
856
|
-
* so incremental state matches what a full rebuild would produce.
|
|
857
|
-
*
|
|
858
|
-
* @param {string} filePath
|
|
859
|
-
* @param {object} p
|
|
860
|
-
* @param {object[]} [p.chunks]
|
|
861
|
-
* @param {string[]} [p.imports]
|
|
862
|
-
* @param {Map<string, Float32Array|number[]>} [p.embeddings] New vectors keyed by embeddingKeyFor(chunk).
|
|
863
|
-
*/
|
|
864
|
-
applyFileUpdate(filePath, { chunks = [], imports = [], embeddings = null } = {}) {
|
|
865
|
-
if (!this.db) this.load();
|
|
866
|
-
this._prepareWrite();
|
|
867
|
-
this._maybeRefresh();
|
|
868
|
-
|
|
869
|
-
// Append new vectors first — orphan bytes on a failed commit are harmless
|
|
870
|
-
// and compacted away by the next full index run.
|
|
871
|
-
let appended = new Map();
|
|
872
|
-
const embCount = embeddings ? (embeddings instanceof Map ? embeddings.size : Object.keys(embeddings).length) : 0;
|
|
873
|
-
if (embCount > 0) {
|
|
874
|
-
appended = appendEmbeddingBinary(this._embeddingPath, embeddings);
|
|
875
|
-
if (this._vecFd < 0) this._openVecFd();
|
|
876
|
-
if (!this._dim && appended.size > 0) this._dim = appended.values().next().value.dim;
|
|
877
|
-
// Keep our own sketch current (tail-only scan of what we just wrote).
|
|
878
|
-
if (this._sketch) {
|
|
879
|
-
try { this._sketch = updateVectorSketch(this._sketch, { fd: this._vecFd }); }
|
|
880
|
-
catch { this._sketch = null; }
|
|
881
|
-
}
|
|
882
|
-
}
|
|
883
|
-
|
|
884
|
-
this.db.exec('BEGIN');
|
|
885
|
-
try {
|
|
886
|
-
// A per-file edit makes the whole-program symbol graph (A4) stale; clear it so
|
|
887
|
-
// findCallers/findReferers fall back to the always-correct scan until the next
|
|
888
|
-
// full `idx-index` rebuilds it (the daemon never rebuilds the graph).
|
|
889
|
-
if (this._hasEdges) { this.db.exec('DELETE FROM edges'); this._hasEdges = false; }
|
|
890
|
-
if (this._hasCentrality) { this.db.exec('DELETE FROM centrality'); this._hasCentrality = false; this._centralityTotal = 0; }
|
|
891
|
-
if (this._hasTaint) { this.db.exec('DELETE FROM taint'); this._hasTaint = false; }
|
|
892
|
-
if (this._hasScipRefs) { this.db.exec('DELETE FROM scip_refs'); this._hasScipRefs = false; this._scipRefsObj = null; }
|
|
893
|
-
if (this._hasSparseModel) { this.db.exec('DELETE FROM sparse_model'); this._hasSparseModel = false; this._sparseModelObj = null; }
|
|
894
|
-
|
|
895
|
-
// Capture reusable vector offsets BEFORE deleting old rows, which may
|
|
896
|
-
// be the only rows that hold those offsets.
|
|
897
|
-
const reuse = new Map();
|
|
898
|
-
for (const chunk of chunks) {
|
|
899
|
-
if (!chunk.content_hash) continue;
|
|
900
|
-
const key = embeddingKeyFor(chunk);
|
|
901
|
-
if (appended.has(key) || reuse.has(key)) continue;
|
|
902
|
-
const row = this._stmtVecByKey.get(key) ?? this._stmtVecByHash.get(chunk.content_hash);
|
|
903
|
-
if (row) reuse.set(key, { offset: row.vec_offset, dim: row.vec_dim });
|
|
904
|
-
}
|
|
905
|
-
|
|
906
|
-
let docCount = this._docCount;
|
|
907
|
-
let totalDocLen = this._totalDocLen;
|
|
908
|
-
for (const row of this._stmtByFile.all(filePath)) {
|
|
909
|
-
for (const { term } of this._stmtPostByChunk.all(row.id)) {
|
|
910
|
-
this._stmtDecTerm.run(term);
|
|
911
|
-
}
|
|
912
|
-
this._stmtDelPostings.run(row.id);
|
|
913
|
-
this._stmtDelEdges.run(row.id);
|
|
914
|
-
this._stmtDelChunk.run(row.id);
|
|
915
|
-
totalDocLen -= (row.doc_len || 0);
|
|
916
|
-
docCount--;
|
|
917
|
-
}
|
|
918
|
-
|
|
919
|
-
for (const chunk of chunks) {
|
|
920
|
-
const key = chunk.content_hash ? embeddingKeyFor(chunk) : null;
|
|
921
|
-
const vecEntry = key ? (appended.get(key) ?? reuse.get(key) ?? null) : null;
|
|
922
|
-
totalDocLen += this._insertChunk(chunk, imports, vecEntry);
|
|
923
|
-
docCount++;
|
|
924
|
-
}
|
|
925
|
-
this._stmtPruneTerms.run();
|
|
926
|
-
|
|
927
|
-
this._stmtDelDeps.run(filePath);
|
|
928
|
-
if (!imports || imports.length === 0) this._stmtInsDep.run(filePath, null);
|
|
929
|
-
else for (const d of imports) this._stmtInsDep.run(filePath, d);
|
|
930
|
-
|
|
931
|
-
docCount = Math.max(0, docCount);
|
|
932
|
-
totalDocLen = Math.max(0, totalDocLen);
|
|
933
|
-
this._stmtInsMeta.run('doc_count', String(docCount));
|
|
934
|
-
this._stmtInsMeta.run('total_doc_len', String(totalDocLen));
|
|
935
|
-
this._stmtInsMeta.run('dim', String(this._dim));
|
|
936
|
-
this._stmtInsMeta.run('built_at', String(Date.now()));
|
|
937
|
-
this.db.exec('COMMIT');
|
|
938
|
-
|
|
939
|
-
this._docCount = docCount;
|
|
940
|
-
this._totalDocLen = totalDocLen;
|
|
941
|
-
this._avgdl = docCount > 0 ? totalDocLen / docCount : 1;
|
|
942
|
-
} catch (err) {
|
|
943
|
-
try { this.db.exec('ROLLBACK'); } catch { /* not in txn */ }
|
|
944
|
-
throw err;
|
|
945
|
-
}
|
|
946
|
-
|
|
947
|
-
const oldDeps = this._graph.dependencies[filePath] || [];
|
|
948
|
-
for (const oldDep of oldDeps) {
|
|
949
|
-
if (this._graph.importedBy[oldDep]) {
|
|
950
|
-
this._graph.importedBy[oldDep] = this._graph.importedBy[oldDep].filter(f => f !== filePath);
|
|
951
|
-
}
|
|
952
|
-
}
|
|
953
|
-
this._graph.dependencies[filePath] = imports || [];
|
|
954
|
-
for (const dep of (imports || [])) {
|
|
955
|
-
(this._graph.importedBy[dep] ||= []);
|
|
956
|
-
if (!this._graph.importedBy[dep].includes(filePath)) this._graph.importedBy[dep].push(filePath);
|
|
957
|
-
}
|
|
958
|
-
// Our own commit bumps data_version for other connections, not this one —
|
|
959
|
-
// re-read so the next _maybeRefresh() doesn't see a spurious change.
|
|
960
|
-
this._dataVersion = this._readDataVersion();
|
|
961
|
-
}
|
|
962
|
-
|
|
963
|
-
/** Remove a deleted file's chunks, postings, edges and graph node. */
|
|
964
|
-
removeFile(filePath) {
|
|
965
|
-
this.applyFileUpdate(filePath, { chunks: [], imports: [] });
|
|
966
|
-
// applyFileUpdate leaves a (file → null) deps row so the file stays a graph
|
|
967
|
-
// node; a deleted file should disappear from the graph entirely.
|
|
968
|
-
this._stmtDelDeps.run(filePath);
|
|
969
|
-
delete this._graph.dependencies[filePath];
|
|
970
|
-
this._dataVersion = this._readDataVersion();
|
|
971
|
-
}
|
|
972
|
-
|
|
973
|
-
// ─── Build (write path, used by the indexer) ───────────────────────────────
|
|
974
|
-
|
|
975
|
-
/**
|
|
976
|
-
* (Re)build the entire database from an in-memory index payload. Runs as a
|
|
977
|
-
* single transaction (concurrent readers see the old index until the commit);
|
|
978
|
-
* mirrors exactly how MemoryGraphIndex derives its BM25 posting lists,
|
|
979
|
-
* document lengths, symbol and call indexes, so the two backends stay
|
|
980
|
-
* rank-consistent.
|
|
981
|
-
*
|
|
982
|
-
* @param {object} payload
|
|
983
|
-
* @param {object[]} payload.chunks
|
|
984
|
-
* @param {{dependencies:Object<string,string[]>}} payload.graph
|
|
985
|
-
* @param {Map<string,Float32Array>|object} [payload.embeddingCache] Keyed by embeddingKeyFor(chunk).
|
|
986
|
-
*/
|
|
987
|
-
buildFrom({ chunks, graph, embeddingCache, edges = null, centrality = null, taint = null, scipRefs = null, sparseModel = null }) {
|
|
988
|
-
this._open();
|
|
989
|
-
this.db.exec('PRAGMA synchronous = OFF;'); // safe only for a full rebuild; not used for incremental writes
|
|
990
|
-
|
|
991
|
-
const offsets = new Map();
|
|
992
|
-
let dim = 0;
|
|
993
|
-
const cacheSize = embeddingCache instanceof Map ? embeddingCache.size : Object.keys(embeddingCache || {}).length;
|
|
994
|
-
if (cacheSize > 0) {
|
|
995
|
-
const buf = writeEmbeddingBinary(embeddingCache);
|
|
996
|
-
fs.writeFileSync(this._embeddingPath, buf);
|
|
997
|
-
let off = 0;
|
|
998
|
-
const count = buf.readUInt32LE(off); off += 4;
|
|
999
|
-
for (let i = 0; i < count; i++) {
|
|
1000
|
-
const hl = buf.readUInt32LE(off); off += 4;
|
|
1001
|
-
const key = buf.subarray(off, off + hl).toString('utf8'); off += hl;
|
|
1002
|
-
const d = buf.readUInt32LE(off); off += 4;
|
|
1003
|
-
offsets.set(key, { offset: off, dim: d });
|
|
1004
|
-
if (!dim) dim = d;
|
|
1005
|
-
off += d * 4;
|
|
1006
|
-
}
|
|
1007
|
-
}
|
|
1008
|
-
|
|
1009
|
-
const df = new Map();
|
|
1010
|
-
let totalDocLen = 0, docCount = 0;
|
|
1011
|
-
|
|
1012
|
-
this.db.exec('BEGIN');
|
|
1013
|
-
// Drop inside the transaction so concurrent readers see the old index
|
|
1014
|
-
// until COMMIT, never briefly-empty tables.
|
|
1015
|
-
this.db.exec(`
|
|
1016
|
-
DROP TABLE IF EXISTS chunks; DROP TABLE IF EXISTS postings;
|
|
1017
|
-
DROP TABLE IF EXISTS terms; DROP TABLE IF EXISTS call_edges;
|
|
1018
|
-
DROP TABLE IF EXISTS deps; DROP TABLE IF EXISTS meta;
|
|
1019
|
-
DROP TABLE IF EXISTS routes; DROP TABLE IF EXISTS edges;
|
|
1020
|
-
DROP TABLE IF EXISTS centrality; DROP TABLE IF EXISTS taint;
|
|
1021
|
-
DROP TABLE IF EXISTS scip_refs; DROP TABLE IF EXISTS sparse_model;
|
|
1022
|
-
`);
|
|
1023
|
-
this.db.exec(SCHEMA_TABLES);
|
|
1024
|
-
this.db.exec(SCHEMA_INDEXES);
|
|
1025
|
-
|
|
1026
|
-
const insChunk = this.db.prepare(
|
|
1027
|
-
`INSERT OR REPLACE INTO chunks (${CHUNK_COLS.join(', ')}) `
|
|
1028
|
-
+ `VALUES (${CHUNK_COLS.map(() => '?').join(', ')})`
|
|
1029
|
-
);
|
|
1030
|
-
const insPost = this.db.prepare('INSERT INTO postings (term, chunk_id, tf) VALUES (?, ?, ?)');
|
|
1031
|
-
const insEdge = this.db.prepare('INSERT INTO call_edges (callee, chunk_id) VALUES (?, ?)');
|
|
1032
|
-
|
|
1033
|
-
for (const chunk of chunks) {
|
|
1034
|
-
const deps = (graph?.dependencies?.[chunk.file_path]) || [];
|
|
1035
|
-
const doc = buildLexicalDocument(chunk, deps);
|
|
1036
|
-
const tokens = tokenize(doc); // raw + additive stems
|
|
1037
|
-
const docLen = tokenize(doc, false).length; // raw-only length for BM25 avgdl (parity w/ memory engine)
|
|
1038
|
-
totalDocLen += docLen; docCount++;
|
|
1039
|
-
|
|
1040
|
-
const termCounts = new Map();
|
|
1041
|
-
for (const t of tokens) termCounts.set(t, (termCounts.get(t) || 0) + 1);
|
|
1042
|
-
for (const [term, cnt] of termCounts) {
|
|
1043
|
-
df.set(term, (df.get(term) || 0) + 1);
|
|
1044
|
-
insPost.run(term, chunk.id, cnt);
|
|
1045
|
-
}
|
|
1046
|
-
|
|
1047
|
-
const pathTokens = Array.from(new Set(tokenize(chunk.file_path.replace(/[/\-_.]/g, ' '), false)));
|
|
1048
|
-
const vecKey = chunk.content_hash ? embeddingKeyFor(chunk) : null;
|
|
1049
|
-
const vec = vecKey ? (offsets.get(vecKey) ?? offsets.get(chunk.content_hash)) : null;
|
|
1050
|
-
|
|
1051
|
-
insChunk.run(
|
|
1052
|
-
chunk.id, chunk.file_path, chunk.node_type ?? null, chunk.name ?? null,
|
|
1053
|
-
nameLowerOf(chunk.name), chunk.docstring ?? '', chunk.code_snippet ?? '',
|
|
1054
|
-
chunk.content_hash ?? null, chunk.start_line ?? null, chunk.end_line ?? null,
|
|
1055
|
-
jsonArr(chunk.calls), jsonArr(chunk.params), chunk.return_type ?? '',
|
|
1056
|
-
chunk.class_context ?? '', jsonArr(chunk.type_refs), jsonArr(chunk.decorators),
|
|
1057
|
-
jsonArr(chunk.extends), chunk.hyde ?? null, chunk.summary ?? null, jsonArr(chunk.concepts),
|
|
1058
|
-
docLen, JSON.stringify(pathTokens), vecKey,
|
|
1059
|
-
vec ? vec.offset : -1, vec ? vec.dim : 0,
|
|
1060
|
-
jsonArr(chunk.call_sites)
|
|
1061
|
-
);
|
|
1062
|
-
for (const callee of (chunk.calls || [])) insEdge.run(callee, chunk.id);
|
|
1063
|
-
}
|
|
1064
|
-
|
|
1065
|
-
const insTerm = this.db.prepare('INSERT OR REPLACE INTO terms (term, df) VALUES (?, ?)');
|
|
1066
|
-
for (const [term, d] of df) insTerm.run(term, d);
|
|
1067
|
-
|
|
1068
|
-
const insDep = this.db.prepare('INSERT INTO deps (file, dep) VALUES (?, ?)');
|
|
1069
|
-
for (const [file, list] of Object.entries(graph?.dependencies || {})) {
|
|
1070
|
-
if (!list || list.length === 0) insDep.run(file, null);
|
|
1071
|
-
else for (const d of list) insDep.run(file, d);
|
|
1072
|
-
}
|
|
1073
|
-
|
|
1074
|
-
// Inserted in array order so the autoincrement id mirrors the in-memory
|
|
1075
|
-
// graph.routes order, keeping both backends rank-identical.
|
|
1076
|
-
const insRoute = this.db.prepare(
|
|
1077
|
-
'INSERT INTO routes (method, path, handler_name, handler_chunk_id, file_path, line, framework) '
|
|
1078
|
-
+ 'VALUES (?, ?, ?, ?, ?, ?, ?)'
|
|
1079
|
-
);
|
|
1080
|
-
for (const r of (graph?.routes || [])) {
|
|
1081
|
-
insRoute.run(
|
|
1082
|
-
String(r.method || ''), String(r.path ?? ''), r.handler_name ?? null,
|
|
1083
|
-
r.handler_chunk_id ?? null, String(r.file_path || ''),
|
|
1084
|
-
r.line ?? null, r.framework ?? null
|
|
1085
|
-
);
|
|
1086
|
-
}
|
|
1087
|
-
|
|
1088
|
-
// A4 resolved symbol graph (opt-in). Inserted in the deterministic order produced
|
|
1089
|
-
// by buildSymbolGraph so the table mirrors the in-memory edge list.
|
|
1090
|
-
if (Array.isArray(edges) && edges.length) {
|
|
1091
|
-
const insEdge = this.db.prepare(
|
|
1092
|
-
'INSERT INTO edges (from_chunk_id, to_chunk_id, kind, confidence) VALUES (?, ?, ?, ?)'
|
|
1093
|
-
);
|
|
1094
|
-
for (const e of edges) insEdge.run(e.from_chunk_id, e.to_chunk_id, e.kind, e.confidence);
|
|
1095
|
-
}
|
|
1096
|
-
|
|
1097
|
-
// A5 symbol centrality (opt-in). Precomputed score/rank per chunk id; both backends
|
|
1098
|
-
// read these exact numbers → parity-free. Inserted in rank order for tidy storage.
|
|
1099
|
-
if (centrality && typeof centrality === 'object') {
|
|
1100
|
-
const insCen = this.db.prepare('INSERT INTO centrality (chunk_id, score, rank) VALUES (?, ?, ?)');
|
|
1101
|
-
const rows = Object.entries(centrality).sort((a, b) => a[1].rank - b[1].rank);
|
|
1102
|
-
for (const [id, v] of rows) insCen.run(id, v.score, v.rank);
|
|
1103
|
-
}
|
|
1104
|
-
|
|
1105
|
-
// C2 serialized taint flows (opt-in). A single JSON payload (id = 0); both backends parse
|
|
1106
|
-
// the identical bytes → parity-free, exactly like edges/centrality.
|
|
1107
|
-
if (taint && typeof taint === 'object') {
|
|
1108
|
-
this.db.prepare('INSERT INTO taint (id, payload) VALUES (0, ?)').run(JSON.stringify(taint));
|
|
1109
|
-
}
|
|
1110
|
-
|
|
1111
|
-
// A2 v2 SCIP precise references (opt-in --resolver scip). A single JSON payload (id = 0) of
|
|
1112
|
-
// `{ defChunkId: [refererChunkId, …] }`; both backends parse identical bytes → parity-free,
|
|
1113
|
-
// exactly like taint. find_references reads it for the cross-file `resolved` reference tier.
|
|
1114
|
-
if (scipRefs && typeof scipRefs === 'object' && Object.keys(scipRefs).length) {
|
|
1115
|
-
this.db.prepare('INSERT INTO scip_refs (id, payload) VALUES (0, ?)').run(JSON.stringify(scipRefs));
|
|
1116
|
-
}
|
|
1117
|
-
|
|
1118
|
-
// B3 learned-sparse model (opt-in --learned-sparse). A single JSON payload (id = 0); both
|
|
1119
|
-
// backends parse identical bytes → parity-free, exactly like taint/scip_refs.
|
|
1120
|
-
if (sparseModel && typeof sparseModel === 'object' && sparseModel.assoc) {
|
|
1121
|
-
this.db.prepare('INSERT INTO sparse_model (id, payload) VALUES (0, ?)').run(JSON.stringify(sparseModel));
|
|
1122
|
-
}
|
|
1123
|
-
|
|
1124
|
-
const insMeta = this.db.prepare('INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)');
|
|
1125
|
-
insMeta.run('doc_count', String(docCount));
|
|
1126
|
-
insMeta.run('total_doc_len', String(totalDocLen));
|
|
1127
|
-
insMeta.run('dim', String(dim));
|
|
1128
|
-
insMeta.run('built_at', String(Date.now()));
|
|
1129
|
-
this.db.exec('COMMIT');
|
|
1130
|
-
|
|
1131
|
-
this.db.close();
|
|
1132
|
-
this.db = null;
|
|
1133
|
-
this._writeReady = false;
|
|
1134
|
-
return { chunks: docCount, terms: df.size, dim };
|
|
1135
|
-
}
|
|
1136
|
-
}
|