@esneiderbravo/speclaw 2.0.9 → 2.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/dist/cli/commands/query.js +12 -2
- package/dist/cli/commands/reindex-file.js +146 -0
- package/dist/cli/commands/update.js +32 -0
- package/dist/cli/index.js +7 -3
- package/dist/cli/lib/help.js +19 -0
- package/dist/modules/compass/affected-config.js +5 -0
- package/dist/modules/compass/affected.js +344 -27
- package/dist/modules/compass/db.js +149 -16
- package/dist/modules/compass/diff-context.js +4 -2
- package/dist/modules/compass/explore-rich.js +14 -7
- package/dist/modules/compass/extract.js +163 -4
- package/dist/modules/compass/hotspots.js +11 -6
- package/dist/modules/compass/hybrid.js +9 -4
- package/dist/modules/compass/indexer.js +898 -202
- package/dist/modules/compass/map.js +2 -2
- package/dist/modules/compass/query.js +70 -24
- package/dist/modules/compass/register.js +1 -1
- package/dist/modules/compass/visualize.js +5 -3
- package/dist/modules/foundation/assets/docs/compass.template.md +22 -2
- package/dist/modules/foundation/hooks.js +48 -10
- package/dist/modules/lawbook/drift.js +2 -2
- package/dist/modules/lawbook/investigate.js +2 -1
- package/dist/shared/output-budget.js +2 -0
- package/dist/shared/paths.js +24 -0
- package/package.json +1 -1
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { createHash } from "node:crypto";
|
|
4
|
-
import { openDb, clearNeedsReindex, needsReindex } from "./db.js";
|
|
4
|
+
import { openDb, openCurrentDb, isCurrentIndex, clearNeedsReindex, needsReindex, postPending, setPostPending, clearPostPending, FILE_NODE_KIND, } from "./db.js";
|
|
5
5
|
import { langForPath } from "./languages.js";
|
|
6
|
-
import { extract } from "./extract.js";
|
|
6
|
+
import { extract, importSpecifier } from "./extract.js";
|
|
7
7
|
import { getEmbedder, toBlob } from "./embedder.js";
|
|
8
8
|
import { contentHashFor, defaultEmbedText } from "./embed-input.js";
|
|
9
|
-
import { buildDirHashMap } from "./merkle.js";
|
|
10
|
-
import { loadAffectedConfig, isTestPath, inferModule } from "./affected-config.js";
|
|
9
|
+
import { buildDirHashMap, dirHash, HASH_EMPTY } from "./merkle.js";
|
|
10
|
+
import { loadAffectedConfig, isTestPath, inferModule, } from "./affected-config.js";
|
|
11
|
+
import { realPathOf } from "../../shared/paths.js";
|
|
11
12
|
import { personalizedPageRank, edgeWeightMul } from "./pagerank.js";
|
|
12
13
|
const SKIP_DIRS = new Set([
|
|
13
14
|
".git",
|
|
@@ -30,112 +31,500 @@ const SKIP_DIRS = new Set([
|
|
|
30
31
|
".cache",
|
|
31
32
|
]);
|
|
32
33
|
const MAX_FILE_BYTES = 1_500_000;
|
|
34
|
+
/**
|
|
35
|
+
* Whether a directory or file name inside the project is walked by a full index.
|
|
36
|
+
* The single source of the skip rule shared by the walk and per-file reindex.
|
|
37
|
+
*/
|
|
38
|
+
function isSkippedDir(name) {
|
|
39
|
+
return SKIP_DIRS.has(name);
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* `p` with symlinks resolved and, where the platform reports it (macOS,
|
|
43
|
+
* Windows), each existing segment spelled as stored on disk. A missing tail
|
|
44
|
+
* keeps its given spelling. Falls back to {@link realPathOf} when the native
|
|
45
|
+
* resolver fails.
|
|
46
|
+
*
|
|
47
|
+
* @param p - An absolute path.
|
|
48
|
+
*/
|
|
49
|
+
function canonicalPathOf(p) {
|
|
50
|
+
const tail = [];
|
|
51
|
+
let cur = path.resolve(p);
|
|
52
|
+
for (;;) {
|
|
53
|
+
try {
|
|
54
|
+
return path.join(fs.realpathSync.native(cur), ...tail.reverse());
|
|
55
|
+
}
|
|
56
|
+
catch (err) {
|
|
57
|
+
const code = err.code;
|
|
58
|
+
if (code !== "ENOENT" && code !== "ENOTDIR")
|
|
59
|
+
return realPathOf(p);
|
|
60
|
+
const parent = path.dirname(cur);
|
|
61
|
+
if (parent === cur)
|
|
62
|
+
return realPathOf(p);
|
|
63
|
+
tail.push(path.basename(cur));
|
|
64
|
+
cur = parent;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Classify one path against the full walk's file-set rules, so a per-file
|
|
70
|
+
* reindex and a full index agree on which files are indexed (and the directory
|
|
71
|
+
* Merkle tree stays the hash of the same file set).
|
|
72
|
+
*
|
|
73
|
+
* A path is eligible when it lies inside `projectPath` (symlinks in its existing
|
|
74
|
+
* prefix resolved, and its spelling taken from the filesystem where the platform
|
|
75
|
+
* reports it, so a case variant on a case-insensitive volume yields the stored
|
|
76
|
+
* spelling), no directory segment is a skipped directory, its extension
|
|
77
|
+
* maps to an indexed language, it is a regular file, and it does not exceed the
|
|
78
|
+
* walk's size cap. A path that does not exist yields `missing` with its
|
|
79
|
+
* repo-relative form, so the caller can drop a deleted file's rows.
|
|
80
|
+
*
|
|
81
|
+
* @param projectPath - Absolute project root.
|
|
82
|
+
* @param absPath - Absolute path to classify.
|
|
83
|
+
*/
|
|
84
|
+
export function classifyIndexPath(projectPath, absPath) {
|
|
85
|
+
const relNative = path.relative(canonicalPathOf(projectPath), canonicalPathOf(absPath));
|
|
86
|
+
if (!relNative || relNative.startsWith("..") || path.isAbsolute(relNative)) {
|
|
87
|
+
return { eligible: false, rel: null, reason: "outside" };
|
|
88
|
+
}
|
|
89
|
+
const rel = relNative.split(path.sep).join("/");
|
|
90
|
+
const parts = rel.split("/");
|
|
91
|
+
if (parts.slice(0, -1).some(isSkippedDir))
|
|
92
|
+
return { eligible: false, rel, reason: "ignored" };
|
|
93
|
+
const lang = langForPath(rel);
|
|
94
|
+
if (!lang)
|
|
95
|
+
return { eligible: false, rel, reason: "language" };
|
|
96
|
+
let stat;
|
|
97
|
+
try {
|
|
98
|
+
stat = fs.statSync(path.join(projectPath, rel));
|
|
99
|
+
}
|
|
100
|
+
catch {
|
|
101
|
+
return { eligible: false, rel, reason: "missing" };
|
|
102
|
+
}
|
|
103
|
+
if (!stat.isFile())
|
|
104
|
+
return { eligible: false, rel, reason: "ignored" };
|
|
105
|
+
if (stat.size > MAX_FILE_BYTES)
|
|
106
|
+
return { eligible: false, rel, reason: "size" };
|
|
107
|
+
return { eligible: true, rel, lang, size: stat.size, mtimeMs: Math.trunc(stat.mtimeMs) };
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Whether a full walk of `projectPath` would yield exactly `rel`: every segment
|
|
111
|
+
* is listed by its parent directory with exactly that spelling (a case-sensitive
|
|
112
|
+
* compare, so a case-only rename on a case-insensitive volume does not match),
|
|
113
|
+
* every directory segment is a real, non-skipped directory and the leaf a
|
|
114
|
+
* regular file (no segment is a symlink, as the walk never follows one), and
|
|
115
|
+
* {@link classifyIndexPath} finds it eligible under the same `rel`.
|
|
116
|
+
*
|
|
117
|
+
* A full run keeps a stored row its walk did not see only when this holds;
|
|
118
|
+
* any other row would duplicate the file's symbols under a second path.
|
|
119
|
+
*
|
|
120
|
+
* @param projectPath - Absolute project root (as walked).
|
|
121
|
+
* @param rel - Repo-relative path with `/` separators.
|
|
122
|
+
* @param listings - Optional cache of directory listings, keyed by absolute
|
|
123
|
+
* directory, shared across calls in one run.
|
|
124
|
+
*/
|
|
125
|
+
export function walkWouldYield(projectPath, rel, listings = new Map()) {
|
|
126
|
+
const parts = rel.split("/");
|
|
127
|
+
if (parts.some((p) => p === "" || p === "." || p === ".."))
|
|
128
|
+
return false;
|
|
129
|
+
let dir = projectPath;
|
|
130
|
+
for (let i = 0; i < parts.length; i++) {
|
|
131
|
+
const name = parts[i];
|
|
132
|
+
let names = listings.get(dir);
|
|
133
|
+
if (names === undefined) {
|
|
134
|
+
try {
|
|
135
|
+
names = new Set(fs.readdirSync(dir));
|
|
136
|
+
}
|
|
137
|
+
catch {
|
|
138
|
+
names = null;
|
|
139
|
+
}
|
|
140
|
+
listings.set(dir, names);
|
|
141
|
+
}
|
|
142
|
+
if (!names?.has(name))
|
|
143
|
+
return false;
|
|
144
|
+
const full = path.join(dir, name);
|
|
145
|
+
let stat;
|
|
146
|
+
try {
|
|
147
|
+
stat = fs.lstatSync(full);
|
|
148
|
+
}
|
|
149
|
+
catch {
|
|
150
|
+
return false;
|
|
151
|
+
}
|
|
152
|
+
const leaf = i === parts.length - 1;
|
|
153
|
+
if (leaf ? !stat.isFile() : !stat.isDirectory() || isSkippedDir(name))
|
|
154
|
+
return false;
|
|
155
|
+
dir = full;
|
|
156
|
+
}
|
|
157
|
+
const c = classifyIndexPath(projectPath, path.join(projectPath, rel));
|
|
158
|
+
return c.eligible && c.rel === rel;
|
|
159
|
+
}
|
|
33
160
|
function hashOf(content) {
|
|
34
161
|
return createHash("sha256").update(content).digest("hex");
|
|
35
162
|
}
|
|
36
163
|
const DEFAULT_MAX_CACHE_MB = 256;
|
|
37
164
|
const DEFAULT_RETENTION_DAYS = 30;
|
|
38
165
|
/**
|
|
39
|
-
*
|
|
40
|
-
*
|
|
166
|
+
* Call names that are ambient globals in common runtimes and test frameworks.
|
|
167
|
+
* A plain call to one of them binds only to a definition in the same file, so
|
|
168
|
+
* a project symbol that happens to share the name never collects every test
|
|
169
|
+
* file as a caller.
|
|
170
|
+
*/
|
|
171
|
+
export const BUILTIN_GLOBALS = [
|
|
172
|
+
"describe",
|
|
173
|
+
"it",
|
|
174
|
+
"test",
|
|
175
|
+
"expect",
|
|
176
|
+
"beforeEach",
|
|
177
|
+
"afterEach",
|
|
178
|
+
"beforeAll",
|
|
179
|
+
"afterAll",
|
|
180
|
+
"setTimeout",
|
|
181
|
+
"setInterval",
|
|
182
|
+
"clearTimeout",
|
|
183
|
+
"clearInterval",
|
|
184
|
+
"queueMicrotask",
|
|
185
|
+
"parseInt",
|
|
186
|
+
"parseFloat",
|
|
187
|
+
"require",
|
|
188
|
+
"structuredClone",
|
|
189
|
+
"fetch",
|
|
190
|
+
"print",
|
|
191
|
+
"len",
|
|
192
|
+
"isinstance",
|
|
193
|
+
"super",
|
|
194
|
+
];
|
|
195
|
+
/** {@link BUILTIN_GLOBALS} as a quoted SQL `IN (...)` list. */
|
|
196
|
+
export const BUILTIN_SQL_LIST = BUILTIN_GLOBALS.map((n) => `'${n}'`).join(", ");
|
|
197
|
+
/** Most bind parameters a scoped {@link resolveEdges} call uses before running unscoped. */
|
|
198
|
+
const MAX_SCOPE_PARAMS = 10_000;
|
|
199
|
+
/**
|
|
200
|
+
* Resolve call and import edges to node ids — the one place edge resolution
|
|
201
|
+
* lives, shared by the full index run and incremental re-indexing.
|
|
202
|
+
*
|
|
203
|
+
* - Edges whose `dst_node_id` points at a node that no longer exists (its file
|
|
204
|
+
* was re-extracted) are reset to unresolved first.
|
|
205
|
+
* - Imports are resolved first (below). A call's "import target" is the file
|
|
206
|
+
* the same-file import edge with the call's `spec` resolved to.
|
|
207
|
+
* - Calls with `is_member = 1` are never resolved by name. Calls with
|
|
208
|
+
* `is_member = 2` (a member call on an import binding) resolve only when their
|
|
209
|
+
* import target exists: an import that no relative/alias/baseUrl resolution
|
|
210
|
+
* maps to a project file is a package, so `path.parse()` stays foreign.
|
|
211
|
+
* - A call to a {@link BUILTIN_GLOBALS} name resolves only to a definition in
|
|
212
|
+
* the same file or in its import target (`import { fetch } from "./http"`).
|
|
213
|
+
* - Otherwise a call prefers its import target, then a definition under the
|
|
214
|
+
* import target's directory (a barrel's subtree), then a same-file
|
|
215
|
+
* definition, else the lowest-id node with that name. File-owner nodes are
|
|
216
|
+
* never call targets. An import of a symbol-less or re-exporting file (a
|
|
217
|
+
* barrel) resolves to that file's file-owner node, so member calls through
|
|
218
|
+
* it bind by name like this.
|
|
219
|
+
* - Imports resolve their specifier relative to the importing file, then
|
|
220
|
+
* through the nearest `tsconfig.json`/`jsconfig.json` `paths`/`baseUrl`, and
|
|
221
|
+
* point at the target file's file-owner node when it has one, otherwise at
|
|
222
|
+
* its first node by `start_line`.
|
|
223
|
+
*
|
|
224
|
+
* The project root is read from the database's own path
|
|
225
|
+
* (`<root>/.speclaw/index.db`), so callers pass only the connection.
|
|
226
|
+
*
|
|
227
|
+
* @param db - Open index database (the caller owns the transaction).
|
|
228
|
+
* @param fileIds - When set, only calls owned by these files or by files whose
|
|
229
|
+
* import this pass resolved, and unresolved calls whose name a node in these
|
|
230
|
+
* files defines; unresolved imports are always all processed (a new file may
|
|
231
|
+
* satisfy any of them). When omitted, every
|
|
232
|
+
* unresolved edge.
|
|
233
|
+
* @returns How many call and import edges this call resolved.
|
|
234
|
+
*/
|
|
235
|
+
export function resolveEdges(db, fileIds) {
|
|
236
|
+
// Covers: req~impact-id-first~1
|
|
237
|
+
db.exec(`UPDATE edges SET dst_node_id = NULL
|
|
238
|
+
WHERE dst_node_id IS NOT NULL AND NOT EXISTS (SELECT 1 FROM nodes n WHERE n.id = edges.dst_node_id)`);
|
|
239
|
+
const scoped = fileIds !== undefined;
|
|
240
|
+
const ids = fileIds ?? [];
|
|
241
|
+
const idList = ids.map(() => "?").join(", ") || "NULL";
|
|
242
|
+
const resolvedCalls = () => Number(db
|
|
243
|
+
.prepare("SELECT COUNT(*) AS n FROM edges WHERE kind = 'call' AND dst_node_id IS NOT NULL")
|
|
244
|
+
.get().n);
|
|
245
|
+
// Imports first: whether a call on an import binding may bind depends on
|
|
246
|
+
// whether that binding's import resolved to a project file.
|
|
247
|
+
const { resolved: imports, srcFiles } = resolveImportEdges(db, projectRootOf(db));
|
|
248
|
+
const before = resolvedCalls();
|
|
249
|
+
// A file whose import just resolved may have pending is_member = 2 calls.
|
|
250
|
+
// Past a bound (SQLite caps bind parameters) the pass is simply unscoped.
|
|
251
|
+
const importScope = [...srcFiles];
|
|
252
|
+
const importList = importScope.map(() => "?").join(", ") || "NULL";
|
|
253
|
+
const narrow = scoped && 2 * ids.length + importScope.length <= MAX_SCOPE_PARAMS;
|
|
254
|
+
const callScope = narrow
|
|
255
|
+
? `AND (src_file_id IN (${idList}) OR src_file_id IN (${importList})
|
|
256
|
+
OR dst_name IN (SELECT name FROM nodes WHERE file_id IN (${idList}) AND kind <> '${FILE_NODE_KIND}'))`
|
|
257
|
+
: "";
|
|
258
|
+
// File of the node the same-file import with this call's spec resolved to;
|
|
259
|
+
// NULL when the call has no spec (the subquery is skipped) or that import
|
|
260
|
+
// stayed unresolved (a package).
|
|
261
|
+
const importTarget = `(CASE WHEN edges.spec IS NULL THEN NULL ELSE (
|
|
262
|
+
SELECT t.file_id FROM edges i JOIN nodes t ON t.id = i.dst_node_id
|
|
263
|
+
WHERE i.src_file_id = edges.src_file_id AND i.kind = 'import' AND i.spec = edges.spec
|
|
264
|
+
ORDER BY i.id LIMIT 1) END)`;
|
|
265
|
+
// Candidate n lies under the import target's directory: a barrel
|
|
266
|
+
// (`libs/core/src/index.ts`) usually re-exports its own subtree, so this
|
|
267
|
+
// stands in for "reachable through the barrel's re-exports" without storing
|
|
268
|
+
// them. rtrim(p, <p without '/'>) is p's directory with its trailing slash.
|
|
269
|
+
const underTargetDir = `EXISTS (
|
|
270
|
+
SELECT 1 FROM files tf, files nf
|
|
271
|
+
WHERE tf.id = ${importTarget} AND nf.id = n.file_id
|
|
272
|
+
AND substr(nf.path, 1, length(rtrim(tf.path, replace(tf.path, '/', ''))))
|
|
273
|
+
= rtrim(tf.path, replace(tf.path, '/', '')))`;
|
|
274
|
+
db.prepare(`UPDATE edges SET dst_node_id = (
|
|
275
|
+
SELECT n.id FROM nodes n
|
|
276
|
+
WHERE n.name = edges.dst_name
|
|
277
|
+
AND n.kind <> '${FILE_NODE_KIND}'
|
|
278
|
+
AND (edges.dst_name NOT IN (${BUILTIN_SQL_LIST}) OR n.file_id = edges.src_file_id
|
|
279
|
+
OR n.file_id = ${importTarget})
|
|
280
|
+
ORDER BY CASE WHEN n.file_id = ${importTarget} THEN 0
|
|
281
|
+
WHEN edges.spec IS NOT NULL AND ${underTargetDir} THEN 1
|
|
282
|
+
WHEN n.file_id = edges.src_file_id THEN 2 ELSE 3 END, n.id
|
|
283
|
+
LIMIT 1
|
|
284
|
+
)
|
|
285
|
+
WHERE kind = 'call' AND dst_node_id IS NULL
|
|
286
|
+
AND (is_member = 0
|
|
287
|
+
OR (is_member = 2 AND edges.spec IS NOT NULL AND ${importTarget} IS NOT NULL))
|
|
288
|
+
${callScope}`).run(...(narrow ? [...ids, ...importScope, ...ids] : []));
|
|
289
|
+
const calls = resolvedCalls() - before;
|
|
290
|
+
return { calls, imports };
|
|
291
|
+
}
|
|
292
|
+
/** Project root of an index database stored at `<root>/.speclaw/index.db`. */
|
|
293
|
+
function projectRootOf(db) {
|
|
294
|
+
const row = db.prepare("SELECT file FROM pragma_database_list WHERE name = 'main'").get();
|
|
295
|
+
return path.dirname(path.dirname(row?.file ?? ""));
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* Point every unresolved import edge at a representative node of the imported
|
|
299
|
+
* file (its file-owner node, else its first node) so reverse reachability walks
|
|
300
|
+
* file-level dependencies, not just calls. Not narrowed by file: a newly
|
|
301
|
+
* indexed file can satisfy an import anywhere.
|
|
302
|
+
*
|
|
303
|
+
* @returns How many import edges were resolved, and the importing files' ids.
|
|
41
304
|
*/
|
|
42
305
|
function resolveImportEdges(db, projectPath) {
|
|
306
|
+
// Covers: req~import-resolution~1
|
|
43
307
|
const files = db.prepare("SELECT id, path FROM files").all();
|
|
44
308
|
const byNorm = new Map();
|
|
45
|
-
for (const f of files)
|
|
309
|
+
for (const f of files)
|
|
46
310
|
byNorm.set(f.path.split("\\").join("/"), f.id);
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
const namedNode = db.prepare("SELECT id FROM nodes WHERE file_id = ? AND name = ? ORDER BY id ASC LIMIT 1");
|
|
311
|
+
const target = db.prepare(`SELECT id FROM nodes WHERE file_id = ?
|
|
312
|
+
ORDER BY kind = '${FILE_NODE_KIND}' DESC, start_line ASC, id ASC LIMIT 1`);
|
|
50
313
|
const upd = db.prepare("UPDATE edges SET dst_node_id = ? WHERE id = ?");
|
|
51
314
|
const imports = db
|
|
52
|
-
.prepare(`SELECT e.id, e.dst_name, e.src_file_id, f.path AS src_path
|
|
315
|
+
.prepare(`SELECT e.id, e.dst_name, e.spec, e.src_file_id, f.path AS src_path
|
|
53
316
|
FROM edges e JOIN files f ON f.id = e.src_file_id
|
|
54
317
|
WHERE e.kind = 'import' AND e.dst_node_id IS NULL`)
|
|
55
318
|
.all();
|
|
319
|
+
const aliases = new AliasResolver(projectPath);
|
|
320
|
+
let resolved = 0;
|
|
321
|
+
const srcFiles = new Set();
|
|
56
322
|
for (const edge of imports) {
|
|
57
|
-
const spec =
|
|
323
|
+
const spec = edge.spec ?? importSpecifier(edge.dst_name);
|
|
58
324
|
if (!spec)
|
|
59
325
|
continue;
|
|
60
|
-
const targetRel = resolveImportPath(projectPath, edge.src_path, spec
|
|
326
|
+
const targetRel = resolveImportPath(projectPath, edge.src_path, spec, byNorm, aliases);
|
|
61
327
|
if (!targetRel)
|
|
62
328
|
continue;
|
|
63
329
|
const fileId = byNorm.get(targetRel);
|
|
64
330
|
if (fileId === undefined)
|
|
65
331
|
continue;
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
332
|
+
const row = target.get(fileId);
|
|
333
|
+
if (!row)
|
|
334
|
+
continue;
|
|
335
|
+
upd.run(row.id, edge.id);
|
|
336
|
+
resolved++;
|
|
337
|
+
srcFiles.add(edge.src_file_id);
|
|
338
|
+
}
|
|
339
|
+
return { resolved, srcFiles };
|
|
340
|
+
}
|
|
341
|
+
/** Repo-relative candidate files for an import target (extension and index probing). */
|
|
342
|
+
function importCandidates(projectPath, absBase) {
|
|
343
|
+
const rel = path.relative(projectPath, absBase).split(path.sep).join("/");
|
|
344
|
+
if (rel.startsWith("..") || path.isAbsolute(rel))
|
|
345
|
+
return [];
|
|
346
|
+
return [
|
|
347
|
+
rel,
|
|
348
|
+
rel.replace(/\.js$/, ".ts"),
|
|
349
|
+
rel.replace(/\.js$/, ".tsx"),
|
|
350
|
+
rel.replace(/\.jsx$/, ".tsx"),
|
|
351
|
+
rel.replace(/\.mjs$/, ".mts"),
|
|
352
|
+
rel.replace(/\.cjs$/, ".cts"),
|
|
353
|
+
`${rel}.ts`,
|
|
354
|
+
`${rel}.tsx`,
|
|
355
|
+
`${rel}.js`,
|
|
356
|
+
`${rel}.jsx`,
|
|
357
|
+
`${rel}.mjs`,
|
|
358
|
+
`${rel}.cjs`,
|
|
359
|
+
`${rel}/index.ts`,
|
|
360
|
+
`${rel}/index.tsx`,
|
|
361
|
+
`${rel}/index.js`,
|
|
362
|
+
];
|
|
363
|
+
}
|
|
364
|
+
/**
|
|
365
|
+
* Resolve an import specifier to an indexed repo-relative path: relative
|
|
366
|
+
* specifiers against the importing file (including `.js` → `.ts`), then
|
|
367
|
+
* tsconfig/jsconfig `paths` aliases and bare `baseUrl` lookups. Bare package
|
|
368
|
+
* names that match no project file stay unresolved (null).
|
|
369
|
+
*/
|
|
370
|
+
function resolveImportPath(projectPath, srcRel, spec, indexed, aliases) {
|
|
371
|
+
const firstIndexed = (absBase) => importCandidates(projectPath, absBase).find((c) => indexed.has(c)) ?? null;
|
|
372
|
+
const srcDir = path.dirname(path.join(projectPath, srcRel));
|
|
373
|
+
if (spec.startsWith(".") || spec.startsWith("/")) {
|
|
374
|
+
return firstIndexed(path.resolve(srcDir, spec));
|
|
375
|
+
}
|
|
376
|
+
for (const absBase of aliases.candidates(srcDir, spec)) {
|
|
377
|
+
const hit = firstIndexed(absBase);
|
|
378
|
+
if (hit)
|
|
379
|
+
return hit;
|
|
380
|
+
}
|
|
381
|
+
return null;
|
|
382
|
+
}
|
|
383
|
+
/**
|
|
384
|
+
* Best-effort tsconfig/jsconfig alias lookup, cached per directory for one
|
|
385
|
+
* resolution pass. Never throws: an unreadable or malformed config disables
|
|
386
|
+
* alias resolution for the files under it, and relative imports still resolve.
|
|
387
|
+
*/
|
|
388
|
+
class AliasResolver {
|
|
389
|
+
root;
|
|
390
|
+
byDir = new Map();
|
|
391
|
+
constructor(root) {
|
|
392
|
+
this.root = root;
|
|
393
|
+
}
|
|
394
|
+
/** Absolute import bases to probe for a non-relative specifier, in order. */
|
|
395
|
+
candidates(fromDir, spec) {
|
|
396
|
+
const cfg = this.configFor(fromDir);
|
|
397
|
+
if (!cfg)
|
|
398
|
+
return [];
|
|
399
|
+
const out = [];
|
|
400
|
+
for (const { pattern, targets } of cfg.paths) {
|
|
401
|
+
const star = pattern.indexOf("*");
|
|
402
|
+
let captured = null;
|
|
403
|
+
if (star < 0) {
|
|
404
|
+
if (spec === pattern)
|
|
405
|
+
captured = "";
|
|
406
|
+
}
|
|
407
|
+
else {
|
|
408
|
+
const pre = pattern.slice(0, star);
|
|
409
|
+
const post = pattern.slice(star + 1);
|
|
410
|
+
if (spec.startsWith(pre) &&
|
|
411
|
+
spec.endsWith(post) &&
|
|
412
|
+
spec.length >= pre.length + post.length) {
|
|
413
|
+
captured = spec.slice(pre.length, spec.length - post.length);
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
if (captured === null)
|
|
417
|
+
continue;
|
|
418
|
+
for (const t of targets) {
|
|
419
|
+
out.push(path.resolve(cfg.baseUrl ?? cfg.pathsBase, t.replace("*", captured)));
|
|
72
420
|
}
|
|
73
421
|
}
|
|
74
|
-
if (
|
|
75
|
-
|
|
76
|
-
|
|
422
|
+
if (cfg.baseUrl)
|
|
423
|
+
out.push(path.resolve(cfg.baseUrl, spec));
|
|
424
|
+
return out;
|
|
425
|
+
}
|
|
426
|
+
/** Nearest tsconfig/jsconfig at or above `dir`, bounded by the project root. */
|
|
427
|
+
configFor(dir) {
|
|
428
|
+
const cached = this.byDir.get(dir);
|
|
429
|
+
if (cached !== undefined)
|
|
430
|
+
return cached;
|
|
431
|
+
let result = null;
|
|
432
|
+
const rel = path.relative(this.root, dir);
|
|
433
|
+
if (!rel.startsWith("..") && !path.isAbsolute(rel)) {
|
|
434
|
+
const own = ["tsconfig.json", "jsconfig.json"]
|
|
435
|
+
.map((n) => path.join(dir, n))
|
|
436
|
+
.find((f) => fs.existsSync(f));
|
|
437
|
+
if (own)
|
|
438
|
+
result = loadAliasConfig(own, this.root);
|
|
439
|
+
else if (dir !== this.root)
|
|
440
|
+
result = this.configFor(path.dirname(dir));
|
|
77
441
|
}
|
|
78
|
-
|
|
79
|
-
|
|
442
|
+
this.byDir.set(dir, result);
|
|
443
|
+
return result;
|
|
80
444
|
}
|
|
81
445
|
}
|
|
82
|
-
/**
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
446
|
+
/**
|
|
447
|
+
* Strip `//` and block comments and trailing commas from JSONC text, leaving
|
|
448
|
+
* string literals untouched.
|
|
449
|
+
*/
|
|
450
|
+
export function stripJsonComments(text) {
|
|
451
|
+
let out = "";
|
|
452
|
+
let i = 0;
|
|
453
|
+
while (i < text.length) {
|
|
454
|
+
const ch = text[i];
|
|
455
|
+
if (ch === '"') {
|
|
456
|
+
let j = i + 1;
|
|
457
|
+
while (j < text.length && text[j] !== '"')
|
|
458
|
+
j += text[j] === "\\" ? 2 : 1;
|
|
459
|
+
out += text.slice(i, j + 1);
|
|
460
|
+
i = j + 1;
|
|
461
|
+
}
|
|
462
|
+
else if (ch === "/" && text[i + 1] === "/") {
|
|
463
|
+
while (i < text.length && text[i] !== "\n")
|
|
464
|
+
i++;
|
|
465
|
+
}
|
|
466
|
+
else if (ch === "/" && text[i + 1] === "*") {
|
|
467
|
+
const close = text.indexOf("*/", i + 2);
|
|
468
|
+
i = close < 0 ? text.length : close + 2;
|
|
469
|
+
}
|
|
470
|
+
else {
|
|
471
|
+
out += ch;
|
|
472
|
+
i++;
|
|
98
473
|
}
|
|
99
474
|
}
|
|
100
|
-
|
|
101
|
-
if (def && !text.includes("{"))
|
|
102
|
-
names.push(def[1]);
|
|
103
|
-
return { from, names };
|
|
475
|
+
return out.replace(/,(\s*[}\]])/g, "$1");
|
|
104
476
|
}
|
|
105
477
|
/**
|
|
106
|
-
*
|
|
478
|
+
* Read a tsconfig and its relative `extends` chain (max depth 5, cycle-safe,
|
|
479
|
+
* inside the project root). The nearest definition of `baseUrl` and of `paths`
|
|
480
|
+
* wins, each resolved against the config file that declares it.
|
|
481
|
+
*
|
|
482
|
+
* @returns The effective alias config, or null when the file cannot be parsed.
|
|
107
483
|
*/
|
|
108
|
-
function
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
const
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
path.
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
484
|
+
function loadAliasConfig(file, root) {
|
|
485
|
+
let baseUrl;
|
|
486
|
+
let paths;
|
|
487
|
+
let pathsBase = path.dirname(file);
|
|
488
|
+
const seen = new Set();
|
|
489
|
+
let current = file;
|
|
490
|
+
for (let depth = 0; current && depth < 6 && !seen.has(current); depth++) {
|
|
491
|
+
seen.add(current);
|
|
492
|
+
let parsed;
|
|
493
|
+
try {
|
|
494
|
+
parsed = JSON.parse(stripJsonComments(fs.readFileSync(current, "utf8")));
|
|
495
|
+
}
|
|
496
|
+
catch {
|
|
497
|
+
if (depth === 0)
|
|
498
|
+
return null;
|
|
499
|
+
break;
|
|
500
|
+
}
|
|
501
|
+
const dir = path.dirname(current);
|
|
502
|
+
const co = parsed.compilerOptions ?? {};
|
|
503
|
+
if (baseUrl === undefined && typeof co.baseUrl === "string") {
|
|
504
|
+
baseUrl = path.resolve(dir, co.baseUrl);
|
|
505
|
+
}
|
|
506
|
+
if (paths === undefined && co.paths && typeof co.paths === "object") {
|
|
507
|
+
paths = co.paths;
|
|
508
|
+
pathsBase = dir;
|
|
509
|
+
}
|
|
510
|
+
current = null;
|
|
511
|
+
if (typeof parsed.extends === "string" && parsed.extends.startsWith(".")) {
|
|
512
|
+
let next = path.resolve(dir, parsed.extends);
|
|
513
|
+
if (!next.endsWith(".json"))
|
|
514
|
+
next += ".json";
|
|
515
|
+
const rel = path.relative(root, next);
|
|
516
|
+
if (!rel.startsWith("..") && !path.isAbsolute(rel) && fs.existsSync(next))
|
|
517
|
+
current = next;
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
const entries = Object.entries(paths ?? {})
|
|
521
|
+
.filter(([, t]) => Array.isArray(t))
|
|
522
|
+
.map(([pattern, targets]) => ({
|
|
523
|
+
pattern,
|
|
524
|
+
targets: targets.filter((t) => typeof t === "string"),
|
|
525
|
+
}))
|
|
526
|
+
.sort((a, b) => b.pattern.split("*")[0].length - a.pattern.split("*")[0].length);
|
|
527
|
+
return { baseUrl, paths: entries, pathsBase };
|
|
139
528
|
}
|
|
140
529
|
/**
|
|
141
530
|
* Infer a covering artifact's type from its project-relative path.
|
|
@@ -169,7 +558,7 @@ function* walkFiles(root) {
|
|
|
169
558
|
for (const entry of entries) {
|
|
170
559
|
const full = path.join(dir, entry.name);
|
|
171
560
|
if (entry.isDirectory()) {
|
|
172
|
-
if (!
|
|
561
|
+
if (!isSkippedDir(entry.name))
|
|
173
562
|
stack.push(full);
|
|
174
563
|
}
|
|
175
564
|
else if (entry.isFile()) {
|
|
@@ -179,6 +568,152 @@ function* walkFiles(root) {
|
|
|
179
568
|
}
|
|
180
569
|
}
|
|
181
570
|
}
|
|
571
|
+
function prepareFileWriter(db, embedder, cfg, touch = true) {
|
|
572
|
+
return {
|
|
573
|
+
db,
|
|
574
|
+
touch,
|
|
575
|
+
collectOwners: !touch,
|
|
576
|
+
embedder,
|
|
577
|
+
cfg,
|
|
578
|
+
insFile: db.prepare("INSERT INTO files(path, hash, lang, is_test, module, mtime_ms, size) VALUES (?, ?, ?, ?, ?, ?, ?)"),
|
|
579
|
+
updFile: db.prepare("UPDATE files SET hash = ?, lang = ?, is_test = ?, module = ?, mtime_ms = ?, size = ? WHERE id = ?"),
|
|
580
|
+
detach: db.prepare("UPDATE edges SET dst_node_id = NULL WHERE dst_node_id IN (SELECT id FROM nodes WHERE file_id = ?)"),
|
|
581
|
+
detachedOwners: db.prepare(`SELECT DISTINCT src_file_id AS id FROM edges
|
|
582
|
+
WHERE src_file_id <> ? AND dst_node_id IN (SELECT id FROM nodes WHERE file_id = ?)`),
|
|
583
|
+
delNodes: db.prepare("DELETE FROM nodes WHERE file_id = ?"),
|
|
584
|
+
delEdges: db.prepare("DELETE FROM edges WHERE src_file_id = ?"),
|
|
585
|
+
delCoverage: db.prepare("DELETE FROM coverage_links WHERE file_path = ?"),
|
|
586
|
+
delFile: db.prepare("DELETE FROM files WHERE id = ?"),
|
|
587
|
+
insNode: db.prepare(`INSERT INTO nodes(file_id, name, kind, start_line, end_line, start_byte, end_byte, parent_id, signature, body_hash, norm_hash, content_hash)
|
|
588
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`),
|
|
589
|
+
insMetrics: db.prepare(`INSERT INTO node_metrics(node_id, loc, max_nesting, branches) VALUES (?, ?, ?, ?)`),
|
|
590
|
+
insEdge: db.prepare(`INSERT INTO edges(src_node_id, src_file_id, dst_name, kind, line, is_member, spec) VALUES (?, ?, ?, ?, ?, ?, ?)`),
|
|
591
|
+
insCoverage: db.prepare(`INSERT OR REPLACE INTO coverage_links(
|
|
592
|
+
artifact_type, name, revision, kind, file_path, line, node_id, source_type, origin
|
|
593
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`),
|
|
594
|
+
insCache: db.prepare(`INSERT INTO embedding_cache(content_hash, model, dim, vec, created_at, last_seen_at)
|
|
595
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
596
|
+
ON CONFLICT(content_hash, model) DO UPDATE SET last_seen_at = excluded.last_seen_at`),
|
|
597
|
+
hasCache: db.prepare(`SELECT 1 AS ok FROM embedding_cache WHERE content_hash = ? AND model = ?`),
|
|
598
|
+
touchCache: db.prepare(`UPDATE embedding_cache SET last_seen_at = ? WHERE content_hash = ? AND model = ?`),
|
|
599
|
+
insNodeText: db.prepare(`INSERT INTO node_text(node_id, name, subtokens, signature, doc) VALUES (?, ?, ?, ?, ?)
|
|
600
|
+
ON CONFLICT(node_id) DO UPDATE SET
|
|
601
|
+
name = excluded.name,
|
|
602
|
+
subtokens = excluded.subtokens,
|
|
603
|
+
signature = excluded.signature,
|
|
604
|
+
doc = excluded.doc`),
|
|
605
|
+
};
|
|
606
|
+
}
|
|
607
|
+
/**
|
|
608
|
+
* Reset every edge whose destination is one of a file's nodes, before those
|
|
609
|
+
* nodes are deleted. Without this, a rowid freed by the delete and reused by a
|
|
610
|
+
* node inserted later in the same run would silently re-point another file's
|
|
611
|
+
* edge at an unrelated symbol; a post-hoc "id not in nodes" check cannot see
|
|
612
|
+
* that. The run's resolution pass then re-binds the edges by their rules.
|
|
613
|
+
*
|
|
614
|
+
* @returns The ids of the other files whose edges were reset (empty on a full run, which does not need them).
|
|
615
|
+
*/
|
|
616
|
+
function detachFileNodes(w, fileId) {
|
|
617
|
+
// Covers: req~edge-ids-survive-reindex~1
|
|
618
|
+
const owners = w.collectOwners
|
|
619
|
+
? w.detachedOwners.all(fileId, fileId).map((r) => r.id)
|
|
620
|
+
: [];
|
|
621
|
+
w.detach.run(fileId);
|
|
622
|
+
return owners;
|
|
623
|
+
}
|
|
624
|
+
/**
|
|
625
|
+
* Drop a removed file: detach edges into it, delete its coverage links (the
|
|
626
|
+
* file-level ones have no node to cascade from), then delete its row (nodes and
|
|
627
|
+
* edges cascade).
|
|
628
|
+
*/
|
|
629
|
+
function removeFileRows(w, fileId, rel) {
|
|
630
|
+
const owners = detachFileNodes(w, fileId);
|
|
631
|
+
w.delCoverage.run(rel);
|
|
632
|
+
w.delFile.run(fileId);
|
|
633
|
+
return owners;
|
|
634
|
+
}
|
|
635
|
+
/**
|
|
636
|
+
* (Re-)extract one file and write its whole fragment: upsert its `files` row,
|
|
637
|
+
* replace its nodes, file-owner node, edges, `node_text`/FTS rows, metrics, and
|
|
638
|
+
* coverage links, and embed cache misses through `embedding_cache`. Edges are
|
|
639
|
+
* written unresolved; the caller runs {@link resolveEdges}.
|
|
640
|
+
*
|
|
641
|
+
* @returns The file id and the ids of other files whose edges were detached.
|
|
642
|
+
*/
|
|
643
|
+
async function writeFileFragment(w, f) {
|
|
644
|
+
const { rel, lang, content, stats } = f;
|
|
645
|
+
let fileId;
|
|
646
|
+
let detachedOwners = [];
|
|
647
|
+
const isTest = isTestPath(rel, w.cfg.testGlobs) ? 1 : 0;
|
|
648
|
+
const mod = inferModule(rel);
|
|
649
|
+
if (f.priorId !== null) {
|
|
650
|
+
w.updFile.run(f.hash, lang.id, isTest, mod, f.mtimeMs, f.size, f.priorId);
|
|
651
|
+
// Covers: req~edge-ids-survive-reindex~1
|
|
652
|
+
detachedOwners = detachFileNodes(w, f.priorId);
|
|
653
|
+
w.delNodes.run(f.priorId);
|
|
654
|
+
w.delEdges.run(f.priorId);
|
|
655
|
+
w.delCoverage.run(rel);
|
|
656
|
+
fileId = f.priorId;
|
|
657
|
+
}
|
|
658
|
+
else {
|
|
659
|
+
fileId = Number(w.insFile.run(rel, f.hash, lang.id, isTest, mod, f.mtimeMs, f.size).lastInsertRowid);
|
|
660
|
+
}
|
|
661
|
+
const { symbols, refs, coverage, reexports } = await extract(content, lang);
|
|
662
|
+
const nodeIds = [];
|
|
663
|
+
const now = Date.now();
|
|
664
|
+
for (const s of symbols) {
|
|
665
|
+
const parentId = s.parentIndex !== null ? nodeIds[s.parentIndex] : null;
|
|
666
|
+
const embedText = defaultEmbedText(s.kind, s.name, s.signature);
|
|
667
|
+
const ch = contentHashFor({
|
|
668
|
+
lang: lang.id,
|
|
669
|
+
kind: s.kind,
|
|
670
|
+
name: s.name,
|
|
671
|
+
signature: s.signature,
|
|
672
|
+
embedText,
|
|
673
|
+
});
|
|
674
|
+
const id = Number(w.insNode.run(fileId, s.name, s.kind, s.startLine, s.endLine, s.startByte, s.endByte, parentId, s.signature, s.bodyHash, s.normHash, ch).lastInsertRowid);
|
|
675
|
+
nodeIds.push(id);
|
|
676
|
+
w.insMetrics.run(id, s.loc, s.maxNesting, s.branches);
|
|
677
|
+
w.insNodeText.run(id, s.name, s.subtokens, s.signature ?? "", s.docstring);
|
|
678
|
+
const hit = w.hasCache.get(ch, w.embedder.id);
|
|
679
|
+
if (hit) {
|
|
680
|
+
if (w.touch)
|
|
681
|
+
w.touchCache.run(now, ch, w.embedder.id);
|
|
682
|
+
stats.fromCache++;
|
|
683
|
+
}
|
|
684
|
+
else {
|
|
685
|
+
const vec = await w.embedder.embed(embedText);
|
|
686
|
+
w.insCache.run(ch, w.embedder.id, w.embedder.dim, toBlob(vec), now, now);
|
|
687
|
+
stats.computed++;
|
|
688
|
+
}
|
|
689
|
+
stats.embeddings++;
|
|
690
|
+
}
|
|
691
|
+
// Covers: req~impact-id-first~1
|
|
692
|
+
// Imports and calls outside any definition belong to the file: give the
|
|
693
|
+
// file one hidden owner node so no edge is ever stored ownerless. A file
|
|
694
|
+
// with no symbols or with re-exports (a barrel: `export * from "./x"`)
|
|
695
|
+
// gets one too, so an import of it resolves to the file instead of
|
|
696
|
+
// looking like a package. It gets no text, embedding, or metrics.
|
|
697
|
+
const isOrphan = (r) => r.kind === "import" || r.ownerIndex === null;
|
|
698
|
+
let fileOwner = null;
|
|
699
|
+
if (refs.some(isOrphan) || symbols.length === 0 || reexports.length > 0) {
|
|
700
|
+
const lastLine = content.length === 0 ? 1 : content.replace(/\n$/, "").split("\n").length;
|
|
701
|
+
fileOwner = Number(w.insNode.run(fileId, rel, FILE_NODE_KIND, 1, lastLine, 0, content.length, null, null, null, null, null).lastInsertRowid);
|
|
702
|
+
}
|
|
703
|
+
for (const r of refs) {
|
|
704
|
+
const srcId = isOrphan(r) ? fileOwner : nodeIds[r.ownerIndex];
|
|
705
|
+
w.insEdge.run(srcId, fileId, r.name, r.kind, r.line, r.member, r.spec);
|
|
706
|
+
stats.edges++;
|
|
707
|
+
}
|
|
708
|
+
const sourceType = inferSourceType(rel);
|
|
709
|
+
for (const c of coverage) {
|
|
710
|
+
const nodeId = c.ownerIndex !== null ? nodeIds[c.ownerIndex] : null;
|
|
711
|
+
w.insCoverage.run(c.artifactType, c.name, c.revision, c.kind, rel, c.line, nodeId, sourceType, "comment");
|
|
712
|
+
}
|
|
713
|
+
stats.files++;
|
|
714
|
+
stats.nodes += symbols.length;
|
|
715
|
+
return { fileId, detachedOwners };
|
|
716
|
+
}
|
|
182
717
|
/**
|
|
183
718
|
* Build or incrementally refresh the index for a project.
|
|
184
719
|
*
|
|
@@ -205,7 +740,7 @@ export async function buildIndex(projectPath, onProgressOrOpts) {
|
|
|
205
740
|
const maxCacheMB = opts.maxCacheMB ?? DEFAULT_MAX_CACHE_MB;
|
|
206
741
|
const retentionDays = opts.retentionDays ?? DEFAULT_RETENTION_DAYS;
|
|
207
742
|
const db = openDb(projectPath);
|
|
208
|
-
|
|
743
|
+
opts.hooks?.onOpen?.(db);
|
|
209
744
|
const embedder = getEmbedder();
|
|
210
745
|
const stats = {
|
|
211
746
|
files: 0,
|
|
@@ -223,61 +758,45 @@ export async function buildIndex(projectPath, onProgressOrOpts) {
|
|
|
223
758
|
nextStep: "",
|
|
224
759
|
};
|
|
225
760
|
const cfg = loadAffectedConfig(projectPath);
|
|
226
|
-
const existing = new Map();
|
|
227
|
-
for (const row of db.prepare("SELECT id, path, hash, mtime_ms, size FROM files").all()) {
|
|
228
|
-
existing.set(row.path, {
|
|
229
|
-
id: row.id,
|
|
230
|
-
hash: row.hash,
|
|
231
|
-
mtime_ms: row.mtime_ms,
|
|
232
|
-
size: row.size,
|
|
233
|
-
});
|
|
234
|
-
}
|
|
235
|
-
const prevRoot = db.prepare("SELECT hash FROM dir_hashes WHERE path = ''").get();
|
|
236
761
|
const seen = new Set();
|
|
237
762
|
const fileHashes = new Map();
|
|
238
|
-
const
|
|
239
|
-
const updFile = db.prepare("UPDATE files SET hash = ?, lang = ?, is_test = ?, module = ?, mtime_ms = ?, size = ? WHERE id = ?");
|
|
240
|
-
const delNodes = db.prepare("DELETE FROM nodes WHERE file_id = ?");
|
|
241
|
-
const delEdges = db.prepare("DELETE FROM edges WHERE src_file_id = ?");
|
|
242
|
-
const delCoverage = db.prepare("DELETE FROM coverage_links WHERE file_path = ?");
|
|
243
|
-
const insNode = db.prepare(`INSERT INTO nodes(file_id, name, kind, start_line, end_line, start_byte, end_byte, parent_id, signature, body_hash, norm_hash, content_hash)
|
|
244
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
245
|
-
const insMetrics = db.prepare(`INSERT INTO node_metrics(node_id, loc, max_nesting, branches) VALUES (?, ?, ?, ?)`);
|
|
246
|
-
const insEdge = db.prepare(`INSERT INTO edges(src_node_id, src_file_id, dst_name, kind, line) VALUES (?, ?, ?, ?, ?)`);
|
|
247
|
-
const insCoverage = db.prepare(`INSERT OR REPLACE INTO coverage_links(
|
|
248
|
-
artifact_type, name, revision, kind, file_path, line, node_id, source_type, origin
|
|
249
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
250
|
-
const insCache = db.prepare(`INSERT INTO embedding_cache(content_hash, model, dim, vec, created_at, last_seen_at)
|
|
251
|
-
VALUES (?, ?, ?, ?, ?, ?)
|
|
252
|
-
ON CONFLICT(content_hash, model) DO UPDATE SET last_seen_at = excluded.last_seen_at`);
|
|
253
|
-
const hasCache = db.prepare(`SELECT 1 AS ok FROM embedding_cache WHERE content_hash = ? AND model = ?`);
|
|
254
|
-
const insNodeText = db.prepare(`INSERT INTO node_text(node_id, name, subtokens, signature, doc) VALUES (?, ?, ?, ?, ?)
|
|
255
|
-
ON CONFLICT(node_id) DO UPDATE SET
|
|
256
|
-
name = excluded.name,
|
|
257
|
-
subtokens = excluded.subtokens,
|
|
258
|
-
signature = excluded.signature,
|
|
259
|
-
doc = excluded.doc`);
|
|
763
|
+
const w = prepareFileWriter(db, embedder, cfg);
|
|
260
764
|
const allFiles = [...walkFiles(projectPath)];
|
|
261
765
|
// Set inside the transaction; read after it to decide whether the map is written.
|
|
262
766
|
let noop;
|
|
263
|
-
db.exec("BEGIN");
|
|
264
767
|
try {
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
768
|
+
await opts.hooks?.afterWalk?.();
|
|
769
|
+
// Take the write lock before reading the stored file set: a per-file
|
|
770
|
+
// reindex committing a new file's row between that read and this run's
|
|
771
|
+
// insert would otherwise fail the whole run on the unique path. Per-file
|
|
772
|
+
// runs wait for (or give up on) this lock instead. Inside the try so a
|
|
773
|
+
// SQLITE_BUSY here still closes the connection.
|
|
774
|
+
db.exec("BEGIN IMMEDIATE");
|
|
775
|
+
const force = Boolean(opts.force) || needsReindex(db);
|
|
776
|
+
const existing = new Map();
|
|
777
|
+
for (const row of db
|
|
778
|
+
.prepare("SELECT id, path, hash, mtime_ms, size FROM files")
|
|
779
|
+
.all()) {
|
|
780
|
+
existing.set(row.path, {
|
|
781
|
+
id: row.id,
|
|
782
|
+
hash: row.hash,
|
|
783
|
+
mtime_ms: row.mtime_ms,
|
|
784
|
+
size: row.size,
|
|
785
|
+
});
|
|
786
|
+
}
|
|
787
|
+
const prevRoot = db.prepare("SELECT hash FROM dir_hashes WHERE path = ''").get();
|
|
788
|
+
// Stat, hash, and (when changed) re-extract one walked file.
|
|
789
|
+
const visit = async (filePath, rel) => {
|
|
271
790
|
seen.add(rel);
|
|
272
791
|
const lang = langForPath(filePath);
|
|
273
792
|
let stat;
|
|
274
793
|
try {
|
|
275
794
|
stat = fs.statSync(filePath);
|
|
276
795
|
if (stat.size > MAX_FILE_BYTES)
|
|
277
|
-
|
|
796
|
+
return;
|
|
278
797
|
}
|
|
279
798
|
catch {
|
|
280
|
-
|
|
799
|
+
return;
|
|
281
800
|
}
|
|
282
801
|
const prior = existing.get(rel);
|
|
283
802
|
const mtimeMs = Math.trunc(stat.mtimeMs);
|
|
@@ -291,89 +810,60 @@ export async function buildIndex(projectPath, onProgressOrOpts) {
|
|
|
291
810
|
fileHashes.set(rel, prior.hash);
|
|
292
811
|
stats.skippedByStat++;
|
|
293
812
|
stats.unchanged++;
|
|
294
|
-
|
|
813
|
+
return;
|
|
295
814
|
}
|
|
296
815
|
let content;
|
|
297
816
|
try {
|
|
298
817
|
content = fs.readFileSync(filePath, "utf8");
|
|
299
818
|
}
|
|
300
819
|
catch {
|
|
301
|
-
|
|
820
|
+
return;
|
|
302
821
|
}
|
|
303
822
|
const hash = hashOf(content);
|
|
304
823
|
fileHashes.set(rel, hash);
|
|
305
824
|
if (!force && prior && prior.hash === hash) {
|
|
306
|
-
updFile.run(hash, lang.id, isTestPath(rel, cfg.testGlobs) ? 1 : 0, inferModule(rel), mtimeMs, size, prior.id);
|
|
825
|
+
w.updFile.run(hash, lang.id, isTestPath(rel, cfg.testGlobs) ? 1 : 0, inferModule(rel), mtimeMs, size, prior.id);
|
|
307
826
|
stats.unchanged++;
|
|
308
|
-
|
|
309
|
-
}
|
|
310
|
-
let fileId;
|
|
311
|
-
const isTest = isTestPath(rel, cfg.testGlobs) ? 1 : 0;
|
|
312
|
-
const mod = inferModule(rel);
|
|
313
|
-
if (prior) {
|
|
314
|
-
updFile.run(hash, lang.id, isTest, mod, mtimeMs, size, prior.id);
|
|
315
|
-
delNodes.run(prior.id);
|
|
316
|
-
delEdges.run(prior.id);
|
|
317
|
-
delCoverage.run(rel);
|
|
318
|
-
fileId = prior.id;
|
|
319
|
-
}
|
|
320
|
-
else {
|
|
321
|
-
fileId = Number(insFile.run(rel, hash, lang.id, isTest, mod, mtimeMs, size).lastInsertRowid);
|
|
827
|
+
return;
|
|
322
828
|
}
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
const hit = hasCache.get(ch, embedder.id);
|
|
342
|
-
if (hit) {
|
|
343
|
-
touchCache.run(now, ch, embedder.id);
|
|
344
|
-
stats.fromCache++;
|
|
345
|
-
}
|
|
346
|
-
else {
|
|
347
|
-
const vec = await embedder.embed(embedText);
|
|
348
|
-
insCache.run(ch, embedder.id, embedder.dim, toBlob(vec), now, now);
|
|
349
|
-
stats.computed++;
|
|
350
|
-
}
|
|
351
|
-
stats.embeddings++;
|
|
352
|
-
}
|
|
353
|
-
const fileOwner = nodeIds[0] ?? null;
|
|
354
|
-
for (const r of refs) {
|
|
355
|
-
let srcId = r.ownerIndex !== null ? nodeIds[r.ownerIndex] : null;
|
|
356
|
-
if (srcId === null && r.kind === "import")
|
|
357
|
-
srcId = fileOwner;
|
|
358
|
-
insEdge.run(srcId, fileId, r.name, r.kind, r.line);
|
|
359
|
-
stats.edges++;
|
|
360
|
-
}
|
|
361
|
-
const sourceType = inferSourceType(rel);
|
|
362
|
-
for (const c of coverage) {
|
|
363
|
-
const nodeId = c.ownerIndex !== null ? nodeIds[c.ownerIndex] : null;
|
|
364
|
-
insCoverage.run(c.artifactType, c.name, c.revision, c.kind, rel, c.line, nodeId, sourceType, "comment");
|
|
365
|
-
}
|
|
366
|
-
stats.files++;
|
|
367
|
-
stats.nodes += symbols.length;
|
|
829
|
+
await writeFileFragment(w, {
|
|
830
|
+
rel,
|
|
831
|
+
lang,
|
|
832
|
+
content,
|
|
833
|
+
hash,
|
|
834
|
+
mtimeMs,
|
|
835
|
+
size,
|
|
836
|
+
priorId: prior?.id ?? null,
|
|
837
|
+
stats,
|
|
838
|
+
});
|
|
839
|
+
};
|
|
840
|
+
let done = 0;
|
|
841
|
+
for (const filePath of allFiles) {
|
|
842
|
+
const rel = path.relative(projectPath, filePath).split(path.sep).join("/");
|
|
843
|
+
done++;
|
|
844
|
+
if (onProgress)
|
|
845
|
+
onProgress({ file: rel, done, total: allFiles.length });
|
|
846
|
+
await visit(filePath, rel);
|
|
368
847
|
}
|
|
848
|
+
const listings = new Map();
|
|
369
849
|
for (const [rel, row] of existing) {
|
|
370
850
|
if (!seen.has(rel)) {
|
|
371
|
-
|
|
372
|
-
|
|
851
|
+
// The walk ran before the lock: a per-file reindex may have committed
|
|
852
|
+
// a row for a file created after the walk passed its directory. Keep
|
|
853
|
+
// and index a row only when the walk would now yield that exact path;
|
|
854
|
+
// drop the rest (deleted, ineligible, renamed by case only, or reached
|
|
855
|
+
// only through a symlink) so no file is held under two paths.
|
|
856
|
+
const abs = path.join(projectPath, rel);
|
|
857
|
+
if (!walkWouldYield(projectPath, rel, listings)) {
|
|
858
|
+
// Covers: req~edge-ids-survive-reindex~1
|
|
859
|
+
removeFileRows(w, row.id, rel);
|
|
860
|
+
stats.removed++;
|
|
861
|
+
continue;
|
|
862
|
+
}
|
|
863
|
+
await visit(abs, rel);
|
|
373
864
|
}
|
|
374
|
-
|
|
865
|
+
if (!fileHashes.has(rel))
|
|
375
866
|
fileHashes.set(rel, row.hash);
|
|
376
|
-
}
|
|
377
867
|
}
|
|
378
868
|
const dirMap = buildDirHashMap(fileHashes);
|
|
379
869
|
const rootHash = dirMap.get("") ?? "";
|
|
@@ -384,7 +874,14 @@ export async function buildIndex(projectPath, onProgressOrOpts) {
|
|
|
384
874
|
// would write — skip them so a run on every session start stays cheap.
|
|
385
875
|
// `rootUnchanged` already implies no force (explicit or needs_reindex); an
|
|
386
876
|
// explicit prune or cache cap is a maintenance request and runs in full.
|
|
387
|
-
|
|
877
|
+
// A per-file reindex already moved the stored hashes to match the tree but
|
|
878
|
+
// deferred PageRank and the map to here: its marker disables the fast path.
|
|
879
|
+
noop =
|
|
880
|
+
stats.rootUnchanged &&
|
|
881
|
+
!prune &&
|
|
882
|
+
opts.maxCacheMB === undefined &&
|
|
883
|
+
stats.removed === 0 &&
|
|
884
|
+
!postPending(db);
|
|
388
885
|
if (!noop) {
|
|
389
886
|
const now = Date.now();
|
|
390
887
|
db.prepare("DELETE FROM dir_hashes").run();
|
|
@@ -393,17 +890,9 @@ export async function buildIndex(projectPath, onProgressOrOpts) {
|
|
|
393
890
|
const nFiles = [...fileHashes.keys()].filter((f) => dir === "" ? true : f === dir || f.startsWith(dir + "/")).length;
|
|
394
891
|
insDir.run(dir, hash, nFiles, now);
|
|
395
892
|
}
|
|
396
|
-
db
|
|
397
|
-
UPDATE edges SET dst_node_id = (
|
|
398
|
-
SELECT n.id FROM nodes n
|
|
399
|
-
WHERE n.name = edges.dst_name
|
|
400
|
-
ORDER BY CASE WHEN n.file_id = edges.src_file_id THEN 0 ELSE 1 END, n.id
|
|
401
|
-
LIMIT 1
|
|
402
|
-
)
|
|
403
|
-
WHERE kind = 'call' AND dst_node_id IS NULL
|
|
404
|
-
`);
|
|
405
|
-
resolveImportEdges(db, projectPath);
|
|
893
|
+
resolveEdges(db);
|
|
406
894
|
recomputeGlobalPagerank(db);
|
|
895
|
+
clearPostPending(db);
|
|
407
896
|
// Touch last_seen for all live content hashes under active model
|
|
408
897
|
db.prepare(`UPDATE embedding_cache SET last_seen_at = ?
|
|
409
898
|
WHERE model = ?
|
|
@@ -426,7 +915,10 @@ export async function buildIndex(projectPath, onProgressOrOpts) {
|
|
|
426
915
|
db.exec("COMMIT");
|
|
427
916
|
}
|
|
428
917
|
catch (err) {
|
|
429
|
-
|
|
918
|
+
// SQLite may already have rolled back (or BEGIN never succeeded): an
|
|
919
|
+
// unguarded ROLLBACK would throw and mask the original error.
|
|
920
|
+
if (db.isTransaction)
|
|
921
|
+
db.exec("ROLLBACK");
|
|
430
922
|
throw err;
|
|
431
923
|
}
|
|
432
924
|
finally {
|
|
@@ -444,9 +936,206 @@ export async function buildIndex(projectPath, onProgressOrOpts) {
|
|
|
444
936
|
}
|
|
445
937
|
return stats;
|
|
446
938
|
}
|
|
939
|
+
/**
|
|
940
|
+
* Re-index exactly the given files in an existing, current index — the
|
|
941
|
+
* per-edit path behind `speclaw reindex-file`.
|
|
942
|
+
*
|
|
943
|
+
* Inside one `BEGIN IMMEDIATE` transaction (so each file is read after the
|
|
944
|
+
* write lock is held, and the last run to begin saw the last edit), each path
|
|
945
|
+
* is classified with the full walk's rules ({@link classifyIndexPath}): a
|
|
946
|
+
* changed file is re-extracted with the same fragment writer as
|
|
947
|
+
* {@link buildIndex}, a deleted file's rows are dropped, an unchanged file only
|
|
948
|
+
* refreshes its stat columns, and anything else is skipped. Edges are then
|
|
949
|
+
* resolved by {@link resolveEdges} scoped to the re-indexed files plus the files
|
|
950
|
+
* whose edges were detached from them (that scope also re-tries NULL edges that
|
|
951
|
+
* name a symbol the re-indexed files now define), and the directory hashes of
|
|
952
|
+
* the touched files' ancestors are recomputed.
|
|
953
|
+
*
|
|
954
|
+
* Global post-processing is deferred: no PageRank recomputation (the old rows of
|
|
955
|
+
* replaced nodes drop by cascade; new symbols have none until the next full
|
|
956
|
+
* run), no embedding-cache touch or eviction, no `docs/compass.md` write, no
|
|
957
|
+
* `meta.indexed_at` change, and no Compass call log entry. When it writes any
|
|
958
|
+
* row it sets `meta.post_pending`, which makes the next full run skip its no-op
|
|
959
|
+
* fast path and do that work.
|
|
960
|
+
*
|
|
961
|
+
* Caveat inherited from scoped resolution: an edge already bound to a node
|
|
962
|
+
* elsewhere is never re-ranked, so when an edit adds a better candidate for an
|
|
963
|
+
* ambiguous name the old binding stays until the next full run that re-extracts
|
|
964
|
+
* the calling file.
|
|
965
|
+
*
|
|
966
|
+
* Never creates, migrates, or wipes the database: a missing index or one that
|
|
967
|
+
* needs a full reindex returns `stale: true` with nothing written.
|
|
968
|
+
*
|
|
969
|
+
* @param projectPath - Absolute project root.
|
|
970
|
+
* @param paths - File paths, absolute or relative to `projectPath`.
|
|
971
|
+
* @throws When the database stays locked past the busy timeout (`SQLITE_BUSY`) or a write fails; the transaction is rolled back.
|
|
972
|
+
*/
|
|
973
|
+
export async function indexFiles(projectPath, paths) {
|
|
974
|
+
// Covers: req~reindex-on-edit~1
|
|
975
|
+
const result = {
|
|
976
|
+
reindexed: [],
|
|
977
|
+
removed: [],
|
|
978
|
+
unchanged: [],
|
|
979
|
+
skipped: [],
|
|
980
|
+
stale: false,
|
|
981
|
+
};
|
|
982
|
+
const db = openCurrentDb(projectPath);
|
|
983
|
+
if (!db) {
|
|
984
|
+
result.stale = true;
|
|
985
|
+
return result;
|
|
986
|
+
}
|
|
987
|
+
try {
|
|
988
|
+
const embedder = getEmbedder();
|
|
989
|
+
const w = prepareFileWriter(db, embedder, loadAffectedConfig(projectPath), false);
|
|
990
|
+
const getFile = db.prepare("SELECT id, hash FROM files WHERE path = ?");
|
|
991
|
+
const stats = {
|
|
992
|
+
files: 0,
|
|
993
|
+
nodes: 0,
|
|
994
|
+
edges: 0,
|
|
995
|
+
embeddings: 0,
|
|
996
|
+
computed: 0,
|
|
997
|
+
fromCache: 0,
|
|
998
|
+
};
|
|
999
|
+
const scope = new Set();
|
|
1000
|
+
const changed = [];
|
|
1001
|
+
const targets = [...new Set(paths.map((p) => path.resolve(projectPath, p)))];
|
|
1002
|
+
db.exec("BEGIN IMMEDIATE");
|
|
1003
|
+
try {
|
|
1004
|
+
// The probe ran before the lock: a full run (or another version) may
|
|
1005
|
+
// have migrated, wiped, or flagged the index since. Re-check under the
|
|
1006
|
+
// lock and leave it to the next full run if so.
|
|
1007
|
+
if (!isCurrentIndex(db)) {
|
|
1008
|
+
db.exec("ROLLBACK");
|
|
1009
|
+
result.stale = true;
|
|
1010
|
+
return result;
|
|
1011
|
+
}
|
|
1012
|
+
for (const abs of targets) {
|
|
1013
|
+
let c = classifyIndexPath(projectPath, abs);
|
|
1014
|
+
// Register only the path a full walk would yield (exact spelling, no
|
|
1015
|
+
// symlink segment); a full run would otherwise drop the row again.
|
|
1016
|
+
if (c.eligible && !walkWouldYield(projectPath, c.rel)) {
|
|
1017
|
+
c = { eligible: false, rel: c.rel, reason: "ignored" };
|
|
1018
|
+
}
|
|
1019
|
+
if (!c.eligible) {
|
|
1020
|
+
const row = c.reason === "missing" && c.rel !== null
|
|
1021
|
+
? getFile.get(c.rel)
|
|
1022
|
+
: undefined;
|
|
1023
|
+
if (row && c.rel !== null) {
|
|
1024
|
+
// Covers: req~edge-ids-survive-reindex~1
|
|
1025
|
+
for (const id of removeFileRows(w, row.id, c.rel))
|
|
1026
|
+
scope.add(id);
|
|
1027
|
+
result.removed.push(c.rel);
|
|
1028
|
+
changed.push(c.rel);
|
|
1029
|
+
}
|
|
1030
|
+
else {
|
|
1031
|
+
result.skipped.push({ path: c.rel ?? abs, reason: c.reason });
|
|
1032
|
+
}
|
|
1033
|
+
continue;
|
|
1034
|
+
}
|
|
1035
|
+
let content;
|
|
1036
|
+
try {
|
|
1037
|
+
content = fs.readFileSync(path.join(projectPath, c.rel), "utf8");
|
|
1038
|
+
}
|
|
1039
|
+
catch {
|
|
1040
|
+
result.skipped.push({ path: c.rel, reason: "missing" });
|
|
1041
|
+
continue;
|
|
1042
|
+
}
|
|
1043
|
+
const hash = hashOf(content);
|
|
1044
|
+
const prior = getFile.get(c.rel);
|
|
1045
|
+
if (prior && prior.hash === hash) {
|
|
1046
|
+
w.updFile.run(hash, c.lang.id, isTestPath(c.rel, w.cfg.testGlobs) ? 1 : 0, inferModule(c.rel), c.mtimeMs, c.size, prior.id);
|
|
1047
|
+
result.unchanged.push(c.rel);
|
|
1048
|
+
continue;
|
|
1049
|
+
}
|
|
1050
|
+
const written = await writeFileFragment(w, {
|
|
1051
|
+
rel: c.rel,
|
|
1052
|
+
lang: c.lang,
|
|
1053
|
+
content,
|
|
1054
|
+
hash,
|
|
1055
|
+
mtimeMs: c.mtimeMs,
|
|
1056
|
+
size: c.size,
|
|
1057
|
+
priorId: prior?.id ?? null,
|
|
1058
|
+
stats,
|
|
1059
|
+
});
|
|
1060
|
+
scope.add(written.fileId);
|
|
1061
|
+
for (const id of written.detachedOwners)
|
|
1062
|
+
scope.add(id);
|
|
1063
|
+
result.reindexed.push(c.rel);
|
|
1064
|
+
changed.push(c.rel);
|
|
1065
|
+
}
|
|
1066
|
+
if (changed.length > 0) {
|
|
1067
|
+
resolveEdges(db, [...scope]);
|
|
1068
|
+
updateAncestorDirHashes(db, changed);
|
|
1069
|
+
setPostPending(db);
|
|
1070
|
+
}
|
|
1071
|
+
db.exec("COMMIT");
|
|
1072
|
+
}
|
|
1073
|
+
catch (err) {
|
|
1074
|
+
if (db.isTransaction)
|
|
1075
|
+
db.exec("ROLLBACK");
|
|
1076
|
+
throw err;
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
finally {
|
|
1080
|
+
db.close();
|
|
1081
|
+
}
|
|
1082
|
+
return result;
|
|
1083
|
+
}
|
|
1084
|
+
/**
|
|
1085
|
+
* Recompute the `dir_hashes` rows of every ancestor directory of the given
|
|
1086
|
+
* files (deepest first, root last) from the stored `files.hash` values and the
|
|
1087
|
+
* child directory rows, with the same hash rule and `n_files` count as the full
|
|
1088
|
+
* walk ({@link buildDirHashMap}). A directory left with no indexed file loses
|
|
1089
|
+
* its row; the root row always exists.
|
|
1090
|
+
*/
|
|
1091
|
+
function updateAncestorDirHashes(db, rels) {
|
|
1092
|
+
const dirs = new Set([""]);
|
|
1093
|
+
for (const rel of rels) {
|
|
1094
|
+
const parts = rel.split("/");
|
|
1095
|
+
for (let i = 1; i < parts.length; i++)
|
|
1096
|
+
dirs.add(parts.slice(0, i).join("/"));
|
|
1097
|
+
}
|
|
1098
|
+
const depth = (d) => (d === "" ? 0 : d.split("/").length);
|
|
1099
|
+
const ordered = [...dirs].sort((a, b) => depth(b) - depth(a));
|
|
1100
|
+
const all = db.prepare("SELECT path, hash FROM files");
|
|
1101
|
+
// '/' + 1 is '0': [dir/, dir0) is every path under dir/ in byte order.
|
|
1102
|
+
const under = db.prepare("SELECT path, hash FROM files WHERE path >= ? AND path < ?");
|
|
1103
|
+
const dirRow = db.prepare("SELECT hash FROM dir_hashes WHERE path = ?");
|
|
1104
|
+
const upsert = db.prepare(`INSERT INTO dir_hashes(path, hash, n_files, updated_at) VALUES (?, ?, ?, ?)
|
|
1105
|
+
ON CONFLICT(path) DO UPDATE SET hash = excluded.hash, n_files = excluded.n_files,
|
|
1106
|
+
updated_at = excluded.updated_at`);
|
|
1107
|
+
const del = db.prepare("DELETE FROM dir_hashes WHERE path = ?");
|
|
1108
|
+
const now = Date.now();
|
|
1109
|
+
for (const dir of ordered) {
|
|
1110
|
+
const prefix = dir === "" ? "" : `${dir}/`;
|
|
1111
|
+
const rows = (dir === "" ? all.all() : under.all(prefix, `${dir}0`));
|
|
1112
|
+
if (rows.length === 0 && dir !== "") {
|
|
1113
|
+
del.run(dir);
|
|
1114
|
+
continue;
|
|
1115
|
+
}
|
|
1116
|
+
const children = new Map();
|
|
1117
|
+
for (const r of rows) {
|
|
1118
|
+
const rest = r.path.slice(prefix.length);
|
|
1119
|
+
const slash = rest.indexOf("/");
|
|
1120
|
+
if (slash < 0) {
|
|
1121
|
+
children.set(rest, r.hash);
|
|
1122
|
+
}
|
|
1123
|
+
else {
|
|
1124
|
+
const name = rest.slice(0, slash);
|
|
1125
|
+
if (!children.has(name)) {
|
|
1126
|
+
const sub = dirRow.get(prefix + name);
|
|
1127
|
+
children.set(name, sub?.hash ?? HASH_EMPTY);
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1131
|
+
const hash = dirHash([...children].map(([name, h]) => ({ name, hash: h })));
|
|
1132
|
+
upsert.run(dir, hash, rows.length, now);
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
447
1135
|
function countTotals(db) {
|
|
448
1136
|
const count = (table) => Number(db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get().n);
|
|
449
|
-
|
|
1137
|
+
const nodes = Number(db.prepare(`SELECT COUNT(*) AS n FROM nodes WHERE kind <> '${FILE_NODE_KIND}'`).get().n);
|
|
1138
|
+
return { files: count("files"), nodes, edges: count("edges") };
|
|
450
1139
|
}
|
|
451
1140
|
/** Next-step hint printed after an index run; names the query tools, not grep. */
|
|
452
1141
|
export function indexNextStep(totals) {
|
|
@@ -482,7 +1171,11 @@ function evictCacheBySize(db, maxCacheMB) {
|
|
|
482
1171
|
*/
|
|
483
1172
|
export function recomputeGlobalPagerank(db) {
|
|
484
1173
|
const files = db.prepare("SELECT id, path FROM files").all();
|
|
485
|
-
|
|
1174
|
+
// File-owner nodes are not ranked symbols; their outgoing calls count as
|
|
1175
|
+
// references from their file in the file–symbol graph.
|
|
1176
|
+
const nodes = db
|
|
1177
|
+
.prepare(`SELECT id, name, file_id FROM nodes WHERE kind <> '${FILE_NODE_KIND}'`)
|
|
1178
|
+
.all();
|
|
486
1179
|
if (nodes.length === 0) {
|
|
487
1180
|
db.exec("DELETE FROM pagerank");
|
|
488
1181
|
return;
|
|
@@ -499,10 +1192,12 @@ export function recomputeGlobalPagerank(db) {
|
|
|
499
1192
|
defCount.set(n.name, (defCount.get(n.name) ?? 0) + 1);
|
|
500
1193
|
const refCount = new Map();
|
|
501
1194
|
const callEdges = db
|
|
502
|
-
.prepare(`SELECT e.src_node_id, e.dst_node_id, e.dst_name,
|
|
1195
|
+
.prepare(`SELECT e.src_node_id, e.src_file_id, src.kind AS src_kind, e.dst_node_id, e.dst_name,
|
|
1196
|
+
f.path AS src_path
|
|
503
1197
|
FROM edges e
|
|
504
1198
|
JOIN files f ON f.id = e.src_file_id
|
|
505
|
-
|
|
1199
|
+
JOIN nodes src ON src.id = e.src_node_id
|
|
1200
|
+
WHERE e.kind = 'call'`)
|
|
506
1201
|
.all();
|
|
507
1202
|
for (const e of callEdges) {
|
|
508
1203
|
refCount.set(e.dst_name, (refCount.get(e.dst_name) ?? 0) + 1);
|
|
@@ -521,7 +1216,8 @@ export function recomputeGlobalPagerank(db) {
|
|
|
521
1216
|
if (e.dst_node_id == null)
|
|
522
1217
|
continue;
|
|
523
1218
|
const w = edgeWeightMul(e.dst_name, e.src_path, ctx);
|
|
524
|
-
|
|
1219
|
+
const from = e.src_kind === FILE_NODE_KIND ? fileNodeId(e.src_file_id) : e.src_node_id;
|
|
1220
|
+
edges.push({ from, to: e.dst_node_id, weight: w });
|
|
525
1221
|
}
|
|
526
1222
|
const scores = personalizedPageRank(nodeIds, edges, []);
|
|
527
1223
|
db.exec("DELETE FROM pagerank");
|