@esneiderbravo/speclaw 0.3.13 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +88 -72
- package/dist/cli/commands/index-build.js +12 -3
- package/dist/cli/commands/lawbook.js +1 -0
- package/dist/cli/commands/laws.js +149 -8
- package/dist/cli/commands/owners.js +44 -0
- package/dist/cli/commands/query.js +52 -10
- package/dist/cli/commands/update.js +35 -5
- package/dist/cli/commands/verify.js +8 -0
- package/dist/cli/index.js +15 -4
- package/dist/modules/compass/budget.js +128 -0
- package/dist/modules/compass/db.js +290 -30
- package/dist/modules/compass/diff-context.js +134 -0
- package/dist/modules/compass/embed-input.js +28 -0
- package/dist/modules/compass/embedder.js +3 -1
- package/dist/modules/compass/explore-rich.js +134 -0
- package/dist/modules/compass/extract.js +86 -0
- package/dist/modules/compass/hybrid.js +318 -0
- package/dist/modules/compass/impact-summary.js +33 -0
- package/dist/modules/compass/indexer.js +204 -33
- package/dist/modules/compass/merkle.js +76 -0
- package/dist/modules/compass/pagerank.js +122 -0
- package/dist/modules/compass/rank.js +95 -0
- package/dist/modules/compass/register.js +169 -75
- package/dist/modules/foundation/check.js +4 -2
- package/dist/modules/foundation/compile-laws.js +212 -0
- package/dist/modules/foundation/context-budget.js +1 -14
- package/dist/modules/foundation/dialects/agentsmd.js +95 -0
- package/dist/modules/foundation/dialects/claude-cursor.js +45 -0
- package/dist/modules/foundation/dialects/coderabbit.js +27 -0
- package/dist/modules/foundation/dialects/copilot.js +35 -0
- package/dist/modules/foundation/dialects/index.js +5 -0
- package/dist/modules/foundation/dialects/types.js +58 -0
- package/dist/modules/foundation/doctor.js +266 -14
- package/dist/modules/foundation/import-rules.js +67 -0
- package/dist/modules/foundation/integrity.js +307 -0
- package/dist/modules/foundation/laws-parse.js +131 -0
- package/dist/modules/foundation/laws.js +5 -0
- package/dist/modules/foundation/lock.js +283 -0
- package/dist/modules/foundation/ownership.js +4 -0
- package/dist/modules/foundation/register-core.js +57 -88
- package/dist/modules/foundation/register.js +1 -21
- package/dist/modules/foundation/scaffold.js +25 -0
- package/dist/modules/foundation/scan.js +227 -0
- package/dist/modules/foundation/setup-tool.js +96 -0
- package/dist/modules/foundation/verify.js +9 -1
- package/dist/modules/lawbook/assets/commands/archive.md +1 -1
- package/dist/modules/lawbook/assets/commands/draft.md +1 -1
- package/dist/modules/lawbook/assets/commands/explore.md +1 -1
- package/dist/modules/lawbook/assets/commands/sync.md +2 -2
- package/dist/modules/lawbook/assets/skills/archive/SKILL.md +1 -1
- package/dist/modules/lawbook/assets/skills/archive/steps/03-validate-and-sync.md +3 -3
- package/dist/modules/lawbook/assets/skills/archive/steps/04-archive.md +1 -1
- package/dist/modules/lawbook/assets/skills/draft/steps/02-understand.md +1 -1
- package/dist/modules/lawbook/assets/skills/draft/steps/05-validate.md +1 -1
- package/dist/modules/lawbook/assets/skills/explore/steps/01-investigate.md +1 -1
- package/dist/modules/lawbook/assets/skills/quick/steps/02-implement.md +1 -1
- package/dist/modules/lawbook/assets/skills/sync/SKILL.md +1 -1
- package/dist/modules/lawbook/assets/skills/sync/steps/03-validate.md +1 -1
- package/dist/modules/lawbook/assets/skills/sync/steps/04-promote.md +1 -1
- package/dist/modules/lawbook/change-tool.js +90 -0
- package/dist/modules/lawbook/coverage.js +45 -6
- package/dist/modules/lawbook/ears.js +417 -0
- package/dist/modules/lawbook/engine.js +29 -0
- package/dist/modules/lawbook/register.js +96 -54
- package/dist/modules/lawbook/spec-items.js +4 -1
- package/dist/modules/team/owners.js +464 -0
- package/dist/modules/tools/register.js +4 -26
- package/dist/shared/deprecation.js +99 -0
- package/dist/shared/exposure.js +4 -19
- package/dist/shared/git.js +25 -0
- package/dist/shared/mcp.js +29 -3
- package/dist/shared/output-budget.js +68 -0
- package/dist/shared/tool-catalog.js +49 -0
- package/package.json +4 -3
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { createHash } from "node:crypto";
|
|
4
|
-
import { openDb, clearNeedsReindex } from "./db.js";
|
|
4
|
+
import { openDb, clearNeedsReindex, needsReindex } from "./db.js";
|
|
5
5
|
import { langForPath } from "./languages.js";
|
|
6
6
|
import { extract } from "./extract.js";
|
|
7
7
|
import { getEmbedder, toBlob } from "./embedder.js";
|
|
8
|
+
import { contentHashFor, defaultEmbedText } from "./embed-input.js";
|
|
9
|
+
import { buildDirHashMap } from "./merkle.js";
|
|
8
10
|
import { loadAffectedConfig, isTestPath, inferModule } from "./affected-config.js";
|
|
11
|
+
import { personalizedPageRank, edgeWeightMul } from "./pagerank.js";
|
|
9
12
|
const SKIP_DIRS = new Set([
|
|
10
13
|
".git",
|
|
11
14
|
"node_modules",
|
|
@@ -30,6 +33,8 @@ const MAX_FILE_BYTES = 1_500_000;
|
|
|
30
33
|
function hashOf(content) {
|
|
31
34
|
return createHash("sha256").update(content).digest("hex");
|
|
32
35
|
}
|
|
36
|
+
const DEFAULT_MAX_CACHE_MB = 256;
|
|
37
|
+
const DEFAULT_RETENTION_DAYS = 30;
|
|
33
38
|
/**
|
|
34
39
|
* Point import edges at a representative node in the imported file so reverse
|
|
35
40
|
* reachability can walk file-level dependencies (not just calls).
|
|
@@ -177,73 +182,117 @@ function* walkFiles(root) {
|
|
|
177
182
|
/**
|
|
178
183
|
* Build or incrementally refresh the index for a project.
|
|
179
184
|
*
|
|
180
|
-
*
|
|
181
|
-
*
|
|
182
|
-
*
|
|
183
|
-
* unchanged are skipped; files that disappeared are pruned. Finally resolves
|
|
184
|
-
* call edges to their target node definitions by name. The whole run executes
|
|
185
|
-
* in a single transaction, rolled back on any error.
|
|
185
|
+
* Uses a stat prefilter and directory Merkle tree to avoid unnecessary reads,
|
|
186
|
+
* and an embedding cache keyed by embedder-input hash so renames/moves do not
|
|
187
|
+
* recompute vectors. The whole run executes in a single transaction.
|
|
186
188
|
*
|
|
187
189
|
* @param projectPath - Absolute path to the project root.
|
|
188
|
-
* @param
|
|
189
|
-
* @returns Counts of files, nodes, edges, embeddings, and pruned/unchanged files.
|
|
190
|
-
* @throws Re-throws any error encountered mid-run after rolling back the transaction.
|
|
190
|
+
* @param onProgressOrOpts - Progress callback (legacy) or {@link BuildIndexOptions}.
|
|
191
191
|
*/
|
|
192
|
-
export async function buildIndex(projectPath,
|
|
192
|
+
export async function buildIndex(projectPath, onProgressOrOpts) {
|
|
193
|
+
const opts = typeof onProgressOrOpts === "function"
|
|
194
|
+
? { onProgress: onProgressOrOpts }
|
|
195
|
+
: (onProgressOrOpts ?? {});
|
|
196
|
+
const onProgress = opts.onProgress;
|
|
197
|
+
const prune = Boolean(opts.prune);
|
|
198
|
+
const maxCacheMB = opts.maxCacheMB ?? DEFAULT_MAX_CACHE_MB;
|
|
199
|
+
const retentionDays = opts.retentionDays ?? DEFAULT_RETENTION_DAYS;
|
|
193
200
|
const db = openDb(projectPath);
|
|
201
|
+
const force = Boolean(opts.force) || needsReindex(db);
|
|
194
202
|
const embedder = getEmbedder();
|
|
195
203
|
const stats = {
|
|
196
204
|
files: 0,
|
|
197
205
|
nodes: 0,
|
|
198
206
|
edges: 0,
|
|
199
207
|
embeddings: 0,
|
|
208
|
+
computed: 0,
|
|
209
|
+
fromCache: 0,
|
|
200
210
|
unchanged: 0,
|
|
211
|
+
skippedByStat: 0,
|
|
201
212
|
removed: 0,
|
|
213
|
+
rootUnchanged: false,
|
|
202
214
|
embedder: embedder.id,
|
|
203
215
|
};
|
|
204
216
|
const cfg = loadAffectedConfig(projectPath);
|
|
205
217
|
const existing = new Map();
|
|
206
|
-
for (const row of db.prepare("SELECT id, path, hash FROM files").all()) {
|
|
207
|
-
existing.set(row.path, {
|
|
218
|
+
for (const row of db.prepare("SELECT id, path, hash, mtime_ms, size FROM files").all()) {
|
|
219
|
+
existing.set(row.path, {
|
|
220
|
+
id: row.id,
|
|
221
|
+
hash: row.hash,
|
|
222
|
+
mtime_ms: row.mtime_ms,
|
|
223
|
+
size: row.size,
|
|
224
|
+
});
|
|
208
225
|
}
|
|
226
|
+
const prevRoot = db.prepare("SELECT hash FROM dir_hashes WHERE path = ''").get();
|
|
209
227
|
const seen = new Set();
|
|
210
|
-
const
|
|
211
|
-
const
|
|
228
|
+
const fileHashes = new Map();
|
|
229
|
+
const insFile = db.prepare("INSERT INTO files(path, hash, lang, is_test, module, mtime_ms, size) VALUES (?, ?, ?, ?, ?, ?, ?)");
|
|
230
|
+
const updFile = db.prepare("UPDATE files SET hash = ?, lang = ?, is_test = ?, module = ?, mtime_ms = ?, size = ? WHERE id = ?");
|
|
212
231
|
const delNodes = db.prepare("DELETE FROM nodes WHERE file_id = ?");
|
|
213
232
|
const delEdges = db.prepare("DELETE FROM edges WHERE src_file_id = ?");
|
|
214
233
|
const delCoverage = db.prepare("DELETE FROM coverage_links WHERE file_path = ?");
|
|
215
|
-
const insNode = db.prepare(`INSERT INTO nodes(file_id, name, kind, start_line, end_line, start_byte, end_byte, parent_id, signature, body_hash, norm_hash)
|
|
216
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
234
|
+
const insNode = db.prepare(`INSERT INTO nodes(file_id, name, kind, start_line, end_line, start_byte, end_byte, parent_id, signature, body_hash, norm_hash, content_hash)
|
|
235
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
217
236
|
const insMetrics = db.prepare(`INSERT INTO node_metrics(node_id, loc, max_nesting, branches) VALUES (?, ?, ?, ?)`);
|
|
218
237
|
const insEdge = db.prepare(`INSERT INTO edges(src_node_id, src_file_id, dst_name, kind, line) VALUES (?, ?, ?, ?, ?)`);
|
|
219
238
|
const insCoverage = db.prepare(`INSERT OR REPLACE INTO coverage_links(
|
|
220
239
|
artifact_type, name, revision, kind, file_path, line, node_id, source_type, origin
|
|
221
240
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
222
|
-
const
|
|
241
|
+
const insCache = db.prepare(`INSERT INTO embedding_cache(content_hash, model, dim, vec, created_at, last_seen_at)
|
|
242
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
243
|
+
ON CONFLICT(content_hash, model) DO UPDATE SET last_seen_at = excluded.last_seen_at`);
|
|
244
|
+
const hasCache = db.prepare(`SELECT 1 AS ok FROM embedding_cache WHERE content_hash = ? AND model = ?`);
|
|
245
|
+
const insNodeText = db.prepare(`INSERT INTO node_text(node_id, name, subtokens, signature, doc) VALUES (?, ?, ?, ?, ?)
|
|
246
|
+
ON CONFLICT(node_id) DO UPDATE SET
|
|
247
|
+
name = excluded.name,
|
|
248
|
+
subtokens = excluded.subtokens,
|
|
249
|
+
signature = excluded.signature,
|
|
250
|
+
doc = excluded.doc`);
|
|
223
251
|
const allFiles = [...walkFiles(projectPath)];
|
|
224
252
|
db.exec("BEGIN");
|
|
225
253
|
try {
|
|
226
254
|
let done = 0;
|
|
227
255
|
for (const filePath of allFiles) {
|
|
228
|
-
const rel = path.relative(projectPath, filePath);
|
|
256
|
+
const rel = path.relative(projectPath, filePath).split(path.sep).join("/");
|
|
229
257
|
done++;
|
|
230
258
|
if (onProgress)
|
|
231
259
|
onProgress({ file: rel, done, total: allFiles.length });
|
|
232
260
|
seen.add(rel);
|
|
233
261
|
const lang = langForPath(filePath);
|
|
234
|
-
let
|
|
262
|
+
let stat;
|
|
235
263
|
try {
|
|
236
|
-
|
|
264
|
+
stat = fs.statSync(filePath);
|
|
237
265
|
if (stat.size > MAX_FILE_BYTES)
|
|
238
266
|
continue;
|
|
267
|
+
}
|
|
268
|
+
catch {
|
|
269
|
+
continue;
|
|
270
|
+
}
|
|
271
|
+
const prior = existing.get(rel);
|
|
272
|
+
const mtimeMs = Math.trunc(stat.mtimeMs);
|
|
273
|
+
const size = stat.size;
|
|
274
|
+
if (!force &&
|
|
275
|
+
prior &&
|
|
276
|
+
prior.mtime_ms != null &&
|
|
277
|
+
prior.size != null &&
|
|
278
|
+
prior.mtime_ms === mtimeMs &&
|
|
279
|
+
prior.size === size) {
|
|
280
|
+
fileHashes.set(rel, prior.hash);
|
|
281
|
+
stats.skippedByStat++;
|
|
282
|
+
stats.unchanged++;
|
|
283
|
+
continue;
|
|
284
|
+
}
|
|
285
|
+
let content;
|
|
286
|
+
try {
|
|
239
287
|
content = fs.readFileSync(filePath, "utf8");
|
|
240
288
|
}
|
|
241
289
|
catch {
|
|
242
290
|
continue;
|
|
243
291
|
}
|
|
244
292
|
const hash = hashOf(content);
|
|
245
|
-
|
|
246
|
-
if (prior && prior.hash === hash) {
|
|
293
|
+
fileHashes.set(rel, hash);
|
|
294
|
+
if (!force && prior && prior.hash === hash) {
|
|
295
|
+
updFile.run(hash, lang.id, isTestPath(rel, cfg.testGlobs) ? 1 : 0, inferModule(rel), mtimeMs, size, prior.id);
|
|
247
296
|
stats.unchanged++;
|
|
248
297
|
continue;
|
|
249
298
|
}
|
|
@@ -251,28 +300,45 @@ export async function buildIndex(projectPath, onProgress) {
|
|
|
251
300
|
const isTest = isTestPath(rel, cfg.testGlobs) ? 1 : 0;
|
|
252
301
|
const mod = inferModule(rel);
|
|
253
302
|
if (prior) {
|
|
254
|
-
updFile.run(hash, lang.id, isTest, mod, prior.id);
|
|
303
|
+
updFile.run(hash, lang.id, isTest, mod, mtimeMs, size, prior.id);
|
|
255
304
|
delNodes.run(prior.id);
|
|
256
305
|
delEdges.run(prior.id);
|
|
257
306
|
delCoverage.run(rel);
|
|
258
307
|
fileId = prior.id;
|
|
259
308
|
}
|
|
260
309
|
else {
|
|
261
|
-
fileId = Number(insFile.run(rel, hash, lang.id, isTest, mod).lastInsertRowid);
|
|
310
|
+
fileId = Number(insFile.run(rel, hash, lang.id, isTest, mod, mtimeMs, size).lastInsertRowid);
|
|
262
311
|
}
|
|
263
312
|
const { symbols, refs, coverage } = await extract(content, lang);
|
|
264
313
|
const nodeIds = [];
|
|
314
|
+
const now = Date.now();
|
|
315
|
+
const touchCache = db.prepare(`UPDATE embedding_cache SET last_seen_at = ? WHERE content_hash = ? AND model = ?`);
|
|
265
316
|
for (const s of symbols) {
|
|
266
317
|
const parentId = s.parentIndex !== null ? nodeIds[s.parentIndex] : null;
|
|
267
|
-
const
|
|
318
|
+
const embedText = defaultEmbedText(s.kind, s.name, s.signature);
|
|
319
|
+
const ch = contentHashFor({
|
|
320
|
+
lang: lang.id,
|
|
321
|
+
kind: s.kind,
|
|
322
|
+
name: s.name,
|
|
323
|
+
signature: s.signature,
|
|
324
|
+
embedText,
|
|
325
|
+
});
|
|
326
|
+
const id = Number(insNode.run(fileId, s.name, s.kind, s.startLine, s.endLine, s.startByte, s.endByte, parentId, s.signature, s.bodyHash, s.normHash, ch).lastInsertRowid);
|
|
268
327
|
nodeIds.push(id);
|
|
269
328
|
insMetrics.run(id, s.loc, s.maxNesting, s.branches);
|
|
270
|
-
|
|
271
|
-
const
|
|
272
|
-
|
|
329
|
+
insNodeText.run(id, s.name, s.subtokens, s.signature ?? "", s.docstring);
|
|
330
|
+
const hit = hasCache.get(ch, embedder.id);
|
|
331
|
+
if (hit) {
|
|
332
|
+
touchCache.run(now, ch, embedder.id);
|
|
333
|
+
stats.fromCache++;
|
|
334
|
+
}
|
|
335
|
+
else {
|
|
336
|
+
const vec = await embedder.embed(embedText);
|
|
337
|
+
insCache.run(ch, embedder.id, embedder.dim, toBlob(vec), now, now);
|
|
338
|
+
stats.computed++;
|
|
339
|
+
}
|
|
273
340
|
stats.embeddings++;
|
|
274
341
|
}
|
|
275
|
-
// Prefer a real symbol as import owner when the AST leaves imports file-scoped.
|
|
276
342
|
const fileOwner = nodeIds[0] ?? null;
|
|
277
343
|
for (const r of refs) {
|
|
278
344
|
let srcId = r.ownerIndex !== null ? nodeIds[r.ownerIndex] : null;
|
|
@@ -289,14 +355,25 @@ export async function buildIndex(projectPath, onProgress) {
|
|
|
289
355
|
stats.files++;
|
|
290
356
|
stats.nodes += symbols.length;
|
|
291
357
|
}
|
|
292
|
-
// prune files that no longer exist
|
|
293
358
|
for (const [rel, row] of existing) {
|
|
294
359
|
if (!seen.has(rel)) {
|
|
295
360
|
db.prepare("DELETE FROM files WHERE id = ?").run(row.id);
|
|
296
361
|
stats.removed++;
|
|
297
362
|
}
|
|
363
|
+
else if (!fileHashes.has(rel)) {
|
|
364
|
+
fileHashes.set(rel, row.hash);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
const dirMap = buildDirHashMap(fileHashes);
|
|
368
|
+
const rootHash = dirMap.get("") ?? "";
|
|
369
|
+
stats.rootUnchanged = Boolean(prevRoot && prevRoot.hash === rootHash && !force && stats.files === 0);
|
|
370
|
+
const now = Date.now();
|
|
371
|
+
db.prepare("DELETE FROM dir_hashes").run();
|
|
372
|
+
const insDir = db.prepare("INSERT INTO dir_hashes(path, hash, n_files, updated_at) VALUES (?, ?, ?, ?)");
|
|
373
|
+
for (const [dir, hash] of dirMap) {
|
|
374
|
+
const nFiles = [...fileHashes.keys()].filter((f) => dir === "" ? true : f === dir || f.startsWith(dir + "/")).length;
|
|
375
|
+
insDir.run(dir, hash, nFiles, now);
|
|
298
376
|
}
|
|
299
|
-
// Prefer same-file callees so colliding names across files do not share one id.
|
|
300
377
|
db.exec(`
|
|
301
378
|
UPDATE edges SET dst_node_id = (
|
|
302
379
|
SELECT n.id FROM nodes n
|
|
@@ -307,6 +384,18 @@ export async function buildIndex(projectPath, onProgress) {
|
|
|
307
384
|
WHERE kind = 'call' AND dst_node_id IS NULL
|
|
308
385
|
`);
|
|
309
386
|
resolveImportEdges(db, projectPath);
|
|
387
|
+
recomputeGlobalPagerank(db);
|
|
388
|
+
// Touch last_seen for all live content hashes under active model
|
|
389
|
+
db.prepare(`UPDATE embedding_cache SET last_seen_at = ?
|
|
390
|
+
WHERE model = ?
|
|
391
|
+
AND content_hash IN (SELECT content_hash FROM nodes WHERE content_hash IS NOT NULL)`).run(now, embedder.id);
|
|
392
|
+
if (prune) {
|
|
393
|
+
const cutoff = now - retentionDays * 24 * 60 * 60 * 1000;
|
|
394
|
+
db.prepare(`DELETE FROM embedding_cache
|
|
395
|
+
WHERE last_seen_at < ?
|
|
396
|
+
AND content_hash NOT IN (SELECT content_hash FROM nodes WHERE content_hash IS NOT NULL)`).run(cutoff);
|
|
397
|
+
}
|
|
398
|
+
evictCacheBySize(db, maxCacheMB);
|
|
310
399
|
db.prepare("INSERT INTO meta(key, value) VALUES ('indexed_at', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(new Date().toISOString());
|
|
311
400
|
clearNeedsReindex(db);
|
|
312
401
|
db.exec("COMMIT");
|
|
@@ -318,7 +407,6 @@ export async function buildIndex(projectPath, onProgress) {
|
|
|
318
407
|
finally {
|
|
319
408
|
db.close();
|
|
320
409
|
}
|
|
321
|
-
// Compact map in committed docs/compass.md (between markers) — zero tool-call cost.
|
|
322
410
|
try {
|
|
323
411
|
const { writeCompactMap } = await import("./map.js");
|
|
324
412
|
writeCompactMap(projectPath);
|
|
@@ -328,3 +416,86 @@ export async function buildIndex(projectPath, onProgress) {
|
|
|
328
416
|
}
|
|
329
417
|
return stats;
|
|
330
418
|
}
|
|
419
|
+
function evictCacheBySize(db, maxCacheMB) {
|
|
420
|
+
const limitBytes = maxCacheMB * 1024 * 1024;
|
|
421
|
+
const row = db
|
|
422
|
+
.prepare("SELECT COALESCE(SUM(LENGTH(vec)), 0) AS bytes FROM embedding_cache")
|
|
423
|
+
.get();
|
|
424
|
+
if (row.bytes <= limitBytes)
|
|
425
|
+
return;
|
|
426
|
+
const target = Math.floor(limitBytes * 0.8);
|
|
427
|
+
let bytes = row.bytes;
|
|
428
|
+
const oldest = db
|
|
429
|
+
.prepare("SELECT content_hash, model, LENGTH(vec) AS len FROM embedding_cache ORDER BY last_seen_at ASC")
|
|
430
|
+
.all();
|
|
431
|
+
const del = db.prepare("DELETE FROM embedding_cache WHERE content_hash = ? AND model = ?");
|
|
432
|
+
for (const e of oldest) {
|
|
433
|
+
if (bytes <= target)
|
|
434
|
+
break;
|
|
435
|
+
del.run(e.content_hash, e.model);
|
|
436
|
+
bytes -= e.len;
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
/**
|
|
440
|
+
* Recompute global (non-personalized) PageRank over the bipartite file↔symbol
|
|
441
|
+
* graph and replace the `pagerank` table.
|
|
442
|
+
*
|
|
443
|
+
* @param db - Open index database.
|
|
444
|
+
*/
|
|
445
|
+
export function recomputeGlobalPagerank(db) {
|
|
446
|
+
const files = db.prepare("SELECT id, path FROM files").all();
|
|
447
|
+
const nodes = db.prepare("SELECT id, name, file_id FROM nodes").all();
|
|
448
|
+
if (nodes.length === 0) {
|
|
449
|
+
db.exec("DELETE FROM pagerank");
|
|
450
|
+
return;
|
|
451
|
+
}
|
|
452
|
+
// Use negative ids for files so they never collide with node ids.
|
|
453
|
+
const fileNodeId = (fileId) => -fileId;
|
|
454
|
+
const nodeIds = [];
|
|
455
|
+
for (const f of files)
|
|
456
|
+
nodeIds.push(fileNodeId(f.id));
|
|
457
|
+
for (const n of nodes)
|
|
458
|
+
nodeIds.push(n.id);
|
|
459
|
+
const defCount = new Map();
|
|
460
|
+
for (const n of nodes)
|
|
461
|
+
defCount.set(n.name, (defCount.get(n.name) ?? 0) + 1);
|
|
462
|
+
const refCount = new Map();
|
|
463
|
+
const callEdges = db
|
|
464
|
+
.prepare(`SELECT e.src_node_id, e.dst_node_id, e.dst_name, f.path AS src_path
|
|
465
|
+
FROM edges e
|
|
466
|
+
JOIN files f ON f.id = e.src_file_id
|
|
467
|
+
WHERE e.kind = 'call' AND e.src_node_id IS NOT NULL`)
|
|
468
|
+
.all();
|
|
469
|
+
for (const e of callEdges) {
|
|
470
|
+
refCount.set(e.dst_name, (refCount.get(e.dst_name) ?? 0) + 1);
|
|
471
|
+
}
|
|
472
|
+
const ctx = {
|
|
473
|
+
mentionedIdents: new Set(),
|
|
474
|
+
focusFiles: new Set(),
|
|
475
|
+
defCount,
|
|
476
|
+
refCount,
|
|
477
|
+
};
|
|
478
|
+
const edges = [];
|
|
479
|
+
for (const n of nodes) {
|
|
480
|
+
edges.push({ from: fileNodeId(n.file_id), to: n.id, weight: 1 });
|
|
481
|
+
}
|
|
482
|
+
for (const e of callEdges) {
|
|
483
|
+
if (e.dst_node_id == null)
|
|
484
|
+
continue;
|
|
485
|
+
const w = edgeWeightMul(e.dst_name, e.src_path, ctx);
|
|
486
|
+
edges.push({ from: e.src_node_id, to: e.dst_node_id, weight: w });
|
|
487
|
+
}
|
|
488
|
+
const scores = personalizedPageRank(nodeIds, edges, []);
|
|
489
|
+
db.exec("DELETE FROM pagerank");
|
|
490
|
+
const ins = db.prepare("INSERT INTO pagerank(node_id, score) VALUES (?, ?)");
|
|
491
|
+
for (const n of nodes) {
|
|
492
|
+
ins.run(n.id, scores.get(n.id) ?? 0);
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
/** @internal exported for tests */
|
|
496
|
+
export async function embedSymbol(embedder, lang, kind, name, signature) {
|
|
497
|
+
const embedText = defaultEmbedText(kind, name, signature);
|
|
498
|
+
const contentHash = contentHashFor({ lang, kind, name, signature, embedText });
|
|
499
|
+
const vec = await embedder.embed(embedText);
|
|
500
|
+
return { contentHash, vec };
|
|
501
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
/** Stable hash of an empty directory (or fully excluded contents). */
|
|
3
|
+
export const HASH_EMPTY = createHash("sha256").update("").digest("hex");
|
|
4
|
+
/**
|
|
5
|
+
* Directory Merkle hash: sha256 of sorted `name\\0childHash\\n` lines.
|
|
6
|
+
* Sort uses UTF-8 byte order (never localeCompare) for cross-platform stability.
|
|
7
|
+
*
|
|
8
|
+
* @param children - Immediate children (files or subdirs) with their hashes.
|
|
9
|
+
*/
|
|
10
|
+
export function dirHash(children) {
|
|
11
|
+
if (children.length === 0)
|
|
12
|
+
return HASH_EMPTY;
|
|
13
|
+
const sorted = [...children].sort((a, b) => Buffer.compare(Buffer.from(a.name, "utf8"), Buffer.from(b.name, "utf8")));
|
|
14
|
+
const h = createHash("sha256");
|
|
15
|
+
for (const c of sorted)
|
|
16
|
+
h.update(`${c.name}\0${c.hash}\n`);
|
|
17
|
+
return h.digest("hex");
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Rebuild directory hashes bottom-up from relative file paths → content hashes.
|
|
21
|
+
*
|
|
22
|
+
* @param fileHashes - Project-relative paths using `/` separators.
|
|
23
|
+
* @returns Map including every ancestor directory; `""` is the project root.
|
|
24
|
+
*/
|
|
25
|
+
export function buildDirHashMap(fileHashes) {
|
|
26
|
+
/** dir path → list of direct children {name, hash} (hash filled later for dirs). */
|
|
27
|
+
const children = new Map();
|
|
28
|
+
const addChild = (parent, name, hash) => {
|
|
29
|
+
if (!children.has(parent))
|
|
30
|
+
children.set(parent, new Map());
|
|
31
|
+
children.get(parent).set(name, hash);
|
|
32
|
+
};
|
|
33
|
+
// Ensure root exists even with zero files
|
|
34
|
+
if (!children.has(""))
|
|
35
|
+
children.set("", new Map());
|
|
36
|
+
for (const [rel, hash] of fileHashes) {
|
|
37
|
+
const parts = rel.split("/").filter(Boolean);
|
|
38
|
+
if (parts.length === 0)
|
|
39
|
+
continue;
|
|
40
|
+
let parent = "";
|
|
41
|
+
for (let i = 0; i < parts.length; i++) {
|
|
42
|
+
const name = parts[i];
|
|
43
|
+
if (i === parts.length - 1) {
|
|
44
|
+
addChild(parent, name, hash);
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
const next = parent ? `${parent}/${name}` : name;
|
|
48
|
+
if (!children.has(next))
|
|
49
|
+
children.set(next, new Map());
|
|
50
|
+
// placeholder — overwritten when we hash the child dir
|
|
51
|
+
if (!children.get(parent).has(name))
|
|
52
|
+
addChild(parent, name, "");
|
|
53
|
+
parent = next;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
const hashes = new Map();
|
|
58
|
+
const dirs = [...children.keys()].sort((a, b) => b.split("/").filter(Boolean).length - a.split("/").filter(Boolean).length);
|
|
59
|
+
for (const dir of dirs) {
|
|
60
|
+
const kids = children.get(dir) ?? new Map();
|
|
61
|
+
const list = [];
|
|
62
|
+
for (const [name, fileHash] of kids) {
|
|
63
|
+
const childPath = dir ? `${dir}/${name}` : name;
|
|
64
|
+
if (children.has(childPath)) {
|
|
65
|
+
list.push({ name, hash: hashes.get(childPath) ?? HASH_EMPTY });
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
list.push({ name, hash: fileHash });
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
hashes.set(dir, dirHash(list));
|
|
72
|
+
}
|
|
73
|
+
if (!hashes.has(""))
|
|
74
|
+
hashes.set("", HASH_EMPTY);
|
|
75
|
+
return hashes;
|
|
76
|
+
}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Personalized PageRank over a bipartite file↔symbol graph (query-time and
|
|
3
|
+
* global precompute). Pure numeric iteration — callers supply adjacency.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Run personalized PageRank.
|
|
7
|
+
*
|
|
8
|
+
* @param nodeIds - All node ids in the graph (files and/or symbols).
|
|
9
|
+
* @param edges - Weighted directed edges.
|
|
10
|
+
* @param personalize - Focus node ids; empty → uniform over `nodeIds`.
|
|
11
|
+
* @param opts - Iteration controls.
|
|
12
|
+
* @returns Map of node id → score (approximately sums to 1).
|
|
13
|
+
*/
|
|
14
|
+
export function personalizedPageRank(nodeIds, edges, personalize = [], opts = {}) {
|
|
15
|
+
const alpha = opts.alpha ?? 0.85;
|
|
16
|
+
const maxIter = opts.maxIter ?? 20;
|
|
17
|
+
const tol = opts.tol ?? 1e-6;
|
|
18
|
+
const ids = [...new Set(nodeIds)];
|
|
19
|
+
if (ids.length === 0)
|
|
20
|
+
return new Map();
|
|
21
|
+
const index = new Map();
|
|
22
|
+
ids.forEach((id, i) => index.set(id, i));
|
|
23
|
+
const n = ids.length;
|
|
24
|
+
const p = new Float64Array(n);
|
|
25
|
+
const focus = personalize.filter((id) => index.has(id));
|
|
26
|
+
if (focus.length === 0) {
|
|
27
|
+
const u = 1 / n;
|
|
28
|
+
for (let i = 0; i < n; i++)
|
|
29
|
+
p[i] = u;
|
|
30
|
+
}
|
|
31
|
+
else {
|
|
32
|
+
const u = 1 / focus.length;
|
|
33
|
+
for (const id of focus)
|
|
34
|
+
p[index.get(id)] = u;
|
|
35
|
+
}
|
|
36
|
+
const outW = new Float64Array(n);
|
|
37
|
+
const adj = Array.from({ length: n }, () => []);
|
|
38
|
+
for (const e of edges) {
|
|
39
|
+
const fi = index.get(e.from);
|
|
40
|
+
const ti = index.get(e.to);
|
|
41
|
+
if (fi === undefined || ti === undefined)
|
|
42
|
+
continue;
|
|
43
|
+
const w = Math.max(e.weight, 0);
|
|
44
|
+
if (w === 0)
|
|
45
|
+
continue;
|
|
46
|
+
adj[fi].push({ to: ti, w });
|
|
47
|
+
outW[fi] += w;
|
|
48
|
+
}
|
|
49
|
+
// Self-loop 0.1 for nodes with no outbound mass so they stay in the walk.
|
|
50
|
+
for (let i = 0; i < n; i++) {
|
|
51
|
+
if (outW[i] === 0) {
|
|
52
|
+
adj[i].push({ to: i, w: 0.1 });
|
|
53
|
+
outW[i] = 0.1;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
let pr = new Float64Array(p);
|
|
57
|
+
let next = new Float64Array(n);
|
|
58
|
+
for (let iter = 0; iter < maxIter; iter++) {
|
|
59
|
+
next.fill(0);
|
|
60
|
+
let dangling = 0;
|
|
61
|
+
for (let i = 0; i < n; i++) {
|
|
62
|
+
if (outW[i] === 0)
|
|
63
|
+
dangling += pr[i];
|
|
64
|
+
}
|
|
65
|
+
for (let i = 0; i < n; i++) {
|
|
66
|
+
next[i] += (1 - alpha) * p[i];
|
|
67
|
+
next[i] += alpha * dangling * p[i];
|
|
68
|
+
}
|
|
69
|
+
for (let i = 0; i < n; i++) {
|
|
70
|
+
const ow = outW[i];
|
|
71
|
+
if (ow === 0)
|
|
72
|
+
continue;
|
|
73
|
+
const share = (alpha * pr[i]) / ow;
|
|
74
|
+
for (const { to, w } of adj[i]) {
|
|
75
|
+
next[to] += share * w;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
let delta = 0;
|
|
79
|
+
for (let i = 0; i < n; i++)
|
|
80
|
+
delta += Math.abs(next[i] - pr[i]);
|
|
81
|
+
const tmp = pr;
|
|
82
|
+
pr = next;
|
|
83
|
+
next = tmp;
|
|
84
|
+
if (delta < tol)
|
|
85
|
+
break;
|
|
86
|
+
}
|
|
87
|
+
const out = new Map();
|
|
88
|
+
for (let i = 0; i < n; i++)
|
|
89
|
+
out.set(ids[i], pr[i]);
|
|
90
|
+
return out;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Heuristic: identifiers that look meaningful (camel/snake, length ≥ 8).
|
|
94
|
+
*
|
|
95
|
+
* @param name - Symbol name.
|
|
96
|
+
*/
|
|
97
|
+
export function isMeaningfulIdent(name) {
|
|
98
|
+
if (name.length < 8)
|
|
99
|
+
return false;
|
|
100
|
+
return /[a-z][A-Z]/.test(name) || name.includes("_");
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Edge weight multiplier adapted from aider's repo-map heuristics.
|
|
104
|
+
*
|
|
105
|
+
* @param dstName - Destination symbol name.
|
|
106
|
+
* @param srcFilePath - Source file path (for focus ×50).
|
|
107
|
+
* @param ctx - Ranking context bags.
|
|
108
|
+
*/
|
|
109
|
+
export function edgeWeightMul(dstName, srcFilePath, ctx) {
|
|
110
|
+
let mul = 1.0;
|
|
111
|
+
if (ctx.mentionedIdents.has(dstName))
|
|
112
|
+
mul *= 10;
|
|
113
|
+
if (isMeaningfulIdent(dstName))
|
|
114
|
+
mul *= 10;
|
|
115
|
+
if (dstName.startsWith("_"))
|
|
116
|
+
mul *= 0.1;
|
|
117
|
+
if ((ctx.defCount.get(dstName) ?? 0) > 5)
|
|
118
|
+
mul *= 0.1;
|
|
119
|
+
if (ctx.focusFiles.has(srcFilePath))
|
|
120
|
+
mul *= 50;
|
|
121
|
+
return mul * Math.sqrt(ctx.refCount.get(dstName) ?? 1);
|
|
122
|
+
}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hybrid retrieval ranking primitives: query routing, RRF fusion, name boost,
|
|
3
|
+
* and structural rerank multipliers. Pure functions — no I/O.
|
|
4
|
+
*/
|
|
5
|
+
/** Reciprocal Rank Fusion constant (Cormack et al., SIGIR 2009). */
|
|
6
|
+
export const RRF_K = 60;
|
|
7
|
+
/** Max neighbours expanded per hub node during ego-graph growth. */
|
|
8
|
+
export const MAX_DEGREE = 200;
|
|
9
|
+
/**
|
|
10
|
+
* True when `q` looks like a single identifier (optionally dotted), not prose.
|
|
11
|
+
*
|
|
12
|
+
* @param q - Raw user query.
|
|
13
|
+
*/
|
|
14
|
+
export function isSymbolQuery(q) {
|
|
15
|
+
const t = q.trim();
|
|
16
|
+
return /^[A-Za-z_$][\w$]*(\.[A-Za-z_$][\w$]*)*$/.test(t) && t.split(/\s+/).length === 1;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Route list weights from query shape (PL→PL sparse-heavy; NL→PL dense-heavy).
|
|
20
|
+
*
|
|
21
|
+
* @param q - Raw user query.
|
|
22
|
+
*/
|
|
23
|
+
export function routeWeights(q) {
|
|
24
|
+
return isSymbolQuery(q) ? { bm25: 1.0, knn: 0.3, name: 1.0 } : { bm25: 0.7, knn: 1.0, name: 0.5 };
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Reciprocal Rank Fusion over named ranked lists.
|
|
28
|
+
*
|
|
29
|
+
* @param lists - Map of list name → ordered node ids (best first).
|
|
30
|
+
* @param weights - Per-list multipliers (missing → 1).
|
|
31
|
+
* @param k - RRF constant (default {@link RRF_K}).
|
|
32
|
+
* @returns Map of node id → fused score (higher is better).
|
|
33
|
+
*/
|
|
34
|
+
export function rrfFuse(lists, weights = {}, k = RRF_K) {
|
|
35
|
+
const scores = new Map();
|
|
36
|
+
for (const [name, ranked] of Object.entries(lists)) {
|
|
37
|
+
const w = weights[name] ?? 1;
|
|
38
|
+
ranked.forEach((id, i) => {
|
|
39
|
+
const rank = i + 1;
|
|
40
|
+
scores.set(id, (scores.get(id) ?? 0) + w / (k + rank));
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
return scores;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Multiplicative name-match boost after RRF.
|
|
47
|
+
*
|
|
48
|
+
* @param name - Candidate symbol name.
|
|
49
|
+
* @param query - Original query string.
|
|
50
|
+
* @returns Multiplier ≥ 1.
|
|
51
|
+
*/
|
|
52
|
+
export function nameBoost(name, query) {
|
|
53
|
+
const q = query.trim();
|
|
54
|
+
if (!q)
|
|
55
|
+
return 1;
|
|
56
|
+
let boost = 1;
|
|
57
|
+
if (name === q)
|
|
58
|
+
boost += 2.0;
|
|
59
|
+
else if (name.toLowerCase() === q.toLowerCase())
|
|
60
|
+
boost += 0.5;
|
|
61
|
+
if (name.startsWith(q) || name.toLowerCase().startsWith(q.toLowerCase()))
|
|
62
|
+
boost += 0.25;
|
|
63
|
+
return boost;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Structural rerank multiplier (PageRank, churn, hops, kind). Never uses
|
|
67
|
+
* directory path-distance.
|
|
68
|
+
*
|
|
69
|
+
* @param seed - Fused seed score (already includes name boost).
|
|
70
|
+
* @param s - Structural signals for the candidate.
|
|
71
|
+
*/
|
|
72
|
+
export function structuralScore(seed, s) {
|
|
73
|
+
const pr = Math.max(s.pagerank, 1e-12);
|
|
74
|
+
const churn = 1 + Math.log1p(Math.max(0, s.commits30d)) / Math.log(30);
|
|
75
|
+
const hops = 1 / (1 + Math.max(0, s.hopsToFocus));
|
|
76
|
+
const kind = s.isDefinition ? 1.0 : 0.4;
|
|
77
|
+
const testBoost = s.coveredByBrokenTest ? 1.5 : 1.0;
|
|
78
|
+
return seed * Math.sqrt(pr) * churn * hops * kind * testBoost;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Escape a free-text query for FTS5 MATCH by quoting each term.
|
|
82
|
+
*
|
|
83
|
+
* @param q - Raw user query (may contain AND, quotes, NEAR, *).
|
|
84
|
+
* @returns Safe MATCH expression, or empty string when no terms remain.
|
|
85
|
+
*/
|
|
86
|
+
export function escapeFtsQuery(q) {
|
|
87
|
+
const terms = q
|
|
88
|
+
.trim()
|
|
89
|
+
.split(/\s+/)
|
|
90
|
+
.map((t) => t.replace(/"/g, '""'))
|
|
91
|
+
.filter((t) => t.length > 0);
|
|
92
|
+
if (terms.length === 0)
|
|
93
|
+
return "";
|
|
94
|
+
return terms.map((t) => `"${t}"`).join(" ");
|
|
95
|
+
}
|