grepmax 0.26.39 → 0.26.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +4 -0
- package/dist/commands/codex.js +22 -16
- package/dist/commands/impact.js +2 -2
- package/dist/commands/mcp.js +37 -14
- package/dist/commands/peek.js +2 -2
- package/dist/commands/status.js +32 -28
- package/dist/commands/test-find.js +2 -2
- package/dist/commands/trace.js +1 -1
- package/dist/commands/watch.js +7 -0
- package/dist/lib/daemon/daemon.js +47 -15
- package/dist/lib/daemon/graph-handler.js +1 -1
- package/dist/lib/daemon/ipc-handler.js +10 -8
- package/dist/lib/daemon/search-handler.js +18 -8
- package/dist/lib/daemon/watcher-manager.js +37 -4
- package/dist/lib/graph/graph-builder.js +52 -4
- package/dist/lib/graph/impact-rollup.js +2 -2
- package/dist/lib/graph/impact.js +20 -8
- package/dist/lib/graph/test-hits.js +13 -7
- package/dist/lib/index/batch-processor.js +70 -7
- package/dist/lib/index/embedding-reuse.js +47 -0
- package/dist/lib/index/grammar-loader.js +22 -9
- package/dist/lib/output/formatter.js +1 -1
- package/dist/lib/output/index-state-footer.js +26 -6
- package/dist/lib/store/vector-db.js +43 -0
- package/dist/lib/utils/resource-snapshot.js +20 -0
- package/dist/lib/utils/tests-footer.js +8 -16
- package/dist/lib/workers/embedding-error.js +16 -0
- package/dist/lib/workers/embeddings/mlx-client.js +94 -66
- package/dist/lib/workers/orchestrator.js +31 -6
- package/dist/lib/workers/pool.js +1 -1
- package/dist/lib/workers/process-child.js +2 -1
- package/package.json +2 -3
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
|
@@ -44,6 +44,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
44
44
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
45
|
exports.handleDaemonSearch = handleDaemonSearch;
|
|
46
46
|
const embedding_generation_1 = require("../index/embedding-generation");
|
|
47
|
+
const index_state_footer_1 = require("../output/index-state-footer");
|
|
47
48
|
const searcher_1 = require("../search/searcher");
|
|
48
49
|
const retriever_1 = require("../skeleton/retriever");
|
|
49
50
|
const project_registry_1 = require("../utils/project-registry");
|
|
@@ -55,7 +56,7 @@ const project_registry_1 = require("../utils/project-registry");
|
|
|
55
56
|
function handleDaemonSearch(deps, payload, signal) {
|
|
56
57
|
return __awaiter(this, void 0, void 0, function* () {
|
|
57
58
|
var _a, _b, _c, _d;
|
|
58
|
-
var _e, _f;
|
|
59
|
+
var _e, _f, _g;
|
|
59
60
|
const { vectorDb, workerPool } = deps;
|
|
60
61
|
if (!vectorDb || !workerPool) {
|
|
61
62
|
return { ok: false, error: "daemon not ready" };
|
|
@@ -116,11 +117,20 @@ function handleDaemonSearch(deps, payload, signal) {
|
|
|
116
117
|
if ((_c = result.warnings) === null || _c === void 0 ? void 0 : _c.length)
|
|
117
118
|
response.warnings = result.warnings;
|
|
118
119
|
// Annotate partial results when the index is still catching up, so an
|
|
119
|
-
// agent can caveat or retry.
|
|
120
|
-
// formatter suppresses the settled case anyway).
|
|
120
|
+
// agent can caveat or retry. Settled failures remain visible too.
|
|
121
121
|
const idx = deps.getIndexState(root);
|
|
122
|
-
|
|
123
|
-
|
|
122
|
+
response.indexState = idx;
|
|
123
|
+
for (const candidateRoot of roots) {
|
|
124
|
+
if (candidateRoot === root)
|
|
125
|
+
continue;
|
|
126
|
+
const health = (0, index_state_footer_1.formatIndexStateFooter)(deps.getIndexState(candidateRoot), {
|
|
127
|
+
agent: true,
|
|
128
|
+
});
|
|
129
|
+
if (health) {
|
|
130
|
+
(_f = response.warnings) !== null && _f !== void 0 ? _f : (response.warnings = []);
|
|
131
|
+
response.warnings.push(`${candidateRoot}: ${health}`);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
124
134
|
// --skeleton support: fetch per-file skeletons inline so the CLI doesn't
|
|
125
135
|
// have to open its own VectorDB. getStoredSkeleton is a single LIMIT-1
|
|
126
136
|
// lookup; cheap enough to call for the top N distinct paths.
|
|
@@ -131,7 +141,7 @@ function handleDaemonSearch(deps, payload, signal) {
|
|
|
131
141
|
const seen = new Set();
|
|
132
142
|
const skeletons = {};
|
|
133
143
|
for (const chunk of result.data) {
|
|
134
|
-
const p = (
|
|
144
|
+
const p = (_g = chunk.path) !== null && _g !== void 0 ? _g : (_d = chunk.metadata) === null || _d === void 0 ? void 0 : _d.path;
|
|
135
145
|
if (!p || seen.has(p))
|
|
136
146
|
continue;
|
|
137
147
|
seen.add(p);
|
|
@@ -142,7 +152,7 @@ function handleDaemonSearch(deps, payload, signal) {
|
|
|
142
152
|
if (sk)
|
|
143
153
|
skeletons[p] = sk;
|
|
144
154
|
}
|
|
145
|
-
catch (
|
|
155
|
+
catch (_h) {
|
|
146
156
|
// best-effort — drop the entry, keep the search result
|
|
147
157
|
}
|
|
148
158
|
}
|
|
@@ -157,7 +167,7 @@ function handleDaemonSearch(deps, payload, signal) {
|
|
|
157
167
|
const builder = new GraphBuilder(vectorDb, root);
|
|
158
168
|
response.graph = yield builder.buildGraphMultiHop(payload.query, 1);
|
|
159
169
|
}
|
|
160
|
-
catch (
|
|
170
|
+
catch (_j) {
|
|
161
171
|
// best-effort — drop graph, keep results
|
|
162
172
|
}
|
|
163
173
|
}
|
|
@@ -104,6 +104,10 @@ class WatcherManager {
|
|
|
104
104
|
this.pollRecoveryTimers = new Map();
|
|
105
105
|
this.recoveryTimeouts = new Map();
|
|
106
106
|
this.lastOverflowMs = new Map();
|
|
107
|
+
this.lastWatcherErrorLogMs = new Map();
|
|
108
|
+
this.overflowCounts = new Map();
|
|
109
|
+
this.reconciledAt = new Map();
|
|
110
|
+
this.catchupDurations = new Map();
|
|
107
111
|
this.lastCatchupEndMs = new Map();
|
|
108
112
|
this.deferredCatchups = new Map();
|
|
109
113
|
this.catchups = new Map();
|
|
@@ -111,6 +115,24 @@ class WatcherManager {
|
|
|
111
115
|
this.terminalFailures = new Map();
|
|
112
116
|
this.watchLifecycles = new Map();
|
|
113
117
|
}
|
|
118
|
+
health(root) {
|
|
119
|
+
var _a;
|
|
120
|
+
var _b, _c;
|
|
121
|
+
return {
|
|
122
|
+
failedFiles: (_b = (_a = this.terminalFailures.get(root)) === null || _a === void 0 ? void 0 : _a.size) !== null && _b !== void 0 ? _b : 0,
|
|
123
|
+
degraded: this.isRootDegraded(root),
|
|
124
|
+
watcherMode: this.pollIntervals.has(root)
|
|
125
|
+
? "polling"
|
|
126
|
+
: this.pendingOps.has(`recover:${root}`) ||
|
|
127
|
+
this.deferredCatchups.has(root)
|
|
128
|
+
? "recovering"
|
|
129
|
+
: "native",
|
|
130
|
+
catchupRunning: this.catchups.has(root),
|
|
131
|
+
lastReconciledAt: this.reconciledAt.get(root),
|
|
132
|
+
overflowCount: (_c = this.overflowCounts.get(root)) !== null && _c !== void 0 ? _c : 0,
|
|
133
|
+
catchupMs: this.catchupDurations.get(root),
|
|
134
|
+
};
|
|
135
|
+
}
|
|
114
136
|
watchProject(root_1) {
|
|
115
137
|
return __awaiter(this, arguments, void 0, function* (root, options = {}) {
|
|
116
138
|
var _a, _b;
|
|
@@ -268,14 +290,20 @@ class WatcherManager {
|
|
|
268
290
|
this.deps.subscriptions.delete(root);
|
|
269
291
|
}
|
|
270
292
|
const sub = yield watcher.subscribe(root, (err, events) => {
|
|
271
|
-
var _a;
|
|
293
|
+
var _a, _b;
|
|
272
294
|
if (err) {
|
|
273
|
-
|
|
295
|
+
this.overflowCounts.set(root, ((_a = this.overflowCounts.get(root)) !== null && _a !== void 0 ? _a : 0) + 1);
|
|
296
|
+
const now = Date.now();
|
|
297
|
+
if (!this.lastWatcherErrorLogMs.has(root) ||
|
|
298
|
+
now - this.lastWatcherErrorLogMs.get(root) >= 60000) {
|
|
299
|
+
this.lastWatcherErrorLogMs.set(root, now);
|
|
300
|
+
console.error(`[daemon:${name}] Watcher error (total=${this.overflowCounts.get(root)}):`, err);
|
|
301
|
+
}
|
|
274
302
|
this.recoverWatcher(root, processor);
|
|
275
303
|
return;
|
|
276
304
|
}
|
|
277
305
|
// Only reset fail counter after sustained health (5min since last overflow)
|
|
278
|
-
const lastOverflow = (
|
|
306
|
+
const lastOverflow = (_b = this.lastOverflowMs.get(root)) !== null && _b !== void 0 ? _b : 0;
|
|
279
307
|
if (Date.now() - lastOverflow > 5 * 60 * 1000) {
|
|
280
308
|
this.watcherFailCount.delete(root);
|
|
281
309
|
}
|
|
@@ -490,6 +518,7 @@ class WatcherManager {
|
|
|
490
518
|
var _a, e_1, _b, _c;
|
|
491
519
|
var _d;
|
|
492
520
|
var _e;
|
|
521
|
+
const scanStart = Date.now();
|
|
493
522
|
const { isFileCached } = yield Promise.resolve().then(() => __importStar(require("../utils/cache-check")));
|
|
494
523
|
const metaCache = this.deps.getMetaCache();
|
|
495
524
|
const vectorDb = this.deps.getVectorDb();
|
|
@@ -641,7 +670,11 @@ class WatcherManager {
|
|
|
641
670
|
parts.push(`${purged} deleted`);
|
|
642
671
|
console.log(`[daemon:${path.basename(root)}] Catchup: ${parts.join(", ")} file(s) while offline`);
|
|
643
672
|
}
|
|
644
|
-
|
|
673
|
+
const ended = Date.now();
|
|
674
|
+
this.lastCatchupEndMs.set(root, ended);
|
|
675
|
+
this.catchupDurations.set(root, ended - scanStart);
|
|
676
|
+
if (complete)
|
|
677
|
+
this.reconciledAt.set(root, ended);
|
|
645
678
|
return complete;
|
|
646
679
|
});
|
|
647
680
|
}
|
|
@@ -11,6 +11,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
12
|
exports.GraphBuilder = void 0;
|
|
13
13
|
const languages_1 = require("../core/languages");
|
|
14
|
+
const arrow_1 = require("../utils/arrow");
|
|
14
15
|
const filter_builder_1 = require("../utils/filter-builder");
|
|
15
16
|
const query_timeout_1 = require("../utils/query-timeout");
|
|
16
17
|
const callsites_1 = require("./callsites");
|
|
@@ -118,19 +119,43 @@ class GraphBuilder {
|
|
|
118
119
|
*/
|
|
119
120
|
getCallees(symbol) {
|
|
120
121
|
return __awaiter(this, void 0, void 0, function* () {
|
|
122
|
+
var _a;
|
|
121
123
|
const table = yield this.db.ensureTable();
|
|
122
124
|
const escaped = (0, filter_builder_1.escapeSqlString)(symbol);
|
|
123
125
|
// Find the definition of the symbol
|
|
124
126
|
const rows = yield table
|
|
125
127
|
.query()
|
|
126
|
-
.select([
|
|
128
|
+
.select([
|
|
129
|
+
"path",
|
|
130
|
+
"referenced_symbols",
|
|
131
|
+
"member_referenced_symbols",
|
|
132
|
+
"content",
|
|
133
|
+
])
|
|
127
134
|
.where(this.scopeWhere(`array_contains(defined_symbols, '${escaped}')`))
|
|
128
135
|
.limit(1)
|
|
129
136
|
.toArray();
|
|
130
137
|
if (rows.length === 0)
|
|
131
138
|
return [];
|
|
132
139
|
const record = rows[0];
|
|
133
|
-
|
|
140
|
+
const memberNames = (0, arrow_1.toArr)(record.member_referenced_symbols);
|
|
141
|
+
const content = String((_a = record.content) !== null && _a !== void 0 ? _a : "");
|
|
142
|
+
let localDefinitions = [];
|
|
143
|
+
if (record.path && memberNames.some(callsites_1.isBuiltinCallee)) {
|
|
144
|
+
const localRows = yield table
|
|
145
|
+
.query()
|
|
146
|
+
.select(["defined_symbols"])
|
|
147
|
+
.where(this.scopeWhere(`path = '${(0, filter_builder_1.escapeSqlString)(String(record.path))}'`))
|
|
148
|
+
.toArray();
|
|
149
|
+
localDefinitions = localRows.flatMap((row) => (0, arrow_1.toArr)(row.defined_symbols));
|
|
150
|
+
}
|
|
151
|
+
return (0, arrow_1.toArr)(record.referenced_symbols).filter((name) => {
|
|
152
|
+
if (!(0, callsites_1.isBuiltinCallee)(name) ||
|
|
153
|
+
!memberNames.includes(name) ||
|
|
154
|
+
localDefinitions.includes(name))
|
|
155
|
+
return true;
|
|
156
|
+
const escapedName = name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
157
|
+
return new RegExp(`(?<![\\w.$])${escapedName}\\s*\\(`).test(content);
|
|
158
|
+
});
|
|
134
159
|
});
|
|
135
160
|
}
|
|
136
161
|
/**
|
|
@@ -138,7 +163,8 @@ class GraphBuilder {
|
|
|
138
163
|
*/
|
|
139
164
|
buildGraph(symbol) {
|
|
140
165
|
return __awaiter(this, void 0, void 0, function* () {
|
|
141
|
-
var _a;
|
|
166
|
+
var _a, _b;
|
|
167
|
+
var _c, _d;
|
|
142
168
|
const table = yield this.db.ensureTable();
|
|
143
169
|
const escaped = (0, filter_builder_1.escapeSqlString)(symbol);
|
|
144
170
|
// 1. Get Center (Definition)
|
|
@@ -149,6 +175,8 @@ class GraphBuilder {
|
|
|
149
175
|
"start_line",
|
|
150
176
|
"defined_symbols",
|
|
151
177
|
"referenced_symbols",
|
|
178
|
+
"member_referenced_symbols",
|
|
179
|
+
"content",
|
|
152
180
|
"role",
|
|
153
181
|
"parent_symbol",
|
|
154
182
|
"complexity",
|
|
@@ -187,6 +215,26 @@ class GraphBuilder {
|
|
|
187
215
|
])
|
|
188
216
|
.limit(25)
|
|
189
217
|
.toArray();
|
|
218
|
+
const memberNames = (0, arrow_1.toArr)((_a = centerRows[0]) === null || _a === void 0 ? void 0 : _a.member_referenced_symbols);
|
|
219
|
+
const content = String((_c = (_b = centerRows[0]) === null || _b === void 0 ? void 0 : _b.content) !== null && _c !== void 0 ? _c : "");
|
|
220
|
+
const bareCall = new RegExp(`(?<![\\w.$])${name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\s*\\(`).test(content);
|
|
221
|
+
const selfDefinition = rows.find((r) => { var _a; return String((_a = r.path) !== null && _a !== void 0 ? _a : "") === centerFile; });
|
|
222
|
+
if ((0, callsites_1.isBuiltinCallee)(name) &&
|
|
223
|
+
memberNames.includes(name) &&
|
|
224
|
+
!bareCall &&
|
|
225
|
+
!selfDefinition) {
|
|
226
|
+
calleeNodes.push({
|
|
227
|
+
symbol: name,
|
|
228
|
+
file: "",
|
|
229
|
+
line: 0,
|
|
230
|
+
role: "",
|
|
231
|
+
calls: [],
|
|
232
|
+
calledBy: [],
|
|
233
|
+
resolution: "ambiguous-member",
|
|
234
|
+
confidence: "INFERRED",
|
|
235
|
+
});
|
|
236
|
+
continue;
|
|
237
|
+
}
|
|
190
238
|
if (rows.length > 0) {
|
|
191
239
|
// Prefer-self-file, then same-language-family. Only fall back to the
|
|
192
240
|
// first row when no candidate shares the center's file or family.
|
|
@@ -195,7 +243,7 @@ class GraphBuilder {
|
|
|
195
243
|
? rows.find((r) => { var _a; return (0, languages_1.languageFamilyForPath)(String((_a = r.path) !== null && _a !== void 0 ? _a : "")) ===
|
|
196
244
|
centerFamily; })
|
|
197
245
|
: undefined;
|
|
198
|
-
calleeNodes.push(this.mapRowToNode(((
|
|
246
|
+
calleeNodes.push(this.mapRowToNode(((_d = selfRow !== null && selfRow !== void 0 ? selfRow : familyRow) !== null && _d !== void 0 ? _d : rows[0]), name, "center"));
|
|
199
247
|
}
|
|
200
248
|
else if (!(0, callsites_1.isBuiltinCallee)(name)) {
|
|
201
249
|
// Unresolved + a known builtin (.map/.get/forEach) → suppress the
|
|
@@ -151,7 +151,7 @@ function formatImpactRollupHuman(rollup, opts) {
|
|
|
151
151
|
}
|
|
152
152
|
else {
|
|
153
153
|
for (const test of rollup.topTests) {
|
|
154
|
-
lines.push(` ${relativeToProject(opts.projectRoot, test.file)}:${test.line + 1} (${(0, test_hits_1.hopLabelHuman)(test.hops)}${(0, test_hits_1.formatViaHuman)(test.via)})`);
|
|
154
|
+
lines.push(` ${relativeToProject(opts.projectRoot, test.file)}:${test.line + 1} (${(0, test_hits_1.hopLabelHuman)(test.hops, test.evidence)}${(0, test_hits_1.formatViaHuman)(test.via)})`);
|
|
155
155
|
}
|
|
156
156
|
}
|
|
157
157
|
}
|
|
@@ -191,7 +191,7 @@ function formatImpactRollupAgent(rollup, opts) {
|
|
|
191
191
|
lines.push([
|
|
192
192
|
"test",
|
|
193
193
|
`${relativeToProject(opts.projectRoot, test.file)}:${test.line + 1}`,
|
|
194
|
-
(0, test_hits_1.hopLabelAgent)(test.hops),
|
|
194
|
+
(0, test_hits_1.hopLabelAgent)(test.hops, test.evidence),
|
|
195
195
|
(0, test_hits_1.formatViaAgent)(test.via).trim(),
|
|
196
196
|
]
|
|
197
197
|
.filter(Boolean)
|
package/dist/lib/graph/impact.js
CHANGED
|
@@ -157,10 +157,20 @@ function findTests(symbols_1, vectorDb_1, projectRoot_1) {
|
|
|
157
157
|
const expanded = yield expandFileSymbols(symbols, vectorDb, projectRoot, excludePrefixes);
|
|
158
158
|
const families = yield resolveSymbolFamilies(expanded, vectorDb, projectRoot, excludePrefixes, symbolFamilies);
|
|
159
159
|
for (const symbol of expanded) {
|
|
160
|
-
|
|
160
|
+
const candidates = new Map();
|
|
161
|
+
yield walkCallers(symbol, graphBuilder, candidates, 0, depth, new Set(), (_a = families.get(symbol)) !== null && _a !== void 0 ? _a : null);
|
|
162
|
+
for (const [key, hit] of candidates) {
|
|
163
|
+
if (!symbols.includes(symbol))
|
|
164
|
+
hit.evidence = "file-candidate";
|
|
165
|
+
const previous = testHits.get(key);
|
|
166
|
+
if (!previous ||
|
|
167
|
+
(previous.evidence && !hit.evidence) ||
|
|
168
|
+
(previous.evidence === hit.evidence && hit.hops < previous.hops))
|
|
169
|
+
testHits.set(key, hit);
|
|
170
|
+
}
|
|
161
171
|
}
|
|
162
|
-
if (testHits.
|
|
163
|
-
const importFiles = yield findImportFallbackTests(
|
|
172
|
+
if (![...testHits.values()].some((hit) => !hit.evidence)) {
|
|
173
|
+
const importFiles = yield findImportFallbackTests(symbols, vectorDb, projectRoot, excludePrefixes, families);
|
|
164
174
|
for (const file of importFiles) {
|
|
165
175
|
testHits.set(`${file}:(referenced)`, {
|
|
166
176
|
file,
|
|
@@ -170,17 +180,19 @@ function findTests(symbols_1, vectorDb_1, projectRoot_1) {
|
|
|
170
180
|
});
|
|
171
181
|
}
|
|
172
182
|
}
|
|
173
|
-
return [...testHits.values()].sort((a, b) => a.
|
|
183
|
+
return [...testHits.values()].sort((a, b) => Number(!!a.evidence) - Number(!!b.evidence) ||
|
|
184
|
+
a.hops - b.hops ||
|
|
185
|
+
a.file.localeCompare(b.file));
|
|
174
186
|
});
|
|
175
187
|
}
|
|
176
|
-
function findImportFallbackTests(
|
|
188
|
+
function findImportFallbackTests(originalSymbols, vectorDb, projectRoot, excludePrefixes, symbolFamilies) {
|
|
177
189
|
return __awaiter(this, void 0, void 0, function* () {
|
|
178
190
|
var _a;
|
|
179
191
|
const files = new Set();
|
|
180
192
|
// Signal 1: referenced_symbols match (precise; works when the chunker
|
|
181
|
-
// captured call references in test bodies).
|
|
182
|
-
//
|
|
183
|
-
const dependents = yield findDependents(
|
|
193
|
+
// captured call references in test bodies). Co-defined symbols are already
|
|
194
|
+
// exposed as file candidates and must not become target coverage here.
|
|
195
|
+
const dependents = yield findDependents(originalSymbols, vectorDb, projectRoot, undefined, 50, excludePrefixes, symbolFamilies);
|
|
184
196
|
for (const d of dependents) {
|
|
185
197
|
if (isTestPath(d.file))
|
|
186
198
|
files.add(d.file);
|
|
@@ -9,20 +9,22 @@ function groupTestHitsByFile(hits) {
|
|
|
9
9
|
const byFile = new Map();
|
|
10
10
|
// Sort by hops so the first hit per file carries the best line/hops and
|
|
11
11
|
// `via` lists closest callers first ((referenced) fallback hits sort last).
|
|
12
|
-
const ordered = [...hits].sort((a, b) => (a.
|
|
13
|
-
(
|
|
12
|
+
const ordered = [...hits].sort((a, b) => Number(!!a.evidence) - Number(!!b.evidence) ||
|
|
13
|
+
(a.hops === -1 ? Number.MAX_SAFE_INTEGER : a.hops) -
|
|
14
|
+
(b.hops === -1 ? Number.MAX_SAFE_INTEGER : b.hops));
|
|
14
15
|
for (const h of ordered) {
|
|
15
16
|
let g = byFile.get(h.file);
|
|
16
17
|
if (!g) {
|
|
17
|
-
g = { file: h.file, line: h.line, hops: h.hops, via: [] };
|
|
18
|
+
g = Object.assign({ file: h.file, line: h.line, hops: h.hops, via: [] }, (h.evidence ? { evidence: h.evidence } : {}));
|
|
18
19
|
byFile.set(h.file, g);
|
|
19
20
|
}
|
|
20
21
|
if (h.symbol && h.symbol !== "(referenced)" && !g.via.includes(h.symbol)) {
|
|
21
22
|
g.via.push(h.symbol);
|
|
22
23
|
}
|
|
23
24
|
}
|
|
24
|
-
return [...byFile.values()].sort((a, b) => (a.
|
|
25
|
-
(
|
|
25
|
+
return [...byFile.values()].sort((a, b) => Number(!!a.evidence) - Number(!!b.evidence) ||
|
|
26
|
+
(a.hops === -1 ? Number.MAX_SAFE_INTEGER : a.hops) -
|
|
27
|
+
(b.hops === -1 ? Number.MAX_SAFE_INTEGER : b.hops) ||
|
|
26
28
|
a.file.localeCompare(b.file));
|
|
27
29
|
}
|
|
28
30
|
const MAX_VIA = 3;
|
|
@@ -42,10 +44,14 @@ function formatViaHuman(via) {
|
|
|
42
44
|
const more = via.length > MAX_VIA ? ` (+${via.length - MAX_VIA} more)` : "";
|
|
43
45
|
return `, via ${shown}${more}`;
|
|
44
46
|
}
|
|
45
|
-
function hopLabelAgent(hops) {
|
|
47
|
+
function hopLabelAgent(hops, evidence) {
|
|
48
|
+
if (evidence)
|
|
49
|
+
return "file-candidate";
|
|
46
50
|
return hops === -1 ? "via-import" : hops === 0 ? "direct" : `${hops}-hop`;
|
|
47
51
|
}
|
|
48
|
-
function hopLabelHuman(hops) {
|
|
52
|
+
function hopLabelHuman(hops, evidence) {
|
|
53
|
+
if (evidence)
|
|
54
|
+
return "same-file candidate";
|
|
49
55
|
return hops === -1
|
|
50
56
|
? "via import"
|
|
51
57
|
: hops === 0
|
|
@@ -43,12 +43,14 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
43
43
|
};
|
|
44
44
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
45
|
exports.ProjectBatchProcessor = void 0;
|
|
46
|
+
exports.applyReusedEmbeddings = applyReusedEmbeddings;
|
|
46
47
|
const path = __importStar(require("node:path"));
|
|
47
48
|
const config_1 = require("../../config");
|
|
48
49
|
const vector_db_1 = require("../store/vector-db");
|
|
49
50
|
const cache_check_1 = require("../utils/cache-check");
|
|
50
51
|
const file_utils_1 = require("../utils/file-utils");
|
|
51
52
|
const logger_1 = require("../utils/logger");
|
|
53
|
+
const embedding_error_1 = require("../workers/embedding-error");
|
|
52
54
|
const pool_1 = require("../workers/pool");
|
|
53
55
|
const cache_coherence_1 = require("./cache-coherence");
|
|
54
56
|
const file_policy_1 = require("./file-policy");
|
|
@@ -80,6 +82,8 @@ class ProjectBatchProcessor {
|
|
|
80
82
|
this.closed = false;
|
|
81
83
|
this.currentBatchAc = null;
|
|
82
84
|
this.lastCorruptionLogMs = 0;
|
|
85
|
+
this.backendFailures = 0;
|
|
86
|
+
this.backendRetryAt = 0;
|
|
83
87
|
this.policyChangedDuringBatch = false;
|
|
84
88
|
/** Rolling window of recent batch outcomes — see `progress`. */
|
|
85
89
|
this.recentBatches = [];
|
|
@@ -172,6 +176,18 @@ class ProjectBatchProcessor {
|
|
|
172
176
|
recentReindexed,
|
|
173
177
|
};
|
|
174
178
|
}
|
|
179
|
+
loadReusableEmbeddings(absPath) {
|
|
180
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
181
|
+
// An optimisation only: a failed lookup costs a full re-embed, nothing more.
|
|
182
|
+
try {
|
|
183
|
+
return yield this.vectorDb.getReusableEmbeddings(absPath);
|
|
184
|
+
}
|
|
185
|
+
catch (err) {
|
|
186
|
+
(0, logger_1.debug)(this.wtag, `embedding reuse lookup failed for ${absPath}: ${err instanceof Error ? err.message : String(err)}`);
|
|
187
|
+
return undefined;
|
|
188
|
+
}
|
|
189
|
+
});
|
|
190
|
+
}
|
|
175
191
|
close() {
|
|
176
192
|
return __awaiter(this, void 0, void 0, function* () {
|
|
177
193
|
var _a;
|
|
@@ -188,8 +204,11 @@ class ProjectBatchProcessor {
|
|
|
188
204
|
});
|
|
189
205
|
}
|
|
190
206
|
scheduleBatch(delayMs = DEBOUNCE_MS) {
|
|
207
|
+
delayMs = Math.max(delayMs, this.backendRetryAt - Date.now());
|
|
191
208
|
const dueMs = Date.now() + Math.max(0, delayMs);
|
|
192
|
-
if (this.debounceTimer &&
|
|
209
|
+
if (this.debounceTimer &&
|
|
210
|
+
this.debounceDueMs <= dueMs &&
|
|
211
|
+
this.debounceDueMs >= this.backendRetryAt)
|
|
193
212
|
return;
|
|
194
213
|
if (this.debounceTimer)
|
|
195
214
|
clearTimeout(this.debounceTimer);
|
|
@@ -221,7 +240,10 @@ class ProjectBatchProcessor {
|
|
|
221
240
|
const run = (this.runOperation
|
|
222
241
|
? this.runOperation(execute)
|
|
223
242
|
: execute(new AbortController().signal)).catch((err) => {
|
|
224
|
-
|
|
243
|
+
if ((err === null || err === void 0 ? void 0 : err.name) === "AbortError")
|
|
244
|
+
(0, logger_1.log)(this.wtag, "Batch cancelled during shutdown");
|
|
245
|
+
else
|
|
246
|
+
console.error(`[${this.wtag}] Batch processing failed:`, err);
|
|
225
247
|
});
|
|
226
248
|
this.activeBatch = run;
|
|
227
249
|
void run.finally(() => {
|
|
@@ -290,6 +312,7 @@ class ProjectBatchProcessor {
|
|
|
290
312
|
const metaDeletes = [];
|
|
291
313
|
const completed = new Set();
|
|
292
314
|
const retryFailures = new Set();
|
|
315
|
+
let backendUnavailable = false;
|
|
293
316
|
const requeuePath = (absPath, event, failed) => {
|
|
294
317
|
var _a;
|
|
295
318
|
var _b;
|
|
@@ -360,11 +383,14 @@ class ProjectBatchProcessor {
|
|
|
360
383
|
return;
|
|
361
384
|
}
|
|
362
385
|
}
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
},
|
|
386
|
+
// A forced reprocess is a repair; it must not trust what is stored.
|
|
387
|
+
const reusable = !forceReprocess && (cached === null || cached === void 0 ? void 0 : cached.hasVectors) !== false && (cached === null || cached === void 0 ? void 0 : cached.hash)
|
|
388
|
+
? yield this.loadReusableEmbeddings(absPath)
|
|
389
|
+
: undefined;
|
|
390
|
+
const result = yield pool.processFile(Object.assign({ path: absPath, absolutePath: absPath, projectRoot: this.projectRoot }, ((reusable === null || reusable === void 0 ? void 0 : reusable.size)
|
|
391
|
+
? { reusableKeys: Array.from(reusable.keys()) }
|
|
392
|
+
: {})), batchAc.signal);
|
|
393
|
+
applyReusedEmbeddings(result, reusable);
|
|
368
394
|
// Policy and filesystem state can change while embedding is in
|
|
369
395
|
// flight. Never commit a result that no longer describes the path.
|
|
370
396
|
const current = yield this.filePolicy.classifyFile(absPath);
|
|
@@ -418,6 +444,11 @@ class ProjectBatchProcessor {
|
|
|
418
444
|
if (batchAc.signal.aborted)
|
|
419
445
|
return;
|
|
420
446
|
const code = err === null || err === void 0 ? void 0 : err.code;
|
|
447
|
+
if ((0, embedding_error_1.isEmbeddingBackendUnavailable)(err)) {
|
|
448
|
+
backendUnavailable = true;
|
|
449
|
+
stopDispatch = true;
|
|
450
|
+
return;
|
|
451
|
+
}
|
|
421
452
|
if (code === "ENOENT") {
|
|
422
453
|
deletes.push(absPath);
|
|
423
454
|
metaDeletes.push(absPath);
|
|
@@ -463,6 +494,16 @@ class ProjectBatchProcessor {
|
|
|
463
494
|
yield schedule(() => processOne(absPath, event));
|
|
464
495
|
}
|
|
465
496
|
yield Promise.allSettled(activeTasks);
|
|
497
|
+
if (backendUnavailable) {
|
|
498
|
+
this.backendFailures++;
|
|
499
|
+
const backoffMs = Math.min(60000, 5000 * Math.pow(2, Math.min(this.backendFailures - 1, 4)));
|
|
500
|
+
this.backendRetryAt = Date.now() + backoffMs;
|
|
501
|
+
(0, logger_1.log)(this.wtag, `Embedding backend unavailable — preserving file retry budgets; retry in ${backoffMs / 1000}s`);
|
|
502
|
+
}
|
|
503
|
+
else {
|
|
504
|
+
this.backendFailures = 0;
|
|
505
|
+
this.backendRetryAt = 0;
|
|
506
|
+
}
|
|
466
507
|
const pureDeleteCandidates = new Set(metaDeletes);
|
|
467
508
|
const authoritativeMetaDeletes = new Set();
|
|
468
509
|
for (const absPath of pureDeleteCandidates) {
|
|
@@ -652,3 +693,25 @@ class ProjectBatchProcessor {
|
|
|
652
693
|
}
|
|
653
694
|
exports.ProjectBatchProcessor = ProjectBatchProcessor;
|
|
654
695
|
ProjectBatchProcessor.RECENT_BATCH_WINDOW = 5;
|
|
696
|
+
/**
|
|
697
|
+
* Fill the records a worker left unembedded from the embeddings offered to it.
|
|
698
|
+
* Throws when one cannot be filled, so the file is retried rather than stored
|
|
699
|
+
* with an empty vector.
|
|
700
|
+
*/
|
|
701
|
+
function applyReusedEmbeddings(result, reusable) {
|
|
702
|
+
var _a;
|
|
703
|
+
if (!((_a = result.reused) === null || _a === void 0 ? void 0 : _a.length))
|
|
704
|
+
return;
|
|
705
|
+
for (const { index, key } of result.reused) {
|
|
706
|
+
const record = result.vectors[index];
|
|
707
|
+
const embedding = reusable === null || reusable === void 0 ? void 0 : reusable.get(key);
|
|
708
|
+
if (!record || !embedding) {
|
|
709
|
+
throw new Error(`worker reused an embedding that was not offered (chunk ${index})`);
|
|
710
|
+
}
|
|
711
|
+
record.vector = embedding.vector;
|
|
712
|
+
record.colbert = embedding.colbert;
|
|
713
|
+
record.colbert_scale = embedding.colbert_scale;
|
|
714
|
+
record.pooled_colbert_48d = embedding.pooled_colbert_48d;
|
|
715
|
+
record.doc_token_ids = embedding.doc_token_ids;
|
|
716
|
+
}
|
|
717
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.embeddingReuseKey = embeddingReuseKey;
|
|
4
|
+
exports.toFloat32 = toFloat32;
|
|
5
|
+
exports.toInt32 = toInt32;
|
|
6
|
+
exports.toBytes = toBytes;
|
|
7
|
+
const node_crypto_1 = require("node:crypto");
|
|
8
|
+
/** Keyed by the exact text passed to the embedder (`PreparedChunk.content`). */
|
|
9
|
+
function embeddingReuseKey(content) {
|
|
10
|
+
return (0, node_crypto_1.createHash)("sha256").update(content).digest("hex");
|
|
11
|
+
}
|
|
12
|
+
function unwrap(val) {
|
|
13
|
+
if (val &&
|
|
14
|
+
!ArrayBuffer.isView(val) &&
|
|
15
|
+
typeof val.toArray === "function") {
|
|
16
|
+
return val.toArray();
|
|
17
|
+
}
|
|
18
|
+
return val;
|
|
19
|
+
}
|
|
20
|
+
// Copies rather than views: an Arrow view pins its whole record batch buffer.
|
|
21
|
+
function toFloat32(val) {
|
|
22
|
+
const v = unwrap(val);
|
|
23
|
+
if (v instanceof Float32Array)
|
|
24
|
+
return v.slice();
|
|
25
|
+
if (ArrayBuffer.isView(v) || Array.isArray(v)) {
|
|
26
|
+
return Float32Array.from(v);
|
|
27
|
+
}
|
|
28
|
+
return new Float32Array(0);
|
|
29
|
+
}
|
|
30
|
+
function toInt32(val) {
|
|
31
|
+
const v = unwrap(val);
|
|
32
|
+
if (v instanceof Int32Array)
|
|
33
|
+
return v.slice();
|
|
34
|
+
if (ArrayBuffer.isView(v) || Array.isArray(v)) {
|
|
35
|
+
return Int32Array.from(v);
|
|
36
|
+
}
|
|
37
|
+
return new Int32Array(0);
|
|
38
|
+
}
|
|
39
|
+
function toBytes(val) {
|
|
40
|
+
const v = unwrap(val);
|
|
41
|
+
if (ArrayBuffer.isView(v)) {
|
|
42
|
+
return Buffer.from(new Uint8Array(v.buffer, v.byteOffset, v.byteLength).slice());
|
|
43
|
+
}
|
|
44
|
+
if (Array.isArray(v))
|
|
45
|
+
return Buffer.from(v);
|
|
46
|
+
return Buffer.alloc(0);
|
|
47
|
+
}
|
|
@@ -56,15 +56,24 @@ for (const lang of languages_1.LANGUAGES) {
|
|
|
56
56
|
}
|
|
57
57
|
}
|
|
58
58
|
const downloadFile = (url, dest) => __awaiter(void 0, void 0, void 0, function* () {
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
59
|
+
for (let attempt = 1; attempt <= 3; attempt++) {
|
|
60
|
+
try {
|
|
61
|
+
const response = yield fetch(url, {
|
|
62
|
+
headers: { "User-Agent": "gmax" },
|
|
63
|
+
signal: AbortSignal.timeout(30000),
|
|
64
|
+
});
|
|
65
|
+
if (!response.ok)
|
|
66
|
+
throw new Error(`Failed to download ${url}: HTTP ${response.status}`);
|
|
67
|
+
const arrayBuffer = yield response.arrayBuffer();
|
|
68
|
+
fs.writeFileSync(dest, Buffer.from(arrayBuffer));
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
catch (error) {
|
|
72
|
+
if (attempt === 3)
|
|
73
|
+
throw error;
|
|
74
|
+
yield new Promise((resolve) => setTimeout(resolve, attempt * 500));
|
|
75
|
+
}
|
|
76
|
+
}
|
|
68
77
|
});
|
|
69
78
|
function ensureGrammars() {
|
|
70
79
|
return __awaiter(this, arguments, void 0, function* (log = console.log, options) {
|
|
@@ -73,6 +82,7 @@ function ensureGrammars() {
|
|
|
73
82
|
fs.mkdirSync(exports.GRAMMARS_DIR, { recursive: true });
|
|
74
83
|
}
|
|
75
84
|
const silent = (_a = options === null || options === void 0 ? void 0 : options.silent) !== null && _a !== void 0 ? _a : false;
|
|
85
|
+
const failed = [];
|
|
76
86
|
for (const [lang, url] of Object.entries(GRAMMAR_URLS)) {
|
|
77
87
|
const dest = path.join(exports.GRAMMARS_DIR, `tree-sitter-${lang}.wasm`);
|
|
78
88
|
if (fs.existsSync(dest)) {
|
|
@@ -99,6 +109,7 @@ function ensureGrammars() {
|
|
|
99
109
|
}
|
|
100
110
|
}
|
|
101
111
|
catch (err) {
|
|
112
|
+
failed.push(lang);
|
|
102
113
|
if (log === console.log) {
|
|
103
114
|
console.log("Failed");
|
|
104
115
|
}
|
|
@@ -106,5 +117,7 @@ function ensureGrammars() {
|
|
|
106
117
|
}
|
|
107
118
|
}
|
|
108
119
|
}
|
|
120
|
+
if ((options === null || options === void 0 ? void 0 : options.strict) && failed.length)
|
|
121
|
+
throw new Error(`Required grammars could not be downloaded: ${failed.join(", ")}`);
|
|
109
122
|
});
|
|
110
123
|
}
|
|
@@ -191,7 +191,7 @@ function formatTrace(graph, options) {
|
|
|
191
191
|
lines.push(` ${style.cyan("↓")} ${style.green(callee.symbol)} ${style.dim(`(${callee.file}:${callee.line})`)}`);
|
|
192
192
|
}
|
|
193
193
|
else {
|
|
194
|
-
lines.push(` ${style.cyan("↓")} ${callee.symbol} ${style.dim("(not indexed)")}`);
|
|
194
|
+
lines.push(` ${style.cyan("↓")} ${callee.symbol} ${style.dim(callee.resolution === "ambiguous-member" ? "(receiver unresolved)" : "(not indexed)")}`);
|
|
195
195
|
}
|
|
196
196
|
});
|
|
197
197
|
}
|