grepmax 0.7.30 → 0.7.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -77,6 +77,8 @@ const DEBOUNCE_MS = 2000;
|
|
|
77
77
|
const FTS_REBUILD_INTERVAL_MS = 5 * 60 * 1000;
|
|
78
78
|
function startWatcher(opts) {
|
|
79
79
|
const { projectRoot, vectorDb, metaCache, dataDir, onReindex } = opts;
|
|
80
|
+
const projectName = path.basename(projectRoot);
|
|
81
|
+
const wtag = `watch:${projectName}`;
|
|
80
82
|
const pending = new Map();
|
|
81
83
|
const retryCount = new Map();
|
|
82
84
|
let debounceTimer = null;
|
|
@@ -96,21 +98,28 @@ function startWatcher(opts) {
|
|
|
96
98
|
binaryInterval: 10000,
|
|
97
99
|
});
|
|
98
100
|
watcher.on("error", (err) => {
|
|
99
|
-
console.error(
|
|
101
|
+
console.error(`[${wtag}] Watcher error:`, err);
|
|
100
102
|
});
|
|
101
103
|
const scheduleBatch = () => {
|
|
102
104
|
if (debounceTimer)
|
|
103
105
|
clearTimeout(debounceTimer);
|
|
104
106
|
debounceTimer = setTimeout(() => processBatch(), DEBOUNCE_MS);
|
|
105
107
|
};
|
|
108
|
+
const BATCH_TIMEOUT_MS = 120000;
|
|
106
109
|
const processBatch = () => __awaiter(this, void 0, void 0, function* () {
|
|
107
110
|
var _a;
|
|
108
111
|
if (closed || processing || pending.size === 0)
|
|
109
112
|
return;
|
|
110
113
|
processing = true;
|
|
114
|
+
const batchTimeout = setTimeout(() => {
|
|
115
|
+
if (processing) {
|
|
116
|
+
console.error(`[${wtag}] Batch processing timed out after 120s, resetting`);
|
|
117
|
+
processing = false;
|
|
118
|
+
}
|
|
119
|
+
}, BATCH_TIMEOUT_MS);
|
|
111
120
|
const batch = new Map(pending);
|
|
112
121
|
pending.clear();
|
|
113
|
-
(0, logger_1.log)(
|
|
122
|
+
(0, logger_1.log)(wtag, `Processing ${batch.size} changed files`);
|
|
114
123
|
const start = Date.now();
|
|
115
124
|
let reindexed = 0;
|
|
116
125
|
const changedIds = [];
|
|
@@ -181,7 +190,11 @@ function startWatcher(opts) {
|
|
|
181
190
|
reindexed++;
|
|
182
191
|
}
|
|
183
192
|
else {
|
|
184
|
-
console.error(`[
|
|
193
|
+
console.error(`[${wtag}] Failed to process ${absPath}:`, err);
|
|
194
|
+
if (!pool.isHealthy()) {
|
|
195
|
+
console.error(`[${wtag}] Worker pool unhealthy, aborting batch`);
|
|
196
|
+
break;
|
|
197
|
+
}
|
|
185
198
|
}
|
|
186
199
|
}
|
|
187
200
|
}
|
|
@@ -209,43 +222,11 @@ function startWatcher(opts) {
|
|
|
209
222
|
finally {
|
|
210
223
|
yield lock.release();
|
|
211
224
|
}
|
|
212
|
-
|
|
213
|
-
if (changedIds.length > 0) {
|
|
214
|
-
try {
|
|
215
|
-
const table = yield vectorDb.ensureTable();
|
|
216
|
-
for (const id of changedIds) {
|
|
217
|
-
const escaped = (0, filter_builder_1.escapeSqlString)(id);
|
|
218
|
-
const rows = yield table
|
|
219
|
-
.query()
|
|
220
|
-
.select(["id", "path", "content"])
|
|
221
|
-
.where(`id = '${escaped}'`)
|
|
222
|
-
.limit(1)
|
|
223
|
-
.toArray();
|
|
224
|
-
if (rows.length === 0)
|
|
225
|
-
continue;
|
|
226
|
-
const r = rows[0];
|
|
227
|
-
const lang = path.extname(String(r.path || "")).replace(/^\./, "") ||
|
|
228
|
-
"unknown";
|
|
229
|
-
const summaries = yield (0, llm_client_1.summarizeChunks)([
|
|
230
|
-
{
|
|
231
|
-
code: String(r.content || ""),
|
|
232
|
-
language: lang,
|
|
233
|
-
file: String(r.path || ""),
|
|
234
|
-
},
|
|
235
|
-
]);
|
|
236
|
-
if (summaries === null || summaries === void 0 ? void 0 : summaries[0]) {
|
|
237
|
-
yield vectorDb.updateRows([id], "summary", [summaries[0]]);
|
|
238
|
-
}
|
|
239
|
-
}
|
|
240
|
-
}
|
|
241
|
-
catch (_b) {
|
|
242
|
-
// Summarizer unavailable — skip silently
|
|
243
|
-
}
|
|
244
|
-
}
|
|
225
|
+
const duration = Date.now() - start;
|
|
245
226
|
if (reindexed > 0) {
|
|
246
|
-
const duration = Date.now() - start;
|
|
247
227
|
onReindex === null || onReindex === void 0 ? void 0 : onReindex(reindexed, duration);
|
|
248
228
|
}
|
|
229
|
+
(0, logger_1.log)(wtag, `Batch complete: ${batch.size} files, ${reindexed} reindexed (${(duration / 1000).toFixed(1)}s)`);
|
|
249
230
|
consecutiveLockFailures = 0;
|
|
250
231
|
for (const absPath of batch.keys()) {
|
|
251
232
|
retryCount.delete(absPath);
|
|
@@ -256,7 +237,7 @@ function startWatcher(opts) {
|
|
|
256
237
|
if (isLockError) {
|
|
257
238
|
consecutiveLockFailures++;
|
|
258
239
|
}
|
|
259
|
-
console.error(
|
|
240
|
+
console.error(`[${wtag}] Batch processing failed:`, err);
|
|
260
241
|
let dropped = 0;
|
|
261
242
|
for (const [absPath, event] of batch) {
|
|
262
243
|
const count = ((_a = retryCount.get(absPath)) !== null && _a !== void 0 ? _a : 0) + 1;
|
|
@@ -270,7 +251,7 @@ function startWatcher(opts) {
|
|
|
270
251
|
}
|
|
271
252
|
}
|
|
272
253
|
if (dropped > 0) {
|
|
273
|
-
console.warn(`[
|
|
254
|
+
console.warn(`[${wtag}] Dropped ${dropped} file(s) after ${MAX_RETRIES} failed retries`);
|
|
274
255
|
}
|
|
275
256
|
if (pending.size > 0) {
|
|
276
257
|
const backoffMs = Math.min(DEBOUNCE_MS * Math.pow(2, consecutiveLockFailures), 30000);
|
|
@@ -280,12 +261,48 @@ function startWatcher(opts) {
|
|
|
280
261
|
}
|
|
281
262
|
}
|
|
282
263
|
finally {
|
|
264
|
+
clearTimeout(batchTimeout);
|
|
283
265
|
processing = false;
|
|
284
266
|
// Process any events that came in while we were processing
|
|
285
267
|
if (pending.size > 0) {
|
|
286
268
|
scheduleBatch();
|
|
287
269
|
}
|
|
288
270
|
}
|
|
271
|
+
// Fire-and-forget summarization — doesn't block the next batch
|
|
272
|
+
if (changedIds.length > 0) {
|
|
273
|
+
(() => __awaiter(this, void 0, void 0, function* () {
|
|
274
|
+
try {
|
|
275
|
+
const table = yield vectorDb.ensureTable();
|
|
276
|
+
for (const id of changedIds) {
|
|
277
|
+
const escaped = (0, filter_builder_1.escapeSqlString)(id);
|
|
278
|
+
const rows = yield table
|
|
279
|
+
.query()
|
|
280
|
+
.select(["id", "path", "content"])
|
|
281
|
+
.where(`id = '${escaped}'`)
|
|
282
|
+
.limit(1)
|
|
283
|
+
.toArray();
|
|
284
|
+
if (rows.length === 0)
|
|
285
|
+
continue;
|
|
286
|
+
const r = rows[0];
|
|
287
|
+
const lang = path.extname(String(r.path || "")).replace(/^\./, "") ||
|
|
288
|
+
"unknown";
|
|
289
|
+
const summaries = yield (0, llm_client_1.summarizeChunks)([
|
|
290
|
+
{
|
|
291
|
+
code: String(r.content || ""),
|
|
292
|
+
language: lang,
|
|
293
|
+
file: String(r.path || ""),
|
|
294
|
+
},
|
|
295
|
+
]);
|
|
296
|
+
if (summaries === null || summaries === void 0 ? void 0 : summaries[0]) {
|
|
297
|
+
yield vectorDb.updateRows([id], "summary", [summaries[0]]);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
catch (_a) {
|
|
302
|
+
// Summarizer unavailable — skip
|
|
303
|
+
}
|
|
304
|
+
}))();
|
|
305
|
+
}
|
|
289
306
|
});
|
|
290
307
|
const onFileEvent = (event, absPath) => {
|
|
291
308
|
if (closed)
|
|
@@ -304,13 +321,13 @@ function startWatcher(opts) {
|
|
|
304
321
|
watcher.on("unlink", (p) => onFileEvent("unlink", p));
|
|
305
322
|
// Periodic FTS rebuild
|
|
306
323
|
const ftsInterval = setInterval(() => __awaiter(this, void 0, void 0, function* () {
|
|
307
|
-
if (closed)
|
|
324
|
+
if (closed || processing)
|
|
308
325
|
return;
|
|
309
326
|
try {
|
|
310
327
|
yield vectorDb.createFTSIndex();
|
|
311
328
|
}
|
|
312
329
|
catch (err) {
|
|
313
|
-
console.error(
|
|
330
|
+
console.error(`[${wtag}] FTS rebuild failed:`, err);
|
|
314
331
|
}
|
|
315
332
|
}), FTS_REBUILD_INTERVAL_MS);
|
|
316
333
|
ftsInterval.unref();
|
package/dist/lib/workers/pool.js
CHANGED
package/package.json
CHANGED