grepmax 0.5.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/index/syncer.js +25 -9
- package/dist/lib/index/watcher.js +2 -0
- package/dist/lib/store/vector-db.js +2 -0
- package/dist/lib/utils/logger.js +23 -0
- package/dist/lib/workers/pool.js +3 -0
- package/dist/lib/workers/summarize/llm-client.js +1 -0
- package/package.json +1 -1
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
package/dist/lib/index/syncer.js
CHANGED
|
@@ -54,6 +54,7 @@ exports.initialSync = initialSync;
|
|
|
54
54
|
const fs = __importStar(require("node:fs"));
|
|
55
55
|
const path = __importStar(require("node:path"));
|
|
56
56
|
const config_1 = require("../../config");
|
|
57
|
+
const logger_1 = require("../utils/logger");
|
|
57
58
|
const meta_cache_1 = require("../store/meta-cache");
|
|
58
59
|
const vector_db_1 = require("../store/vector-db");
|
|
59
60
|
const file_utils_1 = require("../utils/file-utils");
|
|
@@ -183,6 +184,8 @@ function initialSync(options) {
|
|
|
183
184
|
: `${resolvedRoot}/`;
|
|
184
185
|
// Propagate project root to worker processes
|
|
185
186
|
process.env.GMAX_PROJECT_ROOT = paths.root;
|
|
187
|
+
const syncTimer = (0, logger_1.timer)("index", "Total");
|
|
188
|
+
(0, logger_1.log)("index", `Root: ${resolvedRoot}`);
|
|
186
189
|
let lock = null;
|
|
187
190
|
const vectorDb = new vector_db_1.VectorDB(paths.lancedbDir);
|
|
188
191
|
const treatAsEmptyCache = reset && dryRun;
|
|
@@ -199,11 +202,15 @@ function initialSync(options) {
|
|
|
199
202
|
if (!dryRun) {
|
|
200
203
|
// Scope checks to this project's paths only
|
|
201
204
|
const projectKeys = yield metaCache.getKeysWithPrefix(rootPrefix);
|
|
205
|
+
(0, logger_1.log)("index", `Cached files: ${projectKeys.size}`);
|
|
202
206
|
const modelChanged = (0, index_config_1.checkModelMismatch)(paths.configPath);
|
|
203
207
|
if (reset || modelChanged) {
|
|
204
208
|
if (modelChanged) {
|
|
205
209
|
const stored = (0, index_config_1.readIndexConfig)(paths.configPath);
|
|
206
|
-
|
|
210
|
+
(0, logger_1.log)("index", `Reset: model changed (${stored === null || stored === void 0 ? void 0 : stored.embedModel} → ${config_1.MODEL_IDS.embed})`);
|
|
211
|
+
}
|
|
212
|
+
else {
|
|
213
|
+
(0, logger_1.log)("index", "Reset: --reset flag");
|
|
207
214
|
}
|
|
208
215
|
// Only delete this project's data from the centralized store
|
|
209
216
|
yield vectorDb.deletePathsWithPrefix(rootPrefix);
|
|
@@ -230,6 +237,9 @@ function initialSync(options) {
|
|
|
230
237
|
let processed = 0;
|
|
231
238
|
let indexed = 0;
|
|
232
239
|
let failedFiles = 0;
|
|
240
|
+
let cacheHits = 0;
|
|
241
|
+
let walkedFiles = 0;
|
|
242
|
+
const walkTimer = (0, logger_1.timer)("index", "Walk");
|
|
233
243
|
let shouldSkipCleanup = false;
|
|
234
244
|
let flushError;
|
|
235
245
|
let flushPromise = null;
|
|
@@ -326,6 +336,7 @@ function initialSync(options) {
|
|
|
326
336
|
}
|
|
327
337
|
if (!(0, file_utils_1.isIndexableFile)(absPath))
|
|
328
338
|
continue;
|
|
339
|
+
walkedFiles++;
|
|
329
340
|
yield schedule(() => __awaiter(this, void 0, void 0, function* () {
|
|
330
341
|
if (signal === null || signal === void 0 ? void 0 : signal.aborted) {
|
|
331
342
|
shouldSkipCleanup = true;
|
|
@@ -343,11 +354,14 @@ function initialSync(options) {
|
|
|
343
354
|
if (cached &&
|
|
344
355
|
cached.mtimeMs === stats.mtimeMs &&
|
|
345
356
|
cached.size === stats.size) {
|
|
357
|
+
cacheHits++;
|
|
358
|
+
(0, logger_1.debug)("index", `SKIP ${relPath} (cached)`);
|
|
346
359
|
processed += 1;
|
|
347
360
|
seenPaths.add(absPath);
|
|
348
361
|
markProgress(relPath);
|
|
349
362
|
return;
|
|
350
363
|
}
|
|
364
|
+
(0, logger_1.debug)("index", `EMBED ${relPath}`);
|
|
351
365
|
const result = yield processFileWithRetry(absPath);
|
|
352
366
|
const metaEntry = {
|
|
353
367
|
hash: result.hash,
|
|
@@ -426,6 +440,9 @@ function initialSync(options) {
|
|
|
426
440
|
finally { if (e_1) throw e_1.error; }
|
|
427
441
|
}
|
|
428
442
|
yield Promise.allSettled(activeTasks);
|
|
443
|
+
walkTimer();
|
|
444
|
+
(0, logger_1.log)("index", `Walk: ${walkedFiles} files`);
|
|
445
|
+
(0, logger_1.log)("index", `Embed: ${indexed} new, ${cacheHits} cached, ${failedFiles} failed`);
|
|
429
446
|
if (signal === null || signal === void 0 ? void 0 : signal.aborted) {
|
|
430
447
|
shouldSkipCleanup = true;
|
|
431
448
|
}
|
|
@@ -436,6 +453,7 @@ function initialSync(options) {
|
|
|
436
453
|
: new Error(String(flushError));
|
|
437
454
|
}
|
|
438
455
|
if (!dryRun) {
|
|
456
|
+
const ftsTimer = (0, logger_1.timer)("index", "FTS");
|
|
439
457
|
onProgress === null || onProgress === void 0 ? void 0 : onProgress({
|
|
440
458
|
processed,
|
|
441
459
|
indexed,
|
|
@@ -443,10 +461,12 @@ function initialSync(options) {
|
|
|
443
461
|
filePath: "Creating FTS index...",
|
|
444
462
|
});
|
|
445
463
|
yield vectorDb.createFTSIndex();
|
|
464
|
+
ftsTimer();
|
|
446
465
|
}
|
|
447
466
|
// Stale cleanup: only remove paths scoped to this project's root
|
|
448
467
|
const stale = Array.from(cachedPaths).filter((p) => !seenPaths.has(p));
|
|
449
468
|
if (!dryRun && stale.length > 0 && !shouldSkipCleanup) {
|
|
469
|
+
(0, logger_1.log)("index", `Stale cleanup: ${stale.length} paths`);
|
|
450
470
|
yield vectorDb.deletePaths(stale);
|
|
451
471
|
stale.forEach((p) => {
|
|
452
472
|
metaCache.delete(p);
|
|
@@ -454,6 +474,7 @@ function initialSync(options) {
|
|
|
454
474
|
}
|
|
455
475
|
// --- Summary post-processing (sequential, single process) ---
|
|
456
476
|
if (!dryRun && indexed > 0) {
|
|
477
|
+
const sumTimer = (0, logger_1.timer)("index", "Summarize");
|
|
457
478
|
onProgress === null || onProgress === void 0 ? void 0 : onProgress({
|
|
458
479
|
processed,
|
|
459
480
|
indexed,
|
|
@@ -468,15 +489,10 @@ function initialSync(options) {
|
|
|
468
489
|
filePath: `Summarizing... (${count}/${chunkTotal})`,
|
|
469
490
|
});
|
|
470
491
|
});
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
processed,
|
|
474
|
-
indexed,
|
|
475
|
-
total,
|
|
476
|
-
filePath: `Summarized ${summarized} chunks`,
|
|
477
|
-
});
|
|
478
|
-
}
|
|
492
|
+
sumTimer();
|
|
493
|
+
(0, logger_1.log)("index", `Summarize: ${summarized} chunks`);
|
|
479
494
|
}
|
|
495
|
+
syncTimer();
|
|
480
496
|
// Write model config so future runs can detect model changes
|
|
481
497
|
if (!dryRun) {
|
|
482
498
|
(0, index_config_1.writeIndexConfig)(paths.configPath);
|
|
@@ -48,6 +48,7 @@ const fs = __importStar(require("node:fs"));
|
|
|
48
48
|
const path = __importStar(require("node:path"));
|
|
49
49
|
const chokidar_1 = require("chokidar");
|
|
50
50
|
const file_utils_1 = require("../utils/file-utils");
|
|
51
|
+
const logger_1 = require("../utils/logger");
|
|
51
52
|
const lock_1 = require("../utils/lock");
|
|
52
53
|
const pool_1 = require("../workers/pool");
|
|
53
54
|
const llm_client_1 = require("../workers/summarize/llm-client");
|
|
@@ -102,6 +103,7 @@ function startWatcher(opts) {
|
|
|
102
103
|
processing = true;
|
|
103
104
|
const batch = new Map(pending);
|
|
104
105
|
pending.clear();
|
|
106
|
+
(0, logger_1.log)("watch", `Processing ${batch.size} changed files`);
|
|
105
107
|
const start = Date.now();
|
|
106
108
|
let reindexed = 0;
|
|
107
109
|
const changedIds = [];
|
|
@@ -47,6 +47,7 @@ const fs = __importStar(require("node:fs"));
|
|
|
47
47
|
const lancedb = __importStar(require("@lancedb/lancedb"));
|
|
48
48
|
const apache_arrow_1 = require("apache-arrow");
|
|
49
49
|
const config_1 = require("../../config");
|
|
50
|
+
const logger_1 = require("../utils/logger");
|
|
50
51
|
const cleanup_1 = require("../utils/cleanup");
|
|
51
52
|
const TABLE_NAME = "chunks";
|
|
52
53
|
class VectorDB {
|
|
@@ -151,6 +152,7 @@ class VectorDB {
|
|
|
151
152
|
return table;
|
|
152
153
|
}
|
|
153
154
|
catch (_err) {
|
|
155
|
+
(0, logger_1.log)("db", `Creating table (${this.vectorDim}d)`);
|
|
154
156
|
const schema = this.buildSchema();
|
|
155
157
|
const table = yield db.createTable(TABLE_NAME, [this.seedRow()], {
|
|
156
158
|
schema,
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.log = log;
|
|
4
|
+
exports.debug = debug;
|
|
5
|
+
exports.timer = timer;
|
|
6
|
+
const VERBOSE = process.env.GMAX_DEBUG === "1" || process.env.GMAX_VERBOSE === "1";
|
|
7
|
+
function log(tag, msg) {
|
|
8
|
+
process.stderr.write(`[${tag}] ${msg}\n`);
|
|
9
|
+
}
|
|
10
|
+
function debug(tag, msg) {
|
|
11
|
+
if (VERBOSE)
|
|
12
|
+
process.stderr.write(`[${tag}] ${msg}\n`);
|
|
13
|
+
}
|
|
14
|
+
function timer(tag, label) {
|
|
15
|
+
const start = Date.now();
|
|
16
|
+
return () => {
|
|
17
|
+
const ms = Date.now() - start;
|
|
18
|
+
const elapsed = ms > 60000
|
|
19
|
+
? `${(ms / 60000).toFixed(1)}min`
|
|
20
|
+
: `${(ms / 1000).toFixed(1)}s`;
|
|
21
|
+
log(tag, `${label}: ${elapsed}`);
|
|
22
|
+
};
|
|
23
|
+
}
|
package/dist/lib/workers/pool.js
CHANGED
|
@@ -51,6 +51,7 @@ exports.isWorkerPoolInitialized = isWorkerPoolInitialized;
|
|
|
51
51
|
* to ensure the ONNX Runtime segfaults do not crash the main process.
|
|
52
52
|
*/
|
|
53
53
|
const childProcess = __importStar(require("node:child_process"));
|
|
54
|
+
const logger_1 = require("../utils/logger");
|
|
54
55
|
const fs = __importStar(require("node:fs"));
|
|
55
56
|
const path = __importStar(require("node:path"));
|
|
56
57
|
const config_1 = require("../../config");
|
|
@@ -149,6 +150,7 @@ class WorkerPool {
|
|
|
149
150
|
task.reject(new Error(`Worker exited unexpectedly${code ? ` (code ${code})` : ""}${signal ? ` signal ${signal}` : ""}`));
|
|
150
151
|
this.completeTask(task, null);
|
|
151
152
|
}
|
|
153
|
+
(0, logger_1.log)("pool", `Worker PID:${worker.child.pid} exited (code:${code} signal:${signal})`);
|
|
152
154
|
this.workers = this.workers.filter((w) => w !== worker);
|
|
153
155
|
if (!this.destroyed) {
|
|
154
156
|
this.spawnWorker();
|
|
@@ -157,6 +159,7 @@ class WorkerPool {
|
|
|
157
159
|
}
|
|
158
160
|
spawnWorker() {
|
|
159
161
|
const worker = new ProcessWorker(this.modulePath, this.execArgv);
|
|
162
|
+
(0, logger_1.debug)("pool", `Spawned worker PID:${worker.child.pid}`);
|
|
160
163
|
const onMessage = (msg) => {
|
|
161
164
|
const task = this.tasks.get(msg.id);
|
|
162
165
|
if (!task)
|
|
@@ -100,6 +100,7 @@ function summarizeChunks(chunks) {
|
|
|
100
100
|
return [];
|
|
101
101
|
const { ok, data } = yield postJSON("/summarize", { chunks });
|
|
102
102
|
if (!ok || !(data === null || data === void 0 ? void 0 : data.summaries)) {
|
|
103
|
+
process.stderr.write("[summarizer] Request failed or server unavailable\n");
|
|
103
104
|
return null;
|
|
104
105
|
}
|
|
105
106
|
return data.summaries;
|
package/package.json
CHANGED