grepmax 0.16.9 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/watch.js +0 -3
- package/dist/eval.js +50 -21
- package/dist/lib/daemon/daemon.js +25 -5
- package/dist/lib/daemon/ipc-handler.js +21 -0
- package/dist/lib/utils/daemon-client.js +5 -0
- package/dist/lib/utils/watcher-store.js +0 -25
- package/dist/lib/workers/embeddings/mlx-client.js +0 -8
- package/dist/lib/workers/orchestrator.js +39 -22
- package/dist/lib/workers/pool.js +28 -5
- package/package.json +3 -1
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
- package/dist/lib/utils/watcher-registry.js +0 -121
package/dist/commands/watch.js
CHANGED
|
@@ -119,7 +119,6 @@ exports.watch = new commander_1.Command("watch")
|
|
|
119
119
|
process.exit(0);
|
|
120
120
|
}
|
|
121
121
|
// Daemon foreground
|
|
122
|
-
(0, watcher_store_1.migrateFromJson)();
|
|
123
122
|
const { Daemon } = yield Promise.resolve().then(() => __importStar(require("../lib/daemon/daemon")));
|
|
124
123
|
const daemon = new Daemon();
|
|
125
124
|
try {
|
|
@@ -176,8 +175,6 @@ exports.watch = new commander_1.Command("watch")
|
|
|
176
175
|
process.exit(0);
|
|
177
176
|
}
|
|
178
177
|
// --- Per-project foreground mode ---
|
|
179
|
-
// Migrate legacy watchers.json to LMDB on first use
|
|
180
|
-
(0, watcher_store_1.migrateFromJson)();
|
|
181
178
|
// Watcher requires project to be registered
|
|
182
179
|
if (!(0, project_registry_1.getProject)(projectRoot)) {
|
|
183
180
|
console.error(`[watch:${projectName}] Project not registered. Run: gmax add ${projectRoot}`);
|
package/dist/eval.js
CHANGED
|
@@ -572,11 +572,19 @@ function run() {
|
|
|
572
572
|
process.exit(1);
|
|
573
573
|
}
|
|
574
574
|
const results = [];
|
|
575
|
-
|
|
575
|
+
const jsonModeEarly = process.env.GMAX_EVAL_JSON === "1" || process.argv.includes("--json");
|
|
576
|
+
// In JSON mode, route all human-readable preamble to stderr so stdout
|
|
577
|
+
// stays a single parseable JSON object.
|
|
578
|
+
const log = jsonModeEarly ? console.error : console.log;
|
|
579
|
+
log("Starting evaluation...\n");
|
|
576
580
|
const startTime = performance.now();
|
|
581
|
+
// Rerank is OFF by default — measures pre-rerank quality so ranking-only
|
|
582
|
+
// changes show up clearly. Set GMAX_EVAL_RERANK=1 to measure the full
|
|
583
|
+
// production pipeline (slower, but more representative).
|
|
584
|
+
const rerank = process.env.GMAX_EVAL_RERANK === "1";
|
|
577
585
|
for (const c of exports.cases) {
|
|
578
586
|
const queryStart = performance.now();
|
|
579
|
-
const res = yield searcher.search(c.query, topK, { rerank
|
|
587
|
+
const res = yield searcher.search(c.query, topK, { rerank });
|
|
580
588
|
const queryEnd = performance.now();
|
|
581
589
|
const timeMs = queryEnd - queryStart;
|
|
582
590
|
results.push(evaluateCase(res, c, timeMs));
|
|
@@ -585,25 +593,46 @@ function run() {
|
|
|
585
593
|
const mrr = results.reduce((sum, r) => sum + r.rr, 0) / results.length;
|
|
586
594
|
const recallAt10 = results.reduce((sum, r) => sum + r.recall, 0) / results.length;
|
|
587
595
|
const avgTime = results.reduce((sum, r) => sum + r.timeMs, 0) / results.length;
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
596
|
+
const summary = {
|
|
597
|
+
cases: results.length,
|
|
598
|
+
found: results.filter((r) => r.found).length,
|
|
599
|
+
hitsAt1: results.filter((r) => r.rr === 1).length,
|
|
600
|
+
mrrAt10: Number(mrr.toFixed(4)),
|
|
601
|
+
recallAt10: Number(recallAt10.toFixed(4)),
|
|
602
|
+
avgTimeMs: Math.round(avgTime),
|
|
603
|
+
totalTimeMs: Math.round(totalTime),
|
|
604
|
+
storePath: paths.lancedbDir,
|
|
605
|
+
rerank,
|
|
606
|
+
};
|
|
607
|
+
// JSON mode (GMAX_EVAL_JSON=1 or --json arg) emits a single stable object
|
|
608
|
+
// on stdout for tooling. Human output goes to stderr so both can be
|
|
609
|
+
// captured separately. Public-compatible shape — adding fixtures later
|
|
610
|
+
// doesn't change the schema.
|
|
611
|
+
const jsonMode = process.env.GMAX_EVAL_JSON === "1" || process.argv.includes("--json");
|
|
612
|
+
if (jsonMode) {
|
|
613
|
+
process.stdout.write(`${JSON.stringify({ summary, results }, null, 2)}\n`);
|
|
614
|
+
}
|
|
615
|
+
else {
|
|
616
|
+
console.log("=".repeat(80));
|
|
617
|
+
console.log(`Eval results for store at: ${paths.lancedbDir}`);
|
|
618
|
+
console.log("=".repeat(80));
|
|
619
|
+
results.forEach((r) => {
|
|
620
|
+
const status = r.found ? `rank ${(1 / r.rr).toFixed(0)}` : "❌ missed";
|
|
621
|
+
const emoji = r.found ? (r.rr === 1 ? "🎯" : "✓") : "❌";
|
|
622
|
+
console.log(`${emoji} ${r.query}`);
|
|
623
|
+
console.log(` => ${status} (target: ${r.path}) [${r.timeMs.toFixed(0)}ms]`);
|
|
624
|
+
if (r.note) {
|
|
625
|
+
console.log(` // ${r.note}`);
|
|
626
|
+
}
|
|
627
|
+
});
|
|
628
|
+
console.log("=".repeat(80));
|
|
629
|
+
console.log(`MRR: ${mrr.toFixed(3)}`);
|
|
630
|
+
console.log(`Recall@10: ${recallAt10.toFixed(3)}`);
|
|
631
|
+
console.log(`Avg query time: ${avgTime.toFixed(0)}ms`);
|
|
632
|
+
console.log(`Total time: ${totalTime.toFixed(0)}ms`);
|
|
633
|
+
console.log(`Found: ${summary.found}/${results.length}`);
|
|
634
|
+
console.log("=".repeat(80));
|
|
635
|
+
}
|
|
607
636
|
yield (0, exit_1.gracefulExit)(0);
|
|
608
637
|
});
|
|
609
638
|
}
|
|
@@ -398,11 +398,11 @@ class Daemon {
|
|
|
398
398
|
});
|
|
399
399
|
this.processors.set(root, processor);
|
|
400
400
|
// Subscribe with @parcel/watcher — native backend, no polling.
|
|
401
|
-
// If the kernel refuses (e.g. FSEvents slots stuck after a prior kill -9
|
|
402
|
-
//
|
|
403
|
-
//
|
|
404
|
-
//
|
|
405
|
-
//
|
|
401
|
+
// If the kernel refuses (e.g. FSEvents slots stuck after a prior kill -9),
|
|
402
|
+
// fall straight through to poll mode. The retry/backoff path inside
|
|
403
|
+
// recoverWatcher is for transient overflows, not hard kernel-level
|
|
404
|
+
// subscribe failures, so we skip it on startup by priming failCount past
|
|
405
|
+
// MAX before invoking it.
|
|
406
406
|
try {
|
|
407
407
|
yield this.subscribeWatcher(root, processor);
|
|
408
408
|
}
|
|
@@ -910,6 +910,7 @@ class Daemon {
|
|
|
910
910
|
conn.on("close", () => ac.abort());
|
|
911
911
|
this.shutdownAbortControllers.add(ac);
|
|
912
912
|
this.vectorDb.pauseMaintenanceLoop();
|
|
913
|
+
const stopHeartbeat = (0, ipc_handler_1.startHeartbeat)(conn);
|
|
913
914
|
let lastProgressTime = 0;
|
|
914
915
|
try {
|
|
915
916
|
const result = yield (0, syncer_1.initialSync)({
|
|
@@ -934,6 +935,7 @@ class Daemon {
|
|
|
934
935
|
if (!this.shuttingDown) {
|
|
935
936
|
yield this.watchProject(root);
|
|
936
937
|
}
|
|
938
|
+
stopHeartbeat();
|
|
937
939
|
(0, ipc_handler_1.writeDone)(conn, {
|
|
938
940
|
ok: true,
|
|
939
941
|
processed: result.processed,
|
|
@@ -945,9 +947,11 @@ class Daemon {
|
|
|
945
947
|
catch (err) {
|
|
946
948
|
const msg = err instanceof Error ? err.message : String(err);
|
|
947
949
|
console.error(`[daemon] addProject failed for ${path.basename(root)}:`, msg);
|
|
950
|
+
stopHeartbeat();
|
|
948
951
|
(0, ipc_handler_1.writeDone)(conn, { ok: false, error: msg });
|
|
949
952
|
}
|
|
950
953
|
finally {
|
|
954
|
+
stopHeartbeat();
|
|
951
955
|
this.shutdownAbortControllers.delete(ac);
|
|
952
956
|
(_a = this.vectorDb) === null || _a === void 0 ? void 0 : _a.resumeMaintenanceLoop();
|
|
953
957
|
}
|
|
@@ -977,6 +981,7 @@ class Daemon {
|
|
|
977
981
|
conn.on("close", () => ac.abort());
|
|
978
982
|
this.shutdownAbortControllers.add(ac);
|
|
979
983
|
this.vectorDb.pauseMaintenanceLoop();
|
|
984
|
+
const stopHeartbeat = (0, ipc_handler_1.startHeartbeat)(conn);
|
|
980
985
|
let lastProgressTime = 0;
|
|
981
986
|
try {
|
|
982
987
|
const result = yield (0, syncer_1.initialSync)({
|
|
@@ -1000,6 +1005,7 @@ class Daemon {
|
|
|
1000
1005
|
});
|
|
1001
1006
|
},
|
|
1002
1007
|
});
|
|
1008
|
+
stopHeartbeat();
|
|
1003
1009
|
(0, ipc_handler_1.writeDone)(conn, {
|
|
1004
1010
|
ok: true,
|
|
1005
1011
|
processed: result.processed,
|
|
@@ -1011,9 +1017,11 @@ class Daemon {
|
|
|
1011
1017
|
catch (err) {
|
|
1012
1018
|
const msg = err instanceof Error ? err.message : String(err);
|
|
1013
1019
|
console.error(`[daemon] indexProject failed for ${path.basename(root)}:`, msg);
|
|
1020
|
+
stopHeartbeat();
|
|
1014
1021
|
(0, ipc_handler_1.writeDone)(conn, { ok: false, error: msg });
|
|
1015
1022
|
}
|
|
1016
1023
|
finally {
|
|
1024
|
+
stopHeartbeat();
|
|
1017
1025
|
this.shutdownAbortControllers.delete(ac);
|
|
1018
1026
|
(_a = this.vectorDb) === null || _a === void 0 ? void 0 : _a.resumeMaintenanceLoop();
|
|
1019
1027
|
// Re-enable watcher (skip if shutting down)
|
|
@@ -1036,6 +1044,7 @@ class Daemon {
|
|
|
1036
1044
|
(0, ipc_handler_1.writeDone)(conn, { ok: false, error: "daemon resources not ready" });
|
|
1037
1045
|
return;
|
|
1038
1046
|
}
|
|
1047
|
+
const stopHeartbeat = (0, ipc_handler_1.startHeartbeat)(conn);
|
|
1039
1048
|
try {
|
|
1040
1049
|
yield this.unwatchProject(root);
|
|
1041
1050
|
const rootPrefix = root.endsWith("/") ? root : `${root}/`;
|
|
@@ -1044,13 +1053,18 @@ class Daemon {
|
|
|
1044
1053
|
for (const key of keys) {
|
|
1045
1054
|
this.metaCache.delete(key);
|
|
1046
1055
|
}
|
|
1056
|
+
stopHeartbeat();
|
|
1047
1057
|
(0, ipc_handler_1.writeDone)(conn, { ok: true });
|
|
1048
1058
|
}
|
|
1049
1059
|
catch (err) {
|
|
1050
1060
|
const msg = err instanceof Error ? err.message : String(err);
|
|
1051
1061
|
console.error(`[daemon] removeProject failed for ${path.basename(root)}:`, msg);
|
|
1062
|
+
stopHeartbeat();
|
|
1052
1063
|
(0, ipc_handler_1.writeDone)(conn, { ok: false, error: msg });
|
|
1053
1064
|
}
|
|
1065
|
+
finally {
|
|
1066
|
+
stopHeartbeat();
|
|
1067
|
+
}
|
|
1054
1068
|
}));
|
|
1055
1069
|
});
|
|
1056
1070
|
}
|
|
@@ -1063,6 +1077,7 @@ class Daemon {
|
|
|
1063
1077
|
return;
|
|
1064
1078
|
}
|
|
1065
1079
|
const rootPrefix = (_a = opts.pathPrefix) !== null && _a !== void 0 ? _a : (root.endsWith("/") ? root : `${root}/`);
|
|
1080
|
+
const stopHeartbeat = (0, ipc_handler_1.startHeartbeat)(conn);
|
|
1066
1081
|
let lastProgressTime = 0;
|
|
1067
1082
|
try {
|
|
1068
1083
|
const result = yield (0, syncer_1.generateSummaries)(this.vectorDb, rootPrefix, (done, total) => {
|
|
@@ -1073,6 +1088,7 @@ class Daemon {
|
|
|
1073
1088
|
lastProgressTime = now;
|
|
1074
1089
|
(0, ipc_handler_1.writeProgress)(conn, { summarized: done, total });
|
|
1075
1090
|
}, opts.limit);
|
|
1091
|
+
stopHeartbeat();
|
|
1076
1092
|
(0, ipc_handler_1.writeDone)(conn, {
|
|
1077
1093
|
ok: true,
|
|
1078
1094
|
summarized: result.summarized,
|
|
@@ -1082,8 +1098,12 @@ class Daemon {
|
|
|
1082
1098
|
catch (err) {
|
|
1083
1099
|
const msg = err instanceof Error ? err.message : String(err);
|
|
1084
1100
|
console.error(`[daemon] summarizeProject failed for ${path.basename(root)}:`, msg);
|
|
1101
|
+
stopHeartbeat();
|
|
1085
1102
|
(0, ipc_handler_1.writeDone)(conn, { ok: false, error: msg });
|
|
1086
1103
|
}
|
|
1104
|
+
finally {
|
|
1105
|
+
stopHeartbeat();
|
|
1106
|
+
}
|
|
1087
1107
|
}));
|
|
1088
1108
|
});
|
|
1089
1109
|
}
|
|
@@ -44,6 +44,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
44
44
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
45
|
exports.writeProgress = writeProgress;
|
|
46
46
|
exports.writeDone = writeDone;
|
|
47
|
+
exports.startHeartbeat = startHeartbeat;
|
|
47
48
|
exports.handleCommand = handleCommand;
|
|
48
49
|
const fs = __importStar(require("node:fs"));
|
|
49
50
|
const path = __importStar(require("node:path"));
|
|
@@ -73,6 +74,26 @@ function writeDone(conn, data) {
|
|
|
73
74
|
conn.write(`${JSON.stringify(Object.assign({ type: "done" }, data))}\n`);
|
|
74
75
|
conn.end();
|
|
75
76
|
}
|
|
77
|
+
const HEARTBEAT_INTERVAL_MS = 5000;
|
|
78
|
+
/**
|
|
79
|
+
* Emit periodic heartbeat lines so the client's watchdog timer resets even
|
|
80
|
+
* when no progress is reported (e.g., during a long DB flush or compaction).
|
|
81
|
+
* The client treats heartbeat lines as proof-of-life but not as progress.
|
|
82
|
+
*
|
|
83
|
+
* Returns a stop function; caller MUST call it before writeDone to avoid a
|
|
84
|
+
* stray heartbeat racing the done line.
|
|
85
|
+
*/
|
|
86
|
+
function startHeartbeat(conn, intervalMs = HEARTBEAT_INTERVAL_MS) {
|
|
87
|
+
const timer = setInterval(() => {
|
|
88
|
+
if (!conn.writable) {
|
|
89
|
+
clearInterval(timer);
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
conn.write(`${JSON.stringify({ type: "heartbeat", ts: Date.now() })}\n`);
|
|
93
|
+
}, intervalMs);
|
|
94
|
+
conn.once("close", () => clearInterval(timer));
|
|
95
|
+
return () => clearInterval(timer);
|
|
96
|
+
}
|
|
76
97
|
/**
|
|
77
98
|
* Handle a single IPC command.
|
|
78
99
|
*
|
|
@@ -219,6 +219,11 @@ function sendStreamingCommand(cmd, onProgress, opts) {
|
|
|
219
219
|
resetTimer();
|
|
220
220
|
onProgress(msg);
|
|
221
221
|
}
|
|
222
|
+
else if (msg.type === "heartbeat") {
|
|
223
|
+
// Proof-of-life from a daemon doing slow non-emitting work
|
|
224
|
+
// (DB flush, compaction). Reset the watchdog; do not surface.
|
|
225
|
+
resetTimer();
|
|
226
|
+
}
|
|
222
227
|
}
|
|
223
228
|
catch (_a) {
|
|
224
229
|
console.warn("[daemon-client] Malformed response line:", line.slice(0, 200));
|
|
@@ -49,7 +49,6 @@ exports.unregisterWatcher = unregisterWatcher;
|
|
|
49
49
|
exports.getWatcherForProject = getWatcherForProject;
|
|
50
50
|
exports.getWatcherCoveringPath = getWatcherCoveringPath;
|
|
51
51
|
exports.listWatchers = listWatchers;
|
|
52
|
-
exports.migrateFromJson = migrateFromJson;
|
|
53
52
|
exports.registerDaemon = registerDaemon;
|
|
54
53
|
exports.unregisterDaemon = unregisterDaemon;
|
|
55
54
|
exports.getDaemonInfo = getDaemonInfo;
|
|
@@ -173,30 +172,6 @@ function listWatchers() {
|
|
|
173
172
|
}
|
|
174
173
|
return alive;
|
|
175
174
|
}
|
|
176
|
-
/**
|
|
177
|
-
* Migrate from legacy watchers.json if it exists.
|
|
178
|
-
* Call once on startup.
|
|
179
|
-
*/
|
|
180
|
-
function migrateFromJson() {
|
|
181
|
-
const jsonPath = path.join(config_1.PATHS.globalRoot, "watchers.json");
|
|
182
|
-
if (!fs.existsSync(jsonPath))
|
|
183
|
-
return;
|
|
184
|
-
try {
|
|
185
|
-
const raw = fs.readFileSync(jsonPath, "utf-8");
|
|
186
|
-
const entries = JSON.parse(raw);
|
|
187
|
-
const db = getDb();
|
|
188
|
-
for (const entry of entries) {
|
|
189
|
-
if (entry.projectRoot && isProcessRunning(entry.pid)) {
|
|
190
|
-
db.put(entry.projectRoot, Object.assign(Object.assign({}, entry), { lastHeartbeat: Date.now() }));
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
// Remove legacy file
|
|
194
|
-
fs.unlinkSync(jsonPath);
|
|
195
|
-
}
|
|
196
|
-
catch (_a) {
|
|
197
|
-
// Best effort — ignore
|
|
198
|
-
}
|
|
199
|
-
}
|
|
200
175
|
// --- Daemon registry ---
|
|
201
176
|
exports.DAEMON_KEY = "__daemon__";
|
|
202
177
|
function registerDaemon(pid) {
|
|
@@ -49,7 +49,6 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
49
49
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
50
50
|
exports.isMlxUp = isMlxUp;
|
|
51
51
|
exports.mlxEmbed = mlxEmbed;
|
|
52
|
-
exports.resetMlxCache = resetMlxCache;
|
|
53
52
|
const http = __importStar(require("node:http"));
|
|
54
53
|
const logger_1 = require("../../utils/logger");
|
|
55
54
|
const MLX_PORT = parseInt(process.env.MLX_EMBED_PORT || "8100", 10);
|
|
@@ -192,10 +191,3 @@ function mlxEmbed(texts) {
|
|
|
192
191
|
return data.vectors.map((v) => new Float32Array(v));
|
|
193
192
|
});
|
|
194
193
|
}
|
|
195
|
-
/**
|
|
196
|
-
* Reset availability cache (e.g., after starting the server).
|
|
197
|
-
*/
|
|
198
|
-
function resetMlxCache() {
|
|
199
|
-
mlxAvailable = null;
|
|
200
|
-
lastCheck = 0;
|
|
201
|
-
}
|
|
@@ -43,6 +43,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
43
43
|
};
|
|
44
44
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
45
|
exports.WorkerOrchestrator = void 0;
|
|
46
|
+
exports.coerceColbertBytes = coerceColbertBytes;
|
|
46
47
|
const fs = __importStar(require("node:fs"));
|
|
47
48
|
const path = __importStar(require("node:path"));
|
|
48
49
|
const transformers_1 = require("@huggingface/transformers");
|
|
@@ -58,6 +59,43 @@ const granite_1 = require("./embeddings/granite");
|
|
|
58
59
|
const mlx_client_1 = require("./embeddings/mlx-client");
|
|
59
60
|
const logger_1 = require("../utils/logger");
|
|
60
61
|
let mlxFallbackWarned = false;
|
|
62
|
+
/**
|
|
63
|
+
* Normalize a colbert payload received over IPC into an Int8Array.
|
|
64
|
+
*
|
|
65
|
+
* Node `child_process.send` serializes payloads as JSON, which doesn't
|
|
66
|
+
* preserve TypedArrays. An Int8Array sent over IPC arrives as a plain
|
|
67
|
+
* object with numeric keys (`{0: byte, 1: byte, ...}`). Without explicitly
|
|
68
|
+
* handling that shape, the rerank pipeline silently no-ops (empty matrix
|
|
69
|
+
* → maxSim returns 0 → final ranking falls back to fusion-only). Caught
|
|
70
|
+
* 2026-05-25 when the eval harness showed rerank-on producing identical
|
|
71
|
+
* scores to rerank-off across 97 cases.
|
|
72
|
+
*/
|
|
73
|
+
function coerceColbertBytes(col) {
|
|
74
|
+
if (col instanceof Int8Array)
|
|
75
|
+
return col;
|
|
76
|
+
if (Buffer.isBuffer(col)) {
|
|
77
|
+
return new Int8Array(col.buffer, col.byteOffset, col.byteLength);
|
|
78
|
+
}
|
|
79
|
+
if (Array.isArray(col))
|
|
80
|
+
return new Int8Array(col);
|
|
81
|
+
if (col && typeof col === "object") {
|
|
82
|
+
// {type:"Buffer", data:[...]} — Node Buffer.toJSON output
|
|
83
|
+
const asUnknown = col;
|
|
84
|
+
if (asUnknown.type === "Buffer" && Array.isArray(asUnknown.data)) {
|
|
85
|
+
return new Int8Array(asUnknown.data);
|
|
86
|
+
}
|
|
87
|
+
// {0: byte, 1: byte, ...} — Int8Array after IPC JSON serialization
|
|
88
|
+
const keys = Object.keys(col);
|
|
89
|
+
if (keys.length > 0 && keys.every((k) => /^\d+$/.test(k))) {
|
|
90
|
+
const arr = new Int8Array(keys.length);
|
|
91
|
+
for (const k of keys) {
|
|
92
|
+
arr[Number(k)] = col[k];
|
|
93
|
+
}
|
|
94
|
+
return arr;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
return new Int8Array(0);
|
|
98
|
+
}
|
|
61
99
|
const CACHE_DIR = config_1.PATHS.models;
|
|
62
100
|
const LOG_MODELS = process.env.GMAX_DEBUG_MODELS === "1" ||
|
|
63
101
|
process.env.GMAX_DEBUG_MODELS === "true";
|
|
@@ -330,28 +368,7 @@ class WorkerOrchestrator {
|
|
|
330
368
|
yield this.ensureReady();
|
|
331
369
|
const queryMatrix = input.query.map((row) => row instanceof Float32Array ? row : new Float32Array(row));
|
|
332
370
|
return input.docs.map((doc) => {
|
|
333
|
-
const
|
|
334
|
-
let colbert;
|
|
335
|
-
if (col instanceof Int8Array) {
|
|
336
|
-
colbert = col;
|
|
337
|
-
}
|
|
338
|
-
else if (Buffer.isBuffer(col)) {
|
|
339
|
-
colbert = new Int8Array(col.buffer, col.byteOffset, col.byteLength);
|
|
340
|
-
}
|
|
341
|
-
else if (col &&
|
|
342
|
-
typeof col === "object" &&
|
|
343
|
-
"type" in col &&
|
|
344
|
-
col.type === "Buffer" &&
|
|
345
|
-
Array.isArray(col.data)) {
|
|
346
|
-
// IPC serialization fallback (still copies, but unavoidable without SharedArrayBuffer)
|
|
347
|
-
colbert = new Int8Array(col.data);
|
|
348
|
-
}
|
|
349
|
-
else if (Array.isArray(col)) {
|
|
350
|
-
colbert = new Int8Array(col);
|
|
351
|
-
}
|
|
352
|
-
else {
|
|
353
|
-
colbert = new Int8Array(0);
|
|
354
|
-
}
|
|
371
|
+
const colbert = coerceColbertBytes(doc.colbert);
|
|
355
372
|
const seqLen = Math.floor(colbert.length / input.colbertDim);
|
|
356
373
|
const docMatrix = [];
|
|
357
374
|
for (let i = 0; i < seqLen; i++) {
|
package/dist/lib/workers/pool.js
CHANGED
|
@@ -90,6 +90,10 @@ class ProcessWorker {
|
|
|
90
90
|
this.busy = false;
|
|
91
91
|
this.pendingTaskId = null;
|
|
92
92
|
this.lastBusyTime = Date.now();
|
|
93
|
+
// Set when the pool has cleaned up after this worker (via exit or error
|
|
94
|
+
// event). Guards against handleWorkerExit running twice when both events
|
|
95
|
+
// fire for the same crash.
|
|
96
|
+
this.cleanedUp = false;
|
|
93
97
|
const memArgs = maxMemoryMb
|
|
94
98
|
? [`--max-old-space-size=${maxMemoryMb}`]
|
|
95
99
|
: [];
|
|
@@ -166,18 +170,26 @@ class WorkerPool {
|
|
|
166
170
|
worker.lastBusyTime = Date.now();
|
|
167
171
|
}
|
|
168
172
|
}
|
|
169
|
-
handleWorkerExit(worker, code, signal) {
|
|
173
|
+
handleWorkerExit(worker, code, signal, reason = "exit", err) {
|
|
170
174
|
var _a, _b, _c, _d;
|
|
175
|
+
// Crash paths can fire both 'error' and 'exit'. Either is sufficient
|
|
176
|
+
// to clean up; running this twice would double-respawn.
|
|
177
|
+
if (worker.cleanedUp)
|
|
178
|
+
return;
|
|
179
|
+
worker.cleanedUp = true;
|
|
171
180
|
worker.busy = false;
|
|
172
181
|
const failedTasks = Array.from(this.tasks.values()).filter((t) => t.worker === worker);
|
|
173
182
|
for (const task of failedTasks) {
|
|
174
183
|
this.clearTaskTimeout(task);
|
|
175
184
|
const filePath = (_d = (_b = (_a = task.payload) === null || _a === void 0 ? void 0 : _a.path) !== null && _b !== void 0 ? _b : (_c = task.payload) === null || _c === void 0 ? void 0 : _c.absolutePath) !== null && _d !== void 0 ? _d : "unknown";
|
|
176
|
-
(0, logger_1.debug)("pool",
|
|
177
|
-
|
|
185
|
+
(0, logger_1.debug)("pool", `${reason} killed task=${task.id} method=${task.method} file=${filePath}`);
|
|
186
|
+
const exitDetail = err
|
|
187
|
+
? `: ${err.message}`
|
|
188
|
+
: `${code ? ` (code ${code})` : ""}${signal ? ` signal ${signal}` : ""}`;
|
|
189
|
+
task.reject(new Error(`Worker ${reason === "error" ? "errored" : "exited unexpectedly"}${exitDetail}`));
|
|
178
190
|
this.completeTask(task, null);
|
|
179
191
|
}
|
|
180
|
-
(0, logger_1.log)("pool", `Worker PID:${worker.child.pid}
|
|
192
|
+
(0, logger_1.log)("pool", `Worker PID:${worker.child.pid} ${reason} (code:${code} signal:${signal}${err ? ` err:${err.message}` : ""} pending=${failedTasks.length})`);
|
|
181
193
|
this.workers = this.workers.filter((w) => w !== worker);
|
|
182
194
|
if (!this.destroyed) {
|
|
183
195
|
// Only respawn if we have no workers left or there are pending tasks
|
|
@@ -243,9 +255,14 @@ class WorkerPool {
|
|
|
243
255
|
this.consecutiveRespawns = 0;
|
|
244
256
|
this.dispatch();
|
|
245
257
|
};
|
|
246
|
-
const onExit = (code, signal) => this.handleWorkerExit(worker, code, signal);
|
|
258
|
+
const onExit = (code, signal) => this.handleWorkerExit(worker, code, signal, "exit");
|
|
259
|
+
// 'error' fires when spawn fails, IPC send fails async, or the child
|
|
260
|
+
// can't be killed. Without this handler the worker stays in
|
|
261
|
+
// this.workers as a zombie that the next dispatch tries to send to.
|
|
262
|
+
const onError = (err) => this.handleWorkerExit(worker, null, null, "error", err);
|
|
247
263
|
worker.child.on("message", onMessage);
|
|
248
264
|
worker.child.on("exit", onExit);
|
|
265
|
+
worker.child.on("error", onError);
|
|
249
266
|
this.workers.push(worker);
|
|
250
267
|
}
|
|
251
268
|
enqueue(method, payload, signal) {
|
|
@@ -326,8 +343,10 @@ class WorkerPool {
|
|
|
326
343
|
(0, logger_1.log)("pool", `timeout task=${task.id} method=${task.method} file=${filePath} after ${TASK_TIMEOUT_MS}ms — killing worker PID:${worker.child.pid}`);
|
|
327
344
|
this.completeTask(task, null);
|
|
328
345
|
task.reject(new Error(`Worker task ${task.method} timed out after ${TASK_TIMEOUT_MS}ms on ${filePath}`));
|
|
346
|
+
worker.cleanedUp = true;
|
|
329
347
|
worker.child.removeAllListeners("message");
|
|
330
348
|
worker.child.removeAllListeners("exit");
|
|
349
|
+
worker.child.removeAllListeners("error");
|
|
331
350
|
try {
|
|
332
351
|
worker.child.kill("SIGKILL");
|
|
333
352
|
}
|
|
@@ -421,8 +440,10 @@ class WorkerPool {
|
|
|
421
440
|
.slice(0, reapCount)
|
|
422
441
|
.forEach((w) => {
|
|
423
442
|
(0, logger_1.log)("pool", `reap idle worker PID:${w.child.pid} (idle ${Math.round((now - w.lastBusyTime) / 1000)}s, ${this.workers.length - 1} remaining)`);
|
|
443
|
+
w.cleanedUp = true;
|
|
424
444
|
w.child.removeAllListeners("message");
|
|
425
445
|
w.child.removeAllListeners("exit");
|
|
446
|
+
w.child.removeAllListeners("error");
|
|
426
447
|
try {
|
|
427
448
|
w.child.kill("SIGTERM");
|
|
428
449
|
}
|
|
@@ -449,8 +470,10 @@ class WorkerPool {
|
|
|
449
470
|
this.taskQueue = [];
|
|
450
471
|
this.priorityQueue = [];
|
|
451
472
|
const killPromises = this.workers.map((w) => new Promise((resolve) => {
|
|
473
|
+
w.cleanedUp = true;
|
|
452
474
|
w.child.removeAllListeners("message");
|
|
453
475
|
w.child.removeAllListeners("exit");
|
|
476
|
+
w.child.removeAllListeners("error");
|
|
454
477
|
w.child.once("exit", () => resolve());
|
|
455
478
|
w.child.kill("SIGTERM");
|
|
456
479
|
const force = setTimeout(() => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "grepmax",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.17.0",
|
|
4
4
|
"author": "Robert Owens <78518764+reowens@users.noreply.github.com>",
|
|
5
5
|
"homepage": "https://github.com/reowens/grepmax",
|
|
6
6
|
"bugs": {
|
|
@@ -26,6 +26,8 @@
|
|
|
26
26
|
"benchmark:index": "./run-benchmark.sh $HOME/gmax-benchmarks --index",
|
|
27
27
|
"benchmark:agent": "npx tsx src/bench/benchmark-agent.ts",
|
|
28
28
|
"benchmark:chart": "npx tsx src/bench/generate-benchmark-chart.ts",
|
|
29
|
+
"bench:recall": "npx tsx src/eval.ts",
|
|
30
|
+
"bench:recall:json": "GMAX_EVAL_JSON=1 npx tsx src/eval.ts",
|
|
29
31
|
"format": "biome check --write .",
|
|
30
32
|
"format:check": "biome check .",
|
|
31
33
|
"lint": "biome lint .",
|
|
@@ -1,121 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
/**
|
|
3
|
-
* Watcher registry — tracks background watcher processes per project.
|
|
4
|
-
* Ensures only one watcher runs per project root.
|
|
5
|
-
*
|
|
6
|
-
* Stored in ~/.gmax/watchers.json
|
|
7
|
-
*/
|
|
8
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
9
|
-
if (k2 === undefined) k2 = k;
|
|
10
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
11
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
12
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
13
|
-
}
|
|
14
|
-
Object.defineProperty(o, k2, desc);
|
|
15
|
-
}) : (function(o, m, k, k2) {
|
|
16
|
-
if (k2 === undefined) k2 = k;
|
|
17
|
-
o[k2] = m[k];
|
|
18
|
-
}));
|
|
19
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
20
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
21
|
-
}) : function(o, v) {
|
|
22
|
-
o["default"] = v;
|
|
23
|
-
});
|
|
24
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
25
|
-
var ownKeys = function(o) {
|
|
26
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
27
|
-
var ar = [];
|
|
28
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
29
|
-
return ar;
|
|
30
|
-
};
|
|
31
|
-
return ownKeys(o);
|
|
32
|
-
};
|
|
33
|
-
return function (mod) {
|
|
34
|
-
if (mod && mod.__esModule) return mod;
|
|
35
|
-
var result = {};
|
|
36
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
37
|
-
__setModuleDefault(result, mod);
|
|
38
|
-
return result;
|
|
39
|
-
};
|
|
40
|
-
})();
|
|
41
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
42
|
-
exports.isProcessRunning = isProcessRunning;
|
|
43
|
-
exports.registerWatcher = registerWatcher;
|
|
44
|
-
exports.updateWatcherStatus = updateWatcherStatus;
|
|
45
|
-
exports.unregisterWatcher = unregisterWatcher;
|
|
46
|
-
exports.getWatcherForProject = getWatcherForProject;
|
|
47
|
-
exports.getWatcherCoveringPath = getWatcherCoveringPath;
|
|
48
|
-
exports.listWatchers = listWatchers;
|
|
49
|
-
const fs = __importStar(require("node:fs"));
|
|
50
|
-
const path = __importStar(require("node:path"));
|
|
51
|
-
const config_1 = require("../../config");
|
|
52
|
-
const REGISTRY_PATH = path.join(config_1.PATHS.globalRoot, "watchers.json");
|
|
53
|
-
function loadRegistry() {
|
|
54
|
-
try {
|
|
55
|
-
const raw = fs.readFileSync(REGISTRY_PATH, "utf-8");
|
|
56
|
-
return JSON.parse(raw);
|
|
57
|
-
}
|
|
58
|
-
catch (_a) {
|
|
59
|
-
return [];
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
function saveRegistry(entries) {
|
|
63
|
-
fs.mkdirSync(path.dirname(REGISTRY_PATH), { recursive: true });
|
|
64
|
-
fs.writeFileSync(REGISTRY_PATH, `${JSON.stringify(entries, null, 2)}\n`);
|
|
65
|
-
}
|
|
66
|
-
function isProcessRunning(pid) {
|
|
67
|
-
try {
|
|
68
|
-
process.kill(pid, 0);
|
|
69
|
-
return true;
|
|
70
|
-
}
|
|
71
|
-
catch (_a) {
|
|
72
|
-
return false;
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
function registerWatcher(info) {
|
|
76
|
-
const entries = loadRegistry().filter((e) => e.projectRoot !== info.projectRoot);
|
|
77
|
-
entries.push(info);
|
|
78
|
-
saveRegistry(entries);
|
|
79
|
-
}
|
|
80
|
-
function updateWatcherStatus(pid, status, lastReindex) {
|
|
81
|
-
const entries = loadRegistry();
|
|
82
|
-
const match = entries.find((e) => e.pid === pid);
|
|
83
|
-
if (match) {
|
|
84
|
-
match.status = status;
|
|
85
|
-
if (lastReindex)
|
|
86
|
-
match.lastReindex = lastReindex;
|
|
87
|
-
saveRegistry(entries);
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
function unregisterWatcher(pid) {
|
|
91
|
-
const entries = loadRegistry().filter((e) => e.pid !== pid);
|
|
92
|
-
saveRegistry(entries);
|
|
93
|
-
}
|
|
94
|
-
function getWatcherForProject(projectRoot) {
|
|
95
|
-
const entries = loadRegistry();
|
|
96
|
-
const match = entries.find((e) => e.projectRoot === projectRoot);
|
|
97
|
-
if (match && isProcessRunning(match.pid))
|
|
98
|
-
return match;
|
|
99
|
-
// Clean stale entry
|
|
100
|
-
if (match) {
|
|
101
|
-
saveRegistry(entries.filter((e) => e.pid !== match.pid));
|
|
102
|
-
}
|
|
103
|
-
return undefined;
|
|
104
|
-
}
|
|
105
|
-
function getWatcherCoveringPath(dir) {
|
|
106
|
-
const resolved = path.resolve(dir);
|
|
107
|
-
const entries = loadRegistry();
|
|
108
|
-
for (const e of entries) {
|
|
109
|
-
if (resolved.startsWith(e.projectRoot) && isProcessRunning(e.pid))
|
|
110
|
-
return e;
|
|
111
|
-
}
|
|
112
|
-
return undefined;
|
|
113
|
-
}
|
|
114
|
-
function listWatchers() {
|
|
115
|
-
const entries = loadRegistry();
|
|
116
|
-
const active = entries.filter((e) => isProcessRunning(e.pid));
|
|
117
|
-
if (active.length !== entries.length) {
|
|
118
|
-
saveRegistry(active);
|
|
119
|
-
}
|
|
120
|
-
return active;
|
|
121
|
-
}
|