grepmax 0.23.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +25 -6
- package/dist/commands/add.js +38 -31
- package/dist/commands/config.js +16 -15
- package/dist/commands/context.js +38 -13
- package/dist/commands/doctor.js +76 -83
- package/dist/commands/extract.js +12 -1
- package/dist/commands/impact.js +33 -1
- package/dist/commands/index.js +23 -24
- package/dist/commands/list.js +23 -14
- package/dist/commands/mcp.js +494 -162
- package/dist/commands/peek.js +16 -5
- package/dist/commands/repair.js +54 -120
- package/dist/commands/search-output.js +30 -9
- package/dist/commands/search-run.js +75 -47
- package/dist/commands/search-skeletons.js +28 -18
- package/dist/commands/search.js +45 -49
- package/dist/commands/serve.js +415 -373
- package/dist/commands/setup.js +2 -2
- package/dist/commands/similar.js +5 -5
- package/dist/commands/skeleton.js +67 -41
- package/dist/commands/status.js +5 -2
- package/dist/commands/summarize.js +6 -0
- package/dist/commands/surprises.js +150 -0
- package/dist/commands/watch.js +102 -38
- package/dist/config.js +3 -0
- package/dist/eval-surprising-connections.js +191 -0
- package/dist/index.js +2 -0
- package/dist/lib/analysis/surprising-connections.js +600 -0
- package/dist/lib/daemon/daemon.js +1101 -379
- package/dist/lib/daemon/ipc-handler.js +122 -13
- package/dist/lib/daemon/mlx-server-manager.js +268 -125
- package/dist/lib/daemon/process-manager.js +7 -4
- package/dist/lib/daemon/search-handler.js +23 -9
- package/dist/lib/daemon/watcher-manager.js +353 -110
- package/dist/lib/graph/impact-rollup.js +202 -0
- package/dist/lib/graph/impact.js +15 -1
- package/dist/lib/help/agent-cheatsheet.js +1 -0
- package/dist/lib/index/batch-processor.js +231 -67
- package/dist/lib/index/embedding-generation.js +109 -0
- package/dist/lib/index/embedding-status.js +23 -0
- package/dist/lib/index/file-policy.js +273 -0
- package/dist/lib/index/ignore-patterns.js +4 -0
- package/dist/lib/index/index-config.js +18 -4
- package/dist/lib/index/syncer.js +256 -79
- package/dist/lib/index/walker.js +66 -86
- package/dist/lib/index/watcher-batch.js +9 -0
- package/dist/lib/index/watcher.js +129 -3
- package/dist/lib/llm/server.js +6 -0
- package/dist/lib/search/searcher.js +30 -19
- package/dist/lib/skeleton/skeletonizer.js +118 -0
- package/dist/lib/skeleton/summary-formatter.js +8 -1
- package/dist/lib/store/store-lease.js +473 -0
- package/dist/lib/store/vector-db.js +302 -57
- package/dist/lib/utils/blocked-roots.js +63 -0
- package/dist/lib/utils/cross-project.js +7 -3
- package/dist/lib/utils/daemon-client.js +24 -1
- package/dist/lib/utils/daemon-launcher.js +38 -13
- package/dist/lib/utils/doctor-status.js +76 -0
- package/dist/lib/utils/file-utils.js +74 -4
- package/dist/lib/utils/keyed-mutex.js +101 -0
- package/dist/lib/utils/logger.js +57 -1
- package/dist/lib/utils/mlx-hf-cache.js +114 -0
- package/dist/lib/utils/operation-coordinator.js +146 -0
- package/dist/lib/utils/path-containment.js +106 -0
- package/dist/lib/utils/process.js +44 -0
- package/dist/lib/utils/project-registry.js +351 -3
- package/dist/lib/utils/scope-filter.js +3 -9
- package/dist/lib/utils/stale-hint.js +12 -19
- package/dist/lib/utils/watcher-launcher.js +5 -1
- package/dist/lib/utils/watcher-store.js +2 -2
- package/dist/lib/workers/colbert-math.js +15 -12
- package/dist/lib/workers/embeddings/colbert.js +3 -2
- package/dist/lib/workers/embeddings/granite.js +4 -3
- package/dist/lib/workers/embeddings/mlx-client.js +59 -16
- package/dist/lib/workers/orchestrator.js +39 -8
- package/dist/lib/workers/pool.js +150 -83
- package/dist/lib/workers/process-child.js +11 -2
- package/dist/lib/workers/serialized-handler.js +10 -0
- package/dist/lib/workers/worker.js +13 -4
- package/mlx-embed-server/server.py +21 -3
- package/mlx-embed-server/summarizer.py +8 -0
- package/package.json +6 -3
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
- package/plugins/grepmax/hooks/start.js +3 -170
|
@@ -18,7 +18,7 @@ const project_registry_1 = require("./project-registry");
|
|
|
18
18
|
function resolveCrossProjectScope(opts) {
|
|
19
19
|
const active = !!(opts.allProjects || opts.projects);
|
|
20
20
|
if (!active) {
|
|
21
|
-
return { active: false, roots: [], warnings: [] };
|
|
21
|
+
return { active: false, roots: [], projectRoots: [], warnings: [] };
|
|
22
22
|
}
|
|
23
23
|
// Ignore "error"-status projects: the daemon won't search them anyway.
|
|
24
24
|
const all = (0, project_registry_1.listProjects)().filter((p) => p.status !== "error");
|
|
@@ -62,9 +62,12 @@ function resolveCrossProjectScope(opts) {
|
|
|
62
62
|
: undefined;
|
|
63
63
|
}
|
|
64
64
|
else {
|
|
65
|
-
// --all-projects
|
|
66
|
-
//
|
|
65
|
+
// --all-projects still emits an explicit allow-list. The physical table can
|
|
66
|
+
// contain orphan or failed-project rows that are not eligible here.
|
|
67
67
|
included = all.filter((p) => !excludedRoots.has(p.root));
|
|
68
|
+
projectRootsCsv = included.length
|
|
69
|
+
? included.map((p) => p.root).join(",")
|
|
70
|
+
: undefined;
|
|
68
71
|
if (excludedRoots.size) {
|
|
69
72
|
excludeProjectRootsCsv = [...excludedRoots].join(",");
|
|
70
73
|
}
|
|
@@ -72,6 +75,7 @@ function resolveCrossProjectScope(opts) {
|
|
|
72
75
|
return {
|
|
73
76
|
active: true,
|
|
74
77
|
roots: included.map((p) => ({ root: p.root, name: p.name })),
|
|
78
|
+
projectRoots: included.map((p) => p.root),
|
|
75
79
|
projectRootsCsv,
|
|
76
80
|
excludeProjectRootsCsv,
|
|
77
81
|
warnings,
|
|
@@ -71,18 +71,28 @@ function sendDaemonCommand(cmd, opts) {
|
|
|
71
71
|
var _a;
|
|
72
72
|
const timeout = (_a = opts === null || opts === void 0 ? void 0 : opts.timeoutMs) !== null && _a !== void 0 ? _a : DEFAULT_TIMEOUT_MS;
|
|
73
73
|
return new Promise((resolve) => {
|
|
74
|
+
var _a, _b;
|
|
74
75
|
let settled = false;
|
|
75
76
|
const finish = (resp) => {
|
|
77
|
+
var _a;
|
|
76
78
|
if (settled)
|
|
77
79
|
return;
|
|
78
80
|
settled = true;
|
|
81
|
+
clearTimeout(timer);
|
|
82
|
+
(_a = opts === null || opts === void 0 ? void 0 : opts.signal) === null || _a === void 0 ? void 0 : _a.removeEventListener("abort", onAbort);
|
|
79
83
|
socket.destroy();
|
|
80
84
|
resolve(resp);
|
|
81
85
|
};
|
|
82
86
|
const socket = net.createConnection({ path: config_1.PATHS.daemonSocket });
|
|
87
|
+
const onAbort = () => finish({ ok: false, error: "aborted" });
|
|
83
88
|
const timer = setTimeout(() => {
|
|
84
89
|
finish({ ok: false, error: "timeout" });
|
|
85
90
|
}, timeout);
|
|
91
|
+
if ((_a = opts === null || opts === void 0 ? void 0 : opts.signal) === null || _a === void 0 ? void 0 : _a.aborted) {
|
|
92
|
+
finish({ ok: false, error: "aborted" });
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
(_b = opts === null || opts === void 0 ? void 0 : opts.signal) === null || _b === void 0 ? void 0 : _b.addEventListener("abort", onAbort, { once: true });
|
|
86
96
|
socket.on("connect", () => {
|
|
87
97
|
socket.write(`${JSON.stringify(cmd)}\n`);
|
|
88
98
|
});
|
|
@@ -266,7 +276,7 @@ function ensureDaemonRunning() {
|
|
|
266
276
|
return true;
|
|
267
277
|
if (!(yield isDaemonRunning())) {
|
|
268
278
|
const { spawnDaemon } = yield Promise.resolve().then(() => __importStar(require("./daemon-launcher")));
|
|
269
|
-
const pid = spawnDaemon();
|
|
279
|
+
const pid = yield spawnDaemon();
|
|
270
280
|
if (!pid)
|
|
271
281
|
return false;
|
|
272
282
|
}
|
|
@@ -289,13 +299,16 @@ function sendStreamingCommand(cmd, onProgress, opts) {
|
|
|
289
299
|
var _a;
|
|
290
300
|
const timeout = (_a = opts === null || opts === void 0 ? void 0 : opts.timeoutMs) !== null && _a !== void 0 ? _a : DEFAULT_STREAMING_TIMEOUT_MS;
|
|
291
301
|
return new Promise((resolve, reject) => {
|
|
302
|
+
var _a, _b;
|
|
292
303
|
let settled = false;
|
|
293
304
|
let timer;
|
|
294
305
|
const finish = (result) => {
|
|
306
|
+
var _a;
|
|
295
307
|
if (settled)
|
|
296
308
|
return;
|
|
297
309
|
settled = true;
|
|
298
310
|
clearTimeout(timer);
|
|
311
|
+
(_a = opts === null || opts === void 0 ? void 0 : opts.signal) === null || _a === void 0 ? void 0 : _a.removeEventListener("abort", onAbort);
|
|
299
312
|
socket.destroy();
|
|
300
313
|
if (result instanceof Error)
|
|
301
314
|
reject(result);
|
|
@@ -309,7 +322,17 @@ function sendStreamingCommand(cmd, onProgress, opts) {
|
|
|
309
322
|
}, timeout);
|
|
310
323
|
};
|
|
311
324
|
const socket = net.createConnection({ path: config_1.PATHS.daemonSocket });
|
|
325
|
+
const onAbort = () => {
|
|
326
|
+
const error = new Error("Aborted");
|
|
327
|
+
error.name = "AbortError";
|
|
328
|
+
finish(error);
|
|
329
|
+
};
|
|
312
330
|
resetTimer();
|
|
331
|
+
if ((_a = opts === null || opts === void 0 ? void 0 : opts.signal) === null || _a === void 0 ? void 0 : _a.aborted) {
|
|
332
|
+
onAbort();
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
335
|
+
(_b = opts === null || opts === void 0 ? void 0 : opts.signal) === null || _b === void 0 ? void 0 : _b.addEventListener("abort", onAbort, { once: true });
|
|
313
336
|
socket.on("connect", () => {
|
|
314
337
|
socket.write(`${JSON.stringify(cmd)}\n`);
|
|
315
338
|
});
|
|
@@ -32,9 +32,19 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
32
32
|
return result;
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
36
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
37
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
38
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
39
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
40
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
41
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
|
+
});
|
|
43
|
+
};
|
|
35
44
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
45
|
exports.spawnDaemon = spawnDaemon;
|
|
37
46
|
const node_child_process_1 = require("node:child_process");
|
|
47
|
+
const fs = __importStar(require("node:fs"));
|
|
38
48
|
const path = __importStar(require("node:path"));
|
|
39
49
|
const config_1 = require("../../config");
|
|
40
50
|
const log_rotate_1 = require("./log-rotate");
|
|
@@ -43,17 +53,32 @@ const log_rotate_1 = require("./log-rotate");
|
|
|
43
53
|
* Returns the child PID, or null on failure.
|
|
44
54
|
*/
|
|
45
55
|
function spawnDaemon() {
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
56
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
57
|
+
var _a;
|
|
58
|
+
let out = null;
|
|
59
|
+
try {
|
|
60
|
+
const logFile = path.join(config_1.PATHS.logsDir, "daemon.log");
|
|
61
|
+
out = (0, log_rotate_1.openRotatedLog)(logFile);
|
|
62
|
+
const child = (0, node_child_process_1.spawn)(process.argv[0], [process.argv[1], "watch", "--daemon", "-b"], { detached: true, stdio: ["ignore", out, out] });
|
|
63
|
+
yield new Promise((resolve, reject) => {
|
|
64
|
+
child.once("spawn", resolve);
|
|
65
|
+
child.once("error", reject);
|
|
66
|
+
});
|
|
67
|
+
child.unref();
|
|
68
|
+
return (_a = child.pid) !== null && _a !== void 0 ? _a : null;
|
|
69
|
+
}
|
|
70
|
+
catch (err) {
|
|
71
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
72
|
+
console.error(`[daemon-launcher] Failed to spawn daemon: ${msg}`);
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
finally {
|
|
76
|
+
if (out !== null) {
|
|
77
|
+
try {
|
|
78
|
+
fs.closeSync(out);
|
|
79
|
+
}
|
|
80
|
+
catch (_b) { }
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
});
|
|
59
84
|
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Pure presentation helpers for `gmax doctor`. No I/O — callers do the probes
|
|
4
|
+
* (fs existence, MLX/summarizer health) and pass the results in; these decide
|
|
5
|
+
* the severity symbol and message. Kept out of doctor.ts so the severity logic
|
|
6
|
+
* is unit-testable without spinning up the whole command or its dependencies.
|
|
7
|
+
*/
|
|
8
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
+
exports.onnxModelStatus = onnxModelStatus;
|
|
10
|
+
exports.gpuEmbedModelStatus = gpuEmbedModelStatus;
|
|
11
|
+
exports.summaryCoverageStatus = summaryCoverageStatus;
|
|
12
|
+
exports.summarizerServerStatus = summarizerServerStatus;
|
|
13
|
+
/**
|
|
14
|
+
* ONNX model availability from the ~/.gmax/models/<id> dir. Correct for cpu-mode
|
|
15
|
+
* embed models and for ColBERT (always ONNX in-worker). NOT correct for gpu-mode
|
|
16
|
+
* embed models — those are served by MLX from the HF cache; use
|
|
17
|
+
* {@link gpuEmbedModelStatus} for those.
|
|
18
|
+
*/
|
|
19
|
+
function onnxModelStatus(id, exists) {
|
|
20
|
+
return exists
|
|
21
|
+
? { symbol: "ok", message: `${id}: downloaded` }
|
|
22
|
+
: { symbol: "WARN", message: `${id}: will download on first use` };
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* gpu-mode embed model availability, reported from reality rather than the ONNX
|
|
26
|
+
* models dir (where MLX models never live). A live server is proof enough; a
|
|
27
|
+
* down server falls back to the pinned HF cache to distinguish "cached, will
|
|
28
|
+
* load on next start" from "not present, will download" — neither of which is a
|
|
29
|
+
* warning, since the daemon respawns the server on demand.
|
|
30
|
+
*/
|
|
31
|
+
function gpuEmbedModelStatus(id, health, hfCacheExists) {
|
|
32
|
+
if (health.up) {
|
|
33
|
+
return { symbol: "ok", message: `${id}: serving via MLX (port 8100)` };
|
|
34
|
+
}
|
|
35
|
+
if (hfCacheExists) {
|
|
36
|
+
return {
|
|
37
|
+
symbol: "ok",
|
|
38
|
+
message: `${id}: cached (will load on next MLX start)`,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
return {
|
|
42
|
+
symbol: "INFO",
|
|
43
|
+
message: `${id}: will download on first MLX start`,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Summary coverage severity. The summarizer is an opt-in feature that has never
|
|
48
|
+
* been enabled by default, so zero coverage is INFO (an unexercised opt-in), not
|
|
49
|
+
* FAIL (which implies breakage and trains users/agents to ignore doctor FAILs).
|
|
50
|
+
* Any partial coverage below 90% is a genuine WARN (a stalled/incomplete
|
|
51
|
+
* backfill); >=90% is ok.
|
|
52
|
+
*/
|
|
53
|
+
function summaryCoverageStatus(withSummary, totalChunks) {
|
|
54
|
+
const pct = totalChunks > 0 ? Math.round((withSummary / totalChunks) * 100) : 0;
|
|
55
|
+
if (withSummary === 0) {
|
|
56
|
+
return {
|
|
57
|
+
symbol: "INFO",
|
|
58
|
+
message: "Summary coverage: 0% (summarizer never enabled — opt-in)",
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
const symbol = pct >= 90 ? "ok" : "WARN";
|
|
62
|
+
return {
|
|
63
|
+
symbol,
|
|
64
|
+
message: `Summary coverage: ${withSummary}/${totalChunks} (${pct}%)`,
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Summarizer server presence. Being down is INFO, not WARN: the summarizer is
|
|
69
|
+
* opt-in and, even when enabled, its llama-server idles out after 10min and
|
|
70
|
+
* respawns on demand — a stopped server is the normal resting state.
|
|
71
|
+
*/
|
|
72
|
+
function summarizerServerStatus(up) {
|
|
73
|
+
return up
|
|
74
|
+
? { symbol: "ok", message: "Summarizer: running (port 8101)" }
|
|
75
|
+
: { symbol: "INFO", message: "Summarizer: not running (opt-in)" };
|
|
76
|
+
}
|
|
@@ -47,6 +47,7 @@ exports.stripMarkdownFrontmatter = stripMarkdownFrontmatter;
|
|
|
47
47
|
exports.computeContentHash = computeContentHash;
|
|
48
48
|
exports.hasNullByte = hasNullByte;
|
|
49
49
|
exports.readFileSnapshot = readFileSnapshot;
|
|
50
|
+
exports.readContainedTextFileSync = readContainedTextFileSync;
|
|
50
51
|
exports.isIndexableFile = isIndexableFile;
|
|
51
52
|
exports.isIndexablePath = isIndexablePath;
|
|
52
53
|
exports.isGeneratedContent = isGeneratedContent;
|
|
@@ -57,6 +58,7 @@ const fs = __importStar(require("node:fs"));
|
|
|
57
58
|
const path = __importStar(require("node:path"));
|
|
58
59
|
const node_path_1 = require("node:path");
|
|
59
60
|
const config_1 = require("../../config");
|
|
61
|
+
const path_containment_1 = require("./path-containment");
|
|
60
62
|
function computeBufferHash(buffer) {
|
|
61
63
|
return (0, node_crypto_1.createHash)("sha256").update(buffer).digest("hex");
|
|
62
64
|
}
|
|
@@ -99,23 +101,50 @@ function hasNullByte(buffer, sampleLength = 1024) {
|
|
|
99
101
|
}
|
|
100
102
|
return false;
|
|
101
103
|
}
|
|
102
|
-
function readFileSnapshot(
|
|
103
|
-
return __awaiter(this,
|
|
104
|
-
|
|
104
|
+
function readFileSnapshot(filePath_1) {
|
|
105
|
+
return __awaiter(this, arguments, void 0, function* (filePath, options = {}) {
|
|
106
|
+
var _a;
|
|
107
|
+
const resolved = options.projectRoot
|
|
108
|
+
? (0, path_containment_1.resolveContainedPath)(options.projectRoot, filePath, {
|
|
109
|
+
verifyExistingTarget: true,
|
|
110
|
+
})
|
|
111
|
+
: filePath;
|
|
112
|
+
const handle = yield fs.promises.open(resolved, fs.constants.O_RDONLY | ((_a = fs.constants.O_NOFOLLOW) !== null && _a !== void 0 ? _a : 0));
|
|
105
113
|
try {
|
|
106
114
|
const before = yield handle.stat();
|
|
115
|
+
if (!before.isFile())
|
|
116
|
+
throw new Error("Path is not a regular file");
|
|
107
117
|
if (before.size > config_1.MAX_FILE_SIZE_BYTES) {
|
|
108
118
|
throw new Error("File exceeds maximum allowed size");
|
|
109
119
|
}
|
|
110
120
|
const size = before.size;
|
|
111
121
|
const buffer = size > 0 ? Buffer.allocUnsafe(size) : Buffer.alloc(0);
|
|
112
122
|
if (size > 0) {
|
|
113
|
-
|
|
123
|
+
let offset = 0;
|
|
124
|
+
while (offset < size) {
|
|
125
|
+
const { bytesRead } = yield handle.read(buffer, offset, size - offset, offset);
|
|
126
|
+
if (bytesRead <= 0) {
|
|
127
|
+
throw new Error("Unexpected end of file during read");
|
|
128
|
+
}
|
|
129
|
+
offset += bytesRead;
|
|
130
|
+
}
|
|
114
131
|
}
|
|
115
132
|
const after = yield handle.stat();
|
|
116
133
|
if (before.mtimeMs !== after.mtimeMs || before.size !== after.size) {
|
|
117
134
|
throw new Error("File changed during read");
|
|
118
135
|
}
|
|
136
|
+
if (options.projectRoot) {
|
|
137
|
+
(0, path_containment_1.resolveContainedPath)(options.projectRoot, resolved, {
|
|
138
|
+
verifyExistingTarget: true,
|
|
139
|
+
});
|
|
140
|
+
const currentLstat = yield fs.promises.lstat(resolved);
|
|
141
|
+
const current = yield fs.promises.stat(resolved);
|
|
142
|
+
if (currentLstat.isSymbolicLink() ||
|
|
143
|
+
current.dev !== after.dev ||
|
|
144
|
+
current.ino !== after.ino) {
|
|
145
|
+
throw new Error("File identity changed during read");
|
|
146
|
+
}
|
|
147
|
+
}
|
|
119
148
|
return { buffer, mtimeMs: after.mtimeMs, size: after.size };
|
|
120
149
|
}
|
|
121
150
|
finally {
|
|
@@ -123,6 +152,47 @@ function readFileSnapshot(filePath) {
|
|
|
123
152
|
}
|
|
124
153
|
});
|
|
125
154
|
}
|
|
155
|
+
function readContainedTextFileSync(projectRoot, filePath) {
|
|
156
|
+
var _a;
|
|
157
|
+
const resolved = (0, path_containment_1.resolveContainedPath)(projectRoot, filePath, {
|
|
158
|
+
verifyExistingTarget: true,
|
|
159
|
+
});
|
|
160
|
+
const fd = fs.openSync(resolved, fs.constants.O_RDONLY | ((_a = fs.constants.O_NOFOLLOW) !== null && _a !== void 0 ? _a : 0));
|
|
161
|
+
try {
|
|
162
|
+
const before = fs.fstatSync(fd);
|
|
163
|
+
if (!before.isFile())
|
|
164
|
+
throw new Error("Path is not a regular file");
|
|
165
|
+
if (before.size > config_1.MAX_FILE_SIZE_BYTES) {
|
|
166
|
+
throw new Error("File exceeds maximum allowed size");
|
|
167
|
+
}
|
|
168
|
+
const buffer = Buffer.alloc(before.size);
|
|
169
|
+
let offset = 0;
|
|
170
|
+
while (offset < buffer.length) {
|
|
171
|
+
const bytesRead = fs.readSync(fd, buffer, offset, buffer.length - offset, offset);
|
|
172
|
+
if (bytesRead <= 0)
|
|
173
|
+
throw new Error("Unexpected end of file during read");
|
|
174
|
+
offset += bytesRead;
|
|
175
|
+
}
|
|
176
|
+
const after = fs.fstatSync(fd);
|
|
177
|
+
if (before.mtimeMs !== after.mtimeMs || before.size !== after.size) {
|
|
178
|
+
throw new Error("File changed during read");
|
|
179
|
+
}
|
|
180
|
+
(0, path_containment_1.resolveContainedPath)(projectRoot, resolved, {
|
|
181
|
+
verifyExistingTarget: true,
|
|
182
|
+
});
|
|
183
|
+
const currentLstat = fs.lstatSync(resolved);
|
|
184
|
+
const current = fs.statSync(resolved);
|
|
185
|
+
if (currentLstat.isSymbolicLink() ||
|
|
186
|
+
current.dev !== after.dev ||
|
|
187
|
+
current.ino !== after.ino) {
|
|
188
|
+
throw new Error("File identity changed during read");
|
|
189
|
+
}
|
|
190
|
+
return buffer.toString("utf8");
|
|
191
|
+
}
|
|
192
|
+
finally {
|
|
193
|
+
fs.closeSync(fd);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
126
196
|
// Check if a file should be indexed (extension and size).
|
|
127
197
|
function isIndexableFile(filePath, size) {
|
|
128
198
|
const ext = (0, node_path_1.extname)(filePath).toLowerCase();
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.KeyedMutex = void 0;
|
|
13
|
+
function abortError() {
|
|
14
|
+
const error = new Error("Aborted");
|
|
15
|
+
error.name = "AbortError";
|
|
16
|
+
return error;
|
|
17
|
+
}
|
|
18
|
+
class KeyedMutex {
|
|
19
|
+
constructor() {
|
|
20
|
+
this.keys = new Map();
|
|
21
|
+
this.activeTasks = new Set();
|
|
22
|
+
this.closed = false;
|
|
23
|
+
}
|
|
24
|
+
get pending() {
|
|
25
|
+
let count = 0;
|
|
26
|
+
for (const state of this.keys.values()) {
|
|
27
|
+
count += (state.locked ? 1 : 0) + state.queue.length;
|
|
28
|
+
}
|
|
29
|
+
return count;
|
|
30
|
+
}
|
|
31
|
+
run(key, signal, fn) {
|
|
32
|
+
const task = (() => __awaiter(this, void 0, void 0, function* () {
|
|
33
|
+
yield this.acquire(key, signal);
|
|
34
|
+
try {
|
|
35
|
+
if (signal === null || signal === void 0 ? void 0 : signal.aborted)
|
|
36
|
+
throw abortError();
|
|
37
|
+
return yield fn();
|
|
38
|
+
}
|
|
39
|
+
finally {
|
|
40
|
+
this.release(key);
|
|
41
|
+
}
|
|
42
|
+
}))();
|
|
43
|
+
this.activeTasks.add(task);
|
|
44
|
+
const cleanup = () => this.activeTasks.delete(task);
|
|
45
|
+
void task.then(cleanup, cleanup);
|
|
46
|
+
return task;
|
|
47
|
+
}
|
|
48
|
+
close() {
|
|
49
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
50
|
+
var _a;
|
|
51
|
+
this.closed = true;
|
|
52
|
+
for (const [key, state] of this.keys) {
|
|
53
|
+
for (const waiter of state.queue.splice(0)) {
|
|
54
|
+
(_a = waiter.signal) === null || _a === void 0 ? void 0 : _a.removeEventListener("abort", waiter.onAbort);
|
|
55
|
+
waiter.reject(abortError());
|
|
56
|
+
}
|
|
57
|
+
if (!state.locked)
|
|
58
|
+
this.keys.delete(key);
|
|
59
|
+
}
|
|
60
|
+
yield Promise.allSettled([...this.activeTasks]);
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
acquire(key, signal) {
|
|
64
|
+
var _a;
|
|
65
|
+
if (this.closed || (signal === null || signal === void 0 ? void 0 : signal.aborted))
|
|
66
|
+
return Promise.reject(abortError());
|
|
67
|
+
const state = (_a = this.keys.get(key)) !== null && _a !== void 0 ? _a : { locked: false, queue: [] };
|
|
68
|
+
this.keys.set(key, state);
|
|
69
|
+
if (!state.locked) {
|
|
70
|
+
state.locked = true;
|
|
71
|
+
return Promise.resolve();
|
|
72
|
+
}
|
|
73
|
+
return new Promise((resolve, reject) => {
|
|
74
|
+
const waiter = { resolve, reject, signal };
|
|
75
|
+
const onAbort = () => {
|
|
76
|
+
const index = state.queue.indexOf(waiter);
|
|
77
|
+
if (index !== -1)
|
|
78
|
+
state.queue.splice(index, 1);
|
|
79
|
+
reject(abortError());
|
|
80
|
+
};
|
|
81
|
+
waiter.onAbort = onAbort;
|
|
82
|
+
signal === null || signal === void 0 ? void 0 : signal.addEventListener("abort", onAbort, { once: true });
|
|
83
|
+
state.queue.push(waiter);
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
release(key) {
|
|
87
|
+
var _a;
|
|
88
|
+
const state = this.keys.get(key);
|
|
89
|
+
if (!state)
|
|
90
|
+
return;
|
|
91
|
+
const next = state.queue.shift();
|
|
92
|
+
if (next) {
|
|
93
|
+
(_a = next.signal) === null || _a === void 0 ? void 0 : _a.removeEventListener("abort", next.onAbort);
|
|
94
|
+
next.resolve();
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
state.locked = false;
|
|
98
|
+
this.keys.delete(key);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
exports.KeyedMutex = KeyedMutex;
|
package/dist/lib/utils/logger.js
CHANGED
|
@@ -1,12 +1,68 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.VERBOSE = void 0;
|
|
3
|
+
exports.LOG_TIMESTAMPS_ENV = exports.VERBOSE = void 0;
|
|
4
|
+
exports.stampLines = stampLines;
|
|
5
|
+
exports.installTimestampedOutput = installTimestampedOutput;
|
|
4
6
|
exports.log = log;
|
|
5
7
|
exports.debug = debug;
|
|
6
8
|
exports.timer = timer;
|
|
7
9
|
exports.debugTimer = debugTimer;
|
|
8
10
|
exports.debugEvery = debugEvery;
|
|
9
11
|
exports.VERBOSE = process.env.GMAX_DEBUG === "1" || process.env.GMAX_VERBOSE === "1";
|
|
12
|
+
/** Set by the daemon so forked workers (which share its log fd) opt in too. */
|
|
13
|
+
exports.LOG_TIMESTAMPS_ENV = "GMAX_LOG_TIMESTAMPS";
|
|
14
|
+
/** Local time, same shape mlx-embed-server.log uses: 2026-07-06T21:33:26 */
|
|
15
|
+
function timestamp() {
|
|
16
|
+
const d = new Date();
|
|
17
|
+
const pad = (n) => String(n).padStart(2, "0");
|
|
18
|
+
return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}T${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}`;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Prefix a timestamp to each line in `text` that begins at line start.
|
|
22
|
+
* `state.atLineStart` carries the position across chunks so a line split
|
|
23
|
+
* over multiple write() calls is stamped exactly once. Blank lines are
|
|
24
|
+
* left unstamped. Exported for tests.
|
|
25
|
+
*/
|
|
26
|
+
function stampLines(text, state) {
|
|
27
|
+
let out = "";
|
|
28
|
+
let i = 0;
|
|
29
|
+
while (i < text.length) {
|
|
30
|
+
const nl = text.indexOf("\n", i);
|
|
31
|
+
const end = nl === -1 ? text.length : nl + 1;
|
|
32
|
+
const line = text.slice(i, end);
|
|
33
|
+
if (state.atLineStart && line !== "\n")
|
|
34
|
+
out += `${timestamp()} `;
|
|
35
|
+
out += line;
|
|
36
|
+
state.atLineStart = nl !== -1;
|
|
37
|
+
i = end;
|
|
38
|
+
}
|
|
39
|
+
return out;
|
|
40
|
+
}
|
|
41
|
+
let timestampsInstalled = false;
|
|
42
|
+
/**
|
|
43
|
+
* Prefix every stdout/stderr line with a local timestamp. Installed by the
|
|
44
|
+
* daemon (whose stdio becomes daemon.log) and by workers when the daemon's
|
|
45
|
+
* env flag is present — never by interactive CLI commands. Without this,
|
|
46
|
+
* daemon.log lines can't be correlated with the timestamped MLX/LLM server
|
|
47
|
+
* logs when reconstructing an incident.
|
|
48
|
+
*/
|
|
49
|
+
function installTimestampedOutput() {
|
|
50
|
+
if (timestampsInstalled)
|
|
51
|
+
return;
|
|
52
|
+
timestampsInstalled = true;
|
|
53
|
+
process.env[exports.LOG_TIMESTAMPS_ENV] = "1"; // forked workers inherit this
|
|
54
|
+
for (const stream of [process.stdout, process.stderr]) {
|
|
55
|
+
const orig = stream.write.bind(stream);
|
|
56
|
+
const state = { atLineStart: true };
|
|
57
|
+
stream.write = ((chunk, enc, cb) => {
|
|
58
|
+
const out = typeof chunk === "string" || Buffer.isBuffer(chunk)
|
|
59
|
+
? stampLines(chunk.toString(), state)
|
|
60
|
+
: chunk;
|
|
61
|
+
// Forward the (chunk, cb) and (chunk, enc, cb) overloads untouched.
|
|
62
|
+
return orig(out, enc, cb);
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
}
|
|
10
66
|
function log(tag, msg) {
|
|
11
67
|
process.stderr.write(`[${tag}] ${msg}\n`);
|
|
12
68
|
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.DEFAULT_MLX_EMBED_MODEL = void 0;
|
|
37
|
+
exports.isMlxModelCached = isMlxModelCached;
|
|
38
|
+
exports.resolveMlxHfHome = resolveMlxHfHome;
|
|
39
|
+
const node_child_process_1 = require("node:child_process");
|
|
40
|
+
const fs = __importStar(require("node:fs"));
|
|
41
|
+
const os = __importStar(require("node:os"));
|
|
42
|
+
const path = __importStar(require("node:path"));
|
|
43
|
+
const config_1 = require("../../config");
|
|
44
|
+
const logger_1 = require("./logger");
|
|
45
|
+
/** Default model served by mlx-embed-server/server.py (keep in sync). */
|
|
46
|
+
exports.DEFAULT_MLX_EMBED_MODEL = "ibm-granite/granite-embedding-small-english-r2";
|
|
47
|
+
function hasSnapshot(modelDir) {
|
|
48
|
+
try {
|
|
49
|
+
return fs.readdirSync(path.join(modelDir, "snapshots")).length > 0;
|
|
50
|
+
}
|
|
51
|
+
catch (_a) {
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Whether an MLX model already has a snapshot in the local pinned HF cache
|
|
57
|
+
* (~/.gmax/hf). Used by `gmax doctor` to report gpu-mode embed model status
|
|
58
|
+
* from the HF cache rather than the ONNX models dir (where MLX models never
|
|
59
|
+
* live). Independent of whether the embed server is currently running.
|
|
60
|
+
*/
|
|
61
|
+
function isMlxModelCached(modelId = exports.DEFAULT_MLX_EMBED_MODEL, localHfHome = config_1.PATHS.hfDir) {
|
|
62
|
+
const modelDirName = `models--${modelId.replace(/\//g, "--")}`;
|
|
63
|
+
return hasSnapshot(path.join(localHfHome, "hub", modelDirName));
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Resolve the HF_HOME to spawn the MLX embed server with, pinned to internal
|
|
67
|
+
* disk (~/.gmax/hf).
|
|
68
|
+
*
|
|
69
|
+
* The user's shell HF_HOME — and even ~/.cache/huggingface, which may itself
|
|
70
|
+
* be a symlink — can live on an external volume. When that volume is
|
|
71
|
+
* unmounted the embed server dies at model load (PermissionError on the dead
|
|
72
|
+
* mountpoint) and every worker silently degrades to ONNX CPU. The embed
|
|
73
|
+
* model is small (~185MB), so keeping a private copy under ~/.gmax/hf makes
|
|
74
|
+
* the server immune to drive state and shell env.
|
|
75
|
+
*
|
|
76
|
+
* Seeds the local cache from the inherited HF cache when the model is
|
|
77
|
+
* already there (one-time `cp -R`, preserves the hub layout's relative
|
|
78
|
+
* snapshot→blob symlinks) so the first spawn needs no network. Otherwise the
|
|
79
|
+
* server downloads into the local cache on startup. Copies land in a temp
|
|
80
|
+
* dir and are renamed into place so a partial copy never looks complete —
|
|
81
|
+
* server.py flips to offline mode whenever the model dir is non-empty.
|
|
82
|
+
*/
|
|
83
|
+
function resolveMlxHfHome(modelId = exports.DEFAULT_MLX_EMBED_MODEL, opts = {}) {
|
|
84
|
+
var _a, _b, _c;
|
|
85
|
+
const localHfHome = (_a = opts.localHfHome) !== null && _a !== void 0 ? _a : config_1.PATHS.hfDir;
|
|
86
|
+
const hubDir = path.join(localHfHome, "hub");
|
|
87
|
+
const modelDirName = `models--${modelId.replace(/\//g, "--")}`;
|
|
88
|
+
const localModelDir = path.join(hubDir, modelDirName);
|
|
89
|
+
try {
|
|
90
|
+
fs.mkdirSync(hubDir, { recursive: true });
|
|
91
|
+
}
|
|
92
|
+
catch (_d) { }
|
|
93
|
+
if (hasSnapshot(localModelDir))
|
|
94
|
+
return localHfHome;
|
|
95
|
+
const inherited = (_c = (_b = opts.inheritedHfHome) !== null && _b !== void 0 ? _b : process.env.HF_HOME) !== null && _c !== void 0 ? _c : path.join(os.homedir(), ".cache", "huggingface");
|
|
96
|
+
const sourceModelDir = path.join(inherited, "hub", modelDirName);
|
|
97
|
+
const tmpDir = path.join(hubDir, `.seed-${modelDirName}`);
|
|
98
|
+
try {
|
|
99
|
+
if (hasSnapshot(sourceModelDir)) {
|
|
100
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
101
|
+
(0, node_child_process_1.execFileSync)("cp", ["-R", sourceModelDir, tmpDir], { timeout: 300000 });
|
|
102
|
+
fs.renameSync(tmpDir, localModelDir);
|
|
103
|
+
(0, logger_1.log)("mlx", `seeded local HF cache for ${modelId} from ${sourceModelDir}`);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
catch (err) {
|
|
107
|
+
try {
|
|
108
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
109
|
+
}
|
|
110
|
+
catch (_e) { }
|
|
111
|
+
(0, logger_1.log)("mlx", `local HF cache seed failed (${err instanceof Error ? err.message : String(err)}) — model will download on first server start`);
|
|
112
|
+
}
|
|
113
|
+
return localHfHome;
|
|
114
|
+
}
|