grepmax 0.26.54 → 0.26.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +9 -7
- package/dist/commands/add.js +2 -0
- package/dist/commands/doctor.js +9 -0
- package/dist/commands/index.js +2 -0
- package/dist/commands/remove.js +2 -0
- package/dist/commands/repair.js +2 -0
- package/dist/commands/watch.js +12 -0
- package/dist/lib/daemon/daemon.js +201 -82
- package/dist/lib/daemon/mlx-server-manager.js +24 -8
- package/dist/lib/store/maintenance-policy.js +172 -0
- package/dist/lib/store/meta-cache.js +34 -1
- package/dist/lib/store/vector-db.js +76 -29
- package/dist/lib/utils/autostart.js +34 -7
- package/dist/lib/utils/daemon-launcher.js +6 -0
- package/dist/lib/utils/kernel-zone.js +140 -21
- package/dist/lib/utils/safety-latch.js +121 -0
- package/dist/lib/workers/pool.js +97 -10
- package/package.json +1 -1
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
- package/plugins/grepmax/hooks/watch-lease.js +13 -5
|
@@ -48,6 +48,7 @@ const node_child_process_1 = require("node:child_process");
|
|
|
48
48
|
const fs = __importStar(require("node:fs"));
|
|
49
49
|
const path = __importStar(require("node:path"));
|
|
50
50
|
const config_1 = require("../../config");
|
|
51
|
+
const autostart_1 = require("./autostart");
|
|
51
52
|
const log_rotate_1 = require("./log-rotate");
|
|
52
53
|
/**
|
|
53
54
|
* Spawn the daemon in background mode.
|
|
@@ -56,6 +57,8 @@ const log_rotate_1 = require("./log-rotate");
|
|
|
56
57
|
function spawnDaemon() {
|
|
57
58
|
return __awaiter(this, void 0, void 0, function* () {
|
|
58
59
|
var _a;
|
|
60
|
+
if ((0, autostart_1.daemonStartDeniedReason)() !== null)
|
|
61
|
+
return null;
|
|
59
62
|
let out = null;
|
|
60
63
|
try {
|
|
61
64
|
const logFile = path.join(config_1.PATHS.logsDir, "daemon.log");
|
|
@@ -90,6 +93,9 @@ function spawnDaemon() {
|
|
|
90
93
|
*/
|
|
91
94
|
function spawnDaemonProcess() {
|
|
92
95
|
return __awaiter(this, void 0, void 0, function* () {
|
|
96
|
+
const denied = (0, autostart_1.daemonStartDeniedReason)();
|
|
97
|
+
if (denied !== null)
|
|
98
|
+
throw new Error(`gmax: ${denied}`);
|
|
93
99
|
const logFile = path.join(config_1.PATHS.logsDir, "daemon.log");
|
|
94
100
|
const out = (0, log_rotate_1.openRotatedLog)(logFile);
|
|
95
101
|
try {
|
|
@@ -1,6 +1,20 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __rest = (this && this.__rest) || function (s, e) {
|
|
3
|
+
var t = {};
|
|
4
|
+
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
|
|
5
|
+
t[p] = s[p];
|
|
6
|
+
if (s != null && typeof Object.getOwnPropertySymbols === "function")
|
|
7
|
+
for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
|
|
8
|
+
if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
|
|
9
|
+
t[p[i]] = s[p[i]];
|
|
10
|
+
}
|
|
11
|
+
return t;
|
|
12
|
+
};
|
|
2
13
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
14
|
exports.ZONE_THRESHOLDS = void 0;
|
|
15
|
+
exports.formatPressureProbe = formatPressureProbe;
|
|
16
|
+
exports.probeKernelZoneUsage = probeKernelZoneUsage;
|
|
17
|
+
exports.probeMemoryPressure = probeMemoryPressure;
|
|
4
18
|
exports.classifyZonePressure = classifyZonePressure;
|
|
5
19
|
exports.parseZprintOutput = parseZprintOutput;
|
|
6
20
|
exports.readKernelZoneUsage = readKernelZoneUsage;
|
|
@@ -21,11 +35,15 @@ const node_child_process_1 = require("node:child_process");
|
|
|
21
35
|
* docs/2026-08-04-macos-kernel-zone-panic-incident.md.
|
|
22
36
|
*
|
|
23
37
|
* gmax cannot fix the kernel defect, but it can decline to be the thing that
|
|
24
|
-
* detonates it. Sampling the zone
|
|
25
|
-
*
|
|
38
|
+
* detonates it. Sampling the zone has taken 1.584 s on the incident host, so the
|
|
39
|
+
* probe uses a bounded five-second budget. The daemon stands down while there
|
|
40
|
+
* are still hours of headroom.
|
|
26
41
|
*/
|
|
27
42
|
/** The zone that leaks. Named explicitly so `zprint` returns a single line. */
|
|
28
43
|
const ZONE_NAME = "data.kalloc.1024";
|
|
44
|
+
// Two 500 ms probes timed out on the incident host; a subsequent probe completed
|
|
45
|
+
// in 1,584 ms. Keep the previous five-second ceiling to allow scheduling margin.
|
|
46
|
+
const PROBE_TIMEOUT_MS = 5000;
|
|
29
47
|
/**
|
|
30
48
|
* Warn threshold.
|
|
31
49
|
*
|
|
@@ -45,6 +63,122 @@ const WARN_BYTES = 4 * Math.pow(1024, 3);
|
|
|
45
63
|
* is an unplanned reboot.
|
|
46
64
|
*/
|
|
47
65
|
const CRITICAL_BYTES = 8 * Math.pow(1024, 3);
|
|
66
|
+
/** Compact diagnostic metadata; never serializes raw probe output or errors. */
|
|
67
|
+
function formatPressureProbe(result) {
|
|
68
|
+
return JSON.stringify(Object.assign(Object.assign({ status: result.status, sampledAtMs: result.sampledAtMs, durationMs: result.durationMs, outputBytes: result.outputBytes }, (result.status !== "known" ? { reason: result.reason } : {})), (result.status === "unknown"
|
|
69
|
+
? {
|
|
70
|
+
errorCode: result.errorCode,
|
|
71
|
+
exitCode: result.exitCode,
|
|
72
|
+
signal: result.signal,
|
|
73
|
+
}
|
|
74
|
+
: {})));
|
|
75
|
+
}
|
|
76
|
+
const PROBE_ERROR_CODES = new Set([
|
|
77
|
+
"ETIMEDOUT",
|
|
78
|
+
"ENOBUFS",
|
|
79
|
+
"ERR_CHILD_PROCESS_STDIO_MAXBUFFER",
|
|
80
|
+
"ENOENT",
|
|
81
|
+
"EACCES",
|
|
82
|
+
"EPERM",
|
|
83
|
+
]);
|
|
84
|
+
const PROBE_SIGNALS = new Set(["SIGTERM", "SIGKILL", "SIGABRT", "SIGSEGV"]);
|
|
85
|
+
function stdoutBytes(output) {
|
|
86
|
+
return typeof output === "string"
|
|
87
|
+
? Buffer.byteLength(output)
|
|
88
|
+
: Buffer.isBuffer(output)
|
|
89
|
+
? output.length
|
|
90
|
+
: 0;
|
|
91
|
+
}
|
|
92
|
+
function boundedPressureProbe(command, args, timeout, parse, overrides) {
|
|
93
|
+
const deps = Object.assign({ platform: process.platform, wallNow: Date.now, monotonicNow: () => performance.now(), run: (cmd, argv, options) => (0, node_child_process_1.execFileSync)(cmd, argv, options) }, overrides);
|
|
94
|
+
const sampledAtMs = deps.wallNow();
|
|
95
|
+
const started = deps.monotonicNow();
|
|
96
|
+
let outputBytes = 0;
|
|
97
|
+
const observation = () => ({
|
|
98
|
+
sampledAtMs,
|
|
99
|
+
durationMs: Math.max(0, deps.monotonicNow() - started),
|
|
100
|
+
outputBytes,
|
|
101
|
+
});
|
|
102
|
+
if (deps.platform !== "darwin")
|
|
103
|
+
return Object.assign(Object.assign({}, observation()), { status: "unsupported", reason: "unsupported-platform" });
|
|
104
|
+
try {
|
|
105
|
+
const output = deps.run(command, args, {
|
|
106
|
+
encoding: "utf-8",
|
|
107
|
+
timeout,
|
|
108
|
+
maxBuffer: 64 * 1024,
|
|
109
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
110
|
+
});
|
|
111
|
+
outputBytes = stdoutBytes(output);
|
|
112
|
+
const value = parse(output);
|
|
113
|
+
return value === null
|
|
114
|
+
? Object.assign(Object.assign({}, observation()), { status: "unknown", reason: "parse", errorCode: null, exitCode: null, signal: null }) : Object.assign(Object.assign({}, observation()), { status: "known", value });
|
|
115
|
+
}
|
|
116
|
+
catch (error) {
|
|
117
|
+
const details = error && typeof error === "object"
|
|
118
|
+
? error
|
|
119
|
+
: {};
|
|
120
|
+
outputBytes = stdoutBytes(details.stdout);
|
|
121
|
+
const errorCode = typeof details.code === "string" && PROBE_ERROR_CODES.has(details.code)
|
|
122
|
+
? details.code
|
|
123
|
+
: details.code == null
|
|
124
|
+
? null
|
|
125
|
+
: "UNKNOWN";
|
|
126
|
+
const exitCode = typeof details.status === "number" && Number.isInteger(details.status)
|
|
127
|
+
? details.status
|
|
128
|
+
: null;
|
|
129
|
+
const signal = typeof details.signal === "string" && PROBE_SIGNALS.has(details.signal)
|
|
130
|
+
? details.signal
|
|
131
|
+
: null;
|
|
132
|
+
const reason = errorCode === "ETIMEDOUT"
|
|
133
|
+
? "timeout"
|
|
134
|
+
: errorCode === "ENOBUFS" ||
|
|
135
|
+
errorCode === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER"
|
|
136
|
+
? "output-limit"
|
|
137
|
+
: exitCode !== null || signal !== null
|
|
138
|
+
? "exit"
|
|
139
|
+
: "execution";
|
|
140
|
+
return Object.assign(Object.assign({}, observation()), { status: "unknown", reason,
|
|
141
|
+
errorCode,
|
|
142
|
+
exitCode,
|
|
143
|
+
signal });
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
function probeKernelZoneUsage(zoneName = ZONE_NAME, overrides = {}) {
|
|
147
|
+
const result = boundedPressureProbe("zprint", [zoneName], PROBE_TIMEOUT_MS, (output) => {
|
|
148
|
+
const parsed = parseZprintOutput(output, zoneName);
|
|
149
|
+
if (!parsed)
|
|
150
|
+
return null;
|
|
151
|
+
const bytes = parsed.elements * parsed.elementSize;
|
|
152
|
+
if (!Number.isSafeInteger(bytes))
|
|
153
|
+
return null;
|
|
154
|
+
return Object.assign(Object.assign({}, parsed), { bytes, pressure: classifyZonePressure(bytes) });
|
|
155
|
+
}, overrides);
|
|
156
|
+
if (result.status !== "known")
|
|
157
|
+
return result;
|
|
158
|
+
const { value } = result, observation = __rest(result, ["value"]);
|
|
159
|
+
return Object.assign(Object.assign({}, observation), { usage: value });
|
|
160
|
+
}
|
|
161
|
+
/** XNU publishes dispatch flags NORMAL=1, WARN=2, CRITICAL=4 through this sysctl.
|
|
162
|
+
* https://github.com/apple-oss-distributions/xnu/blob/main/bsd/kern/kern_memorystatus_notify.c
|
|
163
|
+
* Other values are unknown, including the unrelated internal kernel enum. */
|
|
164
|
+
function probeMemoryPressure(overrides = {}) {
|
|
165
|
+
const result = boundedPressureProbe("sysctl", ["-n", "kern.memorystatus_vm_pressure_level"], 1000, (output) => {
|
|
166
|
+
switch (output.trim()) {
|
|
167
|
+
case "1":
|
|
168
|
+
return "normal";
|
|
169
|
+
case "2":
|
|
170
|
+
return "warn";
|
|
171
|
+
case "4":
|
|
172
|
+
return "critical";
|
|
173
|
+
default:
|
|
174
|
+
return null;
|
|
175
|
+
}
|
|
176
|
+
}, overrides);
|
|
177
|
+
if (result.status !== "known")
|
|
178
|
+
return result;
|
|
179
|
+
const { value } = result, observation = __rest(result, ["value"]);
|
|
180
|
+
return Object.assign(Object.assign({}, observation), { pressure: value });
|
|
181
|
+
}
|
|
48
182
|
function classifyZonePressure(bytes) {
|
|
49
183
|
if (bytes >= CRITICAL_BYTES)
|
|
50
184
|
return "critical";
|
|
@@ -84,27 +218,12 @@ function parseZprintOutput(output, zoneName = ZONE_NAME) {
|
|
|
84
218
|
}
|
|
85
219
|
/**
|
|
86
220
|
* Sample the zone. Returns null off macOS, or whenever the sample cannot be
|
|
87
|
-
* trusted
|
|
88
|
-
*
|
|
221
|
+
* trusted. An unparseable or timed-out report reads as "unknown", never as
|
|
222
|
+
* "healthy". macOS startup and heavy-operation admission refuse unknown samples.
|
|
89
223
|
*/
|
|
90
224
|
function readKernelZoneUsage(zoneName = ZONE_NAME) {
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
try {
|
|
94
|
-
const output = (0, node_child_process_1.execFileSync)("zprint", [zoneName], {
|
|
95
|
-
encoding: "utf-8",
|
|
96
|
-
timeout: 5000,
|
|
97
|
-
stdio: ["ignore", "pipe", "ignore"],
|
|
98
|
-
});
|
|
99
|
-
const parsed = parseZprintOutput(output, zoneName);
|
|
100
|
-
if (!parsed)
|
|
101
|
-
return null;
|
|
102
|
-
const bytes = parsed.elements * parsed.elementSize;
|
|
103
|
-
return Object.assign(Object.assign({}, parsed), { bytes, pressure: classifyZonePressure(bytes) });
|
|
104
|
-
}
|
|
105
|
-
catch (_a) {
|
|
106
|
-
return null;
|
|
107
|
-
}
|
|
225
|
+
const result = probeKernelZoneUsage(zoneName);
|
|
226
|
+
return result.status === "known" ? result.usage : null;
|
|
108
227
|
}
|
|
109
228
|
function formatZoneUsage(usage) {
|
|
110
229
|
const gib = (usage.bytes / Math.pow(1024, 3)).toFixed(2);
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.SAFETY_LATCH_NAME = void 0;
|
|
37
|
+
exports.safetyStopReason = safetyStopReason;
|
|
38
|
+
exports.latchSafetyStop = latchSafetyStop;
|
|
39
|
+
exports.assertNoSafetyStop = assertNoSafetyStop;
|
|
40
|
+
const node_crypto_1 = require("node:crypto");
|
|
41
|
+
const fs = __importStar(require("node:fs"));
|
|
42
|
+
const path = __importStar(require("node:path"));
|
|
43
|
+
const config_1 = require("../../config");
|
|
44
|
+
/** Machine-wide stop: changing stores or installing a version cannot clear it. */
|
|
45
|
+
exports.SAFETY_LATCH_NAME = "safety-stop.json";
|
|
46
|
+
function safetyRoot() {
|
|
47
|
+
var _a;
|
|
48
|
+
return (_a = config_1.PATHS.sharedRoot) !== null && _a !== void 0 ? _a : path.dirname(config_1.PATHS.autostartDisabledFile);
|
|
49
|
+
}
|
|
50
|
+
/** Absence is distinct from an unreadable/corrupt stop, which stays stopped. */
|
|
51
|
+
function safetyStopReason(root = safetyRoot()) {
|
|
52
|
+
const file = path.join(root, exports.SAFETY_LATCH_NAME);
|
|
53
|
+
try {
|
|
54
|
+
const stat = fs.lstatSync(file);
|
|
55
|
+
if (!stat.isFile() || stat.size > 8192)
|
|
56
|
+
return "unverified safety stop";
|
|
57
|
+
const value = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
58
|
+
if (value.schemaVersion !== 1 ||
|
|
59
|
+
!Number.isFinite(value.at) ||
|
|
60
|
+
typeof value.reason !== "string" ||
|
|
61
|
+
!value.reason.trim())
|
|
62
|
+
return "unverified safety stop";
|
|
63
|
+
return value.reason.replace(/[\r\n\t]/g, " ").slice(0, 512);
|
|
64
|
+
}
|
|
65
|
+
catch (error) {
|
|
66
|
+
return error.code === "ENOENT"
|
|
67
|
+
? null
|
|
68
|
+
: "unreadable safety stop";
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
/** Atomic durable latch; intentionally no automatic expiry or clear operation. */
|
|
72
|
+
function latchSafetyStop(reason, root = safetyRoot()) {
|
|
73
|
+
if (safetyStopReason(root) !== null)
|
|
74
|
+
return;
|
|
75
|
+
fs.mkdirSync(root, { recursive: true, mode: 0o700 });
|
|
76
|
+
fs.chmodSync(root, 0o700);
|
|
77
|
+
const file = path.join(root, exports.SAFETY_LATCH_NAME);
|
|
78
|
+
const temporary = path.join(root, `.safety-stop-${(0, node_crypto_1.randomUUID)()}.tmp`);
|
|
79
|
+
let fd;
|
|
80
|
+
try {
|
|
81
|
+
fd = fs.openSync(temporary, "wx", 0o600);
|
|
82
|
+
const record = {
|
|
83
|
+
schemaVersion: 1,
|
|
84
|
+
at: Date.now(),
|
|
85
|
+
reason: reason.replace(/[\r\n\t]/g, " ").slice(0, 512) || "host safety stop",
|
|
86
|
+
};
|
|
87
|
+
fs.writeFileSync(fd, `${JSON.stringify(record)}\n`);
|
|
88
|
+
fs.fsyncSync(fd);
|
|
89
|
+
fs.closeSync(fd);
|
|
90
|
+
fd = undefined;
|
|
91
|
+
// Publish without replacing another concurrent stop's original cause.
|
|
92
|
+
try {
|
|
93
|
+
fs.linkSync(temporary, file);
|
|
94
|
+
}
|
|
95
|
+
catch (error) {
|
|
96
|
+
if (error.code !== "EEXIST")
|
|
97
|
+
throw error;
|
|
98
|
+
}
|
|
99
|
+
// Persist the directory entry, not only the payload, before stopping.
|
|
100
|
+
const dir = fs.openSync(root, "r");
|
|
101
|
+
try {
|
|
102
|
+
fs.fsyncSync(dir);
|
|
103
|
+
}
|
|
104
|
+
finally {
|
|
105
|
+
fs.closeSync(dir);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
finally {
|
|
109
|
+
if (fd !== undefined)
|
|
110
|
+
fs.closeSync(fd);
|
|
111
|
+
try {
|
|
112
|
+
fs.unlinkSync(temporary);
|
|
113
|
+
}
|
|
114
|
+
catch (_a) { }
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
function assertNoSafetyStop() {
|
|
118
|
+
const reason = safetyStopReason();
|
|
119
|
+
if (reason !== null)
|
|
120
|
+
throw new Error(`gmax safety stop: ${reason}; containment must be reviewed before resuming`);
|
|
121
|
+
}
|
package/dist/lib/workers/pool.js
CHANGED
|
@@ -57,7 +57,10 @@ const path = __importStar(require("node:path"));
|
|
|
57
57
|
const config_1 = require("../../config");
|
|
58
58
|
const embedding_generation_1 = require("../index/embedding-generation");
|
|
59
59
|
const index_config_1 = require("../index/index-config");
|
|
60
|
+
const autostart_1 = require("../utils/autostart");
|
|
61
|
+
const kernel_zone_1 = require("../utils/kernel-zone");
|
|
60
62
|
const logger_1 = require("../utils/logger");
|
|
63
|
+
const safety_latch_1 = require("../utils/safety-latch");
|
|
61
64
|
const TASK_TIMEOUT_MS = (() => {
|
|
62
65
|
var _a;
|
|
63
66
|
const fromEnv = Number.parseInt((_a = process.env.GMAX_WORKER_TASK_TIMEOUT_MS) !== null && _a !== void 0 ? _a : "", 10);
|
|
@@ -221,6 +224,7 @@ class WorkerPool {
|
|
|
221
224
|
this.destroyPromise = null;
|
|
222
225
|
this.consecutiveRespawns = 0;
|
|
223
226
|
this.respawnLimitReached = false;
|
|
227
|
+
this.spawnDeniedReason = null;
|
|
224
228
|
this.idleReapInterval = null;
|
|
225
229
|
// Re-runs dispatch once the oldest unassigned task reaches SCALE_UP_WAIT_MS.
|
|
226
230
|
this.scaleUpTimer = null;
|
|
@@ -237,7 +241,8 @@ class WorkerPool {
|
|
|
237
241
|
// process environment remains live and is copied at each fork.
|
|
238
242
|
this.embeddingEnvironment = Object.freeze(generatedEmbeddingEnv);
|
|
239
243
|
// Lazy spawn: start with 1 worker, scale up on demand
|
|
240
|
-
this.spawnWorker()
|
|
244
|
+
if (!this.spawnWorker())
|
|
245
|
+
throw this.spawnDeniedError();
|
|
241
246
|
// Periodically reap idle workers back to MIN_KEEP, and force-kill any
|
|
242
247
|
// worker wedged in busy=true (the leak backstop — see STUCK_BUSY_MS).
|
|
243
248
|
this.idleReapInterval = setInterval(() => {
|
|
@@ -247,7 +252,7 @@ class WorkerPool {
|
|
|
247
252
|
}, IDLE_WORKER_TIMEOUT_MS);
|
|
248
253
|
}
|
|
249
254
|
isHealthy() {
|
|
250
|
-
return !this.destroyed && this.workers.length > 0;
|
|
255
|
+
return (!this.destroyed && !this.spawnDeniedReason && this.workers.length > 0);
|
|
251
256
|
}
|
|
252
257
|
/** PIDs of workers the pool currently tracks. Used by the daemon's orphan
|
|
253
258
|
* sweep to distinguish live, accounted-for workers from de-listed strays. */
|
|
@@ -287,16 +292,16 @@ class WorkerPool {
|
|
|
287
292
|
});
|
|
288
293
|
return hasUnassigned(this.priorityQueue) || hasUnassigned(this.taskQueue);
|
|
289
294
|
}
|
|
290
|
-
rejectUnassignedTasks(message) {
|
|
295
|
+
rejectUnassignedTasks(message, code) {
|
|
291
296
|
for (const task of Array.from(this.tasks.values())) {
|
|
292
297
|
if (task.worker)
|
|
293
298
|
continue;
|
|
294
|
-
task.reject(new Error(message));
|
|
299
|
+
task.reject(Object.assign(new Error(message), code ? { code } : {}));
|
|
295
300
|
this.completeTask(task, null);
|
|
296
301
|
}
|
|
297
302
|
}
|
|
298
303
|
maybeRespawn(reason, hasPendingTasks) {
|
|
299
|
-
if (this.destroyed || this.respawnLimitReached)
|
|
304
|
+
if (this.destroyed || this.spawnDeniedReason || this.respawnLimitReached)
|
|
300
305
|
return false;
|
|
301
306
|
this.consecutiveRespawns++;
|
|
302
307
|
(0, logger_1.log)("pool", `respawn #${this.consecutiveRespawns} after ${reason} (workers=${this.workers.length} pending=${hasPendingTasks})`);
|
|
@@ -308,8 +313,7 @@ class WorkerPool {
|
|
|
308
313
|
this.rejectUnassignedTasks(message);
|
|
309
314
|
return false;
|
|
310
315
|
}
|
|
311
|
-
this.spawnWorker();
|
|
312
|
-
return true;
|
|
316
|
+
return this.spawnWorker();
|
|
313
317
|
}
|
|
314
318
|
completeTask(task, worker) {
|
|
315
319
|
this.clearTaskTimeout(task);
|
|
@@ -357,8 +361,69 @@ class WorkerPool {
|
|
|
357
361
|
this.dispatch();
|
|
358
362
|
}
|
|
359
363
|
}
|
|
364
|
+
spawnDeniedError() {
|
|
365
|
+
var _a;
|
|
366
|
+
return Object.assign(new Error((_a = this.spawnDeniedReason) !== null && _a !== void 0 ? _a : "Worker spawn denied"), { code: "HOST_SAFETY" });
|
|
367
|
+
}
|
|
368
|
+
denySpawn(reason, persist = true) {
|
|
369
|
+
// A pool cannot automatically recover after a failed admission. In
|
|
370
|
+
// particular, timer/exit callbacks must not retry forks or strand queues.
|
|
371
|
+
this.spawnDeniedReason = `Worker spawn denied: ${reason}`;
|
|
372
|
+
this.clearScaleUpTimer();
|
|
373
|
+
if (persist) {
|
|
374
|
+
try {
|
|
375
|
+
(0, safety_latch_1.latchSafetyStop)(this.spawnDeniedReason);
|
|
376
|
+
}
|
|
377
|
+
catch (_a) {
|
|
378
|
+
try {
|
|
379
|
+
(0, logger_1.log)("pool", "worker admission denied; safety latch persistence failed");
|
|
380
|
+
}
|
|
381
|
+
catch (_b) { }
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
this.rejectUnassignedTasks(this.spawnDeniedReason, "HOST_SAFETY");
|
|
385
|
+
try {
|
|
386
|
+
(0, logger_1.log)("pool", this.spawnDeniedReason);
|
|
387
|
+
}
|
|
388
|
+
catch (_c) { }
|
|
389
|
+
return false;
|
|
390
|
+
}
|
|
360
391
|
spawnWorker() {
|
|
361
|
-
|
|
392
|
+
if (this.destroyed || this.spawnDeniedReason)
|
|
393
|
+
return false;
|
|
394
|
+
let worker;
|
|
395
|
+
try {
|
|
396
|
+
const quarantine = (0, autostart_1.daemonStartDeniedReason)();
|
|
397
|
+
if (quarantine)
|
|
398
|
+
return this.denySpawn(quarantine, false);
|
|
399
|
+
if (process.platform === "darwin") {
|
|
400
|
+
const memory = (0, kernel_zone_1.probeMemoryPressure)();
|
|
401
|
+
if (memory.status !== "known" || memory.pressure !== "normal") {
|
|
402
|
+
return this.denySpawn(`OS memory pressure ${memory.status === "known" ? memory.pressure : memory.status}; ${(0, kernel_zone_1.formatPressureProbe)(memory)}`);
|
|
403
|
+
}
|
|
404
|
+
const kernel = (0, kernel_zone_1.probeKernelZoneUsage)();
|
|
405
|
+
if (kernel.status !== "known" || kernel.usage.pressure !== "ok") {
|
|
406
|
+
return this.denySpawn(`kernel pressure ${kernel.status === "known" ? kernel.usage.pressure : kernel.status}; ${(0, kernel_zone_1.formatPressureProbe)(kernel)}`);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
if (process.platform === "darwin") {
|
|
410
|
+
// Refresh the cheap OS sample after the potentially slow kernel probe.
|
|
411
|
+
const freshMemory = (0, kernel_zone_1.probeMemoryPressure)();
|
|
412
|
+
if (freshMemory.status !== "known" ||
|
|
413
|
+
freshMemory.pressure !== "normal") {
|
|
414
|
+
return this.denySpawn(`OS memory pressure ${freshMemory.status === "known" ? freshMemory.pressure : freshMemory.status}; ${(0, kernel_zone_1.formatPressureProbe)(freshMemory)}`);
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
// Probes are synchronous but another process can quarantine the host
|
|
418
|
+
// while they run. Recheck the shared markers immediately before fork.
|
|
419
|
+
const finalQuarantine = (0, autostart_1.daemonStartDeniedReason)();
|
|
420
|
+
if (finalQuarantine)
|
|
421
|
+
return this.denySpawn(finalQuarantine, false);
|
|
422
|
+
worker = new ProcessWorker(this.modulePath, this.execArgv, this.embeddingEnvironment, config_1.MAX_WORKER_MEMORY_MB);
|
|
423
|
+
}
|
|
424
|
+
catch (_a) {
|
|
425
|
+
return this.denySpawn("host admission or fork failed");
|
|
426
|
+
}
|
|
362
427
|
(0, logger_1.log)("pool", `spawn PID:${worker.child.pid} (${this.workers.length + 1}/${Math.max(1, config_1.CONFIG.WORKER_THREADS)})`);
|
|
363
428
|
const onMessage = (msg) => {
|
|
364
429
|
var _a, _b, _c, _d;
|
|
@@ -409,11 +474,14 @@ class WorkerPool {
|
|
|
409
474
|
worker.child.on("exit", onExit);
|
|
410
475
|
worker.child.on("error", onError);
|
|
411
476
|
this.workers.push(worker);
|
|
477
|
+
return true;
|
|
412
478
|
}
|
|
413
479
|
enqueue(method, payload, signal) {
|
|
414
480
|
if (this.destroyed) {
|
|
415
481
|
return Promise.reject(new Error("Worker pool destroyed"));
|
|
416
482
|
}
|
|
483
|
+
if (this.spawnDeniedReason)
|
|
484
|
+
return Promise.reject(this.spawnDeniedError());
|
|
417
485
|
if (signal === null || signal === void 0 ? void 0 : signal.aborted) {
|
|
418
486
|
const err = new Error("Aborted");
|
|
419
487
|
err.name = "AbortError";
|
|
@@ -502,6 +570,23 @@ class WorkerPool {
|
|
|
502
570
|
var _c, _d, _e;
|
|
503
571
|
if (this.destroyed)
|
|
504
572
|
return;
|
|
573
|
+
if (this.spawnDeniedReason) {
|
|
574
|
+
this.rejectUnassignedTasks(this.spawnDeniedReason, "HOST_SAFETY");
|
|
575
|
+
return;
|
|
576
|
+
}
|
|
577
|
+
// Work admitted before another process quarantines the host may still
|
|
578
|
+
// be queued. Do not send it to an existing worker after that stop.
|
|
579
|
+
try {
|
|
580
|
+
const quarantine = (0, autostart_1.daemonStartDeniedReason)();
|
|
581
|
+
if (quarantine) {
|
|
582
|
+
this.denySpawn(quarantine, false);
|
|
583
|
+
return;
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
catch (_f) {
|
|
587
|
+
this.denySpawn("quarantine check failed");
|
|
588
|
+
return;
|
|
589
|
+
}
|
|
505
590
|
let idle = this.workers.find((w) => !w.busy);
|
|
506
591
|
// Drain priority queue first so search tasks never wait behind an
|
|
507
592
|
// indexing batch.
|
|
@@ -528,7 +613,8 @@ class WorkerPool {
|
|
|
528
613
|
this.armScaleUpTimer(waitMs);
|
|
529
614
|
return;
|
|
530
615
|
}
|
|
531
|
-
this.spawnWorker()
|
|
616
|
+
if (!this.spawnWorker())
|
|
617
|
+
return;
|
|
532
618
|
idle = this.workers[this.workers.length - 1];
|
|
533
619
|
}
|
|
534
620
|
if (!idle)
|
|
@@ -723,7 +809,8 @@ class WorkerPool {
|
|
|
723
809
|
}
|
|
724
810
|
// Replace anything we dropped below the floor with fresh, lean workers.
|
|
725
811
|
while (!this.destroyed && this.workers.length < MIN_KEEP_WORKERS) {
|
|
726
|
-
this.spawnWorker()
|
|
812
|
+
if (!this.spawnWorker())
|
|
813
|
+
break;
|
|
727
814
|
}
|
|
728
815
|
}
|
|
729
816
|
/**
|
package/package.json
CHANGED
|
@@ -15,15 +15,23 @@ const SOCKET = path.join(GMAX_DIR, "daemon.sock");
|
|
|
15
15
|
// Activity refreshes this TTL; idle sessions do not watch forever.
|
|
16
16
|
const SESSION_LEASE_TTL_MS = 4 * 60 * 60 * 1000;
|
|
17
17
|
|
|
18
|
-
function
|
|
19
|
-
if (process.env.GMAX_NO_AUTOSTART === "1") return true;
|
|
18
|
+
function markerPresentOrUnknown(file) {
|
|
20
19
|
try {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
20
|
+
fs.lstatSync(file);
|
|
21
|
+
return true;
|
|
22
|
+
} catch (error) {
|
|
23
|
+
return error.code !== "ENOENT";
|
|
24
24
|
}
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
+
function isAutostartDisabled() {
|
|
28
|
+
if (process.env.GMAX_NO_AUTOSTART === "1") return true;
|
|
29
|
+
return (
|
|
30
|
+
markerPresentOrUnknown(path.join(GMAX_DIR, "autostart-disabled")) ||
|
|
31
|
+
markerPresentOrUnknown(path.join(GMAX_DIR, "safety-stop.json"))
|
|
32
|
+
);
|
|
33
|
+
}
|
|
34
|
+
|
|
27
35
|
function readHookInput(timeoutMs = 1000) {
|
|
28
36
|
return new Promise((resolve) => {
|
|
29
37
|
let data = "";
|