grepmax 0.26.72 → 0.26.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/dist/lib/daemon/daemon.js +28 -14
- package/dist/lib/utils/host-guard-policy.js +1 -0
- package/dist/lib/utils/kernel-zone.js +4 -4
- package/dist/lib/utils/pressure-diagnostics.js +176 -0
- package/dist/lib/utils/resource-budget.js +12 -9
- package/dist/lib/utils/safety-latch.js +5 -1
- package/dist/lib/workers/pool.js +13 -6
- package/package.json +1 -1
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"name": "grepmax",
|
|
11
11
|
"source": "./plugins/grepmax",
|
|
12
12
|
"description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
|
|
13
|
-
"version": "0.26.
|
|
13
|
+
"version": "0.26.73",
|
|
14
14
|
"author": {
|
|
15
15
|
"name": "Robert Owens",
|
|
16
16
|
"email": "robowens@me.com"
|
|
@@ -74,6 +74,7 @@ const keyed_mutex_1 = require("../utils/keyed-mutex");
|
|
|
74
74
|
const log_rotate_1 = require("../utils/log-rotate");
|
|
75
75
|
const logger_1 = require("../utils/logger");
|
|
76
76
|
const operation_coordinator_1 = require("../utils/operation-coordinator");
|
|
77
|
+
const pressure_diagnostics_1 = require("../utils/pressure-diagnostics");
|
|
77
78
|
const process_1 = require("../utils/process");
|
|
78
79
|
const process_footprint_1 = require("../utils/process-footprint");
|
|
79
80
|
const project_registry_1 = require("../utils/project-registry");
|
|
@@ -263,9 +264,9 @@ class Daemon {
|
|
|
263
264
|
if (denied !== null && !options.readOnly)
|
|
264
265
|
throw new Error(`gmax: ${denied}; preserving containment`);
|
|
265
266
|
// Explicit read-only startup preserves every marker and opens no models.
|
|
266
|
-
const pressure = this.hostPressureDeniedReason();
|
|
267
|
+
const pressure = this.hostPressureDeniedReason("startup");
|
|
267
268
|
if (pressure === null || pressure === void 0 ? void 0 : pressure.critical) {
|
|
268
|
-
(0, safety_latch_1.latchSafetyStop)(pressure.reason);
|
|
269
|
+
(0, safety_latch_1.latchSafetyStop)(pressure.reason, undefined, pressure.diagnostics);
|
|
269
270
|
throw new Error(`gmax: ${pressure.reason}; startup safety stop`);
|
|
270
271
|
}
|
|
271
272
|
if (options.readOnly || pressure !== null) {
|
|
@@ -534,7 +535,7 @@ class Daemon {
|
|
|
534
535
|
catch (_a) { }
|
|
535
536
|
(0, log_rotate_1.rotateLogFds)(path.join(config_1.PATHS.logsDir, "daemon.log"));
|
|
536
537
|
this.sweepWatchLeases();
|
|
537
|
-
if (!this.checkKernelZonePressure()) {
|
|
538
|
+
if (!this.checkKernelZonePressure("heartbeat")) {
|
|
538
539
|
this.heartbeatTick++;
|
|
539
540
|
if (this.pausedReason !== null && this.heartbeatTick % 5 === 0)
|
|
540
541
|
this.logResourceSnapshot("paused-heartbeat", null);
|
|
@@ -726,7 +727,7 @@ class Daemon {
|
|
|
726
727
|
yield (0, query_timeout_1.withQueryTimeout)(this.pausePromise, "pause transition", paused_reads_1.PAUSED_QUERY_TIMEOUT_MS);
|
|
727
728
|
if (this.shuttingDown || this.recycling)
|
|
728
729
|
throw new operation_coordinator_1.OperationClosedError();
|
|
729
|
-
this.checkKernelZonePressure();
|
|
730
|
+
this.checkKernelZonePressure("operation");
|
|
730
731
|
if (this.shuttingDown || this.recycling)
|
|
731
732
|
throw new operation_coordinator_1.OperationClosedError();
|
|
732
733
|
return fn(operationSignal);
|
|
@@ -795,7 +796,7 @@ class Daemon {
|
|
|
795
796
|
if (denied !== null)
|
|
796
797
|
throw new Error(`${name} refused: ${denied}`);
|
|
797
798
|
// Probe at operation admission, before setup/cache/worker/model side effects.
|
|
798
|
-
if (!this.checkKernelZonePressure()) {
|
|
799
|
+
if (!this.checkKernelZonePressure("operation")) {
|
|
799
800
|
throw new Error(`${name} refused: host pressure is unsafe or unknown${this.pausedReason ? `: ${this.pausedReason}` : ""}`);
|
|
800
801
|
}
|
|
801
802
|
if (!this.checkAggregateResources())
|
|
@@ -2140,7 +2141,7 @@ class Daemon {
|
|
|
2140
2141
|
}
|
|
2141
2142
|
runHeartbeatMaintenance(sampleFootprint) {
|
|
2142
2143
|
// Both ordinary probes and deferred-recycle ticks check safety first.
|
|
2143
|
-
if (!this.checkKernelZonePressure())
|
|
2144
|
+
if (!this.checkKernelZonePressure("heartbeat"))
|
|
2144
2145
|
return;
|
|
2145
2146
|
if (sampleFootprint && !this.checkAggregateResources())
|
|
2146
2147
|
return;
|
|
@@ -2260,12 +2261,12 @@ class Daemon {
|
|
|
2260
2261
|
* Non-macOS hosts have no such probe. On macOS an unavailable sample is
|
|
2261
2262
|
* explicitly unknown and cannot authorize further mutation.
|
|
2262
2263
|
*/
|
|
2263
|
-
checkKernelZonePressure() {
|
|
2264
|
+
checkKernelZonePressure(action = "pressure-check") {
|
|
2264
2265
|
if (this.shuttingDown || this.recycling)
|
|
2265
2266
|
return false;
|
|
2266
|
-
const pressure = this.hostPressureDeniedReason();
|
|
2267
|
+
const pressure = this.hostPressureDeniedReason(action);
|
|
2267
2268
|
if (pressure === null || pressure === void 0 ? void 0 : pressure.critical) {
|
|
2268
|
-
this.stopForSafety(pressure.reason);
|
|
2269
|
+
this.stopForSafety(pressure.reason, true, pressure.diagnostics);
|
|
2269
2270
|
return false;
|
|
2270
2271
|
}
|
|
2271
2272
|
const denied = (0, autostart_1.daemonStartDeniedReason)();
|
|
@@ -2287,39 +2288,48 @@ class Daemon {
|
|
|
2287
2288
|
? error.message
|
|
2288
2289
|
: "aggregate resource admission unavailable";
|
|
2289
2290
|
if (error instanceof resource_budget_1.ResourceAdmissionError && error.critical)
|
|
2290
|
-
this.stopForSafety(reason);
|
|
2291
|
+
this.stopForSafety(reason, true, error.diagnostics);
|
|
2291
2292
|
else
|
|
2292
2293
|
this.pauseWork(reason);
|
|
2293
2294
|
return false;
|
|
2294
2295
|
}
|
|
2295
2296
|
}
|
|
2296
2297
|
/** Unknown/warning pauses resource expansion; only known critical shuts down. */
|
|
2297
|
-
hostPressureDeniedReason() {
|
|
2298
|
+
hostPressureDeniedReason(action = "pressure-check") {
|
|
2298
2299
|
if (process.platform !== "darwin")
|
|
2299
2300
|
return null;
|
|
2301
|
+
const policy = (0, host_guard_policy_1.hostGuardPolicy)();
|
|
2302
|
+
const probes = {};
|
|
2303
|
+
const diagnostics = () => (0, pressure_diagnostics_1.probeStopDiagnostics)("daemon-pressure", action, policy, probes);
|
|
2300
2304
|
const memory = (0, kernel_zone_1.probeMemoryPressure)();
|
|
2305
|
+
probes.memoryInitial = memory;
|
|
2301
2306
|
if (memory.status === "known" && memory.pressure === "critical")
|
|
2302
2307
|
return {
|
|
2303
2308
|
reason: `critical OS memory pressure (${(0, kernel_zone_1.formatPressureProbe)(memory)})`,
|
|
2304
2309
|
measured: true,
|
|
2305
2310
|
critical: true,
|
|
2311
|
+
diagnostics: diagnostics(),
|
|
2306
2312
|
};
|
|
2307
2313
|
// Paused reads still need the kernel-critical guard, even if OS pressure warns.
|
|
2308
2314
|
const kernel = (0, kernel_zone_1.probeKernelZoneUsage)();
|
|
2315
|
+
probes.kernel = kernel;
|
|
2309
2316
|
if (kernel.status === "known" && kernel.usage.pressure === "critical")
|
|
2310
2317
|
return {
|
|
2311
2318
|
reason: `critical kernel pressure: ${(0, kernel_zone_1.formatZoneUsage)(kernel.usage)} (${(0, kernel_zone_1.formatPressureProbe)(kernel)})`,
|
|
2312
2319
|
measured: true,
|
|
2313
2320
|
critical: true,
|
|
2321
|
+
diagnostics: diagnostics(),
|
|
2314
2322
|
};
|
|
2315
2323
|
const freshMemory = (0, kernel_zone_1.probeMemoryPressure)();
|
|
2324
|
+
probes.memoryFinal = freshMemory;
|
|
2316
2325
|
if (freshMemory.status === "known" && freshMemory.pressure === "critical")
|
|
2317
2326
|
return {
|
|
2318
2327
|
reason: `critical OS memory pressure (${(0, kernel_zone_1.formatPressureProbe)(freshMemory)})`,
|
|
2319
2328
|
measured: true,
|
|
2320
2329
|
critical: true,
|
|
2330
|
+
diagnostics: diagnostics(),
|
|
2321
2331
|
};
|
|
2322
|
-
if (
|
|
2332
|
+
if (policy === "critical-only")
|
|
2323
2333
|
return null;
|
|
2324
2334
|
for (const sample of [memory, freshMemory]) {
|
|
2325
2335
|
if (sample.status !== "known")
|
|
@@ -2327,12 +2337,14 @@ class Daemon {
|
|
|
2327
2337
|
reason: `OS memory pressure unavailable (${(0, kernel_zone_1.formatPressureProbe)(sample)})`,
|
|
2328
2338
|
measured: false,
|
|
2329
2339
|
critical: false,
|
|
2340
|
+
diagnostics: diagnostics(),
|
|
2330
2341
|
};
|
|
2331
2342
|
if (sample.pressure !== "normal")
|
|
2332
2343
|
return {
|
|
2333
2344
|
reason: `warn OS memory pressure (${(0, kernel_zone_1.formatPressureProbe)(sample)})`,
|
|
2334
2345
|
measured: true,
|
|
2335
2346
|
critical: false,
|
|
2347
|
+
diagnostics: diagnostics(),
|
|
2336
2348
|
};
|
|
2337
2349
|
}
|
|
2338
2350
|
if (kernel.status !== "known")
|
|
@@ -2340,21 +2352,23 @@ class Daemon {
|
|
|
2340
2352
|
reason: `kernel pressure unavailable (${(0, kernel_zone_1.formatPressureProbe)(kernel)}); heavy work paused`,
|
|
2341
2353
|
measured: false,
|
|
2342
2354
|
critical: false,
|
|
2355
|
+
diagnostics: diagnostics(),
|
|
2343
2356
|
};
|
|
2344
2357
|
if (kernel.usage.pressure !== "ok")
|
|
2345
2358
|
return {
|
|
2346
2359
|
reason: `warn kernel pressure: ${(0, kernel_zone_1.formatZoneUsage)(kernel.usage)} (${(0, kernel_zone_1.formatPressureProbe)(kernel)})`,
|
|
2347
2360
|
measured: true,
|
|
2348
2361
|
critical: false,
|
|
2362
|
+
diagnostics: diagnostics(),
|
|
2349
2363
|
};
|
|
2350
2364
|
return null;
|
|
2351
2365
|
}
|
|
2352
|
-
stopForSafety(reason, persist = true) {
|
|
2366
|
+
stopForSafety(reason, persist = true, diagnostics) {
|
|
2353
2367
|
// Warning already pauses mutation: a stale index costs less than another
|
|
2354
2368
|
// host crash. Unknown readings are visible stops, never healthy samples.
|
|
2355
2369
|
try {
|
|
2356
2370
|
if (persist)
|
|
2357
|
-
(0, safety_latch_1.latchSafetyStop)(reason);
|
|
2371
|
+
(0, safety_latch_1.latchSafetyStop)(reason, undefined, diagnostics);
|
|
2358
2372
|
}
|
|
2359
2373
|
catch (error) {
|
|
2360
2374
|
console.error("[daemon] Failed to persist safety stop; stopping anyway:", error);
|
|
@@ -77,6 +77,7 @@ function sampleCriticalPressure() {
|
|
|
77
77
|
memoryPressure,
|
|
78
78
|
kernelPressure: kernel.status === "known" ? kernel.usage.pressure : "unknown",
|
|
79
79
|
kernelBytes: kernel.status === "known" ? kernel.usage.bytes : null,
|
|
80
|
+
pressureProbes: { memoryInitial: memory, kernel, memoryFinal: freshMemory },
|
|
80
81
|
incompleteReasons: [
|
|
81
82
|
"aggregate admission disabled by explicit critical-only policy",
|
|
82
83
|
],
|
|
@@ -170,11 +170,11 @@ function probeMemoryPressure(overrides = {}) {
|
|
|
170
170
|
const result = boundedPressureProbe("sysctl", ["-n", "kern.memorystatus_vm_pressure_level"], 1000, (output) => {
|
|
171
171
|
switch (output.trim()) {
|
|
172
172
|
case "1":
|
|
173
|
-
return "normal";
|
|
173
|
+
return { pressure: "normal", level: 1 };
|
|
174
174
|
case "2":
|
|
175
|
-
return "warn";
|
|
175
|
+
return { pressure: "warn", level: 2 };
|
|
176
176
|
case "4":
|
|
177
|
-
return "critical";
|
|
177
|
+
return { pressure: "critical", level: 4 };
|
|
178
178
|
default:
|
|
179
179
|
return null;
|
|
180
180
|
}
|
|
@@ -182,7 +182,7 @@ function probeMemoryPressure(overrides = {}) {
|
|
|
182
182
|
if (result.status !== "known")
|
|
183
183
|
return result;
|
|
184
184
|
const { value } = result, observation = __rest(result, ["value"]);
|
|
185
|
-
return Object.assign(Object.assign({}, observation),
|
|
185
|
+
return Object.assign(Object.assign({}, observation), value);
|
|
186
186
|
}
|
|
187
187
|
function classifyZonePressure(bytes) {
|
|
188
188
|
if (bytes >= CRITICAL_BYTES)
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.sanitizeStopDiagnostics = sanitizeStopDiagnostics;
|
|
4
|
+
exports.probeStopDiagnostics = probeStopDiagnostics;
|
|
5
|
+
exports.resourceStopDiagnostics = resourceStopDiagnostics;
|
|
6
|
+
const sources = new Set([
|
|
7
|
+
"resource-admission",
|
|
8
|
+
"daemon-pressure",
|
|
9
|
+
"worker-spawn",
|
|
10
|
+
]);
|
|
11
|
+
const actions = new Set([
|
|
12
|
+
"check",
|
|
13
|
+
"reserve",
|
|
14
|
+
"startup",
|
|
15
|
+
"pressure-check",
|
|
16
|
+
"heartbeat",
|
|
17
|
+
"operation",
|
|
18
|
+
"worker-spawn",
|
|
19
|
+
]);
|
|
20
|
+
const kinds = new Set([
|
|
21
|
+
"worker",
|
|
22
|
+
"embedding",
|
|
23
|
+
"native-store",
|
|
24
|
+
"llm",
|
|
25
|
+
"prune-helper",
|
|
26
|
+
"prune-runtime-setup",
|
|
27
|
+
"other",
|
|
28
|
+
]);
|
|
29
|
+
const memoryStates = new Set([
|
|
30
|
+
"normal",
|
|
31
|
+
"warn",
|
|
32
|
+
"critical",
|
|
33
|
+
"unknown",
|
|
34
|
+
"unsupported",
|
|
35
|
+
]);
|
|
36
|
+
const kernelStates = new Set([
|
|
37
|
+
"ok",
|
|
38
|
+
"warn",
|
|
39
|
+
"critical",
|
|
40
|
+
"unknown",
|
|
41
|
+
"unsupported",
|
|
42
|
+
]);
|
|
43
|
+
const reasons = new Set([
|
|
44
|
+
"timeout",
|
|
45
|
+
"output-limit",
|
|
46
|
+
"exit",
|
|
47
|
+
"execution",
|
|
48
|
+
"parse",
|
|
49
|
+
"unsupported-platform",
|
|
50
|
+
]);
|
|
51
|
+
const number = (n) => typeof n === "number" && Number.isFinite(n) && n >= 0 ? n : undefined;
|
|
52
|
+
const nullableNumber = (n) => { var _a; return (_a = number(n)) !== null && _a !== void 0 ? _a : null; };
|
|
53
|
+
function cleanProbe(p) {
|
|
54
|
+
if (!["known", "unknown", "unsupported"].includes(p.status))
|
|
55
|
+
return;
|
|
56
|
+
return Object.assign({ status: p.status, sampledAtMs: number(p.sampledAtMs), durationMs: number(p.durationMs), outputBytes: number(p.outputBytes), reason: p.reason && reasons.has(p.reason) ? p.reason : undefined }, (p.status === "known"
|
|
57
|
+
? {
|
|
58
|
+
memoryPressure: p.memoryPressure && memoryStates.has(p.memoryPressure)
|
|
59
|
+
? p.memoryPressure
|
|
60
|
+
: undefined,
|
|
61
|
+
memoryLevel: p.memoryLevel !== undefined && [1, 2, 4].includes(p.memoryLevel)
|
|
62
|
+
? p.memoryLevel
|
|
63
|
+
: undefined,
|
|
64
|
+
kernelPressure: p.kernelPressure && kernelStates.has(p.kernelPressure)
|
|
65
|
+
? p.kernelPressure
|
|
66
|
+
: undefined,
|
|
67
|
+
kernelBytes: number(p.kernelBytes),
|
|
68
|
+
kernelElements: number(p.kernelElements),
|
|
69
|
+
kernelElementSize: number(p.kernelElementSize),
|
|
70
|
+
}
|
|
71
|
+
: {}));
|
|
72
|
+
}
|
|
73
|
+
/** Rebuild the allowlist at persistence, even when a runtime caller supplies extras.
|
|
74
|
+
* Invalid diagnostic data must never prevent the original stop from persisting. */
|
|
75
|
+
function sanitizeStopDiagnostics(input) {
|
|
76
|
+
try {
|
|
77
|
+
if (!sources.has(input.source) ||
|
|
78
|
+
!actions.has(input.action) ||
|
|
79
|
+
!["strict", "critical-only"].includes(input.policy) ||
|
|
80
|
+
!Number.isSafeInteger(input.pid) ||
|
|
81
|
+
input.pid < 1)
|
|
82
|
+
return;
|
|
83
|
+
const s = input.snapshot;
|
|
84
|
+
const p = input.probes;
|
|
85
|
+
return {
|
|
86
|
+
source: input.source,
|
|
87
|
+
action: input.action,
|
|
88
|
+
policy: input.policy,
|
|
89
|
+
pid: input.pid,
|
|
90
|
+
reservationKind: input.reservationKind && kinds.has(input.reservationKind)
|
|
91
|
+
? input.reservationKind
|
|
92
|
+
: undefined,
|
|
93
|
+
reservationMb: number(input.reservationMb),
|
|
94
|
+
snapshot: s
|
|
95
|
+
? {
|
|
96
|
+
at: nullableNumber(s.at),
|
|
97
|
+
completedAt: nullableNumber(s.completedAt),
|
|
98
|
+
memoryPressure: memoryStates.has(s.memoryPressure)
|
|
99
|
+
? s.memoryPressure
|
|
100
|
+
: "unknown",
|
|
101
|
+
kernelPressure: kernelStates.has(s.kernelPressure)
|
|
102
|
+
? s.kernelPressure
|
|
103
|
+
: "unknown",
|
|
104
|
+
kernelBytes: nullableNumber(s.kernelBytes),
|
|
105
|
+
aggregateFootprintMb: nullableNumber(s.aggregateFootprintMb),
|
|
106
|
+
physicalFreeMb: nullableNumber(s.physicalFreeMb),
|
|
107
|
+
swapUsedMb: nullableNumber(s.swapUsedMb),
|
|
108
|
+
}
|
|
109
|
+
: undefined,
|
|
110
|
+
probes: p
|
|
111
|
+
? {
|
|
112
|
+
memoryInitial: p.memoryInitial
|
|
113
|
+
? cleanProbe(p.memoryInitial)
|
|
114
|
+
: undefined,
|
|
115
|
+
kernel: p.kernel ? cleanProbe(p.kernel) : undefined,
|
|
116
|
+
memoryFinal: p.memoryFinal ? cleanProbe(p.memoryFinal) : undefined,
|
|
117
|
+
}
|
|
118
|
+
: undefined,
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
catch (_a) {
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
function probeEvidence(p) {
|
|
126
|
+
return Object.assign({ status: p.status, sampledAtMs: p.sampledAtMs, durationMs: p.durationMs, outputBytes: p.outputBytes }, (p.status !== "known"
|
|
127
|
+
? { reason: p.reason }
|
|
128
|
+
: "usage" in p
|
|
129
|
+
? {
|
|
130
|
+
kernelPressure: p.usage.pressure,
|
|
131
|
+
kernelBytes: p.usage.bytes,
|
|
132
|
+
kernelElements: p.usage.elements,
|
|
133
|
+
kernelElementSize: p.usage.elementSize,
|
|
134
|
+
}
|
|
135
|
+
: { memoryPressure: p.pressure, memoryLevel: p.level }));
|
|
136
|
+
}
|
|
137
|
+
function probeStopDiagnostics(source, action, policy, probes) {
|
|
138
|
+
return {
|
|
139
|
+
source,
|
|
140
|
+
action,
|
|
141
|
+
policy,
|
|
142
|
+
pid: process.pid,
|
|
143
|
+
probes: {
|
|
144
|
+
memoryInitial: probes.memoryInitial && probeEvidence(probes.memoryInitial),
|
|
145
|
+
kernel: probes.kernel && probeEvidence(probes.kernel),
|
|
146
|
+
memoryFinal: probes.memoryFinal && probeEvidence(probes.memoryFinal),
|
|
147
|
+
},
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
function resourceStopDiagnostics(snapshot, policy, action, pid, kind, mb) {
|
|
151
|
+
return {
|
|
152
|
+
source: "resource-admission",
|
|
153
|
+
action,
|
|
154
|
+
policy,
|
|
155
|
+
pid,
|
|
156
|
+
snapshot: {
|
|
157
|
+
at: snapshot.at,
|
|
158
|
+
completedAt: snapshot.completedAt,
|
|
159
|
+
memoryPressure: snapshot.memoryPressure,
|
|
160
|
+
kernelPressure: snapshot.kernelPressure,
|
|
161
|
+
kernelBytes: snapshot.kernelBytes,
|
|
162
|
+
aggregateFootprintMb: snapshot.aggregateFootprintMb,
|
|
163
|
+
physicalFreeMb: snapshot.physicalFreeMb,
|
|
164
|
+
swapUsedMb: snapshot.swapUsedMb,
|
|
165
|
+
},
|
|
166
|
+
reservationKind: kind
|
|
167
|
+
? kinds.has(kind)
|
|
168
|
+
? kind
|
|
169
|
+
: "other"
|
|
170
|
+
: undefined,
|
|
171
|
+
reservationMb: mb,
|
|
172
|
+
probes: snapshot.pressureProbes
|
|
173
|
+
? probeStopDiagnostics("daemon-pressure", "pressure-check", policy, snapshot.pressureProbes).probes
|
|
174
|
+
: undefined,
|
|
175
|
+
};
|
|
176
|
+
}
|
|
@@ -48,6 +48,7 @@ const config_1 = require("../../config");
|
|
|
48
48
|
const autostart_1 = require("./autostart");
|
|
49
49
|
const host_guard_policy_1 = require("./host-guard-policy");
|
|
50
50
|
const host_resource_1 = require("./host-resource");
|
|
51
|
+
const pressure_diagnostics_1 = require("./pressure-diagnostics");
|
|
51
52
|
const safety_latch_1 = require("./safety-latch");
|
|
52
53
|
/** Admission thresholds, not an OS-enforced memory cap. Keep enough room for
|
|
53
54
|
* a worker's measured 1536MiB recycle threshold, and charge pending launches
|
|
@@ -58,9 +59,10 @@ exports.WORKER_RESOURCE_RESERVE_MB = 1536;
|
|
|
58
59
|
exports.EMBEDDING_RESOURCE_RESERVE_MB = 1024;
|
|
59
60
|
exports.HELPER_RESOURCE_RESERVE_MB = 512;
|
|
60
61
|
class ResourceAdmissionError extends Error {
|
|
61
|
-
constructor(message, critical = false) {
|
|
62
|
+
constructor(message, critical = false, diagnostics) {
|
|
62
63
|
super(message);
|
|
63
64
|
this.critical = critical;
|
|
65
|
+
this.diagnostics = diagnostics;
|
|
64
66
|
this.code = "HOST_RESOURCE";
|
|
65
67
|
this.name = "ResourceAdmissionError";
|
|
66
68
|
}
|
|
@@ -108,7 +110,7 @@ class ResourceBudget {
|
|
|
108
110
|
this.deps = Object.assign({ root: path.join(config_1.PATHS.sharedRoot, "resource-budget"), platform: process.platform, pid: process.pid, now: Date.now, sample: (extraPids = []) => {
|
|
109
111
|
const records = this.readExistingRecords();
|
|
110
112
|
return (0, host_resource_1.sampleHostResources)([...records.map((r) => r.pid), ...extraPids], this.deps.samplerOverrides, records.flatMap((r) => (r.groupPid ? [r.groupPid] : [])));
|
|
111
|
-
}, signal: (pid) => process.kill(pid, 0), policy: host_guard_policy_1.hostGuardPolicy, requireClientRegistration: true, samplerOverrides: {}, sampleMaxAgeMs: 5000, criticalSample: host_guard_policy_1.sampleCriticalPressure, quarantine: autostart_1.daemonStartDeniedReason, latch: safety_latch_1.latchSafetyStop, processStart: () => (0, node_child_process_1.execFileSync)("ps", ["-p", String(process.pid), "-o", "lstart="], {
|
|
113
|
+
}, signal: (pid) => process.kill(pid, 0), policy: host_guard_policy_1.hostGuardPolicy, requireClientRegistration: true, samplerOverrides: {}, sampleMaxAgeMs: 5000, criticalSample: host_guard_policy_1.sampleCriticalPressure, quarantine: autostart_1.daemonStartDeniedReason, latch: (reason, diagnostics) => (0, safety_latch_1.latchSafetyStop)(reason, undefined, diagnostics), processStart: () => (0, node_child_process_1.execFileSync)("ps", ["-p", String(process.pid), "-o", "lstart="], {
|
|
112
114
|
encoding: "utf8",
|
|
113
115
|
timeout: 500,
|
|
114
116
|
maxBuffer: 1024,
|
|
@@ -263,7 +265,7 @@ class ResourceBudget {
|
|
|
263
265
|
fs.closeSync(dir);
|
|
264
266
|
}
|
|
265
267
|
}
|
|
266
|
-
checked(snapshot, records) {
|
|
268
|
+
checked(snapshot, records, kind, mb) {
|
|
267
269
|
try {
|
|
268
270
|
assertResourceSnapshot(snapshot, this.charge(snapshot, records), resolveResourceBudgetMb(), this.deps.now(), this.deps.sampleMaxAgeMs);
|
|
269
271
|
const legacy = snapshot.processes.filter((p) => p.role === "mcp" &&
|
|
@@ -273,8 +275,9 @@ class ResourceBudget {
|
|
|
273
275
|
}
|
|
274
276
|
catch (error) {
|
|
275
277
|
if (error instanceof ResourceAdmissionError && error.critical) {
|
|
278
|
+
error.diagnostics = (0, pressure_diagnostics_1.resourceStopDiagnostics)(snapshot, "strict", kind ? "reserve" : "check", this.deps.pid, kind, mb);
|
|
276
279
|
try {
|
|
277
|
-
this.deps.latch(error.message);
|
|
280
|
+
this.deps.latch(error.message, error.diagnostics);
|
|
278
281
|
}
|
|
279
282
|
catch (_a) {
|
|
280
283
|
error.message += "; safety stop persistence failed";
|
|
@@ -346,7 +349,7 @@ class ResourceBudget {
|
|
|
346
349
|
throw new ResourceAdmissionError("host containment");
|
|
347
350
|
return snapshot;
|
|
348
351
|
}
|
|
349
|
-
check(requiredPid) {
|
|
352
|
+
check(requiredPid, reservation) {
|
|
350
353
|
if (this.deps.platform !== "darwin")
|
|
351
354
|
return null;
|
|
352
355
|
if (requiredPid !== undefined &&
|
|
@@ -363,9 +366,9 @@ class ResourceBudget {
|
|
|
363
366
|
const snapshot = this.deps.criticalSample();
|
|
364
367
|
if (snapshot.memoryPressure === "critical" ||
|
|
365
368
|
snapshot.kernelPressure === "critical") {
|
|
366
|
-
const error = new ResourceAdmissionError("critical host pressure; heavy work refused", true);
|
|
369
|
+
const error = new ResourceAdmissionError("critical host pressure; heavy work refused", true, (0, pressure_diagnostics_1.resourceStopDiagnostics)(snapshot, "critical-only", reservation ? "reserve" : "check", this.deps.pid, reservation === null || reservation === void 0 ? void 0 : reservation.kind, reservation === null || reservation === void 0 ? void 0 : reservation.mb));
|
|
367
370
|
try {
|
|
368
|
-
this.deps.latch(error.message);
|
|
371
|
+
this.deps.latch(error.message, error.diagnostics);
|
|
369
372
|
}
|
|
370
373
|
catch (_a) {
|
|
371
374
|
error.message += "; safety stop persistence failed";
|
|
@@ -391,7 +394,7 @@ class ResourceBudget {
|
|
|
391
394
|
if (this.deps.platform !== "darwin")
|
|
392
395
|
return { attach: () => { }, release: () => { } };
|
|
393
396
|
if (this.deps.policy() === "critical-only") {
|
|
394
|
-
this.check();
|
|
397
|
+
this.check(undefined, { kind, mb });
|
|
395
398
|
return { attach: () => { }, release: () => { } };
|
|
396
399
|
}
|
|
397
400
|
let record = this.withLock(() => {
|
|
@@ -418,7 +421,7 @@ class ResourceBudget {
|
|
|
418
421
|
mb: charge,
|
|
419
422
|
kind,
|
|
420
423
|
};
|
|
421
|
-
this.checked(snapshot, [...records, next]);
|
|
424
|
+
this.checked(snapshot, [...records, next], kind, mb);
|
|
422
425
|
this.persist(next);
|
|
423
426
|
return next;
|
|
424
427
|
});
|
|
@@ -41,6 +41,7 @@ const node_crypto_1 = require("node:crypto");
|
|
|
41
41
|
const fs = __importStar(require("node:fs"));
|
|
42
42
|
const path = __importStar(require("node:path"));
|
|
43
43
|
const config_1 = require("../../config");
|
|
44
|
+
const pressure_diagnostics_1 = require("./pressure-diagnostics");
|
|
44
45
|
/** Machine-wide stop: changing stores or installing a version cannot clear it. */
|
|
45
46
|
exports.SAFETY_LATCH_NAME = "safety-stop.json";
|
|
46
47
|
function safetyRoot() {
|
|
@@ -69,7 +70,7 @@ function safetyStopReason(root = safetyRoot()) {
|
|
|
69
70
|
}
|
|
70
71
|
}
|
|
71
72
|
/** Atomic durable latch; intentionally no automatic expiry or clear operation. */
|
|
72
|
-
function latchSafetyStop(reason, root = safetyRoot()) {
|
|
73
|
+
function latchSafetyStop(reason, root = safetyRoot(), diagnostics) {
|
|
73
74
|
if (safetyStopReason(root) !== null)
|
|
74
75
|
return;
|
|
75
76
|
fs.mkdirSync(root, { recursive: true, mode: 0o700 });
|
|
@@ -83,6 +84,9 @@ function latchSafetyStop(reason, root = safetyRoot()) {
|
|
|
83
84
|
schemaVersion: 1,
|
|
84
85
|
at: Date.now(),
|
|
85
86
|
reason: reason.replace(/[\r\n\t]/g, " ").slice(0, 512) || "host safety stop",
|
|
87
|
+
diagnostics: diagnostics
|
|
88
|
+
? (0, pressure_diagnostics_1.sanitizeStopDiagnostics)(diagnostics)
|
|
89
|
+
: undefined,
|
|
86
90
|
};
|
|
87
91
|
fs.writeFileSync(fd, `${JSON.stringify(record)}\n`);
|
|
88
92
|
fs.fsyncSync(fd);
|
package/dist/lib/workers/pool.js
CHANGED
|
@@ -61,6 +61,7 @@ const autostart_1 = require("../utils/autostart");
|
|
|
61
61
|
const host_guard_policy_1 = require("../utils/host-guard-policy");
|
|
62
62
|
const kernel_zone_1 = require("../utils/kernel-zone");
|
|
63
63
|
const logger_1 = require("../utils/logger");
|
|
64
|
+
const pressure_diagnostics_1 = require("../utils/pressure-diagnostics");
|
|
64
65
|
const resource_budget_1 = require("../utils/resource-budget");
|
|
65
66
|
const safety_latch_1 = require("../utils/safety-latch");
|
|
66
67
|
exports.EXISTING_QUERY_WAIT_MS = 2000;
|
|
@@ -407,14 +408,14 @@ class WorkerPool {
|
|
|
407
408
|
var _a;
|
|
408
409
|
return Object.assign(new Error((_a = this.spawnDeniedReason) !== null && _a !== void 0 ? _a : "Worker spawn denied"), { code: "HOST_SAFETY" });
|
|
409
410
|
}
|
|
410
|
-
denySpawn(reason, persist = false) {
|
|
411
|
+
denySpawn(reason, persist = false, diagnostics) {
|
|
411
412
|
// A pool cannot automatically recover after a failed admission. In
|
|
412
413
|
// particular, timer/exit callbacks must not retry forks or strand queues.
|
|
413
414
|
this.spawnDeniedReason = `Worker spawn denied: ${reason}`;
|
|
414
415
|
this.clearScaleUpTimer();
|
|
415
416
|
if (persist) {
|
|
416
417
|
try {
|
|
417
|
-
(0, safety_latch_1.latchSafetyStop)(this.spawnDeniedReason);
|
|
418
|
+
(0, safety_latch_1.latchSafetyStop)(this.spawnDeniedReason, undefined, diagnostics);
|
|
418
419
|
}
|
|
419
420
|
catch (_a) {
|
|
420
421
|
try {
|
|
@@ -438,31 +439,37 @@ class WorkerPool {
|
|
|
438
439
|
const quarantine = (0, autostart_1.daemonStartDeniedReason)();
|
|
439
440
|
if (quarantine)
|
|
440
441
|
return this.denySpawn(quarantine, false);
|
|
441
|
-
const
|
|
442
|
+
const policy = (0, host_guard_policy_1.hostGuardPolicy)();
|
|
443
|
+
const strictHostGuard = policy === "strict";
|
|
444
|
+
const probes = {};
|
|
445
|
+
const diagnostics = () => (0, pressure_diagnostics_1.probeStopDiagnostics)("worker-spawn", "worker-spawn", policy, probes);
|
|
442
446
|
if (process.platform === "darwin") {
|
|
443
447
|
const memory = (0, kernel_zone_1.probeMemoryPressure)();
|
|
448
|
+
probes.memoryInitial = memory;
|
|
444
449
|
if ((memory.status === "known" && memory.pressure === "critical") ||
|
|
445
450
|
(strictHostGuard &&
|
|
446
451
|
(memory.status !== "known" || memory.pressure !== "normal"))) {
|
|
447
|
-
return this.denySpawn(`OS memory pressure ${memory.status === "known" ? memory.pressure : memory.status}; ${(0, kernel_zone_1.formatPressureProbe)(memory)}`, memory.status === "known" && memory.pressure === "critical");
|
|
452
|
+
return this.denySpawn(`OS memory pressure ${memory.status === "known" ? memory.pressure : memory.status}; ${(0, kernel_zone_1.formatPressureProbe)(memory)}`, memory.status === "known" && memory.pressure === "critical", diagnostics());
|
|
448
453
|
}
|
|
449
454
|
const kernel = (0, kernel_zone_1.probeKernelZoneUsage)();
|
|
455
|
+
probes.kernel = kernel;
|
|
450
456
|
if ((kernel.status === "known" && kernel.usage.pressure === "critical") ||
|
|
451
457
|
(strictHostGuard &&
|
|
452
458
|
(kernel.status !== "known" || kernel.usage.pressure !== "ok"))) {
|
|
453
|
-
return this.denySpawn(`kernel pressure ${kernel.status === "known" ? kernel.usage.pressure : kernel.status}; ${(0, kernel_zone_1.formatPressureProbe)(kernel)}`, kernel.status === "known" && kernel.usage.pressure === "critical");
|
|
459
|
+
return this.denySpawn(`kernel pressure ${kernel.status === "known" ? kernel.usage.pressure : kernel.status}; ${(0, kernel_zone_1.formatPressureProbe)(kernel)}`, kernel.status === "known" && kernel.usage.pressure === "critical", diagnostics());
|
|
454
460
|
}
|
|
455
461
|
}
|
|
456
462
|
if (process.platform === "darwin") {
|
|
457
463
|
// Refresh the cheap OS sample after the potentially slow kernel probe.
|
|
458
464
|
const freshMemory = (0, kernel_zone_1.probeMemoryPressure)();
|
|
465
|
+
probes.memoryFinal = freshMemory;
|
|
459
466
|
if ((freshMemory.status === "known" &&
|
|
460
467
|
freshMemory.pressure === "critical") ||
|
|
461
468
|
(strictHostGuard &&
|
|
462
469
|
(freshMemory.status !== "known" ||
|
|
463
470
|
freshMemory.pressure !== "normal"))) {
|
|
464
471
|
return this.denySpawn(`OS memory pressure ${freshMemory.status === "known" ? freshMemory.pressure : freshMemory.status}; ${(0, kernel_zone_1.formatPressureProbe)(freshMemory)}`, freshMemory.status === "known" &&
|
|
465
|
-
freshMemory.pressure === "critical");
|
|
472
|
+
freshMemory.pressure === "critical", diagnostics());
|
|
466
473
|
}
|
|
467
474
|
}
|
|
468
475
|
// Probes are synchronous but another process can quarantine the host
|
package/package.json
CHANGED