grepmax 0.26.72 → 0.26.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/dist/commands/watch.js +15 -4
- package/dist/lib/daemon/daemon.js +28 -14
- package/dist/lib/daemon/watcher-manager.js +209 -170
- package/dist/lib/index/watcher-recovery.js +91 -0
- package/dist/lib/index/watcher.js +97 -61
- package/dist/lib/output/index-state-footer.js +8 -0
- package/dist/lib/utils/host-guard-policy.js +1 -0
- package/dist/lib/utils/kernel-zone.js +4 -4
- package/dist/lib/utils/pressure-diagnostics.js +176 -0
- package/dist/lib/utils/resource-budget.js +12 -9
- package/dist/lib/utils/safety-latch.js +5 -1
- package/dist/lib/utils/watcher-store.js +3 -2
- package/dist/lib/workers/pool.js +13 -6
- package/package.json +1 -1
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
|
@@ -59,6 +59,7 @@ const ignore_patterns_1 = require("./ignore-patterns");
|
|
|
59
59
|
const walker_1 = require("./walker");
|
|
60
60
|
const watcher_errors_1 = require("./watcher-errors");
|
|
61
61
|
const watcher_ignore_1 = require("./watcher-ignore");
|
|
62
|
+
const watcher_recovery_1 = require("./watcher-recovery");
|
|
62
63
|
// Directory names ignored at ANY depth. @parcel/watcher treats a non-glob
|
|
63
64
|
// ignore entry as a literal path resolved against the watch root, so a bare
|
|
64
65
|
// "dist" only covered <root>/dist — nested build dirs (packages/*/dist,
|
|
@@ -114,6 +115,7 @@ function startWatcher(opts) {
|
|
|
114
115
|
const { projectRoot } = opts;
|
|
115
116
|
const wtag = `watch:${projectRoot.split("/").pop()}`;
|
|
116
117
|
const filePolicy = new file_policy_1.ProjectFilePolicy(projectRoot);
|
|
118
|
+
const recovery = new watcher_recovery_1.WatcherRecoveryTracker();
|
|
117
119
|
let reconciliation = null;
|
|
118
120
|
let reconcileRequested = false;
|
|
119
121
|
let closing = false;
|
|
@@ -143,6 +145,7 @@ function startWatcher(opts) {
|
|
|
143
145
|
subscriptionFailed = false;
|
|
144
146
|
yield subscription === null || subscription === void 0 ? void 0 : subscription.unsubscribe();
|
|
145
147
|
subscription = null;
|
|
148
|
+
recovery.detached();
|
|
146
149
|
if (closing)
|
|
147
150
|
return;
|
|
148
151
|
const { subscription: next, policy: attachedPolicy } = yield (0, watcher_ignore_1.subscribeWithNativeExclusions)(projectRoot, (err, events) => {
|
|
@@ -150,6 +153,7 @@ function startWatcher(opts) {
|
|
|
150
153
|
if (closing || generation !== subscriptionGeneration)
|
|
151
154
|
return;
|
|
152
155
|
if (err) {
|
|
156
|
+
recovery.error(err);
|
|
153
157
|
console.error(`[${wtag}] Watcher error:`, err);
|
|
154
158
|
scanHealthy = false;
|
|
155
159
|
(_a = opts.onHealthChange) === null || _a === void 0 ? void 0 : _a.call(opts, false, 1);
|
|
@@ -171,6 +175,7 @@ function startWatcher(opts) {
|
|
|
171
175
|
}
|
|
172
176
|
subscription = next;
|
|
173
177
|
subscriptionPolicy = attachedPolicy;
|
|
178
|
+
recovery.attached(attachedPolicy);
|
|
174
179
|
}));
|
|
175
180
|
subscriptionUpdate = update;
|
|
176
181
|
return update;
|
|
@@ -199,76 +204,88 @@ function startWatcher(opts) {
|
|
|
199
204
|
reconcileRequested = false;
|
|
200
205
|
lastReconcileStart = Date.now();
|
|
201
206
|
const observedGapGeneration = gapGeneration;
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
return;
|
|
206
|
-
const rootPrefix = projectRoot.endsWith("/")
|
|
207
|
-
? projectRoot
|
|
208
|
-
: `${projectRoot}/`;
|
|
209
|
-
const cached = yield opts.metaCache.getKeysWithPrefix(rootPrefix);
|
|
210
|
-
const vectorPaths = yield opts.vectorDb.getDistinctPathsForPrefix(rootPrefix);
|
|
211
|
-
const knownPaths = new Set([...cached, ...vectorPaths]);
|
|
212
|
-
const seen = new Set();
|
|
213
|
-
const state = (0, walker_1.createWalkState)();
|
|
207
|
+
const scan = recovery.beginScan();
|
|
208
|
+
let outcome = "failed";
|
|
209
|
+
let retryDelayMs = 0;
|
|
214
210
|
try {
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
211
|
+
filePolicy.invalidateIgnoreCache();
|
|
212
|
+
yield updateSubscription();
|
|
213
|
+
if (closing)
|
|
214
|
+
return;
|
|
215
|
+
const rootPrefix = projectRoot.endsWith("/")
|
|
216
|
+
? projectRoot
|
|
217
|
+
: `${projectRoot}/`;
|
|
218
|
+
const cached = yield opts.metaCache.getKeysWithPrefix(rootPrefix);
|
|
219
|
+
const vectorPaths = yield opts.vectorDb.getDistinctPathsForPrefix(rootPrefix);
|
|
220
|
+
const knownPaths = new Set([...cached, ...vectorPaths]);
|
|
221
|
+
const seen = new Set();
|
|
222
|
+
const state = (0, walker_1.createWalkState)();
|
|
223
|
+
try {
|
|
224
|
+
for (var _e = true, _f = (e_1 = void 0, __asyncValues((0, walker_1.walk)(projectRoot, {
|
|
225
|
+
policy: filePolicy,
|
|
226
|
+
state,
|
|
227
|
+
}))), _g; _g = yield _f.next(), _a = _g.done, !_a; _e = true) {
|
|
228
|
+
_c = _g.value;
|
|
229
|
+
_e = false;
|
|
230
|
+
const relative = _c;
|
|
231
|
+
if (closing)
|
|
232
|
+
return;
|
|
233
|
+
const absolute = path.join(projectRoot, relative);
|
|
234
|
+
seen.add(absolute);
|
|
235
|
+
const reconciliation = (0, cache_coherence_1.reconcileMetaEntry)(absolute, opts.metaCache.get(absolute), vectorPaths.has(absolute));
|
|
236
|
+
if (reconciliation.action === "stamp") {
|
|
237
|
+
opts.metaCache.put(absolute, reconciliation.entry);
|
|
238
|
+
}
|
|
239
|
+
processor.handleFileEvent("change", absolute, {
|
|
240
|
+
forceReprocess: reconciliation.action === "reprocess",
|
|
241
|
+
workKind: "catchup",
|
|
242
|
+
});
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
246
|
+
finally {
|
|
247
|
+
try {
|
|
248
|
+
if (!_e && !_a && (_b = _f.return)) yield _b.call(_f);
|
|
249
|
+
}
|
|
250
|
+
finally { if (e_1) throw e_1.error; }
|
|
251
|
+
}
|
|
252
|
+
for (const cachedPath of knownPaths) {
|
|
222
253
|
if (closing)
|
|
223
254
|
return;
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
255
|
+
if (!seen.has(cachedPath) &&
|
|
256
|
+
!(0, walker_1.isPathProtectedByWalkState)(cachedPath, state)) {
|
|
257
|
+
processor.handleFileEvent("unlink", cachedPath, {
|
|
258
|
+
forceDelete: true,
|
|
259
|
+
workKind: "cleanup",
|
|
260
|
+
});
|
|
229
261
|
}
|
|
230
|
-
processor.handleFileEvent("change", absolute, {
|
|
231
|
-
forceReprocess: reconciliation.action === "reprocess",
|
|
232
|
-
workKind: "catchup",
|
|
233
|
-
});
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
237
|
-
finally {
|
|
238
|
-
try {
|
|
239
|
-
if (!_e && !_a && (_b = _f.return)) yield _b.call(_f);
|
|
240
262
|
}
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
});
|
|
263
|
+
if (!state.rootComplete || state.errors.length > 0) {
|
|
264
|
+
outcome = "incomplete";
|
|
265
|
+
scanHealthy = false;
|
|
266
|
+
(_d = opts.onHealthChange) === null || _d === void 0 ? void 0 : _d.call(opts, false, state.errors.length);
|
|
267
|
+
console.error(`[${wtag}] Reconciliation incomplete at ${state.errors.length} path(s)`);
|
|
268
|
+
if (incompleteRetries < 3) {
|
|
269
|
+
incompleteRetries++;
|
|
270
|
+
reconcileRequested = true;
|
|
271
|
+
retryDelayMs = 1000 * Math.pow(2, (incompleteRetries - 1));
|
|
272
|
+
}
|
|
252
273
|
}
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
yield new Promise((resolve) => setTimeout(resolve, 1000 * Math.pow(2, (incompleteRetries - 1))));
|
|
274
|
+
else {
|
|
275
|
+
outcome = "complete";
|
|
276
|
+
scanHealthy = observedGapGeneration === gapGeneration;
|
|
277
|
+
if (ingestionDegraded && processor.progress.pendingFiles > 0) {
|
|
278
|
+
degradedRepairQueued = true;
|
|
279
|
+
}
|
|
280
|
+
reportHealthyIfSettled();
|
|
281
|
+
incompleteRetries = 0;
|
|
262
282
|
}
|
|
263
283
|
}
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
if (ingestionDegraded && processor.progress.pendingFiles > 0) {
|
|
267
|
-
degradedRepairQueued = true;
|
|
268
|
-
}
|
|
269
|
-
reportHealthyIfSettled();
|
|
270
|
-
incompleteRetries = 0;
|
|
284
|
+
finally {
|
|
285
|
+
recovery.finishScan(scan, closing ? "aborted" : outcome);
|
|
271
286
|
}
|
|
287
|
+
if (retryDelayMs > 0)
|
|
288
|
+
yield new Promise((resolve) => setTimeout(resolve, retryDelayMs));
|
|
272
289
|
} while (reconcileRequested && !closing);
|
|
273
290
|
}))()
|
|
274
291
|
.catch((err) => console.error(`[${wtag}] Reconciliation failed:`, err))
|
|
@@ -313,6 +330,24 @@ function startWatcher(opts) {
|
|
|
313
330
|
yield updateSubscription();
|
|
314
331
|
reconcile();
|
|
315
332
|
return {
|
|
333
|
+
get health() {
|
|
334
|
+
const progress = processor.progress;
|
|
335
|
+
const state = recovery.snapshot();
|
|
336
|
+
return {
|
|
337
|
+
indexing: progress.processing || progress.pendingFiles > 0,
|
|
338
|
+
pendingFiles: progress.pendingFiles,
|
|
339
|
+
queue: progress.queue,
|
|
340
|
+
failedFiles: Math.max(terminalFailures.size, progress.failedFiles),
|
|
341
|
+
degraded: !scanHealthy || ingestionDegraded || terminalFailures.size > 0,
|
|
342
|
+
watcherMode: closing
|
|
343
|
+
? undefined
|
|
344
|
+
: subscriptionFailed || gapScanTimer || state.reconciliationNeeded
|
|
345
|
+
? "recovering"
|
|
346
|
+
: "native",
|
|
347
|
+
catchupRunning: reconciliation !== null,
|
|
348
|
+
watcherRecovery: state,
|
|
349
|
+
};
|
|
350
|
+
},
|
|
316
351
|
get progress() {
|
|
317
352
|
return processor.progress;
|
|
318
353
|
},
|
|
@@ -322,6 +357,7 @@ function startWatcher(opts) {
|
|
|
322
357
|
clearTimeout(gapScanTimer);
|
|
323
358
|
yield subscriptionUpdate.catch(() => { });
|
|
324
359
|
yield subscription === null || subscription === void 0 ? void 0 : subscription.unsubscribe();
|
|
360
|
+
recovery.detached();
|
|
325
361
|
yield reconciliation;
|
|
326
362
|
yield processor.close();
|
|
327
363
|
}),
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
// agent can decide to caveat its answer or retry once indexing settles.
|
|
8
8
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
9
|
exports.formatWatchQueue = formatWatchQueue;
|
|
10
|
+
exports.formatWatcherRecovery = formatWatcherRecovery;
|
|
10
11
|
exports.formatIndexStateFooter = formatIndexStateFooter;
|
|
11
12
|
function formatWatchQueue(queue) {
|
|
12
13
|
const age = queue.oldestLiveEditAgeMs === null
|
|
@@ -14,6 +15,13 @@ function formatWatchQueue(queue) {
|
|
|
14
15
|
: `${Math.ceil(queue.oldestLiveEditAgeMs / 1000)}s`;
|
|
15
16
|
return `live=${queue.live} catchup=${queue.catchup} cleanup=${queue.cleanup} active=${queue.activeFiles} oldestLiveEdit=${age}`;
|
|
16
17
|
}
|
|
18
|
+
function formatWatcherRecovery(state) {
|
|
19
|
+
var _a, _b;
|
|
20
|
+
var _c, _d;
|
|
21
|
+
if (!state)
|
|
22
|
+
return null;
|
|
23
|
+
return `gaps=${state.gapCount} covered=${state.coveredGapCount} outstanding=${state.outstandingGapCount} terminalErrors=${state.terminalErrorCount} reconciliationNeeded=${state.reconciliationNeeded} scan=${(_c = (_a = state.lastScan) === null || _a === void 0 ? void 0 : _a.id) !== null && _c !== void 0 ? _c : "none"}/${(_d = (_b = state.lastScan) === null || _b === void 0 ? void 0 : _b.outcome) !== null && _d !== void 0 ? _d : "none"} exclusions=${state.exclusions ? `${state.exclusions.attached ? "attached" : "detached"}:${state.exclusions.literalCount}+${state.exclusions.filterCount}:${state.exclusions.fingerprint}` : "unknown"}`;
|
|
24
|
+
}
|
|
17
25
|
/**
|
|
18
26
|
* One-line footer describing an in-progress index, or null when there's
|
|
19
27
|
* nothing to say (no state, or the index is settled). Suppressing the
|
|
@@ -77,6 +77,7 @@ function sampleCriticalPressure() {
|
|
|
77
77
|
memoryPressure,
|
|
78
78
|
kernelPressure: kernel.status === "known" ? kernel.usage.pressure : "unknown",
|
|
79
79
|
kernelBytes: kernel.status === "known" ? kernel.usage.bytes : null,
|
|
80
|
+
pressureProbes: { memoryInitial: memory, kernel, memoryFinal: freshMemory },
|
|
80
81
|
incompleteReasons: [
|
|
81
82
|
"aggregate admission disabled by explicit critical-only policy",
|
|
82
83
|
],
|
|
@@ -170,11 +170,11 @@ function probeMemoryPressure(overrides = {}) {
|
|
|
170
170
|
const result = boundedPressureProbe("sysctl", ["-n", "kern.memorystatus_vm_pressure_level"], 1000, (output) => {
|
|
171
171
|
switch (output.trim()) {
|
|
172
172
|
case "1":
|
|
173
|
-
return "normal";
|
|
173
|
+
return { pressure: "normal", level: 1 };
|
|
174
174
|
case "2":
|
|
175
|
-
return "warn";
|
|
175
|
+
return { pressure: "warn", level: 2 };
|
|
176
176
|
case "4":
|
|
177
|
-
return "critical";
|
|
177
|
+
return { pressure: "critical", level: 4 };
|
|
178
178
|
default:
|
|
179
179
|
return null;
|
|
180
180
|
}
|
|
@@ -182,7 +182,7 @@ function probeMemoryPressure(overrides = {}) {
|
|
|
182
182
|
if (result.status !== "known")
|
|
183
183
|
return result;
|
|
184
184
|
const { value } = result, observation = __rest(result, ["value"]);
|
|
185
|
-
return Object.assign(Object.assign({}, observation),
|
|
185
|
+
return Object.assign(Object.assign({}, observation), value);
|
|
186
186
|
}
|
|
187
187
|
function classifyZonePressure(bytes) {
|
|
188
188
|
if (bytes >= CRITICAL_BYTES)
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.sanitizeStopDiagnostics = sanitizeStopDiagnostics;
|
|
4
|
+
exports.probeStopDiagnostics = probeStopDiagnostics;
|
|
5
|
+
exports.resourceStopDiagnostics = resourceStopDiagnostics;
|
|
6
|
+
const sources = new Set([
|
|
7
|
+
"resource-admission",
|
|
8
|
+
"daemon-pressure",
|
|
9
|
+
"worker-spawn",
|
|
10
|
+
]);
|
|
11
|
+
const actions = new Set([
|
|
12
|
+
"check",
|
|
13
|
+
"reserve",
|
|
14
|
+
"startup",
|
|
15
|
+
"pressure-check",
|
|
16
|
+
"heartbeat",
|
|
17
|
+
"operation",
|
|
18
|
+
"worker-spawn",
|
|
19
|
+
]);
|
|
20
|
+
const kinds = new Set([
|
|
21
|
+
"worker",
|
|
22
|
+
"embedding",
|
|
23
|
+
"native-store",
|
|
24
|
+
"llm",
|
|
25
|
+
"prune-helper",
|
|
26
|
+
"prune-runtime-setup",
|
|
27
|
+
"other",
|
|
28
|
+
]);
|
|
29
|
+
const memoryStates = new Set([
|
|
30
|
+
"normal",
|
|
31
|
+
"warn",
|
|
32
|
+
"critical",
|
|
33
|
+
"unknown",
|
|
34
|
+
"unsupported",
|
|
35
|
+
]);
|
|
36
|
+
const kernelStates = new Set([
|
|
37
|
+
"ok",
|
|
38
|
+
"warn",
|
|
39
|
+
"critical",
|
|
40
|
+
"unknown",
|
|
41
|
+
"unsupported",
|
|
42
|
+
]);
|
|
43
|
+
const reasons = new Set([
|
|
44
|
+
"timeout",
|
|
45
|
+
"output-limit",
|
|
46
|
+
"exit",
|
|
47
|
+
"execution",
|
|
48
|
+
"parse",
|
|
49
|
+
"unsupported-platform",
|
|
50
|
+
]);
|
|
51
|
+
const number = (n) => typeof n === "number" && Number.isFinite(n) && n >= 0 ? n : undefined;
|
|
52
|
+
const nullableNumber = (n) => { var _a; return (_a = number(n)) !== null && _a !== void 0 ? _a : null; };
|
|
53
|
+
function cleanProbe(p) {
|
|
54
|
+
if (!["known", "unknown", "unsupported"].includes(p.status))
|
|
55
|
+
return;
|
|
56
|
+
return Object.assign({ status: p.status, sampledAtMs: number(p.sampledAtMs), durationMs: number(p.durationMs), outputBytes: number(p.outputBytes), reason: p.reason && reasons.has(p.reason) ? p.reason : undefined }, (p.status === "known"
|
|
57
|
+
? {
|
|
58
|
+
memoryPressure: p.memoryPressure && memoryStates.has(p.memoryPressure)
|
|
59
|
+
? p.memoryPressure
|
|
60
|
+
: undefined,
|
|
61
|
+
memoryLevel: p.memoryLevel !== undefined && [1, 2, 4].includes(p.memoryLevel)
|
|
62
|
+
? p.memoryLevel
|
|
63
|
+
: undefined,
|
|
64
|
+
kernelPressure: p.kernelPressure && kernelStates.has(p.kernelPressure)
|
|
65
|
+
? p.kernelPressure
|
|
66
|
+
: undefined,
|
|
67
|
+
kernelBytes: number(p.kernelBytes),
|
|
68
|
+
kernelElements: number(p.kernelElements),
|
|
69
|
+
kernelElementSize: number(p.kernelElementSize),
|
|
70
|
+
}
|
|
71
|
+
: {}));
|
|
72
|
+
}
|
|
73
|
+
/** Rebuild the allowlist at persistence, even when a runtime caller supplies extras.
|
|
74
|
+
* Invalid diagnostic data must never prevent the original stop from persisting. */
|
|
75
|
+
function sanitizeStopDiagnostics(input) {
|
|
76
|
+
try {
|
|
77
|
+
if (!sources.has(input.source) ||
|
|
78
|
+
!actions.has(input.action) ||
|
|
79
|
+
!["strict", "critical-only"].includes(input.policy) ||
|
|
80
|
+
!Number.isSafeInteger(input.pid) ||
|
|
81
|
+
input.pid < 1)
|
|
82
|
+
return;
|
|
83
|
+
const s = input.snapshot;
|
|
84
|
+
const p = input.probes;
|
|
85
|
+
return {
|
|
86
|
+
source: input.source,
|
|
87
|
+
action: input.action,
|
|
88
|
+
policy: input.policy,
|
|
89
|
+
pid: input.pid,
|
|
90
|
+
reservationKind: input.reservationKind && kinds.has(input.reservationKind)
|
|
91
|
+
? input.reservationKind
|
|
92
|
+
: undefined,
|
|
93
|
+
reservationMb: number(input.reservationMb),
|
|
94
|
+
snapshot: s
|
|
95
|
+
? {
|
|
96
|
+
at: nullableNumber(s.at),
|
|
97
|
+
completedAt: nullableNumber(s.completedAt),
|
|
98
|
+
memoryPressure: memoryStates.has(s.memoryPressure)
|
|
99
|
+
? s.memoryPressure
|
|
100
|
+
: "unknown",
|
|
101
|
+
kernelPressure: kernelStates.has(s.kernelPressure)
|
|
102
|
+
? s.kernelPressure
|
|
103
|
+
: "unknown",
|
|
104
|
+
kernelBytes: nullableNumber(s.kernelBytes),
|
|
105
|
+
aggregateFootprintMb: nullableNumber(s.aggregateFootprintMb),
|
|
106
|
+
physicalFreeMb: nullableNumber(s.physicalFreeMb),
|
|
107
|
+
swapUsedMb: nullableNumber(s.swapUsedMb),
|
|
108
|
+
}
|
|
109
|
+
: undefined,
|
|
110
|
+
probes: p
|
|
111
|
+
? {
|
|
112
|
+
memoryInitial: p.memoryInitial
|
|
113
|
+
? cleanProbe(p.memoryInitial)
|
|
114
|
+
: undefined,
|
|
115
|
+
kernel: p.kernel ? cleanProbe(p.kernel) : undefined,
|
|
116
|
+
memoryFinal: p.memoryFinal ? cleanProbe(p.memoryFinal) : undefined,
|
|
117
|
+
}
|
|
118
|
+
: undefined,
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
catch (_a) {
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
function probeEvidence(p) {
|
|
126
|
+
return Object.assign({ status: p.status, sampledAtMs: p.sampledAtMs, durationMs: p.durationMs, outputBytes: p.outputBytes }, (p.status !== "known"
|
|
127
|
+
? { reason: p.reason }
|
|
128
|
+
: "usage" in p
|
|
129
|
+
? {
|
|
130
|
+
kernelPressure: p.usage.pressure,
|
|
131
|
+
kernelBytes: p.usage.bytes,
|
|
132
|
+
kernelElements: p.usage.elements,
|
|
133
|
+
kernelElementSize: p.usage.elementSize,
|
|
134
|
+
}
|
|
135
|
+
: { memoryPressure: p.pressure, memoryLevel: p.level }));
|
|
136
|
+
}
|
|
137
|
+
function probeStopDiagnostics(source, action, policy, probes) {
|
|
138
|
+
return {
|
|
139
|
+
source,
|
|
140
|
+
action,
|
|
141
|
+
policy,
|
|
142
|
+
pid: process.pid,
|
|
143
|
+
probes: {
|
|
144
|
+
memoryInitial: probes.memoryInitial && probeEvidence(probes.memoryInitial),
|
|
145
|
+
kernel: probes.kernel && probeEvidence(probes.kernel),
|
|
146
|
+
memoryFinal: probes.memoryFinal && probeEvidence(probes.memoryFinal),
|
|
147
|
+
},
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
function resourceStopDiagnostics(snapshot, policy, action, pid, kind, mb) {
|
|
151
|
+
return {
|
|
152
|
+
source: "resource-admission",
|
|
153
|
+
action,
|
|
154
|
+
policy,
|
|
155
|
+
pid,
|
|
156
|
+
snapshot: {
|
|
157
|
+
at: snapshot.at,
|
|
158
|
+
completedAt: snapshot.completedAt,
|
|
159
|
+
memoryPressure: snapshot.memoryPressure,
|
|
160
|
+
kernelPressure: snapshot.kernelPressure,
|
|
161
|
+
kernelBytes: snapshot.kernelBytes,
|
|
162
|
+
aggregateFootprintMb: snapshot.aggregateFootprintMb,
|
|
163
|
+
physicalFreeMb: snapshot.physicalFreeMb,
|
|
164
|
+
swapUsedMb: snapshot.swapUsedMb,
|
|
165
|
+
},
|
|
166
|
+
reservationKind: kind
|
|
167
|
+
? kinds.has(kind)
|
|
168
|
+
? kind
|
|
169
|
+
: "other"
|
|
170
|
+
: undefined,
|
|
171
|
+
reservationMb: mb,
|
|
172
|
+
probes: snapshot.pressureProbes
|
|
173
|
+
? probeStopDiagnostics("daemon-pressure", "pressure-check", policy, snapshot.pressureProbes).probes
|
|
174
|
+
: undefined,
|
|
175
|
+
};
|
|
176
|
+
}
|
|
@@ -48,6 +48,7 @@ const config_1 = require("../../config");
|
|
|
48
48
|
const autostart_1 = require("./autostart");
|
|
49
49
|
const host_guard_policy_1 = require("./host-guard-policy");
|
|
50
50
|
const host_resource_1 = require("./host-resource");
|
|
51
|
+
const pressure_diagnostics_1 = require("./pressure-diagnostics");
|
|
51
52
|
const safety_latch_1 = require("./safety-latch");
|
|
52
53
|
/** Admission thresholds, not an OS-enforced memory cap. Keep enough room for
|
|
53
54
|
* a worker's measured 1536MiB recycle threshold, and charge pending launches
|
|
@@ -58,9 +59,10 @@ exports.WORKER_RESOURCE_RESERVE_MB = 1536;
|
|
|
58
59
|
exports.EMBEDDING_RESOURCE_RESERVE_MB = 1024;
|
|
59
60
|
exports.HELPER_RESOURCE_RESERVE_MB = 512;
|
|
60
61
|
class ResourceAdmissionError extends Error {
|
|
61
|
-
constructor(message, critical = false) {
|
|
62
|
+
constructor(message, critical = false, diagnostics) {
|
|
62
63
|
super(message);
|
|
63
64
|
this.critical = critical;
|
|
65
|
+
this.diagnostics = diagnostics;
|
|
64
66
|
this.code = "HOST_RESOURCE";
|
|
65
67
|
this.name = "ResourceAdmissionError";
|
|
66
68
|
}
|
|
@@ -108,7 +110,7 @@ class ResourceBudget {
|
|
|
108
110
|
this.deps = Object.assign({ root: path.join(config_1.PATHS.sharedRoot, "resource-budget"), platform: process.platform, pid: process.pid, now: Date.now, sample: (extraPids = []) => {
|
|
109
111
|
const records = this.readExistingRecords();
|
|
110
112
|
return (0, host_resource_1.sampleHostResources)([...records.map((r) => r.pid), ...extraPids], this.deps.samplerOverrides, records.flatMap((r) => (r.groupPid ? [r.groupPid] : [])));
|
|
111
|
-
}, signal: (pid) => process.kill(pid, 0), policy: host_guard_policy_1.hostGuardPolicy, requireClientRegistration: true, samplerOverrides: {}, sampleMaxAgeMs: 5000, criticalSample: host_guard_policy_1.sampleCriticalPressure, quarantine: autostart_1.daemonStartDeniedReason, latch: safety_latch_1.latchSafetyStop, processStart: () => (0, node_child_process_1.execFileSync)("ps", ["-p", String(process.pid), "-o", "lstart="], {
|
|
113
|
+
}, signal: (pid) => process.kill(pid, 0), policy: host_guard_policy_1.hostGuardPolicy, requireClientRegistration: true, samplerOverrides: {}, sampleMaxAgeMs: 5000, criticalSample: host_guard_policy_1.sampleCriticalPressure, quarantine: autostart_1.daemonStartDeniedReason, latch: (reason, diagnostics) => (0, safety_latch_1.latchSafetyStop)(reason, undefined, diagnostics), processStart: () => (0, node_child_process_1.execFileSync)("ps", ["-p", String(process.pid), "-o", "lstart="], {
|
|
112
114
|
encoding: "utf8",
|
|
113
115
|
timeout: 500,
|
|
114
116
|
maxBuffer: 1024,
|
|
@@ -263,7 +265,7 @@ class ResourceBudget {
|
|
|
263
265
|
fs.closeSync(dir);
|
|
264
266
|
}
|
|
265
267
|
}
|
|
266
|
-
checked(snapshot, records) {
|
|
268
|
+
checked(snapshot, records, kind, mb) {
|
|
267
269
|
try {
|
|
268
270
|
assertResourceSnapshot(snapshot, this.charge(snapshot, records), resolveResourceBudgetMb(), this.deps.now(), this.deps.sampleMaxAgeMs);
|
|
269
271
|
const legacy = snapshot.processes.filter((p) => p.role === "mcp" &&
|
|
@@ -273,8 +275,9 @@ class ResourceBudget {
|
|
|
273
275
|
}
|
|
274
276
|
catch (error) {
|
|
275
277
|
if (error instanceof ResourceAdmissionError && error.critical) {
|
|
278
|
+
error.diagnostics = (0, pressure_diagnostics_1.resourceStopDiagnostics)(snapshot, "strict", kind ? "reserve" : "check", this.deps.pid, kind, mb);
|
|
276
279
|
try {
|
|
277
|
-
this.deps.latch(error.message);
|
|
280
|
+
this.deps.latch(error.message, error.diagnostics);
|
|
278
281
|
}
|
|
279
282
|
catch (_a) {
|
|
280
283
|
error.message += "; safety stop persistence failed";
|
|
@@ -346,7 +349,7 @@ class ResourceBudget {
|
|
|
346
349
|
throw new ResourceAdmissionError("host containment");
|
|
347
350
|
return snapshot;
|
|
348
351
|
}
|
|
349
|
-
check(requiredPid) {
|
|
352
|
+
check(requiredPid, reservation) {
|
|
350
353
|
if (this.deps.platform !== "darwin")
|
|
351
354
|
return null;
|
|
352
355
|
if (requiredPid !== undefined &&
|
|
@@ -363,9 +366,9 @@ class ResourceBudget {
|
|
|
363
366
|
const snapshot = this.deps.criticalSample();
|
|
364
367
|
if (snapshot.memoryPressure === "critical" ||
|
|
365
368
|
snapshot.kernelPressure === "critical") {
|
|
366
|
-
const error = new ResourceAdmissionError("critical host pressure; heavy work refused", true);
|
|
369
|
+
const error = new ResourceAdmissionError("critical host pressure; heavy work refused", true, (0, pressure_diagnostics_1.resourceStopDiagnostics)(snapshot, "critical-only", reservation ? "reserve" : "check", this.deps.pid, reservation === null || reservation === void 0 ? void 0 : reservation.kind, reservation === null || reservation === void 0 ? void 0 : reservation.mb));
|
|
367
370
|
try {
|
|
368
|
-
this.deps.latch(error.message);
|
|
371
|
+
this.deps.latch(error.message, error.diagnostics);
|
|
369
372
|
}
|
|
370
373
|
catch (_a) {
|
|
371
374
|
error.message += "; safety stop persistence failed";
|
|
@@ -391,7 +394,7 @@ class ResourceBudget {
|
|
|
391
394
|
if (this.deps.platform !== "darwin")
|
|
392
395
|
return { attach: () => { }, release: () => { } };
|
|
393
396
|
if (this.deps.policy() === "critical-only") {
|
|
394
|
-
this.check();
|
|
397
|
+
this.check(undefined, { kind, mb });
|
|
395
398
|
return { attach: () => { }, release: () => { } };
|
|
396
399
|
}
|
|
397
400
|
let record = this.withLock(() => {
|
|
@@ -418,7 +421,7 @@ class ResourceBudget {
|
|
|
418
421
|
mb: charge,
|
|
419
422
|
kind,
|
|
420
423
|
};
|
|
421
|
-
this.checked(snapshot, [...records, next]);
|
|
424
|
+
this.checked(snapshot, [...records, next], kind, mb);
|
|
422
425
|
this.persist(next);
|
|
423
426
|
return next;
|
|
424
427
|
});
|
|
@@ -41,6 +41,7 @@ const node_crypto_1 = require("node:crypto");
|
|
|
41
41
|
const fs = __importStar(require("node:fs"));
|
|
42
42
|
const path = __importStar(require("node:path"));
|
|
43
43
|
const config_1 = require("../../config");
|
|
44
|
+
const pressure_diagnostics_1 = require("./pressure-diagnostics");
|
|
44
45
|
/** Machine-wide stop: changing stores or installing a version cannot clear it. */
|
|
45
46
|
exports.SAFETY_LATCH_NAME = "safety-stop.json";
|
|
46
47
|
function safetyRoot() {
|
|
@@ -69,7 +70,7 @@ function safetyStopReason(root = safetyRoot()) {
|
|
|
69
70
|
}
|
|
70
71
|
}
|
|
71
72
|
/** Atomic durable latch; intentionally no automatic expiry or clear operation. */
|
|
72
|
-
function latchSafetyStop(reason, root = safetyRoot()) {
|
|
73
|
+
function latchSafetyStop(reason, root = safetyRoot(), diagnostics) {
|
|
73
74
|
if (safetyStopReason(root) !== null)
|
|
74
75
|
return;
|
|
75
76
|
fs.mkdirSync(root, { recursive: true, mode: 0o700 });
|
|
@@ -83,6 +84,9 @@ function latchSafetyStop(reason, root = safetyRoot()) {
|
|
|
83
84
|
schemaVersion: 1,
|
|
84
85
|
at: Date.now(),
|
|
85
86
|
reason: reason.replace(/[\r\n\t]/g, " ").slice(0, 512) || "host safety stop",
|
|
87
|
+
diagnostics: diagnostics
|
|
88
|
+
? (0, pressure_diagnostics_1.sanitizeStopDiagnostics)(diagnostics)
|
|
89
|
+
: undefined,
|
|
86
90
|
};
|
|
87
91
|
fs.writeFileSync(fd, `${JSON.stringify(record)}\n`);
|
|
88
92
|
fs.fsyncSync(fd);
|
|
@@ -109,11 +109,12 @@ function updateWatcherStatus(pid, status, lastReindex, lastError) {
|
|
|
109
109
|
}
|
|
110
110
|
}
|
|
111
111
|
}
|
|
112
|
-
function heartbeat(pid) {
|
|
112
|
+
function heartbeat(pid, indexState) {
|
|
113
113
|
const db = getDb();
|
|
114
114
|
for (const { key, value } of db.getRange()) {
|
|
115
115
|
if (value && value.pid === pid) {
|
|
116
|
-
|
|
116
|
+
const now = Date.now();
|
|
117
|
+
db.put(String(key), Object.assign(Object.assign(Object.assign({}, value), { lastHeartbeat: now }), (indexState ? { indexState, indexStateAt: now } : {})));
|
|
117
118
|
}
|
|
118
119
|
}
|
|
119
120
|
}
|
package/dist/lib/workers/pool.js
CHANGED
|
@@ -61,6 +61,7 @@ const autostart_1 = require("../utils/autostart");
|
|
|
61
61
|
const host_guard_policy_1 = require("../utils/host-guard-policy");
|
|
62
62
|
const kernel_zone_1 = require("../utils/kernel-zone");
|
|
63
63
|
const logger_1 = require("../utils/logger");
|
|
64
|
+
const pressure_diagnostics_1 = require("../utils/pressure-diagnostics");
|
|
64
65
|
const resource_budget_1 = require("../utils/resource-budget");
|
|
65
66
|
const safety_latch_1 = require("../utils/safety-latch");
|
|
66
67
|
exports.EXISTING_QUERY_WAIT_MS = 2000;
|
|
@@ -407,14 +408,14 @@ class WorkerPool {
|
|
|
407
408
|
var _a;
|
|
408
409
|
return Object.assign(new Error((_a = this.spawnDeniedReason) !== null && _a !== void 0 ? _a : "Worker spawn denied"), { code: "HOST_SAFETY" });
|
|
409
410
|
}
|
|
410
|
-
denySpawn(reason, persist = false) {
|
|
411
|
+
denySpawn(reason, persist = false, diagnostics) {
|
|
411
412
|
// A pool cannot automatically recover after a failed admission. In
|
|
412
413
|
// particular, timer/exit callbacks must not retry forks or strand queues.
|
|
413
414
|
this.spawnDeniedReason = `Worker spawn denied: ${reason}`;
|
|
414
415
|
this.clearScaleUpTimer();
|
|
415
416
|
if (persist) {
|
|
416
417
|
try {
|
|
417
|
-
(0, safety_latch_1.latchSafetyStop)(this.spawnDeniedReason);
|
|
418
|
+
(0, safety_latch_1.latchSafetyStop)(this.spawnDeniedReason, undefined, diagnostics);
|
|
418
419
|
}
|
|
419
420
|
catch (_a) {
|
|
420
421
|
try {
|
|
@@ -438,31 +439,37 @@ class WorkerPool {
|
|
|
438
439
|
const quarantine = (0, autostart_1.daemonStartDeniedReason)();
|
|
439
440
|
if (quarantine)
|
|
440
441
|
return this.denySpawn(quarantine, false);
|
|
441
|
-
const
|
|
442
|
+
const policy = (0, host_guard_policy_1.hostGuardPolicy)();
|
|
443
|
+
const strictHostGuard = policy === "strict";
|
|
444
|
+
const probes = {};
|
|
445
|
+
const diagnostics = () => (0, pressure_diagnostics_1.probeStopDiagnostics)("worker-spawn", "worker-spawn", policy, probes);
|
|
442
446
|
if (process.platform === "darwin") {
|
|
443
447
|
const memory = (0, kernel_zone_1.probeMemoryPressure)();
|
|
448
|
+
probes.memoryInitial = memory;
|
|
444
449
|
if ((memory.status === "known" && memory.pressure === "critical") ||
|
|
445
450
|
(strictHostGuard &&
|
|
446
451
|
(memory.status !== "known" || memory.pressure !== "normal"))) {
|
|
447
|
-
return this.denySpawn(`OS memory pressure ${memory.status === "known" ? memory.pressure : memory.status}; ${(0, kernel_zone_1.formatPressureProbe)(memory)}`, memory.status === "known" && memory.pressure === "critical");
|
|
452
|
+
return this.denySpawn(`OS memory pressure ${memory.status === "known" ? memory.pressure : memory.status}; ${(0, kernel_zone_1.formatPressureProbe)(memory)}`, memory.status === "known" && memory.pressure === "critical", diagnostics());
|
|
448
453
|
}
|
|
449
454
|
const kernel = (0, kernel_zone_1.probeKernelZoneUsage)();
|
|
455
|
+
probes.kernel = kernel;
|
|
450
456
|
if ((kernel.status === "known" && kernel.usage.pressure === "critical") ||
|
|
451
457
|
(strictHostGuard &&
|
|
452
458
|
(kernel.status !== "known" || kernel.usage.pressure !== "ok"))) {
|
|
453
|
-
return this.denySpawn(`kernel pressure ${kernel.status === "known" ? kernel.usage.pressure : kernel.status}; ${(0, kernel_zone_1.formatPressureProbe)(kernel)}`, kernel.status === "known" && kernel.usage.pressure === "critical");
|
|
459
|
+
return this.denySpawn(`kernel pressure ${kernel.status === "known" ? kernel.usage.pressure : kernel.status}; ${(0, kernel_zone_1.formatPressureProbe)(kernel)}`, kernel.status === "known" && kernel.usage.pressure === "critical", diagnostics());
|
|
454
460
|
}
|
|
455
461
|
}
|
|
456
462
|
if (process.platform === "darwin") {
|
|
457
463
|
// Refresh the cheap OS sample after the potentially slow kernel probe.
|
|
458
464
|
const freshMemory = (0, kernel_zone_1.probeMemoryPressure)();
|
|
465
|
+
probes.memoryFinal = freshMemory;
|
|
459
466
|
if ((freshMemory.status === "known" &&
|
|
460
467
|
freshMemory.pressure === "critical") ||
|
|
461
468
|
(strictHostGuard &&
|
|
462
469
|
(freshMemory.status !== "known" ||
|
|
463
470
|
freshMemory.pressure !== "normal"))) {
|
|
464
471
|
return this.denySpawn(`OS memory pressure ${freshMemory.status === "known" ? freshMemory.pressure : freshMemory.status}; ${(0, kernel_zone_1.formatPressureProbe)(freshMemory)}`, freshMemory.status === "known" &&
|
|
465
|
-
freshMemory.pressure === "critical");
|
|
472
|
+
freshMemory.pressure === "critical", diagnostics());
|
|
466
473
|
}
|
|
467
474
|
}
|
|
468
475
|
// Probes are synchronous but another process can quarantine the host
|