grepmax 0.26.71 → 0.26.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/DOCUMENT-SEARCH.md +30 -9
- package/README.md +82 -0
- package/dist/lib/daemon/daemon.js +28 -14
- package/dist/lib/utils/host-guard-policy.js +1 -0
- package/dist/lib/utils/kernel-zone.js +4 -4
- package/dist/lib/utils/pressure-diagnostics.js +176 -0
- package/dist/lib/utils/resource-budget.js +23 -10
- package/dist/lib/utils/safety-latch.js +5 -1
- package/dist/lib/workers/pool.js +148 -35
- package/package.json +1 -1
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"name": "grepmax",
|
|
11
11
|
"source": "./plugins/grepmax",
|
|
12
12
|
"description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
|
|
13
|
-
"version": "0.26.
|
|
13
|
+
"version": "0.26.73",
|
|
14
14
|
"author": {
|
|
15
15
|
"name": "Robert Owens",
|
|
16
16
|
"email": "robowens@me.com"
|
package/DOCUMENT-SEARCH.md
CHANGED
|
@@ -63,16 +63,37 @@ confirmation of the existing configured MLX model. Idle time alone does not revo
|
|
|
63
63
|
that identity: each bounded `/embed` response revalidates model and dimensions,
|
|
64
64
|
and successful inference renews the normal health cache. Failure revokes restricted
|
|
65
65
|
eligibility until external work confirms readiness. A worker rechecks before inference;
|
|
66
|
-
no initialization, health polling, retry or backend fallback is allowed. Cold
|
|
67
|
-
|
|
66
|
+
no initialization, health polling, retry or backend fallback is allowed. Cold
|
|
67
|
+
workers refuse immediately. When compatible warm workers are occupied, at most
|
|
68
|
+
four document queries may wait FIFO for up to two seconds for a warm worker.
|
|
69
|
+
They get a turn between indexing files; after four consecutive document-query
|
|
70
|
+
assignments, waiting ordinary work gets a turn. A full queue or expired worker
|
|
71
|
+
wait returns `busy`. `queryState: ready` describes admission availability, including
|
|
72
|
+
this bounded wait, and does not guarantee immediate execution or query success.
|
|
73
|
+
Waiting requests live outside the ordinary worker queues and cannot trigger
|
|
74
|
+
worker startup, replacement or scaling. Existing ordinary work retains its own
|
|
75
|
+
admitted lifecycle, including independent expansion/replacement.
|
|
68
76
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
77
|
+
Admission and warm readiness are rechecked when a worker becomes available.
|
|
78
|
+
Queued cancellation/deadlines withdraw the request; cancellation after dispatch
|
|
79
|
+
keeps the shared worker assigned until its actual terminal response or exit.
|
|
80
|
+
The ten-second request bound includes waiting rather than restarting at dispatch.
|
|
81
|
+
Synchronous admission has its own probe bound and is checked against the absolute
|
|
82
|
+
request deadline before any query is sent. Native retrieval still uses the
|
|
83
|
+
remaining overall deadline. An in-flight indexing file is never interrupted;
|
|
84
|
+
long files or backend outages can still exceed the worker wait and return `busy`.
|
|
85
|
+
|
|
86
|
+
Read-only admission honors the configured host guard policy without creating locks,
|
|
87
|
+
reservations, pruning records or changing containment markers. Under the default
|
|
88
|
+
`strict` policy, existing Darwin inference requires fresh known-normal aggregate
|
|
89
|
+
resources and reads the existing ledger; absent ledgers are treated as empty and
|
|
90
|
+
corrupt or symlinked ledgers refuse. Explicit `critical-only` uses fresh OS/kernel
|
|
91
|
+
pressure probes without aggregate sampling or ledger access: warning/unknown
|
|
92
|
+
measurements remain diagnostic, while confirmed critical pressure refuses.
|
|
93
|
+
Containment is checked before and after sampling in either policy. This read-only
|
|
94
|
+
path never creates a safety latch; the daemon's safety machinery remains authoritative.
|
|
95
|
+
Other platforms retain the existing no-footprint-probe policy; this is not a
|
|
96
|
+
cross-platform native qualification claim. Markdown coverage
|
|
76
97
|
and returned pointers require regular files, including after symlink resolution;
|
|
77
98
|
a directory whose name ends in `.md` can be a prefix but never a source pointer.
|
|
78
99
|
|
package/README.md
CHANGED
|
@@ -241,6 +241,19 @@ index and already-warm compatible query worker. It cannot start a daemon, watch
|
|
|
241
241
|
project, initialize a model, fall back to another backend or mutate the index.
|
|
242
242
|
Cold, busy or unavailable resources return an explicit unavailable state.
|
|
243
243
|
|
|
244
|
+
Document query admission follows the configured host guard policy. On macOS,
|
|
245
|
+
default `strict` requires fresh aggregate resource admission; explicit
|
|
246
|
+
`critical-only` checks OS/kernel critical pressure without an aggregate scan or
|
|
247
|
+
ledger access. Warning or unavailable measurements stay diagnostic in that mode.
|
|
248
|
+
Both policies preserve containment checks. Up to four document queries can wait
|
|
249
|
+
FIFO for an already-warm compatible worker for up to two seconds, taking a turn
|
|
250
|
+
between indexing files. After four consecutive document queries, waiting ordinary
|
|
251
|
+
work gets a turn. Saturation or wait expiry returns `busy`; cold workers still
|
|
252
|
+
refuse. This wait cannot start, replace or scale workers, and it never interrupts
|
|
253
|
+
an in-flight file. Admission/readiness are checked again before dispatch, and the
|
|
254
|
+
overall ten-second deadline includes waiting. Readiness means a request can be
|
|
255
|
+
admitted, including this bounded wait. Long indexing files can still return `busy`.
|
|
256
|
+
|
|
244
257
|
Results contain canonical paths and one-based source ranges, without cached source
|
|
245
258
|
bodies. The first slice uses dense retrieval with explicit path scope and bounded
|
|
246
259
|
deadlines. Consumers must reread and authorize current source before using a
|
|
@@ -594,6 +607,75 @@ The candidate trace contains pointers and stage ranks for at most the first 200
|
|
|
594
607
|
|
|
595
608
|
Each traced candidate also has an `outcome`: returned, stage-one/pooled cut, deduplicated, per-file limit or display limit. Display limit means the production loop filled its result window before examining that candidate; it does not assert that the candidate would pass the per-file cap.
|
|
596
609
|
|
|
610
|
+
#### Existing-index document retrieval evaluation
|
|
611
|
+
|
|
612
|
+
In a source checkout, [scripts/eval-documents.cjs](scripts/eval-documents.cjs) measures the released [document MCP contract](DOCUMENT-SEARCH.md) through an explicit installed entry point. It uses only Node built-ins and sequential `document_search_status` / `semantic_search` calls. It performs no model warmup, daemon startup, indexing, refusal retries or fallback to normal hybrid search.
|
|
613
|
+
|
|
614
|
+
```bash
|
|
615
|
+
node scripts/eval-documents.cjs \
|
|
616
|
+
--fixture docs/measurements/documents/fixture-v1.json \
|
|
617
|
+
--sha256 <reviewed-fixture-digest> \
|
|
618
|
+
--root /absolute/already-indexed/project \
|
|
619
|
+
--entry /absolute/installed/grepmax/dist/bin.js \
|
|
620
|
+
--output docs/measurements/documents/baseline-v1.json
|
|
621
|
+
```
|
|
622
|
+
|
|
623
|
+
Freeze the JSON fixture before retrieving results. It contains `schemaVersion: 1`, `purpose`, explicit project-relative `prefixes`, and 1–100 `cases`. Each case has a unique `id`, a `query` (maximum 500 characters), and one or more `expected` targets: `{file, startLine, endLine, sourceSha256}`. Files are contained relative Markdown paths; ranges are one-based and answer-bearing; hashes cover exact source bytes. Keep private fixtures and evidence in ignored `docs/measurements/`. Optional `--repeats 1-3` repeats the same frozen cases without retries or warmup.
|
|
624
|
+
|
|
625
|
+
The report separates the rank of the expected document among distinct returned documents from the rank of an overlapping expected section among chunk pointers. Document Recall@10 credits the fraction of declared target documents retrieved; MRR@10 credits only ranks 1–10. Section overlap indicates source-context discovery, not complete answer coverage. Only the first 50 returned candidates are available; document deduplication cannot recover targets outside that window. Reviewed targets are not exhaustive judgments of all useful documents, so these metrics describe the frozen targets, not general recall.
|
|
626
|
+
|
|
627
|
+
Source and indexed hashes must match the fixture before querying; generation and project/store identity must agree across coverage and retrieval. Returned source ranges require reread/hash verification; new artifacts retain both the indexed pointer digest and current-source digest so later reviews can check provenance. Refusals, changed targets and unverified expected pointers are excluded from relevance metrics and counted separately; `usableFraction` includes every planned sample. Background indexing and partial/degraded index context are recorded without excluding targets whose coverage and hashes remain current. No queries, source bodies or raw diagnostics are copied into result artifacts. Reads are bounded to 2 MiB per source; MCP frames to 1 MiB. The report and sibling `.samples.jsonl` checkpoint refuse overwrite and use owner-only permissions. Exit 0 means an evaluation with usable samples completed, not that a quality threshold passed; exit 2 means no usable samples; exit 1 means preflight or transport/contract failure. No consumer rollout is implied.
|
|
628
|
+
|
|
629
|
+
To investigate scope competition, [scripts/eval-document-scopes.cjs](scripts/eval-document-scopes.cjs) compares the original fixture prefixes against declared narrower directories without changing queries or targets:
|
|
630
|
+
|
|
631
|
+
```bash
|
|
632
|
+
node scripts/eval-document-scopes.cjs \
|
|
633
|
+
--fixture docs/measurements/documents/fixture-v1.json --sha256 <digest> \
|
|
634
|
+
--root /absolute/already-indexed/project \
|
|
635
|
+
--entry /absolute/installed/grepmax/dist/bin.js \
|
|
636
|
+
--output docs/measurements/documents/paired-scope-v1.json \
|
|
637
|
+
--narrow-prefix docs/services --narrow-prefix docs/modules
|
|
638
|
+
```
|
|
639
|
+
|
|
640
|
+
Declare and freeze the scope hypothesis before querying. Narrower prefixes must stay within the original scope and retain every target. Arm order alternates per case, with a one-second gap between pairs and no retries. Reports show each arm's standalone observations and separately compare only pairs where both arms are usable under the same resource generation, project and store. The live competing corpus can still change; this is a diagnostic comparison rather than an isolated causal test. Scopes chosen from known target families introduce a useful prior, so gains do not establish unscoped or held-out retrieval quality. Owner-only checkpoints, overwrite refusal and source/transport bounds are shared with the base evaluator. Exit 2 means no comparable pairs.
|
|
641
|
+
|
|
642
|
+
### Offline lexical candidate experiment
|
|
643
|
+
|
|
644
|
+
[scripts/eval-document-lexical.cjs](scripts/eval-document-lexical.cjs) tests lexical candidates and fixed document reciprocal-rank fusion against a prior paired scope report. It uses Node built-ins and filesystem Markdown only; it never contacts a daemon, opens a native store, loads a model or changes production retrieval.
|
|
645
|
+
|
|
646
|
+
Freeze an owner-only study plan before scoring, including the original fixture checksum, archived report checksum, canonical root, narrower relative prefixes, fixed algorithm parameters and resource limits. The selected scope must retain every target and match the archived narrow arm. Pass that plan's checksum explicitly:
|
|
647
|
+
|
|
648
|
+
```bash
|
|
649
|
+
NODE_OPTIONS=--max-old-space-size=384 node scripts/eval-document-lexical.cjs \
|
|
650
|
+
--fixture docs/measurements/documents/fixture-v1.json \
|
|
651
|
+
--plan docs/measurements/documents/lexical-plan.json --plan-sha256 <digest> \
|
|
652
|
+
--archive docs/measurements/documents/paired-scope-v1.json \
|
|
653
|
+
--snapshot docs/measurements/documents/new-private-snapshot \
|
|
654
|
+
--output docs/measurements/documents/new-lexical-report.json
|
|
655
|
+
```
|
|
656
|
+
|
|
657
|
+
The prototype uses fixed 48-line windows with eight-line overlap, BM25 `k1=1.2` / `b=0.75`, 50 positive-score lexical candidates and equal-weight document RRF with constant 60. Targets are used for grading only. Stable path/line ties, no per-case overrides and no zero-score padding keep the experiment reproducible. Bounds are 512 files, 16 MiB total source, 2 MiB per file, 10,000 windows, one million token occurrences and 20,000 directory entries. Missing, aliased, changed or oversized source fails snapshot creation rather than silently dropping competing documents. A private snapshot preserves exact source bytes and a digest manifest; the report stores pointers, digests, IDs and scores without questions or source bodies. Existing output/snapshot paths are refused.
|
|
658
|
+
|
|
659
|
+
Lexical metrics cover cases whose frozen targets match the snapshot. Archive replay additionally requires original ready state/resource identity and every competitor pointer's indexed/current digest and range to match the snapshot; an invalid competitor excludes the entire case. Refused archive cases stay separate. Offline lexical availability does not exercise deployed busy/pressure admission. Archived dense candidates are not re-encoded over this snapshot, and narrower families chosen from known target locations introduce a prior. Report document gains/losses on the verified shared replay cohort; section overlap is diagnostic because window sizes and hybrid representatives differ from production chunks. A passing exploratory gate only supports proposing further matched qualification. No production integration or rollout follows automatically. Exit 0 means scoring completed with usable lexical cases, even when the gate fails; exit 2 means no usable lexical cases, and exit 1 means preflight or evaluation failure.
|
|
660
|
+
|
|
661
|
+
### Protected document fusion qualification
|
|
662
|
+
|
|
663
|
+
[scripts/eval-document-fusion.cjs](scripts/eval-document-fusion.cjs) qualifies a fixed candidate ordering in two modes: replay cached verified lexical/dense candidates, or collect one broader pass through the released existing-only MCP. It interleaves each arm's first three distinct documents (`lex1,dense1,lex2,dense2,lex3,dense3`), deduplicates while preserving the selected arm's pointer, then fills to 50 with the fixed RRF tail. This protects high-ranked lexical-only evidence; spending up to six slots can displace useful lower-ranked dense candidates. Expected targets never affect ranking.
|
|
664
|
+
|
|
665
|
+
```bash
|
|
666
|
+
NODE_OPTIONS=--max-old-space-size=384 node scripts/eval-document-fusion.cjs \
|
|
667
|
+
--mode live --design <frozen-design.json> --design-sha256 <digest> \
|
|
668
|
+
--fixture <frozen-fixture.json> --fixture-sha256 <digest> \
|
|
669
|
+
--entry /absolute/path/to/installed/grepmax/dist/bin.js --entry-sha256 <digest> \
|
|
670
|
+
--snapshot <new-private-snapshot-directory> --output <new-private-report.json>
|
|
671
|
+
```
|
|
672
|
+
|
|
673
|
+
For `--mode replay`, replace the entry arguments with `--input <prior-lexical-report.json> --input-sha256 <digest>` and point `--snapshot` at its immutable snapshot. Replay uses saved lexical candidates without rescoring queries. Live mode snapshots the complete predefined scope and builds the bounded filesystem index before MCP initialization; it permits one case-order pass with one-second gaps, without retries, warmup, runtime startup or native fallback. The existing-only entry must expose the expected restricted capabilities. Coverage, generation, project/store identity and every indexed/current competitor digest and range must verify. Refusals remain separate from relevance. A final full source-byte/file-set fence invalidates comparisons if the corpus changed; checkpoints are provisional and the final JSON is authoritative.
|
|
674
|
+
|
|
675
|
+
The lexical CLI retains its one-million-token default. A frozen fusion design may explicitly allow 1,100,000 token occurrences; all other source/window limits and the 384 MiB Node heap setting remain unchanged. The heap setting is not an operating-system physical memory cap. Outputs and snapshots refuse overwrite and use private permissions; reports exclude question text and source bodies. The broader study requires at least 16 of 20 comparable cases, no dense top-ten target losses, and protected document recall at least as high as either arm on the same cohort. Other predefined positive cohort bounds are supported. Exit 0 means collection completed with some comparable cases, even if the gate fails; exit 2 means none, and exit 1 means interrupted/preflight failure.
|
|
676
|
+
|
|
677
|
+
The October 8 experiment recovered both original residuals: Redis rank 1, People rank 3, 15/15 replay document hits with no dense top-ten target loss. Its broader 393-file, 20-case pass produced only two comparable cases (15 busy and three pressure refusals; one busy case also lacked indexed coverage), below the frozen 16-case minimum. Broader qualification is **insufficient**. Questions were source-curated after the rule freeze and held out from parameter selection, not independently adjudicated real-user questions. Representative/window differences prevent like-for-like section accuracy claims. This evaluation does not change production retrieval or authorize integration, release or rollout.
|
|
678
|
+
|
|
597
679
|
## Attribution
|
|
598
680
|
|
|
599
681
|
grepmax is built upon the foundation of [mgrep](https://github.com/mixedbread-ai/mgrep) by MixedBread. See the [NOTICE](NOTICE) file for details.
|
|
@@ -74,6 +74,7 @@ const keyed_mutex_1 = require("../utils/keyed-mutex");
|
|
|
74
74
|
const log_rotate_1 = require("../utils/log-rotate");
|
|
75
75
|
const logger_1 = require("../utils/logger");
|
|
76
76
|
const operation_coordinator_1 = require("../utils/operation-coordinator");
|
|
77
|
+
const pressure_diagnostics_1 = require("../utils/pressure-diagnostics");
|
|
77
78
|
const process_1 = require("../utils/process");
|
|
78
79
|
const process_footprint_1 = require("../utils/process-footprint");
|
|
79
80
|
const project_registry_1 = require("../utils/project-registry");
|
|
@@ -263,9 +264,9 @@ class Daemon {
|
|
|
263
264
|
if (denied !== null && !options.readOnly)
|
|
264
265
|
throw new Error(`gmax: ${denied}; preserving containment`);
|
|
265
266
|
// Explicit read-only startup preserves every marker and opens no models.
|
|
266
|
-
const pressure = this.hostPressureDeniedReason();
|
|
267
|
+
const pressure = this.hostPressureDeniedReason("startup");
|
|
267
268
|
if (pressure === null || pressure === void 0 ? void 0 : pressure.critical) {
|
|
268
|
-
(0, safety_latch_1.latchSafetyStop)(pressure.reason);
|
|
269
|
+
(0, safety_latch_1.latchSafetyStop)(pressure.reason, undefined, pressure.diagnostics);
|
|
269
270
|
throw new Error(`gmax: ${pressure.reason}; startup safety stop`);
|
|
270
271
|
}
|
|
271
272
|
if (options.readOnly || pressure !== null) {
|
|
@@ -534,7 +535,7 @@ class Daemon {
|
|
|
534
535
|
catch (_a) { }
|
|
535
536
|
(0, log_rotate_1.rotateLogFds)(path.join(config_1.PATHS.logsDir, "daemon.log"));
|
|
536
537
|
this.sweepWatchLeases();
|
|
537
|
-
if (!this.checkKernelZonePressure()) {
|
|
538
|
+
if (!this.checkKernelZonePressure("heartbeat")) {
|
|
538
539
|
this.heartbeatTick++;
|
|
539
540
|
if (this.pausedReason !== null && this.heartbeatTick % 5 === 0)
|
|
540
541
|
this.logResourceSnapshot("paused-heartbeat", null);
|
|
@@ -726,7 +727,7 @@ class Daemon {
|
|
|
726
727
|
yield (0, query_timeout_1.withQueryTimeout)(this.pausePromise, "pause transition", paused_reads_1.PAUSED_QUERY_TIMEOUT_MS);
|
|
727
728
|
if (this.shuttingDown || this.recycling)
|
|
728
729
|
throw new operation_coordinator_1.OperationClosedError();
|
|
729
|
-
this.checkKernelZonePressure();
|
|
730
|
+
this.checkKernelZonePressure("operation");
|
|
730
731
|
if (this.shuttingDown || this.recycling)
|
|
731
732
|
throw new operation_coordinator_1.OperationClosedError();
|
|
732
733
|
return fn(operationSignal);
|
|
@@ -795,7 +796,7 @@ class Daemon {
|
|
|
795
796
|
if (denied !== null)
|
|
796
797
|
throw new Error(`${name} refused: ${denied}`);
|
|
797
798
|
// Probe at operation admission, before setup/cache/worker/model side effects.
|
|
798
|
-
if (!this.checkKernelZonePressure()) {
|
|
799
|
+
if (!this.checkKernelZonePressure("operation")) {
|
|
799
800
|
throw new Error(`${name} refused: host pressure is unsafe or unknown${this.pausedReason ? `: ${this.pausedReason}` : ""}`);
|
|
800
801
|
}
|
|
801
802
|
if (!this.checkAggregateResources())
|
|
@@ -2140,7 +2141,7 @@ class Daemon {
|
|
|
2140
2141
|
}
|
|
2141
2142
|
runHeartbeatMaintenance(sampleFootprint) {
|
|
2142
2143
|
// Both ordinary probes and deferred-recycle ticks check safety first.
|
|
2143
|
-
if (!this.checkKernelZonePressure())
|
|
2144
|
+
if (!this.checkKernelZonePressure("heartbeat"))
|
|
2144
2145
|
return;
|
|
2145
2146
|
if (sampleFootprint && !this.checkAggregateResources())
|
|
2146
2147
|
return;
|
|
@@ -2260,12 +2261,12 @@ class Daemon {
|
|
|
2260
2261
|
* Non-macOS hosts have no such probe. On macOS an unavailable sample is
|
|
2261
2262
|
* explicitly unknown and cannot authorize further mutation.
|
|
2262
2263
|
*/
|
|
2263
|
-
checkKernelZonePressure() {
|
|
2264
|
+
checkKernelZonePressure(action = "pressure-check") {
|
|
2264
2265
|
if (this.shuttingDown || this.recycling)
|
|
2265
2266
|
return false;
|
|
2266
|
-
const pressure = this.hostPressureDeniedReason();
|
|
2267
|
+
const pressure = this.hostPressureDeniedReason(action);
|
|
2267
2268
|
if (pressure === null || pressure === void 0 ? void 0 : pressure.critical) {
|
|
2268
|
-
this.stopForSafety(pressure.reason);
|
|
2269
|
+
this.stopForSafety(pressure.reason, true, pressure.diagnostics);
|
|
2269
2270
|
return false;
|
|
2270
2271
|
}
|
|
2271
2272
|
const denied = (0, autostart_1.daemonStartDeniedReason)();
|
|
@@ -2287,39 +2288,48 @@ class Daemon {
|
|
|
2287
2288
|
? error.message
|
|
2288
2289
|
: "aggregate resource admission unavailable";
|
|
2289
2290
|
if (error instanceof resource_budget_1.ResourceAdmissionError && error.critical)
|
|
2290
|
-
this.stopForSafety(reason);
|
|
2291
|
+
this.stopForSafety(reason, true, error.diagnostics);
|
|
2291
2292
|
else
|
|
2292
2293
|
this.pauseWork(reason);
|
|
2293
2294
|
return false;
|
|
2294
2295
|
}
|
|
2295
2296
|
}
|
|
2296
2297
|
/** Unknown/warning pauses resource expansion; only known critical shuts down. */
|
|
2297
|
-
hostPressureDeniedReason() {
|
|
2298
|
+
hostPressureDeniedReason(action = "pressure-check") {
|
|
2298
2299
|
if (process.platform !== "darwin")
|
|
2299
2300
|
return null;
|
|
2301
|
+
const policy = (0, host_guard_policy_1.hostGuardPolicy)();
|
|
2302
|
+
const probes = {};
|
|
2303
|
+
const diagnostics = () => (0, pressure_diagnostics_1.probeStopDiagnostics)("daemon-pressure", action, policy, probes);
|
|
2300
2304
|
const memory = (0, kernel_zone_1.probeMemoryPressure)();
|
|
2305
|
+
probes.memoryInitial = memory;
|
|
2301
2306
|
if (memory.status === "known" && memory.pressure === "critical")
|
|
2302
2307
|
return {
|
|
2303
2308
|
reason: `critical OS memory pressure (${(0, kernel_zone_1.formatPressureProbe)(memory)})`,
|
|
2304
2309
|
measured: true,
|
|
2305
2310
|
critical: true,
|
|
2311
|
+
diagnostics: diagnostics(),
|
|
2306
2312
|
};
|
|
2307
2313
|
// Paused reads still need the kernel-critical guard, even if OS pressure warns.
|
|
2308
2314
|
const kernel = (0, kernel_zone_1.probeKernelZoneUsage)();
|
|
2315
|
+
probes.kernel = kernel;
|
|
2309
2316
|
if (kernel.status === "known" && kernel.usage.pressure === "critical")
|
|
2310
2317
|
return {
|
|
2311
2318
|
reason: `critical kernel pressure: ${(0, kernel_zone_1.formatZoneUsage)(kernel.usage)} (${(0, kernel_zone_1.formatPressureProbe)(kernel)})`,
|
|
2312
2319
|
measured: true,
|
|
2313
2320
|
critical: true,
|
|
2321
|
+
diagnostics: diagnostics(),
|
|
2314
2322
|
};
|
|
2315
2323
|
const freshMemory = (0, kernel_zone_1.probeMemoryPressure)();
|
|
2324
|
+
probes.memoryFinal = freshMemory;
|
|
2316
2325
|
if (freshMemory.status === "known" && freshMemory.pressure === "critical")
|
|
2317
2326
|
return {
|
|
2318
2327
|
reason: `critical OS memory pressure (${(0, kernel_zone_1.formatPressureProbe)(freshMemory)})`,
|
|
2319
2328
|
measured: true,
|
|
2320
2329
|
critical: true,
|
|
2330
|
+
diagnostics: diagnostics(),
|
|
2321
2331
|
};
|
|
2322
|
-
if (
|
|
2332
|
+
if (policy === "critical-only")
|
|
2323
2333
|
return null;
|
|
2324
2334
|
for (const sample of [memory, freshMemory]) {
|
|
2325
2335
|
if (sample.status !== "known")
|
|
@@ -2327,12 +2337,14 @@ class Daemon {
|
|
|
2327
2337
|
reason: `OS memory pressure unavailable (${(0, kernel_zone_1.formatPressureProbe)(sample)})`,
|
|
2328
2338
|
measured: false,
|
|
2329
2339
|
critical: false,
|
|
2340
|
+
diagnostics: diagnostics(),
|
|
2330
2341
|
};
|
|
2331
2342
|
if (sample.pressure !== "normal")
|
|
2332
2343
|
return {
|
|
2333
2344
|
reason: `warn OS memory pressure (${(0, kernel_zone_1.formatPressureProbe)(sample)})`,
|
|
2334
2345
|
measured: true,
|
|
2335
2346
|
critical: false,
|
|
2347
|
+
diagnostics: diagnostics(),
|
|
2336
2348
|
};
|
|
2337
2349
|
}
|
|
2338
2350
|
if (kernel.status !== "known")
|
|
@@ -2340,21 +2352,23 @@ class Daemon {
|
|
|
2340
2352
|
reason: `kernel pressure unavailable (${(0, kernel_zone_1.formatPressureProbe)(kernel)}); heavy work paused`,
|
|
2341
2353
|
measured: false,
|
|
2342
2354
|
critical: false,
|
|
2355
|
+
diagnostics: diagnostics(),
|
|
2343
2356
|
};
|
|
2344
2357
|
if (kernel.usage.pressure !== "ok")
|
|
2345
2358
|
return {
|
|
2346
2359
|
reason: `warn kernel pressure: ${(0, kernel_zone_1.formatZoneUsage)(kernel.usage)} (${(0, kernel_zone_1.formatPressureProbe)(kernel)})`,
|
|
2347
2360
|
measured: true,
|
|
2348
2361
|
critical: false,
|
|
2362
|
+
diagnostics: diagnostics(),
|
|
2349
2363
|
};
|
|
2350
2364
|
return null;
|
|
2351
2365
|
}
|
|
2352
|
-
stopForSafety(reason, persist = true) {
|
|
2366
|
+
stopForSafety(reason, persist = true, diagnostics) {
|
|
2353
2367
|
// Warning already pauses mutation: a stale index costs less than another
|
|
2354
2368
|
// host crash. Unknown readings are visible stops, never healthy samples.
|
|
2355
2369
|
try {
|
|
2356
2370
|
if (persist)
|
|
2357
|
-
(0, safety_latch_1.latchSafetyStop)(reason);
|
|
2371
|
+
(0, safety_latch_1.latchSafetyStop)(reason, undefined, diagnostics);
|
|
2358
2372
|
}
|
|
2359
2373
|
catch (error) {
|
|
2360
2374
|
console.error("[daemon] Failed to persist safety stop; stopping anyway:", error);
|
|
@@ -77,6 +77,7 @@ function sampleCriticalPressure() {
|
|
|
77
77
|
memoryPressure,
|
|
78
78
|
kernelPressure: kernel.status === "known" ? kernel.usage.pressure : "unknown",
|
|
79
79
|
kernelBytes: kernel.status === "known" ? kernel.usage.bytes : null,
|
|
80
|
+
pressureProbes: { memoryInitial: memory, kernel, memoryFinal: freshMemory },
|
|
80
81
|
incompleteReasons: [
|
|
81
82
|
"aggregate admission disabled by explicit critical-only policy",
|
|
82
83
|
],
|
|
@@ -170,11 +170,11 @@ function probeMemoryPressure(overrides = {}) {
|
|
|
170
170
|
const result = boundedPressureProbe("sysctl", ["-n", "kern.memorystatus_vm_pressure_level"], 1000, (output) => {
|
|
171
171
|
switch (output.trim()) {
|
|
172
172
|
case "1":
|
|
173
|
-
return "normal";
|
|
173
|
+
return { pressure: "normal", level: 1 };
|
|
174
174
|
case "2":
|
|
175
|
-
return "warn";
|
|
175
|
+
return { pressure: "warn", level: 2 };
|
|
176
176
|
case "4":
|
|
177
|
-
return "critical";
|
|
177
|
+
return { pressure: "critical", level: 4 };
|
|
178
178
|
default:
|
|
179
179
|
return null;
|
|
180
180
|
}
|
|
@@ -182,7 +182,7 @@ function probeMemoryPressure(overrides = {}) {
|
|
|
182
182
|
if (result.status !== "known")
|
|
183
183
|
return result;
|
|
184
184
|
const { value } = result, observation = __rest(result, ["value"]);
|
|
185
|
-
return Object.assign(Object.assign({}, observation),
|
|
185
|
+
return Object.assign(Object.assign({}, observation), value);
|
|
186
186
|
}
|
|
187
187
|
function classifyZonePressure(bytes) {
|
|
188
188
|
if (bytes >= CRITICAL_BYTES)
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.sanitizeStopDiagnostics = sanitizeStopDiagnostics;
|
|
4
|
+
exports.probeStopDiagnostics = probeStopDiagnostics;
|
|
5
|
+
exports.resourceStopDiagnostics = resourceStopDiagnostics;
|
|
6
|
+
const sources = new Set([
|
|
7
|
+
"resource-admission",
|
|
8
|
+
"daemon-pressure",
|
|
9
|
+
"worker-spawn",
|
|
10
|
+
]);
|
|
11
|
+
const actions = new Set([
|
|
12
|
+
"check",
|
|
13
|
+
"reserve",
|
|
14
|
+
"startup",
|
|
15
|
+
"pressure-check",
|
|
16
|
+
"heartbeat",
|
|
17
|
+
"operation",
|
|
18
|
+
"worker-spawn",
|
|
19
|
+
]);
|
|
20
|
+
const kinds = new Set([
|
|
21
|
+
"worker",
|
|
22
|
+
"embedding",
|
|
23
|
+
"native-store",
|
|
24
|
+
"llm",
|
|
25
|
+
"prune-helper",
|
|
26
|
+
"prune-runtime-setup",
|
|
27
|
+
"other",
|
|
28
|
+
]);
|
|
29
|
+
const memoryStates = new Set([
|
|
30
|
+
"normal",
|
|
31
|
+
"warn",
|
|
32
|
+
"critical",
|
|
33
|
+
"unknown",
|
|
34
|
+
"unsupported",
|
|
35
|
+
]);
|
|
36
|
+
const kernelStates = new Set([
|
|
37
|
+
"ok",
|
|
38
|
+
"warn",
|
|
39
|
+
"critical",
|
|
40
|
+
"unknown",
|
|
41
|
+
"unsupported",
|
|
42
|
+
]);
|
|
43
|
+
const reasons = new Set([
|
|
44
|
+
"timeout",
|
|
45
|
+
"output-limit",
|
|
46
|
+
"exit",
|
|
47
|
+
"execution",
|
|
48
|
+
"parse",
|
|
49
|
+
"unsupported-platform",
|
|
50
|
+
]);
|
|
51
|
+
const number = (n) => typeof n === "number" && Number.isFinite(n) && n >= 0 ? n : undefined;
|
|
52
|
+
const nullableNumber = (n) => { var _a; return (_a = number(n)) !== null && _a !== void 0 ? _a : null; };
|
|
53
|
+
function cleanProbe(p) {
|
|
54
|
+
if (!["known", "unknown", "unsupported"].includes(p.status))
|
|
55
|
+
return;
|
|
56
|
+
return Object.assign({ status: p.status, sampledAtMs: number(p.sampledAtMs), durationMs: number(p.durationMs), outputBytes: number(p.outputBytes), reason: p.reason && reasons.has(p.reason) ? p.reason : undefined }, (p.status === "known"
|
|
57
|
+
? {
|
|
58
|
+
memoryPressure: p.memoryPressure && memoryStates.has(p.memoryPressure)
|
|
59
|
+
? p.memoryPressure
|
|
60
|
+
: undefined,
|
|
61
|
+
memoryLevel: p.memoryLevel !== undefined && [1, 2, 4].includes(p.memoryLevel)
|
|
62
|
+
? p.memoryLevel
|
|
63
|
+
: undefined,
|
|
64
|
+
kernelPressure: p.kernelPressure && kernelStates.has(p.kernelPressure)
|
|
65
|
+
? p.kernelPressure
|
|
66
|
+
: undefined,
|
|
67
|
+
kernelBytes: number(p.kernelBytes),
|
|
68
|
+
kernelElements: number(p.kernelElements),
|
|
69
|
+
kernelElementSize: number(p.kernelElementSize),
|
|
70
|
+
}
|
|
71
|
+
: {}));
|
|
72
|
+
}
|
|
73
|
+
/** Rebuild the allowlist at persistence, even when a runtime caller supplies extras.
|
|
74
|
+
* Invalid diagnostic data must never prevent the original stop from persisting. */
|
|
75
|
+
function sanitizeStopDiagnostics(input) {
|
|
76
|
+
try {
|
|
77
|
+
if (!sources.has(input.source) ||
|
|
78
|
+
!actions.has(input.action) ||
|
|
79
|
+
!["strict", "critical-only"].includes(input.policy) ||
|
|
80
|
+
!Number.isSafeInteger(input.pid) ||
|
|
81
|
+
input.pid < 1)
|
|
82
|
+
return;
|
|
83
|
+
const s = input.snapshot;
|
|
84
|
+
const p = input.probes;
|
|
85
|
+
return {
|
|
86
|
+
source: input.source,
|
|
87
|
+
action: input.action,
|
|
88
|
+
policy: input.policy,
|
|
89
|
+
pid: input.pid,
|
|
90
|
+
reservationKind: input.reservationKind && kinds.has(input.reservationKind)
|
|
91
|
+
? input.reservationKind
|
|
92
|
+
: undefined,
|
|
93
|
+
reservationMb: number(input.reservationMb),
|
|
94
|
+
snapshot: s
|
|
95
|
+
? {
|
|
96
|
+
at: nullableNumber(s.at),
|
|
97
|
+
completedAt: nullableNumber(s.completedAt),
|
|
98
|
+
memoryPressure: memoryStates.has(s.memoryPressure)
|
|
99
|
+
? s.memoryPressure
|
|
100
|
+
: "unknown",
|
|
101
|
+
kernelPressure: kernelStates.has(s.kernelPressure)
|
|
102
|
+
? s.kernelPressure
|
|
103
|
+
: "unknown",
|
|
104
|
+
kernelBytes: nullableNumber(s.kernelBytes),
|
|
105
|
+
aggregateFootprintMb: nullableNumber(s.aggregateFootprintMb),
|
|
106
|
+
physicalFreeMb: nullableNumber(s.physicalFreeMb),
|
|
107
|
+
swapUsedMb: nullableNumber(s.swapUsedMb),
|
|
108
|
+
}
|
|
109
|
+
: undefined,
|
|
110
|
+
probes: p
|
|
111
|
+
? {
|
|
112
|
+
memoryInitial: p.memoryInitial
|
|
113
|
+
? cleanProbe(p.memoryInitial)
|
|
114
|
+
: undefined,
|
|
115
|
+
kernel: p.kernel ? cleanProbe(p.kernel) : undefined,
|
|
116
|
+
memoryFinal: p.memoryFinal ? cleanProbe(p.memoryFinal) : undefined,
|
|
117
|
+
}
|
|
118
|
+
: undefined,
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
catch (_a) {
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
function probeEvidence(p) {
|
|
126
|
+
return Object.assign({ status: p.status, sampledAtMs: p.sampledAtMs, durationMs: p.durationMs, outputBytes: p.outputBytes }, (p.status !== "known"
|
|
127
|
+
? { reason: p.reason }
|
|
128
|
+
: "usage" in p
|
|
129
|
+
? {
|
|
130
|
+
kernelPressure: p.usage.pressure,
|
|
131
|
+
kernelBytes: p.usage.bytes,
|
|
132
|
+
kernelElements: p.usage.elements,
|
|
133
|
+
kernelElementSize: p.usage.elementSize,
|
|
134
|
+
}
|
|
135
|
+
: { memoryPressure: p.pressure, memoryLevel: p.level }));
|
|
136
|
+
}
|
|
137
|
+
function probeStopDiagnostics(source, action, policy, probes) {
|
|
138
|
+
return {
|
|
139
|
+
source,
|
|
140
|
+
action,
|
|
141
|
+
policy,
|
|
142
|
+
pid: process.pid,
|
|
143
|
+
probes: {
|
|
144
|
+
memoryInitial: probes.memoryInitial && probeEvidence(probes.memoryInitial),
|
|
145
|
+
kernel: probes.kernel && probeEvidence(probes.kernel),
|
|
146
|
+
memoryFinal: probes.memoryFinal && probeEvidence(probes.memoryFinal),
|
|
147
|
+
},
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
function resourceStopDiagnostics(snapshot, policy, action, pid, kind, mb) {
|
|
151
|
+
return {
|
|
152
|
+
source: "resource-admission",
|
|
153
|
+
action,
|
|
154
|
+
policy,
|
|
155
|
+
pid,
|
|
156
|
+
snapshot: {
|
|
157
|
+
at: snapshot.at,
|
|
158
|
+
completedAt: snapshot.completedAt,
|
|
159
|
+
memoryPressure: snapshot.memoryPressure,
|
|
160
|
+
kernelPressure: snapshot.kernelPressure,
|
|
161
|
+
kernelBytes: snapshot.kernelBytes,
|
|
162
|
+
aggregateFootprintMb: snapshot.aggregateFootprintMb,
|
|
163
|
+
physicalFreeMb: snapshot.physicalFreeMb,
|
|
164
|
+
swapUsedMb: snapshot.swapUsedMb,
|
|
165
|
+
},
|
|
166
|
+
reservationKind: kind
|
|
167
|
+
? kinds.has(kind)
|
|
168
|
+
? kind
|
|
169
|
+
: "other"
|
|
170
|
+
: undefined,
|
|
171
|
+
reservationMb: mb,
|
|
172
|
+
probes: snapshot.pressureProbes
|
|
173
|
+
? probeStopDiagnostics("daemon-pressure", "pressure-check", policy, snapshot.pressureProbes).probes
|
|
174
|
+
: undefined,
|
|
175
|
+
};
|
|
176
|
+
}
|
|
@@ -48,6 +48,7 @@ const config_1 = require("../../config");
|
|
|
48
48
|
const autostart_1 = require("./autostart");
|
|
49
49
|
const host_guard_policy_1 = require("./host-guard-policy");
|
|
50
50
|
const host_resource_1 = require("./host-resource");
|
|
51
|
+
const pressure_diagnostics_1 = require("./pressure-diagnostics");
|
|
51
52
|
const safety_latch_1 = require("./safety-latch");
|
|
52
53
|
/** Admission thresholds, not an OS-enforced memory cap. Keep enough room for
|
|
53
54
|
* a worker's measured 1536MiB recycle threshold, and charge pending launches
|
|
@@ -58,9 +59,10 @@ exports.WORKER_RESOURCE_RESERVE_MB = 1536;
|
|
|
58
59
|
exports.EMBEDDING_RESOURCE_RESERVE_MB = 1024;
|
|
59
60
|
exports.HELPER_RESOURCE_RESERVE_MB = 512;
|
|
60
61
|
class ResourceAdmissionError extends Error {
|
|
61
|
-
constructor(message, critical = false) {
|
|
62
|
+
constructor(message, critical = false, diagnostics) {
|
|
62
63
|
super(message);
|
|
63
64
|
this.critical = critical;
|
|
65
|
+
this.diagnostics = diagnostics;
|
|
64
66
|
this.code = "HOST_RESOURCE";
|
|
65
67
|
this.name = "ResourceAdmissionError";
|
|
66
68
|
}
|
|
@@ -108,7 +110,7 @@ class ResourceBudget {
|
|
|
108
110
|
this.deps = Object.assign({ root: path.join(config_1.PATHS.sharedRoot, "resource-budget"), platform: process.platform, pid: process.pid, now: Date.now, sample: (extraPids = []) => {
|
|
109
111
|
const records = this.readExistingRecords();
|
|
110
112
|
return (0, host_resource_1.sampleHostResources)([...records.map((r) => r.pid), ...extraPids], this.deps.samplerOverrides, records.flatMap((r) => (r.groupPid ? [r.groupPid] : [])));
|
|
111
|
-
}, signal: (pid) => process.kill(pid, 0), policy: host_guard_policy_1.hostGuardPolicy, requireClientRegistration: true, samplerOverrides: {}, sampleMaxAgeMs: 5000, criticalSample: host_guard_policy_1.sampleCriticalPressure, quarantine: autostart_1.daemonStartDeniedReason, latch: safety_latch_1.latchSafetyStop, processStart: () => (0, node_child_process_1.execFileSync)("ps", ["-p", String(process.pid), "-o", "lstart="], {
|
|
113
|
+
}, signal: (pid) => process.kill(pid, 0), policy: host_guard_policy_1.hostGuardPolicy, requireClientRegistration: true, samplerOverrides: {}, sampleMaxAgeMs: 5000, criticalSample: host_guard_policy_1.sampleCriticalPressure, quarantine: autostart_1.daemonStartDeniedReason, latch: (reason, diagnostics) => (0, safety_latch_1.latchSafetyStop)(reason, undefined, diagnostics), processStart: () => (0, node_child_process_1.execFileSync)("ps", ["-p", String(process.pid), "-o", "lstart="], {
|
|
112
114
|
encoding: "utf8",
|
|
113
115
|
timeout: 500,
|
|
114
116
|
maxBuffer: 1024,
|
|
@@ -263,7 +265,7 @@ class ResourceBudget {
|
|
|
263
265
|
fs.closeSync(dir);
|
|
264
266
|
}
|
|
265
267
|
}
|
|
266
|
-
checked(snapshot, records) {
|
|
268
|
+
checked(snapshot, records, kind, mb) {
|
|
267
269
|
try {
|
|
268
270
|
assertResourceSnapshot(snapshot, this.charge(snapshot, records), resolveResourceBudgetMb(), this.deps.now(), this.deps.sampleMaxAgeMs);
|
|
269
271
|
const legacy = snapshot.processes.filter((p) => p.role === "mcp" &&
|
|
@@ -273,8 +275,9 @@ class ResourceBudget {
|
|
|
273
275
|
}
|
|
274
276
|
catch (error) {
|
|
275
277
|
if (error instanceof ResourceAdmissionError && error.critical) {
|
|
278
|
+
error.diagnostics = (0, pressure_diagnostics_1.resourceStopDiagnostics)(snapshot, "strict", kind ? "reserve" : "check", this.deps.pid, kind, mb);
|
|
276
279
|
try {
|
|
277
|
-
this.deps.latch(error.message);
|
|
280
|
+
this.deps.latch(error.message, error.diagnostics);
|
|
278
281
|
}
|
|
279
282
|
catch (_a) {
|
|
280
283
|
error.message += "; safety stop persistence failed";
|
|
@@ -323,12 +326,22 @@ class ResourceBudget {
|
|
|
323
326
|
},
|
|
324
327
|
};
|
|
325
328
|
}
|
|
326
|
-
/** Read-only admission for existing inference
|
|
329
|
+
/** Read-only admission for existing inference, honoring the selected host policy.
|
|
330
|
+
* Never seed/lock/prune the ledger or latch. */
|
|
327
331
|
checkExisting() {
|
|
328
332
|
if (this.deps.quarantine())
|
|
329
333
|
throw new ResourceAdmissionError("host containment");
|
|
330
334
|
if (this.deps.platform !== "darwin")
|
|
331
335
|
return null;
|
|
336
|
+
if (this.deps.policy() === "critical-only") {
|
|
337
|
+
const snapshot = this.deps.criticalSample();
|
|
338
|
+
if (snapshot.memoryPressure === "critical" ||
|
|
339
|
+
snapshot.kernelPressure === "critical")
|
|
340
|
+
throw new ResourceAdmissionError("critical host pressure; existing inference refused", true);
|
|
341
|
+
if (this.deps.quarantine())
|
|
342
|
+
throw new ResourceAdmissionError("host containment");
|
|
343
|
+
return snapshot;
|
|
344
|
+
}
|
|
332
345
|
const records = this.readExistingRecords();
|
|
333
346
|
const snapshot = this.deps.sample();
|
|
334
347
|
assertResourceSnapshot(snapshot, this.charge(snapshot, records), resolveResourceBudgetMb(), this.deps.now(), this.deps.sampleMaxAgeMs);
|
|
@@ -336,7 +349,7 @@ class ResourceBudget {
|
|
|
336
349
|
throw new ResourceAdmissionError("host containment");
|
|
337
350
|
return snapshot;
|
|
338
351
|
}
|
|
339
|
-
check(requiredPid) {
|
|
352
|
+
check(requiredPid, reservation) {
|
|
340
353
|
if (this.deps.platform !== "darwin")
|
|
341
354
|
return null;
|
|
342
355
|
if (requiredPid !== undefined &&
|
|
@@ -353,9 +366,9 @@ class ResourceBudget {
|
|
|
353
366
|
const snapshot = this.deps.criticalSample();
|
|
354
367
|
if (snapshot.memoryPressure === "critical" ||
|
|
355
368
|
snapshot.kernelPressure === "critical") {
|
|
356
|
-
const error = new ResourceAdmissionError("critical host pressure; heavy work refused", true);
|
|
369
|
+
const error = new ResourceAdmissionError("critical host pressure; heavy work refused", true, (0, pressure_diagnostics_1.resourceStopDiagnostics)(snapshot, "critical-only", reservation ? "reserve" : "check", this.deps.pid, reservation === null || reservation === void 0 ? void 0 : reservation.kind, reservation === null || reservation === void 0 ? void 0 : reservation.mb));
|
|
357
370
|
try {
|
|
358
|
-
this.deps.latch(error.message);
|
|
371
|
+
this.deps.latch(error.message, error.diagnostics);
|
|
359
372
|
}
|
|
360
373
|
catch (_a) {
|
|
361
374
|
error.message += "; safety stop persistence failed";
|
|
@@ -381,7 +394,7 @@ class ResourceBudget {
|
|
|
381
394
|
if (this.deps.platform !== "darwin")
|
|
382
395
|
return { attach: () => { }, release: () => { } };
|
|
383
396
|
if (this.deps.policy() === "critical-only") {
|
|
384
|
-
this.check();
|
|
397
|
+
this.check(undefined, { kind, mb });
|
|
385
398
|
return { attach: () => { }, release: () => { } };
|
|
386
399
|
}
|
|
387
400
|
let record = this.withLock(() => {
|
|
@@ -408,7 +421,7 @@ class ResourceBudget {
|
|
|
408
421
|
mb: charge,
|
|
409
422
|
kind,
|
|
410
423
|
};
|
|
411
|
-
this.checked(snapshot, [...records, next]);
|
|
424
|
+
this.checked(snapshot, [...records, next], kind, mb);
|
|
412
425
|
this.persist(next);
|
|
413
426
|
return next;
|
|
414
427
|
});
|
|
@@ -41,6 +41,7 @@ const node_crypto_1 = require("node:crypto");
|
|
|
41
41
|
const fs = __importStar(require("node:fs"));
|
|
42
42
|
const path = __importStar(require("node:path"));
|
|
43
43
|
const config_1 = require("../../config");
|
|
44
|
+
const pressure_diagnostics_1 = require("./pressure-diagnostics");
|
|
44
45
|
/** Machine-wide stop: changing stores or installing a version cannot clear it. */
|
|
45
46
|
exports.SAFETY_LATCH_NAME = "safety-stop.json";
|
|
46
47
|
function safetyRoot() {
|
|
@@ -69,7 +70,7 @@ function safetyStopReason(root = safetyRoot()) {
|
|
|
69
70
|
}
|
|
70
71
|
}
|
|
71
72
|
/** Atomic durable latch; intentionally no automatic expiry or clear operation. */
|
|
72
|
-
function latchSafetyStop(reason, root = safetyRoot()) {
|
|
73
|
+
function latchSafetyStop(reason, root = safetyRoot(), diagnostics) {
|
|
73
74
|
if (safetyStopReason(root) !== null)
|
|
74
75
|
return;
|
|
75
76
|
fs.mkdirSync(root, { recursive: true, mode: 0o700 });
|
|
@@ -83,6 +84,9 @@ function latchSafetyStop(reason, root = safetyRoot()) {
|
|
|
83
84
|
schemaVersion: 1,
|
|
84
85
|
at: Date.now(),
|
|
85
86
|
reason: reason.replace(/[\r\n\t]/g, " ").slice(0, 512) || "host safety stop",
|
|
87
|
+
diagnostics: diagnostics
|
|
88
|
+
? (0, pressure_diagnostics_1.sanitizeStopDiagnostics)(diagnostics)
|
|
89
|
+
: undefined,
|
|
86
90
|
};
|
|
87
91
|
fs.writeFileSync(fd, `${JSON.stringify(record)}\n`);
|
|
88
92
|
fs.fsyncSync(fd);
|
package/dist/lib/workers/pool.js
CHANGED
|
@@ -42,7 +42,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
42
42
|
});
|
|
43
43
|
};
|
|
44
44
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
|
-
exports.WorkerPool = void 0;
|
|
45
|
+
exports.WorkerPool = exports.EXISTING_QUERY_QUEUE_LIMIT = exports.EXISTING_QUERY_WAIT_MS = void 0;
|
|
46
46
|
exports.embeddingEnv = embeddingEnv;
|
|
47
47
|
exports.getWorkerPool = getWorkerPool;
|
|
48
48
|
exports.destroyWorkerPool = destroyWorkerPool;
|
|
@@ -61,8 +61,12 @@ const autostart_1 = require("../utils/autostart");
|
|
|
61
61
|
const host_guard_policy_1 = require("../utils/host-guard-policy");
|
|
62
62
|
const kernel_zone_1 = require("../utils/kernel-zone");
|
|
63
63
|
const logger_1 = require("../utils/logger");
|
|
64
|
+
const pressure_diagnostics_1 = require("../utils/pressure-diagnostics");
|
|
64
65
|
const resource_budget_1 = require("../utils/resource-budget");
|
|
65
66
|
const safety_latch_1 = require("../utils/safety-latch");
|
|
67
|
+
exports.EXISTING_QUERY_WAIT_MS = 2000;
|
|
68
|
+
exports.EXISTING_QUERY_QUEUE_LIMIT = 4;
|
|
69
|
+
const EXISTING_QUERY_DEADLINE_MS = 10000;
|
|
66
70
|
const TASK_TIMEOUT_MS = (() => {
|
|
67
71
|
var _a;
|
|
68
72
|
const fromEnv = Number.parseInt((_a = process.env.GMAX_WORKER_TASK_TIMEOUT_MS) !== null && _a !== void 0 ? _a : "", 10);
|
|
@@ -249,6 +253,9 @@ class WorkerPool {
|
|
|
249
253
|
this.priorityQueue = [];
|
|
250
254
|
this.taskQueue = [];
|
|
251
255
|
this.tasks = new Map();
|
|
256
|
+
// Separate from ordinary queues: these requests must never trigger expansion.
|
|
257
|
+
this.existingWaiters = new Map();
|
|
258
|
+
this.existingBurst = 0;
|
|
252
259
|
this.nextId = 1;
|
|
253
260
|
this.destroyed = false;
|
|
254
261
|
this.destroyPromise = null;
|
|
@@ -298,6 +305,10 @@ class WorkerPool {
|
|
|
298
305
|
: [{ pid: worker.child.pid, rssBytes: worker.lastRssBytes }]);
|
|
299
306
|
}
|
|
300
307
|
clearTaskTimeout(task) {
|
|
308
|
+
if (task.waitTimeout) {
|
|
309
|
+
clearTimeout(task.waitTimeout);
|
|
310
|
+
task.waitTimeout = undefined;
|
|
311
|
+
}
|
|
301
312
|
if (task.timeout) {
|
|
302
313
|
clearTimeout(task.timeout);
|
|
303
314
|
task.timeout = undefined;
|
|
@@ -352,6 +363,7 @@ class WorkerPool {
|
|
|
352
363
|
task.abortListener = undefined;
|
|
353
364
|
}
|
|
354
365
|
this.tasks.delete(task.id);
|
|
366
|
+
this.existingWaiters.delete(task.id);
|
|
355
367
|
this.removeFromQueue(task.id);
|
|
356
368
|
if (worker) {
|
|
357
369
|
worker.busy = false;
|
|
@@ -396,14 +408,14 @@ class WorkerPool {
|
|
|
396
408
|
var _a;
|
|
397
409
|
return Object.assign(new Error((_a = this.spawnDeniedReason) !== null && _a !== void 0 ? _a : "Worker spawn denied"), { code: "HOST_SAFETY" });
|
|
398
410
|
}
|
|
399
|
-
denySpawn(reason, persist = false) {
|
|
411
|
+
denySpawn(reason, persist = false, diagnostics) {
|
|
400
412
|
// A pool cannot automatically recover after a failed admission. In
|
|
401
413
|
// particular, timer/exit callbacks must not retry forks or strand queues.
|
|
402
414
|
this.spawnDeniedReason = `Worker spawn denied: ${reason}`;
|
|
403
415
|
this.clearScaleUpTimer();
|
|
404
416
|
if (persist) {
|
|
405
417
|
try {
|
|
406
|
-
(0, safety_latch_1.latchSafetyStop)(this.spawnDeniedReason);
|
|
418
|
+
(0, safety_latch_1.latchSafetyStop)(this.spawnDeniedReason, undefined, diagnostics);
|
|
407
419
|
}
|
|
408
420
|
catch (_a) {
|
|
409
421
|
try {
|
|
@@ -427,31 +439,37 @@ class WorkerPool {
|
|
|
427
439
|
const quarantine = (0, autostart_1.daemonStartDeniedReason)();
|
|
428
440
|
if (quarantine)
|
|
429
441
|
return this.denySpawn(quarantine, false);
|
|
430
|
-
const
|
|
442
|
+
const policy = (0, host_guard_policy_1.hostGuardPolicy)();
|
|
443
|
+
const strictHostGuard = policy === "strict";
|
|
444
|
+
const probes = {};
|
|
445
|
+
const diagnostics = () => (0, pressure_diagnostics_1.probeStopDiagnostics)("worker-spawn", "worker-spawn", policy, probes);
|
|
431
446
|
if (process.platform === "darwin") {
|
|
432
447
|
const memory = (0, kernel_zone_1.probeMemoryPressure)();
|
|
448
|
+
probes.memoryInitial = memory;
|
|
433
449
|
if ((memory.status === "known" && memory.pressure === "critical") ||
|
|
434
450
|
(strictHostGuard &&
|
|
435
451
|
(memory.status !== "known" || memory.pressure !== "normal"))) {
|
|
436
|
-
return this.denySpawn(`OS memory pressure ${memory.status === "known" ? memory.pressure : memory.status}; ${(0, kernel_zone_1.formatPressureProbe)(memory)}`, memory.status === "known" && memory.pressure === "critical");
|
|
452
|
+
return this.denySpawn(`OS memory pressure ${memory.status === "known" ? memory.pressure : memory.status}; ${(0, kernel_zone_1.formatPressureProbe)(memory)}`, memory.status === "known" && memory.pressure === "critical", diagnostics());
|
|
437
453
|
}
|
|
438
454
|
const kernel = (0, kernel_zone_1.probeKernelZoneUsage)();
|
|
455
|
+
probes.kernel = kernel;
|
|
439
456
|
if ((kernel.status === "known" && kernel.usage.pressure === "critical") ||
|
|
440
457
|
(strictHostGuard &&
|
|
441
458
|
(kernel.status !== "known" || kernel.usage.pressure !== "ok"))) {
|
|
442
|
-
return this.denySpawn(`kernel pressure ${kernel.status === "known" ? kernel.usage.pressure : kernel.status}; ${(0, kernel_zone_1.formatPressureProbe)(kernel)}`, kernel.status === "known" && kernel.usage.pressure === "critical");
|
|
459
|
+
return this.denySpawn(`kernel pressure ${kernel.status === "known" ? kernel.usage.pressure : kernel.status}; ${(0, kernel_zone_1.formatPressureProbe)(kernel)}`, kernel.status === "known" && kernel.usage.pressure === "critical", diagnostics());
|
|
443
460
|
}
|
|
444
461
|
}
|
|
445
462
|
if (process.platform === "darwin") {
|
|
446
463
|
// Refresh the cheap OS sample after the potentially slow kernel probe.
|
|
447
464
|
const freshMemory = (0, kernel_zone_1.probeMemoryPressure)();
|
|
465
|
+
probes.memoryFinal = freshMemory;
|
|
448
466
|
if ((freshMemory.status === "known" &&
|
|
449
467
|
freshMemory.pressure === "critical") ||
|
|
450
468
|
(strictHostGuard &&
|
|
451
469
|
(freshMemory.status !== "known" ||
|
|
452
470
|
freshMemory.pressure !== "normal"))) {
|
|
453
471
|
return this.denySpawn(`OS memory pressure ${freshMemory.status === "known" ? freshMemory.pressure : freshMemory.status}; ${(0, kernel_zone_1.formatPressureProbe)(freshMemory)}`, freshMemory.status === "known" &&
|
|
454
|
-
freshMemory.pressure === "critical");
|
|
472
|
+
freshMemory.pressure === "critical", diagnostics());
|
|
455
473
|
}
|
|
456
474
|
}
|
|
457
475
|
// Probes are synchronous but another process can quarantine the host
|
|
@@ -617,19 +635,25 @@ class WorkerPool {
|
|
|
617
635
|
if (this.destroyed)
|
|
618
636
|
return;
|
|
619
637
|
if (this.spawnDeniedReason) {
|
|
638
|
+
this.rejectExistingWaiters("host_pressure");
|
|
620
639
|
this.rejectUnassignedTasks(this.spawnDeniedReason, "HOST_SAFETY");
|
|
621
640
|
return;
|
|
622
641
|
}
|
|
623
642
|
// Work admitted before another process quarantines the host may still
|
|
624
643
|
// be queued. Do not send it to an existing worker after that stop.
|
|
625
644
|
try {
|
|
645
|
+
this.dispatchExistingWaiters();
|
|
646
|
+
if (this.destroyed)
|
|
647
|
+
return;
|
|
626
648
|
const quarantine = (0, autostart_1.daemonStartDeniedReason)();
|
|
627
649
|
if (quarantine) {
|
|
650
|
+
this.rejectExistingWaiters("host_pressure");
|
|
628
651
|
this.denySpawn(quarantine, false);
|
|
629
652
|
return;
|
|
630
653
|
}
|
|
631
654
|
}
|
|
632
655
|
catch (_f) {
|
|
656
|
+
this.rejectExistingWaiters("host_pressure");
|
|
633
657
|
this.denySpawn("quarantine check failed");
|
|
634
658
|
return;
|
|
635
659
|
}
|
|
@@ -681,6 +705,7 @@ class WorkerPool {
|
|
|
681
705
|
return;
|
|
682
706
|
}
|
|
683
707
|
idle.suppressReplacement = false;
|
|
708
|
+
this.existingBurst = 0;
|
|
684
709
|
idle.busy = true;
|
|
685
710
|
idle.pendingTaskId = task.id;
|
|
686
711
|
idle.busySince = Date.now();
|
|
@@ -752,8 +777,7 @@ class WorkerPool {
|
|
|
752
777
|
processFile(input, signal) {
|
|
753
778
|
return this.enqueue("processFile", input, signal);
|
|
754
779
|
}
|
|
755
|
-
|
|
756
|
-
existingQueryState() {
|
|
780
|
+
existingWorkerState() {
|
|
757
781
|
if (this.destroyed || this.spawnDeniedReason || (0, autostart_1.daemonStartDeniedReason)())
|
|
758
782
|
return "host_pressure";
|
|
759
783
|
const warm = this.workers.filter((w) => w.queryReady && !w.cleanedUp && w.child.connected !== false);
|
|
@@ -763,23 +787,112 @@ class WorkerPool {
|
|
|
763
787
|
? "busy"
|
|
764
788
|
: "embedding_unavailable";
|
|
765
789
|
}
|
|
790
|
+
/** Admission readiness includes a short bounded wait for an already-warm worker. */
|
|
791
|
+
existingQueryState() {
|
|
792
|
+
const state = this.existingWorkerState();
|
|
793
|
+
return state === "busy" &&
|
|
794
|
+
this.existingWaiters.size < exports.EXISTING_QUERY_QUEUE_LIMIT
|
|
795
|
+
? "ready"
|
|
796
|
+
: this.existingWaiters.size >= exports.EXISTING_QUERY_QUEUE_LIMIT &&
|
|
797
|
+
state === "ready"
|
|
798
|
+
? "busy"
|
|
799
|
+
: state;
|
|
800
|
+
}
|
|
801
|
+
rejectExistingWaiters(state) {
|
|
802
|
+
for (const task of this.existingWaiters.values()) {
|
|
803
|
+
task.reject(Object.assign(new Error(state), { code: state }));
|
|
804
|
+
this.completeTask(task, null);
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
dispatchExistingWaiters() {
|
|
808
|
+
while (this.existingWaiters.size) {
|
|
809
|
+
const state = this.existingWorkerState();
|
|
810
|
+
if (state === "host_pressure" || state === "embedding_unavailable") {
|
|
811
|
+
this.rejectExistingWaiters(state);
|
|
812
|
+
return;
|
|
813
|
+
}
|
|
814
|
+
// A continuous caller stream cannot keep indexing behind this queue.
|
|
815
|
+
if (this.existingBurst >= exports.EXISTING_QUERY_QUEUE_LIMIT &&
|
|
816
|
+
[...this.tasks.values()].some((t) => !t.worker && !t.existingOnly))
|
|
817
|
+
return;
|
|
818
|
+
if (state === "busy")
|
|
819
|
+
return;
|
|
820
|
+
const task = this.existingWaiters.values().next().value;
|
|
821
|
+
const age = Date.now() - task.queuedAt;
|
|
822
|
+
if (age < 0 || age >= exports.EXISTING_QUERY_WAIT_MS) {
|
|
823
|
+
task.reject(Object.assign(new Error("busy"), { code: "busy" }));
|
|
824
|
+
this.completeTask(task, null);
|
|
825
|
+
continue;
|
|
826
|
+
}
|
|
827
|
+
// The worker wait ends here; synchronous admission has its own bound and
|
|
828
|
+
// the request's overall timer/signal still applies during inference.
|
|
829
|
+
if (task.waitTimeout)
|
|
830
|
+
clearTimeout(task.waitTimeout);
|
|
831
|
+
task.waitTimeout = undefined;
|
|
832
|
+
try {
|
|
833
|
+
resource_budget_1.resourceBudget.checkExisting();
|
|
834
|
+
}
|
|
835
|
+
catch (_a) {
|
|
836
|
+
this.rejectExistingWaiters("host_pressure");
|
|
837
|
+
// Ordinary work performs its own admission; a document refusal must
|
|
838
|
+
// not strand that queue. The dispatcher rechecks containment next.
|
|
839
|
+
return;
|
|
840
|
+
}
|
|
841
|
+
// Cancellation during admission may have removed this waiter.
|
|
842
|
+
if (!this.existingWaiters.has(task.id))
|
|
843
|
+
continue;
|
|
844
|
+
const admittedAge = Date.now() - task.queuedAt;
|
|
845
|
+
if (admittedAge < 0 || admittedAge >= EXISTING_QUERY_DEADLINE_MS) {
|
|
846
|
+
task.reject(new Error("timeout"));
|
|
847
|
+
this.completeTask(task, null);
|
|
848
|
+
continue;
|
|
849
|
+
}
|
|
850
|
+
const after = this.existingWorkerState();
|
|
851
|
+
if (after !== "ready") {
|
|
852
|
+
if (after === "busy") {
|
|
853
|
+
task.reject(Object.assign(new Error("busy"), { code: "busy" }));
|
|
854
|
+
this.completeTask(task, null);
|
|
855
|
+
continue;
|
|
856
|
+
}
|
|
857
|
+
this.rejectExistingWaiters(after);
|
|
858
|
+
return;
|
|
859
|
+
}
|
|
860
|
+
const worker = this.workers.find((w) => w.queryReady &&
|
|
861
|
+
!w.busy &&
|
|
862
|
+
!w.cleanedUp &&
|
|
863
|
+
w.child.connected !== false);
|
|
864
|
+
this.existingWaiters.delete(task.id);
|
|
865
|
+
if (task.waitTimeout)
|
|
866
|
+
clearTimeout(task.waitTimeout);
|
|
867
|
+
task.waitTimeout = undefined;
|
|
868
|
+
task.worker = worker;
|
|
869
|
+
task.startTime = Date.now();
|
|
870
|
+
worker.busy = true;
|
|
871
|
+
worker.suppressReplacement = true;
|
|
872
|
+
worker.busySince = Date.now();
|
|
873
|
+
worker.pendingTaskId = task.id;
|
|
874
|
+
this.tasks.set(task.id, task);
|
|
875
|
+
this.existingBurst++;
|
|
876
|
+
try {
|
|
877
|
+
worker.child.send({
|
|
878
|
+
id: task.id,
|
|
879
|
+
method: task.method,
|
|
880
|
+
payload: task.payload,
|
|
881
|
+
});
|
|
882
|
+
}
|
|
883
|
+
catch (_b) {
|
|
884
|
+
this.completeTask(task, worker);
|
|
885
|
+
worker.queryReady = false;
|
|
886
|
+
task.reject(new Error("embedding_unavailable"));
|
|
887
|
+
}
|
|
888
|
+
}
|
|
889
|
+
}
|
|
766
890
|
encodeQueryExisting(text, signal) {
|
|
767
891
|
const state = this.existingQueryState();
|
|
768
892
|
if (state !== "ready")
|
|
769
893
|
return Promise.reject(Object.assign(new Error(state), { code: state }));
|
|
770
894
|
if (signal === null || signal === void 0 ? void 0 : signal.aborted)
|
|
771
895
|
return Promise.reject(Object.assign(new Error("Aborted"), { name: "AbortError" }));
|
|
772
|
-
try {
|
|
773
|
-
resource_budget_1.resourceBudget.checkExisting();
|
|
774
|
-
}
|
|
775
|
-
catch (_a) {
|
|
776
|
-
return Promise.reject(Object.assign(new Error("host_pressure"), { code: "host_pressure" }));
|
|
777
|
-
}
|
|
778
|
-
// Recheck after admission: quarantine or worker readiness may have changed.
|
|
779
|
-
const after = this.existingQueryState();
|
|
780
|
-
if (after !== "ready")
|
|
781
|
-
return Promise.reject(Object.assign(new Error(after), { code: after }));
|
|
782
|
-
const worker = this.workers.find((w) => w.queryReady && !w.busy && !w.cleanedUp && w.child.connected !== false);
|
|
783
896
|
return new Promise((resolve, reject) => {
|
|
784
897
|
const id = this.nextId++;
|
|
785
898
|
const task = {
|
|
@@ -795,11 +908,14 @@ class WorkerPool {
|
|
|
795
908
|
queuedAt: Date.now(),
|
|
796
909
|
signal,
|
|
797
910
|
existingOnly: true,
|
|
798
|
-
worker,
|
|
799
911
|
};
|
|
800
912
|
const cancel = () => {
|
|
801
913
|
task.callerAborted = true;
|
|
802
914
|
reject(Object.assign(new Error("Aborted"), { name: "AbortError" }));
|
|
915
|
+
if (!task.worker) {
|
|
916
|
+
this.completeTask(task, null);
|
|
917
|
+
this.dispatch();
|
|
918
|
+
}
|
|
803
919
|
};
|
|
804
920
|
task.abortListener = cancel;
|
|
805
921
|
signal === null || signal === void 0 ? void 0 : signal.addEventListener("abort", cancel, { once: true });
|
|
@@ -808,20 +924,16 @@ class WorkerPool {
|
|
|
808
924
|
task.timeout = setTimeout(() => {
|
|
809
925
|
task.callerAborted = true;
|
|
810
926
|
reject(new Error("timeout"));
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
this.completeTask(task, worker);
|
|
822
|
-
worker.queryReady = false;
|
|
823
|
-
reject(new Error("embedding_unavailable"));
|
|
824
|
-
}
|
|
927
|
+
if (!task.worker)
|
|
928
|
+
this.completeTask(task, null);
|
|
929
|
+
}, EXISTING_QUERY_DEADLINE_MS);
|
|
930
|
+
task.waitTimeout = setTimeout(() => {
|
|
931
|
+
reject(Object.assign(new Error("busy"), { code: "busy" }));
|
|
932
|
+
this.completeTask(task, null);
|
|
933
|
+
this.dispatch();
|
|
934
|
+
}, exports.EXISTING_QUERY_WAIT_MS);
|
|
935
|
+
this.existingWaiters.set(id, task);
|
|
936
|
+
this.dispatch();
|
|
825
937
|
});
|
|
826
938
|
}
|
|
827
939
|
encodeQuery(text, signal) {
|
|
@@ -1006,6 +1118,7 @@ class WorkerPool {
|
|
|
1006
1118
|
this.idleReapInterval = null;
|
|
1007
1119
|
}
|
|
1008
1120
|
this.clearScaleUpTimer();
|
|
1121
|
+
this.rejectExistingWaiters("embedding_unavailable");
|
|
1009
1122
|
for (const task of this.tasks.values()) {
|
|
1010
1123
|
task.reject(new Error("Worker pool destroyed"));
|
|
1011
1124
|
this.completeTask(task, null);
|
package/package.json
CHANGED