grepmax 0.26.55 → 0.26.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +1 -1
- package/dist/lib/daemon/daemon.js +129 -66
- package/dist/lib/daemon/mlx-server-manager.js +24 -8
- package/dist/lib/store/vector-db.js +4 -0
- package/dist/lib/utils/kernel-zone.js +132 -18
- package/dist/lib/workers/pool.js +97 -10
- package/package.json +1 -1
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"name": "grepmax",
|
|
11
11
|
"source": "./plugins/grepmax",
|
|
12
12
|
"description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
|
|
13
|
-
"version": "0.26.
|
|
13
|
+
"version": "0.26.56",
|
|
14
14
|
"author": {
|
|
15
15
|
"name": "Robert Owens",
|
|
16
16
|
"email": "robowens@me.com"
|
package/README.md
CHANGED
|
@@ -57,7 +57,7 @@ gmax doctor # Health check
|
|
|
57
57
|
gmax doctor --fix # Repair checks; full-table maintenance is disabled
|
|
58
58
|
```
|
|
59
59
|
|
|
60
|
-
**Storage containment:** full-table compaction is disabled, including forced maintenance and doctor repair. Existing retained index copies are not reclaimed by this release. The verified prune-only recovery path is separate work; normal incremental indexing remains available when the host is healthy and not quarantined. A persistent host safety stop blocks mutation and daemon restart. Installing an update preserves existing quarantine.
|
|
60
|
+
**Storage containment:** full-table compaction is disabled, including forced maintenance and doctor repair. Existing retained index copies are not reclaimed by this release. The verified prune-only recovery path is separate work; normal incremental indexing remains available when the host is healthy and not quarantined. A persistent host safety stop blocks mutation and daemon restart. Installing an update preserves existing quarantine. Speculative embedding warmup and maintenance timers are disabled. On macOS, startup, heavy queries, worker forks and MLX launches require known normal OS memory pressure and a healthy kernel-zone sample; unavailable probes report bounded diagnostic details and refuse work.
|
|
61
61
|
|
|
62
62
|
### Core Commands
|
|
63
63
|
|
|
@@ -57,6 +57,8 @@ const embedding_status_1 = require("../index/embedding-status");
|
|
|
57
57
|
const index_config_1 = require("../index/index-config");
|
|
58
58
|
const syncer_1 = require("../index/syncer");
|
|
59
59
|
const server_1 = require("../llm/server");
|
|
60
|
+
const compaction_result_1 = require("../store/compaction-result");
|
|
61
|
+
const maintenance_policy_1 = require("../store/maintenance-policy");
|
|
60
62
|
const meta_cache_1 = require("../store/meta-cache");
|
|
61
63
|
const store_lease_1 = require("../store/store-lease");
|
|
62
64
|
const vector_db_1 = require("../store/vector-db");
|
|
@@ -165,6 +167,7 @@ class Daemon {
|
|
|
165
167
|
});
|
|
166
168
|
this.mlxServerManager = new mlx_server_manager_1.MlxServerManager({
|
|
167
169
|
getShuttingDown: () => this.shuttingDown,
|
|
170
|
+
assertExpansionAdmission: () => this.assertHeavyOperationAdmission("mlx-start"),
|
|
168
171
|
});
|
|
169
172
|
this.watcherManager = new watcher_manager_1.WatcherManager({
|
|
170
173
|
processors: this.processors,
|
|
@@ -242,14 +245,11 @@ class Daemon {
|
|
|
242
245
|
if (denied !== null)
|
|
243
246
|
throw new Error(`gmax: ${denied}; preserving containment`);
|
|
244
247
|
// Check before stale-process cleanup, store opening, workers or model setup.
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
if (
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
(0, safety_latch_1.latchSafetyStop)((0, kernel_zone_1.formatZoneUsage)(usage));
|
|
251
|
-
throw new Error(`gmax: ${(0, kernel_zone_1.formatZoneUsage)(usage)}; startup safety stop`);
|
|
252
|
-
}
|
|
248
|
+
const pressure = this.hostPressureDeniedReason();
|
|
249
|
+
if (pressure !== null) {
|
|
250
|
+
if (pressure.measured)
|
|
251
|
+
(0, safety_latch_1.latchSafetyStop)(pressure.reason);
|
|
252
|
+
throw new Error(`gmax: ${pressure.reason}; startup safety stop`);
|
|
253
253
|
}
|
|
254
254
|
process.title = "gmax-daemon";
|
|
255
255
|
// Read verbs register here rather than at read-verbs.ts module scope: the
|
|
@@ -404,7 +404,7 @@ class Daemon {
|
|
|
404
404
|
const globalConfig = (_a = this.activeConfig) !== null && _a !== void 0 ? _a : (0, index_config_1.readGlobalConfig)();
|
|
405
405
|
const isAppleSilicon = process.arch === "arm64" && process.platform === "darwin";
|
|
406
406
|
if (isAppleSilicon && globalConfig.embedMode === "gpu") {
|
|
407
|
-
yield this.
|
|
407
|
+
yield this.ensureAdmittedMlxServer(globalConfig.mlxModel);
|
|
408
408
|
this.assertStartupActive();
|
|
409
409
|
}
|
|
410
410
|
this.publishResourceGeneration(globalConfig, this.activeGeneration, this.vectorDb, this.workerPool, this.mlxMode(globalConfig));
|
|
@@ -496,52 +496,59 @@ class Daemon {
|
|
|
496
496
|
console.log("[daemon] Idle shutdown disabled (GMAX_DAEMON_IDLE_TIMEOUT_MS<=0)");
|
|
497
497
|
}
|
|
498
498
|
console.log(`[daemon] Started (PID: ${process.pid}, ${this.processors.size} projects)`);
|
|
499
|
-
|
|
500
|
-
// pay daemon-side cold costs:
|
|
501
|
-
// - LanceDB connection open + first openTable() (~10–15s on a 5GB
|
|
502
|
-
// index — this is the dominant cost)
|
|
503
|
-
// - FTS index "already exists" round-trip
|
|
504
|
-
// - Two parallel encodeQuery calls so the worker pool spawns + warms
|
|
505
|
-
// workers ahead of the first real search. The reaper floor is
|
|
506
|
-
// MIN_KEEP_WORKERS = 1 (pool.ts), so only one worker stays resident
|
|
507
|
-
// long-term — but the prewarm still pays the model-load cost up front.
|
|
508
|
-
// Fire-and-forget; failures are non-fatal — the next real search just
|
|
509
|
-
// pays the cost once. Delay a few seconds so we don't compete with the
|
|
510
|
-
// catchup scans dispatched on startup.
|
|
511
|
-
setTimeout(() => {
|
|
512
|
-
if (this.shuttingDown)
|
|
513
|
-
return;
|
|
514
|
-
void this.runSharedOperation("search-warmup", undefined, () => __awaiter(this, void 0, void 0, function* () {
|
|
515
|
-
const t0 = Date.now();
|
|
516
|
-
try {
|
|
517
|
-
if (this.vectorDb) {
|
|
518
|
-
yield this.vectorDb.ensureTable();
|
|
519
|
-
yield this.vectorDb.createFTSIndex();
|
|
520
|
-
yield this.vectorDb.createVectorIndex(false, 5, true);
|
|
521
|
-
}
|
|
522
|
-
const pool = this.workerPool;
|
|
523
|
-
if (!pool)
|
|
524
|
-
throw new Error("worker pool not ready");
|
|
525
|
-
// Two parallel encodes force the pool to spawn two workers and
|
|
526
|
-
// warm both (each worker loads Granite + ColBERT lazily on first
|
|
527
|
-
// encode — once warmed, subsequent encodes are ~13ms).
|
|
528
|
-
yield Promise.all([
|
|
529
|
-
pool.encodeQuery("warmup-a"),
|
|
530
|
-
pool.encodeQuery("warmup-b"),
|
|
531
|
-
]);
|
|
532
|
-
console.log(`[daemon] Search hot path pre-warmed (${Date.now() - t0}ms)`);
|
|
533
|
-
}
|
|
534
|
-
catch (err) {
|
|
535
|
-
console.log(`[daemon] Search warmup failed (non-fatal): ${err}`);
|
|
536
|
-
}
|
|
537
|
-
})).catch((err) => {
|
|
538
|
-
if (err instanceof operation_coordinator_1.OperationClosedError)
|
|
539
|
-
return;
|
|
540
|
-
console.log(`[daemon] Search warmup skipped: ${err}`);
|
|
541
|
-
});
|
|
542
|
-
}, 5000).unref();
|
|
499
|
+
this.scheduleSearchWarmup();
|
|
543
500
|
});
|
|
544
501
|
}
|
|
502
|
+
scheduleSearchWarmup() {
|
|
503
|
+
// Speculative encodes can spawn/load models even though existing FTS
|
|
504
|
+
// adoption is read-only. Containment schedules neither work nor admission.
|
|
505
|
+
if ((0, maintenance_policy_1.fullTableMaintenanceDisabled)())
|
|
506
|
+
return;
|
|
507
|
+
// Pre-warm the search hot path so the first user-facing search doesn't
|
|
508
|
+
// pay daemon-side cold costs:
|
|
509
|
+
// - LanceDB connection open + first openTable() (~10–15s on a 5GB
|
|
510
|
+
// index — this is the dominant cost)
|
|
511
|
+
// - FTS index "already exists" round-trip
|
|
512
|
+
// - Two parallel encodeQuery calls so the worker pool spawns + warms
|
|
513
|
+
// workers ahead of the first real search. The reaper floor is
|
|
514
|
+
// MIN_KEEP_WORKERS = 1 (pool.ts), so only one worker stays resident
|
|
515
|
+
// long-term — but the prewarm still pays the model-load cost up front.
|
|
516
|
+
// Fire-and-forget; failures are non-fatal — the next real search just
|
|
517
|
+
// pays the cost once. Delay a few seconds so we don't compete with the
|
|
518
|
+
// catchup scans dispatched on startup.
|
|
519
|
+
setTimeout(() => {
|
|
520
|
+
if (this.shuttingDown)
|
|
521
|
+
return;
|
|
522
|
+
void this.runSharedOperation("search-warmup", undefined, () => __awaiter(this, void 0, void 0, function* () {
|
|
523
|
+
const t0 = Date.now();
|
|
524
|
+
try {
|
|
525
|
+
if (this.vectorDb) {
|
|
526
|
+
yield this.vectorDb.ensureTable();
|
|
527
|
+
yield this.vectorDb.createFTSIndex();
|
|
528
|
+
yield this.vectorDb.createVectorIndex(false, 5, true);
|
|
529
|
+
}
|
|
530
|
+
const pool = this.workerPool;
|
|
531
|
+
if (!pool)
|
|
532
|
+
throw new Error("worker pool not ready");
|
|
533
|
+
// Two parallel encodes force the pool to spawn two workers and
|
|
534
|
+
// warm both (each worker loads Granite + ColBERT lazily on first
|
|
535
|
+
// encode — once warmed, subsequent encodes are ~13ms).
|
|
536
|
+
yield Promise.all([
|
|
537
|
+
pool.encodeQuery("warmup-a"),
|
|
538
|
+
pool.encodeQuery("warmup-b"),
|
|
539
|
+
]);
|
|
540
|
+
console.log(`[daemon] Search hot path pre-warmed (${Date.now() - t0}ms)`);
|
|
541
|
+
}
|
|
542
|
+
catch (err) {
|
|
543
|
+
console.log(`[daemon] Search warmup failed (non-fatal): ${err}`);
|
|
544
|
+
}
|
|
545
|
+
})).catch((err) => {
|
|
546
|
+
if (err instanceof operation_coordinator_1.OperationClosedError)
|
|
547
|
+
return;
|
|
548
|
+
console.log(`[daemon] Search warmup skipped: ${err}`);
|
|
549
|
+
});
|
|
550
|
+
}, 5000).unref();
|
|
551
|
+
}
|
|
545
552
|
watchProject(root, signal) {
|
|
546
553
|
return this.withProjectLock(root, signal, () => this.runSharedOperation("watch", signal, () => this.watchProjectWithinOperation(root)));
|
|
547
554
|
}
|
|
@@ -616,14 +623,25 @@ class Daemon {
|
|
|
616
623
|
}
|
|
617
624
|
}
|
|
618
625
|
runSharedOperation(name, signal, fn) {
|
|
619
|
-
const readOnly = (0, read_verbs_1.getReadVerb)(name) !== undefined ||
|
|
620
|
-
["
|
|
626
|
+
const readOnly = ((0, read_verbs_1.getReadVerb)(name) !== undefined && !(0, read_verbs_1.isHeavyReadVerb)(name)) ||
|
|
627
|
+
["project-stats", "unwatch", "llm-stop"].includes(name);
|
|
621
628
|
return this.operations.runShared(name, signal, (operationSignal) => __awaiter(this, void 0, void 0, function* () {
|
|
629
|
+
if (name === "store-maintenance" && (0, maintenance_policy_1.fullTableMaintenanceDisabled)()) {
|
|
630
|
+
// Retain coordinator close/exclusive checks, but never admit a disabled
|
|
631
|
+
// callback as heavy work or probe the kernel for a known no-op.
|
|
632
|
+
return (0, compaction_result_1.skippedCompaction)(maintenance_policy_1.FULL_TABLE_MAINTENANCE_DISABLED_REASON);
|
|
633
|
+
}
|
|
622
634
|
if (!readOnly)
|
|
623
635
|
this.assertHeavyOperationAdmission(name);
|
|
624
636
|
return fn(operationSignal);
|
|
625
637
|
}));
|
|
626
638
|
}
|
|
639
|
+
ensureAdmittedMlxServer(model) {
|
|
640
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
641
|
+
this.assertHeavyOperationAdmission("mlx-start");
|
|
642
|
+
yield this.mlxServerManager.ensureMlxServer(model);
|
|
643
|
+
});
|
|
644
|
+
}
|
|
627
645
|
assertHeavyOperationAdmission(name) {
|
|
628
646
|
const denied = (0, autostart_1.daemonStartDeniedReason)();
|
|
629
647
|
if (denied !== null)
|
|
@@ -632,6 +650,10 @@ class Daemon {
|
|
|
632
650
|
if (!this.checkKernelZonePressure()) {
|
|
633
651
|
throw new Error(`${name} refused: host pressure is unsafe or unknown`);
|
|
634
652
|
}
|
|
653
|
+
// Another process can quarantine the machine while the probes run.
|
|
654
|
+
const finalDenied = (0, autostart_1.daemonStartDeniedReason)();
|
|
655
|
+
if (finalDenied !== null)
|
|
656
|
+
throw new Error(`${name} refused: ${finalDenied}`);
|
|
635
657
|
}
|
|
636
658
|
/**
|
|
637
659
|
* Compact + prune on behalf of a CLI caller.
|
|
@@ -1474,7 +1496,7 @@ class Daemon {
|
|
|
1474
1496
|
throw new Error(`rebuilt table dimension ${physicalDim !== null && physicalDim !== void 0 ? physicalDim : "missing"} does not match target ${targetGeneration.vectorDim}`);
|
|
1475
1497
|
}
|
|
1476
1498
|
if (targetConfig.embedMode === "gpu") {
|
|
1477
|
-
yield this.
|
|
1499
|
+
yield this.ensureAdmittedMlxServer(targetGeneration.mlxModel);
|
|
1478
1500
|
}
|
|
1479
1501
|
const targetMlx = this.mlxMode(targetConfig);
|
|
1480
1502
|
targetPool = this.createWorkerPool(targetGeneration, targetConfig.embedMode);
|
|
@@ -1588,7 +1610,7 @@ class Daemon {
|
|
|
1588
1610
|
: oldResources.vectorDb;
|
|
1589
1611
|
if (oldResources.config.embedMode === "gpu" &&
|
|
1590
1612
|
oldResources.mlx === "owned") {
|
|
1591
|
-
yield this.
|
|
1613
|
+
yield this.ensureAdmittedMlxServer(oldResources.embedding.mlxModel);
|
|
1592
1614
|
}
|
|
1593
1615
|
restoredPool = this.createWorkerPool(oldResources.embedding, oldResources.config.embedMode);
|
|
1594
1616
|
yield restoredDb.downgradeStoreLease();
|
|
@@ -1889,7 +1911,9 @@ class Daemon {
|
|
|
1889
1911
|
this.maybeRecycle(sampleFootprint);
|
|
1890
1912
|
if (this.shuttingDown || this.recycling || !sampleFootprint)
|
|
1891
1913
|
return;
|
|
1892
|
-
void this.mlxServerManager.checkMlxHealth()
|
|
1914
|
+
void this.mlxServerManager.checkMlxHealth().catch((error) => {
|
|
1915
|
+
console.error("[daemon] MLX health/restart refused:", error instanceof Error ? error.message : "unknown failure");
|
|
1916
|
+
});
|
|
1893
1917
|
this.processManager.sweepOrphanWorkers();
|
|
1894
1918
|
}
|
|
1895
1919
|
logResourceSnapshot(reason, footprintMb) {
|
|
@@ -2007,17 +2031,56 @@ class Daemon {
|
|
|
2007
2031
|
this.stopForSafety(denied, false);
|
|
2008
2032
|
return false;
|
|
2009
2033
|
}
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
const usage = (0, kernel_zone_1.readKernelZoneUsage)();
|
|
2013
|
-
if ((usage === null || usage === void 0 ? void 0 : usage.pressure) === "ok")
|
|
2034
|
+
const pressure = this.hostPressureDeniedReason();
|
|
2035
|
+
if (pressure === null)
|
|
2014
2036
|
return true;
|
|
2015
|
-
|
|
2016
|
-
? `${usage.pressure} kernel pressure: ${(0, kernel_zone_1.formatZoneUsage)(usage)}`
|
|
2017
|
-
: "kernel pressure unavailable; heavy work cannot be admitted";
|
|
2018
|
-
this.stopForSafety(reason);
|
|
2037
|
+
this.stopForSafety(pressure.reason);
|
|
2019
2038
|
return false;
|
|
2020
2039
|
}
|
|
2040
|
+
/** Both measurements must be known and normal before any resource expansion. */
|
|
2041
|
+
hostPressureDeniedReason() {
|
|
2042
|
+
if (process.platform !== "darwin")
|
|
2043
|
+
return null;
|
|
2044
|
+
// This inexpensive probe rejects OS warning before a worker/model starts,
|
|
2045
|
+
// or before paying the bounded but relatively slow kernel-zone probe cost.
|
|
2046
|
+
const memory = (0, kernel_zone_1.probeMemoryPressure)();
|
|
2047
|
+
if (memory.status !== "known") {
|
|
2048
|
+
return {
|
|
2049
|
+
reason: `OS memory pressure unavailable (${(0, kernel_zone_1.formatPressureProbe)(memory)})`,
|
|
2050
|
+
measured: false,
|
|
2051
|
+
};
|
|
2052
|
+
}
|
|
2053
|
+
if (memory.pressure !== "normal") {
|
|
2054
|
+
return {
|
|
2055
|
+
reason: `${memory.pressure} OS memory pressure (${(0, kernel_zone_1.formatPressureProbe)(memory)})`,
|
|
2056
|
+
measured: true,
|
|
2057
|
+
};
|
|
2058
|
+
}
|
|
2059
|
+
const kernel = (0, kernel_zone_1.probeKernelZoneUsage)();
|
|
2060
|
+
if (kernel.status !== "known") {
|
|
2061
|
+
return {
|
|
2062
|
+
reason: `kernel pressure unavailable (${(0, kernel_zone_1.formatPressureProbe)(kernel)}); heavy work cannot be admitted`,
|
|
2063
|
+
measured: false,
|
|
2064
|
+
};
|
|
2065
|
+
}
|
|
2066
|
+
if (kernel.usage.pressure === "ok") {
|
|
2067
|
+
// Kernel sampling can take seconds; use a new inexpensive OS observation
|
|
2068
|
+
// at the actual admission boundary, rather than trusting its first sample.
|
|
2069
|
+
const freshMemory = (0, kernel_zone_1.probeMemoryPressure)();
|
|
2070
|
+
if (freshMemory.status === "known" && freshMemory.pressure === "normal")
|
|
2071
|
+
return null;
|
|
2072
|
+
return {
|
|
2073
|
+
reason: freshMemory.status === "known"
|
|
2074
|
+
? `${freshMemory.pressure} OS memory pressure (${(0, kernel_zone_1.formatPressureProbe)(freshMemory)})`
|
|
2075
|
+
: `OS memory pressure unavailable (${(0, kernel_zone_1.formatPressureProbe)(freshMemory)})`,
|
|
2076
|
+
measured: freshMemory.status === "known",
|
|
2077
|
+
};
|
|
2078
|
+
}
|
|
2079
|
+
return {
|
|
2080
|
+
reason: `${kernel.usage.pressure} kernel pressure: ${(0, kernel_zone_1.formatZoneUsage)(kernel.usage)} (${(0, kernel_zone_1.formatPressureProbe)(kernel)})`,
|
|
2081
|
+
measured: true,
|
|
2082
|
+
};
|
|
2083
|
+
}
|
|
2021
2084
|
stopForSafety(reason, persist = true) {
|
|
2022
2085
|
// Warning already pauses mutation: a stale index costs less than another
|
|
2023
2086
|
// host crash. Unknown readings are visible stops, never healthy samples.
|
|
@@ -203,8 +203,8 @@ class MlxServerManager {
|
|
|
203
203
|
}
|
|
204
204
|
ensureInternal(model) {
|
|
205
205
|
return __awaiter(this, void 0, void 0, function* () {
|
|
206
|
-
var _a, _b, _c;
|
|
207
|
-
var
|
|
206
|
+
var _a, _b, _c, _d, _e;
|
|
207
|
+
var _f, _g, _h, _j, _k;
|
|
208
208
|
this.phase = "probing";
|
|
209
209
|
const health = yield this.probeHealth(this.port);
|
|
210
210
|
if (this.shouldStop())
|
|
@@ -256,15 +256,31 @@ class MlxServerManager {
|
|
|
256
256
|
console.warn("[daemon] MLX embed server not found — falling back to CPU embeddings");
|
|
257
257
|
return;
|
|
258
258
|
}
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
259
|
+
// A health/termination await may outlive the daemon's outer admission.
|
|
260
|
+
// Refusal must not leak a log descriptor or look like a successful startup.
|
|
261
|
+
// Disable automatic retries until an explicit admitted ensure occurs;
|
|
262
|
+
// fire-and-forget health callers must catch the propagated rejection.
|
|
263
|
+
try {
|
|
264
|
+
(_b = (_a = this.deps).assertExpansionAdmission) === null || _b === void 0 ? void 0 : _b.call(_a);
|
|
265
|
+
}
|
|
266
|
+
catch (error) {
|
|
267
|
+
this.enabled = false;
|
|
268
|
+
this.phase = "failed";
|
|
269
|
+
this.lastError = `MLX expansion admission refused: ${error instanceof Error ? error.message : "host admission unavailable"}`;
|
|
270
|
+
console.error(`[daemon] ${this.lastError}`);
|
|
271
|
+
throw new Error(this.lastError);
|
|
272
|
+
}
|
|
273
|
+
if (this.shouldStop())
|
|
274
|
+
return;
|
|
275
|
+
const owner = (_f = (_d = (_c = this.deps).createOwnerToken) === null || _d === void 0 ? void 0 : _d.call(_c)) !== null && _f !== void 0 ? _f : (0, node_crypto_1.randomUUID)();
|
|
276
|
+
const openLog = (_g = this.deps.openLog) !== null && _g !== void 0 ? _g : log_rotate_1.openRotatedLog;
|
|
277
|
+
const closeFd = (_h = this.deps.closeFd) !== null && _h !== void 0 ? _h : fs.closeSync;
|
|
262
278
|
const logFd = openLog(path.join(config_1.PATHS.logsDir, "mlx-embed-server.log"));
|
|
263
279
|
const env = Object.assign(Object.assign({}, process.env), { MLX_EMBED_MODEL: model, GMAX_EMBED_OWNER_TOKEN: owner, HF_HOME: (0, mlx_hf_cache_1.resolveMlxHfHome)(model) });
|
|
264
280
|
this.phase = "starting";
|
|
265
281
|
let child;
|
|
266
282
|
try {
|
|
267
|
-
child = ((
|
|
283
|
+
child = ((_j = this.deps.spawn) !== null && _j !== void 0 ? _j : node_child_process_1.spawn)("uv", ["run", "python", "server.py"], {
|
|
268
284
|
cwd: serverDir,
|
|
269
285
|
detached: true,
|
|
270
286
|
stdio: ["ignore", logFd, logFd],
|
|
@@ -281,7 +297,7 @@ class MlxServerManager {
|
|
|
281
297
|
try {
|
|
282
298
|
closeFd(logFd);
|
|
283
299
|
}
|
|
284
|
-
catch (
|
|
300
|
+
catch (_l) { }
|
|
285
301
|
}
|
|
286
302
|
if (!child.pid) {
|
|
287
303
|
this.lastError = "spawned MLX process has no PID";
|
|
@@ -340,7 +356,7 @@ class MlxServerManager {
|
|
|
340
356
|
child.off("error", onError);
|
|
341
357
|
child.off("exit", onExit);
|
|
342
358
|
yield this.terminateOwned(candidate);
|
|
343
|
-
this.lastError = (
|
|
359
|
+
this.lastError = (_k = (_e = startup.error) === null || _e === void 0 ? void 0 : _e.message) !== null && _k !== void 0 ? _k : (this.shouldStop() ? "MLX startup cancelled" : "MLX startup timed out");
|
|
344
360
|
if (!this.shouldStop()) {
|
|
345
361
|
this.phase = "failed";
|
|
346
362
|
console.error(`[daemon] ${this.lastError} — falling back to CPU embeddings`);
|
|
@@ -275,6 +275,10 @@ class VectorDB {
|
|
|
275
275
|
* Call once from the daemon — replaces per-processor maintenance intervals.
|
|
276
276
|
*/
|
|
277
277
|
startMaintenanceLoop(runOperation) {
|
|
278
|
+
if ((0, maintenance_policy_1.fullTableMaintenanceDisabled)()) {
|
|
279
|
+
this.recordSkippedCompaction((0, maintenance_policy_1.recordMaintenanceContainment)(this.lancedbDir));
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
278
282
|
if (runOperation)
|
|
279
283
|
this.maintenanceRunner = runOperation;
|
|
280
284
|
if (this.maintenanceTimer)
|
|
@@ -1,6 +1,20 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __rest = (this && this.__rest) || function (s, e) {
|
|
3
|
+
var t = {};
|
|
4
|
+
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
|
|
5
|
+
t[p] = s[p];
|
|
6
|
+
if (s != null && typeof Object.getOwnPropertySymbols === "function")
|
|
7
|
+
for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
|
|
8
|
+
if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
|
|
9
|
+
t[p[i]] = s[p[i]];
|
|
10
|
+
}
|
|
11
|
+
return t;
|
|
12
|
+
};
|
|
2
13
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
14
|
exports.ZONE_THRESHOLDS = void 0;
|
|
15
|
+
exports.formatPressureProbe = formatPressureProbe;
|
|
16
|
+
exports.probeKernelZoneUsage = probeKernelZoneUsage;
|
|
17
|
+
exports.probeMemoryPressure = probeMemoryPressure;
|
|
4
18
|
exports.classifyZonePressure = classifyZonePressure;
|
|
5
19
|
exports.parseZprintOutput = parseZprintOutput;
|
|
6
20
|
exports.readKernelZoneUsage = readKernelZoneUsage;
|
|
@@ -49,6 +63,122 @@ const WARN_BYTES = 4 * Math.pow(1024, 3);
|
|
|
49
63
|
* is an unplanned reboot.
|
|
50
64
|
*/
|
|
51
65
|
const CRITICAL_BYTES = 8 * Math.pow(1024, 3);
|
|
66
|
+
/** Compact diagnostic metadata; never serializes raw probe output or errors. */
|
|
67
|
+
function formatPressureProbe(result) {
|
|
68
|
+
return JSON.stringify(Object.assign(Object.assign({ status: result.status, sampledAtMs: result.sampledAtMs, durationMs: result.durationMs, outputBytes: result.outputBytes }, (result.status !== "known" ? { reason: result.reason } : {})), (result.status === "unknown"
|
|
69
|
+
? {
|
|
70
|
+
errorCode: result.errorCode,
|
|
71
|
+
exitCode: result.exitCode,
|
|
72
|
+
signal: result.signal,
|
|
73
|
+
}
|
|
74
|
+
: {})));
|
|
75
|
+
}
|
|
76
|
+
const PROBE_ERROR_CODES = new Set([
|
|
77
|
+
"ETIMEDOUT",
|
|
78
|
+
"ENOBUFS",
|
|
79
|
+
"ERR_CHILD_PROCESS_STDIO_MAXBUFFER",
|
|
80
|
+
"ENOENT",
|
|
81
|
+
"EACCES",
|
|
82
|
+
"EPERM",
|
|
83
|
+
]);
|
|
84
|
+
const PROBE_SIGNALS = new Set(["SIGTERM", "SIGKILL", "SIGABRT", "SIGSEGV"]);
|
|
85
|
+
function stdoutBytes(output) {
|
|
86
|
+
return typeof output === "string"
|
|
87
|
+
? Buffer.byteLength(output)
|
|
88
|
+
: Buffer.isBuffer(output)
|
|
89
|
+
? output.length
|
|
90
|
+
: 0;
|
|
91
|
+
}
|
|
92
|
+
function boundedPressureProbe(command, args, timeout, parse, overrides) {
|
|
93
|
+
const deps = Object.assign({ platform: process.platform, wallNow: Date.now, monotonicNow: () => performance.now(), run: (cmd, argv, options) => (0, node_child_process_1.execFileSync)(cmd, argv, options) }, overrides);
|
|
94
|
+
const sampledAtMs = deps.wallNow();
|
|
95
|
+
const started = deps.monotonicNow();
|
|
96
|
+
let outputBytes = 0;
|
|
97
|
+
const observation = () => ({
|
|
98
|
+
sampledAtMs,
|
|
99
|
+
durationMs: Math.max(0, deps.monotonicNow() - started),
|
|
100
|
+
outputBytes,
|
|
101
|
+
});
|
|
102
|
+
if (deps.platform !== "darwin")
|
|
103
|
+
return Object.assign(Object.assign({}, observation()), { status: "unsupported", reason: "unsupported-platform" });
|
|
104
|
+
try {
|
|
105
|
+
const output = deps.run(command, args, {
|
|
106
|
+
encoding: "utf-8",
|
|
107
|
+
timeout,
|
|
108
|
+
maxBuffer: 64 * 1024,
|
|
109
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
110
|
+
});
|
|
111
|
+
outputBytes = stdoutBytes(output);
|
|
112
|
+
const value = parse(output);
|
|
113
|
+
return value === null
|
|
114
|
+
? Object.assign(Object.assign({}, observation()), { status: "unknown", reason: "parse", errorCode: null, exitCode: null, signal: null }) : Object.assign(Object.assign({}, observation()), { status: "known", value });
|
|
115
|
+
}
|
|
116
|
+
catch (error) {
|
|
117
|
+
const details = error && typeof error === "object"
|
|
118
|
+
? error
|
|
119
|
+
: {};
|
|
120
|
+
outputBytes = stdoutBytes(details.stdout);
|
|
121
|
+
const errorCode = typeof details.code === "string" && PROBE_ERROR_CODES.has(details.code)
|
|
122
|
+
? details.code
|
|
123
|
+
: details.code == null
|
|
124
|
+
? null
|
|
125
|
+
: "UNKNOWN";
|
|
126
|
+
const exitCode = typeof details.status === "number" && Number.isInteger(details.status)
|
|
127
|
+
? details.status
|
|
128
|
+
: null;
|
|
129
|
+
const signal = typeof details.signal === "string" && PROBE_SIGNALS.has(details.signal)
|
|
130
|
+
? details.signal
|
|
131
|
+
: null;
|
|
132
|
+
const reason = errorCode === "ETIMEDOUT"
|
|
133
|
+
? "timeout"
|
|
134
|
+
: errorCode === "ENOBUFS" ||
|
|
135
|
+
errorCode === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER"
|
|
136
|
+
? "output-limit"
|
|
137
|
+
: exitCode !== null || signal !== null
|
|
138
|
+
? "exit"
|
|
139
|
+
: "execution";
|
|
140
|
+
return Object.assign(Object.assign({}, observation()), { status: "unknown", reason,
|
|
141
|
+
errorCode,
|
|
142
|
+
exitCode,
|
|
143
|
+
signal });
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
function probeKernelZoneUsage(zoneName = ZONE_NAME, overrides = {}) {
|
|
147
|
+
const result = boundedPressureProbe("zprint", [zoneName], PROBE_TIMEOUT_MS, (output) => {
|
|
148
|
+
const parsed = parseZprintOutput(output, zoneName);
|
|
149
|
+
if (!parsed)
|
|
150
|
+
return null;
|
|
151
|
+
const bytes = parsed.elements * parsed.elementSize;
|
|
152
|
+
if (!Number.isSafeInteger(bytes))
|
|
153
|
+
return null;
|
|
154
|
+
return Object.assign(Object.assign({}, parsed), { bytes, pressure: classifyZonePressure(bytes) });
|
|
155
|
+
}, overrides);
|
|
156
|
+
if (result.status !== "known")
|
|
157
|
+
return result;
|
|
158
|
+
const { value } = result, observation = __rest(result, ["value"]);
|
|
159
|
+
return Object.assign(Object.assign({}, observation), { usage: value });
|
|
160
|
+
}
|
|
161
|
+
/** XNU publishes dispatch flags NORMAL=1, WARN=2, CRITICAL=4 through this sysctl.
|
|
162
|
+
* https://github.com/apple-oss-distributions/xnu/blob/main/bsd/kern/kern_memorystatus_notify.c
|
|
163
|
+
* Other values are unknown, including the unrelated internal kernel enum. */
|
|
164
|
+
function probeMemoryPressure(overrides = {}) {
|
|
165
|
+
const result = boundedPressureProbe("sysctl", ["-n", "kern.memorystatus_vm_pressure_level"], 1000, (output) => {
|
|
166
|
+
switch (output.trim()) {
|
|
167
|
+
case "1":
|
|
168
|
+
return "normal";
|
|
169
|
+
case "2":
|
|
170
|
+
return "warn";
|
|
171
|
+
case "4":
|
|
172
|
+
return "critical";
|
|
173
|
+
default:
|
|
174
|
+
return null;
|
|
175
|
+
}
|
|
176
|
+
}, overrides);
|
|
177
|
+
if (result.status !== "known")
|
|
178
|
+
return result;
|
|
179
|
+
const { value } = result, observation = __rest(result, ["value"]);
|
|
180
|
+
return Object.assign(Object.assign({}, observation), { pressure: value });
|
|
181
|
+
}
|
|
52
182
|
function classifyZonePressure(bytes) {
|
|
53
183
|
if (bytes >= CRITICAL_BYTES)
|
|
54
184
|
return "critical";
|
|
@@ -92,24 +222,8 @@ function parseZprintOutput(output, zoneName = ZONE_NAME) {
|
|
|
92
222
|
* "healthy". macOS startup and heavy-operation admission refuse unknown samples.
|
|
93
223
|
*/
|
|
94
224
|
function readKernelZoneUsage(zoneName = ZONE_NAME) {
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
try {
|
|
98
|
-
const output = (0, node_child_process_1.execFileSync)("zprint", [zoneName], {
|
|
99
|
-
encoding: "utf-8",
|
|
100
|
-
timeout: PROBE_TIMEOUT_MS,
|
|
101
|
-
maxBuffer: 64 * 1024,
|
|
102
|
-
stdio: ["ignore", "pipe", "ignore"],
|
|
103
|
-
});
|
|
104
|
-
const parsed = parseZprintOutput(output, zoneName);
|
|
105
|
-
if (!parsed)
|
|
106
|
-
return null;
|
|
107
|
-
const bytes = parsed.elements * parsed.elementSize;
|
|
108
|
-
return Object.assign(Object.assign({}, parsed), { bytes, pressure: classifyZonePressure(bytes) });
|
|
109
|
-
}
|
|
110
|
-
catch (_a) {
|
|
111
|
-
return null;
|
|
112
|
-
}
|
|
225
|
+
const result = probeKernelZoneUsage(zoneName);
|
|
226
|
+
return result.status === "known" ? result.usage : null;
|
|
113
227
|
}
|
|
114
228
|
function formatZoneUsage(usage) {
|
|
115
229
|
const gib = (usage.bytes / Math.pow(1024, 3)).toFixed(2);
|
package/dist/lib/workers/pool.js
CHANGED
|
@@ -57,7 +57,10 @@ const path = __importStar(require("node:path"));
|
|
|
57
57
|
const config_1 = require("../../config");
|
|
58
58
|
const embedding_generation_1 = require("../index/embedding-generation");
|
|
59
59
|
const index_config_1 = require("../index/index-config");
|
|
60
|
+
const autostart_1 = require("../utils/autostart");
|
|
61
|
+
const kernel_zone_1 = require("../utils/kernel-zone");
|
|
60
62
|
const logger_1 = require("../utils/logger");
|
|
63
|
+
const safety_latch_1 = require("../utils/safety-latch");
|
|
61
64
|
const TASK_TIMEOUT_MS = (() => {
|
|
62
65
|
var _a;
|
|
63
66
|
const fromEnv = Number.parseInt((_a = process.env.GMAX_WORKER_TASK_TIMEOUT_MS) !== null && _a !== void 0 ? _a : "", 10);
|
|
@@ -221,6 +224,7 @@ class WorkerPool {
|
|
|
221
224
|
this.destroyPromise = null;
|
|
222
225
|
this.consecutiveRespawns = 0;
|
|
223
226
|
this.respawnLimitReached = false;
|
|
227
|
+
this.spawnDeniedReason = null;
|
|
224
228
|
this.idleReapInterval = null;
|
|
225
229
|
// Re-runs dispatch once the oldest unassigned task reaches SCALE_UP_WAIT_MS.
|
|
226
230
|
this.scaleUpTimer = null;
|
|
@@ -237,7 +241,8 @@ class WorkerPool {
|
|
|
237
241
|
// process environment remains live and is copied at each fork.
|
|
238
242
|
this.embeddingEnvironment = Object.freeze(generatedEmbeddingEnv);
|
|
239
243
|
// Lazy spawn: start with 1 worker, scale up on demand
|
|
240
|
-
this.spawnWorker()
|
|
244
|
+
if (!this.spawnWorker())
|
|
245
|
+
throw this.spawnDeniedError();
|
|
241
246
|
// Periodically reap idle workers back to MIN_KEEP, and force-kill any
|
|
242
247
|
// worker wedged in busy=true (the leak backstop — see STUCK_BUSY_MS).
|
|
243
248
|
this.idleReapInterval = setInterval(() => {
|
|
@@ -247,7 +252,7 @@ class WorkerPool {
|
|
|
247
252
|
}, IDLE_WORKER_TIMEOUT_MS);
|
|
248
253
|
}
|
|
249
254
|
isHealthy() {
|
|
250
|
-
return !this.destroyed && this.workers.length > 0;
|
|
255
|
+
return (!this.destroyed && !this.spawnDeniedReason && this.workers.length > 0);
|
|
251
256
|
}
|
|
252
257
|
/** PIDs of workers the pool currently tracks. Used by the daemon's orphan
|
|
253
258
|
* sweep to distinguish live, accounted-for workers from de-listed strays. */
|
|
@@ -287,16 +292,16 @@ class WorkerPool {
|
|
|
287
292
|
});
|
|
288
293
|
return hasUnassigned(this.priorityQueue) || hasUnassigned(this.taskQueue);
|
|
289
294
|
}
|
|
290
|
-
rejectUnassignedTasks(message) {
|
|
295
|
+
rejectUnassignedTasks(message, code) {
|
|
291
296
|
for (const task of Array.from(this.tasks.values())) {
|
|
292
297
|
if (task.worker)
|
|
293
298
|
continue;
|
|
294
|
-
task.reject(new Error(message));
|
|
299
|
+
task.reject(Object.assign(new Error(message), code ? { code } : {}));
|
|
295
300
|
this.completeTask(task, null);
|
|
296
301
|
}
|
|
297
302
|
}
|
|
298
303
|
maybeRespawn(reason, hasPendingTasks) {
|
|
299
|
-
if (this.destroyed || this.respawnLimitReached)
|
|
304
|
+
if (this.destroyed || this.spawnDeniedReason || this.respawnLimitReached)
|
|
300
305
|
return false;
|
|
301
306
|
this.consecutiveRespawns++;
|
|
302
307
|
(0, logger_1.log)("pool", `respawn #${this.consecutiveRespawns} after ${reason} (workers=${this.workers.length} pending=${hasPendingTasks})`);
|
|
@@ -308,8 +313,7 @@ class WorkerPool {
|
|
|
308
313
|
this.rejectUnassignedTasks(message);
|
|
309
314
|
return false;
|
|
310
315
|
}
|
|
311
|
-
this.spawnWorker();
|
|
312
|
-
return true;
|
|
316
|
+
return this.spawnWorker();
|
|
313
317
|
}
|
|
314
318
|
completeTask(task, worker) {
|
|
315
319
|
this.clearTaskTimeout(task);
|
|
@@ -357,8 +361,69 @@ class WorkerPool {
|
|
|
357
361
|
this.dispatch();
|
|
358
362
|
}
|
|
359
363
|
}
|
|
364
|
+
spawnDeniedError() {
|
|
365
|
+
var _a;
|
|
366
|
+
return Object.assign(new Error((_a = this.spawnDeniedReason) !== null && _a !== void 0 ? _a : "Worker spawn denied"), { code: "HOST_SAFETY" });
|
|
367
|
+
}
|
|
368
|
+
denySpawn(reason, persist = true) {
|
|
369
|
+
// A pool cannot automatically recover after a failed admission. In
|
|
370
|
+
// particular, timer/exit callbacks must not retry forks or strand queues.
|
|
371
|
+
this.spawnDeniedReason = `Worker spawn denied: ${reason}`;
|
|
372
|
+
this.clearScaleUpTimer();
|
|
373
|
+
if (persist) {
|
|
374
|
+
try {
|
|
375
|
+
(0, safety_latch_1.latchSafetyStop)(this.spawnDeniedReason);
|
|
376
|
+
}
|
|
377
|
+
catch (_a) {
|
|
378
|
+
try {
|
|
379
|
+
(0, logger_1.log)("pool", "worker admission denied; safety latch persistence failed");
|
|
380
|
+
}
|
|
381
|
+
catch (_b) { }
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
this.rejectUnassignedTasks(this.spawnDeniedReason, "HOST_SAFETY");
|
|
385
|
+
try {
|
|
386
|
+
(0, logger_1.log)("pool", this.spawnDeniedReason);
|
|
387
|
+
}
|
|
388
|
+
catch (_c) { }
|
|
389
|
+
return false;
|
|
390
|
+
}
|
|
360
391
|
spawnWorker() {
|
|
361
|
-
|
|
392
|
+
if (this.destroyed || this.spawnDeniedReason)
|
|
393
|
+
return false;
|
|
394
|
+
let worker;
|
|
395
|
+
try {
|
|
396
|
+
const quarantine = (0, autostart_1.daemonStartDeniedReason)();
|
|
397
|
+
if (quarantine)
|
|
398
|
+
return this.denySpawn(quarantine, false);
|
|
399
|
+
if (process.platform === "darwin") {
|
|
400
|
+
const memory = (0, kernel_zone_1.probeMemoryPressure)();
|
|
401
|
+
if (memory.status !== "known" || memory.pressure !== "normal") {
|
|
402
|
+
return this.denySpawn(`OS memory pressure ${memory.status === "known" ? memory.pressure : memory.status}; ${(0, kernel_zone_1.formatPressureProbe)(memory)}`);
|
|
403
|
+
}
|
|
404
|
+
const kernel = (0, kernel_zone_1.probeKernelZoneUsage)();
|
|
405
|
+
if (kernel.status !== "known" || kernel.usage.pressure !== "ok") {
|
|
406
|
+
return this.denySpawn(`kernel pressure ${kernel.status === "known" ? kernel.usage.pressure : kernel.status}; ${(0, kernel_zone_1.formatPressureProbe)(kernel)}`);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
if (process.platform === "darwin") {
|
|
410
|
+
// Refresh the cheap OS sample after the potentially slow kernel probe.
|
|
411
|
+
const freshMemory = (0, kernel_zone_1.probeMemoryPressure)();
|
|
412
|
+
if (freshMemory.status !== "known" ||
|
|
413
|
+
freshMemory.pressure !== "normal") {
|
|
414
|
+
return this.denySpawn(`OS memory pressure ${freshMemory.status === "known" ? freshMemory.pressure : freshMemory.status}; ${(0, kernel_zone_1.formatPressureProbe)(freshMemory)}`);
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
// Probes are synchronous but another process can quarantine the host
|
|
418
|
+
// while they run. Recheck the shared markers immediately before fork.
|
|
419
|
+
const finalQuarantine = (0, autostart_1.daemonStartDeniedReason)();
|
|
420
|
+
if (finalQuarantine)
|
|
421
|
+
return this.denySpawn(finalQuarantine, false);
|
|
422
|
+
worker = new ProcessWorker(this.modulePath, this.execArgv, this.embeddingEnvironment, config_1.MAX_WORKER_MEMORY_MB);
|
|
423
|
+
}
|
|
424
|
+
catch (_a) {
|
|
425
|
+
return this.denySpawn("host admission or fork failed");
|
|
426
|
+
}
|
|
362
427
|
(0, logger_1.log)("pool", `spawn PID:${worker.child.pid} (${this.workers.length + 1}/${Math.max(1, config_1.CONFIG.WORKER_THREADS)})`);
|
|
363
428
|
const onMessage = (msg) => {
|
|
364
429
|
var _a, _b, _c, _d;
|
|
@@ -409,11 +474,14 @@ class WorkerPool {
|
|
|
409
474
|
worker.child.on("exit", onExit);
|
|
410
475
|
worker.child.on("error", onError);
|
|
411
476
|
this.workers.push(worker);
|
|
477
|
+
return true;
|
|
412
478
|
}
|
|
413
479
|
enqueue(method, payload, signal) {
|
|
414
480
|
if (this.destroyed) {
|
|
415
481
|
return Promise.reject(new Error("Worker pool destroyed"));
|
|
416
482
|
}
|
|
483
|
+
if (this.spawnDeniedReason)
|
|
484
|
+
return Promise.reject(this.spawnDeniedError());
|
|
417
485
|
if (signal === null || signal === void 0 ? void 0 : signal.aborted) {
|
|
418
486
|
const err = new Error("Aborted");
|
|
419
487
|
err.name = "AbortError";
|
|
@@ -502,6 +570,23 @@ class WorkerPool {
|
|
|
502
570
|
var _c, _d, _e;
|
|
503
571
|
if (this.destroyed)
|
|
504
572
|
return;
|
|
573
|
+
if (this.spawnDeniedReason) {
|
|
574
|
+
this.rejectUnassignedTasks(this.spawnDeniedReason, "HOST_SAFETY");
|
|
575
|
+
return;
|
|
576
|
+
}
|
|
577
|
+
// Work admitted before another process quarantines the host may still
|
|
578
|
+
// be queued. Do not send it to an existing worker after that stop.
|
|
579
|
+
try {
|
|
580
|
+
const quarantine = (0, autostart_1.daemonStartDeniedReason)();
|
|
581
|
+
if (quarantine) {
|
|
582
|
+
this.denySpawn(quarantine, false);
|
|
583
|
+
return;
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
catch (_f) {
|
|
587
|
+
this.denySpawn("quarantine check failed");
|
|
588
|
+
return;
|
|
589
|
+
}
|
|
505
590
|
let idle = this.workers.find((w) => !w.busy);
|
|
506
591
|
// Drain priority queue first so search tasks never wait behind an
|
|
507
592
|
// indexing batch.
|
|
@@ -528,7 +613,8 @@ class WorkerPool {
|
|
|
528
613
|
this.armScaleUpTimer(waitMs);
|
|
529
614
|
return;
|
|
530
615
|
}
|
|
531
|
-
this.spawnWorker()
|
|
616
|
+
if (!this.spawnWorker())
|
|
617
|
+
return;
|
|
532
618
|
idle = this.workers[this.workers.length - 1];
|
|
533
619
|
}
|
|
534
620
|
if (!idle)
|
|
@@ -723,7 +809,8 @@ class WorkerPool {
|
|
|
723
809
|
}
|
|
724
810
|
// Replace anything we dropped below the floor with fresh, lean workers.
|
|
725
811
|
while (!this.destroyed && this.workers.length < MIN_KEEP_WORKERS) {
|
|
726
|
-
this.spawnWorker()
|
|
812
|
+
if (!this.spawnWorker())
|
|
813
|
+
break;
|
|
727
814
|
}
|
|
728
815
|
}
|
|
729
816
|
/**
|
package/package.json
CHANGED