grepmax 0.26.57 → 0.26.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +5 -1
- package/dist/commands/doctor.js +29 -13
- package/dist/commands/status.js +27 -14
- package/dist/commands/watch.js +22 -3
- package/dist/lib/daemon/daemon.js +214 -76
- package/dist/lib/daemon/ipc-handler.js +35 -14
- package/dist/lib/daemon/paused-reads.js +155 -0
- package/dist/lib/daemon/rows-handler.js +14 -8
- package/dist/lib/skeleton/retriever.js +3 -3
- package/dist/lib/store/vector-db.js +23 -14
- package/dist/lib/utils/daemon-launcher.js +8 -3
- package/dist/lib/utils/operation-coordinator.js +6 -0
- package/package.json +1 -1
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
|
@@ -62,6 +62,7 @@ const maintenance_policy_1 = require("../store/maintenance-policy");
|
|
|
62
62
|
const meta_cache_1 = require("../store/meta-cache");
|
|
63
63
|
const store_lease_1 = require("../store/store-lease");
|
|
64
64
|
const vector_db_1 = require("../store/vector-db");
|
|
65
|
+
const async_semaphore_1 = require("../utils/async-semaphore");
|
|
65
66
|
const autostart_1 = require("../utils/autostart");
|
|
66
67
|
const blocked_roots_1 = require("../utils/blocked-roots");
|
|
67
68
|
const daemon_client_1 = require("../utils/daemon-client");
|
|
@@ -74,6 +75,7 @@ const operation_coordinator_1 = require("../utils/operation-coordinator");
|
|
|
74
75
|
const process_1 = require("../utils/process");
|
|
75
76
|
const process_footprint_1 = require("../utils/process-footprint");
|
|
76
77
|
const project_registry_1 = require("../utils/project-registry");
|
|
78
|
+
const query_timeout_1 = require("../utils/query-timeout");
|
|
77
79
|
const resource_snapshot_1 = require("../utils/resource-snapshot");
|
|
78
80
|
const root_availability_1 = require("../utils/root-availability");
|
|
79
81
|
const safety_latch_1 = require("../utils/safety-latch");
|
|
@@ -81,6 +83,7 @@ const watcher_store_1 = require("../utils/watcher-store");
|
|
|
81
83
|
const pool_1 = require("../workers/pool");
|
|
82
84
|
const ipc_handler_1 = require("./ipc-handler");
|
|
83
85
|
const mlx_server_manager_1 = require("./mlx-server-manager");
|
|
86
|
+
const paused_reads_1 = require("./paused-reads");
|
|
84
87
|
const process_manager_1 = require("./process-manager");
|
|
85
88
|
const read_verbs_1 = require("./read-verbs");
|
|
86
89
|
const rows_handler_1 = require("./rows-handler");
|
|
@@ -161,6 +164,9 @@ class Daemon {
|
|
|
161
164
|
// (so liveness probes succeed during slow init), so commands that need those
|
|
162
165
|
// resources must be gated on this to avoid hitting null stores mid-startup.
|
|
163
166
|
this.ready = false;
|
|
167
|
+
this.pausedReason = null;
|
|
168
|
+
this.pausePromise = null;
|
|
169
|
+
this.pausedReadGate = new async_semaphore_1.AsyncSemaphore(1);
|
|
164
170
|
this.processManager = new process_manager_1.ProcessManager({
|
|
165
171
|
getShuttingDown: () => this.shuttingDown,
|
|
166
172
|
getWorkerPids: () => { var _a; return (_a = this.workerPool) === null || _a === void 0 ? void 0 : _a.getWorkerPids(); },
|
|
@@ -239,18 +245,20 @@ class Daemon {
|
|
|
239
245
|
return resources;
|
|
240
246
|
}
|
|
241
247
|
start() {
|
|
242
|
-
return __awaiter(this,
|
|
243
|
-
var _a;
|
|
248
|
+
return __awaiter(this, arguments, void 0, function* (options = {}) {
|
|
249
|
+
var _a, _b;
|
|
244
250
|
const denied = (0, autostart_1.daemonStartDeniedReason)();
|
|
245
|
-
if (denied !== null)
|
|
251
|
+
if (denied !== null && !options.readOnly)
|
|
246
252
|
throw new Error(`gmax: ${denied}; preserving containment`);
|
|
247
|
-
//
|
|
253
|
+
// Explicit read-only startup preserves every marker and opens no models.
|
|
248
254
|
const pressure = this.hostPressureDeniedReason();
|
|
249
|
-
if (pressure
|
|
250
|
-
|
|
251
|
-
(0, safety_latch_1.latchSafetyStop)(pressure.reason);
|
|
255
|
+
if (pressure === null || pressure === void 0 ? void 0 : pressure.critical) {
|
|
256
|
+
(0, safety_latch_1.latchSafetyStop)(pressure.reason);
|
|
252
257
|
throw new Error(`gmax: ${pressure.reason}; startup safety stop`);
|
|
253
258
|
}
|
|
259
|
+
if (options.readOnly || pressure !== null) {
|
|
260
|
+
this.pausedReason = (_a = denied !== null && denied !== void 0 ? denied : pressure === null || pressure === void 0 ? void 0 : pressure.reason) !== null && _a !== void 0 ? _a : "read-only service requested";
|
|
261
|
+
}
|
|
254
262
|
process.title = "gmax-daemon";
|
|
255
263
|
// Read verbs register here rather than at read-verbs.ts module scope: the
|
|
256
264
|
// registry is asserted on exactly in tests/ipc-read-verbs.test.ts, and a
|
|
@@ -262,7 +270,8 @@ class Daemon {
|
|
|
262
270
|
(0, rows_handler_1.registerRowsVerbs)();
|
|
263
271
|
(0, vector_handler_1.registerVectorVerbs)();
|
|
264
272
|
// 0. Singleton enforcement: find and kill ALL stale daemon/worker processes
|
|
265
|
-
|
|
273
|
+
if (this.pausedReason === null)
|
|
274
|
+
yield this.processManager.killStaleProcesses();
|
|
266
275
|
this.assertStartupActive();
|
|
267
276
|
// 1. Acquire exclusive lock — kernel-enforced, atomic, auto-released on death
|
|
268
277
|
const ipcHome = path.dirname(config_1.PATHS.daemonLockFile);
|
|
@@ -302,7 +311,7 @@ class Daemon {
|
|
|
302
311
|
try {
|
|
303
312
|
fs.unlinkSync(config_1.PATHS.daemonSocket);
|
|
304
313
|
}
|
|
305
|
-
catch (
|
|
314
|
+
catch (_c) { }
|
|
306
315
|
this.server = net.createServer((conn) => {
|
|
307
316
|
(0, logger_1.debug)("daemon", "client connected");
|
|
308
317
|
this.connections.add(conn);
|
|
@@ -370,7 +379,7 @@ class Daemon {
|
|
|
370
379
|
// reads the PID can immediately ping this daemon and get a response.
|
|
371
380
|
fs.writeFileSync(config_1.PATHS.daemonPidFile, String(process.pid));
|
|
372
381
|
// 4. Kill existing per-project watchers
|
|
373
|
-
const existing = (0, watcher_store_1.listWatchers)();
|
|
382
|
+
const existing = this.pausedReason === null ? (0, watcher_store_1.listWatchers)() : [];
|
|
374
383
|
for (const w of existing) {
|
|
375
384
|
console.log(`[daemon] Taking over from per-project watcher (PID: ${w.pid}, ${path.basename(w.projectRoot)})`);
|
|
376
385
|
yield (0, process_1.killProcess)(w.pid);
|
|
@@ -379,17 +388,34 @@ class Daemon {
|
|
|
379
388
|
}
|
|
380
389
|
// 5. Open shared resources
|
|
381
390
|
try {
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
391
|
+
if (this.pausedReason === null) {
|
|
392
|
+
fs.mkdirSync(config_1.PATHS.cacheDir, { recursive: true });
|
|
393
|
+
fs.mkdirSync(config_1.PATHS.lancedbDir, { recursive: true });
|
|
394
|
+
}
|
|
395
|
+
console.log(this.pausedReason === null
|
|
396
|
+
? "[daemon] Opening LanceDB:"
|
|
397
|
+
: "[daemon] Preparing bounded retained-index reader:", config_1.PATHS.lancedbDir);
|
|
385
398
|
this.activeConfig = (0, index_config_1.readGlobalConfig)();
|
|
386
399
|
this.activeGeneration = (0, embedding_generation_1.resolveEmbeddingGeneration)(this.activeConfig);
|
|
387
|
-
this.vectorDb = new vector_db_1.VectorDB(config_1.PATHS.lancedbDir, this.activeGeneration.vectorDim);
|
|
388
|
-
this.
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
400
|
+
this.vectorDb = new vector_db_1.VectorDB(config_1.PATHS.lancedbDir, this.activeGeneration.vectorDim, undefined, this.pausedReason === null ? {} : paused_reads_1.PAUSED_CACHE_OPTIONS);
|
|
401
|
+
if (this.pausedReason === null) {
|
|
402
|
+
this.vectorDb.markIndexOwner();
|
|
403
|
+
try {
|
|
404
|
+
this.workerPool = this.createWorkerPool(this.activeGeneration, this.activeConfig.embedMode);
|
|
405
|
+
}
|
|
406
|
+
catch (error) {
|
|
407
|
+
const freshPressure = this.hostPressureDeniedReason();
|
|
408
|
+
if (freshPressure === null || freshPressure.critical)
|
|
409
|
+
throw error;
|
|
410
|
+
this.pauseWork(freshPressure.reason);
|
|
411
|
+
yield this.pausePromise;
|
|
412
|
+
}
|
|
413
|
+
if (this.pausedReason === null) {
|
|
414
|
+
this.vectorDb.startMaintenanceLoop((fn) => this.runSharedOperation("store-maintenance", undefined, () => fn()));
|
|
415
|
+
console.log("[daemon] Opening MetaCache:", config_1.PATHS.lmdbPath);
|
|
416
|
+
this.metaCache = new meta_cache_1.MetaCache(config_1.PATHS.lmdbPath);
|
|
417
|
+
}
|
|
418
|
+
}
|
|
393
419
|
this.assertStartupActive();
|
|
394
420
|
// Resources are open — only now may resource-dependent IPC commands run.
|
|
395
421
|
this.ready = true;
|
|
@@ -401,20 +427,32 @@ class Daemon {
|
|
|
401
427
|
// 6. LLM server manager (constructed, not started — starts on first request)
|
|
402
428
|
this.llmServer = new server_1.LlmServer();
|
|
403
429
|
// 6b. MLX embed server — start if GPU mode is active
|
|
404
|
-
const globalConfig = (
|
|
430
|
+
const globalConfig = (_b = this.activeConfig) !== null && _b !== void 0 ? _b : (0, index_config_1.readGlobalConfig)();
|
|
405
431
|
const isAppleSilicon = process.arch === "arm64" && process.platform === "darwin";
|
|
406
|
-
if (
|
|
407
|
-
|
|
432
|
+
if (this.pausedReason === null &&
|
|
433
|
+
isAppleSilicon &&
|
|
434
|
+
globalConfig.embedMode === "gpu") {
|
|
435
|
+
try {
|
|
436
|
+
yield this.ensureAdmittedMlxServer(globalConfig.mlxModel);
|
|
437
|
+
}
|
|
438
|
+
catch (error) {
|
|
439
|
+
if (this.pausedReason === null || this.shuttingDown || this.recycling)
|
|
440
|
+
throw error;
|
|
441
|
+
yield this.pausePromise;
|
|
442
|
+
}
|
|
408
443
|
this.assertStartupActive();
|
|
409
444
|
}
|
|
410
|
-
|
|
445
|
+
if (this.pausedReason === null)
|
|
446
|
+
this.publishResourceGeneration(globalConfig, this.activeGeneration, this.vectorDb, this.workerPool, this.mlxMode(globalConfig));
|
|
411
447
|
// 7. Register daemon (only after resources are open)
|
|
412
448
|
(0, watcher_store_1.registerDaemon)(process.pid);
|
|
413
449
|
// 8. Subscribe to the registered projects some session still holds a lease
|
|
414
450
|
// on (skip missing directories). Everything else waits for a lease.
|
|
415
451
|
const allProjects = (0, project_registry_1.listProjects)();
|
|
416
452
|
this.watchLeases.load();
|
|
417
|
-
const indexed =
|
|
453
|
+
const indexed = this.pausedReason !== null
|
|
454
|
+
? []
|
|
455
|
+
: allProjects.filter((p) => p.status === "indexed" && this.shouldWatch(p.root));
|
|
418
456
|
console.log(`[daemon] Watching ${indexed.length} of ${allProjects.length} registered projects` +
|
|
419
457
|
(WATCH_ALL ? " (GMAX_WATCH_ALL=1)" : " (leased by active sessions)"));
|
|
420
458
|
for (const p of indexed) {
|
|
@@ -439,12 +477,14 @@ class Daemon {
|
|
|
439
477
|
// Re-check shuttingDown each iteration: shutdown's pendingLocks drain is a
|
|
440
478
|
// snapshot, so a new project op kicked off after the snapshot would race
|
|
441
479
|
// with vectorDb.close() and fail with "VectorDB connection is closed".
|
|
442
|
-
const pending =
|
|
443
|
-
|
|
444
|
-
|
|
480
|
+
const pending = this.pausedReason !== null
|
|
481
|
+
? []
|
|
482
|
+
: allProjects.filter((p) => (p.status === "pending" || p.status === "error") &&
|
|
483
|
+
!p.rebuildId &&
|
|
484
|
+
fs.existsSync(p.root));
|
|
445
485
|
void (() => __awaiter(this, void 0, void 0, function* () {
|
|
446
486
|
for (const p of pending) {
|
|
447
|
-
if (this.shuttingDown)
|
|
487
|
+
if (this.shuttingDown || this.pausedReason !== null)
|
|
448
488
|
return;
|
|
449
489
|
try {
|
|
450
490
|
yield this.indexPendingProject(p.root);
|
|
@@ -464,6 +504,12 @@ class Daemon {
|
|
|
464
504
|
catch (_a) { }
|
|
465
505
|
(0, log_rotate_1.rotateLogFds)(path.join(config_1.PATHS.logsDir, "daemon.log"));
|
|
466
506
|
this.sweepWatchLeases();
|
|
507
|
+
if (!this.checkKernelZonePressure()) {
|
|
508
|
+
this.heartbeatTick++;
|
|
509
|
+
if (this.pausedReason !== null && this.heartbeatTick % 5 === 0)
|
|
510
|
+
this.logResourceSnapshot("paused-heartbeat", null);
|
|
511
|
+
return;
|
|
512
|
+
}
|
|
467
513
|
// Every 5 ticks (5 min), probe the MLX embed server and respawn if
|
|
468
514
|
// it's gone zombie (port held but /health unresponsive). Closes the
|
|
469
515
|
// 42h-degradation window where workers silently fell back to ONNX CPU
|
|
@@ -480,7 +526,8 @@ class Daemon {
|
|
|
480
526
|
if (IDLE_TIMEOUT_MS > 0) {
|
|
481
527
|
this.idleInterval = setInterval(() => {
|
|
482
528
|
var _a;
|
|
483
|
-
if (
|
|
529
|
+
if (this.pausedReason !== null ||
|
|
530
|
+
Date.now() - this.lastActivity <= IDLE_TIMEOUT_MS)
|
|
484
531
|
return;
|
|
485
532
|
// Don't kick off shutdown on top of a live maintenance pass — let it
|
|
486
533
|
// finish and check again next tick. close() awaits this anyway, but
|
|
@@ -496,7 +543,12 @@ class Daemon {
|
|
|
496
543
|
console.log("[daemon] Idle shutdown disabled (GMAX_DAEMON_IDLE_TIMEOUT_MS<=0)");
|
|
497
544
|
}
|
|
498
545
|
console.log(`[daemon] Started (PID: ${process.pid}, ${this.processors.size} projects)`);
|
|
499
|
-
this.
|
|
546
|
+
if (this.pausedReason === null)
|
|
547
|
+
this.scheduleSearchWarmup();
|
|
548
|
+
else {
|
|
549
|
+
console.log(`[daemon] Serving bounded reads; heavy work paused: ${this.pausedReason}`);
|
|
550
|
+
this.logResourceSnapshot("paused-startup", null);
|
|
551
|
+
}
|
|
500
552
|
});
|
|
501
553
|
}
|
|
502
554
|
scheduleSearchWarmup() {
|
|
@@ -555,7 +607,7 @@ class Daemon {
|
|
|
555
607
|
watchProjectWithinOperation(root) {
|
|
556
608
|
// Every path that re-watches after an index/rebuild/remove goes through
|
|
557
609
|
// here, so an unleased project stays unwatched after the operation too.
|
|
558
|
-
if (!this.shouldWatch(root))
|
|
610
|
+
if (this.pausedReason !== null || !this.shouldWatch(root))
|
|
559
611
|
return Promise.resolve();
|
|
560
612
|
const project = (0, project_registry_1.getProject)(root);
|
|
561
613
|
if ((project === null || project === void 0 ? void 0 : project.status) === "indexed" &&
|
|
@@ -581,7 +633,7 @@ class Daemon {
|
|
|
581
633
|
pid: lease.pid,
|
|
582
634
|
ttlMs: (_a = lease.ttlMs) !== null && _a !== void 0 ? _a : watch_leases_1.DEFAULT_LEASE_TTL_MS,
|
|
583
635
|
});
|
|
584
|
-
if (this.processors.has(root))
|
|
636
|
+
if (this.pausedReason !== null || this.processors.has(root))
|
|
585
637
|
return Promise.resolve();
|
|
586
638
|
return this.watchProject(root, signal);
|
|
587
639
|
}
|
|
@@ -631,6 +683,23 @@ class Daemon {
|
|
|
631
683
|
// callback as heavy work or probe the kernel for a known no-op.
|
|
632
684
|
return (0, compaction_result_1.skippedCompaction)(maintenance_policy_1.FULL_TABLE_MAINTENANCE_DISABLED_REASON);
|
|
633
685
|
}
|
|
686
|
+
if (this.pausedReason !== null) {
|
|
687
|
+
if (!(0, paused_reads_1.isPausedRead)(name))
|
|
688
|
+
throw new paused_reads_1.DaemonPausedError(this.pausedReason);
|
|
689
|
+
if (this.pausedReadGate.waiting >= 16)
|
|
690
|
+
throw new paused_reads_1.DaemonPausedError("bounded read queue is full; retry shortly");
|
|
691
|
+
return this.pausedReadGate.run(operationSignal, () => __awaiter(this, void 0, void 0, function* () {
|
|
692
|
+
operationSignal.throwIfAborted();
|
|
693
|
+
if (this.pausePromise)
|
|
694
|
+
yield (0, query_timeout_1.withQueryTimeout)(this.pausePromise, "pause transition", paused_reads_1.PAUSED_QUERY_TIMEOUT_MS);
|
|
695
|
+
if (this.shuttingDown || this.recycling)
|
|
696
|
+
throw new operation_coordinator_1.OperationClosedError();
|
|
697
|
+
this.checkKernelZonePressure();
|
|
698
|
+
if (this.shuttingDown || this.recycling)
|
|
699
|
+
throw new operation_coordinator_1.OperationClosedError();
|
|
700
|
+
return fn(operationSignal);
|
|
701
|
+
}));
|
|
702
|
+
}
|
|
634
703
|
if (!readOnly)
|
|
635
704
|
this.assertHeavyOperationAdmission(name);
|
|
636
705
|
return fn(operationSignal);
|
|
@@ -642,13 +711,59 @@ class Daemon {
|
|
|
642
711
|
yield this.mlxServerManager.ensureMlxServer(model);
|
|
643
712
|
});
|
|
644
713
|
}
|
|
714
|
+
serviceStatus() {
|
|
715
|
+
return this.pausedReason === null
|
|
716
|
+
? { mode: "active" }
|
|
717
|
+
: { mode: "paused", reason: this.pausedReason };
|
|
718
|
+
}
|
|
719
|
+
pauseWork(reason) {
|
|
720
|
+
var _a, _b;
|
|
721
|
+
if (this.pausedReason !== null || this.shuttingDown)
|
|
722
|
+
return;
|
|
723
|
+
this.pausedReason = reason;
|
|
724
|
+
console.log(`[daemon] Heavy work paused; service stays available: ${reason}`);
|
|
725
|
+
const error = new paused_reads_1.DaemonPausedError(reason);
|
|
726
|
+
for (const ac of this.shutdownAbortControllers)
|
|
727
|
+
ac.abort(error);
|
|
728
|
+
for (const timer of this.pendingIndexRetryTimers.values())
|
|
729
|
+
clearTimeout(timer);
|
|
730
|
+
this.pendingIndexRetryTimers.clear();
|
|
731
|
+
(_a = this.vectorDb) === null || _a === void 0 ? void 0 : _a.pauseMaintenanceLoop();
|
|
732
|
+
(_b = this.vectorDb) === null || _b === void 0 ? void 0 : _b.abortLeaseWaits();
|
|
733
|
+
const drain = this.operations.abortAndDrain(error);
|
|
734
|
+
const pool = this.workerPool;
|
|
735
|
+
this.workerPool = null;
|
|
736
|
+
this.resources = null;
|
|
737
|
+
this.searchers.clear();
|
|
738
|
+
this.pausePromise = (() => __awaiter(this, void 0, void 0, function* () {
|
|
739
|
+
var _a, _b, _c, _d;
|
|
740
|
+
yield Promise.all([
|
|
741
|
+
this.watcherManager.quiesceAll(),
|
|
742
|
+
pool === null || pool === void 0 ? void 0 : pool.destroy(),
|
|
743
|
+
drain,
|
|
744
|
+
]);
|
|
745
|
+
yield this.mlxServerManager.stopMlxServer();
|
|
746
|
+
yield (_a = this.llmServer) === null || _a === void 0 ? void 0 : _a.stop();
|
|
747
|
+
yield (_b = this.metaCache) === null || _b === void 0 ? void 0 : _b.close();
|
|
748
|
+
this.metaCache = null;
|
|
749
|
+
yield (_c = this.vectorDb) === null || _c === void 0 ? void 0 : _c.close();
|
|
750
|
+
this.vectorDb = null;
|
|
751
|
+
if (this.shuttingDown)
|
|
752
|
+
return;
|
|
753
|
+
this.vectorDb = new vector_db_1.VectorDB(config_1.PATHS.lancedbDir, (_d = this.activeGeneration) === null || _d === void 0 ? void 0 : _d.vectorDim, undefined, paused_reads_1.PAUSED_CACHE_OPTIONS);
|
|
754
|
+
this.logResourceSnapshot("paused-transition", null);
|
|
755
|
+
}))();
|
|
756
|
+
void this.pausePromise.catch((error) => {
|
|
757
|
+
console.error("[daemon] Paused reader transition failed; status remains available:", error);
|
|
758
|
+
});
|
|
759
|
+
}
|
|
645
760
|
assertHeavyOperationAdmission(name) {
|
|
646
761
|
const denied = (0, autostart_1.daemonStartDeniedReason)();
|
|
647
762
|
if (denied !== null)
|
|
648
763
|
throw new Error(`${name} refused: ${denied}`);
|
|
649
764
|
// Probe at operation admission, before setup/cache/worker/model side effects.
|
|
650
765
|
if (!this.checkKernelZonePressure()) {
|
|
651
|
-
throw new Error(`${name} refused: host pressure is unsafe or unknown`);
|
|
766
|
+
throw new Error(`${name} refused: host pressure is unsafe or unknown${this.pausedReason ? `: ${this.pausedReason}` : ""}`);
|
|
652
767
|
}
|
|
653
768
|
// Another process can quarantine the machine while the probes run.
|
|
654
769
|
const finalDenied = (0, autostart_1.daemonStartDeniedReason)();
|
|
@@ -866,14 +981,27 @@ class Daemon {
|
|
|
866
981
|
recentReindexed === 0 &&
|
|
867
982
|
pendingFiles > 0;
|
|
868
983
|
const health = this.watcherManager.health(root);
|
|
869
|
-
return Object.assign(Object.assign({ indexing:
|
|
870
|
-
|
|
984
|
+
return Object.assign(Object.assign({ indexing: this.pausedReason == null &&
|
|
985
|
+
(!!fullIdx || processing || batchPending > 0 || initialPending), pendingFiles: this.pausedReason == null ? pendingFiles : 0, verifying: this.pausedReason == null && verifying }, health), { failedFiles: Math.max((_h = (_d = processor === null || processor === void 0 ? void 0 : processor.progress) === null || _d === void 0 ? void 0 : _d.failedFiles) !== null && _h !== void 0 ? _h : 0, (_j = health.failedFiles) !== null && _j !== void 0 ? _j : 0) });
|
|
871
986
|
}
|
|
872
987
|
search(payload, signal) {
|
|
873
988
|
return __awaiter(this, void 0, void 0, function* () {
|
|
874
989
|
// Search handling lives in search-handler.ts (Phase 12 split). The daemon
|
|
875
990
|
// supplies its warm VectorDB + watcher/index bookkeeping; the handler runs
|
|
876
991
|
// the query and assembles the response.
|
|
992
|
+
if (this.pausedReason === null)
|
|
993
|
+
this.checkKernelZonePressure();
|
|
994
|
+
if (this.shuttingDown || this.recycling)
|
|
995
|
+
return { ok: false, error: "daemon stopping for critical pressure" };
|
|
996
|
+
if (this.pausedReason !== null) {
|
|
997
|
+
return this.runSharedOperation("keyword-search", signal, (operationSignal) => __awaiter(this, void 0, void 0, function* () {
|
|
998
|
+
if (!this.vectorDb)
|
|
999
|
+
return { ok: false, error: "paused reader unavailable" };
|
|
1000
|
+
this.resetActivity();
|
|
1001
|
+
const result = yield (0, paused_reads_1.searchPausedIndex)(this.vectorDb, payload, this.pausedReason, operationSignal);
|
|
1002
|
+
return Object.assign(Object.assign({}, result), { indexState: this.indexState(payload.projectRoot) });
|
|
1003
|
+
}));
|
|
1004
|
+
}
|
|
877
1005
|
return this.runSharedOperation("search", signal, (operationSignal) => __awaiter(this, void 0, void 0, function* () {
|
|
878
1006
|
return (0, search_handler_1.handleDaemonSearch)({
|
|
879
1007
|
vectorDb: this.vectorDb,
|
|
@@ -902,6 +1030,7 @@ class Daemon {
|
|
|
902
1030
|
touchActivity: () => {
|
|
903
1031
|
this.lastActivity = Date.now();
|
|
904
1032
|
},
|
|
1033
|
+
queryTimeoutMs: this.pausedReason === null ? 15000 : paused_reads_1.PAUSED_QUERY_TIMEOUT_MS,
|
|
905
1034
|
};
|
|
906
1035
|
}
|
|
907
1036
|
listProjects() {
|
|
@@ -2014,9 +2143,9 @@ class Daemon {
|
|
|
2014
2143
|
* This host lost three sessions that way before the compactor was identified as
|
|
2015
2144
|
* the largest write source on it.
|
|
2016
2145
|
*
|
|
2017
|
-
* Warning
|
|
2018
|
-
*
|
|
2019
|
-
*
|
|
2146
|
+
* Warning or unknown readings pause heavy work while bounded reads remain
|
|
2147
|
+
* available. Critical readings persist a stop before shutting down so hooks
|
|
2148
|
+
* cannot revive the writer. Nothing here reclaims kernel memory.
|
|
2020
2149
|
*
|
|
2021
2150
|
* Non-macOS hosts have no such probe. On macOS an unavailable sample is
|
|
2022
2151
|
* explicitly unknown and cannot authorize further mutation.
|
|
@@ -2024,62 +2153,71 @@ class Daemon {
|
|
|
2024
2153
|
checkKernelZonePressure() {
|
|
2025
2154
|
if (this.shuttingDown || this.recycling)
|
|
2026
2155
|
return false;
|
|
2156
|
+
const pressure = this.hostPressureDeniedReason();
|
|
2157
|
+
if (pressure === null || pressure === void 0 ? void 0 : pressure.critical) {
|
|
2158
|
+
this.stopForSafety(pressure.reason);
|
|
2159
|
+
return false;
|
|
2160
|
+
}
|
|
2027
2161
|
const denied = (0, autostart_1.daemonStartDeniedReason)();
|
|
2028
|
-
if (denied !== null) {
|
|
2029
|
-
|
|
2030
|
-
// do not replace it with a new marker that an operator must also clear.
|
|
2031
|
-
this.stopForSafety(denied, false);
|
|
2162
|
+
if (denied !== null || pressure !== null) {
|
|
2163
|
+
this.pauseWork(denied !== null && denied !== void 0 ? denied : pressure.reason);
|
|
2032
2164
|
return false;
|
|
2033
2165
|
}
|
|
2034
|
-
|
|
2035
|
-
if (pressure === null)
|
|
2036
|
-
return true;
|
|
2037
|
-
this.stopForSafety(pressure.reason);
|
|
2038
|
-
return false;
|
|
2166
|
+
return this.pausedReason === null;
|
|
2039
2167
|
}
|
|
2040
|
-
/**
|
|
2168
|
+
/** Unknown/warning pauses resource expansion; only known critical shuts down. */
|
|
2041
2169
|
hostPressureDeniedReason() {
|
|
2042
2170
|
if (process.platform !== "darwin")
|
|
2043
2171
|
return null;
|
|
2044
|
-
// This inexpensive probe rejects OS warning before a worker/model starts,
|
|
2045
|
-
// or before paying the bounded but relatively slow kernel-zone probe cost.
|
|
2046
2172
|
const memory = (0, kernel_zone_1.probeMemoryPressure)();
|
|
2047
|
-
if (memory.status
|
|
2173
|
+
if (memory.status === "known" && memory.pressure === "critical")
|
|
2048
2174
|
return {
|
|
2049
|
-
reason: `OS memory pressure
|
|
2050
|
-
measured:
|
|
2175
|
+
reason: `critical OS memory pressure (${(0, kernel_zone_1.formatPressureProbe)(memory)})`,
|
|
2176
|
+
measured: true,
|
|
2177
|
+
critical: true,
|
|
2051
2178
|
};
|
|
2052
|
-
|
|
2053
|
-
|
|
2179
|
+
// Paused reads still need the kernel-critical guard, even if OS pressure warns.
|
|
2180
|
+
const kernel = (0, kernel_zone_1.probeKernelZoneUsage)();
|
|
2181
|
+
if (kernel.status === "known" && kernel.usage.pressure === "critical")
|
|
2182
|
+
return {
|
|
2183
|
+
reason: `critical kernel pressure: ${(0, kernel_zone_1.formatZoneUsage)(kernel.usage)} (${(0, kernel_zone_1.formatPressureProbe)(kernel)})`,
|
|
2184
|
+
measured: true,
|
|
2185
|
+
critical: true,
|
|
2186
|
+
};
|
|
2187
|
+
const freshMemory = (0, kernel_zone_1.probeMemoryPressure)();
|
|
2188
|
+
if (freshMemory.status === "known" && freshMemory.pressure === "critical")
|
|
2054
2189
|
return {
|
|
2055
|
-
reason:
|
|
2190
|
+
reason: `critical OS memory pressure (${(0, kernel_zone_1.formatPressureProbe)(freshMemory)})`,
|
|
2056
2191
|
measured: true,
|
|
2192
|
+
critical: true,
|
|
2057
2193
|
};
|
|
2194
|
+
for (const sample of [memory, freshMemory]) {
|
|
2195
|
+
if (sample.status !== "known")
|
|
2196
|
+
return {
|
|
2197
|
+
reason: `OS memory pressure unavailable (${(0, kernel_zone_1.formatPressureProbe)(sample)})`,
|
|
2198
|
+
measured: false,
|
|
2199
|
+
critical: false,
|
|
2200
|
+
};
|
|
2201
|
+
if (sample.pressure !== "normal")
|
|
2202
|
+
return {
|
|
2203
|
+
reason: `warn OS memory pressure (${(0, kernel_zone_1.formatPressureProbe)(sample)})`,
|
|
2204
|
+
measured: true,
|
|
2205
|
+
critical: false,
|
|
2206
|
+
};
|
|
2058
2207
|
}
|
|
2059
|
-
|
|
2060
|
-
if (kernel.status !== "known") {
|
|
2208
|
+
if (kernel.status !== "known")
|
|
2061
2209
|
return {
|
|
2062
|
-
reason: `kernel pressure unavailable (${(0, kernel_zone_1.formatPressureProbe)(kernel)}); heavy work
|
|
2210
|
+
reason: `kernel pressure unavailable (${(0, kernel_zone_1.formatPressureProbe)(kernel)}); heavy work paused`,
|
|
2063
2211
|
measured: false,
|
|
2212
|
+
critical: false,
|
|
2064
2213
|
};
|
|
2065
|
-
|
|
2066
|
-
if (kernel.usage.pressure === "ok") {
|
|
2067
|
-
// Kernel sampling can take seconds; use a new inexpensive OS observation
|
|
2068
|
-
// at the actual admission boundary, rather than trusting its first sample.
|
|
2069
|
-
const freshMemory = (0, kernel_zone_1.probeMemoryPressure)();
|
|
2070
|
-
if (freshMemory.status === "known" && freshMemory.pressure === "normal")
|
|
2071
|
-
return null;
|
|
2214
|
+
if (kernel.usage.pressure !== "ok")
|
|
2072
2215
|
return {
|
|
2073
|
-
reason:
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
measured: freshMemory.status === "known",
|
|
2216
|
+
reason: `warn kernel pressure: ${(0, kernel_zone_1.formatZoneUsage)(kernel.usage)} (${(0, kernel_zone_1.formatPressureProbe)(kernel)})`,
|
|
2217
|
+
measured: true,
|
|
2218
|
+
critical: false,
|
|
2077
2219
|
};
|
|
2078
|
-
|
|
2079
|
-
return {
|
|
2080
|
-
reason: `${kernel.usage.pressure} kernel pressure: ${(0, kernel_zone_1.formatZoneUsage)(kernel.usage)} (${(0, kernel_zone_1.formatPressureProbe)(kernel)})`,
|
|
2081
|
-
measured: true,
|
|
2082
|
-
};
|
|
2220
|
+
return null;
|
|
2083
2221
|
}
|
|
2084
2222
|
stopForSafety(reason, persist = true) {
|
|
2085
2223
|
// Warning already pauses mutation: a stale index costs less than another
|
|
@@ -2206,7 +2344,7 @@ class Daemon {
|
|
|
2206
2344
|
const projectDrain = this.projectMutex.close();
|
|
2207
2345
|
// Abort catchup/recovery and remove each processor before worker teardown.
|
|
2208
2346
|
yield this.watcherManager.quiesceAll();
|
|
2209
|
-
yield this.drainOperationsBounded(operationDrain, projectDrain);
|
|
2347
|
+
yield this.drainOperationsBounded(operationDrain, projectDrain, ...(this.pausePromise ? [this.pausePromise.catch(() => { })] : []));
|
|
2210
2348
|
// Stop LLM server if running
|
|
2211
2349
|
try {
|
|
2212
2350
|
yield (_b = this.llmServer) === null || _b === void 0 ? void 0 : _b.stop();
|
|
@@ -55,6 +55,7 @@ const logger_1 = require("../utils/logger");
|
|
|
55
55
|
const operation_coordinator_1 = require("../utils/operation-coordinator");
|
|
56
56
|
const path_containment_1 = require("../utils/path-containment");
|
|
57
57
|
const project_registry_1 = require("../utils/project-registry");
|
|
58
|
+
const paused_reads_1 = require("./paused-reads");
|
|
58
59
|
const read_verbs_1 = require("./read-verbs");
|
|
59
60
|
const DAEMON_VERSION = (() => {
|
|
60
61
|
try {
|
|
@@ -83,7 +84,8 @@ function writeDone(conn, data) {
|
|
|
83
84
|
conn.end();
|
|
84
85
|
}
|
|
85
86
|
function errorResponse(err) {
|
|
86
|
-
if (err instanceof
|
|
87
|
+
if (err instanceof paused_reads_1.DaemonPausedError ||
|
|
88
|
+
err instanceof operation_coordinator_1.OperationBusyError ||
|
|
87
89
|
err instanceof operation_coordinator_1.OperationClosedError) {
|
|
88
90
|
return { ok: false, error: err.message, code: err.code };
|
|
89
91
|
}
|
|
@@ -123,8 +125,8 @@ function startHeartbeat(conn, intervalMs = HEARTBEAT_INTERVAL_MS) {
|
|
|
123
125
|
*/
|
|
124
126
|
function handleCommand(daemon, cmd, conn) {
|
|
125
127
|
return __awaiter(this, void 0, void 0, function* () {
|
|
126
|
-
var _a, _b;
|
|
127
|
-
var
|
|
128
|
+
var _a, _b, _c, _d;
|
|
129
|
+
var _e, _f, _g, _h, _j, _k, _l, _m, _o, _p;
|
|
128
130
|
try {
|
|
129
131
|
(0, logger_1.debug)("daemon", `ipc cmd=${cmd.cmd}${cmd.root ? ` root=${cmd.root}` : ""}`);
|
|
130
132
|
if (cmd.cmd !== "ping" &&
|
|
@@ -148,6 +150,7 @@ function handleCommand(daemon, cmd, conn) {
|
|
|
148
150
|
uptime: daemon.uptime(),
|
|
149
151
|
version: DAEMON_VERSION,
|
|
150
152
|
operationStatus: daemon.operationStatus(),
|
|
153
|
+
service: (_a = daemon.serviceStatus) === null || _a === void 0 ? void 0 : _a.call(daemon),
|
|
151
154
|
resourceGeneration: daemon.resourceGenerationId(),
|
|
152
155
|
rebuildPending: daemon.hasUnfinishedRebuild(),
|
|
153
156
|
capabilities: {
|
|
@@ -189,10 +192,14 @@ function handleCommand(daemon, cmd, conn) {
|
|
|
189
192
|
diskPressure: daemon.getDiskPressure(),
|
|
190
193
|
mlx: daemon.getMlxStatus(),
|
|
191
194
|
workers: daemon.workerCount(),
|
|
192
|
-
resources: (
|
|
193
|
-
compaction: (
|
|
195
|
+
resources: (_e = (_b = daemon.resourceSnapshot) === null || _b === void 0 ? void 0 : _b.call(daemon)) !== null && _e !== void 0 ? _e : null,
|
|
196
|
+
compaction: (_f = (_c = daemon.compactionStatus) === null || _c === void 0 ? void 0 : _c.call(daemon)) !== null && _f !== void 0 ? _f : null,
|
|
194
197
|
workerThreads: config_1.WORKER_THREADS_SETTING,
|
|
198
|
+
service: (_d = daemon.serviceStatus) === null || _d === void 0 ? void 0 : _d.call(daemon),
|
|
195
199
|
};
|
|
200
|
+
case "pause":
|
|
201
|
+
daemon.pauseWork(String((_g = cmd.reason) !== null && _g !== void 0 ? _g : "read-only service requested"));
|
|
202
|
+
return { ok: true, service: daemon.serviceStatus() };
|
|
196
203
|
case "optimize":
|
|
197
204
|
// Keeps compaction single-writer: see Daemon.runOptimize.
|
|
198
205
|
return yield daemon.runOptimize();
|
|
@@ -206,14 +213,14 @@ function handleCommand(daemon, cmd, conn) {
|
|
|
206
213
|
return Object.assign({ ok: true }, (yield daemon.projectStats(root)));
|
|
207
214
|
}
|
|
208
215
|
case "shutdown": {
|
|
209
|
-
const reason = String((
|
|
210
|
-
const fromPid = (
|
|
211
|
-
const fromPpid = (
|
|
212
|
-
const fromVer = (
|
|
216
|
+
const reason = String((_h = cmd.reason) !== null && _h !== void 0 ? _h : "unknown");
|
|
217
|
+
const fromPid = (_j = cmd.from_pid) !== null && _j !== void 0 ? _j : "?";
|
|
218
|
+
const fromPpid = (_k = cmd.from_ppid) !== null && _k !== void 0 ? _k : "?";
|
|
219
|
+
const fromVer = (_l = cmd.from_version) !== null && _l !== void 0 ? _l : "?";
|
|
213
220
|
const fromArgv = Array.isArray(cmd.from_argv)
|
|
214
221
|
? cmd.from_argv.join(" ")
|
|
215
222
|
: "?";
|
|
216
|
-
const fromParentCmd = (
|
|
223
|
+
const fromParentCmd = (_m = cmd.from_parent_cmd) !== null && _m !== void 0 ? _m : "?";
|
|
217
224
|
console.log(`[daemon] shutdown command received via IPC: reason=${reason} from_pid=${fromPid} from_ppid=${fromPpid} from_version=${fromVer} from_argv=[${fromArgv}] from_parent_cmd=[${fromParentCmd}]`);
|
|
218
225
|
setImmediate(() => daemon.shutdown());
|
|
219
226
|
return { ok: true };
|
|
@@ -238,7 +245,7 @@ function handleCommand(daemon, cmd, conn) {
|
|
|
238
245
|
try {
|
|
239
246
|
maxPerFile = (0, per_file_1.validateMaxPerFile)(cmd.maxPerFile);
|
|
240
247
|
}
|
|
241
|
-
catch (
|
|
248
|
+
catch (_q) {
|
|
242
249
|
return { ok: false, error: "invalid maxPerFile" };
|
|
243
250
|
}
|
|
244
251
|
if (!Number.isInteger(limitRaw) || limitRaw < 1 || limitRaw > 1000) {
|
|
@@ -285,8 +292,8 @@ function handleCommand(daemon, cmd, conn) {
|
|
|
285
292
|
return { ok: false, error: "invalid project_roots" };
|
|
286
293
|
}
|
|
287
294
|
for (const prefix of [
|
|
288
|
-
...((
|
|
289
|
-
...((
|
|
295
|
+
...((_o = filters === null || filters === void 0 ? void 0 : filters.inPrefixes) !== null && _o !== void 0 ? _o : []),
|
|
296
|
+
...((_p = filters === null || filters === void 0 ? void 0 : filters.excludePrefixes) !== null && _p !== void 0 ? _p : []),
|
|
290
297
|
]) {
|
|
291
298
|
(0, path_containment_1.resolveContainedPath)(projectRoot, prefix);
|
|
292
299
|
}
|
|
@@ -419,7 +426,21 @@ function handleCommand(daemon, cmd, conn) {
|
|
|
419
426
|
try {
|
|
420
427
|
// Heavy verbs queue on a global gate before admission; see
|
|
421
428
|
// HEAVY_READ_VERBS in read-verbs.ts.
|
|
422
|
-
return yield (0, read_verbs_1.runReadVerb)(name, ac.signal, () => daemon.runSharedOperation(name, ac.signal, (signal) =>
|
|
429
|
+
return yield (0, read_verbs_1.runReadVerb)(name, ac.signal, () => daemon.runSharedOperation(name, ac.signal, (signal) => __awaiter(this, void 0, void 0, function* () {
|
|
430
|
+
var _a;
|
|
431
|
+
// Validate after admission: pressure can change while queued.
|
|
432
|
+
const paused = ((_a = daemon.serviceStatus) === null || _a === void 0 ? void 0 : _a.call(daemon).mode) === "paused";
|
|
433
|
+
if (paused) {
|
|
434
|
+
(0, paused_reads_1.validatePausedRead)(name, cmd);
|
|
435
|
+
if (name === "rows.locate" && cmd.limit === undefined)
|
|
436
|
+
cmd = Object.assign(Object.assign({}, cmd), { limit: paused_reads_1.PAUSED_RESULT_LIMIT });
|
|
437
|
+
}
|
|
438
|
+
const response = yield verb(cmd, { daemon, conn, signal });
|
|
439
|
+
if (paused &&
|
|
440
|
+
Buffer.byteLength(JSON.stringify(response)) > 262144)
|
|
441
|
+
throw new paused_reads_1.DaemonPausedError("read response exceeds the paused service budget; narrow the request");
|
|
442
|
+
return response;
|
|
443
|
+
})));
|
|
423
444
|
}
|
|
424
445
|
finally {
|
|
425
446
|
conn.off("close", onClose);
|