grepmax 0.26.27 → 0.26.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +40 -0
- package/dist/commands/audit.js +28 -260
- package/dist/commands/context.js +42 -26
- package/dist/commands/dead.js +37 -35
- package/dist/commands/doctor.js +280 -10
- package/dist/commands/extract.js +79 -49
- package/dist/commands/impact.js +74 -20
- package/dist/commands/log.js +25 -25
- package/dist/commands/mcp.js +639 -475
- package/dist/commands/peek.js +49 -47
- package/dist/commands/project.js +29 -107
- package/dist/commands/related.js +43 -52
- package/dist/commands/search-run.js +23 -8
- package/dist/commands/search.js +7 -2
- package/dist/commands/similar.js +50 -76
- package/dist/commands/skeleton.js +137 -83
- package/dist/commands/status.js +90 -25
- package/dist/commands/surprises.js +73 -16
- package/dist/commands/symbols.js +58 -77
- package/dist/commands/test-find.js +51 -17
- package/dist/commands/trace.js +26 -11
- package/dist/commands/watch.js +1 -1
- package/dist/config.js +8 -2
- package/dist/lib/analysis/surprising-connections.js +135 -15
- package/dist/lib/daemon/daemon.js +157 -13
- package/dist/lib/daemon/graph-handler.js +859 -0
- package/dist/lib/daemon/ipc-handler.js +36 -4
- package/dist/lib/daemon/read-verbs.js +156 -0
- package/dist/lib/daemon/rows-handler.js +665 -0
- package/dist/lib/daemon/search-handler.js +7 -4
- package/dist/lib/daemon/vector-handler.js +199 -0
- package/dist/lib/daemon/watch-leases.js +179 -0
- package/dist/lib/index/batch-processor.js +7 -1
- package/dist/lib/index/watcher.js +4 -0
- package/dist/lib/search/pagerank.js +80 -19
- package/dist/lib/store/vector-db.js +35 -1
- package/dist/lib/utils/async-semaphore.js +141 -0
- package/dist/lib/utils/daemon-client.js +11 -0
- package/dist/lib/utils/process-footprint.js +51 -0
- package/dist/lib/utils/store-access.js +381 -0
- package/dist/lib/utils/tests-footer.js +19 -2
- package/dist/lib/utils/watcher-launcher.js +46 -19
- package/dist/lib/workers/pool.js +72 -2
- package/package.json +1 -1
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
- package/plugins/grepmax/hooks/cwd-changed.js +19 -36
- package/plugins/grepmax/hooks/start.js +15 -19
- package/plugins/grepmax/hooks/stop.js +13 -4
- package/plugins/grepmax/hooks/watch-lease.js +163 -0
|
@@ -62,6 +62,7 @@ const path = __importStar(require("node:path"));
|
|
|
62
62
|
const callsites_1 = require("../graph/callsites");
|
|
63
63
|
const ann_config_1 = require("../store/ann-config");
|
|
64
64
|
const arrow_1 = require("../utils/arrow");
|
|
65
|
+
const filter_builder_1 = require("../utils/filter-builder");
|
|
65
66
|
const scope_filter_1 = require("../utils/scope-filter");
|
|
66
67
|
exports.DEFAULT_SURPRISE_OPTIONS = {
|
|
67
68
|
sample: 160,
|
|
@@ -73,6 +74,11 @@ exports.DEFAULT_SURPRISE_OPTIONS = {
|
|
|
73
74
|
includeEval: false,
|
|
74
75
|
};
|
|
75
76
|
exports.MAX_SURPRISE_ROWS = 100000;
|
|
77
|
+
/**
|
|
78
|
+
* Columns a pair row carries: the scan's metadata plus `content`, which the
|
|
79
|
+
* weak-code filter and the scorer read. `vector` is not here — only sampled
|
|
80
|
+
* anchors need one, and they fetch it by id (see `hydrateRows`).
|
|
81
|
+
*/
|
|
76
82
|
exports.SURPRISE_COLUMNS = [
|
|
77
83
|
"id",
|
|
78
84
|
"path",
|
|
@@ -83,8 +89,40 @@ exports.SURPRISE_COLUMNS = [
|
|
|
83
89
|
"type_referenced_symbols",
|
|
84
90
|
"role",
|
|
85
91
|
"content",
|
|
86
|
-
"vector",
|
|
87
92
|
];
|
|
93
|
+
const MIN_CODE_CONTENT_LENGTH = 80;
|
|
94
|
+
/**
|
|
95
|
+
* The row scan reads up to MAX_SURPRISE_ROWS rows. Selecting `content` and
|
|
96
|
+
* `vector` there held every chunk body and embedding in memory for an answer
|
|
97
|
+
* that needs ~160 vectors. It now selects metadata plus two bounds on the
|
|
98
|
+
* weak-code test (`content.trim().length >= 80`), computed in the store:
|
|
99
|
+
*
|
|
100
|
+
* - `content_min_len` trims a *superset* of JS whitespace (Unicode White_Space
|
|
101
|
+
* plus U+FEFF) and counts code points, so it never exceeds the JS length —
|
|
102
|
+
* at or above the threshold, the row passes.
|
|
103
|
+
* - `content_max_len` trims a *subset* (ASCII whitespace) and counts UTF-8
|
|
104
|
+
* bytes, so it is never below the JS length — under the threshold, it fails.
|
|
105
|
+
*
|
|
106
|
+
* Anything between the two (non-ASCII text or whitespace near the threshold)
|
|
107
|
+
* fetches its content by id and takes the exact JS test, so `codeRows` and the
|
|
108
|
+
* anchor sample are unchanged. Backslashes are literal in DataFusion strings.
|
|
109
|
+
*/
|
|
110
|
+
const CONTENT_MIN_LEN_EXPR = "char_length(regexp_replace(content, '^[\\s\\x{FEFF}]+|[\\s\\x{FEFF}]+$', '', 'g'))";
|
|
111
|
+
const CONTENT_MAX_LEN_EXPR = "octet_length(regexp_replace(content, '^[\\t\\n\\x0B\\f\\r ]+|[\\t\\n\\x0B\\f\\r ]+$', '', 'g'))";
|
|
112
|
+
const SCAN_SELECT = {
|
|
113
|
+
id: "id",
|
|
114
|
+
path: "path",
|
|
115
|
+
start_line: "start_line",
|
|
116
|
+
end_line: "end_line",
|
|
117
|
+
defined_symbols: "defined_symbols",
|
|
118
|
+
referenced_symbols: "referenced_symbols",
|
|
119
|
+
type_referenced_symbols: "type_referenced_symbols",
|
|
120
|
+
role: "role",
|
|
121
|
+
content_min_len: CONTENT_MIN_LEN_EXPR,
|
|
122
|
+
content_max_len: CONTENT_MAX_LEN_EXPR,
|
|
123
|
+
};
|
|
124
|
+
/** Ids per `id IN (...)` hydration query. */
|
|
125
|
+
const HYDRATE_BATCH = 200;
|
|
88
126
|
const CODE_EXTENSIONS = new Set([
|
|
89
127
|
".ts",
|
|
90
128
|
".tsx",
|
|
@@ -430,19 +468,61 @@ function buildFileEdges(rows) {
|
|
|
430
468
|
function hasDirectFileEdge(edges, a, b) {
|
|
431
469
|
return edges.has(`${a.path}\0${b.path}`) || edges.has(`${b.path}\0${a.path}`);
|
|
432
470
|
}
|
|
433
|
-
|
|
471
|
+
/** The anchor filter minus its content and vector tests (see below). */
|
|
472
|
+
function filterableCodeMeta(row, opts) {
|
|
434
473
|
if (!row.path || !isCodePath(row.path))
|
|
435
474
|
return false;
|
|
436
475
|
if (!opts.includeTests && isTestPath(row.relPath))
|
|
437
476
|
return false;
|
|
438
477
|
if (!opts.includeEval && isEvalPath(row.relPath))
|
|
439
478
|
return false;
|
|
440
|
-
if (vectorLength(row.vector) === 0)
|
|
441
|
-
return false;
|
|
442
|
-
if (row.content.trim().length < 80)
|
|
443
|
-
return false;
|
|
444
479
|
return row.definedSymbols.length > 0;
|
|
445
480
|
}
|
|
481
|
+
function hasCodeContent(content) {
|
|
482
|
+
return content.trim().length >= MIN_CODE_CONTENT_LENGTH;
|
|
483
|
+
}
|
|
484
|
+
/**
|
|
485
|
+
* The weak-code verdict from the scan alone: true/false when the store-side
|
|
486
|
+
* bounds decide it, null when the row needs its content fetched. A row that
|
|
487
|
+
* already carries `content` (a caller-supplied table) is decided exactly.
|
|
488
|
+
*/
|
|
489
|
+
function contentVerdict(raw) {
|
|
490
|
+
if (typeof raw.content === "string")
|
|
491
|
+
return hasCodeContent(raw.content);
|
|
492
|
+
const min = Number(raw.content_min_len);
|
|
493
|
+
const max = Number(raw.content_max_len);
|
|
494
|
+
if (Number.isFinite(min) && min >= MIN_CODE_CONTENT_LENGTH)
|
|
495
|
+
return true;
|
|
496
|
+
if (Number.isFinite(max) && max < MIN_CODE_CONTENT_LENGTH)
|
|
497
|
+
return false;
|
|
498
|
+
return null;
|
|
499
|
+
}
|
|
500
|
+
/**
|
|
501
|
+
* Fetch `columns` for the given ids, keyed by id. The scope clause is ANDed
|
|
502
|
+
* on so a hydration can never reach outside the scan that produced the ids.
|
|
503
|
+
*/
|
|
504
|
+
function hydrateRows(table, where, ids, columns) {
|
|
505
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
506
|
+
const out = new Map();
|
|
507
|
+
for (let i = 0; i < ids.length; i += HYDRATE_BATCH) {
|
|
508
|
+
const batch = ids.slice(i, i + HYDRATE_BATCH);
|
|
509
|
+
const list = batch.map((id) => `'${(0, filter_builder_1.escapeSqlString)(id)}'`).join(", ");
|
|
510
|
+
const rows = (yield table
|
|
511
|
+
.query()
|
|
512
|
+
.select(["id", ...columns])
|
|
513
|
+
.where(`(${where}) AND id IN (${list})`)
|
|
514
|
+
.limit(batch.length)
|
|
515
|
+
.toArray());
|
|
516
|
+
for (const row of rows)
|
|
517
|
+
out.set(String(row.id || ""), row);
|
|
518
|
+
}
|
|
519
|
+
return out;
|
|
520
|
+
});
|
|
521
|
+
}
|
|
522
|
+
/** Pair rows carry no vector: nothing downstream reads one, and findings cross the socket. */
|
|
523
|
+
function withoutVector(row) {
|
|
524
|
+
return row.vector === undefined ? row : Object.assign(Object.assign({}, row), { vector: undefined });
|
|
525
|
+
}
|
|
446
526
|
function buildFindings(pairs) {
|
|
447
527
|
var _a;
|
|
448
528
|
const groups = new Map();
|
|
@@ -497,16 +577,53 @@ function analyzeSurprisingConnections(table_1, projectRoot_1) {
|
|
|
497
577
|
const where = (0, scope_filter_1.buildScopeWhere)(scope);
|
|
498
578
|
const rawRows = (yield table
|
|
499
579
|
.query()
|
|
500
|
-
.select(
|
|
580
|
+
.select(SCAN_SELECT)
|
|
501
581
|
.where(where)
|
|
502
582
|
.limit(opts.maxRows)
|
|
503
583
|
.toArray());
|
|
504
584
|
const rows = rawRows.map((row) => toChunkRow(row, prefix));
|
|
505
|
-
const codeRows = rows.filter((row) => filterableCodeRow(row, opts));
|
|
506
585
|
const fileEdges = buildFileEdges(rows);
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
586
|
+
// Weak-code test: decided from the scan's bounds where possible, otherwise
|
|
587
|
+
// from content fetched for just the undecided rows.
|
|
588
|
+
const candidates = [];
|
|
589
|
+
for (let i = 0; i < rows.length; i++) {
|
|
590
|
+
if (!filterableCodeMeta(rows[i], opts))
|
|
591
|
+
continue;
|
|
592
|
+
candidates.push({ row: rows[i], verdict: contentVerdict(rawRows[i]) });
|
|
593
|
+
}
|
|
594
|
+
const undecidedContent = yield hydrateRows(table, where, candidates
|
|
595
|
+
.filter((c) => c.verdict === null && c.row.id)
|
|
596
|
+
.map((c) => c.row.id), ["content"]);
|
|
597
|
+
const codeRows = candidates
|
|
598
|
+
.filter(({ row, verdict }) => {
|
|
599
|
+
if (verdict !== null)
|
|
600
|
+
return verdict;
|
|
601
|
+
const hydrated = undecidedContent.get(row.id);
|
|
602
|
+
return hydrated ? hasCodeContent(String(hydrated.content || "")) : false;
|
|
603
|
+
})
|
|
604
|
+
.map(({ row }) => row);
|
|
605
|
+
// Anchors are the first `sample` code rows in hash order, and only they need
|
|
606
|
+
// content and a vector. Fetch those a page at a time, skipping any row that
|
|
607
|
+
// comes back without a vector (the column is non-nullable, so this is
|
|
608
|
+
// defensive) just as the old whole-scan filter did.
|
|
609
|
+
const ordered = [...codeRows].sort((a, b) => stableHash(rowKey(a)) - stableHash(rowKey(b)));
|
|
610
|
+
const anchors = [];
|
|
611
|
+
for (let i = 0; i < ordered.length && anchors.length < opts.sample; i += opts.sample) {
|
|
612
|
+
const page = ordered.slice(i, i + opts.sample);
|
|
613
|
+
const fetched = yield hydrateRows(table, where, page
|
|
614
|
+
.filter((row) => row.id && vectorLength(row.vector) === 0)
|
|
615
|
+
.map((row) => row.id), ["content", "vector"]);
|
|
616
|
+
for (const row of page) {
|
|
617
|
+
if (anchors.length >= opts.sample)
|
|
618
|
+
break;
|
|
619
|
+
const extra = fetched.get(row.id);
|
|
620
|
+
const anchor = extra
|
|
621
|
+
? Object.assign(Object.assign({}, row), { content: String(extra.content || ""), vector: extra.vector }) : row;
|
|
622
|
+
if (vectorLength(anchor.vector) === 0)
|
|
623
|
+
continue;
|
|
624
|
+
anchors.push(anchor);
|
|
625
|
+
}
|
|
626
|
+
}
|
|
510
627
|
const filters = {
|
|
511
628
|
rawNeighbors: 0,
|
|
512
629
|
sameChunk: 0,
|
|
@@ -520,15 +637,16 @@ function analyzeSurprisingConnections(table_1, projectRoot_1) {
|
|
|
520
637
|
belowThreshold: 0,
|
|
521
638
|
};
|
|
522
639
|
const pairs = new Map();
|
|
523
|
-
for (const
|
|
524
|
-
const
|
|
640
|
+
for (const anchor of anchors) {
|
|
641
|
+
const source = withoutVector(anchor);
|
|
642
|
+
const neighbors = (yield (0, ann_config_1.configureAnnVectorQuery)(table.vectorSearch(anchor.vector))
|
|
525
643
|
.select([...exports.SURPRISE_COLUMNS, "_distance"])
|
|
526
644
|
.where(where)
|
|
527
645
|
.limit(opts.neighbors + 8)
|
|
528
646
|
.toArray());
|
|
529
647
|
for (const rawTarget of neighbors) {
|
|
530
648
|
filters.rawNeighbors++;
|
|
531
|
-
const target = toChunkRow(rawTarget, prefix);
|
|
649
|
+
const target = withoutVector(toChunkRow(rawTarget, prefix));
|
|
532
650
|
if (rowKey(source) === rowKey(target)) {
|
|
533
651
|
filters.sameChunk++;
|
|
534
652
|
continue;
|
|
@@ -537,7 +655,9 @@ function analyzeSurprisingConnections(table_1, projectRoot_1) {
|
|
|
537
655
|
filters.sameFile++;
|
|
538
656
|
continue;
|
|
539
657
|
}
|
|
540
|
-
|
|
658
|
+
// A vector-search hit has a vector by construction, and `vector` is not
|
|
659
|
+
// selected, so only the path decides "non-code" here.
|
|
660
|
+
if (!isCodePath(target.path)) {
|
|
541
661
|
filters.nonCode++;
|
|
542
662
|
continue;
|
|
543
663
|
}
|
|
@@ -69,6 +69,7 @@ const log_rotate_1 = require("../utils/log-rotate");
|
|
|
69
69
|
const logger_1 = require("../utils/logger");
|
|
70
70
|
const operation_coordinator_1 = require("../utils/operation-coordinator");
|
|
71
71
|
const process_1 = require("../utils/process");
|
|
72
|
+
const process_footprint_1 = require("../utils/process-footprint");
|
|
72
73
|
const project_registry_1 = require("../utils/project-registry");
|
|
73
74
|
const root_availability_1 = require("../utils/root-availability");
|
|
74
75
|
const watcher_store_1 = require("../utils/watcher-store");
|
|
@@ -76,7 +77,11 @@ const pool_1 = require("../workers/pool");
|
|
|
76
77
|
const ipc_handler_1 = require("./ipc-handler");
|
|
77
78
|
const mlx_server_manager_1 = require("./mlx-server-manager");
|
|
78
79
|
const process_manager_1 = require("./process-manager");
|
|
80
|
+
const read_verbs_1 = require("./read-verbs");
|
|
81
|
+
const rows_handler_1 = require("./rows-handler");
|
|
79
82
|
const search_handler_1 = require("./search-handler");
|
|
83
|
+
const vector_handler_1 = require("./vector-handler");
|
|
84
|
+
const watch_leases_1 = require("./watch-leases");
|
|
80
85
|
const watcher_manager_1 = require("./watcher-manager");
|
|
81
86
|
// 30 min was too aggressive — every shutdown is a chance for races, FSEvents
|
|
82
87
|
// drops, and orphan MLX cleanup. 4 hours keeps the daemon resident through a
|
|
@@ -97,6 +102,9 @@ const IDLE_TIMEOUT_MS = (() => {
|
|
|
97
102
|
return parsed; // <= 0 disables the idle check below
|
|
98
103
|
})();
|
|
99
104
|
const HEARTBEAT_INTERVAL_MS = 60 * 1000;
|
|
105
|
+
// Escape hatch for the pre-lease behavior: watch every indexed project for the
|
|
106
|
+
// daemon's whole life. See WatchLeases for why that is no longer the default.
|
|
107
|
+
const WATCH_ALL = process.env.GMAX_WATCH_ALL === "1";
|
|
100
108
|
// Self-recycle. Under continuous load (a busy monorepo) the idle timeout never
|
|
101
109
|
// fires, so a long-lived daemon never gets a fresh start. The 24h age trigger
|
|
102
110
|
// is the primary hygiene mechanism. The RSS trigger is a backstop for a genuine
|
|
@@ -113,7 +121,11 @@ const envNum = (name, fallback) => {
|
|
|
113
121
|
return Number.isFinite(parsed) ? parsed : fallback;
|
|
114
122
|
};
|
|
115
123
|
const MAX_LIFETIME_MS = envNum("GMAX_DAEMON_MAX_LIFETIME_MS", 24 * 60 * 60 * 1000);
|
|
124
|
+
// Compared against the physical footprint (compressed + swapped included), not
|
|
125
|
+
// RSS — see readFootprintMb. The env name is kept for existing overrides.
|
|
116
126
|
const RSS_WATERMARK_MB = envNum("GMAX_DAEMON_RSS_WATERMARK_MB", 2560);
|
|
127
|
+
const FOOTPRINT_RECYCLE_MIN_AGE_MS = 30 * 60 * 1000;
|
|
128
|
+
const RECYCLE_FORCE_AFTER_MS = envNum("GMAX_DAEMON_RECYCLE_FORCE_AFTER_MS", 60 * 60 * 1000);
|
|
117
129
|
class Daemon {
|
|
118
130
|
constructor() {
|
|
119
131
|
this.processors = new Map();
|
|
@@ -138,6 +150,9 @@ class Daemon {
|
|
|
138
150
|
this.lastZoneWarnMs = 0;
|
|
139
151
|
this.shuttingDown = false;
|
|
140
152
|
this.recycling = false;
|
|
153
|
+
this.recycleDueSinceMs = null;
|
|
154
|
+
this.recycleReason = "";
|
|
155
|
+
this.lastRecycleDeferLogMs = 0;
|
|
141
156
|
// False until LanceDB + MetaCache are open. The socket starts listening early
|
|
142
157
|
// (so liveness probes succeed during slow init), so commands that need those
|
|
143
158
|
// resources must be gated on this to avoid hitting null stores mid-startup.
|
|
@@ -165,6 +180,8 @@ class Daemon {
|
|
|
165
180
|
runProjectOperation: (root, name, signal, fn) => this.withProjectLock(root, signal, () => this.runSharedOperation(name, signal, fn)),
|
|
166
181
|
});
|
|
167
182
|
this.projectMutex = new keyed_mutex_1.KeyedMutex();
|
|
183
|
+
// Tests construct daemons against the real HOME; keep their leases in memory.
|
|
184
|
+
this.watchLeases = new watch_leases_1.WatchLeases(process.env.VITEST ? null : config_1.PATHS.watchLeasesFile);
|
|
168
185
|
this.operations = new operation_coordinator_1.OperationCoordinator();
|
|
169
186
|
this.shutdownPromise = null;
|
|
170
187
|
// Full-index progress per root while initialSync runs (--reset / initial
|
|
@@ -220,6 +237,15 @@ class Daemon {
|
|
|
220
237
|
return __awaiter(this, void 0, void 0, function* () {
|
|
221
238
|
var _a;
|
|
222
239
|
process.title = "gmax-daemon";
|
|
240
|
+
// Read verbs register here rather than at read-verbs.ts module scope: the
|
|
241
|
+
// registry is asserted on exactly in tests/ipc-read-verbs.test.ts, and a
|
|
242
|
+
// module-scope registration would make it non-empty on import. Register
|
|
243
|
+
// before the socket server starts (step 2) so no client can race in and
|
|
244
|
+
// get "unknown command" for a verb this daemon does support. Idempotent:
|
|
245
|
+
// re-registering replaces by name.
|
|
246
|
+
(0, read_verbs_1.registerGraphVerbs)();
|
|
247
|
+
(0, rows_handler_1.registerRowsVerbs)();
|
|
248
|
+
(0, vector_handler_1.registerVectorVerbs)();
|
|
223
249
|
// 0. Singleton enforcement: find and kill ALL stale daemon/worker processes
|
|
224
250
|
yield this.processManager.killStaleProcesses();
|
|
225
251
|
this.assertStartupActive();
|
|
@@ -357,9 +383,13 @@ class Daemon {
|
|
|
357
383
|
this.publishResourceGeneration(globalConfig, this.activeGeneration, this.vectorDb, this.workerPool, this.mlxMode(globalConfig));
|
|
358
384
|
// 7. Register daemon (only after resources are open)
|
|
359
385
|
(0, watcher_store_1.registerDaemon)(process.pid);
|
|
360
|
-
// 8. Subscribe to
|
|
386
|
+
// 8. Subscribe to the registered projects some session still holds a lease
|
|
387
|
+
// on (skip missing directories). Everything else waits for a lease.
|
|
361
388
|
const allProjects = (0, project_registry_1.listProjects)();
|
|
362
|
-
|
|
389
|
+
this.watchLeases.load();
|
|
390
|
+
const indexed = allProjects.filter((p) => p.status === "indexed" && this.shouldWatch(p.root));
|
|
391
|
+
console.log(`[daemon] Watching ${indexed.length} of ${allProjects.length} registered projects` +
|
|
392
|
+
(WATCH_ALL ? " (GMAX_WATCH_ALL=1)" : " (leased by active sessions)"));
|
|
363
393
|
for (const p of indexed) {
|
|
364
394
|
if (!fs.existsSync(p.root)) {
|
|
365
395
|
// Skipping is already the safe behavior — the project keeps its registry
|
|
@@ -406,6 +436,7 @@ class Daemon {
|
|
|
406
436
|
}
|
|
407
437
|
catch (_a) { }
|
|
408
438
|
(0, log_rotate_1.rotateLogFds)(path.join(config_1.PATHS.logsDir, "daemon.log"));
|
|
439
|
+
this.sweepWatchLeases();
|
|
409
440
|
// Every 5 ticks (5 min), probe the MLX embed server and respawn if
|
|
410
441
|
// it's gone zombie (port held but /health unresponsive). Closes the
|
|
411
442
|
// 42h-degradation window where workers silently fell back to ONNX CPU
|
|
@@ -417,6 +448,9 @@ class Daemon {
|
|
|
417
448
|
this.maybeRecycle();
|
|
418
449
|
this.checkKernelZonePressure();
|
|
419
450
|
}
|
|
451
|
+
else if (this.recycleDueSinceMs !== null) {
|
|
452
|
+
this.maybeRecycle(false);
|
|
453
|
+
}
|
|
420
454
|
}, HEARTBEAT_INTERVAL_MS);
|
|
421
455
|
// 10. Idle timeout (skip when disabled via env)
|
|
422
456
|
if (IDLE_TIMEOUT_MS > 0) {
|
|
@@ -488,6 +522,10 @@ class Daemon {
|
|
|
488
522
|
return this.withProjectLock(root, signal, () => this.runSharedOperation("watch", signal, () => this.watchProjectWithinOperation(root)));
|
|
489
523
|
}
|
|
490
524
|
watchProjectWithinOperation(root) {
|
|
525
|
+
// Every path that re-watches after an index/rebuild/remove goes through
|
|
526
|
+
// here, so an unleased project stays unwatched after the operation too.
|
|
527
|
+
if (!this.shouldWatch(root))
|
|
528
|
+
return Promise.resolve();
|
|
491
529
|
const project = (0, project_registry_1.getProject)(root);
|
|
492
530
|
if ((project === null || project === void 0 ? void 0 : project.status) === "indexed" &&
|
|
493
531
|
this.activeGeneration &&
|
|
@@ -497,6 +535,62 @@ class Daemon {
|
|
|
497
535
|
}
|
|
498
536
|
return this.watcherManager.watchProject(root);
|
|
499
537
|
}
|
|
538
|
+
shouldWatch(root) {
|
|
539
|
+
return WATCH_ALL || this.watchLeases.isWanted(root);
|
|
540
|
+
}
|
|
541
|
+
/**
|
|
542
|
+
* Take (or renew) a lease on `root` and make sure it is watched. Called for
|
|
543
|
+
* every `watch` IPC — MCP servers renew theirs every few minutes, so the
|
|
544
|
+
* common case is a lease bump on an already-watched project.
|
|
545
|
+
*/
|
|
546
|
+
requestWatch(root, lease = {}, signal) {
|
|
547
|
+
var _a;
|
|
548
|
+
this.watchLeases.acquire(root, {
|
|
549
|
+
holder: lease.holder || "cli",
|
|
550
|
+
pid: lease.pid,
|
|
551
|
+
ttlMs: (_a = lease.ttlMs) !== null && _a !== void 0 ? _a : watch_leases_1.DEFAULT_LEASE_TTL_MS,
|
|
552
|
+
});
|
|
553
|
+
if (this.processors.has(root))
|
|
554
|
+
return Promise.resolve();
|
|
555
|
+
return this.watchProject(root, signal);
|
|
556
|
+
}
|
|
557
|
+
/**
|
|
558
|
+
* Drop a lease — every lease on the root when `holder` is omitted, which is
|
|
559
|
+
* what an explicit `unwatch` means — and unwatch once nothing wants it.
|
|
560
|
+
*/
|
|
561
|
+
releaseWatch(root, holder, signal) {
|
|
562
|
+
this.watchLeases.release(root, holder);
|
|
563
|
+
if (this.shouldWatch(root) || !this.processors.has(root)) {
|
|
564
|
+
return Promise.resolve();
|
|
565
|
+
}
|
|
566
|
+
return this.unwatchProject(root, signal);
|
|
567
|
+
}
|
|
568
|
+
/**
|
|
569
|
+
* A short lease for a CLI add/index: the project stays watched after the
|
|
570
|
+
* operation re-watches it, and lapses unless a session picks it up.
|
|
571
|
+
*/
|
|
572
|
+
leaseForCli(root) {
|
|
573
|
+
this.watchLeases.acquire(root, {
|
|
574
|
+
holder: "cli",
|
|
575
|
+
ttlMs: watch_leases_1.DEFAULT_LEASE_TTL_MS,
|
|
576
|
+
});
|
|
577
|
+
}
|
|
578
|
+
listWatchLeases() {
|
|
579
|
+
return this.watchLeases.list();
|
|
580
|
+
}
|
|
581
|
+
/** Unwatch projects whose last lease expired or whose holder exited. */
|
|
582
|
+
sweepWatchLeases() {
|
|
583
|
+
if (this.shuttingDown || WATCH_ALL)
|
|
584
|
+
return;
|
|
585
|
+
for (const root of this.watchLeases.sweep()) {
|
|
586
|
+
if (!this.processors.has(root) || this.shouldWatch(root))
|
|
587
|
+
continue;
|
|
588
|
+
console.log(`[daemon] No session holds ${path.basename(root)} any more — unwatching`);
|
|
589
|
+
void this.unwatchProject(root).catch((err) => {
|
|
590
|
+
console.error(`[daemon] Failed to unwatch ${path.basename(root)}:`, err);
|
|
591
|
+
});
|
|
592
|
+
}
|
|
593
|
+
}
|
|
500
594
|
runSharedOperation(name, signal, fn) {
|
|
501
595
|
return this.operations.runShared(name, signal, fn);
|
|
502
596
|
}
|
|
@@ -724,6 +818,20 @@ class Daemon {
|
|
|
724
818
|
}));
|
|
725
819
|
});
|
|
726
820
|
}
|
|
821
|
+
/**
|
|
822
|
+
* The slice of the daemon a read verb handler needs: the warm store and the
|
|
823
|
+
* activity touch that keeps the idle timer honest. One accessor for every
|
|
824
|
+
* read verb (graph, rows, vector) — extend this rather than adding a second;
|
|
825
|
+
* handlers get no other access to daemon internals.
|
|
826
|
+
*/
|
|
827
|
+
storeReadDeps() {
|
|
828
|
+
return {
|
|
829
|
+
vectorDb: this.vectorDb,
|
|
830
|
+
touchActivity: () => {
|
|
831
|
+
this.lastActivity = Date.now();
|
|
832
|
+
},
|
|
833
|
+
};
|
|
834
|
+
}
|
|
727
835
|
listProjects() {
|
|
728
836
|
return [...this.processors.keys()].map((root) => ({
|
|
729
837
|
root,
|
|
@@ -1546,6 +1654,7 @@ class Daemon {
|
|
|
1546
1654
|
const keys = yield this.metaCache.getKeysWithPrefix(rootPrefix);
|
|
1547
1655
|
for (const key of keys)
|
|
1548
1656
|
this.metaCache.delete(key);
|
|
1657
|
+
this.watchLeases.release(root);
|
|
1549
1658
|
(0, ipc_handler_1.writeDone)(conn, { ok: true });
|
|
1550
1659
|
}
|
|
1551
1660
|
catch (err) {
|
|
@@ -1693,26 +1802,61 @@ class Daemon {
|
|
|
1693
1802
|
* too large. Only fires when quiet — no active compaction and no in-flight
|
|
1694
1803
|
* project operations — so a recycle never interrupts indexing work. The
|
|
1695
1804
|
* successor re-runs catchup on startup, so nothing is lost.
|
|
1805
|
+
*
|
|
1806
|
+
* "Quiet" alone was not enough: with a dozen watched projects some batch or
|
|
1807
|
+
* catchup was in flight at nearly every 5-minute probe, and one daemon ran for
|
|
1808
|
+
* 8 days past its 24h limit while its footprint sat at 4 GB. So once a recycle
|
|
1809
|
+
* is due it is re-checked every heartbeat to catch a quiet minute, and after
|
|
1810
|
+
* RECYCLE_FORCE_AFTER_MS of deferral it proceeds over in-flight project work
|
|
1811
|
+
* (shutdown aborts it; the successor's catchup redoes it). Compaction is still
|
|
1812
|
+
* waited out — killing it strands scratch files and wastes the rewrite.
|
|
1813
|
+
*
|
|
1814
|
+
* @param sampleFootprint false on the in-between ticks: `footprint` costs ~50 ms,
|
|
1815
|
+
* so it is only sampled on the 5-minute probe.
|
|
1696
1816
|
*/
|
|
1697
|
-
maybeRecycle() {
|
|
1817
|
+
maybeRecycle(sampleFootprint = true) {
|
|
1698
1818
|
var _a;
|
|
1699
1819
|
if (this.shuttingDown || this.recycling)
|
|
1700
1820
|
return;
|
|
1701
1821
|
const ageMs = process.uptime() * 1000;
|
|
1702
|
-
const rssMb = process.memoryUsage().rss / (1024 * 1024);
|
|
1703
1822
|
const ageExceeded = MAX_LIFETIME_MS > 0 && ageMs > MAX_LIFETIME_MS;
|
|
1704
|
-
|
|
1705
|
-
if (
|
|
1706
|
-
|
|
1707
|
-
|
|
1823
|
+
let footprintMb = null;
|
|
1824
|
+
if (this.recycleDueSinceMs === null &&
|
|
1825
|
+
!ageExceeded &&
|
|
1826
|
+
sampleFootprint &&
|
|
1827
|
+
RSS_WATERMARK_MB > 0 &&
|
|
1828
|
+
// A fresh daemon above the watermark would recycle in a loop; give the
|
|
1829
|
+
// caches a chance to settle before judging the footprint.
|
|
1830
|
+
ageMs > FOOTPRINT_RECYCLE_MIN_AGE_MS) {
|
|
1831
|
+
footprintMb = (0, process_footprint_1.readFootprintMb)();
|
|
1832
|
+
}
|
|
1833
|
+
const footprintExceeded = footprintMb !== null && footprintMb > RSS_WATERMARK_MB;
|
|
1834
|
+
if (this.recycleDueSinceMs === null) {
|
|
1835
|
+
if (!ageExceeded && !footprintExceeded)
|
|
1836
|
+
return;
|
|
1837
|
+
this.recycleDueSinceMs = Date.now();
|
|
1838
|
+
this.recycleReason = ageExceeded
|
|
1839
|
+
? `age ${(ageMs / 3600000).toFixed(1)}h > ${(MAX_LIFETIME_MS / 3600000).toFixed(1)}h`
|
|
1840
|
+
: `footprint ${Math.round(footprintMb !== null && footprintMb !== void 0 ? footprintMb : 0)}MB > ${RSS_WATERMARK_MB}MB`;
|
|
1841
|
+
}
|
|
1842
|
+
// Defer while busy; re-checked on every heartbeat until it goes through.
|
|
1708
1843
|
if ((_a = this.vectorDb) === null || _a === void 0 ? void 0 : _a.isMaintenanceActive())
|
|
1709
1844
|
return;
|
|
1710
|
-
|
|
1845
|
+
const busy = this.projectMutex.pending > 0 || this.operations.activeCount > 0;
|
|
1846
|
+
const deferredMs = Date.now() - this.recycleDueSinceMs;
|
|
1847
|
+
if (busy && deferredMs < RECYCLE_FORCE_AFTER_MS) {
|
|
1848
|
+
if (Date.now() - this.lastRecycleDeferLogMs >= 30 * 60 * 1000) {
|
|
1849
|
+
this.lastRecycleDeferLogMs = Date.now();
|
|
1850
|
+
console.log(`[daemon] Recycle due (${this.recycleReason}) — deferred ${Math.round(deferredMs / 60000)}min, busy: ` +
|
|
1851
|
+
[
|
|
1852
|
+
...this.operations.activeOperationNames(),
|
|
1853
|
+
...this.projectMutex.pendingKeys().map((k) => path.basename(k)),
|
|
1854
|
+
].join(", "));
|
|
1855
|
+
}
|
|
1711
1856
|
return;
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
console.log(`[daemon] Recycling (${reason}) — handing off to a fresh daemon`);
|
|
1857
|
+
}
|
|
1858
|
+
const forced = busy ? " (forced over in-flight work)" : "";
|
|
1859
|
+
console.log(`[daemon] Recycling (${this.recycleReason})${forced} — handing off to a fresh daemon`);
|
|
1716
1860
|
this.recycling = true;
|
|
1717
1861
|
void this.shutdown({ relaunch: true }).finally(() => process.exit(0));
|
|
1718
1862
|
}
|