grepmax 0.26.28 → 0.26.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/dist/commands/mcp.js +17 -1
- package/dist/commands/watch.js +1 -1
- package/dist/config.js +8 -2
- package/dist/lib/analysis/surprising-connections.js +135 -15
- package/dist/lib/daemon/daemon.js +131 -13
- package/dist/lib/daemon/graph-handler.js +12 -6
- package/dist/lib/daemon/ipc-handler.js +18 -3
- package/dist/lib/daemon/read-verbs.js +52 -1
- package/dist/lib/daemon/search-handler.js +7 -4
- package/dist/lib/daemon/watch-leases.js +179 -0
- package/dist/lib/index/batch-processor.js +7 -1
- package/dist/lib/index/watcher.js +4 -0
- package/dist/lib/search/pagerank.js +80 -19
- package/dist/lib/store/vector-db.js +35 -1
- package/dist/lib/utils/async-semaphore.js +141 -0
- package/dist/lib/utils/process-footprint.js +51 -0
- package/dist/lib/utils/watcher-launcher.js +36 -19
- package/dist/lib/workers/pool.js +72 -2
- package/package.json +1 -1
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
- package/plugins/grepmax/hooks/cwd-changed.js +19 -36
- package/plugins/grepmax/hooks/start.js +15 -19
- package/plugins/grepmax/hooks/stop.js +13 -4
- package/plugins/grepmax/hooks/watch-lease.js +163 -0
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"name": "grepmax",
|
|
11
11
|
"source": "./plugins/grepmax",
|
|
12
12
|
"description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
|
|
13
|
-
"version": "0.26.
|
|
13
|
+
"version": "0.26.30",
|
|
14
14
|
"author": {
|
|
15
15
|
"name": "Robert Owens",
|
|
16
16
|
"email": "robowens@me.com"
|
package/dist/commands/mcp.js
CHANGED
|
@@ -151,6 +151,12 @@ function isMcpSessionFallback(error) {
|
|
|
151
151
|
* a live daemon saying no, and is reported. Falling back there would put a
|
|
152
152
|
* second opener on the store while the daemon is working.
|
|
153
153
|
*/
|
|
154
|
+
/**
|
|
155
|
+
* Watch-lease cadence. Renewal is cheap (one IPC line), and the TTL is three
|
|
156
|
+
* renewals long so a slow tick or a daemon restart does not drop the project.
|
|
157
|
+
*/
|
|
158
|
+
const MCP_WATCH_LEASE_RENEW_MS = 5 * 60 * 1000;
|
|
159
|
+
const MCP_WATCH_LEASE_TTL_MS = 3 * MCP_WATCH_LEASE_RENEW_MS;
|
|
154
160
|
const MCP_STORE_FALLBACK = {
|
|
155
161
|
fallbackOnUnknownVerb: true,
|
|
156
162
|
fallbackOnOversize: true,
|
|
@@ -573,6 +579,13 @@ exports.mcp = new commander_1.Command("mcp")
|
|
|
573
579
|
});
|
|
574
580
|
}
|
|
575
581
|
// --- Background watcher ---
|
|
582
|
+
// This server's watch lease. It names our PID, so the daemon drops it the
|
|
583
|
+
// minute this session exits; the TTL only covers a wedged server.
|
|
584
|
+
const WATCH_LEASE = {
|
|
585
|
+
holder: `mcp:${process.pid}`,
|
|
586
|
+
pid: process.pid,
|
|
587
|
+
ttlMs: MCP_WATCH_LEASE_TTL_MS,
|
|
588
|
+
};
|
|
576
589
|
function ensureWatcher(root) {
|
|
577
590
|
return __awaiter(this, void 0, void 0, function* () {
|
|
578
591
|
var _a;
|
|
@@ -580,7 +593,7 @@ exports.mcp = new commander_1.Command("mcp")
|
|
|
580
593
|
const watchRoot = root !== null && root !== void 0 ? root : (_a = resolveMcpProject(undefined, projectRoot)) === null || _a === void 0 ? void 0 : _a.root;
|
|
581
594
|
if (!watchRoot)
|
|
582
595
|
return;
|
|
583
|
-
const result = yield (0, watcher_launcher_1.launchWatcher)(watchRoot);
|
|
596
|
+
const result = yield (0, watcher_launcher_1.launchWatcher)(watchRoot, WATCH_LEASE);
|
|
584
597
|
if (result.ok && !result.reused) {
|
|
585
598
|
console.log(`[MCP] Started background watcher for ${watchRoot} (PID: ${result.pid})`);
|
|
586
599
|
}
|
|
@@ -2965,4 +2978,7 @@ exports.mcp = new commander_1.Command("mcp")
|
|
|
2965
2978
|
// Kick off index readiness check and watcher in background
|
|
2966
2979
|
ensureIndexReady().catch((e) => console.error("[MCP] Index readiness check failed:", e));
|
|
2967
2980
|
ensureWatcher();
|
|
2981
|
+
// Keep this session's project leased while the session lives. The daemon
|
|
2982
|
+
// only watches projects some session holds.
|
|
2983
|
+
setInterval(() => void ensureWatcher(), MCP_WATCH_LEASE_RENEW_MS).unref();
|
|
2968
2984
|
}));
|
package/dist/commands/watch.js
CHANGED
|
@@ -67,7 +67,7 @@ const IDLE_CHECK_INTERVAL_MS = 60 * 1000; // check every minute
|
|
|
67
67
|
exports.watch = new commander_1.Command("watch")
|
|
68
68
|
.description("Start background file watcher for live reindexing")
|
|
69
69
|
.option("-b, --background", "Run watcher in background and exit")
|
|
70
|
-
.option("-d, --daemon", "Run as centralized daemon watching
|
|
70
|
+
.option("-d, --daemon", "Run as centralized daemon watching projects that active sessions hold")
|
|
71
71
|
.option("-p, --path <dir>", "Directory to watch (defaults to project root)")
|
|
72
72
|
.option("--no-idle-timeout", "Disable the 30-minute idle shutdown")
|
|
73
73
|
.action((options) => __awaiter(void 0, void 0, void 0, function* () {
|
package/dist/config.js
CHANGED
|
@@ -68,9 +68,14 @@ const DEFAULT_WORKER_THREADS = (() => {
|
|
|
68
68
|
const fromEnv = Number.parseInt((_a = process.env.GMAX_WORKER_THREADS) !== null && _a !== void 0 ? _a : "", 10);
|
|
69
69
|
if (Number.isFinite(fromEnv) && fromEnv > 0)
|
|
70
70
|
return fromEnv;
|
|
71
|
+
// Each worker is a separate process that loads the ONNX models, peaking at
|
|
72
|
+
// ~800 MB-1 GB. The old max(4, cores/2) let a 14-core host run 7 of them;
|
|
73
|
+
// together with the daemon that froze a 48 GB machine. Four is the ceiling;
|
|
74
|
+
// two is the floor where the cores exist, so one worker can stay free for
|
|
75
|
+
// search. GMAX_WORKER_THREADS still overrides.
|
|
71
76
|
const cores = os.cpus().length || 1;
|
|
72
|
-
const
|
|
73
|
-
return Math.max(1, Math.min(
|
|
77
|
+
const target = Math.min(4, Math.max(2, Math.floor(cores * 0.5)));
|
|
78
|
+
return Math.max(1, Math.min(target, cores));
|
|
74
79
|
})();
|
|
75
80
|
exports.CONFIG = {
|
|
76
81
|
VECTOR_DIM: 384,
|
|
@@ -224,6 +229,7 @@ exports.PATHS = {
|
|
|
224
229
|
daemonSocket: path.join(GLOBAL_ROOT, "daemon.sock"),
|
|
225
230
|
daemonPidFile: path.join(GLOBAL_ROOT, "daemon.pid"),
|
|
226
231
|
daemonLockFile: path.join(GLOBAL_ROOT, "daemon.lock"),
|
|
232
|
+
watchLeasesFile: path.join(GLOBAL_ROOT, "watch-leases.json"),
|
|
227
233
|
// Written by a daemon while it is gracefully shutting down (draining workers,
|
|
228
234
|
// closing LanceDB). A successor's killStaleProcesses() respects this so it
|
|
229
235
|
// never SIGKILLs a peer mid-cleanup once the peer has already dropped its
|
|
@@ -62,6 +62,7 @@ const path = __importStar(require("node:path"));
|
|
|
62
62
|
const callsites_1 = require("../graph/callsites");
|
|
63
63
|
const ann_config_1 = require("../store/ann-config");
|
|
64
64
|
const arrow_1 = require("../utils/arrow");
|
|
65
|
+
const filter_builder_1 = require("../utils/filter-builder");
|
|
65
66
|
const scope_filter_1 = require("../utils/scope-filter");
|
|
66
67
|
exports.DEFAULT_SURPRISE_OPTIONS = {
|
|
67
68
|
sample: 160,
|
|
@@ -73,6 +74,11 @@ exports.DEFAULT_SURPRISE_OPTIONS = {
|
|
|
73
74
|
includeEval: false,
|
|
74
75
|
};
|
|
75
76
|
exports.MAX_SURPRISE_ROWS = 100000;
|
|
77
|
+
/**
|
|
78
|
+
* Columns a pair row carries: the scan's metadata plus `content`, which the
|
|
79
|
+
* weak-code filter and the scorer read. `vector` is not here — only sampled
|
|
80
|
+
* anchors need one, and they fetch it by id (see `hydrateRows`).
|
|
81
|
+
*/
|
|
76
82
|
exports.SURPRISE_COLUMNS = [
|
|
77
83
|
"id",
|
|
78
84
|
"path",
|
|
@@ -83,8 +89,40 @@ exports.SURPRISE_COLUMNS = [
|
|
|
83
89
|
"type_referenced_symbols",
|
|
84
90
|
"role",
|
|
85
91
|
"content",
|
|
86
|
-
"vector",
|
|
87
92
|
];
|
|
93
|
+
const MIN_CODE_CONTENT_LENGTH = 80;
|
|
94
|
+
/**
|
|
95
|
+
* The row scan reads up to MAX_SURPRISE_ROWS rows. Selecting `content` and
|
|
96
|
+
* `vector` there held every chunk body and embedding in memory for an answer
|
|
97
|
+
* that needs ~160 vectors. It now selects metadata plus two bounds on the
|
|
98
|
+
* weak-code test (`content.trim().length >= 80`), computed in the store:
|
|
99
|
+
*
|
|
100
|
+
* - `content_min_len` trims a *superset* of JS whitespace (Unicode White_Space
|
|
101
|
+
* plus U+FEFF) and counts code points, so it never exceeds the JS length —
|
|
102
|
+
* at or above the threshold, the row passes.
|
|
103
|
+
* - `content_max_len` trims a *subset* (ASCII whitespace) and counts UTF-8
|
|
104
|
+
* bytes, so it is never below the JS length — under the threshold, it fails.
|
|
105
|
+
*
|
|
106
|
+
* Anything between the two (non-ASCII text or whitespace near the threshold)
|
|
107
|
+
* fetches its content by id and takes the exact JS test, so `codeRows` and the
|
|
108
|
+
* anchor sample are unchanged. Backslashes are literal in DataFusion strings.
|
|
109
|
+
*/
|
|
110
|
+
const CONTENT_MIN_LEN_EXPR = "char_length(regexp_replace(content, '^[\\s\\x{FEFF}]+|[\\s\\x{FEFF}]+$', '', 'g'))";
|
|
111
|
+
const CONTENT_MAX_LEN_EXPR = "octet_length(regexp_replace(content, '^[\\t\\n\\x0B\\f\\r ]+|[\\t\\n\\x0B\\f\\r ]+$', '', 'g'))";
|
|
112
|
+
const SCAN_SELECT = {
|
|
113
|
+
id: "id",
|
|
114
|
+
path: "path",
|
|
115
|
+
start_line: "start_line",
|
|
116
|
+
end_line: "end_line",
|
|
117
|
+
defined_symbols: "defined_symbols",
|
|
118
|
+
referenced_symbols: "referenced_symbols",
|
|
119
|
+
type_referenced_symbols: "type_referenced_symbols",
|
|
120
|
+
role: "role",
|
|
121
|
+
content_min_len: CONTENT_MIN_LEN_EXPR,
|
|
122
|
+
content_max_len: CONTENT_MAX_LEN_EXPR,
|
|
123
|
+
};
|
|
124
|
+
/** Ids per `id IN (...)` hydration query. */
|
|
125
|
+
const HYDRATE_BATCH = 200;
|
|
88
126
|
const CODE_EXTENSIONS = new Set([
|
|
89
127
|
".ts",
|
|
90
128
|
".tsx",
|
|
@@ -430,19 +468,61 @@ function buildFileEdges(rows) {
|
|
|
430
468
|
function hasDirectFileEdge(edges, a, b) {
|
|
431
469
|
return edges.has(`${a.path}\0${b.path}`) || edges.has(`${b.path}\0${a.path}`);
|
|
432
470
|
}
|
|
433
|
-
|
|
471
|
+
/** The anchor filter minus its content and vector tests (see below). */
|
|
472
|
+
function filterableCodeMeta(row, opts) {
|
|
434
473
|
if (!row.path || !isCodePath(row.path))
|
|
435
474
|
return false;
|
|
436
475
|
if (!opts.includeTests && isTestPath(row.relPath))
|
|
437
476
|
return false;
|
|
438
477
|
if (!opts.includeEval && isEvalPath(row.relPath))
|
|
439
478
|
return false;
|
|
440
|
-
if (vectorLength(row.vector) === 0)
|
|
441
|
-
return false;
|
|
442
|
-
if (row.content.trim().length < 80)
|
|
443
|
-
return false;
|
|
444
479
|
return row.definedSymbols.length > 0;
|
|
445
480
|
}
|
|
481
|
+
function hasCodeContent(content) {
|
|
482
|
+
return content.trim().length >= MIN_CODE_CONTENT_LENGTH;
|
|
483
|
+
}
|
|
484
|
+
/**
|
|
485
|
+
* The weak-code verdict from the scan alone: true/false when the store-side
|
|
486
|
+
* bounds decide it, null when the row needs its content fetched. A row that
|
|
487
|
+
* already carries `content` (a caller-supplied table) is decided exactly.
|
|
488
|
+
*/
|
|
489
|
+
function contentVerdict(raw) {
|
|
490
|
+
if (typeof raw.content === "string")
|
|
491
|
+
return hasCodeContent(raw.content);
|
|
492
|
+
const min = Number(raw.content_min_len);
|
|
493
|
+
const max = Number(raw.content_max_len);
|
|
494
|
+
if (Number.isFinite(min) && min >= MIN_CODE_CONTENT_LENGTH)
|
|
495
|
+
return true;
|
|
496
|
+
if (Number.isFinite(max) && max < MIN_CODE_CONTENT_LENGTH)
|
|
497
|
+
return false;
|
|
498
|
+
return null;
|
|
499
|
+
}
|
|
500
|
+
/**
|
|
501
|
+
* Fetch `columns` for the given ids, keyed by id. The scope clause is ANDed
|
|
502
|
+
* on so a hydration can never reach outside the scan that produced the ids.
|
|
503
|
+
*/
|
|
504
|
+
function hydrateRows(table, where, ids, columns) {
|
|
505
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
506
|
+
const out = new Map();
|
|
507
|
+
for (let i = 0; i < ids.length; i += HYDRATE_BATCH) {
|
|
508
|
+
const batch = ids.slice(i, i + HYDRATE_BATCH);
|
|
509
|
+
const list = batch.map((id) => `'${(0, filter_builder_1.escapeSqlString)(id)}'`).join(", ");
|
|
510
|
+
const rows = (yield table
|
|
511
|
+
.query()
|
|
512
|
+
.select(["id", ...columns])
|
|
513
|
+
.where(`(${where}) AND id IN (${list})`)
|
|
514
|
+
.limit(batch.length)
|
|
515
|
+
.toArray());
|
|
516
|
+
for (const row of rows)
|
|
517
|
+
out.set(String(row.id || ""), row);
|
|
518
|
+
}
|
|
519
|
+
return out;
|
|
520
|
+
});
|
|
521
|
+
}
|
|
522
|
+
/** Pair rows carry no vector: nothing downstream reads one, and findings cross the socket. */
|
|
523
|
+
function withoutVector(row) {
|
|
524
|
+
return row.vector === undefined ? row : Object.assign(Object.assign({}, row), { vector: undefined });
|
|
525
|
+
}
|
|
446
526
|
function buildFindings(pairs) {
|
|
447
527
|
var _a;
|
|
448
528
|
const groups = new Map();
|
|
@@ -497,16 +577,53 @@ function analyzeSurprisingConnections(table_1, projectRoot_1) {
|
|
|
497
577
|
const where = (0, scope_filter_1.buildScopeWhere)(scope);
|
|
498
578
|
const rawRows = (yield table
|
|
499
579
|
.query()
|
|
500
|
-
.select(
|
|
580
|
+
.select(SCAN_SELECT)
|
|
501
581
|
.where(where)
|
|
502
582
|
.limit(opts.maxRows)
|
|
503
583
|
.toArray());
|
|
504
584
|
const rows = rawRows.map((row) => toChunkRow(row, prefix));
|
|
505
|
-
const codeRows = rows.filter((row) => filterableCodeRow(row, opts));
|
|
506
585
|
const fileEdges = buildFileEdges(rows);
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
586
|
+
// Weak-code test: decided from the scan's bounds where possible, otherwise
|
|
587
|
+
// from content fetched for just the undecided rows.
|
|
588
|
+
const candidates = [];
|
|
589
|
+
for (let i = 0; i < rows.length; i++) {
|
|
590
|
+
if (!filterableCodeMeta(rows[i], opts))
|
|
591
|
+
continue;
|
|
592
|
+
candidates.push({ row: rows[i], verdict: contentVerdict(rawRows[i]) });
|
|
593
|
+
}
|
|
594
|
+
const undecidedContent = yield hydrateRows(table, where, candidates
|
|
595
|
+
.filter((c) => c.verdict === null && c.row.id)
|
|
596
|
+
.map((c) => c.row.id), ["content"]);
|
|
597
|
+
const codeRows = candidates
|
|
598
|
+
.filter(({ row, verdict }) => {
|
|
599
|
+
if (verdict !== null)
|
|
600
|
+
return verdict;
|
|
601
|
+
const hydrated = undecidedContent.get(row.id);
|
|
602
|
+
return hydrated ? hasCodeContent(String(hydrated.content || "")) : false;
|
|
603
|
+
})
|
|
604
|
+
.map(({ row }) => row);
|
|
605
|
+
// Anchors are the first `sample` code rows in hash order, and only they need
|
|
606
|
+
// content and a vector. Fetch those a page at a time, skipping any row that
|
|
607
|
+
// comes back without a vector (the column is non-nullable, so this is
|
|
608
|
+
// defensive) just as the old whole-scan filter did.
|
|
609
|
+
const ordered = [...codeRows].sort((a, b) => stableHash(rowKey(a)) - stableHash(rowKey(b)));
|
|
610
|
+
const anchors = [];
|
|
611
|
+
for (let i = 0; i < ordered.length && anchors.length < opts.sample; i += opts.sample) {
|
|
612
|
+
const page = ordered.slice(i, i + opts.sample);
|
|
613
|
+
const fetched = yield hydrateRows(table, where, page
|
|
614
|
+
.filter((row) => row.id && vectorLength(row.vector) === 0)
|
|
615
|
+
.map((row) => row.id), ["content", "vector"]);
|
|
616
|
+
for (const row of page) {
|
|
617
|
+
if (anchors.length >= opts.sample)
|
|
618
|
+
break;
|
|
619
|
+
const extra = fetched.get(row.id);
|
|
620
|
+
const anchor = extra
|
|
621
|
+
? Object.assign(Object.assign({}, row), { content: String(extra.content || ""), vector: extra.vector }) : row;
|
|
622
|
+
if (vectorLength(anchor.vector) === 0)
|
|
623
|
+
continue;
|
|
624
|
+
anchors.push(anchor);
|
|
625
|
+
}
|
|
626
|
+
}
|
|
510
627
|
const filters = {
|
|
511
628
|
rawNeighbors: 0,
|
|
512
629
|
sameChunk: 0,
|
|
@@ -520,15 +637,16 @@ function analyzeSurprisingConnections(table_1, projectRoot_1) {
|
|
|
520
637
|
belowThreshold: 0,
|
|
521
638
|
};
|
|
522
639
|
const pairs = new Map();
|
|
523
|
-
for (const
|
|
524
|
-
const
|
|
640
|
+
for (const anchor of anchors) {
|
|
641
|
+
const source = withoutVector(anchor);
|
|
642
|
+
const neighbors = (yield (0, ann_config_1.configureAnnVectorQuery)(table.vectorSearch(anchor.vector))
|
|
525
643
|
.select([...exports.SURPRISE_COLUMNS, "_distance"])
|
|
526
644
|
.where(where)
|
|
527
645
|
.limit(opts.neighbors + 8)
|
|
528
646
|
.toArray());
|
|
529
647
|
for (const rawTarget of neighbors) {
|
|
530
648
|
filters.rawNeighbors++;
|
|
531
|
-
const target = toChunkRow(rawTarget, prefix);
|
|
649
|
+
const target = withoutVector(toChunkRow(rawTarget, prefix));
|
|
532
650
|
if (rowKey(source) === rowKey(target)) {
|
|
533
651
|
filters.sameChunk++;
|
|
534
652
|
continue;
|
|
@@ -537,7 +655,9 @@ function analyzeSurprisingConnections(table_1, projectRoot_1) {
|
|
|
537
655
|
filters.sameFile++;
|
|
538
656
|
continue;
|
|
539
657
|
}
|
|
540
|
-
|
|
658
|
+
// A vector-search hit has a vector by construction, and `vector` is not
|
|
659
|
+
// selected, so only the path decides "non-code" here.
|
|
660
|
+
if (!isCodePath(target.path)) {
|
|
541
661
|
filters.nonCode++;
|
|
542
662
|
continue;
|
|
543
663
|
}
|
|
@@ -69,6 +69,7 @@ const log_rotate_1 = require("../utils/log-rotate");
|
|
|
69
69
|
const logger_1 = require("../utils/logger");
|
|
70
70
|
const operation_coordinator_1 = require("../utils/operation-coordinator");
|
|
71
71
|
const process_1 = require("../utils/process");
|
|
72
|
+
const process_footprint_1 = require("../utils/process-footprint");
|
|
72
73
|
const project_registry_1 = require("../utils/project-registry");
|
|
73
74
|
const root_availability_1 = require("../utils/root-availability");
|
|
74
75
|
const watcher_store_1 = require("../utils/watcher-store");
|
|
@@ -80,6 +81,7 @@ const read_verbs_1 = require("./read-verbs");
|
|
|
80
81
|
const rows_handler_1 = require("./rows-handler");
|
|
81
82
|
const search_handler_1 = require("./search-handler");
|
|
82
83
|
const vector_handler_1 = require("./vector-handler");
|
|
84
|
+
const watch_leases_1 = require("./watch-leases");
|
|
83
85
|
const watcher_manager_1 = require("./watcher-manager");
|
|
84
86
|
// 30 min was too aggressive — every shutdown is a chance for races, FSEvents
|
|
85
87
|
// drops, and orphan MLX cleanup. 4 hours keeps the daemon resident through a
|
|
@@ -100,6 +102,9 @@ const IDLE_TIMEOUT_MS = (() => {
|
|
|
100
102
|
return parsed; // <= 0 disables the idle check below
|
|
101
103
|
})();
|
|
102
104
|
const HEARTBEAT_INTERVAL_MS = 60 * 1000;
|
|
105
|
+
// Escape hatch for the pre-lease behavior: watch every indexed project for the
|
|
106
|
+
// daemon's whole life. See WatchLeases for why that is no longer the default.
|
|
107
|
+
const WATCH_ALL = process.env.GMAX_WATCH_ALL === "1";
|
|
103
108
|
// Self-recycle. Under continuous load (a busy monorepo) the idle timeout never
|
|
104
109
|
// fires, so a long-lived daemon never gets a fresh start. The 24h age trigger
|
|
105
110
|
// is the primary hygiene mechanism. The RSS trigger is a backstop for a genuine
|
|
@@ -116,7 +121,11 @@ const envNum = (name, fallback) => {
|
|
|
116
121
|
return Number.isFinite(parsed) ? parsed : fallback;
|
|
117
122
|
};
|
|
118
123
|
const MAX_LIFETIME_MS = envNum("GMAX_DAEMON_MAX_LIFETIME_MS", 24 * 60 * 60 * 1000);
|
|
124
|
+
// Compared against the physical footprint (compressed + swapped included), not
|
|
125
|
+
// RSS — see readFootprintMb. The env name is kept for existing overrides.
|
|
119
126
|
const RSS_WATERMARK_MB = envNum("GMAX_DAEMON_RSS_WATERMARK_MB", 2560);
|
|
127
|
+
const FOOTPRINT_RECYCLE_MIN_AGE_MS = 30 * 60 * 1000;
|
|
128
|
+
const RECYCLE_FORCE_AFTER_MS = envNum("GMAX_DAEMON_RECYCLE_FORCE_AFTER_MS", 60 * 60 * 1000);
|
|
120
129
|
class Daemon {
|
|
121
130
|
constructor() {
|
|
122
131
|
this.processors = new Map();
|
|
@@ -141,6 +150,9 @@ class Daemon {
|
|
|
141
150
|
this.lastZoneWarnMs = 0;
|
|
142
151
|
this.shuttingDown = false;
|
|
143
152
|
this.recycling = false;
|
|
153
|
+
this.recycleDueSinceMs = null;
|
|
154
|
+
this.recycleReason = "";
|
|
155
|
+
this.lastRecycleDeferLogMs = 0;
|
|
144
156
|
// False until LanceDB + MetaCache are open. The socket starts listening early
|
|
145
157
|
// (so liveness probes succeed during slow init), so commands that need those
|
|
146
158
|
// resources must be gated on this to avoid hitting null stores mid-startup.
|
|
@@ -168,6 +180,8 @@ class Daemon {
|
|
|
168
180
|
runProjectOperation: (root, name, signal, fn) => this.withProjectLock(root, signal, () => this.runSharedOperation(name, signal, fn)),
|
|
169
181
|
});
|
|
170
182
|
this.projectMutex = new keyed_mutex_1.KeyedMutex();
|
|
183
|
+
// Tests construct daemons against the real HOME; keep their leases in memory.
|
|
184
|
+
this.watchLeases = new watch_leases_1.WatchLeases(process.env.VITEST ? null : config_1.PATHS.watchLeasesFile);
|
|
171
185
|
this.operations = new operation_coordinator_1.OperationCoordinator();
|
|
172
186
|
this.shutdownPromise = null;
|
|
173
187
|
// Full-index progress per root while initialSync runs (--reset / initial
|
|
@@ -369,9 +383,13 @@ class Daemon {
|
|
|
369
383
|
this.publishResourceGeneration(globalConfig, this.activeGeneration, this.vectorDb, this.workerPool, this.mlxMode(globalConfig));
|
|
370
384
|
// 7. Register daemon (only after resources are open)
|
|
371
385
|
(0, watcher_store_1.registerDaemon)(process.pid);
|
|
372
|
-
// 8. Subscribe to
|
|
386
|
+
// 8. Subscribe to the registered projects some session still holds a lease
|
|
387
|
+
// on (skip missing directories). Everything else waits for a lease.
|
|
373
388
|
const allProjects = (0, project_registry_1.listProjects)();
|
|
374
|
-
|
|
389
|
+
this.watchLeases.load();
|
|
390
|
+
const indexed = allProjects.filter((p) => p.status === "indexed" && this.shouldWatch(p.root));
|
|
391
|
+
console.log(`[daemon] Watching ${indexed.length} of ${allProjects.length} registered projects` +
|
|
392
|
+
(WATCH_ALL ? " (GMAX_WATCH_ALL=1)" : " (leased by active sessions)"));
|
|
375
393
|
for (const p of indexed) {
|
|
376
394
|
if (!fs.existsSync(p.root)) {
|
|
377
395
|
// Skipping is already the safe behavior — the project keeps its registry
|
|
@@ -418,6 +436,7 @@ class Daemon {
|
|
|
418
436
|
}
|
|
419
437
|
catch (_a) { }
|
|
420
438
|
(0, log_rotate_1.rotateLogFds)(path.join(config_1.PATHS.logsDir, "daemon.log"));
|
|
439
|
+
this.sweepWatchLeases();
|
|
421
440
|
// Every 5 ticks (5 min), probe the MLX embed server and respawn if
|
|
422
441
|
// it's gone zombie (port held but /health unresponsive). Closes the
|
|
423
442
|
// 42h-degradation window where workers silently fell back to ONNX CPU
|
|
@@ -429,6 +448,9 @@ class Daemon {
|
|
|
429
448
|
this.maybeRecycle();
|
|
430
449
|
this.checkKernelZonePressure();
|
|
431
450
|
}
|
|
451
|
+
else if (this.recycleDueSinceMs !== null) {
|
|
452
|
+
this.maybeRecycle(false);
|
|
453
|
+
}
|
|
432
454
|
}, HEARTBEAT_INTERVAL_MS);
|
|
433
455
|
// 10. Idle timeout (skip when disabled via env)
|
|
434
456
|
if (IDLE_TIMEOUT_MS > 0) {
|
|
@@ -500,6 +522,10 @@ class Daemon {
|
|
|
500
522
|
return this.withProjectLock(root, signal, () => this.runSharedOperation("watch", signal, () => this.watchProjectWithinOperation(root)));
|
|
501
523
|
}
|
|
502
524
|
watchProjectWithinOperation(root) {
|
|
525
|
+
// Every path that re-watches after an index/rebuild/remove goes through
|
|
526
|
+
// here, so an unleased project stays unwatched after the operation too.
|
|
527
|
+
if (!this.shouldWatch(root))
|
|
528
|
+
return Promise.resolve();
|
|
503
529
|
const project = (0, project_registry_1.getProject)(root);
|
|
504
530
|
if ((project === null || project === void 0 ? void 0 : project.status) === "indexed" &&
|
|
505
531
|
this.activeGeneration &&
|
|
@@ -509,6 +535,62 @@ class Daemon {
|
|
|
509
535
|
}
|
|
510
536
|
return this.watcherManager.watchProject(root);
|
|
511
537
|
}
|
|
538
|
+
shouldWatch(root) {
|
|
539
|
+
return WATCH_ALL || this.watchLeases.isWanted(root);
|
|
540
|
+
}
|
|
541
|
+
/**
|
|
542
|
+
* Take (or renew) a lease on `root` and make sure it is watched. Called for
|
|
543
|
+
* every `watch` IPC — MCP servers renew theirs every few minutes, so the
|
|
544
|
+
* common case is a lease bump on an already-watched project.
|
|
545
|
+
*/
|
|
546
|
+
requestWatch(root, lease = {}, signal) {
|
|
547
|
+
var _a;
|
|
548
|
+
this.watchLeases.acquire(root, {
|
|
549
|
+
holder: lease.holder || "cli",
|
|
550
|
+
pid: lease.pid,
|
|
551
|
+
ttlMs: (_a = lease.ttlMs) !== null && _a !== void 0 ? _a : watch_leases_1.DEFAULT_LEASE_TTL_MS,
|
|
552
|
+
});
|
|
553
|
+
if (this.processors.has(root))
|
|
554
|
+
return Promise.resolve();
|
|
555
|
+
return this.watchProject(root, signal);
|
|
556
|
+
}
|
|
557
|
+
/**
|
|
558
|
+
* Drop a lease — every lease on the root when `holder` is omitted, which is
|
|
559
|
+
* what an explicit `unwatch` means — and unwatch once nothing wants it.
|
|
560
|
+
*/
|
|
561
|
+
releaseWatch(root, holder, signal) {
|
|
562
|
+
this.watchLeases.release(root, holder);
|
|
563
|
+
if (this.shouldWatch(root) || !this.processors.has(root)) {
|
|
564
|
+
return Promise.resolve();
|
|
565
|
+
}
|
|
566
|
+
return this.unwatchProject(root, signal);
|
|
567
|
+
}
|
|
568
|
+
/**
|
|
569
|
+
* A short lease for a CLI add/index: the project stays watched after the
|
|
570
|
+
* operation re-watches it, and lapses unless a session picks it up.
|
|
571
|
+
*/
|
|
572
|
+
leaseForCli(root) {
|
|
573
|
+
this.watchLeases.acquire(root, {
|
|
574
|
+
holder: "cli",
|
|
575
|
+
ttlMs: watch_leases_1.DEFAULT_LEASE_TTL_MS,
|
|
576
|
+
});
|
|
577
|
+
}
|
|
578
|
+
listWatchLeases() {
|
|
579
|
+
return this.watchLeases.list();
|
|
580
|
+
}
|
|
581
|
+
/** Unwatch projects whose last lease expired or whose holder exited. */
|
|
582
|
+
sweepWatchLeases() {
|
|
583
|
+
if (this.shuttingDown || WATCH_ALL)
|
|
584
|
+
return;
|
|
585
|
+
for (const root of this.watchLeases.sweep()) {
|
|
586
|
+
if (!this.processors.has(root) || this.shouldWatch(root))
|
|
587
|
+
continue;
|
|
588
|
+
console.log(`[daemon] No session holds ${path.basename(root)} any more — unwatching`);
|
|
589
|
+
void this.unwatchProject(root).catch((err) => {
|
|
590
|
+
console.error(`[daemon] Failed to unwatch ${path.basename(root)}:`, err);
|
|
591
|
+
});
|
|
592
|
+
}
|
|
593
|
+
}
|
|
512
594
|
runSharedOperation(name, signal, fn) {
|
|
513
595
|
return this.operations.runShared(name, signal, fn);
|
|
514
596
|
}
|
|
@@ -1572,6 +1654,7 @@ class Daemon {
|
|
|
1572
1654
|
const keys = yield this.metaCache.getKeysWithPrefix(rootPrefix);
|
|
1573
1655
|
for (const key of keys)
|
|
1574
1656
|
this.metaCache.delete(key);
|
|
1657
|
+
this.watchLeases.release(root);
|
|
1575
1658
|
(0, ipc_handler_1.writeDone)(conn, { ok: true });
|
|
1576
1659
|
}
|
|
1577
1660
|
catch (err) {
|
|
@@ -1719,26 +1802,61 @@ class Daemon {
|
|
|
1719
1802
|
* too large. Only fires when quiet — no active compaction and no in-flight
|
|
1720
1803
|
* project operations — so a recycle never interrupts indexing work. The
|
|
1721
1804
|
* successor re-runs catchup on startup, so nothing is lost.
|
|
1805
|
+
*
|
|
1806
|
+
* "Quiet" alone was not enough: with a dozen watched projects some batch or
|
|
1807
|
+
* catchup was in flight at nearly every 5-minute probe, and one daemon ran for
|
|
1808
|
+
* 8 days past its 24h limit while its footprint sat at 4 GB. So once a recycle
|
|
1809
|
+
* is due it is re-checked every heartbeat to catch a quiet minute, and after
|
|
1810
|
+
* RECYCLE_FORCE_AFTER_MS of deferral it proceeds over in-flight project work
|
|
1811
|
+
* (shutdown aborts it; the successor's catchup redoes it). Compaction is still
|
|
1812
|
+
* waited out — killing it strands scratch files and wastes the rewrite.
|
|
1813
|
+
*
|
|
1814
|
+
* @param sampleFootprint false on the in-between ticks: `footprint` costs ~50 ms,
|
|
1815
|
+
* so it is only sampled on the 5-minute probe.
|
|
1722
1816
|
*/
|
|
1723
|
-
maybeRecycle() {
|
|
1817
|
+
maybeRecycle(sampleFootprint = true) {
|
|
1724
1818
|
var _a;
|
|
1725
1819
|
if (this.shuttingDown || this.recycling)
|
|
1726
1820
|
return;
|
|
1727
1821
|
const ageMs = process.uptime() * 1000;
|
|
1728
|
-
const rssMb = process.memoryUsage().rss / (1024 * 1024);
|
|
1729
1822
|
const ageExceeded = MAX_LIFETIME_MS > 0 && ageMs > MAX_LIFETIME_MS;
|
|
1730
|
-
|
|
1731
|
-
if (
|
|
1732
|
-
|
|
1733
|
-
|
|
1823
|
+
let footprintMb = null;
|
|
1824
|
+
if (this.recycleDueSinceMs === null &&
|
|
1825
|
+
!ageExceeded &&
|
|
1826
|
+
sampleFootprint &&
|
|
1827
|
+
RSS_WATERMARK_MB > 0 &&
|
|
1828
|
+
// A fresh daemon above the watermark would recycle in a loop; give the
|
|
1829
|
+
// caches a chance to settle before judging the footprint.
|
|
1830
|
+
ageMs > FOOTPRINT_RECYCLE_MIN_AGE_MS) {
|
|
1831
|
+
footprintMb = (0, process_footprint_1.readFootprintMb)();
|
|
1832
|
+
}
|
|
1833
|
+
const footprintExceeded = footprintMb !== null && footprintMb > RSS_WATERMARK_MB;
|
|
1834
|
+
if (this.recycleDueSinceMs === null) {
|
|
1835
|
+
if (!ageExceeded && !footprintExceeded)
|
|
1836
|
+
return;
|
|
1837
|
+
this.recycleDueSinceMs = Date.now();
|
|
1838
|
+
this.recycleReason = ageExceeded
|
|
1839
|
+
? `age ${(ageMs / 3600000).toFixed(1)}h > ${(MAX_LIFETIME_MS / 3600000).toFixed(1)}h`
|
|
1840
|
+
: `footprint ${Math.round(footprintMb !== null && footprintMb !== void 0 ? footprintMb : 0)}MB > ${RSS_WATERMARK_MB}MB`;
|
|
1841
|
+
}
|
|
1842
|
+
// Defer while busy; re-checked on every heartbeat until it goes through.
|
|
1734
1843
|
if ((_a = this.vectorDb) === null || _a === void 0 ? void 0 : _a.isMaintenanceActive())
|
|
1735
1844
|
return;
|
|
1736
|
-
|
|
1845
|
+
const busy = this.projectMutex.pending > 0 || this.operations.activeCount > 0;
|
|
1846
|
+
const deferredMs = Date.now() - this.recycleDueSinceMs;
|
|
1847
|
+
if (busy && deferredMs < RECYCLE_FORCE_AFTER_MS) {
|
|
1848
|
+
if (Date.now() - this.lastRecycleDeferLogMs >= 30 * 60 * 1000) {
|
|
1849
|
+
this.lastRecycleDeferLogMs = Date.now();
|
|
1850
|
+
console.log(`[daemon] Recycle due (${this.recycleReason}) — deferred ${Math.round(deferredMs / 60000)}min, busy: ` +
|
|
1851
|
+
[
|
|
1852
|
+
...this.operations.activeOperationNames(),
|
|
1853
|
+
...this.projectMutex.pendingKeys().map((k) => path.basename(k)),
|
|
1854
|
+
].join(", "));
|
|
1855
|
+
}
|
|
1737
1856
|
return;
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
console.log(`[daemon] Recycling (${reason}) — handing off to a fresh daemon`);
|
|
1857
|
+
}
|
|
1858
|
+
const forced = busy ? " (forced over in-flight work)" : "";
|
|
1859
|
+
console.log(`[daemon] Recycling (${this.recycleReason})${forced} — handing off to a fresh daemon`);
|
|
1742
1860
|
this.recycling = true;
|
|
1743
1861
|
void this.shutdown({ relaunch: true }).finally(() => process.exit(0));
|
|
1744
1862
|
}
|
|
@@ -94,6 +94,7 @@ const callsites_1 = require("../graph/callsites");
|
|
|
94
94
|
const graph_builder_1 = require("../graph/graph-builder");
|
|
95
95
|
const impact_1 = require("../graph/impact");
|
|
96
96
|
const arrow_1 = require("../utils/arrow");
|
|
97
|
+
const async_semaphore_1 = require("../utils/async-semaphore");
|
|
97
98
|
const daemon_client_1 = require("../utils/daemon-client");
|
|
98
99
|
const filter_builder_1 = require("../utils/filter-builder");
|
|
99
100
|
const path_containment_1 = require("../utils/path-containment");
|
|
@@ -113,6 +114,8 @@ const MAX_PATH_HOPS = 10;
|
|
|
113
114
|
const MAX_SUBGRAPH_FILES = 500;
|
|
114
115
|
/** A diff can name many symbols; each costs three graph queries. */
|
|
115
116
|
const MAX_RISK_SYMBOLS = 200;
|
|
117
|
+
/** Symbols `graph.risk` resolves at once (three store scans each). */
|
|
118
|
+
const RISK_SYMBOL_CONCURRENCY = 4;
|
|
116
119
|
function clampInt(value, min, max, fallback) {
|
|
117
120
|
const n = typeof value === "number" ? value : Number.NaN;
|
|
118
121
|
if (!Number.isFinite(n))
|
|
@@ -430,10 +433,13 @@ function runGraphSubgraph(db, args) {
|
|
|
430
433
|
* The caller keeps the git half and assembles the `RiskInput` rows, so the
|
|
431
434
|
* scoring stays exactly where it was.
|
|
432
435
|
*/
|
|
433
|
-
function runGraphRisk(db, args) {
|
|
436
|
+
function runGraphRisk(db, args, signal) {
|
|
434
437
|
return __awaiter(this, void 0, void 0, function* () {
|
|
435
438
|
const builder = new graph_builder_1.GraphBuilder(db, args.scope.pathPrefix, args.scope.excludePrefixes);
|
|
436
|
-
|
|
439
|
+
// Each symbol costs three unindexed content scans. Fanning all of them out
|
|
440
|
+
// at once put up to MAX_RISK_SYMBOLS x 3 scans in native memory together;
|
|
441
|
+
// the pool keeps it at RISK_SYMBOL_CONCURRENCY x 3.
|
|
442
|
+
return (0, async_semaphore_1.mapWithConcurrency)(args.symbols.slice(0, MAX_RISK_SYMBOLS), RISK_SYMBOL_CONCURRENCY, (symbol) => __awaiter(this, void 0, void 0, function* () {
|
|
437
443
|
var _a, _b;
|
|
438
444
|
const [callers, loc, tests] = yield Promise.all([
|
|
439
445
|
builder.callersOf(symbol).catch(() => []),
|
|
@@ -447,7 +453,7 @@ function runGraphRisk(db, args) {
|
|
|
447
453
|
callerCount: callers.length,
|
|
448
454
|
hasTests: tests.length > 0,
|
|
449
455
|
};
|
|
450
|
-
}))
|
|
456
|
+
}), signal);
|
|
451
457
|
});
|
|
452
458
|
}
|
|
453
459
|
// Names too generic to be useful as god-node signal (`id`, `el`, `fn`, …).
|
|
@@ -810,13 +816,13 @@ function createGraphVerbs(getDeps) {
|
|
|
810
816
|
});
|
|
811
817
|
return { ok: true, subgraph };
|
|
812
818
|
}),
|
|
813
|
-
"graph.risk": (payload, db) => __awaiter(this, void 0, void 0, function* () {
|
|
819
|
+
"graph.risk": (payload, db, signal) => __awaiter(this, void 0, void 0, function* () {
|
|
814
820
|
const decoded = decodeScopePayload(payload);
|
|
815
821
|
const facts = yield runGraphRisk(db, {
|
|
816
822
|
symbols: requireSymbols(payload),
|
|
817
823
|
queryRoot: queryRootFor(decoded),
|
|
818
824
|
scope: decoded.scope,
|
|
819
|
-
});
|
|
825
|
+
}, signal);
|
|
820
826
|
return { ok: true, facts };
|
|
821
827
|
}),
|
|
822
828
|
"graph.audit": (payload, db) => __awaiter(this, void 0, void 0, function* () {
|
|
@@ -831,7 +837,7 @@ function createGraphVerbs(getDeps) {
|
|
|
831
837
|
};
|
|
832
838
|
const wired = {};
|
|
833
839
|
for (const [name, run] of Object.entries(handlers)) {
|
|
834
|
-
wired[name] = (payload, ctx) => __awaiter(this, void 0, void 0, function* () { return run(payload, requireStore(getDeps(ctx))); });
|
|
840
|
+
wired[name] = (payload, ctx) => __awaiter(this, void 0, void 0, function* () { return run(payload, requireStore(getDeps(ctx)), ctx.signal); });
|
|
835
841
|
}
|
|
836
842
|
return wired;
|
|
837
843
|
}
|