grepmax 0.19.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +23 -0
- package/README.md +1 -1
- package/dist/commands/codex.js +5 -2
- package/dist/commands/context.js +1 -1
- package/dist/commands/doctor.js +37 -7
- package/dist/commands/droid.js +61 -9
- package/dist/commands/mcp.js +7 -7
- package/dist/commands/plugin.js +35 -21
- package/dist/commands/project.js +1 -1
- package/dist/commands/remove.js +9 -2
- package/dist/commands/repair.js +186 -0
- package/dist/commands/search-run.js +12 -1
- package/dist/commands/setup.js +4 -2
- package/dist/commands/status.js +1 -1
- package/dist/commands/symbols.js +1 -1
- package/dist/commands/watch.js +25 -4
- package/dist/config.js +45 -1
- package/dist/index.js +2 -0
- package/dist/lib/daemon/daemon.js +177 -48
- package/dist/lib/daemon/ipc-handler.js +12 -0
- package/dist/lib/daemon/process-manager.js +21 -3
- package/dist/lib/daemon/watcher-manager.js +18 -2
- package/dist/lib/graph/graph-builder.js +2 -2
- package/dist/lib/graph/impact.js +6 -6
- package/dist/lib/index/batch-processor.js +10 -1
- package/dist/lib/index/syncer.js +15 -4
- package/dist/lib/index/watcher-batch.js +10 -1
- package/dist/lib/llm/config.js +2 -1
- package/dist/lib/llm/investigate.js +40 -6
- package/dist/lib/llm/server.js +9 -2
- package/dist/lib/llm/tools.js +1 -1
- package/dist/lib/search/pagerank.js +1 -1
- package/dist/lib/search/searcher.js +6 -6
- package/dist/lib/store/vector-db.js +49 -10
- package/dist/lib/utils/daemon-client.js +87 -0
- package/dist/lib/utils/filter-builder.js +22 -0
- package/dist/lib/utils/scope-filter.js +3 -5
- package/dist/lib/workers/embeddings/granite.js +4 -1
- package/dist/lib/workers/pool.js +21 -1
- package/package.json +2 -1
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
- package/plugins/grepmax/agents/semantic-explore.md +1 -1
- package/plugins/grepmax/skills/grepmax/SKILL.md +1 -1
package/dist/commands/status.js
CHANGED
|
@@ -121,7 +121,7 @@ Examples:
|
|
|
121
121
|
const rows = yield table
|
|
122
122
|
.query()
|
|
123
123
|
.select(["id"])
|
|
124
|
-
.where(
|
|
124
|
+
.where((0, filter_builder_1.pathStartsWith)(prefix))
|
|
125
125
|
.toArray();
|
|
126
126
|
chunkCounts.set(project.root, rows.length);
|
|
127
127
|
}
|
package/dist/commands/symbols.js
CHANGED
|
@@ -86,7 +86,7 @@ function collectSymbols(options) {
|
|
|
86
86
|
const absPrefix = path.isAbsolute(options.pathPrefix)
|
|
87
87
|
? options.pathPrefix
|
|
88
88
|
: path.resolve(options.projectRoot, options.pathPrefix);
|
|
89
|
-
query = query.where(
|
|
89
|
+
query = query.where((0, filter_builder_1.pathStartsWith)((0, filter_builder_1.normalizePath)(absPrefix)));
|
|
90
90
|
}
|
|
91
91
|
const rows = yield query.toArray();
|
|
92
92
|
const map = new Map();
|
package/dist/commands/watch.js
CHANGED
|
@@ -81,7 +81,7 @@ exports.watch = new commander_1.Command("watch")
|
|
|
81
81
|
// Skip spawn if daemon already running at the same version.
|
|
82
82
|
// If version mismatches (e.g. after npm install -g), shut down the old
|
|
83
83
|
// daemon so we can start a fresh one with the new code.
|
|
84
|
-
const { isDaemonRunning, isDaemonHeartbeatFresh, sendDaemonCommand } = yield Promise.resolve().then(() => __importStar(require("../lib/utils/daemon-client")));
|
|
84
|
+
const { isDaemonRunning, isDaemonHeartbeatFresh, sendDaemonCommand, readDaemonPid, waitForProcessExit, } = yield Promise.resolve().then(() => __importStar(require("../lib/utils/daemon-client")));
|
|
85
85
|
if (yield isDaemonRunning()) {
|
|
86
86
|
const cliVersion = JSON.parse(fs.readFileSync(path.join(__dirname, "../../package.json"), "utf-8")).version;
|
|
87
87
|
const resp = yield sendDaemonCommand({ cmd: "ping" });
|
|
@@ -89,6 +89,12 @@ exports.watch = new commander_1.Command("watch")
|
|
|
89
89
|
process.exit(0);
|
|
90
90
|
}
|
|
91
91
|
console.log(`Daemon version mismatch (${resp.version} → ${cliVersion}), restarting...`);
|
|
92
|
+
// Capture the old PID before shutdown removes the PID file, so we
|
|
93
|
+
// can wait for the process to fully EXIT rather than guessing with a
|
|
94
|
+
// fixed sleep. Spawning the successor while the old daemon is still
|
|
95
|
+
// draining lets the successor's killStaleProcesses() classify it as
|
|
96
|
+
// stale and SIGKILL it mid-cleanup.
|
|
97
|
+
const oldPid = readDaemonPid();
|
|
92
98
|
yield sendDaemonCommand({
|
|
93
99
|
cmd: "shutdown",
|
|
94
100
|
reason: "version-mismatch",
|
|
@@ -97,8 +103,16 @@ exports.watch = new commander_1.Command("watch")
|
|
|
97
103
|
from_version: cliVersion,
|
|
98
104
|
from_argv: process.argv.slice(0, 4),
|
|
99
105
|
});
|
|
100
|
-
|
|
101
|
-
|
|
106
|
+
if (oldPid) {
|
|
107
|
+
const exited = yield waitForProcessExit(oldPid, 20000);
|
|
108
|
+
if (!exited) {
|
|
109
|
+
console.log(`Old daemon (PID ${oldPid}) still draining after 20s — starting anyway ` +
|
|
110
|
+
`(its draining marker keeps the successor from killing it mid-cleanup)`);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
else {
|
|
114
|
+
yield new Promise((r) => setTimeout(r, 2000));
|
|
115
|
+
}
|
|
102
116
|
}
|
|
103
117
|
else if (isDaemonHeartbeatFresh()) {
|
|
104
118
|
// Ping failed but daemon.lock mtime is fresh — another daemon is
|
|
@@ -131,6 +145,13 @@ exports.watch = new commander_1.Command("watch")
|
|
|
131
145
|
process.exit(0);
|
|
132
146
|
}
|
|
133
147
|
console.error("[daemon] Failed to start:", err);
|
|
148
|
+
// Tear down any half-opened state — the listening socket, PID, and
|
|
149
|
+
// lock — so we don't leave a zombie that answers pings but has no
|
|
150
|
+
// resources. The socket also keeps the event loop alive otherwise.
|
|
151
|
+
try {
|
|
152
|
+
yield daemon.shutdown();
|
|
153
|
+
}
|
|
154
|
+
catch (_c) { }
|
|
134
155
|
process.exitCode = 1;
|
|
135
156
|
return;
|
|
136
157
|
}
|
|
@@ -201,7 +222,7 @@ exports.watch = new commander_1.Command("watch")
|
|
|
201
222
|
const indexed = yield table
|
|
202
223
|
.query()
|
|
203
224
|
.select(["id"])
|
|
204
|
-
.where(
|
|
225
|
+
.where((0, filter_builder_1.pathStartsWith)(prefix))
|
|
205
226
|
.limit(1)
|
|
206
227
|
.toArray();
|
|
207
228
|
if (indexed.length === 0) {
|
package/dist/config.js
CHANGED
|
@@ -33,9 +33,11 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.INDEXABLE_EXTENSIONS = exports.FRAGMENT_COMPACT_THRESHOLD = exports.DISK_LOW_BYTES = exports.DISK_CRITICAL_BYTES = exports.MAX_FILE_SIZE_BYTES = exports.PATHS = exports.MAX_WORKER_MEMORY_MB = exports.WORKER_BOOT_TIMEOUT_MS = exports.WORKER_TIMEOUT_MS = exports.CHUNKER_VERSION_HISTORY = exports.CONFIG = exports.MODEL_IDS = exports.DEFAULT_MODEL_TIER = exports.MODEL_TIERS = void 0;
|
|
36
|
+
exports.INDEXABLE_EXTENSIONS = exports.FRAGMENT_COMPACT_THRESHOLD = exports.DISK_LOW_BYTES = exports.DISK_CRITICAL_BYTES = exports.MAX_FILE_SIZE_BYTES = exports.PATHS = exports.MAX_WORKER_MEMORY_MB = exports.WORKER_BOOT_TIMEOUT_MS = exports.WORKER_TIMEOUT_MS = exports.REBUILD_COMMAND = exports.CHUNKER_VERSION_HISTORY = exports.CONFIG = exports.MODEL_IDS = exports.DEFAULT_MODEL_TIER = exports.MODEL_TIERS = void 0;
|
|
37
37
|
exports.describeChunkerGap = describeChunkerGap;
|
|
38
38
|
exports.describeEmbeddingGap = describeEmbeddingGap;
|
|
39
|
+
exports.describeSchemaDimGap = describeSchemaDimGap;
|
|
40
|
+
exports.schemaDimAgentRow = schemaDimAgentRow;
|
|
39
41
|
const os = __importStar(require("node:os"));
|
|
40
42
|
const path = __importStar(require("node:path"));
|
|
41
43
|
exports.MODEL_TIERS = {
|
|
@@ -157,6 +159,43 @@ function describeEmbeddingGap(stored, current) {
|
|
|
157
159
|
severity: dimChanged ? "breaking" : "additive",
|
|
158
160
|
};
|
|
159
161
|
}
|
|
162
|
+
/**
|
|
163
|
+
* The single sanctioned recovery for a physical table-width mismatch. The shared
|
|
164
|
+
* `chunks` table is fixed-width at creation, so a tier/dim change strands it at
|
|
165
|
+
* the old width and every write throws. A per-project `gmax index --reset` only
|
|
166
|
+
* deletes rows — it can't change the shared table's width — so the real fix is a
|
|
167
|
+
* global drop-and-rebuild. doctor, the insertBatch failure, and the staleness
|
|
168
|
+
* hint all point here so the guidance never contradicts itself.
|
|
169
|
+
*/
|
|
170
|
+
exports.REBUILD_COMMAND = "gmax repair --rebuild";
|
|
171
|
+
/**
|
|
172
|
+
* Describe the gap between the LanceDB table's PHYSICAL `vector` width and the
|
|
173
|
+
* width the current global config would produce, or null when they agree (or no
|
|
174
|
+
* table exists yet). This is distinct from {@link describeEmbeddingGap}: that one
|
|
175
|
+
* compares the project REGISTRY's recorded `{modelTier, vectorDim}` to config
|
|
176
|
+
* (logical drift, fixable per project), while this compares the actual on-disk
|
|
177
|
+
* table schema to config (physical drift — every write throws until a global
|
|
178
|
+
* rebuild). A table can match the registry yet still be physically stranded, so
|
|
179
|
+
* doctor reports both independently.
|
|
180
|
+
*/
|
|
181
|
+
function describeSchemaDimGap(tableDim, configDim) {
|
|
182
|
+
if (tableDim == null)
|
|
183
|
+
return null; // no table on disk yet — nothing to compare
|
|
184
|
+
if (tableDim === configDim)
|
|
185
|
+
return null;
|
|
186
|
+
return { tableDim, configDim };
|
|
187
|
+
}
|
|
188
|
+
/** Stable, tab-delimited doctor `--agent` row for a physical schema-dim
|
|
189
|
+
* mismatch. Kept here (pure) so the wire format is testable without running the
|
|
190
|
+
* full doctor command. */
|
|
191
|
+
function schemaDimAgentRow(gap) {
|
|
192
|
+
return [
|
|
193
|
+
"schema_dim_mismatch",
|
|
194
|
+
`table_dim=${gap.tableDim}`,
|
|
195
|
+
`current_dim=${gap.configDim}`,
|
|
196
|
+
`fix=${exports.REBUILD_COMMAND}`,
|
|
197
|
+
].join("\t");
|
|
198
|
+
}
|
|
160
199
|
exports.WORKER_TIMEOUT_MS = Number.parseInt(process.env.GMAX_WORKER_TIMEOUT_MS || "60000", 10);
|
|
161
200
|
exports.WORKER_BOOT_TIMEOUT_MS = Number.parseInt(process.env.GMAX_WORKER_BOOT_TIMEOUT_MS || "300000", 10);
|
|
162
201
|
exports.MAX_WORKER_MEMORY_MB = (() => {
|
|
@@ -177,6 +216,11 @@ exports.PATHS = {
|
|
|
177
216
|
daemonSocket: path.join(GLOBAL_ROOT, "daemon.sock"),
|
|
178
217
|
daemonPidFile: path.join(GLOBAL_ROOT, "daemon.pid"),
|
|
179
218
|
daemonLockFile: path.join(GLOBAL_ROOT, "daemon.lock"),
|
|
219
|
+
// Written by a daemon while it is gracefully shutting down (draining workers,
|
|
220
|
+
// closing LanceDB). A successor's killStaleProcesses() respects this so it
|
|
221
|
+
// never SIGKILLs a peer mid-cleanup once the peer has already dropped its
|
|
222
|
+
// socket/PID/lock liveness markers.
|
|
223
|
+
daemonDrainingFile: path.join(GLOBAL_ROOT, "daemon.draining"),
|
|
180
224
|
// Centralized index storage — one database for all indexed directories
|
|
181
225
|
lancedbDir: path.join(GLOBAL_ROOT, "lancedb"),
|
|
182
226
|
cacheDir: path.join(GLOBAL_ROOT, "cache"),
|
package/dist/index.js
CHANGED
|
@@ -64,6 +64,7 @@ const project_1 = require("./commands/project");
|
|
|
64
64
|
const recent_1 = require("./commands/recent");
|
|
65
65
|
const related_1 = require("./commands/related");
|
|
66
66
|
const remove_1 = require("./commands/remove");
|
|
67
|
+
const repair_1 = require("./commands/repair");
|
|
67
68
|
const review_1 = require("./commands/review");
|
|
68
69
|
const search_1 = require("./commands/search");
|
|
69
70
|
const serve_1 = require("./commands/serve");
|
|
@@ -141,6 +142,7 @@ commander_1.program.addCommand(review_1.review);
|
|
|
141
142
|
commander_1.program.addCommand(setup_1.setup);
|
|
142
143
|
commander_1.program.addCommand(config_1.config);
|
|
143
144
|
commander_1.program.addCommand(doctor_1.doctor);
|
|
145
|
+
commander_1.program.addCommand(repair_1.repair);
|
|
144
146
|
// Plugins
|
|
145
147
|
commander_1.program.addCommand(plugin_1.plugin);
|
|
146
148
|
// Legacy plugin installers (hidden — use `gmax plugin` instead)
|
|
@@ -56,6 +56,7 @@ const syncer_1 = require("../index/syncer");
|
|
|
56
56
|
const server_1 = require("../llm/server");
|
|
57
57
|
const meta_cache_1 = require("../store/meta-cache");
|
|
58
58
|
const vector_db_1 = require("../store/vector-db");
|
|
59
|
+
const daemon_client_1 = require("../utils/daemon-client");
|
|
59
60
|
const daemon_launcher_1 = require("../utils/daemon-launcher");
|
|
60
61
|
const log_rotate_1 = require("../utils/log-rotate");
|
|
61
62
|
const logger_1 = require("../utils/logger");
|
|
@@ -116,6 +117,10 @@ class Daemon {
|
|
|
116
117
|
this.heartbeatTick = 0;
|
|
117
118
|
this.shuttingDown = false;
|
|
118
119
|
this.recycling = false;
|
|
120
|
+
// False until LanceDB + MetaCache are open. The socket starts listening early
|
|
121
|
+
// (so liveness probes succeed during slow init), so commands that need those
|
|
122
|
+
// resources must be gated on this to avoid hitting null stores mid-startup.
|
|
123
|
+
this.ready = false;
|
|
119
124
|
this.processManager = new process_manager_1.ProcessManager({
|
|
120
125
|
getShuttingDown: () => this.shuttingDown,
|
|
121
126
|
});
|
|
@@ -253,6 +258,8 @@ class Daemon {
|
|
|
253
258
|
this.vectorDb.startMaintenanceLoop();
|
|
254
259
|
console.log("[daemon] Opening MetaCache:", config_1.PATHS.lmdbPath);
|
|
255
260
|
this.metaCache = new meta_cache_1.MetaCache(config_1.PATHS.lmdbPath);
|
|
261
|
+
// Resources are open — only now may resource-dependent IPC commands run.
|
|
262
|
+
this.ready = true;
|
|
256
263
|
}
|
|
257
264
|
catch (err) {
|
|
258
265
|
console.error("[daemon] Failed to open shared resources:", err);
|
|
@@ -499,6 +506,10 @@ class Daemon {
|
|
|
499
506
|
uptime() {
|
|
500
507
|
return Math.floor((Date.now() - this.startTime) / 1000);
|
|
501
508
|
}
|
|
509
|
+
/** True once shared resources (LanceDB + MetaCache) are open. */
|
|
510
|
+
isReady() {
|
|
511
|
+
return this.ready;
|
|
512
|
+
}
|
|
502
513
|
getDiskPressure() {
|
|
503
514
|
var _a, _b;
|
|
504
515
|
return (_b = (_a = this.vectorDb) === null || _a === void 0 ? void 0 : _a.diskPressure) !== null && _b !== void 0 ? _b : "unknown";
|
|
@@ -590,6 +601,66 @@ class Daemon {
|
|
|
590
601
|
}));
|
|
591
602
|
});
|
|
592
603
|
}
|
|
604
|
+
/**
|
|
605
|
+
* Core full-(re)index of one project: quiesce its batch processor + watcher,
|
|
606
|
+
* run initialSync, then re-watch in the finally. Shared by indexProject (one
|
|
607
|
+
* project per IPC connection) and repairRebuild (all projects after a global
|
|
608
|
+
* table drop). The caller owns the project lock, the maintenance pause, the
|
|
609
|
+
* heartbeat, and the AbortController; this method owns the watcher handoff and
|
|
610
|
+
* the indexProgress bookkeeping (so concurrent searches get a partial-result
|
|
611
|
+
* footer while it runs — Phase 6).
|
|
612
|
+
*/
|
|
613
|
+
reindexOneProject(root, opts, signal, onProgress) {
|
|
614
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
615
|
+
if (!this.vectorDb || !this.metaCache) {
|
|
616
|
+
throw new Error("daemon resources not ready");
|
|
617
|
+
}
|
|
618
|
+
// Pause the project's batch processor during full index
|
|
619
|
+
const processor = this.processors.get(root);
|
|
620
|
+
if (processor) {
|
|
621
|
+
yield processor.close();
|
|
622
|
+
this.processors.delete(root);
|
|
623
|
+
}
|
|
624
|
+
const sub = this.subscriptions.get(root);
|
|
625
|
+
if (sub) {
|
|
626
|
+
yield sub.unsubscribe();
|
|
627
|
+
this.subscriptions.delete(root);
|
|
628
|
+
}
|
|
629
|
+
// Mark this root as full-indexing so concurrent searches get a
|
|
630
|
+
// partial-result footer (Phase 6); seeded at 0/0 until the first tick.
|
|
631
|
+
this.indexProgress.set(root, { processed: 0, total: 0 });
|
|
632
|
+
try {
|
|
633
|
+
return yield (0, syncer_1.initialSync)({
|
|
634
|
+
projectRoot: root,
|
|
635
|
+
reset: opts.reset,
|
|
636
|
+
dryRun: opts.dryRun,
|
|
637
|
+
vectorDb: this.vectorDb,
|
|
638
|
+
metaCache: this.metaCache,
|
|
639
|
+
signal,
|
|
640
|
+
onProgress: (info) => {
|
|
641
|
+
this.resetActivity();
|
|
642
|
+
this.indexProgress.set(root, {
|
|
643
|
+
processed: info.processed,
|
|
644
|
+
total: info.total,
|
|
645
|
+
});
|
|
646
|
+
onProgress(info);
|
|
647
|
+
},
|
|
648
|
+
});
|
|
649
|
+
}
|
|
650
|
+
finally {
|
|
651
|
+
this.indexProgress.delete(root);
|
|
652
|
+
// Re-enable watcher (skip if shutting down)
|
|
653
|
+
if (!this.shuttingDown) {
|
|
654
|
+
try {
|
|
655
|
+
yield this.watchProject(root);
|
|
656
|
+
}
|
|
657
|
+
catch (err) {
|
|
658
|
+
console.error(`[daemon] Failed to re-watch ${path.basename(root)}:`, err);
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
});
|
|
663
|
+
}
|
|
593
664
|
indexProject(root, conn, opts) {
|
|
594
665
|
return __awaiter(this, void 0, void 0, function* () {
|
|
595
666
|
yield this.withProjectLock(root, () => __awaiter(this, void 0, void 0, function* () {
|
|
@@ -598,51 +669,24 @@ class Daemon {
|
|
|
598
669
|
(0, ipc_handler_1.writeDone)(conn, { ok: false, error: "daemon resources not ready" });
|
|
599
670
|
return;
|
|
600
671
|
}
|
|
601
|
-
// Pause the project's batch processor during full index
|
|
602
|
-
const processor = this.processors.get(root);
|
|
603
|
-
if (processor) {
|
|
604
|
-
yield processor.close();
|
|
605
|
-
this.processors.delete(root);
|
|
606
|
-
}
|
|
607
|
-
const sub = this.subscriptions.get(root);
|
|
608
|
-
if (sub) {
|
|
609
|
-
yield sub.unsubscribe();
|
|
610
|
-
this.subscriptions.delete(root);
|
|
611
|
-
}
|
|
612
672
|
const ac = new AbortController();
|
|
613
673
|
conn.on("close", () => ac.abort());
|
|
614
674
|
this.shutdownAbortControllers.add(ac);
|
|
615
675
|
this.vectorDb.pauseMaintenanceLoop();
|
|
616
676
|
const stopHeartbeat = (0, ipc_handler_1.startHeartbeat)(conn);
|
|
617
677
|
let lastProgressTime = 0;
|
|
618
|
-
// Mark this root as full-indexing so concurrent searches get a
|
|
619
|
-
// partial-result footer (Phase 6); seeded at 0/0 until the first tick.
|
|
620
|
-
this.indexProgress.set(root, { processed: 0, total: 0 });
|
|
621
678
|
try {
|
|
622
|
-
const result = yield (
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
total: info.total,
|
|
634
|
-
});
|
|
635
|
-
const now = Date.now();
|
|
636
|
-
if (now - lastProgressTime < 100)
|
|
637
|
-
return;
|
|
638
|
-
lastProgressTime = now;
|
|
639
|
-
(0, ipc_handler_1.writeProgress)(conn, {
|
|
640
|
-
processed: info.processed,
|
|
641
|
-
indexed: info.indexed,
|
|
642
|
-
total: info.total,
|
|
643
|
-
filePath: info.filePath,
|
|
644
|
-
});
|
|
645
|
-
},
|
|
679
|
+
const result = yield this.reindexOneProject(root, opts, ac.signal, (info) => {
|
|
680
|
+
const now = Date.now();
|
|
681
|
+
if (now - lastProgressTime < 100)
|
|
682
|
+
return;
|
|
683
|
+
lastProgressTime = now;
|
|
684
|
+
(0, ipc_handler_1.writeProgress)(conn, {
|
|
685
|
+
processed: info.processed,
|
|
686
|
+
indexed: info.indexed,
|
|
687
|
+
total: info.total,
|
|
688
|
+
filePath: info.filePath,
|
|
689
|
+
});
|
|
646
690
|
});
|
|
647
691
|
stopHeartbeat();
|
|
648
692
|
(0, ipc_handler_1.writeDone)(conn, {
|
|
@@ -661,22 +705,100 @@ class Daemon {
|
|
|
661
705
|
}
|
|
662
706
|
finally {
|
|
663
707
|
stopHeartbeat();
|
|
664
|
-
this.indexProgress.delete(root);
|
|
665
708
|
this.shutdownAbortControllers.delete(ac);
|
|
666
709
|
(_a = this.vectorDb) === null || _a === void 0 ? void 0 : _a.resumeMaintenanceLoop();
|
|
667
|
-
// Re-enable watcher (skip if shutting down)
|
|
668
|
-
if (!this.shuttingDown) {
|
|
669
|
-
try {
|
|
670
|
-
yield this.watchProject(root);
|
|
671
|
-
}
|
|
672
|
-
catch (err) {
|
|
673
|
-
console.error(`[daemon] Failed to re-watch ${path.basename(root)}:`, err);
|
|
674
|
-
}
|
|
675
|
-
}
|
|
676
710
|
}
|
|
677
711
|
}));
|
|
678
712
|
});
|
|
679
713
|
}
|
|
714
|
+
/**
|
|
715
|
+
* Global recovery for a physical table-width mismatch (the chosen "global
|
|
716
|
+
* rebuild" strategy): drop the shared `chunks` table and re-index every
|
|
717
|
+
* registered project at the configured dim. The table is fixed-width at
|
|
718
|
+
* creation, so this is the only way to move it from e.g. 384d to 768d after a
|
|
719
|
+
* tier change. Streams per-project progress over `conn`. Each project is
|
|
720
|
+
* reindexed under its own lock with `reset: true`, which also clears that
|
|
721
|
+
* project's stale MetaCache entries so everything re-embeds into the freshly
|
|
722
|
+
* recreated (lazily, at config width) table.
|
|
723
|
+
*/
|
|
724
|
+
repairRebuild(conn) {
|
|
725
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
726
|
+
var _a;
|
|
727
|
+
if (!this.vectorDb || !this.metaCache) {
|
|
728
|
+
(0, ipc_handler_1.writeDone)(conn, { ok: false, error: "daemon resources not ready" });
|
|
729
|
+
return;
|
|
730
|
+
}
|
|
731
|
+
const ac = new AbortController();
|
|
732
|
+
conn.on("close", () => ac.abort());
|
|
733
|
+
this.shutdownAbortControllers.add(ac);
|
|
734
|
+
this.vectorDb.pauseMaintenanceLoop();
|
|
735
|
+
const stopHeartbeat = (0, ipc_handler_1.startHeartbeat)(conn);
|
|
736
|
+
// Reindex every project the daemon would normally index on startup.
|
|
737
|
+
const projects = (0, project_registry_1.listProjects)().filter((p) => p.status === "indexed" || p.status === "pending");
|
|
738
|
+
const globalConfig = (0, index_config_1.readGlobalConfig)();
|
|
739
|
+
let rebuilt = 0;
|
|
740
|
+
let totalIndexed = 0;
|
|
741
|
+
let lastProgressTime = 0;
|
|
742
|
+
try {
|
|
743
|
+
// Drop the shared table — recreated lazily at the configured width on the
|
|
744
|
+
// first insert of the reindex below.
|
|
745
|
+
(0, ipc_handler_1.writeProgress)(conn, { phase: "drop", projectsTotal: projects.length });
|
|
746
|
+
yield this.vectorDb.drop();
|
|
747
|
+
for (const p of projects) {
|
|
748
|
+
if (ac.signal.aborted)
|
|
749
|
+
break;
|
|
750
|
+
const name = p.name || path.basename(p.root);
|
|
751
|
+
yield this.withProjectLock(p.root, () => __awaiter(this, void 0, void 0, function* () {
|
|
752
|
+
const result = yield this.reindexOneProject(p.root, { reset: true }, ac.signal, (info) => {
|
|
753
|
+
const now = Date.now();
|
|
754
|
+
if (now - lastProgressTime < 100)
|
|
755
|
+
return;
|
|
756
|
+
lastProgressTime = now;
|
|
757
|
+
(0, ipc_handler_1.writeProgress)(conn, {
|
|
758
|
+
phase: "reindex",
|
|
759
|
+
project: name,
|
|
760
|
+
projectsDone: rebuilt,
|
|
761
|
+
projectsTotal: projects.length,
|
|
762
|
+
processed: info.processed,
|
|
763
|
+
indexed: info.indexed,
|
|
764
|
+
total: info.total,
|
|
765
|
+
filePath: info.filePath,
|
|
766
|
+
});
|
|
767
|
+
});
|
|
768
|
+
totalIndexed += result.indexed;
|
|
769
|
+
const proj = (0, project_registry_1.getProject)(p.root);
|
|
770
|
+
if (proj) {
|
|
771
|
+
(0, project_registry_1.registerProject)(Object.assign(Object.assign({}, proj), { vectorDim: globalConfig.vectorDim, modelTier: globalConfig.modelTier, embedMode: globalConfig.embedMode, lastIndexed: new Date().toISOString(), chunkCount: result.indexed, status: "indexed", chunkerVersion: config_1.CONFIG.CHUNKER_VERSION }));
|
|
772
|
+
}
|
|
773
|
+
}));
|
|
774
|
+
rebuilt++;
|
|
775
|
+
}
|
|
776
|
+
stopHeartbeat();
|
|
777
|
+
(0, ipc_handler_1.writeDone)(conn, {
|
|
778
|
+
ok: true,
|
|
779
|
+
projects: rebuilt,
|
|
780
|
+
projectsTotal: projects.length,
|
|
781
|
+
indexed: totalIndexed,
|
|
782
|
+
});
|
|
783
|
+
}
|
|
784
|
+
catch (err) {
|
|
785
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
786
|
+
console.error("[daemon] repairRebuild failed:", msg);
|
|
787
|
+
stopHeartbeat();
|
|
788
|
+
(0, ipc_handler_1.writeDone)(conn, {
|
|
789
|
+
ok: false,
|
|
790
|
+
error: msg,
|
|
791
|
+
projects: rebuilt,
|
|
792
|
+
indexed: totalIndexed,
|
|
793
|
+
});
|
|
794
|
+
}
|
|
795
|
+
finally {
|
|
796
|
+
stopHeartbeat();
|
|
797
|
+
this.shutdownAbortControllers.delete(ac);
|
|
798
|
+
(_a = this.vectorDb) === null || _a === void 0 ? void 0 : _a.resumeMaintenanceLoop();
|
|
799
|
+
}
|
|
800
|
+
});
|
|
801
|
+
}
|
|
680
802
|
removeProject(root, conn) {
|
|
681
803
|
return __awaiter(this, void 0, void 0, function* () {
|
|
682
804
|
yield this.withProjectLock(root, () => __awaiter(this, void 0, void 0, function* () {
|
|
@@ -839,6 +961,10 @@ class Daemon {
|
|
|
839
961
|
return;
|
|
840
962
|
this.shuttingDown = true;
|
|
841
963
|
console.log("[daemon] Shutting down...");
|
|
964
|
+
// Announce graceful shutdown BEFORE dropping the liveness markers below, so a
|
|
965
|
+
// successor spawned during the (possibly long) drain sees the draining marker
|
|
966
|
+
// and defers instead of SIGKILLing us mid-cleanup.
|
|
967
|
+
(0, daemon_client_1.writeDrainingMarker)(process.pid);
|
|
842
968
|
// Drop external liveness markers FIRST so the next daemon start isn't
|
|
843
969
|
// fooled by leftover state if the long cleanup below is interrupted
|
|
844
970
|
// (uncaught exception, second SIGTERM, OOM kill mid-shutdown). The
|
|
@@ -916,6 +1042,9 @@ class Daemon {
|
|
|
916
1042
|
const pid = (0, daemon_launcher_1.spawnDaemon)();
|
|
917
1043
|
console.log(`[daemon] Spawned successor daemon${pid ? ` (PID: ${pid})` : " (spawn failed)"}`);
|
|
918
1044
|
}
|
|
1045
|
+
// Cleanly drained — drop the marker so a later start doesn't defer to a
|
|
1046
|
+
// process that's already gone (it self-expires after DRAIN_GRACE_MS anyway).
|
|
1047
|
+
(0, daemon_client_1.clearDrainingMarker)();
|
|
919
1048
|
console.log("[daemon] Shutdown complete");
|
|
920
1049
|
});
|
|
921
1050
|
}
|
|
@@ -105,6 +105,12 @@ function handleCommand(daemon, cmd, conn) {
|
|
|
105
105
|
var _a, _b, _c, _d, _e;
|
|
106
106
|
try {
|
|
107
107
|
(0, logger_1.debug)("daemon", `ipc cmd=${cmd.cmd}${cmd.root ? ` root=${cmd.root}` : ""}`);
|
|
108
|
+
// The socket listens before LanceDB/MetaCache are open so liveness probes
|
|
109
|
+
// succeed during slow startup. Gate resource-dependent commands until those
|
|
110
|
+
// stores exist; ping/shutdown must always work (probes + restart).
|
|
111
|
+
if (cmd.cmd !== "ping" && cmd.cmd !== "shutdown" && !daemon.isReady()) {
|
|
112
|
+
return { ok: false, error: "daemon initializing" };
|
|
113
|
+
}
|
|
108
114
|
switch (cmd.cmd) {
|
|
109
115
|
case "ping":
|
|
110
116
|
return {
|
|
@@ -222,6 +228,12 @@ function handleCommand(daemon, cmd, conn) {
|
|
|
222
228
|
daemon.removeProject(root, conn);
|
|
223
229
|
return null;
|
|
224
230
|
}
|
|
231
|
+
case "repair": {
|
|
232
|
+
// Global drop-and-rebuild recovery for a physical table-width mismatch.
|
|
233
|
+
// Streams per-project progress; daemon manages the connection.
|
|
234
|
+
daemon.repairRebuild(conn);
|
|
235
|
+
return null;
|
|
236
|
+
}
|
|
225
237
|
case "summarize": {
|
|
226
238
|
const root = String(cmd.root || "");
|
|
227
239
|
if (!root)
|
|
@@ -89,8 +89,19 @@ class ProcessManager {
|
|
|
89
89
|
(0, logger_1.log)("daemon", "No stale processes found");
|
|
90
90
|
return;
|
|
91
91
|
}
|
|
92
|
+
// A peer that's gracefully shutting down has already dropped its
|
|
93
|
+
// socket/PID/lock (so the liveness probes below read "stale") yet is still
|
|
94
|
+
// mid-cleanup — destroying workers, closing LanceDB. Killing it now corrupts
|
|
95
|
+
// that teardown. Defer to its draining marker: don't kill it, and don't
|
|
96
|
+
// sweep its workers (it's reaping them itself); just take over the free lock.
|
|
97
|
+
let anyDraining = false;
|
|
92
98
|
for (const pid of daemonPids) {
|
|
93
99
|
(0, logger_1.log)("daemon", `found daemon PID:${pid}, checking liveness...`);
|
|
100
|
+
if ((0, daemon_client_1.isDaemonDraining)(pid)) {
|
|
101
|
+
anyDraining = true;
|
|
102
|
+
(0, logger_1.log)("daemon", `daemon PID:${pid} is gracefully draining — leaving it to exit, taking over`);
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
94
105
|
// A busy daemon (mid-index, compaction, big LMDB write) can block the
|
|
95
106
|
// event loop long enough to miss a ping. Two independent liveness
|
|
96
107
|
// probes — if either says "alive", defer to the running peer instead
|
|
@@ -109,9 +120,16 @@ class ProcessManager {
|
|
|
109
120
|
}
|
|
110
121
|
// 2. Kill orphaned workers from previous daemon instances.
|
|
111
122
|
// Safe because this runs before the new daemon's worker pool is initialized.
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
123
|
+
// Skipped while a peer is draining — those workers belong to it and it's
|
|
124
|
+
// tearing them down; sweeping them here would race its own cleanup.
|
|
125
|
+
if (anyDraining) {
|
|
126
|
+
(0, logger_1.log)("daemon", "skipping orphan-worker sweep — a peer is draining its own workers");
|
|
127
|
+
}
|
|
128
|
+
else {
|
|
129
|
+
for (const pid of workerPids) {
|
|
130
|
+
(0, logger_1.log)("daemon", `killing orphaned worker PID:${pid}`);
|
|
131
|
+
yield (0, process_1.killProcess)(pid);
|
|
132
|
+
}
|
|
115
133
|
}
|
|
116
134
|
(0, logger_1.log)("daemon", `Cleaned up ${daemonPids.length} stale daemon(s), ${workerPids.length} orphaned worker(s)`);
|
|
117
135
|
});
|
|
@@ -368,12 +368,15 @@ class WatcherManager {
|
|
|
368
368
|
const bn = path.basename(absPath).toLowerCase();
|
|
369
369
|
if (!INDEXABLE_EXTENSIONS.has(ext) && !INDEXABLE_EXTENSIONS.has(bn))
|
|
370
370
|
continue;
|
|
371
|
-
seenPaths.add(absPath);
|
|
372
371
|
try {
|
|
373
372
|
const stats = yield fs.promises.stat(absPath);
|
|
374
|
-
// Skip files that are too large or empty — they'll never be indexed
|
|
373
|
+
// Skip files that are too large or empty — they'll never be indexed.
|
|
374
|
+
// Leave them OUT of seenPaths so a file that became empty/oversized is
|
|
375
|
+
// treated as deleted by the purge sweep below and its now-stale chunks
|
|
376
|
+
// are removed, rather than lingering as unsearchable-but-present rows.
|
|
375
377
|
if (stats.size === 0 || stats.size > MAX_FILE_SIZE_BYTES)
|
|
376
378
|
continue;
|
|
379
|
+
seenPaths.add(absPath);
|
|
377
380
|
const cached = metaCache.get(absPath);
|
|
378
381
|
if (!isFileCached(cached, stats)) {
|
|
379
382
|
// Fast path: if only mtime changed but size is identical and we have a hash,
|
|
@@ -439,6 +442,19 @@ class WatcherManager {
|
|
|
439
442
|
}
|
|
440
443
|
unwatchProject(root) {
|
|
441
444
|
return __awaiter(this, void 0, void 0, function* () {
|
|
445
|
+
// Stop poll-mode timers + their FSEvents recovery probe first, so a removed
|
|
446
|
+
// project can't keep scanning until full daemon shutdown. These live
|
|
447
|
+
// independently of the processor, so clear them even on the early return.
|
|
448
|
+
const pollInterval = this.pollIntervals.get(root);
|
|
449
|
+
if (pollInterval) {
|
|
450
|
+
clearInterval(pollInterval);
|
|
451
|
+
this.pollIntervals.delete(root);
|
|
452
|
+
}
|
|
453
|
+
const recoveryTimer = this.pollRecoveryTimers.get(root);
|
|
454
|
+
if (recoveryTimer) {
|
|
455
|
+
clearInterval(recoveryTimer);
|
|
456
|
+
this.pollRecoveryTimers.delete(root);
|
|
457
|
+
}
|
|
442
458
|
const processor = this.deps.processors.get(root);
|
|
443
459
|
if (!processor)
|
|
444
460
|
return;
|
|
@@ -27,10 +27,10 @@ class GraphBuilder {
|
|
|
27
27
|
scopeWhere(condition) {
|
|
28
28
|
let result = condition;
|
|
29
29
|
if (this.pathPrefix) {
|
|
30
|
-
result = `${result} AND
|
|
30
|
+
result = `${result} AND ${(0, filter_builder_1.pathStartsWith)(this.pathPrefix)}`;
|
|
31
31
|
}
|
|
32
32
|
for (const ex of this.excludePrefixes) {
|
|
33
|
-
result = `${result} AND
|
|
33
|
+
result = `${result} AND ${(0, filter_builder_1.pathNotStartsWith)(ex)}`;
|
|
34
34
|
}
|
|
35
35
|
return result;
|
|
36
36
|
}
|
package/dist/lib/graph/impact.js
CHANGED
|
@@ -66,10 +66,10 @@ function expandFileSymbols(symbols, vectorDb, projectRoot, excludePrefixes) {
|
|
|
66
66
|
return symbols;
|
|
67
67
|
const table = yield vectorDb.ensureTable();
|
|
68
68
|
const prefix = projectRoot.endsWith("/") ? projectRoot : `${projectRoot}/`;
|
|
69
|
-
let where = `array_contains(defined_symbols, '${(0, filter_builder_1.escapeSqlString)(symbols[0])}') AND
|
|
69
|
+
let where = `array_contains(defined_symbols, '${(0, filter_builder_1.escapeSqlString)(symbols[0])}') AND ${(0, filter_builder_1.pathStartsWith)(prefix)}`;
|
|
70
70
|
for (const ex of excludePrefixes !== null && excludePrefixes !== void 0 ? excludePrefixes : []) {
|
|
71
71
|
const exNorm = ex.endsWith("/") ? ex : `${ex}/`;
|
|
72
|
-
where += ` AND
|
|
72
|
+
where += ` AND ${(0, filter_builder_1.pathNotStartsWith)(exNorm)}`;
|
|
73
73
|
}
|
|
74
74
|
// Find the file that defines this symbol
|
|
75
75
|
const defRows = yield table
|
|
@@ -146,10 +146,10 @@ function findImportFallbackTests(expandedSymbols, originalSymbols, vectorDb, pro
|
|
|
146
146
|
// test body's referenced_symbols ends up empty).
|
|
147
147
|
const table = yield vectorDb.ensureTable();
|
|
148
148
|
const prefix = projectRoot.endsWith("/") ? projectRoot : `${projectRoot}/`;
|
|
149
|
-
let pathScope =
|
|
149
|
+
let pathScope = (0, filter_builder_1.pathStartsWith)(prefix);
|
|
150
150
|
for (const ex of excludePrefixes !== null && excludePrefixes !== void 0 ? excludePrefixes : []) {
|
|
151
151
|
const exNorm = ex.endsWith("/") ? ex : `${ex}/`;
|
|
152
|
-
pathScope += ` AND
|
|
152
|
+
pathScope += ` AND ${(0, filter_builder_1.pathNotStartsWith)(exNorm)}`;
|
|
153
153
|
}
|
|
154
154
|
// Textual matching runs on the ORIGINAL targets only: matching co-defined
|
|
155
155
|
// file symbols (helpers like `log`) textually drags in every test file
|
|
@@ -203,10 +203,10 @@ function walkCallers(symbol, graphBuilder, testHits, currentHop, maxDepth, visit
|
|
|
203
203
|
function findDependents(symbols_1, vectorDb_1, projectRoot_1, excludePaths_1) {
|
|
204
204
|
return __awaiter(this, arguments, void 0, function* (symbols, vectorDb, projectRoot, excludePaths, limit = 10, excludePrefixes) {
|
|
205
205
|
const table = yield vectorDb.ensureTable();
|
|
206
|
-
let pathScope =
|
|
206
|
+
let pathScope = (0, filter_builder_1.pathStartsWith)(`${projectRoot}/`);
|
|
207
207
|
for (const ex of excludePrefixes !== null && excludePrefixes !== void 0 ? excludePrefixes : []) {
|
|
208
208
|
const exNorm = ex.endsWith("/") ? ex : `${ex}/`;
|
|
209
|
-
pathScope += ` AND
|
|
209
|
+
pathScope += ` AND ${(0, filter_builder_1.pathNotStartsWith)(exNorm)}`;
|
|
210
210
|
}
|
|
211
211
|
const counts = new Map();
|
|
212
212
|
for (const sym of symbols) {
|
|
@@ -177,8 +177,17 @@ class ProjectBatchProcessor {
|
|
|
177
177
|
// change or add
|
|
178
178
|
try {
|
|
179
179
|
const stats = yield fs.promises.stat(absPath);
|
|
180
|
-
if (!(0, file_utils_1.isIndexableFile)(absPath, stats.size))
|
|
180
|
+
if (!(0, file_utils_1.isIndexableFile)(absPath, stats.size)) {
|
|
181
|
+
// File became non-indexable (emptied or now too large). If we
|
|
182
|
+
// indexed it before, drop its chunks + meta so search stops
|
|
183
|
+
// returning stale content; otherwise there's nothing to clean up.
|
|
184
|
+
if (this.metaCache.get(absPath)) {
|
|
185
|
+
deletes.push(absPath);
|
|
186
|
+
metaDeletes.push(absPath);
|
|
187
|
+
reindexed++;
|
|
188
|
+
}
|
|
181
189
|
continue;
|
|
190
|
+
}
|
|
182
191
|
const cached = this.metaCache.get(absPath);
|
|
183
192
|
if ((0, cache_check_1.isFileCached)(cached, stats)) {
|
|
184
193
|
continue;
|