grepmax 0.20.0 → 0.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +41 -0
- package/dist/commands/codex.js +5 -2
- package/dist/commands/config.js +7 -1
- package/dist/commands/doctor.js +37 -7
- package/dist/commands/droid.js +61 -9
- package/dist/commands/repair.js +186 -0
- package/dist/commands/search-run.js +12 -1
- package/dist/commands/watch.js +2 -1
- package/dist/config.js +45 -1
- package/dist/index.js +2 -0
- package/dist/lib/daemon/daemon.js +167 -48
- package/dist/lib/daemon/ipc-handler.js +6 -0
- package/dist/lib/daemon/process-manager.js +21 -3
- package/dist/lib/store/vector-db.js +29 -1
- package/dist/lib/utils/daemon-client.js +53 -0
- package/dist/lib/utils/stale-hint.js +7 -2
- package/package.json +1 -1
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
- package/plugins/grepmax/agents/semantic-explore.md +1 -1
- package/plugins/grepmax/skills/grepmax/SKILL.md +1 -1
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"name": "grepmax",
|
|
11
11
|
"source": "./plugins/grepmax",
|
|
12
12
|
"description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
|
|
13
|
-
"version": "0.
|
|
13
|
+
"version": "0.21.1",
|
|
14
14
|
"author": {
|
|
15
15
|
"name": "Robert Owens",
|
|
16
16
|
"email": "robowens@me.com"
|
package/README.md
CHANGED
|
@@ -289,6 +289,45 @@ All data lives in `~/.gmax/`:
|
|
|
289
289
|
}
|
|
290
290
|
```
|
|
291
291
|
|
|
292
|
+
### Model Tier
|
|
293
|
+
|
|
294
|
+
gmax embeds with IBM's Granite r2 code-embedding models. Two tiers are available:
|
|
295
|
+
|
|
296
|
+
| Tier | Model | Dim | Params |
|
|
297
|
+
| --- | --- | --- | --- |
|
|
298
|
+
| `small` (default) | `granite-embedding-small-english-r2` | 384 | 47M |
|
|
299
|
+
| `standard` | `granite-embedding-english-r2` | 768 | 149M |
|
|
300
|
+
|
|
301
|
+
**384d is the default — and benchmarking says keep it.** On gmax's own 97-case
|
|
302
|
+
retrieval eval (`pnpm bench:recall`), the larger 768d model scored **~10 points
|
|
303
|
+
*worse* on Recall@10** than 384d, consistently on both the MLX GPU and ONNX CPU
|
|
304
|
+
embedding paths, with and without ColBERT rerank:
|
|
305
|
+
|
|
306
|
+
| Model | Recall@10 | MRR@10 |
|
|
307
|
+
| --- | --- | --- |
|
|
308
|
+
| `small` / 384d | **0.72** | **0.51** |
|
|
309
|
+
| `standard` / 768d | 0.62 | 0.44 |
|
|
310
|
+
|
|
311
|
+
<sub>gmax repo, 97 cases, MLX GPU, rerank off (the shipped default). The CPU/q4
|
|
312
|
+
path and the rerank-on path show the same ~0.10 Recall@10 gap, so it isn't a
|
|
313
|
+
quantization artifact — the bigger model just retrieves worse on code here.</sub>
|
|
314
|
+
|
|
315
|
+
Bigger isn't better for this workload: 768d roughly doubles embedding compute,
|
|
316
|
+
worker RAM, and dense-vector storage while *lowering* recall on code search.
|
|
317
|
+
Unless you have a measured reason to switch (e.g. a recall complaint on a very
|
|
318
|
+
large repo — benchmark it first), stay on 384d.
|
|
319
|
+
|
|
320
|
+
To change tiers:
|
|
321
|
+
|
|
322
|
+
```bash
|
|
323
|
+
gmax config --model-tier standard # a dim change needs a full re-embed
|
|
324
|
+
gmax repair --rebuild # drops the shared table, re-embeds every project at the new dim
|
|
325
|
+
```
|
|
326
|
+
|
|
327
|
+
A vector-dimension change can't be applied by a per-project `gmax index --reset`
|
|
328
|
+
— the shared `chunks` table is fixed-width, so `gmax repair --rebuild` is the
|
|
329
|
+
sanctioned path.
|
|
330
|
+
|
|
292
331
|
### Ignoring Files
|
|
293
332
|
|
|
294
333
|
gmax respects `.gitignore` and `.gmaxignore`:
|
|
@@ -339,6 +378,8 @@ GMAX_EVAL_RERANK=1 pnpm bench:oss # toggle ColBERT rerank
|
|
|
339
378
|
|
|
340
379
|
The OSS bench requires the fixture repos to be indexed first — see [`docs/known-limitations.md`](docs/known-limitations.md) for the most recent rerank-on-vs-off comparison across 4 datasets / 131 cases.
|
|
341
380
|
|
|
381
|
+
`pnpm bench:recall` also drives the model-tier comparison behind the 384d default — see [Model Tier](#model-tier) for why the larger 768d model is *not* the default.
|
|
382
|
+
|
|
342
383
|
## Attribution
|
|
343
384
|
|
|
344
385
|
grepmax is built upon the foundation of [mgrep](https://github.com/mixedbread-ai/mgrep) by MixedBread. See the [NOTICE](NOTICE) file for details.
|
package/dist/commands/codex.js
CHANGED
|
@@ -69,8 +69,11 @@ function writeSkillToAgents(skill) {
|
|
|
69
69
|
function installPlugin() {
|
|
70
70
|
return __awaiter(this, void 0, void 0, function* () {
|
|
71
71
|
try {
|
|
72
|
-
// 1. Register MCP tool
|
|
73
|
-
|
|
72
|
+
// 1. Register MCP tool. Codex requires the stdio command after `--`:
|
|
73
|
+
// codex mcp add [OPTIONS] <NAME> -- <COMMAND>...
|
|
74
|
+
// Without the separator the launch command is misparsed. AGENTS.md is only
|
|
75
|
+
// written after this resolves, so a failed registration leaves it untouched.
|
|
76
|
+
yield execAsync("codex mcp add gmax -- gmax mcp", {
|
|
74
77
|
shell,
|
|
75
78
|
env: process.env,
|
|
76
79
|
});
|
package/dist/commands/config.js
CHANGED
|
@@ -84,6 +84,7 @@ Examples:
|
|
|
84
84
|
const newMode = (_c = options.embedMode) !== null && _c !== void 0 ? _c : globalConfig.embedMode;
|
|
85
85
|
const tier = (_d = config_1.MODEL_TIERS[newTier]) !== null && _d !== void 0 ? _d : config_1.MODEL_TIERS.small;
|
|
86
86
|
const tierChanged = newTier !== globalConfig.modelTier;
|
|
87
|
+
const dimChanged = tier.vectorDim !== globalConfig.vectorDim;
|
|
87
88
|
(0, index_config_1.writeGlobalConfig)({
|
|
88
89
|
modelTier: newTier,
|
|
89
90
|
vectorDim: tier.vectorDim,
|
|
@@ -98,7 +99,12 @@ Examples:
|
|
|
98
99
|
});
|
|
99
100
|
console.log(`Updated: embed-mode=${newMode}, model-tier=${newTier} (${tier.vectorDim}d)`);
|
|
100
101
|
if (tierChanged) {
|
|
101
|
-
|
|
102
|
+
// A dimension change can't be fixed by a per-project `gmax index --reset`:
|
|
103
|
+
// the shared `chunks` table is fixed-width, so it must be dropped and
|
|
104
|
+
// rebuilt. A same-dim model swap (future tiers) can use a per-project reset.
|
|
105
|
+
console.log(dimChanged
|
|
106
|
+
? `⚠️ Model tier changed (${globalConfig.vectorDim}d → ${tier.vectorDim}d). The shared vector table is fixed-width — run \`${config_1.REBUILD_COMMAND}\` to drop it and re-embed every project at the new dim.`
|
|
107
|
+
: "⚠️ Model tier changed — run `gmax index --reset` per project to re-embed with the new model.");
|
|
102
108
|
}
|
|
103
109
|
yield (0, exit_1.gracefulExit)();
|
|
104
110
|
}));
|
package/dist/commands/doctor.js
CHANGED
|
@@ -179,6 +179,13 @@ exports.doctor = new commander_1.Command("doctor")
|
|
|
179
179
|
const db = new VectorDB(config_1.PATHS.lancedbDir);
|
|
180
180
|
const table = yield db.ensureTable();
|
|
181
181
|
const totalChunks = yield table.countRows();
|
|
182
|
+
// Physical schema-width check: the shared `chunks` table is fixed-width at
|
|
183
|
+
// creation, so a tier/dim change strands it at the old width and every
|
|
184
|
+
// write throws. This is independent of the per-project registry drift
|
|
185
|
+
// checked below — the table can match the registry yet still be physically
|
|
186
|
+
// stranded — so we surface it as its own line.
|
|
187
|
+
const physicalDim = yield db.getSchemaVectorDim();
|
|
188
|
+
const schemaGap = (0, config_1.describeSchemaDimGap)(physicalDim, globalConfig.vectorDim);
|
|
182
189
|
// Summary coverage (existing check)
|
|
183
190
|
if (!opts.agent && totalChunks > 0) {
|
|
184
191
|
const withSummary = (yield table
|
|
@@ -275,8 +282,13 @@ exports.doctor = new commander_1.Command("doctor")
|
|
|
275
282
|
`orphaned=${orphanedProjects.length}`,
|
|
276
283
|
`stale_chunker=${staleChunkerProjects.length}`,
|
|
277
284
|
`stale_embedding=${staleEmbeddingProjects.length}`,
|
|
285
|
+
`schema_dim=${physicalDim !== null && physicalDim !== void 0 ? physicalDim : "none"}`,
|
|
286
|
+
`schema_dim_ok=${schemaGap ? "false" : "true"}`,
|
|
278
287
|
];
|
|
279
288
|
console.log(fields.join("\t"));
|
|
289
|
+
if (schemaGap) {
|
|
290
|
+
console.log((0, config_1.schemaDimAgentRow)(schemaGap));
|
|
291
|
+
}
|
|
280
292
|
for (const p of staleChunkerProjects) {
|
|
281
293
|
const gap = (0, config_1.describeChunkerGap)(p.chunkerVersion);
|
|
282
294
|
if (!gap)
|
|
@@ -307,12 +319,24 @@ exports.doctor = new commander_1.Command("doctor")
|
|
|
307
319
|
`current_dim=${gap.toDim}`,
|
|
308
320
|
`dim_changed=${gap.dimChanged}`,
|
|
309
321
|
`severity=${gap.severity}`,
|
|
310
|
-
|
|
322
|
+
// A dim change can't be fixed by a per-project reset (the shared
|
|
323
|
+
// table is fixed-width) — point at the global rebuild instead.
|
|
324
|
+
`fix=${gap.dimChanged ? config_1.REBUILD_COMMAND : `gmax index --reset (in ${p.root})`}`,
|
|
311
325
|
].join("\t"));
|
|
312
326
|
}
|
|
313
327
|
}
|
|
314
328
|
else {
|
|
315
329
|
console.log("\nIndex Health\n");
|
|
330
|
+
// Physical schema width — a mismatch means every write throws until a
|
|
331
|
+
// global rebuild (the shared fixed-width table can't be reshaped by a
|
|
332
|
+
// per-project reset). Surfaced first because it's the most severe.
|
|
333
|
+
if (schemaGap) {
|
|
334
|
+
console.log(`FAIL Schema: vector table is ${schemaGap.tableDim}d, config expects ${schemaGap.configDim}d`);
|
|
335
|
+
console.log(` run '${config_1.REBUILD_COMMAND}' (drops + reindexes all projects at the new width)`);
|
|
336
|
+
}
|
|
337
|
+
else if (physicalDim) {
|
|
338
|
+
console.log(`ok Schema: vector table is ${physicalDim}d`);
|
|
339
|
+
}
|
|
316
340
|
// Disk space
|
|
317
341
|
if (diskLevel !== "ok") {
|
|
318
342
|
console.log(`WARN Disk: ${formatSize(availBytes)} available (${diskLevel})`);
|
|
@@ -370,17 +394,21 @@ exports.doctor = new commander_1.Command("doctor")
|
|
|
370
394
|
}
|
|
371
395
|
// Index built with a different embedding model/dim than the current
|
|
372
396
|
// config. A dim change is breaking (search scores are invalid until a
|
|
373
|
-
// re-embed); a same-dim model swap is additive.
|
|
374
|
-
//
|
|
375
|
-
//
|
|
376
|
-
//
|
|
397
|
+
// re-embed); a same-dim model swap is additive. Recovery differs by kind:
|
|
398
|
+
// a same-dim model swap is fixed by a per-project `gmax index --reset`,
|
|
399
|
+
// but a dim change can't be — the shared table is fixed-width, so it
|
|
400
|
+
// needs the global rebuild (see the Schema check above).
|
|
377
401
|
if (staleEmbeddingProjects.length > 0) {
|
|
378
402
|
const gaps = staleEmbeddingProjects.map((p) => (0, config_1.describeEmbeddingGap)({ modelTier: p.modelTier, vectorDim: p.vectorDim }, {
|
|
379
403
|
modelTier: globalConfig.modelTier,
|
|
380
404
|
vectorDim: globalConfig.vectorDim,
|
|
381
405
|
}));
|
|
382
406
|
const anyBreaking = gaps.some((g) => (g === null || g === void 0 ? void 0 : g.severity) === "breaking");
|
|
383
|
-
|
|
407
|
+
const anyDimChange = gaps.some((g) => g === null || g === void 0 ? void 0 : g.dimChanged);
|
|
408
|
+
const headerFix = anyDimChange
|
|
409
|
+
? `run '${config_1.REBUILD_COMMAND}' (dim change needs a full rebuild)`
|
|
410
|
+
: "run 'gmax index --reset' per project";
|
|
411
|
+
console.log(`${anyBreaking ? "WARN" : "INFO"} Stale embedding: ${staleEmbeddingProjects.length} project(s) indexed with a different embedding model/dim — ${headerFix}`);
|
|
384
412
|
staleEmbeddingProjects.forEach((p, i) => {
|
|
385
413
|
const gap = gaps[i];
|
|
386
414
|
if (!gap)
|
|
@@ -389,7 +417,9 @@ exports.doctor = new commander_1.Command("doctor")
|
|
|
389
417
|
? `${gap.fromDim}d→${gap.toDim}d`
|
|
390
418
|
: `model ${gap.fromModel}→${gap.toModel}`;
|
|
391
419
|
console.log(` - ${p.name || path.basename(p.root)} (${change}, ${gap.severity})`);
|
|
392
|
-
console.log(
|
|
420
|
+
console.log(gap.dimChanged
|
|
421
|
+
? ` run '${config_1.REBUILD_COMMAND}'`
|
|
422
|
+
: ` run 'gmax index --reset' in ${p.root}`);
|
|
393
423
|
});
|
|
394
424
|
}
|
|
395
425
|
// Projects
|
package/dist/commands/droid.js
CHANGED
|
@@ -69,17 +69,50 @@ function parseJsonWithComments(content) {
|
|
|
69
69
|
.split("\n")
|
|
70
70
|
.map((line) => line.replace(/^\s*\/\/.*$/, ""))
|
|
71
71
|
.join("\n");
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
}
|
|
75
|
-
|
|
72
|
+
// An empty / whitespace-only file is a legitimate "no settings yet" -> {}.
|
|
73
|
+
// Anything else that fails to parse is a real, user-owned file we must NOT
|
|
74
|
+
// silently coerce to {} (that would clobber it on the next save) — let the
|
|
75
|
+
// caller decide.
|
|
76
|
+
if (stripped.trim() === "")
|
|
76
77
|
return {};
|
|
77
|
-
|
|
78
|
+
return JSON.parse(stripped);
|
|
78
79
|
}
|
|
79
80
|
function loadSettings(settingsPath) {
|
|
80
81
|
if (!node_fs_1.default.existsSync(settingsPath))
|
|
81
82
|
return {};
|
|
82
|
-
|
|
83
|
+
const raw = node_fs_1.default.readFileSync(settingsPath, "utf-8");
|
|
84
|
+
try {
|
|
85
|
+
return parseJsonWithComments(raw);
|
|
86
|
+
}
|
|
87
|
+
catch (err) {
|
|
88
|
+
throw new Error(`Refusing to touch ${settingsPath}: it contains invalid JSON ` +
|
|
89
|
+
`(${err.message}). Fix or remove the file, then re-run.`);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
/** True when a hook entry was installed by gmax — its command points at our
|
|
93
|
+
* hooks dir. Used to remove only gmax entries on uninstall. */
|
|
94
|
+
function isGmaxHookEntry(entry, hooksDir) {
|
|
95
|
+
var _a, _b;
|
|
96
|
+
return (_b = (_a = entry.hooks) === null || _a === void 0 ? void 0 : _a.some((h) => { var _a; return (_a = h.command) === null || _a === void 0 ? void 0 : _a.includes(hooksDir); })) !== null && _b !== void 0 ? _b : false;
|
|
97
|
+
}
|
|
98
|
+
/** Strip gmax hook entries from settings.hooks in place, preserving unrelated
|
|
99
|
+
* user hooks. Returns true if anything was removed. */
|
|
100
|
+
function removeGmaxHooks(settings, hooksDir) {
|
|
101
|
+
if (!settings.hooks)
|
|
102
|
+
return false;
|
|
103
|
+
let changed = false;
|
|
104
|
+
for (const [event, entries] of Object.entries(settings.hooks)) {
|
|
105
|
+
const kept = entries.filter((e) => !isGmaxHookEntry(e, hooksDir));
|
|
106
|
+
if (kept.length !== entries.length)
|
|
107
|
+
changed = true;
|
|
108
|
+
if (kept.length > 0)
|
|
109
|
+
settings.hooks[event] = kept;
|
|
110
|
+
else
|
|
111
|
+
delete settings.hooks[event];
|
|
112
|
+
}
|
|
113
|
+
if (Object.keys(settings.hooks).length === 0)
|
|
114
|
+
delete settings.hooks;
|
|
115
|
+
return changed;
|
|
83
116
|
}
|
|
84
117
|
function saveSettings(settingsPath, settings) {
|
|
85
118
|
node_fs_1.default.mkdirSync(node_path_1.default.dirname(settingsPath), { recursive: true });
|
|
@@ -103,10 +136,14 @@ function mergeHooks(existing, incoming) {
|
|
|
103
136
|
function installPlugin() {
|
|
104
137
|
return __awaiter(this, void 0, void 0, function* () {
|
|
105
138
|
const root = resolveDroidRoot();
|
|
139
|
+
const settingsPath = node_path_1.default.join(root, "settings.json");
|
|
140
|
+
// Validate/parse settings BEFORE writing anything. A malformed, user-owned
|
|
141
|
+
// settings.json aborts the install here — otherwise we'd either clobber it
|
|
142
|
+
// with {} or leave half-written hook scripts behind on the failure path.
|
|
143
|
+
const settings = loadSettings(settingsPath);
|
|
106
144
|
const gmaxBin = resolveGmaxBin();
|
|
107
145
|
const hooksDir = node_path_1.default.join(root, "hooks", "gmax");
|
|
108
146
|
const skillsDir = node_path_1.default.join(root, "skills", "gmax");
|
|
109
|
-
const settingsPath = node_path_1.default.join(root, "settings.json");
|
|
110
147
|
// 1. Install hook scripts (start/stop daemon)
|
|
111
148
|
const startScript = `
|
|
112
149
|
const { execFileSync } = require("child_process");
|
|
@@ -166,7 +203,6 @@ try { execFileSync("gmax", ["watch", "stop", "--all"], { stdio: "ignore", timeou
|
|
|
166
203
|
},
|
|
167
204
|
],
|
|
168
205
|
};
|
|
169
|
-
const settings = loadSettings(settingsPath);
|
|
170
206
|
settings.enableHooks = true;
|
|
171
207
|
settings.allowBackgroundProcesses = true;
|
|
172
208
|
settings.hooks = mergeHooks(settings.hooks, hookConfig);
|
|
@@ -179,12 +215,28 @@ function uninstallPlugin() {
|
|
|
179
215
|
const root = resolveDroidRoot();
|
|
180
216
|
const hooksDir = node_path_1.default.join(root, "hooks", "gmax");
|
|
181
217
|
const skillsDir = node_path_1.default.join(root, "skills", "gmax");
|
|
218
|
+
const settingsPath = node_path_1.default.join(root, "settings.json");
|
|
182
219
|
if (node_fs_1.default.existsSync(hooksDir))
|
|
183
220
|
node_fs_1.default.rmSync(hooksDir, { recursive: true, force: true });
|
|
184
221
|
if (node_fs_1.default.existsSync(skillsDir))
|
|
185
222
|
node_fs_1.default.rmSync(skillsDir, { recursive: true, force: true });
|
|
223
|
+
// Remove only gmax hook entries from settings.json, preserving unrelated user
|
|
224
|
+
// hooks. enableHooks/allowBackgroundProcesses are left alone — other hooks may
|
|
225
|
+
// depend on them.
|
|
226
|
+
if (node_fs_1.default.existsSync(settingsPath)) {
|
|
227
|
+
try {
|
|
228
|
+
const settings = loadSettings(settingsPath);
|
|
229
|
+
if (removeGmaxHooks(settings, hooksDir)) {
|
|
230
|
+
saveSettings(settingsPath, settings);
|
|
231
|
+
console.log("✅ Removed gmax hooks from settings.json");
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
catch (err) {
|
|
235
|
+
// Don't clobber an invalid settings file on uninstall either.
|
|
236
|
+
console.warn(`⚠️ Skipped settings cleanup: ${err.message}`);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
186
239
|
console.log("✅ gmax removed from Factory Droid");
|
|
187
|
-
console.log("NOTE: You may want to manually clean up 'hooks' in ~/.factory/settings.json");
|
|
188
240
|
});
|
|
189
241
|
}
|
|
190
242
|
exports.installDroid = new commander_1.Command("install-droid")
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
36
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
37
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
38
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
39
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
40
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
41
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
45
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
46
|
+
};
|
|
47
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
48
|
+
exports.repair = void 0;
|
|
49
|
+
const readline = __importStar(require("node:readline"));
|
|
50
|
+
const commander_1 = require("commander");
|
|
51
|
+
const ora_1 = __importDefault(require("ora"));
|
|
52
|
+
const config_1 = require("../config");
|
|
53
|
+
const index_config_1 = require("../lib/index/index-config");
|
|
54
|
+
const vector_db_1 = require("../lib/store/vector-db");
|
|
55
|
+
const exit_1 = require("../lib/utils/exit");
|
|
56
|
+
const project_registry_1 = require("../lib/utils/project-registry");
|
|
57
|
+
function confirm(message) {
|
|
58
|
+
const rl = readline.createInterface({
|
|
59
|
+
input: process.stdin,
|
|
60
|
+
output: process.stdout,
|
|
61
|
+
});
|
|
62
|
+
return new Promise((resolve) => {
|
|
63
|
+
rl.question(`${message} [y/N] `, (answer) => {
|
|
64
|
+
rl.close();
|
|
65
|
+
resolve(answer.toLowerCase() === "y");
|
|
66
|
+
});
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
exports.repair = new commander_1.Command("repair")
|
|
70
|
+
.description("Repair the centralized index (recover from a schema mismatch)")
|
|
71
|
+
.option("--rebuild", "Drop the shared vector table and re-index every project at the configured embedding dim", false)
|
|
72
|
+
.option("-y, --yes", "Skip the confirmation prompt", false)
|
|
73
|
+
.addHelpText("after", `
|
|
74
|
+
The shared LanceDB \`chunks\` table is fixed-width at creation. Switching model
|
|
75
|
+
tiers (e.g. small 384d -> standard 768d) strands the table at the old width, so
|
|
76
|
+
every write then fails. A per-project \`gmax index --reset\` only deletes rows —
|
|
77
|
+
it can't change the table width. \`${config_1.REBUILD_COMMAND}\` drops the table and
|
|
78
|
+
re-embeds all projects at the current dim.
|
|
79
|
+
|
|
80
|
+
Examples:
|
|
81
|
+
gmax repair Show schema status and what a rebuild would do
|
|
82
|
+
gmax repair --rebuild Drop the table and re-index every project
|
|
83
|
+
gmax repair --rebuild -y Rebuild without the confirmation prompt
|
|
84
|
+
`)
|
|
85
|
+
.action((opts) => __awaiter(void 0, void 0, void 0, function* () {
|
|
86
|
+
var _a, _b, _c;
|
|
87
|
+
try {
|
|
88
|
+
const globalConfig = (0, index_config_1.readGlobalConfig)();
|
|
89
|
+
const configDim = globalConfig.vectorDim;
|
|
90
|
+
// Physical width of the on-disk table (null if no table yet). Opened
|
|
91
|
+
// read-only; safe to inspect even while the daemon holds the table.
|
|
92
|
+
let physicalDim = null;
|
|
93
|
+
const probe = new vector_db_1.VectorDB(config_1.PATHS.lancedbDir);
|
|
94
|
+
try {
|
|
95
|
+
physicalDim = yield probe.getSchemaVectorDim();
|
|
96
|
+
}
|
|
97
|
+
finally {
|
|
98
|
+
yield probe.close();
|
|
99
|
+
}
|
|
100
|
+
const projects = (0, project_registry_1.listProjects)().filter((p) => p.status === "indexed" || p.status === "pending");
|
|
101
|
+
const dimLine = physicalDim == null
|
|
102
|
+
? "Vector table: none yet"
|
|
103
|
+
: physicalDim === configDim
|
|
104
|
+
? `Vector table: ${physicalDim}d (matches config)`
|
|
105
|
+
: `Vector table: ${physicalDim}d, config expects ${configDim}d (MISMATCH)`;
|
|
106
|
+
console.log(dimLine);
|
|
107
|
+
if (!opts.rebuild) {
|
|
108
|
+
if (physicalDim != null && physicalDim !== configDim) {
|
|
109
|
+
console.log(`\nRun '${config_1.REBUILD_COMMAND}' to drop the table and re-index ` +
|
|
110
|
+
`${projects.length} project(s) at ${configDim}d.`);
|
|
111
|
+
}
|
|
112
|
+
else {
|
|
113
|
+
console.log(`\nNothing to repair. Pass --rebuild to force a full drop + re-index anyway.`);
|
|
114
|
+
}
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
if (projects.length === 0) {
|
|
118
|
+
console.log("\nNo indexed projects to rebuild.");
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
const totalChunks = projects.reduce((sum, p) => { var _a; return sum + ((_a = p.chunkCount) !== null && _a !== void 0 ? _a : 0); }, 0);
|
|
122
|
+
console.log(`\nThis will DROP the shared vector table and re-embed ${projects.length} project(s)` +
|
|
123
|
+
(totalChunks > 0
|
|
124
|
+
? ` (~${totalChunks.toLocaleString()} chunks).`
|
|
125
|
+
: "."));
|
|
126
|
+
for (const p of projects) {
|
|
127
|
+
console.log(` - ${p.name}`);
|
|
128
|
+
}
|
|
129
|
+
if (!opts.yes) {
|
|
130
|
+
const ok = yield confirm("\nContinue?");
|
|
131
|
+
if (!ok) {
|
|
132
|
+
console.log("Cancelled.");
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
// The rebuild must run through the daemon — it is the single writer for
|
|
137
|
+
// the shared table. Refuse rather than risk a torn drop alongside a
|
|
138
|
+
// daemon mid-flush.
|
|
139
|
+
const { ensureDaemonRunning, sendStreamingCommand } = yield Promise.resolve().then(() => __importStar(require("../lib/utils/daemon-client")));
|
|
140
|
+
if (!(yield ensureDaemonRunning())) {
|
|
141
|
+
console.error("Could not start the gmax daemon. Start it with 'gmax watch --daemon -b' and retry.");
|
|
142
|
+
process.exitCode = 1;
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
const spinner = (0, ora_1.default)({
|
|
146
|
+
text: "Dropping vector table...",
|
|
147
|
+
isSilent: !process.stdout.isTTY,
|
|
148
|
+
}).start();
|
|
149
|
+
try {
|
|
150
|
+
const done = yield sendStreamingCommand({ cmd: "repair" }, (msg) => {
|
|
151
|
+
var _a, _b, _c, _d;
|
|
152
|
+
if (msg.phase === "drop") {
|
|
153
|
+
spinner.text = "Dropping vector table...";
|
|
154
|
+
}
|
|
155
|
+
else if (msg.phase === "reindex") {
|
|
156
|
+
const doneN = (_a = msg.projectsDone) !== null && _a !== void 0 ? _a : 0;
|
|
157
|
+
const totalN = (_b = msg.projectsTotal) !== null && _b !== void 0 ? _b : projects.length;
|
|
158
|
+
const processed = (_c = msg.processed) !== null && _c !== void 0 ? _c : 0;
|
|
159
|
+
const total = (_d = msg.total) !== null && _d !== void 0 ? _d : 0;
|
|
160
|
+
const counts = total > 0 ? ` ${processed}/${total}` : "";
|
|
161
|
+
spinner.text = `Re-indexing ${msg.project} (${doneN + 1}/${totalN})${counts}`;
|
|
162
|
+
}
|
|
163
|
+
});
|
|
164
|
+
if (!done.ok) {
|
|
165
|
+
spinner.fail("Rebuild failed");
|
|
166
|
+
throw new Error((_a = done.error) !== null && _a !== void 0 ? _a : "daemon repair failed");
|
|
167
|
+
}
|
|
168
|
+
const rebuilt = (_b = done.projects) !== null && _b !== void 0 ? _b : 0;
|
|
169
|
+
const indexed = (_c = done.indexed) !== null && _c !== void 0 ? _c : 0;
|
|
170
|
+
spinner.succeed(`Rebuilt ${rebuilt} project(s) at ${configDim}d • ${indexed.toLocaleString()} chunks`);
|
|
171
|
+
}
|
|
172
|
+
catch (e) {
|
|
173
|
+
if (spinner.isSpinning)
|
|
174
|
+
spinner.fail("Rebuild failed");
|
|
175
|
+
throw e;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
catch (error) {
|
|
179
|
+
const message = error instanceof Error ? error.message : "Unknown error";
|
|
180
|
+
console.error("Failed to repair:", message);
|
|
181
|
+
process.exitCode = 1;
|
|
182
|
+
}
|
|
183
|
+
finally {
|
|
184
|
+
yield (0, exit_1.gracefulExit)();
|
|
185
|
+
}
|
|
186
|
+
}));
|
|
@@ -130,7 +130,18 @@ function runSearch(params) {
|
|
|
130
130
|
if (!indexState && (locked || projectStatus === "pending")) {
|
|
131
131
|
indexState = { indexing: true, pendingFiles: 0 };
|
|
132
132
|
}
|
|
133
|
-
|
|
133
|
+
// Decide first-run auto-index by whether the project being searched has
|
|
134
|
+
// rows — NOT whether the shared store has any rows at all. The store is
|
|
135
|
+
// centralized, so a global hasAnyRows() lets a sibling project's rows
|
|
136
|
+
// suppress this project's first-run index. Cross-project mode keeps the
|
|
137
|
+
// global check: it spans already-indexed projects and must not first-run
|
|
138
|
+
// a single directory just because the cwd happens to be unindexed.
|
|
139
|
+
const crossProject = !!options.allProjects ||
|
|
140
|
+
!!options.projects ||
|
|
141
|
+
!!options.excludeProjects;
|
|
142
|
+
const hasRows = crossProject
|
|
143
|
+
? yield vectorDb.hasAnyRows()
|
|
144
|
+
: yield vectorDb.hasRowsForPath(effectiveRoot);
|
|
134
145
|
const needsSync = options.sync || !hasRows;
|
|
135
146
|
if (needsSync) {
|
|
136
147
|
const isTTY = process.stdout.isTTY;
|
package/dist/commands/watch.js
CHANGED
|
@@ -106,7 +106,8 @@ exports.watch = new commander_1.Command("watch")
|
|
|
106
106
|
if (oldPid) {
|
|
107
107
|
const exited = yield waitForProcessExit(oldPid, 20000);
|
|
108
108
|
if (!exited) {
|
|
109
|
-
console.log(`Old daemon (PID ${oldPid}) still draining after 20s — starting anyway`
|
|
109
|
+
console.log(`Old daemon (PID ${oldPid}) still draining after 20s — starting anyway ` +
|
|
110
|
+
`(its draining marker keeps the successor from killing it mid-cleanup)`);
|
|
110
111
|
}
|
|
111
112
|
}
|
|
112
113
|
else {
|
package/dist/config.js
CHANGED
|
@@ -33,9 +33,11 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.INDEXABLE_EXTENSIONS = exports.FRAGMENT_COMPACT_THRESHOLD = exports.DISK_LOW_BYTES = exports.DISK_CRITICAL_BYTES = exports.MAX_FILE_SIZE_BYTES = exports.PATHS = exports.MAX_WORKER_MEMORY_MB = exports.WORKER_BOOT_TIMEOUT_MS = exports.WORKER_TIMEOUT_MS = exports.CHUNKER_VERSION_HISTORY = exports.CONFIG = exports.MODEL_IDS = exports.DEFAULT_MODEL_TIER = exports.MODEL_TIERS = void 0;
|
|
36
|
+
exports.INDEXABLE_EXTENSIONS = exports.FRAGMENT_COMPACT_THRESHOLD = exports.DISK_LOW_BYTES = exports.DISK_CRITICAL_BYTES = exports.MAX_FILE_SIZE_BYTES = exports.PATHS = exports.MAX_WORKER_MEMORY_MB = exports.WORKER_BOOT_TIMEOUT_MS = exports.WORKER_TIMEOUT_MS = exports.REBUILD_COMMAND = exports.CHUNKER_VERSION_HISTORY = exports.CONFIG = exports.MODEL_IDS = exports.DEFAULT_MODEL_TIER = exports.MODEL_TIERS = void 0;
|
|
37
37
|
exports.describeChunkerGap = describeChunkerGap;
|
|
38
38
|
exports.describeEmbeddingGap = describeEmbeddingGap;
|
|
39
|
+
exports.describeSchemaDimGap = describeSchemaDimGap;
|
|
40
|
+
exports.schemaDimAgentRow = schemaDimAgentRow;
|
|
39
41
|
const os = __importStar(require("node:os"));
|
|
40
42
|
const path = __importStar(require("node:path"));
|
|
41
43
|
exports.MODEL_TIERS = {
|
|
@@ -157,6 +159,43 @@ function describeEmbeddingGap(stored, current) {
|
|
|
157
159
|
severity: dimChanged ? "breaking" : "additive",
|
|
158
160
|
};
|
|
159
161
|
}
|
|
162
|
+
/**
|
|
163
|
+
* The single sanctioned recovery for a physical table-width mismatch. The shared
|
|
164
|
+
* `chunks` table is fixed-width at creation, so a tier/dim change strands it at
|
|
165
|
+
* the old width and every write throws. A per-project `gmax index --reset` only
|
|
166
|
+
* deletes rows — it can't change the shared table's width — so the real fix is a
|
|
167
|
+
* global drop-and-rebuild. doctor, the insertBatch failure, and the staleness
|
|
168
|
+
* hint all point here so the guidance never contradicts itself.
|
|
169
|
+
*/
|
|
170
|
+
exports.REBUILD_COMMAND = "gmax repair --rebuild";
|
|
171
|
+
/**
|
|
172
|
+
* Describe the gap between the LanceDB table's PHYSICAL `vector` width and the
|
|
173
|
+
* width the current global config would produce, or null when they agree (or no
|
|
174
|
+
* table exists yet). This is distinct from {@link describeEmbeddingGap}: that one
|
|
175
|
+
* compares the project REGISTRY's recorded `{modelTier, vectorDim}` to config
|
|
176
|
+
* (logical drift, fixable per project), while this compares the actual on-disk
|
|
177
|
+
* table schema to config (physical drift — every write throws until a global
|
|
178
|
+
* rebuild). A table can match the registry yet still be physically stranded, so
|
|
179
|
+
* doctor reports both independently.
|
|
180
|
+
*/
|
|
181
|
+
function describeSchemaDimGap(tableDim, configDim) {
|
|
182
|
+
if (tableDim == null)
|
|
183
|
+
return null; // no table on disk yet — nothing to compare
|
|
184
|
+
if (tableDim === configDim)
|
|
185
|
+
return null;
|
|
186
|
+
return { tableDim, configDim };
|
|
187
|
+
}
|
|
188
|
+
/** Stable, tab-delimited doctor `--agent` row for a physical schema-dim
|
|
189
|
+
* mismatch. Kept here (pure) so the wire format is testable without running the
|
|
190
|
+
* full doctor command. */
|
|
191
|
+
function schemaDimAgentRow(gap) {
|
|
192
|
+
return [
|
|
193
|
+
"schema_dim_mismatch",
|
|
194
|
+
`table_dim=${gap.tableDim}`,
|
|
195
|
+
`current_dim=${gap.configDim}`,
|
|
196
|
+
`fix=${exports.REBUILD_COMMAND}`,
|
|
197
|
+
].join("\t");
|
|
198
|
+
}
|
|
160
199
|
exports.WORKER_TIMEOUT_MS = Number.parseInt(process.env.GMAX_WORKER_TIMEOUT_MS || "60000", 10);
|
|
161
200
|
exports.WORKER_BOOT_TIMEOUT_MS = Number.parseInt(process.env.GMAX_WORKER_BOOT_TIMEOUT_MS || "300000", 10);
|
|
162
201
|
exports.MAX_WORKER_MEMORY_MB = (() => {
|
|
@@ -177,6 +216,11 @@ exports.PATHS = {
|
|
|
177
216
|
daemonSocket: path.join(GLOBAL_ROOT, "daemon.sock"),
|
|
178
217
|
daemonPidFile: path.join(GLOBAL_ROOT, "daemon.pid"),
|
|
179
218
|
daemonLockFile: path.join(GLOBAL_ROOT, "daemon.lock"),
|
|
219
|
+
// Written by a daemon while it is gracefully shutting down (draining workers,
|
|
220
|
+
// closing LanceDB). A successor's killStaleProcesses() respects this so it
|
|
221
|
+
// never SIGKILLs a peer mid-cleanup once the peer has already dropped its
|
|
222
|
+
// socket/PID/lock liveness markers.
|
|
223
|
+
daemonDrainingFile: path.join(GLOBAL_ROOT, "daemon.draining"),
|
|
180
224
|
// Centralized index storage — one database for all indexed directories
|
|
181
225
|
lancedbDir: path.join(GLOBAL_ROOT, "lancedb"),
|
|
182
226
|
cacheDir: path.join(GLOBAL_ROOT, "cache"),
|
package/dist/index.js
CHANGED
|
@@ -64,6 +64,7 @@ const project_1 = require("./commands/project");
|
|
|
64
64
|
const recent_1 = require("./commands/recent");
|
|
65
65
|
const related_1 = require("./commands/related");
|
|
66
66
|
const remove_1 = require("./commands/remove");
|
|
67
|
+
const repair_1 = require("./commands/repair");
|
|
67
68
|
const review_1 = require("./commands/review");
|
|
68
69
|
const search_1 = require("./commands/search");
|
|
69
70
|
const serve_1 = require("./commands/serve");
|
|
@@ -141,6 +142,7 @@ commander_1.program.addCommand(review_1.review);
|
|
|
141
142
|
commander_1.program.addCommand(setup_1.setup);
|
|
142
143
|
commander_1.program.addCommand(config_1.config);
|
|
143
144
|
commander_1.program.addCommand(doctor_1.doctor);
|
|
145
|
+
commander_1.program.addCommand(repair_1.repair);
|
|
144
146
|
// Plugins
|
|
145
147
|
commander_1.program.addCommand(plugin_1.plugin);
|
|
146
148
|
// Legacy plugin installers (hidden — use `gmax plugin` instead)
|
|
@@ -56,6 +56,7 @@ const syncer_1 = require("../index/syncer");
|
|
|
56
56
|
const server_1 = require("../llm/server");
|
|
57
57
|
const meta_cache_1 = require("../store/meta-cache");
|
|
58
58
|
const vector_db_1 = require("../store/vector-db");
|
|
59
|
+
const daemon_client_1 = require("../utils/daemon-client");
|
|
59
60
|
const daemon_launcher_1 = require("../utils/daemon-launcher");
|
|
60
61
|
const log_rotate_1 = require("../utils/log-rotate");
|
|
61
62
|
const logger_1 = require("../utils/logger");
|
|
@@ -600,6 +601,66 @@ class Daemon {
|
|
|
600
601
|
}));
|
|
601
602
|
});
|
|
602
603
|
}
|
|
604
|
+
/**
|
|
605
|
+
* Core full-(re)index of one project: quiesce its batch processor + watcher,
|
|
606
|
+
* run initialSync, then re-watch in the finally. Shared by indexProject (one
|
|
607
|
+
* project per IPC connection) and repairRebuild (all projects after a global
|
|
608
|
+
* table drop). The caller owns the project lock, the maintenance pause, the
|
|
609
|
+
* heartbeat, and the AbortController; this method owns the watcher handoff and
|
|
610
|
+
* the indexProgress bookkeeping (so concurrent searches get a partial-result
|
|
611
|
+
* footer while it runs — Phase 6).
|
|
612
|
+
*/
|
|
613
|
+
reindexOneProject(root, opts, signal, onProgress) {
|
|
614
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
615
|
+
if (!this.vectorDb || !this.metaCache) {
|
|
616
|
+
throw new Error("daemon resources not ready");
|
|
617
|
+
}
|
|
618
|
+
// Pause the project's batch processor during full index
|
|
619
|
+
const processor = this.processors.get(root);
|
|
620
|
+
if (processor) {
|
|
621
|
+
yield processor.close();
|
|
622
|
+
this.processors.delete(root);
|
|
623
|
+
}
|
|
624
|
+
const sub = this.subscriptions.get(root);
|
|
625
|
+
if (sub) {
|
|
626
|
+
yield sub.unsubscribe();
|
|
627
|
+
this.subscriptions.delete(root);
|
|
628
|
+
}
|
|
629
|
+
// Mark this root as full-indexing so concurrent searches get a
|
|
630
|
+
// partial-result footer (Phase 6); seeded at 0/0 until the first tick.
|
|
631
|
+
this.indexProgress.set(root, { processed: 0, total: 0 });
|
|
632
|
+
try {
|
|
633
|
+
return yield (0, syncer_1.initialSync)({
|
|
634
|
+
projectRoot: root,
|
|
635
|
+
reset: opts.reset,
|
|
636
|
+
dryRun: opts.dryRun,
|
|
637
|
+
vectorDb: this.vectorDb,
|
|
638
|
+
metaCache: this.metaCache,
|
|
639
|
+
signal,
|
|
640
|
+
onProgress: (info) => {
|
|
641
|
+
this.resetActivity();
|
|
642
|
+
this.indexProgress.set(root, {
|
|
643
|
+
processed: info.processed,
|
|
644
|
+
total: info.total,
|
|
645
|
+
});
|
|
646
|
+
onProgress(info);
|
|
647
|
+
},
|
|
648
|
+
});
|
|
649
|
+
}
|
|
650
|
+
finally {
|
|
651
|
+
this.indexProgress.delete(root);
|
|
652
|
+
// Re-enable watcher (skip if shutting down)
|
|
653
|
+
if (!this.shuttingDown) {
|
|
654
|
+
try {
|
|
655
|
+
yield this.watchProject(root);
|
|
656
|
+
}
|
|
657
|
+
catch (err) {
|
|
658
|
+
console.error(`[daemon] Failed to re-watch ${path.basename(root)}:`, err);
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
});
|
|
663
|
+
}
|
|
603
664
|
indexProject(root, conn, opts) {
|
|
604
665
|
return __awaiter(this, void 0, void 0, function* () {
|
|
605
666
|
yield this.withProjectLock(root, () => __awaiter(this, void 0, void 0, function* () {
|
|
@@ -608,51 +669,24 @@ class Daemon {
|
|
|
608
669
|
(0, ipc_handler_1.writeDone)(conn, { ok: false, error: "daemon resources not ready" });
|
|
609
670
|
return;
|
|
610
671
|
}
|
|
611
|
-
// Pause the project's batch processor during full index
|
|
612
|
-
const processor = this.processors.get(root);
|
|
613
|
-
if (processor) {
|
|
614
|
-
yield processor.close();
|
|
615
|
-
this.processors.delete(root);
|
|
616
|
-
}
|
|
617
|
-
const sub = this.subscriptions.get(root);
|
|
618
|
-
if (sub) {
|
|
619
|
-
yield sub.unsubscribe();
|
|
620
|
-
this.subscriptions.delete(root);
|
|
621
|
-
}
|
|
622
672
|
const ac = new AbortController();
|
|
623
673
|
conn.on("close", () => ac.abort());
|
|
624
674
|
this.shutdownAbortControllers.add(ac);
|
|
625
675
|
this.vectorDb.pauseMaintenanceLoop();
|
|
626
676
|
const stopHeartbeat = (0, ipc_handler_1.startHeartbeat)(conn);
|
|
627
677
|
let lastProgressTime = 0;
|
|
628
|
-
// Mark this root as full-indexing so concurrent searches get a
|
|
629
|
-
// partial-result footer (Phase 6); seeded at 0/0 until the first tick.
|
|
630
|
-
this.indexProgress.set(root, { processed: 0, total: 0 });
|
|
631
678
|
try {
|
|
632
|
-
const result = yield (
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
total: info.total,
|
|
644
|
-
});
|
|
645
|
-
const now = Date.now();
|
|
646
|
-
if (now - lastProgressTime < 100)
|
|
647
|
-
return;
|
|
648
|
-
lastProgressTime = now;
|
|
649
|
-
(0, ipc_handler_1.writeProgress)(conn, {
|
|
650
|
-
processed: info.processed,
|
|
651
|
-
indexed: info.indexed,
|
|
652
|
-
total: info.total,
|
|
653
|
-
filePath: info.filePath,
|
|
654
|
-
});
|
|
655
|
-
},
|
|
679
|
+
const result = yield this.reindexOneProject(root, opts, ac.signal, (info) => {
|
|
680
|
+
const now = Date.now();
|
|
681
|
+
if (now - lastProgressTime < 100)
|
|
682
|
+
return;
|
|
683
|
+
lastProgressTime = now;
|
|
684
|
+
(0, ipc_handler_1.writeProgress)(conn, {
|
|
685
|
+
processed: info.processed,
|
|
686
|
+
indexed: info.indexed,
|
|
687
|
+
total: info.total,
|
|
688
|
+
filePath: info.filePath,
|
|
689
|
+
});
|
|
656
690
|
});
|
|
657
691
|
stopHeartbeat();
|
|
658
692
|
(0, ipc_handler_1.writeDone)(conn, {
|
|
@@ -671,22 +705,100 @@ class Daemon {
|
|
|
671
705
|
}
|
|
672
706
|
finally {
|
|
673
707
|
stopHeartbeat();
|
|
674
|
-
this.indexProgress.delete(root);
|
|
675
708
|
this.shutdownAbortControllers.delete(ac);
|
|
676
709
|
(_a = this.vectorDb) === null || _a === void 0 ? void 0 : _a.resumeMaintenanceLoop();
|
|
677
|
-
// Re-enable watcher (skip if shutting down)
|
|
678
|
-
if (!this.shuttingDown) {
|
|
679
|
-
try {
|
|
680
|
-
yield this.watchProject(root);
|
|
681
|
-
}
|
|
682
|
-
catch (err) {
|
|
683
|
-
console.error(`[daemon] Failed to re-watch ${path.basename(root)}:`, err);
|
|
684
|
-
}
|
|
685
|
-
}
|
|
686
710
|
}
|
|
687
711
|
}));
|
|
688
712
|
});
|
|
689
713
|
}
|
|
714
|
+
/**
|
|
715
|
+
* Global recovery for a physical table-width mismatch (the chosen "global
|
|
716
|
+
* rebuild" strategy): drop the shared `chunks` table and re-index every
|
|
717
|
+
* registered project at the configured dim. The table is fixed-width at
|
|
718
|
+
* creation, so this is the only way to move it from e.g. 384d to 768d after a
|
|
719
|
+
* tier change. Streams per-project progress over `conn`. Each project is
|
|
720
|
+
* reindexed under its own lock with `reset: true`, which also clears that
|
|
721
|
+
* project's stale MetaCache entries so everything re-embeds into the freshly
|
|
722
|
+
* recreated (lazily, at config width) table.
|
|
723
|
+
*/
|
|
724
|
+
repairRebuild(conn) {
|
|
725
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
726
|
+
var _a;
|
|
727
|
+
if (!this.vectorDb || !this.metaCache) {
|
|
728
|
+
(0, ipc_handler_1.writeDone)(conn, { ok: false, error: "daemon resources not ready" });
|
|
729
|
+
return;
|
|
730
|
+
}
|
|
731
|
+
const ac = new AbortController();
|
|
732
|
+
conn.on("close", () => ac.abort());
|
|
733
|
+
this.shutdownAbortControllers.add(ac);
|
|
734
|
+
this.vectorDb.pauseMaintenanceLoop();
|
|
735
|
+
const stopHeartbeat = (0, ipc_handler_1.startHeartbeat)(conn);
|
|
736
|
+
// Reindex every project the daemon would normally index on startup.
|
|
737
|
+
const projects = (0, project_registry_1.listProjects)().filter((p) => p.status === "indexed" || p.status === "pending");
|
|
738
|
+
const globalConfig = (0, index_config_1.readGlobalConfig)();
|
|
739
|
+
let rebuilt = 0;
|
|
740
|
+
let totalIndexed = 0;
|
|
741
|
+
let lastProgressTime = 0;
|
|
742
|
+
try {
|
|
743
|
+
// Drop the shared table — recreated lazily at the configured width on the
|
|
744
|
+
// first insert of the reindex below.
|
|
745
|
+
(0, ipc_handler_1.writeProgress)(conn, { phase: "drop", projectsTotal: projects.length });
|
|
746
|
+
yield this.vectorDb.drop();
|
|
747
|
+
for (const p of projects) {
|
|
748
|
+
if (ac.signal.aborted)
|
|
749
|
+
break;
|
|
750
|
+
const name = p.name || path.basename(p.root);
|
|
751
|
+
yield this.withProjectLock(p.root, () => __awaiter(this, void 0, void 0, function* () {
|
|
752
|
+
const result = yield this.reindexOneProject(p.root, { reset: true }, ac.signal, (info) => {
|
|
753
|
+
const now = Date.now();
|
|
754
|
+
if (now - lastProgressTime < 100)
|
|
755
|
+
return;
|
|
756
|
+
lastProgressTime = now;
|
|
757
|
+
(0, ipc_handler_1.writeProgress)(conn, {
|
|
758
|
+
phase: "reindex",
|
|
759
|
+
project: name,
|
|
760
|
+
projectsDone: rebuilt,
|
|
761
|
+
projectsTotal: projects.length,
|
|
762
|
+
processed: info.processed,
|
|
763
|
+
indexed: info.indexed,
|
|
764
|
+
total: info.total,
|
|
765
|
+
filePath: info.filePath,
|
|
766
|
+
});
|
|
767
|
+
});
|
|
768
|
+
totalIndexed += result.indexed;
|
|
769
|
+
const proj = (0, project_registry_1.getProject)(p.root);
|
|
770
|
+
if (proj) {
|
|
771
|
+
(0, project_registry_1.registerProject)(Object.assign(Object.assign({}, proj), { vectorDim: globalConfig.vectorDim, modelTier: globalConfig.modelTier, embedMode: globalConfig.embedMode, lastIndexed: new Date().toISOString(), chunkCount: result.indexed, status: "indexed", chunkerVersion: config_1.CONFIG.CHUNKER_VERSION }));
|
|
772
|
+
}
|
|
773
|
+
}));
|
|
774
|
+
rebuilt++;
|
|
775
|
+
}
|
|
776
|
+
stopHeartbeat();
|
|
777
|
+
(0, ipc_handler_1.writeDone)(conn, {
|
|
778
|
+
ok: true,
|
|
779
|
+
projects: rebuilt,
|
|
780
|
+
projectsTotal: projects.length,
|
|
781
|
+
indexed: totalIndexed,
|
|
782
|
+
});
|
|
783
|
+
}
|
|
784
|
+
catch (err) {
|
|
785
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
786
|
+
console.error("[daemon] repairRebuild failed:", msg);
|
|
787
|
+
stopHeartbeat();
|
|
788
|
+
(0, ipc_handler_1.writeDone)(conn, {
|
|
789
|
+
ok: false,
|
|
790
|
+
error: msg,
|
|
791
|
+
projects: rebuilt,
|
|
792
|
+
indexed: totalIndexed,
|
|
793
|
+
});
|
|
794
|
+
}
|
|
795
|
+
finally {
|
|
796
|
+
stopHeartbeat();
|
|
797
|
+
this.shutdownAbortControllers.delete(ac);
|
|
798
|
+
(_a = this.vectorDb) === null || _a === void 0 ? void 0 : _a.resumeMaintenanceLoop();
|
|
799
|
+
}
|
|
800
|
+
});
|
|
801
|
+
}
|
|
690
802
|
removeProject(root, conn) {
|
|
691
803
|
return __awaiter(this, void 0, void 0, function* () {
|
|
692
804
|
yield this.withProjectLock(root, () => __awaiter(this, void 0, void 0, function* () {
|
|
@@ -849,6 +961,10 @@ class Daemon {
|
|
|
849
961
|
return;
|
|
850
962
|
this.shuttingDown = true;
|
|
851
963
|
console.log("[daemon] Shutting down...");
|
|
964
|
+
// Announce graceful shutdown BEFORE dropping the liveness markers below, so a
|
|
965
|
+
// successor spawned during the (possibly long) drain sees the draining marker
|
|
966
|
+
// and defers instead of SIGKILLing us mid-cleanup.
|
|
967
|
+
(0, daemon_client_1.writeDrainingMarker)(process.pid);
|
|
852
968
|
// Drop external liveness markers FIRST so the next daemon start isn't
|
|
853
969
|
// fooled by leftover state if the long cleanup below is interrupted
|
|
854
970
|
// (uncaught exception, second SIGTERM, OOM kill mid-shutdown). The
|
|
@@ -926,6 +1042,9 @@ class Daemon {
|
|
|
926
1042
|
const pid = (0, daemon_launcher_1.spawnDaemon)();
|
|
927
1043
|
console.log(`[daemon] Spawned successor daemon${pid ? ` (PID: ${pid})` : " (spawn failed)"}`);
|
|
928
1044
|
}
|
|
1045
|
+
// Cleanly drained — drop the marker so a later start doesn't defer to a
|
|
1046
|
+
// process that's already gone (it self-expires after DRAIN_GRACE_MS anyway).
|
|
1047
|
+
(0, daemon_client_1.clearDrainingMarker)();
|
|
929
1048
|
console.log("[daemon] Shutdown complete");
|
|
930
1049
|
});
|
|
931
1050
|
}
|
|
@@ -228,6 +228,12 @@ function handleCommand(daemon, cmd, conn) {
|
|
|
228
228
|
daemon.removeProject(root, conn);
|
|
229
229
|
return null;
|
|
230
230
|
}
|
|
231
|
+
case "repair": {
|
|
232
|
+
// Global drop-and-rebuild recovery for a physical table-width mismatch.
|
|
233
|
+
// Streams per-project progress; daemon manages the connection.
|
|
234
|
+
daemon.repairRebuild(conn);
|
|
235
|
+
return null;
|
|
236
|
+
}
|
|
231
237
|
case "summarize": {
|
|
232
238
|
const root = String(cmd.root || "");
|
|
233
239
|
if (!root)
|
|
@@ -89,8 +89,19 @@ class ProcessManager {
|
|
|
89
89
|
(0, logger_1.log)("daemon", "No stale processes found");
|
|
90
90
|
return;
|
|
91
91
|
}
|
|
92
|
+
// A peer that's gracefully shutting down has already dropped its
|
|
93
|
+
// socket/PID/lock (so the liveness probes below read "stale") yet is still
|
|
94
|
+
// mid-cleanup — destroying workers, closing LanceDB. Killing it now corrupts
|
|
95
|
+
// that teardown. Defer to its draining marker: don't kill it, and don't
|
|
96
|
+
// sweep its workers (it's reaping them itself); just take over the free lock.
|
|
97
|
+
let anyDraining = false;
|
|
92
98
|
for (const pid of daemonPids) {
|
|
93
99
|
(0, logger_1.log)("daemon", `found daemon PID:${pid}, checking liveness...`);
|
|
100
|
+
if ((0, daemon_client_1.isDaemonDraining)(pid)) {
|
|
101
|
+
anyDraining = true;
|
|
102
|
+
(0, logger_1.log)("daemon", `daemon PID:${pid} is gracefully draining — leaving it to exit, taking over`);
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
94
105
|
// A busy daemon (mid-index, compaction, big LMDB write) can block the
|
|
95
106
|
// event loop long enough to miss a ping. Two independent liveness
|
|
96
107
|
// probes — if either says "alive", defer to the running peer instead
|
|
@@ -109,9 +120,16 @@ class ProcessManager {
|
|
|
109
120
|
}
|
|
110
121
|
// 2. Kill orphaned workers from previous daemon instances.
|
|
111
122
|
// Safe because this runs before the new daemon's worker pool is initialized.
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
123
|
+
// Skipped while a peer is draining — those workers belong to it and it's
|
|
124
|
+
// tearing them down; sweeping them here would race its own cleanup.
|
|
125
|
+
if (anyDraining) {
|
|
126
|
+
(0, logger_1.log)("daemon", "skipping orphan-worker sweep — a peer is draining its own workers");
|
|
127
|
+
}
|
|
128
|
+
else {
|
|
129
|
+
for (const pid of workerPids) {
|
|
130
|
+
(0, logger_1.log)("daemon", `killing orphaned worker PID:${pid}`);
|
|
131
|
+
yield (0, process_1.killProcess)(pid);
|
|
132
|
+
}
|
|
115
133
|
}
|
|
116
134
|
(0, logger_1.log)("daemon", `Cleaned up ${daemonPids.length} stale daemon(s), ${workerPids.length} orphaned worker(s)`);
|
|
117
135
|
});
|
|
@@ -279,6 +279,32 @@ class VectorDB {
|
|
|
279
279
|
summary: "",
|
|
280
280
|
};
|
|
281
281
|
}
|
|
282
|
+
/**
|
|
283
|
+
* Read the physical width of the on-disk `vector` column, or null if the
|
|
284
|
+
* table doesn't exist yet. Non-throwing and validation-free on purpose: doctor
|
|
285
|
+
* uses it to detect a table stranded at an old width after a tier change, and
|
|
286
|
+
* must see the truth even when the table is incompatible with the current
|
|
287
|
+
* config (a throwing ensureTable would mask exactly the mismatch we're hunting).
|
|
288
|
+
*/
|
|
289
|
+
getSchemaVectorDim() {
|
|
290
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
291
|
+
const db = yield this.getDb();
|
|
292
|
+
let table;
|
|
293
|
+
try {
|
|
294
|
+
table = yield db.openTable(TABLE_NAME);
|
|
295
|
+
}
|
|
296
|
+
catch (_a) {
|
|
297
|
+
return null; // no table on disk yet
|
|
298
|
+
}
|
|
299
|
+
const schema = yield table.schema();
|
|
300
|
+
const field = schema.fields.find((f) => f.name === "vector");
|
|
301
|
+
if (!field)
|
|
302
|
+
return null;
|
|
303
|
+
// The `vector` column is a FixedSizeList; its listSize is the vector width.
|
|
304
|
+
const listSize = field.type.listSize;
|
|
305
|
+
return typeof listSize === "number" ? listSize : null;
|
|
306
|
+
});
|
|
307
|
+
}
|
|
282
308
|
validateSchema(table) {
|
|
283
309
|
return __awaiter(this, void 0, void 0, function* () {
|
|
284
310
|
const schema = yield table.schema();
|
|
@@ -421,7 +447,9 @@ class VectorDB {
|
|
|
421
447
|
// Fail loudly and point at the fix instead.
|
|
422
448
|
if (vec.length !== this.vectorDim) {
|
|
423
449
|
throw new Error(`Vector dimension mismatch: got ${vec.length}d, expected ${this.vectorDim}d. ` +
|
|
424
|
-
"The embedding model tier likely changed without a rebuild —
|
|
450
|
+
"The embedding model tier likely changed without a rebuild — the shared " +
|
|
451
|
+
`table is fixed-width, so run \`${config_1.REBUILD_COMMAND}\` (a per-project ` +
|
|
452
|
+
"`gmax index --reset` cannot change the table width).");
|
|
425
453
|
}
|
|
426
454
|
rec.vector = vec;
|
|
427
455
|
rec.colbert = toBuffer(rec.colbert);
|
|
@@ -47,6 +47,9 @@ exports.isDaemonRunning = isDaemonRunning;
|
|
|
47
47
|
exports.isDaemonHeartbeatFresh = isDaemonHeartbeatFresh;
|
|
48
48
|
exports.readDaemonPid = readDaemonPid;
|
|
49
49
|
exports.waitForProcessExit = waitForProcessExit;
|
|
50
|
+
exports.writeDrainingMarker = writeDrainingMarker;
|
|
51
|
+
exports.clearDrainingMarker = clearDrainingMarker;
|
|
52
|
+
exports.isDaemonDraining = isDaemonDraining;
|
|
50
53
|
exports.ensureDaemonRunning = ensureDaemonRunning;
|
|
51
54
|
exports.sendStreamingCommand = sendStreamingCommand;
|
|
52
55
|
const fs = __importStar(require("node:fs"));
|
|
@@ -186,6 +189,56 @@ function waitForProcessExit(pid, timeoutMs) {
|
|
|
186
189
|
return false;
|
|
187
190
|
});
|
|
188
191
|
}
|
|
192
|
+
// A daemon's graceful shutdown can run well past the 20s restart wait — worker
|
|
193
|
+
// SIGTERM→SIGKILL escalation (5s), maintenance drain (10s), LanceDB close (5s),
|
|
194
|
+
// LLM/MLX teardown. Treat a draining marker as authoritative for this long; past
|
|
195
|
+
// it, assume the draining daemon wedged and let killStaleProcesses reclaim it.
|
|
196
|
+
const DRAIN_GRACE_MS = 90000;
|
|
197
|
+
/**
|
|
198
|
+
* Record that this daemon (pid) has begun graceful shutdown, so a successor's
|
|
199
|
+
* killStaleProcesses() won't SIGKILL it mid-cleanup after it drops its
|
|
200
|
+
* socket/PID/lock liveness markers. Best-effort; cleared by clearDrainingMarker
|
|
201
|
+
* on a clean exit and otherwise self-expires after DRAIN_GRACE_MS.
|
|
202
|
+
*/
|
|
203
|
+
function writeDrainingMarker(pid) {
|
|
204
|
+
try {
|
|
205
|
+
fs.mkdirSync(config_1.PATHS.globalRoot, { recursive: true });
|
|
206
|
+
fs.writeFileSync(config_1.PATHS.daemonDrainingFile, JSON.stringify({ pid, ts: Date.now() }));
|
|
207
|
+
}
|
|
208
|
+
catch (_a) { }
|
|
209
|
+
}
|
|
210
|
+
/** Remove the draining marker (clean end of shutdown). */
|
|
211
|
+
function clearDrainingMarker() {
|
|
212
|
+
try {
|
|
213
|
+
fs.unlinkSync(config_1.PATHS.daemonDrainingFile);
|
|
214
|
+
}
|
|
215
|
+
catch (_a) { }
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* True if `pid` is a daemon currently inside graceful shutdown: a fresh draining
|
|
219
|
+
* marker naming that exact PID, and the process is still alive. A stale marker
|
|
220
|
+
* (older than DRAIN_GRACE_MS), a mismatched PID, or a dead process all read as
|
|
221
|
+
* "not draining" so a wedged or already-exited predecessor is still reclaimable.
|
|
222
|
+
*/
|
|
223
|
+
function isDaemonDraining(pid) {
|
|
224
|
+
try {
|
|
225
|
+
const { pid: markerPid, ts } = JSON.parse(fs.readFileSync(config_1.PATHS.daemonDrainingFile, "utf-8"));
|
|
226
|
+
if (markerPid !== pid)
|
|
227
|
+
return false;
|
|
228
|
+
if (typeof ts !== "number" || Date.now() - ts > DRAIN_GRACE_MS)
|
|
229
|
+
return false;
|
|
230
|
+
try {
|
|
231
|
+
process.kill(pid, 0);
|
|
232
|
+
return true;
|
|
233
|
+
}
|
|
234
|
+
catch (_a) {
|
|
235
|
+
return false; // process already gone — done draining
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
catch (_b) {
|
|
239
|
+
return false; // no marker / unreadable
|
|
240
|
+
}
|
|
241
|
+
}
|
|
189
242
|
/**
|
|
190
243
|
* Ensure the daemon is running — start it if needed, poll up to 5s.
|
|
191
244
|
* Returns true if daemon is ready, false if it couldn't be started.
|
|
@@ -104,7 +104,9 @@ function maybeWarnStaleEmbedding(projectRoot, opts) {
|
|
|
104
104
|
`current_dim=${gap.toDim}`,
|
|
105
105
|
`dim_changed=${gap.dimChanged}`,
|
|
106
106
|
`severity=${gap.severity}`,
|
|
107
|
-
|
|
107
|
+
// A dim change needs the global rebuild (shared table is fixed-width); a
|
|
108
|
+
// same-dim model swap can use a per-project reset.
|
|
109
|
+
`fix=${gap.dimChanged ? config_1.REBUILD_COMMAND : "gmax index --reset"}`,
|
|
108
110
|
].join("\t");
|
|
109
111
|
process.stderr.write(`${fields}\n`);
|
|
110
112
|
return;
|
|
@@ -113,7 +115,10 @@ function maybeWarnStaleEmbedding(projectRoot, opts) {
|
|
|
113
115
|
const detail = gap.dimChanged
|
|
114
116
|
? `vector dim ${gap.fromDim}→${gap.toDim} (incompatible — scores invalid until re-embed)`
|
|
115
117
|
: `model '${gap.fromModel}'→'${gap.toModel}' (same dim; results mix models until re-embed)`;
|
|
116
|
-
|
|
118
|
+
const fix = gap.dimChanged
|
|
119
|
+
? `Run '${config_1.REBUILD_COMMAND}'`
|
|
120
|
+
: "Run 'gmax index --reset'";
|
|
121
|
+
process.stderr.write(`${label} gmax: '${name}' indexed with embedding ${detail}. ${fix}. (silence: GMAX_NO_STALE_HINT=1)\n`);
|
|
117
122
|
}
|
|
118
123
|
/**
|
|
119
124
|
* Cross-project guard: when a `--all-projects` / `--projects` search spans
|
package/package.json
CHANGED
|
@@ -116,7 +116,7 @@ Agent output ends with `t: <test-file>:line\t<test-symbol>\t<hop-label>` rows wh
|
|
|
116
116
|
### Skeleton — `gmax skeleton <target>`
|
|
117
117
|
```
|
|
118
118
|
gmax skeleton src/lib/auth.ts # single file
|
|
119
|
-
gmax skeleton
|
|
119
|
+
gmax skeleton handleAuth # by symbol name (finds its file)
|
|
120
120
|
gmax skeleton src/a.ts,src/b.ts # batch
|
|
121
121
|
gmax skeleton src/lib/auth.ts --json # structured JSON output
|
|
122
122
|
```
|