grepmax 0.23.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +25 -6
- package/dist/commands/add.js +38 -31
- package/dist/commands/config.js +16 -15
- package/dist/commands/context.js +38 -13
- package/dist/commands/doctor.js +76 -83
- package/dist/commands/extract.js +12 -1
- package/dist/commands/impact.js +33 -1
- package/dist/commands/index.js +23 -24
- package/dist/commands/list.js +23 -14
- package/dist/commands/mcp.js +494 -162
- package/dist/commands/peek.js +16 -5
- package/dist/commands/repair.js +54 -120
- package/dist/commands/search-output.js +30 -9
- package/dist/commands/search-run.js +75 -47
- package/dist/commands/search-skeletons.js +28 -18
- package/dist/commands/search.js +45 -49
- package/dist/commands/serve.js +415 -373
- package/dist/commands/setup.js +2 -2
- package/dist/commands/similar.js +5 -5
- package/dist/commands/skeleton.js +67 -41
- package/dist/commands/status.js +5 -2
- package/dist/commands/summarize.js +6 -0
- package/dist/commands/surprises.js +150 -0
- package/dist/commands/watch.js +102 -38
- package/dist/config.js +3 -0
- package/dist/eval-surprising-connections.js +191 -0
- package/dist/index.js +2 -0
- package/dist/lib/analysis/surprising-connections.js +600 -0
- package/dist/lib/daemon/daemon.js +1101 -379
- package/dist/lib/daemon/ipc-handler.js +122 -13
- package/dist/lib/daemon/mlx-server-manager.js +268 -125
- package/dist/lib/daemon/process-manager.js +7 -4
- package/dist/lib/daemon/search-handler.js +23 -9
- package/dist/lib/daemon/watcher-manager.js +353 -110
- package/dist/lib/graph/impact-rollup.js +202 -0
- package/dist/lib/graph/impact.js +15 -1
- package/dist/lib/help/agent-cheatsheet.js +1 -0
- package/dist/lib/index/batch-processor.js +231 -67
- package/dist/lib/index/embedding-generation.js +109 -0
- package/dist/lib/index/embedding-status.js +23 -0
- package/dist/lib/index/file-policy.js +273 -0
- package/dist/lib/index/ignore-patterns.js +4 -0
- package/dist/lib/index/index-config.js +18 -4
- package/dist/lib/index/syncer.js +256 -79
- package/dist/lib/index/walker.js +66 -86
- package/dist/lib/index/watcher-batch.js +9 -0
- package/dist/lib/index/watcher.js +129 -3
- package/dist/lib/llm/server.js +6 -0
- package/dist/lib/search/searcher.js +30 -19
- package/dist/lib/skeleton/skeletonizer.js +118 -0
- package/dist/lib/skeleton/summary-formatter.js +8 -1
- package/dist/lib/store/store-lease.js +473 -0
- package/dist/lib/store/vector-db.js +302 -57
- package/dist/lib/utils/blocked-roots.js +63 -0
- package/dist/lib/utils/cross-project.js +7 -3
- package/dist/lib/utils/daemon-client.js +24 -1
- package/dist/lib/utils/daemon-launcher.js +38 -13
- package/dist/lib/utils/doctor-status.js +76 -0
- package/dist/lib/utils/file-utils.js +74 -4
- package/dist/lib/utils/keyed-mutex.js +101 -0
- package/dist/lib/utils/logger.js +57 -1
- package/dist/lib/utils/mlx-hf-cache.js +114 -0
- package/dist/lib/utils/operation-coordinator.js +146 -0
- package/dist/lib/utils/path-containment.js +106 -0
- package/dist/lib/utils/process.js +44 -0
- package/dist/lib/utils/project-registry.js +351 -3
- package/dist/lib/utils/scope-filter.js +3 -9
- package/dist/lib/utils/stale-hint.js +12 -19
- package/dist/lib/utils/watcher-launcher.js +5 -1
- package/dist/lib/utils/watcher-store.js +2 -2
- package/dist/lib/workers/colbert-math.js +15 -12
- package/dist/lib/workers/embeddings/colbert.js +3 -2
- package/dist/lib/workers/embeddings/granite.js +4 -3
- package/dist/lib/workers/embeddings/mlx-client.js +59 -16
- package/dist/lib/workers/orchestrator.js +39 -8
- package/dist/lib/workers/pool.js +150 -83
- package/dist/lib/workers/process-child.js +11 -2
- package/dist/lib/workers/serialized-handler.js +10 -0
- package/dist/lib/workers/worker.js +13 -4
- package/mlx-embed-server/server.py +21 -3
- package/mlx-embed-server/summarizer.py +8 -0
- package/package.json +6 -3
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
- package/plugins/grepmax/hooks/start.js +3 -170
|
@@ -42,7 +42,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
42
42
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
43
43
|
};
|
|
44
44
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
|
+
exports.ProjectRegistryConflictError = void 0;
|
|
45
46
|
exports.registerProject = registerProject;
|
|
47
|
+
exports.reserveProjectsForRebuild = reserveProjectsForRebuild;
|
|
48
|
+
exports.restoreProjectsAfterRebuild = restoreProjectsAfterRebuild;
|
|
49
|
+
exports.markProjectRebuildDropping = markProjectRebuildDropping;
|
|
50
|
+
exports.completeProjectRebuild = completeProjectRebuild;
|
|
51
|
+
exports.hasUnfinishedProjectRebuild = hasUnfinishedProjectRebuild;
|
|
52
|
+
exports.stampProjectFullSync = stampProjectFullSync;
|
|
46
53
|
exports.listProjects = listProjects;
|
|
47
54
|
exports.getProject = getProject;
|
|
48
55
|
exports.removeProject = removeProject;
|
|
@@ -50,20 +57,125 @@ exports.getParentProject = getParentProject;
|
|
|
50
57
|
exports.getChildProjects = getChildProjects;
|
|
51
58
|
exports.resolveRootOrExit = resolveRootOrExit;
|
|
52
59
|
exports.resolveProjectRoot = resolveProjectRoot;
|
|
60
|
+
const node_crypto_1 = require("node:crypto");
|
|
53
61
|
const fs = __importStar(require("node:fs"));
|
|
54
62
|
const path = __importStar(require("node:path"));
|
|
55
63
|
const proper_lockfile_1 = __importDefault(require("proper-lockfile"));
|
|
56
64
|
const config_1 = require("../../config");
|
|
65
|
+
const embedding_generation_1 = require("../index/embedding-generation");
|
|
57
66
|
const REGISTRY_PATH = path.join(config_1.PATHS.globalRoot, "projects.json");
|
|
67
|
+
const REBUILD_JOURNAL_PATH = path.join(config_1.PATHS.globalRoot, "rebuild-journal.json");
|
|
68
|
+
class ProjectRegistryConflictError extends Error {
|
|
69
|
+
constructor(message) {
|
|
70
|
+
super(message);
|
|
71
|
+
this.code = "PROJECT_REGISTRY_CONFLICT";
|
|
72
|
+
this.name = "ProjectRegistryConflictError";
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
exports.ProjectRegistryConflictError = ProjectRegistryConflictError;
|
|
58
76
|
function loadRegistry() {
|
|
77
|
+
let raw;
|
|
78
|
+
try {
|
|
79
|
+
raw = fs.readFileSync(REGISTRY_PATH, "utf-8");
|
|
80
|
+
}
|
|
81
|
+
catch (err) {
|
|
82
|
+
if (err.code === "ENOENT")
|
|
83
|
+
return [];
|
|
84
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
85
|
+
throw new Error(`Failed to read project registry ${REGISTRY_PATH}: ${message}`);
|
|
86
|
+
}
|
|
87
|
+
let parsed;
|
|
59
88
|
try {
|
|
60
|
-
|
|
61
|
-
|
|
89
|
+
parsed = JSON.parse(raw);
|
|
90
|
+
}
|
|
91
|
+
catch (err) {
|
|
92
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
93
|
+
throw new Error(`Invalid project registry JSON ${REGISTRY_PATH}: ${message}`);
|
|
94
|
+
}
|
|
95
|
+
if (!Array.isArray(parsed)) {
|
|
96
|
+
throw new Error(`Invalid project registry ${REGISTRY_PATH}: expected an array`);
|
|
97
|
+
}
|
|
98
|
+
for (const [index, entry] of parsed.entries()) {
|
|
99
|
+
if (!isProjectEntry(entry)) {
|
|
100
|
+
throw new Error(`Invalid project registry entry at index ${index} in ${REGISTRY_PATH}`);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return parsed;
|
|
104
|
+
}
|
|
105
|
+
function isProjectEntry(entry) {
|
|
106
|
+
if (!entry || typeof entry !== "object" || Array.isArray(entry))
|
|
107
|
+
return false;
|
|
108
|
+
const value = entry;
|
|
109
|
+
const valid = typeof value.root === "string" &&
|
|
110
|
+
value.root.length > 0 &&
|
|
111
|
+
typeof value.name === "string" &&
|
|
112
|
+
value.name.length > 0 &&
|
|
113
|
+
typeof value.vectorDim === "number" &&
|
|
114
|
+
Number.isFinite(value.vectorDim) &&
|
|
115
|
+
value.vectorDim > 0 &&
|
|
116
|
+
typeof value.modelTier === "string" &&
|
|
117
|
+
typeof value.embedMode === "string" &&
|
|
118
|
+
typeof value.lastIndexed === "string" &&
|
|
119
|
+
(value.status === undefined ||
|
|
120
|
+
value.status === "pending" ||
|
|
121
|
+
value.status === "indexed" ||
|
|
122
|
+
value.status === "error") &&
|
|
123
|
+
(value.chunkCount === undefined ||
|
|
124
|
+
(typeof value.chunkCount === "number" &&
|
|
125
|
+
Number.isFinite(value.chunkCount) &&
|
|
126
|
+
value.chunkCount >= 0)) &&
|
|
127
|
+
(value.chunkerVersion === undefined ||
|
|
128
|
+
(typeof value.chunkerVersion === "number" &&
|
|
129
|
+
Number.isInteger(value.chunkerVersion) &&
|
|
130
|
+
value.chunkerVersion >= 0)) &&
|
|
131
|
+
optionalNonEmptyString(value.embedModel) &&
|
|
132
|
+
optionalNonEmptyString(value.mlxModel) &&
|
|
133
|
+
optionalNonEmptyString(value.colbertModel) &&
|
|
134
|
+
(value.embeddingFingerprint === undefined ||
|
|
135
|
+
(typeof value.embeddingFingerprint === "string" &&
|
|
136
|
+
/^[a-f0-9]{64}$/.test(value.embeddingFingerprint))) &&
|
|
137
|
+
optionalNonEmptyString(value.rebuildId);
|
|
138
|
+
if (!valid)
|
|
139
|
+
return false;
|
|
140
|
+
try {
|
|
141
|
+
if (value.embeddingFingerprint !== undefined) {
|
|
142
|
+
if (value.embedModel === undefined ||
|
|
143
|
+
value.mlxModel === undefined ||
|
|
144
|
+
value.colbertModel === undefined) {
|
|
145
|
+
return false;
|
|
146
|
+
}
|
|
147
|
+
return (value.embeddingFingerprint ===
|
|
148
|
+
(0, embedding_generation_1.computeEmbeddingFingerprint)({
|
|
149
|
+
tier: value.modelTier,
|
|
150
|
+
vectorDim: value.vectorDim,
|
|
151
|
+
onnxModel: value.embedModel,
|
|
152
|
+
mlxModel: value.mlxModel,
|
|
153
|
+
colbertModel: value.colbertModel,
|
|
154
|
+
}));
|
|
155
|
+
}
|
|
156
|
+
// Unstamped records predate exact generation identities. Keep accepting
|
|
157
|
+
// their historical shape while retaining the old tier sanity checks.
|
|
158
|
+
const generation = (0, embedding_generation_1.resolveEmbeddingGeneration)(Object.assign({ modelTier: value.modelTier, vectorDim: value.vectorDim }, (value.mlxModel === undefined
|
|
159
|
+
? {}
|
|
160
|
+
: { mlxModel: value.mlxModel })));
|
|
161
|
+
if ((value.embedModel !== undefined &&
|
|
162
|
+
value.embedModel !== generation.onnxModel) ||
|
|
163
|
+
(value.mlxModel !== undefined &&
|
|
164
|
+
value.mlxModel !== generation.mlxModel) ||
|
|
165
|
+
(value.colbertModel !== undefined &&
|
|
166
|
+
value.colbertModel !== generation.colbertModel)) {
|
|
167
|
+
return false;
|
|
168
|
+
}
|
|
169
|
+
return true;
|
|
62
170
|
}
|
|
63
171
|
catch (_a) {
|
|
64
|
-
return
|
|
172
|
+
return false;
|
|
65
173
|
}
|
|
66
174
|
}
|
|
175
|
+
function optionalNonEmptyString(value) {
|
|
176
|
+
return (value === undefined ||
|
|
177
|
+
(typeof value === "string" && value.length > 0 && value === value.trim()));
|
|
178
|
+
}
|
|
67
179
|
function saveRegistry(entries) {
|
|
68
180
|
fs.mkdirSync(path.dirname(REGISTRY_PATH), { recursive: true });
|
|
69
181
|
const tmp = `${REGISTRY_PATH}.tmp`;
|
|
@@ -90,8 +202,31 @@ function withRegistryLock(fn) {
|
|
|
90
202
|
}
|
|
91
203
|
}
|
|
92
204
|
function registerProject(entry) {
|
|
205
|
+
if (!isProjectEntry(entry)) {
|
|
206
|
+
throw new Error("Invalid project registry entry");
|
|
207
|
+
}
|
|
93
208
|
withRegistryLock(() => {
|
|
94
209
|
const entries = loadRegistry();
|
|
210
|
+
let canonicalRoot;
|
|
211
|
+
try {
|
|
212
|
+
canonicalRoot = fs.realpathSync(entry.root);
|
|
213
|
+
}
|
|
214
|
+
catch (_a) { }
|
|
215
|
+
if (canonicalRoot) {
|
|
216
|
+
const duplicate = entries.find((existing) => {
|
|
217
|
+
if (existing.root === entry.root)
|
|
218
|
+
return false;
|
|
219
|
+
try {
|
|
220
|
+
return fs.realpathSync(existing.root) === canonicalRoot;
|
|
221
|
+
}
|
|
222
|
+
catch (_a) {
|
|
223
|
+
return false;
|
|
224
|
+
}
|
|
225
|
+
});
|
|
226
|
+
if (duplicate) {
|
|
227
|
+
throw new Error(`Project root resolves to already registered project ${duplicate.root}`);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
95
230
|
const idx = entries.findIndex((e) => e.root === entry.root);
|
|
96
231
|
if (idx >= 0) {
|
|
97
232
|
entries[idx] = entry;
|
|
@@ -102,6 +237,219 @@ function registerProject(entry) {
|
|
|
102
237
|
saveRegistry(entries);
|
|
103
238
|
});
|
|
104
239
|
}
|
|
240
|
+
function immutableEntries(entries) {
|
|
241
|
+
return Object.freeze(entries.map((entry) => Object.freeze(Object.assign({}, entry))));
|
|
242
|
+
}
|
|
243
|
+
function saveRebuildJournal(journal) {
|
|
244
|
+
fs.mkdirSync(path.dirname(REBUILD_JOURNAL_PATH), { recursive: true });
|
|
245
|
+
const temp = `${REBUILD_JOURNAL_PATH}.${journal.rebuildId}.tmp`;
|
|
246
|
+
fs.writeFileSync(temp, `${JSON.stringify(journal, null, 2)}\n`, {
|
|
247
|
+
flag: "wx",
|
|
248
|
+
});
|
|
249
|
+
fs.renameSync(temp, REBUILD_JOURNAL_PATH);
|
|
250
|
+
}
|
|
251
|
+
function parseRebuildJournal(filePath) {
|
|
252
|
+
let parsed;
|
|
253
|
+
try {
|
|
254
|
+
parsed = JSON.parse(fs.readFileSync(filePath, "utf8"));
|
|
255
|
+
}
|
|
256
|
+
catch (error) {
|
|
257
|
+
if (error.code === "ENOENT")
|
|
258
|
+
throw error;
|
|
259
|
+
throw new Error(`Invalid rebuild journal ${filePath}: ${error instanceof Error ? error.message : String(error)}`);
|
|
260
|
+
}
|
|
261
|
+
if (!parsed || typeof parsed !== "object") {
|
|
262
|
+
throw new Error(`Invalid rebuild journal ${filePath}`);
|
|
263
|
+
}
|
|
264
|
+
const journal = parsed;
|
|
265
|
+
if (typeof journal.rebuildId !== "string" ||
|
|
266
|
+
(journal.phase !== "reserved" && journal.phase !== "dropping") ||
|
|
267
|
+
typeof journal.targetFingerprint !== "string" ||
|
|
268
|
+
!Array.isArray(journal.previous) ||
|
|
269
|
+
!Array.isArray(journal.reserved) ||
|
|
270
|
+
!journal.previous.every(isProjectEntry) ||
|
|
271
|
+
!journal.reserved.every(isProjectEntry)) {
|
|
272
|
+
throw new Error(`Invalid rebuild journal ${filePath}`);
|
|
273
|
+
}
|
|
274
|
+
return journal;
|
|
275
|
+
}
|
|
276
|
+
function loadRebuildJournal() {
|
|
277
|
+
const dir = path.dirname(REBUILD_JOURNAL_PATH);
|
|
278
|
+
const prefix = `${path.basename(REBUILD_JOURNAL_PATH)}.`;
|
|
279
|
+
let temps = [];
|
|
280
|
+
try {
|
|
281
|
+
temps = fs
|
|
282
|
+
.readdirSync(dir)
|
|
283
|
+
.filter((name) => name.startsWith(prefix) && name.endsWith(".tmp"))
|
|
284
|
+
.map((name) => path.join(dir, name));
|
|
285
|
+
}
|
|
286
|
+
catch (error) {
|
|
287
|
+
if (error.code !== "ENOENT")
|
|
288
|
+
throw error;
|
|
289
|
+
}
|
|
290
|
+
const validTemps = [];
|
|
291
|
+
for (const temp of temps) {
|
|
292
|
+
try {
|
|
293
|
+
validTemps.push({ path: temp, journal: parseRebuildJournal(temp) });
|
|
294
|
+
}
|
|
295
|
+
catch (_a) {
|
|
296
|
+
fs.rmSync(temp, { force: true });
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
if (validTemps.length > 1) {
|
|
300
|
+
throw new Error(`Multiple rebuild journal temp files found in ${dir}`);
|
|
301
|
+
}
|
|
302
|
+
if (validTemps.length === 1) {
|
|
303
|
+
const candidate = validTemps[0];
|
|
304
|
+
try {
|
|
305
|
+
const current = parseRebuildJournal(REBUILD_JOURNAL_PATH);
|
|
306
|
+
if (current.rebuildId !== candidate.journal.rebuildId) {
|
|
307
|
+
throw new Error("Rebuild journal temp identity changed");
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
catch (error) {
|
|
311
|
+
if (error.code !== "ENOENT")
|
|
312
|
+
throw error;
|
|
313
|
+
}
|
|
314
|
+
fs.renameSync(candidate.path, REBUILD_JOURNAL_PATH);
|
|
315
|
+
}
|
|
316
|
+
try {
|
|
317
|
+
return parseRebuildJournal(REBUILD_JOURNAL_PATH);
|
|
318
|
+
}
|
|
319
|
+
catch (error) {
|
|
320
|
+
if (error.code === "ENOENT")
|
|
321
|
+
return null;
|
|
322
|
+
throw error;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
function removeRebuildJournal(rebuildId) {
|
|
326
|
+
const current = loadRebuildJournal();
|
|
327
|
+
if ((current === null || current === void 0 ? void 0 : current.rebuildId) !== rebuildId)
|
|
328
|
+
return;
|
|
329
|
+
fs.rmSync(REBUILD_JOURNAL_PATH, { force: true });
|
|
330
|
+
}
|
|
331
|
+
function reserveProjectsForRebuild(generation) {
|
|
332
|
+
return withRegistryLock(() => {
|
|
333
|
+
const entries = loadRegistry();
|
|
334
|
+
const existingJournal = loadRebuildJournal();
|
|
335
|
+
if (existingJournal) {
|
|
336
|
+
if (existingJournal.targetFingerprint !== generation.fingerprint) {
|
|
337
|
+
throw new ProjectRegistryConflictError("Configured embedding changed during an unfinished rebuild");
|
|
338
|
+
}
|
|
339
|
+
const previousByRoot = new Map(existingJournal.previous.map((entry) => [entry.root, entry]));
|
|
340
|
+
const previous = immutableEntries(entries.map((entry) => { var _a; return (_a = previousByRoot.get(entry.root)) !== null && _a !== void 0 ? _a : entry; }));
|
|
341
|
+
const roots = new Set(entries.map((entry) => entry.root));
|
|
342
|
+
const resumed = entries.map((entry) => roots.has(entry.root)
|
|
343
|
+
? Object.assign(Object.assign({}, entry), { vectorDim: generation.vectorDim, modelTier: generation.tier, embedModel: generation.onnxModel, mlxModel: generation.mlxModel, colbertModel: generation.colbertModel, embeddingFingerprint: generation.fingerprint, status: "pending", rebuildId: existingJournal.rebuildId }) : entry);
|
|
344
|
+
saveRebuildJournal(Object.assign(Object.assign({}, existingJournal), { previous, reserved: immutableEntries(resumed) }));
|
|
345
|
+
saveRegistry(resumed);
|
|
346
|
+
return Object.freeze({
|
|
347
|
+
rebuildId: existingJournal.rebuildId,
|
|
348
|
+
previous,
|
|
349
|
+
reserved: immutableEntries(resumed),
|
|
350
|
+
});
|
|
351
|
+
}
|
|
352
|
+
if (entries.some((entry) => entry.rebuildId)) {
|
|
353
|
+
throw new ProjectRegistryConflictError("A project is already reserved by an active rebuild");
|
|
354
|
+
}
|
|
355
|
+
const rebuildId = (0, node_crypto_1.randomUUID)();
|
|
356
|
+
const previous = immutableEntries(entries);
|
|
357
|
+
const reservedEntries = entries.map((entry) => (Object.assign(Object.assign({}, entry), { vectorDim: generation.vectorDim, modelTier: generation.tier, embedModel: generation.onnxModel, mlxModel: generation.mlxModel, colbertModel: generation.colbertModel, embeddingFingerprint: generation.fingerprint, status: "pending", rebuildId })));
|
|
358
|
+
if (!reservedEntries.every(isProjectEntry)) {
|
|
359
|
+
throw new Error("Invalid rebuild generation reservation");
|
|
360
|
+
}
|
|
361
|
+
const journal = {
|
|
362
|
+
rebuildId,
|
|
363
|
+
phase: "reserved",
|
|
364
|
+
targetFingerprint: generation.fingerprint,
|
|
365
|
+
previous,
|
|
366
|
+
reserved: immutableEntries(reservedEntries),
|
|
367
|
+
};
|
|
368
|
+
saveRebuildJournal(journal);
|
|
369
|
+
try {
|
|
370
|
+
saveRegistry(reservedEntries);
|
|
371
|
+
}
|
|
372
|
+
catch (error) {
|
|
373
|
+
removeRebuildJournal(rebuildId);
|
|
374
|
+
throw error;
|
|
375
|
+
}
|
|
376
|
+
return Object.freeze({
|
|
377
|
+
rebuildId,
|
|
378
|
+
previous,
|
|
379
|
+
reserved: immutableEntries(reservedEntries),
|
|
380
|
+
});
|
|
381
|
+
});
|
|
382
|
+
}
|
|
383
|
+
function restoreProjectsAfterRebuild(reservation) {
|
|
384
|
+
withRegistryLock(() => {
|
|
385
|
+
const entries = loadRegistry();
|
|
386
|
+
const previousByRoot = new Map(reservation.previous.map((entry) => [entry.root, entry]));
|
|
387
|
+
let changed = false;
|
|
388
|
+
const restored = entries.map((entry) => {
|
|
389
|
+
if (entry.rebuildId !== reservation.rebuildId)
|
|
390
|
+
return entry;
|
|
391
|
+
const previous = previousByRoot.get(entry.root);
|
|
392
|
+
if (!previous)
|
|
393
|
+
return entry;
|
|
394
|
+
changed = true;
|
|
395
|
+
return Object.assign({}, previous);
|
|
396
|
+
});
|
|
397
|
+
if (changed)
|
|
398
|
+
saveRegistry(restored);
|
|
399
|
+
removeRebuildJournal(reservation.rebuildId);
|
|
400
|
+
});
|
|
401
|
+
}
|
|
402
|
+
function markProjectRebuildDropping(reservation) {
|
|
403
|
+
withRegistryLock(() => {
|
|
404
|
+
const journal = loadRebuildJournal();
|
|
405
|
+
if (!journal || journal.rebuildId !== reservation.rebuildId) {
|
|
406
|
+
throw new ProjectRegistryConflictError("Project rebuild journal identity changed");
|
|
407
|
+
}
|
|
408
|
+
saveRebuildJournal(Object.assign(Object.assign({}, journal), { phase: "dropping" }));
|
|
409
|
+
});
|
|
410
|
+
}
|
|
411
|
+
function completeProjectRebuild(rebuildId) {
|
|
412
|
+
withRegistryLock(() => {
|
|
413
|
+
const remaining = loadRegistry().some((entry) => entry.rebuildId === rebuildId);
|
|
414
|
+
if (remaining) {
|
|
415
|
+
throw new ProjectRegistryConflictError("Cannot complete rebuild while projects remain pending");
|
|
416
|
+
}
|
|
417
|
+
removeRebuildJournal(rebuildId);
|
|
418
|
+
});
|
|
419
|
+
}
|
|
420
|
+
function hasUnfinishedProjectRebuild() {
|
|
421
|
+
return withRegistryLock(() => loadRebuildJournal() !== null);
|
|
422
|
+
}
|
|
423
|
+
function stampProjectFullSync(stamp) {
|
|
424
|
+
return withRegistryLock(() => {
|
|
425
|
+
var _a, _b, _c, _d, _e;
|
|
426
|
+
const entries = loadRegistry();
|
|
427
|
+
const idx = entries.findIndex((entry) => entry.root === stamp.root);
|
|
428
|
+
const previous = idx >= 0 ? entries[idx] : undefined;
|
|
429
|
+
if ((previous === null || previous === void 0 ? void 0 : previous.rebuildId) && stamp.expectedRebuildId === undefined) {
|
|
430
|
+
throw new ProjectRegistryConflictError("Project is reserved by an active rebuild");
|
|
431
|
+
}
|
|
432
|
+
if (stamp.expectedFingerprint !== undefined &&
|
|
433
|
+
((_a = previous === null || previous === void 0 ? void 0 : previous.embeddingFingerprint) !== null && _a !== void 0 ? _a : null) !== stamp.expectedFingerprint) {
|
|
434
|
+
throw new ProjectRegistryConflictError("Project embedding fingerprint changed during indexing");
|
|
435
|
+
}
|
|
436
|
+
if (stamp.expectedRebuildId !== undefined &&
|
|
437
|
+
((_b = previous === null || previous === void 0 ? void 0 : previous.rebuildId) !== null && _b !== void 0 ? _b : null) !== stamp.expectedRebuildId) {
|
|
438
|
+
throw new ProjectRegistryConflictError("Project rebuild identity changed during indexing");
|
|
439
|
+
}
|
|
440
|
+
const entry = Object.assign(Object.assign({}, previous), { root: stamp.root, name: (_d = (_c = stamp.name) !== null && _c !== void 0 ? _c : previous === null || previous === void 0 ? void 0 : previous.name) !== null && _d !== void 0 ? _d : path.basename(stamp.root), vectorDim: stamp.generation.vectorDim, modelTier: stamp.generation.tier, embedMode: stamp.embedMode, embedModel: stamp.generation.onnxModel, mlxModel: stamp.generation.mlxModel, colbertModel: stamp.generation.colbertModel, embeddingFingerprint: stamp.generation.fingerprint, lastIndexed: (_e = stamp.indexedAt) !== null && _e !== void 0 ? _e : new Date().toISOString(), chunkCount: stamp.chunkCount, status: "indexed", chunkerVersion: stamp.chunkerVersion });
|
|
441
|
+
delete entry.rebuildId;
|
|
442
|
+
if (!isProjectEntry(entry)) {
|
|
443
|
+
throw new Error("Invalid full-sync project stamp");
|
|
444
|
+
}
|
|
445
|
+
if (idx >= 0)
|
|
446
|
+
entries[idx] = entry;
|
|
447
|
+
else
|
|
448
|
+
entries.push(entry);
|
|
449
|
+
saveRegistry(entries);
|
|
450
|
+
return entry;
|
|
451
|
+
});
|
|
452
|
+
}
|
|
105
453
|
function listProjects() {
|
|
106
454
|
return loadRegistry();
|
|
107
455
|
}
|
|
@@ -37,6 +37,7 @@ exports.resolveScope = resolveScope;
|
|
|
37
37
|
exports.buildScopeWhere = buildScopeWhere;
|
|
38
38
|
const path = __importStar(require("node:path"));
|
|
39
39
|
const filter_builder_1 = require("./filter-builder");
|
|
40
|
+
const path_containment_1 = require("./path-containment");
|
|
40
41
|
function toArray(value) {
|
|
41
42
|
if (value === undefined)
|
|
42
43
|
return [];
|
|
@@ -49,15 +50,8 @@ function toArray(value) {
|
|
|
49
50
|
.filter(Boolean);
|
|
50
51
|
}
|
|
51
52
|
function joinSubpath(projectRoot, sub) {
|
|
52
|
-
const
|
|
53
|
-
|
|
54
|
-
: `${projectRoot}/`;
|
|
55
|
-
if (path.isAbsolute(sub))
|
|
56
|
-
return sub.endsWith("/") ? sub : `${sub}/`;
|
|
57
|
-
if (sub.startsWith(rootWithSlash))
|
|
58
|
-
return sub.endsWith("/") ? sub : `${sub}/`;
|
|
59
|
-
const joined = path.join(rootWithSlash, sub);
|
|
60
|
-
return joined.endsWith("/") ? joined : `${joined}/`;
|
|
53
|
+
const resolved = (0, path_containment_1.resolveContainedPath)(projectRoot, sub);
|
|
54
|
+
return resolved.endsWith(path.sep) ? resolved : `${resolved}${path.sep}`;
|
|
61
55
|
}
|
|
62
56
|
function resolveScope(opts) {
|
|
63
57
|
const { projectRoot } = opts;
|
|
@@ -19,6 +19,7 @@ exports.maybeWarnCrossProjectDim = maybeWarnCrossProjectDim;
|
|
|
19
19
|
* chunker does not mask a stale embedding). Suppress all with GMAX_NO_STALE_HINT=1.
|
|
20
20
|
*/
|
|
21
21
|
const config_1 = require("../../config");
|
|
22
|
+
const embedding_status_1 = require("../index/embedding-status");
|
|
22
23
|
const index_config_1 = require("../index/index-config");
|
|
23
24
|
const project_registry_1 = require("./project-registry");
|
|
24
25
|
let chunkerEmitted = false;
|
|
@@ -89,8 +90,8 @@ function maybeWarnStaleEmbedding(projectRoot, opts) {
|
|
|
89
90
|
if (project.status && project.status !== "indexed")
|
|
90
91
|
return;
|
|
91
92
|
const current = (0, index_config_1.readGlobalConfig)();
|
|
92
|
-
const
|
|
93
|
-
if (!
|
|
93
|
+
const identity = (0, embedding_status_1.projectEmbeddingStatus)(project, current);
|
|
94
|
+
if (identity.state !== "stale" || !identity.built)
|
|
94
95
|
return;
|
|
95
96
|
embeddingEmitted = true;
|
|
96
97
|
const name = project.name || projectRoot;
|
|
@@ -98,27 +99,19 @@ function maybeWarnStaleEmbedding(projectRoot, opts) {
|
|
|
98
99
|
const fields = [
|
|
99
100
|
"stale_embedding",
|
|
100
101
|
`project=${name}`,
|
|
101
|
-
`indexed_model=${
|
|
102
|
-
`current_model=${
|
|
103
|
-
`indexed_dim=${
|
|
104
|
-
`current_dim=${
|
|
105
|
-
`
|
|
106
|
-
`
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
`fix=${gap.dimChanged ? config_1.REBUILD_COMMAND : "gmax index --reset"}`,
|
|
102
|
+
`indexed_model=${identity.built.tier}`,
|
|
103
|
+
`current_model=${identity.configured.tier}`,
|
|
104
|
+
`indexed_dim=${identity.built.vectorDim}`,
|
|
105
|
+
`current_dim=${identity.configured.vectorDim}`,
|
|
106
|
+
`indexed_fingerprint=${identity.built.fingerprint}`,
|
|
107
|
+
`current_fingerprint=${identity.configured.fingerprint}`,
|
|
108
|
+
"severity=breaking",
|
|
109
|
+
"fix=gmax repair --rebuild",
|
|
110
110
|
].join("\t");
|
|
111
111
|
process.stderr.write(`${fields}\n`);
|
|
112
112
|
return;
|
|
113
113
|
}
|
|
114
|
-
|
|
115
|
-
const detail = gap.dimChanged
|
|
116
|
-
? `vector dim ${gap.fromDim}→${gap.toDim} (incompatible — scores invalid until re-embed)`
|
|
117
|
-
: `model '${gap.fromModel}'→'${gap.toModel}' (same dim; results mix models until re-embed)`;
|
|
118
|
-
const fix = gap.dimChanged
|
|
119
|
-
? `Run '${config_1.REBUILD_COMMAND}'`
|
|
120
|
-
: "Run 'gmax index --reset'";
|
|
121
|
-
process.stderr.write(`${label} gmax: '${name}' indexed with embedding ${detail}. ${fix}. (silence: GMAX_NO_STALE_HINT=1)\n`);
|
|
114
|
+
process.stderr.write(`WARN gmax: '${name}' uses a stale embedding generation (${identity.built.tier} ${identity.built.vectorDim}d → ${identity.configured.tier} ${identity.configured.vectorDim}d). Run 'gmax repair --rebuild' to rebuild the whole corpus. (silence: GMAX_NO_STALE_HINT=1)\n`);
|
|
122
115
|
}
|
|
123
116
|
/**
|
|
124
117
|
* Cross-project guard: when a `--all-projects` / `--projects` search spans
|
|
@@ -51,7 +51,7 @@ function launchWatcher(projectRoot) {
|
|
|
51
51
|
// 4. Daemon not running — try to start it, poll until ready
|
|
52
52
|
const error = resp.error;
|
|
53
53
|
if (error === "ENOENT" || error === "ECONNREFUSED") {
|
|
54
|
-
const daemonPid = (0, daemon_launcher_1.spawnDaemon)();
|
|
54
|
+
const daemonPid = yield (0, daemon_launcher_1.spawnDaemon)();
|
|
55
55
|
if (daemonPid) {
|
|
56
56
|
for (let i = 0; i < 25; i++) {
|
|
57
57
|
yield new Promise((r) => setTimeout(r, 200));
|
|
@@ -71,6 +71,10 @@ function launchWatcher(projectRoot) {
|
|
|
71
71
|
// 5. Fall back to per-project spawn
|
|
72
72
|
try {
|
|
73
73
|
const child = (0, node_child_process_1.spawn)(process.argv[0], [process.argv[1], "watch", "--path", projectRoot, "-b"], { detached: true, stdio: "ignore" });
|
|
74
|
+
yield new Promise((resolve, reject) => {
|
|
75
|
+
child.once("spawn", resolve);
|
|
76
|
+
child.once("error", reject);
|
|
77
|
+
});
|
|
74
78
|
child.unref();
|
|
75
79
|
if (child.pid) {
|
|
76
80
|
return { ok: true, pid: child.pid, reused: false };
|
|
@@ -99,12 +99,12 @@ function registerWatcher(info) {
|
|
|
99
99
|
}
|
|
100
100
|
db.put(info.projectRoot, Object.assign(Object.assign({}, info), { lastHeartbeat: Date.now() }));
|
|
101
101
|
}
|
|
102
|
-
function updateWatcherStatus(pid, status, lastReindex) {
|
|
102
|
+
function updateWatcherStatus(pid, status, lastReindex, lastError) {
|
|
103
103
|
const db = getDb();
|
|
104
104
|
// Find entry by PID (iterate since key is projectRoot)
|
|
105
105
|
for (const { key, value } of db.getRange()) {
|
|
106
106
|
if (value && value.pid === pid) {
|
|
107
|
-
db.put(String(key), Object.assign(Object.assign(Object.assign({}, value), { status, lastHeartbeat: Date.now() }), (lastReindex ? { lastReindex } : {})));
|
|
107
|
+
db.put(String(key), Object.assign(Object.assign(Object.assign(Object.assign({}, value), { status, lastHeartbeat: Date.now() }), (lastReindex ? { lastReindex } : {})), (lastError ? { lastError } : { lastError: undefined })));
|
|
108
108
|
return;
|
|
109
109
|
}
|
|
110
110
|
}
|
|
@@ -39,19 +39,20 @@ const fs = __importStar(require("node:fs"));
|
|
|
39
39
|
const path = __importStar(require("node:path"));
|
|
40
40
|
const simsimd_1 = require("simsimd");
|
|
41
41
|
const config_1 = require("../../config");
|
|
42
|
-
|
|
43
|
-
function loadSkipIds() {
|
|
44
|
-
|
|
45
|
-
|
|
42
|
+
const SKIP_IDS = new Map();
|
|
43
|
+
function loadSkipIds(modelId) {
|
|
44
|
+
const cached = SKIP_IDS.get(modelId);
|
|
45
|
+
if (cached)
|
|
46
|
+
return cached;
|
|
46
47
|
// Check local models first (same logic as orchestrator)
|
|
47
48
|
const PROJECT_ROOT = process.env.GMAX_PROJECT_ROOT
|
|
48
49
|
? path.resolve(process.env.GMAX_PROJECT_ROOT)
|
|
49
50
|
: process.cwd();
|
|
50
51
|
const localModels = path.join(PROJECT_ROOT, "models");
|
|
51
|
-
const localColbert = path.join(localModels, ...
|
|
52
|
+
const localColbert = path.join(localModels, ...modelId.split("/"));
|
|
52
53
|
const localSkipPath = path.join(localColbert, "skiplist.json");
|
|
53
54
|
// Try local first, then global
|
|
54
|
-
const globalBasePath = path.join(config_1.PATHS.models, ...
|
|
55
|
+
const globalBasePath = path.join(config_1.PATHS.models, ...modelId.split("/"));
|
|
55
56
|
const globalSkipPath = path.join(globalBasePath, "skiplist.json");
|
|
56
57
|
const skipPath = fs.existsSync(localSkipPath)
|
|
57
58
|
? localSkipPath
|
|
@@ -59,24 +60,26 @@ function loadSkipIds() {
|
|
|
59
60
|
if (fs.existsSync(skipPath)) {
|
|
60
61
|
try {
|
|
61
62
|
const parsed = JSON.parse(fs.readFileSync(skipPath, "utf8"));
|
|
62
|
-
|
|
63
|
-
|
|
63
|
+
const ids = new Set(parsed.map((n) => Number(n)));
|
|
64
|
+
SKIP_IDS.set(modelId, ids);
|
|
65
|
+
return ids;
|
|
64
66
|
}
|
|
65
67
|
catch (_e) {
|
|
66
68
|
// fall through to empty set
|
|
67
69
|
}
|
|
68
70
|
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
+
const ids = new Set();
|
|
72
|
+
SKIP_IDS.set(modelId, ids);
|
|
73
|
+
return ids;
|
|
71
74
|
}
|
|
72
|
-
function maxSim(queryEmbeddings, docEmbeddings, docTokenIds) {
|
|
75
|
+
function maxSim(queryEmbeddings, docEmbeddings, docTokenIds, modelId = config_1.MODEL_IDS.colbert) {
|
|
73
76
|
if (queryEmbeddings.length === 0 || docEmbeddings.length === 0) {
|
|
74
77
|
return 0;
|
|
75
78
|
}
|
|
76
79
|
const qVecs = queryEmbeddings.map((v) => v instanceof Float32Array ? v : new Float32Array(v));
|
|
77
80
|
const dVecs = docEmbeddings.map((v) => v instanceof Float32Array ? v : new Float32Array(v));
|
|
78
81
|
const dTokenIds = docTokenIds && docTokenIds.length === dVecs.length ? docTokenIds : null;
|
|
79
|
-
const skipIds = loadSkipIds();
|
|
82
|
+
const skipIds = loadSkipIds(modelId);
|
|
80
83
|
let totalScore = 0;
|
|
81
84
|
for (const qVec of qVecs) {
|
|
82
85
|
let maxDotProduct = -Infinity;
|
|
@@ -57,7 +57,8 @@ const log = (...args) => {
|
|
|
57
57
|
console.log(...args);
|
|
58
58
|
};
|
|
59
59
|
class ColbertModel {
|
|
60
|
-
constructor() {
|
|
60
|
+
constructor(modelId = config_1.MODEL_IDS.colbert) {
|
|
61
|
+
this.modelId = modelId;
|
|
61
62
|
this.session = null;
|
|
62
63
|
this.tokenizer = null;
|
|
63
64
|
}
|
|
@@ -66,7 +67,7 @@ class ColbertModel {
|
|
|
66
67
|
if (this.session && this.tokenizer)
|
|
67
68
|
return;
|
|
68
69
|
this.tokenizer = new colbert_tokenizer_1.ColBERTTokenizer();
|
|
69
|
-
const basePath = path.join(CACHE_DIR,
|
|
70
|
+
const basePath = path.join(CACHE_DIR, this.modelId);
|
|
70
71
|
const onnxDir = path.join(basePath, "onnx");
|
|
71
72
|
const modelPath = path.join(onnxDir, "model_int8.onnx");
|
|
72
73
|
if (!fs.existsSync(modelPath)) {
|
|
@@ -57,7 +57,9 @@ const log = (...args) => {
|
|
|
57
57
|
console.log(...args);
|
|
58
58
|
};
|
|
59
59
|
class GraniteModel {
|
|
60
|
-
constructor(vectorDim
|
|
60
|
+
constructor(vectorDim, modelId = process.env.GMAX_EMBED_ONNX_MODEL ||
|
|
61
|
+
config_1.MODEL_IDS.embed) {
|
|
62
|
+
this.modelId = modelId;
|
|
61
63
|
this.session = null;
|
|
62
64
|
this.tokenizer = null;
|
|
63
65
|
this.vectorDimensions = vectorDim !== null && vectorDim !== void 0 ? vectorDim : config_1.CONFIG.VECTOR_DIM;
|
|
@@ -65,8 +67,7 @@ class GraniteModel {
|
|
|
65
67
|
resolvePaths() {
|
|
66
68
|
// Honor the active tier's ONNX model (propagated from the pool); fall back
|
|
67
69
|
// to the small-tier default when unset (standalone/test invocations).
|
|
68
|
-
const
|
|
69
|
-
const basePath = path.join(CACHE_DIR, modelId);
|
|
70
|
+
const basePath = path.join(CACHE_DIR, this.modelId);
|
|
70
71
|
const onnxDir = path.join(basePath, "onnx");
|
|
71
72
|
const candidates = ["model_q4.onnx", "model.onnx"];
|
|
72
73
|
for (const candidate of candidates) {
|