grepmax 0.26.39 → 0.26.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +4 -0
- package/dist/commands/codex.js +22 -16
- package/dist/commands/impact.js +2 -2
- package/dist/commands/mcp.js +37 -14
- package/dist/commands/peek.js +2 -2
- package/dist/commands/status.js +32 -28
- package/dist/commands/test-find.js +2 -2
- package/dist/commands/trace.js +1 -1
- package/dist/commands/watch.js +7 -0
- package/dist/lib/daemon/daemon.js +47 -15
- package/dist/lib/daemon/graph-handler.js +1 -1
- package/dist/lib/daemon/ipc-handler.js +10 -8
- package/dist/lib/daemon/search-handler.js +18 -8
- package/dist/lib/daemon/watcher-manager.js +37 -4
- package/dist/lib/graph/graph-builder.js +52 -4
- package/dist/lib/graph/impact-rollup.js +2 -2
- package/dist/lib/graph/impact.js +20 -8
- package/dist/lib/graph/test-hits.js +13 -7
- package/dist/lib/index/batch-processor.js +70 -7
- package/dist/lib/index/embedding-reuse.js +47 -0
- package/dist/lib/index/grammar-loader.js +22 -9
- package/dist/lib/output/formatter.js +1 -1
- package/dist/lib/output/index-state-footer.js +26 -6
- package/dist/lib/store/vector-db.js +43 -0
- package/dist/lib/utils/resource-snapshot.js +20 -0
- package/dist/lib/utils/tests-footer.js +8 -16
- package/dist/lib/workers/embedding-error.js +16 -0
- package/dist/lib/workers/embeddings/mlx-client.js +94 -66
- package/dist/lib/workers/orchestrator.js +31 -6
- package/dist/lib/workers/pool.js +1 -1
- package/dist/lib/workers/process-child.js +2 -1
- package/package.json +2 -3
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
|
@@ -14,17 +14,37 @@ exports.formatIndexStateFooter = formatIndexStateFooter;
|
|
|
14
14
|
* while results may actually be incomplete.
|
|
15
15
|
*/
|
|
16
16
|
function formatIndexStateFooter(state, opts) {
|
|
17
|
-
if (!
|
|
17
|
+
if (!state)
|
|
18
18
|
return null;
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
19
|
+
const issues = [];
|
|
20
|
+
if (state.failedFiles)
|
|
21
|
+
issues.push(`${state.failedFiles} files failed`);
|
|
22
|
+
if (state.degraded && !state.failedFiles)
|
|
23
|
+
issues.push("reconciliation incomplete");
|
|
24
|
+
if (state.watcherMode === "polling")
|
|
25
|
+
issues.push("polling; changes may lag up to 5min");
|
|
26
|
+
if (state.watcherMode === "recovering")
|
|
27
|
+
issues.push("watcher recovering");
|
|
28
|
+
if (state.catchupRunning)
|
|
29
|
+
issues.push("reconciling filesystem changes");
|
|
30
|
+
if (issues.length) {
|
|
31
|
+
const next = state.failedFiles || state.degraded
|
|
32
|
+
? "inspect gmax watch status; retry after recovery"
|
|
33
|
+
: "retry after reconciliation";
|
|
34
|
+
return opts.agent
|
|
35
|
+
? `[index: ${issues.join(" · ")} · results may be incomplete · ${next}]`
|
|
36
|
+
: `Index: ${issues.join("; ")} — results may be incomplete; ${next}.`;
|
|
37
|
+
}
|
|
38
|
+
if (!state.indexing)
|
|
39
|
+
return null;
|
|
40
|
+
// Cache-hit samples suggest verification, but do not prove the remaining
|
|
41
|
+
// files are unchanged. Keep that distinction without claiming full coverage.
|
|
22
42
|
if (state.verifying) {
|
|
23
43
|
const n = state.pendingFiles;
|
|
24
44
|
if (opts.agent) {
|
|
25
|
-
return `[index: verifying ~${n}
|
|
45
|
+
return `[index: verifying ~${n} files · coverage not yet verified]`;
|
|
26
46
|
}
|
|
27
|
-
return `Index verifying ${n}
|
|
47
|
+
return `Index verifying ${n} file${n === 1 ? "" : "s"} — coverage is not yet verified.`;
|
|
28
48
|
}
|
|
29
49
|
const count = state.pendingFiles > 0 ? `~${state.pendingFiles} files pending` : null;
|
|
30
50
|
if (opts.agent) {
|
|
@@ -51,6 +51,7 @@ const fs = __importStar(require("node:fs"));
|
|
|
51
51
|
const lancedb = __importStar(require("@lancedb/lancedb"));
|
|
52
52
|
const apache_arrow_1 = require("apache-arrow");
|
|
53
53
|
const config_1 = require("../../config");
|
|
54
|
+
const embedding_reuse_1 = require("../index/embedding-reuse");
|
|
54
55
|
const index_config_1 = require("../index/index-config");
|
|
55
56
|
const cleanup_1 = require("../utils/cleanup");
|
|
56
57
|
const filter_builder_1 = require("../utils/filter-builder");
|
|
@@ -1435,6 +1436,48 @@ class VectorDB {
|
|
|
1435
1436
|
return unique;
|
|
1436
1437
|
});
|
|
1437
1438
|
}
|
|
1439
|
+
/**
|
|
1440
|
+
* The stored embeddings of one file, keyed by `embeddingReuseKey(content)`.
|
|
1441
|
+
* Rows whose dense width disagrees with this table are left out rather than
|
|
1442
|
+
* offered — reuse must never be the way a wrong-width vector gets copied.
|
|
1443
|
+
*/
|
|
1444
|
+
getReusableEmbeddings(filePath) {
|
|
1445
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1446
|
+
var _a;
|
|
1447
|
+
const table = yield this.ensureTable();
|
|
1448
|
+
const rows = yield table
|
|
1449
|
+
.query()
|
|
1450
|
+
.select([
|
|
1451
|
+
"content",
|
|
1452
|
+
"vector",
|
|
1453
|
+
"colbert",
|
|
1454
|
+
"colbert_scale",
|
|
1455
|
+
"pooled_colbert_48d",
|
|
1456
|
+
"doc_token_ids",
|
|
1457
|
+
])
|
|
1458
|
+
.where(`path = '${(0, filter_builder_1.escapeSqlString)(filePath)}'`)
|
|
1459
|
+
.toArray();
|
|
1460
|
+
const reusable = new Map();
|
|
1461
|
+
for (const row of rows) {
|
|
1462
|
+
const vector = (0, embedding_reuse_1.toFloat32)(row.vector);
|
|
1463
|
+
const colbert = (0, embedding_reuse_1.toBytes)(row.colbert);
|
|
1464
|
+
if (vector.length !== this.vectorDim || colbert.length === 0)
|
|
1465
|
+
continue;
|
|
1466
|
+
const pooled = row.pooled_colbert_48d == null
|
|
1467
|
+
? undefined
|
|
1468
|
+
: (0, embedding_reuse_1.toFloat32)(row.pooled_colbert_48d);
|
|
1469
|
+
const tokenIds = row.doc_token_ids == null ? undefined : (0, embedding_reuse_1.toInt32)(row.doc_token_ids);
|
|
1470
|
+
reusable.set((0, embedding_reuse_1.embeddingReuseKey)(String((_a = row.content) !== null && _a !== void 0 ? _a : "")), {
|
|
1471
|
+
vector,
|
|
1472
|
+
colbert,
|
|
1473
|
+
colbert_scale: typeof row.colbert_scale === "number" ? row.colbert_scale : 1,
|
|
1474
|
+
pooled_colbert_48d: (pooled === null || pooled === void 0 ? void 0 : pooled.length) ? pooled : undefined,
|
|
1475
|
+
doc_token_ids: tokenIds,
|
|
1476
|
+
});
|
|
1477
|
+
}
|
|
1478
|
+
return reusable;
|
|
1479
|
+
});
|
|
1480
|
+
}
|
|
1438
1481
|
getStats() {
|
|
1439
1482
|
return __awaiter(this, void 0, void 0, function* () {
|
|
1440
1483
|
const table = yield this.ensureTable();
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildResourceSnapshot = buildResourceSnapshot;
|
|
4
|
+
function buildResourceSnapshot(counters, memory = process.memoryUsage(), at = Date.now()) {
|
|
5
|
+
const mb = (bytes) => Math.round(bytes / 1024 / 1024);
|
|
6
|
+
return {
|
|
7
|
+
at,
|
|
8
|
+
reason: counters.reason,
|
|
9
|
+
footprintMb: counters.footprintMb,
|
|
10
|
+
rssMb: mb(memory.rss),
|
|
11
|
+
heapUsedMb: mb(memory.heapUsed),
|
|
12
|
+
externalMb: mb(memory.external),
|
|
13
|
+
arrayBuffersMb: mb(memory.arrayBuffers),
|
|
14
|
+
lanceCacheMb: counters.lanceCacheBytes === null ? null : mb(counters.lanceCacheBytes),
|
|
15
|
+
workers: counters.workers,
|
|
16
|
+
pendingFiles: counters.pendingFiles,
|
|
17
|
+
operations: counters.operations,
|
|
18
|
+
maintenance: counters.maintenance,
|
|
19
|
+
};
|
|
20
|
+
}
|
|
@@ -15,6 +15,7 @@ exports.fetchTestsForFooter = fetchTestsForFooter;
|
|
|
15
15
|
exports.renderTestsFooterAgent = renderTestsFooterAgent;
|
|
16
16
|
exports.renderTestsFooterHuman = renderTestsFooterHuman;
|
|
17
17
|
const impact_1 = require("../graph/impact");
|
|
18
|
+
const test_hits_1 = require("../graph/test-hits");
|
|
18
19
|
/**
|
|
19
20
|
* The footer is a courtesy line under a body the user asked for; it must never
|
|
20
21
|
* make the command feel slow. Past this budget the lookup is abandoned and the
|
|
@@ -57,24 +58,10 @@ function fetchTestsForFooter(symbol, vectorDb, pathPrefix, excludePrefixes) {
|
|
|
57
58
|
function relPath(p, projectRoot) {
|
|
58
59
|
return p.startsWith(`${projectRoot}/`) ? p.slice(projectRoot.length + 1) : p;
|
|
59
60
|
}
|
|
60
|
-
function hopLabelAgent(hops) {
|
|
61
|
-
if (hops === -1)
|
|
62
|
-
return "via-import";
|
|
63
|
-
if (hops === 0)
|
|
64
|
-
return "direct";
|
|
65
|
-
return `${hops}-hop`;
|
|
66
|
-
}
|
|
67
|
-
function hopLabelHuman(hops) {
|
|
68
|
-
if (hops === -1)
|
|
69
|
-
return "via import";
|
|
70
|
-
if (hops === 0)
|
|
71
|
-
return "direct";
|
|
72
|
-
return `${hops} hop${hops > 1 ? "s" : ""}`;
|
|
73
|
-
}
|
|
74
61
|
function renderTestsFooterAgent(tests, projectRoot) {
|
|
75
62
|
const lines = [];
|
|
76
63
|
for (const t of tests.slice(0, MAX_SHOWN)) {
|
|
77
|
-
lines.push(`t: ${relPath(t.file, projectRoot)}:${t.line + 1}\t${t.symbol}\t${hopLabelAgent(t.hops)}`);
|
|
64
|
+
lines.push(`t: ${relPath(t.file, projectRoot)}:${t.line + 1}\t${t.symbol}\t${(0, test_hits_1.hopLabelAgent)(t.hops, t.evidence)}`);
|
|
78
65
|
}
|
|
79
66
|
if (tests.length > MAX_SHOWN) {
|
|
80
67
|
lines.push(`t: ... ${tests.length - MAX_SHOWN} more`);
|
|
@@ -86,7 +73,12 @@ function renderTestsFooterHuman(tests, projectRoot) {
|
|
|
86
73
|
lines.push("");
|
|
87
74
|
lines.push(`tests (${tests.length}):`);
|
|
88
75
|
for (const t of tests.slice(0, MAX_SHOWN)) {
|
|
89
|
-
|
|
76
|
+
const label = t.evidence
|
|
77
|
+
? "same-file candidate"
|
|
78
|
+
: t.hops === -1
|
|
79
|
+
? "via import"
|
|
80
|
+
: (0, test_hits_1.hopLabelAgent)(t.hops);
|
|
81
|
+
lines.push(` ${t.symbol.padEnd(25)} ${dim(`${relPath(t.file, projectRoot)}:${t.line + 1}`)} ${dim(`(${label})`)}`);
|
|
90
82
|
}
|
|
91
83
|
if (tests.length > MAX_SHOWN) {
|
|
92
84
|
lines.push(dim(` ... and ${tests.length - MAX_SHOWN} more`));
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.EmbeddingBackendUnavailableError = void 0;
|
|
4
|
+
exports.isEmbeddingBackendUnavailable = isEmbeddingBackendUnavailable;
|
|
5
|
+
/** Shared backend outages are retryable; malformed inputs/protocols are not. */
|
|
6
|
+
class EmbeddingBackendUnavailableError extends Error {
|
|
7
|
+
constructor(message) {
|
|
8
|
+
super(message);
|
|
9
|
+
this.code = "EMBED_BACKEND_UNAVAILABLE";
|
|
10
|
+
this.name = "EmbeddingBackendUnavailableError";
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
exports.EmbeddingBackendUnavailableError = EmbeddingBackendUnavailableError;
|
|
14
|
+
function isEmbeddingBackendUnavailable(error) {
|
|
15
|
+
return ((error === null || error === void 0 ? void 0 : error.code) === "EMBED_BACKEND_UNAVAILABLE");
|
|
16
|
+
}
|
|
@@ -48,6 +48,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
48
48
|
};
|
|
49
49
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
50
50
|
exports.validateMlxEmbeddingResponse = validateMlxEmbeddingResponse;
|
|
51
|
+
exports.requestMlxJSON = requestMlxJSON;
|
|
51
52
|
exports.isMlxUp = isMlxUp;
|
|
52
53
|
exports.mlxEmbed = mlxEmbed;
|
|
53
54
|
const http = __importStar(require("node:http"));
|
|
@@ -81,81 +82,108 @@ function validateMlxEmbeddingResponse(data, textCount, expectedModel, expectedDi
|
|
|
81
82
|
Number.isFinite(element) &&
|
|
82
83
|
Math.abs(element) <= FLOAT32_MAX)));
|
|
83
84
|
}
|
|
84
|
-
|
|
85
|
+
/** A hard deadline also bounds stalled/truncated response bodies. */
|
|
86
|
+
function requestMlxJSON(reqPath, body) {
|
|
85
87
|
const start = performance.now();
|
|
86
88
|
return new Promise((resolve) => {
|
|
87
|
-
const payload = JSON.stringify(body);
|
|
89
|
+
const payload = body === undefined ? undefined : JSON.stringify(body);
|
|
90
|
+
let settled = false;
|
|
91
|
+
let deadline;
|
|
92
|
+
const finish = (result) => {
|
|
93
|
+
if (settled)
|
|
94
|
+
return;
|
|
95
|
+
settled = true;
|
|
96
|
+
clearTimeout(deadline);
|
|
97
|
+
resolve(Object.assign(Object.assign({}, result), { ms: Math.round(performance.now() - start) }));
|
|
98
|
+
};
|
|
88
99
|
const req = http.request({
|
|
89
100
|
hostname: MLX_HOST,
|
|
90
101
|
port: MLX_PORT,
|
|
91
102
|
path: reqPath,
|
|
92
|
-
method: "POST",
|
|
93
|
-
headers:
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
103
|
+
method: payload === undefined ? "GET" : "POST",
|
|
104
|
+
headers: payload === undefined
|
|
105
|
+
? undefined
|
|
106
|
+
: {
|
|
107
|
+
"Content-Type": "application/json",
|
|
108
|
+
"Content-Length": Buffer.byteLength(payload),
|
|
109
|
+
},
|
|
98
110
|
}, (res) => {
|
|
99
111
|
const chunks = [];
|
|
100
|
-
|
|
112
|
+
let bytes = 0;
|
|
113
|
+
res.on("data", (chunk) => {
|
|
114
|
+
bytes += chunk.length;
|
|
115
|
+
if (bytes > 16 * 1024 * 1024) {
|
|
116
|
+
finish({
|
|
117
|
+
ok: false,
|
|
118
|
+
category: "protocol",
|
|
119
|
+
status: res.statusCode,
|
|
120
|
+
detail: "response exceeds 16MiB",
|
|
121
|
+
});
|
|
122
|
+
req.destroy();
|
|
123
|
+
}
|
|
124
|
+
else
|
|
125
|
+
chunks.push(chunk);
|
|
126
|
+
});
|
|
127
|
+
res.on("aborted", () => finish({
|
|
128
|
+
ok: false,
|
|
129
|
+
category: "transport",
|
|
130
|
+
status: res.statusCode,
|
|
131
|
+
detail: "response aborted",
|
|
132
|
+
}));
|
|
133
|
+
res.on("error", (err) => finish({
|
|
134
|
+
ok: false,
|
|
135
|
+
category: "transport",
|
|
136
|
+
status: res.statusCode,
|
|
137
|
+
detail: err.message.slice(0, 512),
|
|
138
|
+
}));
|
|
101
139
|
res.on("end", () => {
|
|
140
|
+
const raw = Buffer.concat(chunks).toString("utf8");
|
|
141
|
+
if (res.statusCode !== 200) {
|
|
142
|
+
finish({
|
|
143
|
+
ok: false,
|
|
144
|
+
category: "http",
|
|
145
|
+
status: res.statusCode,
|
|
146
|
+
detail: raw.slice(0, 512),
|
|
147
|
+
});
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
102
150
|
try {
|
|
103
|
-
|
|
104
|
-
const ok = res.statusCode === 200;
|
|
105
|
-
(0, logger_1.debug)("mlx", `POST ${reqPath} → ${res.statusCode} ${(performance.now() - start).toFixed(0)}ms payload=${payload.length}B`);
|
|
106
|
-
resolve({ ok, data });
|
|
151
|
+
finish({ ok: true, status: res.statusCode, data: JSON.parse(raw) });
|
|
107
152
|
}
|
|
108
153
|
catch (_a) {
|
|
109
|
-
(
|
|
110
|
-
|
|
154
|
+
finish({
|
|
155
|
+
ok: false,
|
|
156
|
+
category: "protocol",
|
|
157
|
+
status: res.statusCode,
|
|
158
|
+
detail: "invalid JSON",
|
|
159
|
+
});
|
|
111
160
|
}
|
|
112
161
|
});
|
|
113
162
|
});
|
|
114
|
-
req.on("error", (err) => {
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
163
|
+
req.on("error", (err) => finish({
|
|
164
|
+
ok: false,
|
|
165
|
+
category: "transport",
|
|
166
|
+
detail: err.message.slice(0, 512),
|
|
167
|
+
}));
|
|
168
|
+
deadline = setTimeout(() => {
|
|
169
|
+
finish({
|
|
170
|
+
ok: false,
|
|
171
|
+
category: "timeout",
|
|
172
|
+
detail: "request deadline exceeded",
|
|
173
|
+
});
|
|
120
174
|
req.destroy();
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
req.write(payload);
|
|
124
|
-
req.end();
|
|
175
|
+
}, reqPath === "/health" ? 2000 : MLX_TIMEOUT_MS);
|
|
176
|
+
req.end(payload);
|
|
125
177
|
});
|
|
126
178
|
}
|
|
127
|
-
/**
|
|
128
|
-
* Check if MLX server is reachable. Caches result for CHECK_INTERVAL_MS.
|
|
129
|
-
*/
|
|
130
179
|
function checkHealth(expectedModel) {
|
|
131
180
|
return __awaiter(this, void 0, void 0, function* () {
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
let model;
|
|
139
|
-
try {
|
|
140
|
-
model = JSON.parse(Buffer.concat(chunks).toString("utf8")).model;
|
|
141
|
-
}
|
|
142
|
-
catch (_a) { }
|
|
143
|
-
const ok = res.statusCode === 200 &&
|
|
144
|
-
(!expectedModel || model === expectedModel);
|
|
145
|
-
(0, logger_1.debug)("mlx", `health → ${ok ? "ok" : `status=${res.statusCode} model=${String(model)}`} ${(performance.now() - start).toFixed(0)}ms`);
|
|
146
|
-
resolve(ok);
|
|
147
|
-
});
|
|
148
|
-
});
|
|
149
|
-
req.on("error", (err) => {
|
|
150
|
-
(0, logger_1.debug)("mlx", `health → error: ${err.message} ${(performance.now() - start).toFixed(0)}ms`);
|
|
151
|
-
resolve(false);
|
|
152
|
-
});
|
|
153
|
-
req.on("timeout", () => {
|
|
154
|
-
(0, logger_1.debug)("mlx", `health → timeout ${(performance.now() - start).toFixed(0)}ms`);
|
|
155
|
-
req.destroy();
|
|
156
|
-
resolve(false);
|
|
157
|
-
});
|
|
158
|
-
});
|
|
181
|
+
var _a;
|
|
182
|
+
var _b, _c;
|
|
183
|
+
const result = yield requestMlxJSON("/health");
|
|
184
|
+
const ok = result.ok && (!expectedModel || ((_a = result.data) === null || _a === void 0 ? void 0 : _a.model) === expectedModel);
|
|
185
|
+
(0, logger_1.debug)("mlx", `health ok=${ok} category=${(_b = result.category) !== null && _b !== void 0 ? _b : "none"} status=${(_c = result.status) !== null && _c !== void 0 ? _c : "none"} ms=${result.ms}`);
|
|
186
|
+
return ok;
|
|
159
187
|
});
|
|
160
188
|
}
|
|
161
189
|
function isMlxUp(expectedModel) {
|
|
@@ -192,7 +220,7 @@ function isMlxUp(expectedModel) {
|
|
|
192
220
|
*/
|
|
193
221
|
function mlxEmbed(texts, options) {
|
|
194
222
|
return __awaiter(this, void 0, void 0, function* () {
|
|
195
|
-
var _a;
|
|
223
|
+
var _a, _b, _c, _d, _e, _f, _g;
|
|
196
224
|
const mode = (_a = options === null || options === void 0 ? void 0 : options.mode) !== null && _a !== void 0 ? _a : EMBED_MODE;
|
|
197
225
|
if (mode === "cpu")
|
|
198
226
|
return null;
|
|
@@ -201,7 +229,7 @@ function mlxEmbed(texts, options) {
|
|
|
201
229
|
(0, logger_1.debug)("mlx", `embed ${texts.length} texts`);
|
|
202
230
|
let postResult;
|
|
203
231
|
try {
|
|
204
|
-
postResult = yield
|
|
232
|
+
postResult = yield requestMlxJSON("/embed", {
|
|
205
233
|
texts,
|
|
206
234
|
expected_model: options === null || options === void 0 ? void 0 : options.expectedModel,
|
|
207
235
|
});
|
|
@@ -223,17 +251,17 @@ function mlxEmbed(texts, options) {
|
|
|
223
251
|
const now = Date.now();
|
|
224
252
|
if (wasPreviouslyAvailable ||
|
|
225
253
|
now - lastMlxWarning >= MLX_WARNING_INTERVAL_MS) {
|
|
226
|
-
console.error(
|
|
227
|
-
ok +
|
|
228
|
-
", validResponse=" +
|
|
229
|
-
responseMatches +
|
|
230
|
-
", dim=" +
|
|
231
|
-
String(data === null || data === void 0 ? void 0 : data.dim) +
|
|
232
|
-
", model=" +
|
|
233
|
-
String(data === null || data === void 0 ? void 0 : data.model) +
|
|
234
|
-
")");
|
|
254
|
+
console.error(`[mlx] Embed failed: category=${(_b = postResult.category) !== null && _b !== void 0 ? _b : "protocol"} status=${(_c = postResult.status) !== null && _c !== void 0 ? _c : "none"} ms=${postResult.ms} batch=${texts.length} detail=${JSON.stringify((_d = postResult.detail) !== null && _d !== void 0 ? _d : `invalid vectors/model: dim=${String(data === null || data === void 0 ? void 0 : data.dim)} model=${String(data === null || data === void 0 ? void 0 : data.model)}`).slice(0, 600)}`);
|
|
235
255
|
lastMlxWarning = now;
|
|
236
256
|
}
|
|
257
|
+
if ((postResult.ok && !responseMatches) ||
|
|
258
|
+
postResult.category === "protocol" ||
|
|
259
|
+
(postResult.category === "http" &&
|
|
260
|
+
((_e = postResult.status) !== null && _e !== void 0 ? _e : 500) < 500 &&
|
|
261
|
+
postResult.status !== 429 &&
|
|
262
|
+
postResult.status !== 408)) {
|
|
263
|
+
throw new Error(`MLX embedding protocol failure: status=${(_f = postResult.status) !== null && _f !== void 0 ? _f : "none"} ${(_g = postResult.detail) !== null && _g !== void 0 ? _g : "invalid vectors or model identity"}`);
|
|
264
|
+
}
|
|
237
265
|
return null;
|
|
238
266
|
}
|
|
239
267
|
return data.vectors.map((v) => new Float32Array(v));
|
|
@@ -52,11 +52,13 @@ const uuid_1 = require("uuid");
|
|
|
52
52
|
const config_1 = require("../../config");
|
|
53
53
|
const chunker_1 = require("../index/chunker");
|
|
54
54
|
const embedding_generation_1 = require("../index/embedding-generation");
|
|
55
|
+
const embedding_reuse_1 = require("../index/embedding-reuse");
|
|
55
56
|
const skeleton_1 = require("../skeleton");
|
|
56
57
|
const file_utils_1 = require("../utils/file-utils");
|
|
57
58
|
const logger_1 = require("../utils/logger");
|
|
58
59
|
const path_containment_1 = require("../utils/path-containment");
|
|
59
60
|
const colbert_math_1 = require("./colbert-math");
|
|
61
|
+
const embedding_error_1 = require("./embedding-error");
|
|
60
62
|
const colbert_1 = require("./embeddings/colbert");
|
|
61
63
|
const granite_1 = require("./embeddings/granite");
|
|
62
64
|
const mlx_client_1 = require("./embeddings/mlx-client");
|
|
@@ -157,10 +159,10 @@ class WorkerOrchestrator {
|
|
|
157
159
|
expectedDim: this.generation.vectorDim,
|
|
158
160
|
});
|
|
159
161
|
if (!mlxResult && !this.allowOnnxFallback) {
|
|
160
|
-
throw new
|
|
162
|
+
throw new embedding_error_1.EmbeddingBackendUnavailableError(`MLX embedding model ${this.generation.mlxModel} is unavailable; ONNX fallback is disabled for this custom embedding generation`);
|
|
161
163
|
}
|
|
162
164
|
if (!mlxResult && denseBackend === "mlx") {
|
|
163
|
-
throw new
|
|
165
|
+
throw new embedding_error_1.EmbeddingBackendUnavailableError(`MLX embedding model ${this.generation.mlxModel} became unavailable during this embedding operation`);
|
|
164
166
|
}
|
|
165
167
|
if (!mlxResult && !mlxFallbackWarned) {
|
|
166
168
|
console.warn("[embed] MLX unavailable, falling back to CPU (ONNX)");
|
|
@@ -234,6 +236,7 @@ class WorkerOrchestrator {
|
|
|
234
236
|
}
|
|
235
237
|
processFile(input, onProgress) {
|
|
236
238
|
return __awaiter(this, void 0, void 0, function* () {
|
|
239
|
+
var _a;
|
|
237
240
|
const fileStart = performance.now();
|
|
238
241
|
const absolutePath = path.isAbsolute(input.path)
|
|
239
242
|
? input.path
|
|
@@ -281,12 +284,32 @@ class WorkerOrchestrator {
|
|
|
281
284
|
return { vectors: [], hash, mtimeMs, size };
|
|
282
285
|
}
|
|
283
286
|
const preparedChunks = this.toPreparedChunks(input.path, hash, chunks, skeletonResult.success ? skeletonResult.skeleton : undefined);
|
|
287
|
+
const reusableKeys = ((_a = input.reusableKeys) === null || _a === void 0 ? void 0 : _a.length)
|
|
288
|
+
? new Set(input.reusableKeys)
|
|
289
|
+
: null;
|
|
290
|
+
const reused = [];
|
|
291
|
+
const toEmbed = [];
|
|
292
|
+
preparedChunks.forEach((chunk, idx) => {
|
|
293
|
+
if (reusableKeys) {
|
|
294
|
+
const key = (0, embedding_reuse_1.embeddingReuseKey)(chunk.content);
|
|
295
|
+
if (reusableKeys.has(key)) {
|
|
296
|
+
reused.push({ index: idx, key });
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
toEmbed.push(idx);
|
|
301
|
+
});
|
|
284
302
|
const embedStart = performance.now();
|
|
285
|
-
const
|
|
286
|
-
|
|
303
|
+
const embedded = yield this.computeHybrid(toEmbed.map((idx) => preparedChunks[idx].content), onProgress);
|
|
304
|
+
const hybrids = new Map();
|
|
305
|
+
toEmbed.forEach((chunkIdx, i) => {
|
|
306
|
+
if (embedded[i])
|
|
307
|
+
hybrids.set(chunkIdx, embedded[i]);
|
|
308
|
+
});
|
|
309
|
+
(0, logger_1.debug)("orch", `embedded ${input.path} → ${embedded.length} vectors (${reused.length} reused) ${(performance.now() - embedStart).toFixed(0)}ms`);
|
|
287
310
|
const vectors = preparedChunks.map((chunk, idx) => {
|
|
288
311
|
var _a;
|
|
289
|
-
const hybrid = (_a = hybrids
|
|
312
|
+
const hybrid = (_a = hybrids.get(idx)) !== null && _a !== void 0 ? _a : {
|
|
290
313
|
dense: new Float32Array(),
|
|
291
314
|
colbert: new Int8Array(),
|
|
292
315
|
scale: 1,
|
|
@@ -295,7 +318,9 @@ class WorkerOrchestrator {
|
|
|
295
318
|
});
|
|
296
319
|
onProgress === null || onProgress === void 0 ? void 0 : onProgress();
|
|
297
320
|
(0, logger_1.debug)("orch", `processFile done: ${input.path} ${vectors.length} vectors ${(performance.now() - fileStart).toFixed(0)}ms`);
|
|
298
|
-
return
|
|
321
|
+
return reused.length
|
|
322
|
+
? { vectors, hash, mtimeMs, size, reused }
|
|
323
|
+
: { vectors, hash, mtimeMs, size };
|
|
299
324
|
});
|
|
300
325
|
}
|
|
301
326
|
encodeQuery(text) {
|
package/dist/lib/workers/pool.js
CHANGED
|
@@ -384,7 +384,7 @@ class WorkerPool {
|
|
|
384
384
|
if ("error" in msg) {
|
|
385
385
|
const filePath = (_f = (_e = (_a = task.payload) === null || _a === void 0 ? void 0 : _a.path) !== null && _e !== void 0 ? _e : (_b = task.payload) === null || _b === void 0 ? void 0 : _b.absolutePath) !== null && _f !== void 0 ? _f : "unknown";
|
|
386
386
|
(0, logger_1.debug)("pool", `error task=${task.id} method=${task.method} file=${filePath}: ${msg.error}`);
|
|
387
|
-
task.reject(new Error(msg.error));
|
|
387
|
+
task.reject(Object.assign(new Error(msg.error), typeof msg.code === "string" ? { code: msg.code } : {}));
|
|
388
388
|
}
|
|
389
389
|
else {
|
|
390
390
|
const elapsed = task.startTime
|
|
@@ -93,7 +93,8 @@ const handleMessage = (msg) => __awaiter(void 0, void 0, void 0, function* () {
|
|
|
93
93
|
catch (err) {
|
|
94
94
|
const message = err instanceof Error ? err.message : String(err);
|
|
95
95
|
(0, logger_1.debug)("worker", `fail task=${id} method=${method} ${(performance.now() - start).toFixed(0)}ms: ${message}`);
|
|
96
|
-
|
|
96
|
+
const code = err === null || err === void 0 ? void 0 : err.code;
|
|
97
|
+
send(Object.assign({ id, error: message }, (typeof code === "string" ? { code } : {})));
|
|
97
98
|
}
|
|
98
99
|
});
|
|
99
100
|
const handleMessageSerially = (0, serialized_handler_1.createSerializedHandler)(handleMessage);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "grepmax",
|
|
3
|
-
"version": "0.26.
|
|
3
|
+
"version": "0.26.41",
|
|
4
4
|
"author": "Robert Owens <78518764+reowens@users.noreply.github.com>",
|
|
5
5
|
"homepage": "https://github.com/reowens/grepmax",
|
|
6
6
|
"bugs": {
|
|
@@ -40,7 +40,7 @@
|
|
|
40
40
|
"typecheck": "tsc --noEmit",
|
|
41
41
|
"test:typecheck": "tsc -p tsconfig.tests.json",
|
|
42
42
|
"prepublishOnly": "pnpm run test:typecheck && pnpm build",
|
|
43
|
-
"preversion": "pnpm test && pnpm typecheck && pnpm run test:typecheck && pnpm run format:check",
|
|
43
|
+
"preversion": "pnpm audit --prod && pnpm test && pnpm typecheck && pnpm run test:typecheck && pnpm run format:check",
|
|
44
44
|
"version": "bash scripts/sync-versions.sh && git add -A",
|
|
45
45
|
"postversion": "bash scripts/postrelease.sh"
|
|
46
46
|
},
|
|
@@ -86,7 +86,6 @@
|
|
|
86
86
|
"cli-highlight": "^2.1.11",
|
|
87
87
|
"commander": "^15.0.0",
|
|
88
88
|
"dotenv": "^17.4.2",
|
|
89
|
-
"fast-glob": "^3.3.3",
|
|
90
89
|
"ignore": "^7.0.9",
|
|
91
90
|
"lmdb": "^3.5.6",
|
|
92
91
|
"numkong": "^7.8.2",
|