grepmax 0.26.42 → 0.26.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/NOTICE +6 -0
- package/README.md +7 -1
- package/dist/bin.js +4 -0
- package/dist/commands/doctor.js +23 -33
- package/dist/commands/mcp.js +144 -67
- package/dist/commands/project.js +0 -1
- package/dist/commands/similar.js +0 -1
- package/dist/commands/skeleton.js +0 -1
- package/dist/commands/status.js +35 -19
- package/dist/commands/surprises.js +0 -1
- package/dist/commands/symbols.js +0 -1
- package/dist/commands/watch.js +2 -1
- package/dist/eval-like-limit-probe.js +60 -41
- package/dist/lib/analysis/surprising-connections.js +4 -3
- package/dist/lib/daemon/daemon.js +36 -9
- package/dist/lib/daemon/graph-handler.js +33 -33
- package/dist/lib/daemon/ipc-handler.js +11 -10
- package/dist/lib/daemon/rows-handler.js +13 -51
- package/dist/lib/daemon/vector-handler.js +18 -17
- package/dist/lib/graph/graph-builder.js +33 -16
- package/dist/lib/graph/impact.js +9 -8
- package/dist/lib/index/index-config.js +2 -2
- package/dist/lib/llm/config.js +9 -3
- package/dist/lib/llm/tools.js +5 -4
- package/dist/lib/search/pagerank.js +2 -1
- package/dist/lib/search/searcher.js +5 -4
- package/dist/lib/skeleton/retriever.js +2 -1
- package/dist/lib/store/compaction-result.js +35 -0
- package/dist/lib/store/lance-sdk.js +46 -0
- package/dist/lib/store/vector-db.js +118 -31
- package/dist/lib/utils/cross-project.js +2 -2
- package/dist/lib/utils/doctor-optimize.js +81 -0
- package/dist/lib/utils/project-registry.js +8 -8
- package/dist/lib/utils/query-timeout.js +53 -13
- package/dist/lib/utils/store-access.js +6 -5
- package/dist/lib/utils/store-context.js +102 -0
- package/dist/lib/utils/stores.js +1 -1
- package/dist/vendor/lancedb/LICENSE +202 -0
- package/dist/vendor/lancedb/NODEJS_THIRD_PARTY_LICENSES.md +668 -0
- package/dist/vendor/lancedb/RUST_THIRD_PARTY_LICENSES.html +14607 -0
- package/dist/vendor/lancedb/arrow.d.ts +296 -0
- package/dist/vendor/lancedb/arrow.js +1145 -0
- package/dist/vendor/lancedb/arrow_type.d.ts +9 -0
- package/dist/vendor/lancedb/arrow_type.js +29 -0
- package/dist/vendor/lancedb/connection.d.ts +508 -0
- package/dist/vendor/lancedb/connection.js +298 -0
- package/dist/vendor/lancedb/embedding/embedding_function.d.ts +103 -0
- package/dist/vendor/lancedb/embedding/embedding_function.js +192 -0
- package/dist/vendor/lancedb/embedding/index.d.ts +37 -0
- package/dist/vendor/lancedb/embedding/index.js +108 -0
- package/dist/vendor/lancedb/embedding/openai.d.ts +16 -0
- package/dist/vendor/lancedb/embedding/openai.js +81 -0
- package/dist/vendor/lancedb/embedding/registry.d.ts +94 -0
- package/dist/vendor/lancedb/embedding/registry.js +221 -0
- package/dist/vendor/lancedb/embedding/transformers.d.ts +36 -0
- package/dist/vendor/lancedb/embedding/transformers.js +110 -0
- package/dist/vendor/lancedb/header.d.ts +162 -0
- package/dist/vendor/lancedb/header.js +217 -0
- package/dist/vendor/lancedb/index.d.ts +246 -0
- package/dist/vendor/lancedb/index.js +176 -0
- package/dist/vendor/lancedb/indices.d.ts +721 -0
- package/dist/vendor/lancedb/indices.js +166 -0
- package/dist/vendor/lancedb/license_header.txt +2 -0
- package/dist/vendor/lancedb/materialized_view.d.ts +69 -0
- package/dist/vendor/lancedb/materialized_view.js +112 -0
- package/dist/vendor/lancedb/merge.d.ts +104 -0
- package/dist/vendor/lancedb/merge.js +120 -0
- package/dist/vendor/lancedb/native.d.ts +1193 -0
- package/dist/vendor/lancedb/native.js +688 -0
- package/dist/vendor/lancedb/oauth.d.ts +66 -0
- package/dist/vendor/lancedb/oauth.js +15 -0
- package/dist/vendor/lancedb/otel.d.ts +26 -0
- package/dist/vendor/lancedb/otel.js +114 -0
- package/dist/vendor/lancedb/permutation.d.ts +143 -0
- package/dist/vendor/lancedb/permutation.js +184 -0
- package/dist/vendor/lancedb/provenance.json +6 -0
- package/dist/vendor/lancedb/query.d.ts +663 -0
- package/dist/vendor/lancedb/query.js +966 -0
- package/dist/vendor/lancedb/rerankers/index.d.ts +5 -0
- package/dist/vendor/lancedb/rerankers/index.js +19 -0
- package/dist/vendor/lancedb/rerankers/rrf.d.ts +14 -0
- package/dist/vendor/lancedb/rerankers/rrf.js +28 -0
- package/dist/vendor/lancedb/sanitize.d.ts +32 -0
- package/dist/vendor/lancedb/sanitize.js +544 -0
- package/dist/vendor/lancedb/scannable.d.ts +92 -0
- package/dist/vendor/lancedb/scannable.js +200 -0
- package/dist/vendor/lancedb/schema.d.ts +16 -0
- package/dist/vendor/lancedb/schema.js +387 -0
- package/dist/vendor/lancedb/table.d.ts +1069 -0
- package/dist/vendor/lancedb/table.js +498 -0
- package/dist/vendor/lancedb/util.d.ts +14 -0
- package/dist/vendor/lancedb/util.js +77 -0
- package/package.json +19 -7
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
|
@@ -41,6 +41,13 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
41
41
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
42
|
});
|
|
43
43
|
};
|
|
44
|
+
var __asyncValues = (this && this.__asyncValues) || function (o) {
|
|
45
|
+
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
|
|
46
|
+
var m = o[Symbol.asyncIterator], i;
|
|
47
|
+
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
|
|
48
|
+
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
|
|
49
|
+
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
|
|
50
|
+
};
|
|
44
51
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
52
|
exports.VectorDB = exports.DiskPressureError = void 0;
|
|
46
53
|
exports.findStaleLanceTempFiles = findStaleLanceTempFiles;
|
|
@@ -48,7 +55,6 @@ exports.sweepStaleLanceTempFiles = sweepStaleLanceTempFiles;
|
|
|
48
55
|
exports.isLanceCorruptionError = isLanceCorruptionError;
|
|
49
56
|
exports.isMissingTableError = isMissingTableError;
|
|
50
57
|
const fs = __importStar(require("node:fs"));
|
|
51
|
-
const lancedb = __importStar(require("@lancedb/lancedb"));
|
|
52
58
|
const apache_arrow_1 = require("apache-arrow");
|
|
53
59
|
const config_1 = require("../../config");
|
|
54
60
|
const embedding_reuse_1 = require("../index/embedding-reuse");
|
|
@@ -56,7 +62,10 @@ const index_config_1 = require("../index/index-config");
|
|
|
56
62
|
const cleanup_1 = require("../utils/cleanup");
|
|
57
63
|
const filter_builder_1 = require("../utils/filter-builder");
|
|
58
64
|
const logger_1 = require("../utils/logger");
|
|
65
|
+
const query_timeout_1 = require("../utils/query-timeout");
|
|
59
66
|
const ann_config_1 = require("./ann-config");
|
|
67
|
+
const compaction_result_1 = require("./compaction-result");
|
|
68
|
+
const lancedb = __importStar(require("./lance-sdk"));
|
|
60
69
|
const store_lease_1 = require("./store-lease");
|
|
61
70
|
/**
|
|
62
71
|
* Find abandoned Lance scratch files under `<lancedbDir>/ *.lance/data`.
|
|
@@ -230,6 +239,7 @@ class VectorDB {
|
|
|
230
239
|
/** Only an index owner may create/rebuild shared indexes — see markIndexOwner. */
|
|
231
240
|
this.indexOwner = false;
|
|
232
241
|
this.lastOptimizeDidWork = false;
|
|
242
|
+
this.lastCompactionResult = null;
|
|
233
243
|
/** Compaction rate limiter — see COMPACTION_MIN_INTERVAL_MS. */
|
|
234
244
|
this.lastCompactionMs = 0;
|
|
235
245
|
this.compactionIntervalMs = COMPACTION_MIN_INTERVAL_MS;
|
|
@@ -367,7 +377,11 @@ class VectorDB {
|
|
|
367
377
|
if (!this.db) {
|
|
368
378
|
fs.mkdirSync(this.lancedbDir, { recursive: true });
|
|
369
379
|
this.session = createLanceSession();
|
|
370
|
-
|
|
380
|
+
// 0.38 accepts the legacy third argument in its types but drops it in
|
|
381
|
+
// the JS wrapper. Put the session in native ConnectionOptions instead.
|
|
382
|
+
this.db = yield lancedb.connect(this.lancedbDir, {
|
|
383
|
+
session: this.session,
|
|
384
|
+
});
|
|
371
385
|
}
|
|
372
386
|
return this.db;
|
|
373
387
|
});
|
|
@@ -1085,8 +1099,18 @@ class VectorDB {
|
|
|
1085
1099
|
}
|
|
1086
1100
|
});
|
|
1087
1101
|
}
|
|
1102
|
+
compactionStatus() {
|
|
1103
|
+
return this.lastCompactionResult ? Object.assign({}, this.lastCompactionResult) : null;
|
|
1104
|
+
}
|
|
1105
|
+
recordSkippedCompaction(reason) {
|
|
1106
|
+
const result = (0, compaction_result_1.skippedCompaction)(reason);
|
|
1107
|
+
this.lastCompactionResult = result;
|
|
1108
|
+
(0, logger_1.log)("vectordb", `Compaction result: ${JSON.stringify(result)}`);
|
|
1109
|
+
return Object.assign({}, result);
|
|
1110
|
+
}
|
|
1088
1111
|
optimize() {
|
|
1089
1112
|
return __awaiter(this, arguments, void 0, function* (retries = COMPACTION_MAX_ATTEMPTS, retentionMs = 0, bypassExclusiveMutation = false) {
|
|
1113
|
+
var _a;
|
|
1090
1114
|
if (!bypassExclusiveMutation) {
|
|
1091
1115
|
while (this.exclusiveMutationPromise)
|
|
1092
1116
|
yield this.exclusiveMutationPromise;
|
|
@@ -1096,7 +1120,7 @@ class VectorDB {
|
|
|
1096
1120
|
if (this.compactingPromise) {
|
|
1097
1121
|
(0, logger_1.debug)("vectordb", "Optimize already in progress, skipping");
|
|
1098
1122
|
yield this.compactingPromise;
|
|
1099
|
-
return;
|
|
1123
|
+
return ((_a = this.compactionStatus()) !== null && _a !== void 0 ? _a : (0, compaction_result_1.skippedCompaction)("compaction unavailable"));
|
|
1100
1124
|
}
|
|
1101
1125
|
this.lastOptimizeDidWork = false;
|
|
1102
1126
|
let resolveCompacting;
|
|
@@ -1104,6 +1128,29 @@ class VectorDB {
|
|
|
1104
1128
|
resolveCompacting = resolve;
|
|
1105
1129
|
});
|
|
1106
1130
|
this.activeCompactions++;
|
|
1131
|
+
const startedAt = Date.now();
|
|
1132
|
+
let attempts = 0;
|
|
1133
|
+
let logicalBytes;
|
|
1134
|
+
let diskBytesBefore;
|
|
1135
|
+
let freeBytesBefore;
|
|
1136
|
+
const finish = (status, reason, bytesReclaimed) => {
|
|
1137
|
+
const result = {
|
|
1138
|
+
status,
|
|
1139
|
+
at: Date.now(),
|
|
1140
|
+
attempts,
|
|
1141
|
+
elapsedMs: Date.now() - startedAt,
|
|
1142
|
+
reason: reason === null || reason === void 0 ? void 0 : reason.replace(/[\r\n\t]/g, " ").slice(0, 512),
|
|
1143
|
+
logicalBytes,
|
|
1144
|
+
diskBytesBefore,
|
|
1145
|
+
freeBytesBefore,
|
|
1146
|
+
diskBytesAfter: this.getDirectorySize(this.lancedbDir),
|
|
1147
|
+
freeBytesAfter: this.getAvailableBytes(),
|
|
1148
|
+
bytesReclaimed,
|
|
1149
|
+
};
|
|
1150
|
+
this.lastCompactionResult = result;
|
|
1151
|
+
(0, logger_1.log)("vectordb", `Compaction result: ${JSON.stringify(result)}`);
|
|
1152
|
+
return Object.assign({}, result);
|
|
1153
|
+
};
|
|
1107
1154
|
try {
|
|
1108
1155
|
const maxAttempts = Math.min(retries, COMPACTION_MAX_ATTEMPTS);
|
|
1109
1156
|
let rebuiltFts = false;
|
|
@@ -1123,10 +1170,17 @@ class VectorDB {
|
|
|
1123
1170
|
// for rewrite/index overhead, leaving the critical reserve untouched.
|
|
1124
1171
|
const availableBytes = this.getAvailableBytes();
|
|
1125
1172
|
const requiredBytes = totalBytes * 2 + config_1.DISK_CRITICAL_BYTES;
|
|
1173
|
+
logicalBytes = totalBytes;
|
|
1174
|
+
const diskBytes = this.getDirectorySize(this.lancedbDir);
|
|
1175
|
+
diskBytesBefore !== null && diskBytesBefore !== void 0 ? diskBytesBefore : (diskBytesBefore = diskBytes);
|
|
1176
|
+
freeBytesBefore !== null && freeBytesBefore !== void 0 ? freeBytesBefore : (freeBytesBefore = availableBytes);
|
|
1126
1177
|
if (availableBytes < requiredBytes) {
|
|
1127
|
-
|
|
1128
|
-
|
|
1178
|
+
const reason = `insufficient rewrite headroom (${(availableBytes / Math.pow(1024, 3)).toFixed(1)}GB free, ${(requiredBytes / Math.pow(1024, 3)).toFixed(1)}GB required)`;
|
|
1179
|
+
(0, logger_1.log)("vectordb", `Optimize skipped: ${reason}`);
|
|
1180
|
+
return finish("skipped", reason);
|
|
1129
1181
|
}
|
|
1182
|
+
attempts++;
|
|
1183
|
+
(0, logger_1.log)("vectordb", `Compaction attempt: ${JSON.stringify({ attempt, maxAttempts, logicalBytes, diskBytes, freeBytes: availableBytes, requiredBytes })}`);
|
|
1130
1184
|
const done = (0, logger_1.timer)("vectordb", "optimize");
|
|
1131
1185
|
// deleteUnverified deletes files Lance cannot prove are unreferenced.
|
|
1132
1186
|
// LanceDB's own docs: only safe "if you can guarantee that no other
|
|
@@ -1156,19 +1210,20 @@ class VectorDB {
|
|
|
1156
1210
|
else {
|
|
1157
1211
|
(0, logger_1.debug)("vectordb", "Optimize: nothing to compact or prune");
|
|
1158
1212
|
}
|
|
1159
|
-
return;
|
|
1213
|
+
return finish("completed", undefined, prune.bytesRemoved);
|
|
1160
1214
|
}
|
|
1161
1215
|
catch (e) {
|
|
1162
1216
|
const msg = e instanceof Error ? e.message : String(e);
|
|
1163
1217
|
if (msg.includes("Nothing to do")) {
|
|
1164
1218
|
(0, logger_1.debug)("vectordb", "Optimize: nothing to do");
|
|
1165
|
-
return;
|
|
1219
|
+
return finish("completed", "nothing to compact or prune", 0);
|
|
1166
1220
|
}
|
|
1221
|
+
(0, logger_1.log)("vectordb", `Compaction attempt failed: ${JSON.stringify({ attempt, elapsedMs: Date.now() - startedAt, diskBytes: this.getDirectorySize(this.lancedbDir), freeBytes: this.getAvailableBytes(), reason: msg.slice(0, 512) })}`);
|
|
1167
1222
|
// ENOSPC: return immediately — retrying will only make things worse
|
|
1168
1223
|
if (msg.includes("No space left on device") ||
|
|
1169
1224
|
msg.includes("os error 28")) {
|
|
1170
1225
|
(0, logger_1.log)("vectordb", `Optimize failed (ENOSPC): disk full — skipping retries`);
|
|
1171
|
-
return;
|
|
1226
|
+
return finish("failed", "disk full — skipping retries");
|
|
1172
1227
|
}
|
|
1173
1228
|
if (attempt < maxAttempts &&
|
|
1174
1229
|
(msg.includes("conflict") || msg.includes("Retryable"))) {
|
|
@@ -1198,7 +1253,7 @@ class VectorDB {
|
|
|
1198
1253
|
catch (rebuildErr) {
|
|
1199
1254
|
this.ftsPanicRecoveryExhausted = true;
|
|
1200
1255
|
(0, logger_1.log)("vectordb", `FTS rebuild failed: ${rebuildErr}`);
|
|
1201
|
-
return;
|
|
1256
|
+
return finish("failed", `FTS rebuild failed: ${rebuildErr}`);
|
|
1202
1257
|
}
|
|
1203
1258
|
}
|
|
1204
1259
|
if (rebuiltFts) {
|
|
@@ -1207,9 +1262,10 @@ class VectorDB {
|
|
|
1207
1262
|
}
|
|
1208
1263
|
}
|
|
1209
1264
|
(0, logger_1.log)("vectordb", `Optimize failed: ${msg}`);
|
|
1210
|
-
return;
|
|
1265
|
+
return finish("failed", msg);
|
|
1211
1266
|
}
|
|
1212
1267
|
}
|
|
1268
|
+
return finish("skipped", "no compaction attempts requested");
|
|
1213
1269
|
}
|
|
1214
1270
|
finally {
|
|
1215
1271
|
this.compactingPromise = null;
|
|
@@ -1232,16 +1288,17 @@ class VectorDB {
|
|
|
1232
1288
|
return __awaiter(this, arguments, void 0, function* (options = {}) {
|
|
1233
1289
|
if (this.maintenanceRunning) {
|
|
1234
1290
|
(0, logger_1.debug)("vectordb", "Maintenance already running, skipping");
|
|
1235
|
-
return;
|
|
1291
|
+
return (0, compaction_result_1.skippedCompaction)("maintenance already running");
|
|
1236
1292
|
}
|
|
1237
1293
|
this.maintenanceRunning = true;
|
|
1238
1294
|
const epochSnapshot = this.writeEpoch;
|
|
1295
|
+
let result;
|
|
1239
1296
|
try {
|
|
1240
1297
|
const pressure = this.checkDiskPressure();
|
|
1241
1298
|
if (pressure === "critical") {
|
|
1242
1299
|
const freeGb = (this.getAvailableBytes() / 1024 / 1024 / 1024).toFixed(1);
|
|
1243
1300
|
(0, logger_1.log)("vectordb", `Maintenance skipped: disk critically low (${freeGb}GB free)`);
|
|
1244
|
-
return;
|
|
1301
|
+
return this.recordSkippedCompaction(`disk critically low (${freeGb}GB free)`);
|
|
1245
1302
|
}
|
|
1246
1303
|
if (!options.force &&
|
|
1247
1304
|
epochSnapshot === this.maintainedEpoch &&
|
|
@@ -1270,12 +1327,12 @@ class VectorDB {
|
|
|
1270
1327
|
}
|
|
1271
1328
|
else if (pressure === "low") {
|
|
1272
1329
|
(0, logger_1.log)("vectordb", `Low disk — single-pass optimize (no bloat retry)`);
|
|
1273
|
-
yield this.optimize(1, 0, true);
|
|
1330
|
+
result = yield this.optimize(1, 0, true);
|
|
1274
1331
|
this.noteCompaction(this.lastOptimizeDidWork);
|
|
1275
1332
|
}
|
|
1276
1333
|
else {
|
|
1277
1334
|
// Normal maintenance: full optimize + bloat check
|
|
1278
|
-
yield this.optimize(5, 0, true);
|
|
1335
|
+
result = yield this.optimize(5, 0, true);
|
|
1279
1336
|
// Track across both passes: `lastOptimizeDidWork` is reset per optimize()
|
|
1280
1337
|
// call, so a productive first pass followed by a barren bloat retry would
|
|
1281
1338
|
// otherwise read as unproductive and trigger a spurious backoff.
|
|
@@ -1300,7 +1357,7 @@ class VectorDB {
|
|
|
1300
1357
|
if (bloatRatio > 2.0 && this.checkDiskPressure() === "ok") {
|
|
1301
1358
|
(0, logger_1.log)("vectordb", `Bloat detected after optimize: ${(diskSize / 1024 / 1024).toFixed(0)}MB disk vs ${(logicalSize / 1024 / 1024).toFixed(0)}MB logical (${bloatRatio.toFixed(1)}x) — retrying`);
|
|
1302
1359
|
yield new Promise((r) => setTimeout(r, 2000));
|
|
1303
|
-
yield this.optimize(5, 0, true);
|
|
1360
|
+
result = yield this.optimize(5, 0, true);
|
|
1304
1361
|
didWork = didWork || this.lastOptimizeDidWork;
|
|
1305
1362
|
}
|
|
1306
1363
|
}
|
|
@@ -1312,6 +1369,7 @@ class VectorDB {
|
|
|
1312
1369
|
yield this.readTableVersion(maintainedTable);
|
|
1313
1370
|
this.maintainedEpoch = epochSnapshot;
|
|
1314
1371
|
this.lastMaintenanceMs = Date.now();
|
|
1372
|
+
return result;
|
|
1315
1373
|
}
|
|
1316
1374
|
finally {
|
|
1317
1375
|
this.maintenanceRunning = false;
|
|
@@ -1376,9 +1434,9 @@ class VectorDB {
|
|
|
1376
1434
|
return false;
|
|
1377
1435
|
}
|
|
1378
1436
|
(0, logger_1.log)("vectordb", `Fragment threshold exceeded (${stats.fragmentStats.numSmallFragments} > ${threshold}) — compacting`);
|
|
1379
|
-
yield this.optimize(2, 0, true);
|
|
1437
|
+
const result = yield this.optimize(2, 0, true);
|
|
1380
1438
|
this.noteCompaction(this.lastOptimizeDidWork);
|
|
1381
|
-
return
|
|
1439
|
+
return result.status === "completed";
|
|
1382
1440
|
}
|
|
1383
1441
|
}
|
|
1384
1442
|
catch (err) {
|
|
@@ -1411,7 +1469,11 @@ class VectorDB {
|
|
|
1411
1469
|
hasAnyRows() {
|
|
1412
1470
|
return __awaiter(this, void 0, void 0, function* () {
|
|
1413
1471
|
const table = yield this.ensureTable();
|
|
1414
|
-
const rows = yield table
|
|
1472
|
+
const rows = yield table
|
|
1473
|
+
.query()
|
|
1474
|
+
.select(["id"])
|
|
1475
|
+
.limit(1)
|
|
1476
|
+
.toArray(query_timeout_1.QUERY_EXECUTION_OPTIONS);
|
|
1415
1477
|
return rows.length > 0;
|
|
1416
1478
|
});
|
|
1417
1479
|
}
|
|
@@ -1424,7 +1486,7 @@ class VectorDB {
|
|
|
1424
1486
|
.select(["id"])
|
|
1425
1487
|
.where((0, filter_builder_1.pathStartsWith)(prefix))
|
|
1426
1488
|
.limit(1)
|
|
1427
|
-
.toArray();
|
|
1489
|
+
.toArray(query_timeout_1.QUERY_EXECUTION_OPTIONS);
|
|
1428
1490
|
return rows.length > 0;
|
|
1429
1491
|
});
|
|
1430
1492
|
}
|
|
@@ -1442,16 +1504,25 @@ class VectorDB {
|
|
|
1442
1504
|
}
|
|
1443
1505
|
getDistinctPathsForPrefix(pathPrefix) {
|
|
1444
1506
|
return __awaiter(this, void 0, void 0, function* () {
|
|
1507
|
+
var _a, e_1, _b, _c;
|
|
1445
1508
|
const table = yield this.ensureTable();
|
|
1446
1509
|
const prefix = pathPrefix.endsWith("/") ? pathPrefix : `${pathPrefix}/`;
|
|
1447
|
-
const rows =
|
|
1448
|
-
.query()
|
|
1449
|
-
.select(["path"])
|
|
1450
|
-
.where((0, filter_builder_1.pathStartsWith)(prefix))
|
|
1451
|
-
.toArray();
|
|
1510
|
+
const rows = (0, query_timeout_1.streamQueryRows)(table.query().select(["path"]).where((0, filter_builder_1.pathStartsWith)(prefix)), "distinct project paths");
|
|
1452
1511
|
const unique = new Set();
|
|
1453
|
-
|
|
1454
|
-
|
|
1512
|
+
try {
|
|
1513
|
+
for (var _d = true, rows_1 = __asyncValues(rows), rows_1_1; rows_1_1 = yield rows_1.next(), _a = rows_1_1.done, !_a; _d = true) {
|
|
1514
|
+
_c = rows_1_1.value;
|
|
1515
|
+
_d = false;
|
|
1516
|
+
const r = _c;
|
|
1517
|
+
unique.add(String(r.path));
|
|
1518
|
+
}
|
|
1519
|
+
}
|
|
1520
|
+
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
1521
|
+
finally {
|
|
1522
|
+
try {
|
|
1523
|
+
if (!_d && !_a && (_b = rows_1.return)) yield _b.call(rows_1);
|
|
1524
|
+
}
|
|
1525
|
+
finally { if (e_1) throw e_1.error; }
|
|
1455
1526
|
}
|
|
1456
1527
|
return unique;
|
|
1457
1528
|
});
|
|
@@ -1476,7 +1547,7 @@ class VectorDB {
|
|
|
1476
1547
|
"doc_token_ids",
|
|
1477
1548
|
])
|
|
1478
1549
|
.where(`path = '${(0, filter_builder_1.escapeSqlString)(filePath)}'`)
|
|
1479
|
-
.toArray();
|
|
1550
|
+
.toArray(query_timeout_1.QUERY_EXECUTION_OPTIONS);
|
|
1480
1551
|
const reusable = new Map();
|
|
1481
1552
|
for (const row of rows) {
|
|
1482
1553
|
const vector = (0, embedding_reuse_1.toFloat32)(row.vector);
|
|
@@ -1510,9 +1581,25 @@ class VectorDB {
|
|
|
1510
1581
|
}
|
|
1511
1582
|
getDistinctFileCount() {
|
|
1512
1583
|
return __awaiter(this, void 0, void 0, function* () {
|
|
1584
|
+
var _a, e_2, _b, _c;
|
|
1513
1585
|
const table = yield this.ensureTable();
|
|
1514
|
-
const
|
|
1515
|
-
|
|
1586
|
+
const paths = new Set();
|
|
1587
|
+
try {
|
|
1588
|
+
for (var _d = true, _e = __asyncValues((0, query_timeout_1.streamQueryRows)(table.query().select(["path"]), "distinct file count")), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {
|
|
1589
|
+
_c = _f.value;
|
|
1590
|
+
_d = false;
|
|
1591
|
+
const row = _c;
|
|
1592
|
+
paths.add(String(row.path));
|
|
1593
|
+
}
|
|
1594
|
+
}
|
|
1595
|
+
catch (e_2_1) { e_2 = { error: e_2_1 }; }
|
|
1596
|
+
finally {
|
|
1597
|
+
try {
|
|
1598
|
+
if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
|
|
1599
|
+
}
|
|
1600
|
+
finally { if (e_2) throw e_2.error; }
|
|
1601
|
+
}
|
|
1602
|
+
return paths.size;
|
|
1516
1603
|
});
|
|
1517
1604
|
}
|
|
1518
1605
|
deletePaths(paths) {
|
|
@@ -1535,7 +1622,7 @@ class VectorDB {
|
|
|
1535
1622
|
.select(["id"])
|
|
1536
1623
|
.where(where)
|
|
1537
1624
|
.limit(1)
|
|
1538
|
-
.toArray();
|
|
1625
|
+
.toArray(query_timeout_1.QUERY_EXECUTION_OPTIONS);
|
|
1539
1626
|
if (existing.length > 0) {
|
|
1540
1627
|
yield table.delete(where);
|
|
1541
1628
|
this.markWriteCommitted();
|
|
@@ -1584,7 +1671,7 @@ class VectorDB {
|
|
|
1584
1671
|
.select(["id"])
|
|
1585
1672
|
.where(where)
|
|
1586
1673
|
.limit(1)
|
|
1587
|
-
.toArray();
|
|
1674
|
+
.toArray(query_timeout_1.QUERY_EXECUTION_OPTIONS);
|
|
1588
1675
|
if (existing.length > 0) {
|
|
1589
1676
|
yield table.delete(where);
|
|
1590
1677
|
this.markWriteCommitted();
|
|
@@ -15,13 +15,13 @@ exports.resolveCrossProjectScope = resolveCrossProjectScope;
|
|
|
15
15
|
exports.projectForPath = projectForPath;
|
|
16
16
|
exports.groupResultsByProject = groupResultsByProject;
|
|
17
17
|
const project_registry_1 = require("./project-registry");
|
|
18
|
-
function resolveCrossProjectScope(opts) {
|
|
18
|
+
function resolveCrossProjectScope(opts, registeredProjects = (0, project_registry_1.listProjects)()) {
|
|
19
19
|
const active = !!(opts.allProjects || opts.projects);
|
|
20
20
|
if (!active) {
|
|
21
21
|
return { active: false, roots: [], projectRoots: [], warnings: [] };
|
|
22
22
|
}
|
|
23
23
|
// Ignore "error"-status projects: the daemon won't search them anyway.
|
|
24
|
-
const all =
|
|
24
|
+
const all = registeredProjects.filter((p) => p.status !== "error");
|
|
25
25
|
const byName = new Map(all.map((p) => [p.name, p]));
|
|
26
26
|
const warnings = [];
|
|
27
27
|
const resolveNames = (csv) => {
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.runDoctorOptimize = runDoctorOptimize;
|
|
13
|
+
exports.formatDoctorOptimize = formatDoctorOptimize;
|
|
14
|
+
const compaction_result_1 = require("../store/compaction-result");
|
|
15
|
+
/** A live daemon's failed repair must never become a competing local rewrite. */
|
|
16
|
+
function runDoctorOptimize(send, optimizeInProcess) {
|
|
17
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
18
|
+
var _a, _b;
|
|
19
|
+
const response = yield send();
|
|
20
|
+
const result = (0, compaction_result_1.parseCompactionResult)(response.compaction);
|
|
21
|
+
if (result) {
|
|
22
|
+
return {
|
|
23
|
+
status: !response.ok && result.status === "completed"
|
|
24
|
+
? "failed"
|
|
25
|
+
: result.status,
|
|
26
|
+
via: "daemon",
|
|
27
|
+
reason: (_a = result.reason) !== null && _a !== void 0 ? _a : (response.ok ? undefined : String(response.error)),
|
|
28
|
+
result,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
if (response.ok) {
|
|
32
|
+
return {
|
|
33
|
+
status: "unverified",
|
|
34
|
+
via: "daemon",
|
|
35
|
+
reason: "daemon did not report a compaction outcome; restart it with the current gmax version",
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
const reason = String((_b = response.error) !== null && _b !== void 0 ? _b : "unknown failure");
|
|
39
|
+
// Socket errors are returned as exact errno codes by sendDaemonCommand.
|
|
40
|
+
// An ENOENT inside a daemon's failure message is not proof it is absent.
|
|
41
|
+
if (reason === "ECONNREFUSED" || reason === "ENOENT") {
|
|
42
|
+
try {
|
|
43
|
+
const local = yield optimizeInProcess();
|
|
44
|
+
return {
|
|
45
|
+
status: local.status,
|
|
46
|
+
via: "in-process",
|
|
47
|
+
reason: local.reason,
|
|
48
|
+
result: local,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
return { status: "failed", via: "in-process", reason: String(error) };
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return {
|
|
56
|
+
status: reason.includes("unknown command") ? "skipped" : "failed",
|
|
57
|
+
via: "daemon",
|
|
58
|
+
reason,
|
|
59
|
+
};
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
function formatDoctorOptimize(report, agent) {
|
|
63
|
+
var _a;
|
|
64
|
+
const reason = (_a = report.reason) === null || _a === void 0 ? void 0 : _a.replace(/[\r\n\t]/g, " ").slice(0, 512);
|
|
65
|
+
if (agent) {
|
|
66
|
+
const r = report.result;
|
|
67
|
+
return [
|
|
68
|
+
`optimize`,
|
|
69
|
+
`status=${report.status}`,
|
|
70
|
+
`via=${report.via}`,
|
|
71
|
+
...(r ? [`attempts=${r.attempts}`, `elapsed_ms=${r.elapsedMs}`] : []),
|
|
72
|
+
...(typeof (r === null || r === void 0 ? void 0 : r.bytesReclaimed) === "number"
|
|
73
|
+
? [`bytes_reclaimed=${r.bytesReclaimed}`]
|
|
74
|
+
: []),
|
|
75
|
+
...(reason ? [`reason=${reason}`] : []),
|
|
76
|
+
].join("\t");
|
|
77
|
+
}
|
|
78
|
+
if (report.status === "completed")
|
|
79
|
+
return `ok Optimize complete (${report.via})`;
|
|
80
|
+
return `${report.status === "failed" ? "FAIL" : "WARN"} Optimize ${report.status} (${report.via}): ${reason !== null && reason !== void 0 ? reason : "no outcome reported"}`;
|
|
81
|
+
}
|
|
@@ -74,16 +74,16 @@ class ProjectRegistryConflictError extends Error {
|
|
|
74
74
|
}
|
|
75
75
|
}
|
|
76
76
|
exports.ProjectRegistryConflictError = ProjectRegistryConflictError;
|
|
77
|
-
function loadRegistry() {
|
|
77
|
+
function loadRegistry(registryPath = REGISTRY_PATH) {
|
|
78
78
|
let raw;
|
|
79
79
|
try {
|
|
80
|
-
raw = fs.readFileSync(
|
|
80
|
+
raw = fs.readFileSync(registryPath, "utf-8");
|
|
81
81
|
}
|
|
82
82
|
catch (err) {
|
|
83
83
|
if (err.code === "ENOENT")
|
|
84
84
|
return [];
|
|
85
85
|
const message = err instanceof Error ? err.message : String(err);
|
|
86
|
-
throw new Error(`Failed to read project registry ${
|
|
86
|
+
throw new Error(`Failed to read project registry ${registryPath}: ${message}`);
|
|
87
87
|
}
|
|
88
88
|
let parsed;
|
|
89
89
|
try {
|
|
@@ -91,14 +91,14 @@ function loadRegistry() {
|
|
|
91
91
|
}
|
|
92
92
|
catch (err) {
|
|
93
93
|
const message = err instanceof Error ? err.message : String(err);
|
|
94
|
-
throw new Error(`Invalid project registry JSON ${
|
|
94
|
+
throw new Error(`Invalid project registry JSON ${registryPath}: ${message}`);
|
|
95
95
|
}
|
|
96
96
|
if (!Array.isArray(parsed)) {
|
|
97
|
-
throw new Error(`Invalid project registry ${
|
|
97
|
+
throw new Error(`Invalid project registry ${registryPath}: expected an array`);
|
|
98
98
|
}
|
|
99
99
|
for (const [index, entry] of parsed.entries()) {
|
|
100
100
|
if (!isProjectEntry(entry)) {
|
|
101
|
-
throw new Error(`Invalid project registry entry at index ${index} in ${
|
|
101
|
+
throw new Error(`Invalid project registry entry at index ${index} in ${registryPath}`);
|
|
102
102
|
}
|
|
103
103
|
}
|
|
104
104
|
return parsed;
|
|
@@ -455,8 +455,8 @@ function stampProjectFullSync(stamp) {
|
|
|
455
455
|
return entry;
|
|
456
456
|
});
|
|
457
457
|
}
|
|
458
|
-
function listProjects() {
|
|
459
|
-
return loadRegistry();
|
|
458
|
+
function listProjects(home) {
|
|
459
|
+
return loadRegistry(home ? path.join(home, "projects.json") : undefined);
|
|
460
460
|
}
|
|
461
461
|
function getProject(root) {
|
|
462
462
|
return loadRegistry().find((e) => e.root === root);
|
|
@@ -8,10 +8,28 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
8
8
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
9
|
});
|
|
10
10
|
};
|
|
11
|
+
var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); }
|
|
12
|
+
var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
|
|
13
|
+
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
|
|
14
|
+
var g = generator.apply(thisArg, _arguments || []), i, q = [];
|
|
15
|
+
return i = Object.create((typeof AsyncIterator === "function" ? AsyncIterator : Object).prototype), verb("next"), verb("throw"), verb("return", awaitReturn), i[Symbol.asyncIterator] = function () { return this; }, i;
|
|
16
|
+
function awaitReturn(f) { return function (v) { return Promise.resolve(v).then(f, reject); }; }
|
|
17
|
+
function verb(n, f) { if (g[n]) { i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; if (f) i[n] = f(i[n]); } }
|
|
18
|
+
function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
|
|
19
|
+
function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
|
|
20
|
+
function fulfill(value) { resume("next", value); }
|
|
21
|
+
function reject(value) { resume("throw", value); }
|
|
22
|
+
function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
|
|
23
|
+
};
|
|
11
24
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.QueryTimeoutError = void 0;
|
|
25
|
+
exports.QueryTimeoutError = exports.QUERY_EXECUTION_OPTIONS = exports.QUERY_TIMEOUT_MS = void 0;
|
|
13
26
|
exports.withQueryTimeout = withQueryTimeout;
|
|
14
|
-
|
|
27
|
+
exports.streamQueryRows = streamQueryRows;
|
|
28
|
+
const configuredTimeout = Number(process.env.GMAX_QUERY_TIMEOUT_MS);
|
|
29
|
+
exports.QUERY_TIMEOUT_MS = Number.isFinite(configuredTimeout) && configuredTimeout > 0
|
|
30
|
+
? Math.max(1, Math.min(Math.floor(configuredTimeout), 2147483647))
|
|
31
|
+
: 15000;
|
|
32
|
+
exports.QUERY_EXECUTION_OPTIONS = { timeoutMs: exports.QUERY_TIMEOUT_MS };
|
|
15
33
|
class QueryTimeoutError extends Error {
|
|
16
34
|
constructor(label, ms) {
|
|
17
35
|
super(`LanceDB query timed out after ${ms}ms (${label}). ` +
|
|
@@ -24,19 +42,13 @@ exports.QueryTimeoutError = QueryTimeoutError;
|
|
|
24
42
|
* Race a LanceDB query against a wall-clock timeout so a native-layer deadlock
|
|
25
43
|
* surfaces as a loud error instead of hanging the process forever.
|
|
26
44
|
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
* a limit and cap in JS); this wrapper is the backstop for shapes we missed.
|
|
32
|
-
*
|
|
33
|
-
* The timed-out native promise is NOT cancelled — its tokio task may stay
|
|
34
|
-
* parked. CLI commands exit via gracefulExit() (process.exit), so the leak is
|
|
35
|
-
* bounded to the command's lifetime. Long-lived callers (daemon) should treat
|
|
36
|
-
* a QueryTimeoutError as a signal that the connection may be wedged.
|
|
45
|
+
* This JavaScript backstop does not cancel its input. Native query callers
|
|
46
|
+
* must also pass QUERY_EXECUTION_OPTIONS to toArray/execute so Lance enforces
|
|
47
|
+
* an execution deadline. The historical 0.27 LIKE+limit hang has a native
|
|
48
|
+
* temporary-store regression on the pinned SDK.
|
|
37
49
|
*/
|
|
38
50
|
function withQueryTimeout(promise_1, label_1) {
|
|
39
|
-
return __awaiter(this, arguments, void 0, function* (promise, label, ms =
|
|
51
|
+
return __awaiter(this, arguments, void 0, function* (promise, label, ms = exports.QUERY_TIMEOUT_MS) {
|
|
40
52
|
let timer;
|
|
41
53
|
const timeout = new Promise((_, reject) => {
|
|
42
54
|
timer = setTimeout(() => reject(new QueryTimeoutError(label, ms)), ms);
|
|
@@ -50,3 +62,31 @@ function withQueryTimeout(promise_1, label_1) {
|
|
|
50
62
|
}
|
|
51
63
|
});
|
|
52
64
|
}
|
|
65
|
+
/** Stream bounded Arrow batches with a native and overall wall-clock deadline.
|
|
66
|
+
* SDK 0.38 exposes execute(options) at runtime but marks it protected in its
|
|
67
|
+
* declarations. Keep this version-specific adapter here and cover it natively.
|
|
68
|
+
*/
|
|
69
|
+
function streamQueryRows(query_1, label_1) {
|
|
70
|
+
return __asyncGenerator(this, arguments, function* streamQueryRows_1(query, label, ms = exports.QUERY_TIMEOUT_MS) {
|
|
71
|
+
var _a;
|
|
72
|
+
const streaming = query;
|
|
73
|
+
const iterator = streaming.execute({ timeoutMs: ms, maxBatchLength: 512 });
|
|
74
|
+
const deadline = Date.now() + ms;
|
|
75
|
+
try {
|
|
76
|
+
for (;;) {
|
|
77
|
+
const remaining = deadline - Date.now();
|
|
78
|
+
if (remaining <= 0)
|
|
79
|
+
throw new QueryTimeoutError(label, ms);
|
|
80
|
+
const batch = yield __await(withQueryTimeout(iterator.next(), label, remaining));
|
|
81
|
+
if (batch.done)
|
|
82
|
+
return yield __await(void 0);
|
|
83
|
+
for (const row of batch.value.toArray())
|
|
84
|
+
yield yield __await(row);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
finally {
|
|
88
|
+
// Do not let a broken native iterator's pending next() hang cleanup.
|
|
89
|
+
void ((_a = iterator.return) === null || _a === void 0 ? void 0 : _a.call(iterator).catch(() => { }));
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
}
|
|
@@ -189,15 +189,14 @@ function classifyDaemonError(error) {
|
|
|
189
189
|
*
|
|
190
190
|
* Kept as a named predicate because it is the exact rule `search-run.ts` has
|
|
191
191
|
* always applied (and re-exports); `unknown command` is deliberately NOT in the
|
|
192
|
-
* set
|
|
192
|
+
* set; an unsupported verb is a live daemon refusing the request.
|
|
193
193
|
*/
|
|
194
194
|
function shouldFallbackFromDaemonError(error) {
|
|
195
195
|
return classifyDaemonError(error) === "no-daemon";
|
|
196
196
|
}
|
|
197
197
|
/**
|
|
198
|
-
*
|
|
199
|
-
*
|
|
200
|
-
* command, then remove the allowance. Mirrors the `search-v2` transition rule.
|
|
198
|
+
* An unsupported verb means a live daemon is outdated or incompatible. It must
|
|
199
|
+
* be restarted, never bypassed by opening a second reader.
|
|
201
200
|
*/
|
|
202
201
|
function isUnknownCommandError(error) {
|
|
203
202
|
return typeof error === "string" && error.startsWith("unknown command");
|
|
@@ -304,8 +303,10 @@ function withStoreRead(name, opts) {
|
|
|
304
303
|
const cls = classifyDaemonError(resp.error);
|
|
305
304
|
if (cls === "sandboxed")
|
|
306
305
|
throw refuseStoreAccess("socket");
|
|
306
|
+
if (isUnknownCommandError(resp.error)) {
|
|
307
|
+
throw new Error(`${defaultDaemonErrorMessage(name, resp)}. The daemon is outdated or incompatible; run: gmax watch restart`);
|
|
308
|
+
}
|
|
307
309
|
if (cls === "no-daemon" ||
|
|
308
|
-
(opts.fallbackOnUnknownVerb && isUnknownCommandError(resp.error)) ||
|
|
309
310
|
(opts.fallbackOnOversize && isOversizeError(resp.error)) ||
|
|
310
311
|
((_a = opts.extraFallback) === null || _a === void 0 ? void 0 : _a.call(opts, resp.error)) === true) {
|
|
311
312
|
if (process.env.GMAX_DEBUG === "1") {
|