grepmax 0.26.52 → 0.26.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/NOTICE +1 -1
- package/README.md +1 -1
- package/dist/lib/output/mcp-results.js +3 -0
- package/dist/lib/store/vector-db.js +94 -4
- package/dist/lib/utils/doctor-optimize.js +6 -0
- package/dist/vendor/lancedb/arrow.d.ts +1 -1
- package/dist/vendor/lancedb/arrow.js +81 -3
- package/dist/vendor/lancedb/blob.d.ts +92 -0
- package/dist/vendor/lancedb/blob.js +165 -0
- package/dist/vendor/lancedb/connection.d.ts +10 -22
- package/dist/vendor/lancedb/connection.js +4 -14
- package/dist/vendor/lancedb/index.d.ts +4 -1
- package/dist/vendor/lancedb/index.js +7 -2
- package/dist/vendor/lancedb/indices.d.ts +3 -3
- package/dist/vendor/lancedb/job.d.ts +93 -0
- package/dist/vendor/lancedb/job.js +163 -0
- package/dist/vendor/lancedb/materialized_view.d.ts +2 -0
- package/dist/vendor/lancedb/materialized_view.js +3 -1
- package/dist/vendor/lancedb/merge.d.ts +1 -1
- package/dist/vendor/lancedb/merge.js +1 -1
- package/dist/vendor/lancedb/native.d.ts +50 -34
- package/dist/vendor/lancedb/native.js +53 -52
- package/dist/vendor/lancedb/provenance.json +2 -2
- package/dist/vendor/lancedb/sanitize.js +1 -1
- package/dist/vendor/lancedb/table.d.ts +47 -11
- package/dist/vendor/lancedb/table.js +36 -25
- package/package.json +9 -9
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"name": "grepmax",
|
|
11
11
|
"source": "./plugins/grepmax",
|
|
12
12
|
"description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
|
|
13
|
-
"version": "0.26.
|
|
13
|
+
"version": "0.26.54",
|
|
14
14
|
"author": {
|
|
15
15
|
"name": "Robert Owens",
|
|
16
16
|
"email": "robowens@me.com"
|
package/NOTICE
CHANGED
|
@@ -33,7 +33,7 @@ This software depends on various open-source libraries. See package.json for
|
|
|
33
33
|
a complete list of dependencies and their respective licenses.
|
|
34
34
|
|
|
35
35
|
The distribution includes the unmodified JavaScript runtime from
|
|
36
|
-
@lancedb/lancedb 0.
|
|
36
|
+
@lancedb/lancedb 0.39.0 (https://github.com/lancedb/lancedb), licensed under
|
|
37
37
|
Apache-2.0. Its license and build provenance are included in dist/vendor/lancedb/.
|
|
38
38
|
Its optional provider dependency manifest is not distributed. Platform native
|
|
39
39
|
binaries are installed as separately pinned @lancedb packages.
|
package/README.md
CHANGED
|
@@ -484,7 +484,7 @@ whether you need to act.
|
|
|
484
484
|
|
|
485
485
|
| You see | What it means |
|
|
486
486
|
|---|---|
|
|
487
|
-
| `Optimize panicked ... inverted/builder.rs` | The historical FTS merge defect is fixed in the shipped LanceDB 0.
|
|
487
|
+
| `Optimize panicked ... inverted/builder.rs` | The historical FTS merge defect is fixed in the shipped LanceDB 0.39.0 runtime. If it recurs on a current release, preserve the version and logs and report a regression; the rebuild guard remains a recovery tripwire. |
|
|
488
488
|
| `disabling auto-rebuild until an optimize succeeds` | Transient, not a wedge. The next successful optimize clears it. Search stays available. |
|
|
489
489
|
| `ANN: vector index not built` | Normal — exact search is the default. |
|
|
490
490
|
| `cannot reach the daemon socket from this sandbox` (exit 2) | The shell is sandboxed. Add the two keys in [Running under the Claude Code sandbox](#running-under-the-claude-code-sandbox); `gmax doctor` warns about the same gap. |
|
|
@@ -47,6 +47,9 @@ exports.compactionSchema = zod_1.z.object({
|
|
|
47
47
|
freeBytesBefore: zod_1.z.number().optional(),
|
|
48
48
|
freeBytesAfter: zod_1.z.number().optional(),
|
|
49
49
|
bytesReclaimed: zod_1.z.number().optional(),
|
|
50
|
+
netBytesReclaimed: zod_1.z.number().optional(),
|
|
51
|
+
cleanupPasses: zod_1.z.number().int().min(0).max(1).optional(),
|
|
52
|
+
cleanupReason: zod_1.z.string().optional(),
|
|
50
53
|
});
|
|
51
54
|
const embedding = zod_1.z.object({
|
|
52
55
|
tier: zod_1.z.string(),
|
|
@@ -1108,9 +1108,80 @@ class VectorDB {
|
|
|
1108
1108
|
(0, logger_1.log)("vectordb", `Compaction result: ${JSON.stringify(result)}`);
|
|
1109
1109
|
return Object.assign({}, result);
|
|
1110
1110
|
}
|
|
1111
|
+
/** Prune the reserve-ID snapshot without another data rewrite. Caller holds
|
|
1112
|
+
* compactingPromise and has drained writes; never use this for retention > 0.
|
|
1113
|
+
* Lance 12 treats one deletion-free fragment as a compaction no-op. */
|
|
1114
|
+
cleanupCompactionReservation(table) {
|
|
1115
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1116
|
+
var _a;
|
|
1117
|
+
let passes = 0;
|
|
1118
|
+
const deferred = (reason) => ({
|
|
1119
|
+
passes: 0,
|
|
1120
|
+
bytesRemoved: 0,
|
|
1121
|
+
reason,
|
|
1122
|
+
});
|
|
1123
|
+
try {
|
|
1124
|
+
const stats = yield table.stats();
|
|
1125
|
+
if (((_a = stats.fragmentStats) === null || _a === void 0 ? void 0 : _a.numFragments) !== 1) {
|
|
1126
|
+
return deferred("cleanup deferred: table is not a single fragment");
|
|
1127
|
+
}
|
|
1128
|
+
const version = yield table.version();
|
|
1129
|
+
const fresh = yield this.openExistingTableUnsafe();
|
|
1130
|
+
if (!fresh || (yield fresh.version()) !== version) {
|
|
1131
|
+
return deferred("cleanup deferred: table changed after compaction");
|
|
1132
|
+
}
|
|
1133
|
+
const latest = (yield fresh.listVersions()).find((v) => v.version === version);
|
|
1134
|
+
if ((latest === null || latest === void 0 ? void 0 : latest.metadata.total_fragments) !== "1" ||
|
|
1135
|
+
latest.metadata.total_deletion_files !== "0" ||
|
|
1136
|
+
latest.metadata.total_deletion_file_rows !== "0") {
|
|
1137
|
+
return deferred("cleanup deferred: deletion-free fragment not verified");
|
|
1138
|
+
}
|
|
1139
|
+
// Manifest timestamps have sub-ms precision; wait for a real later tick,
|
|
1140
|
+
// never invent a future cutoff. No gmax writes can enter while gated.
|
|
1141
|
+
yield new Promise((resolve) => setTimeout(resolve, 1));
|
|
1142
|
+
const cutoff = new Date();
|
|
1143
|
+
if (!Number.isFinite(latest.timestamp.getTime()) ||
|
|
1144
|
+
cutoff.getTime() <= latest.timestamp.getTime()) {
|
|
1145
|
+
return deferred("cleanup deferred: clock has not passed the commit");
|
|
1146
|
+
}
|
|
1147
|
+
const freeBytes = this.getAvailableBytes();
|
|
1148
|
+
const requiredBytes = stats.totalBytes * 2 + config_1.DISK_CRITICAL_BYTES;
|
|
1149
|
+
if (!Number.isFinite(stats.totalBytes) ||
|
|
1150
|
+
stats.totalBytes < 0 ||
|
|
1151
|
+
freeBytes < requiredBytes) {
|
|
1152
|
+
return deferred("cleanup deferred: insufficient rewrite headroom");
|
|
1153
|
+
}
|
|
1154
|
+
(0, logger_1.log)("vectordb", `Compaction cleanup attempt: ${JSON.stringify({ version, cutoff: cutoff.getTime(), freeBytes, requiredBytes })}`);
|
|
1155
|
+
passes = 1;
|
|
1156
|
+
const result = yield fresh.optimize({
|
|
1157
|
+
cleanupOlderThan: cutoff,
|
|
1158
|
+
deleteUnverified: true,
|
|
1159
|
+
});
|
|
1160
|
+
if (result.compaction.fragmentsRemoved !== 0 ||
|
|
1161
|
+
result.compaction.fragmentsAdded !== 0) {
|
|
1162
|
+
return {
|
|
1163
|
+
passes,
|
|
1164
|
+
bytesRemoved: result.prune.bytesRemoved,
|
|
1165
|
+
failed: true,
|
|
1166
|
+
reason: "cleanup unexpectedly rewrote data; no retry",
|
|
1167
|
+
};
|
|
1168
|
+
}
|
|
1169
|
+
return { passes, bytesRemoved: result.prune.bytesRemoved };
|
|
1170
|
+
}
|
|
1171
|
+
catch (error) {
|
|
1172
|
+
return {
|
|
1173
|
+
passes,
|
|
1174
|
+
bytesRemoved: 0,
|
|
1175
|
+
failed: true,
|
|
1176
|
+
reason: `cleanup failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
1177
|
+
};
|
|
1178
|
+
}
|
|
1179
|
+
});
|
|
1180
|
+
}
|
|
1111
1181
|
optimize() {
|
|
1112
1182
|
return __awaiter(this, arguments, void 0, function* (retries = COMPACTION_MAX_ATTEMPTS, retentionMs = 0, bypassExclusiveMutation = false) {
|
|
1113
1183
|
var _a;
|
|
1184
|
+
var _b;
|
|
1114
1185
|
if (!bypassExclusiveMutation) {
|
|
1115
1186
|
while (this.exclusiveMutationPromise)
|
|
1116
1187
|
yield this.exclusiveMutationPromise;
|
|
@@ -1120,7 +1191,7 @@ class VectorDB {
|
|
|
1120
1191
|
if (this.compactingPromise) {
|
|
1121
1192
|
(0, logger_1.debug)("vectordb", "Optimize already in progress, skipping");
|
|
1122
1193
|
yield this.compactingPromise;
|
|
1123
|
-
return ((
|
|
1194
|
+
return ((_b = this.compactionStatus()) !== null && _b !== void 0 ? _b : (0, compaction_result_1.skippedCompaction)("compaction unavailable"));
|
|
1124
1195
|
}
|
|
1125
1196
|
this.lastOptimizeDidWork = false;
|
|
1126
1197
|
let resolveCompacting;
|
|
@@ -1133,7 +1204,10 @@ class VectorDB {
|
|
|
1133
1204
|
let logicalBytes;
|
|
1134
1205
|
let diskBytesBefore;
|
|
1135
1206
|
let freeBytesBefore;
|
|
1207
|
+
let cleanupPasses = 0;
|
|
1208
|
+
let cleanupReason;
|
|
1136
1209
|
const finish = (status, reason, bytesReclaimed) => {
|
|
1210
|
+
const diskBytesAfter = this.getDirectorySize(this.lancedbDir);
|
|
1137
1211
|
const result = {
|
|
1138
1212
|
status,
|
|
1139
1213
|
at: Date.now(),
|
|
@@ -1143,9 +1217,14 @@ class VectorDB {
|
|
|
1143
1217
|
logicalBytes,
|
|
1144
1218
|
diskBytesBefore,
|
|
1145
1219
|
freeBytesBefore,
|
|
1146
|
-
diskBytesAfter
|
|
1220
|
+
diskBytesAfter,
|
|
1147
1221
|
freeBytesAfter: this.getAvailableBytes(),
|
|
1148
1222
|
bytesReclaimed,
|
|
1223
|
+
netBytesReclaimed: diskBytesBefore === undefined
|
|
1224
|
+
? undefined
|
|
1225
|
+
: diskBytesBefore - diskBytesAfter,
|
|
1226
|
+
cleanupPasses,
|
|
1227
|
+
cleanupReason,
|
|
1149
1228
|
};
|
|
1150
1229
|
this.lastCompactionResult = result;
|
|
1151
1230
|
(0, logger_1.log)("vectordb", `Compaction result: ${JSON.stringify(result)}`);
|
|
@@ -1210,7 +1289,18 @@ class VectorDB {
|
|
|
1210
1289
|
else {
|
|
1211
1290
|
(0, logger_1.debug)("vectordb", "Optimize: nothing to compact or prune");
|
|
1212
1291
|
}
|
|
1213
|
-
|
|
1292
|
+
let bytesReclaimed = prune.bytesRemoved;
|
|
1293
|
+
if (retentionMs === 0 && compaction.fragmentsRemoved > 0) {
|
|
1294
|
+
const cleanup = yield this.cleanupCompactionReservation(table);
|
|
1295
|
+
cleanupPasses = cleanup.passes;
|
|
1296
|
+
cleanupReason = (_a = cleanup.reason) === null || _a === void 0 ? void 0 : _a.replace(/[\r\n\t]/g, " ").slice(0, 512);
|
|
1297
|
+
bytesReclaimed += cleanup.bytesRemoved;
|
|
1298
|
+
this.lastOptimizeDidWork || (this.lastOptimizeDidWork = cleanup.bytesRemoved > 0);
|
|
1299
|
+
(0, logger_1.log)("vectordb", `Compaction cleanup result: ${JSON.stringify(Object.assign(Object.assign({}, cleanup), { reason: cleanupReason }))}`);
|
|
1300
|
+
if (cleanup.failed)
|
|
1301
|
+
return finish("failed", cleanupReason, bytesReclaimed);
|
|
1302
|
+
}
|
|
1303
|
+
return finish("completed", undefined, bytesReclaimed);
|
|
1214
1304
|
}
|
|
1215
1305
|
catch (e) {
|
|
1216
1306
|
const msg = e instanceof Error ? e.message : String(e);
|
|
@@ -1337,7 +1427,7 @@ class VectorDB {
|
|
|
1337
1427
|
// call, so a productive first pass followed by a barren bloat retry would
|
|
1338
1428
|
// otherwise read as unproductive and trigger a spurious backoff.
|
|
1339
1429
|
let didWork = this.lastOptimizeDidWork;
|
|
1340
|
-
if (this.lastOptimizeDidWork) {
|
|
1430
|
+
if (result.status === "completed" && this.lastOptimizeDidWork) {
|
|
1341
1431
|
const table = yield this.openExistingTableUnsafe();
|
|
1342
1432
|
if (table) {
|
|
1343
1433
|
// Collect abandoned scratch files before measuring. They are pure
|
|
@@ -72,6 +72,12 @@ function formatDoctorOptimize(report, agent) {
|
|
|
72
72
|
...(typeof (r === null || r === void 0 ? void 0 : r.bytesReclaimed) === "number"
|
|
73
73
|
? [`bytes_reclaimed=${r.bytesReclaimed}`]
|
|
74
74
|
: []),
|
|
75
|
+
...(typeof (r === null || r === void 0 ? void 0 : r.netBytesReclaimed) === "number"
|
|
76
|
+
? [`net_bytes_reclaimed=${r.netBytesReclaimed}`]
|
|
77
|
+
: []),
|
|
78
|
+
...(typeof (r === null || r === void 0 ? void 0 : r.cleanupPasses) === "number"
|
|
79
|
+
? [`cleanup_passes=${r.cleanupPasses}`]
|
|
80
|
+
: []),
|
|
75
81
|
...(reason ? [`reason=${reason}`] : []),
|
|
76
82
|
].join("\t");
|
|
77
83
|
}
|
|
@@ -208,7 +208,7 @@ export declare function makeEmptyTable(schema: SchemaLike, metadata?: Map<string
|
|
|
208
208
|
* customized by the `embeddingDataType` property of the embedding function.
|
|
209
209
|
*
|
|
210
210
|
* If a schema is provided in `makeTableOptions` then it should include the
|
|
211
|
-
* embedding columns. If no schema is
|
|
211
|
+
* embedding columns. If no schema is provided then embedding columns will
|
|
212
212
|
* be placed at the end of the table, after all of the input columns.
|
|
213
213
|
*/
|
|
214
214
|
export declare function convertToTable(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunctionConfig, makeTableOptions?: Partial<MakeArrowTableOptions>): Promise<ArrowTable>;
|
|
@@ -57,6 +57,7 @@ exports.ensureNestedFieldsExist = ensureNestedFieldsExist;
|
|
|
57
57
|
exports.dataTypeToJson = dataTypeToJson;
|
|
58
58
|
const apache_arrow_1 = require("apache-arrow");
|
|
59
59
|
const arrow_type_1 = require("./arrow_type");
|
|
60
|
+
const blob_1 = require("./blob");
|
|
60
61
|
const registry_1 = require("./embedding/registry");
|
|
61
62
|
const sanitize_1 = require("./sanitize");
|
|
62
63
|
const schema_1 = require("./schema");
|
|
@@ -348,17 +349,39 @@ function makeArrowTable(data, options, metadata) {
|
|
|
348
349
|
}
|
|
349
350
|
else {
|
|
350
351
|
schema = new apache_arrow_1.Schema(schema.fields, schemaMetadata);
|
|
352
|
+
validateBlobSchema(schema);
|
|
351
353
|
return new apache_arrow_1.Table(schema);
|
|
352
354
|
}
|
|
353
355
|
}
|
|
354
356
|
let inferredSchema = (0, schema_1.inferSchema)(data, schema, opt);
|
|
355
357
|
inferredSchema = new apache_arrow_1.Schema(inferredSchema.fields, schemaMetadata);
|
|
358
|
+
validateBlobSchema(inferredSchema);
|
|
356
359
|
const finalColumns = {};
|
|
357
360
|
for (const field of inferredSchema.fields) {
|
|
358
361
|
finalColumns[field.name] = transposeData(data, field);
|
|
359
362
|
}
|
|
360
363
|
return new apache_arrow_1.Table(inferredSchema, finalColumns);
|
|
361
364
|
}
|
|
365
|
+
function validateBlobSchema(schema) {
|
|
366
|
+
for (const field of schema.fields) {
|
|
367
|
+
validateBlobField(field);
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
function validateBlobField(field) {
|
|
371
|
+
if (isFixedSizeList(field.type) &&
|
|
372
|
+
containsBlobField(field.type.children[0])) {
|
|
373
|
+
throw new Error("Blob fields inside FixedSizeList are not supported. Use List instead.");
|
|
374
|
+
}
|
|
375
|
+
for (const child of field.type.children ?? []) {
|
|
376
|
+
validateBlobField(child);
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
function containsBlobField(field) {
|
|
380
|
+
if ((0, blob_1.isBlobField)(field)) {
|
|
381
|
+
return true;
|
|
382
|
+
}
|
|
383
|
+
return (field.type.children ?? []).some((child) => containsBlobField(child));
|
|
384
|
+
}
|
|
362
385
|
function isObject(value) {
|
|
363
386
|
return (typeof value === "object" &&
|
|
364
387
|
value !== null &&
|
|
@@ -387,6 +410,27 @@ function valueAtPath(datum, path) {
|
|
|
387
410
|
}
|
|
388
411
|
function transposeData(data, field, path = []) {
|
|
389
412
|
const valuesPath = [...path, field.name];
|
|
413
|
+
if ((0, blob_1.isBlobField)(field) && field.type instanceof apache_arrow_1.Struct) {
|
|
414
|
+
const blobRows = data.map((datum) => (0, blob_1.coerceBlobValue)(valueAtPath(datum, valuesPath)));
|
|
415
|
+
const childVectors = field.type.children.map((child) => {
|
|
416
|
+
const values = blobRows.map((row) => row == null ? null : (row[child.name] ?? null));
|
|
417
|
+
return makeVector(values, child.type, undefined, child.nullable);
|
|
418
|
+
});
|
|
419
|
+
const nullCount = blobRows.filter((row) => row === null).length;
|
|
420
|
+
const structData = (0, apache_arrow_1.makeData)({
|
|
421
|
+
type: field.type,
|
|
422
|
+
length: blobRows.length,
|
|
423
|
+
nullCount,
|
|
424
|
+
nullBitmap: nullCount > 0
|
|
425
|
+
? apache_arrow_1.util.packBools(blobRows.map((row) => row !== null))
|
|
426
|
+
: undefined,
|
|
427
|
+
children: childVectors.map((v) => v.data[0]),
|
|
428
|
+
});
|
|
429
|
+
return (0, apache_arrow_1.makeVector)(structData);
|
|
430
|
+
}
|
|
431
|
+
if (isList(field.type) && containsBlobField(field.type.children[0])) {
|
|
432
|
+
return transposeListData(data, field, valuesPath);
|
|
433
|
+
}
|
|
390
434
|
const values = data.map((datum) => valueAtPath(datum, valuesPath));
|
|
391
435
|
if (field.type instanceof apache_arrow_1.Struct) {
|
|
392
436
|
const childFields = field.type.children;
|
|
@@ -401,7 +445,7 @@ function transposeData(data, field, path = []) {
|
|
|
401
445
|
nullBitmap: nullCount > 0
|
|
402
446
|
? apache_arrow_1.util.packBools(values.map((value) => value !== null))
|
|
403
447
|
: undefined,
|
|
404
|
-
children: childVectors,
|
|
448
|
+
children: childVectors.map((v) => v.data[0]),
|
|
405
449
|
});
|
|
406
450
|
return (0, apache_arrow_1.makeVector)(structData);
|
|
407
451
|
}
|
|
@@ -409,6 +453,39 @@ function transposeData(data, field, path = []) {
|
|
|
409
453
|
return makeVector(values, field.type, undefined, field.nullable);
|
|
410
454
|
}
|
|
411
455
|
}
|
|
456
|
+
function transposeListData(data, field, valuesPath) {
|
|
457
|
+
const listType = field.type;
|
|
458
|
+
const childField = listType.children[0];
|
|
459
|
+
const lists = data.map((datum) => valueAtPath(datum, valuesPath));
|
|
460
|
+
const flattened = [];
|
|
461
|
+
const validity = [];
|
|
462
|
+
const offsets = [0];
|
|
463
|
+
for (const list of lists) {
|
|
464
|
+
if (list == null) {
|
|
465
|
+
validity.push(false);
|
|
466
|
+
offsets.push(flattened.length);
|
|
467
|
+
continue;
|
|
468
|
+
}
|
|
469
|
+
if (!Array.isArray(list)) {
|
|
470
|
+
throw new Error(`expected an array for list field '${field.name}'`);
|
|
471
|
+
}
|
|
472
|
+
validity.push(true);
|
|
473
|
+
for (const element of list) {
|
|
474
|
+
flattened.push({ [childField.name]: element });
|
|
475
|
+
}
|
|
476
|
+
offsets.push(flattened.length);
|
|
477
|
+
}
|
|
478
|
+
const childVector = transposeData(flattened, childField, []);
|
|
479
|
+
const nullCount = validity.filter((valid) => !valid).length;
|
|
480
|
+
return (0, apache_arrow_1.makeVector)((0, apache_arrow_1.makeData)({
|
|
481
|
+
type: listType,
|
|
482
|
+
length: lists.length,
|
|
483
|
+
nullCount,
|
|
484
|
+
nullBitmap: nullCount > 0 ? apache_arrow_1.util.packBools(validity) : undefined,
|
|
485
|
+
valueOffsets: Int32Array.from(offsets),
|
|
486
|
+
child: childVector.data[0],
|
|
487
|
+
}));
|
|
488
|
+
}
|
|
412
489
|
/**
|
|
413
490
|
* Create an empty Arrow table with the provided schema
|
|
414
491
|
*/
|
|
@@ -497,7 +574,7 @@ function makeVector(values, type, stringAsDictionary, nullable) {
|
|
|
497
574
|
return vectorFromArray(values, type);
|
|
498
575
|
}
|
|
499
576
|
if (values.length === 0) {
|
|
500
|
-
throw Error("makeVector requires at least one value or the type must be
|
|
577
|
+
throw Error("makeVector requires at least one value or the type must be specified");
|
|
501
578
|
}
|
|
502
579
|
const sampleValue = values.find((val) => val !== null && val !== undefined);
|
|
503
580
|
if (sampleValue === undefined) {
|
|
@@ -688,7 +765,7 @@ async function applyEmbeddings(table, embeddings, schema) {
|
|
|
688
765
|
* customized by the `embeddingDataType` property of the embedding function.
|
|
689
766
|
*
|
|
690
767
|
* If a schema is provided in `makeTableOptions` then it should include the
|
|
691
|
-
* embedding columns. If no schema is
|
|
768
|
+
* embedding columns. If no schema is provided then embedding columns will
|
|
692
769
|
* be placed at the end of the table, after all of the input columns.
|
|
693
770
|
*/
|
|
694
771
|
async function convertToTable(data, embeddings, makeTableOptions) {
|
|
@@ -752,6 +829,7 @@ async function fromTableToBuffer(table, embeddings, schema) {
|
|
|
752
829
|
schema = (0, sanitize_1.sanitizeSchema)(schema);
|
|
753
830
|
}
|
|
754
831
|
const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
|
|
832
|
+
validateBlobSchema(tableWithEmbeddings.schema);
|
|
755
833
|
const writer = apache_arrow_1.RecordBatchFileWriter.writeAll(tableWithEmbeddings);
|
|
756
834
|
return Buffer.from(await writer.toUint8Array());
|
|
757
835
|
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { Field } from "apache-arrow";
|
|
2
|
+
import { BlobFile as NativeBlobFile } from "./native";
|
|
3
|
+
export type BlobInput = {
|
|
4
|
+
data: Buffer | Uint8Array | null;
|
|
5
|
+
uri: string | null;
|
|
6
|
+
};
|
|
7
|
+
export type BlobOptions = {
|
|
8
|
+
/** Defaults to true. */
|
|
9
|
+
nullable?: boolean;
|
|
10
|
+
/**
|
|
11
|
+
* Max payload bytes kept inline in the data file. Zero is allowed. Must be a
|
|
12
|
+
* safe integer.
|
|
13
|
+
*/
|
|
14
|
+
inlineSizeThreshold?: number;
|
|
15
|
+
/**
|
|
16
|
+
* Max payload bytes stored in a packed sidecar before a dedicated file. Must
|
|
17
|
+
* be a positive safe integer.
|
|
18
|
+
*/
|
|
19
|
+
dedicatedSizeThreshold?: number;
|
|
20
|
+
/**
|
|
21
|
+
* Max bytes in one packed sidecar before starting another. Must be a positive
|
|
22
|
+
* safe integer.
|
|
23
|
+
*/
|
|
24
|
+
packFileSizeThreshold?: number;
|
|
25
|
+
};
|
|
26
|
+
/**
|
|
27
|
+
* Declares a `lance.blob.v2` column.
|
|
28
|
+
*
|
|
29
|
+
* Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
|
|
30
|
+
* or {@link Table.fetchBlobFiles} to read bytes.
|
|
31
|
+
*
|
|
32
|
+
* @example
|
|
33
|
+
* ```ts
|
|
34
|
+
* import { readFile } from "node:fs/promises";
|
|
35
|
+
* import { Field, Int64, Schema } from "apache-arrow";
|
|
36
|
+
* import { blob, connect } from "@lancedb/lancedb";
|
|
37
|
+
*
|
|
38
|
+
* const db = await connect("./data");
|
|
39
|
+
* const video = await readFile("clip.mp4");
|
|
40
|
+
* const table = await db.createTable(
|
|
41
|
+
* "videos",
|
|
42
|
+
* [{ id: 1n, video }],
|
|
43
|
+
* {
|
|
44
|
+
* schema: new Schema([
|
|
45
|
+
* new Field("id", new Int64()),
|
|
46
|
+
* blob("video"),
|
|
47
|
+
* ]),
|
|
48
|
+
* },
|
|
49
|
+
* );
|
|
50
|
+
*
|
|
51
|
+
* const rows = await table.query().select(["id"]).withRowId().toArray();
|
|
52
|
+
* const rowIds = rows.map((row) => row._rowid as bigint);
|
|
53
|
+
* const bytes = await table.fetchBlobs("video", rowIds);
|
|
54
|
+
*
|
|
55
|
+
* const [handle] = await table.fetchBlobFiles("video", rowIds);
|
|
56
|
+
* const size = handle!.size();
|
|
57
|
+
* const header = await handle!.readRange(0n, size < 65536n ? size : 65536n);
|
|
58
|
+
* ```
|
|
59
|
+
*/
|
|
60
|
+
export declare function blob(name: string, options?: BlobOptions): Field;
|
|
61
|
+
/**
|
|
62
|
+
* Checks for the `lance.blob.v2` extension marker. Does not validate the
|
|
63
|
+
* field's storage type.
|
|
64
|
+
*/
|
|
65
|
+
export declare function isBlobField(field: Field): boolean;
|
|
66
|
+
/**
|
|
67
|
+
* A lazy handle to blob bytes. Create one with {@link Table.fetchBlobFiles}.
|
|
68
|
+
*
|
|
69
|
+
* @hideconstructor
|
|
70
|
+
*/
|
|
71
|
+
export declare class BlobFile {
|
|
72
|
+
private readonly inner;
|
|
73
|
+
private constructor();
|
|
74
|
+
/** @ignore */
|
|
75
|
+
static fromNative(inner: NativeBlobFile): BlobFile;
|
|
76
|
+
/** Returns the blob size in bytes. */
|
|
77
|
+
size(): bigint;
|
|
78
|
+
/**
|
|
79
|
+
* Reads from the cursor to the end and advances the cursor.
|
|
80
|
+
*
|
|
81
|
+
* A second call returns an empty buffer. {@link BlobFile.readRange} does
|
|
82
|
+
* not move the cursor.
|
|
83
|
+
*/
|
|
84
|
+
read(): Promise<Buffer>;
|
|
85
|
+
/**
|
|
86
|
+
* Reads the half-open byte range `[start, end)`.
|
|
87
|
+
*
|
|
88
|
+
* Fails when `end` is past the blob size. Does not move the cursor.
|
|
89
|
+
*/
|
|
90
|
+
readRange(start: bigint, end: bigint): Promise<Buffer>;
|
|
91
|
+
}
|
|
92
|
+
export declare function coerceBlobValue(value: unknown): BlobInput | null;
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
4
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
+
exports.BlobFile = void 0;
|
|
6
|
+
exports.blob = blob;
|
|
7
|
+
exports.isBlobField = isBlobField;
|
|
8
|
+
exports.coerceBlobValue = coerceBlobValue;
|
|
9
|
+
const apache_arrow_1 = require("apache-arrow");
|
|
10
|
+
const native_1 = require("./native");
|
|
11
|
+
const BLOB_V2_EXTENSION_NAME = "lance.blob.v2";
|
|
12
|
+
const INLINE_SIZE_THRESHOLD_KEY = "lance-encoding:blob-inline-size-threshold";
|
|
13
|
+
const DEDICATED_SIZE_THRESHOLD_KEY = "lance-encoding:blob-dedicated-size-threshold";
|
|
14
|
+
const PACK_FILE_SIZE_THRESHOLD_KEY = "lance-encoding:blob-pack-file-size-threshold";
|
|
15
|
+
/**
|
|
16
|
+
* Declares a `lance.blob.v2` column.
|
|
17
|
+
*
|
|
18
|
+
* Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
|
|
19
|
+
* or {@link Table.fetchBlobFiles} to read bytes.
|
|
20
|
+
*
|
|
21
|
+
* @example
|
|
22
|
+
* ```ts
|
|
23
|
+
* import { readFile } from "node:fs/promises";
|
|
24
|
+
* import { Field, Int64, Schema } from "apache-arrow";
|
|
25
|
+
* import { blob, connect } from "@lancedb/lancedb";
|
|
26
|
+
*
|
|
27
|
+
* const db = await connect("./data");
|
|
28
|
+
* const video = await readFile("clip.mp4");
|
|
29
|
+
* const table = await db.createTable(
|
|
30
|
+
* "videos",
|
|
31
|
+
* [{ id: 1n, video }],
|
|
32
|
+
* {
|
|
33
|
+
* schema: new Schema([
|
|
34
|
+
* new Field("id", new Int64()),
|
|
35
|
+
* blob("video"),
|
|
36
|
+
* ]),
|
|
37
|
+
* },
|
|
38
|
+
* );
|
|
39
|
+
*
|
|
40
|
+
* const rows = await table.query().select(["id"]).withRowId().toArray();
|
|
41
|
+
* const rowIds = rows.map((row) => row._rowid as bigint);
|
|
42
|
+
* const bytes = await table.fetchBlobs("video", rowIds);
|
|
43
|
+
*
|
|
44
|
+
* const [handle] = await table.fetchBlobFiles("video", rowIds);
|
|
45
|
+
* const size = handle!.size();
|
|
46
|
+
* const header = await handle!.readRange(0n, size < 65536n ? size : 65536n);
|
|
47
|
+
* ```
|
|
48
|
+
*/
|
|
49
|
+
function blob(name, options = {}) {
|
|
50
|
+
const metadata = new Map([
|
|
51
|
+
["ARROW:extension:name", BLOB_V2_EXTENSION_NAME],
|
|
52
|
+
]);
|
|
53
|
+
setThreshold(metadata, INLINE_SIZE_THRESHOLD_KEY, "inlineSizeThreshold", options.inlineSizeThreshold, 0);
|
|
54
|
+
setThreshold(metadata, DEDICATED_SIZE_THRESHOLD_KEY, "dedicatedSizeThreshold", options.dedicatedSizeThreshold, 1);
|
|
55
|
+
setThreshold(metadata, PACK_FILE_SIZE_THRESHOLD_KEY, "packFileSizeThreshold", options.packFileSizeThreshold, 1);
|
|
56
|
+
return new apache_arrow_1.Field(name, new apache_arrow_1.Struct([
|
|
57
|
+
new apache_arrow_1.Field("data", new apache_arrow_1.LargeBinary(), true),
|
|
58
|
+
new apache_arrow_1.Field("uri", new apache_arrow_1.Utf8(), true),
|
|
59
|
+
]), options.nullable ?? true, metadata);
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Checks for the `lance.blob.v2` extension marker. Does not validate the
|
|
63
|
+
* field's storage type.
|
|
64
|
+
*/
|
|
65
|
+
function isBlobField(field) {
|
|
66
|
+
return field.metadata?.get("ARROW:extension:name") === BLOB_V2_EXTENSION_NAME;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* A lazy handle to blob bytes. Create one with {@link Table.fetchBlobFiles}.
|
|
70
|
+
*
|
|
71
|
+
* @hideconstructor
|
|
72
|
+
*/
|
|
73
|
+
class BlobFile {
|
|
74
|
+
inner;
|
|
75
|
+
constructor(inner) {
|
|
76
|
+
if (!(inner instanceof native_1.BlobFile)) {
|
|
77
|
+
throw new Error("BlobFile handles come from Table.fetchBlobFiles");
|
|
78
|
+
}
|
|
79
|
+
this.inner = inner;
|
|
80
|
+
}
|
|
81
|
+
/** @ignore */
|
|
82
|
+
static fromNative(inner) {
|
|
83
|
+
return new BlobFile(inner);
|
|
84
|
+
}
|
|
85
|
+
/** Returns the blob size in bytes. */
|
|
86
|
+
size() {
|
|
87
|
+
return this.inner.size();
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Reads from the cursor to the end and advances the cursor.
|
|
91
|
+
*
|
|
92
|
+
* A second call returns an empty buffer. {@link BlobFile.readRange} does
|
|
93
|
+
* not move the cursor.
|
|
94
|
+
*/
|
|
95
|
+
read() {
|
|
96
|
+
return this.inner.read();
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Reads the half-open byte range `[start, end)`.
|
|
100
|
+
*
|
|
101
|
+
* Fails when `end` is past the blob size. Does not move the cursor.
|
|
102
|
+
*/
|
|
103
|
+
readRange(start, end) {
|
|
104
|
+
return this.inner.readRange(start, end);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
exports.BlobFile = BlobFile;
|
|
108
|
+
function coerceBlobValue(value) {
|
|
109
|
+
if (value == null) {
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
if (isBlobBytes(value)) {
|
|
113
|
+
return { data: value, uri: null };
|
|
114
|
+
}
|
|
115
|
+
if (ArrayBuffer.isView(value)) {
|
|
116
|
+
throw new Error("Blob data must be Buffer or Uint8Array");
|
|
117
|
+
}
|
|
118
|
+
if (typeof value === "string") {
|
|
119
|
+
if (value === "") {
|
|
120
|
+
throw new Error("Blob uri cannot be empty");
|
|
121
|
+
}
|
|
122
|
+
return { data: null, uri: value };
|
|
123
|
+
}
|
|
124
|
+
if (typeof value === "object") {
|
|
125
|
+
const record = value;
|
|
126
|
+
if (!("data" in record) && !("uri" in record)) {
|
|
127
|
+
throw new Error("Blob struct values must include a 'data' or 'uri' field");
|
|
128
|
+
}
|
|
129
|
+
const uri = record.uri;
|
|
130
|
+
if (uri === "") {
|
|
131
|
+
throw new Error("Blob uri cannot be empty");
|
|
132
|
+
}
|
|
133
|
+
if (uri != null && typeof uri !== "string") {
|
|
134
|
+
throw new Error(`Blob uri must be a string or null, got ${typeof uri}`);
|
|
135
|
+
}
|
|
136
|
+
const data = record.data;
|
|
137
|
+
if (data != null && !isBlobBytes(data)) {
|
|
138
|
+
throw new Error("Blob data must be Buffer, Uint8Array, or null");
|
|
139
|
+
}
|
|
140
|
+
const bytes = data ?? null;
|
|
141
|
+
const uriValue = uri ?? null;
|
|
142
|
+
if ((bytes == null) === (uriValue == null)) {
|
|
143
|
+
throw new Error("Blob struct values must set exactly one of 'data' or 'uri'");
|
|
144
|
+
}
|
|
145
|
+
return { data: bytes, uri: uriValue };
|
|
146
|
+
}
|
|
147
|
+
throw new Error("Blob column values must be Buffer, Uint8Array, a URI string, null, or { data?, uri? }");
|
|
148
|
+
}
|
|
149
|
+
function isBlobBytes(value) {
|
|
150
|
+
return Buffer.isBuffer(value) || value instanceof Uint8Array;
|
|
151
|
+
}
|
|
152
|
+
function setThreshold(metadata, key, optionName, value, minimum) {
|
|
153
|
+
if (value === undefined) {
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
if (!Number.isSafeInteger(value)) {
|
|
157
|
+
throw new Error(`${optionName} must be a safe integer`);
|
|
158
|
+
}
|
|
159
|
+
if (value < minimum) {
|
|
160
|
+
throw new Error(minimum <= 0
|
|
161
|
+
? `${optionName} must be non-negative`
|
|
162
|
+
: `${optionName} must be positive`);
|
|
163
|
+
}
|
|
164
|
+
metadata.set(key, String(value));
|
|
165
|
+
}
|