grepmax 0.26.52 → 0.26.54

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,7 +10,7 @@
10
10
  "name": "grepmax",
11
11
  "source": "./plugins/grepmax",
12
12
  "description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
13
- "version": "0.26.52",
13
+ "version": "0.26.54",
14
14
  "author": {
15
15
  "name": "Robert Owens",
16
16
  "email": "robowens@me.com"
package/NOTICE CHANGED
@@ -33,7 +33,7 @@ This software depends on various open-source libraries. See package.json for
33
33
  a complete list of dependencies and their respective licenses.
34
34
 
35
35
  The distribution includes the unmodified JavaScript runtime from
36
- @lancedb/lancedb 0.38.0 (https://github.com/lancedb/lancedb), licensed under
36
+ @lancedb/lancedb 0.39.0 (https://github.com/lancedb/lancedb), licensed under
37
37
  Apache-2.0. Its license and build provenance are included in dist/vendor/lancedb/.
38
38
  Its optional provider dependency manifest is not distributed. Platform native
39
39
  binaries are installed as separately pinned @lancedb packages.
package/README.md CHANGED
@@ -484,7 +484,7 @@ whether you need to act.
484
484
 
485
485
  | You see | What it means |
486
486
  |---|---|
487
- | `Optimize panicked ... inverted/builder.rs` | The historical FTS merge defect is fixed in the shipped LanceDB 0.38.0 runtime. If it recurs on a current release, preserve the version and logs and report a regression; the rebuild guard remains a recovery tripwire. |
487
+ | `Optimize panicked ... inverted/builder.rs` | The historical FTS merge defect is fixed in the shipped LanceDB 0.39.0 runtime. If it recurs on a current release, preserve the version and logs and report a regression; the rebuild guard remains a recovery tripwire. |
488
488
  | `disabling auto-rebuild until an optimize succeeds` | Transient, not a wedge. The next successful optimize clears it. Search stays available. |
489
489
  | `ANN: vector index not built` | Normal — exact search is the default. |
490
490
  | `cannot reach the daemon socket from this sandbox` (exit 2) | The shell is sandboxed. Add the two keys in [Running under the Claude Code sandbox](#running-under-the-claude-code-sandbox); `gmax doctor` warns about the same gap. |
@@ -47,6 +47,9 @@ exports.compactionSchema = zod_1.z.object({
47
47
  freeBytesBefore: zod_1.z.number().optional(),
48
48
  freeBytesAfter: zod_1.z.number().optional(),
49
49
  bytesReclaimed: zod_1.z.number().optional(),
50
+ netBytesReclaimed: zod_1.z.number().optional(),
51
+ cleanupPasses: zod_1.z.number().int().min(0).max(1).optional(),
52
+ cleanupReason: zod_1.z.string().optional(),
50
53
  });
51
54
  const embedding = zod_1.z.object({
52
55
  tier: zod_1.z.string(),
@@ -1108,9 +1108,80 @@ class VectorDB {
1108
1108
  (0, logger_1.log)("vectordb", `Compaction result: ${JSON.stringify(result)}`);
1109
1109
  return Object.assign({}, result);
1110
1110
  }
1111
+ /** Prune the reserve-ID snapshot without another data rewrite. Caller holds
1112
+ * compactingPromise and has drained writes; never use this for retention > 0.
1113
+ * Lance 12 treats one deletion-free fragment as a compaction no-op. */
1114
+ cleanupCompactionReservation(table) {
1115
+ return __awaiter(this, void 0, void 0, function* () {
1116
+ var _a;
1117
+ let passes = 0;
1118
+ const deferred = (reason) => ({
1119
+ passes: 0,
1120
+ bytesRemoved: 0,
1121
+ reason,
1122
+ });
1123
+ try {
1124
+ const stats = yield table.stats();
1125
+ if (((_a = stats.fragmentStats) === null || _a === void 0 ? void 0 : _a.numFragments) !== 1) {
1126
+ return deferred("cleanup deferred: table is not a single fragment");
1127
+ }
1128
+ const version = yield table.version();
1129
+ const fresh = yield this.openExistingTableUnsafe();
1130
+ if (!fresh || (yield fresh.version()) !== version) {
1131
+ return deferred("cleanup deferred: table changed after compaction");
1132
+ }
1133
+ const latest = (yield fresh.listVersions()).find((v) => v.version === version);
1134
+ if ((latest === null || latest === void 0 ? void 0 : latest.metadata.total_fragments) !== "1" ||
1135
+ latest.metadata.total_deletion_files !== "0" ||
1136
+ latest.metadata.total_deletion_file_rows !== "0") {
1137
+ return deferred("cleanup deferred: deletion-free fragment not verified");
1138
+ }
1139
+ // Manifest timestamps have sub-ms precision; wait for a real later tick,
1140
+ // never invent a future cutoff. No gmax writes can enter while gated.
1141
+ yield new Promise((resolve) => setTimeout(resolve, 1));
1142
+ const cutoff = new Date();
1143
+ if (!Number.isFinite(latest.timestamp.getTime()) ||
1144
+ cutoff.getTime() <= latest.timestamp.getTime()) {
1145
+ return deferred("cleanup deferred: clock has not passed the commit");
1146
+ }
1147
+ const freeBytes = this.getAvailableBytes();
1148
+ const requiredBytes = stats.totalBytes * 2 + config_1.DISK_CRITICAL_BYTES;
1149
+ if (!Number.isFinite(stats.totalBytes) ||
1150
+ stats.totalBytes < 0 ||
1151
+ freeBytes < requiredBytes) {
1152
+ return deferred("cleanup deferred: insufficient rewrite headroom");
1153
+ }
1154
+ (0, logger_1.log)("vectordb", `Compaction cleanup attempt: ${JSON.stringify({ version, cutoff: cutoff.getTime(), freeBytes, requiredBytes })}`);
1155
+ passes = 1;
1156
+ const result = yield fresh.optimize({
1157
+ cleanupOlderThan: cutoff,
1158
+ deleteUnverified: true,
1159
+ });
1160
+ if (result.compaction.fragmentsRemoved !== 0 ||
1161
+ result.compaction.fragmentsAdded !== 0) {
1162
+ return {
1163
+ passes,
1164
+ bytesRemoved: result.prune.bytesRemoved,
1165
+ failed: true,
1166
+ reason: "cleanup unexpectedly rewrote data; no retry",
1167
+ };
1168
+ }
1169
+ return { passes, bytesRemoved: result.prune.bytesRemoved };
1170
+ }
1171
+ catch (error) {
1172
+ return {
1173
+ passes,
1174
+ bytesRemoved: 0,
1175
+ failed: true,
1176
+ reason: `cleanup failed: ${error instanceof Error ? error.message : String(error)}`,
1177
+ };
1178
+ }
1179
+ });
1180
+ }
1111
1181
  optimize() {
1112
1182
  return __awaiter(this, arguments, void 0, function* (retries = COMPACTION_MAX_ATTEMPTS, retentionMs = 0, bypassExclusiveMutation = false) {
1113
1183
  var _a;
1184
+ var _b;
1114
1185
  if (!bypassExclusiveMutation) {
1115
1186
  while (this.exclusiveMutationPromise)
1116
1187
  yield this.exclusiveMutationPromise;
@@ -1120,7 +1191,7 @@ class VectorDB {
1120
1191
  if (this.compactingPromise) {
1121
1192
  (0, logger_1.debug)("vectordb", "Optimize already in progress, skipping");
1122
1193
  yield this.compactingPromise;
1123
- return ((_a = this.compactionStatus()) !== null && _a !== void 0 ? _a : (0, compaction_result_1.skippedCompaction)("compaction unavailable"));
1194
+ return ((_b = this.compactionStatus()) !== null && _b !== void 0 ? _b : (0, compaction_result_1.skippedCompaction)("compaction unavailable"));
1124
1195
  }
1125
1196
  this.lastOptimizeDidWork = false;
1126
1197
  let resolveCompacting;
@@ -1133,7 +1204,10 @@ class VectorDB {
1133
1204
  let logicalBytes;
1134
1205
  let diskBytesBefore;
1135
1206
  let freeBytesBefore;
1207
+ let cleanupPasses = 0;
1208
+ let cleanupReason;
1136
1209
  const finish = (status, reason, bytesReclaimed) => {
1210
+ const diskBytesAfter = this.getDirectorySize(this.lancedbDir);
1137
1211
  const result = {
1138
1212
  status,
1139
1213
  at: Date.now(),
@@ -1143,9 +1217,14 @@ class VectorDB {
1143
1217
  logicalBytes,
1144
1218
  diskBytesBefore,
1145
1219
  freeBytesBefore,
1146
- diskBytesAfter: this.getDirectorySize(this.lancedbDir),
1220
+ diskBytesAfter,
1147
1221
  freeBytesAfter: this.getAvailableBytes(),
1148
1222
  bytesReclaimed,
1223
+ netBytesReclaimed: diskBytesBefore === undefined
1224
+ ? undefined
1225
+ : diskBytesBefore - diskBytesAfter,
1226
+ cleanupPasses,
1227
+ cleanupReason,
1149
1228
  };
1150
1229
  this.lastCompactionResult = result;
1151
1230
  (0, logger_1.log)("vectordb", `Compaction result: ${JSON.stringify(result)}`);
@@ -1210,7 +1289,18 @@ class VectorDB {
1210
1289
  else {
1211
1290
  (0, logger_1.debug)("vectordb", "Optimize: nothing to compact or prune");
1212
1291
  }
1213
- return finish("completed", undefined, prune.bytesRemoved);
1292
+ let bytesReclaimed = prune.bytesRemoved;
1293
+ if (retentionMs === 0 && compaction.fragmentsRemoved > 0) {
1294
+ const cleanup = yield this.cleanupCompactionReservation(table);
1295
+ cleanupPasses = cleanup.passes;
1296
+ cleanupReason = (_a = cleanup.reason) === null || _a === void 0 ? void 0 : _a.replace(/[\r\n\t]/g, " ").slice(0, 512);
1297
+ bytesReclaimed += cleanup.bytesRemoved;
1298
+ this.lastOptimizeDidWork || (this.lastOptimizeDidWork = cleanup.bytesRemoved > 0);
1299
+ (0, logger_1.log)("vectordb", `Compaction cleanup result: ${JSON.stringify(Object.assign(Object.assign({}, cleanup), { reason: cleanupReason }))}`);
1300
+ if (cleanup.failed)
1301
+ return finish("failed", cleanupReason, bytesReclaimed);
1302
+ }
1303
+ return finish("completed", undefined, bytesReclaimed);
1214
1304
  }
1215
1305
  catch (e) {
1216
1306
  const msg = e instanceof Error ? e.message : String(e);
@@ -1337,7 +1427,7 @@ class VectorDB {
1337
1427
  // call, so a productive first pass followed by a barren bloat retry would
1338
1428
  // otherwise read as unproductive and trigger a spurious backoff.
1339
1429
  let didWork = this.lastOptimizeDidWork;
1340
- if (this.lastOptimizeDidWork) {
1430
+ if (result.status === "completed" && this.lastOptimizeDidWork) {
1341
1431
  const table = yield this.openExistingTableUnsafe();
1342
1432
  if (table) {
1343
1433
  // Collect abandoned scratch files before measuring. They are pure
@@ -72,6 +72,12 @@ function formatDoctorOptimize(report, agent) {
72
72
  ...(typeof (r === null || r === void 0 ? void 0 : r.bytesReclaimed) === "number"
73
73
  ? [`bytes_reclaimed=${r.bytesReclaimed}`]
74
74
  : []),
75
+ ...(typeof (r === null || r === void 0 ? void 0 : r.netBytesReclaimed) === "number"
76
+ ? [`net_bytes_reclaimed=${r.netBytesReclaimed}`]
77
+ : []),
78
+ ...(typeof (r === null || r === void 0 ? void 0 : r.cleanupPasses) === "number"
79
+ ? [`cleanup_passes=${r.cleanupPasses}`]
80
+ : []),
75
81
  ...(reason ? [`reason=${reason}`] : []),
76
82
  ].join("\t");
77
83
  }
@@ -208,7 +208,7 @@ export declare function makeEmptyTable(schema: SchemaLike, metadata?: Map<string
208
208
  * customized by the `embeddingDataType` property of the embedding function.
209
209
  *
210
210
  * If a schema is provided in `makeTableOptions` then it should include the
211
- * embedding columns. If no schema is provded then embedding columns will
211
+ * embedding columns. If no schema is provided then embedding columns will
212
212
  * be placed at the end of the table, after all of the input columns.
213
213
  */
214
214
  export declare function convertToTable(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunctionConfig, makeTableOptions?: Partial<MakeArrowTableOptions>): Promise<ArrowTable>;
@@ -57,6 +57,7 @@ exports.ensureNestedFieldsExist = ensureNestedFieldsExist;
57
57
  exports.dataTypeToJson = dataTypeToJson;
58
58
  const apache_arrow_1 = require("apache-arrow");
59
59
  const arrow_type_1 = require("./arrow_type");
60
+ const blob_1 = require("./blob");
60
61
  const registry_1 = require("./embedding/registry");
61
62
  const sanitize_1 = require("./sanitize");
62
63
  const schema_1 = require("./schema");
@@ -348,17 +349,39 @@ function makeArrowTable(data, options, metadata) {
348
349
  }
349
350
  else {
350
351
  schema = new apache_arrow_1.Schema(schema.fields, schemaMetadata);
352
+ validateBlobSchema(schema);
351
353
  return new apache_arrow_1.Table(schema);
352
354
  }
353
355
  }
354
356
  let inferredSchema = (0, schema_1.inferSchema)(data, schema, opt);
355
357
  inferredSchema = new apache_arrow_1.Schema(inferredSchema.fields, schemaMetadata);
358
+ validateBlobSchema(inferredSchema);
356
359
  const finalColumns = {};
357
360
  for (const field of inferredSchema.fields) {
358
361
  finalColumns[field.name] = transposeData(data, field);
359
362
  }
360
363
  return new apache_arrow_1.Table(inferredSchema, finalColumns);
361
364
  }
365
+ function validateBlobSchema(schema) {
366
+ for (const field of schema.fields) {
367
+ validateBlobField(field);
368
+ }
369
+ }
370
+ function validateBlobField(field) {
371
+ if (isFixedSizeList(field.type) &&
372
+ containsBlobField(field.type.children[0])) {
373
+ throw new Error("Blob fields inside FixedSizeList are not supported. Use List instead.");
374
+ }
375
+ for (const child of field.type.children ?? []) {
376
+ validateBlobField(child);
377
+ }
378
+ }
379
+ function containsBlobField(field) {
380
+ if ((0, blob_1.isBlobField)(field)) {
381
+ return true;
382
+ }
383
+ return (field.type.children ?? []).some((child) => containsBlobField(child));
384
+ }
362
385
  function isObject(value) {
363
386
  return (typeof value === "object" &&
364
387
  value !== null &&
@@ -387,6 +410,27 @@ function valueAtPath(datum, path) {
387
410
  }
388
411
  function transposeData(data, field, path = []) {
389
412
  const valuesPath = [...path, field.name];
413
+ if ((0, blob_1.isBlobField)(field) && field.type instanceof apache_arrow_1.Struct) {
414
+ const blobRows = data.map((datum) => (0, blob_1.coerceBlobValue)(valueAtPath(datum, valuesPath)));
415
+ const childVectors = field.type.children.map((child) => {
416
+ const values = blobRows.map((row) => row == null ? null : (row[child.name] ?? null));
417
+ return makeVector(values, child.type, undefined, child.nullable);
418
+ });
419
+ const nullCount = blobRows.filter((row) => row === null).length;
420
+ const structData = (0, apache_arrow_1.makeData)({
421
+ type: field.type,
422
+ length: blobRows.length,
423
+ nullCount,
424
+ nullBitmap: nullCount > 0
425
+ ? apache_arrow_1.util.packBools(blobRows.map((row) => row !== null))
426
+ : undefined,
427
+ children: childVectors.map((v) => v.data[0]),
428
+ });
429
+ return (0, apache_arrow_1.makeVector)(structData);
430
+ }
431
+ if (isList(field.type) && containsBlobField(field.type.children[0])) {
432
+ return transposeListData(data, field, valuesPath);
433
+ }
390
434
  const values = data.map((datum) => valueAtPath(datum, valuesPath));
391
435
  if (field.type instanceof apache_arrow_1.Struct) {
392
436
  const childFields = field.type.children;
@@ -401,7 +445,7 @@ function transposeData(data, field, path = []) {
401
445
  nullBitmap: nullCount > 0
402
446
  ? apache_arrow_1.util.packBools(values.map((value) => value !== null))
403
447
  : undefined,
404
- children: childVectors,
448
+ children: childVectors.map((v) => v.data[0]),
405
449
  });
406
450
  return (0, apache_arrow_1.makeVector)(structData);
407
451
  }
@@ -409,6 +453,39 @@ function transposeData(data, field, path = []) {
409
453
  return makeVector(values, field.type, undefined, field.nullable);
410
454
  }
411
455
  }
456
+ function transposeListData(data, field, valuesPath) {
457
+ const listType = field.type;
458
+ const childField = listType.children[0];
459
+ const lists = data.map((datum) => valueAtPath(datum, valuesPath));
460
+ const flattened = [];
461
+ const validity = [];
462
+ const offsets = [0];
463
+ for (const list of lists) {
464
+ if (list == null) {
465
+ validity.push(false);
466
+ offsets.push(flattened.length);
467
+ continue;
468
+ }
469
+ if (!Array.isArray(list)) {
470
+ throw new Error(`expected an array for list field '${field.name}'`);
471
+ }
472
+ validity.push(true);
473
+ for (const element of list) {
474
+ flattened.push({ [childField.name]: element });
475
+ }
476
+ offsets.push(flattened.length);
477
+ }
478
+ const childVector = transposeData(flattened, childField, []);
479
+ const nullCount = validity.filter((valid) => !valid).length;
480
+ return (0, apache_arrow_1.makeVector)((0, apache_arrow_1.makeData)({
481
+ type: listType,
482
+ length: lists.length,
483
+ nullCount,
484
+ nullBitmap: nullCount > 0 ? apache_arrow_1.util.packBools(validity) : undefined,
485
+ valueOffsets: Int32Array.from(offsets),
486
+ child: childVector.data[0],
487
+ }));
488
+ }
412
489
  /**
413
490
  * Create an empty Arrow table with the provided schema
414
491
  */
@@ -497,7 +574,7 @@ function makeVector(values, type, stringAsDictionary, nullable) {
497
574
  return vectorFromArray(values, type);
498
575
  }
499
576
  if (values.length === 0) {
500
- throw Error("makeVector requires at least one value or the type must be specfied");
577
+ throw Error("makeVector requires at least one value or the type must be specified");
501
578
  }
502
579
  const sampleValue = values.find((val) => val !== null && val !== undefined);
503
580
  if (sampleValue === undefined) {
@@ -688,7 +765,7 @@ async function applyEmbeddings(table, embeddings, schema) {
688
765
  * customized by the `embeddingDataType` property of the embedding function.
689
766
  *
690
767
  * If a schema is provided in `makeTableOptions` then it should include the
691
- * embedding columns. If no schema is provded then embedding columns will
768
+ * embedding columns. If no schema is provided then embedding columns will
692
769
  * be placed at the end of the table, after all of the input columns.
693
770
  */
694
771
  async function convertToTable(data, embeddings, makeTableOptions) {
@@ -752,6 +829,7 @@ async function fromTableToBuffer(table, embeddings, schema) {
752
829
  schema = (0, sanitize_1.sanitizeSchema)(schema);
753
830
  }
754
831
  const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
832
+ validateBlobSchema(tableWithEmbeddings.schema);
755
833
  const writer = apache_arrow_1.RecordBatchFileWriter.writeAll(tableWithEmbeddings);
756
834
  return Buffer.from(await writer.toUint8Array());
757
835
  }
@@ -0,0 +1,92 @@
1
+ import { Field } from "apache-arrow";
2
+ import { BlobFile as NativeBlobFile } from "./native";
3
+ export type BlobInput = {
4
+ data: Buffer | Uint8Array | null;
5
+ uri: string | null;
6
+ };
7
+ export type BlobOptions = {
8
+ /** Defaults to true. */
9
+ nullable?: boolean;
10
+ /**
11
+ * Max payload bytes kept inline in the data file. Zero is allowed. Must be a
12
+ * safe integer.
13
+ */
14
+ inlineSizeThreshold?: number;
15
+ /**
16
+ * Max payload bytes stored in a packed sidecar before a dedicated file. Must
17
+ * be a positive safe integer.
18
+ */
19
+ dedicatedSizeThreshold?: number;
20
+ /**
21
+ * Max bytes in one packed sidecar before starting another. Must be a positive
22
+ * safe integer.
23
+ */
24
+ packFileSizeThreshold?: number;
25
+ };
26
+ /**
27
+ * Declares a `lance.blob.v2` column.
28
+ *
29
+ * Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
30
+ * or {@link Table.fetchBlobFiles} to read bytes.
31
+ *
32
+ * @example
33
+ * ```ts
34
+ * import { readFile } from "node:fs/promises";
35
+ * import { Field, Int64, Schema } from "apache-arrow";
36
+ * import { blob, connect } from "@lancedb/lancedb";
37
+ *
38
+ * const db = await connect("./data");
39
+ * const video = await readFile("clip.mp4");
40
+ * const table = await db.createTable(
41
+ * "videos",
42
+ * [{ id: 1n, video }],
43
+ * {
44
+ * schema: new Schema([
45
+ * new Field("id", new Int64()),
46
+ * blob("video"),
47
+ * ]),
48
+ * },
49
+ * );
50
+ *
51
+ * const rows = await table.query().select(["id"]).withRowId().toArray();
52
+ * const rowIds = rows.map((row) => row._rowid as bigint);
53
+ * const bytes = await table.fetchBlobs("video", rowIds);
54
+ *
55
+ * const [handle] = await table.fetchBlobFiles("video", rowIds);
56
+ * const size = handle!.size();
57
+ * const header = await handle!.readRange(0n, size < 65536n ? size : 65536n);
58
+ * ```
59
+ */
60
+ export declare function blob(name: string, options?: BlobOptions): Field;
61
+ /**
62
+ * Checks for the `lance.blob.v2` extension marker. Does not validate the
63
+ * field's storage type.
64
+ */
65
+ export declare function isBlobField(field: Field): boolean;
66
+ /**
67
+ * A lazy handle to blob bytes. Create one with {@link Table.fetchBlobFiles}.
68
+ *
69
+ * @hideconstructor
70
+ */
71
+ export declare class BlobFile {
72
+ private readonly inner;
73
+ private constructor();
74
+ /** @ignore */
75
+ static fromNative(inner: NativeBlobFile): BlobFile;
76
+ /** Returns the blob size in bytes. */
77
+ size(): bigint;
78
+ /**
79
+ * Reads from the cursor to the end and advances the cursor.
80
+ *
81
+ * A second call returns an empty buffer. {@link BlobFile.readRange} does
82
+ * not move the cursor.
83
+ */
84
+ read(): Promise<Buffer>;
85
+ /**
86
+ * Reads the half-open byte range `[start, end)`.
87
+ *
88
+ * Fails when `end` is past the blob size. Does not move the cursor.
89
+ */
90
+ readRange(start: bigint, end: bigint): Promise<Buffer>;
91
+ }
92
+ export declare function coerceBlobValue(value: unknown): BlobInput | null;
@@ -0,0 +1,165 @@
1
+ "use strict";
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ // SPDX-FileCopyrightText: Copyright The LanceDB Authors
4
+ Object.defineProperty(exports, "__esModule", { value: true });
5
+ exports.BlobFile = void 0;
6
+ exports.blob = blob;
7
+ exports.isBlobField = isBlobField;
8
+ exports.coerceBlobValue = coerceBlobValue;
9
+ const apache_arrow_1 = require("apache-arrow");
10
+ const native_1 = require("./native");
11
+ const BLOB_V2_EXTENSION_NAME = "lance.blob.v2";
12
+ const INLINE_SIZE_THRESHOLD_KEY = "lance-encoding:blob-inline-size-threshold";
13
+ const DEDICATED_SIZE_THRESHOLD_KEY = "lance-encoding:blob-dedicated-size-threshold";
14
+ const PACK_FILE_SIZE_THRESHOLD_KEY = "lance-encoding:blob-pack-file-size-threshold";
15
+ /**
16
+ * Declares a `lance.blob.v2` column.
17
+ *
18
+ * Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
19
+ * or {@link Table.fetchBlobFiles} to read bytes.
20
+ *
21
+ * @example
22
+ * ```ts
23
+ * import { readFile } from "node:fs/promises";
24
+ * import { Field, Int64, Schema } from "apache-arrow";
25
+ * import { blob, connect } from "@lancedb/lancedb";
26
+ *
27
+ * const db = await connect("./data");
28
+ * const video = await readFile("clip.mp4");
29
+ * const table = await db.createTable(
30
+ * "videos",
31
+ * [{ id: 1n, video }],
32
+ * {
33
+ * schema: new Schema([
34
+ * new Field("id", new Int64()),
35
+ * blob("video"),
36
+ * ]),
37
+ * },
38
+ * );
39
+ *
40
+ * const rows = await table.query().select(["id"]).withRowId().toArray();
41
+ * const rowIds = rows.map((row) => row._rowid as bigint);
42
+ * const bytes = await table.fetchBlobs("video", rowIds);
43
+ *
44
+ * const [handle] = await table.fetchBlobFiles("video", rowIds);
45
+ * const size = handle!.size();
46
+ * const header = await handle!.readRange(0n, size < 65536n ? size : 65536n);
47
+ * ```
48
+ */
49
+ function blob(name, options = {}) {
50
+ const metadata = new Map([
51
+ ["ARROW:extension:name", BLOB_V2_EXTENSION_NAME],
52
+ ]);
53
+ setThreshold(metadata, INLINE_SIZE_THRESHOLD_KEY, "inlineSizeThreshold", options.inlineSizeThreshold, 0);
54
+ setThreshold(metadata, DEDICATED_SIZE_THRESHOLD_KEY, "dedicatedSizeThreshold", options.dedicatedSizeThreshold, 1);
55
+ setThreshold(metadata, PACK_FILE_SIZE_THRESHOLD_KEY, "packFileSizeThreshold", options.packFileSizeThreshold, 1);
56
+ return new apache_arrow_1.Field(name, new apache_arrow_1.Struct([
57
+ new apache_arrow_1.Field("data", new apache_arrow_1.LargeBinary(), true),
58
+ new apache_arrow_1.Field("uri", new apache_arrow_1.Utf8(), true),
59
+ ]), options.nullable ?? true, metadata);
60
+ }
61
+ /**
62
+ * Checks for the `lance.blob.v2` extension marker. Does not validate the
63
+ * field's storage type.
64
+ */
65
+ function isBlobField(field) {
66
+ return field.metadata?.get("ARROW:extension:name") === BLOB_V2_EXTENSION_NAME;
67
+ }
68
+ /**
69
+ * A lazy handle to blob bytes. Create one with {@link Table.fetchBlobFiles}.
70
+ *
71
+ * @hideconstructor
72
+ */
73
+ class BlobFile {
74
+ inner;
75
+ constructor(inner) {
76
+ if (!(inner instanceof native_1.BlobFile)) {
77
+ throw new Error("BlobFile handles come from Table.fetchBlobFiles");
78
+ }
79
+ this.inner = inner;
80
+ }
81
+ /** @ignore */
82
+ static fromNative(inner) {
83
+ return new BlobFile(inner);
84
+ }
85
+ /** Returns the blob size in bytes. */
86
+ size() {
87
+ return this.inner.size();
88
+ }
89
+ /**
90
+ * Reads from the cursor to the end and advances the cursor.
91
+ *
92
+ * A second call returns an empty buffer. {@link BlobFile.readRange} does
93
+ * not move the cursor.
94
+ */
95
+ read() {
96
+ return this.inner.read();
97
+ }
98
+ /**
99
+ * Reads the half-open byte range `[start, end)`.
100
+ *
101
+ * Fails when `end` is past the blob size. Does not move the cursor.
102
+ */
103
+ readRange(start, end) {
104
+ return this.inner.readRange(start, end);
105
+ }
106
+ }
107
+ exports.BlobFile = BlobFile;
108
+ function coerceBlobValue(value) {
109
+ if (value == null) {
110
+ return null;
111
+ }
112
+ if (isBlobBytes(value)) {
113
+ return { data: value, uri: null };
114
+ }
115
+ if (ArrayBuffer.isView(value)) {
116
+ throw new Error("Blob data must be Buffer or Uint8Array");
117
+ }
118
+ if (typeof value === "string") {
119
+ if (value === "") {
120
+ throw new Error("Blob uri cannot be empty");
121
+ }
122
+ return { data: null, uri: value };
123
+ }
124
+ if (typeof value === "object") {
125
+ const record = value;
126
+ if (!("data" in record) && !("uri" in record)) {
127
+ throw new Error("Blob struct values must include a 'data' or 'uri' field");
128
+ }
129
+ const uri = record.uri;
130
+ if (uri === "") {
131
+ throw new Error("Blob uri cannot be empty");
132
+ }
133
+ if (uri != null && typeof uri !== "string") {
134
+ throw new Error(`Blob uri must be a string or null, got ${typeof uri}`);
135
+ }
136
+ const data = record.data;
137
+ if (data != null && !isBlobBytes(data)) {
138
+ throw new Error("Blob data must be Buffer, Uint8Array, or null");
139
+ }
140
+ const bytes = data ?? null;
141
+ const uriValue = uri ?? null;
142
+ if ((bytes == null) === (uriValue == null)) {
143
+ throw new Error("Blob struct values must set exactly one of 'data' or 'uri'");
144
+ }
145
+ return { data: bytes, uri: uriValue };
146
+ }
147
+ throw new Error("Blob column values must be Buffer, Uint8Array, a URI string, null, or { data?, uri? }");
148
+ }
149
+ function isBlobBytes(value) {
150
+ return Buffer.isBuffer(value) || value instanceof Uint8Array;
151
+ }
152
+ function setThreshold(metadata, key, optionName, value, minimum) {
153
+ if (value === undefined) {
154
+ return;
155
+ }
156
+ if (!Number.isSafeInteger(value)) {
157
+ throw new Error(`${optionName} must be a safe integer`);
158
+ }
159
+ if (value < minimum) {
160
+ throw new Error(minimum <= 0
161
+ ? `${optionName} must be non-negative`
162
+ : `${optionName} must be positive`);
163
+ }
164
+ metadata.set(key, String(value));
165
+ }