@lancedb/lancedb 0.40.0 → 0.41.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NODEJS_THIRD_PARTY_LICENSES.md +8 -0
- package/dist/arrow.d.ts +18 -1
- package/dist/arrow.js +194 -14
- package/dist/blob.d.ts +81 -2
- package/dist/blob.js +159 -6
- package/dist/catalog.d.ts +40 -6
- package/dist/catalog.js +58 -9
- package/dist/connection.d.ts +19 -13
- package/dist/connection.js +12 -7
- package/dist/index.d.ts +6 -5
- package/dist/index.js +4 -1
- package/dist/indices.d.ts +3 -1
- package/dist/listing.d.ts +41 -0
- package/dist/listing.js +57 -0
- package/dist/materialized_view.d.ts +5 -12
- package/dist/materialized_view.js +6 -12
- package/dist/merge.d.ts +7 -7
- package/dist/merge.js +8 -8
- package/dist/native.d.ts +151 -114
- package/dist/native.js +55 -52
- package/dist/query.d.ts +94 -26
- package/dist/query.js +176 -52
- package/dist/sanitize.d.ts +19 -18
- package/dist/sanitize.js +74 -11
- package/dist/schema.js +31 -3
- package/dist/table.d.ts +65 -60
- package/dist/table.js +26 -20
- package/package.json +13 -9
|
@@ -275,6 +275,7 @@
|
|
|
275
275
|
[@types/node@20.16.10](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
|
|
276
276
|
[@types/node@20.17.9](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
|
|
277
277
|
[@types/node@22.7.4](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
|
|
278
|
+
[@types/node@25.9.8](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
|
|
278
279
|
[@types/semver@7.5.6](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
|
|
279
280
|
[@types/stack-utils@2.0.3](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
|
|
280
281
|
[@types/tmp@0.2.6](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
|
|
@@ -306,6 +307,9 @@
|
|
|
306
307
|
[apache-arrow@16.0.0](https://github.com/apache/arrow) - Apache-2.0
|
|
307
308
|
[apache-arrow@17.0.0](https://github.com/apache/arrow) - Apache-2.0
|
|
308
309
|
[apache-arrow@18.0.0](https://github.com/apache/arrow) - Apache-2.0
|
|
310
|
+
[apache-arrow@19.0.1](https://github.com/apache/arrow) - Apache-2.0
|
|
311
|
+
[apache-arrow@20.0.0](https://github.com/apache/arrow) - Apache-2.0
|
|
312
|
+
[apache-arrow@21.2.0](https://github.com/apache/arrow-js) - Apache-2.0
|
|
309
313
|
[argparse@1.0.10](https://github.com/nodeca/argparse) - MIT
|
|
310
314
|
[argparse@2.0.1](https://github.com/nodeca/argparse) - Python-2.0
|
|
311
315
|
[array-back@3.1.0](https://github.com/75lb/array-back) - MIT
|
|
@@ -348,6 +352,7 @@
|
|
|
348
352
|
[color@4.2.3](https://github.com/Qix-/color) - MIT
|
|
349
353
|
[combined-stream@1.0.8](https://github.com/felixge/node-combined-stream) - MIT
|
|
350
354
|
[command-line-args@5.2.1](https://github.com/75lb/command-line-args) - MIT
|
|
355
|
+
[command-line-args@6.0.2](https://github.com/75lb/command-line-args) - MIT
|
|
351
356
|
[command-line-usage@7.0.1](https://github.com/75lb/command-line-usage) - MIT
|
|
352
357
|
[concat-map@0.0.1](https://github.com/substack/node-concat-map) - MIT
|
|
353
358
|
[convert-source-map@2.0.0](https://github.com/thlorenz/convert-source-map) - MIT
|
|
@@ -398,12 +403,14 @@
|
|
|
398
403
|
[file-entry-cache@6.0.1](https://github.com/royriojas/file-entry-cache) - MIT
|
|
399
404
|
[fill-range@7.1.1](https://github.com/jonschlinkert/fill-range) - MIT
|
|
400
405
|
[find-replace@3.0.0](https://github.com/75lb/find-replace) - MIT
|
|
406
|
+
[find-replace@5.0.2](https://github.com/75lb/find-replace) - MIT
|
|
401
407
|
[find-up@4.1.0](https://github.com/sindresorhus/find-up) - MIT
|
|
402
408
|
[find-up@5.0.0](https://github.com/sindresorhus/find-up) - MIT
|
|
403
409
|
[flat-cache@3.2.0](https://github.com/jaredwray/flat-cache) - MIT
|
|
404
410
|
[flatbuffers@1.12.0](https://github.com/google/flatbuffers) - Apache*
|
|
405
411
|
[flatbuffers@23.5.26](https://github.com/google/flatbuffers) - Apache*
|
|
406
412
|
[flatbuffers@24.3.25](https://github.com/google/flatbuffers) - Apache-2.0
|
|
413
|
+
[flatbuffers@25.9.23](https://github.com/google/flatbuffers) - Apache-2.0
|
|
407
414
|
[flatted@3.2.9](https://github.com/WebReflection/flatted) - ISC
|
|
408
415
|
[follow-redirects@1.15.6](https://github.com/follow-redirects/follow-redirects) - MIT
|
|
409
416
|
[foreground-child@3.3.0](https://github.com/tapjs/foreground-child) - ISC
|
|
@@ -644,6 +651,7 @@
|
|
|
644
651
|
[uc.micro@2.1.0](https://github.com/markdown-it/uc.micro) - MIT
|
|
645
652
|
[undici-types@5.26.5](https://github.com/nodejs/undici) - MIT
|
|
646
653
|
[undici-types@6.19.8](https://github.com/nodejs/undici) - MIT
|
|
654
|
+
[undici-types@7.24.6](https://github.com/nodejs/undici) - MIT
|
|
647
655
|
[update-browserslist-db@1.0.13](https://github.com/browserslist/update-db) - MIT
|
|
648
656
|
[uri-js@4.4.1](https://github.com/garycourt/uri-js) - BSD-2-Clause
|
|
649
657
|
[uuid@9.0.1](https://github.com/uuidjs/uuid) - MIT
|
package/dist/arrow.d.ts
CHANGED
|
@@ -9,11 +9,22 @@ export type SchemaLike = Schema | {
|
|
|
9
9
|
get names(): unknown[];
|
|
10
10
|
};
|
|
11
11
|
export type FieldLike = Field | {
|
|
12
|
-
type: string;
|
|
12
|
+
type: string | DataTypeLike;
|
|
13
13
|
name: string;
|
|
14
14
|
nullable: boolean;
|
|
15
15
|
metadata?: Map<string, string>;
|
|
16
16
|
};
|
|
17
|
+
/**
|
|
18
|
+
* A `DataType` from any copy or version of apache-arrow.
|
|
19
|
+
*
|
|
20
|
+
* Arrow 21 brands its classes with `unique symbol` properties, so a type
|
|
21
|
+
* object from a second copy of the library no longer satisfies the `DataType`
|
|
22
|
+
* type of this one even though it is structurally identical. Inputs that only
|
|
23
|
+
* need to be sanitized accept this looser shape instead.
|
|
24
|
+
*/
|
|
25
|
+
export type DataTypeLike = DataType | {
|
|
26
|
+
readonly typeId: number;
|
|
27
|
+
};
|
|
17
28
|
/**
|
|
18
29
|
* Create an Arrow field backed by LanceDB's JSON extension type.
|
|
19
30
|
*
|
|
@@ -205,6 +216,12 @@ export declare class MakeArrowTableOptions {
|
|
|
205
216
|
* ```
|
|
206
217
|
*/
|
|
207
218
|
export declare function makeArrowTable(data: Array<Record<string, unknown>>, options?: Partial<MakeArrowTableOptions>, metadata?: Map<string, string>): ArrowTable;
|
|
219
|
+
/**
|
|
220
|
+
* Reads `Blob` / `File` values in the blob columns of `schema` into bytes so
|
|
221
|
+
* the synchronous conversion in {@link makeArrowTable} can accept them.
|
|
222
|
+
* Returns `data` itself when there is nothing to read.
|
|
223
|
+
*/
|
|
224
|
+
export declare function resolveBlobInputs(data: Array<Record<string, unknown>>, schema?: SchemaLike): Promise<Array<Record<string, unknown>>>;
|
|
208
225
|
/**
|
|
209
226
|
* Create an empty Arrow table with the provided schema
|
|
210
227
|
*/
|
package/dist/arrow.js
CHANGED
|
@@ -42,6 +42,7 @@ exports.isUnion = isUnion;
|
|
|
42
42
|
exports.isFixedSizeBinary = isFixedSizeBinary;
|
|
43
43
|
exports.isFixedSizeList = isFixedSizeList;
|
|
44
44
|
exports.makeArrowTable = makeArrowTable;
|
|
45
|
+
exports.resolveBlobInputs = resolveBlobInputs;
|
|
45
46
|
exports.makeEmptyTable = makeEmptyTable;
|
|
46
47
|
exports.convertToTable = convertToTable;
|
|
47
48
|
exports.newVectorType = newVectorType;
|
|
@@ -354,6 +355,33 @@ function makeArrowTable(data, options, metadata) {
|
|
|
354
355
|
schema = (0, sanitize_1.sanitizeSchema)(opt.schema);
|
|
355
356
|
schema = validateSchemaEmbeddings(schema, data, options?.embeddingFunction);
|
|
356
357
|
}
|
|
358
|
+
if (schema !== undefined) {
|
|
359
|
+
// Validate blob values up front and give every one the full
|
|
360
|
+
// storage shape, so inference sees the same struct whether a row
|
|
361
|
+
// passed bytes, a URI, or a partial struct.
|
|
362
|
+
data = mapBlobInputs(data, schema, (value, field, row, storageField) => {
|
|
363
|
+
if (value === undefined) {
|
|
364
|
+
return value;
|
|
365
|
+
}
|
|
366
|
+
try {
|
|
367
|
+
const normalized = (0, blob_1.coerceBlobValue)(value, storageField);
|
|
368
|
+
if (normalized === null ||
|
|
369
|
+
!(storageField.type instanceof apache_arrow_1.Struct) ||
|
|
370
|
+
storageField.type.children.length === 2) {
|
|
371
|
+
return normalized;
|
|
372
|
+
}
|
|
373
|
+
// Keep even unused range children during schema inference. Otherwise
|
|
374
|
+
// creating a range-capable table with inline data would lose them.
|
|
375
|
+
return Object.fromEntries(storageField.type.children.map((child) => [
|
|
376
|
+
child.name,
|
|
377
|
+
normalized[child.name] ?? null,
|
|
378
|
+
]));
|
|
379
|
+
}
|
|
380
|
+
catch (e) {
|
|
381
|
+
throw new Error(`Invalid value for blob field ${field} at row ${row}: ${e.message}`);
|
|
382
|
+
}
|
|
383
|
+
});
|
|
384
|
+
}
|
|
357
385
|
let schemaMetadata = schema?.metadata || new Map();
|
|
358
386
|
if (metadata !== undefined) {
|
|
359
387
|
schemaMetadata = new Map([...schemaMetadata, ...metadata]);
|
|
@@ -401,6 +429,87 @@ function containsBlobField(field) {
|
|
|
401
429
|
}
|
|
402
430
|
return (field.type.children ?? []).some((child) => containsBlobField(child));
|
|
403
431
|
}
|
|
432
|
+
/**
|
|
433
|
+
* Calls `visit` on every value that lands in a blob column of `schema`,
|
|
434
|
+
* including blobs nested in structs and lists, and replaces it with the
|
|
435
|
+
* result. Records and arrays are copied only where a value changed, so the
|
|
436
|
+
* caller's data is never mutated.
|
|
437
|
+
*/
|
|
438
|
+
function mapBlobInputs(data, schema, visit) {
|
|
439
|
+
if (!schema.fields.some(containsBlobField)) {
|
|
440
|
+
return data;
|
|
441
|
+
}
|
|
442
|
+
return data.map((record, row) => isObject(record)
|
|
443
|
+
? mapBlobFields(record, schema.fields, "", row, visit)
|
|
444
|
+
: record);
|
|
445
|
+
}
|
|
446
|
+
function mapBlobFields(record, fields, prefix, row, visit) {
|
|
447
|
+
let out;
|
|
448
|
+
for (const field of fields) {
|
|
449
|
+
if (!containsBlobField(field) || !Object.hasOwn(record, field.name)) {
|
|
450
|
+
continue;
|
|
451
|
+
}
|
|
452
|
+
const value = record[field.name];
|
|
453
|
+
const mapped = mapBlobField(value, field, `${prefix}${field.name}`, row, visit);
|
|
454
|
+
if (mapped !== value) {
|
|
455
|
+
out ??= { ...record };
|
|
456
|
+
out[field.name] = mapped;
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
return out ?? record;
|
|
460
|
+
}
|
|
461
|
+
function mapBlobField(value, field, label, row, visit) {
|
|
462
|
+
if ((0, blob_1.isBlobField)(field)) {
|
|
463
|
+
return visit(value, label, row, field);
|
|
464
|
+
}
|
|
465
|
+
if (field.type instanceof apache_arrow_1.Struct && isObject(value)) {
|
|
466
|
+
return mapBlobFields(value, field.type.children, `${label}.`, row, visit);
|
|
467
|
+
}
|
|
468
|
+
if (isList(field.type) && Array.isArray(value)) {
|
|
469
|
+
const child = field.type.children[0];
|
|
470
|
+
let out;
|
|
471
|
+
for (const [index, element] of value.entries()) {
|
|
472
|
+
const mapped = mapBlobField(element, child, `${label}[${index}]`, row, visit);
|
|
473
|
+
if (mapped !== element) {
|
|
474
|
+
out ??= [...value];
|
|
475
|
+
out[index] = mapped;
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
return out ?? value;
|
|
479
|
+
}
|
|
480
|
+
return value;
|
|
481
|
+
}
|
|
482
|
+
/**
|
|
483
|
+
* Reads `Blob` / `File` values in the blob columns of `schema` into bytes so
|
|
484
|
+
* the synchronous conversion in {@link makeArrowTable} can accept them.
|
|
485
|
+
* Returns `data` itself when there is nothing to read.
|
|
486
|
+
*/
|
|
487
|
+
async function resolveBlobInputs(data, schema) {
|
|
488
|
+
if (schema === undefined || schema === null) {
|
|
489
|
+
return data;
|
|
490
|
+
}
|
|
491
|
+
const sanitized = (0, sanitize_1.sanitizeSchema)(schema);
|
|
492
|
+
// Keyed by the Blob itself, so rows sharing one Blob (bare or as
|
|
493
|
+
// `{ data }`) read it once and share the bytes.
|
|
494
|
+
const reads = new Map();
|
|
495
|
+
mapBlobInputs(data, sanitized, (value) => {
|
|
496
|
+
const source = (0, blob_1.blobToRead)(value);
|
|
497
|
+
if (source !== undefined && !reads.has(source)) {
|
|
498
|
+
reads.set(source, source.arrayBuffer().then((buffer) => new Uint8Array(buffer)));
|
|
499
|
+
}
|
|
500
|
+
return value;
|
|
501
|
+
});
|
|
502
|
+
if (reads.size === 0) {
|
|
503
|
+
return data;
|
|
504
|
+
}
|
|
505
|
+
const bytes = new Map(await Promise.all([...reads].map(async ([source, read]) => [source, await read])));
|
|
506
|
+
return mapBlobInputs(data, sanitized, (value) => {
|
|
507
|
+
const source = (0, blob_1.blobToRead)(value);
|
|
508
|
+
return source === undefined
|
|
509
|
+
? value
|
|
510
|
+
: (0, blob_1.withBlobBytes)(value, bytes.get(source));
|
|
511
|
+
});
|
|
512
|
+
}
|
|
404
513
|
function isObject(value) {
|
|
405
514
|
return (typeof value === "object" &&
|
|
406
515
|
value !== null &&
|
|
@@ -430,7 +539,7 @@ function valueAtPath(datum, path) {
|
|
|
430
539
|
function transposeData(data, field, path = []) {
|
|
431
540
|
const valuesPath = [...path, field.name];
|
|
432
541
|
if ((0, blob_1.isBlobField)(field) && field.type instanceof apache_arrow_1.Struct) {
|
|
433
|
-
const blobRows = data.map((datum) => (0, blob_1.coerceBlobValue)(valueAtPath(datum, valuesPath)));
|
|
542
|
+
const blobRows = data.map((datum) => (0, blob_1.coerceBlobValue)(valueAtPath(datum, valuesPath), field));
|
|
434
543
|
const childVectors = field.type.children.map((child) => {
|
|
435
544
|
const values = blobRows.map((row) => row == null ? null : (row[child.name] ?? null));
|
|
436
545
|
return makeVector(values, child.type, undefined, child.nullable);
|
|
@@ -795,6 +904,7 @@ async function convertToTable(data, embeddings, makeTableOptions) {
|
|
|
795
904
|
makeTableOptions.schema.metadata?.has("embedding_functions")) {
|
|
796
905
|
processedData = ensureNestedFieldsExist(data, makeTableOptions.schema);
|
|
797
906
|
}
|
|
907
|
+
processedData = await resolveBlobInputs(processedData, makeTableOptions?.schema);
|
|
798
908
|
const table = makeArrowTable(processedData, makeTableOptions);
|
|
799
909
|
return await applyEmbeddings(table, embeddings, makeTableOptions?.schema);
|
|
800
910
|
}
|
|
@@ -835,6 +945,69 @@ async function fromRecordsToStreamBuffer(data, embeddings, schema) {
|
|
|
835
945
|
const writer = apache_arrow_1.RecordBatchStreamWriter.writeAll(table);
|
|
836
946
|
return Buffer.from(await writer.toUint8Array());
|
|
837
947
|
}
|
|
948
|
+
// `Type.Utf8View` / `Type.BinaryView` as numbers: the enum members only exist
|
|
949
|
+
// in Arrow 21+, and this module compiles against every supported release.
|
|
950
|
+
const UTF8_VIEW_TYPE_ID = 24;
|
|
951
|
+
const BINARY_VIEW_TYPE_ID = 23;
|
|
952
|
+
/**
|
|
953
|
+
* Copy a Utf8View / BinaryView `Data` into a single `Data` of `type`.
|
|
954
|
+
*/
|
|
955
|
+
function materializeViewData(child, type) {
|
|
956
|
+
const builder = (0, apache_arrow_1.makeBuilder)({ type, nullValues: [null] });
|
|
957
|
+
for (const value of new apache_arrow_1.Vector([child])) {
|
|
958
|
+
builder.append(value);
|
|
959
|
+
}
|
|
960
|
+
return builder.finish().flush();
|
|
961
|
+
}
|
|
962
|
+
/**
|
|
963
|
+
* Rebuild any top-level Utf8View / BinaryView column as Utf8 / Binary.
|
|
964
|
+
*
|
|
965
|
+
* Lance stores the view types as their offset-based equivalents anyway, so
|
|
966
|
+
* nothing is lost. Doing it here also sidesteps an Arrow JS 21 bug: its IPC
|
|
967
|
+
* writer emits a truncated views buffer for a *sliced* view array, which the
|
|
968
|
+
* Rust reader rejects with "Need at least N bytes in buffers[0]".
|
|
969
|
+
*
|
|
970
|
+
* The record batches are rebuilt positionally rather than through a
|
|
971
|
+
* `Record<string, Vector>`: JavaScript enumerates integer-like keys first, so
|
|
972
|
+
* a field named e.g. `"1"` would otherwise be paired with the wrong column.
|
|
973
|
+
*
|
|
974
|
+
* Tables without view columns are returned as-is.
|
|
975
|
+
*/
|
|
976
|
+
function materializeViewColumns(table) {
|
|
977
|
+
const replacements = new Map();
|
|
978
|
+
table.schema.fields.forEach((field, i) => {
|
|
979
|
+
if (field.type.typeId === UTF8_VIEW_TYPE_ID) {
|
|
980
|
+
replacements.set(i, new apache_arrow_1.Utf8());
|
|
981
|
+
}
|
|
982
|
+
else if (field.type.typeId === BINARY_VIEW_TYPE_ID) {
|
|
983
|
+
replacements.set(i, new apache_arrow_1.Binary());
|
|
984
|
+
}
|
|
985
|
+
});
|
|
986
|
+
if (replacements.size === 0) {
|
|
987
|
+
return table;
|
|
988
|
+
}
|
|
989
|
+
const fields = table.schema.fields.map((field, i) => {
|
|
990
|
+
const type = replacements.get(i);
|
|
991
|
+
return type === undefined
|
|
992
|
+
? field
|
|
993
|
+
: new apache_arrow_1.Field(field.name, type, field.nullable, field.metadata);
|
|
994
|
+
});
|
|
995
|
+
const schema = new apache_arrow_1.Schema(fields, table.schema.metadata);
|
|
996
|
+
const batches = table.batches.map((batch) => {
|
|
997
|
+
const children = batch.data.children.map((child, i) => {
|
|
998
|
+
const type = replacements.get(i);
|
|
999
|
+
return type === undefined ? child : materializeViewData(child, type);
|
|
1000
|
+
});
|
|
1001
|
+
const data = (0, apache_arrow_1.makeData)({
|
|
1002
|
+
type: new apache_arrow_1.Struct(fields),
|
|
1003
|
+
length: batch.numRows,
|
|
1004
|
+
nullCount: 0,
|
|
1005
|
+
children,
|
|
1006
|
+
});
|
|
1007
|
+
return new apache_arrow_1.RecordBatch(schema, data);
|
|
1008
|
+
});
|
|
1009
|
+
return new apache_arrow_1.Table(schema, batches);
|
|
1010
|
+
}
|
|
838
1011
|
/**
|
|
839
1012
|
* Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
|
|
840
1013
|
*
|
|
@@ -847,7 +1020,7 @@ async function fromTableToBuffer(table, embeddings, schema) {
|
|
|
847
1020
|
if (schema !== undefined && schema !== null) {
|
|
848
1021
|
schema = (0, sanitize_1.sanitizeSchema)(schema);
|
|
849
1022
|
}
|
|
850
|
-
const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
|
|
1023
|
+
const tableWithEmbeddings = materializeViewColumns(await applyEmbeddings(table, embeddings, schema));
|
|
851
1024
|
validateBlobSchema(tableWithEmbeddings.schema);
|
|
852
1025
|
const writer = apache_arrow_1.RecordBatchFileWriter.writeAll(tableWithEmbeddings);
|
|
853
1026
|
return Buffer.from(await writer.toUint8Array());
|
|
@@ -906,7 +1079,8 @@ async function fromRecordBatchToBuffer(batch) {
|
|
|
906
1079
|
* batch + EOS) suitable for incremental decode by `arrow_ipc::reader::StreamReader`.
|
|
907
1080
|
*/
|
|
908
1081
|
async function fromRecordBatchToStreamBuffer(batch) {
|
|
909
|
-
const
|
|
1082
|
+
const table = materializeViewColumns(new apache_arrow_1.Table([batch]));
|
|
1083
|
+
const writer = apache_arrow_1.RecordBatchStreamWriter.writeAll(table);
|
|
910
1084
|
return Buffer.from(await writer.toUint8Array());
|
|
911
1085
|
}
|
|
912
1086
|
/**
|
|
@@ -918,7 +1092,7 @@ async function fromRecordBatchToStreamBuffer(batch) {
|
|
|
918
1092
|
* `schema` is required if the table is empty
|
|
919
1093
|
*/
|
|
920
1094
|
async function fromTableToStreamBuffer(table, embeddings, schema) {
|
|
921
|
-
const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
|
|
1095
|
+
const tableWithEmbeddings = materializeViewColumns(await applyEmbeddings(table, embeddings, schema));
|
|
922
1096
|
const writer = apache_arrow_1.RecordBatchStreamWriter.writeAll(tableWithEmbeddings);
|
|
923
1097
|
return Buffer.from(await writer.toUint8Array());
|
|
924
1098
|
}
|
|
@@ -1019,7 +1193,7 @@ function ensureNestedFieldsExist(data, schema) {
|
|
|
1019
1193
|
const completeRow = {};
|
|
1020
1194
|
for (const field of schema.fields) {
|
|
1021
1195
|
if (field.name in row) {
|
|
1022
|
-
if (field
|
|
1196
|
+
if (isPlainStructField(field) &&
|
|
1023
1197
|
row[field.name] !== null &&
|
|
1024
1198
|
row[field.name] !== undefined) {
|
|
1025
1199
|
// Handle nested struct
|
|
@@ -1034,15 +1208,22 @@ function ensureNestedFieldsExist(data, schema) {
|
|
|
1034
1208
|
else {
|
|
1035
1209
|
// Keep a missing struct valid while filling each of its children with
|
|
1036
1210
|
// null. This is distinct from an explicitly null struct value.
|
|
1037
|
-
completeRow[field.name] =
|
|
1038
|
-
field.type
|
|
1039
|
-
|
|
1040
|
-
: null;
|
|
1211
|
+
completeRow[field.name] = isPlainStructField(field)
|
|
1212
|
+
? ensureStructFieldsExist({}, field.type)
|
|
1213
|
+
: null;
|
|
1041
1214
|
}
|
|
1042
1215
|
}
|
|
1043
1216
|
return completeRow;
|
|
1044
1217
|
});
|
|
1045
1218
|
}
|
|
1219
|
+
/**
|
|
1220
|
+
* Blob fields are Arrow structs, but their values are bytes, URIs, or
|
|
1221
|
+
* `{ data } | { uri }` inputs that the blob coercion in `makeArrowTable`
|
|
1222
|
+
* handles, so they must not be filled in like ordinary structs.
|
|
1223
|
+
*/
|
|
1224
|
+
function isPlainStructField(field) {
|
|
1225
|
+
return field.type.constructor.name === "Struct" && !(0, blob_1.isBlobField)(field);
|
|
1226
|
+
}
|
|
1046
1227
|
/**
|
|
1047
1228
|
* Recursively ensures that all fields in a struct type exist in the data,
|
|
1048
1229
|
* filling missing fields with null values.
|
|
@@ -1051,7 +1232,7 @@ function ensureStructFieldsExist(data, structType) {
|
|
|
1051
1232
|
const completeStruct = {};
|
|
1052
1233
|
for (const childField of structType.children) {
|
|
1053
1234
|
if (childField.name in data) {
|
|
1054
|
-
if (childField
|
|
1235
|
+
if (isPlainStructField(childField) &&
|
|
1055
1236
|
data[childField.name] !== null &&
|
|
1056
1237
|
data[childField.name] !== undefined) {
|
|
1057
1238
|
// Recursively handle nested struct
|
|
@@ -1065,10 +1246,9 @@ function ensureStructFieldsExist(data, structType) {
|
|
|
1065
1246
|
else {
|
|
1066
1247
|
// Keep a missing struct valid while filling each of its children with
|
|
1067
1248
|
// null. This is distinct from an explicitly null struct value.
|
|
1068
|
-
completeStruct[childField.name] =
|
|
1069
|
-
childField.type
|
|
1070
|
-
|
|
1071
|
-
: null;
|
|
1249
|
+
completeStruct[childField.name] = isPlainStructField(childField)
|
|
1250
|
+
? ensureStructFieldsExist({}, childField.type)
|
|
1251
|
+
: null;
|
|
1072
1252
|
}
|
|
1073
1253
|
}
|
|
1074
1254
|
return completeStruct;
|
package/dist/blob.d.ts
CHANGED
|
@@ -1,8 +1,58 @@
|
|
|
1
1
|
import { Field } from "apache-arrow";
|
|
2
2
|
import { BlobFile as NativeBlobFile } from "./native";
|
|
3
|
-
|
|
3
|
+
/**
|
|
4
|
+
* Bytes accepted for a blob value. `ArrayBuffer` is wrapped without copying.
|
|
5
|
+
* `Blob` and `File` are read with `arrayBuffer()`, which only the async write
|
|
6
|
+
* paths ({@link Connection.createTable}, {@link Table.add},
|
|
7
|
+
* {@link Table.mergeInsert}) can do.
|
|
8
|
+
*/
|
|
9
|
+
export type BlobData = Buffer | Uint8Array | ArrayBuffer | Blob;
|
|
10
|
+
/** A URI accepted for a blob value. A `URL` is stored as its `href`. */
|
|
11
|
+
export type BlobUri = string | URL;
|
|
12
|
+
/**
|
|
13
|
+
* A value accepted for a `lance.blob.v2` column (see {@link blob}).
|
|
14
|
+
*
|
|
15
|
+
* Either inline bytes, a URI pointing at external bytes, or a struct that sets
|
|
16
|
+
* exactly one of `data` or `uri`. `null` and `undefined` write a null blob.
|
|
17
|
+
* External byte ranges require both
|
|
18
|
+
* `position` (non-negative) and `size` (positive). Use safe integer numbers or
|
|
19
|
+
* `bigint` values for these unsigned 64-bit byte counts.
|
|
20
|
+
*
|
|
21
|
+
* @example
|
|
22
|
+
* ```ts
|
|
23
|
+
* import { pathToFileURL } from "node:url";
|
|
24
|
+
* import type { BlobInput } from "@lancedb/lancedb";
|
|
25
|
+
*
|
|
26
|
+
* const rows: { id: bigint; image: BlobInput }[] = [
|
|
27
|
+
* { id: 1n, image: await (await fetch(url)).arrayBuffer() },
|
|
28
|
+
* { id: 2n, image: pathToFileURL("/data/cat.png") },
|
|
29
|
+
* { id: 3n, image: { data: new Blob(["hello"]) } },
|
|
30
|
+
* ];
|
|
31
|
+
* await table.add(rows);
|
|
32
|
+
* ```
|
|
33
|
+
*/
|
|
34
|
+
export type BlobInput = BlobData | BlobUri | {
|
|
35
|
+
data: BlobData;
|
|
36
|
+
uri?: null;
|
|
37
|
+
position?: null;
|
|
38
|
+
size?: null;
|
|
39
|
+
} | {
|
|
40
|
+
data?: null;
|
|
41
|
+
uri: BlobUri;
|
|
42
|
+
position?: null;
|
|
43
|
+
size?: null;
|
|
44
|
+
} | {
|
|
45
|
+
data?: null;
|
|
46
|
+
uri: BlobUri;
|
|
47
|
+
position: number | bigint;
|
|
48
|
+
size: number | bigint;
|
|
49
|
+
} | null | undefined;
|
|
50
|
+
/** A blob value normalized to the columns of the blob struct. */
|
|
51
|
+
export type BlobValue = {
|
|
4
52
|
data: Buffer | Uint8Array | null;
|
|
5
53
|
uri: string | null;
|
|
54
|
+
position?: bigint;
|
|
55
|
+
size?: bigint;
|
|
6
56
|
};
|
|
7
57
|
export type BlobOptions = {
|
|
8
58
|
/** Defaults to true. */
|
|
@@ -25,9 +75,20 @@ export type BlobOptions = {
|
|
|
25
75
|
};
|
|
26
76
|
/**
|
|
27
77
|
* Declares a `lance.blob.v2` column.
|
|
78
|
+
* Uses nullable `data`, `uri`, `position`, and `size` fields for inline bytes,
|
|
79
|
+
* whole external files, and external byte ranges. Set both `position` and
|
|
80
|
+
* positive `size` for a range, or omit both for a whole file.
|
|
81
|
+
*
|
|
82
|
+
* Existing two-field columns remain writable. Appends and merge inserts fill
|
|
83
|
+
* missing range fields with nulls for four-field targets and drop all-null
|
|
84
|
+
* range fields for two-field targets. Non-null ranges require a four-field
|
|
85
|
+
* target. Overwrite with Arrow input replaces the table's schema; row objects
|
|
86
|
+
* are converted using the current table schema.
|
|
28
87
|
*
|
|
29
88
|
* Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
|
|
30
89
|
* or {@link Table.fetchBlobFiles} to read bytes.
|
|
90
|
+
* Blobs inside lists can be written and queried as descriptors, but cannot
|
|
91
|
+
* currently be fetched because a row ID does not identify a list element.
|
|
31
92
|
*
|
|
32
93
|
* @example
|
|
33
94
|
* ```ts
|
|
@@ -123,4 +184,22 @@ export type BlobRange = {
|
|
|
123
184
|
start: bigint;
|
|
124
185
|
end: bigint;
|
|
125
186
|
};
|
|
126
|
-
|
|
187
|
+
/**
|
|
188
|
+
* Rewrites the synchronous-only widenings of {@link BlobInput} (`ArrayBuffer`,
|
|
189
|
+
* `URL`) into `Uint8Array` and URI strings, keeping the value's shape. Other
|
|
190
|
+
* values are returned unchanged and validated later by {@link coerceBlobValue}.
|
|
191
|
+
*
|
|
192
|
+
* Throws on `Blob` / `File`, which must be read into bytes first.
|
|
193
|
+
*/
|
|
194
|
+
export declare function normalizeBlobInput(value: unknown): unknown;
|
|
195
|
+
/**
|
|
196
|
+
* Returns the `Blob` / `File` a blob value needs read, either the value itself
|
|
197
|
+
* or its `data` field, or `undefined` when there is nothing to read.
|
|
198
|
+
*/
|
|
199
|
+
export declare function blobToRead(value: unknown): Blob | undefined;
|
|
200
|
+
/**
|
|
201
|
+
* Replaces the `Blob` that {@link blobToRead} found in `value` with `bytes`,
|
|
202
|
+
* keeping the value's shape.
|
|
203
|
+
*/
|
|
204
|
+
export declare function withBlobBytes(value: unknown, bytes: Uint8Array): unknown;
|
|
205
|
+
export declare function coerceBlobValue(input: unknown, field?: Field): BlobValue | null;
|