@lancedb/lancedb 0.37.1 → 0.38.0-beta.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow.d.ts +4 -1
- package/dist/arrow.js +35 -282
- package/dist/arrow_type.d.ts +9 -0
- package/dist/arrow_type.js +29 -0
- package/dist/connection.d.ts +104 -2
- package/dist/connection.js +26 -0
- package/dist/embedding/index.d.ts +11 -1
- package/dist/embedding/index.js +24 -17
- package/dist/embedding/openai.js +3 -15
- package/dist/embedding/registry.d.ts +21 -1
- package/dist/embedding/registry.js +76 -20
- package/dist/embedding/transformers.js +3 -15
- package/dist/index.d.ts +5 -4
- package/dist/index.js +4 -1
- package/dist/materialized_view.d.ts +69 -0
- package/dist/materialized_view.js +112 -0
- package/dist/native.d.ts +121 -2
- package/dist/native.js +52 -52
- package/dist/query.d.ts +38 -2
- package/dist/query.js +154 -101
- package/dist/sanitize.js +10 -4
- package/dist/schema.d.ts +16 -0
- package/dist/schema.js +387 -0
- package/dist/table.d.ts +170 -27
- package/dist/table.js +55 -15
- package/package.json +8 -8
package/dist/arrow.d.ts
CHANGED
|
@@ -14,13 +14,16 @@ export type FieldLike = Field | {
|
|
|
14
14
|
nullable: boolean;
|
|
15
15
|
metadata?: Map<string, string>;
|
|
16
16
|
};
|
|
17
|
-
export type DataLike = import("apache-arrow").Data
|
|
17
|
+
export type DataLike = import("apache-arrow").Data | {
|
|
18
18
|
type: any;
|
|
19
19
|
length: number;
|
|
20
20
|
offset: number;
|
|
21
21
|
stride: number;
|
|
22
22
|
nullable: boolean;
|
|
23
23
|
children: DataLike[];
|
|
24
|
+
dictionary?: {
|
|
25
|
+
data: readonly DataLike[];
|
|
26
|
+
};
|
|
24
27
|
get nullCount(): number;
|
|
25
28
|
values: Buffers<any>[BufferType.DATA];
|
|
26
29
|
typeIds: Buffers<any>[BufferType.TYPE];
|
package/dist/arrow.js
CHANGED
|
@@ -56,15 +56,10 @@ exports.createEmptyTable = createEmptyTable;
|
|
|
56
56
|
exports.ensureNestedFieldsExist = ensureNestedFieldsExist;
|
|
57
57
|
exports.dataTypeToJson = dataTypeToJson;
|
|
58
58
|
const apache_arrow_1 = require("apache-arrow");
|
|
59
|
+
const arrow_type_1 = require("./arrow_type");
|
|
59
60
|
const registry_1 = require("./embedding/registry");
|
|
60
61
|
const sanitize_1 = require("./sanitize");
|
|
61
|
-
|
|
62
|
-
* Check if a field name indicates a vector column.
|
|
63
|
-
*/
|
|
64
|
-
function nameSuggestsVectorColumn(fieldName) {
|
|
65
|
-
const nameLower = fieldName.toLowerCase();
|
|
66
|
-
return nameLower.includes("vector") || nameLower.includes("embedding");
|
|
67
|
-
}
|
|
62
|
+
const schema_1 = require("./schema");
|
|
68
63
|
__exportStar(require("apache-arrow"), exports);
|
|
69
64
|
function isMultiVector(value) {
|
|
70
65
|
return Array.isArray(value) && isIntoVector(value[0]);
|
|
@@ -356,7 +351,7 @@ function makeArrowTable(data, options, metadata) {
|
|
|
356
351
|
return new apache_arrow_1.Table(schema);
|
|
357
352
|
}
|
|
358
353
|
}
|
|
359
|
-
let inferredSchema = inferSchema(data, schema, opt);
|
|
354
|
+
let inferredSchema = (0, schema_1.inferSchema)(data, schema, opt);
|
|
360
355
|
inferredSchema = new apache_arrow_1.Schema(inferredSchema.fields, schemaMetadata);
|
|
361
356
|
const finalColumns = {};
|
|
362
357
|
for (const field of inferredSchema.fields) {
|
|
@@ -364,95 +359,6 @@ function makeArrowTable(data, options, metadata) {
|
|
|
364
359
|
}
|
|
365
360
|
return new apache_arrow_1.Table(inferredSchema, finalColumns);
|
|
366
361
|
}
|
|
367
|
-
function inferSchema(data, schema, opts) {
|
|
368
|
-
// We will collect all fields we see in the data.
|
|
369
|
-
const pathTree = new PathTree();
|
|
370
|
-
for (const [rowI, row] of data.entries()) {
|
|
371
|
-
for (const [path, value] of rowPathsAndValues(row)) {
|
|
372
|
-
if (!pathTree.has(path)) {
|
|
373
|
-
// First time seeing this field.
|
|
374
|
-
if (schema !== undefined) {
|
|
375
|
-
const field = getFieldForPath(schema, path);
|
|
376
|
-
if (field === undefined) {
|
|
377
|
-
throw new Error(`Found field not in schema: ${path.join(".")} at row ${rowI}`);
|
|
378
|
-
}
|
|
379
|
-
else {
|
|
380
|
-
pathTree.set(path, field.type);
|
|
381
|
-
}
|
|
382
|
-
}
|
|
383
|
-
else {
|
|
384
|
-
const inferredType = inferType(value, path, opts);
|
|
385
|
-
if (inferredType === undefined) {
|
|
386
|
-
throw new Error(`Failed to infer data type for field ${path.join(".")} at row ${rowI}. \
|
|
387
|
-
Consider providing an explicit schema.`);
|
|
388
|
-
}
|
|
389
|
-
pathTree.set(path, inferredType);
|
|
390
|
-
}
|
|
391
|
-
}
|
|
392
|
-
else if (schema === undefined) {
|
|
393
|
-
const currentType = pathTree.get(path);
|
|
394
|
-
const newType = inferType(value, path, opts);
|
|
395
|
-
if (currentType !== newType) {
|
|
396
|
-
new Error(`Failed to infer schema for data. Previously inferred type \
|
|
397
|
-
${currentType} but found ${newType} at row ${rowI}. Consider \
|
|
398
|
-
providing an explicit schema.`);
|
|
399
|
-
}
|
|
400
|
-
}
|
|
401
|
-
}
|
|
402
|
-
}
|
|
403
|
-
if (schema === undefined) {
|
|
404
|
-
function fieldsFromPathTree(pathTree) {
|
|
405
|
-
const fields = [];
|
|
406
|
-
for (const [name, value] of pathTree.map.entries()) {
|
|
407
|
-
if (value instanceof PathTree) {
|
|
408
|
-
const children = fieldsFromPathTree(value);
|
|
409
|
-
fields.push(new apache_arrow_1.Field(name, new apache_arrow_1.Struct(children), true));
|
|
410
|
-
}
|
|
411
|
-
else {
|
|
412
|
-
fields.push(new apache_arrow_1.Field(name, value, true));
|
|
413
|
-
}
|
|
414
|
-
}
|
|
415
|
-
return fields;
|
|
416
|
-
}
|
|
417
|
-
const fields = fieldsFromPathTree(pathTree);
|
|
418
|
-
return new apache_arrow_1.Schema(fields);
|
|
419
|
-
}
|
|
420
|
-
else {
|
|
421
|
-
function takeMatchingFields(fields, pathTree) {
|
|
422
|
-
const outFields = [];
|
|
423
|
-
for (const field of fields) {
|
|
424
|
-
if (pathTree.map.has(field.name)) {
|
|
425
|
-
const value = pathTree.get([field.name]);
|
|
426
|
-
if (value instanceof PathTree) {
|
|
427
|
-
const struct = field.type;
|
|
428
|
-
const children = takeMatchingFields(struct.children, value);
|
|
429
|
-
outFields.push(new apache_arrow_1.Field(field.name, new apache_arrow_1.Struct(children), field.nullable));
|
|
430
|
-
}
|
|
431
|
-
else {
|
|
432
|
-
outFields.push(new apache_arrow_1.Field(field.name, value, field.nullable));
|
|
433
|
-
}
|
|
434
|
-
}
|
|
435
|
-
}
|
|
436
|
-
return outFields;
|
|
437
|
-
}
|
|
438
|
-
const fields = takeMatchingFields(schema.fields, pathTree);
|
|
439
|
-
return new apache_arrow_1.Schema(fields);
|
|
440
|
-
}
|
|
441
|
-
}
|
|
442
|
-
function* rowPathsAndValues(row, basePath = []) {
|
|
443
|
-
for (const [key, value] of Object.entries(row)) {
|
|
444
|
-
if (isObject(value)) {
|
|
445
|
-
yield* rowPathsAndValues(value, [...basePath, key]);
|
|
446
|
-
}
|
|
447
|
-
else {
|
|
448
|
-
// Skip undefined values - they should be treated the same as missing fields
|
|
449
|
-
// for embedding function purposes
|
|
450
|
-
if (value !== undefined) {
|
|
451
|
-
yield [[...basePath, key], value];
|
|
452
|
-
}
|
|
453
|
-
}
|
|
454
|
-
}
|
|
455
|
-
}
|
|
456
362
|
function isObject(value) {
|
|
457
363
|
return (typeof value === "object" &&
|
|
458
364
|
value !== null &&
|
|
@@ -464,179 +370,42 @@ function isObject(value) {
|
|
|
464
370
|
!(value instanceof Buffer) &&
|
|
465
371
|
!ArrayBuffer.isView(value));
|
|
466
372
|
}
|
|
467
|
-
function
|
|
468
|
-
let current =
|
|
373
|
+
function valueAtPath(datum, path) {
|
|
374
|
+
let current = datum;
|
|
469
375
|
for (const key of path) {
|
|
470
|
-
if (current
|
|
471
|
-
|
|
472
|
-
if (field === undefined) {
|
|
473
|
-
return undefined;
|
|
474
|
-
}
|
|
475
|
-
current = field;
|
|
476
|
-
}
|
|
477
|
-
else if (current instanceof apache_arrow_1.Field && apache_arrow_1.DataType.isStruct(current.type)) {
|
|
478
|
-
const struct = current.type;
|
|
479
|
-
const field = struct.children.find((f) => f.name === key);
|
|
480
|
-
if (field === undefined) {
|
|
481
|
-
return undefined;
|
|
482
|
-
}
|
|
483
|
-
current = field;
|
|
376
|
+
if (current == null) {
|
|
377
|
+
return null;
|
|
484
378
|
}
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
}
|
|
488
|
-
}
|
|
489
|
-
if (current instanceof apache_arrow_1.Field) {
|
|
490
|
-
return current;
|
|
491
|
-
}
|
|
492
|
-
else {
|
|
493
|
-
return undefined;
|
|
494
|
-
}
|
|
495
|
-
}
|
|
496
|
-
/**
|
|
497
|
-
* Try to infer which Arrow type to use for a given value.
|
|
498
|
-
*
|
|
499
|
-
* May return undefined if the type cannot be inferred.
|
|
500
|
-
*/
|
|
501
|
-
function inferType(value, path, opts) {
|
|
502
|
-
if (typeof value === "bigint") {
|
|
503
|
-
return new apache_arrow_1.Int64();
|
|
504
|
-
}
|
|
505
|
-
else if (typeof value === "number") {
|
|
506
|
-
// Even if it's an integer, it's safer to assume Float64. Users can
|
|
507
|
-
// always provide an explicit schema or use BigInt if they mean integer.
|
|
508
|
-
return new apache_arrow_1.Float64();
|
|
509
|
-
}
|
|
510
|
-
else if (typeof value === "string") {
|
|
511
|
-
if (opts.dictionaryEncodeStrings) {
|
|
512
|
-
return new apache_arrow_1.Dictionary(new apache_arrow_1.Utf8(), new apache_arrow_1.Int32());
|
|
379
|
+
if (isObject(current) && (Object.hasOwn(current, key) || key in current)) {
|
|
380
|
+
current = current[key];
|
|
513
381
|
}
|
|
514
382
|
else {
|
|
515
|
-
return new apache_arrow_1.Utf8();
|
|
516
|
-
}
|
|
517
|
-
}
|
|
518
|
-
else if (typeof value === "boolean") {
|
|
519
|
-
return new apache_arrow_1.Bool();
|
|
520
|
-
}
|
|
521
|
-
else if (value instanceof Buffer) {
|
|
522
|
-
return new apache_arrow_1.Binary();
|
|
523
|
-
}
|
|
524
|
-
else if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
|
|
525
|
-
const info = typedArrayToArrowType(value);
|
|
526
|
-
if (info !== undefined) {
|
|
527
|
-
const child = new apache_arrow_1.Field("item", info.elementType, true);
|
|
528
|
-
return new apache_arrow_1.FixedSizeList(info.length, child);
|
|
529
|
-
}
|
|
530
|
-
return undefined;
|
|
531
|
-
}
|
|
532
|
-
else if (Array.isArray(value)) {
|
|
533
|
-
if (value.length === 0) {
|
|
534
|
-
return undefined; // Without any values we can't infer the type
|
|
535
|
-
}
|
|
536
|
-
if (path.length === 1 && Object.hasOwn(opts.vectorColumns, path[0])) {
|
|
537
|
-
const floatType = (0, sanitize_1.sanitizeType)(opts.vectorColumns[path[0]].type);
|
|
538
|
-
return new apache_arrow_1.FixedSizeList(value.length, new apache_arrow_1.Field("item", floatType, true));
|
|
539
|
-
}
|
|
540
|
-
const valueType = inferType(value[0], path, opts);
|
|
541
|
-
if (valueType === undefined) {
|
|
542
383
|
return undefined;
|
|
543
384
|
}
|
|
544
|
-
// Try to automatically detect embedding columns.
|
|
545
|
-
if (nameSuggestsVectorColumn(path[path.length - 1])) {
|
|
546
|
-
// Check if value is a Uint8Array for integer vector type determination
|
|
547
|
-
if (value instanceof Uint8Array) {
|
|
548
|
-
// For integer vectors, we default to Uint8 (matching Python implementation)
|
|
549
|
-
const child = new apache_arrow_1.Field("item", new apache_arrow_1.Uint8(), true);
|
|
550
|
-
return new apache_arrow_1.FixedSizeList(value.length, child);
|
|
551
|
-
}
|
|
552
|
-
else {
|
|
553
|
-
// For float vectors, we default to Float32
|
|
554
|
-
const child = new apache_arrow_1.Field("item", new apache_arrow_1.Float32(), true);
|
|
555
|
-
return new apache_arrow_1.FixedSizeList(value.length, child);
|
|
556
|
-
}
|
|
557
|
-
}
|
|
558
|
-
else {
|
|
559
|
-
const child = new apache_arrow_1.Field("item", valueType, true);
|
|
560
|
-
return new apache_arrow_1.List(child);
|
|
561
|
-
}
|
|
562
|
-
}
|
|
563
|
-
else {
|
|
564
|
-
// TODO: timestamp
|
|
565
|
-
return undefined;
|
|
566
|
-
}
|
|
567
|
-
}
|
|
568
|
-
class PathTree {
|
|
569
|
-
map;
|
|
570
|
-
constructor(entries) {
|
|
571
|
-
this.map = new Map();
|
|
572
|
-
if (entries !== undefined) {
|
|
573
|
-
for (const [path, value] of entries) {
|
|
574
|
-
this.set(path, value);
|
|
575
|
-
}
|
|
576
|
-
}
|
|
577
|
-
}
|
|
578
|
-
has(path) {
|
|
579
|
-
let ref = this;
|
|
580
|
-
for (const part of path) {
|
|
581
|
-
if (!(ref instanceof PathTree) || !ref.map.has(part)) {
|
|
582
|
-
return false;
|
|
583
|
-
}
|
|
584
|
-
ref = ref.map.get(part);
|
|
585
|
-
}
|
|
586
|
-
return true;
|
|
587
|
-
}
|
|
588
|
-
get(path) {
|
|
589
|
-
let ref = this;
|
|
590
|
-
for (const part of path) {
|
|
591
|
-
if (!(ref instanceof PathTree) || !ref.map.has(part)) {
|
|
592
|
-
return undefined;
|
|
593
|
-
}
|
|
594
|
-
ref = ref.map.get(part);
|
|
595
|
-
}
|
|
596
|
-
return ref;
|
|
597
|
-
}
|
|
598
|
-
set(path, value) {
|
|
599
|
-
let ref = this;
|
|
600
|
-
for (const part of path.slice(0, path.length - 1)) {
|
|
601
|
-
if (!ref.map.has(part)) {
|
|
602
|
-
ref.map.set(part, new PathTree());
|
|
603
|
-
}
|
|
604
|
-
ref = ref.map.get(part);
|
|
605
|
-
}
|
|
606
|
-
ref.map.set(path[path.length - 1], value);
|
|
607
385
|
}
|
|
386
|
+
return current;
|
|
608
387
|
}
|
|
609
388
|
function transposeData(data, field, path = []) {
|
|
389
|
+
const valuesPath = [...path, field.name];
|
|
390
|
+
const values = data.map((datum) => valueAtPath(datum, valuesPath));
|
|
610
391
|
if (field.type instanceof apache_arrow_1.Struct) {
|
|
611
392
|
const childFields = field.type.children;
|
|
612
|
-
const fullPath = [...path, field.name];
|
|
613
393
|
const childVectors = childFields.map((child) => {
|
|
614
|
-
return transposeData(data, child,
|
|
394
|
+
return transposeData(data, child, valuesPath);
|
|
615
395
|
});
|
|
396
|
+
const nullCount = values.filter((value) => value === null).length;
|
|
616
397
|
const structData = (0, apache_arrow_1.makeData)({
|
|
617
398
|
type: field.type,
|
|
399
|
+
length: values.length,
|
|
400
|
+
nullCount,
|
|
401
|
+
nullBitmap: nullCount > 0
|
|
402
|
+
? apache_arrow_1.util.packBools(values.map((value) => value !== null))
|
|
403
|
+
: undefined,
|
|
618
404
|
children: childVectors,
|
|
619
405
|
});
|
|
620
406
|
return (0, apache_arrow_1.makeVector)(structData);
|
|
621
407
|
}
|
|
622
408
|
else {
|
|
623
|
-
const valuesPath = [...path, field.name];
|
|
624
|
-
const values = data.map((datum) => {
|
|
625
|
-
let current = datum;
|
|
626
|
-
for (const key of valuesPath) {
|
|
627
|
-
if (current == null) {
|
|
628
|
-
return null;
|
|
629
|
-
}
|
|
630
|
-
if (isObject(current) &&
|
|
631
|
-
(Object.hasOwn(current, key) || key in current)) {
|
|
632
|
-
current = current[key];
|
|
633
|
-
}
|
|
634
|
-
else {
|
|
635
|
-
return null;
|
|
636
|
-
}
|
|
637
|
-
}
|
|
638
|
-
return current;
|
|
639
|
-
});
|
|
640
409
|
return makeVector(values, field.type, undefined, field.nullable);
|
|
641
410
|
}
|
|
642
411
|
}
|
|
@@ -673,29 +442,6 @@ function makeListVector(lists) {
|
|
|
673
442
|
}
|
|
674
443
|
return listBuilder.finish().toVector();
|
|
675
444
|
}
|
|
676
|
-
/**
|
|
677
|
-
* Map a JS TypedArray instance to the corresponding Arrow element DataType
|
|
678
|
-
* and its length. Returns undefined if the value is not a recognized TypedArray.
|
|
679
|
-
*/
|
|
680
|
-
function typedArrayToArrowType(value) {
|
|
681
|
-
if (value instanceof Float32Array)
|
|
682
|
-
return { elementType: new apache_arrow_1.Float32(), length: value.length };
|
|
683
|
-
if (value instanceof Float64Array)
|
|
684
|
-
return { elementType: new apache_arrow_1.Float64(), length: value.length };
|
|
685
|
-
if (value instanceof Uint8Array)
|
|
686
|
-
return { elementType: new apache_arrow_1.Uint8(), length: value.length };
|
|
687
|
-
if (value instanceof Uint16Array)
|
|
688
|
-
return { elementType: new apache_arrow_1.Uint16(), length: value.length };
|
|
689
|
-
if (value instanceof Uint32Array)
|
|
690
|
-
return { elementType: new apache_arrow_1.Uint32(), length: value.length };
|
|
691
|
-
if (value instanceof Int8Array)
|
|
692
|
-
return { elementType: new apache_arrow_1.Int8(), length: value.length };
|
|
693
|
-
if (value instanceof Int16Array)
|
|
694
|
-
return { elementType: new apache_arrow_1.Int16(), length: value.length };
|
|
695
|
-
if (value instanceof Int32Array)
|
|
696
|
-
return { elementType: new apache_arrow_1.Int32(), length: value.length };
|
|
697
|
-
return undefined;
|
|
698
|
-
}
|
|
699
445
|
/** Helper function to convert an Array of JS values to an Arrow Vector */
|
|
700
446
|
function makeVector(values, type, stringAsDictionary, nullable) {
|
|
701
447
|
if (type !== undefined) {
|
|
@@ -758,7 +504,7 @@ function makeVector(values, type, stringAsDictionary, nullable) {
|
|
|
758
504
|
throw Error("makeVector cannot infer the type if all values are null or undefined");
|
|
759
505
|
}
|
|
760
506
|
if (ArrayBuffer.isView(sampleValue) && !(sampleValue instanceof DataView)) {
|
|
761
|
-
const info = typedArrayToArrowType(sampleValue);
|
|
507
|
+
const info = (0, arrow_type_1.typedArrayToArrowType)(sampleValue);
|
|
762
508
|
if (info !== undefined) {
|
|
763
509
|
const fslType = new apache_arrow_1.FixedSizeList(info.length, new apache_arrow_1.Field("item", info.elementType, true));
|
|
764
510
|
return vectorFromArray(values, fslType);
|
|
@@ -793,7 +539,7 @@ async function applyEmbeddingsFromMetadata(table, schema) {
|
|
|
793
539
|
]));
|
|
794
540
|
for (const functionEntry of functions.values()) {
|
|
795
541
|
const sourceColumn = columns[functionEntry.sourceColumn];
|
|
796
|
-
const destColumn = functionEntry.vectorColumn
|
|
542
|
+
const destColumn = functionEntry.vectorColumn;
|
|
797
543
|
if (sourceColumn === undefined) {
|
|
798
544
|
throw new Error(`Cannot apply embedding function because the source column '${functionEntry.sourceColumn}' was not present in the data`);
|
|
799
545
|
}
|
|
@@ -1128,9 +874,8 @@ function validateSchemaEmbeddings(schema, data, embeddings) {
|
|
|
1128
874
|
let hasEmbeddingFunction = false;
|
|
1129
875
|
// Check schema metadata for embedding functions
|
|
1130
876
|
if (schema.metadata.has("embedding_functions")) {
|
|
1131
|
-
const
|
|
1132
|
-
|
|
1133
|
-
if (embeddings.find((f) => f["vectorColumn"] === field.name)) {
|
|
877
|
+
const entries = (0, registry_1.parseEmbeddingMetadata)(schema.metadata.get("embedding_functions"));
|
|
878
|
+
if (entries.some((f) => f.vectorColumn === field.name)) {
|
|
1134
879
|
hasEmbeddingFunction = true;
|
|
1135
880
|
}
|
|
1136
881
|
}
|
|
@@ -1190,8 +935,12 @@ function ensureNestedFieldsExist(data, schema) {
|
|
|
1190
935
|
}
|
|
1191
936
|
}
|
|
1192
937
|
else {
|
|
1193
|
-
//
|
|
1194
|
-
|
|
938
|
+
// Keep a missing struct valid while filling each of its children with
|
|
939
|
+
// null. This is distinct from an explicitly null struct value.
|
|
940
|
+
completeRow[field.name] =
|
|
941
|
+
field.type.constructor.name === "Struct"
|
|
942
|
+
? ensureStructFieldsExist({}, field.type)
|
|
943
|
+
: null;
|
|
1195
944
|
}
|
|
1196
945
|
}
|
|
1197
946
|
return completeRow;
|
|
@@ -1217,8 +966,12 @@ function ensureStructFieldsExist(data, structType) {
|
|
|
1217
966
|
}
|
|
1218
967
|
}
|
|
1219
968
|
else {
|
|
1220
|
-
//
|
|
1221
|
-
|
|
969
|
+
// Keep a missing struct valid while filling each of its children with
|
|
970
|
+
// null. This is distinct from an explicitly null struct value.
|
|
971
|
+
completeStruct[childField.name] =
|
|
972
|
+
childField.type.constructor.name === "Struct"
|
|
973
|
+
? ensureStructFieldsExist({}, childField.type)
|
|
974
|
+
: null;
|
|
1222
975
|
}
|
|
1223
976
|
}
|
|
1224
977
|
return completeStruct;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { type DataType } from "apache-arrow";
|
|
2
|
+
/**
|
|
3
|
+
* Map a JS TypedArray instance to the corresponding Arrow element type and
|
|
4
|
+
* length. Returns undefined when the view is not a supported TypedArray.
|
|
5
|
+
*/
|
|
6
|
+
export declare function typedArrayToArrowType(value: ArrayBufferView): {
|
|
7
|
+
elementType: DataType;
|
|
8
|
+
length: number;
|
|
9
|
+
} | undefined;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
4
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
+
exports.typedArrayToArrowType = typedArrayToArrowType;
|
|
6
|
+
const apache_arrow_1 = require("apache-arrow");
|
|
7
|
+
/**
|
|
8
|
+
* Map a JS TypedArray instance to the corresponding Arrow element type and
|
|
9
|
+
* length. Returns undefined when the view is not a supported TypedArray.
|
|
10
|
+
*/
|
|
11
|
+
function typedArrayToArrowType(value) {
|
|
12
|
+
if (value instanceof Float32Array)
|
|
13
|
+
return { elementType: new apache_arrow_1.Float32(), length: value.length };
|
|
14
|
+
if (value instanceof Float64Array)
|
|
15
|
+
return { elementType: new apache_arrow_1.Float64(), length: value.length };
|
|
16
|
+
if (value instanceof Uint8Array)
|
|
17
|
+
return { elementType: new apache_arrow_1.Uint8(), length: value.length };
|
|
18
|
+
if (value instanceof Uint16Array)
|
|
19
|
+
return { elementType: new apache_arrow_1.Uint16(), length: value.length };
|
|
20
|
+
if (value instanceof Uint32Array)
|
|
21
|
+
return { elementType: new apache_arrow_1.Uint32(), length: value.length };
|
|
22
|
+
if (value instanceof Int8Array)
|
|
23
|
+
return { elementType: new apache_arrow_1.Int8(), length: value.length };
|
|
24
|
+
if (value instanceof Int16Array)
|
|
25
|
+
return { elementType: new apache_arrow_1.Int16(), length: value.length };
|
|
26
|
+
if (value instanceof Int32Array)
|
|
27
|
+
return { elementType: new apache_arrow_1.Int32(), length: value.length };
|
|
28
|
+
return undefined;
|
|
29
|
+
}
|
package/dist/connection.d.ts
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { Data, SchemaLike, TableLike } from "./arrow";
|
|
2
2
|
import { Table as ArrowTable } from "./arrow";
|
|
3
3
|
import { EmbeddingFunctionConfig } from "./embedding/registry";
|
|
4
|
+
import { MaterializedView, MaterializedViewSelect } from "./materialized_view";
|
|
4
5
|
import { Connection as LanceDbConnection } from "./native";
|
|
5
|
-
import type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse, Job, JobDescription, JobInfo, ListNamespacesResponse } from "./native";
|
|
6
|
-
export type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse, ListNamespacesResponse, };
|
|
6
|
+
import type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse, Job, JobDescription, JobInfo, ListNamespacesResponse, ListTablesResponse } from "./native";
|
|
7
|
+
export type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse, ListNamespacesResponse, ListTablesResponse, };
|
|
7
8
|
import { Table } from "./table";
|
|
8
9
|
export interface CreateTableOptions {
|
|
9
10
|
/**
|
|
@@ -92,6 +93,10 @@ export interface OpenTableOptions {
|
|
|
92
93
|
*/
|
|
93
94
|
indexCacheSize?: number;
|
|
94
95
|
}
|
|
96
|
+
/**
|
|
97
|
+
* @deprecated Use {@link ListTablesOptions} with {@link Connection.listTables}
|
|
98
|
+
* instead.
|
|
99
|
+
*/
|
|
95
100
|
export interface TableNamesOptions {
|
|
96
101
|
/**
|
|
97
102
|
* If present, only return names that come lexicographically after the
|
|
@@ -104,6 +109,23 @@ export interface TableNamesOptions {
|
|
|
104
109
|
/** An optional limit to the number of results to return. */
|
|
105
110
|
limit?: number;
|
|
106
111
|
}
|
|
112
|
+
export interface ListTablesOptions {
|
|
113
|
+
/**
|
|
114
|
+
* Token from a previous response, to resume listing where it left off.
|
|
115
|
+
*
|
|
116
|
+
* The token is opaque: it carries whatever the database needs to resume, and
|
|
117
|
+
* callers should not construct or interpret one.
|
|
118
|
+
*/
|
|
119
|
+
pageToken?: string;
|
|
120
|
+
/**
|
|
121
|
+
* An upper bound on how many tables to return.
|
|
122
|
+
*
|
|
123
|
+
* A page may hold fewer than this and still not be the last one, so keep
|
|
124
|
+
* going while the response carries a page token rather than while pages are
|
|
125
|
+
* full.
|
|
126
|
+
*/
|
|
127
|
+
limit?: number;
|
|
128
|
+
}
|
|
107
129
|
export interface ListNamespacesOptions {
|
|
108
130
|
/** Token from a previous response for pagination. */
|
|
109
131
|
pageToken?: string;
|
|
@@ -177,6 +199,7 @@ export declare abstract class Connection {
|
|
|
177
199
|
* @param {Partial<TableNamesOptions>} options - options to control the
|
|
178
200
|
* paging / start point (backwards compatibility)
|
|
179
201
|
*
|
|
202
|
+
* @deprecated Use {@link Connection.listTables} instead.
|
|
180
203
|
*/
|
|
181
204
|
abstract tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
|
|
182
205
|
/**
|
|
@@ -187,14 +210,77 @@ export declare abstract class Connection {
|
|
|
187
210
|
* @param {Partial<TableNamesOptions>} options - options to control the
|
|
188
211
|
* paging / start point
|
|
189
212
|
*
|
|
213
|
+
* @deprecated Use {@link Connection.listTables} instead.
|
|
190
214
|
*/
|
|
191
215
|
abstract tableNames(namespacePath?: string[], options?: Partial<TableNamesOptions>): Promise<string[]>;
|
|
216
|
+
/**
|
|
217
|
+
* List a page of the tables in this database.
|
|
218
|
+
*
|
|
219
|
+
* To retrieve the tables after the page, pass the `pageToken` the response
|
|
220
|
+
* carries back in. A page can be shorter than `limit` without being the last
|
|
221
|
+
* one, so walk until a response carries no page token:
|
|
222
|
+
*
|
|
223
|
+
* ```ts
|
|
224
|
+
* const names = [];
|
|
225
|
+
* let pageToken = undefined;
|
|
226
|
+
* do {
|
|
227
|
+
* const page = await conn.listTables({ pageToken, limit: 100 });
|
|
228
|
+
* names.push(...page.tables);
|
|
229
|
+
* pageToken = page.pageToken;
|
|
230
|
+
* } while (pageToken);
|
|
231
|
+
* ```
|
|
232
|
+
*
|
|
233
|
+
* @param {Partial<ListTablesOptions>} options - Pagination options
|
|
234
|
+
* (`pageToken`, `limit`).
|
|
235
|
+
* @returns {Promise<ListTablesResponse>} A page of table names and an
|
|
236
|
+
* optional token for the tables after it.
|
|
237
|
+
*/
|
|
238
|
+
abstract listTables(options?: Partial<ListTablesOptions>): Promise<ListTablesResponse>;
|
|
239
|
+
/**
|
|
240
|
+
* List a page of the tables in this database.
|
|
241
|
+
*
|
|
242
|
+
* @param {string[]} namespacePath - The namespace path to list tables from
|
|
243
|
+
* (defaults to root namespace)
|
|
244
|
+
* @param {Partial<ListTablesOptions>} options - Pagination options
|
|
245
|
+
* (`pageToken`, `limit`).
|
|
246
|
+
* @returns {Promise<ListTablesResponse>} A page of table names and an
|
|
247
|
+
* optional token for the tables after it.
|
|
248
|
+
*/
|
|
249
|
+
abstract listTables(namespacePath?: string[], options?: Partial<ListTablesOptions>): Promise<ListTablesResponse>;
|
|
192
250
|
/**
|
|
193
251
|
* Open a table in the database.
|
|
194
252
|
* @param {string} name - The name of the table
|
|
195
253
|
* @param {string[]} namespacePath - The namespace path of the table (defaults to root namespace)
|
|
196
254
|
* @param {Partial<OpenTableOptions>} options - Additional options
|
|
197
255
|
*/
|
|
256
|
+
/**
|
|
257
|
+
* Define a materialized view named `name` over the table `source`.
|
|
258
|
+
*
|
|
259
|
+
* The view is created empty, with the query recorded in its schema
|
|
260
|
+
* metadata; `view.refresh()` computes the rows. The view is a normal
|
|
261
|
+
* table: it can be queried, indexed and searched, and it appears in
|
|
262
|
+
* `tableNames`. The source table must have stable row ids (create it with
|
|
263
|
+
* the `newTableEnableStableRowIds` storage option); they keep the view's
|
|
264
|
+
* provenance valid across source compactions and cannot be enabled after
|
|
265
|
+
* a table exists. Local databases only.
|
|
266
|
+
*/
|
|
267
|
+
abstract createMaterializedView(name: string, source: string, options?: {
|
|
268
|
+
select?: MaterializedViewSelect;
|
|
269
|
+
where?: string;
|
|
270
|
+
limit?: number;
|
|
271
|
+
}): Promise<MaterializedView>;
|
|
272
|
+
/**
|
|
273
|
+
* Open the materialized view named `name`.
|
|
274
|
+
*
|
|
275
|
+
* Rejects a table that exists but is not a materialized view.
|
|
276
|
+
*/
|
|
277
|
+
abstract openMaterializedView(name: string): Promise<MaterializedView>;
|
|
278
|
+
/**
|
|
279
|
+
* The names of the materialized views in this database.
|
|
280
|
+
*
|
|
281
|
+
* Found by reading every table's schema, so this costs an open per table.
|
|
282
|
+
*/
|
|
283
|
+
abstract listMaterializedViews(): Promise<string[]>;
|
|
198
284
|
abstract openTable(name: string, namespacePath?: string[], options?: Partial<OpenTableOptions>): Promise<Table>;
|
|
199
285
|
/**
|
|
200
286
|
* Creates a new Table and initialize it with new data.
|
|
@@ -246,6 +332,13 @@ export declare abstract class Connection {
|
|
|
246
332
|
* @param {string[]} namespacePath The namespace path of the table (defaults to root namespace).
|
|
247
333
|
*/
|
|
248
334
|
abstract dropTable(name: string, namespacePath?: string[]): Promise<void>;
|
|
335
|
+
/**
|
|
336
|
+
* Start dropping a table and return its cleanup job.
|
|
337
|
+
*
|
|
338
|
+
* The table may become unavailable before its data files are removed. Wait
|
|
339
|
+
* on the returned job to know when cleanup has finished.
|
|
340
|
+
*/
|
|
341
|
+
abstract dropTableAsync(name: string, namespacePath?: string[]): Promise<Job>;
|
|
249
342
|
/**
|
|
250
343
|
* Drop all tables in the database.
|
|
251
344
|
* @param {string[]} namespacePath The namespace path to drop tables from (defaults to root namespace).
|
|
@@ -373,6 +466,14 @@ export declare class LocalConnection extends Connection {
|
|
|
373
466
|
close(): void;
|
|
374
467
|
display(): string;
|
|
375
468
|
tableNames(namespacePathOrOptions?: string[] | Partial<TableNamesOptions>, options?: Partial<TableNamesOptions>): Promise<string[]>;
|
|
469
|
+
createMaterializedView(name: string, source: string, options?: {
|
|
470
|
+
select?: MaterializedViewSelect;
|
|
471
|
+
where?: string;
|
|
472
|
+
limit?: number;
|
|
473
|
+
}): Promise<MaterializedView>;
|
|
474
|
+
openMaterializedView(name: string): Promise<MaterializedView>;
|
|
475
|
+
listMaterializedViews(): Promise<string[]>;
|
|
476
|
+
listTables(namespacePathOrOptions?: string[] | Partial<ListTablesOptions>, options?: Partial<ListTablesOptions>): Promise<ListTablesResponse>;
|
|
376
477
|
openTable(name: string, namespacePath?: string[], options?: Partial<OpenTableOptions>): Promise<Table>;
|
|
377
478
|
cloneTable(targetTableName: string, sourceUri: string, options?: {
|
|
378
479
|
targetNamespacePath?: string[];
|
|
@@ -388,6 +489,7 @@ export declare class LocalConnection extends Connection {
|
|
|
388
489
|
private _createTableImpl;
|
|
389
490
|
createEmptyTable(name: string, schema: import("./arrow").SchemaLike, namespacePathOrOptions?: string[] | Partial<CreateTableOptions>, options?: Partial<CreateTableOptions>): Promise<Table>;
|
|
390
491
|
dropTable(name: string, namespacePath?: string[]): Promise<void>;
|
|
492
|
+
dropTableAsync(name: string, namespacePath?: string[]): Promise<Job>;
|
|
391
493
|
dropAllTables(namespacePath?: string[]): Promise<void>;
|
|
392
494
|
describeNamespace(namespacePath: string[]): Promise<DescribeNamespaceResponse>;
|
|
393
495
|
listNamespaces(namespacePath?: string[], options?: Partial<ListNamespacesOptions>): Promise<ListNamespacesResponse>;
|
package/dist/connection.js
CHANGED
|
@@ -8,6 +8,7 @@ const apache_arrow_1 = require("apache-arrow");
|
|
|
8
8
|
const arrow_1 = require("./arrow");
|
|
9
9
|
const arrow_2 = require("./arrow");
|
|
10
10
|
const registry_1 = require("./embedding/registry");
|
|
11
|
+
const materialized_view_1 = require("./materialized_view");
|
|
11
12
|
const sanitize_1 = require("./sanitize");
|
|
12
13
|
const table_1 = require("./table");
|
|
13
14
|
/**
|
|
@@ -68,6 +69,28 @@ class LocalConnection extends Connection {
|
|
|
68
69
|
}
|
|
69
70
|
return this.inner.tableNames(namespacePath ?? [], tableNamesOptions?.startAfter, tableNamesOptions?.limit);
|
|
70
71
|
}
|
|
72
|
+
async createMaterializedView(name, source, options) {
|
|
73
|
+
(0, materialized_view_1.validateNonNegativeInteger)(options?.limit, "limit");
|
|
74
|
+
const innerTable = await this.inner.createMaterializedView(name, source, (0, materialized_view_1.normalizeSelect)(options?.select), options?.where, options?.limit);
|
|
75
|
+
return new materialized_view_1.MaterializedView(new table_1.LocalTable(innerTable));
|
|
76
|
+
}
|
|
77
|
+
async openMaterializedView(name) {
|
|
78
|
+
const innerTable = await this.inner.openMaterializedView(name);
|
|
79
|
+
return new materialized_view_1.MaterializedView(new table_1.LocalTable(innerTable));
|
|
80
|
+
}
|
|
81
|
+
async listMaterializedViews() {
|
|
82
|
+
return await this.inner.listMaterializedViews();
|
|
83
|
+
}
|
|
84
|
+
async listTables(namespacePathOrOptions, options) {
|
|
85
|
+
// Detect if first argument is namespacePath array or options object
|
|
86
|
+
const namespacePath = Array.isArray(namespacePathOrOptions)
|
|
87
|
+
? namespacePathOrOptions
|
|
88
|
+
: undefined;
|
|
89
|
+
const listTablesOptions = Array.isArray(namespacePathOrOptions)
|
|
90
|
+
? options
|
|
91
|
+
: namespacePathOrOptions;
|
|
92
|
+
return this.inner.listTables(namespacePath ?? [], listTablesOptions?.pageToken, listTablesOptions?.limit);
|
|
93
|
+
}
|
|
71
94
|
async openTable(name, namespacePath, options) {
|
|
72
95
|
const innerTable = await this.inner.openTable(name, namespacePath ?? [], cleanseStorageOptions(options?.storageOptions), options?.indexCacheSize);
|
|
73
96
|
let table = new table_1.LocalTable(innerTable);
|
|
@@ -174,6 +197,9 @@ class LocalConnection extends Connection {
|
|
|
174
197
|
async dropTable(name, namespacePath) {
|
|
175
198
|
return this.inner.dropTable(name, namespacePath ?? []);
|
|
176
199
|
}
|
|
200
|
+
async dropTableAsync(name, namespacePath) {
|
|
201
|
+
return this.inner.dropTableAsync(name, namespacePath ?? []);
|
|
202
|
+
}
|
|
177
203
|
async dropAllTables(namespacePath) {
|
|
178
204
|
return this.inner.dropAllTables(namespacePath ?? []);
|
|
179
205
|
}
|