@lancedb/lancedb 0.15.1-beta.3 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow.d.ts +4 -2
- package/dist/arrow.js +293 -83
- package/dist/connection.d.ts +10 -8
- package/dist/connection.js +24 -16
- package/dist/native.d.ts +4 -2
- package/package.json +9 -9
package/dist/arrow.d.ts
CHANGED
|
@@ -91,8 +91,6 @@ export declare class MakeArrowTableOptions {
|
|
|
91
91
|
* This function converts an array of Record<String, any> (row-major JS objects)
|
|
92
92
|
* to an Arrow Table (a columnar structure)
|
|
93
93
|
*
|
|
94
|
-
* Note that it currently does not support nulls.
|
|
95
|
-
*
|
|
96
94
|
* If a schema is provided then it will be used to determine the resulting array
|
|
97
95
|
* types. Fields will also be reordered to fit the order defined by the schema.
|
|
98
96
|
*
|
|
@@ -100,6 +98,9 @@ export declare class MakeArrowTableOptions {
|
|
|
100
98
|
* will be controlled by the order of properties in the first record. If a type
|
|
101
99
|
* is inferred it will always be nullable.
|
|
102
100
|
*
|
|
101
|
+
* If not all fields are found in the data, then a subset of the schema will be
|
|
102
|
+
* returned.
|
|
103
|
+
*
|
|
103
104
|
* If the input is empty then a schema must be provided to create an empty table.
|
|
104
105
|
*
|
|
105
106
|
* When a schema is not specified then data types will be inferred. The inference
|
|
@@ -107,6 +108,7 @@ export declare class MakeArrowTableOptions {
|
|
|
107
108
|
*
|
|
108
109
|
* - boolean => Bool
|
|
109
110
|
* - number => Float64
|
|
111
|
+
* - bigint => Int64
|
|
110
112
|
* - String => Utf8
|
|
111
113
|
* - Buffer => Binary
|
|
112
114
|
* - Record<String, any> => Struct
|
package/dist/arrow.js
CHANGED
|
@@ -186,8 +186,6 @@ exports.MakeArrowTableOptions = MakeArrowTableOptions;
|
|
|
186
186
|
* This function converts an array of Record<String, any> (row-major JS objects)
|
|
187
187
|
* to an Arrow Table (a columnar structure)
|
|
188
188
|
*
|
|
189
|
-
* Note that it currently does not support nulls.
|
|
190
|
-
*
|
|
191
189
|
* If a schema is provided then it will be used to determine the resulting array
|
|
192
190
|
* types. Fields will also be reordered to fit the order defined by the schema.
|
|
193
191
|
*
|
|
@@ -195,6 +193,9 @@ exports.MakeArrowTableOptions = MakeArrowTableOptions;
|
|
|
195
193
|
* will be controlled by the order of properties in the first record. If a type
|
|
196
194
|
* is inferred it will always be nullable.
|
|
197
195
|
*
|
|
196
|
+
* If not all fields are found in the data, then a subset of the schema will be
|
|
197
|
+
* returned.
|
|
198
|
+
*
|
|
198
199
|
* If the input is empty then a schema must be provided to create an empty table.
|
|
199
200
|
*
|
|
200
201
|
* When a schema is not specified then data types will be inferred. The inference
|
|
@@ -202,6 +203,7 @@ exports.MakeArrowTableOptions = MakeArrowTableOptions;
|
|
|
202
203
|
*
|
|
203
204
|
* - boolean => Bool
|
|
204
205
|
* - number => Float64
|
|
206
|
+
* - bigint => Int64
|
|
205
207
|
* - String => Utf8
|
|
206
208
|
* - Buffer => Binary
|
|
207
209
|
* - Record<String, any> => Struct
|
|
@@ -268,108 +270,286 @@ exports.MakeArrowTableOptions = MakeArrowTableOptions;
|
|
|
268
270
|
* ```
|
|
269
271
|
*/
|
|
270
272
|
function makeArrowTable(data, options, metadata) {
|
|
273
|
+
const opt = new MakeArrowTableOptions(options !== undefined ? options : {});
|
|
274
|
+
let schema = undefined;
|
|
275
|
+
if (opt.schema !== undefined && opt.schema !== null) {
|
|
276
|
+
schema = (0, sanitize_1.sanitizeSchema)(opt.schema);
|
|
277
|
+
schema = validateSchemaEmbeddings(schema, data, options?.embeddingFunction);
|
|
278
|
+
}
|
|
279
|
+
let schemaMetadata = schema?.metadata || new Map();
|
|
280
|
+
if (metadata !== undefined) {
|
|
281
|
+
schemaMetadata = new Map([...schemaMetadata, ...metadata]);
|
|
282
|
+
}
|
|
271
283
|
if (data.length === 0 &&
|
|
272
284
|
(options?.schema === undefined || options?.schema === null)) {
|
|
273
285
|
throw new Error("At least one record or a schema needs to be provided");
|
|
274
286
|
}
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
if (
|
|
298
|
-
//
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
287
|
+
else if (data.length === 0) {
|
|
288
|
+
if (schema === undefined) {
|
|
289
|
+
throw new Error("A schema must be provided if data is empty");
|
|
290
|
+
}
|
|
291
|
+
else {
|
|
292
|
+
schema = new apache_arrow_1.Schema(schema.fields, schemaMetadata);
|
|
293
|
+
return new apache_arrow_1.Table(schema);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
let inferredSchema = inferSchema(data, schema, opt);
|
|
297
|
+
inferredSchema = new apache_arrow_1.Schema(inferredSchema.fields, schemaMetadata);
|
|
298
|
+
const finalColumns = {};
|
|
299
|
+
for (const field of inferredSchema.fields) {
|
|
300
|
+
finalColumns[field.name] = transposeData(data, field);
|
|
301
|
+
}
|
|
302
|
+
return new apache_arrow_1.Table(inferredSchema, finalColumns);
|
|
303
|
+
}
|
|
304
|
+
function inferSchema(data, schema, opts) {
|
|
305
|
+
// We will collect all fields we see in the data.
|
|
306
|
+
const pathTree = new PathTree();
|
|
307
|
+
for (const [rowI, row] of data.entries()) {
|
|
308
|
+
for (const [path, value] of rowPathsAndValues(row)) {
|
|
309
|
+
if (!pathTree.has(path)) {
|
|
310
|
+
// First time seeing this field.
|
|
311
|
+
if (schema !== undefined) {
|
|
312
|
+
const field = getFieldForPath(schema, path);
|
|
313
|
+
if (field === undefined) {
|
|
314
|
+
throw new Error(`Found field not in schema: ${path.join(".")} at row ${rowI}`);
|
|
302
315
|
}
|
|
303
|
-
|
|
304
|
-
|
|
316
|
+
else {
|
|
317
|
+
pathTree.set(path, field.type);
|
|
305
318
|
}
|
|
306
|
-
|
|
307
|
-
|
|
319
|
+
}
|
|
320
|
+
else {
|
|
321
|
+
const inferredType = inferType(value, path, opts);
|
|
322
|
+
if (inferredType === undefined) {
|
|
323
|
+
throw new Error(`Failed to infer data type for field ${path.join(".")} at row ${rowI}. \
|
|
324
|
+
Consider providing an explicit schema.`);
|
|
308
325
|
}
|
|
309
|
-
|
|
310
|
-
}
|
|
326
|
+
pathTree.set(path, inferredType);
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
else if (schema === undefined) {
|
|
330
|
+
const currentType = pathTree.get(path);
|
|
331
|
+
const newType = inferType(value, path, opts);
|
|
332
|
+
if (currentType !== newType) {
|
|
333
|
+
new Error(`Failed to infer schema for data. Previously inferred type \
|
|
334
|
+
${currentType} but found ${newType} at row ${rowI}. Consider \
|
|
335
|
+
providing an explicit schema.`);
|
|
336
|
+
}
|
|
311
337
|
}
|
|
312
338
|
}
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
const
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
339
|
+
}
|
|
340
|
+
if (schema === undefined) {
|
|
341
|
+
function fieldsFromPathTree(pathTree) {
|
|
342
|
+
const fields = [];
|
|
343
|
+
for (const [name, value] of pathTree.map.entries()) {
|
|
344
|
+
if (value instanceof PathTree) {
|
|
345
|
+
const children = fieldsFromPathTree(value);
|
|
346
|
+
fields.push(new apache_arrow_1.Field(name, new apache_arrow_1.Struct(children), true));
|
|
321
347
|
}
|
|
322
348
|
else {
|
|
323
|
-
|
|
349
|
+
fields.push(new apache_arrow_1.Field(name, value, true));
|
|
324
350
|
}
|
|
325
351
|
}
|
|
352
|
+
return fields;
|
|
326
353
|
}
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
// To work around this we first create a table with the wrong schema and
|
|
345
|
-
// then patch the schema of the batches so we can use
|
|
346
|
-
// `new ArrowTable(schema, batches)` which does not do any schema inference
|
|
347
|
-
const firstTable = new apache_arrow_1.Table(columns);
|
|
348
|
-
const batchesFixed = firstTable.batches.map((batch) => new apache_arrow_1.RecordBatch(opt.schema, batch.data));
|
|
349
|
-
let schema;
|
|
350
|
-
if (metadata !== undefined) {
|
|
351
|
-
let schemaMetadata = opt.schema.metadata;
|
|
352
|
-
if (schemaMetadata.size === 0) {
|
|
353
|
-
schemaMetadata = metadata;
|
|
354
|
-
}
|
|
355
|
-
else {
|
|
356
|
-
for (const [key, entry] of schemaMetadata.entries()) {
|
|
357
|
-
schemaMetadata.set(key, entry);
|
|
354
|
+
const fields = fieldsFromPathTree(pathTree);
|
|
355
|
+
return new apache_arrow_1.Schema(fields);
|
|
356
|
+
}
|
|
357
|
+
else {
|
|
358
|
+
function takeMatchingFields(fields, pathTree) {
|
|
359
|
+
const outFields = [];
|
|
360
|
+
for (const field of fields) {
|
|
361
|
+
if (pathTree.map.has(field.name)) {
|
|
362
|
+
const value = pathTree.get([field.name]);
|
|
363
|
+
if (value instanceof PathTree) {
|
|
364
|
+
const struct = field.type;
|
|
365
|
+
const children = takeMatchingFields(struct.children, value);
|
|
366
|
+
outFields.push(new apache_arrow_1.Field(field.name, new apache_arrow_1.Struct(children), field.nullable));
|
|
367
|
+
}
|
|
368
|
+
else {
|
|
369
|
+
outFields.push(new apache_arrow_1.Field(field.name, value, field.nullable));
|
|
370
|
+
}
|
|
358
371
|
}
|
|
359
372
|
}
|
|
360
|
-
|
|
373
|
+
return outFields;
|
|
374
|
+
}
|
|
375
|
+
const fields = takeMatchingFields(schema.fields, pathTree);
|
|
376
|
+
return new apache_arrow_1.Schema(fields);
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
function* rowPathsAndValues(row, basePath = []) {
|
|
380
|
+
for (const [key, value] of Object.entries(row)) {
|
|
381
|
+
if (isObject(value)) {
|
|
382
|
+
yield* rowPathsAndValues(value, [...basePath, key]);
|
|
361
383
|
}
|
|
362
384
|
else {
|
|
363
|
-
|
|
385
|
+
yield [[...basePath, key], value];
|
|
364
386
|
}
|
|
365
|
-
return new apache_arrow_1.Table(schema, batchesFixed);
|
|
366
387
|
}
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
388
|
+
}
|
|
389
|
+
function isObject(value) {
|
|
390
|
+
return (typeof value === "object" &&
|
|
391
|
+
value !== null &&
|
|
392
|
+
!Array.isArray(value) &&
|
|
393
|
+
!(value instanceof RegExp) &&
|
|
394
|
+
!(value instanceof Date) &&
|
|
395
|
+
!(value instanceof Set) &&
|
|
396
|
+
!(value instanceof Map) &&
|
|
397
|
+
!(value instanceof Buffer));
|
|
398
|
+
}
|
|
399
|
+
function getFieldForPath(schema, path) {
|
|
400
|
+
let current = schema;
|
|
401
|
+
for (const key of path) {
|
|
402
|
+
if (current instanceof apache_arrow_1.Schema) {
|
|
403
|
+
const field = current.fields.find((f) => f.name === key);
|
|
404
|
+
if (field === undefined) {
|
|
405
|
+
return undefined;
|
|
406
|
+
}
|
|
407
|
+
current = field;
|
|
408
|
+
}
|
|
409
|
+
else if (current instanceof apache_arrow_1.Field && apache_arrow_1.DataType.isStruct(current.type)) {
|
|
410
|
+
const struct = current.type;
|
|
411
|
+
const field = struct.children.find((f) => f.name === key);
|
|
412
|
+
if (field === undefined) {
|
|
413
|
+
return undefined;
|
|
414
|
+
}
|
|
415
|
+
current = field;
|
|
416
|
+
}
|
|
417
|
+
else {
|
|
418
|
+
return undefined;
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
if (current instanceof apache_arrow_1.Field) {
|
|
422
|
+
return current;
|
|
423
|
+
}
|
|
424
|
+
else {
|
|
425
|
+
return undefined;
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
/**
|
|
429
|
+
* Try to infer which Arrow type to use for a given value.
|
|
430
|
+
*
|
|
431
|
+
* May return undefined if the type cannot be inferred.
|
|
432
|
+
*/
|
|
433
|
+
function inferType(value, path, opts) {
|
|
434
|
+
if (typeof value === "bigint") {
|
|
435
|
+
return new apache_arrow_1.Int64();
|
|
436
|
+
}
|
|
437
|
+
else if (typeof value === "number") {
|
|
438
|
+
// Even if it's an integer, it's safer to assume Float64. Users can
|
|
439
|
+
// always provide an explicit schema or use BigInt if they mean integer.
|
|
440
|
+
return new apache_arrow_1.Float64();
|
|
441
|
+
}
|
|
442
|
+
else if (typeof value === "string") {
|
|
443
|
+
if (opts.dictionaryEncodeStrings) {
|
|
444
|
+
return new apache_arrow_1.Dictionary(new apache_arrow_1.Utf8(), new apache_arrow_1.Int32());
|
|
445
|
+
}
|
|
446
|
+
else {
|
|
447
|
+
return new apache_arrow_1.Utf8();
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
else if (typeof value === "boolean") {
|
|
451
|
+
return new apache_arrow_1.Bool();
|
|
452
|
+
}
|
|
453
|
+
else if (value instanceof Buffer) {
|
|
454
|
+
return new apache_arrow_1.Binary();
|
|
455
|
+
}
|
|
456
|
+
else if (Array.isArray(value)) {
|
|
457
|
+
if (value.length === 0) {
|
|
458
|
+
return undefined; // Without any values we can't infer the type
|
|
459
|
+
}
|
|
460
|
+
if (path.length === 1 && Object.hasOwn(opts.vectorColumns, path[0])) {
|
|
461
|
+
const floatType = (0, sanitize_1.sanitizeType)(opts.vectorColumns[path[0]].type);
|
|
462
|
+
return new apache_arrow_1.FixedSizeList(value.length, new apache_arrow_1.Field("item", floatType, true));
|
|
463
|
+
}
|
|
464
|
+
const valueType = inferType(value[0], path, opts);
|
|
465
|
+
if (valueType === undefined) {
|
|
466
|
+
return undefined;
|
|
467
|
+
}
|
|
468
|
+
// Try to automatically detect embedding columns.
|
|
469
|
+
if (valueType instanceof apache_arrow_1.Float && path[path.length - 1] === "vector") {
|
|
470
|
+
// We default to Float32 for vectors.
|
|
471
|
+
const child = new apache_arrow_1.Field("item", new apache_arrow_1.Float32(), true);
|
|
472
|
+
return new apache_arrow_1.FixedSizeList(value.length, child);
|
|
473
|
+
}
|
|
474
|
+
else {
|
|
475
|
+
const child = new apache_arrow_1.Field("item", valueType, true);
|
|
476
|
+
return new apache_arrow_1.List(child);
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
else {
|
|
480
|
+
// TODO: timestamp
|
|
481
|
+
return undefined;
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
class PathTree {
|
|
485
|
+
map;
|
|
486
|
+
constructor(entries) {
|
|
487
|
+
this.map = new Map();
|
|
488
|
+
if (entries !== undefined) {
|
|
489
|
+
for (const [path, value] of entries) {
|
|
490
|
+
this.set(path, value);
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
has(path) {
|
|
495
|
+
let ref = this;
|
|
496
|
+
for (const part of path) {
|
|
497
|
+
if (!(ref instanceof PathTree) || !ref.map.has(part)) {
|
|
498
|
+
return false;
|
|
499
|
+
}
|
|
500
|
+
ref = ref.map.get(part);
|
|
501
|
+
}
|
|
502
|
+
return true;
|
|
503
|
+
}
|
|
504
|
+
get(path) {
|
|
505
|
+
let ref = this;
|
|
506
|
+
for (const part of path) {
|
|
507
|
+
if (!(ref instanceof PathTree) || !ref.map.has(part)) {
|
|
508
|
+
return undefined;
|
|
509
|
+
}
|
|
510
|
+
ref = ref.map.get(part);
|
|
511
|
+
}
|
|
512
|
+
return ref;
|
|
513
|
+
}
|
|
514
|
+
set(path, value) {
|
|
515
|
+
let ref = this;
|
|
516
|
+
for (const part of path.slice(0, path.length - 1)) {
|
|
517
|
+
if (!ref.map.has(part)) {
|
|
518
|
+
ref.map.set(part, new PathTree());
|
|
519
|
+
}
|
|
520
|
+
ref = ref.map.get(part);
|
|
521
|
+
}
|
|
522
|
+
ref.map.set(path[path.length - 1], value);
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
function transposeData(data, field, path = []) {
|
|
526
|
+
if (field.type instanceof apache_arrow_1.Struct) {
|
|
527
|
+
const childFields = field.type.children;
|
|
528
|
+
const childVectors = childFields.map((child) => {
|
|
529
|
+
return transposeData(data, child, [...path, child.name]);
|
|
530
|
+
});
|
|
531
|
+
const structData = (0, apache_arrow_1.makeData)({
|
|
532
|
+
type: field.type,
|
|
533
|
+
children: childVectors,
|
|
534
|
+
});
|
|
535
|
+
return (0, apache_arrow_1.makeVector)(structData);
|
|
536
|
+
}
|
|
537
|
+
else {
|
|
538
|
+
const valuesPath = [...path, field.name];
|
|
539
|
+
const values = data.map((datum) => {
|
|
540
|
+
let current = datum;
|
|
541
|
+
for (const key of valuesPath) {
|
|
542
|
+
if (isObject(current) && Object.hasOwn(current, key)) {
|
|
543
|
+
current = current[key];
|
|
544
|
+
}
|
|
545
|
+
else {
|
|
546
|
+
return null;
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
return current;
|
|
550
|
+
});
|
|
551
|
+
return makeVector(values, field.type);
|
|
371
552
|
}
|
|
372
|
-
return tbl;
|
|
373
553
|
}
|
|
374
554
|
/**
|
|
375
555
|
* Create an empty Arrow table with the provided schema
|
|
@@ -408,6 +588,36 @@ function makeListVector(lists) {
|
|
|
408
588
|
function makeVector(values, type, stringAsDictionary) {
|
|
409
589
|
if (type !== undefined) {
|
|
410
590
|
// No need for inference, let Arrow create it
|
|
591
|
+
if (type instanceof apache_arrow_1.Int) {
|
|
592
|
+
if (apache_arrow_1.DataType.isInt(type) && type.bitWidth === 64) {
|
|
593
|
+
// wrap in BigInt to avoid bug: https://github.com/apache/arrow/issues/40051
|
|
594
|
+
values = values.map((v) => {
|
|
595
|
+
if (v === null) {
|
|
596
|
+
return v;
|
|
597
|
+
}
|
|
598
|
+
else if (typeof v === "bigint") {
|
|
599
|
+
return v;
|
|
600
|
+
}
|
|
601
|
+
else if (typeof v === "number") {
|
|
602
|
+
return BigInt(v);
|
|
603
|
+
}
|
|
604
|
+
else {
|
|
605
|
+
return v;
|
|
606
|
+
}
|
|
607
|
+
});
|
|
608
|
+
}
|
|
609
|
+
else {
|
|
610
|
+
// Similarly, bigint isn't supported for 16 or 32-bit ints.
|
|
611
|
+
values = values.map((v) => {
|
|
612
|
+
if (typeof v == "bigint") {
|
|
613
|
+
return Number(v);
|
|
614
|
+
}
|
|
615
|
+
else {
|
|
616
|
+
return v;
|
|
617
|
+
}
|
|
618
|
+
});
|
|
619
|
+
}
|
|
620
|
+
}
|
|
411
621
|
return (0, apache_arrow_1.vectorFromArray)(values, type);
|
|
412
622
|
}
|
|
413
623
|
if (values.length === 0) {
|
package/dist/connection.d.ts
CHANGED
|
@@ -32,6 +32,8 @@ export interface CreateTableOptions {
|
|
|
32
32
|
*
|
|
33
33
|
* The default is `stable`.
|
|
34
34
|
* Set to "legacy" to use the old format.
|
|
35
|
+
*
|
|
36
|
+
* @deprecated Pass `new_table_data_storage_version` to storageOptions instead.
|
|
35
37
|
*/
|
|
36
38
|
dataStorageVersion?: string;
|
|
37
39
|
/**
|
|
@@ -40,16 +42,10 @@ export interface CreateTableOptions {
|
|
|
40
42
|
* turning this on will make the dataset unreadable for older versions
|
|
41
43
|
* of LanceDB (prior to 0.10.0). To migrate an existing dataset, instead
|
|
42
44
|
* use the {@link LocalTable#migrateManifestPathsV2} method.
|
|
43
|
-
*/
|
|
44
|
-
enableV2ManifestPaths?: boolean;
|
|
45
|
-
/**
|
|
46
|
-
* If true then data files will be written with the legacy format
|
|
47
45
|
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
* Deprecated. Use data storage version instead.
|
|
46
|
+
* @deprecated Pass `new_table_enable_v2_manifest_paths` to storageOptions instead.
|
|
51
47
|
*/
|
|
52
|
-
|
|
48
|
+
enableV2ManifestPaths?: boolean;
|
|
53
49
|
schema?: SchemaLike;
|
|
54
50
|
embeddingFunction?: EmbeddingFunctionConfig;
|
|
55
51
|
}
|
|
@@ -167,6 +163,10 @@ export declare abstract class Connection {
|
|
|
167
163
|
* @param {string} name The name of the table to drop.
|
|
168
164
|
*/
|
|
169
165
|
abstract dropTable(name: string): Promise<void>;
|
|
166
|
+
/**
|
|
167
|
+
* Drop all tables in the database.
|
|
168
|
+
*/
|
|
169
|
+
abstract dropAllTables(): Promise<void>;
|
|
170
170
|
}
|
|
171
171
|
/** @hideconstructor */
|
|
172
172
|
export declare class LocalConnection extends Connection {
|
|
@@ -178,12 +178,14 @@ export declare class LocalConnection extends Connection {
|
|
|
178
178
|
display(): string;
|
|
179
179
|
tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
|
|
180
180
|
openTable(name: string, options?: Partial<OpenTableOptions>): Promise<Table>;
|
|
181
|
+
private getStorageOptions;
|
|
181
182
|
createTable(nameOrOptions: string | ({
|
|
182
183
|
name: string;
|
|
183
184
|
data: Data;
|
|
184
185
|
} & Partial<CreateTableOptions>), data?: Record<string, unknown>[] | TableLike, options?: Partial<CreateTableOptions>): Promise<Table>;
|
|
185
186
|
createEmptyTable(name: string, schema: import("./arrow").SchemaLike, options?: Partial<CreateTableOptions>): Promise<Table>;
|
|
186
187
|
dropTable(name: string): Promise<void>;
|
|
188
|
+
dropAllTables(): Promise<void>;
|
|
187
189
|
}
|
|
188
190
|
/**
|
|
189
191
|
* Takes storage options and makes all the keys snake case.
|
package/dist/connection.js
CHANGED
|
@@ -58,6 +58,23 @@ class LocalConnection extends Connection {
|
|
|
58
58
|
const innerTable = await this.inner.openTable(name, cleanseStorageOptions(options?.storageOptions), options?.indexCacheSize);
|
|
59
59
|
return new table_1.LocalTable(innerTable);
|
|
60
60
|
}
|
|
61
|
+
getStorageOptions(options) {
|
|
62
|
+
if (options?.dataStorageVersion !== undefined) {
|
|
63
|
+
if (options.storageOptions === undefined) {
|
|
64
|
+
options.storageOptions = {};
|
|
65
|
+
}
|
|
66
|
+
options.storageOptions["newTableDataStorageVersion"] =
|
|
67
|
+
options.dataStorageVersion;
|
|
68
|
+
}
|
|
69
|
+
if (options?.enableV2ManifestPaths !== undefined) {
|
|
70
|
+
if (options.storageOptions === undefined) {
|
|
71
|
+
options.storageOptions = {};
|
|
72
|
+
}
|
|
73
|
+
options.storageOptions["newTableEnableV2ManifestPaths"] =
|
|
74
|
+
options.enableV2ManifestPaths ? "true" : "false";
|
|
75
|
+
}
|
|
76
|
+
return cleanseStorageOptions(options?.storageOptions);
|
|
77
|
+
}
|
|
61
78
|
async createTable(nameOrOptions, data, options) {
|
|
62
79
|
if (typeof nameOrOptions !== "string" && "name" in nameOrOptions) {
|
|
63
80
|
const { name, data, ...options } = nameOrOptions;
|
|
@@ -67,14 +84,8 @@ class LocalConnection extends Connection {
|
|
|
67
84
|
throw new Error("data is required");
|
|
68
85
|
}
|
|
69
86
|
const { buf, mode } = await parseTableData(data, options);
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
dataStorageVersion = options.dataStorageVersion;
|
|
73
|
-
}
|
|
74
|
-
else if (options?.useLegacyFormat !== undefined) {
|
|
75
|
-
dataStorageVersion = options.useLegacyFormat ? "legacy" : "stable";
|
|
76
|
-
}
|
|
77
|
-
const innerTable = await this.inner.createTable(nameOrOptions, buf, mode, cleanseStorageOptions(options?.storageOptions), dataStorageVersion, options?.enableV2ManifestPaths);
|
|
87
|
+
const storageOptions = this.getStorageOptions(options);
|
|
88
|
+
const innerTable = await this.inner.createTable(nameOrOptions, buf, mode, storageOptions);
|
|
78
89
|
return new table_1.LocalTable(innerTable);
|
|
79
90
|
}
|
|
80
91
|
async createEmptyTable(name, schema, options) {
|
|
@@ -89,21 +100,18 @@ class LocalConnection extends Connection {
|
|
|
89
100
|
const registry = (0, registry_1.getRegistry)();
|
|
90
101
|
metadata = registry.getTableMetadata([embeddingFunction]);
|
|
91
102
|
}
|
|
92
|
-
|
|
93
|
-
if (options?.dataStorageVersion !== undefined) {
|
|
94
|
-
dataStorageVersion = options.dataStorageVersion;
|
|
95
|
-
}
|
|
96
|
-
else if (options?.useLegacyFormat !== undefined) {
|
|
97
|
-
dataStorageVersion = options.useLegacyFormat ? "legacy" : "stable";
|
|
98
|
-
}
|
|
103
|
+
const storageOptions = this.getStorageOptions(options);
|
|
99
104
|
const table = (0, arrow_2.makeEmptyTable)(schema, metadata);
|
|
100
105
|
const buf = await (0, arrow_2.fromTableToBuffer)(table);
|
|
101
|
-
const innerTable = await this.inner.createEmptyTable(name, buf, mode,
|
|
106
|
+
const innerTable = await this.inner.createEmptyTable(name, buf, mode, storageOptions);
|
|
102
107
|
return new table_1.LocalTable(innerTable);
|
|
103
108
|
}
|
|
104
109
|
async dropTable(name) {
|
|
105
110
|
return this.inner.dropTable(name);
|
|
106
111
|
}
|
|
112
|
+
async dropAllTables() {
|
|
113
|
+
return this.inner.dropAllTables();
|
|
114
|
+
}
|
|
107
115
|
}
|
|
108
116
|
exports.LocalConnection = LocalConnection;
|
|
109
117
|
/**
|
package/dist/native.d.ts
CHANGED
|
@@ -77,6 +77,7 @@ export interface ClientConfig {
|
|
|
77
77
|
userAgent?: string
|
|
78
78
|
retryConfig?: RetryConfig
|
|
79
79
|
timeoutConfig?: TimeoutConfig
|
|
80
|
+
extraHeaders?: Record<string, string>
|
|
80
81
|
}
|
|
81
82
|
export interface RerankerCallbacks {
|
|
82
83
|
rerankHybrid: (...args: any[]) => any
|
|
@@ -247,11 +248,12 @@ export class Connection {
|
|
|
247
248
|
* - buf: The buffer containing the IPC file.
|
|
248
249
|
*
|
|
249
250
|
*/
|
|
250
|
-
createTable(name: string, buf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null
|
|
251
|
-
createEmptyTable(name: string, schemaBuf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null
|
|
251
|
+
createTable(name: string, buf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null): Promise<Table>
|
|
252
|
+
createEmptyTable(name: string, schemaBuf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null): Promise<Table>
|
|
252
253
|
openTable(name: string, storageOptions?: Record<string, string> | undefined | null, indexCacheSize?: number | undefined | null): Promise<Table>
|
|
253
254
|
/** Drop table with the name. Or raise an error if the table does not exist. */
|
|
254
255
|
dropTable(name: string): Promise<void>
|
|
256
|
+
dropAllTables(): Promise<void>
|
|
255
257
|
}
|
|
256
258
|
export class Index {
|
|
257
259
|
static ivfPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, numBits?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
|
package/package.json
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
"ann"
|
|
12
12
|
],
|
|
13
13
|
"private": false,
|
|
14
|
-
"version": "0.
|
|
14
|
+
"version": "0.16.0",
|
|
15
15
|
"main": "dist/index.js",
|
|
16
16
|
"exports": {
|
|
17
17
|
".": "./dist/index.js",
|
|
@@ -98,14 +98,14 @@
|
|
|
98
98
|
"reflect-metadata": "^0.2.2"
|
|
99
99
|
},
|
|
100
100
|
"optionalDependencies": {
|
|
101
|
-
"@lancedb/lancedb-darwin-x64": "0.
|
|
102
|
-
"@lancedb/lancedb-darwin-arm64": "0.
|
|
103
|
-
"@lancedb/lancedb-linux-x64-gnu": "0.
|
|
104
|
-
"@lancedb/lancedb-linux-arm64-gnu": "0.
|
|
105
|
-
"@lancedb/lancedb-linux-x64-musl": "0.
|
|
106
|
-
"@lancedb/lancedb-linux-arm64-musl": "0.
|
|
107
|
-
"@lancedb/lancedb-win32-x64-msvc": "0.
|
|
108
|
-
"@lancedb/lancedb-win32-arm64-msvc": "0.
|
|
101
|
+
"@lancedb/lancedb-darwin-x64": "0.16.0",
|
|
102
|
+
"@lancedb/lancedb-darwin-arm64": "0.16.0",
|
|
103
|
+
"@lancedb/lancedb-linux-x64-gnu": "0.16.0",
|
|
104
|
+
"@lancedb/lancedb-linux-arm64-gnu": "0.16.0",
|
|
105
|
+
"@lancedb/lancedb-linux-x64-musl": "0.16.0",
|
|
106
|
+
"@lancedb/lancedb-linux-arm64-musl": "0.16.0",
|
|
107
|
+
"@lancedb/lancedb-win32-x64-msvc": "0.16.0",
|
|
108
|
+
"@lancedb/lancedb-win32-arm64-msvc": "0.16.0"
|
|
109
109
|
},
|
|
110
110
|
"peerDependencies": {
|
|
111
111
|
"apache-arrow": ">=15.0.0 <=18.1.0"
|