@lancedb/lancedb 0.40.0 → 0.41.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NODEJS_THIRD_PARTY_LICENSES.md +8 -0
- package/dist/arrow.d.ts +18 -1
- package/dist/arrow.js +194 -14
- package/dist/blob.d.ts +81 -2
- package/dist/blob.js +159 -6
- package/dist/catalog.d.ts +40 -6
- package/dist/catalog.js +58 -9
- package/dist/connection.d.ts +19 -13
- package/dist/connection.js +12 -7
- package/dist/index.d.ts +6 -5
- package/dist/index.js +4 -1
- package/dist/indices.d.ts +3 -1
- package/dist/listing.d.ts +41 -0
- package/dist/listing.js +57 -0
- package/dist/materialized_view.d.ts +5 -12
- package/dist/materialized_view.js +6 -12
- package/dist/merge.d.ts +7 -7
- package/dist/merge.js +8 -8
- package/dist/native.d.ts +151 -114
- package/dist/native.js +55 -52
- package/dist/query.d.ts +94 -26
- package/dist/query.js +176 -52
- package/dist/sanitize.d.ts +19 -18
- package/dist/sanitize.js +74 -11
- package/dist/schema.js +31 -3
- package/dist/table.d.ts +65 -60
- package/dist/table.js +26 -20
- package/package.json +13 -9
package/dist/query.js
CHANGED
|
@@ -21,17 +21,61 @@ async function* RecordBatchIterator(promisedInner) {
|
|
|
21
21
|
yield batches[0];
|
|
22
22
|
}
|
|
23
23
|
}
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
24
|
+
const BLOB_BYTES_ATTEMPTS = 3;
|
|
25
|
+
function blobSources(blobColumns, projection) {
|
|
26
|
+
const columns = new Set(blobColumns);
|
|
27
|
+
const pairs = projection ??
|
|
28
|
+
blobColumns.map((column) => [column, column]);
|
|
29
|
+
return new Map(pairs.filter(([, source]) => columns.has(source)));
|
|
30
|
+
}
|
|
31
|
+
async function readBlobBytes(table, sources, snapshot, dropRowId) {
|
|
32
|
+
const rowIdColumn = table.getChild("_rowid");
|
|
33
|
+
if (table.numRows > 0 && rowIdColumn === null) {
|
|
34
|
+
throw new Error('blobMode "bytes" needs _rowid in the query result');
|
|
30
35
|
}
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
36
|
+
const rowIds = rowIdColumn === null
|
|
37
|
+
? []
|
|
38
|
+
: Array.from(rowIdColumn.toArray());
|
|
39
|
+
const replacements = new Map();
|
|
40
|
+
for (const [output, source] of sources) {
|
|
41
|
+
const index = table.schema.fields.findIndex((field) => field.name === output);
|
|
42
|
+
if (index < 0) {
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
const values = rowIds.length === 0 ? [] : await snapshot.fetchBlobs(source, rowIds);
|
|
46
|
+
replacements.set(index, values.map((value) => value ?? null));
|
|
34
47
|
}
|
|
48
|
+
const result = withBinaryColumns(table, replacements);
|
|
49
|
+
if (!dropRowId) {
|
|
50
|
+
return result;
|
|
51
|
+
}
|
|
52
|
+
return result.select(result.schema.fields
|
|
53
|
+
.map((field) => field.name)
|
|
54
|
+
.filter((name) => name !== "_rowid"));
|
|
55
|
+
}
|
|
56
|
+
// A fresh field, not a clone: the descriptor field's metadata must not follow
|
|
57
|
+
// the bytes.
|
|
58
|
+
function withBinaryColumns(table, replacements) {
|
|
59
|
+
if (replacements.size === 0) {
|
|
60
|
+
return table;
|
|
61
|
+
}
|
|
62
|
+
const fields = table.schema.fields.map((field, index) => replacements.has(index)
|
|
63
|
+
? new arrow_1.Field(field.name, new arrow_1.LargeBinary(), true)
|
|
64
|
+
: field);
|
|
65
|
+
const schema = new arrow_1.Schema(fields, table.schema.metadata);
|
|
66
|
+
let offset = 0;
|
|
67
|
+
const batches = table.batches.map((batch) => {
|
|
68
|
+
const children = batch.data.children.map((child, index) => {
|
|
69
|
+
const values = replacements.get(index);
|
|
70
|
+
if (values === undefined) {
|
|
71
|
+
return child;
|
|
72
|
+
}
|
|
73
|
+
return (0, arrow_1.vectorFromArray)(values.slice(offset, offset + batch.numRows), new arrow_1.LargeBinary()).data[0];
|
|
74
|
+
});
|
|
75
|
+
offset += batch.numRows;
|
|
76
|
+
return new arrow_1.RecordBatch(schema, (0, arrow_1.makeData)({ type: new arrow_1.Struct(fields), length: batch.numRows, children }));
|
|
77
|
+
});
|
|
78
|
+
return new arrow_1.Table(schema, batches);
|
|
35
79
|
}
|
|
36
80
|
function nearestToNative(inner, vector) {
|
|
37
81
|
const raw = Array.isArray(vector) ? null : (0, arrow_1.extractVectorBuffer)(vector);
|
|
@@ -61,10 +105,15 @@ class QueryBase {
|
|
|
61
105
|
/**
|
|
62
106
|
* @hidden
|
|
63
107
|
*/
|
|
64
|
-
|
|
108
|
+
blobContext;
|
|
109
|
+
/**
|
|
110
|
+
* @hidden
|
|
111
|
+
*/
|
|
112
|
+
constructor(inner, blobContext) {
|
|
65
113
|
if (inner !== undefined) {
|
|
66
114
|
this.inner = inner;
|
|
67
115
|
}
|
|
116
|
+
this.blobContext = blobContext;
|
|
68
117
|
}
|
|
69
118
|
// call a function on the inner (either a promise or the actual object)
|
|
70
119
|
/**
|
|
@@ -130,17 +179,25 @@ class QueryBase {
|
|
|
130
179
|
inner.select(columnTuples);
|
|
131
180
|
});
|
|
132
181
|
};
|
|
182
|
+
let projection;
|
|
133
183
|
if (typeof columns === "string") {
|
|
134
184
|
selectColumns([columns]);
|
|
185
|
+
projection = [[columns, columns]];
|
|
135
186
|
}
|
|
136
187
|
else if (Array.isArray(columns)) {
|
|
137
188
|
selectColumns(columns);
|
|
189
|
+
projection = columns.map((column) => [column, column]);
|
|
138
190
|
}
|
|
139
191
|
else if (columns instanceof Map) {
|
|
140
|
-
|
|
192
|
+
projection = Array.from(columns.entries());
|
|
193
|
+
selectMapping(projection);
|
|
141
194
|
}
|
|
142
195
|
else {
|
|
143
|
-
|
|
196
|
+
projection = Object.entries(columns);
|
|
197
|
+
selectMapping(projection);
|
|
198
|
+
}
|
|
199
|
+
if (this.blobContext !== undefined) {
|
|
200
|
+
this.blobContext = { ...this.blobContext, projection };
|
|
144
201
|
}
|
|
145
202
|
return this;
|
|
146
203
|
}
|
|
@@ -153,14 +210,17 @@ class QueryBase {
|
|
|
153
210
|
*/
|
|
154
211
|
withRowId() {
|
|
155
212
|
this.doCall((inner) => inner.withRowId());
|
|
213
|
+
if (this.blobContext !== undefined) {
|
|
214
|
+
this.blobContext = { ...this.blobContext, withRowId: true };
|
|
215
|
+
}
|
|
156
216
|
return this;
|
|
157
217
|
}
|
|
158
218
|
/**
|
|
159
219
|
* @hidden
|
|
160
220
|
*/
|
|
161
|
-
async nativeExecute(options) {
|
|
221
|
+
async nativeExecute(options, withRowId = false) {
|
|
162
222
|
const inner = await this.getInner();
|
|
163
|
-
return inner.execute(options?.maxBatchLength, options?.timeoutMs);
|
|
223
|
+
return inner.execute(options?.maxBatchLength, options?.timeoutMs, withRowId);
|
|
164
224
|
}
|
|
165
225
|
/**
|
|
166
226
|
* Execute the query and return the results as an @see {@link AsyncIterator}
|
|
@@ -183,15 +243,58 @@ class QueryBase {
|
|
|
183
243
|
[Symbol.asyncIterator]() {
|
|
184
244
|
return RecordBatchIterator(this.nativeExecute());
|
|
185
245
|
}
|
|
186
|
-
/**
|
|
246
|
+
/**
|
|
247
|
+
* Collect the results as an Arrow @see {@link ArrowTable}.
|
|
248
|
+
*
|
|
249
|
+
* Empty results retain the query's output schema.
|
|
250
|
+
*/
|
|
187
251
|
async toArrow(options) {
|
|
252
|
+
const context = this.blobBytesContext(options?.blobMode);
|
|
253
|
+
if (context === undefined) {
|
|
254
|
+
return this.collect(options, false);
|
|
255
|
+
}
|
|
256
|
+
for (let attempt = 1;; attempt++) {
|
|
257
|
+
// The snapshot pins the bytes. The query runs on the live table, so an
|
|
258
|
+
// unchanged version afterwards proves it read the snapshot's version.
|
|
259
|
+
const snapshot = await context.table.querySnapshot();
|
|
260
|
+
const version = await snapshot.version();
|
|
261
|
+
const sources = blobSources(await snapshot.blobColumns(), context.projection);
|
|
262
|
+
if (sources.size === 0) {
|
|
263
|
+
return this.collect(options, false);
|
|
264
|
+
}
|
|
265
|
+
const addRowId = !context.withRowId;
|
|
266
|
+
const table = await this.collect(options, addRowId);
|
|
267
|
+
if ((await context.table.version()) === version) {
|
|
268
|
+
return readBlobBytes(table, sources, snapshot, addRowId);
|
|
269
|
+
}
|
|
270
|
+
if (attempt === BLOB_BYTES_ATTEMPTS) {
|
|
271
|
+
throw new Error(`blobMode "bytes": the table changed during each of ${BLOB_BYTES_ATTEMPTS} attempts to read it`);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
async collect(options, withRowId) {
|
|
188
276
|
const batches = [];
|
|
189
|
-
const
|
|
190
|
-
for await (const batch of
|
|
277
|
+
const iterator = this.nativeExecute(options, withRowId);
|
|
278
|
+
for await (const batch of RecordBatchIterator(iterator)) {
|
|
191
279
|
batches.push(batch);
|
|
192
280
|
}
|
|
281
|
+
if (batches.length === 0) {
|
|
282
|
+
return (0, arrow_1.tableFromIPC)((await iterator).schema());
|
|
283
|
+
}
|
|
193
284
|
return new arrow_1.Table(batches);
|
|
194
285
|
}
|
|
286
|
+
blobBytesContext(mode) {
|
|
287
|
+
if (mode === undefined || mode === "descriptions") {
|
|
288
|
+
return undefined;
|
|
289
|
+
}
|
|
290
|
+
if (mode !== "bytes") {
|
|
291
|
+
throw new Error(`blobMode must be "descriptions" or "bytes", got "${String(mode)}"`);
|
|
292
|
+
}
|
|
293
|
+
if (this.blobContext === undefined) {
|
|
294
|
+
throw new Error('blobMode "bytes" is not supported for this query');
|
|
295
|
+
}
|
|
296
|
+
return this.blobContext;
|
|
297
|
+
}
|
|
195
298
|
/** Collect the results as an array of objects. */
|
|
196
299
|
// biome-ignore lint/suspicious/noExplicitAny: arrow.toArrow() returns any[]
|
|
197
300
|
async toArray(options) {
|
|
@@ -269,8 +372,8 @@ class QueryBase {
|
|
|
269
372
|
}
|
|
270
373
|
exports.QueryBase = QueryBase;
|
|
271
374
|
class StandardQueryBase extends QueryBase {
|
|
272
|
-
constructor(inner) {
|
|
273
|
-
super(inner);
|
|
375
|
+
constructor(inner, blobContext) {
|
|
376
|
+
super(inner, blobContext);
|
|
274
377
|
}
|
|
275
378
|
/**
|
|
276
379
|
* A filter statement to be applied to this query.
|
|
@@ -366,26 +469,26 @@ class StandardQueryBase extends QueryBase {
|
|
|
366
469
|
return this;
|
|
367
470
|
}
|
|
368
471
|
/**
|
|
369
|
-
* Control
|
|
472
|
+
* Control RTE read routing for this query.
|
|
370
473
|
*
|
|
371
|
-
* By default (unset), when the table carries
|
|
372
|
-
* {@link Table#
|
|
474
|
+
* By default (unset), when the table carries an RTE write spec (see
|
|
475
|
+
* {@link Table#enableRTE}), reads are routed through the LSM scanner so
|
|
373
476
|
* they also return data written via the `mergeInsert` LSM path that has not yet
|
|
374
477
|
* been compacted into the base table (the active/frozen in-memory memtables and
|
|
375
|
-
* the
|
|
478
|
+
* the SSTables), deduplicated by primary key; a table without a spec
|
|
376
479
|
* reads the base table.
|
|
377
480
|
*
|
|
378
481
|
* @param enable - `true` forces the LSM scanner and errors if the table has no
|
|
379
|
-
*
|
|
482
|
+
* RTE write spec. `false` bypasses the RTE and reads the base table only,
|
|
380
483
|
* even when a spec is present.
|
|
381
484
|
*
|
|
382
485
|
* Note: the LSM scanner does not support every query shape (e.g. reranking,
|
|
383
|
-
* hybrid search, `orderBy`). On
|
|
384
|
-
* `
|
|
385
|
-
* un-compacted
|
|
486
|
+
* hybrid search, `orderBy`). On an RTE table those shapes error unless
|
|
487
|
+
* `useRTE(false)` is set, because a base-only read would silently exclude
|
|
488
|
+
* un-compacted RTE data.
|
|
386
489
|
*/
|
|
387
|
-
|
|
388
|
-
this.doCall((inner) => inner.
|
|
490
|
+
useRTE(enable) {
|
|
491
|
+
this.doCall((inner) => inner.useRTE(enable));
|
|
389
492
|
return this;
|
|
390
493
|
}
|
|
391
494
|
}
|
|
@@ -403,8 +506,8 @@ class VectorQuery extends StandardQueryBase {
|
|
|
403
506
|
/**
|
|
404
507
|
* @hidden
|
|
405
508
|
*/
|
|
406
|
-
constructor(inner) {
|
|
407
|
-
super(inner);
|
|
509
|
+
constructor(inner, blobContext) {
|
|
510
|
+
super(inner, blobContext);
|
|
408
511
|
}
|
|
409
512
|
/**
|
|
410
513
|
* @hidden
|
|
@@ -415,6 +518,8 @@ class VectorQuery extends StandardQueryBase {
|
|
|
415
518
|
/**
|
|
416
519
|
* Set the number of partitions to search (probe)
|
|
417
520
|
*
|
|
521
|
+
* The number of probes must be greater than 0.
|
|
522
|
+
*
|
|
418
523
|
* This argument is only used when the vector column has an IVF PQ index.
|
|
419
524
|
* If there is no index then this value is ignored.
|
|
420
525
|
*
|
|
@@ -426,22 +531,41 @@ class VectorQuery extends StandardQueryBase {
|
|
|
426
531
|
* partitions should be searched.
|
|
427
532
|
*
|
|
428
533
|
* Increasing this value will increase the recall of your query but will
|
|
429
|
-
* also increase the latency of your query.
|
|
430
|
-
*
|
|
431
|
-
* your data and the recall that you need to achieve.
|
|
534
|
+
* also increase the latency of your query. If this method is not called,
|
|
535
|
+
* Lance's adaptive probe defaults are used.
|
|
432
536
|
*
|
|
433
537
|
* For best results we recommend tuning this parameter with a benchmark against
|
|
434
538
|
* your actual data to find the smallest possible value that will still give
|
|
435
539
|
* you the desired recall.
|
|
436
540
|
*
|
|
437
|
-
*
|
|
438
|
-
*
|
|
439
|
-
*
|
|
541
|
+
* The value is retained as `nprobes` through client and server request
|
|
542
|
+
* construction. Lance sets both probe bounds to this value. Explicit minimum
|
|
543
|
+
* or maximum settings can override their respective bounds.
|
|
440
544
|
*/
|
|
441
545
|
nprobes(nprobes) {
|
|
442
546
|
this.doVectorCall((inner) => inner.nprobes(nprobes));
|
|
443
547
|
return this;
|
|
444
548
|
}
|
|
549
|
+
/**
|
|
550
|
+
* Set the initial IVF search effort to a finite value in [0, 1].
|
|
551
|
+
*
|
|
552
|
+
* The default, null, uses Auto (internally 0.5). Zero starts at the caller
|
|
553
|
+
* minimum; one searches all partitions within the caller maximum. Later
|
|
554
|
+
* probing may continue when too few candidates are found.
|
|
555
|
+
* Any explicit effort, including 0.5, overrides nprobes in either call order.
|
|
556
|
+
* Explicit minimum and maximum bounds still apply. This is not a recall
|
|
557
|
+
* guarantee and does not change refinement or HNSW ef. The RTE LSM
|
|
558
|
+
* scanner does not yet support this option.
|
|
559
|
+
*
|
|
560
|
+
* @example
|
|
561
|
+
* ```ts
|
|
562
|
+
* table.vectorSearch([0.1, 0.2]).searchEffort(0.75).limit(10);
|
|
563
|
+
* ```
|
|
564
|
+
*/
|
|
565
|
+
searchEffort(searchEffort) {
|
|
566
|
+
this.doVectorCall((inner) => inner.searchEffort(searchEffort));
|
|
567
|
+
return this;
|
|
568
|
+
}
|
|
445
569
|
/**
|
|
446
570
|
* Set the minimum number of probes used.
|
|
447
571
|
*
|
|
@@ -618,7 +742,7 @@ class VectorQuery extends StandardQueryBase {
|
|
|
618
742
|
addQueryVectorToNative(inner, outcome.value);
|
|
619
743
|
return inner;
|
|
620
744
|
})();
|
|
621
|
-
return new VectorQuery(res);
|
|
745
|
+
return new VectorQuery(res, this.blobContext);
|
|
622
746
|
}
|
|
623
747
|
else {
|
|
624
748
|
this.doVectorCall((inner) => addQueryVectorToNative(inner, vector));
|
|
@@ -680,7 +804,7 @@ function createAutoQuery(table, query, columns, getVector) {
|
|
|
680
804
|
}
|
|
681
805
|
return nearestToNative(route.table.query(), vector);
|
|
682
806
|
};
|
|
683
|
-
return new AutoQuery(createInner);
|
|
807
|
+
return new AutoQuery(createInner, { table, withRowId: false });
|
|
684
808
|
}
|
|
685
809
|
/**
|
|
686
810
|
* A query that returns a subset of the rows in the table.
|
|
@@ -688,20 +812,20 @@ function createAutoQuery(table, query, columns, getVector) {
|
|
|
688
812
|
* @hideconstructor
|
|
689
813
|
*/
|
|
690
814
|
class TakeQuery extends QueryBase {
|
|
691
|
-
constructor(inner) {
|
|
692
|
-
super(inner);
|
|
815
|
+
constructor(inner, blobContext) {
|
|
816
|
+
super(inner, blobContext);
|
|
693
817
|
}
|
|
694
818
|
/**
|
|
695
|
-
* Control
|
|
819
|
+
* Control RTE read routing for this take query.
|
|
696
820
|
*
|
|
697
|
-
* `false` bypasses the
|
|
821
|
+
* `false` bypasses the RTE and reads the base table only — the escape hatch,
|
|
698
822
|
* since take-by-row-id/offset is not supported on the LSM scanner and, on a
|
|
699
|
-
*
|
|
823
|
+
* RTE table, auto-routes to it and errors otherwise.
|
|
700
824
|
*
|
|
701
825
|
* @param enable - `false` reads the base table only.
|
|
702
826
|
*/
|
|
703
|
-
|
|
704
|
-
this.doCall((inner) => inner.
|
|
827
|
+
useRTE(enable) {
|
|
828
|
+
this.doCall((inner) => inner.useRTE(enable));
|
|
705
829
|
return this;
|
|
706
830
|
}
|
|
707
831
|
}
|
|
@@ -719,8 +843,8 @@ class AutoQuery extends StandardQueryBase {
|
|
|
719
843
|
createInner;
|
|
720
844
|
calls = [];
|
|
721
845
|
/** @hidden */
|
|
722
|
-
constructor(createInner) {
|
|
723
|
-
super();
|
|
846
|
+
constructor(createInner, blobContext) {
|
|
847
|
+
super(undefined, blobContext);
|
|
724
848
|
this.createInner = createInner;
|
|
725
849
|
}
|
|
726
850
|
/** @hidden */
|
|
@@ -749,7 +873,7 @@ class Query extends StandardQueryBase {
|
|
|
749
873
|
* @hidden
|
|
750
874
|
*/
|
|
751
875
|
constructor(tbl) {
|
|
752
|
-
super(tbl.query());
|
|
876
|
+
super(tbl.query(), { table: tbl, withRowId: false });
|
|
753
877
|
}
|
|
754
878
|
/**
|
|
755
879
|
* Find the nearest vectors to the given query vector.
|
|
@@ -792,12 +916,12 @@ class Query extends StandardQueryBase {
|
|
|
792
916
|
const inner = this.inner;
|
|
793
917
|
if (inner instanceof Promise) {
|
|
794
918
|
const nativeQuery = inner.then(async (resolvedInner) => nearestToNative(resolvedInner, await vector));
|
|
795
|
-
return new VectorQuery(nativeQuery);
|
|
919
|
+
return new VectorQuery(nativeQuery, this.blobContext);
|
|
796
920
|
}
|
|
797
921
|
if (vector instanceof Promise) {
|
|
798
|
-
return new VectorQuery(vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)));
|
|
922
|
+
return new VectorQuery(vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)), this.blobContext);
|
|
799
923
|
}
|
|
800
|
-
return new VectorQuery(nearestToNative(inner, vector));
|
|
924
|
+
return new VectorQuery(nearestToNative(inner, vector), this.blobContext);
|
|
801
925
|
}
|
|
802
926
|
nearestToText(query, columns) {
|
|
803
927
|
this.doCall((inner) => {
|
package/dist/sanitize.d.ts
CHANGED
|
@@ -1,23 +1,24 @@
|
|
|
1
|
+
import * as arrowModule from "apache-arrow";
|
|
1
2
|
import type { TKeys } from "apache-arrow/type";
|
|
2
|
-
import { DataType,
|
|
3
|
+
import { DataType, DenseUnion, Field, Schema, SchemaLike, SparseUnion, Table, TableLike, TimestampMicrosecond, TimestampMillisecond, TimestampNanosecond, TimestampSecond } from "./arrow";
|
|
3
4
|
export declare function sanitizeMetadata(metadataLike?: unknown): Map<string, string> | undefined;
|
|
4
|
-
export declare function sanitizeInt(typeLike: object): Int<Type.Int | Type.Int8 | Type.Int16 | Type.Int32 | Type.Int64 | Type.Uint8 | Type.Uint16 | Type.Uint32 | Type.Uint64>;
|
|
5
|
-
export declare function sanitizeFloat(typeLike: object): Float<Type.Float | Type.Float16 | Type.Float32 | Type.Float64>;
|
|
6
|
-
export declare function sanitizeDecimal(typeLike: object): Decimal;
|
|
7
|
-
export declare function sanitizeDate(typeLike: object): Date_<import("apache-arrow/type").Dates>;
|
|
8
|
-
export declare function sanitizeTime(typeLike: object): Time<Type.Time | Type.TimeSecond | Type.TimeMillisecond | Type.TimeMicrosecond | Type.TimeNanosecond>;
|
|
9
|
-
export declare function sanitizeTimestamp(typeLike: object): Timestamp<Type.Timestamp | Type.TimestampSecond | Type.TimestampMillisecond | Type.TimestampMicrosecond | Type.TimestampNanosecond>;
|
|
10
|
-
export declare function sanitizeTypedTimestamp(typeLike: object, Datatype: typeof TimestampNanosecond | typeof TimestampMicrosecond | typeof TimestampMillisecond | typeof TimestampSecond):
|
|
11
|
-
export declare function sanitizeInterval(typeLike: object): Interval<Type.Interval | Type.IntervalDayTime | Type.IntervalYearMonth>;
|
|
12
|
-
export declare function sanitizeList(typeLike: object): List<any>;
|
|
13
|
-
export declare function sanitizeStruct(typeLike: object): Struct<any>;
|
|
14
|
-
export declare function sanitizeUnion(typeLike: object): Union<Type.Union | Type.DenseUnion | Type.SparseUnion>;
|
|
15
|
-
export declare function sanitizeTypedUnion(typeLike: object, UnionType: typeof DenseUnion | typeof SparseUnion):
|
|
16
|
-
export declare function sanitizeFixedSizeBinary(typeLike: object): FixedSizeBinary;
|
|
17
|
-
export declare function sanitizeFixedSizeList(typeLike: object): FixedSizeList<any>;
|
|
18
|
-
export declare function sanitizeMap(typeLike: object): Map_<any, any>;
|
|
19
|
-
export declare function sanitizeDuration(typeLike: object): Duration<Type.Duration | Type.DurationSecond | Type.DurationMillisecond | Type.DurationMicrosecond | Type.DurationNanosecond>;
|
|
20
|
-
export declare function sanitizeDictionary(typeLike: object): Dictionary<DataType<Type, any>, TKeys>;
|
|
5
|
+
export declare function sanitizeInt(typeLike: object): arrowModule.Int<arrowModule.Type.Int | arrowModule.Type.Int8 | arrowModule.Type.Int16 | arrowModule.Type.Int32 | arrowModule.Type.Int64 | arrowModule.Type.Uint8 | arrowModule.Type.Uint16 | arrowModule.Type.Uint32 | arrowModule.Type.Uint64>;
|
|
6
|
+
export declare function sanitizeFloat(typeLike: object): arrowModule.Float<arrowModule.Type.Float | arrowModule.Type.Float16 | arrowModule.Type.Float32 | arrowModule.Type.Float64>;
|
|
7
|
+
export declare function sanitizeDecimal(typeLike: object): arrowModule.Decimal;
|
|
8
|
+
export declare function sanitizeDate(typeLike: object): arrowModule.Date_<import("apache-arrow/type").Dates>;
|
|
9
|
+
export declare function sanitizeTime(typeLike: object): arrowModule.Time<arrowModule.Type.Time | arrowModule.Type.TimeSecond | arrowModule.Type.TimeMillisecond | arrowModule.Type.TimeMicrosecond | arrowModule.Type.TimeNanosecond>;
|
|
10
|
+
export declare function sanitizeTimestamp(typeLike: object): arrowModule.Timestamp<arrowModule.Type.Timestamp | arrowModule.Type.TimestampSecond | arrowModule.Type.TimestampMillisecond | arrowModule.Type.TimestampMicrosecond | arrowModule.Type.TimestampNanosecond>;
|
|
11
|
+
export declare function sanitizeTypedTimestamp(typeLike: object, Datatype: typeof TimestampNanosecond | typeof TimestampMicrosecond | typeof TimestampMillisecond | typeof TimestampSecond): arrowModule.TimestampNanosecond | arrowModule.TimestampMicrosecond | arrowModule.TimestampMillisecond | arrowModule.TimestampSecond;
|
|
12
|
+
export declare function sanitizeInterval(typeLike: object): arrowModule.Interval<arrowModule.Type.Interval | arrowModule.Type.IntervalDayTime | arrowModule.Type.IntervalYearMonth | arrowModule.Type.IntervalMonthDayNano>;
|
|
13
|
+
export declare function sanitizeList(typeLike: object): arrowModule.List<any>;
|
|
14
|
+
export declare function sanitizeStruct(typeLike: object): arrowModule.Struct<any>;
|
|
15
|
+
export declare function sanitizeUnion(typeLike: object): arrowModule.Union<arrowModule.Type.Union | arrowModule.Type.DenseUnion | arrowModule.Type.SparseUnion>;
|
|
16
|
+
export declare function sanitizeTypedUnion(typeLike: object, UnionType: typeof DenseUnion | typeof SparseUnion): arrowModule.DenseUnion | arrowModule.SparseUnion;
|
|
17
|
+
export declare function sanitizeFixedSizeBinary(typeLike: object): arrowModule.FixedSizeBinary;
|
|
18
|
+
export declare function sanitizeFixedSizeList(typeLike: object): arrowModule.FixedSizeList<any>;
|
|
19
|
+
export declare function sanitizeMap(typeLike: object): arrowModule.Map_<any, any>;
|
|
20
|
+
export declare function sanitizeDuration(typeLike: object): arrowModule.Duration<arrowModule.Type.Duration | arrowModule.Type.DurationSecond | arrowModule.Type.DurationMillisecond | arrowModule.Type.DurationMicrosecond | arrowModule.Type.DurationNanosecond>;
|
|
21
|
+
export declare function sanitizeDictionary(typeLike: object): arrowModule.Dictionary<arrowModule.DataType<arrowModule.Type, any>, TKeys>;
|
|
21
22
|
export declare function sanitizeType(typeLike: unknown): DataType<any>;
|
|
22
23
|
export declare function sanitizeField(fieldLike: unknown): Field;
|
|
23
24
|
/**
|
package/dist/sanitize.js
CHANGED
|
@@ -32,8 +32,42 @@ exports.dataTypeFromName = dataTypeFromName;
|
|
|
32
32
|
// time. However, arrow-js uses instanceof which expected that the input
|
|
33
33
|
// comes from the exact same library instance. This is not always the case
|
|
34
34
|
// and so we must sanitize the input to ensure that it is compatible.
|
|
35
|
+
const arrowModule = require("apache-arrow");
|
|
35
36
|
const apache_arrow_1 = require("apache-arrow");
|
|
36
37
|
const arrow_1 = require("./arrow");
|
|
38
|
+
/**
|
|
39
|
+
* Whether `value` was created by *this* copy of apache-arrow's `ctor`.
|
|
40
|
+
*
|
|
41
|
+
* Arrow 21 overrides `Symbol.hasInstance` on `Schema`, `Field`, `Table`,
|
|
42
|
+
* `RecordBatch`, `Vector` and `Data` so that `instanceof` also matches objects
|
|
43
|
+
* built by a different copy of the library. Those are exactly the objects
|
|
44
|
+
* this module has to rebuild: the `DataType` instances nested inside them are
|
|
45
|
+
* deliberately *not* given that treatment upstream, so Arrow's own builders and
|
|
46
|
+
* writers reject them. Walk the real prototype chain instead of trusting
|
|
47
|
+
* `instanceof`.
|
|
48
|
+
*/
|
|
49
|
+
function isOwnInstance(value,
|
|
50
|
+
// biome-ignore lint/suspicious/noExplicitAny: matches any class constructor
|
|
51
|
+
ctor) {
|
|
52
|
+
return (typeof value === "object" &&
|
|
53
|
+
value !== null &&
|
|
54
|
+
Object.prototype.isPrototypeOf.call(ctor.prototype, value));
|
|
55
|
+
}
|
|
56
|
+
const NEWER_TYPES = [
|
|
57
|
+
{ name: "IntervalMonthDayNano", typeId: -31, since: 20 },
|
|
58
|
+
{ name: "LargeList", typeId: 21, since: 21 },
|
|
59
|
+
{ name: "BinaryView", typeId: 23, since: 21 },
|
|
60
|
+
{ name: "Utf8View", typeId: 24, since: 21 },
|
|
61
|
+
];
|
|
62
|
+
function newerTypeConstructor(name) {
|
|
63
|
+
const ctor = arrowModule[name];
|
|
64
|
+
if (typeof ctor !== "function") {
|
|
65
|
+
const since = NEWER_TYPES.find((t) => t.name === name)?.since;
|
|
66
|
+
throw new Error(`The Arrow type ${name} requires apache-arrow ${since} or newer, ` +
|
|
67
|
+
"but the installed apache-arrow does not provide it");
|
|
68
|
+
}
|
|
69
|
+
return ctor;
|
|
70
|
+
}
|
|
37
71
|
function createSanitizationContext() {
|
|
38
72
|
return {
|
|
39
73
|
types: new WeakMap(),
|
|
@@ -130,14 +164,21 @@ function sanitizeInterval(typeLike) {
|
|
|
130
164
|
function sanitizeList(typeLike) {
|
|
131
165
|
return sanitizeListWithContext(typeLike, createSanitizationContext());
|
|
132
166
|
}
|
|
133
|
-
function
|
|
167
|
+
function sanitizeListChildWithContext(typeLike, context, typeName) {
|
|
134
168
|
if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
|
|
135
|
-
throw Error(
|
|
169
|
+
throw Error(`Expected a ${typeName} type to have an array-like \`children\` property`);
|
|
136
170
|
}
|
|
137
171
|
if (typeLike.children.length !== 1) {
|
|
138
|
-
throw Error(
|
|
172
|
+
throw Error(`Expected a ${typeName} type to have exactly one child`);
|
|
139
173
|
}
|
|
140
|
-
return
|
|
174
|
+
return sanitizeFieldWithContext(typeLike.children[0], context);
|
|
175
|
+
}
|
|
176
|
+
function sanitizeListWithContext(typeLike, context) {
|
|
177
|
+
return new arrow_1.List(sanitizeListChildWithContext(typeLike, context, "List"));
|
|
178
|
+
}
|
|
179
|
+
function sanitizeLargeListWithContext(typeLike, context) {
|
|
180
|
+
const LargeList = newerTypeConstructor("LargeList");
|
|
181
|
+
return new LargeList(sanitizeListChildWithContext(typeLike, context, "LargeList"));
|
|
141
182
|
}
|
|
142
183
|
function sanitizeStruct(typeLike) {
|
|
143
184
|
return sanitizeStructWithContext(typeLike, createSanitizationContext());
|
|
@@ -274,7 +315,21 @@ function sanitizeTypeWithContext(typeLike, context) {
|
|
|
274
315
|
context.types.set(typeLike, type);
|
|
275
316
|
return type;
|
|
276
317
|
}
|
|
318
|
+
function sanitizeNewerTypeById(typeLike, typeId, context) {
|
|
319
|
+
const entry = NEWER_TYPES.find((t) => t.typeId === typeId);
|
|
320
|
+
if (entry === undefined) {
|
|
321
|
+
return undefined;
|
|
322
|
+
}
|
|
323
|
+
if (entry.name === "LargeList") {
|
|
324
|
+
return sanitizeLargeListWithContext(typeLike, context);
|
|
325
|
+
}
|
|
326
|
+
return new (newerTypeConstructor(entry.name))();
|
|
327
|
+
}
|
|
277
328
|
function sanitizeTypeById(typeLike, typeId, context) {
|
|
329
|
+
const newer = sanitizeNewerTypeById(typeLike, typeId, context);
|
|
330
|
+
if (newer !== undefined) {
|
|
331
|
+
return newer;
|
|
332
|
+
}
|
|
278
333
|
switch (typeId) {
|
|
279
334
|
case arrow_1.Type.NONE:
|
|
280
335
|
throw Error("Received a Type with a typeId of NONE");
|
|
@@ -386,7 +441,7 @@ function sanitizeField(fieldLike) {
|
|
|
386
441
|
return sanitizeFieldWithContext(fieldLike, createSanitizationContext());
|
|
387
442
|
}
|
|
388
443
|
function sanitizeFieldWithContext(fieldLike, context) {
|
|
389
|
-
if (fieldLike
|
|
444
|
+
if (isOwnInstance(fieldLike, arrow_1.Field)) {
|
|
390
445
|
return fieldLike;
|
|
391
446
|
}
|
|
392
447
|
if (typeof fieldLike !== "object" || fieldLike === null) {
|
|
@@ -429,7 +484,7 @@ function sanitizeSchema(schemaLike) {
|
|
|
429
484
|
return sanitizeSchemaWithContext(schemaLike, createSanitizationContext());
|
|
430
485
|
}
|
|
431
486
|
function sanitizeSchemaWithContext(schemaLike, context) {
|
|
432
|
-
if (schemaLike
|
|
487
|
+
if (isOwnInstance(schemaLike, arrow_1.Schema)) {
|
|
433
488
|
return schemaLike;
|
|
434
489
|
}
|
|
435
490
|
if (typeof schemaLike !== "object" || schemaLike === null) {
|
|
@@ -449,7 +504,7 @@ function sanitizeSchemaWithContext(schemaLike, context) {
|
|
|
449
504
|
return new arrow_1.Schema(sanitizedFields, metadata);
|
|
450
505
|
}
|
|
451
506
|
function sanitizeTable(tableLike) {
|
|
452
|
-
if (tableLike
|
|
507
|
+
if (isOwnInstance(tableLike, arrow_1.Table)) {
|
|
453
508
|
return tableLike;
|
|
454
509
|
}
|
|
455
510
|
if (typeof tableLike !== "object" || tableLike === null) {
|
|
@@ -467,7 +522,7 @@ function sanitizeTable(tableLike) {
|
|
|
467
522
|
return new arrow_1.Table(schema, batches);
|
|
468
523
|
}
|
|
469
524
|
function sanitizeRecordBatch(batchLike, context) {
|
|
470
|
-
if (batchLike
|
|
525
|
+
if (isOwnInstance(batchLike, arrow_1.RecordBatch)) {
|
|
471
526
|
return batchLike;
|
|
472
527
|
}
|
|
473
528
|
if (typeof batchLike !== "object" || batchLike === null) {
|
|
@@ -484,7 +539,7 @@ function sanitizeRecordBatch(batchLike, context) {
|
|
|
484
539
|
return new arrow_1.RecordBatch(schema, data);
|
|
485
540
|
}
|
|
486
541
|
function sanitizeData(dataLike, context) {
|
|
487
|
-
if (dataLike
|
|
542
|
+
if (isOwnInstance(dataLike, apache_arrow_1.Data)) {
|
|
488
543
|
return dataLike;
|
|
489
544
|
}
|
|
490
545
|
const cachedData = context.data.get(dataLike);
|
|
@@ -500,19 +555,26 @@ function sanitizeData(dataLike, context) {
|
|
|
500
555
|
context.vectors.set(dictionaryLike, dictionary);
|
|
501
556
|
}
|
|
502
557
|
}
|
|
503
|
-
|
|
558
|
+
// Arrow 21 added an eighth constructor parameter (`variadicBuffers`); older
|
|
559
|
+
// releases ignore the extra argument but their typings do not declare it.
|
|
560
|
+
const DataCtor = apache_arrow_1.Data;
|
|
561
|
+
const data = new DataCtor(sanitizeTypeWithContext(dataLike.type, context), dataLike.offset, dataLike.length, dataLike.nullCount, {
|
|
504
562
|
[apache_arrow_1.BufferType.OFFSET]: dataLike.valueOffsets,
|
|
505
563
|
[apache_arrow_1.BufferType.DATA]: dataLike.values,
|
|
506
564
|
[apache_arrow_1.BufferType.VALIDITY]: dataLike.nullBitmap,
|
|
507
565
|
[apache_arrow_1.BufferType.TYPE]: dataLike.typeIds,
|
|
508
|
-
}, dataLike.children.map((child) => sanitizeData(child, context)), dictionary);
|
|
566
|
+
}, dataLike.children.map((child) => sanitizeData(child, context)), dictionary, dataLike.variadicBuffers);
|
|
509
567
|
context.data.set(dataLike, data);
|
|
510
568
|
return data;
|
|
511
569
|
}
|
|
512
570
|
const constructorsByTypeName = {
|
|
513
571
|
null: () => new arrow_1.Null(),
|
|
514
572
|
binary: () => new arrow_1.Binary(),
|
|
573
|
+
largebinary: () => new arrow_1.LargeBinary(),
|
|
574
|
+
binaryview: () => new (newerTypeConstructor("BinaryView"))(),
|
|
515
575
|
utf8: () => new arrow_1.Utf8(),
|
|
576
|
+
largeutf8: () => new arrow_1.LargeUtf8(),
|
|
577
|
+
utf8view: () => new (newerTypeConstructor("Utf8View"))(),
|
|
516
578
|
bool: () => new arrow_1.Bool(),
|
|
517
579
|
int8: () => new arrow_1.Int8(),
|
|
518
580
|
int16: () => new arrow_1.Int16(),
|
|
@@ -533,6 +595,7 @@ const constructorsByTypeName = {
|
|
|
533
595
|
timesecond: () => new arrow_1.TimeSecond(),
|
|
534
596
|
intervaldaytime: () => new arrow_1.IntervalDayTime(),
|
|
535
597
|
intervalyearmonth: () => new arrow_1.IntervalYearMonth(),
|
|
598
|
+
intervalmonthdaynano: () => new (newerTypeConstructor("IntervalMonthDayNano"))(),
|
|
536
599
|
durationnanosecond: () => new arrow_1.DurationNanosecond(),
|
|
537
600
|
durationmicrosecond: () => new arrow_1.DurationMicrosecond(),
|
|
538
601
|
durationmillisecond: () => new arrow_1.DurationMillisecond(),
|