@lancedb/lancedb 0.40.0 → 0.41.0-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/query.js CHANGED
@@ -21,17 +21,61 @@ async function* RecordBatchIterator(promisedInner) {
21
21
  yield batches[0];
22
22
  }
23
23
  }
24
- class RecordBatchIterable {
25
- inner;
26
- options;
27
- constructor(inner, options) {
28
- this.inner = inner;
29
- this.options = options;
24
+ const BLOB_BYTES_ATTEMPTS = 3;
25
+ function blobSources(blobColumns, projection) {
26
+ const columns = new Set(blobColumns);
27
+ const pairs = projection ??
28
+ blobColumns.map((column) => [column, column]);
29
+ return new Map(pairs.filter(([, source]) => columns.has(source)));
30
+ }
31
+ async function readBlobBytes(table, sources, snapshot, dropRowId) {
32
+ const rowIdColumn = table.getChild("_rowid");
33
+ if (table.numRows > 0 && rowIdColumn === null) {
34
+ throw new Error('blobMode "bytes" needs _rowid in the query result');
30
35
  }
31
- // biome-ignore lint/suspicious/noExplicitAny: skip
32
- [Symbol.asyncIterator]() {
33
- return RecordBatchIterator(this.inner.execute(this.options?.maxBatchLength, this.options?.timeoutMs));
36
+ const rowIds = rowIdColumn === null
37
+ ? []
38
+ : Array.from(rowIdColumn.toArray());
39
+ const replacements = new Map();
40
+ for (const [output, source] of sources) {
41
+ const index = table.schema.fields.findIndex((field) => field.name === output);
42
+ if (index < 0) {
43
+ continue;
44
+ }
45
+ const values = rowIds.length === 0 ? [] : await snapshot.fetchBlobs(source, rowIds);
46
+ replacements.set(index, values.map((value) => value ?? null));
34
47
  }
48
+ const result = withBinaryColumns(table, replacements);
49
+ if (!dropRowId) {
50
+ return result;
51
+ }
52
+ return result.select(result.schema.fields
53
+ .map((field) => field.name)
54
+ .filter((name) => name !== "_rowid"));
55
+ }
56
+ // A fresh field, not a clone: the descriptor field's metadata must not follow
57
+ // the bytes.
58
+ function withBinaryColumns(table, replacements) {
59
+ if (replacements.size === 0) {
60
+ return table;
61
+ }
62
+ const fields = table.schema.fields.map((field, index) => replacements.has(index)
63
+ ? new arrow_1.Field(field.name, new arrow_1.LargeBinary(), true)
64
+ : field);
65
+ const schema = new arrow_1.Schema(fields, table.schema.metadata);
66
+ let offset = 0;
67
+ const batches = table.batches.map((batch) => {
68
+ const children = batch.data.children.map((child, index) => {
69
+ const values = replacements.get(index);
70
+ if (values === undefined) {
71
+ return child;
72
+ }
73
+ return (0, arrow_1.vectorFromArray)(values.slice(offset, offset + batch.numRows), new arrow_1.LargeBinary()).data[0];
74
+ });
75
+ offset += batch.numRows;
76
+ return new arrow_1.RecordBatch(schema, (0, arrow_1.makeData)({ type: new arrow_1.Struct(fields), length: batch.numRows, children }));
77
+ });
78
+ return new arrow_1.Table(schema, batches);
35
79
  }
36
80
  function nearestToNative(inner, vector) {
37
81
  const raw = Array.isArray(vector) ? null : (0, arrow_1.extractVectorBuffer)(vector);
@@ -61,10 +105,15 @@ class QueryBase {
61
105
  /**
62
106
  * @hidden
63
107
  */
64
- constructor(inner) {
108
+ blobContext;
109
+ /**
110
+ * @hidden
111
+ */
112
+ constructor(inner, blobContext) {
65
113
  if (inner !== undefined) {
66
114
  this.inner = inner;
67
115
  }
116
+ this.blobContext = blobContext;
68
117
  }
69
118
  // call a function on the inner (either a promise or the actual object)
70
119
  /**
@@ -130,17 +179,25 @@ class QueryBase {
130
179
  inner.select(columnTuples);
131
180
  });
132
181
  };
182
+ let projection;
133
183
  if (typeof columns === "string") {
134
184
  selectColumns([columns]);
185
+ projection = [[columns, columns]];
135
186
  }
136
187
  else if (Array.isArray(columns)) {
137
188
  selectColumns(columns);
189
+ projection = columns.map((column) => [column, column]);
138
190
  }
139
191
  else if (columns instanceof Map) {
140
- selectMapping(Array.from(columns.entries()));
192
+ projection = Array.from(columns.entries());
193
+ selectMapping(projection);
141
194
  }
142
195
  else {
143
- selectMapping(Object.entries(columns));
196
+ projection = Object.entries(columns);
197
+ selectMapping(projection);
198
+ }
199
+ if (this.blobContext !== undefined) {
200
+ this.blobContext = { ...this.blobContext, projection };
144
201
  }
145
202
  return this;
146
203
  }
@@ -153,14 +210,17 @@ class QueryBase {
153
210
  */
154
211
  withRowId() {
155
212
  this.doCall((inner) => inner.withRowId());
213
+ if (this.blobContext !== undefined) {
214
+ this.blobContext = { ...this.blobContext, withRowId: true };
215
+ }
156
216
  return this;
157
217
  }
158
218
  /**
159
219
  * @hidden
160
220
  */
161
- async nativeExecute(options) {
221
+ async nativeExecute(options, withRowId = false) {
162
222
  const inner = await this.getInner();
163
- return inner.execute(options?.maxBatchLength, options?.timeoutMs);
223
+ return inner.execute(options?.maxBatchLength, options?.timeoutMs, withRowId);
164
224
  }
165
225
  /**
166
226
  * Execute the query and return the results as an @see {@link AsyncIterator}
@@ -183,15 +243,58 @@ class QueryBase {
183
243
  [Symbol.asyncIterator]() {
184
244
  return RecordBatchIterator(this.nativeExecute());
185
245
  }
186
- /** Collect the results as an Arrow @see {@link ArrowTable}. */
246
+ /**
247
+ * Collect the results as an Arrow @see {@link ArrowTable}.
248
+ *
249
+ * Empty results retain the query's output schema.
250
+ */
187
251
  async toArrow(options) {
252
+ const context = this.blobBytesContext(options?.blobMode);
253
+ if (context === undefined) {
254
+ return this.collect(options, false);
255
+ }
256
+ for (let attempt = 1;; attempt++) {
257
+ // The snapshot pins the bytes. The query runs on the live table, so an
258
+ // unchanged version afterwards proves it read the snapshot's version.
259
+ const snapshot = await context.table.querySnapshot();
260
+ const version = await snapshot.version();
261
+ const sources = blobSources(await snapshot.blobColumns(), context.projection);
262
+ if (sources.size === 0) {
263
+ return this.collect(options, false);
264
+ }
265
+ const addRowId = !context.withRowId;
266
+ const table = await this.collect(options, addRowId);
267
+ if ((await context.table.version()) === version) {
268
+ return readBlobBytes(table, sources, snapshot, addRowId);
269
+ }
270
+ if (attempt === BLOB_BYTES_ATTEMPTS) {
271
+ throw new Error(`blobMode "bytes": the table changed during each of ${BLOB_BYTES_ATTEMPTS} attempts to read it`);
272
+ }
273
+ }
274
+ }
275
+ async collect(options, withRowId) {
188
276
  const batches = [];
189
- const inner = await this.getInner();
190
- for await (const batch of new RecordBatchIterable(inner, options)) {
277
+ const iterator = this.nativeExecute(options, withRowId);
278
+ for await (const batch of RecordBatchIterator(iterator)) {
191
279
  batches.push(batch);
192
280
  }
281
+ if (batches.length === 0) {
282
+ return (0, arrow_1.tableFromIPC)((await iterator).schema());
283
+ }
193
284
  return new arrow_1.Table(batches);
194
285
  }
286
+ blobBytesContext(mode) {
287
+ if (mode === undefined || mode === "descriptions") {
288
+ return undefined;
289
+ }
290
+ if (mode !== "bytes") {
291
+ throw new Error(`blobMode must be "descriptions" or "bytes", got "${String(mode)}"`);
292
+ }
293
+ if (this.blobContext === undefined) {
294
+ throw new Error('blobMode "bytes" is not supported for this query');
295
+ }
296
+ return this.blobContext;
297
+ }
195
298
  /** Collect the results as an array of objects. */
196
299
  // biome-ignore lint/suspicious/noExplicitAny: arrow.toArrow() returns any[]
197
300
  async toArray(options) {
@@ -269,8 +372,8 @@ class QueryBase {
269
372
  }
270
373
  exports.QueryBase = QueryBase;
271
374
  class StandardQueryBase extends QueryBase {
272
- constructor(inner) {
273
- super(inner);
375
+ constructor(inner, blobContext) {
376
+ super(inner, blobContext);
274
377
  }
275
378
  /**
276
379
  * A filter statement to be applied to this query.
@@ -366,26 +469,26 @@ class StandardQueryBase extends QueryBase {
366
469
  return this;
367
470
  }
368
471
  /**
369
- * Control MemWAL read routing for this query.
472
+ * Control RTE read routing for this query.
370
473
  *
371
- * By default (unset), when the table carries a MemWAL write spec (see
372
- * {@link Table#setLsmWriteSpec}), reads are routed through the LSM scanner so
474
+ * By default (unset), when the table carries an RTE write spec (see
475
+ * {@link Table#enableRTE}), reads are routed through the LSM scanner so
373
476
  * they also return data written via the `mergeInsert` LSM path that has not yet
374
477
  * been compacted into the base table (the active/frozen in-memory memtables and
375
- * the flushed generations), deduplicated by primary key; a table without a spec
478
+ * the SSTables), deduplicated by primary key; a table without a spec
376
479
  * reads the base table.
377
480
  *
378
481
  * @param enable - `true` forces the LSM scanner and errors if the table has no
379
- * MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
482
+ * RTE write spec. `false` bypasses the RTE and reads the base table only,
380
483
  * even when a spec is present.
381
484
  *
382
485
  * Note: the LSM scanner does not support every query shape (e.g. reranking,
383
- * hybrid search, `orderBy`). On a MemWAL table those shapes error unless
384
- * `useLsm(false)` is set, because a base-only read would silently exclude
385
- * un-compacted MemWAL data.
486
+ * hybrid search, `orderBy`). On an RTE table those shapes error unless
487
+ * `useRTE(false)` is set, because a base-only read would silently exclude
488
+ * un-compacted RTE data.
386
489
  */
387
- useLsm(enable) {
388
- this.doCall((inner) => inner.useLsm(enable));
490
+ useRTE(enable) {
491
+ this.doCall((inner) => inner.useRTE(enable));
389
492
  return this;
390
493
  }
391
494
  }
@@ -403,8 +506,8 @@ class VectorQuery extends StandardQueryBase {
403
506
  /**
404
507
  * @hidden
405
508
  */
406
- constructor(inner) {
407
- super(inner);
509
+ constructor(inner, blobContext) {
510
+ super(inner, blobContext);
408
511
  }
409
512
  /**
410
513
  * @hidden
@@ -415,6 +518,8 @@ class VectorQuery extends StandardQueryBase {
415
518
  /**
416
519
  * Set the number of partitions to search (probe)
417
520
  *
521
+ * The number of probes must be greater than 0.
522
+ *
418
523
  * This argument is only used when the vector column has an IVF PQ index.
419
524
  * If there is no index then this value is ignored.
420
525
  *
@@ -426,22 +531,41 @@ class VectorQuery extends StandardQueryBase {
426
531
  * partitions should be searched.
427
532
  *
428
533
  * Increasing this value will increase the recall of your query but will
429
- * also increase the latency of your query. The default value is 20. This
430
- * default is good for many cases but the best value to use will depend on
431
- * your data and the recall that you need to achieve.
534
+ * also increase the latency of your query. If this method is not called,
535
+ * Lance's adaptive probe defaults are used.
432
536
  *
433
537
  * For best results we recommend tuning this parameter with a benchmark against
434
538
  * your actual data to find the smallest possible value that will still give
435
539
  * you the desired recall.
436
540
  *
437
- * For more fine grained control over behavior when you have a very narrow filter
438
- * you can use `minimumNprobes` and `maximumNprobes`. This method sets both
439
- * the minimum and maximum to the same value.
541
+ * The value is retained as `nprobes` through client and server request
542
+ * construction. Lance sets both probe bounds to this value. Explicit minimum
543
+ * or maximum settings can override their respective bounds.
440
544
  */
441
545
  nprobes(nprobes) {
442
546
  this.doVectorCall((inner) => inner.nprobes(nprobes));
443
547
  return this;
444
548
  }
549
+ /**
550
+ * Set the initial IVF search effort to a finite value in [0, 1].
551
+ *
552
+ * The default, null, uses Auto (internally 0.5). Zero starts at the caller
553
+ * minimum; one searches all partitions within the caller maximum. Later
554
+ * probing may continue when too few candidates are found.
555
+ * Any explicit effort, including 0.5, overrides nprobes in either call order.
556
+ * Explicit minimum and maximum bounds still apply. This is not a recall
557
+ * guarantee and does not change refinement or HNSW ef. The RTE LSM
558
+ * scanner does not yet support this option.
559
+ *
560
+ * @example
561
+ * ```ts
562
+ * table.vectorSearch([0.1, 0.2]).searchEffort(0.75).limit(10);
563
+ * ```
564
+ */
565
+ searchEffort(searchEffort) {
566
+ this.doVectorCall((inner) => inner.searchEffort(searchEffort));
567
+ return this;
568
+ }
445
569
  /**
446
570
  * Set the minimum number of probes used.
447
571
  *
@@ -618,7 +742,7 @@ class VectorQuery extends StandardQueryBase {
618
742
  addQueryVectorToNative(inner, outcome.value);
619
743
  return inner;
620
744
  })();
621
- return new VectorQuery(res);
745
+ return new VectorQuery(res, this.blobContext);
622
746
  }
623
747
  else {
624
748
  this.doVectorCall((inner) => addQueryVectorToNative(inner, vector));
@@ -680,7 +804,7 @@ function createAutoQuery(table, query, columns, getVector) {
680
804
  }
681
805
  return nearestToNative(route.table.query(), vector);
682
806
  };
683
- return new AutoQuery(createInner);
807
+ return new AutoQuery(createInner, { table, withRowId: false });
684
808
  }
685
809
  /**
686
810
  * A query that returns a subset of the rows in the table.
@@ -688,20 +812,20 @@ function createAutoQuery(table, query, columns, getVector) {
688
812
  * @hideconstructor
689
813
  */
690
814
  class TakeQuery extends QueryBase {
691
- constructor(inner) {
692
- super(inner);
815
+ constructor(inner, blobContext) {
816
+ super(inner, blobContext);
693
817
  }
694
818
  /**
695
- * Control MemWAL read routing for this take query.
819
+ * Control RTE read routing for this take query.
696
820
  *
697
- * `false` bypasses the MemWAL and reads the base table only — the escape hatch,
821
+ * `false` bypasses the RTE and reads the base table only — the escape hatch,
698
822
  * since take-by-row-id/offset is not supported on the LSM scanner and, on a
699
- * MemWAL table, auto-routes to it and errors otherwise.
823
+ * RTE table, auto-routes to it and errors otherwise.
700
824
  *
701
825
  * @param enable - `false` reads the base table only.
702
826
  */
703
- useLsm(enable) {
704
- this.doCall((inner) => inner.useLsm(enable));
827
+ useRTE(enable) {
828
+ this.doCall((inner) => inner.useRTE(enable));
705
829
  return this;
706
830
  }
707
831
  }
@@ -719,8 +843,8 @@ class AutoQuery extends StandardQueryBase {
719
843
  createInner;
720
844
  calls = [];
721
845
  /** @hidden */
722
- constructor(createInner) {
723
- super();
846
+ constructor(createInner, blobContext) {
847
+ super(undefined, blobContext);
724
848
  this.createInner = createInner;
725
849
  }
726
850
  /** @hidden */
@@ -749,7 +873,7 @@ class Query extends StandardQueryBase {
749
873
  * @hidden
750
874
  */
751
875
  constructor(tbl) {
752
- super(tbl.query());
876
+ super(tbl.query(), { table: tbl, withRowId: false });
753
877
  }
754
878
  /**
755
879
  * Find the nearest vectors to the given query vector.
@@ -792,12 +916,12 @@ class Query extends StandardQueryBase {
792
916
  const inner = this.inner;
793
917
  if (inner instanceof Promise) {
794
918
  const nativeQuery = inner.then(async (resolvedInner) => nearestToNative(resolvedInner, await vector));
795
- return new VectorQuery(nativeQuery);
919
+ return new VectorQuery(nativeQuery, this.blobContext);
796
920
  }
797
921
  if (vector instanceof Promise) {
798
- return new VectorQuery(vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)));
922
+ return new VectorQuery(vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)), this.blobContext);
799
923
  }
800
- return new VectorQuery(nearestToNative(inner, vector));
924
+ return new VectorQuery(nearestToNative(inner, vector), this.blobContext);
801
925
  }
802
926
  nearestToText(query, columns) {
803
927
  this.doCall((inner) => {
@@ -1,23 +1,24 @@
1
+ import * as arrowModule from "apache-arrow";
1
2
  import type { TKeys } from "apache-arrow/type";
2
- import { DataType, Date_, Decimal, DenseUnion, Dictionary, Duration, Field, FixedSizeBinary, FixedSizeList, Float, Int, Interval, List, Map_, Schema, SchemaLike, SparseUnion, Struct, Table, TableLike, Time, Timestamp, TimestampMicrosecond, TimestampMillisecond, TimestampNanosecond, TimestampSecond, Type, Union } from "./arrow";
3
+ import { DataType, DenseUnion, Field, Schema, SchemaLike, SparseUnion, Table, TableLike, TimestampMicrosecond, TimestampMillisecond, TimestampNanosecond, TimestampSecond } from "./arrow";
3
4
  export declare function sanitizeMetadata(metadataLike?: unknown): Map<string, string> | undefined;
4
- export declare function sanitizeInt(typeLike: object): Int<Type.Int | Type.Int8 | Type.Int16 | Type.Int32 | Type.Int64 | Type.Uint8 | Type.Uint16 | Type.Uint32 | Type.Uint64>;
5
- export declare function sanitizeFloat(typeLike: object): Float<Type.Float | Type.Float16 | Type.Float32 | Type.Float64>;
6
- export declare function sanitizeDecimal(typeLike: object): Decimal;
7
- export declare function sanitizeDate(typeLike: object): Date_<import("apache-arrow/type").Dates>;
8
- export declare function sanitizeTime(typeLike: object): Time<Type.Time | Type.TimeSecond | Type.TimeMillisecond | Type.TimeMicrosecond | Type.TimeNanosecond>;
9
- export declare function sanitizeTimestamp(typeLike: object): Timestamp<Type.Timestamp | Type.TimestampSecond | Type.TimestampMillisecond | Type.TimestampMicrosecond | Type.TimestampNanosecond>;
10
- export declare function sanitizeTypedTimestamp(typeLike: object, Datatype: typeof TimestampNanosecond | typeof TimestampMicrosecond | typeof TimestampMillisecond | typeof TimestampSecond): TimestampSecond | TimestampMillisecond | TimestampMicrosecond | TimestampNanosecond;
11
- export declare function sanitizeInterval(typeLike: object): Interval<Type.Interval | Type.IntervalDayTime | Type.IntervalYearMonth>;
12
- export declare function sanitizeList(typeLike: object): List<any>;
13
- export declare function sanitizeStruct(typeLike: object): Struct<any>;
14
- export declare function sanitizeUnion(typeLike: object): Union<Type.Union | Type.DenseUnion | Type.SparseUnion>;
15
- export declare function sanitizeTypedUnion(typeLike: object, UnionType: typeof DenseUnion | typeof SparseUnion): SparseUnion | DenseUnion;
16
- export declare function sanitizeFixedSizeBinary(typeLike: object): FixedSizeBinary;
17
- export declare function sanitizeFixedSizeList(typeLike: object): FixedSizeList<any>;
18
- export declare function sanitizeMap(typeLike: object): Map_<any, any>;
19
- export declare function sanitizeDuration(typeLike: object): Duration<Type.Duration | Type.DurationSecond | Type.DurationMillisecond | Type.DurationMicrosecond | Type.DurationNanosecond>;
20
- export declare function sanitizeDictionary(typeLike: object): Dictionary<DataType<Type, any>, TKeys>;
5
+ export declare function sanitizeInt(typeLike: object): arrowModule.Int<arrowModule.Type.Int | arrowModule.Type.Int8 | arrowModule.Type.Int16 | arrowModule.Type.Int32 | arrowModule.Type.Int64 | arrowModule.Type.Uint8 | arrowModule.Type.Uint16 | arrowModule.Type.Uint32 | arrowModule.Type.Uint64>;
6
+ export declare function sanitizeFloat(typeLike: object): arrowModule.Float<arrowModule.Type.Float | arrowModule.Type.Float16 | arrowModule.Type.Float32 | arrowModule.Type.Float64>;
7
+ export declare function sanitizeDecimal(typeLike: object): arrowModule.Decimal;
8
+ export declare function sanitizeDate(typeLike: object): arrowModule.Date_<import("apache-arrow/type").Dates>;
9
+ export declare function sanitizeTime(typeLike: object): arrowModule.Time<arrowModule.Type.Time | arrowModule.Type.TimeSecond | arrowModule.Type.TimeMillisecond | arrowModule.Type.TimeMicrosecond | arrowModule.Type.TimeNanosecond>;
10
+ export declare function sanitizeTimestamp(typeLike: object): arrowModule.Timestamp<arrowModule.Type.Timestamp | arrowModule.Type.TimestampSecond | arrowModule.Type.TimestampMillisecond | arrowModule.Type.TimestampMicrosecond | arrowModule.Type.TimestampNanosecond>;
11
+ export declare function sanitizeTypedTimestamp(typeLike: object, Datatype: typeof TimestampNanosecond | typeof TimestampMicrosecond | typeof TimestampMillisecond | typeof TimestampSecond): arrowModule.TimestampNanosecond | arrowModule.TimestampMicrosecond | arrowModule.TimestampMillisecond | arrowModule.TimestampSecond;
12
+ export declare function sanitizeInterval(typeLike: object): arrowModule.Interval<arrowModule.Type.Interval | arrowModule.Type.IntervalDayTime | arrowModule.Type.IntervalYearMonth | arrowModule.Type.IntervalMonthDayNano>;
13
+ export declare function sanitizeList(typeLike: object): arrowModule.List<any>;
14
+ export declare function sanitizeStruct(typeLike: object): arrowModule.Struct<any>;
15
+ export declare function sanitizeUnion(typeLike: object): arrowModule.Union<arrowModule.Type.Union | arrowModule.Type.DenseUnion | arrowModule.Type.SparseUnion>;
16
+ export declare function sanitizeTypedUnion(typeLike: object, UnionType: typeof DenseUnion | typeof SparseUnion): arrowModule.DenseUnion | arrowModule.SparseUnion;
17
+ export declare function sanitizeFixedSizeBinary(typeLike: object): arrowModule.FixedSizeBinary;
18
+ export declare function sanitizeFixedSizeList(typeLike: object): arrowModule.FixedSizeList<any>;
19
+ export declare function sanitizeMap(typeLike: object): arrowModule.Map_<any, any>;
20
+ export declare function sanitizeDuration(typeLike: object): arrowModule.Duration<arrowModule.Type.Duration | arrowModule.Type.DurationSecond | arrowModule.Type.DurationMillisecond | arrowModule.Type.DurationMicrosecond | arrowModule.Type.DurationNanosecond>;
21
+ export declare function sanitizeDictionary(typeLike: object): arrowModule.Dictionary<arrowModule.DataType<arrowModule.Type, any>, TKeys>;
21
22
  export declare function sanitizeType(typeLike: unknown): DataType<any>;
22
23
  export declare function sanitizeField(fieldLike: unknown): Field;
23
24
  /**
package/dist/sanitize.js CHANGED
@@ -32,8 +32,42 @@ exports.dataTypeFromName = dataTypeFromName;
32
32
  // time. However, arrow-js uses instanceof which expected that the input
33
33
  // comes from the exact same library instance. This is not always the case
34
34
  // and so we must sanitize the input to ensure that it is compatible.
35
+ const arrowModule = require("apache-arrow");
35
36
  const apache_arrow_1 = require("apache-arrow");
36
37
  const arrow_1 = require("./arrow");
38
+ /**
39
+ * Whether `value` was created by *this* copy of apache-arrow's `ctor`.
40
+ *
41
+ * Arrow 21 overrides `Symbol.hasInstance` on `Schema`, `Field`, `Table`,
42
+ * `RecordBatch`, `Vector` and `Data` so that `instanceof` also matches objects
43
+ * built by a different copy of the library. Those are exactly the objects
44
+ * this module has to rebuild: the `DataType` instances nested inside them are
45
+ * deliberately *not* given that treatment upstream, so Arrow's own builders and
46
+ * writers reject them. Walk the real prototype chain instead of trusting
47
+ * `instanceof`.
48
+ */
49
+ function isOwnInstance(value,
50
+ // biome-ignore lint/suspicious/noExplicitAny: matches any class constructor
51
+ ctor) {
52
+ return (typeof value === "object" &&
53
+ value !== null &&
54
+ Object.prototype.isPrototypeOf.call(ctor.prototype, value));
55
+ }
56
+ const NEWER_TYPES = [
57
+ { name: "IntervalMonthDayNano", typeId: -31, since: 20 },
58
+ { name: "LargeList", typeId: 21, since: 21 },
59
+ { name: "BinaryView", typeId: 23, since: 21 },
60
+ { name: "Utf8View", typeId: 24, since: 21 },
61
+ ];
62
+ function newerTypeConstructor(name) {
63
+ const ctor = arrowModule[name];
64
+ if (typeof ctor !== "function") {
65
+ const since = NEWER_TYPES.find((t) => t.name === name)?.since;
66
+ throw new Error(`The Arrow type ${name} requires apache-arrow ${since} or newer, ` +
67
+ "but the installed apache-arrow does not provide it");
68
+ }
69
+ return ctor;
70
+ }
37
71
  function createSanitizationContext() {
38
72
  return {
39
73
  types: new WeakMap(),
@@ -130,14 +164,21 @@ function sanitizeInterval(typeLike) {
130
164
  function sanitizeList(typeLike) {
131
165
  return sanitizeListWithContext(typeLike, createSanitizationContext());
132
166
  }
133
- function sanitizeListWithContext(typeLike, context) {
167
+ function sanitizeListChildWithContext(typeLike, context, typeName) {
134
168
  if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
135
- throw Error("Expected a List type to have an array-like `children` property");
169
+ throw Error(`Expected a ${typeName} type to have an array-like \`children\` property`);
136
170
  }
137
171
  if (typeLike.children.length !== 1) {
138
- throw Error("Expected a List type to have exactly one child");
172
+ throw Error(`Expected a ${typeName} type to have exactly one child`);
139
173
  }
140
- return new arrow_1.List(sanitizeFieldWithContext(typeLike.children[0], context));
174
+ return sanitizeFieldWithContext(typeLike.children[0], context);
175
+ }
176
+ function sanitizeListWithContext(typeLike, context) {
177
+ return new arrow_1.List(sanitizeListChildWithContext(typeLike, context, "List"));
178
+ }
179
+ function sanitizeLargeListWithContext(typeLike, context) {
180
+ const LargeList = newerTypeConstructor("LargeList");
181
+ return new LargeList(sanitizeListChildWithContext(typeLike, context, "LargeList"));
141
182
  }
142
183
  function sanitizeStruct(typeLike) {
143
184
  return sanitizeStructWithContext(typeLike, createSanitizationContext());
@@ -274,7 +315,21 @@ function sanitizeTypeWithContext(typeLike, context) {
274
315
  context.types.set(typeLike, type);
275
316
  return type;
276
317
  }
318
+ function sanitizeNewerTypeById(typeLike, typeId, context) {
319
+ const entry = NEWER_TYPES.find((t) => t.typeId === typeId);
320
+ if (entry === undefined) {
321
+ return undefined;
322
+ }
323
+ if (entry.name === "LargeList") {
324
+ return sanitizeLargeListWithContext(typeLike, context);
325
+ }
326
+ return new (newerTypeConstructor(entry.name))();
327
+ }
277
328
  function sanitizeTypeById(typeLike, typeId, context) {
329
+ const newer = sanitizeNewerTypeById(typeLike, typeId, context);
330
+ if (newer !== undefined) {
331
+ return newer;
332
+ }
278
333
  switch (typeId) {
279
334
  case arrow_1.Type.NONE:
280
335
  throw Error("Received a Type with a typeId of NONE");
@@ -386,7 +441,7 @@ function sanitizeField(fieldLike) {
386
441
  return sanitizeFieldWithContext(fieldLike, createSanitizationContext());
387
442
  }
388
443
  function sanitizeFieldWithContext(fieldLike, context) {
389
- if (fieldLike instanceof arrow_1.Field) {
444
+ if (isOwnInstance(fieldLike, arrow_1.Field)) {
390
445
  return fieldLike;
391
446
  }
392
447
  if (typeof fieldLike !== "object" || fieldLike === null) {
@@ -429,7 +484,7 @@ function sanitizeSchema(schemaLike) {
429
484
  return sanitizeSchemaWithContext(schemaLike, createSanitizationContext());
430
485
  }
431
486
  function sanitizeSchemaWithContext(schemaLike, context) {
432
- if (schemaLike instanceof arrow_1.Schema) {
487
+ if (isOwnInstance(schemaLike, arrow_1.Schema)) {
433
488
  return schemaLike;
434
489
  }
435
490
  if (typeof schemaLike !== "object" || schemaLike === null) {
@@ -449,7 +504,7 @@ function sanitizeSchemaWithContext(schemaLike, context) {
449
504
  return new arrow_1.Schema(sanitizedFields, metadata);
450
505
  }
451
506
  function sanitizeTable(tableLike) {
452
- if (tableLike instanceof arrow_1.Table) {
507
+ if (isOwnInstance(tableLike, arrow_1.Table)) {
453
508
  return tableLike;
454
509
  }
455
510
  if (typeof tableLike !== "object" || tableLike === null) {
@@ -467,7 +522,7 @@ function sanitizeTable(tableLike) {
467
522
  return new arrow_1.Table(schema, batches);
468
523
  }
469
524
  function sanitizeRecordBatch(batchLike, context) {
470
- if (batchLike instanceof arrow_1.RecordBatch) {
525
+ if (isOwnInstance(batchLike, arrow_1.RecordBatch)) {
471
526
  return batchLike;
472
527
  }
473
528
  if (typeof batchLike !== "object" || batchLike === null) {
@@ -484,7 +539,7 @@ function sanitizeRecordBatch(batchLike, context) {
484
539
  return new arrow_1.RecordBatch(schema, data);
485
540
  }
486
541
  function sanitizeData(dataLike, context) {
487
- if (dataLike instanceof apache_arrow_1.Data) {
542
+ if (isOwnInstance(dataLike, apache_arrow_1.Data)) {
488
543
  return dataLike;
489
544
  }
490
545
  const cachedData = context.data.get(dataLike);
@@ -500,19 +555,26 @@ function sanitizeData(dataLike, context) {
500
555
  context.vectors.set(dictionaryLike, dictionary);
501
556
  }
502
557
  }
503
- const data = new apache_arrow_1.Data(sanitizeTypeWithContext(dataLike.type, context), dataLike.offset, dataLike.length, dataLike.nullCount, {
558
+ // Arrow 21 added an eighth constructor parameter (`variadicBuffers`); older
559
+ // releases ignore the extra argument but their typings do not declare it.
560
+ const DataCtor = apache_arrow_1.Data;
561
+ const data = new DataCtor(sanitizeTypeWithContext(dataLike.type, context), dataLike.offset, dataLike.length, dataLike.nullCount, {
504
562
  [apache_arrow_1.BufferType.OFFSET]: dataLike.valueOffsets,
505
563
  [apache_arrow_1.BufferType.DATA]: dataLike.values,
506
564
  [apache_arrow_1.BufferType.VALIDITY]: dataLike.nullBitmap,
507
565
  [apache_arrow_1.BufferType.TYPE]: dataLike.typeIds,
508
- }, dataLike.children.map((child) => sanitizeData(child, context)), dictionary);
566
+ }, dataLike.children.map((child) => sanitizeData(child, context)), dictionary, dataLike.variadicBuffers);
509
567
  context.data.set(dataLike, data);
510
568
  return data;
511
569
  }
512
570
  const constructorsByTypeName = {
513
571
  null: () => new arrow_1.Null(),
514
572
  binary: () => new arrow_1.Binary(),
573
+ largebinary: () => new arrow_1.LargeBinary(),
574
+ binaryview: () => new (newerTypeConstructor("BinaryView"))(),
515
575
  utf8: () => new arrow_1.Utf8(),
576
+ largeutf8: () => new arrow_1.LargeUtf8(),
577
+ utf8view: () => new (newerTypeConstructor("Utf8View"))(),
516
578
  bool: () => new arrow_1.Bool(),
517
579
  int8: () => new arrow_1.Int8(),
518
580
  int16: () => new arrow_1.Int16(),
@@ -533,6 +595,7 @@ const constructorsByTypeName = {
533
595
  timesecond: () => new arrow_1.TimeSecond(),
534
596
  intervaldaytime: () => new arrow_1.IntervalDayTime(),
535
597
  intervalyearmonth: () => new arrow_1.IntervalYearMonth(),
598
+ intervalmonthdaynano: () => new (newerTypeConstructor("IntervalMonthDayNano"))(),
536
599
  durationnanosecond: () => new arrow_1.DurationNanosecond(),
537
600
  durationmicrosecond: () => new arrow_1.DurationMicrosecond(),
538
601
  durationmillisecond: () => new arrow_1.DurationMillisecond(),