@lancedb/lancedb 0.40.0-beta.12 → 0.40.0-beta.13

package/dist/query.js

@@ -21,18 +21,6 @@ async function* RecordBatchIterator(promisedInner) {
         yield batches[0];
     }
 }
-class RecordBatchIterable {
-    inner;
-    options;
-    constructor(inner, options) {
-        this.inner = inner;
-        this.options = options;
-    }
-    // biome-ignore lint/suspicious/noExplicitAny: skip
-    [Symbol.asyncIterator]() {
-        return RecordBatchIterator(this.inner.execute(this.options?.maxBatchLength, this.options?.timeoutMs));
-    }
-}
 function nearestToNative(inner, vector) {
     const raw = Array.isArray(vector) ? null : (0, arrow_1.extractVectorBuffer)(vector);
     if (raw) {
@@ -183,13 +171,20 @@ class QueryBase {
     [Symbol.asyncIterator]() {
         return RecordBatchIterator(this.nativeExecute());
     }
-    /** Collect the results as an Arrow @see {@link ArrowTable}. */
+    /**
+     * Collect the results as an Arrow @see {@link ArrowTable}.
+     *
+     * Empty results retain the query's output schema.
+     */
     async toArrow(options) {
         const batches = [];
-        const inner = await this.getInner();
-        for await (const batch of new RecordBatchIterable(inner, options)) {
+        const iterator = this.nativeExecute(options);
+        for await (const batch of RecordBatchIterator(iterator)) {
             batches.push(batch);
         }
+        if (batches.length === 0) {
+            return (0, arrow_1.tableFromIPC)((await iterator).schema());
+        }
         return new arrow_1.Table(batches);
     }
     /** Collect the results as an array of objects. */
@@ -415,6 +410,8 @@ class VectorQuery extends StandardQueryBase {
     /**
      * Set the number of partitions to search (probe)
      *
+     * The number of probes must be greater than 0.
+     *
      * This argument is only used when the vector column has an IVF PQ index.
      * If there is no index then this value is ignored.
      *

package/dist/sanitize.d.ts

@@ -1,23 +1,24 @@
+import * as arrowModule from "apache-arrow";
 import type { TKeys } from "apache-arrow/type";
-import { DataType, Date_, Decimal, DenseUnion, Dictionary, Duration, Field, FixedSizeBinary, FixedSizeList, Float, Int, Interval, List, Map_, Schema, SchemaLike, SparseUnion, Struct, Table, TableLike, Time, Timestamp, TimestampMicrosecond, TimestampMillisecond, TimestampNanosecond, TimestampSecond, Type, Union } from "./arrow";
+import { DataType, DenseUnion, Field, Schema, SchemaLike, SparseUnion, Table, TableLike, TimestampMicrosecond, TimestampMillisecond, TimestampNanosecond, TimestampSecond } from "./arrow";
 export declare function sanitizeMetadata(metadataLike?: unknown): Map<string, string> | undefined;
-export declare function sanitizeInt(typeLike: object): Int<Type.Int | Type.Int8 | Type.Int16 | Type.Int32 | Type.Int64 | Type.Uint8 | Type.Uint16 | Type.Uint32 | Type.Uint64>;
-export declare function sanitizeFloat(typeLike: object): Float<Type.Float | Type.Float16 | Type.Float32 | Type.Float64>;
-export declare function sanitizeDecimal(typeLike: object): Decimal;
-export declare function sanitizeDate(typeLike: object): Date_<import("apache-arrow/type").Dates>;
-export declare function sanitizeTime(typeLike: object): Time<Type.Time | Type.TimeSecond | Type.TimeMillisecond | Type.TimeMicrosecond | Type.TimeNanosecond>;
-export declare function sanitizeTimestamp(typeLike: object): Timestamp<Type.Timestamp | Type.TimestampSecond | Type.TimestampMillisecond | Type.TimestampMicrosecond | Type.TimestampNanosecond>;
-export declare function sanitizeTypedTimestamp(typeLike: object, Datatype: typeof TimestampNanosecond | typeof TimestampMicrosecond | typeof TimestampMillisecond | typeof TimestampSecond): TimestampSecond | TimestampMillisecond | TimestampMicrosecond | TimestampNanosecond;
-export declare function sanitizeInterval(typeLike: object): Interval<Type.Interval | Type.IntervalDayTime | Type.IntervalYearMonth>;
-export declare function sanitizeList(typeLike: object): List<any>;
-export declare function sanitizeStruct(typeLike: object): Struct<any>;
-export declare function sanitizeUnion(typeLike: object): Union<Type.Union | Type.DenseUnion | Type.SparseUnion>;
-export declare function sanitizeTypedUnion(typeLike: object, UnionType: typeof DenseUnion | typeof SparseUnion): SparseUnion | DenseUnion;
-export declare function sanitizeFixedSizeBinary(typeLike: object): FixedSizeBinary;
-export declare function sanitizeFixedSizeList(typeLike: object): FixedSizeList<any>;
-export declare function sanitizeMap(typeLike: object): Map_<any, any>;
-export declare function sanitizeDuration(typeLike: object): Duration<Type.Duration | Type.DurationSecond | Type.DurationMillisecond | Type.DurationMicrosecond | Type.DurationNanosecond>;
-export declare function sanitizeDictionary(typeLike: object): Dictionary<DataType<Type, any>, TKeys>;
+export declare function sanitizeInt(typeLike: object): arrowModule.Int<arrowModule.Type.Int | arrowModule.Type.Int8 | arrowModule.Type.Int16 | arrowModule.Type.Int32 | arrowModule.Type.Int64 | arrowModule.Type.Uint8 | arrowModule.Type.Uint16 | arrowModule.Type.Uint32 | arrowModule.Type.Uint64>;
+export declare function sanitizeFloat(typeLike: object): arrowModule.Float<arrowModule.Type.Float | arrowModule.Type.Float16 | arrowModule.Type.Float32 | arrowModule.Type.Float64>;
+export declare function sanitizeDecimal(typeLike: object): arrowModule.Decimal;
+export declare function sanitizeDate(typeLike: object): arrowModule.Date_<import("apache-arrow/type").Dates>;
+export declare function sanitizeTime(typeLike: object): arrowModule.Time<arrowModule.Type.Time | arrowModule.Type.TimeSecond | arrowModule.Type.TimeMillisecond | arrowModule.Type.TimeMicrosecond | arrowModule.Type.TimeNanosecond>;
+export declare function sanitizeTimestamp(typeLike: object): arrowModule.Timestamp<arrowModule.Type.Timestamp | arrowModule.Type.TimestampSecond | arrowModule.Type.TimestampMillisecond | arrowModule.Type.TimestampMicrosecond | arrowModule.Type.TimestampNanosecond>;
+export declare function sanitizeTypedTimestamp(typeLike: object, Datatype: typeof TimestampNanosecond | typeof TimestampMicrosecond | typeof TimestampMillisecond | typeof TimestampSecond): arrowModule.TimestampNanosecond | arrowModule.TimestampMicrosecond | arrowModule.TimestampMillisecond | arrowModule.TimestampSecond;
+export declare function sanitizeInterval(typeLike: object): arrowModule.Interval<arrowModule.Type.Interval | arrowModule.Type.IntervalDayTime | arrowModule.Type.IntervalYearMonth | arrowModule.Type.IntervalMonthDayNano>;
+export declare function sanitizeList(typeLike: object): arrowModule.List<any>;
+export declare function sanitizeStruct(typeLike: object): arrowModule.Struct<any>;
+export declare function sanitizeUnion(typeLike: object): arrowModule.Union<arrowModule.Type.Union | arrowModule.Type.DenseUnion | arrowModule.Type.SparseUnion>;
+export declare function sanitizeTypedUnion(typeLike: object, UnionType: typeof DenseUnion | typeof SparseUnion): arrowModule.DenseUnion | arrowModule.SparseUnion;
+export declare function sanitizeFixedSizeBinary(typeLike: object): arrowModule.FixedSizeBinary;
+export declare function sanitizeFixedSizeList(typeLike: object): arrowModule.FixedSizeList<any>;
+export declare function sanitizeMap(typeLike: object): arrowModule.Map_<any, any>;
+export declare function sanitizeDuration(typeLike: object): arrowModule.Duration<arrowModule.Type.Duration | arrowModule.Type.DurationSecond | arrowModule.Type.DurationMillisecond | arrowModule.Type.DurationMicrosecond | arrowModule.Type.DurationNanosecond>;
+export declare function sanitizeDictionary(typeLike: object): arrowModule.Dictionary<arrowModule.DataType<arrowModule.Type, any>, TKeys>;
 export declare function sanitizeType(typeLike: unknown): DataType<any>;
 export declare function sanitizeField(fieldLike: unknown): Field;
 /**

package/dist/sanitize.js

@@ -32,8 +32,42 @@ exports.dataTypeFromName = dataTypeFromName;
 // time.  However, arrow-js uses instanceof which expected that the input
 // comes from the exact same library instance.  This is not always the case
 // and so we must sanitize the input to ensure that it is compatible.
+const arrowModule = require("apache-arrow");
 const apache_arrow_1 = require("apache-arrow");
 const arrow_1 = require("./arrow");
+/**
+ * Whether `value` was created by *this* copy of apache-arrow's `ctor`.
+ *
+ * Arrow 21 overrides `Symbol.hasInstance` on `Schema`, `Field`, `Table`,
+ * `RecordBatch`, `Vector` and `Data` so that `instanceof` also matches objects
+ * built by a different copy of the library.  Those are exactly the objects
+ * this module has to rebuild: the `DataType` instances nested inside them are
+ * deliberately *not* given that treatment upstream, so Arrow's own builders and
+ * writers reject them.  Walk the real prototype chain instead of trusting
+ * `instanceof`.
+ */
+function isOwnInstance(value, 
+// biome-ignore lint/suspicious/noExplicitAny: matches any class constructor
+ctor) {
+    return (typeof value === "object" &&
+        value !== null &&
+        Object.prototype.isPrototypeOf.call(ctor.prototype, value));
+}
+const NEWER_TYPES = [
+    { name: "IntervalMonthDayNano", typeId: -31, since: 20 },
+    { name: "LargeList", typeId: 21, since: 21 },
+    { name: "BinaryView", typeId: 23, since: 21 },
+    { name: "Utf8View", typeId: 24, since: 21 },
+];
+function newerTypeConstructor(name) {
+    const ctor = arrowModule[name];
+    if (typeof ctor !== "function") {
+        const since = NEWER_TYPES.find((t) => t.name === name)?.since;
+        throw new Error(`The Arrow type ${name} requires apache-arrow ${since} or newer, ` +
+            "but the installed apache-arrow does not provide it");
+    }
+    return ctor;
+}
 function createSanitizationContext() {
     return {
         types: new WeakMap(),
@@ -130,14 +164,21 @@ function sanitizeInterval(typeLike) {
 function sanitizeList(typeLike) {
     return sanitizeListWithContext(typeLike, createSanitizationContext());
 }
-function sanitizeListWithContext(typeLike, context) {
+function sanitizeListChildWithContext(typeLike, context, typeName) {
     if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
-        throw Error("Expected a List type to have an array-like `children` property");
+        throw Error(`Expected a ${typeName} type to have an array-like \`children\` property`);
     }
     if (typeLike.children.length !== 1) {
-        throw Error("Expected a List type to have exactly one child");
+        throw Error(`Expected a ${typeName} type to have exactly one child`);
     }
-    return new arrow_1.List(sanitizeFieldWithContext(typeLike.children[0], context));
+    return sanitizeFieldWithContext(typeLike.children[0], context);
+}
+function sanitizeListWithContext(typeLike, context) {
+    return new arrow_1.List(sanitizeListChildWithContext(typeLike, context, "List"));
+}
+function sanitizeLargeListWithContext(typeLike, context) {
+    const LargeList = newerTypeConstructor("LargeList");
+    return new LargeList(sanitizeListChildWithContext(typeLike, context, "LargeList"));
 }
 function sanitizeStruct(typeLike) {
     return sanitizeStructWithContext(typeLike, createSanitizationContext());
@@ -274,7 +315,21 @@ function sanitizeTypeWithContext(typeLike, context) {
     context.types.set(typeLike, type);
     return type;
 }
+function sanitizeNewerTypeById(typeLike, typeId, context) {
+    const entry = NEWER_TYPES.find((t) => t.typeId === typeId);
+    if (entry === undefined) {
+        return undefined;
+    }
+    if (entry.name === "LargeList") {
+        return sanitizeLargeListWithContext(typeLike, context);
+    }
+    return new (newerTypeConstructor(entry.name))();
+}
 function sanitizeTypeById(typeLike, typeId, context) {
+    const newer = sanitizeNewerTypeById(typeLike, typeId, context);
+    if (newer !== undefined) {
+        return newer;
+    }
     switch (typeId) {
         case arrow_1.Type.NONE:
             throw Error("Received a Type with a typeId of NONE");
@@ -386,7 +441,7 @@ function sanitizeField(fieldLike) {
     return sanitizeFieldWithContext(fieldLike, createSanitizationContext());
 }
 function sanitizeFieldWithContext(fieldLike, context) {
-    if (fieldLike instanceof arrow_1.Field) {
+    if (isOwnInstance(fieldLike, arrow_1.Field)) {
         return fieldLike;
     }
     if (typeof fieldLike !== "object" || fieldLike === null) {
@@ -429,7 +484,7 @@ function sanitizeSchema(schemaLike) {
     return sanitizeSchemaWithContext(schemaLike, createSanitizationContext());
 }
 function sanitizeSchemaWithContext(schemaLike, context) {
-    if (schemaLike instanceof arrow_1.Schema) {
+    if (isOwnInstance(schemaLike, arrow_1.Schema)) {
         return schemaLike;
     }
     if (typeof schemaLike !== "object" || schemaLike === null) {
@@ -449,7 +504,7 @@ function sanitizeSchemaWithContext(schemaLike, context) {
     return new arrow_1.Schema(sanitizedFields, metadata);
 }
 function sanitizeTable(tableLike) {
-    if (tableLike instanceof arrow_1.Table) {
+    if (isOwnInstance(tableLike, arrow_1.Table)) {
         return tableLike;
     }
     if (typeof tableLike !== "object" || tableLike === null) {
@@ -467,7 +522,7 @@ function sanitizeTable(tableLike) {
     return new arrow_1.Table(schema, batches);
 }
 function sanitizeRecordBatch(batchLike, context) {
-    if (batchLike instanceof arrow_1.RecordBatch) {
+    if (isOwnInstance(batchLike, arrow_1.RecordBatch)) {
         return batchLike;
     }
     if (typeof batchLike !== "object" || batchLike === null) {
@@ -484,7 +539,7 @@ function sanitizeRecordBatch(batchLike, context) {
     return new arrow_1.RecordBatch(schema, data);
 }
 function sanitizeData(dataLike, context) {
-    if (dataLike instanceof apache_arrow_1.Data) {
+    if (isOwnInstance(dataLike, apache_arrow_1.Data)) {
         return dataLike;
     }
     const cachedData = context.data.get(dataLike);
@@ -500,19 +555,26 @@ function sanitizeData(dataLike, context) {
             context.vectors.set(dictionaryLike, dictionary);
         }
     }
-    const data = new apache_arrow_1.Data(sanitizeTypeWithContext(dataLike.type, context), dataLike.offset, dataLike.length, dataLike.nullCount, {
+    // Arrow 21 added an eighth constructor parameter (`variadicBuffers`); older
+    // releases ignore the extra argument but their typings do not declare it.
+    const DataCtor = apache_arrow_1.Data;
+    const data = new DataCtor(sanitizeTypeWithContext(dataLike.type, context), dataLike.offset, dataLike.length, dataLike.nullCount, {
         [apache_arrow_1.BufferType.OFFSET]: dataLike.valueOffsets,
         [apache_arrow_1.BufferType.DATA]: dataLike.values,
         [apache_arrow_1.BufferType.VALIDITY]: dataLike.nullBitmap,
         [apache_arrow_1.BufferType.TYPE]: dataLike.typeIds,
-    }, dataLike.children.map((child) => sanitizeData(child, context)), dictionary);
+    }, dataLike.children.map((child) => sanitizeData(child, context)), dictionary, dataLike.variadicBuffers);
     context.data.set(dataLike, data);
     return data;
 }
 const constructorsByTypeName = {
     null: () => new arrow_1.Null(),
     binary: () => new arrow_1.Binary(),
+    largebinary: () => new arrow_1.LargeBinary(),
+    binaryview: () => new (newerTypeConstructor("BinaryView"))(),
     utf8: () => new arrow_1.Utf8(),
+    largeutf8: () => new arrow_1.LargeUtf8(),
+    utf8view: () => new (newerTypeConstructor("Utf8View"))(),
     bool: () => new arrow_1.Bool(),
     int8: () => new arrow_1.Int8(),
     int16: () => new arrow_1.Int16(),
@@ -533,6 +595,7 @@ const constructorsByTypeName = {
     timesecond: () => new arrow_1.TimeSecond(),
     intervaldaytime: () => new arrow_1.IntervalDayTime(),
     intervalyearmonth: () => new arrow_1.IntervalYearMonth(),
+    intervalmonthdaynano: () => new (newerTypeConstructor("IntervalMonthDayNano"))(),
     durationnanosecond: () => new arrow_1.DurationNanosecond(),
     durationmicrosecond: () => new arrow_1.DurationMicrosecond(),
     durationmillisecond: () => new arrow_1.DurationMillisecond(),

package/dist/schema.js

@@ -43,10 +43,18 @@ class SchemaInferrer {
         else if (this.providedSchema === undefined) {
             this.updateInferredField(path, value, row, current);
         }
+        else if (current instanceof DeferredTypeEvidence) {
+            if (value != null) {
+                this.addSchemaField(this.providedSchema, path, value, row);
+            }
+        }
+        else if (current instanceof FieldTree || apache_arrow_1.DataType.isStruct(current)) {
+            assertStructValue(path, value, row);
+        }
     }
     addField(path, value, row) {
         if (this.providedSchema !== undefined) {
-            this.addSchemaField(this.providedSchema, path, row);
+            this.addSchemaField(this.providedSchema, path, value, row);
             return;
         }
         const evidence = this.inferType(value, path) ?? DeferredTypeEvidence.from(value, row);
@@ -58,12 +66,18 @@ class SchemaInferrer {
             throw branchConflictError(conflict, row, "Struct");
         }
     }
-    addSchemaField(schema, path, row) {
+    addSchemaField(schema, path, value, row) {
         const field = fieldAtPath(schema, path);
         if (field === undefined) {
             throw new Error(`Found field not in schema: ${path.join(".")} at row ${row}`);
         }
-        const conflict = this.fields.set(path, field.type);
+        if (apache_arrow_1.DataType.isStruct(field.type)) {
+            assertStructValue(path, value, row);
+        }
+        // Nulls are kept as deferred evidence so that a struct field seen first as
+        // null can be replaced by a record in a later row, just as a record seen
+        // first already ignores later nulls.
+        const conflict = this.fields.set(path, DeferredTypeEvidence.from(value, row) ?? field.type, (existing) => existing instanceof DeferredTypeEvidence && existing.isOnlyNulls());
         if (conflict !== undefined) {
             throw branchConflictError(conflict, row, "Struct");
         }
@@ -332,6 +346,20 @@ function assertSupportedValue(value, field, row) {
         throw new Error(`Unsupported object value for field ${field} at row ${row}.`);
     }
 }
+/**
+ * Rejects primitives, arrays, and typed arrays given for a struct field, which
+ * would otherwise be written as a struct with every child null.
+ */
+function assertStructValue(path, value, row) {
+    if (value == null ||
+        (typeof value === "object" &&
+            !Array.isArray(value) &&
+            !ArrayBuffer.isView(value))) {
+        return;
+    }
+    const kind = typeof value === "object" ? value.constructor.name : typeof value;
+    throw new Error(`Expected a struct value for field ${path.join(".")} at row ${row}, got ${kind}.`);
+}
 function fieldAtPath(schema, path) {
     let fields = schema.fields;
     let field;

package/dist/table.d.ts

@@ -556,7 +556,7 @@ export declare abstract class Table {
      * it: rejects tables that carry no view definition.
      * @ignore
      */
-    abstract refreshMaterializedView(full?: boolean, sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
+    abstract refreshMaterializedView(sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
     /** @ignore */
     abstract materializedViewDefinition(): Promise<string>;
     /**
@@ -929,7 +929,7 @@ export declare class LocalTable extends Table {
     refreshColumn(column: string): Promise<RefreshColumnResult>;
     refreshColumnAsync(column: string): Promise<Job>;
     functionErrors(options?: FunctionErrorsOptions): Promise<FunctionErrors>;
-    refreshMaterializedView(full?: boolean, sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
+    refreshMaterializedView(sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
     materializedViewDefinition(): Promise<string>;
     alterColumns(columnAlterations: ColumnAlteration[]): Promise<AlterColumnsResult>;
     updateFieldMetadata(updates: FieldMetadataUpdate[]): Promise<UpdateFieldMetadataResult>;

package/dist/table.js

@@ -280,8 +280,8 @@ class LocalTable extends Table {
         (0, materialized_view_1.validateNonNegativeInteger)(options?.limit, "limit");
         return await this.inner.functionErrors(options);
     }
-    async refreshMaterializedView(full, sourceVersion) {
-        return await this.inner.refreshMaterializedView(full, sourceVersion);
+    async refreshMaterializedView(sourceVersion) {
+        return await this.inner.refreshMaterializedView(sourceVersion);
     }
     async materializedViewDefinition() {
         return await this.inner.materializedViewDefinition();

package/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.40.0-beta.12",
+  "version": "0.40.0-beta.13",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -49,10 +49,14 @@
     "@types/jest": "^29.1.2",
     "@types/node": "22.7.4",
     "@types/tmp": "^0.2.6",
+    "apache-arrow": "21.2.0",
     "apache-arrow-15": "npm:apache-arrow@15.0.0",
     "apache-arrow-16": "npm:apache-arrow@16.0.0",
     "apache-arrow-17": "npm:apache-arrow@17.0.0",
     "apache-arrow-18": "npm:apache-arrow@18.0.0",
+    "apache-arrow-19": "npm:apache-arrow@19.0.1",
+    "apache-arrow-20": "npm:apache-arrow@20.0.0",
+    "apache-arrow-21": "npm:apache-arrow@21.2.0",
     "eslint": "^8.57.0",
     "jest": "^29.7.0",
     "shx": "^0.3.4",
@@ -106,17 +110,17 @@
   "optionalDependencies": {
     "@huggingface/transformers": "3.0.2",
     "openai": "4.29.2",
-    "@lancedb/lancedb-darwin-arm64": "0.40.0-beta.12",
-    "@lancedb/lancedb-linux-x64-gnu": "0.40.0-beta.12",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.40.0-beta.12",
-    "@lancedb/lancedb-linux-x64-musl": "0.40.0-beta.12",
-    "@lancedb/lancedb-linux-arm64-musl": "0.40.0-beta.12",
-    "@lancedb/lancedb-win32-x64-msvc": "0.40.0-beta.12",
-    "@lancedb/lancedb-win32-arm64-msvc": "0.40.0-beta.12"
+    "@lancedb/lancedb-darwin-arm64": "0.40.0-beta.13",
+    "@lancedb/lancedb-linux-x64-gnu": "0.40.0-beta.13",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.40.0-beta.13",
+    "@lancedb/lancedb-linux-x64-musl": "0.40.0-beta.13",
+    "@lancedb/lancedb-linux-arm64-musl": "0.40.0-beta.13",
+    "@lancedb/lancedb-win32-x64-msvc": "0.40.0-beta.13",
+    "@lancedb/lancedb-win32-arm64-msvc": "0.40.0-beta.13"
   },
   "peerDependencies": {
     "@types/node": ">=22",
-    "apache-arrow": ">=15.0.0 <=18.1.0"
+    "apache-arrow": ">=15.0.0 <22.0.0"
   },
   "peerDependenciesMeta": {
     "@types/node": {