@lancedb/lancedb 0.40.0-beta.11 → 0.40.0-beta.13

package/dist/sanitize.js

@@ -32,8 +32,42 @@ exports.dataTypeFromName = dataTypeFromName;
 // time.  However, arrow-js uses instanceof which expected that the input
 // comes from the exact same library instance.  This is not always the case
 // and so we must sanitize the input to ensure that it is compatible.
+const arrowModule = require("apache-arrow");
 const apache_arrow_1 = require("apache-arrow");
 const arrow_1 = require("./arrow");
+/**
+ * Whether `value` was created by *this* copy of apache-arrow's `ctor`.
+ *
+ * Arrow 21 overrides `Symbol.hasInstance` on `Schema`, `Field`, `Table`,
+ * `RecordBatch`, `Vector` and `Data` so that `instanceof` also matches objects
+ * built by a different copy of the library.  Those are exactly the objects
+ * this module has to rebuild: the `DataType` instances nested inside them are
+ * deliberately *not* given that treatment upstream, so Arrow's own builders and
+ * writers reject them.  Walk the real prototype chain instead of trusting
+ * `instanceof`.
+ */
+function isOwnInstance(value, 
+// biome-ignore lint/suspicious/noExplicitAny: matches any class constructor
+ctor) {
+    return (typeof value === "object" &&
+        value !== null &&
+        Object.prototype.isPrototypeOf.call(ctor.prototype, value));
+}
+const NEWER_TYPES = [
+    { name: "IntervalMonthDayNano", typeId: -31, since: 20 },
+    { name: "LargeList", typeId: 21, since: 21 },
+    { name: "BinaryView", typeId: 23, since: 21 },
+    { name: "Utf8View", typeId: 24, since: 21 },
+];
+function newerTypeConstructor(name) {
+    const ctor = arrowModule[name];
+    if (typeof ctor !== "function") {
+        const since = NEWER_TYPES.find((t) => t.name === name)?.since;
+        throw new Error(`The Arrow type ${name} requires apache-arrow ${since} or newer, ` +
+            "but the installed apache-arrow does not provide it");
+    }
+    return ctor;
+}
 function createSanitizationContext() {
     return {
         types: new WeakMap(),
@@ -130,14 +164,21 @@ function sanitizeInterval(typeLike) {
 function sanitizeList(typeLike) {
     return sanitizeListWithContext(typeLike, createSanitizationContext());
 }
-function sanitizeListWithContext(typeLike, context) {
+function sanitizeListChildWithContext(typeLike, context, typeName) {
     if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
-        throw Error("Expected a List type to have an array-like `children` property");
+        throw Error(`Expected a ${typeName} type to have an array-like \`children\` property`);
     }
     if (typeLike.children.length !== 1) {
-        throw Error("Expected a List type to have exactly one child");
+        throw Error(`Expected a ${typeName} type to have exactly one child`);
     }
-    return new arrow_1.List(sanitizeFieldWithContext(typeLike.children[0], context));
+    return sanitizeFieldWithContext(typeLike.children[0], context);
+}
+function sanitizeListWithContext(typeLike, context) {
+    return new arrow_1.List(sanitizeListChildWithContext(typeLike, context, "List"));
+}
+function sanitizeLargeListWithContext(typeLike, context) {
+    const LargeList = newerTypeConstructor("LargeList");
+    return new LargeList(sanitizeListChildWithContext(typeLike, context, "LargeList"));
 }
 function sanitizeStruct(typeLike) {
     return sanitizeStructWithContext(typeLike, createSanitizationContext());
@@ -274,7 +315,21 @@ function sanitizeTypeWithContext(typeLike, context) {
     context.types.set(typeLike, type);
     return type;
 }
+function sanitizeNewerTypeById(typeLike, typeId, context) {
+    const entry = NEWER_TYPES.find((t) => t.typeId === typeId);
+    if (entry === undefined) {
+        return undefined;
+    }
+    if (entry.name === "LargeList") {
+        return sanitizeLargeListWithContext(typeLike, context);
+    }
+    return new (newerTypeConstructor(entry.name))();
+}
 function sanitizeTypeById(typeLike, typeId, context) {
+    const newer = sanitizeNewerTypeById(typeLike, typeId, context);
+    if (newer !== undefined) {
+        return newer;
+    }
     switch (typeId) {
         case arrow_1.Type.NONE:
             throw Error("Received a Type with a typeId of NONE");
@@ -286,8 +341,12 @@ function sanitizeTypeById(typeLike, typeId, context) {
             return sanitizeFloat(typeLike);
         case arrow_1.Type.Binary:
             return new arrow_1.Binary();
+        case arrow_1.Type.LargeBinary:
+            return new arrow_1.LargeBinary();
         case arrow_1.Type.Utf8:
             return new arrow_1.Utf8();
+        case arrow_1.Type.LargeUtf8:
+            return new arrow_1.LargeUtf8();
         case arrow_1.Type.Bool:
             return new arrow_1.Bool();
         case arrow_1.Type.Decimal:
@@ -382,7 +441,7 @@ function sanitizeField(fieldLike) {
     return sanitizeFieldWithContext(fieldLike, createSanitizationContext());
 }
 function sanitizeFieldWithContext(fieldLike, context) {
-    if (fieldLike instanceof arrow_1.Field) {
+    if (isOwnInstance(fieldLike, arrow_1.Field)) {
         return fieldLike;
     }
     if (typeof fieldLike !== "object" || fieldLike === null) {
@@ -425,7 +484,7 @@ function sanitizeSchema(schemaLike) {
     return sanitizeSchemaWithContext(schemaLike, createSanitizationContext());
 }
 function sanitizeSchemaWithContext(schemaLike, context) {
-    if (schemaLike instanceof arrow_1.Schema) {
+    if (isOwnInstance(schemaLike, arrow_1.Schema)) {
         return schemaLike;
     }
     if (typeof schemaLike !== "object" || schemaLike === null) {
@@ -445,7 +504,7 @@ function sanitizeSchemaWithContext(schemaLike, context) {
     return new arrow_1.Schema(sanitizedFields, metadata);
 }
 function sanitizeTable(tableLike) {
-    if (tableLike instanceof arrow_1.Table) {
+    if (isOwnInstance(tableLike, arrow_1.Table)) {
         return tableLike;
     }
     if (typeof tableLike !== "object" || tableLike === null) {
@@ -463,7 +522,7 @@ function sanitizeTable(tableLike) {
     return new arrow_1.Table(schema, batches);
 }
 function sanitizeRecordBatch(batchLike, context) {
-    if (batchLike instanceof arrow_1.RecordBatch) {
+    if (isOwnInstance(batchLike, arrow_1.RecordBatch)) {
         return batchLike;
     }
     if (typeof batchLike !== "object" || batchLike === null) {
@@ -480,7 +539,7 @@ function sanitizeRecordBatch(batchLike, context) {
     return new arrow_1.RecordBatch(schema, data);
 }
 function sanitizeData(dataLike, context) {
-    if (dataLike instanceof apache_arrow_1.Data) {
+    if (isOwnInstance(dataLike, apache_arrow_1.Data)) {
         return dataLike;
     }
     const cachedData = context.data.get(dataLike);
@@ -496,19 +555,26 @@ function sanitizeData(dataLike, context) {
             context.vectors.set(dictionaryLike, dictionary);
         }
     }
-    const data = new apache_arrow_1.Data(sanitizeTypeWithContext(dataLike.type, context), dataLike.offset, dataLike.length, dataLike.nullCount, {
+    // Arrow 21 added an eighth constructor parameter (`variadicBuffers`); older
+    // releases ignore the extra argument but their typings do not declare it.
+    const DataCtor = apache_arrow_1.Data;
+    const data = new DataCtor(sanitizeTypeWithContext(dataLike.type, context), dataLike.offset, dataLike.length, dataLike.nullCount, {
         [apache_arrow_1.BufferType.OFFSET]: dataLike.valueOffsets,
         [apache_arrow_1.BufferType.DATA]: dataLike.values,
         [apache_arrow_1.BufferType.VALIDITY]: dataLike.nullBitmap,
         [apache_arrow_1.BufferType.TYPE]: dataLike.typeIds,
-    }, dataLike.children.map((child) => sanitizeData(child, context)), dictionary);
+    }, dataLike.children.map((child) => sanitizeData(child, context)), dictionary, dataLike.variadicBuffers);
     context.data.set(dataLike, data);
     return data;
 }
 const constructorsByTypeName = {
     null: () => new arrow_1.Null(),
     binary: () => new arrow_1.Binary(),
+    largebinary: () => new arrow_1.LargeBinary(),
+    binaryview: () => new (newerTypeConstructor("BinaryView"))(),
     utf8: () => new arrow_1.Utf8(),
+    largeutf8: () => new arrow_1.LargeUtf8(),
+    utf8view: () => new (newerTypeConstructor("Utf8View"))(),
     bool: () => new arrow_1.Bool(),
     int8: () => new arrow_1.Int8(),
     int16: () => new arrow_1.Int16(),
@@ -529,6 +595,7 @@ const constructorsByTypeName = {
     timesecond: () => new arrow_1.TimeSecond(),
     intervaldaytime: () => new arrow_1.IntervalDayTime(),
     intervalyearmonth: () => new arrow_1.IntervalYearMonth(),
+    intervalmonthdaynano: () => new (newerTypeConstructor("IntervalMonthDayNano"))(),
     durationnanosecond: () => new arrow_1.DurationNanosecond(),
     durationmicrosecond: () => new arrow_1.DurationMicrosecond(),
     durationmillisecond: () => new arrow_1.DurationMillisecond(),

package/dist/schema.js

@@ -43,10 +43,18 @@ class SchemaInferrer {
         else if (this.providedSchema === undefined) {
             this.updateInferredField(path, value, row, current);
         }
+        else if (current instanceof DeferredTypeEvidence) {
+            if (value != null) {
+                this.addSchemaField(this.providedSchema, path, value, row);
+            }
+        }
+        else if (current instanceof FieldTree || apache_arrow_1.DataType.isStruct(current)) {
+            assertStructValue(path, value, row);
+        }
     }
     addField(path, value, row) {
         if (this.providedSchema !== undefined) {
-            this.addSchemaField(this.providedSchema, path, row);
+            this.addSchemaField(this.providedSchema, path, value, row);
             return;
         }
         const evidence = this.inferType(value, path) ?? DeferredTypeEvidence.from(value, row);
@@ -58,12 +66,18 @@ class SchemaInferrer {
             throw branchConflictError(conflict, row, "Struct");
         }
     }
-    addSchemaField(schema, path, row) {
+    addSchemaField(schema, path, value, row) {
         const field = fieldAtPath(schema, path);
         if (field === undefined) {
             throw new Error(`Found field not in schema: ${path.join(".")} at row ${row}`);
         }
-        const conflict = this.fields.set(path, field.type);
+        if (apache_arrow_1.DataType.isStruct(field.type)) {
+            assertStructValue(path, value, row);
+        }
+        // Nulls are kept as deferred evidence so that a struct field seen first as
+        // null can be replaced by a record in a later row, just as a record seen
+        // first already ignores later nulls.
+        const conflict = this.fields.set(path, DeferredTypeEvidence.from(value, row) ?? field.type, (existing) => existing instanceof DeferredTypeEvidence && existing.isOnlyNulls());
         if (conflict !== undefined) {
             throw branchConflictError(conflict, row, "Struct");
         }
@@ -332,6 +346,20 @@ function assertSupportedValue(value, field, row) {
         throw new Error(`Unsupported object value for field ${field} at row ${row}.`);
     }
 }
+/**
+ * Rejects primitives, arrays, and typed arrays given for a struct field, which
+ * would otherwise be written as a struct with every child null.
+ */
+function assertStructValue(path, value, row) {
+    if (value == null ||
+        (typeof value === "object" &&
+            !Array.isArray(value) &&
+            !ArrayBuffer.isView(value))) {
+        return;
+    }
+    const kind = typeof value === "object" ? value.constructor.name : typeof value;
+    throw new Error(`Expected a struct value for field ${path.join(".")} at row ${row}, got ${kind}.`);
+}
 function fieldAtPath(schema, path) {
     let fields = schema.fields;
     let field;

package/dist/table.d.ts

@@ -1,5 +1,5 @@
 import { Table as ArrowTable, Data, DataType, Field, IntoVector, MultiVector, Schema } from "./arrow";
-import { BlobFile } from "./blob";
+import { BlobFile, BlobReadOptions } from "./blob";
 import { IndexOptions } from "./indices";
 import { Job } from "./job";
 import { MergeInsertBuilder } from "./merge";
@@ -65,6 +65,16 @@ export interface AddDataOptions {
      * ```
      */
     progress: (progress: WriteProgress) => void;
+    /**
+     * Whether blob URIs outside registered external bases may be written.
+     *
+     * Defaults to `false`. This option is supported only for local/native
+     * tables; remote tables return an error when it is enabled. An enabled write
+     * stores the absolute URI reference without registering a base or copying
+     * the external object into the database. The object must remain accessible
+     * when the blob is read later.
+     */
+    allowExternalBlobOutsideBases?: boolean;
 }
 export interface UpdateOptions {
     /**
@@ -424,8 +434,12 @@ export declare abstract class Table {
      * Reads the table's current checkout. IDs from another version can fail after
      * compaction unless stable row ids are enabled. Results keep input order and
      * duplicates. Null blobs are `null`. Empty blobs are empty buffers.
+     * Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes.
+     * The client splits requests automatically and reads an individual larger
+     * blob through the Range route. This method still materializes all bytes in
+     * memory; use {@link Table.fetchBlobFiles} for large values.
      */
-    abstract fetchBlobs(column: string, rowIds: readonly (bigint | number)[]): Promise<(Buffer | null)[]>;
+    abstract fetchBlobs(column: string, rowIds: readonly (bigint | number)[], options?: BlobReadOptions): Promise<(Buffer | null)[]>;
     /**
      * Opens lazy blob handles for `column` at the given row IDs using the
      * table's current checkout.
@@ -433,7 +447,7 @@ export declare abstract class Table {
      * Preserves input order, duplicates, and nulls. Use this for large payloads.
      * See {@link Table.fetchBlobs} for row-ID validity across versions.
      */
-    abstract fetchBlobFiles(column: string, rowIds: readonly (bigint | number)[]): Promise<(BlobFile | null)[]>;
+    abstract fetchBlobFiles(column: string, rowIds: readonly (bigint | number)[], options?: BlobReadOptions): Promise<(BlobFile | null)[]>;
     /**
      * Create a search query to find the nearest neighbors
      * of the given query
@@ -542,7 +556,7 @@ export declare abstract class Table {
      * it: rejects tables that carry no view definition.
      * @ignore
      */
-    abstract refreshMaterializedView(full?: boolean, sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
+    abstract refreshMaterializedView(sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
     /** @ignore */
     abstract materializedViewDefinition(): Promise<string>;
     /**
@@ -904,8 +918,8 @@ export declare class LocalTable extends Table {
     takeOffsets(offsets: number[]): TakeQuery;
     takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery;
     blobColumns(): Promise<string[]>;
-    fetchBlobs(column: string, rowIds: readonly (bigint | number)[]): Promise<(Buffer | null)[]>;
-    fetchBlobFiles(column: string, rowIds: readonly (bigint | number)[]): Promise<(BlobFile | null)[]>;
+    fetchBlobs(column: string, rowIds: readonly (bigint | number)[], options?: BlobReadOptions): Promise<(Buffer | null)[]>;
+    fetchBlobFiles(column: string, rowIds: readonly (bigint | number)[], options?: BlobReadOptions): Promise<(BlobFile | null)[]>;
     query(): Query;
     search(query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query | AutoQuery;
     vectorSearch(vector: IntoVector | MultiVector): VectorQuery;
@@ -915,7 +929,7 @@ export declare class LocalTable extends Table {
     refreshColumn(column: string): Promise<RefreshColumnResult>;
     refreshColumnAsync(column: string): Promise<Job>;
     functionErrors(options?: FunctionErrorsOptions): Promise<FunctionErrors>;
-    refreshMaterializedView(full?: boolean, sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
+    refreshMaterializedView(sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
     materializedViewDefinition(): Promise<string>;
     alterColumns(columnAlterations: ColumnAlteration[]): Promise<AlterColumnsResult>;
     updateFieldMetadata(updates: FieldMetadataUpdate[]): Promise<UpdateFieldMetadataResult>;

package/dist/table.js

@@ -84,7 +84,7 @@ class LocalTable extends Table {
                 }
             }
             : undefined;
-        return await this.inner.add(buffer, mode, progress);
+        return await this.inner.add(buffer, mode, progress, options?.allowExternalBlobOutsideBases);
     }
     async update(optsOrUpdates, options) {
         const isValues = "values" in optsOrUpdates && typeof optsOrUpdates.values !== "string";
@@ -169,13 +169,13 @@ class LocalTable extends Table {
     blobColumns() {
         return this.inner.blobColumns();
     }
-    async fetchBlobs(column, rowIds) {
-        const values = await this.inner.fetchBlobs(column, rowIdsToBigInts(rowIds));
+    async fetchBlobs(column, rowIds, options) {
+        const values = await (0, blob_1.runWithSignal)(options?.signal, (signal) => this.inner.fetchBlobs(column, rowIdsToBigInts(rowIds), signal));
         // N-API Option maps missing values to undefined. Collapse those to null.
         return values.map((value) => value ?? null);
     }
-    async fetchBlobFiles(column, rowIds) {
-        const files = await this.inner.fetchBlobFiles(column, rowIdsToBigInts(rowIds));
+    async fetchBlobFiles(column, rowIds, options) {
+        const files = await (0, blob_1.runWithSignal)(options?.signal, (signal) => this.inner.fetchBlobFiles(column, rowIdsToBigInts(rowIds), signal));
         // N-API Option maps missing values to undefined. Collapse those to null.
         return files.map((file) => file == null ? null : blob_1.BlobFile.fromNative(file));
     }
@@ -280,8 +280,8 @@ class LocalTable extends Table {
         (0, materialized_view_1.validateNonNegativeInteger)(options?.limit, "limit");
         return await this.inner.functionErrors(options);
     }
-    async refreshMaterializedView(full, sourceVersion) {
-        return await this.inner.refreshMaterializedView(full, sourceVersion);
+    async refreshMaterializedView(sourceVersion) {
+        return await this.inner.refreshMaterializedView(sourceVersion);
     }
     async materializedViewDefinition() {
         return await this.inner.materializedViewDefinition();

package/dist/util.js

@@ -10,6 +10,15 @@ function toSQL(value) {
         return `'${value.replace(/'/g, "''")}'`;
     }
     else if (typeof value === "number") {
+        // toString() gives "NaN" / "Infinity", which SQL reads as column names.
+        if (Number.isNaN(value)) {
+            return "CAST('NaN' AS DOUBLE)";
+        }
+        else if (!Number.isFinite(value)) {
+            return value > 0
+                ? "CAST('Infinity' AS DOUBLE)"
+                : "CAST('-Infinity' AS DOUBLE)";
+        }
         return value.toString();
     }
     else if (typeof value === "boolean") {

package/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.40.0-beta.11",
+  "version": "0.40.0-beta.13",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -49,10 +49,14 @@
     "@types/jest": "^29.1.2",
     "@types/node": "22.7.4",
     "@types/tmp": "^0.2.6",
+    "apache-arrow": "21.2.0",
     "apache-arrow-15": "npm:apache-arrow@15.0.0",
     "apache-arrow-16": "npm:apache-arrow@16.0.0",
     "apache-arrow-17": "npm:apache-arrow@17.0.0",
     "apache-arrow-18": "npm:apache-arrow@18.0.0",
+    "apache-arrow-19": "npm:apache-arrow@19.0.1",
+    "apache-arrow-20": "npm:apache-arrow@20.0.0",
+    "apache-arrow-21": "npm:apache-arrow@21.2.0",
     "eslint": "^8.57.0",
     "jest": "^29.7.0",
     "shx": "^0.3.4",
@@ -106,17 +110,17 @@
   "optionalDependencies": {
     "@huggingface/transformers": "3.0.2",
     "openai": "4.29.2",
-    "@lancedb/lancedb-darwin-arm64": "0.40.0-beta.11",
-    "@lancedb/lancedb-linux-x64-gnu": "0.40.0-beta.11",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.40.0-beta.11",
-    "@lancedb/lancedb-linux-x64-musl": "0.40.0-beta.11",
-    "@lancedb/lancedb-linux-arm64-musl": "0.40.0-beta.11",
-    "@lancedb/lancedb-win32-x64-msvc": "0.40.0-beta.11",
-    "@lancedb/lancedb-win32-arm64-msvc": "0.40.0-beta.11"
+    "@lancedb/lancedb-darwin-arm64": "0.40.0-beta.13",
+    "@lancedb/lancedb-linux-x64-gnu": "0.40.0-beta.13",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.40.0-beta.13",
+    "@lancedb/lancedb-linux-x64-musl": "0.40.0-beta.13",
+    "@lancedb/lancedb-linux-arm64-musl": "0.40.0-beta.13",
+    "@lancedb/lancedb-win32-x64-msvc": "0.40.0-beta.13",
+    "@lancedb/lancedb-win32-arm64-msvc": "0.40.0-beta.13"
   },
   "peerDependencies": {
     "@types/node": ">=22",
-    "apache-arrow": ">=15.0.0 <=18.1.0"
+    "apache-arrow": ">=15.0.0 <22.0.0"
   },
   "peerDependenciesMeta": {
     "@types/node": {