@lancedb/lancedb 0.5.0 → 0.5.2

package/nodejs-artifacts/embedding/index.js

@@ -1,7 +1,114 @@
 "use strict";
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.OpenAIEmbeddingFunction = exports.isEmbeddingFunction = void 0;
+exports.LanceSchema = exports.EmbeddingFunction = void 0;
+const arrow_1 = require("../arrow");
+const arrow_2 = require("../arrow");
+const sanitize_1 = require("../sanitize");
+const registry_1 = require("./registry");
 var embedding_function_1 = require("./embedding_function");
-Object.defineProperty(exports, "isEmbeddingFunction", { enumerable: true, get: function () { return embedding_function_1.isEmbeddingFunction; } });
-var openai_1 = require("./openai");
-Object.defineProperty(exports, "OpenAIEmbeddingFunction", { enumerable: true, get: function () { return openai_1.OpenAIEmbeddingFunction; } });
+Object.defineProperty(exports, "EmbeddingFunction", { enumerable: true, get: function () { return embedding_function_1.EmbeddingFunction; } });
+// We need to explicitly export '*' so that the `register` decorator actually registers the class.
+__exportStar(require("./openai"), exports);
+__exportStar(require("./registry"), exports);
+/**
+ * Create a schema with embedding functions.
+ *
+ * @param fields
+ * @returns Schema
+ * @example
+ * ```ts
+ * class MyEmbeddingFunction extends EmbeddingFunction {
+ * // ...
+ * }
+ * const func = new MyEmbeddingFunction();
+ * const schema = LanceSchema({
+ *   id: new Int32(),
+ *   text: func.sourceField(new Utf8()),
+ *   vector: func.vectorField(),
+ *   // optional: specify the datatype and/or dimensions
+ *   vector2: func.vectorField({ datatype: new Float32(), dims: 3}),
+ * });
+ *
+ * const table = await db.createTable("my_table", data, { schema });
+ * ```
+ */
+function LanceSchema(fields) {
+    const arrowFields = [];
+    const embeddingFunctions = new Map();
+    Object.entries(fields).forEach(([key, value]) => {
+        if ((0, arrow_2.isDataType)(value)) {
+            arrowFields.push(new arrow_1.Field(key, (0, sanitize_1.sanitizeType)(value), true));
+        }
+        else {
+            const [dtype, metadata] = value;
+            arrowFields.push(new arrow_1.Field(key, (0, sanitize_1.sanitizeType)(dtype), true));
+            parseEmbeddingFunctions(embeddingFunctions, key, metadata);
+        }
+    });
+    const registry = (0, registry_1.getRegistry)();
+    const metadata = registry.getTableMetadata(Array.from(embeddingFunctions.values()));
+    const schema = new arrow_1.Schema(arrowFields, metadata);
+    return schema;
+}
+exports.LanceSchema = LanceSchema;
+function parseEmbeddingFunctions(embeddingFunctions, key, metadata) {
+    if (metadata.has("source_column_for")) {
+        const embedFunction = metadata.get("source_column_for");
+        const current = embeddingFunctions.get(embedFunction);
+        if (current !== undefined) {
+            embeddingFunctions.set(embedFunction, {
+                ...current,
+                sourceColumn: key,
+            });
+        }
+        else {
+            embeddingFunctions.set(embedFunction, {
+                sourceColumn: key,
+                function: embedFunction,
+            });
+        }
+    }
+    else if (metadata.has("vector_column_for")) {
+        const embedFunction = metadata.get("vector_column_for");
+        const current = embeddingFunctions.get(embedFunction);
+        if (current !== undefined) {
+            embeddingFunctions.set(embedFunction, {
+                ...current,
+                vectorColumn: key,
+            });
+        }
+        else {
+            embeddingFunctions.set(embedFunction, {
+                vectorColumn: key,
+                function: embedFunction,
+            });
+        }
+    }
+}

package/nodejs-artifacts/embedding/openai.d.ts

@@ -1,8 +1,17 @@
-import { type EmbeddingFunction } from "./embedding_function";
-export declare class OpenAIEmbeddingFunction implements EmbeddingFunction<string> {
-    private readonly _openai;
-    private readonly _modelName;
-    constructor(sourceColumn: string, openAIKey: string, modelName?: string);
-    embed(data: string[]): Promise<number[][]>;
-    sourceColumn: string;
+import { Float } from "../arrow";
+import { EmbeddingFunction } from "./embedding_function";
+export type OpenAIOptions = {
+    apiKey?: string;
+    model?: string;
+};
+export declare class OpenAIEmbeddingFunction extends EmbeddingFunction<string, OpenAIOptions> {
+    #private;
+    constructor(options?: OpenAIOptions);
+    toJSON(): {
+        model: string;
+    };
+    ndims(): number;
+    embeddingDataType(): Float;
+    computeSourceEmbeddings(data: string[]): Promise<number[][]>;
+    computeQueryEmbeddings(data: string): Promise<number[]>;
 }

package/nodejs-artifacts/embedding/openai.js

@@ -12,12 +12,30 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.OpenAIEmbeddingFunction = void 0;
-class OpenAIEmbeddingFunction {
-    _openai;
-    _modelName;
-    constructor(sourceColumn, openAIKey, modelName = "text-embedding-ada-002") {
+const arrow_1 = require("../arrow");
+const embedding_function_1 = require("./embedding_function");
+const registry_1 = require("./registry");
+let OpenAIEmbeddingFunction = class OpenAIEmbeddingFunction extends embedding_function_1.EmbeddingFunction {
+    #openai;
+    #modelName;
+    constructor(options = { model: "text-embedding-ada-002" }) {
+        super();
+        const openAIKey = options?.apiKey ?? process.env.OPENAI_API_KEY;
+        if (!openAIKey) {
+            throw new Error("OpenAI API key is required");
+        }
+        const modelName = options?.model ?? "text-embedding-ada-002";
         /**
          * @type {import("openai").default}
          */
@@ -30,16 +48,35 @@ class OpenAIEmbeddingFunction {
         catch {
             throw new Error("please install openai@^4.24.1 using npm install openai");
         }
-        this.sourceColumn = sourceColumn;
         const configuration = {
             apiKey: openAIKey,
         };
-        this._openai = new Openai(configuration);
-        this._modelName = modelName;
+        this.#openai = new Openai(configuration);
+        this.#modelName = modelName;
+    }
+    toJSON() {
+        return {
+            model: this.#modelName,
+        };
+    }
+    ndims() {
+        switch (this.#modelName) {
+            case "text-embedding-ada-002":
+                return 1536;
+            case "text-embedding-3-large":
+                return 3072;
+            case "text-embedding-3-small":
+                return 1536;
+            default:
+                return null;
+        }
     }
-    async embed(data) {
-        const response = await this._openai.embeddings.create({
-            model: this._modelName,
+    embeddingDataType() {
+        return new arrow_1.Float32();
+    }
+    async computeSourceEmbeddings(data) {
+        const response = await this.#openai.embeddings.create({
+            model: this.#modelName,
             input: data,
         });
         const embeddings = [];
@@ -48,6 +85,19 @@ class OpenAIEmbeddingFunction {
         }
         return embeddings;
     }
-    sourceColumn;
-}
+    async computeQueryEmbeddings(data) {
+        if (typeof data !== "string") {
+            throw new Error("Data must be a string");
+        }
+        const response = await this.#openai.embeddings.create({
+            model: this.#modelName,
+            input: data,
+        });
+        return response.data[0].embedding;
+    }
+};
 exports.OpenAIEmbeddingFunction = OpenAIEmbeddingFunction;
+exports.OpenAIEmbeddingFunction = OpenAIEmbeddingFunction = __decorate([
+    (0, registry_1.register)("openai"),
+    __metadata("design:paramtypes", [Object])
+], OpenAIEmbeddingFunction);

package/nodejs-artifacts/embedding/registry.d.ts

@@ -0,0 +1,58 @@
+import type { EmbeddingFunction } from "./embedding_function";
+import "reflect-metadata";
+export interface EmbeddingFunctionOptions {
+    [key: string]: unknown;
+}
+export interface EmbeddingFunctionFactory<T extends EmbeddingFunction = EmbeddingFunction> {
+    new (modelOptions?: EmbeddingFunctionOptions): T;
+}
+interface EmbeddingFunctionCreate<T extends EmbeddingFunction> {
+    create(options?: EmbeddingFunctionOptions): T;
+}
+/**
+ * This is a singleton class used to register embedding functions
+ * and fetch them by name. It also handles serializing and deserializing.
+ * You can implement your own embedding function by subclassing EmbeddingFunction
+ * or TextEmbeddingFunction and registering it with the registry
+ */
+export declare class EmbeddingFunctionRegistry {
+    #private;
+    /**
+     * Register an embedding function
+     * @param name The name of the function
+     * @param func The function to register
+     * @throws Error if the function is already registered
+     */
+    register<T extends EmbeddingFunctionFactory = EmbeddingFunctionFactory>(this: EmbeddingFunctionRegistry, alias?: string): (ctor: T) => any;
+    /**
+     * Fetch an embedding function by name
+     * @param name The name of the function
+     */
+    get<T extends EmbeddingFunction<unknown> = EmbeddingFunction>(name: string): EmbeddingFunctionCreate<T> | undefined;
+    /**
+     * reset the registry to the initial state
+     */
+    reset(this: EmbeddingFunctionRegistry): void;
+    /**
+     * @ignore
+     */
+    parseFunctions(this: EmbeddingFunctionRegistry, metadata: Map<string, string>): Map<string, EmbeddingFunctionConfig>;
+    functionToMetadata(conf: EmbeddingFunctionConfig): Record<string, any>;
+    getTableMetadata(functions: EmbeddingFunctionConfig[]): Map<string, string>;
+}
+export declare function register(name?: string): (ctor: EmbeddingFunctionFactory<EmbeddingFunction<any, import("./embedding_function").FunctionOptions>>) => any;
+/**
+ * Utility function to get the global instance of the registry
+ * @returns `EmbeddingFunctionRegistry` The global instance of the registry
+ * @example
+ * ```ts
+ * const registry = getRegistry();
+ * const openai = registry.get("openai").create();
+ */
+export declare function getRegistry(): EmbeddingFunctionRegistry;
+export interface EmbeddingFunctionConfig {
+    sourceColumn: string;
+    vectorColumn?: string;
+    function: EmbeddingFunction;
+}
+export {};

package/nodejs-artifacts/embedding/registry.js

@@ -0,0 +1,127 @@
+"use strict";
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getRegistry = exports.register = exports.EmbeddingFunctionRegistry = void 0;
+require("reflect-metadata");
+/**
+ * This is a singleton class used to register embedding functions
+ * and fetch them by name. It also handles serializing and deserializing.
+ * You can implement your own embedding function by subclassing EmbeddingFunction
+ * or TextEmbeddingFunction and registering it with the registry
+ */
+class EmbeddingFunctionRegistry {
+    #functions = new Map();
+    /**
+     * Register an embedding function
+     * @param name The name of the function
+     * @param func The function to register
+     * @throws Error if the function is already registered
+     */
+    register(alias) {
+        const self = this;
+        return function (ctor) {
+            if (!alias) {
+                alias = ctor.name;
+            }
+            if (self.#functions.has(alias)) {
+                throw new Error(`Embedding function with alias "${alias}" already exists`);
+            }
+            self.#functions.set(alias, ctor);
+            Reflect.defineMetadata("lancedb::embedding::name", alias, ctor);
+            return ctor;
+        };
+    }
+    /**
+     * Fetch an embedding function by name
+     * @param name The name of the function
+     */
+    get(name) {
+        const factory = this.#functions.get(name);
+        if (!factory) {
+            return undefined;
+        }
+        return {
+            create: function (options) {
+                return new factory(options);
+            },
+        };
+    }
+    /**
+     * reset the registry to the initial state
+     */
+    reset() {
+        this.#functions.clear();
+    }
+    /**
+     * @ignore
+     */
+    parseFunctions(metadata) {
+        if (!metadata.has("embedding_functions")) {
+            return new Map();
+        }
+        else {
+            const functions = (JSON.parse(metadata.get("embedding_functions")));
+            return new Map(functions.map((f) => {
+                const fn = this.get(f.name);
+                if (!fn) {
+                    throw new Error(`Function "${f.name}" not found in registry`);
+                }
+                return [
+                    f.name,
+                    {
+                        sourceColumn: f.sourceColumn,
+                        vectorColumn: f.vectorColumn,
+                        function: this.get(f.name).create(f.model),
+                    },
+                ];
+            }));
+        }
+    }
+    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+    functionToMetadata(conf) {
+        // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+        const metadata = {};
+        const name = Reflect.getMetadata("lancedb::embedding::name", conf.function.constructor);
+        metadata["sourceColumn"] = conf.sourceColumn;
+        metadata["vectorColumn"] = conf.vectorColumn ?? "vector";
+        metadata["name"] = name ?? conf.function.constructor.name;
+        metadata["model"] = conf.function.toJSON();
+        return metadata;
+    }
+    getTableMetadata(functions) {
+        const metadata = new Map();
+        const jsonData = functions.map((conf) => this.functionToMetadata(conf));
+        metadata.set("embedding_functions", JSON.stringify(jsonData));
+        return metadata;
+    }
+}
+exports.EmbeddingFunctionRegistry = EmbeddingFunctionRegistry;
+const _REGISTRY = new EmbeddingFunctionRegistry();
+function register(name) {
+    return _REGISTRY.register(name);
+}
+exports.register = register;
+/**
+ * Utility function to get the global instance of the registry
+ * @returns `EmbeddingFunctionRegistry` The global instance of the registry
+ * @example
+ * ```ts
+ * const registry = getRegistry();
+ * const openai = registry.get("openai").create();
+ */
+function getRegistry() {
+    return _REGISTRY;
+}
+exports.getRegistry = getRegistry;

package/nodejs-artifacts/native.d.ts

@@ -102,6 +102,7 @@ export const enum WriteMode {
 }
 /** Write options when creating a Table. */
 export interface WriteOptions {
+  /** Write mode for writing to a table. */
   mode?: WriteMode
 }
 export interface OpenTableOptions {
@@ -123,8 +124,8 @@ export class Connection {
    * - buf: The buffer containing the IPC file.
    *
    */
-  createTable(name: string, buf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null): Promise<Table>
-  createEmptyTable(name: string, schemaBuf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null): Promise<Table>
+  createTable(name: string, buf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, useLegacyFormat?: boolean | undefined | null): Promise<Table>
+  createEmptyTable(name: string, schemaBuf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, useLegacyFormat?: boolean | undefined | null): Promise<Table>
   openTable(name: string, storageOptions?: Record<string, string> | undefined | null, indexCacheSize?: number | undefined | null): Promise<Table>
   /** Drop table with the name. Or raise an error if the table does not exist. */
   dropTable(name: string): Promise<void>
@@ -142,7 +143,7 @@ export class Query {
   select(columns: Array<[string, string]>): void
   limit(limit: number): void
   nearestTo(vector: Float32Array): VectorQuery
-  execute(): Promise<RecordBatchIterator>
+  execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
 }
 export class VectorQuery {
   column(column: string): void
@@ -154,7 +155,7 @@ export class VectorQuery {
   onlyIf(predicate: string): void
   select(columns: Array<[string, string]>): void
   limit(limit: number): void
-  execute(): Promise<RecordBatchIterator>
+  execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
 }
 export class Table {
   display(): string

package/nodejs-artifacts/query.d.ts

@@ -1,4 +1,4 @@
-import { Table as ArrowTable, RecordBatch } from "apache-arrow";
+import { Table as ArrowTable, type IntoVector, RecordBatch } from "./arrow";
 import { RecordBatchIterator as NativeBatchIterator, Query as NativeQuery, Table as NativeTable, VectorQuery as NativeVectorQuery } from "./native";
 export declare class RecordBatchIterator implements AsyncIterator<RecordBatch> {
     private promisedInner?;
@@ -6,6 +6,18 @@ export declare class RecordBatchIterator implements AsyncIterator<RecordBatch> {
     constructor(promise?: Promise<NativeBatchIterator>);
     next(): Promise<IteratorResult<RecordBatch<any>>>;
 }
+/**
+ * Options that control the behavior of a particular query execution
+ */
+export interface QueryExecutionOptions {
+    /**
+     * The maximum number of rows to return in a single batch
+     *
+     * Batches may have fewer rows if the underlying data is stored
+     * in smaller chunks.
+     */
+    maxBatchLength?: number;
+}
 /** Common methods supported by all query types */
 export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery, QueryType> implements AsyncIterable<RecordBatch> {
     protected inner: NativeQueryType;
@@ -53,7 +65,7 @@ export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVecto
      * uses `Object.entries` which should preserve the insertion order of the object.  However,
      * object insertion order is easy to get wrong and `Map` is more foolproof.
      */
-    select(columns: string[] | Map<string, string> | Record<string, string>): QueryType;
+    select(columns: string[] | Map<string, string> | Record<string, string> | string): QueryType;
     /**
      * Set the maximum number of results to return.
      *
@@ -61,7 +73,7 @@ export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVecto
      * called then every valid row from the table will be returned.
      */
     limit(limit: number): QueryType;
-    protected nativeExecute(): Promise<NativeBatchIterator>;
+    protected nativeExecute(options?: Partial<QueryExecutionOptions>): Promise<NativeBatchIterator>;
     /**
      * Execute the query and return the results as an @see {@link AsyncIterator}
      * of @see {@link RecordBatch}.
@@ -73,12 +85,12 @@ export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVecto
      * single query)
      *
      */
-    protected execute(): RecordBatchIterator;
+    protected execute(options?: Partial<QueryExecutionOptions>): RecordBatchIterator;
     [Symbol.asyncIterator](): AsyncIterator<RecordBatch<any>>;
     /** Collect the results as an Arrow @see {@link ArrowTable}. */
-    toArrow(): Promise<ArrowTable>;
+    toArrow(options?: Partial<QueryExecutionOptions>): Promise<ArrowTable>;
     /** Collect the results as an array of objects. */
-    toArray(): Promise<unknown[]>;
+    toArray(options?: Partial<QueryExecutionOptions>): Promise<any[]>;
 }
 /**
  * An interface for a query that can be executed
@@ -244,5 +256,5 @@ export declare class Query extends QueryBase<NativeQuery, Query> {
      * Vector searches always have a `limit`.  If `limit` has not been called then
      * a default `limit` of 10 will be used.  @see {@link Query#limit}
      */
-    nearestTo(vector: unknown): VectorQuery;
+    nearestTo(vector: IntoVector): VectorQuery;
 }

package/nodejs-artifacts/query.js

@@ -14,7 +14,7 @@
 // limitations under the License.
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Query = exports.VectorQuery = exports.QueryBase = exports.RecordBatchIterator = void 0;
-const apache_arrow_1 = require("apache-arrow");
+const arrow_1 = require("./arrow");
 class RecordBatchIterator {
     promisedInner;
     inner;
@@ -34,7 +34,7 @@ class RecordBatchIterator {
         if (n == null) {
             return Promise.resolve({ done: true, value: null });
         }
-        const tbl = (0, apache_arrow_1.tableFromIPC)(n);
+        const tbl = (0, arrow_1.tableFromIPC)(n);
         if (tbl.batches.length != 1) {
             throw new Error("Expected only one batch");
         }
@@ -43,6 +43,18 @@ class RecordBatchIterator {
 }
 exports.RecordBatchIterator = RecordBatchIterator;
 /* eslint-enable */
+class RecordBatchIterable {
+    inner;
+    options;
+    constructor(inner, options) {
+        this.inner = inner;
+        this.options = options;
+    }
+    // biome-ignore lint/suspicious/noExplicitAny: skip
+    [Symbol.asyncIterator]() {
+        return new RecordBatchIterator(this.inner.execute(this.options?.maxBatchLength));
+    }
+}
 /** Common methods supported by all query types */
 class QueryBase {
     inner;
@@ -98,6 +110,9 @@ class QueryBase {
      */
     select(columns) {
         let columnTuples;
+        if (typeof columns === "string") {
+            columns = [columns];
+        }
         if (Array.isArray(columns)) {
             columnTuples = columns.map((c) => [c, c]);
         }
@@ -120,8 +135,8 @@ class QueryBase {
         this.inner.limit(limit);
         return this;
     }
-    nativeExecute() {
-        return this.inner.execute();
+    nativeExecute(options) {
+        return this.inner.execute(options?.maxBatchLength);
     }
     /**
      * Execute the query and return the results as an @see {@link AsyncIterator}
@@ -134,8 +149,8 @@ class QueryBase {
      * single query)
      *
      */
-    execute() {
-        return new RecordBatchIterator(this.nativeExecute());
+    execute(options) {
+        return new RecordBatchIterator(this.nativeExecute(options));
     }
     // biome-ignore lint/suspicious/noExplicitAny: skip
     [Symbol.asyncIterator]() {
@@ -143,17 +158,17 @@ class QueryBase {
         return new RecordBatchIterator(promise);
     }
     /** Collect the results as an Arrow @see {@link ArrowTable}. */
-    async toArrow() {
+    async toArrow(options) {
         const batches = [];
-        for await (const batch of this) {
+        for await (const batch of new RecordBatchIterable(this.inner, options)) {
             batches.push(batch);
         }
-        return new apache_arrow_1.Table(batches);
+        return new arrow_1.Table(batches);
     }
     /** Collect the results as an array of objects. */
-    async toArray() {
-        const tbl = await this.toArrow();
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+    // biome-ignore lint/suspicious/noExplicitAny: arrow.toArrow() returns any[]
+    async toArray(options) {
+        const tbl = await this.toArrow(options);
         return tbl.toArray();
     }
 }
@@ -339,7 +354,6 @@ class Query extends QueryBase {
      * a default `limit` of 10 will be used.  @see {@link Query#limit}
      */
     nearestTo(vector) {
-        // biome-ignore lint/suspicious/noExplicitAny: skip
         const vectorQuery = this.inner.nearestTo(Float32Array.from(vector));
         return new VectorQuery(vectorQuery);
     }

package/nodejs-artifacts/sanitize.d.ts

@@ -1,4 +1,25 @@
-import { Schema } from "apache-arrow";
+import type { TKeys } from "apache-arrow/type";
+import { DataType, Date_, Decimal, DenseUnion, Dictionary, Duration, Field, FixedSizeBinary, FixedSizeList, Float, Int, Interval, List, Map_, Schema, SparseUnion, Struct, Time, Timestamp, TimestampMicrosecond, TimestampMillisecond, TimestampNanosecond, TimestampSecond, Type, Union } from "./arrow";
+export declare function sanitizeMetadata(metadataLike?: unknown): Map<string, string> | undefined;
+export declare function sanitizeInt(typeLike: object): Int<Type.Int | Type.Int8 | Type.Int16 | Type.Int32 | Type.Int64 | Type.Uint8 | Type.Uint16 | Type.Uint32 | Type.Uint64>;
+export declare function sanitizeFloat(typeLike: object): Float<Type.Float | Type.Float16 | Type.Float32 | Type.Float64>;
+export declare function sanitizeDecimal(typeLike: object): Decimal;
+export declare function sanitizeDate(typeLike: object): Date_<import("apache-arrow/type").Dates>;
+export declare function sanitizeTime(typeLike: object): Time<Type.Time | Type.TimeSecond | Type.TimeMillisecond | Type.TimeMicrosecond | Type.TimeNanosecond>;
+export declare function sanitizeTimestamp(typeLike: object): Timestamp<Type.Timestamp | Type.TimestampSecond | Type.TimestampMillisecond | Type.TimestampMicrosecond | Type.TimestampNanosecond>;
+export declare function sanitizeTypedTimestamp(typeLike: object, Datatype: typeof TimestampNanosecond | typeof TimestampMicrosecond | typeof TimestampMillisecond | typeof TimestampSecond): TimestampSecond | TimestampMillisecond | TimestampMicrosecond | TimestampNanosecond;
+export declare function sanitizeInterval(typeLike: object): Interval<Type.Interval | Type.IntervalDayTime | Type.IntervalYearMonth>;
+export declare function sanitizeList(typeLike: object): List<any>;
+export declare function sanitizeStruct(typeLike: object): Struct<any>;
+export declare function sanitizeUnion(typeLike: object): Union<Type.Union | Type.DenseUnion | Type.SparseUnion>;
+export declare function sanitizeTypedUnion(typeLike: object, UnionType: typeof DenseUnion | typeof SparseUnion): SparseUnion | DenseUnion;
+export declare function sanitizeFixedSizeBinary(typeLike: object): FixedSizeBinary;
+export declare function sanitizeFixedSizeList(typeLike: object): FixedSizeList<any>;
+export declare function sanitizeMap(typeLike: object): Map_<any, any>;
+export declare function sanitizeDuration(typeLike: object): Duration<Type.Duration | Type.DurationSecond | Type.DurationMillisecond | Type.DurationMicrosecond | Type.DurationNanosecond>;
+export declare function sanitizeDictionary(typeLike: object): Dictionary<DataType<any, any>, TKeys>;
+export declare function sanitizeType(typeLike: unknown): DataType<any>;
+export declare function sanitizeField(fieldLike: unknown): Field;
 /**
  * Convert something schemaLike into a Schema instance
  *