@lancedb/lancedb 0.7.1 → 0.9.0

package/dist/arrow.d.ts

@@ -1,5 +1,5 @@
 /// <reference types="node" />
-import { Table as ArrowTable, Binary, BufferType, DataType, Field, FixedSizeBinary, FixedSizeList, Float, Int, LargeBinary, List, Null, RecordBatch, Schema, Struct, Utf8 } from "apache-arrow";
+import { Table as ArrowTable, Binary, BufferType, Field, FixedSizeBinary, FixedSizeList, Float, Int, LargeBinary, List, Null, RecordBatch, Schema, Struct, Utf8 } from "apache-arrow";
 import { Buffers } from "apache-arrow/data";
 import { type EmbeddingFunction } from "./embedding/embedding_function";
 import { EmbeddingFunctionConfig } from "./embedding/registry";
@@ -38,7 +38,6 @@ export type TableLike = ArrowTable | {
 };
 export type IntoVector = Float32Array | Float64Array | number[] | Promise<Float32Array | Float64Array | number[]>;
 export declare function isArrowTable(value: object): value is TableLike;
-export declare function isDataType(value: unknown): value is DataType;
 export declare function isNull(value: unknown): value is Null;
 export declare function isInt(value: unknown): value is Int;
 export declare function isFloat(value: unknown): value is Float;
@@ -200,7 +199,7 @@ export declare function makeEmptyTable(schema: SchemaLike, metadata?: Map<string
  */
 export declare function convertToTable(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunctionConfig, makeTableOptions?: Partial<MakeArrowTableOptions>): Promise<ArrowTable>;
 /** Creates the Arrow Type for a Vector column with dimension `dim` */
-export declare function newVectorType<T extends Float>(dim: number, innerType: T): FixedSizeList<T>;
+export declare function newVectorType<T extends Float>(dim: number, innerType: unknown): FixedSizeList<T>;
 /**
  * Serialize an Array of records into a buffer using the Arrow IPC File serialization
  *

package/dist/arrow.js

@@ -27,7 +27,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.createEmptyTable = exports.fromTableToStreamBuffer = exports.fromDataToBuffer = exports.fromTableToBuffer = exports.fromRecordsToStreamBuffer = exports.fromRecordsToBuffer = exports.newVectorType = exports.convertToTable = exports.makeEmptyTable = exports.makeArrowTable = exports.MakeArrowTableOptions = exports.VectorColumnOptions = exports.isFixedSizeList = exports.isFixedSizeBinary = exports.isUnion = exports.isStruct = exports.isList = exports.isDuration = exports.isInterval = exports.isTimestamp = exports.isTime = exports.isDate = exports.isDecimal = exports.isBool = exports.isLargeUtf8 = exports.isUtf8 = exports.isLargeBinary = exports.isBinary = exports.isFloat = exports.isInt = exports.isNull = exports.isDataType = exports.isArrowTable = void 0;
+exports.createEmptyTable = exports.fromTableToStreamBuffer = exports.fromDataToBuffer = exports.fromTableToBuffer = exports.fromRecordsToStreamBuffer = exports.fromRecordsToBuffer = exports.newVectorType = exports.convertToTable = exports.makeEmptyTable = exports.makeArrowTable = exports.MakeArrowTableOptions = exports.VectorColumnOptions = exports.isFixedSizeList = exports.isFixedSizeBinary = exports.isUnion = exports.isStruct = exports.isList = exports.isDuration = exports.isInterval = exports.isTimestamp = exports.isTime = exports.isDate = exports.isDecimal = exports.isBool = exports.isLargeUtf8 = exports.isUtf8 = exports.isLargeBinary = exports.isBinary = exports.isFloat = exports.isInt = exports.isNull = exports.isArrowTable = void 0;
 const apache_arrow_1 = require("apache-arrow");
 const registry_1 = require("./embedding/registry");
 const sanitize_1 = require("./sanitize");
@@ -38,31 +38,6 @@ function isArrowTable(value) {
     return "schema" in value && "batches" in value;
 }
 exports.isArrowTable = isArrowTable;
-function isDataType(value) {
-    return (value instanceof apache_arrow_1.DataType ||
-        apache_arrow_1.DataType.isNull(value) ||
-        apache_arrow_1.DataType.isInt(value) ||
-        apache_arrow_1.DataType.isFloat(value) ||
-        apache_arrow_1.DataType.isBinary(value) ||
-        apache_arrow_1.DataType.isLargeBinary(value) ||
-        apache_arrow_1.DataType.isUtf8(value) ||
-        apache_arrow_1.DataType.isLargeUtf8(value) ||
-        apache_arrow_1.DataType.isBool(value) ||
-        apache_arrow_1.DataType.isDecimal(value) ||
-        apache_arrow_1.DataType.isDate(value) ||
-        apache_arrow_1.DataType.isTime(value) ||
-        apache_arrow_1.DataType.isTimestamp(value) ||
-        apache_arrow_1.DataType.isInterval(value) ||
-        apache_arrow_1.DataType.isDuration(value) ||
-        apache_arrow_1.DataType.isList(value) ||
-        apache_arrow_1.DataType.isStruct(value) ||
-        apache_arrow_1.DataType.isUnion(value) ||
-        apache_arrow_1.DataType.isFixedSizeBinary(value) ||
-        apache_arrow_1.DataType.isFixedSizeList(value) ||
-        apache_arrow_1.DataType.isMap(value) ||
-        apache_arrow_1.DataType.isDictionary(value));
-}
-exports.isDataType = isDataType;
 function isNull(value) {
     return value instanceof apache_arrow_1.Null || apache_arrow_1.DataType.isNull(value);
 }
@@ -466,7 +441,7 @@ function makeVector(values, type, stringAsDictionary) {
 /** Helper function to apply embeddings from metadata to an input table */
 async function applyEmbeddingsFromMetadata(table, schema) {
     const registry = (0, registry_1.getRegistry)();
-    const functions = registry.parseFunctions(schema.metadata);
+    const functions = await registry.parseFunctions(schema.metadata);
     const columns = Object.fromEntries(table.schema.fields.map((field) => [
         field.name,
         table.getChild(field.name),

package/dist/connection.d.ts

@@ -27,10 +27,19 @@ export interface CreateTableOptions {
      * The available options are described at https://lancedb.github.io/lancedb/guides/storage/
      */
     storageOptions?: Record<string, string>;
+    /**
+     * The version of the data storage format to use.
+     *
+     * The default is `legacy`, which is Lance format v1.
+     * `stable` is the new format, which is Lance format v2.
+     */
+    dataStorageVersion?: string;
     /**
      * If true then data files will be written with the legacy format
      *
      * The default is true while the new format is in beta
+     *
+     * Deprecated.
      */
     useLegacyFormat?: boolean;
     schema?: SchemaLike;

package/dist/connection.js

@@ -72,7 +72,14 @@ class LocalConnection extends Connection {
             throw new Error("data is required");
         }
         const { buf, mode } = await table_1.Table.parseTableData(data, options);
-        const innerTable = await this.inner.createTable(nameOrOptions, buf, mode, cleanseStorageOptions(options?.storageOptions), options?.useLegacyFormat);
+        let dataStorageVersion = "legacy";
+        if (options?.dataStorageVersion !== undefined) {
+            dataStorageVersion = options.dataStorageVersion;
+        }
+        else if (options?.useLegacyFormat !== undefined) {
+            dataStorageVersion = options.useLegacyFormat ? "legacy" : "stable";
+        }
+        const innerTable = await this.inner.createTable(nameOrOptions, buf, mode, cleanseStorageOptions(options?.storageOptions), dataStorageVersion);
         return new table_1.LocalTable(innerTable);
     }
     async createEmptyTable(name, schema, options) {
@@ -87,9 +94,16 @@ class LocalConnection extends Connection {
             const registry = (0, registry_1.getRegistry)();
             metadata = registry.getTableMetadata([embeddingFunction]);
         }
+        let dataStorageVersion = "legacy";
+        if (options?.dataStorageVersion !== undefined) {
+            dataStorageVersion = options.dataStorageVersion;
+        }
+        else if (options?.useLegacyFormat !== undefined) {
+            dataStorageVersion = options.useLegacyFormat ? "legacy" : "stable";
+        }
         const table = (0, arrow_1.makeEmptyTable)(schema, metadata);
         const buf = await (0, arrow_1.fromTableToBuffer)(table);
-        const innerTable = await this.inner.createEmptyTable(name, buf, mode, cleanseStorageOptions(options?.storageOptions), options?.useLegacyFormat);
+        const innerTable = await this.inner.createEmptyTable(name, buf, mode, cleanseStorageOptions(options?.storageOptions), dataStorageVersion);
         return new table_1.LocalTable(innerTable);
     }
     async dropTable(name) {

package/dist/embedding/embedding_function.d.ts

@@ -44,6 +44,7 @@ export declare abstract class EmbeddingFunction<T = any, M extends FunctionOptio
      * ```
      */
     abstract toJSON(): Partial<M>;
+    init?(): Promise<void>;
     /**
      * sourceField is used in combination with `LanceSchema` to provide a declarative data model
      *
@@ -73,6 +74,16 @@ export declare abstract class EmbeddingFunction<T = any, M extends FunctionOptio
    */
     computeQueryEmbeddings(data: T): Promise<Awaited<IntoVector>>;
 }
+/**
+ * an abstract class for implementing embedding functions that take text as input
+ */
+export declare abstract class TextEmbeddingFunction<M extends FunctionOptions = FunctionOptions> extends EmbeddingFunction<string, M> {
+    abstract generateEmbeddings(texts: string[], ...args: any[]): Promise<number[][] | Float32Array[] | Float64Array[]>;
+    computeQueryEmbeddings(data: string): Promise<Awaited<IntoVector>>;
+    embeddingDataType(): Float;
+    sourceField(): [DataType, Map<string, EmbeddingFunction>];
+    computeSourceEmbeddings(data: string[]): Promise<number[][] | Float32Array[] | Float64Array[]>;
+}
 export interface FieldOptions<T extends DataType = DataType> {
     datatype: T;
     dims?: number;

package/dist/embedding/embedding_function.js

@@ -13,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.EmbeddingFunction = void 0;
+exports.TextEmbeddingFunction = exports.EmbeddingFunction = void 0;
 require("reflect-metadata");
 const arrow_1 = require("../arrow");
 const sanitize_1 = require("../sanitize");
@@ -35,9 +35,9 @@ class EmbeddingFunction {
      * @see {@link lancedb.LanceSchema}
      */
     sourceField(optionsOrDatatype) {
-        let datatype = (0, arrow_1.isDataType)(optionsOrDatatype)
-            ? optionsOrDatatype
-            : optionsOrDatatype?.datatype;
+        let datatype = "datatype" in optionsOrDatatype
+            ? optionsOrDatatype.datatype
+            : optionsOrDatatype;
         if (!datatype) {
             throw new Error("Datatype is required");
         }
@@ -58,8 +58,11 @@ class EmbeddingFunction {
         let vectorType;
         let dims = this.ndims();
         // `func.vectorField(new Float32())`
-        if ((0, arrow_1.isDataType)(optionsOrDatatype)) {
-            dtype = optionsOrDatatype;
+        if (optionsOrDatatype === undefined) {
+            dtype = new arrow_1.Float32();
+        }
+        else if (!("datatype" in optionsOrDatatype)) {
+            dtype = (0, sanitize_1.sanitizeType)(optionsOrDatatype);
         }
         else {
             // `func.vectorField({
@@ -67,7 +70,7 @@ class EmbeddingFunction {
             //  dims: 10
             // })`
             dims = dims ?? optionsOrDatatype?.dims;
-            dtype = optionsOrDatatype?.datatype;
+            dtype = (0, sanitize_1.sanitizeType)(optionsOrDatatype?.datatype);
         }
         if (dtype !== undefined) {
             // `func.vectorField(new FixedSizeList(dims, new Field("item", new Float32(), true)))`
@@ -110,3 +113,21 @@ class EmbeddingFunction {
     }
 }
 exports.EmbeddingFunction = EmbeddingFunction;
+/**
+ * an abstract class for implementing embedding functions that take text as input
+ */
+class TextEmbeddingFunction extends EmbeddingFunction {
+    async computeQueryEmbeddings(data) {
+        return this.generateEmbeddings([data]).then((data) => data[0]);
+    }
+    embeddingDataType() {
+        return new arrow_1.Float32();
+    }
+    sourceField() {
+        return super.sourceField(new arrow_1.Utf8());
+    }
+    computeSourceEmbeddings(data) {
+        return this.generateEmbeddings(data);
+    }
+}
+exports.TextEmbeddingFunction = TextEmbeddingFunction;

package/dist/embedding/index.d.ts

@@ -1,7 +1,8 @@
 import { Schema } from "../arrow";
 import { EmbeddingFunction } from "./embedding_function";
-export { EmbeddingFunction } from "./embedding_function";
+export { EmbeddingFunction, TextEmbeddingFunction } from "./embedding_function";
 export * from "./openai";
+export * from "./transformers";
 export * from "./registry";
 /**
  * Create a schema with embedding functions.

package/dist/embedding/index.js

@@ -27,15 +27,16 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.LanceSchema = exports.EmbeddingFunction = void 0;
+exports.LanceSchema = exports.TextEmbeddingFunction = exports.EmbeddingFunction = void 0;
 const arrow_1 = require("../arrow");
-const arrow_2 = require("../arrow");
 const sanitize_1 = require("../sanitize");
 const registry_1 = require("./registry");
 var embedding_function_1 = require("./embedding_function");
 Object.defineProperty(exports, "EmbeddingFunction", { enumerable: true, get: function () { return embedding_function_1.EmbeddingFunction; } });
+Object.defineProperty(exports, "TextEmbeddingFunction", { enumerable: true, get: function () { return embedding_function_1.TextEmbeddingFunction; } });
 // We need to explicitly export '*' so that the `register` decorator actually registers the class.
 __exportStar(require("./openai"), exports);
+__exportStar(require("./transformers"), exports);
 __exportStar(require("./registry"), exports);
 /**
  * Create a schema with embedding functions.
@@ -63,14 +64,14 @@ function LanceSchema(fields) {
     const arrowFields = [];
     const embeddingFunctions = new Map();
     Object.entries(fields).forEach(([key, value]) => {
-        if ((0, arrow_2.isDataType)(value)) {
-            arrowFields.push(new arrow_1.Field(key, (0, sanitize_1.sanitizeType)(value), true));
-        }
-        else {
+        if (Array.isArray(value)) {
             const [dtype, metadata] = value;
             arrowFields.push(new arrow_1.Field(key, (0, sanitize_1.sanitizeType)(dtype), true));
             parseEmbeddingFunctions(embeddingFunctions, key, metadata);
         }
+        else {
+            arrowFields.push(new arrow_1.Field(key, (0, sanitize_1.sanitizeType)(value), true));
+        }
     });
     const registry = (0, registry_1.getRegistry)();
     const metadata = registry.getTableMetadata(Array.from(embeddingFunctions.values()));

package/dist/embedding/openai.d.ts

@@ -1,4 +1,4 @@
-import { type EmbeddingCreateParams } from "openai/resources";
+import type { EmbeddingCreateParams } from "openai/resources/index";
 import { Float } from "../arrow";
 import { EmbeddingFunction } from "./embedding_function";
 export type OpenAIOptions = {

package/dist/embedding/registry.d.ts

@@ -1,8 +1,12 @@
 import { type EmbeddingFunction, type EmbeddingFunctionConstructor } from "./embedding_function";
 import "reflect-metadata";
 import { OpenAIEmbeddingFunction } from "./openai";
+import { TransformersEmbeddingFunction } from "./transformers";
+type CreateReturnType<T> = T extends {
+    init: () => Promise<void>;
+} ? Promise<T> : T;
 interface EmbeddingFunctionCreate<T extends EmbeddingFunction> {
-    create(options?: T["TOptions"]): T;
+    create(options?: T["TOptions"]): CreateReturnType<T>;
 }
 /**
  * This is a singleton class used to register embedding functions
@@ -19,11 +23,9 @@ export declare class EmbeddingFunctionRegistry {
      * @throws Error if the function is already registered
      */
     register<T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor>(this: EmbeddingFunctionRegistry, alias?: string): (ctor: T) => any;
-    /**
-     * Fetch an embedding function by name
-     * @param name The name of the function
-     */
-    get<T extends EmbeddingFunction<unknown>, Name extends string = "">(name: Name extends "openai" ? "openai" : string): Name extends "openai" ? EmbeddingFunctionCreate<OpenAIEmbeddingFunction> : EmbeddingFunctionCreate<T> | undefined;
+    get(name: "openai"): EmbeddingFunctionCreate<OpenAIEmbeddingFunction>;
+    get(name: "huggingface"): EmbeddingFunctionCreate<TransformersEmbeddingFunction>;
+    get<T extends EmbeddingFunction<unknown>>(name: string): EmbeddingFunctionCreate<T> | undefined;
     /**
      * reset the registry to the initial state
      */
@@ -31,7 +33,7 @@ export declare class EmbeddingFunctionRegistry {
     /**
      * @ignore
      */
-    parseFunctions(this: EmbeddingFunctionRegistry, metadata: Map<string, string>): Map<string, EmbeddingFunctionConfig>;
+    parseFunctions(this: EmbeddingFunctionRegistry, metadata: Map<string, string>): Promise<Map<string, EmbeddingFunctionConfig>>;
     functionToMetadata(conf: EmbeddingFunctionConfig): Record<string, any>;
     getTableMetadata(functions: EmbeddingFunctionConfig[]): Map<string, string>;
 }

package/dist/embedding/registry.js

@@ -50,12 +50,28 @@ class EmbeddingFunctionRegistry {
     get(name) {
         const factory = this.#functions.get(name);
         if (!factory) {
+            // biome-ignore lint/suspicious/noExplicitAny: <explanation>
             return undefined;
         }
+        // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+        let create;
+        if (factory.prototype.init) {
+            // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+            create = async function (options) {
+                const instance = new factory(options);
+                await instance.init();
+                return instance;
+            };
+        }
+        else {
+            // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+            create = function (options) {
+                const instance = new factory(options);
+                return instance;
+            };
+        }
         return {
-            create: function (options) {
-                return new factory(options);
-            },
+            create,
         };
     }
     /**
@@ -67,26 +83,28 @@ class EmbeddingFunctionRegistry {
     /**
      * @ignore
      */
-    parseFunctions(metadata) {
+    async parseFunctions(metadata) {
         if (!metadata.has("embedding_functions")) {
             return new Map();
         }
         else {
             const functions = (JSON.parse(metadata.get("embedding_functions")));
-            return new Map(functions.map((f) => {
+            const items = await Promise.all(functions.map(async (f) => {
                 const fn = this.get(f.name);
                 if (!fn) {
                     throw new Error(`Function "${f.name}" not found in registry`);
                 }
+                const func = await this.get(f.name).create(f.model);
                 return [
                     f.name,
                     {
                         sourceColumn: f.sourceColumn,
                         vectorColumn: f.vectorColumn,
-                        function: this.get(f.name).create(f.model),
+                        function: func,
                     },
                 ];
             }));
+            return new Map(items);
         }
     }
     // biome-ignore lint/suspicious/noExplicitAny: <explanation>

package/dist/embedding/transformers.d.ts

@@ -0,0 +1,37 @@
+import { Float } from "../arrow";
+import { EmbeddingFunction } from "./embedding_function";
+export type XenovaTransformerOptions = {
+    /** The wasm compatible model to use */
+    model: string;
+    /**
+     * The wasm compatible tokenizer to use
+     * If not provided, it will use the default tokenizer for the model
+     */
+    tokenizer?: string;
+    /**
+     * The number of dimensions of the embeddings
+     *
+     * We will attempt to infer this from the model config if not provided.
+     * Since there isn't a standard way to get this information from the model,
+     * you may need to manually specify this if using a model that doesn't have a 'hidden_size' in the config.
+     * */
+    ndims?: number;
+    /** Options for the tokenizer */
+    tokenizerOptions?: {
+        textPair?: string | string[];
+        padding?: boolean | "max_length";
+        addSpecialTokens?: boolean;
+        truncation?: boolean;
+        maxLength?: number;
+    };
+};
+export declare class TransformersEmbeddingFunction extends EmbeddingFunction<string, Partial<XenovaTransformerOptions>> {
+    #private;
+    constructor(options?: Partial<XenovaTransformerOptions>);
+    toJSON(): Record<string, any>;
+    init(): Promise<void>;
+    ndims(): number;
+    embeddingDataType(): Float;
+    computeSourceEmbeddings(data: string[]): Promise<number[][]>;
+    computeQueryEmbeddings(data: string): Promise<number[]>;
+}

package/dist/embedding/transformers.js

@@ -0,0 +1,147 @@
+"use strict";
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TransformersEmbeddingFunction = void 0;
+const arrow_1 = require("../arrow");
+const embedding_function_1 = require("./embedding_function");
+const registry_1 = require("./registry");
+let TransformersEmbeddingFunction = class TransformersEmbeddingFunction extends embedding_function_1.EmbeddingFunction {
+    #model;
+    #tokenizer;
+    #modelName;
+    #initialized = false;
+    #tokenizerOptions;
+    #ndims;
+    constructor(options = {
+        model: "Xenova/all-MiniLM-L6-v2",
+    }) {
+        super();
+        const modelName = options?.model ?? "Xenova/all-MiniLM-L6-v2";
+        this.#tokenizerOptions = {
+            padding: true,
+            ...options.tokenizerOptions,
+        };
+        this.#ndims = options.ndims;
+        this.#modelName = modelName;
+    }
+    toJSON() {
+        // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+        const obj = {
+            model: this.#modelName,
+        };
+        if (this.#ndims) {
+            obj["ndims"] = this.#ndims;
+        }
+        if (this.#tokenizerOptions) {
+            obj["tokenizerOptions"] = this.#tokenizerOptions;
+        }
+        if (this.#tokenizer) {
+            obj["tokenizer"] = this.#tokenizer.name;
+        }
+        return obj;
+    }
+    async init() {
+        let transformers;
+        try {
+            // SAFETY:
+            // since typescript transpiles `import` to `require`, we need to do this in an unsafe way
+            // We can't use `require` because `@xenova/transformers` is an ESM module
+            // and we can't use `import` directly because typescript will transpile it to `require`.
+            // and we want to remain compatible with both ESM and CJS modules
+            // so we use `eval` to bypass typescript for this specific import.
+            transformers = await eval('import("@xenova/transformers")');
+        }
+        catch (e) {
+            throw new Error(`error loading @xenova/transformers\nReason: ${e}`);
+        }
+        try {
+            this.#model = await transformers.AutoModel.from_pretrained(this.#modelName);
+        }
+        catch (e) {
+            throw new Error(`error loading model ${this.#modelName}. Make sure you are using a wasm compatible model.\nReason: ${e}`);
+        }
+        try {
+            this.#tokenizer = await transformers.AutoTokenizer.from_pretrained(this.#modelName);
+        }
+        catch (e) {
+            throw new Error(`error loading tokenizer for ${this.#modelName}. Make sure you are using a wasm compatible model:\nReason: ${e}`);
+        }
+        this.#initialized = true;
+    }
+    ndims() {
+        if (this.#ndims) {
+            return this.#ndims;
+        }
+        else {
+            const config = this.#model.config;
+            const ndims = config["hidden_size"];
+            if (!ndims) {
+                throw new Error("hidden_size not found in model config, you may need to manually specify the embedding dimensions. ");
+            }
+            return ndims;
+        }
+    }
+    embeddingDataType() {
+        return new arrow_1.Float32();
+    }
+    async computeSourceEmbeddings(data) {
+        // this should only happen if the user is trying to use the function directly.
+        // Anything going through the registry should already be initialized.
+        if (!this.#initialized) {
+            return Promise.reject(new Error("something went wrong: embedding function not initialized. Please call init()"));
+        }
+        const tokenizer = this.#tokenizer;
+        const model = this.#model;
+        const inputs = await tokenizer(data, this.#tokenizerOptions);
+        let tokens = await model.forward(inputs);
+        tokens = tokens[Object.keys(tokens)[0]];
+        const [nItems, nTokens] = tokens.dims;
+        tokens = tensorDiv(tokens.sum(1), nTokens);
+        // TODO: support other data types
+        const tokenData = tokens.data;
+        const stride = this.ndims();
+        const embeddings = [];
+        for (let i = 0; i < nItems; i++) {
+            const start = i * stride;
+            const end = start + stride;
+            const slice = tokenData.slice(start, end);
+            embeddings.push(Array.from(slice)); // TODO: Avoid copy here
+        }
+        return embeddings;
+    }
+    async computeQueryEmbeddings(data) {
+        return (await this.computeSourceEmbeddings([data]))[0];
+    }
+};
+exports.TransformersEmbeddingFunction = TransformersEmbeddingFunction;
+exports.TransformersEmbeddingFunction = TransformersEmbeddingFunction = __decorate([
+    (0, registry_1.register)("huggingface"),
+    __metadata("design:paramtypes", [Object])
+], TransformersEmbeddingFunction);
+const tensorDiv = (src, divBy) => {
+    for (let i = 0; i < src.data.length; ++i) {
+        src.data[i] /= divBy;
+    }
+    return src;
+};

package/dist/index.d.ts

@@ -6,7 +6,7 @@ export { makeArrowTable, MakeArrowTableOptions, Data, VectorColumnOptions, } fro
 export { Connection, CreateTableOptions, TableNamesOptions, } from "./connection";
 export { ExecutableQuery, Query, QueryBase, VectorQuery, RecordBatchIterator, } from "./query";
 export { Index, IndexOptions, IvfPqOptions } from "./indices";
-export { Table, AddDataOptions, UpdateOptions } from "./table";
+export { Table, AddDataOptions, UpdateOptions, OptimizeOptions } from "./table";
 export * as embedding from "./embedding";
 /**
  * Connect to a LanceDB instance at the given URI.

package/dist/query.js

@@ -129,22 +129,28 @@ class QueryBase {
      * object insertion order is easy to get wrong and `Map` is more foolproof.
      */
     select(columns) {
-        let columnTuples;
+        const selectColumns = (columnArray) => {
+            this.doCall((inner) => {
+                inner.selectColumns(columnArray);
+            });
+        };
+        const selectMapping = (columnTuples) => {
+            this.doCall((inner) => {
+                inner.select(columnTuples);
+            });
+        };
         if (typeof columns === "string") {
-            columns = [columns];
+            selectColumns([columns]);
         }
-        if (Array.isArray(columns)) {
-            columnTuples = columns.map((c) => [c, c]);
+        else if (Array.isArray(columns)) {
+            selectColumns(columns);
         }
         else if (columns instanceof Map) {
-            columnTuples = Array.from(columns.entries());
+            selectMapping(Array.from(columns.entries()));
         }
         else {
-            columnTuples = Object.entries(columns);
+            selectMapping(Object.entries(columns));
         }
-        this.doCall((inner) => {
-            inner.select(columnTuples);
-        });
         return this;
     }
     /**

package/dist/remote/client.d.ts

@@ -4,7 +4,7 @@ import { Table as ArrowTable } from "../arrow";
 import { VectorQuery } from "../query";
 export declare class RestfulLanceDBClient {
     #private;
-    constructor(dbName: string, apiKey: string, region: string, hostOverride?: string, connectionTimeout?: number, readTimeout?: number);
+    constructor(dbName: string, apiKey: string, region: string, hostOverride?: string, timeout?: number);
     get session(): import("axios").AxiosInstance;
     get url(): string;
     get headers(): {