@lancedb/lancedb 0.8.0 → 0.9.0

package/dist/arrow.d.ts

@@ -37,10 +37,7 @@ export type TableLike = ArrowTable | {
     batches: RecordBatchLike[];
 };
 export type IntoVector = Float32Array | Float64Array | number[] | Promise<Float32Array | Float64Array | number[]>;
-export type FloatLike = import("apache-arrow-13").Float | import("apache-arrow-14").Float | import("apache-arrow-15").Float | import("apache-arrow-16").Float | import("apache-arrow-17").Float;
-export type DataTypeLike = import("apache-arrow-13").DataType | import("apache-arrow-14").DataType | import("apache-arrow-15").DataType | import("apache-arrow-16").DataType | import("apache-arrow-17").DataType;
 export declare function isArrowTable(value: object): value is TableLike;
-export declare function isDataType(value: unknown): value is DataTypeLike;
 export declare function isNull(value: unknown): value is Null;
 export declare function isInt(value: unknown): value is Int;
 export declare function isFloat(value: unknown): value is Float;

package/dist/arrow.js

@@ -27,7 +27,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.createEmptyTable = exports.fromTableToStreamBuffer = exports.fromDataToBuffer = exports.fromTableToBuffer = exports.fromRecordsToStreamBuffer = exports.fromRecordsToBuffer = exports.newVectorType = exports.convertToTable = exports.makeEmptyTable = exports.makeArrowTable = exports.MakeArrowTableOptions = exports.VectorColumnOptions = exports.isFixedSizeList = exports.isFixedSizeBinary = exports.isUnion = exports.isStruct = exports.isList = exports.isDuration = exports.isInterval = exports.isTimestamp = exports.isTime = exports.isDate = exports.isDecimal = exports.isBool = exports.isLargeUtf8 = exports.isUtf8 = exports.isLargeBinary = exports.isBinary = exports.isFloat = exports.isInt = exports.isNull = exports.isDataType = exports.isArrowTable = void 0;
+exports.createEmptyTable = exports.fromTableToStreamBuffer = exports.fromDataToBuffer = exports.fromTableToBuffer = exports.fromRecordsToStreamBuffer = exports.fromRecordsToBuffer = exports.newVectorType = exports.convertToTable = exports.makeEmptyTable = exports.makeArrowTable = exports.MakeArrowTableOptions = exports.VectorColumnOptions = exports.isFixedSizeList = exports.isFixedSizeBinary = exports.isUnion = exports.isStruct = exports.isList = exports.isDuration = exports.isInterval = exports.isTimestamp = exports.isTime = exports.isDate = exports.isDecimal = exports.isBool = exports.isLargeUtf8 = exports.isUtf8 = exports.isLargeBinary = exports.isBinary = exports.isFloat = exports.isInt = exports.isNull = exports.isArrowTable = void 0;
 const apache_arrow_1 = require("apache-arrow");
 const registry_1 = require("./embedding/registry");
 const sanitize_1 = require("./sanitize");
@@ -38,31 +38,6 @@ function isArrowTable(value) {
     return "schema" in value && "batches" in value;
 }
 exports.isArrowTable = isArrowTable;
-function isDataType(value) {
-    return (value instanceof apache_arrow_1.DataType ||
-        apache_arrow_1.DataType.isNull(value) ||
-        apache_arrow_1.DataType.isInt(value) ||
-        apache_arrow_1.DataType.isFloat(value) ||
-        apache_arrow_1.DataType.isBinary(value) ||
-        apache_arrow_1.DataType.isLargeBinary(value) ||
-        apache_arrow_1.DataType.isUtf8(value) ||
-        apache_arrow_1.DataType.isLargeUtf8(value) ||
-        apache_arrow_1.DataType.isBool(value) ||
-        apache_arrow_1.DataType.isDecimal(value) ||
-        apache_arrow_1.DataType.isDate(value) ||
-        apache_arrow_1.DataType.isTime(value) ||
-        apache_arrow_1.DataType.isTimestamp(value) ||
-        apache_arrow_1.DataType.isInterval(value) ||
-        apache_arrow_1.DataType.isDuration(value) ||
-        apache_arrow_1.DataType.isList(value) ||
-        apache_arrow_1.DataType.isStruct(value) ||
-        apache_arrow_1.DataType.isUnion(value) ||
-        apache_arrow_1.DataType.isFixedSizeBinary(value) ||
-        apache_arrow_1.DataType.isFixedSizeList(value) ||
-        apache_arrow_1.DataType.isMap(value) ||
-        apache_arrow_1.DataType.isDictionary(value));
-}
-exports.isDataType = isDataType;
 function isNull(value) {
     return value instanceof apache_arrow_1.Null || apache_arrow_1.DataType.isNull(value);
 }

package/dist/connection.d.ts

@@ -27,10 +27,19 @@ export interface CreateTableOptions {
      * The available options are described at https://lancedb.github.io/lancedb/guides/storage/
      */
     storageOptions?: Record<string, string>;
+    /**
+     * The version of the data storage format to use.
+     *
+     * The default is `legacy`, which is Lance format v1.
+     * `stable` is the new format, which is Lance format v2.
+     */
+    dataStorageVersion?: string;
     /**
      * If true then data files will be written with the legacy format
      *
      * The default is true while the new format is in beta
+     *
+     * Deprecated.
      */
     useLegacyFormat?: boolean;
     schema?: SchemaLike;

package/dist/connection.js

@@ -72,7 +72,14 @@ class LocalConnection extends Connection {
             throw new Error("data is required");
         }
         const { buf, mode } = await table_1.Table.parseTableData(data, options);
-        const innerTable = await this.inner.createTable(nameOrOptions, buf, mode, cleanseStorageOptions(options?.storageOptions), options?.useLegacyFormat);
+        let dataStorageVersion = "legacy";
+        if (options?.dataStorageVersion !== undefined) {
+            dataStorageVersion = options.dataStorageVersion;
+        }
+        else if (options?.useLegacyFormat !== undefined) {
+            dataStorageVersion = options.useLegacyFormat ? "legacy" : "stable";
+        }
+        const innerTable = await this.inner.createTable(nameOrOptions, buf, mode, cleanseStorageOptions(options?.storageOptions), dataStorageVersion);
         return new table_1.LocalTable(innerTable);
     }
     async createEmptyTable(name, schema, options) {
@@ -87,9 +94,16 @@ class LocalConnection extends Connection {
             const registry = (0, registry_1.getRegistry)();
             metadata = registry.getTableMetadata([embeddingFunction]);
         }
+        let dataStorageVersion = "legacy";
+        if (options?.dataStorageVersion !== undefined) {
+            dataStorageVersion = options.dataStorageVersion;
+        }
+        else if (options?.useLegacyFormat !== undefined) {
+            dataStorageVersion = options.useLegacyFormat ? "legacy" : "stable";
+        }
         const table = (0, arrow_1.makeEmptyTable)(schema, metadata);
         const buf = await (0, arrow_1.fromTableToBuffer)(table);
-        const innerTable = await this.inner.createEmptyTable(name, buf, mode, cleanseStorageOptions(options?.storageOptions), options?.useLegacyFormat);
+        const innerTable = await this.inner.createEmptyTable(name, buf, mode, cleanseStorageOptions(options?.storageOptions), dataStorageVersion);
         return new table_1.LocalTable(innerTable);
     }
     async dropTable(name) {

package/dist/embedding/embedding_function.d.ts

@@ -1,5 +1,5 @@
 import "reflect-metadata";
-import { DataType, DataTypeLike, FloatLike, type IntoVector } from "../arrow";
+import { DataType, Float, type IntoVector } from "../arrow";
 /**
  * Options for a given embedding function
  */
@@ -52,7 +52,7 @@ export declare abstract class EmbeddingFunction<T = any, M extends FunctionOptio
      *
      * @see {@link lancedb.LanceSchema}
      */
-    sourceField(optionsOrDatatype: Partial<FieldOptions> | DataTypeLike): [DataTypeLike, Map<string, EmbeddingFunction>];
+    sourceField(optionsOrDatatype: Partial<FieldOptions> | DataType): [DataType, Map<string, EmbeddingFunction>];
     /**
      * vectorField is used in combination with `LanceSchema` to provide a declarative data model
      *
@@ -64,7 +64,7 @@ export declare abstract class EmbeddingFunction<T = any, M extends FunctionOptio
     /** The number of dimensions of the embeddings */
     ndims(): number | undefined;
     /** The datatype of the embeddings */
-    abstract embeddingDataType(): FloatLike;
+    abstract embeddingDataType(): Float;
     /**
      * Creates a vector representation for the given values.
      */
@@ -74,6 +74,16 @@ export declare abstract class EmbeddingFunction<T = any, M extends FunctionOptio
    */
     computeQueryEmbeddings(data: T): Promise<Awaited<IntoVector>>;
 }
+/**
+ * an abstract class for implementing embedding functions that take text as input
+ */
+export declare abstract class TextEmbeddingFunction<M extends FunctionOptions = FunctionOptions> extends EmbeddingFunction<string, M> {
+    abstract generateEmbeddings(texts: string[], ...args: any[]): Promise<number[][] | Float32Array[] | Float64Array[]>;
+    computeQueryEmbeddings(data: string): Promise<Awaited<IntoVector>>;
+    embeddingDataType(): Float;
+    sourceField(): [DataType, Map<string, EmbeddingFunction>];
+    computeSourceEmbeddings(data: string[]): Promise<number[][] | Float32Array[] | Float64Array[]>;
+}
 export interface FieldOptions<T extends DataType = DataType> {
     datatype: T;
     dims?: number;

package/dist/embedding/embedding_function.js

@@ -13,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.EmbeddingFunction = void 0;
+exports.TextEmbeddingFunction = exports.EmbeddingFunction = void 0;
 require("reflect-metadata");
 const arrow_1 = require("../arrow");
 const sanitize_1 = require("../sanitize");
@@ -35,9 +35,9 @@ class EmbeddingFunction {
      * @see {@link lancedb.LanceSchema}
      */
     sourceField(optionsOrDatatype) {
-        let datatype = (0, arrow_1.isDataType)(optionsOrDatatype)
-            ? optionsOrDatatype
-            : optionsOrDatatype?.datatype;
+        let datatype = "datatype" in optionsOrDatatype
+            ? optionsOrDatatype.datatype
+            : optionsOrDatatype;
         if (!datatype) {
             throw new Error("Datatype is required");
         }
@@ -58,8 +58,11 @@ class EmbeddingFunction {
         let vectorType;
         let dims = this.ndims();
         // `func.vectorField(new Float32())`
-        if ((0, arrow_1.isDataType)(optionsOrDatatype)) {
-            dtype = optionsOrDatatype;
+        if (optionsOrDatatype === undefined) {
+            dtype = new arrow_1.Float32();
+        }
+        else if (!("datatype" in optionsOrDatatype)) {
+            dtype = (0, sanitize_1.sanitizeType)(optionsOrDatatype);
         }
         else {
             // `func.vectorField({
@@ -67,7 +70,7 @@ class EmbeddingFunction {
             //  dims: 10
             // })`
             dims = dims ?? optionsOrDatatype?.dims;
-            dtype = optionsOrDatatype?.datatype;
+            dtype = (0, sanitize_1.sanitizeType)(optionsOrDatatype?.datatype);
         }
         if (dtype !== undefined) {
             // `func.vectorField(new FixedSizeList(dims, new Field("item", new Float32(), true)))`
@@ -110,3 +113,21 @@ class EmbeddingFunction {
     }
 }
 exports.EmbeddingFunction = EmbeddingFunction;
+/**
+ * an abstract class for implementing embedding functions that take text as input
+ */
+class TextEmbeddingFunction extends EmbeddingFunction {
+    async computeQueryEmbeddings(data) {
+        return this.generateEmbeddings([data]).then((data) => data[0]);
+    }
+    embeddingDataType() {
+        return new arrow_1.Float32();
+    }
+    sourceField() {
+        return super.sourceField(new arrow_1.Utf8());
+    }
+    computeSourceEmbeddings(data) {
+        return this.generateEmbeddings(data);
+    }
+}
+exports.TextEmbeddingFunction = TextEmbeddingFunction;

package/dist/embedding/index.d.ts

@@ -1,6 +1,6 @@
 import { Schema } from "../arrow";
 import { EmbeddingFunction } from "./embedding_function";
-export { EmbeddingFunction } from "./embedding_function";
+export { EmbeddingFunction, TextEmbeddingFunction } from "./embedding_function";
 export * from "./openai";
 export * from "./transformers";
 export * from "./registry";

package/dist/embedding/index.js

@@ -27,13 +27,13 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.LanceSchema = exports.EmbeddingFunction = void 0;
+exports.LanceSchema = exports.TextEmbeddingFunction = exports.EmbeddingFunction = void 0;
 const arrow_1 = require("../arrow");
-const arrow_2 = require("../arrow");
 const sanitize_1 = require("../sanitize");
 const registry_1 = require("./registry");
 var embedding_function_1 = require("./embedding_function");
 Object.defineProperty(exports, "EmbeddingFunction", { enumerable: true, get: function () { return embedding_function_1.EmbeddingFunction; } });
+Object.defineProperty(exports, "TextEmbeddingFunction", { enumerable: true, get: function () { return embedding_function_1.TextEmbeddingFunction; } });
 // We need to explicitly export '*' so that the `register` decorator actually registers the class.
 __exportStar(require("./openai"), exports);
 __exportStar(require("./transformers"), exports);
@@ -64,14 +64,14 @@ function LanceSchema(fields) {
     const arrowFields = [];
     const embeddingFunctions = new Map();
     Object.entries(fields).forEach(([key, value]) => {
-        if ((0, arrow_2.isDataType)(value)) {
-            arrowFields.push(new arrow_1.Field(key, (0, sanitize_1.sanitizeType)(value), true));
-        }
-        else {
+        if (Array.isArray(value)) {
             const [dtype, metadata] = value;
             arrowFields.push(new arrow_1.Field(key, (0, sanitize_1.sanitizeType)(dtype), true));
             parseEmbeddingFunctions(embeddingFunctions, key, metadata);
         }
+        else {
+            arrowFields.push(new arrow_1.Field(key, (0, sanitize_1.sanitizeType)(value), true));
+        }
     });
     const registry = (0, registry_1.getRegistry)();
     const metadata = registry.getTableMetadata(Array.from(embeddingFunctions.values()));

package/dist/embedding/openai.d.ts

@@ -1,4 +1,4 @@
-import { type EmbeddingCreateParams } from "openai/resources";
+import type { EmbeddingCreateParams } from "openai/resources/index";
 import { Float } from "../arrow";
 import { EmbeddingFunction } from "./embedding_function";
 export type OpenAIOptions = {

package/dist/index.d.ts

@@ -6,7 +6,7 @@ export { makeArrowTable, MakeArrowTableOptions, Data, VectorColumnOptions, } fro
 export { Connection, CreateTableOptions, TableNamesOptions, } from "./connection";
 export { ExecutableQuery, Query, QueryBase, VectorQuery, RecordBatchIterator, } from "./query";
 export { Index, IndexOptions, IvfPqOptions } from "./indices";
-export { Table, AddDataOptions, UpdateOptions } from "./table";
+export { Table, AddDataOptions, UpdateOptions, OptimizeOptions } from "./table";
 export * as embedding from "./embedding";
 /**
  * Connect to a LanceDB instance at the given URI.

package/dist/sanitize.js

@@ -215,8 +215,10 @@ function sanitizeType(typeLike) {
     if (typeof typeLike !== "object" || typeLike === null) {
         throw Error("Expected a Type but object was null/undefined");
     }
-    if (!("typeId" in typeLike) || !(typeof typeLike.typeId !== "function")) {
-        throw Error("Expected a Type to have a typeId function");
+    if (!("typeId" in typeLike) ||
+        !(typeof typeLike.typeId !== "function" ||
+            typeof typeLike.typeId !== "number")) {
+        throw Error("Expected a Type to have a typeId property");
     }
     let typeId;
     if (typeof typeLike.typeId === "function") {

package/package.json

@@ -10,7 +10,7 @@
     "vector database",
     "ann"
   ],
-  "version": "0.8.0",
+  "version": "0.9.0",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -81,6 +81,7 @@
     "docs": "typedoc --plugin typedoc-plugin-markdown --out ../docs/src/js lancedb/index.ts",
     "lint": "biome check . && biome format .",
     "lint-fix": "biome check --write . && biome format --write .",
+    "prepublishOnly": "napi prepublish -t npm",
     "test": "jest --verbose",
     "integration": "S3_TEST=1 npm run test",
     "universal": "napi universal",
@@ -91,11 +92,11 @@
     "reflect-metadata": "^0.2.2"
   },
   "optionalDependencies": {
-    "@lancedb/lancedb-darwin-arm64": "0.8.0",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.8.0",
-    "@lancedb/lancedb-darwin-x64": "0.8.0",
-    "@lancedb/lancedb-linux-x64-gnu": "0.8.0",
-    "@lancedb/lancedb-win32-x64-msvc": "0.8.0"
+    "@lancedb/lancedb-darwin-arm64": "0.9.0",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.9.0",
+    "@lancedb/lancedb-darwin-x64": "0.9.0",
+    "@lancedb/lancedb-linux-x64-gnu": "0.9.0",
+    "@lancedb/lancedb-win32-x64-msvc": "0.9.0"
   },
   "peerDependencies": {
     "apache-arrow": ">=13.0.0 <=17.0.0"