@lancedb/lancedb 0.5.2 → 0.7.0

package/nodejs-artifacts/embedding/openai.js

@@ -29,7 +29,9 @@ const registry_1 = require("./registry");
 let OpenAIEmbeddingFunction = class OpenAIEmbeddingFunction extends embedding_function_1.EmbeddingFunction {
     #openai;
     #modelName;
-    constructor(options = { model: "text-embedding-ada-002" }) {
+    constructor(options = {
+        model: "text-embedding-ada-002",
+    }) {
         super();
         const openAIKey = options?.apiKey ?? process.env.OPENAI_API_KEY;
         if (!openAIKey) {
@@ -68,7 +70,7 @@ let OpenAIEmbeddingFunction = class OpenAIEmbeddingFunction extends embedding_fu
             case "text-embedding-3-small":
                 return 1536;
             default:
-                return null;
+                throw new Error(`Unknown model: ${this.#modelName}`);
         }
     }
     embeddingDataType() {

package/nodejs-artifacts/embedding/registry.d.ts

@@ -1,13 +1,8 @@
-import type { EmbeddingFunction } from "./embedding_function";
+import { type EmbeddingFunction, type EmbeddingFunctionConstructor } from "./embedding_function";
 import "reflect-metadata";
-export interface EmbeddingFunctionOptions {
-    [key: string]: unknown;
-}
-export interface EmbeddingFunctionFactory<T extends EmbeddingFunction = EmbeddingFunction> {
-    new (modelOptions?: EmbeddingFunctionOptions): T;
-}
+import { OpenAIEmbeddingFunction } from "./openai";
 interface EmbeddingFunctionCreate<T extends EmbeddingFunction> {
-    create(options?: EmbeddingFunctionOptions): T;
+    create(options?: T["TOptions"]): T;
 }
 /**
  * This is a singleton class used to register embedding functions
@@ -23,12 +18,12 @@ export declare class EmbeddingFunctionRegistry {
      * @param func The function to register
      * @throws Error if the function is already registered
      */
-    register<T extends EmbeddingFunctionFactory = EmbeddingFunctionFactory>(this: EmbeddingFunctionRegistry, alias?: string): (ctor: T) => any;
+    register<T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor>(this: EmbeddingFunctionRegistry, alias?: string): (ctor: T) => any;
     /**
      * Fetch an embedding function by name
      * @param name The name of the function
      */
-    get<T extends EmbeddingFunction<unknown> = EmbeddingFunction>(name: string): EmbeddingFunctionCreate<T> | undefined;
+    get<T extends EmbeddingFunction<unknown>, Name extends string = "">(name: Name extends "openai" ? "openai" : string): Name extends "openai" ? EmbeddingFunctionCreate<OpenAIEmbeddingFunction> : EmbeddingFunctionCreate<T> | undefined;
     /**
      * reset the registry to the initial state
      */
@@ -40,7 +35,7 @@ export declare class EmbeddingFunctionRegistry {
     functionToMetadata(conf: EmbeddingFunctionConfig): Record<string, any>;
     getTableMetadata(functions: EmbeddingFunctionConfig[]): Map<string, string>;
 }
-export declare function register(name?: string): (ctor: EmbeddingFunctionFactory<EmbeddingFunction<any, import("./embedding_function").FunctionOptions>>) => any;
+export declare function register(name?: string): (ctor: EmbeddingFunctionConstructor<EmbeddingFunction<any, import("./embedding_function").FunctionOptions>>) => any;
 /**
  * Utility function to get the global instance of the registry
  * @returns `EmbeddingFunctionRegistry` The global instance of the registry

package/nodejs-artifacts/index.d.ts

@@ -1,7 +1,55 @@
-export { WriteOptions, WriteMode, AddColumnsSql, ColumnAlteration, ConnectionOptions, } from "./native.js";
+import { Connection } from "./connection";
+import { ConnectionOptions } from "./native.js";
+import { RemoteConnectionOptions } from "./remote";
+export { WriteOptions, WriteMode, AddColumnsSql, ColumnAlteration, ConnectionOptions, IndexStatistics, IndexMetadata, IndexConfig, } from "./native.js";
 export { makeArrowTable, MakeArrowTableOptions, Data, VectorColumnOptions, } from "./arrow";
-export { connect, Connection, CreateTableOptions, TableNamesOptions, } from "./connection";
+export { Connection, CreateTableOptions, TableNamesOptions, } from "./connection";
 export { ExecutableQuery, Query, QueryBase, VectorQuery, RecordBatchIterator, } from "./query";
 export { Index, IndexOptions, IvfPqOptions } from "./indices";
-export { Table, AddDataOptions, IndexConfig, UpdateOptions } from "./table";
+export { Table, AddDataOptions, UpdateOptions } from "./table";
 export * as embedding from "./embedding";
+/**
+ * Connect to a LanceDB instance at the given URI.
+ *
+ * Accepted formats:
+ *
+ * - `/path/to/database` - local database
+ * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
+ * - `db://host:port` - remote database (LanceDB cloud)
+ * @param {string} uri - The uri of the database. If the database uri starts
+ * with `db://` then it connects to a remote database.
+ * @see {@link ConnectionOptions} for more details on the URI format.
+ * @example
+ * ```ts
+ * const conn = await connect("/path/to/database");
+ * ```
+ * @example
+ * ```ts
+ * const conn = await connect(
+ *   "s3://bucket/path/to/database",
+ *   {storageOptions: {timeout: "60s"}
+ * });
+ * ```
+ */
+export declare function connect(uri: string, opts?: Partial<ConnectionOptions | RemoteConnectionOptions>): Promise<Connection>;
+/**
+ * Connect to a LanceDB instance at the given URI.
+ *
+ * Accepted formats:
+ *
+ * - `/path/to/database` - local database
+ * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
+ * - `db://host:port` - remote database (LanceDB cloud)
+ * @param  options - The options to use when connecting to the database
+ * @see {@link ConnectionOptions} for more details on the URI format.
+ * @example
+ * ```ts
+ * const conn = await connect({
+ *   uri: "/path/to/database",
+ *   storageOptions: {timeout: "60s"}
+ * });
+ * ```
+ */
+export declare function connect(opts: Partial<RemoteConnectionOptions | ConnectionOptions> & {
+    uri: string;
+}): Promise<Connection>;

package/nodejs-artifacts/index.js

@@ -13,14 +13,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.embedding = exports.Table = exports.Index = exports.RecordBatchIterator = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Connection = exports.connect = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = void 0;
+exports.connect = exports.embedding = exports.Table = exports.Index = exports.RecordBatchIterator = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Connection = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = void 0;
+const connection_1 = require("./connection");
+const native_js_1 = require("./native.js");
+const remote_1 = require("./remote");
 var arrow_1 = require("./arrow");
 Object.defineProperty(exports, "makeArrowTable", { enumerable: true, get: function () { return arrow_1.makeArrowTable; } });
 Object.defineProperty(exports, "MakeArrowTableOptions", { enumerable: true, get: function () { return arrow_1.MakeArrowTableOptions; } });
 Object.defineProperty(exports, "VectorColumnOptions", { enumerable: true, get: function () { return arrow_1.VectorColumnOptions; } });
-var connection_1 = require("./connection");
-Object.defineProperty(exports, "connect", { enumerable: true, get: function () { return connection_1.connect; } });
-Object.defineProperty(exports, "Connection", { enumerable: true, get: function () { return connection_1.Connection; } });
+var connection_2 = require("./connection");
+Object.defineProperty(exports, "Connection", { enumerable: true, get: function () { return connection_2.Connection; } });
 var query_1 = require("./query");
 Object.defineProperty(exports, "Query", { enumerable: true, get: function () { return query_1.Query; } });
 Object.defineProperty(exports, "QueryBase", { enumerable: true, get: function () { return query_1.QueryBase; } });
@@ -31,3 +33,25 @@ Object.defineProperty(exports, "Index", { enumerable: true, get: function () { r
 var table_1 = require("./table");
 Object.defineProperty(exports, "Table", { enumerable: true, get: function () { return table_1.Table; } });
 exports.embedding = require("./embedding");
+async function connect(uriOrOptions, opts = {}) {
+    let uri;
+    if (typeof uriOrOptions !== "string") {
+        const { uri: uri_, ...options } = uriOrOptions;
+        uri = uri_;
+        opts = options;
+    }
+    else {
+        uri = uriOrOptions;
+    }
+    if (!uri) {
+        throw new Error("uri is required");
+    }
+    if (uri?.startsWith("db://")) {
+        return new remote_1.RemoteConnection(uri, opts);
+    }
+    opts = opts ?? {};
+    opts.storageOptions = (0, connection_1.cleanseStorageOptions)(opts.storageOptions);
+    const nativeConn = await native_js_1.Connection.new(uri, opts);
+    return new connection_1.LocalConnection(nativeConn);
+}
+exports.connect = connect;

package/nodejs-artifacts/merge.d.ts

@@ -0,0 +1,54 @@
+import { Data } from "./arrow";
+import { NativeMergeInsertBuilder } from "./native";
+/** A builder used to create and run a merge insert operation */
+export declare class MergeInsertBuilder {
+    #private;
+    /** Construct a MergeInsertBuilder. __Internal use only.__ */
+    constructor(native: NativeMergeInsertBuilder);
+    /**
+     * Rows that exist in both the source table (new data) and
+     * the target table (old data) will be updated, replacing
+     * the old row with the corresponding matching row.
+     *
+     * If there are multiple matches then the behavior is undefined.
+     * Currently this causes multiple copies of the row to be created
+     * but that behavior is subject to change.
+     *
+     * An optional condition may be specified.  If it is, then only
+     * matched rows that satisfy the condtion will be updated.  Any
+     * rows that do not satisfy the condition will be left as they
+     * are.  Failing to satisfy the condition does not cause a
+     * "matched row" to become a "not matched" row.
+     *
+     * The condition should be an SQL string.  Use the prefix
+     * target. to refer to rows in the target table (old data)
+     * and the prefix source. to refer to rows in the source
+     * table (new data).
+     *
+     * For example, "target.last_update < source.last_update"
+     */
+    whenMatchedUpdateAll(options?: {
+        where: string;
+    }): MergeInsertBuilder;
+    /**
+     * Rows that exist only in the source table (new data) should
+     * be inserted into the target table.
+     */
+    whenNotMatchedInsertAll(): MergeInsertBuilder;
+    /**
+     * Rows that exist only in the target table (old data) will be
+     * deleted.  An optional condition can be provided to limit what
+     * data is deleted.
+     *
+     * @param options.where - An optional condition to limit what data is deleted
+     */
+    whenNotMatchedBySourceDelete(options?: {
+        where: string;
+    }): MergeInsertBuilder;
+    /**
+     * Executes the merge insert operation
+     *
+     * Nothing is returned but the `Table` is updated
+     */
+    execute(data: Data): Promise<void>;
+}

package/nodejs-artifacts/merge.js

@@ -0,0 +1,64 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MergeInsertBuilder = void 0;
+const arrow_1 = require("./arrow");
+/** A builder used to create and run a merge insert operation */
+class MergeInsertBuilder {
+    #native;
+    /** Construct a MergeInsertBuilder. __Internal use only.__ */
+    constructor(native) {
+        this.#native = native;
+    }
+    /**
+     * Rows that exist in both the source table (new data) and
+     * the target table (old data) will be updated, replacing
+     * the old row with the corresponding matching row.
+     *
+     * If there are multiple matches then the behavior is undefined.
+     * Currently this causes multiple copies of the row to be created
+     * but that behavior is subject to change.
+     *
+     * An optional condition may be specified.  If it is, then only
+     * matched rows that satisfy the condtion will be updated.  Any
+     * rows that do not satisfy the condition will be left as they
+     * are.  Failing to satisfy the condition does not cause a
+     * "matched row" to become a "not matched" row.
+     *
+     * The condition should be an SQL string.  Use the prefix
+     * target. to refer to rows in the target table (old data)
+     * and the prefix source. to refer to rows in the source
+     * table (new data).
+     *
+     * For example, "target.last_update < source.last_update"
+     */
+    whenMatchedUpdateAll(options) {
+        return new MergeInsertBuilder(this.#native.whenMatchedUpdateAll(options?.where));
+    }
+    /**
+     * Rows that exist only in the source table (new data) should
+     * be inserted into the target table.
+     */
+    whenNotMatchedInsertAll() {
+        return new MergeInsertBuilder(this.#native.whenNotMatchedInsertAll());
+    }
+    /**
+     * Rows that exist only in the target table (old data) will be
+     * deleted.  An optional condition can be provided to limit what
+     * data is deleted.
+     *
+     * @param options.where - An optional condition to limit what data is deleted
+     */
+    whenNotMatchedBySourceDelete(options) {
+        return new MergeInsertBuilder(this.#native.whenNotMatchedBySourceDelete(options?.where));
+    }
+    /**
+     * Executes the merge insert operation
+     *
+     * Nothing is returned but the `Table` is updated
+     */
+    async execute(data) {
+        const buffer = await (0, arrow_1.fromDataToBuffer)(data);
+        await this.#native.execute(buffer);
+    }
+}
+exports.MergeInsertBuilder = MergeInsertBuilder;

package/nodejs-artifacts/native.d.ts

@@ -5,12 +5,14 @@
 
 /** A description of an index currently configured on a column */
 export interface IndexConfig {
+  /** The name of the index */
+  name: string
   /** The type of the index */
   indexType: string
   /**
    * The columns in the index
    *
-   * Currently this is always an array of size 1.  In the future there may
+   * Currently this is always an array of size 1. In the future there may
    * be more columns to represent composite indices.
    */
   columns: Array<string>
@@ -72,9 +74,21 @@ export interface AddColumnsSql {
    */
   valueSql: string
 }
+export interface IndexStatistics {
+  /** The number of rows indexed by the index */
+  numIndexedRows: number
+  /** The number of rows not indexed */
+  numUnindexedRows: number
+  /** The type of the index */
+  indexType?: string
+  /** The metadata for each index */
+  indices: Array<IndexMetadata>
+}
+export interface IndexMetadata {
+  metricType?: string
+  indexType?: string
+}
 export interface ConnectionOptions {
-  apiKey?: string
-  hostOverride?: string
   /**
    * (For LanceDB OSS only): The interval, in seconds, at which to check for
    * updates to the table from other processes. If None, then consistency is not
@@ -138,12 +152,20 @@ export class Index {
 export class RecordBatchIterator {
   next(): Promise<Buffer | null>
 }
+/** A builder used to create and run a merge insert operation */
+export class NativeMergeInsertBuilder {
+  whenMatchedUpdateAll(condition?: string | undefined | null): NativeMergeInsertBuilder
+  whenNotMatchedInsertAll(): NativeMergeInsertBuilder
+  whenNotMatchedBySourceDelete(filter?: string | undefined | null): NativeMergeInsertBuilder
+  execute(buf: Buffer): Promise<void>
+}
 export class Query {
   onlyIf(predicate: string): void
   select(columns: Array<[string, string]>): void
   limit(limit: number): void
   nearestTo(vector: Float32Array): VectorQuery
   execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
+  explainPlan(verbose: boolean): Promise<string>
 }
 export class VectorQuery {
   column(column: string): void
@@ -156,8 +178,10 @@ export class VectorQuery {
   select(columns: Array<[string, string]>): void
   limit(limit: number): void
   execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
+  explainPlan(verbose: boolean): Promise<string>
 }
 export class Table {
+  name: string
   display(): string
   isOpen(): boolean
   close(): void
@@ -179,4 +203,6 @@ export class Table {
   restore(): Promise<void>
   optimize(olderThanMs?: number | undefined | null): Promise<OptimizeStats>
   listIndices(): Promise<Array<IndexConfig>>
+  indexStats(indexName: string): Promise<IndexStatistics | null>
+  mergeInsert(on: Array<string>): NativeMergeInsertBuilder
 }

package/nodejs-artifacts/native.js

@@ -233,17 +233,33 @@ switch (platform) {
                 }
                 break;
             case 'arm':
-                localFileExisted = existsSync(join(__dirname, 'lancedb.linux-arm-gnueabihf.node'));
-                try {
-                    if (localFileExisted) {
-                        nativeBinding = require('./lancedb.linux-arm-gnueabihf.node');
+                if (isMusl()) {
+                    localFileExisted = existsSync(join(__dirname, 'lancedb.linux-arm-musleabihf.node'));
+                    try {
+                        if (localFileExisted) {
+                            nativeBinding = require('./lancedb.linux-arm-musleabihf.node');
+                        }
+                        else {
+                            nativeBinding = require('@lancedb/lancedb-linux-arm-musleabihf');
+                        }
                     }
-                    else {
-                        nativeBinding = require('@lancedb/lancedb-linux-arm-gnueabihf');
+                    catch (e) {
+                        loadError = e;
                     }
                 }
-                catch (e) {
-                    loadError = e;
+                else {
+                    localFileExisted = existsSync(join(__dirname, 'lancedb.linux-arm-gnueabihf.node'));
+                    try {
+                        if (localFileExisted) {
+                            nativeBinding = require('./lancedb.linux-arm-gnueabihf.node');
+                        }
+                        else {
+                            nativeBinding = require('@lancedb/lancedb-linux-arm-gnueabihf');
+                        }
+                    }
+                    catch (e) {
+                        loadError = e;
+                    }
                 }
                 break;
             case 'riscv64':
@@ -303,10 +319,11 @@ if (!nativeBinding) {
     }
     throw new Error(`Failed to load native binding`);
 }
-const { Connection, Index, RecordBatchIterator, Query, VectorQuery, Table, WriteMode } = nativeBinding;
+const { Connection, Index, RecordBatchIterator, NativeMergeInsertBuilder, Query, VectorQuery, Table, WriteMode } = nativeBinding;
 module.exports.Connection = Connection;
 module.exports.Index = Index;
 module.exports.RecordBatchIterator = RecordBatchIterator;
+module.exports.NativeMergeInsertBuilder = NativeMergeInsertBuilder;
 module.exports.Query = Query;
 module.exports.VectorQuery = VectorQuery;
 module.exports.Table = Table;

package/nodejs-artifacts/query.d.ts

@@ -1,4 +1,5 @@
 import { Table as ArrowTable, type IntoVector, RecordBatch } from "./arrow";
+import { type IvfPqOptions } from "./indices";
 import { RecordBatchIterator as NativeBatchIterator, Query as NativeQuery, Table as NativeTable, VectorQuery as NativeVectorQuery } from "./native";
 export declare class RecordBatchIterator implements AsyncIterator<RecordBatch> {
     private promisedInner?;
@@ -19,9 +20,10 @@ export interface QueryExecutionOptions {
     maxBatchLength?: number;
 }
 /** Common methods supported by all query types */
-export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery, QueryType> implements AsyncIterable<RecordBatch> {
-    protected inner: NativeQueryType;
-    protected constructor(inner: NativeQueryType);
+export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery> implements AsyncIterable<RecordBatch> {
+    protected inner: NativeQueryType | Promise<NativeQueryType>;
+    protected constructor(inner: NativeQueryType | Promise<NativeQueryType>);
+    protected doCall(fn: (inner: NativeQueryType) => void): void;
     /**
      * A filter statement to be applied to this query.
      *
@@ -34,7 +36,13 @@ export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVecto
      * Filtering performance can often be improved by creating a scalar index
      * on the filter column(s).
      */
-    where(predicate: string): QueryType;
+    where(predicate: string): this;
+    /**
+     * A filter statement to be applied to this query.
+     * @alias where
+     * @deprecated Use `where` instead
+     */
+    filter(predicate: string): this;
     /**
      * Return only the specified columns.
      *
@@ -65,14 +73,14 @@ export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVecto
      * uses `Object.entries` which should preserve the insertion order of the object.  However,
      * object insertion order is easy to get wrong and `Map` is more foolproof.
      */
-    select(columns: string[] | Map<string, string> | Record<string, string> | string): QueryType;
+    select(columns: string[] | Map<string, string> | Record<string, string> | string): this;
     /**
      * Set the maximum number of results to return.
      *
      * By default, a plain search has no limit.  If this method is not
      * called then every valid row from the table will be returned.
      */
-    limit(limit: number): QueryType;
+    limit(limit: number): this;
     protected nativeExecute(options?: Partial<QueryExecutionOptions>): Promise<NativeBatchIterator>;
     /**
      * Execute the query and return the results as an @see {@link AsyncIterator}
@@ -91,6 +99,21 @@ export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVecto
     toArrow(options?: Partial<QueryExecutionOptions>): Promise<ArrowTable>;
     /** Collect the results as an array of objects. */
     toArray(options?: Partial<QueryExecutionOptions>): Promise<any[]>;
+    /**
+     * Generates an explanation of the query execution plan.
+     *
+     * @example
+     * import * as lancedb from "@lancedb/lancedb"
+     * const db = await lancedb.connect("./.lancedb");
+     * const table = await db.createTable("my_table", [
+     *   { vector: [1.1, 0.9], id: "1" },
+     * ]);
+     * const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
+     *
+     * @param verbose - If true, provides a more detailed explanation. Defaults to false.
+     * @returns A Promise that resolves to a string containing the query execution plan explanation.
+     */
+    explainPlan(verbose?: boolean): Promise<string>;
 }
 /**
  * An interface for a query that can be executed
@@ -104,8 +127,8 @@ export interface ExecutableQuery {
  *
  * This builder can be reused to execute the query many times.
  */
-export declare class VectorQuery extends QueryBase<NativeVectorQuery, VectorQuery> {
-    constructor(inner: NativeVectorQuery);
+export declare class VectorQuery extends QueryBase<NativeVectorQuery> {
+    constructor(inner: NativeVectorQuery | Promise<NativeVectorQuery>);
     /**
      * Set the number of partitions to search (probe)
      *
@@ -153,7 +176,7 @@ export declare class VectorQuery extends QueryBase<NativeVectorQuery, VectorQuer
      *
      * By default "l2" is used.
      */
-    distanceType(distanceType: string): VectorQuery;
+    distanceType(distanceType: Required<IvfPqOptions>["distanceType"]): VectorQuery;
     /**
      * A multiplier to control how many additional rows are taken during the refine step
      *
@@ -217,7 +240,7 @@ export declare class VectorQuery extends QueryBase<NativeVectorQuery, VectorQuer
     bypassVectorIndex(): VectorQuery;
 }
 /** A builder for LanceDB queries. */
-export declare class Query extends QueryBase<NativeQuery, Query> {
+export declare class Query extends QueryBase<NativeQuery> {
     constructor(tbl: NativeTable);
     /**
      * Find the nearest vectors to the given query vector.