@dengxifeng/lancedb 0.26.2

package/dist/connection.d.ts

@@ -0,0 +1,259 @@
+import { Data, SchemaLike, TableLike } from "./arrow";
+import { EmbeddingFunctionConfig } from "./embedding/registry";
+import { Connection as LanceDbConnection } from "./native";
+import { Table } from "./table";
+export interface CreateTableOptions {
+    /**
+     * The mode to use when creating the table.
+     *
+     * If this is set to "create" and the table already exists then either
+     * an error will be thrown or, if existOk is true, then nothing will
+     * happen.  Any provided data will be ignored.
+     *
+     * If this is set to "overwrite" then any existing table will be replaced.
+     */
+    mode: "create" | "overwrite";
+    /**
+     * If this is true and the table already exists and the mode is "create"
+     * then no error will be raised.
+     */
+    existOk: boolean;
+    /**
+     * Configuration for object storage.
+     *
+     * Options already set on the connection will be inherited by the table,
+     * but can be overridden here.
+     *
+     * The available options are described at https://lancedb.com/docs/storage/
+     */
+    storageOptions?: Record<string, string>;
+    /**
+     * The version of the data storage format to use.
+     *
+     * The default is `stable`.
+     * Set to "legacy" to use the old format.
+     *
+     * @deprecated Pass `new_table_data_storage_version` to storageOptions instead.
+     */
+    dataStorageVersion?: string;
+    /**
+     * Use the new V2 manifest paths. These paths provide more efficient
+     * opening of datasets with many versions on object stores.  WARNING:
+     * turning this on will make the dataset unreadable for older versions
+     * of LanceDB (prior to 0.10.0). To migrate an existing dataset, instead
+     * use the {@link LocalTable#migrateManifestPathsV2} method.
+     *
+     * @deprecated Pass `new_table_enable_v2_manifest_paths` to storageOptions instead.
+     */
+    enableV2ManifestPaths?: boolean;
+    schema?: SchemaLike;
+    embeddingFunction?: EmbeddingFunctionConfig;
+}
+export interface OpenTableOptions {
+    /**
+     * Configuration for object storage.
+     *
+     * Options already set on the connection will be inherited by the table,
+     * but can be overridden here.
+     *
+     * The available options are described at https://lancedb.com/docs/storage/
+     */
+    storageOptions?: Record<string, string>;
+    /**
+     * Set the size of the index cache, specified as a number of entries
+     *
+     * @deprecated Use session-level cache configuration instead.
+     * Create a Session with custom cache sizes and pass it to the connect() function.
+     *
+     * The exact meaning of an "entry" will depend on the type of index:
+     * - IVF: there is one entry for each IVF partition
+     * - BTREE: there is one entry for the entire index
+     *
+     * This cache applies to the entire opened table, across all indices.
+     * Setting this value higher will increase performance on larger datasets
+     * at the expense of more RAM
+     */
+    indexCacheSize?: number;
+}
+export interface TableNamesOptions {
+    /**
+     * If present, only return names that come lexicographically after the
+     * supplied value.
+     *
+     * This can be combined with limit to implement pagination by setting this to
+     * the last table name from the previous page.
+     */
+    startAfter?: string;
+    /** An optional limit to the number of results to return. */
+    limit?: number;
+}
+/**
+ * A LanceDB Connection that allows you to open tables and create new ones.
+ *
+ * Connection could be local against filesystem or remote against a server.
+ *
+ * A Connection is intended to be a long lived object and may hold open
+ * resources such as HTTP connection pools.  This is generally fine and
+ * a single connection should be shared if it is going to be used many
+ * times. However, if you are finished with a connection, you may call
+ * close to eagerly free these resources.  Any call to a Connection
+ * method after it has been closed will result in an error.
+ *
+ * Closing a connection is optional.  Connections will automatically
+ * be closed when they are garbage collected.
+ *
+ * Any created tables are independent and will continue to work even if
+ * the underlying connection has been closed.
+ * @hideconstructor
+ */
+export declare abstract class Connection {
+    /**
+     * Return true if the connection has not been closed
+     */
+    abstract isOpen(): boolean;
+    /**
+     * Close the connection, releasing any underlying resources.
+     *
+     * It is safe to call this method multiple times.
+     *
+     * Any attempt to use the connection after it is closed will result in an error.
+     */
+    abstract close(): void;
+    /**
+     * Return a brief description of the connection
+     */
+    abstract display(): string;
+    /**
+     * List all the table names in this database.
+     *
+     * Tables will be returned in lexicographical order.
+     * @param {Partial<TableNamesOptions>} options - options to control the
+     * paging / start point (backwards compatibility)
+     *
+     */
+    abstract tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
+    /**
+     * List all the table names in this database.
+     *
+     * Tables will be returned in lexicographical order.
+     * @param {string[]} namespace - The namespace to list tables from (defaults to root namespace)
+     * @param {Partial<TableNamesOptions>} options - options to control the
+     * paging / start point
+     *
+     */
+    abstract tableNames(namespace?: string[], options?: Partial<TableNamesOptions>): Promise<string[]>;
+    /**
+     * Open a table in the database.
+     * @param {string} name - The name of the table
+     * @param {string[]} namespace - The namespace of the table (defaults to root namespace)
+     * @param {Partial<OpenTableOptions>} options - Additional options
+     */
+    abstract openTable(name: string, namespace?: string[], options?: Partial<OpenTableOptions>): Promise<Table>;
+    /**
+     * Creates a new Table and initialize it with new data.
+     * @param {object} options - The options object.
+     * @param {string} options.name - The name of the table.
+     * @param {Data} options.data - Non-empty Array of Records to be inserted into the table
+     * @param {string[]} namespace - The namespace to create the table in (defaults to root namespace)
+     *
+     */
+    abstract createTable(options: {
+        name: string;
+        data: Data;
+    } & Partial<CreateTableOptions>, namespace?: string[]): Promise<Table>;
+    /**
+     * Creates a new Table and initialize it with new data.
+     * @param {string} name - The name of the table.
+     * @param {Record<string, unknown>[] | TableLike} data - Non-empty Array of Records
+     * to be inserted into the table
+     * @param {Partial<CreateTableOptions>} options - Additional options (backwards compatibility)
+     */
+    abstract createTable(name: string, data: Record<string, unknown>[] | TableLike, options?: Partial<CreateTableOptions>): Promise<Table>;
+    /**
+     * Creates a new Table and initialize it with new data.
+     * @param {string} name - The name of the table.
+     * @param {Record<string, unknown>[] | TableLike} data - Non-empty Array of Records
+     * to be inserted into the table
+     * @param {string[]} namespace - The namespace to create the table in (defaults to root namespace)
+     * @param {Partial<CreateTableOptions>} options - Additional options
+     */
+    abstract createTable(name: string, data: Record<string, unknown>[] | TableLike, namespace?: string[], options?: Partial<CreateTableOptions>): Promise<Table>;
+    /**
+     * Creates a new empty Table
+     * @param {string} name - The name of the table.
+     * @param {Schema} schema - The schema of the table
+     * @param {Partial<CreateTableOptions>} options - Additional options (backwards compatibility)
+     */
+    abstract createEmptyTable(name: string, schema: import("./arrow").SchemaLike, options?: Partial<CreateTableOptions>): Promise<Table>;
+    /**
+     * Creates a new empty Table
+     * @param {string} name - The name of the table.
+     * @param {Schema} schema - The schema of the table
+     * @param {string[]} namespace - The namespace to create the table in (defaults to root namespace)
+     * @param {Partial<CreateTableOptions>} options - Additional options
+     */
+    abstract createEmptyTable(name: string, schema: import("./arrow").SchemaLike, namespace?: string[], options?: Partial<CreateTableOptions>): Promise<Table>;
+    /**
+     * Drop an existing table.
+     * @param {string} name The name of the table to drop.
+     * @param {string[]} namespace The namespace of the table (defaults to root namespace).
+     */
+    abstract dropTable(name: string, namespace?: string[]): Promise<void>;
+    /**
+     * Drop all tables in the database.
+     * @param {string[]} namespace The namespace to drop tables from (defaults to root namespace).
+     */
+    abstract dropAllTables(namespace?: string[]): Promise<void>;
+    /**
+     * Clone a table from a source table.
+     *
+     * A shallow clone creates a new table that shares the underlying data files
+     * with the source table but has its own independent manifest. This allows
+     * both the source and cloned tables to evolve independently while initially
+     * sharing the same data, deletion, and index files.
+     *
+     * @param {string} targetTableName - The name of the target table to create.
+     * @param {string} sourceUri - The URI of the source table to clone from.
+     * @param {object} options - Clone options.
+     * @param {string[]} options.targetNamespace - The namespace for the target table (defaults to root namespace).
+     * @param {number} options.sourceVersion - The version of the source table to clone.
+     * @param {string} options.sourceTag - The tag of the source table to clone.
+     * @param {boolean} options.isShallow - Whether to perform a shallow clone (defaults to true).
+     */
+    abstract cloneTable(targetTableName: string, sourceUri: string, options?: {
+        targetNamespace?: string[];
+        sourceVersion?: number;
+        sourceTag?: string;
+        isShallow?: boolean;
+    }): Promise<Table>;
+}
+/** @hideconstructor */
+export declare class LocalConnection extends Connection {
+    readonly inner: LanceDbConnection;
+    /** @hidden */
+    constructor(inner: LanceDbConnection);
+    isOpen(): boolean;
+    close(): void;
+    display(): string;
+    tableNames(namespaceOrOptions?: string[] | Partial<TableNamesOptions>, options?: Partial<TableNamesOptions>): Promise<string[]>;
+    openTable(name: string, namespace?: string[], options?: Partial<OpenTableOptions>): Promise<Table>;
+    cloneTable(targetTableName: string, sourceUri: string, options?: {
+        targetNamespace?: string[];
+        sourceVersion?: number;
+        sourceTag?: string;
+        isShallow?: boolean;
+    }): Promise<Table>;
+    private getStorageOptions;
+    createTable(nameOrOptions: string | ({
+        name: string;
+        data: Data;
+    } & Partial<CreateTableOptions>), dataOrNamespace?: Record<string, unknown>[] | TableLike | string[], namespaceOrOptions?: string[] | Partial<CreateTableOptions>, options?: Partial<CreateTableOptions>): Promise<Table>;
+    private _createTableImpl;
+    createEmptyTable(name: string, schema: import("./arrow").SchemaLike, namespaceOrOptions?: string[] | Partial<CreateTableOptions>, options?: Partial<CreateTableOptions>): Promise<Table>;
+    dropTable(name: string, namespace?: string[]): Promise<void>;
+    dropAllTables(namespace?: string[]): Promise<void>;
+}
+/**
+ * Takes storage options and makes all the keys snake case.
+ */
+export declare function cleanseStorageOptions(options?: Record<string, string>): Record<string, string> | undefined;

package/dist/connection.js

@@ -0,0 +1,224 @@
+"use strict";
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalConnection = exports.Connection = void 0;
+exports.cleanseStorageOptions = cleanseStorageOptions;
+const arrow_1 = require("./arrow");
+const arrow_2 = require("./arrow");
+const registry_1 = require("./embedding/registry");
+const sanitize_1 = require("./sanitize");
+const table_1 = require("./table");
+/**
+ * A LanceDB Connection that allows you to open tables and create new ones.
+ *
+ * Connection could be local against filesystem or remote against a server.
+ *
+ * A Connection is intended to be a long lived object and may hold open
+ * resources such as HTTP connection pools.  This is generally fine and
+ * a single connection should be shared if it is going to be used many
+ * times. However, if you are finished with a connection, you may call
+ * close to eagerly free these resources.  Any call to a Connection
+ * method after it has been closed will result in an error.
+ *
+ * Closing a connection is optional.  Connections will automatically
+ * be closed when they are garbage collected.
+ *
+ * Any created tables are independent and will continue to work even if
+ * the underlying connection has been closed.
+ * @hideconstructor
+ */
+class Connection {
+    [Symbol.for("nodejs.util.inspect.custom")]() {
+        return this.display();
+    }
+}
+exports.Connection = Connection;
+/** @hideconstructor */
+class LocalConnection extends Connection {
+    inner;
+    /** @hidden */
+    constructor(inner) {
+        super();
+        this.inner = inner;
+    }
+    isOpen() {
+        return this.inner.isOpen();
+    }
+    close() {
+        this.inner.close();
+    }
+    display() {
+        return this.inner.display();
+    }
+    async tableNames(namespaceOrOptions, options) {
+        // Detect if first argument is namespace array or options object
+        let namespace;
+        let tableNamesOptions;
+        if (Array.isArray(namespaceOrOptions)) {
+            // First argument is namespace array
+            namespace = namespaceOrOptions;
+            tableNamesOptions = options;
+        }
+        else {
+            // First argument is options object (backwards compatibility)
+            namespace = undefined;
+            tableNamesOptions = namespaceOrOptions;
+        }
+        return this.inner.tableNames(namespace ?? [], tableNamesOptions?.startAfter, tableNamesOptions?.limit);
+    }
+    async openTable(name, namespace, options) {
+        const innerTable = await this.inner.openTable(name, namespace ?? [], cleanseStorageOptions(options?.storageOptions), options?.indexCacheSize);
+        return new table_1.LocalTable(innerTable);
+    }
+    async cloneTable(targetTableName, sourceUri, options) {
+        const innerTable = await this.inner.cloneTable(targetTableName, sourceUri, options?.targetNamespace ?? [], options?.sourceVersion ?? null, options?.sourceTag ?? null, options?.isShallow ?? true);
+        return new table_1.LocalTable(innerTable);
+    }
+    getStorageOptions(options) {
+        if (options?.dataStorageVersion !== undefined) {
+            if (options.storageOptions === undefined) {
+                options.storageOptions = {};
+            }
+            options.storageOptions["newTableDataStorageVersion"] =
+                options.dataStorageVersion;
+        }
+        if (options?.enableV2ManifestPaths !== undefined) {
+            if (options.storageOptions === undefined) {
+                options.storageOptions = {};
+            }
+            options.storageOptions["newTableEnableV2ManifestPaths"] =
+                options.enableV2ManifestPaths ? "true" : "false";
+        }
+        return cleanseStorageOptions(options?.storageOptions);
+    }
+    async createTable(nameOrOptions, dataOrNamespace, namespaceOrOptions, options) {
+        if (typeof nameOrOptions !== "string" && "name" in nameOrOptions) {
+            // First overload: createTable(options, namespace?)
+            const { name, data, ...createOptions } = nameOrOptions;
+            const namespace = dataOrNamespace;
+            return this._createTableImpl(name, data, namespace, createOptions);
+        }
+        // Second overload: createTable(name, data, namespace?, options?)
+        const name = nameOrOptions;
+        const data = dataOrNamespace;
+        // Detect if third argument is namespace array or options object
+        let namespace;
+        let createOptions;
+        if (Array.isArray(namespaceOrOptions)) {
+            // Third argument is namespace array
+            namespace = namespaceOrOptions;
+            createOptions = options;
+        }
+        else {
+            // Third argument is options object (backwards compatibility)
+            namespace = undefined;
+            createOptions = namespaceOrOptions;
+        }
+        return this._createTableImpl(name, data, namespace, createOptions);
+    }
+    async _createTableImpl(name, data, namespace, options) {
+        if (data === undefined) {
+            throw new Error("data is required");
+        }
+        const { buf, mode } = await parseTableData(data, options);
+        const storageOptions = this.getStorageOptions(options);
+        const innerTable = await this.inner.createTable(name, buf, mode, namespace ?? [], storageOptions);
+        return new table_1.LocalTable(innerTable);
+    }
+    async createEmptyTable(name, schema, namespaceOrOptions, options) {
+        // Detect if third argument is namespace array or options object
+        let namespace;
+        let createOptions;
+        if (Array.isArray(namespaceOrOptions)) {
+            // Third argument is namespace array
+            namespace = namespaceOrOptions;
+            createOptions = options;
+        }
+        else {
+            // Third argument is options object (backwards compatibility)
+            namespace = undefined;
+            createOptions = namespaceOrOptions;
+        }
+        let mode = createOptions?.mode ?? "create";
+        const existOk = createOptions?.existOk ?? false;
+        if (mode === "create" && existOk) {
+            mode = "exist_ok";
+        }
+        let metadata = undefined;
+        if (createOptions?.embeddingFunction !== undefined) {
+            const embeddingFunction = createOptions.embeddingFunction;
+            const registry = (0, registry_1.getRegistry)();
+            metadata = registry.getTableMetadata([embeddingFunction]);
+        }
+        const storageOptions = this.getStorageOptions(createOptions);
+        const table = (0, arrow_2.makeEmptyTable)(schema, metadata);
+        const buf = await (0, arrow_2.fromTableToBuffer)(table);
+        const innerTable = await this.inner.createEmptyTable(name, buf, mode, namespace ?? [], storageOptions);
+        return new table_1.LocalTable(innerTable);
+    }
+    async dropTable(name, namespace) {
+        return this.inner.dropTable(name, namespace ?? []);
+    }
+    async dropAllTables(namespace) {
+        return this.inner.dropAllTables(namespace ?? []);
+    }
+}
+exports.LocalConnection = LocalConnection;
+/**
+ * Takes storage options and makes all the keys snake case.
+ */
+function cleanseStorageOptions(options) {
+    if (options === undefined) {
+        return undefined;
+    }
+    const result = {};
+    for (const [key, value] of Object.entries(options)) {
+        if (value !== undefined) {
+            const newKey = camelToSnakeCase(key);
+            result[newKey] = value;
+        }
+    }
+    return result;
+}
+/**
+ * Convert a string to snake case. It might already be snake case, in which case it is
+ * returned unchanged.
+ */
+function camelToSnakeCase(camel) {
+    if (camel.includes("_")) {
+        // Assume if there is at least one underscore, it is already snake case
+        return camel;
+    }
+    if (camel.toLocaleUpperCase() === camel) {
+        // Assume if the string is all uppercase, it is already snake case
+        return camel;
+    }
+    let result = camel.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
+    if (result.startsWith("_")) {
+        result = result.slice(1);
+    }
+    return result;
+}
+async function parseTableData(data, options, streaming = false) {
+    let mode = options?.mode ?? "create";
+    const existOk = options?.existOk ?? false;
+    if (mode === "create" && existOk) {
+        mode = "exist_ok";
+    }
+    let table;
+    if ((0, arrow_1.isArrowTable)(data)) {
+        table = (0, sanitize_1.sanitizeTable)(data);
+    }
+    else {
+        table = (0, arrow_1.makeArrowTable)(data, options);
+    }
+    if (streaming) {
+        const buf = await (0, arrow_1.fromTableToStreamBuffer)(table, options?.embeddingFunction, options?.schema);
+        return { buf, mode };
+    }
+    else {
+        const buf = await (0, arrow_2.fromTableToBuffer)(table, options?.embeddingFunction, options?.schema);
+        return { buf, mode };
+    }
+}

package/dist/embedding/embedding_function.d.ts

@@ -0,0 +1,103 @@
+import "reflect-metadata";
+import { DataType, Float, type IntoVector } from "../arrow";
+/**
+ * Options for a given embedding function
+ */
+export interface FunctionOptions {
+    [key: string]: any;
+}
+export interface EmbeddingFunctionConstructor<T extends EmbeddingFunction = EmbeddingFunction> {
+    new (modelOptions?: T["TOptions"]): T;
+}
+/**
+ * An embedding function that automatically creates vector representation for a given column.
+ *
+ * It's important subclasses pass the **original** options to the super constructor
+ * and then pass those options to `resolveVariables` to resolve any variables before
+ * using them.
+ *
+ * @example
+ * ```ts
+ * class MyEmbeddingFunction extends EmbeddingFunction {
+ *   constructor(options: {model: string, timeout: number}) {
+ *     super(optionsRaw);
+ *     const options = this.resolveVariables(optionsRaw);
+ *     this.model = options.model;
+ *     this.timeout = options.timeout;
+ *   }
+ * }
+ * ```
+ */
+export declare abstract class EmbeddingFunction<T = any, M extends FunctionOptions = FunctionOptions> {
+    #private;
+    /**
+     * @ignore
+     *  This is only used for associating the options type with the class for type checking
+     */
+    readonly TOptions: M;
+    /**
+     * Get the original arguments to the constructor, to serialize them so they
+     * can be used to recreate the embedding function later.
+     */
+    toJSON(): Record<string, any>;
+    constructor();
+    /**
+     * Provide a list of keys in the function options that should be treated as
+     * sensitive. If users pass raw values for these keys, they will be rejected.
+     */
+    protected getSensitiveKeys(): string[];
+    /**
+     * Apply variables to the config.
+     */
+    protected resolveVariables(config: Partial<M>): Partial<M>;
+    /**
+     * Optionally load any resources needed for the embedding function.
+     *
+     * This method is called after the embedding function has been initialized
+     * but before any embeddings are computed. It is useful for loading local models
+     * or other resources that are needed for the embedding function to work.
+     */
+    init?(): Promise<void>;
+    /**
+     * sourceField is used in combination with `LanceSchema` to provide a declarative data model
+     *
+     * @param optionsOrDatatype - The options for the field or the datatype
+     *
+     * @see {@link LanceSchema}
+     */
+    sourceField(optionsOrDatatype: Partial<FieldOptions> | DataType): [DataType, Map<string, EmbeddingFunction>];
+    /**
+     * vectorField is used in combination with `LanceSchema` to provide a declarative data model
+     *
+     * @param optionsOrDatatype - The options for the field
+     *
+     * @see {@link LanceSchema}
+     */
+    vectorField(optionsOrDatatype?: Partial<FieldOptions> | DataType): [DataType, Map<string, EmbeddingFunction>];
+    /** The number of dimensions of the embeddings */
+    ndims(): number | undefined;
+    /** The datatype of the embeddings */
+    abstract embeddingDataType(): Float;
+    /**
+     * Creates a vector representation for the given values.
+     */
+    abstract computeSourceEmbeddings(data: T[]): Promise<number[][] | Float32Array[] | Float64Array[]>;
+    /**
+    Compute the embeddings for a single query
+   */
+    computeQueryEmbeddings(data: T): Promise<Awaited<IntoVector>>;
+}
+/**
+ * an abstract class for implementing embedding functions that take text as input
+ */
+export declare abstract class TextEmbeddingFunction<M extends FunctionOptions = FunctionOptions> extends EmbeddingFunction<string, M> {
+    abstract generateEmbeddings(texts: string[], ...args: any[]): Promise<number[][] | Float32Array[] | Float64Array[]>;
+    computeQueryEmbeddings(data: string): Promise<Awaited<IntoVector>>;
+    embeddingDataType(): Float;
+    sourceField(): [DataType, Map<string, EmbeddingFunction>];
+    computeSourceEmbeddings(data: string[]): Promise<number[][] | Float32Array[] | Float64Array[]>;
+}
+export interface FieldOptions<T extends DataType = DataType> {
+    datatype: T;
+    dims?: number;
+}