@lancedb/lancedb 0.7.0 → 0.8.0

package/nodejs-artifacts/connection.d.ts

@@ -1,171 +0,0 @@
-import { Data, SchemaLike, TableLike } from "./arrow";
-import { EmbeddingFunctionConfig } from "./embedding/registry";
-import { Connection as LanceDbConnection } from "./native";
-import { Table } from "./table";
-export interface CreateTableOptions {
-    /**
-     * The mode to use when creating the table.
-     *
-     * If this is set to "create" and the table already exists then either
-     * an error will be thrown or, if existOk is true, then nothing will
-     * happen.  Any provided data will be ignored.
-     *
-     * If this is set to "overwrite" then any existing table will be replaced.
-     */
-    mode: "create" | "overwrite";
-    /**
-     * If this is true and the table already exists and the mode is "create"
-     * then no error will be raised.
-     */
-    existOk: boolean;
-    /**
-     * Configuration for object storage.
-     *
-     * Options already set on the connection will be inherited by the table,
-     * but can be overridden here.
-     *
-     * The available options are described at https://lancedb.github.io/lancedb/guides/storage/
-     */
-    storageOptions?: Record<string, string>;
-    /**
-     * If true then data files will be written with the legacy format
-     *
-     * The default is true while the new format is in beta
-     */
-    useLegacyFormat?: boolean;
-    schema?: SchemaLike;
-    embeddingFunction?: EmbeddingFunctionConfig;
-}
-export interface OpenTableOptions {
-    /**
-     * Configuration for object storage.
-     *
-     * Options already set on the connection will be inherited by the table,
-     * but can be overridden here.
-     *
-     * The available options are described at https://lancedb.github.io/lancedb/guides/storage/
-     */
-    storageOptions?: Record<string, string>;
-    /**
-     * Set the size of the index cache, specified as a number of entries
-     *
-     * The exact meaning of an "entry" will depend on the type of index:
-     * - IVF: there is one entry for each IVF partition
-     * - BTREE: there is one entry for the entire index
-     *
-     * This cache applies to the entire opened table, across all indices.
-     * Setting this value higher will increase performance on larger datasets
-     * at the expense of more RAM
-     */
-    indexCacheSize?: number;
-}
-export interface TableNamesOptions {
-    /**
-     * If present, only return names that come lexicographically after the
-     * supplied value.
-     *
-     * This can be combined with limit to implement pagination by setting this to
-     * the last table name from the previous page.
-     */
-    startAfter?: string;
-    /** An optional limit to the number of results to return. */
-    limit?: number;
-}
-/**
- * A LanceDB Connection that allows you to open tables and create new ones.
- *
- * Connection could be local against filesystem or remote against a server.
- *
- * A Connection is intended to be a long lived object and may hold open
- * resources such as HTTP connection pools.  This is generally fine and
- * a single connection should be shared if it is going to be used many
- * times. However, if you are finished with a connection, you may call
- * close to eagerly free these resources.  Any call to a Connection
- * method after it has been closed will result in an error.
- *
- * Closing a connection is optional.  Connections will automatically
- * be closed when they are garbage collected.
- *
- * Any created tables are independent and will continue to work even if
- * the underlying connection has been closed.
- */
-export declare abstract class Connection {
-    /**
-     * Return true if the connection has not been closed
-     */
-    abstract isOpen(): boolean;
-    /**
-     * Close the connection, releasing any underlying resources.
-     *
-     * It is safe to call this method multiple times.
-     *
-     * Any attempt to use the connection after it is closed will result in an error.
-     */
-    abstract close(): void;
-    /**
-     * Return a brief description of the connection
-     */
-    abstract display(): string;
-    /**
-     * List all the table names in this database.
-     *
-     * Tables will be returned in lexicographical order.
-     * @param {Partial<TableNamesOptions>} options - options to control the
-     * paging / start point
-     *
-     */
-    abstract tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
-    /**
-     * Open a table in the database.
-     * @param {string} name - The name of the table
-     */
-    abstract openTable(name: string, options?: Partial<OpenTableOptions>): Promise<Table>;
-    /**
-     * Creates a new Table and initialize it with new data.
-     * @param {object} options - The options object.
-     * @param {string} options.name - The name of the table.
-     * @param {Data} options.data - Non-empty Array of Records to be inserted into the table
-     *
-     */
-    abstract createTable(options: {
-        name: string;
-        data: Data;
-    } & Partial<CreateTableOptions>): Promise<Table>;
-    /**
-     * Creates a new Table and initialize it with new data.
-     * @param {string} name - The name of the table.
-     * @param {Record<string, unknown>[] | TableLike} data - Non-empty Array of Records
-     * to be inserted into the table
-     */
-    abstract createTable(name: string, data: Record<string, unknown>[] | TableLike, options?: Partial<CreateTableOptions>): Promise<Table>;
-    /**
-     * Creates a new empty Table
-     * @param {string} name - The name of the table.
-     * @param {Schema} schema - The schema of the table
-     */
-    abstract createEmptyTable(name: string, schema: import("./arrow").SchemaLike, options?: Partial<CreateTableOptions>): Promise<Table>;
-    /**
-     * Drop an existing table.
-     * @param {string} name The name of the table to drop.
-     */
-    abstract dropTable(name: string): Promise<void>;
-}
-export declare class LocalConnection extends Connection {
-    readonly inner: LanceDbConnection;
-    constructor(inner: LanceDbConnection);
-    isOpen(): boolean;
-    close(): void;
-    display(): string;
-    tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
-    openTable(name: string, options?: Partial<OpenTableOptions>): Promise<Table>;
-    createTable(nameOrOptions: string | ({
-        name: string;
-        data: Data;
-    } & Partial<CreateTableOptions>), data?: Record<string, unknown>[] | TableLike, options?: Partial<CreateTableOptions>): Promise<Table>;
-    createEmptyTable(name: string, schema: import("./arrow").SchemaLike, options?: Partial<CreateTableOptions>): Promise<Table>;
-    dropTable(name: string): Promise<void>;
-}
-/**
- * Takes storage options and makes all the keys snake case.
- */
-export declare function cleanseStorageOptions(options?: Record<string, string>): Record<string, string> | undefined;

package/nodejs-artifacts/connection.js

@@ -1,135 +0,0 @@
-"use strict";
-// Copyright 2024 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.cleanseStorageOptions = exports.LocalConnection = exports.Connection = void 0;
-const arrow_1 = require("./arrow");
-const registry_1 = require("./embedding/registry");
-const table_1 = require("./table");
-/**
- * A LanceDB Connection that allows you to open tables and create new ones.
- *
- * Connection could be local against filesystem or remote against a server.
- *
- * A Connection is intended to be a long lived object and may hold open
- * resources such as HTTP connection pools.  This is generally fine and
- * a single connection should be shared if it is going to be used many
- * times. However, if you are finished with a connection, you may call
- * close to eagerly free these resources.  Any call to a Connection
- * method after it has been closed will result in an error.
- *
- * Closing a connection is optional.  Connections will automatically
- * be closed when they are garbage collected.
- *
- * Any created tables are independent and will continue to work even if
- * the underlying connection has been closed.
- */
-class Connection {
-    [Symbol.for("nodejs.util.inspect.custom")]() {
-        return this.display();
-    }
-}
-exports.Connection = Connection;
-class LocalConnection extends Connection {
-    inner;
-    constructor(inner) {
-        super();
-        this.inner = inner;
-    }
-    isOpen() {
-        return this.inner.isOpen();
-    }
-    close() {
-        this.inner.close();
-    }
-    display() {
-        return this.inner.display();
-    }
-    async tableNames(options) {
-        return this.inner.tableNames(options?.startAfter, options?.limit);
-    }
-    async openTable(name, options) {
-        const innerTable = await this.inner.openTable(name, cleanseStorageOptions(options?.storageOptions), options?.indexCacheSize);
-        return new table_1.LocalTable(innerTable);
-    }
-    async createTable(nameOrOptions, data, options) {
-        if (typeof nameOrOptions !== "string" && "name" in nameOrOptions) {
-            const { name, data, ...options } = nameOrOptions;
-            return this.createTable(name, data, options);
-        }
-        if (data === undefined) {
-            throw new Error("data is required");
-        }
-        const { buf, mode } = await table_1.Table.parseTableData(data, options);
-        const innerTable = await this.inner.createTable(nameOrOptions, buf, mode, cleanseStorageOptions(options?.storageOptions), options?.useLegacyFormat);
-        return new table_1.LocalTable(innerTable);
-    }
-    async createEmptyTable(name, schema, options) {
-        let mode = options?.mode ?? "create";
-        const existOk = options?.existOk ?? false;
-        if (mode === "create" && existOk) {
-            mode = "exist_ok";
-        }
-        let metadata = undefined;
-        if (options?.embeddingFunction !== undefined) {
-            const embeddingFunction = options.embeddingFunction;
-            const registry = (0, registry_1.getRegistry)();
-            metadata = registry.getTableMetadata([embeddingFunction]);
-        }
-        const table = (0, arrow_1.makeEmptyTable)(schema, metadata);
-        const buf = await (0, arrow_1.fromTableToBuffer)(table);
-        const innerTable = await this.inner.createEmptyTable(name, buf, mode, cleanseStorageOptions(options?.storageOptions), options?.useLegacyFormat);
-        return new table_1.LocalTable(innerTable);
-    }
-    async dropTable(name) {
-        return this.inner.dropTable(name);
-    }
-}
-exports.LocalConnection = LocalConnection;
-/**
- * Takes storage options and makes all the keys snake case.
- */
-function cleanseStorageOptions(options) {
-    if (options === undefined) {
-        return undefined;
-    }
-    const result = {};
-    for (const [key, value] of Object.entries(options)) {
-        if (value !== undefined) {
-            const newKey = camelToSnakeCase(key);
-            result[newKey] = value;
-        }
-    }
-    return result;
-}
-exports.cleanseStorageOptions = cleanseStorageOptions;
-/**
- * Convert a string to snake case. It might already be snake case, in which case it is
- * returned unchanged.
- */
-function camelToSnakeCase(camel) {
-    if (camel.includes("_")) {
-        // Assume if there is at least one underscore, it is already snake case
-        return camel;
-    }
-    if (camel.toLocaleUpperCase() === camel) {
-        // Assume if the string is all uppercase, it is already snake case
-        return camel;
-    }
-    let result = camel.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
-    if (result.startsWith("_")) {
-        result = result.slice(1);
-    }
-    return result;
-}

package/nodejs-artifacts/embedding/embedding_function.d.ts

@@ -1,79 +0,0 @@
-import "reflect-metadata";
-import { DataType, Float, type IntoVector } from "../arrow";
-/**
- * Options for a given embedding function
- */
-export interface FunctionOptions {
-    [key: string]: any;
-}
-export interface EmbeddingFunctionConstructor<T extends EmbeddingFunction = EmbeddingFunction> {
-    new (modelOptions?: T["TOptions"]): T;
-}
-/**
- * An embedding function that automatically creates vector representation for a given column.
- */
-export declare abstract class EmbeddingFunction<T = any, M extends FunctionOptions = FunctionOptions> {
-    /**
-     * @ignore
-     *  This is only used for associating the options type with the class for type checking
-     */
-    readonly TOptions: M;
-    /**
-     * Convert the embedding function to a JSON object
-     * It is used to serialize the embedding function to the schema
-     * It's important that any object returned by this method contains all the necessary
-     * information to recreate the embedding function
-     *
-     * It should return the same object that was passed to the constructor
-     * If it does not, the embedding function will not be able to be recreated, or could be recreated incorrectly
-     *
-     * @example
-     * ```ts
-     * class MyEmbeddingFunction extends EmbeddingFunction {
-     *   constructor(options: {model: string, timeout: number}) {
-     *     super();
-     *     this.model = options.model;
-     *     this.timeout = options.timeout;
-     *   }
-     *   toJSON() {
-     *     return {
-     *       model: this.model,
-     *       timeout: this.timeout,
-     *     };
-     * }
-     * ```
-     */
-    abstract toJSON(): Partial<M>;
-    /**
-     * sourceField is used in combination with `LanceSchema` to provide a declarative data model
-     *
-     * @param optionsOrDatatype - The options for the field or the datatype
-     *
-     * @see {@link lancedb.LanceSchema}
-     */
-    sourceField(optionsOrDatatype: Partial<FieldOptions> | DataType): [DataType, Map<string, EmbeddingFunction>];
-    /**
-     * vectorField is used in combination with `LanceSchema` to provide a declarative data model
-     *
-     * @param options - The options for the field
-     *
-     * @see {@link lancedb.LanceSchema}
-     */
-    vectorField(optionsOrDatatype?: Partial<FieldOptions> | DataType): [DataType, Map<string, EmbeddingFunction>];
-    /** The number of dimensions of the embeddings */
-    ndims(): number | undefined;
-    /** The datatype of the embeddings */
-    abstract embeddingDataType(): Float;
-    /**
-     * Creates a vector representation for the given values.
-     */
-    abstract computeSourceEmbeddings(data: T[]): Promise<number[][] | Float32Array[] | Float64Array[]>;
-    /**
-    Compute the embeddings for a single query
-   */
-    computeQueryEmbeddings(data: T): Promise<Awaited<IntoVector>>;
-}
-export interface FieldOptions<T extends DataType = DataType> {
-    datatype: T;
-    dims?: number;
-}

package/nodejs-artifacts/embedding/embedding_function.js

@@ -1,112 +0,0 @@
-"use strict";
-// Copyright 2024 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.EmbeddingFunction = void 0;
-require("reflect-metadata");
-const arrow_1 = require("../arrow");
-const sanitize_1 = require("../sanitize");
-/**
- * An embedding function that automatically creates vector representation for a given column.
- */
-class EmbeddingFunction {
-    /**
-     * @ignore
-     *  This is only used for associating the options type with the class for type checking
-     */
-    // biome-ignore lint/style/useNamingConvention: we want to keep the name as it is
-    TOptions;
-    /**
-     * sourceField is used in combination with `LanceSchema` to provide a declarative data model
-     *
-     * @param optionsOrDatatype - The options for the field or the datatype
-     *
-     * @see {@link lancedb.LanceSchema}
-     */
-    sourceField(optionsOrDatatype) {
-        let datatype = (0, arrow_1.isDataType)(optionsOrDatatype)
-            ? optionsOrDatatype
-            : optionsOrDatatype?.datatype;
-        if (!datatype) {
-            throw new Error("Datatype is required");
-        }
-        datatype = (0, sanitize_1.sanitizeType)(datatype);
-        const metadata = new Map();
-        metadata.set("source_column_for", this);
-        return [datatype, metadata];
-    }
-    /**
-     * vectorField is used in combination with `LanceSchema` to provide a declarative data model
-     *
-     * @param options - The options for the field
-     *
-     * @see {@link lancedb.LanceSchema}
-     */
-    vectorField(optionsOrDatatype) {
-        let dtype;
-        let vectorType;
-        let dims = this.ndims();
-        // `func.vectorField(new Float32())`
-        if ((0, arrow_1.isDataType)(optionsOrDatatype)) {
-            dtype = optionsOrDatatype;
-        }
-        else {
-            // `func.vectorField({
-            //  datatype: new Float32(),
-            //  dims: 10
-            // })`
-            dims = dims ?? optionsOrDatatype?.dims;
-            dtype = optionsOrDatatype?.datatype;
-        }
-        if (dtype !== undefined) {
-            // `func.vectorField(new FixedSizeList(dims, new Field("item", new Float32(), true)))`
-            // or `func.vectorField({datatype: new FixedSizeList(dims, new Field("item", new Float32(), true))})`
-            if ((0, arrow_1.isFixedSizeList)(dtype)) {
-                vectorType = dtype;
-                // `func.vectorField(new Float32())`
-                // or `func.vectorField({datatype: new Float32()})`
-            }
-            else if ((0, arrow_1.isFloat)(dtype)) {
-                // No `ndims` impl and no `{dims: n}` provided;
-                if (dims === undefined) {
-                    throw new Error("ndims is required for vector field");
-                }
-                vectorType = (0, arrow_1.newVectorType)(dims, dtype);
-            }
-            else {
-                throw new Error("Expected FixedSizeList or Float as datatype for vector field");
-            }
-        }
-        else {
-            if (dims === undefined) {
-                throw new Error("ndims is required for vector field");
-            }
-            vectorType = new arrow_1.FixedSizeList(dims, new arrow_1.Field("item", new arrow_1.Float32(), true));
-        }
-        const metadata = new Map();
-        metadata.set("vector_column_for", this);
-        return [vectorType, metadata];
-    }
-    /** The number of dimensions of the embeddings */
-    ndims() {
-        return undefined;
-    }
-    /**
-    Compute the embeddings for a single query
-   */
-    async computeQueryEmbeddings(data) {
-        return this.computeSourceEmbeddings([data]).then((embeddings) => embeddings[0]);
-    }
-}
-exports.EmbeddingFunction = EmbeddingFunction;

package/nodejs-artifacts/embedding/index.d.ts

@@ -1,28 +0,0 @@
-import { Schema } from "../arrow";
-import { EmbeddingFunction } from "./embedding_function";
-export { EmbeddingFunction } from "./embedding_function";
-export * from "./openai";
-export * from "./registry";
-/**
- * Create a schema with embedding functions.
- *
- * @param fields
- * @returns Schema
- * @example
- * ```ts
- * class MyEmbeddingFunction extends EmbeddingFunction {
- * // ...
- * }
- * const func = new MyEmbeddingFunction();
- * const schema = LanceSchema({
- *   id: new Int32(),
- *   text: func.sourceField(new Utf8()),
- *   vector: func.vectorField(),
- *   // optional: specify the datatype and/or dimensions
- *   vector2: func.vectorField({ datatype: new Float32(), dims: 3}),
- * });
- *
- * const table = await db.createTable("my_table", data, { schema });
- * ```
- */
-export declare function LanceSchema(fields: Record<string, [object, Map<string, EmbeddingFunction>] | object>): Schema;

package/nodejs-artifacts/embedding/index.js

@@ -1,114 +0,0 @@
-"use strict";
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.LanceSchema = exports.EmbeddingFunction = void 0;
-const arrow_1 = require("../arrow");
-const arrow_2 = require("../arrow");
-const sanitize_1 = require("../sanitize");
-const registry_1 = require("./registry");
-var embedding_function_1 = require("./embedding_function");
-Object.defineProperty(exports, "EmbeddingFunction", { enumerable: true, get: function () { return embedding_function_1.EmbeddingFunction; } });
-// We need to explicitly export '*' so that the `register` decorator actually registers the class.
-__exportStar(require("./openai"), exports);
-__exportStar(require("./registry"), exports);
-/**
- * Create a schema with embedding functions.
- *
- * @param fields
- * @returns Schema
- * @example
- * ```ts
- * class MyEmbeddingFunction extends EmbeddingFunction {
- * // ...
- * }
- * const func = new MyEmbeddingFunction();
- * const schema = LanceSchema({
- *   id: new Int32(),
- *   text: func.sourceField(new Utf8()),
- *   vector: func.vectorField(),
- *   // optional: specify the datatype and/or dimensions
- *   vector2: func.vectorField({ datatype: new Float32(), dims: 3}),
- * });
- *
- * const table = await db.createTable("my_table", data, { schema });
- * ```
- */
-function LanceSchema(fields) {
-    const arrowFields = [];
-    const embeddingFunctions = new Map();
-    Object.entries(fields).forEach(([key, value]) => {
-        if ((0, arrow_2.isDataType)(value)) {
-            arrowFields.push(new arrow_1.Field(key, (0, sanitize_1.sanitizeType)(value), true));
-        }
-        else {
-            const [dtype, metadata] = value;
-            arrowFields.push(new arrow_1.Field(key, (0, sanitize_1.sanitizeType)(dtype), true));
-            parseEmbeddingFunctions(embeddingFunctions, key, metadata);
-        }
-    });
-    const registry = (0, registry_1.getRegistry)();
-    const metadata = registry.getTableMetadata(Array.from(embeddingFunctions.values()));
-    const schema = new arrow_1.Schema(arrowFields, metadata);
-    return schema;
-}
-exports.LanceSchema = LanceSchema;
-function parseEmbeddingFunctions(embeddingFunctions, key, metadata) {
-    if (metadata.has("source_column_for")) {
-        const embedFunction = metadata.get("source_column_for");
-        const current = embeddingFunctions.get(embedFunction);
-        if (current !== undefined) {
-            embeddingFunctions.set(embedFunction, {
-                ...current,
-                sourceColumn: key,
-            });
-        }
-        else {
-            embeddingFunctions.set(embedFunction, {
-                sourceColumn: key,
-                function: embedFunction,
-            });
-        }
-    }
-    else if (metadata.has("vector_column_for")) {
-        const embedFunction = metadata.get("vector_column_for");
-        const current = embeddingFunctions.get(embedFunction);
-        if (current !== undefined) {
-            embeddingFunctions.set(embedFunction, {
-                ...current,
-                vectorColumn: key,
-            });
-        }
-        else {
-            embeddingFunctions.set(embedFunction, {
-                vectorColumn: key,
-                function: embedFunction,
-            });
-        }
-    }
-}

package/nodejs-artifacts/embedding/openai.d.ts

@@ -1,18 +0,0 @@
-import { type EmbeddingCreateParams } from "openai/resources";
-import { Float } from "../arrow";
-import { EmbeddingFunction } from "./embedding_function";
-export type OpenAIOptions = {
-    apiKey: string;
-    model: EmbeddingCreateParams["model"];
-};
-export declare class OpenAIEmbeddingFunction extends EmbeddingFunction<string, Partial<OpenAIOptions>> {
-    #private;
-    constructor(options?: Partial<OpenAIOptions>);
-    toJSON(): {
-        model: (string & {}) | "text-embedding-ada-002" | "text-embedding-3-small" | "text-embedding-3-large";
-    };
-    ndims(): number;
-    embeddingDataType(): Float;
-    computeSourceEmbeddings(data: string[]): Promise<number[][]>;
-    computeQueryEmbeddings(data: string): Promise<number[]>;
-}