@lancedb/lancedb 0.4.3 → 0.4.15

package/dist/connection.js

@@ -0,0 +1,126 @@
+"use strict";
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Connection = void 0;
+const arrow_1 = require("./arrow");
+const table_1 = require("./table");
+const apache_arrow_1 = require("apache-arrow");
+/**
+ * A LanceDB Connection that allows you to open tables and create new ones.
+ *
+ * Connection could be local against filesystem or remote against a server.
+ *
+ * A Connection is intended to be a long lived object and may hold open
+ * resources such as HTTP connection pools.  This is generally fine and
+ * a single connection should be shared if it is going to be used many
+ * times. However, if you are finished with a connection, you may call
+ * close to eagerly free these resources.  Any call to a Connection
+ * method after it has been closed will result in an error.
+ *
+ * Closing a connection is optional.  Connections will automatically
+ * be closed when they are garbage collected.
+ *
+ * Any created tables are independent and will continue to work even if
+ * the underlying connection has been closed.
+ */
+class Connection {
+    inner;
+    constructor(inner) {
+        this.inner = inner;
+    }
+    /** Return true if the connection has not been closed */
+    isOpen() {
+        return this.inner.isOpen();
+    }
+    /**
+     * Close the connection, releasing any underlying resources.
+     *
+     * It is safe to call this method multiple times.
+     *
+     * Any attempt to use the connection after it is closed will result in an error.
+     */
+    close() {
+        this.inner.close();
+    }
+    /** Return a brief description of the connection */
+    display() {
+        return this.inner.display();
+    }
+    /**
+     * List all the table names in this database.
+     *
+     * Tables will be returned in lexicographical order.
+     * @param {Partial<TableNamesOptions>} options - options to control the
+     * paging / start point
+     */
+    async tableNames(options) {
+        return this.inner.tableNames(options?.startAfter, options?.limit);
+    }
+    /**
+     * Open a table in the database.
+     * @param {string} name - The name of the table
+     */
+    async openTable(name) {
+        const innerTable = await this.inner.openTable(name);
+        return new table_1.Table(innerTable);
+    }
+    /**
+     * Creates a new Table and initialize it with new data.
+     * @param {string} name - The name of the table.
+     * @param {Record<string, unknown>[] | ArrowTable} data - Non-empty Array of Records
+     * to be inserted into the table
+     */
+    async createTable(name, data, options) {
+        let mode = options?.mode ?? "create";
+        const existOk = options?.existOk ?? false;
+        if (mode === "create" && existOk) {
+            mode = "exist_ok";
+        }
+        let table;
+        if (data instanceof apache_arrow_1.Table) {
+            table = data;
+        }
+        else {
+            table = (0, arrow_1.makeArrowTable)(data);
+        }
+        const buf = await (0, arrow_1.fromTableToBuffer)(table);
+        const innerTable = await this.inner.createTable(name, buf, mode);
+        return new table_1.Table(innerTable);
+    }
+    /**
+     * Creates a new empty Table
+     * @param {string} name - The name of the table.
+     * @param {Schema} schema - The schema of the table
+     */
+    async createEmptyTable(name, schema, options) {
+        let mode = options?.mode ?? "create";
+        const existOk = options?.existOk ?? false;
+        if (mode === "create" && existOk) {
+            mode = "exist_ok";
+        }
+        const table = (0, arrow_1.makeEmptyTable)(schema);
+        const buf = await (0, arrow_1.fromTableToBuffer)(table);
+        const innerTable = await this.inner.createEmptyTable(name, buf, mode);
+        return new table_1.Table(innerTable);
+    }
+    /**
+     * Drop an existing table.
+     * @param {string} name The name of the table to drop.
+     */
+    async dropTable(name) {
+        return this.inner.dropTable(name);
+    }
+}
+exports.Connection = Connection;

package/dist/embedding/embedding_function.d.ts

@@ -0,0 +1,45 @@
+import { type Float } from "apache-arrow";
+/**
+ * An embedding function that automatically creates vector representation for a given column.
+ */
+export interface EmbeddingFunction<T> {
+    /**
+     * The name of the column that will be used as input for the Embedding Function.
+     */
+    sourceColumn: string;
+    /**
+     * The data type of the embedding
+     *
+     * The embedding function should return `number`.  This will be converted into
+     * an Arrow float array.  By default this will be Float32 but this property can
+     * be used to control the conversion.
+     */
+    embeddingDataType?: Float;
+    /**
+     * The dimension of the embedding
+     *
+     * This is optional, normally this can be determined by looking at the results of
+     * `embed`.  If this is not specified, and there is an attempt to apply the embedding
+     * to an empty table, then that process will fail.
+     */
+    embeddingDimension?: number;
+    /**
+     * The name of the column that will contain the embedding
+     *
+     * By default this is "vector"
+     */
+    destColumn?: string;
+    /**
+     * Should the source column be excluded from the resulting table
+     *
+     * By default the source column is included.  Set this to true and
+     * only the embedding will be stored.
+     */
+    excludeSource?: boolean;
+    /**
+     * Creates a vector representation for the given values.
+     */
+    embed: (data: T[]) => Promise<number[][]>;
+}
+/** Test if the input seems to be an embedding function */
+export declare function isEmbeddingFunction<T>(value: unknown): value is EmbeddingFunction<T>;

package/dist/embedding/embedding_function.js

@@ -0,0 +1,27 @@
+"use strict";
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isEmbeddingFunction = void 0;
+/** Test if the input seems to be an embedding function */
+function isEmbeddingFunction(value) {
+    if (typeof value !== "object" || value === null) {
+        return false;
+    }
+    if (!("sourceColumn" in value) || !("embed" in value)) {
+        return false;
+    }
+    return (typeof value.sourceColumn === "string" && typeof value.embed === "function");
+}
+exports.isEmbeddingFunction = isEmbeddingFunction;

package/dist/embedding/index.d.ts

@@ -0,0 +1,2 @@
+export { EmbeddingFunction, isEmbeddingFunction } from "./embedding_function";
+export { OpenAIEmbeddingFunction } from "./openai";

package/dist/embedding/index.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenAIEmbeddingFunction = exports.isEmbeddingFunction = void 0;
+var embedding_function_1 = require("./embedding_function");
+Object.defineProperty(exports, "isEmbeddingFunction", { enumerable: true, get: function () { return embedding_function_1.isEmbeddingFunction; } });
+var openai_1 = require("./openai");
+Object.defineProperty(exports, "OpenAIEmbeddingFunction", { enumerable: true, get: function () { return openai_1.OpenAIEmbeddingFunction; } });

package/dist/embedding/openai.d.ts

@@ -0,0 +1,8 @@
+import { type EmbeddingFunction } from "./embedding_function";
+export declare class OpenAIEmbeddingFunction implements EmbeddingFunction<string> {
+    private readonly _openai;
+    private readonly _modelName;
+    constructor(sourceColumn: string, openAIKey: string, modelName?: string);
+    embed(data: string[]): Promise<number[][]>;
+    sourceColumn: string;
+}

package/dist/embedding/openai.js

@@ -0,0 +1,53 @@
+"use strict";
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenAIEmbeddingFunction = void 0;
+class OpenAIEmbeddingFunction {
+    _openai;
+    _modelName;
+    constructor(sourceColumn, openAIKey, modelName = "text-embedding-ada-002") {
+        /**
+         * @type {import("openai").default}
+         */
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        let Openai;
+        try {
+            // eslint-disable-next-line @typescript-eslint/no-var-requires
+            Openai = require("openai");
+        }
+        catch {
+            throw new Error("please install openai@^4.24.1 using npm install openai");
+        }
+        this.sourceColumn = sourceColumn;
+        const configuration = {
+            apiKey: openAIKey,
+        };
+        this._openai = new Openai(configuration);
+        this._modelName = modelName;
+    }
+    async embed(data) {
+        const response = await this._openai.embeddings.create({
+            model: this._modelName,
+            input: data,
+        });
+        const embeddings = [];
+        for (let i = 0; i < response.data.length; i++) {
+            embeddings.push(response.data[i].embedding);
+        }
+        return embeddings;
+    }
+    sourceColumn;
+}
+exports.OpenAIEmbeddingFunction = OpenAIEmbeddingFunction;

package/dist/index.d.ts

@@ -0,0 +1,22 @@
+import { Connection } from "./connection";
+import { ConnectionOptions } from "./native.js";
+export { WriteOptions, WriteMode, AddColumnsSql, ColumnAlteration, ConnectionOptions, } from "./native.js";
+export { makeArrowTable, MakeArrowTableOptions, Data, VectorColumnOptions, } from "./arrow";
+export { Connection, CreateTableOptions, TableNamesOptions, } from "./connection";
+export { ExecutableQuery, Query, QueryBase, VectorQuery, RecordBatchIterator, } from "./query";
+export { Index, IndexOptions, IvfPqOptions } from "./indices";
+export { Table, AddDataOptions, IndexConfig, UpdateOptions } from "./table";
+export * as embedding from "./embedding";
+/**
+ * Connect to a LanceDB instance at the given URI.
+ *
+ * Accpeted formats:
+ *
+ * - `/path/to/database` - local database
+ * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
+ * - `db://host:port` - remote database (LanceDB cloud)
+ * @param {string} uri - The uri of the database. If the database uri starts
+ * with `db://` then it connects to a remote database.
+ * @see {@link ConnectionOptions} for more details on the URI format.
+ */
+export declare function connect(uri: string, opts?: Partial<ConnectionOptions>): Promise<Connection>;

package/dist/index.js

@@ -0,0 +1,52 @@
+"use strict";
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.connect = exports.embedding = exports.Table = exports.Index = exports.RecordBatchIterator = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Connection = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = void 0;
+const connection_1 = require("./connection");
+const native_js_1 = require("./native.js");
+var arrow_1 = require("./arrow");
+Object.defineProperty(exports, "makeArrowTable", { enumerable: true, get: function () { return arrow_1.makeArrowTable; } });
+Object.defineProperty(exports, "MakeArrowTableOptions", { enumerable: true, get: function () { return arrow_1.MakeArrowTableOptions; } });
+Object.defineProperty(exports, "VectorColumnOptions", { enumerable: true, get: function () { return arrow_1.VectorColumnOptions; } });
+var connection_2 = require("./connection");
+Object.defineProperty(exports, "Connection", { enumerable: true, get: function () { return connection_2.Connection; } });
+var query_1 = require("./query");
+Object.defineProperty(exports, "Query", { enumerable: true, get: function () { return query_1.Query; } });
+Object.defineProperty(exports, "QueryBase", { enumerable: true, get: function () { return query_1.QueryBase; } });
+Object.defineProperty(exports, "VectorQuery", { enumerable: true, get: function () { return query_1.VectorQuery; } });
+Object.defineProperty(exports, "RecordBatchIterator", { enumerable: true, get: function () { return query_1.RecordBatchIterator; } });
+var indices_1 = require("./indices");
+Object.defineProperty(exports, "Index", { enumerable: true, get: function () { return indices_1.Index; } });
+var table_1 = require("./table");
+Object.defineProperty(exports, "Table", { enumerable: true, get: function () { return table_1.Table; } });
+exports.embedding = require("./embedding");
+/**
+ * Connect to a LanceDB instance at the given URI.
+ *
+ * Accpeted formats:
+ *
+ * - `/path/to/database` - local database
+ * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
+ * - `db://host:port` - remote database (LanceDB cloud)
+ * @param {string} uri - The uri of the database. If the database uri starts
+ * with `db://` then it connects to a remote database.
+ * @see {@link ConnectionOptions} for more details on the URI format.
+ */
+async function connect(uri, opts) {
+    opts = opts ?? {};
+    const nativeConn = await native_js_1.Connection.new(uri, opts);
+    return new connection_1.Connection(nativeConn);
+}
+exports.connect = connect;

package/dist/indices.d.ts

@@ -0,0 +1,165 @@
+/**
+ * Options to create an `IVF_PQ` index
+ */
+export interface IvfPqOptions {
+    /**
+     * The number of IVF partitions to create.
+     *
+     * This value should generally scale with the number of rows in the dataset.
+     * By default the number of partitions is the square root of the number of
+     * rows.
+     *
+     * If this value is too large then the first part of the search (picking the
+     * right partition) will be slow.  If this value is too small then the second
+     * part of the search (searching within a partition) will be slow.
+     */
+    numPartitions?: number;
+    /**
+     * Number of sub-vectors of PQ.
+     *
+     * This value controls how much the vector is compressed during the quantization step.
+     * The more sub vectors there are the less the vector is compressed.  The default is
+     * the dimension of the vector divided by 16.  If the dimension is not evenly divisible
+     * by 16 we use the dimension divded by 8.
+     *
+     * The above two cases are highly preferred.  Having 8 or 16 values per subvector allows
+     * us to use efficient SIMD instructions.
+     *
+     * If the dimension is not visible by 8 then we use 1 subvector.  This is not ideal and
+     * will likely result in poor performance.
+     */
+    numSubVectors?: number;
+    /**
+     * Distance type to use to build the index.
+     *
+     * Default value is "l2".
+     *
+     * This is used when training the index to calculate the IVF partitions
+     * (vectors are grouped in partitions with similar vectors according to this
+     * distance type) and to calculate a subvector's code during quantization.
+     *
+     * The distance type used to train an index MUST match the distance type used
+     * to search the index.  Failure to do so will yield inaccurate results.
+     *
+     * The following distance types are available:
+     *
+     * "l2" - Euclidean distance. This is a very common distance metric that
+     * accounts for both magnitude and direction when determining the distance
+     * between vectors. L2 distance has a range of [0, ∞).
+     *
+     * "cosine" - Cosine distance.  Cosine distance is a distance metric
+     * calculated from the cosine similarity between two vectors. Cosine
+     * similarity is a measure of similarity between two non-zero vectors of an
+     * inner product space. It is defined to equal the cosine of the angle
+     * between them.  Unlike L2, the cosine distance is not affected by the
+     * magnitude of the vectors.  Cosine distance has a range of [0, 2].
+     *
+     * Note: the cosine distance is undefined when one (or both) of the vectors
+     * are all zeros (there is no direction).  These vectors are invalid and may
+     * never be returned from a vector search.
+     *
+     * "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
+     * distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
+     * L2 norm is 1), then dot distance is equivalent to the cosine distance.
+     */
+    distanceType?: "l2" | "cosine" | "dot";
+    /**
+     * Max iteration to train IVF kmeans.
+     *
+     * When training an IVF PQ index we use kmeans to calculate the partitions.  This parameter
+     * controls how many iterations of kmeans to run.
+     *
+     * Increasing this might improve the quality of the index but in most cases these extra
+     * iterations have diminishing returns.
+     *
+     * The default value is 50.
+     */
+    maxIterations?: number;
+    /**
+     * The number of vectors, per partition, to sample when training IVF kmeans.
+     *
+     * When an IVF PQ index is trained, we need to calculate partitions.  These are groups
+     * of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
+     *
+     * Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
+     * random sample of the data.  This parameter controls the size of the sample.  The total
+     * number of vectors used to train the index is `sample_rate * num_partitions`.
+     *
+     * Increasing this value might improve the quality of the index but in most cases the
+     * default should be sufficient.
+     *
+     * The default value is 256.
+     */
+    sampleRate?: number;
+}
+export declare class Index {
+    private readonly inner;
+    private constructor();
+    /**
+     * Create an IvfPq index
+     *
+     * This index stores a compressed (quantized) copy of every vector.  These vectors
+     * are grouped into partitions of similar vectors.  Each partition keeps track of
+     * a centroid which is the average value of all vectors in the group.
+     *
+     * During a query the centroids are compared with the query vector to find the closest
+     * partitions.  The compressed vectors in these partitions are then searched to find
+     * the closest vectors.
+     *
+     * The compression scheme is called product quantization.  Each vector is divided into
+     * subvectors and then each subvector is quantized into a small number of bits.  the
+     * parameters `num_bits` and `num_subvectors` control this process, providing a tradeoff
+     * between index size (and thus search speed) and index accuracy.
+     *
+     * The partitioning process is called IVF and the `num_partitions` parameter controls how
+     * many groups to create.
+     *
+     * Note that training an IVF PQ index on a large dataset is a slow operation and
+     * currently is also a memory intensive operation.
+     */
+    static ivfPq(options?: Partial<IvfPqOptions>): Index;
+    /**
+     * Create a btree index
+     *
+     * A btree index is an index on a scalar columns.  The index stores a copy of the column
+     * in sorted order.  A header entry is created for each block of rows (currently the
+     * block size is fixed at 4096).  These header entries are stored in a separate
+     * cacheable structure (a btree).  To search for data the header is used to determine
+     * which blocks need to be read from disk.
+     *
+     * For example, a btree index in a table with 1Bi rows requires sizeof(Scalar) * 256Ki
+     * bytes of memory and will generally need to read sizeof(Scalar) * 4096 bytes to find
+     * the correct row ids.
+     *
+     * This index is good for scalar columns with mostly distinct values and does best when
+     * the query is highly selective.
+     *
+     * The btree index does not currently have any parameters though parameters such as the
+     * block size may be added in the future.
+     */
+    static btree(): Index;
+}
+export interface IndexOptions {
+    /**
+     * Advanced index configuration
+     *
+     * This option allows you to specify a specfic index to create and also
+     * allows you to pass in configuration for training the index.
+     *
+     * See the static methods on Index for details on the various index types.
+     *
+     * If this is not supplied then column data type(s) and column statistics
+     * will be used to determine the most useful kind of index to create.
+     */
+    config?: Index;
+    /**
+     * Whether to replace the existing index
+     *
+     * If this is false, and another index already exists on the same columns
+     * and the same name, then an error will be returned.  This is true even if
+     * that index is out of date.
+     *
+     * The default is true
+     */
+    replace?: boolean;
+}

package/dist/indices.js

@@ -0,0 +1,71 @@
+"use strict";
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Index = void 0;
+const native_1 = require("./native");
+class Index {
+    inner;
+    constructor(inner) {
+        this.inner = inner;
+    }
+    /**
+     * Create an IvfPq index
+     *
+     * This index stores a compressed (quantized) copy of every vector.  These vectors
+     * are grouped into partitions of similar vectors.  Each partition keeps track of
+     * a centroid which is the average value of all vectors in the group.
+     *
+     * During a query the centroids are compared with the query vector to find the closest
+     * partitions.  The compressed vectors in these partitions are then searched to find
+     * the closest vectors.
+     *
+     * The compression scheme is called product quantization.  Each vector is divided into
+     * subvectors and then each subvector is quantized into a small number of bits.  the
+     * parameters `num_bits` and `num_subvectors` control this process, providing a tradeoff
+     * between index size (and thus search speed) and index accuracy.
+     *
+     * The partitioning process is called IVF and the `num_partitions` parameter controls how
+     * many groups to create.
+     *
+     * Note that training an IVF PQ index on a large dataset is a slow operation and
+     * currently is also a memory intensive operation.
+     */
+    static ivfPq(options) {
+        return new Index(native_1.Index.ivfPq(options?.distanceType, options?.numPartitions, options?.numSubVectors, options?.maxIterations, options?.sampleRate));
+    }
+    /**
+     * Create a btree index
+     *
+     * A btree index is an index on a scalar columns.  The index stores a copy of the column
+     * in sorted order.  A header entry is created for each block of rows (currently the
+     * block size is fixed at 4096).  These header entries are stored in a separate
+     * cacheable structure (a btree).  To search for data the header is used to determine
+     * which blocks need to be read from disk.
+     *
+     * For example, a btree index in a table with 1Bi rows requires sizeof(Scalar) * 256Ki
+     * bytes of memory and will generally need to read sizeof(Scalar) * 4096 bytes to find
+     * the correct row ids.
+     *
+     * This index is good for scalar columns with mostly distinct values and does best when
+     * the query is highly selective.
+     *
+     * The btree index does not currently have any parameters though parameters such as the
+     * block size may be added in the future.
+     */
+    static btree() {
+        return new Index(native_1.Index.btree());
+    }
+}
+exports.Index = Index;