@dengxifeng/lancedb 0.26.2

package/dist/indices.js

@@ -0,0 +1,156 @@
+"use strict";
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Index = void 0;
+const native_1 = require("./native");
+class Index {
+    inner;
+    constructor(inner) {
+        this.inner = inner;
+    }
+    /**
+     * Create an IvfPq index
+     *
+     * This index stores a compressed (quantized) copy of every vector.  These vectors
+     * are grouped into partitions of similar vectors.  Each partition keeps track of
+     * a centroid which is the average value of all vectors in the group.
+     *
+     * During a query the centroids are compared with the query vector to find the closest
+     * partitions.  The compressed vectors in these partitions are then searched to find
+     * the closest vectors.
+     *
+     * The compression scheme is called product quantization.  Each vector is divided into
+     * subvectors and then each subvector is quantized into a small number of bits.  the
+     * parameters `num_bits` and `num_subvectors` control this process, providing a tradeoff
+     * between index size (and thus search speed) and index accuracy.
+     *
+     * The partitioning process is called IVF and the `num_partitions` parameter controls how
+     * many groups to create.
+     *
+     * Note that training an IVF PQ index on a large dataset is a slow operation and
+     * currently is also a memory intensive operation.
+     */
+    static ivfPq(options) {
+        return new Index(native_1.Index.ivfPq(options?.distanceType, options?.numPartitions, options?.numSubVectors, options?.numBits, options?.maxIterations, options?.sampleRate));
+    }
+    /**
+     * Create an IvfRq index
+     *
+     * IVF-RQ (RabitQ Quantization) compresses vectors using RabitQ quantization
+     * and organizes them into IVF partitions.
+     *
+     * The compression scheme is called RabitQ quantization. Each dimension is quantized into a small number of bits.
+     * The parameters `num_bits` and `num_partitions` control this process, providing a tradeoff
+     * between index size (and thus search speed) and index accuracy.
+     *
+     * The partitioning process is called IVF and the `num_partitions` parameter controls how
+     * many groups to create.
+     *
+     * Note that training an IVF RQ index on a large dataset is a slow operation and
+     * currently is also a memory intensive operation.
+     */
+    static ivfRq(options) {
+        return new Index(native_1.Index.ivfRq(options?.distanceType, options?.numPartitions, options?.numBits, options?.maxIterations, options?.sampleRate));
+    }
+    /**
+     * Create an IvfFlat index
+     *
+     * This index groups vectors into partitions of similar vectors.  Each partition keeps track of
+     * a centroid which is the average value of all vectors in the group.
+     *
+     * During a query the centroids are compared with the query vector to find the closest
+     * partitions.  The vectors in these partitions are then searched to find
+     * the closest vectors.
+     *
+     * The partitioning process is called IVF and the `num_partitions` parameter controls how
+     * many groups to create.
+     *
+     * Note that training an IVF FLAT index on a large dataset is a slow operation and
+     * currently is also a memory intensive operation.
+     */
+    static ivfFlat(options) {
+        return new Index(native_1.Index.ivfFlat(options?.distanceType, options?.numPartitions, options?.maxIterations, options?.sampleRate));
+    }
+    /**
+     * Create a btree index
+     *
+     * A btree index is an index on a scalar columns.  The index stores a copy of the column
+     * in sorted order.  A header entry is created for each block of rows (currently the
+     * block size is fixed at 4096).  These header entries are stored in a separate
+     * cacheable structure (a btree).  To search for data the header is used to determine
+     * which blocks need to be read from disk.
+     *
+     * For example, a btree index in a table with 1Bi rows requires sizeof(Scalar) * 256Ki
+     * bytes of memory and will generally need to read sizeof(Scalar) * 4096 bytes to find
+     * the correct row ids.
+     *
+     * This index is good for scalar columns with mostly distinct values and does best when
+     * the query is highly selective.
+     *
+     * The btree index does not currently have any parameters though parameters such as the
+     * block size may be added in the future.
+     */
+    static btree() {
+        return new Index(native_1.Index.btree());
+    }
+    /**
+     * Create a bitmap index.
+     *
+     * A `Bitmap` index stores a bitmap for each distinct value in the column for every row.
+     *
+     * This index works best for low-cardinality columns, where the number of unique values
+     * is small (i.e., less than a few hundreds).
+     */
+    static bitmap() {
+        return new Index(native_1.Index.bitmap());
+    }
+    /**
+     * Create a label list index.
+     *
+     * LabelList index is a scalar index that can be used on `List<T>` columns to
+     * support queries with `array_contains_all` and `array_contains_any`
+     * using an underlying bitmap index.
+     */
+    static labelList() {
+        return new Index(native_1.Index.labelList());
+    }
+    /**
+     * Create a full text search index
+     *
+     * A full text search index is an index on a string column, so that you can conduct full
+     * text searches on the column.
+     *
+     * The results of a full text search are ordered by relevance measured by BM25.
+     *
+     * You can combine filters with full text search.
+     */
+    static fts(options) {
+        return new Index(native_1.Index.fts(options?.withPosition, options?.baseTokenizer, options?.language, options?.maxTokenLength, options?.lowercase, options?.stem, options?.removeStopWords, options?.asciiFolding, options?.ngramMinLength, options?.ngramMaxLength, options?.prefixOnly));
+    }
+    /**
+     *
+     * Create a hnswPq index
+     *
+     * HNSW-PQ stands for Hierarchical Navigable Small World - Product Quantization.
+     * It is a variant of the HNSW algorithm that uses product quantization to compress
+     * the vectors.
+     *
+     */
+    static hnswPq(options) {
+        return new Index(native_1.Index.hnswPq(options?.distanceType, options?.numPartitions, options?.numSubVectors, options?.maxIterations, options?.sampleRate, options?.m, options?.efConstruction));
+    }
+    /**
+     *
+     * Create a hnswSq index
+     *
+     * HNSW-SQ stands for Hierarchical Navigable Small World - Scalar Quantization.
+     * It is a variant of the HNSW algorithm that uses scalar quantization to compress
+     * the vectors.
+     *
+     */
+    static hnswSq(options) {
+        return new Index(native_1.Index.hnswSq(options?.distanceType, options?.numPartitions, options?.maxIterations, options?.sampleRate, options?.m, options?.efConstruction));
+    }
+}
+exports.Index = Index;

package/dist/merge.d.ts

@@ -0,0 +1,80 @@
+import { Data, Schema } from "./arrow";
+import { MergeResult, NativeMergeInsertBuilder } from "./native";
+/** A builder used to create and run a merge insert operation */
+export declare class MergeInsertBuilder {
+    #private;
+    /** Construct a MergeInsertBuilder. __Internal use only.__ */
+    constructor(native: NativeMergeInsertBuilder, schema: Schema | Promise<Schema>);
+    /**
+     * Rows that exist in both the source table (new data) and
+     * the target table (old data) will be updated, replacing
+     * the old row with the corresponding matching row.
+     *
+     * If there are multiple matches then the behavior is undefined.
+     * Currently this causes multiple copies of the row to be created
+     * but that behavior is subject to change.
+     *
+     * An optional condition may be specified.  If it is, then only
+     * matched rows that satisfy the condtion will be updated.  Any
+     * rows that do not satisfy the condition will be left as they
+     * are.  Failing to satisfy the condition does not cause a
+     * "matched row" to become a "not matched" row.
+     *
+     * The condition should be an SQL string.  Use the prefix
+     * target. to refer to rows in the target table (old data)
+     * and the prefix source. to refer to rows in the source
+     * table (new data).
+     *
+     * For example, "target.last_update < source.last_update"
+     */
+    whenMatchedUpdateAll(options?: {
+        where: string;
+    }): MergeInsertBuilder;
+    /**
+     * Rows that exist only in the source table (new data) should
+     * be inserted into the target table.
+     */
+    whenNotMatchedInsertAll(): MergeInsertBuilder;
+    /**
+     * Rows that exist only in the target table (old data) will be
+     * deleted.  An optional condition can be provided to limit what
+     * data is deleted.
+     *
+     * @param options.where - An optional condition to limit what data is deleted
+     */
+    whenNotMatchedBySourceDelete(options?: {
+        where: string;
+    }): MergeInsertBuilder;
+    /**
+     * Controls whether to use indexes for the merge operation.
+     *
+     * When set to `true` (the default), the operation will use an index if available
+     * on the join key for improved performance. When set to `false`, it forces a full
+     * table scan even if an index exists. This can be useful for benchmarking or when
+     * the query optimizer chooses a suboptimal path.
+     *
+     * @param useIndex - Whether to use indices for the merge operation. Defaults to `true`.
+     */
+    useIndex(useIndex: boolean): MergeInsertBuilder;
+    /**
+     * Executes the merge insert operation
+     *
+     * @returns {Promise<MergeResult>} the merge result
+     */
+    execute(data: Data, execOptions?: Partial<WriteExecutionOptions>): Promise<MergeResult>;
+}
+export interface WriteExecutionOptions {
+    /**
+     * Maximum time to run the operation before cancelling it.
+     *
+     * By default, there is a 30-second timeout that is only enforced after the
+     * first attempt. This is to prevent spending too long retrying to resolve
+     * conflicts. For example, if a write attempt takes 20 seconds and fails,
+     * the second attempt will be cancelled after 10 seconds, hitting the
+     * 30-second timeout. However, a write that takes one hour and succeeds on the
+     * first attempt will not be cancelled.
+     *
+     * When this is set, the timeout is enforced on all attempts, including the first.
+     */
+    timeoutMs?: number;
+}

package/dist/merge.js

@@ -0,0 +1,92 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MergeInsertBuilder = void 0;
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+const arrow_1 = require("./arrow");
+/** A builder used to create and run a merge insert operation */
+class MergeInsertBuilder {
+    #native;
+    #schema;
+    /** Construct a MergeInsertBuilder. __Internal use only.__ */
+    constructor(native, schema) {
+        this.#native = native;
+        this.#schema = schema;
+    }
+    /**
+     * Rows that exist in both the source table (new data) and
+     * the target table (old data) will be updated, replacing
+     * the old row with the corresponding matching row.
+     *
+     * If there are multiple matches then the behavior is undefined.
+     * Currently this causes multiple copies of the row to be created
+     * but that behavior is subject to change.
+     *
+     * An optional condition may be specified.  If it is, then only
+     * matched rows that satisfy the condtion will be updated.  Any
+     * rows that do not satisfy the condition will be left as they
+     * are.  Failing to satisfy the condition does not cause a
+     * "matched row" to become a "not matched" row.
+     *
+     * The condition should be an SQL string.  Use the prefix
+     * target. to refer to rows in the target table (old data)
+     * and the prefix source. to refer to rows in the source
+     * table (new data).
+     *
+     * For example, "target.last_update < source.last_update"
+     */
+    whenMatchedUpdateAll(options) {
+        return new MergeInsertBuilder(this.#native.whenMatchedUpdateAll(options?.where), this.#schema);
+    }
+    /**
+     * Rows that exist only in the source table (new data) should
+     * be inserted into the target table.
+     */
+    whenNotMatchedInsertAll() {
+        return new MergeInsertBuilder(this.#native.whenNotMatchedInsertAll(), this.#schema);
+    }
+    /**
+     * Rows that exist only in the target table (old data) will be
+     * deleted.  An optional condition can be provided to limit what
+     * data is deleted.
+     *
+     * @param options.where - An optional condition to limit what data is deleted
+     */
+    whenNotMatchedBySourceDelete(options) {
+        return new MergeInsertBuilder(this.#native.whenNotMatchedBySourceDelete(options?.where), this.#schema);
+    }
+    /**
+     * Controls whether to use indexes for the merge operation.
+     *
+     * When set to `true` (the default), the operation will use an index if available
+     * on the join key for improved performance. When set to `false`, it forces a full
+     * table scan even if an index exists. This can be useful for benchmarking or when
+     * the query optimizer chooses a suboptimal path.
+     *
+     * @param useIndex - Whether to use indices for the merge operation. Defaults to `true`.
+     */
+    useIndex(useIndex) {
+        return new MergeInsertBuilder(this.#native.useIndex(useIndex), this.#schema);
+    }
+    /**
+     * Executes the merge insert operation
+     *
+     * @returns {Promise<MergeResult>} the merge result
+     */
+    async execute(data, execOptions) {
+        let schema;
+        if (this.#schema instanceof Promise) {
+            schema = await this.#schema;
+            this.#schema = schema; // In case of future calls
+        }
+        else {
+            schema = this.#schema;
+        }
+        if (execOptions?.timeoutMs !== undefined) {
+            this.#native.setTimeout(execOptions.timeoutMs);
+        }
+        const buffer = await (0, arrow_1.fromDataToBuffer)(data, undefined, schema);
+        return await this.#native.execute(buffer);
+    }
+}
+exports.MergeInsertBuilder = MergeInsertBuilder;