@dengxifeng/lancedb 0.26.2

package/dist/table.d.ts

@@ -0,0 +1,581 @@
+import { Table as ArrowTable, Data, DataType, IntoVector, MultiVector, Schema } from "./arrow";
+import { IndexOptions } from "./indices";
+import { MergeInsertBuilder } from "./merge";
+import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, DeleteResult, DropColumnsResult, IndexConfig, IndexStatistics, OptimizeStats, TableStatistics, Tags, UpdateResult, Table as _NativeTable } from "./native";
+import { FullTextQuery, Query, TakeQuery, VectorQuery } from "./query";
+import { IntoSql } from "./util";
+export { IndexConfig } from "./native";
+/**
+ * Options for adding data to a table.
+ */
+export interface AddDataOptions {
+    /**
+     * If "append" (the default) then the new data will be added to the table
+     *
+     * If "overwrite" then the new data will replace the existing data in the table.
+     */
+    mode: "append" | "overwrite";
+}
+export interface UpdateOptions {
+    /**
+     * A filter that limits the scope of the update.
+     *
+     * This should be an SQL filter expression.
+     *
+     * Only rows that satisfy the expression will be updated.
+     *
+     * For example, this could be 'my_col == 0' to replace all instances
+     * of 0 in a column with some other default value.
+     */
+    where: string;
+}
+export interface OptimizeOptions {
+    /**
+     * If set then all versions older than the given date
+     * be removed.  The current version will never be removed.
+     * The default is 7 days
+     * @example
+     * // Delete all versions older than 1 day
+     * const olderThan = new Date();
+     * olderThan.setDate(olderThan.getDate() - 1));
+     * tbl.optimize({cleanupOlderThan: olderThan});
+     *
+     * // Delete all versions except the current version
+     * tbl.optimize({cleanupOlderThan: new Date()});
+     */
+    cleanupOlderThan: Date;
+    deleteUnverified: boolean;
+}
+export interface Version {
+    version: number;
+    timestamp: Date;
+    metadata: Record<string, string>;
+}
+/**
+ * A Table is a collection of Records in a LanceDB Database.
+ *
+ * A Table object is expected to be long lived and reused for multiple operations.
+ * Table objects will cache a certain amount of index data in memory.  This cache
+ * will be freed when the Table is garbage collected.  To eagerly free the cache you
+ * can call the `close` method.  Once the Table is closed, it cannot be used for any
+ * further operations.
+ *
+ * Tables are created using the methods {@link Connection#createTable}
+ * and {@link Connection#createEmptyTable}. Existing tables are opened
+ * using {@link Connection#openTable}.
+ *
+ * Closing a table is optional.  It not closed, it will be closed when it is garbage
+ * collected.
+ *
+ * @hideconstructor
+ */
+export declare abstract class Table {
+    /** Returns the name of the table */
+    abstract get name(): string;
+    /** Return true if the table has not been closed */
+    abstract isOpen(): boolean;
+    /**
+     * Close the table, releasing any underlying resources.
+     *
+     * It is safe to call this method multiple times.
+     *
+     * Any attempt to use the table after it is closed will result in an error.
+     */
+    abstract close(): void;
+    /** Return a brief description of the table */
+    abstract display(): string;
+    /** Get the schema of the table. */
+    abstract schema(): Promise<Schema>;
+    /**
+     * Insert records into this Table.
+     * @param {Data} data Records to be inserted into the Table
+     * @returns {Promise<AddResult>} A promise that resolves to an object
+     * containing the new version number of the table
+     */
+    abstract add(data: Data, options?: Partial<AddDataOptions>): Promise<AddResult>;
+    /**
+     * Update existing records in the Table
+     * @param opts.values The values to update. The keys are the column names and the values
+     * are the values to set.
+     * @returns {Promise<UpdateResult>} A promise that resolves to an object containing
+     * the number of rows updated and the new version number
+     * @example
+     * ```ts
+     * table.update({where:"x = 2", values:{"vector": [10, 10]}})
+     * ```
+     */
+    abstract update(opts: {
+        values: Map<string, IntoSql> | Record<string, IntoSql>;
+    } & Partial<UpdateOptions>): Promise<UpdateResult>;
+    /**
+     * Update existing records in the Table
+     * @param opts.valuesSql The values to update. The keys are the column names and the values
+     * are the values to set. The values are SQL expressions.
+     * @returns {Promise<UpdateResult>} A promise that resolves to an object containing
+     * the number of rows updated and the new version number
+     * @example
+     * ```ts
+     * table.update({where:"x = 2", valuesSql:{"x": "x + 1"}})
+     * ```
+     */
+    abstract update(opts: {
+        valuesSql: Map<string, string> | Record<string, string>;
+    } & Partial<UpdateOptions>): Promise<UpdateResult>;
+    /**
+     * Update existing records in the Table
+     *
+     * An update operation can be used to adjust existing values.  Use the
+     * returned builder to specify which columns to update.  The new value
+     * can be a literal value (e.g. replacing nulls with some default value)
+     * or an expression applied to the old value (e.g. incrementing a value)
+     *
+     * An optional condition can be specified (e.g. "only update if the old
+     * value is 0")
+     *
+     * Note: if your condition is something like "some_id_column == 7" and
+     * you are updating many rows (with different ids) then you will get
+     * better performance with a single [`merge_insert`] call instead of
+     * repeatedly calilng this method.
+     * @param {Map<string, string> | Record<string, string>} updates - the
+     * columns to update
+     * @returns {Promise<UpdateResult>} A promise that resolves to an object
+     * containing the number of rows updated and the new version number
+     *
+     * Keys in the map should specify the name of the column to update.
+     * Values in the map provide the new value of the column.  These can
+     * be SQL literal strings (e.g. "7" or "'foo'") or they can be expressions
+     * based on the row being updated (e.g. "my_col + 1")
+     * @param {Partial<UpdateOptions>} options - additional options to control
+     * the update behavior
+     */
+    abstract update(updates: Map<string, string> | Record<string, string>, options?: Partial<UpdateOptions>): Promise<UpdateResult>;
+    /** Count the total number of rows in the dataset. */
+    abstract countRows(filter?: string): Promise<number>;
+    /**
+     * Delete the rows that satisfy the predicate.
+     * @returns {Promise<DeleteResult>} A promise that resolves to an object
+     * containing the new version number of the table
+     */
+    abstract delete(predicate: string): Promise<DeleteResult>;
+    /**
+     * Create an index to speed up queries.
+     *
+     * Indices can be created on vector columns or scalar columns.
+     * Indices on vector columns will speed up vector searches.
+     * Indices on scalar columns will speed up filtering (in both
+     * vector and non-vector searches)
+     *
+     * We currently don't support custom named indexes.
+     * The index name will always be `${column}_idx`.
+     *
+     * @example
+     * // If the column has a vector (fixed size list) data type then
+     * // an IvfPq vector index will be created.
+     * const table = await conn.openTable("my_table");
+     * await table.createIndex("vector");
+     * @example
+     * // For advanced control over vector index creation you can specify
+     * // the index type and options.
+     * const table = await conn.openTable("my_table");
+     * await table.createIndex("vector", {
+     *   config: lancedb.Index.ivfPq({
+     *     numPartitions: 128,
+     *     numSubVectors: 16,
+     *   }),
+     * });
+     * @example
+     * // Or create a Scalar index
+     * await table.createIndex("my_float_col");
+     */
+    abstract createIndex(column: string, options?: Partial<IndexOptions>): Promise<void>;
+    /**
+     * Drop an index from the table.
+     *
+     * @param name The name of the index.
+     *
+     * This does not delete the index from disk, it just removes it from the table.
+     * To delete the index, run {@link Table#optimize} after dropping the index.
+     *
+     * Use {@link Table.listIndices} to find the names of the indices.
+     */
+    abstract dropIndex(name: string): Promise<void>;
+    /**
+     * Prewarm an index in the table.
+     *
+     * @param name The name of the index.
+     *
+     * This will load the index into memory.  This may reduce the cold-start time for
+     * future queries.  If the index does not fit in the cache then this call may be
+     * wasteful.
+     */
+    abstract prewarmIndex(name: string): Promise<void>;
+    /**
+     * Waits for asynchronous indexing to complete on the table.
+     *
+     * @param indexNames The name of the indices to wait for
+     * @param timeoutSeconds The number of seconds to wait before timing out
+     *
+     * This will raise an error if the indices are not created and fully indexed within the timeout.
+     */
+    abstract waitForIndex(indexNames: string[], timeoutSeconds: number): Promise<void>;
+    /**
+     * Create a {@link Query} Builder.
+     *
+     * Queries allow you to search your existing data.  By default the query will
+     * return all the data in the table in no particular order.  The builder
+     * returned by this method can be used to control the query using filtering,
+     * vector similarity, sorting, and more.
+     *
+     * Note: By default, all columns are returned.  For best performance, you should
+     * only fetch the columns you need.
+     *
+     * When appropriate, various indices and statistics based pruning will be used to
+     * accelerate the query.
+     * @example
+     * // SQL-style filtering
+     * //
+     * // This query will return up to 1000 rows whose value in the `id` column
+     * // is greater than 5. LanceDb supports a broad set of filtering functions.
+     * for await (const batch of table
+     *   .query()
+     *   .where("id > 1")
+     *   .select(["id"])
+     *   .limit(20)) {
+     *   console.log(batch);
+     * }
+     * @example
+     * // Vector Similarity Search
+     * //
+     * // This example will find the 10 rows whose value in the "vector" column are
+     * // closest to the query vector [1.0, 2.0, 3.0].  If an index has been created
+     * // on the "vector" column then this will perform an ANN search.
+     * //
+     * // The `refineFactor` and `nprobes` methods are used to control the recall /
+     * // latency tradeoff of the search.
+     * for await (const batch of table
+     *   .query()
+     *   .where("id > 1")
+     *   .select(["id"])
+     *   .limit(20)) {
+     *   console.log(batch);
+     * }
+     * @example
+     * // Scan the full dataset
+     * //
+     * // This query will return everything in the table in no particular order.
+     * for await (const batch of table.query()) {
+     *   console.log(batch);
+     * }
+     * @returns {Query} A builder that can be used to parameterize the query
+     */
+    abstract query(): Query;
+    /**
+     * Create a query that returns a subset of the rows in the table.
+     * @param offsets The offsets of the rows to return.
+     * @returns A builder that can be used to parameterize the query.
+     */
+    abstract takeOffsets(offsets: number[]): TakeQuery;
+    /**
+     * Create a query that returns a subset of the rows in the table.
+     * @param rowIds The row ids of the rows to return.
+     *
+     * Row ids returned by `withRowId()` are `bigint`, so `bigint[]` is supported.
+     * For convenience / backwards compatibility, `number[]` is also accepted (for
+     * small row ids that fit in a safe integer).
+     * @returns A builder that can be used to parameterize the query.
+     */
+    abstract takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery;
+    /**
+     * Create a search query to find the nearest neighbors
+     * of the given query
+     * @param {string | IntoVector} query - the query, a vector or string
+     * @param {string} queryType - the type of the query, "vector", "fts", or "auto"
+     * @param {string | string[]} ftsColumns - the columns to search in for full text search
+     *    for now, only one column can be searched at a time.
+     *
+     * when "auto" is used, if the query is a string and an embedding function is defined, it will be treated as a vector query
+     * if the query is a string and no embedding function is defined, it will be treated as a full text search query
+     */
+    abstract search(query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query;
+    /**
+     * Search the table with a given query vector.
+     *
+     * This is a convenience method for preparing a vector query and
+     * is the same thing as calling `nearestTo` on the builder returned
+     * by `query`.  @see {@link Query#nearestTo} for more details.
+     */
+    abstract vectorSearch(vector: IntoVector | MultiVector): VectorQuery;
+    /**
+     * Add new columns with defined values.
+     * @param {AddColumnsSql[]} newColumnTransforms pairs of column names and
+     * the SQL expression to use to calculate the value of the new column. These
+     * expressions will be evaluated for each row in the table, and can
+     * reference existing columns in the table.
+     * @returns {Promise<AddColumnsResult>} A promise that resolves to an object
+     * containing the new version number of the table after adding the columns.
+     */
+    abstract addColumns(newColumnTransforms: AddColumnsSql[]): Promise<AddColumnsResult>;
+    /**
+     * Alter the name or nullability of columns.
+     * @param {ColumnAlteration[]} columnAlterations One or more alterations to
+     * apply to columns.
+     * @returns {Promise<AlterColumnsResult>} A promise that resolves to an object
+     * containing the new version number of the table after altering the columns.
+     */
+    abstract alterColumns(columnAlterations: ColumnAlteration[]): Promise<AlterColumnsResult>;
+    /**
+     * Drop one or more columns from the dataset
+     *
+     * This is a metadata-only operation and does not remove the data from the
+     * underlying storage. In order to remove the data, you must subsequently
+     * call ``compact_files`` to rewrite the data without the removed columns and
+     * then call ``cleanup_files`` to remove the old files.
+     * @param {string[]} columnNames The names of the columns to drop. These can
+     * be nested column references (e.g. "a.b.c") or top-level column names
+     * (e.g. "a").
+     * @returns {Promise<DropColumnsResult>} A promise that resolves to an object
+     * containing the new version number of the table after dropping the columns.
+     */
+    abstract dropColumns(columnNames: string[]): Promise<DropColumnsResult>;
+    /** Retrieve the version of the table */
+    abstract version(): Promise<number>;
+    /**
+     * Checks out a specific version of the table _This is an in-place operation._
+     *
+     * This allows viewing previous versions of the table. If you wish to
+     * keep writing to the dataset starting from an old version, then use
+     * the `restore` function.
+     *
+     * Calling this method will set the table into time-travel mode. If you
+     * wish to return to standard mode, call `checkoutLatest`.
+     * @param {number | string} version The version to checkout, could be version number or tag
+     * @example
+     * ```typescript
+     * import * as lancedb from "@lancedb/lancedb"
+     * const db = await lancedb.connect("./.lancedb");
+     * const table = await db.createTable("my_table", [
+     *   { vector: [1.1, 0.9], type: "vector" },
+     * ]);
+     *
+     * console.log(await table.version()); // 1
+     * console.log(table.display());
+     * await table.add([{ vector: [0.5, 0.2], type: "vector" }]);
+     * await table.checkout(1);
+     * console.log(await table.version()); // 2
+     * ```
+     */
+    abstract checkout(version: number | string): Promise<void>;
+    /**
+     * Checkout the latest version of the table. _This is an in-place operation._
+     *
+     * The table will be set back into standard mode, and will track the latest
+     * version of the table.
+     */
+    abstract checkoutLatest(): Promise<void>;
+    /**
+     * List all the versions of the table
+     */
+    abstract listVersions(): Promise<Version[]>;
+    /**
+     * Get a tags manager for this table.
+     *
+     * Tags allow you to label specific versions of a table with a human-readable name.
+     * The returned tags manager can be used to list, create, update, or delete tags.
+     *
+     * @returns {Tags} A tags manager for this table
+     * @example
+     * ```typescript
+     * const tagsManager = await table.tags();
+     * await tagsManager.create("v1", 1);
+     * const tags = await tagsManager.list();
+     * console.log(tags); // { "v1": { version: 1, manifestSize: ... } }
+     * ```
+     */
+    abstract tags(): Promise<Tags>;
+    /**
+     * Restore the table to the currently checked out version
+     *
+     * This operation will fail if checkout has not been called previously
+     *
+     * This operation will overwrite the latest version of the table with a
+     * previous version.  Any changes made since the checked out version will
+     * no longer be visible.
+     *
+     * Once the operation concludes the table will no longer be in a checked
+     * out state and the read_consistency_interval, if any, will apply.
+     */
+    abstract restore(): Promise<void>;
+    /**
+     * Optimize the on-disk data and indices for better performance.
+     *
+     * Modeled after ``VACUUM`` in PostgreSQL.
+     *
+     *  Optimization covers three operations:
+     *
+     *  - Compaction: Merges small files into larger ones
+     *  - Prune: Removes old versions of the dataset
+     *  - Index: Optimizes the indices, adding new data to existing indices
+     *
+     *
+     *  Experimental API
+     *  ----------------
+     *
+     *  The optimization process is undergoing active development and may change.
+     *  Our goal with these changes is to improve the performance of optimization and
+     *  reduce the complexity.
+     *
+     *  That being said, it is essential today to run optimize if you want the best
+     *  performance.  It should be stable and safe to use in production, but it our
+     *  hope that the API may be simplified (or not even need to be called) in the
+     *  future.
+     *
+     *  The frequency an application shoudl call optimize is based on the frequency of
+     *  data modifications.  If data is frequently added, deleted, or updated then
+     *  optimize should be run frequently.  A good rule of thumb is to run optimize if
+     *  you have added or modified 100,000 or more records or run more than 20 data
+     *  modification operations.
+     */
+    abstract optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
+    /** List all indices that have been created with {@link Table.createIndex} */
+    abstract listIndices(): Promise<IndexConfig[]>;
+    /** Return the table as an arrow table */
+    abstract toArrow(): Promise<ArrowTable>;
+    abstract mergeInsert(on: string | string[]): MergeInsertBuilder;
+    /** List all the stats of a specified index
+     *
+     * @param {string} name The name of the index.
+     * @returns {IndexStatistics | undefined} The stats of the index. If the index does not exist, it will return undefined
+     *
+     * Use {@link Table.listIndices} to find the names of the indices.
+     */
+    abstract indexStats(name: string): Promise<IndexStatistics | undefined>;
+    /** Returns table and fragment statistics
+     *
+     * @returns {TableStatistics} The table and fragment statistics
+     *
+     */
+    abstract stats(): Promise<TableStatistics>;
+    /**
+     * Get the initial storage options that were passed in when opening this table.
+     *
+     * For dynamically refreshed options (e.g., credential vending), use
+     * {@link Table.latestStorageOptions}.
+     *
+     * Warning: This is an internal API and the return value is subject to change.
+     *
+     * @returns The storage options, or undefined if no storage options were configured.
+     */
+    abstract initialStorageOptions(): Promise<Record<string, string> | null | undefined>;
+    /**
+     * Get the latest storage options, refreshing from provider if configured.
+     *
+     * This method is useful for credential vending scenarios where storage options
+     * may be refreshed dynamically. If no dynamic provider is configured, this
+     * returns the initial static options.
+     *
+     * Warning: This is an internal API and the return value is subject to change.
+     *
+     * @returns The storage options, or undefined if no storage options were configured.
+     */
+    abstract latestStorageOptions(): Promise<Record<string, string> | null | undefined>;
+}
+export declare class LocalTable extends Table {
+    private readonly inner;
+    constructor(inner: _NativeTable);
+    get name(): string;
+    isOpen(): boolean;
+    close(): void;
+    display(): string;
+    private getEmbeddingFunctions;
+    /** Get the schema of the table. */
+    schema(): Promise<Schema>;
+    add(data: Data, options?: Partial<AddDataOptions>): Promise<AddResult>;
+    update(optsOrUpdates: (Map<string, string> | Record<string, string>) | ({
+        values: Map<string, IntoSql> | Record<string, IntoSql>;
+    } & Partial<UpdateOptions>) | ({
+        valuesSql: Map<string, string> | Record<string, string>;
+    } & Partial<UpdateOptions>), options?: Partial<UpdateOptions>): Promise<UpdateResult>;
+    countRows(filter?: string): Promise<number>;
+    delete(predicate: string): Promise<DeleteResult>;
+    createIndex(column: string, options?: Partial<IndexOptions>): Promise<void>;
+    dropIndex(name: string): Promise<void>;
+    prewarmIndex(name: string): Promise<void>;
+    waitForIndex(indexNames: string[], timeoutSeconds: number): Promise<void>;
+    takeOffsets(offsets: number[]): TakeQuery;
+    takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery;
+    query(): Query;
+    search(query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query;
+    vectorSearch(vector: IntoVector | MultiVector): VectorQuery;
+    addColumns(newColumnTransforms: AddColumnsSql[]): Promise<AddColumnsResult>;
+    alterColumns(columnAlterations: ColumnAlteration[]): Promise<AlterColumnsResult>;
+    dropColumns(columnNames: string[]): Promise<DropColumnsResult>;
+    version(): Promise<number>;
+    checkout(version: number | string): Promise<void>;
+    checkoutLatest(): Promise<void>;
+    listVersions(): Promise<Version[]>;
+    restore(): Promise<void>;
+    tags(): Promise<Tags>;
+    optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
+    listIndices(): Promise<IndexConfig[]>;
+    toArrow(): Promise<ArrowTable>;
+    indexStats(name: string): Promise<IndexStatistics | undefined>;
+    stats(): Promise<TableStatistics>;
+    initialStorageOptions(): Promise<Record<string, string> | null | undefined>;
+    latestStorageOptions(): Promise<Record<string, string> | null | undefined>;
+    mergeInsert(on: string | string[]): MergeInsertBuilder;
+    /**
+     * Check if the table uses the new manifest path scheme.
+     *
+     * This function will return true if the table uses the V2 manifest
+     * path scheme.
+     */
+    usesV2ManifestPaths(): Promise<boolean>;
+    /**
+     * Migrate the table to use the new manifest path scheme.
+     *
+     * This function will rename all V1 manifests to V2 manifest paths.
+     * These paths provide more efficient opening of datasets with many versions
+     * on object stores.
+     *
+     * This function is idempotent, and can be run multiple times without
+     * changing the state of the object store.
+     *
+     * However, it should not be run while other concurrent operations are happening.
+     * And it should also run until completion before resuming other operations.
+     */
+    migrateManifestPathsV2(): Promise<void>;
+}
+/**
+ *  A definition of a column alteration. The alteration changes the column at
+ * `path` to have the new name `name`, to be nullable if `nullable` is true,
+ * and to have the data type `data_type`. At least one of `rename` or `nullable`
+ * must be provided.
+ */
+export interface ColumnAlteration {
+    /**
+     * The path to the column to alter. This is a dot-separated path to the column.
+     * If it is a top-level column then it is just the name of the column. If it is
+     * a nested column then it is the path to the column, e.g. "a.b.c" for a column
+     * `c` nested inside a column `b` nested inside a column `a`.
+     */
+    path: string;
+    /**
+     * The new name of the column. If not provided then the name will not be changed.
+     * This must be distinct from the names of all other columns in the table.
+     */
+    rename?: string;
+    /**
+     * A new data type for the column. If not provided then the data type will not be changed.
+     * Changing data types is limited to casting to the same general type. For example, these
+     * changes are valid:
+     * * `int32` -> `int64` (integers)
+     * * `double` -> `float` (floats)
+     * * `string` -> `large_string` (strings)
+     * But these changes are not:
+     * * `int32` -> `double` (mix integers and floats)
+     * * `string` -> `int32` (mix strings and integers)
+     */
+    dataType?: string | DataType;
+    /** Set the new nullability. Note that a nullable column cannot be made non-nullable. */
+    nullable?: boolean;
+}