@lancedb/lancedb 0.5.1 → 0.7.0

package/dist/remote/table.js

@@ -0,0 +1,179 @@
+"use strict";
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RemoteTable = void 0;
+const table_1 = require("../table");
+const util_1 = require("../util");
+class RemoteTable extends table_1.Table {
+    #client;
+    #name;
+    // Used in the display() method
+    #dbName;
+    get #tablePrefix() {
+        return `/v1/table/${encodeURIComponent(this.#name)}/`;
+    }
+    get name() {
+        return this.#name;
+    }
+    constructor(client, tableName, dbName) {
+        super();
+        this.#client = client;
+        this.#name = tableName;
+        this.#dbName = dbName;
+    }
+    isOpen() {
+        return !this.#client.isOpen();
+    }
+    close() {
+        this.#client.close();
+    }
+    display() {
+        return `RemoteTable(${this.#dbName}; ${this.#name})`;
+    }
+    async schema() {
+        const resp = await this.#client.post(`${this.#tablePrefix}/describe/`);
+        // TODO: parse this into a valid arrow schema
+        return resp.schema;
+    }
+    async add(data, options) {
+        const { buf, mode } = await table_1.Table.parseTableData(data, options, true);
+        await this.#client.post(`${this.#tablePrefix}/insert/`, buf, {
+            params: {
+                mode,
+            },
+            headers: {
+                "Content-Type": "application/vnd.apache.arrow.stream",
+            },
+        });
+    }
+    async update(optsOrUpdates, options) {
+        const isValues = "values" in optsOrUpdates && typeof optsOrUpdates.values !== "string";
+        const isValuesSql = "valuesSql" in optsOrUpdates &&
+            typeof optsOrUpdates.valuesSql !== "string";
+        const isMap = (obj) => {
+            return obj instanceof Map;
+        };
+        let predicate;
+        let columns;
+        switch (true) {
+            case isMap(optsOrUpdates):
+                columns = Array.from(optsOrUpdates.entries());
+                predicate = options?.where;
+                break;
+            case isValues && isMap(optsOrUpdates.values):
+                columns = Array.from(optsOrUpdates.values.entries()).map(([k, v]) => [
+                    k,
+                    (0, util_1.toSQL)(v),
+                ]);
+                predicate = optsOrUpdates.where;
+                break;
+            case isValues && !isMap(optsOrUpdates.values):
+                columns = Object.entries(optsOrUpdates.values).map(([k, v]) => [
+                    k,
+                    (0, util_1.toSQL)(v),
+                ]);
+                predicate = optsOrUpdates.where;
+                break;
+            case isValuesSql && isMap(optsOrUpdates.valuesSql):
+                columns = Array.from(optsOrUpdates.valuesSql.entries());
+                predicate = optsOrUpdates.where;
+                break;
+            case isValuesSql && !isMap(optsOrUpdates.valuesSql):
+                columns = Object.entries(optsOrUpdates.valuesSql).map(([k, v]) => [
+                    k,
+                    v,
+                ]);
+                predicate = optsOrUpdates.where;
+                break;
+            default:
+                columns = Object.entries(optsOrUpdates);
+                predicate = options?.where;
+        }
+        await this.#client.post(`${this.#tablePrefix}/update/`, {
+            predicate: predicate ?? null,
+            updates: columns,
+        });
+    }
+    async countRows(filter) {
+        const payload = { predicate: filter };
+        return await this.#client.post(`${this.#tablePrefix}/count_rows/`, payload);
+    }
+    async delete(predicate) {
+        const payload = { predicate };
+        await this.#client.post(`${this.#tablePrefix}/delete/`, payload);
+    }
+    async createIndex(column, options) {
+        if (options !== undefined) {
+            console.warn("options are not yet supported on the LanceDB cloud");
+        }
+        const indexType = "vector";
+        const metric = "L2";
+        const data = {
+            column,
+            // biome-ignore lint/style/useNamingConvention: external API
+            index_type: indexType,
+            // biome-ignore lint/style/useNamingConvention: external API
+            metric_type: metric,
+        };
+        await this.#client.post(`${this.#tablePrefix}/create_index`, data);
+    }
+    query() {
+        throw new Error("query() is not yet supported on the LanceDB cloud");
+    }
+    search(_query) {
+        throw new Error("search() is not yet supported on the LanceDB cloud");
+    }
+    vectorSearch(_vector) {
+        throw new Error("vectorSearch() is not yet supported on the LanceDB cloud");
+    }
+    addColumns(_newColumnTransforms) {
+        throw new Error("addColumns() is not yet supported on the LanceDB cloud");
+    }
+    alterColumns(_columnAlterations) {
+        throw new Error("alterColumns() is not yet supported on the LanceDB cloud");
+    }
+    dropColumns(_columnNames) {
+        throw new Error("dropColumns() is not yet supported on the LanceDB cloud");
+    }
+    async version() {
+        const resp = await this.#client.post(`${this.#tablePrefix}/describe/`);
+        return resp.version;
+    }
+    checkout(_version) {
+        throw new Error("checkout() is not yet supported on the LanceDB cloud");
+    }
+    checkoutLatest() {
+        throw new Error("checkoutLatest() is not yet supported on the LanceDB cloud");
+    }
+    restore() {
+        throw new Error("restore() is not yet supported on the LanceDB cloud");
+    }
+    optimize(_options) {
+        throw new Error("optimize() is not yet supported on the LanceDB cloud");
+    }
+    async listIndices() {
+        return await this.#client.post(`${this.#tablePrefix}/index/list/`);
+    }
+    toArrow() {
+        throw new Error("toArrow() is not yet supported on the LanceDB cloud");
+    }
+    mergeInsert(_on) {
+        throw new Error("mergeInsert() is not yet supported on the LanceDB cloud");
+    }
+    async indexStats(_name) {
+        throw new Error("indexStats() is not yet supported on the LanceDB cloud");
+    }
+}
+exports.RemoteTable = RemoteTable;

package/dist/sanitize.d.ts

@@ -1,5 +1,5 @@
 import type { TKeys } from "apache-arrow/type";
-import { DataType, Date_, Decimal, DenseUnion, Dictionary, Duration, Field, FixedSizeBinary, FixedSizeList, Float, Int, Interval, List, Map_, Schema, SparseUnion, Struct, Time, Timestamp, TimestampMicrosecond, TimestampMillisecond, TimestampNanosecond, TimestampSecond, Type, Union } from "./arrow";
+import { DataType, Date_, Decimal, DenseUnion, Dictionary, Duration, Field, FixedSizeBinary, FixedSizeList, Float, Int, Interval, List, Map_, Schema, SchemaLike, SparseUnion, Struct, Table, TableLike, Time, Timestamp, TimestampMicrosecond, TimestampMillisecond, TimestampNanosecond, TimestampSecond, Type, Union } from "./arrow";
 export declare function sanitizeMetadata(metadataLike?: unknown): Map<string, string> | undefined;
 export declare function sanitizeInt(typeLike: object): Int<Type.Int | Type.Int8 | Type.Int16 | Type.Int32 | Type.Int64 | Type.Uint8 | Type.Uint16 | Type.Uint32 | Type.Uint64>;
 export declare function sanitizeFloat(typeLike: object): Float<Type.Float | Type.Float16 | Type.Float32 | Type.Float64>;
@@ -27,4 +27,5 @@ export declare function sanitizeField(fieldLike: unknown): Field;
  * instance because they might be using a different instance of apache-arrow
  * than lancedb is using.
  */
-export declare function sanitizeSchema(schemaLike: unknown): Schema;
+export declare function sanitizeSchema(schemaLike: SchemaLike): Schema;
+export declare function sanitizeTable(tableLike: TableLike): Table;

package/dist/sanitize.js

@@ -13,7 +13,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.sanitizeSchema = exports.sanitizeField = exports.sanitizeType = exports.sanitizeDictionary = exports.sanitizeDuration = exports.sanitizeMap = exports.sanitizeFixedSizeList = exports.sanitizeFixedSizeBinary = exports.sanitizeTypedUnion = exports.sanitizeUnion = exports.sanitizeStruct = exports.sanitizeList = exports.sanitizeInterval = exports.sanitizeTypedTimestamp = exports.sanitizeTimestamp = exports.sanitizeTime = exports.sanitizeDate = exports.sanitizeDecimal = exports.sanitizeFloat = exports.sanitizeInt = exports.sanitizeMetadata = void 0;
+exports.sanitizeTable = exports.sanitizeSchema = exports.sanitizeField = exports.sanitizeType = exports.sanitizeDictionary = exports.sanitizeDuration = exports.sanitizeMap = exports.sanitizeFixedSizeList = exports.sanitizeFixedSizeBinary = exports.sanitizeTypedUnion = exports.sanitizeUnion = exports.sanitizeStruct = exports.sanitizeList = exports.sanitizeInterval = exports.sanitizeTypedTimestamp = exports.sanitizeTimestamp = exports.sanitizeTime = exports.sanitizeDate = exports.sanitizeDecimal = exports.sanitizeFloat = exports.sanitizeInt = exports.sanitizeMetadata = void 0;
+// The utilities in this file help sanitize data from the user's arrow
+// library into the types expected by vectordb's arrow library.  Node
+// generally allows for mulitple versions of the same library (and sometimes
+// even multiple copies of the same version) to be installed at the same
+// time.  However, arrow-js uses instanceof which expected that the input
+// comes from the exact same library instance.  This is not always the case
+// and so we must sanitize the input to ensure that it is compatible.
+const apache_arrow_1 = require("apache-arrow");
 const arrow_1 = require("./arrow");
 function sanitizeMetadata(metadataLike) {
     if (metadataLike === undefined || metadataLike === null) {
@@ -380,3 +388,49 @@ function sanitizeSchema(schemaLike) {
     return new arrow_1.Schema(sanitizedFields, metadata);
 }
 exports.sanitizeSchema = sanitizeSchema;
+function sanitizeTable(tableLike) {
+    if (tableLike instanceof arrow_1.Table) {
+        return tableLike;
+    }
+    if (typeof tableLike !== "object" || tableLike === null) {
+        throw Error("Expected a Table but object was null/undefined");
+    }
+    if (!("schema" in tableLike)) {
+        throw Error("The table passed in does not appear to be a table (no 'schema' property)");
+    }
+    if (!("batches" in tableLike)) {
+        throw Error("The table passed in does not appear to be a table (no 'columns' property)");
+    }
+    const schema = sanitizeSchema(tableLike.schema);
+    const batches = tableLike.batches.map(sanitizeRecordBatch);
+    return new arrow_1.Table(schema, batches);
+}
+exports.sanitizeTable = sanitizeTable;
+function sanitizeRecordBatch(batchLike) {
+    if (batchLike instanceof arrow_1.RecordBatch) {
+        return batchLike;
+    }
+    if (typeof batchLike !== "object" || batchLike === null) {
+        throw Error("Expected a RecordBatch but object was null/undefined");
+    }
+    if (!("schema" in batchLike)) {
+        throw Error("The record batch passed in does not appear to be a record batch (no 'schema' property)");
+    }
+    if (!("data" in batchLike)) {
+        throw Error("The record batch passed in does not appear to be a record batch (no 'data' property)");
+    }
+    const schema = sanitizeSchema(batchLike.schema);
+    const data = sanitizeData(batchLike.data);
+    return new arrow_1.RecordBatch(schema, data);
+}
+function sanitizeData(dataLike) {
+    if (dataLike instanceof apache_arrow_1.Data) {
+        return dataLike;
+    }
+    return new apache_arrow_1.Data(dataLike.type, dataLike.offset, dataLike.length, dataLike.nullCount, {
+        [apache_arrow_1.BufferType.OFFSET]: dataLike.valueOffsets,
+        [apache_arrow_1.BufferType.DATA]: dataLike.values,
+        [apache_arrow_1.BufferType.VALIDITY]: dataLike.nullBitmap,
+        [apache_arrow_1.BufferType.TYPE]: dataLike.typeIds,
+    });
+}

package/dist/table.d.ts

@@ -1,7 +1,11 @@
-import { Data, Schema } from "./arrow";
+/// <reference types="node" />
+import { Table as ArrowTable, Data, IntoVector, Schema, TableLike } from "./arrow";
+import { CreateTableOptions } from "./connection";
 import { IndexOptions } from "./indices";
-import { AddColumnsSql, ColumnAlteration, IndexConfig, OptimizeStats, Table as _NativeTable } from "./native";
+import { MergeInsertBuilder } from "./merge";
+import { AddColumnsSql, ColumnAlteration, IndexConfig, IndexStatistics, OptimizeStats, Table as _NativeTable } from "./native";
 import { Query, VectorQuery } from "./query";
+import { IntoSql } from "./util";
 export { IndexConfig } from "./native";
 /**
  * Options for adding data to a table.
@@ -55,12 +59,11 @@ export interface OptimizeOptions {
  * Closing a table is optional.  It not closed, it will be closed when it is garbage
  * collected.
  */
-export declare class Table {
-    private readonly inner;
-    /** Construct a Table. Internal use only. */
-    constructor(inner: _NativeTable);
+export declare abstract class Table {
+    /** Returns the name of the table */
+    abstract get name(): string;
     /** Return true if the table has not been closed */
-    isOpen(): boolean;
+    abstract isOpen(): boolean;
     /**
      * Close the table, releasing any underlying resources.
      *
@@ -68,16 +71,40 @@ export declare class Table {
      *
      * Any attempt to use the table after it is closed will result in an error.
      */
-    close(): void;
+    abstract close(): void;
     /** Return a brief description of the table */
-    display(): string;
+    abstract display(): string;
     /** Get the schema of the table. */
-    schema(): Promise<Schema>;
+    abstract schema(): Promise<Schema>;
     /**
      * Insert records into this Table.
      * @param {Data} data Records to be inserted into the Table
      */
-    add(data: Data, options?: Partial<AddDataOptions>): Promise<void>;
+    abstract add(data: Data, options?: Partial<AddDataOptions>): Promise<void>;
+    /**
+     * Update existing records in the Table
+     * @param opts.values The values to update. The keys are the column names and the values
+     * are the values to set.
+     * @example
+     * ```ts
+     * table.update({where:"x = 2", values:{"vector": [10, 10]}})
+     * ```
+     */
+    abstract update(opts: {
+        values: Map<string, IntoSql> | Record<string, IntoSql>;
+    } & Partial<UpdateOptions>): Promise<void>;
+    /**
+     * Update existing records in the Table
+     * @param opts.valuesSql The values to update. The keys are the column names and the values
+     * are the values to set. The values are SQL expressions.
+     * @example
+     * ```ts
+     * table.update({where:"x = 2", valuesSql:{"x": "x + 1"}})
+     * ```
+     */
+    abstract update(opts: {
+        valuesSql: Map<string, string> | Record<string, string>;
+    } & Partial<UpdateOptions>): Promise<void>;
     /**
      * Update existing records in the Table
      *
@@ -103,11 +130,11 @@ export declare class Table {
      * @param {Partial<UpdateOptions>} options - additional options to control
      * the update behavior
      */
-    update(updates: Map<string, string> | Record<string, string>, options?: Partial<UpdateOptions>): Promise<void>;
+    abstract update(updates: Map<string, string> | Record<string, string>, options?: Partial<UpdateOptions>): Promise<void>;
     /** Count the total number of rows in the dataset. */
-    countRows(filter?: string): Promise<number>;
+    abstract countRows(filter?: string): Promise<number>;
     /** Delete the rows that satisfy the predicate. */
-    delete(predicate: string): Promise<void>;
+    abstract delete(predicate: string): Promise<void>;
     /**
      * Create an index to speed up queries.
      *
@@ -115,6 +142,9 @@ export declare class Table {
      * Indices on vector columns will speed up vector searches.
      * Indices on scalar columns will speed up filtering (in both
      * vector and non-vector searches)
+     *
+     * @note We currently don't support custom named indexes,
+     * The index name will always be `${column}_idx`
      * @example
      * // If the column has a vector (fixed size list) data type then
      * // an IvfPq vector index will be created.
@@ -134,7 +164,7 @@ export declare class Table {
      * // Or create a Scalar index
      * await table.createIndex("my_float_col");
      */
-    createIndex(column: string, options?: Partial<IndexOptions>): Promise<void>;
+    abstract createIndex(column: string, options?: Partial<IndexOptions>): Promise<void>;
     /**
      * Create a {@link Query} Builder.
      *
@@ -185,7 +215,20 @@ export declare class Table {
      * }
      * @returns {Query} A builder that can be used to parameterize the query
      */
-    query(): Query;
+    abstract query(): Query;
+    /**
+     * Create a search query to find the nearest neighbors
+     * of the given query vector
+     * @param {string} query - the query. This will be converted to a vector using the table's provided embedding function
+     * @note  If no embedding functions are defined in the table, this will error when collecting the results.
+     */
+    abstract search(query: string): VectorQuery;
+    /**
+     * Create a search query to find the nearest neighbors
+     * of the given query vector
+     * @param {IntoVector} query - the query vector
+     */
+    abstract search(query: IntoVector): VectorQuery;
     /**
      * Search the table with a given query vector.
      *
@@ -193,7 +236,7 @@ export declare class Table {
      * is the same thing as calling `nearestTo` on the builder returned
      * by `query`.  @see {@link Query#nearestTo} for more details.
      */
-    vectorSearch(vector: unknown): VectorQuery;
+    abstract vectorSearch(vector: IntoVector): VectorQuery;
     /**
      * Add new columns with defined values.
      * @param {AddColumnsSql[]} newColumnTransforms pairs of column names and
@@ -201,13 +244,13 @@ export declare class Table {
      * expressions will be evaluated for each row in the table, and can
      * reference existing columns in the table.
      */
-    addColumns(newColumnTransforms: AddColumnsSql[]): Promise<void>;
+    abstract addColumns(newColumnTransforms: AddColumnsSql[]): Promise<void>;
     /**
      * Alter the name or nullability of columns.
      * @param {ColumnAlteration[]} columnAlterations One or more alterations to
      * apply to columns.
      */
-    alterColumns(columnAlterations: ColumnAlteration[]): Promise<void>;
+    abstract alterColumns(columnAlterations: ColumnAlteration[]): Promise<void>;
     /**
      * Drop one or more columns from the dataset
      *
@@ -219,9 +262,9 @@ export declare class Table {
      * be nested column references (e.g. "a.b.c") or top-level column names
      * (e.g. "a").
      */
-    dropColumns(columnNames: string[]): Promise<void>;
+    abstract dropColumns(columnNames: string[]): Promise<void>;
     /** Retrieve the version of the table */
-    version(): Promise<number>;
+    abstract version(): Promise<number>;
     /**
      * Checks out a specific version of the table _This is an in-place operation._
      *
@@ -247,14 +290,14 @@ export declare class Table {
      * console.log(await table.version()); // 2
      * ```
      */
-    checkout(version: number): Promise<void>;
+    abstract checkout(version: number): Promise<void>;
     /**
      * Checkout the latest version of the table. _This is an in-place operation._
      *
      * The table will be set back into standard mode, and will track the latest
      * version of the table.
      */
-    checkoutLatest(): Promise<void>;
+    abstract checkoutLatest(): Promise<void>;
     /**
      * Restore the table to the currently checked out version
      *
@@ -267,7 +310,7 @@ export declare class Table {
      * Once the operation concludes the table will no longer be in a checked
      * out state and the read_consistency_interval, if any, will apply.
      */
-    restore(): Promise<void>;
+    abstract restore(): Promise<void>;
     /**
      * Optimize the on-disk data and indices for better performance.
      *
@@ -298,7 +341,55 @@ export declare class Table {
      *  you have added or modified 100,000 or more records or run more than 20 data
      *  modification operations.
      */
-    optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
+    abstract optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
     /** List all indices that have been created with {@link Table.createIndex} */
+    abstract listIndices(): Promise<IndexConfig[]>;
+    /** Return the table as an arrow table */
+    abstract toArrow(): Promise<ArrowTable>;
+    abstract mergeInsert(on: string | string[]): MergeInsertBuilder;
+    /** List all the stats of a specified index
+     *
+     * @param {string} name The name of the index.
+     * @returns {IndexStatistics | undefined} The stats of the index. If the index does not exist, it will return undefined
+     */
+    abstract indexStats(name: string): Promise<IndexStatistics | undefined>;
+    static parseTableData(data: Record<string, unknown>[] | TableLike, options?: Partial<CreateTableOptions>, streaming?: boolean): Promise<{
+        buf: Buffer;
+        mode: string;
+    }>;
+}
+export declare class LocalTable extends Table {
+    private readonly inner;
+    constructor(inner: _NativeTable);
+    get name(): string;
+    isOpen(): boolean;
+    close(): void;
+    display(): string;
+    private getEmbeddingFunctions;
+    /** Get the schema of the table. */
+    schema(): Promise<Schema>;
+    add(data: Data, options?: Partial<AddDataOptions>): Promise<void>;
+    update(optsOrUpdates: (Map<string, string> | Record<string, string>) | ({
+        values: Map<string, IntoSql> | Record<string, IntoSql>;
+    } & Partial<UpdateOptions>) | ({
+        valuesSql: Map<string, string> | Record<string, string>;
+    } & Partial<UpdateOptions>), options?: Partial<UpdateOptions>): Promise<void>;
+    countRows(filter?: string): Promise<number>;
+    delete(predicate: string): Promise<void>;
+    createIndex(column: string, options?: Partial<IndexOptions>): Promise<void>;
+    query(): Query;
+    search(query: string | IntoVector): VectorQuery;
+    vectorSearch(vector: IntoVector): VectorQuery;
+    addColumns(newColumnTransforms: AddColumnsSql[]): Promise<void>;
+    alterColumns(columnAlterations: ColumnAlteration[]): Promise<void>;
+    dropColumns(columnNames: string[]): Promise<void>;
+    version(): Promise<number>;
+    checkout(version: number): Promise<void>;
+    checkoutLatest(): Promise<void>;
+    restore(): Promise<void>;
+    optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
     listIndices(): Promise<IndexConfig[]>;
+    toArrow(): Promise<ArrowTable>;
+    indexStats(name: string): Promise<IndexStatistics | undefined>;
+    mergeInsert(on: string | string[]): MergeInsertBuilder;
 }