@lancedb/lancedb 0.5.1 → 0.7.0

package/dist/query.js

@@ -43,6 +43,18 @@ class RecordBatchIterator {
 }
 exports.RecordBatchIterator = RecordBatchIterator;
 /* eslint-enable */
+class RecordBatchIterable {
+    inner;
+    options;
+    constructor(inner, options) {
+        this.inner = inner;
+        this.options = options;
+    }
+    // biome-ignore lint/suspicious/noExplicitAny: skip
+    [Symbol.asyncIterator]() {
+        return new RecordBatchIterator(this.inner.execute(this.options?.maxBatchLength));
+    }
+}
 /** Common methods supported by all query types */
 class QueryBase {
     inner;
@@ -50,6 +62,18 @@ class QueryBase {
         this.inner = inner;
         // intentionally empty
     }
+    // call a function on the inner (either a promise or the actual object)
+    doCall(fn) {
+        if (this.inner instanceof Promise) {
+            this.inner = this.inner.then((inner) => {
+                fn(inner);
+                return inner;
+            });
+        }
+        else {
+            fn(this.inner);
+        }
+    }
     /**
      * A filter statement to be applied to this query.
      *
@@ -63,9 +87,17 @@ class QueryBase {
      * on the filter column(s).
      */
     where(predicate) {
-        this.inner.onlyIf(predicate);
+        this.doCall((inner) => inner.onlyIf(predicate));
         return this;
     }
+    /**
+     * A filter statement to be applied to this query.
+     * @alias where
+     * @deprecated Use `where` instead
+     */
+    filter(predicate) {
+        return this.where(predicate);
+    }
     /**
      * Return only the specified columns.
      *
@@ -98,6 +130,9 @@ class QueryBase {
      */
     select(columns) {
         let columnTuples;
+        if (typeof columns === "string") {
+            columns = [columns];
+        }
         if (Array.isArray(columns)) {
             columnTuples = columns.map((c) => [c, c]);
         }
@@ -107,7 +142,9 @@ class QueryBase {
         else {
             columnTuples = Object.entries(columns);
         }
-        this.inner.select(columnTuples);
+        this.doCall((inner) => {
+            inner.select(columnTuples);
+        });
         return this;
     }
     /**
@@ -117,11 +154,16 @@ class QueryBase {
      * called then every valid row from the table will be returned.
      */
     limit(limit) {
-        this.inner.limit(limit);
+        this.doCall((inner) => inner.limit(limit));
         return this;
     }
-    nativeExecute() {
-        return this.inner.execute();
+    nativeExecute(options) {
+        if (this.inner instanceof Promise) {
+            return this.inner.then((inner) => inner.execute(options?.maxBatchLength));
+        }
+        else {
+            return this.inner.execute(options?.maxBatchLength);
+        }
     }
     /**
      * Execute the query and return the results as an @see {@link AsyncIterator}
@@ -134,8 +176,8 @@ class QueryBase {
      * single query)
      *
      */
-    execute() {
-        return new RecordBatchIterator(this.nativeExecute());
+    execute(options) {
+        return new RecordBatchIterator(this.nativeExecute(options));
     }
     // biome-ignore lint/suspicious/noExplicitAny: skip
     [Symbol.asyncIterator]() {
@@ -143,19 +185,48 @@ class QueryBase {
         return new RecordBatchIterator(promise);
     }
     /** Collect the results as an Arrow @see {@link ArrowTable}. */
-    async toArrow() {
+    async toArrow(options) {
         const batches = [];
-        for await (const batch of this) {
+        let inner;
+        if (this.inner instanceof Promise) {
+            inner = await this.inner;
+        }
+        else {
+            inner = this.inner;
+        }
+        for await (const batch of new RecordBatchIterable(inner, options)) {
             batches.push(batch);
         }
         return new arrow_1.Table(batches);
     }
     /** Collect the results as an array of objects. */
-    async toArray() {
-        const tbl = await this.toArrow();
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+    // biome-ignore lint/suspicious/noExplicitAny: arrow.toArrow() returns any[]
+    async toArray(options) {
+        const tbl = await this.toArrow(options);
         return tbl.toArray();
     }
+    /**
+     * Generates an explanation of the query execution plan.
+     *
+     * @example
+     * import * as lancedb from "@lancedb/lancedb"
+     * const db = await lancedb.connect("./.lancedb");
+     * const table = await db.createTable("my_table", [
+     *   { vector: [1.1, 0.9], id: "1" },
+     * ]);
+     * const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
+     *
+     * @param verbose - If true, provides a more detailed explanation. Defaults to false.
+     * @returns A Promise that resolves to a string containing the query execution plan explanation.
+     */
+    async explainPlan(verbose = false) {
+        if (this.inner instanceof Promise) {
+            return this.inner.then((inner) => inner.explainPlan(verbose));
+        }
+        else {
+            return this.inner.explainPlan(verbose);
+        }
+    }
 }
 exports.QueryBase = QueryBase;
 /**
@@ -190,7 +261,7 @@ class VectorQuery extends QueryBase {
      * you the desired recall.
      */
     nprobes(nprobes) {
-        this.inner.nprobes(nprobes);
+        super.doCall((inner) => inner.nprobes(nprobes));
         return this;
     }
     /**
@@ -203,7 +274,7 @@ class VectorQuery extends QueryBase {
      * whose data type is a fixed-size-list of floats.
      */
     column(column) {
-        this.inner.column(column);
+        super.doCall((inner) => inner.column(column));
         return this;
     }
     /**
@@ -221,7 +292,7 @@ class VectorQuery extends QueryBase {
      * By default "l2" is used.
      */
     distanceType(distanceType) {
-        this.inner.distanceType(distanceType);
+        super.doCall((inner) => inner.distanceType(distanceType));
         return this;
     }
     /**
@@ -254,7 +325,7 @@ class VectorQuery extends QueryBase {
      * distance between the query vector and the actual uncompressed vector.
      */
     refineFactor(refineFactor) {
-        this.inner.refineFactor(refineFactor);
+        super.doCall((inner) => inner.refineFactor(refineFactor));
         return this;
     }
     /**
@@ -278,7 +349,7 @@ class VectorQuery extends QueryBase {
      * factor can often help restore some of the results lost by post filtering.
      */
     postfilter() {
-        this.inner.postfilter();
+        super.doCall((inner) => inner.postfilter());
         return this;
     }
     /**
@@ -291,7 +362,7 @@ class VectorQuery extends QueryBase {
      * calculate your recall to select an appropriate value for nprobes.
      */
     bypassVectorIndex() {
-        this.inner.bypassVectorIndex();
+        super.doCall((inner) => inner.bypassVectorIndex());
         return this;
     }
 }
@@ -339,9 +410,39 @@ class Query extends QueryBase {
      * a default `limit` of 10 will be used.  @see {@link Query#limit}
      */
     nearestTo(vector) {
-        // biome-ignore lint/suspicious/noExplicitAny: skip
-        const vectorQuery = this.inner.nearestTo(Float32Array.from(vector));
-        return new VectorQuery(vectorQuery);
+        if (this.inner instanceof Promise) {
+            const nativeQuery = this.inner.then(async (inner) => {
+                if (vector instanceof Promise) {
+                    const arr = await vector.then((v) => Float32Array.from(v));
+                    return inner.nearestTo(arr);
+                }
+                else {
+                    return inner.nearestTo(Float32Array.from(vector));
+                }
+            });
+            return new VectorQuery(nativeQuery);
+        }
+        if (vector instanceof Promise) {
+            const res = (async () => {
+                try {
+                    const v = await vector;
+                    const arr = Float32Array.from(v);
+                    //
+                    // biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
+                    const value = this.nearestTo(arr);
+                    const inner = value.inner;
+                    return inner;
+                }
+                catch (e) {
+                    return Promise.reject(e);
+                }
+            })();
+            return new VectorQuery(res);
+        }
+        else {
+            const vectorQuery = this.inner.nearestTo(Float32Array.from(vector));
+            return new VectorQuery(vectorQuery);
+        }
     }
 }
 exports.Query = Query;

package/dist/remote/client.d.ts

@@ -0,0 +1,28 @@
+/// <reference types="node" />
+import { type AxiosResponse } from "axios";
+import { Table as ArrowTable } from "../arrow";
+import { VectorQuery } from "../query";
+export declare class RestfulLanceDBClient {
+    #private;
+    constructor(dbName: string, apiKey: string, region: string, hostOverride?: string, connectionTimeout?: number, readTimeout?: number);
+    get session(): import("axios").AxiosInstance;
+    get url(): string;
+    get headers(): {
+        [key: string]: string;
+    };
+    isOpen(): boolean;
+    private checkNotClosed;
+    close(): void;
+    get(uri: string, params?: Record<string, any>): Promise<any>;
+    post(uri: string, body?: any): Promise<any>;
+    post(uri: string, body: any, additional: {
+        config?: {
+            responseType: "arraybuffer";
+        };
+        headers?: Record<string, string>;
+        params?: Record<string, string>;
+    }): Promise<Buffer>;
+    listTables(limit?: number, pageToken?: string): Promise<string[]>;
+    query(tableName: string, query: VectorQuery): Promise<ArrowTable>;
+    static checkStatus(response: AxiosResponse): void;
+}

package/dist/remote/client.js

@@ -0,0 +1,172 @@
+"use strict";
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RestfulLanceDBClient = void 0;
+const axios_1 = require("axios");
+const arrow_1 = require("../arrow");
+class RestfulLanceDBClient {
+    #dbName;
+    #region;
+    #apiKey;
+    #hostOverride;
+    #closed = false;
+    #connectionTimeout = 12 * 1000; // 12 seconds;
+    #readTimeout = 30 * 1000; // 30 seconds;
+    #session;
+    constructor(dbName, apiKey, region, hostOverride, connectionTimeout, readTimeout) {
+        this.#dbName = dbName;
+        this.#apiKey = apiKey;
+        this.#region = region;
+        this.#hostOverride = hostOverride ?? this.#hostOverride;
+        this.#connectionTimeout = connectionTimeout ?? this.#connectionTimeout;
+        this.#readTimeout = readTimeout ?? this.#readTimeout;
+    }
+    // todo: cache the session.
+    get session() {
+        if (this.#session !== undefined) {
+            return this.#session;
+        }
+        else {
+            return axios_1.default.create({
+                baseURL: this.url,
+                headers: {
+                    // biome-ignore lint: external API
+                    Authorization: `Bearer ${this.#apiKey}`,
+                },
+                transformResponse: decodeErrorData,
+                timeout: this.#connectionTimeout,
+            });
+        }
+    }
+    get url() {
+        return (this.#hostOverride ??
+            `https://${this.#dbName}.${this.#region}.api.lancedb.com`);
+    }
+    get headers() {
+        const headers = {
+            "x-api-key": this.#apiKey,
+            "x-request-id": "na",
+        };
+        if (this.#region == "local") {
+            headers["Host"] = `${this.#dbName}.${this.#region}.api.lancedb.com`;
+        }
+        if (this.#hostOverride) {
+            headers["x-lancedb-database"] = this.#dbName;
+        }
+        return headers;
+    }
+    isOpen() {
+        return !this.#closed;
+    }
+    checkNotClosed() {
+        if (this.#closed) {
+            throw new Error("Connection is closed");
+        }
+    }
+    close() {
+        this.#session = undefined;
+        this.#closed = true;
+    }
+    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+    async get(uri, params) {
+        this.checkNotClosed();
+        uri = new URL(uri, this.url).toString();
+        let response;
+        try {
+            response = await this.session.get(uri, {
+                headers: this.headers,
+                params,
+            });
+        }
+        catch (e) {
+            if (e instanceof axios_1.AxiosError) {
+                response = e.response;
+            }
+            else {
+                throw e;
+            }
+        }
+        RestfulLanceDBClient.checkStatus(response);
+        return response.data;
+    }
+    async post(uri, 
+    // biome-ignore lint/suspicious/noExplicitAny: api request
+    body, additional) {
+        this.checkNotClosed();
+        uri = new URL(uri, this.url).toString();
+        additional = Object.assign({ config: { responseType: "json" } }, additional);
+        const headers = { ...this.headers, ...additional.headers };
+        if (!headers["Content-Type"]) {
+            headers["Content-Type"] = "application/json";
+        }
+        let response;
+        try {
+            response = await this.session.post(uri, body, {
+                headers,
+                responseType: additional.config.responseType,
+                params: new Map(Object.entries(additional.params ?? {})),
+            });
+        }
+        catch (e) {
+            if (e instanceof axios_1.AxiosError) {
+                response = e.response;
+            }
+            else {
+                throw e;
+            }
+        }
+        RestfulLanceDBClient.checkStatus(response);
+        if (additional.config.responseType === "arraybuffer") {
+            return response.data;
+        }
+        else {
+            return JSON.parse(response.data);
+        }
+    }
+    async listTables(limit = 10, pageToken = "") {
+        const json = await this.get("/v1/table", { limit, pageToken });
+        return json.tables;
+    }
+    async query(tableName, query) {
+        const tbl = await this.post(`/v1/table/${tableName}/query`, query, {
+            config: {
+                responseType: "arraybuffer",
+            },
+        });
+        return (0, arrow_1.tableFromIPC)(tbl);
+    }
+    static checkStatus(response) {
+        if (response.status === 404) {
+            throw new Error(`Not found: ${response.data}`);
+        }
+        else if (response.status >= 400 && response.status < 500) {
+            throw new Error(`Bad Request: ${response.status}, error: ${response.data}`);
+        }
+        else if (response.status >= 500 && response.status < 600) {
+            throw new Error(`Internal Server Error: ${response.status}, error: ${response.data}`);
+        }
+        else if (response.status !== 200) {
+            throw new Error(`Unknown Error: ${response.status}, error: ${response.data}`);
+        }
+    }
+}
+exports.RestfulLanceDBClient = RestfulLanceDBClient;
+function decodeErrorData(data) {
+    if (Buffer.isBuffer(data)) {
+        const decoded = data.toString("utf-8");
+        return decoded;
+    }
+    return data;
+}

package/dist/remote/connection.d.ts

@@ -0,0 +1,25 @@
+import { Data, SchemaLike } from "../arrow";
+import { Connection, CreateTableOptions, OpenTableOptions, TableNamesOptions } from "../connection";
+import { Table } from "../table";
+export interface RemoteConnectionOptions {
+    apiKey?: string;
+    region?: string;
+    hostOverride?: string;
+    connectionTimeout?: number;
+    readTimeout?: number;
+}
+export declare class RemoteConnection extends Connection {
+    #private;
+    constructor(url: string, { apiKey, region, hostOverride, connectionTimeout, readTimeout, }: RemoteConnectionOptions);
+    isOpen(): boolean;
+    close(): void;
+    display(): string;
+    tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
+    openTable(name: string, _options?: Partial<OpenTableOptions> | undefined): Promise<Table>;
+    createTable(nameOrOptions: string | ({
+        name: string;
+        data: Data;
+    } & Partial<CreateTableOptions>), data?: Data, options?: Partial<CreateTableOptions> | undefined): Promise<Table>;
+    createEmptyTable(name: string, schema: SchemaLike, options?: Partial<CreateTableOptions> | undefined): Promise<Table>;
+    dropTable(name: string): Promise<void>;
+}

package/dist/remote/connection.js

@@ -0,0 +1,110 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RemoteConnection = void 0;
+const arrow_1 = require("../arrow");
+const connection_1 = require("../connection");
+const table_1 = require("../table");
+const util_1 = require("../util");
+const client_1 = require("./client");
+const table_2 = require("./table");
+class RemoteConnection extends connection_1.Connection {
+    #dbName;
+    #apiKey;
+    #region;
+    #client;
+    #tableCache = new util_1.TTLCache(300000);
+    constructor(url, { apiKey, region, hostOverride, connectionTimeout, readTimeout, }) {
+        super();
+        apiKey = apiKey ?? process.env.LANCEDB_API_KEY;
+        region = region ?? process.env.LANCEDB_REGION;
+        if (!apiKey) {
+            throw new Error("apiKey is required when connecting to LanceDB Cloud");
+        }
+        if (!region) {
+            throw new Error("region is required when connecting to LanceDB Cloud");
+        }
+        const parsed = new URL(url);
+        if (parsed.protocol !== "db:") {
+            throw new Error(`invalid protocol: ${parsed.protocol}, only accepts db://`);
+        }
+        this.#dbName = parsed.hostname;
+        this.#apiKey = apiKey;
+        this.#region = region;
+        this.#client = new client_1.RestfulLanceDBClient(this.#dbName, this.#apiKey, this.#region, hostOverride, connectionTimeout, readTimeout);
+    }
+    isOpen() {
+        return this.#client.isOpen();
+    }
+    close() {
+        return this.#client.close();
+    }
+    display() {
+        return `RemoteConnection(${this.#dbName})`;
+    }
+    async tableNames(options) {
+        const response = await this.#client.get("/v1/table/", {
+            limit: options?.limit ?? 10,
+            // biome-ignore lint/style/useNamingConvention: <explanation>
+            page_token: options?.startAfter ?? "",
+        });
+        const body = await response.body();
+        for (const table of body.tables) {
+            this.#tableCache.set(table, true);
+        }
+        return body.tables;
+    }
+    async openTable(name, _options) {
+        if (this.#tableCache.get(name) === undefined) {
+            await this.#client.post(`/v1/table/${encodeURIComponent(name)}/describe/`);
+            this.#tableCache.set(name, true);
+        }
+        return new table_2.RemoteTable(this.#client, name, this.#dbName);
+    }
+    async createTable(nameOrOptions, data, options) {
+        if (typeof nameOrOptions !== "string" && "name" in nameOrOptions) {
+            const { name, data, ...options } = nameOrOptions;
+            return this.createTable(name, data, options);
+        }
+        if (data === undefined) {
+            throw new Error("data is required");
+        }
+        if (options?.mode) {
+            console.warn("option 'mode' is not supported in LanceDB Cloud", "LanceDB Cloud only supports the default 'create' mode.", "If the table already exists, an error will be thrown.");
+        }
+        if (options?.embeddingFunction) {
+            console.warn("embedding_functions is not yet supported on LanceDB Cloud.", "Please vote https://github.com/lancedb/lancedb/issues/626 ", "for this feature.");
+        }
+        const { buf } = await table_1.Table.parseTableData(data, options, true /** streaming */);
+        await this.#client.post(`/v1/table/${encodeURIComponent(nameOrOptions)}/create/`, buf, {
+            config: {
+                responseType: "arraybuffer",
+            },
+            headers: { "Content-Type": "application/vnd.apache.arrow.stream" },
+        });
+        this.#tableCache.set(nameOrOptions, true);
+        return new table_2.RemoteTable(this.#client, nameOrOptions, this.#dbName);
+    }
+    async createEmptyTable(name, schema, options) {
+        if (options?.mode) {
+            console.warn(`mode is not supported on LanceDB Cloud`);
+        }
+        if (options?.embeddingFunction) {
+            console.warn("embeddingFunction is not yet supported on LanceDB Cloud.", "Please vote https://github.com/lancedb/lancedb/issues/626 ", "for this feature.");
+        }
+        const emptyTable = (0, arrow_1.makeEmptyTable)(schema);
+        const buf = await (0, arrow_1.fromTableToStreamBuffer)(emptyTable);
+        await this.#client.post(`/v1/table/${encodeURIComponent(name)}/create/`, buf, {
+            config: {
+                responseType: "arraybuffer",
+            },
+            headers: { "Content-Type": "application/vnd.apache.arrow.stream" },
+        });
+        this.#tableCache.set(name, true);
+        return new table_2.RemoteTable(this.#client, name, this.#dbName);
+    }
+    async dropTable(name) {
+        await this.#client.post(`/v1/table/${encodeURIComponent(name)}/drop/`);
+        this.#tableCache.delete(name);
+    }
+}
+exports.RemoteConnection = RemoteConnection;

package/dist/remote/index.d.ts

@@ -0,0 +1,3 @@
+export { RestfulLanceDBClient } from "./client";
+export { type RemoteConnectionOptions, RemoteConnection } from "./connection";
+export { RemoteTable } from "./table";

package/dist/remote/index.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RemoteTable = exports.RemoteConnection = exports.RestfulLanceDBClient = void 0;
+var client_1 = require("./client");
+Object.defineProperty(exports, "RestfulLanceDBClient", { enumerable: true, get: function () { return client_1.RestfulLanceDBClient; } });
+var connection_1 = require("./connection");
+Object.defineProperty(exports, "RemoteConnection", { enumerable: true, get: function () { return connection_1.RemoteConnection; } });
+var table_1 = require("./table");
+Object.defineProperty(exports, "RemoteTable", { enumerable: true, get: function () { return table_1.RemoteTable; } });

package/dist/remote/table.d.ts

@@ -0,0 +1,42 @@
+import { Table as ArrowTable } from "apache-arrow";
+import { Data, IntoVector } from "../arrow";
+import { IndexStatistics } from "..";
+import { IndexOptions } from "../indices";
+import { MergeInsertBuilder } from "../merge";
+import { VectorQuery } from "../query";
+import { AddDataOptions, Table, UpdateOptions } from "../table";
+import { IntoSql } from "../util";
+import { RestfulLanceDBClient } from "./client";
+export declare class RemoteTable extends Table {
+    #private;
+    get name(): string;
+    constructor(client: RestfulLanceDBClient, tableName: string, dbName: string);
+    isOpen(): boolean;
+    close(): void;
+    display(): string;
+    schema(): Promise<import("apache-arrow").Schema>;
+    add(data: Data, options?: Partial<AddDataOptions>): Promise<void>;
+    update(optsOrUpdates: (Map<string, string> | Record<string, string>) | ({
+        values: Map<string, IntoSql> | Record<string, IntoSql>;
+    } & Partial<UpdateOptions>) | ({
+        valuesSql: Map<string, string> | Record<string, string>;
+    } & Partial<UpdateOptions>), options?: Partial<UpdateOptions>): Promise<void>;
+    countRows(filter?: unknown): Promise<number>;
+    delete(predicate: unknown): Promise<void>;
+    createIndex(column: string, options?: Partial<IndexOptions>): Promise<void>;
+    query(): import("..").Query;
+    search(_query: string | IntoVector): VectorQuery;
+    vectorSearch(_vector: unknown): import("..").VectorQuery;
+    addColumns(_newColumnTransforms: unknown): Promise<void>;
+    alterColumns(_columnAlterations: unknown): Promise<void>;
+    dropColumns(_columnNames: unknown): Promise<void>;
+    version(): Promise<number>;
+    checkout(_version: unknown): Promise<void>;
+    checkoutLatest(): Promise<void>;
+    restore(): Promise<void>;
+    optimize(_options?: unknown): Promise<import("../native").OptimizeStats>;
+    listIndices(): Promise<import("../native").IndexConfig[]>;
+    toArrow(): Promise<ArrowTable>;
+    mergeInsert(_on: string | string[]): MergeInsertBuilder;
+    indexStats(_name: string): Promise<IndexStatistics | undefined>;
+}