@lancedb/lancedb 0.5.1 → 0.7.0

package/nodejs-artifacts/query.d.ts

@@ -1,4 +1,5 @@
-import { Table as ArrowTable, RecordBatch } from "./arrow";
+import { Table as ArrowTable, type IntoVector, RecordBatch } from "./arrow";
+import { type IvfPqOptions } from "./indices";
 import { RecordBatchIterator as NativeBatchIterator, Query as NativeQuery, Table as NativeTable, VectorQuery as NativeVectorQuery } from "./native";
 export declare class RecordBatchIterator implements AsyncIterator<RecordBatch> {
     private promisedInner?;
@@ -6,10 +7,23 @@ export declare class RecordBatchIterator implements AsyncIterator<RecordBatch> {
     constructor(promise?: Promise<NativeBatchIterator>);
     next(): Promise<IteratorResult<RecordBatch<any>>>;
 }
+/**
+ * Options that control the behavior of a particular query execution
+ */
+export interface QueryExecutionOptions {
+    /**
+     * The maximum number of rows to return in a single batch
+     *
+     * Batches may have fewer rows if the underlying data is stored
+     * in smaller chunks.
+     */
+    maxBatchLength?: number;
+}
 /** Common methods supported by all query types */
-export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery, QueryType> implements AsyncIterable<RecordBatch> {
-    protected inner: NativeQueryType;
-    protected constructor(inner: NativeQueryType);
+export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery> implements AsyncIterable<RecordBatch> {
+    protected inner: NativeQueryType | Promise<NativeQueryType>;
+    protected constructor(inner: NativeQueryType | Promise<NativeQueryType>);
+    protected doCall(fn: (inner: NativeQueryType) => void): void;
     /**
      * A filter statement to be applied to this query.
      *
@@ -22,7 +36,13 @@ export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVecto
      * Filtering performance can often be improved by creating a scalar index
      * on the filter column(s).
      */
-    where(predicate: string): QueryType;
+    where(predicate: string): this;
+    /**
+     * A filter statement to be applied to this query.
+     * @alias where
+     * @deprecated Use `where` instead
+     */
+    filter(predicate: string): this;
     /**
      * Return only the specified columns.
      *
@@ -53,15 +73,15 @@ export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVecto
      * uses `Object.entries` which should preserve the insertion order of the object.  However,
      * object insertion order is easy to get wrong and `Map` is more foolproof.
      */
-    select(columns: string[] | Map<string, string> | Record<string, string>): QueryType;
+    select(columns: string[] | Map<string, string> | Record<string, string> | string): this;
     /**
      * Set the maximum number of results to return.
      *
      * By default, a plain search has no limit.  If this method is not
      * called then every valid row from the table will be returned.
      */
-    limit(limit: number): QueryType;
-    protected nativeExecute(): Promise<NativeBatchIterator>;
+    limit(limit: number): this;
+    protected nativeExecute(options?: Partial<QueryExecutionOptions>): Promise<NativeBatchIterator>;
     /**
      * Execute the query and return the results as an @see {@link AsyncIterator}
      * of @see {@link RecordBatch}.
@@ -73,12 +93,27 @@ export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVecto
      * single query)
      *
      */
-    protected execute(): RecordBatchIterator;
+    protected execute(options?: Partial<QueryExecutionOptions>): RecordBatchIterator;
     [Symbol.asyncIterator](): AsyncIterator<RecordBatch<any>>;
     /** Collect the results as an Arrow @see {@link ArrowTable}. */
-    toArrow(): Promise<ArrowTable>;
+    toArrow(options?: Partial<QueryExecutionOptions>): Promise<ArrowTable>;
     /** Collect the results as an array of objects. */
-    toArray(): Promise<unknown[]>;
+    toArray(options?: Partial<QueryExecutionOptions>): Promise<any[]>;
+    /**
+     * Generates an explanation of the query execution plan.
+     *
+     * @example
+     * import * as lancedb from "@lancedb/lancedb"
+     * const db = await lancedb.connect("./.lancedb");
+     * const table = await db.createTable("my_table", [
+     *   { vector: [1.1, 0.9], id: "1" },
+     * ]);
+     * const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
+     *
+     * @param verbose - If true, provides a more detailed explanation. Defaults to false.
+     * @returns A Promise that resolves to a string containing the query execution plan explanation.
+     */
+    explainPlan(verbose?: boolean): Promise<string>;
 }
 /**
  * An interface for a query that can be executed
@@ -92,8 +127,8 @@ export interface ExecutableQuery {
  *
  * This builder can be reused to execute the query many times.
  */
-export declare class VectorQuery extends QueryBase<NativeVectorQuery, VectorQuery> {
-    constructor(inner: NativeVectorQuery);
+export declare class VectorQuery extends QueryBase<NativeVectorQuery> {
+    constructor(inner: NativeVectorQuery | Promise<NativeVectorQuery>);
     /**
      * Set the number of partitions to search (probe)
      *
@@ -141,7 +176,7 @@ export declare class VectorQuery extends QueryBase<NativeVectorQuery, VectorQuer
      *
      * By default "l2" is used.
      */
-    distanceType(distanceType: string): VectorQuery;
+    distanceType(distanceType: Required<IvfPqOptions>["distanceType"]): VectorQuery;
     /**
      * A multiplier to control how many additional rows are taken during the refine step
      *
@@ -205,7 +240,7 @@ export declare class VectorQuery extends QueryBase<NativeVectorQuery, VectorQuer
     bypassVectorIndex(): VectorQuery;
 }
 /** A builder for LanceDB queries. */
-export declare class Query extends QueryBase<NativeQuery, Query> {
+export declare class Query extends QueryBase<NativeQuery> {
     constructor(tbl: NativeTable);
     /**
      * Find the nearest vectors to the given query vector.
@@ -244,5 +279,5 @@ export declare class Query extends QueryBase<NativeQuery, Query> {
      * Vector searches always have a `limit`.  If `limit` has not been called then
      * a default `limit` of 10 will be used.  @see {@link Query#limit}
      */
-    nearestTo(vector: unknown): VectorQuery;
+    nearestTo(vector: IntoVector): VectorQuery;
 }

package/nodejs-artifacts/query.js

@@ -43,6 +43,18 @@ class RecordBatchIterator {
 }
 exports.RecordBatchIterator = RecordBatchIterator;
 /* eslint-enable */
+class RecordBatchIterable {
+    inner;
+    options;
+    constructor(inner, options) {
+        this.inner = inner;
+        this.options = options;
+    }
+    // biome-ignore lint/suspicious/noExplicitAny: skip
+    [Symbol.asyncIterator]() {
+        return new RecordBatchIterator(this.inner.execute(this.options?.maxBatchLength));
+    }
+}
 /** Common methods supported by all query types */
 class QueryBase {
     inner;
@@ -50,6 +62,18 @@ class QueryBase {
         this.inner = inner;
         // intentionally empty
     }
+    // call a function on the inner (either a promise or the actual object)
+    doCall(fn) {
+        if (this.inner instanceof Promise) {
+            this.inner = this.inner.then((inner) => {
+                fn(inner);
+                return inner;
+            });
+        }
+        else {
+            fn(this.inner);
+        }
+    }
     /**
      * A filter statement to be applied to this query.
      *
@@ -63,9 +87,17 @@ class QueryBase {
      * on the filter column(s).
      */
     where(predicate) {
-        this.inner.onlyIf(predicate);
+        this.doCall((inner) => inner.onlyIf(predicate));
         return this;
     }
+    /**
+     * A filter statement to be applied to this query.
+     * @alias where
+     * @deprecated Use `where` instead
+     */
+    filter(predicate) {
+        return this.where(predicate);
+    }
     /**
      * Return only the specified columns.
      *
@@ -98,6 +130,9 @@ class QueryBase {
      */
     select(columns) {
         let columnTuples;
+        if (typeof columns === "string") {
+            columns = [columns];
+        }
         if (Array.isArray(columns)) {
             columnTuples = columns.map((c) => [c, c]);
         }
@@ -107,7 +142,9 @@ class QueryBase {
         else {
             columnTuples = Object.entries(columns);
         }
-        this.inner.select(columnTuples);
+        this.doCall((inner) => {
+            inner.select(columnTuples);
+        });
         return this;
     }
     /**
@@ -117,11 +154,16 @@ class QueryBase {
      * called then every valid row from the table will be returned.
      */
     limit(limit) {
-        this.inner.limit(limit);
+        this.doCall((inner) => inner.limit(limit));
         return this;
     }
-    nativeExecute() {
-        return this.inner.execute();
+    nativeExecute(options) {
+        if (this.inner instanceof Promise) {
+            return this.inner.then((inner) => inner.execute(options?.maxBatchLength));
+        }
+        else {
+            return this.inner.execute(options?.maxBatchLength);
+        }
     }
     /**
      * Execute the query and return the results as an @see {@link AsyncIterator}
@@ -134,8 +176,8 @@ class QueryBase {
      * single query)
      *
      */
-    execute() {
-        return new RecordBatchIterator(this.nativeExecute());
+    execute(options) {
+        return new RecordBatchIterator(this.nativeExecute(options));
     }
     // biome-ignore lint/suspicious/noExplicitAny: skip
     [Symbol.asyncIterator]() {
@@ -143,19 +185,48 @@ class QueryBase {
         return new RecordBatchIterator(promise);
     }
     /** Collect the results as an Arrow @see {@link ArrowTable}. */
-    async toArrow() {
+    async toArrow(options) {
         const batches = [];
-        for await (const batch of this) {
+        let inner;
+        if (this.inner instanceof Promise) {
+            inner = await this.inner;
+        }
+        else {
+            inner = this.inner;
+        }
+        for await (const batch of new RecordBatchIterable(inner, options)) {
             batches.push(batch);
         }
         return new arrow_1.Table(batches);
     }
     /** Collect the results as an array of objects. */
-    async toArray() {
-        const tbl = await this.toArrow();
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+    // biome-ignore lint/suspicious/noExplicitAny: arrow.toArrow() returns any[]
+    async toArray(options) {
+        const tbl = await this.toArrow(options);
         return tbl.toArray();
     }
+    /**
+     * Generates an explanation of the query execution plan.
+     *
+     * @example
+     * import * as lancedb from "@lancedb/lancedb"
+     * const db = await lancedb.connect("./.lancedb");
+     * const table = await db.createTable("my_table", [
+     *   { vector: [1.1, 0.9], id: "1" },
+     * ]);
+     * const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
+     *
+     * @param verbose - If true, provides a more detailed explanation. Defaults to false.
+     * @returns A Promise that resolves to a string containing the query execution plan explanation.
+     */
+    async explainPlan(verbose = false) {
+        if (this.inner instanceof Promise) {
+            return this.inner.then((inner) => inner.explainPlan(verbose));
+        }
+        else {
+            return this.inner.explainPlan(verbose);
+        }
+    }
 }
 exports.QueryBase = QueryBase;
 /**
@@ -190,7 +261,7 @@ class VectorQuery extends QueryBase {
      * you the desired recall.
      */
     nprobes(nprobes) {
-        this.inner.nprobes(nprobes);
+        super.doCall((inner) => inner.nprobes(nprobes));
         return this;
     }
     /**
@@ -203,7 +274,7 @@ class VectorQuery extends QueryBase {
      * whose data type is a fixed-size-list of floats.
      */
     column(column) {
-        this.inner.column(column);
+        super.doCall((inner) => inner.column(column));
         return this;
     }
     /**
@@ -221,7 +292,7 @@ class VectorQuery extends QueryBase {
      * By default "l2" is used.
      */
     distanceType(distanceType) {
-        this.inner.distanceType(distanceType);
+        super.doCall((inner) => inner.distanceType(distanceType));
         return this;
     }
     /**
@@ -254,7 +325,7 @@ class VectorQuery extends QueryBase {
      * distance between the query vector and the actual uncompressed vector.
      */
     refineFactor(refineFactor) {
-        this.inner.refineFactor(refineFactor);
+        super.doCall((inner) => inner.refineFactor(refineFactor));
         return this;
     }
     /**
@@ -278,7 +349,7 @@ class VectorQuery extends QueryBase {
      * factor can often help restore some of the results lost by post filtering.
      */
     postfilter() {
-        this.inner.postfilter();
+        super.doCall((inner) => inner.postfilter());
         return this;
     }
     /**
@@ -291,7 +362,7 @@ class VectorQuery extends QueryBase {
      * calculate your recall to select an appropriate value for nprobes.
      */
     bypassVectorIndex() {
-        this.inner.bypassVectorIndex();
+        super.doCall((inner) => inner.bypassVectorIndex());
         return this;
     }
 }
@@ -339,9 +410,39 @@ class Query extends QueryBase {
      * a default `limit` of 10 will be used.  @see {@link Query#limit}
      */
     nearestTo(vector) {
-        // biome-ignore lint/suspicious/noExplicitAny: skip
-        const vectorQuery = this.inner.nearestTo(Float32Array.from(vector));
-        return new VectorQuery(vectorQuery);
+        if (this.inner instanceof Promise) {
+            const nativeQuery = this.inner.then(async (inner) => {
+                if (vector instanceof Promise) {
+                    const arr = await vector.then((v) => Float32Array.from(v));
+                    return inner.nearestTo(arr);
+                }
+                else {
+                    return inner.nearestTo(Float32Array.from(vector));
+                }
+            });
+            return new VectorQuery(nativeQuery);
+        }
+        if (vector instanceof Promise) {
+            const res = (async () => {
+                try {
+                    const v = await vector;
+                    const arr = Float32Array.from(v);
+                    //
+                    // biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
+                    const value = this.nearestTo(arr);
+                    const inner = value.inner;
+                    return inner;
+                }
+                catch (e) {
+                    return Promise.reject(e);
+                }
+            })();
+            return new VectorQuery(res);
+        }
+        else {
+            const vectorQuery = this.inner.nearestTo(Float32Array.from(vector));
+            return new VectorQuery(vectorQuery);
+        }
     }
 }
 exports.Query = Query;

package/nodejs-artifacts/remote/client.d.ts

@@ -0,0 +1,28 @@
+/// <reference types="node" />
+import { type AxiosResponse } from "axios";
+import { Table as ArrowTable } from "../arrow";
+import { VectorQuery } from "../query";
+export declare class RestfulLanceDBClient {
+    #private;
+    constructor(dbName: string, apiKey: string, region: string, hostOverride?: string, connectionTimeout?: number, readTimeout?: number);
+    get session(): import("axios").AxiosInstance;
+    get url(): string;
+    get headers(): {
+        [key: string]: string;
+    };
+    isOpen(): boolean;
+    private checkNotClosed;
+    close(): void;
+    get(uri: string, params?: Record<string, any>): Promise<any>;
+    post(uri: string, body?: any): Promise<any>;
+    post(uri: string, body: any, additional: {
+        config?: {
+            responseType: "arraybuffer";
+        };
+        headers?: Record<string, string>;
+        params?: Record<string, string>;
+    }): Promise<Buffer>;
+    listTables(limit?: number, pageToken?: string): Promise<string[]>;
+    query(tableName: string, query: VectorQuery): Promise<ArrowTable>;
+    static checkStatus(response: AxiosResponse): void;
+}

package/nodejs-artifacts/remote/client.js

@@ -0,0 +1,172 @@
+"use strict";
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RestfulLanceDBClient = void 0;
+const axios_1 = require("axios");
+const arrow_1 = require("../arrow");
+class RestfulLanceDBClient {
+    #dbName;
+    #region;
+    #apiKey;
+    #hostOverride;
+    #closed = false;
+    #connectionTimeout = 12 * 1000; // 12 seconds;
+    #readTimeout = 30 * 1000; // 30 seconds;
+    #session;
+    constructor(dbName, apiKey, region, hostOverride, connectionTimeout, readTimeout) {
+        this.#dbName = dbName;
+        this.#apiKey = apiKey;
+        this.#region = region;
+        this.#hostOverride = hostOverride ?? this.#hostOverride;
+        this.#connectionTimeout = connectionTimeout ?? this.#connectionTimeout;
+        this.#readTimeout = readTimeout ?? this.#readTimeout;
+    }
+    // todo: cache the session.
+    get session() {
+        if (this.#session !== undefined) {
+            return this.#session;
+        }
+        else {
+            return axios_1.default.create({
+                baseURL: this.url,
+                headers: {
+                    // biome-ignore lint: external API
+                    Authorization: `Bearer ${this.#apiKey}`,
+                },
+                transformResponse: decodeErrorData,
+                timeout: this.#connectionTimeout,
+            });
+        }
+    }
+    get url() {
+        return (this.#hostOverride ??
+            `https://${this.#dbName}.${this.#region}.api.lancedb.com`);
+    }
+    get headers() {
+        const headers = {
+            "x-api-key": this.#apiKey,
+            "x-request-id": "na",
+        };
+        if (this.#region == "local") {
+            headers["Host"] = `${this.#dbName}.${this.#region}.api.lancedb.com`;
+        }
+        if (this.#hostOverride) {
+            headers["x-lancedb-database"] = this.#dbName;
+        }
+        return headers;
+    }
+    isOpen() {
+        return !this.#closed;
+    }
+    checkNotClosed() {
+        if (this.#closed) {
+            throw new Error("Connection is closed");
+        }
+    }
+    close() {
+        this.#session = undefined;
+        this.#closed = true;
+    }
+    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+    async get(uri, params) {
+        this.checkNotClosed();
+        uri = new URL(uri, this.url).toString();
+        let response;
+        try {
+            response = await this.session.get(uri, {
+                headers: this.headers,
+                params,
+            });
+        }
+        catch (e) {
+            if (e instanceof axios_1.AxiosError) {
+                response = e.response;
+            }
+            else {
+                throw e;
+            }
+        }
+        RestfulLanceDBClient.checkStatus(response);
+        return response.data;
+    }
+    async post(uri, 
+    // biome-ignore lint/suspicious/noExplicitAny: api request
+    body, additional) {
+        this.checkNotClosed();
+        uri = new URL(uri, this.url).toString();
+        additional = Object.assign({ config: { responseType: "json" } }, additional);
+        const headers = { ...this.headers, ...additional.headers };
+        if (!headers["Content-Type"]) {
+            headers["Content-Type"] = "application/json";
+        }
+        let response;
+        try {
+            response = await this.session.post(uri, body, {
+                headers,
+                responseType: additional.config.responseType,
+                params: new Map(Object.entries(additional.params ?? {})),
+            });
+        }
+        catch (e) {
+            if (e instanceof axios_1.AxiosError) {
+                response = e.response;
+            }
+            else {
+                throw e;
+            }
+        }
+        RestfulLanceDBClient.checkStatus(response);
+        if (additional.config.responseType === "arraybuffer") {
+            return response.data;
+        }
+        else {
+            return JSON.parse(response.data);
+        }
+    }
+    async listTables(limit = 10, pageToken = "") {
+        const json = await this.get("/v1/table", { limit, pageToken });
+        return json.tables;
+    }
+    async query(tableName, query) {
+        const tbl = await this.post(`/v1/table/${tableName}/query`, query, {
+            config: {
+                responseType: "arraybuffer",
+            },
+        });
+        return (0, arrow_1.tableFromIPC)(tbl);
+    }
+    static checkStatus(response) {
+        if (response.status === 404) {
+            throw new Error(`Not found: ${response.data}`);
+        }
+        else if (response.status >= 400 && response.status < 500) {
+            throw new Error(`Bad Request: ${response.status}, error: ${response.data}`);
+        }
+        else if (response.status >= 500 && response.status < 600) {
+            throw new Error(`Internal Server Error: ${response.status}, error: ${response.data}`);
+        }
+        else if (response.status !== 200) {
+            throw new Error(`Unknown Error: ${response.status}, error: ${response.data}`);
+        }
+    }
+}
+exports.RestfulLanceDBClient = RestfulLanceDBClient;
+function decodeErrorData(data) {
+    if (Buffer.isBuffer(data)) {
+        const decoded = data.toString("utf-8");
+        return decoded;
+    }
+    return data;
+}

package/nodejs-artifacts/remote/connection.d.ts

@@ -0,0 +1,25 @@
+import { Data, SchemaLike } from "../arrow";
+import { Connection, CreateTableOptions, OpenTableOptions, TableNamesOptions } from "../connection";
+import { Table } from "../table";
+export interface RemoteConnectionOptions {
+    apiKey?: string;
+    region?: string;
+    hostOverride?: string;
+    connectionTimeout?: number;
+    readTimeout?: number;
+}
+export declare class RemoteConnection extends Connection {
+    #private;
+    constructor(url: string, { apiKey, region, hostOverride, connectionTimeout, readTimeout, }: RemoteConnectionOptions);
+    isOpen(): boolean;
+    close(): void;
+    display(): string;
+    tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
+    openTable(name: string, _options?: Partial<OpenTableOptions> | undefined): Promise<Table>;
+    createTable(nameOrOptions: string | ({
+        name: string;
+        data: Data;
+    } & Partial<CreateTableOptions>), data?: Data, options?: Partial<CreateTableOptions> | undefined): Promise<Table>;
+    createEmptyTable(name: string, schema: SchemaLike, options?: Partial<CreateTableOptions> | undefined): Promise<Table>;
+    dropTable(name: string): Promise<void>;
+}