@lancedb/lancedb 0.5.1 → 0.7.0

package/nodejs-artifacts/remote/connection.js

@@ -0,0 +1,110 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RemoteConnection = void 0;
+const arrow_1 = require("../arrow");
+const connection_1 = require("../connection");
+const table_1 = require("../table");
+const util_1 = require("../util");
+const client_1 = require("./client");
+const table_2 = require("./table");
+class RemoteConnection extends connection_1.Connection {
+    #dbName;
+    #apiKey;
+    #region;
+    #client;
+    #tableCache = new util_1.TTLCache(300000);
+    constructor(url, { apiKey, region, hostOverride, connectionTimeout, readTimeout, }) {
+        super();
+        apiKey = apiKey ?? process.env.LANCEDB_API_KEY;
+        region = region ?? process.env.LANCEDB_REGION;
+        if (!apiKey) {
+            throw new Error("apiKey is required when connecting to LanceDB Cloud");
+        }
+        if (!region) {
+            throw new Error("region is required when connecting to LanceDB Cloud");
+        }
+        const parsed = new URL(url);
+        if (parsed.protocol !== "db:") {
+            throw new Error(`invalid protocol: ${parsed.protocol}, only accepts db://`);
+        }
+        this.#dbName = parsed.hostname;
+        this.#apiKey = apiKey;
+        this.#region = region;
+        this.#client = new client_1.RestfulLanceDBClient(this.#dbName, this.#apiKey, this.#region, hostOverride, connectionTimeout, readTimeout);
+    }
+    isOpen() {
+        return this.#client.isOpen();
+    }
+    close() {
+        return this.#client.close();
+    }
+    display() {
+        return `RemoteConnection(${this.#dbName})`;
+    }
+    async tableNames(options) {
+        const response = await this.#client.get("/v1/table/", {
+            limit: options?.limit ?? 10,
+            // biome-ignore lint/style/useNamingConvention: <explanation>
+            page_token: options?.startAfter ?? "",
+        });
+        const body = await response.body();
+        for (const table of body.tables) {
+            this.#tableCache.set(table, true);
+        }
+        return body.tables;
+    }
+    async openTable(name, _options) {
+        if (this.#tableCache.get(name) === undefined) {
+            await this.#client.post(`/v1/table/${encodeURIComponent(name)}/describe/`);
+            this.#tableCache.set(name, true);
+        }
+        return new table_2.RemoteTable(this.#client, name, this.#dbName);
+    }
+    async createTable(nameOrOptions, data, options) {
+        if (typeof nameOrOptions !== "string" && "name" in nameOrOptions) {
+            const { name, data, ...options } = nameOrOptions;
+            return this.createTable(name, data, options);
+        }
+        if (data === undefined) {
+            throw new Error("data is required");
+        }
+        if (options?.mode) {
+            console.warn("option 'mode' is not supported in LanceDB Cloud", "LanceDB Cloud only supports the default 'create' mode.", "If the table already exists, an error will be thrown.");
+        }
+        if (options?.embeddingFunction) {
+            console.warn("embedding_functions is not yet supported on LanceDB Cloud.", "Please vote https://github.com/lancedb/lancedb/issues/626 ", "for this feature.");
+        }
+        const { buf } = await table_1.Table.parseTableData(data, options, true /** streaming */);
+        await this.#client.post(`/v1/table/${encodeURIComponent(nameOrOptions)}/create/`, buf, {
+            config: {
+                responseType: "arraybuffer",
+            },
+            headers: { "Content-Type": "application/vnd.apache.arrow.stream" },
+        });
+        this.#tableCache.set(nameOrOptions, true);
+        return new table_2.RemoteTable(this.#client, nameOrOptions, this.#dbName);
+    }
+    async createEmptyTable(name, schema, options) {
+        if (options?.mode) {
+            console.warn(`mode is not supported on LanceDB Cloud`);
+        }
+        if (options?.embeddingFunction) {
+            console.warn("embeddingFunction is not yet supported on LanceDB Cloud.", "Please vote https://github.com/lancedb/lancedb/issues/626 ", "for this feature.");
+        }
+        const emptyTable = (0, arrow_1.makeEmptyTable)(schema);
+        const buf = await (0, arrow_1.fromTableToStreamBuffer)(emptyTable);
+        await this.#client.post(`/v1/table/${encodeURIComponent(name)}/create/`, buf, {
+            config: {
+                responseType: "arraybuffer",
+            },
+            headers: { "Content-Type": "application/vnd.apache.arrow.stream" },
+        });
+        this.#tableCache.set(name, true);
+        return new table_2.RemoteTable(this.#client, name, this.#dbName);
+    }
+    async dropTable(name) {
+        await this.#client.post(`/v1/table/${encodeURIComponent(name)}/drop/`);
+        this.#tableCache.delete(name);
+    }
+}
+exports.RemoteConnection = RemoteConnection;

package/nodejs-artifacts/remote/index.d.ts

@@ -0,0 +1,3 @@
+export { RestfulLanceDBClient } from "./client";
+export { type RemoteConnectionOptions, RemoteConnection } from "./connection";
+export { RemoteTable } from "./table";

package/nodejs-artifacts/remote/index.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RemoteTable = exports.RemoteConnection = exports.RestfulLanceDBClient = void 0;
+var client_1 = require("./client");
+Object.defineProperty(exports, "RestfulLanceDBClient", { enumerable: true, get: function () { return client_1.RestfulLanceDBClient; } });
+var connection_1 = require("./connection");
+Object.defineProperty(exports, "RemoteConnection", { enumerable: true, get: function () { return connection_1.RemoteConnection; } });
+var table_1 = require("./table");
+Object.defineProperty(exports, "RemoteTable", { enumerable: true, get: function () { return table_1.RemoteTable; } });

package/nodejs-artifacts/remote/table.d.ts

@@ -0,0 +1,42 @@
+import { Table as ArrowTable } from "apache-arrow";
+import { Data, IntoVector } from "../arrow";
+import { IndexStatistics } from "..";
+import { IndexOptions } from "../indices";
+import { MergeInsertBuilder } from "../merge";
+import { VectorQuery } from "../query";
+import { AddDataOptions, Table, UpdateOptions } from "../table";
+import { IntoSql } from "../util";
+import { RestfulLanceDBClient } from "./client";
+export declare class RemoteTable extends Table {
+    #private;
+    get name(): string;
+    constructor(client: RestfulLanceDBClient, tableName: string, dbName: string);
+    isOpen(): boolean;
+    close(): void;
+    display(): string;
+    schema(): Promise<import("apache-arrow").Schema>;
+    add(data: Data, options?: Partial<AddDataOptions>): Promise<void>;
+    update(optsOrUpdates: (Map<string, string> | Record<string, string>) | ({
+        values: Map<string, IntoSql> | Record<string, IntoSql>;
+    } & Partial<UpdateOptions>) | ({
+        valuesSql: Map<string, string> | Record<string, string>;
+    } & Partial<UpdateOptions>), options?: Partial<UpdateOptions>): Promise<void>;
+    countRows(filter?: unknown): Promise<number>;
+    delete(predicate: unknown): Promise<void>;
+    createIndex(column: string, options?: Partial<IndexOptions>): Promise<void>;
+    query(): import("..").Query;
+    search(_query: string | IntoVector): VectorQuery;
+    vectorSearch(_vector: unknown): import("..").VectorQuery;
+    addColumns(_newColumnTransforms: unknown): Promise<void>;
+    alterColumns(_columnAlterations: unknown): Promise<void>;
+    dropColumns(_columnNames: unknown): Promise<void>;
+    version(): Promise<number>;
+    checkout(_version: unknown): Promise<void>;
+    checkoutLatest(): Promise<void>;
+    restore(): Promise<void>;
+    optimize(_options?: unknown): Promise<import("../native").OptimizeStats>;
+    listIndices(): Promise<import("../native").IndexConfig[]>;
+    toArrow(): Promise<ArrowTable>;
+    mergeInsert(_on: string | string[]): MergeInsertBuilder;
+    indexStats(_name: string): Promise<IndexStatistics | undefined>;
+}

package/nodejs-artifacts/remote/table.js

@@ -0,0 +1,179 @@
+"use strict";
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RemoteTable = void 0;
+const table_1 = require("../table");
+const util_1 = require("../util");
+class RemoteTable extends table_1.Table {
+    #client;
+    #name;
+    // Used in the display() method
+    #dbName;
+    get #tablePrefix() {
+        return `/v1/table/${encodeURIComponent(this.#name)}/`;
+    }
+    get name() {
+        return this.#name;
+    }
+    constructor(client, tableName, dbName) {
+        super();
+        this.#client = client;
+        this.#name = tableName;
+        this.#dbName = dbName;
+    }
+    isOpen() {
+        return !this.#client.isOpen();
+    }
+    close() {
+        this.#client.close();
+    }
+    display() {
+        return `RemoteTable(${this.#dbName}; ${this.#name})`;
+    }
+    async schema() {
+        const resp = await this.#client.post(`${this.#tablePrefix}/describe/`);
+        // TODO: parse this into a valid arrow schema
+        return resp.schema;
+    }
+    async add(data, options) {
+        const { buf, mode } = await table_1.Table.parseTableData(data, options, true);
+        await this.#client.post(`${this.#tablePrefix}/insert/`, buf, {
+            params: {
+                mode,
+            },
+            headers: {
+                "Content-Type": "application/vnd.apache.arrow.stream",
+            },
+        });
+    }
+    async update(optsOrUpdates, options) {
+        const isValues = "values" in optsOrUpdates && typeof optsOrUpdates.values !== "string";
+        const isValuesSql = "valuesSql" in optsOrUpdates &&
+            typeof optsOrUpdates.valuesSql !== "string";
+        const isMap = (obj) => {
+            return obj instanceof Map;
+        };
+        let predicate;
+        let columns;
+        switch (true) {
+            case isMap(optsOrUpdates):
+                columns = Array.from(optsOrUpdates.entries());
+                predicate = options?.where;
+                break;
+            case isValues && isMap(optsOrUpdates.values):
+                columns = Array.from(optsOrUpdates.values.entries()).map(([k, v]) => [
+                    k,
+                    (0, util_1.toSQL)(v),
+                ]);
+                predicate = optsOrUpdates.where;
+                break;
+            case isValues && !isMap(optsOrUpdates.values):
+                columns = Object.entries(optsOrUpdates.values).map(([k, v]) => [
+                    k,
+                    (0, util_1.toSQL)(v),
+                ]);
+                predicate = optsOrUpdates.where;
+                break;
+            case isValuesSql && isMap(optsOrUpdates.valuesSql):
+                columns = Array.from(optsOrUpdates.valuesSql.entries());
+                predicate = optsOrUpdates.where;
+                break;
+            case isValuesSql && !isMap(optsOrUpdates.valuesSql):
+                columns = Object.entries(optsOrUpdates.valuesSql).map(([k, v]) => [
+                    k,
+                    v,
+                ]);
+                predicate = optsOrUpdates.where;
+                break;
+            default:
+                columns = Object.entries(optsOrUpdates);
+                predicate = options?.where;
+        }
+        await this.#client.post(`${this.#tablePrefix}/update/`, {
+            predicate: predicate ?? null,
+            updates: columns,
+        });
+    }
+    async countRows(filter) {
+        const payload = { predicate: filter };
+        return await this.#client.post(`${this.#tablePrefix}/count_rows/`, payload);
+    }
+    async delete(predicate) {
+        const payload = { predicate };
+        await this.#client.post(`${this.#tablePrefix}/delete/`, payload);
+    }
+    async createIndex(column, options) {
+        if (options !== undefined) {
+            console.warn("options are not yet supported on the LanceDB cloud");
+        }
+        const indexType = "vector";
+        const metric = "L2";
+        const data = {
+            column,
+            // biome-ignore lint/style/useNamingConvention: external API
+            index_type: indexType,
+            // biome-ignore lint/style/useNamingConvention: external API
+            metric_type: metric,
+        };
+        await this.#client.post(`${this.#tablePrefix}/create_index`, data);
+    }
+    query() {
+        throw new Error("query() is not yet supported on the LanceDB cloud");
+    }
+    search(_query) {
+        throw new Error("search() is not yet supported on the LanceDB cloud");
+    }
+    vectorSearch(_vector) {
+        throw new Error("vectorSearch() is not yet supported on the LanceDB cloud");
+    }
+    addColumns(_newColumnTransforms) {
+        throw new Error("addColumns() is not yet supported on the LanceDB cloud");
+    }
+    alterColumns(_columnAlterations) {
+        throw new Error("alterColumns() is not yet supported on the LanceDB cloud");
+    }
+    dropColumns(_columnNames) {
+        throw new Error("dropColumns() is not yet supported on the LanceDB cloud");
+    }
+    async version() {
+        const resp = await this.#client.post(`${this.#tablePrefix}/describe/`);
+        return resp.version;
+    }
+    checkout(_version) {
+        throw new Error("checkout() is not yet supported on the LanceDB cloud");
+    }
+    checkoutLatest() {
+        throw new Error("checkoutLatest() is not yet supported on the LanceDB cloud");
+    }
+    restore() {
+        throw new Error("restore() is not yet supported on the LanceDB cloud");
+    }
+    optimize(_options) {
+        throw new Error("optimize() is not yet supported on the LanceDB cloud");
+    }
+    async listIndices() {
+        return await this.#client.post(`${this.#tablePrefix}/index/list/`);
+    }
+    toArrow() {
+        throw new Error("toArrow() is not yet supported on the LanceDB cloud");
+    }
+    mergeInsert(_on) {
+        throw new Error("mergeInsert() is not yet supported on the LanceDB cloud");
+    }
+    async indexStats(_name) {
+        throw new Error("indexStats() is not yet supported on the LanceDB cloud");
+    }
+}
+exports.RemoteTable = RemoteTable;

package/nodejs-artifacts/sanitize.d.ts

@@ -1,5 +1,5 @@
 import type { TKeys } from "apache-arrow/type";
-import { DataType, Date_, Decimal, DenseUnion, Dictionary, Duration, Field, FixedSizeBinary, FixedSizeList, Float, Int, Interval, List, Map_, Schema, SparseUnion, Struct, Time, Timestamp, TimestampMicrosecond, TimestampMillisecond, TimestampNanosecond, TimestampSecond, Type, Union } from "./arrow";
+import { DataType, Date_, Decimal, DenseUnion, Dictionary, Duration, Field, FixedSizeBinary, FixedSizeList, Float, Int, Interval, List, Map_, Schema, SchemaLike, SparseUnion, Struct, Table, TableLike, Time, Timestamp, TimestampMicrosecond, TimestampMillisecond, TimestampNanosecond, TimestampSecond, Type, Union } from "./arrow";
 export declare function sanitizeMetadata(metadataLike?: unknown): Map<string, string> | undefined;
 export declare function sanitizeInt(typeLike: object): Int<Type.Int | Type.Int8 | Type.Int16 | Type.Int32 | Type.Int64 | Type.Uint8 | Type.Uint16 | Type.Uint32 | Type.Uint64>;
 export declare function sanitizeFloat(typeLike: object): Float<Type.Float | Type.Float16 | Type.Float32 | Type.Float64>;
@@ -27,4 +27,5 @@ export declare function sanitizeField(fieldLike: unknown): Field;
  * instance because they might be using a different instance of apache-arrow
  * than lancedb is using.
  */
-export declare function sanitizeSchema(schemaLike: unknown): Schema;
+export declare function sanitizeSchema(schemaLike: SchemaLike): Schema;
+export declare function sanitizeTable(tableLike: TableLike): Table;

package/nodejs-artifacts/sanitize.js

@@ -13,7 +13,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.sanitizeSchema = exports.sanitizeField = exports.sanitizeType = exports.sanitizeDictionary = exports.sanitizeDuration = exports.sanitizeMap = exports.sanitizeFixedSizeList = exports.sanitizeFixedSizeBinary = exports.sanitizeTypedUnion = exports.sanitizeUnion = exports.sanitizeStruct = exports.sanitizeList = exports.sanitizeInterval = exports.sanitizeTypedTimestamp = exports.sanitizeTimestamp = exports.sanitizeTime = exports.sanitizeDate = exports.sanitizeDecimal = exports.sanitizeFloat = exports.sanitizeInt = exports.sanitizeMetadata = void 0;
+exports.sanitizeTable = exports.sanitizeSchema = exports.sanitizeField = exports.sanitizeType = exports.sanitizeDictionary = exports.sanitizeDuration = exports.sanitizeMap = exports.sanitizeFixedSizeList = exports.sanitizeFixedSizeBinary = exports.sanitizeTypedUnion = exports.sanitizeUnion = exports.sanitizeStruct = exports.sanitizeList = exports.sanitizeInterval = exports.sanitizeTypedTimestamp = exports.sanitizeTimestamp = exports.sanitizeTime = exports.sanitizeDate = exports.sanitizeDecimal = exports.sanitizeFloat = exports.sanitizeInt = exports.sanitizeMetadata = void 0;
+// The utilities in this file help sanitize data from the user's arrow
+// library into the types expected by vectordb's arrow library.  Node
+// generally allows for mulitple versions of the same library (and sometimes
+// even multiple copies of the same version) to be installed at the same
+// time.  However, arrow-js uses instanceof which expected that the input
+// comes from the exact same library instance.  This is not always the case
+// and so we must sanitize the input to ensure that it is compatible.
+const apache_arrow_1 = require("apache-arrow");
 const arrow_1 = require("./arrow");
 function sanitizeMetadata(metadataLike) {
     if (metadataLike === undefined || metadataLike === null) {
@@ -380,3 +388,49 @@ function sanitizeSchema(schemaLike) {
     return new arrow_1.Schema(sanitizedFields, metadata);
 }
 exports.sanitizeSchema = sanitizeSchema;
+function sanitizeTable(tableLike) {
+    if (tableLike instanceof arrow_1.Table) {
+        return tableLike;
+    }
+    if (typeof tableLike !== "object" || tableLike === null) {
+        throw Error("Expected a Table but object was null/undefined");
+    }
+    if (!("schema" in tableLike)) {
+        throw Error("The table passed in does not appear to be a table (no 'schema' property)");
+    }
+    if (!("batches" in tableLike)) {
+        throw Error("The table passed in does not appear to be a table (no 'columns' property)");
+    }
+    const schema = sanitizeSchema(tableLike.schema);
+    const batches = tableLike.batches.map(sanitizeRecordBatch);
+    return new arrow_1.Table(schema, batches);
+}
+exports.sanitizeTable = sanitizeTable;
+function sanitizeRecordBatch(batchLike) {
+    if (batchLike instanceof arrow_1.RecordBatch) {
+        return batchLike;
+    }
+    if (typeof batchLike !== "object" || batchLike === null) {
+        throw Error("Expected a RecordBatch but object was null/undefined");
+    }
+    if (!("schema" in batchLike)) {
+        throw Error("The record batch passed in does not appear to be a record batch (no 'schema' property)");
+    }
+    if (!("data" in batchLike)) {
+        throw Error("The record batch passed in does not appear to be a record batch (no 'data' property)");
+    }
+    const schema = sanitizeSchema(batchLike.schema);
+    const data = sanitizeData(batchLike.data);
+    return new arrow_1.RecordBatch(schema, data);
+}
+function sanitizeData(dataLike) {
+    if (dataLike instanceof apache_arrow_1.Data) {
+        return dataLike;
+    }
+    return new apache_arrow_1.Data(dataLike.type, dataLike.offset, dataLike.length, dataLike.nullCount, {
+        [apache_arrow_1.BufferType.OFFSET]: dataLike.valueOffsets,
+        [apache_arrow_1.BufferType.DATA]: dataLike.values,
+        [apache_arrow_1.BufferType.VALIDITY]: dataLike.nullBitmap,
+        [apache_arrow_1.BufferType.TYPE]: dataLike.typeIds,
+    });
+}