@lancedb/lancedb 0.4.16 → 0.4.19

package/dist/connection.d.ts

@@ -1,6 +1,19 @@
-import { Connection as LanceDbConnection } from "./native";
+import { ConnectionOptions, Connection as LanceDbConnection } from "./native";
 import { Table } from "./table";
 import { Table as ArrowTable, Schema } from "apache-arrow";
+/**
+ * Connect to a LanceDB instance at the given URI.
+ *
+ * Accepted formats:
+ *
+ * - `/path/to/database` - local database
+ * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
+ * - `db://host:port` - remote database (LanceDB cloud)
+ * @param {string} uri - The uri of the database. If the database uri starts
+ * with `db://` then it connects to a remote database.
+ * @see {@link ConnectionOptions} for more details on the URI format.
+ */
+export declare function connect(uri: string, opts?: Partial<ConnectionOptions>): Promise<Connection>;
 export interface CreateTableOptions {
     /**
      * The mode to use when creating the table.
@@ -17,6 +30,38 @@ export interface CreateTableOptions {
      * then no error will be raised.
      */
     existOk: boolean;
+    /**
+     * Configuration for object storage.
+     *
+     * Options already set on the connection will be inherited by the table,
+     * but can be overridden here.
+     *
+     * The available options are described at https://lancedb.github.io/lancedb/guides/storage/
+     */
+    storageOptions?: Record<string, string>;
+}
+export interface OpenTableOptions {
+    /**
+     * Configuration for object storage.
+     *
+     * Options already set on the connection will be inherited by the table,
+     * but can be overridden here.
+     *
+     * The available options are described at https://lancedb.github.io/lancedb/guides/storage/
+     */
+    storageOptions?: Record<string, string>;
+    /**
+     * Set the size of the index cache, specified as a number of entries
+     *
+     * The exact meaning of an "entry" will depend on the type of index:
+     * - IVF: there is one entry for each IVF partition
+     * - BTREE: there is one entry for the entire index
+     *
+     * This cache applies to the entire opened table, across all indices.
+     * Setting this value higher will increase performance on larger datasets
+     * at the expense of more RAM
+     */
+    indexCacheSize?: number;
 }
 export interface TableNamesOptions {
     /**
@@ -75,7 +120,7 @@ export declare class Connection {
      * Open a table in the database.
      * @param {string} name - The name of the table
      */
-    openTable(name: string): Promise<Table>;
+    openTable(name: string, options?: Partial<OpenTableOptions>): Promise<Table>;
     /**
      * Creates a new Table and initialize it with new data.
      * @param {string} name - The name of the table.

package/dist/connection.js

@@ -13,10 +13,30 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Connection = void 0;
+exports.Connection = exports.connect = void 0;
 const arrow_1 = require("./arrow");
+const native_1 = require("./native");
 const table_1 = require("./table");
 const apache_arrow_1 = require("apache-arrow");
+/**
+ * Connect to a LanceDB instance at the given URI.
+ *
+ * Accepted formats:
+ *
+ * - `/path/to/database` - local database
+ * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
+ * - `db://host:port` - remote database (LanceDB cloud)
+ * @param {string} uri - The uri of the database. If the database uri starts
+ * with `db://` then it connects to a remote database.
+ * @see {@link ConnectionOptions} for more details on the URI format.
+ */
+async function connect(uri, opts) {
+    opts = opts ?? {};
+    opts.storageOptions = cleanseStorageOptions(opts.storageOptions);
+    const nativeConn = await native_1.Connection.new(uri, opts);
+    return new Connection(nativeConn);
+}
+exports.connect = connect;
 /**
  * A LanceDB Connection that allows you to open tables and create new ones.
  *
@@ -72,8 +92,8 @@ class Connection {
      * Open a table in the database.
      * @param {string} name - The name of the table
      */
-    async openTable(name) {
-        const innerTable = await this.inner.openTable(name);
+    async openTable(name, options) {
+        const innerTable = await this.inner.openTable(name, cleanseStorageOptions(options?.storageOptions), options?.indexCacheSize);
         return new table_1.Table(innerTable);
     }
     /**
@@ -96,7 +116,7 @@ class Connection {
             table = (0, arrow_1.makeArrowTable)(data);
         }
         const buf = await (0, arrow_1.fromTableToBuffer)(table);
-        const innerTable = await this.inner.createTable(name, buf, mode);
+        const innerTable = await this.inner.createTable(name, buf, mode, cleanseStorageOptions(options?.storageOptions));
         return new table_1.Table(innerTable);
     }
     /**
@@ -112,7 +132,7 @@ class Connection {
         }
         const table = (0, arrow_1.makeEmptyTable)(schema);
         const buf = await (0, arrow_1.fromTableToBuffer)(table);
-        const innerTable = await this.inner.createEmptyTable(name, buf, mode);
+        const innerTable = await this.inner.createEmptyTable(name, buf, mode, cleanseStorageOptions(options?.storageOptions));
         return new table_1.Table(innerTable);
     }
     /**
@@ -124,3 +144,38 @@ class Connection {
     }
 }
 exports.Connection = Connection;
+/**
+ * Takes storage options and makes all the keys snake case.
+ */
+function cleanseStorageOptions(options) {
+    if (options === undefined) {
+        return undefined;
+    }
+    const result = {};
+    for (const [key, value] of Object.entries(options)) {
+        if (value !== undefined) {
+            const newKey = camelToSnakeCase(key);
+            result[newKey] = value;
+        }
+    }
+    return result;
+}
+/**
+ * Convert a string to snake case. It might already be snake case, in which case it is
+ * returned unchanged.
+ */
+function camelToSnakeCase(camel) {
+    if (camel.includes("_")) {
+        // Assume if there is at least one underscore, it is already snake case
+        return camel;
+    }
+    if (camel.toLocaleUpperCase() === camel) {
+        // Assume if the string is all uppercase, it is already snake case
+        return camel;
+    }
+    let result = camel.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
+    if (result.startsWith("_")) {
+        result = result.slice(1);
+    }
+    return result;
+}

package/dist/index.d.ts

@@ -1,22 +1,7 @@
-import { Connection } from "./connection";
-import { ConnectionOptions } from "./native.js";
 export { WriteOptions, WriteMode, AddColumnsSql, ColumnAlteration, ConnectionOptions, } from "./native.js";
 export { makeArrowTable, MakeArrowTableOptions, Data, VectorColumnOptions, } from "./arrow";
-export { Connection, CreateTableOptions, TableNamesOptions, } from "./connection";
+export { connect, Connection, CreateTableOptions, TableNamesOptions, } from "./connection";
 export { ExecutableQuery, Query, QueryBase, VectorQuery, RecordBatchIterator, } from "./query";
 export { Index, IndexOptions, IvfPqOptions } from "./indices";
 export { Table, AddDataOptions, IndexConfig, UpdateOptions } from "./table";
 export * as embedding from "./embedding";
-/**
- * Connect to a LanceDB instance at the given URI.
- *
- * Accpeted formats:
- *
- * - `/path/to/database` - local database
- * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
- * - `db://host:port` - remote database (LanceDB cloud)
- * @param {string} uri - The uri of the database. If the database uri starts
- * with `db://` then it connects to a remote database.
- * @see {@link ConnectionOptions} for more details on the URI format.
- */
-export declare function connect(uri: string, opts?: Partial<ConnectionOptions>): Promise<Connection>;

package/dist/index.js

@@ -13,15 +13,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.connect = exports.embedding = exports.Table = exports.Index = exports.RecordBatchIterator = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Connection = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = void 0;
-const connection_1 = require("./connection");
-const native_js_1 = require("./native.js");
+exports.embedding = exports.Table = exports.Index = exports.RecordBatchIterator = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Connection = exports.connect = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = void 0;
 var arrow_1 = require("./arrow");
 Object.defineProperty(exports, "makeArrowTable", { enumerable: true, get: function () { return arrow_1.makeArrowTable; } });
 Object.defineProperty(exports, "MakeArrowTableOptions", { enumerable: true, get: function () { return arrow_1.MakeArrowTableOptions; } });
 Object.defineProperty(exports, "VectorColumnOptions", { enumerable: true, get: function () { return arrow_1.VectorColumnOptions; } });
-var connection_2 = require("./connection");
-Object.defineProperty(exports, "Connection", { enumerable: true, get: function () { return connection_2.Connection; } });
+var connection_1 = require("./connection");
+Object.defineProperty(exports, "connect", { enumerable: true, get: function () { return connection_1.connect; } });
+Object.defineProperty(exports, "Connection", { enumerable: true, get: function () { return connection_1.Connection; } });
 var query_1 = require("./query");
 Object.defineProperty(exports, "Query", { enumerable: true, get: function () { return query_1.Query; } });
 Object.defineProperty(exports, "QueryBase", { enumerable: true, get: function () { return query_1.QueryBase; } });
@@ -32,21 +31,3 @@ Object.defineProperty(exports, "Index", { enumerable: true, get: function () { r
 var table_1 = require("./table");
 Object.defineProperty(exports, "Table", { enumerable: true, get: function () { return table_1.Table; } });
 exports.embedding = require("./embedding");
-/**
- * Connect to a LanceDB instance at the given URI.
- *
- * Accpeted formats:
- *
- * - `/path/to/database` - local database
- * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
- * - `db://host:port` - remote database (LanceDB cloud)
- * @param {string} uri - The uri of the database. If the database uri starts
- * with `db://` then it connects to a remote database.
- * @see {@link ConnectionOptions} for more details on the URI format.
- */
-async function connect(uri, opts) {
-    opts = opts ?? {};
-    const nativeConn = await native_js_1.Connection.new(uri, opts);
-    return new connection_1.Connection(nativeConn);
-}
-exports.connect = connect;

package/dist/native.d.ts

@@ -62,6 +62,12 @@ export interface ConnectionOptions {
    * always consistent.
    */
   readConsistencyInterval?: number
+  /**
+   * (For LanceDB OSS only): configuration for object storage.
+   *
+   * The available options are described at https://lancedb.github.io/lancedb/guides/storage/
+   */
+  storageOptions?: Record<string, string>
 }
 /** Write mode for writing a table. */
 export const enum WriteMode {
@@ -73,7 +79,9 @@ export const enum WriteMode {
 export interface WriteOptions {
   mode?: WriteMode
 }
-export function connect(uri: string, options: ConnectionOptions): Promise<Connection>
+export interface OpenTableOptions {
+  storageOptions?: Record<string, string>
+}
 export class Connection {
   /** Create a new Connection instance from the given URI. */
   static new(uri: string, options: ConnectionOptions): Promise<Connection>
@@ -90,9 +98,9 @@ export class Connection {
    * - buf: The buffer containing the IPC file.
    *
    */
-  createTable(name: string, buf: Buffer, mode: string): Promise<Table>
-  createEmptyTable(name: string, schemaBuf: Buffer, mode: string): Promise<Table>
-  openTable(name: string): Promise<Table>
+  createTable(name: string, buf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null): Promise<Table>
+  createEmptyTable(name: string, schemaBuf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null): Promise<Table>
+  openTable(name: string, storageOptions?: Record<string, string> | undefined | null, indexCacheSize?: number | undefined | null): Promise<Table>
   /** Drop table with the name. Or raise an error if the table does not exist. */
   dropTable(name: string): Promise<void>
 }

package/dist/native.js

@@ -303,7 +303,7 @@ if (!nativeBinding) {
     }
     throw new Error(`Failed to load native binding`);
 }
-const { Connection, Index, RecordBatchIterator, Query, VectorQuery, Table, WriteMode, connect } = nativeBinding;
+const { Connection, Index, RecordBatchIterator, Query, VectorQuery, Table, WriteMode } = nativeBinding;
 module.exports.Connection = Connection;
 module.exports.Index = Index;
 module.exports.RecordBatchIterator = RecordBatchIterator;
@@ -311,4 +311,3 @@ module.exports.Query = Query;
 module.exports.VectorQuery = VectorQuery;
 module.exports.Table = Table;
 module.exports.WriteMode = WriteMode;
-module.exports.connect = connect;

package/dist/table.d.ts

@@ -104,17 +104,20 @@ export declare class Table {
      * // If the column has a vector (fixed size list) data type then
      * // an IvfPq vector index will be created.
      * const table = await conn.openTable("my_table");
-     * await table.createIndex(["vector"]);
+     * await table.createIndex("vector");
      * @example
      * // For advanced control over vector index creation you can specify
      * // the index type and options.
      * const table = await conn.openTable("my_table");
-     * await table.createIndex(["vector"], I)
-     *   .ivf_pq({ num_partitions: 128, num_sub_vectors: 16 })
-     *   .build();
+     * await table.createIndex("vector", {
+     *   config: lancedb.Index.ivfPq({
+     *     numPartitions: 128,
+     *     numSubVectors: 16,
+     *   }),
+     * });
      * @example
      * // Or create a Scalar index
-     * await table.createIndex("my_float_col").build();
+     * await table.createIndex("my_float_col");
      */
     createIndex(column: string, options?: Partial<IndexOptions>): Promise<void>;
     /**
@@ -126,8 +129,7 @@ export declare class Table {
      * vector similarity, sorting, and more.
      *
      * Note: By default, all columns are returned.  For best performance, you should
-     * only fetch the columns you need.  See [`Query::select_with_projection`] for
-     * more details.
+     * only fetch the columns you need.
      *
      * When appropriate, various indices and statistics based pruning will be used to
      * accelerate the query.
@@ -135,10 +137,13 @@ export declare class Table {
      * // SQL-style filtering
      * //
      * // This query will return up to 1000 rows whose value in the `id` column
-     * // is greater than 5.  LanceDb supports a broad set of filtering functions.
-     * for await (const batch of table.query()
-     *                          .filter("id > 1").select(["id"]).limit(20)) {
-     *  console.log(batch);
+     * // is greater than 5. LanceDb supports a broad set of filtering functions.
+     * for await (const batch of table
+     *   .query()
+     *   .where("id > 1")
+     *   .select(["id"])
+     *   .limit(20)) {
+     *   console.log(batch);
      * }
      * @example
      * // Vector Similarity Search
@@ -147,13 +152,14 @@ export declare class Table {
      * // closest to the query vector [1.0, 2.0, 3.0].  If an index has been created
      * // on the "vector" column then this will perform an ANN search.
      * //
-     * // The `refine_factor` and `nprobes` methods are used to control the recall /
+     * // The `refineFactor` and `nprobes` methods are used to control the recall /
      * // latency tradeoff of the search.
-     * for await (const batch of table.query()
-     *                    .nearestTo([1, 2, 3])
-     *                    .refineFactor(5).nprobe(10)
-     *                    .limit(10)) {
-     *  console.log(batch);
+     * for await (const batch of table
+     *   .query()
+     *   .where("id > 1")
+     *   .select(["id"])
+     *   .limit(20)) {
+     *   console.log(batch);
      * }
      * @example
      * // Scan the full dataset
@@ -199,37 +205,39 @@ export declare class Table {
      * (e.g. "a").
      */
     dropColumns(columnNames: string[]): Promise<void>;
-    /**
-     * Retrieve the version of the table
-     *
-     * LanceDb supports versioning.  Every operation that modifies the table increases
-     * version.  As long as a version hasn't been deleted you can `[Self::checkout]` that
-     * version to view the data at that point.  In addition, you can `[Self::restore]` the
-     * version to replace the current table with a previous version.
-     */
+    /** Retrieve the version of the table */
     version(): Promise<number>;
     /**
-     * Checks out a specific version of the Table
+     * Checks out a specific version of the table _This is an in-place operation._
      *
-     * Any read operation on the table will now access the data at the checked out version.
-     * As a consequence, calling this method will disable any read consistency interval
-     * that was previously set.
+     * This allows viewing previous versions of the table. If you wish to
+     * keep writing to the dataset starting from an old version, then use
+     * the `restore` function.
      *
-     * This is a read-only operation that turns the table into a sort of "view"
-     * or "detached head".  Other table instances will not be affected.  To make the change
-     * permanent you can use the `[Self::restore]` method.
-     *
-     * Any operation that modifies the table will fail while the table is in a checked
-     * out state.
+     * Calling this method will set the table into time-travel mode. If you
+     * wish to return to standard mode, call `checkoutLatest`.
+     * @param {number} version The version to checkout
+     * @example
+     * ```typescript
+     * import * as lancedb from "@lancedb/lancedb"
+     * const db = await lancedb.connect("./.lancedb");
+     * const table = await db.createTable("my_table", [
+     *   { vector: [1.1, 0.9], type: "vector" },
+     * ]);
      *
-     * To return the table to a normal state use `[Self::checkout_latest]`
+     * console.log(await table.version()); // 1
+     * console.log(table.display());
+     * await table.add([{ vector: [0.5, 0.2], type: "vector" }]);
+     * await table.checkout(1);
+     * console.log(await table.version()); // 2
+     * ```
      */
     checkout(version: number): Promise<void>;
     /**
-     * Ensures the table is pointing at the latest version
+     * Checkout the latest version of the table. _This is an in-place operation._
      *
-     * This can be used to manually update a table when the read_consistency_interval is None
-     * It can also be used to undo a `[Self::checkout]` operation
+     * The table will be set back into standard mode, and will track the latest
+     * version of the table.
      */
     checkoutLatest(): Promise<void>;
     /**
@@ -245,8 +253,6 @@ export declare class Table {
      * out state and the read_consistency_interval, if any, will apply.
      */
     restore(): Promise<void>;
-    /**
-     * List all indices that have been created with Self::create_index
-     */
+    /** List all indices that have been created with {@link Table.createIndex} */
     listIndices(): Promise<IndexConfig[]>;
 }

package/dist/table.js

@@ -123,17 +123,20 @@ class Table {
      * // If the column has a vector (fixed size list) data type then
      * // an IvfPq vector index will be created.
      * const table = await conn.openTable("my_table");
-     * await table.createIndex(["vector"]);
+     * await table.createIndex("vector");
      * @example
      * // For advanced control over vector index creation you can specify
      * // the index type and options.
      * const table = await conn.openTable("my_table");
-     * await table.createIndex(["vector"], I)
-     *   .ivf_pq({ num_partitions: 128, num_sub_vectors: 16 })
-     *   .build();
+     * await table.createIndex("vector", {
+     *   config: lancedb.Index.ivfPq({
+     *     numPartitions: 128,
+     *     numSubVectors: 16,
+     *   }),
+     * });
      * @example
      * // Or create a Scalar index
-     * await table.createIndex("my_float_col").build();
+     * await table.createIndex("my_float_col");
      */
     async createIndex(column, options) {
         // Bit of a hack to get around the fact that TS has no package-scope.
@@ -150,8 +153,7 @@ class Table {
      * vector similarity, sorting, and more.
      *
      * Note: By default, all columns are returned.  For best performance, you should
-     * only fetch the columns you need.  See [`Query::select_with_projection`] for
-     * more details.
+     * only fetch the columns you need.
      *
      * When appropriate, various indices and statistics based pruning will be used to
      * accelerate the query.
@@ -159,10 +161,13 @@ class Table {
      * // SQL-style filtering
      * //
      * // This query will return up to 1000 rows whose value in the `id` column
-     * // is greater than 5.  LanceDb supports a broad set of filtering functions.
-     * for await (const batch of table.query()
-     *                          .filter("id > 1").select(["id"]).limit(20)) {
-     *  console.log(batch);
+     * // is greater than 5. LanceDb supports a broad set of filtering functions.
+     * for await (const batch of table
+     *   .query()
+     *   .where("id > 1")
+     *   .select(["id"])
+     *   .limit(20)) {
+     *   console.log(batch);
      * }
      * @example
      * // Vector Similarity Search
@@ -171,13 +176,14 @@ class Table {
      * // closest to the query vector [1.0, 2.0, 3.0].  If an index has been created
      * // on the "vector" column then this will perform an ANN search.
      * //
-     * // The `refine_factor` and `nprobes` methods are used to control the recall /
+     * // The `refineFactor` and `nprobes` methods are used to control the recall /
      * // latency tradeoff of the search.
-     * for await (const batch of table.query()
-     *                    .nearestTo([1, 2, 3])
-     *                    .refineFactor(5).nprobe(10)
-     *                    .limit(10)) {
-     *  console.log(batch);
+     * for await (const batch of table
+     *   .query()
+     *   .where("id > 1")
+     *   .select(["id"])
+     *   .limit(20)) {
+     *   console.log(batch);
      * }
      * @example
      * // Scan the full dataset
@@ -234,41 +240,43 @@ class Table {
     async dropColumns(columnNames) {
         await this.inner.dropColumns(columnNames);
     }
-    /**
-     * Retrieve the version of the table
-     *
-     * LanceDb supports versioning.  Every operation that modifies the table increases
-     * version.  As long as a version hasn't been deleted you can `[Self::checkout]` that
-     * version to view the data at that point.  In addition, you can `[Self::restore]` the
-     * version to replace the current table with a previous version.
-     */
+    /** Retrieve the version of the table */
     async version() {
         return await this.inner.version();
     }
     /**
-     * Checks out a specific version of the Table
+     * Checks out a specific version of the table _This is an in-place operation._
      *
-     * Any read operation on the table will now access the data at the checked out version.
-     * As a consequence, calling this method will disable any read consistency interval
-     * that was previously set.
+     * This allows viewing previous versions of the table. If you wish to
+     * keep writing to the dataset starting from an old version, then use
+     * the `restore` function.
      *
-     * This is a read-only operation that turns the table into a sort of "view"
-     * or "detached head".  Other table instances will not be affected.  To make the change
-     * permanent you can use the `[Self::restore]` method.
-     *
-     * Any operation that modifies the table will fail while the table is in a checked
-     * out state.
+     * Calling this method will set the table into time-travel mode. If you
+     * wish to return to standard mode, call `checkoutLatest`.
+     * @param {number} version The version to checkout
+     * @example
+     * ```typescript
+     * import * as lancedb from "@lancedb/lancedb"
+     * const db = await lancedb.connect("./.lancedb");
+     * const table = await db.createTable("my_table", [
+     *   { vector: [1.1, 0.9], type: "vector" },
+     * ]);
      *
-     * To return the table to a normal state use `[Self::checkout_latest]`
+     * console.log(await table.version()); // 1
+     * console.log(table.display());
+     * await table.add([{ vector: [0.5, 0.2], type: "vector" }]);
+     * await table.checkout(1);
+     * console.log(await table.version()); // 2
+     * ```
      */
     async checkout(version) {
         await this.inner.checkout(version);
     }
     /**
-     * Ensures the table is pointing at the latest version
+     * Checkout the latest version of the table. _This is an in-place operation._
      *
-     * This can be used to manually update a table when the read_consistency_interval is None
-     * It can also be used to undo a `[Self::checkout]` operation
+     * The table will be set back into standard mode, and will track the latest
+     * version of the table.
      */
     async checkoutLatest() {
         await this.inner.checkoutLatest();
@@ -288,9 +296,7 @@ class Table {
     async restore() {
         await this.inner.restore();
     }
-    /**
-     * List all indices that have been created with Self::create_index
-     */
+    /** List all indices that have been created with {@link Table.createIndex} */
     async listIndices() {
         return await this.inner.listIndices();
     }