@lancedb/lancedb 0.5.1 → 0.7.0

package/dist/table.js

@@ -13,10 +13,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Table = void 0;
+exports.LocalTable = exports.Table = void 0;
 const arrow_1 = require("./arrow");
 const registry_1 = require("./embedding/registry");
+const merge_1 = require("./merge");
 const query_1 = require("./query");
+const sanitize_1 = require("./sanitize");
+const util_1 = require("./util");
 /**
  * A Table is a collection of Records in a LanceDB Database.
  *
@@ -30,305 +33,173 @@ const query_1 = require("./query");
  * collected.
  */
 class Table {
+    [Symbol.for("nodejs.util.inspect.custom")]() {
+        return this.display();
+    }
+    static async parseTableData(data, options, streaming = false) {
+        let mode = options?.mode ?? "create";
+        const existOk = options?.existOk ?? false;
+        if (mode === "create" && existOk) {
+            mode = "exist_ok";
+        }
+        let table;
+        if ((0, arrow_1.isArrowTable)(data)) {
+            table = (0, sanitize_1.sanitizeTable)(data);
+        }
+        else {
+            table = (0, arrow_1.makeArrowTable)(data, options);
+        }
+        if (streaming) {
+            const buf = await (0, arrow_1.fromTableToStreamBuffer)(table, options?.embeddingFunction, options?.schema);
+            return { buf, mode };
+        }
+        else {
+            const buf = await (0, arrow_1.fromTableToBuffer)(table, options?.embeddingFunction, options?.schema);
+            return { buf, mode };
+        }
+    }
+}
+exports.Table = Table;
+class LocalTable extends Table {
     inner;
-    /** Construct a Table. Internal use only. */
     constructor(inner) {
+        super();
         this.inner = inner;
     }
-    /** Return true if the table has not been closed */
+    get name() {
+        return this.inner.name;
+    }
     isOpen() {
         return this.inner.isOpen();
     }
-    /**
-     * Close the table, releasing any underlying resources.
-     *
-     * It is safe to call this method multiple times.
-     *
-     * Any attempt to use the table after it is closed will result in an error.
-     */
     close() {
         this.inner.close();
     }
-    /** Return a brief description of the table */
     display() {
         return this.inner.display();
     }
+    async getEmbeddingFunctions() {
+        const schema = await this.schema();
+        const registry = (0, registry_1.getRegistry)();
+        return registry.parseFunctions(schema.metadata);
+    }
     /** Get the schema of the table. */
     async schema() {
         const schemaBuf = await this.inner.schema();
         const tbl = (0, arrow_1.tableFromIPC)(schemaBuf);
         return tbl.schema;
     }
-    /**
-     * Insert records into this Table.
-     * @param {Data} data Records to be inserted into the Table
-     */
     async add(data, options) {
         const mode = options?.mode ?? "append";
         const schema = await this.schema();
         const registry = (0, registry_1.getRegistry)();
         const functions = registry.parseFunctions(schema.metadata);
-        const buffer = await (0, arrow_1.fromDataToBuffer)(data, functions.values().next().value);
+        const buffer = await (0, arrow_1.fromDataToBuffer)(data, functions.values().next().value, schema);
         await this.inner.add(buffer, mode);
     }
-    /**
-     * Update existing records in the Table
-     *
-     * An update operation can be used to adjust existing values.  Use the
-     * returned builder to specify which columns to update.  The new value
-     * can be a literal value (e.g. replacing nulls with some default value)
-     * or an expression applied to the old value (e.g. incrementing a value)
-     *
-     * An optional condition can be specified (e.g. "only update if the old
-     * value is 0")
-     *
-     * Note: if your condition is something like "some_id_column == 7" and
-     * you are updating many rows (with different ids) then you will get
-     * better performance with a single [`merge_insert`] call instead of
-     * repeatedly calilng this method.
-     * @param {Map<string, string> | Record<string, string>} updates - the
-     * columns to update
-     *
-     * Keys in the map should specify the name of the column to update.
-     * Values in the map provide the new value of the column.  These can
-     * be SQL literal strings (e.g. "7" or "'foo'") or they can be expressions
-     * based on the row being updated (e.g. "my_col + 1")
-     * @param {Partial<UpdateOptions>} options - additional options to control
-     * the update behavior
-     */
-    async update(updates, options) {
-        const onlyIf = options?.where;
+    async update(optsOrUpdates, options) {
+        const isValues = "values" in optsOrUpdates && typeof optsOrUpdates.values !== "string";
+        const isValuesSql = "valuesSql" in optsOrUpdates &&
+            typeof optsOrUpdates.valuesSql !== "string";
+        const isMap = (obj) => {
+            return obj instanceof Map;
+        };
+        let predicate;
         let columns;
-        if (updates instanceof Map) {
-            columns = Array.from(updates.entries());
-        }
-        else {
-            columns = Object.entries(updates);
+        switch (true) {
+            case isMap(optsOrUpdates):
+                columns = Array.from(optsOrUpdates.entries());
+                predicate = options?.where;
+                break;
+            case isValues && isMap(optsOrUpdates.values):
+                columns = Array.from(optsOrUpdates.values.entries()).map(([k, v]) => [
+                    k,
+                    (0, util_1.toSQL)(v),
+                ]);
+                predicate = optsOrUpdates.where;
+                break;
+            case isValues && !isMap(optsOrUpdates.values):
+                columns = Object.entries(optsOrUpdates.values).map(([k, v]) => [
+                    k,
+                    (0, util_1.toSQL)(v),
+                ]);
+                predicate = optsOrUpdates.where;
+                break;
+            case isValuesSql && isMap(optsOrUpdates.valuesSql):
+                columns = Array.from(optsOrUpdates.valuesSql.entries());
+                predicate = optsOrUpdates.where;
+                break;
+            case isValuesSql && !isMap(optsOrUpdates.valuesSql):
+                columns = Object.entries(optsOrUpdates.valuesSql).map(([k, v]) => [
+                    k,
+                    v,
+                ]);
+                predicate = optsOrUpdates.where;
+                break;
+            default:
+                columns = Object.entries(optsOrUpdates);
+                predicate = options?.where;
         }
-        await this.inner.update(onlyIf, columns);
+        await this.inner.update(predicate, columns);
     }
-    /** Count the total number of rows in the dataset. */
     async countRows(filter) {
         return await this.inner.countRows(filter);
     }
-    /** Delete the rows that satisfy the predicate. */
     async delete(predicate) {
         await this.inner.delete(predicate);
     }
-    /**
-     * Create an index to speed up queries.
-     *
-     * Indices can be created on vector columns or scalar columns.
-     * Indices on vector columns will speed up vector searches.
-     * Indices on scalar columns will speed up filtering (in both
-     * vector and non-vector searches)
-     * @example
-     * // If the column has a vector (fixed size list) data type then
-     * // an IvfPq vector index will be created.
-     * const table = await conn.openTable("my_table");
-     * await table.createIndex("vector");
-     * @example
-     * // For advanced control over vector index creation you can specify
-     * // the index type and options.
-     * const table = await conn.openTable("my_table");
-     * await table.createIndex("vector", {
-     *   config: lancedb.Index.ivfPq({
-     *     numPartitions: 128,
-     *     numSubVectors: 16,
-     *   }),
-     * });
-     * @example
-     * // Or create a Scalar index
-     * await table.createIndex("my_float_col");
-     */
     async createIndex(column, options) {
         // Bit of a hack to get around the fact that TS has no package-scope.
         // biome-ignore lint/suspicious/noExplicitAny: skip
         const nativeIndex = options?.config?.inner;
         await this.inner.createIndex(nativeIndex, column, options?.replace);
     }
-    /**
-     * Create a {@link Query} Builder.
-     *
-     * Queries allow you to search your existing data.  By default the query will
-     * return all the data in the table in no particular order.  The builder
-     * returned by this method can be used to control the query using filtering,
-     * vector similarity, sorting, and more.
-     *
-     * Note: By default, all columns are returned.  For best performance, you should
-     * only fetch the columns you need.
-     *
-     * When appropriate, various indices and statistics based pruning will be used to
-     * accelerate the query.
-     * @example
-     * // SQL-style filtering
-     * //
-     * // This query will return up to 1000 rows whose value in the `id` column
-     * // is greater than 5. LanceDb supports a broad set of filtering functions.
-     * for await (const batch of table
-     *   .query()
-     *   .where("id > 1")
-     *   .select(["id"])
-     *   .limit(20)) {
-     *   console.log(batch);
-     * }
-     * @example
-     * // Vector Similarity Search
-     * //
-     * // This example will find the 10 rows whose value in the "vector" column are
-     * // closest to the query vector [1.0, 2.0, 3.0].  If an index has been created
-     * // on the "vector" column then this will perform an ANN search.
-     * //
-     * // The `refineFactor` and `nprobes` methods are used to control the recall /
-     * // latency tradeoff of the search.
-     * for await (const batch of table
-     *   .query()
-     *   .where("id > 1")
-     *   .select(["id"])
-     *   .limit(20)) {
-     *   console.log(batch);
-     * }
-     * @example
-     * // Scan the full dataset
-     * //
-     * // This query will return everything in the table in no particular order.
-     * for await (const batch of table.query()) {
-     *   console.log(batch);
-     * }
-     * @returns {Query} A builder that can be used to parameterize the query
-     */
     query() {
         return new query_1.Query(this.inner);
     }
-    /**
-     * Search the table with a given query vector.
-     *
-     * This is a convenience method for preparing a vector query and
-     * is the same thing as calling `nearestTo` on the builder returned
-     * by `query`.  @see {@link Query#nearestTo} for more details.
-     */
+    search(query) {
+        if (typeof query !== "string") {
+            return this.vectorSearch(query);
+        }
+        else {
+            const queryPromise = this.getEmbeddingFunctions().then(async (functions) => {
+                // TODO: Support multiple embedding functions
+                const embeddingFunc = functions
+                    .values()
+                    .next().value;
+                if (!embeddingFunc) {
+                    return Promise.reject(new Error("No embedding functions are defined in the table"));
+                }
+                return await embeddingFunc.function.computeQueryEmbeddings(query);
+            });
+            return this.query().nearestTo(queryPromise);
+        }
+    }
     vectorSearch(vector) {
         return this.query().nearestTo(vector);
     }
     // TODO: Support BatchUDF
-    /**
-     * Add new columns with defined values.
-     * @param {AddColumnsSql[]} newColumnTransforms pairs of column names and
-     * the SQL expression to use to calculate the value of the new column. These
-     * expressions will be evaluated for each row in the table, and can
-     * reference existing columns in the table.
-     */
     async addColumns(newColumnTransforms) {
         await this.inner.addColumns(newColumnTransforms);
     }
-    /**
-     * Alter the name or nullability of columns.
-     * @param {ColumnAlteration[]} columnAlterations One or more alterations to
-     * apply to columns.
-     */
     async alterColumns(columnAlterations) {
         await this.inner.alterColumns(columnAlterations);
     }
-    /**
-     * Drop one or more columns from the dataset
-     *
-     * This is a metadata-only operation and does not remove the data from the
-     * underlying storage. In order to remove the data, you must subsequently
-     * call ``compact_files`` to rewrite the data without the removed columns and
-     * then call ``cleanup_files`` to remove the old files.
-     * @param {string[]} columnNames The names of the columns to drop. These can
-     * be nested column references (e.g. "a.b.c") or top-level column names
-     * (e.g. "a").
-     */
     async dropColumns(columnNames) {
         await this.inner.dropColumns(columnNames);
     }
-    /** Retrieve the version of the table */
     async version() {
         return await this.inner.version();
     }
-    /**
-     * Checks out a specific version of the table _This is an in-place operation._
-     *
-     * This allows viewing previous versions of the table. If you wish to
-     * keep writing to the dataset starting from an old version, then use
-     * the `restore` function.
-     *
-     * Calling this method will set the table into time-travel mode. If you
-     * wish to return to standard mode, call `checkoutLatest`.
-     * @param {number} version The version to checkout
-     * @example
-     * ```typescript
-     * import * as lancedb from "@lancedb/lancedb"
-     * const db = await lancedb.connect("./.lancedb");
-     * const table = await db.createTable("my_table", [
-     *   { vector: [1.1, 0.9], type: "vector" },
-     * ]);
-     *
-     * console.log(await table.version()); // 1
-     * console.log(table.display());
-     * await table.add([{ vector: [0.5, 0.2], type: "vector" }]);
-     * await table.checkout(1);
-     * console.log(await table.version()); // 2
-     * ```
-     */
     async checkout(version) {
         await this.inner.checkout(version);
     }
-    /**
-     * Checkout the latest version of the table. _This is an in-place operation._
-     *
-     * The table will be set back into standard mode, and will track the latest
-     * version of the table.
-     */
     async checkoutLatest() {
         await this.inner.checkoutLatest();
     }
-    /**
-     * Restore the table to the currently checked out version
-     *
-     * This operation will fail if checkout has not been called previously
-     *
-     * This operation will overwrite the latest version of the table with a
-     * previous version.  Any changes made since the checked out version will
-     * no longer be visible.
-     *
-     * Once the operation concludes the table will no longer be in a checked
-     * out state and the read_consistency_interval, if any, will apply.
-     */
     async restore() {
         await this.inner.restore();
     }
-    /**
-     * Optimize the on-disk data and indices for better performance.
-     *
-     * Modeled after ``VACUUM`` in PostgreSQL.
-     *
-     *  Optimization covers three operations:
-     *
-     *  - Compaction: Merges small files into larger ones
-     *  - Prune: Removes old versions of the dataset
-     *  - Index: Optimizes the indices, adding new data to existing indices
-     *
-     *
-     *  Experimental API
-     *  ----------------
-     *
-     *  The optimization process is undergoing active development and may change.
-     *  Our goal with these changes is to improve the performance of optimization and
-     *  reduce the complexity.
-     *
-     *  That being said, it is essential today to run optimize if you want the best
-     *  performance.  It should be stable and safe to use in production, but it our
-     *  hope that the API may be simplified (or not even need to be called) in the
-     *  future.
-     *
-     *  The frequency an application shoudl call optimize is based on the frequency of
-     *  data modifications.  If data is frequently added, deleted, or updated then
-     *  optimize should be run frequently.  A good rule of thumb is to run optimize if
-     *  you have added or modified 100,000 or more records or run more than 20 data
-     *  modification operations.
-     */
     async optimize(options) {
         let cleanupOlderThanMs;
         if (options?.cleanupOlderThan !== undefined &&
@@ -338,9 +209,22 @@ class Table {
         }
         return await this.inner.optimize(cleanupOlderThanMs);
     }
-    /** List all indices that have been created with {@link Table.createIndex} */
     async listIndices() {
         return await this.inner.listIndices();
     }
+    async toArrow() {
+        return await this.query().toArrow();
+    }
+    async indexStats(name) {
+        const stats = await this.inner.indexStats(name);
+        if (stats === null) {
+            return undefined;
+        }
+        return stats;
+    }
+    mergeInsert(on) {
+        on = Array.isArray(on) ? on : [on];
+        return new merge_1.MergeInsertBuilder(this.inner.mergeInsert(on));
+    }
 }
-exports.Table = Table;
+exports.LocalTable = LocalTable;

package/dist/util.d.ts

@@ -0,0 +1,14 @@
+/// <reference types="node" />
+export type IntoSql = string | number | boolean | null | Date | ArrayBufferLike | Buffer | IntoSql[];
+export declare function toSQL(value: IntoSql): string;
+export declare class TTLCache {
+    private readonly ttl;
+    private readonly cache;
+    /**
+     * @param ttl Time to live in milliseconds
+     */
+    constructor(ttl: number);
+    get(key: string): any | undefined;
+    set(key: string, value: any): void;
+    delete(key: string): void;
+}

package/dist/util.js

@@ -0,0 +1,65 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TTLCache = exports.toSQL = void 0;
+function toSQL(value) {
+    if (typeof value === "string") {
+        return `'${value.replace(/'/g, "''")}'`;
+    }
+    else if (typeof value === "number") {
+        return value.toString();
+    }
+    else if (typeof value === "boolean") {
+        return value ? "TRUE" : "FALSE";
+    }
+    else if (value === null) {
+        return "NULL";
+    }
+    else if (value instanceof Date) {
+        return `'${value.toISOString()}'`;
+    }
+    else if (Array.isArray(value)) {
+        return `[${value.map(toSQL).join(", ")}]`;
+    }
+    else if (Buffer.isBuffer(value)) {
+        return `X'${value.toString("hex")}'`;
+    }
+    else if (value instanceof ArrayBuffer) {
+        return `X'${Buffer.from(value).toString("hex")}'`;
+    }
+    else {
+        throw new Error(`Unsupported value type: ${typeof value} value: (${value})`);
+    }
+}
+exports.toSQL = toSQL;
+class TTLCache {
+    ttl;
+    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+    cache;
+    /**
+     * @param ttl Time to live in milliseconds
+     */
+    constructor(ttl) {
+        this.ttl = ttl;
+        this.cache = new Map();
+    }
+    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+    get(key) {
+        const entry = this.cache.get(key);
+        if (entry === undefined) {
+            return undefined;
+        }
+        if (entry.expires < Date.now()) {
+            this.cache.delete(key);
+            return undefined;
+        }
+        return entry.value;
+    }
+    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+    set(key, value) {
+        this.cache.set(key, { value, expires: Date.now() + this.ttl });
+    }
+    delete(key) {
+        this.cache.delete(key);
+    }
+}
+exports.TTLCache = TTLCache;

package/examples/ann_indexes.ts

@@ -0,0 +1,49 @@
+// --8<-- [start:import]
+import * as lancedb from "@lancedb/lancedb";
+// --8<-- [end:import]
+
+// --8<-- [start:ingest]
+const db = await lancedb.connect("/tmp/lancedb/");
+
+const data = Array.from({ length: 10_000 }, (_, i) => ({
+  vector: Array(1536).fill(i),
+  id: `${i}`,
+  content: "",
+  longId: `${i}`,
+}));
+
+const table = await db.createTable("my_vectors", data, { mode: "overwrite" });
+await table.createIndex("vector", {
+  config: lancedb.Index.ivfPq({
+    numPartitions: 16,
+    numSubVectors: 48,
+  }),
+});
+// --8<-- [end:ingest]
+
+// --8<-- [start:search1]
+const _results1 = await table
+  .search(Array(1536).fill(1.2))
+  .limit(2)
+  .nprobes(20)
+  .refineFactor(10)
+  .toArray();
+// --8<-- [end:search1]
+
+// --8<-- [start:search2]
+const _results2 = await table
+  .search(Array(1536).fill(1.2))
+  .where("id != '1141'")
+  .limit(2)
+  .toArray();
+// --8<-- [end:search2]
+
+// --8<-- [start:search3]
+const _results3 = await table
+  .search(Array(1536).fill(1.2))
+  .select(["id"])
+  .limit(2)
+  .toArray();
+// --8<-- [end:search3]
+
+console.log("Ann indexes: done");