@lancedb/lancedb 0.37.1 → 0.38.0-beta.12

package/dist/table.d.ts

@@ -1,10 +1,11 @@
 import { Table as ArrowTable, Data, DataType, Field, IntoVector, MultiVector, Schema } from "./arrow";
 import { IndexOptions } from "./indices";
 import { MergeInsertBuilder } from "./merge";
-import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, IndexConfig, IndexStatistics, Job, Branches as NativeBranches, OptimizeStats, TableStatistics, Tags, UpdateFieldMetadataResult, UpdateResult, Table as _NativeTable } from "./native";
-import { FullTextQuery, Query, TakeQuery, VectorQuery } from "./query";
+import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, IndexConfig, IndexStatistics, Job, LsmStats, Branches as NativeBranches, OptimizeStats, RefreshColumnResult, RefreshMaterializedViewResult, TableStatistics, Tags, UpdateFieldMetadataResult, UpdateResult, Table as _NativeTable } from "./native";
+import { AutoQuery, FullTextQuery, Query, TakeQuery, VectorQuery } from "./query";
 import { IntoSql } from "./util";
 export { IndexConfig } from "./native";
+export { BucketStats, GenerationStats, LsmStats, MemtableStats, } from "./native";
 /**
  * Progress snapshot for a write operation, delivered to the `progress`
  * callback passed to {@link Table.add}.
@@ -142,7 +143,11 @@ export interface LsmWriteSpec {
     column?: string;
     /** Bucket variant: the number of buckets, in `[1, 1024]`. */
     numBuckets?: number;
-    /** Names of indexes the MemWAL should keep up to date during writes. */
+    /**
+     * Indexes the MemWAL keeps up to date. Omit to maintain every supported
+     * index, resolved on install — a snapshot, so indexes created later are not
+     * maintained. Pass `[]` for none.
+     */
     maintainedIndexes?: string[];
     /** Default `ShardWriter` configuration recorded in the MemWAL index. */
     writerConfigDefaults?: Record<string, string>;
@@ -417,7 +422,7 @@ export declare abstract class Table {
      * when "auto" is used, if the query is a string and an embedding function is defined, it will be treated as a vector query
      * if the query is a string and no embedding function is defined, it will be treated as a full text search query
      */
-    abstract search(query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query;
+    abstract search(query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query | AutoQuery;
     /**
      * Search the table with a given query vector.
      *
@@ -428,15 +433,75 @@ export declare abstract class Table {
     abstract vectorSearch(vector: IntoVector | MultiVector): VectorQuery;
     /**
      * Add new columns with defined values.
+     *
+     * The `{ computed }` form stores the expression rather than evaluating it
+     * now: the column is committed with no values, and rows get them from
+     * {@link Table#refreshColumn}. Declaring one therefore costs the same on a
+     * large table as on an empty one.
+     *
+     * A refresh does not revisit rows it has already filled, so mutating an
+     * input leaves the value computed at fill time; recomputing means dropping
+     * the column and declaring it again. While a declaration reads a column,
+     * that column cannot be renamed, retyped or dropped.
+     *
+     * On LanceDB Cloud and Enterprise the expression is planned by the
+     * server, and the refresh runs as a server job -- see
+     * {@link Table#refreshColumnAsync}.
      * @param {AddColumnsSql[] | Field | Field[] | Schema} newColumnTransforms Either:
      *   - An array of objects with column names and SQL expressions to calculate values
      *   - A single Arrow Field defining one column with its data type (column will be initialized with null values)
      *   - An array of Arrow Fields defining columns with their data types (columns will be initialized with null values)
      *   - An Arrow Schema defining columns with their data types (columns will be initialized with null values)
+     *   - `{ computed }`, declaring columns defined by a SQL expression whose type and inputs are derived from it
      * @returns {Promise<AddColumnsResult>} A promise that resolves to an object
      * containing the new version number of the table after adding the columns.
+     * @example
+     * ```ts
+     * await table.addColumns({ computed: [{ name: "doubled", valueSql: "x * 2" }] });
+     * const { rowsFilled } = await table.refreshColumn("doubled");
+     * ```
+     */
+    abstract addColumns(newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema | {
+        computed: AddColumnsSql[];
+    }): Promise<AddColumnsResult>;
+    /**
+     * Fill the rows of a computed column that hold no value yet.
+     *
+     * Rows appended since the last refresh are filled by the next one; rows
+     * already filled are left as they are, so the call is idempotent and does
+     * not observe a mutated input. Local tables only: a remote refresh runs
+     * as a server job, through {@link Table#refreshColumnAsync}.
+     * @param {string} column The name of the computed column to fill.
+     * @returns {Promise<RefreshColumnResult>} A promise that resolves to the
+     * number of rows filled and the new version number of the table.
+     */
+    abstract refreshColumn(column: string): Promise<RefreshColumnResult>;
+    /**
+     * Like {@link Table#refreshColumn}, but returns a handle to the refresh
+     * job instead of blocking until it completes.
+     *
+     * The job may already be complete when returned; callers must not assume
+     * the column is filled until {@link Job.wait} resolves. Invalid input --
+     * an unknown column, or one that is not computed -- rejects here rather
+     * than failing the job. On local tables the job runs in-process; on
+     * LanceDB Cloud and Enterprise it is the server's backfill job.
+     * @param {string} column The name of the computed column to fill.
+     * @example
+     * ```ts
+     * const job = await table.refreshColumnAsync("doubled");
+     * await job.wait();
+     * console.log(await job.status()); // "finished"
+     * ```
+     */
+    abstract refreshColumnAsync(column: string): Promise<Job>;
+    /**
+     * Recompute this table's contents from its materialized-view definition.
+     *
+     * Plumbing for {@link MaterializedView.refresh}, which is the way to call
+     * it: rejects tables that carry no view definition. Local tables only.
+     * @ignore
      */
-    abstract addColumns(newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema): Promise<AddColumnsResult>;
+    abstract refreshMaterializedView(full?: boolean, sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
     /**
      * Alter the name or nullability of columns.
      * @param {ColumnAlteration[]} columnAlterations One or more alterations to
@@ -447,6 +512,18 @@ export declare abstract class Table {
     abstract alterColumns(columnAlterations: ColumnAlteration[]): Promise<AlterColumnsResult>;
     /**
      * Update per-field (column) metadata.
+     *
+     * The following keys are treated specially, by convention, and should be
+     * used when appropriate:
+     *
+     * - `lancedb:description`: for a human-readable description of a field.
+     * - `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
+     *   names the tag category; e.g. `lancedb:tag:model: "clip"`.
+     * - `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
+     *   `feature_v2` might be in the same logical column.
+     * - `lancedb:status`: for status options (`production`, `candidate`,
+     *   `deprecated`, `archived`) to designate the current life cycle state of
+     *   this column.
      * @param {FieldMetadataUpdate[]} updates One or more per-field updates. Each
      * update's metadata is merged into the field's existing metadata by default;
      * a value of `null` deletes that key, and `replace: true` swaps the whole map.
@@ -493,6 +570,11 @@ export declare abstract class Table {
      * All variants require the table to have an unenforced primary key
      * ({@link Table#setUnenforcedPrimaryKey}); bucket sharding additionally
      * requires it to be the single column being bucketed.
+     *
+     * Omitting `maintainedIndexes` maintains every index on the table, resolved
+     * here, failing if one cannot be maintained — name them to install anyway.
+     * Naming them pins an exact set, and a still-building index is rejected
+     * rather than quietly omitted.
      * @param {LsmWriteSpec} spec The sharding spec to install.
      * @returns {Promise<void>}
      * @example
@@ -520,9 +602,10 @@ export declare abstract class Table {
      *
      * Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
      * spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}).
-     * The returned spec — including its `maintainedIndexes` and
-     * `writerConfigDefaults` — mirrors what was passed to
-     * {@link Table#setLsmWriteSpec}.
+     * The returned spec mirrors what was passed to
+     * {@link Table#setLsmWriteSpec}, except that `maintainedIndexes` always
+     * reports the concrete list resolved when the spec was set — `undefined`
+     * never round-trips.
      * @returns {Promise<LsmWriteSpec | undefined>}
      */
     abstract getLsmWriteSpec(): Promise<LsmWriteSpec | undefined>;
@@ -536,6 +619,57 @@ export declare abstract class Table {
      * @returns {Promise<void>}
      */
     abstract closeLsmWriters(): Promise<void>;
+    /**
+     * Seal every bucket's active memtable into a new L0 generation.
+     *
+     * Returns once the seal is committed. Sealing an empty memtable is a no-op,
+     * so this is safe to call repeatedly.
+     * @returns {Promise<void>}
+     */
+    abstract flushLsm(): Promise<void>;
+    /**
+     * Trigger a background L0 → base compaction pass per bucket.
+     *
+     * Returns once the passes are *dispatched*, not once they finish — watch
+     * {@link Table#getLsmStats} for progress, or use
+     * {@link Table#checkpointLsm} to wait for convergence.
+     * @returns {Promise<void>}
+     */
+    abstract compactLsm(): Promise<void>;
+    /**
+     * Converge this table's LSM write path into its base table.
+     *
+     * Seals once, then triggers compaction and polls until the L0 that existed
+     * at the start is gone. The target set is fixed at the start, so
+     * generations created *during* the checkpoint are ignored — that is what
+     * lets it terminate under write load, and what makes it best-effort: it
+     * converges the fresh tier as of some instant. Idempotent, abandonable at
+     * any point, and safe to run on a cadence.
+     *
+     * There is no liveness bound — the compactor pool is shared across tables,
+     * so a checkpoint queued behind unrelated work looks exactly like one that
+     * is merging. The caller owns the deadline.
+     * @returns {Promise<void>}
+     * @example
+     * ```ts
+     * const before = await table.getLsmStats();
+     * await table.checkpointLsm();
+     * const after = await table.getLsmStats();
+     * ```
+     */
+    abstract checkpointLsm(): Promise<void>;
+    /**
+     * Read live per-bucket LSM state.
+     *
+     * Answers "how far behind is my fresh tier", "which bucket is hot", and
+     * "why is my fresh-tier vector search brute-force". Mutates no table state.
+     *
+     * Resolves to `undefined` only when the LSM write path is not enabled.
+     * @param {boolean} includeGenerationRows Also count rows per L0 generation.
+     *   Off by default because each count opens an uncached Lance dataset.
+     * @returns {Promise<LsmStats | undefined>}
+     */
+    abstract getLsmStats(includeGenerationRows?: boolean): Promise<LsmStats | undefined>;
     /** Retrieve the version of the table */
     abstract version(): Promise<number>;
     /**
@@ -717,9 +851,14 @@ export declare class LocalTable extends Table {
     takeOffsets(offsets: number[]): TakeQuery;
     takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery;
     query(): Query;
-    search(query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query;
+    search(query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query | AutoQuery;
     vectorSearch(vector: IntoVector | MultiVector): VectorQuery;
-    addColumns(newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema): Promise<AddColumnsResult>;
+    addColumns(newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema | {
+        computed: AddColumnsSql[];
+    }): Promise<AddColumnsResult>;
+    refreshColumn(column: string): Promise<RefreshColumnResult>;
+    refreshColumnAsync(column: string): Promise<Job>;
+    refreshMaterializedView(full?: boolean, sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
     alterColumns(columnAlterations: ColumnAlteration[]): Promise<AlterColumnsResult>;
     updateFieldMetadata(updates: FieldMetadataUpdate[]): Promise<UpdateFieldMetadataResult>;
     dropColumns(columnNames: string[]): Promise<DropColumnsResult>;
@@ -728,6 +867,10 @@ export declare class LocalTable extends Table {
     unsetLsmWriteSpec(): Promise<void>;
     getLsmWriteSpec(): Promise<LsmWriteSpec | undefined>;
     closeLsmWriters(): Promise<void>;
+    flushLsm(): Promise<void>;
+    compactLsm(): Promise<void>;
+    checkpointLsm(): Promise<void>;
+    getLsmStats(includeGenerationRows?: boolean): Promise<LsmStats | undefined>;
     version(): Promise<number>;
     checkout(version: number | string): Promise<void>;
     checkoutLatest(): Promise<void>;
@@ -810,7 +953,8 @@ export interface FieldMetadataUpdate {
     path: string;
     /**
      * Metadata key/value pairs. Merged into the field's existing metadata by
-     * default; a value of `null` deletes that key.
+     * default; a value of `null` deletes that key. See
+     * {@link Table.updateFieldMetadata} for the conventional `lancedb:*` keys.
      */
     metadata: Record<string, string | null>;
     /** If true, replace the field's entire metadata map instead of merging. */
@@ -844,8 +988,8 @@ export interface BranchRowCountSummary {
     inputsChanged: number;
     deltaAvailable: boolean;
 }
-/** A reason why a branch cannot currently be merged. */
-export interface MergeBlocker {
+/** A reason why a cherry-pick cannot currently land. */
+export interface CherryPickError {
     code: string;
     message: string;
 }
@@ -864,18 +1008,17 @@ export interface BranchDiff {
     changedColumns: BranchColumnChange[];
     addedIndexes: BranchIndexSummary[];
     removedIndexes: BranchIndexSummary[];
-    mergeable: boolean;
-    mergeBlockers: MergeBlocker[];
+    errors: CherryPickError[];
 }
-/** Changes that would be, or were, promoted by a branch merge. */
-export interface MergePreview {
+/** Changes that would be, or were, promoted by a cherry-pick. */
+export interface CherryPickPreview {
     promotedColumns: string[];
 }
-/** Result of previewing or attempting a branch merge. */
-export interface MergeBranchResult {
-    status: "ready" | "rejected" | "notImplemented" | "merged" | "unknown";
+/** Result of previewing or attempting a cherry-pick. */
+export interface CherryPickResult {
+    status: "ready" | "failed" | "notImplemented" | "cherryPicked" | "unknown";
     diff: BranchDiff;
-    preview: MergePreview;
+    preview: CherryPickPreview;
     mainVersionAfter?: number;
 }
 /**
@@ -914,13 +1057,13 @@ export declare class Branches {
     /** Compare a branch against main without modifying either branch. */
     diff(fromBranch: string): Promise<BranchDiff>;
     /**
-     * Merge a branch into main.
+     * Cherry-pick a branch onto main.
      *
-     * Set `dryRun` to `true` to preview the merge. A rejected merge resolves
-     * with `status: "rejected"` instead of throwing.
+     * Set `dryRun` to `true` to preview. A failed cherry-pick resolves
+     * with `status: "failed"` instead of throwing.
      *
-     * @param fromBranch Branch to merge from.
-     * @param dryRun When true, only preview the merge. Defaults to false.
+     * @param fromBranch Branch to cherry-pick from.
+     * @param dryRun When true, only preview. Defaults to false.
      */
-    merge(fromBranch: string, dryRun?: boolean): Promise<MergeBranchResult>;
+    cherryPick(fromBranch: string, dryRun?: boolean): Promise<CherryPickResult>;
 }

package/dist/table.js

@@ -51,8 +51,9 @@ class LocalTable extends Table {
     display() {
         return this.inner.display();
     }
-    async getEmbeddingFunctions() {
-        const schema = await this.schema();
+    async getEmbeddingFunctions(inner = this.inner) {
+        const schemaBuf = await inner.schema();
+        const schema = (0, arrow_1.tableFromIPC)(schemaBuf).schema;
         const registry = (0, registry_1.getRegistry)();
         return registry.parseFunctions(schema.metadata);
     }
@@ -193,12 +194,24 @@ class LocalTable extends Table {
                 columns: ftsColumns,
             });
         }
-        // The query type is auto or vector
-        // fall back to full text search if no embedding functions are defined and the query is a string
-        if (queryType === "auto" &&
-            ((0, registry_1.getRegistry)().length() === 0 || (0, query_1.instanceOfFullTextQuery)(query))) {
-            return this.query().fullTextSearch(query, {
-                columns: ftsColumns,
+        if (queryType === "auto") {
+            if ((0, query_1.instanceOfFullTextQuery)(query)) {
+                return this.query().fullTextSearch(query, {
+                    columns: ftsColumns,
+                });
+            }
+            const columns = typeof ftsColumns === "string" ? [ftsColumns] : (ftsColumns ?? null);
+            return (0, query_1.createAutoQuery)(this.inner, query, columns, async (metadata) => {
+                const functions = await (0, registry_1.getRegistry)().parseFunctions(new Map([["embedding_functions", metadata]]));
+                // TODO: Support multiple embedding functions
+                const embeddingFunc = functions
+                    .values()
+                    .next().value;
+                // The route only calls this callback when embedding metadata exists.
+                // parseFunctions either yields a provider or reports malformed metadata.
+                if (!embeddingFunc)
+                    throw new Error("Invalid embedding function metadata");
+                return await embeddingFunc.function.computeQueryEmbeddings(query);
             });
         }
         const queryPromise = this.getEmbeddingFunctions().then(async (functions) => {
@@ -225,6 +238,12 @@ class LocalTable extends Table {
     }
     // TODO: Support BatchUDF
     async addColumns(newColumnTransforms) {
+        // Columns defined by an expression are declared, not materialized here.
+        if (typeof newColumnTransforms === "object" &&
+            !Array.isArray(newColumnTransforms) &&
+            "computed" in newColumnTransforms) {
+            return await this.inner.addComputedColumns(newColumnTransforms.computed);
+        }
         // Handle single Field -> convert to array of Fields
         if (newColumnTransforms instanceof arrow_1.Field) {
             newColumnTransforms = [newColumnTransforms];
@@ -250,6 +269,15 @@ class LocalTable extends Table {
         }
         throw new Error("Invalid input type for addColumns");
     }
+    async refreshColumn(column) {
+        return await this.inner.refreshColumn(column);
+    }
+    async refreshColumnAsync(column) {
+        return await this.inner.refreshColumnAsync(column);
+    }
+    async refreshMaterializedView(full, sourceVersion) {
+        return await this.inner.refreshMaterializedView(full, sourceVersion);
+    }
     async alterColumns(columnAlterations) {
         const processedAlterations = columnAlterations.map((alteration) => {
             if (typeof alteration.dataType === "string") {
@@ -299,6 +327,18 @@ class LocalTable extends Table {
     async closeLsmWriters() {
         return await this.inner.closeLsmWriters();
     }
+    async flushLsm() {
+        return await this.inner.flushLsm();
+    }
+    async compactLsm() {
+        return await this.inner.compactLsm();
+    }
+    async checkpointLsm() {
+        return await this.inner.checkpointLsm();
+    }
+    async getLsmStats(includeGenerationRows = false) {
+        return (await this.inner.getLsmStats(includeGenerationRows)) ?? undefined;
+    }
     async version() {
         return await this.inner.version();
     }
@@ -443,16 +483,16 @@ class Branches {
         return (await this.#inner.diff(fromBranch));
     }
     /**
-     * Merge a branch into main.
+     * Cherry-pick a branch onto main.
      *
-     * Set `dryRun` to `true` to preview the merge. A rejected merge resolves
-     * with `status: "rejected"` instead of throwing.
+     * Set `dryRun` to `true` to preview. A failed cherry-pick resolves
+     * with `status: "failed"` instead of throwing.
      *
-     * @param fromBranch Branch to merge from.
-     * @param dryRun When true, only preview the merge. Defaults to false.
+     * @param fromBranch Branch to cherry-pick from.
+     * @param dryRun When true, only preview. Defaults to false.
      */
-    async merge(fromBranch, dryRun = false) {
-        return (await this.#inner.merge(fromBranch, dryRun));
+    async cherryPick(fromBranch, dryRun = false) {
+        return (await this.#inner.cherryPick(fromBranch, dryRun));
     }
 }
 exports.Branches = Branches;

package/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.37.1",
+  "version": "0.38.0-beta.12",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -106,13 +106,13 @@
   "optionalDependencies": {
     "@huggingface/transformers": "3.0.2",
     "openai": "4.29.2",
-    "@lancedb/lancedb-darwin-arm64": "0.37.1",
-    "@lancedb/lancedb-linux-x64-gnu": "0.37.1",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.37.1",
-    "@lancedb/lancedb-linux-x64-musl": "0.37.1",
-    "@lancedb/lancedb-linux-arm64-musl": "0.37.1",
-    "@lancedb/lancedb-win32-x64-msvc": "0.37.1",
-    "@lancedb/lancedb-win32-arm64-msvc": "0.37.1"
+    "@lancedb/lancedb-darwin-arm64": "0.38.0-beta.12",
+    "@lancedb/lancedb-linux-x64-gnu": "0.38.0-beta.12",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.38.0-beta.12",
+    "@lancedb/lancedb-linux-x64-musl": "0.38.0-beta.12",
+    "@lancedb/lancedb-linux-arm64-musl": "0.38.0-beta.12",
+    "@lancedb/lancedb-win32-x64-msvc": "0.38.0-beta.12",
+    "@lancedb/lancedb-win32-arm64-msvc": "0.38.0-beta.12"
   },
   "peerDependencies": {
     "@types/node": ">=18",