@lancedb/lancedb 0.4.20 → 0.5.0

package/lancedb/query.ts

@@ -12,14 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { RecordBatch, tableFromIPC, Table as ArrowTable } from "apache-arrow";
+import { Table as ArrowTable, RecordBatch, tableFromIPC } from "apache-arrow";
+import { type IvfPqOptions } from "./indices";
 import {
   RecordBatchIterator as NativeBatchIterator,
   Query as NativeQuery,
   Table as NativeTable,
   VectorQuery as NativeVectorQuery,
 } from "./native";
-import { type IvfPqOptions } from "./indices";
 export class RecordBatchIterator implements AsyncIterator<RecordBatch> {
   private promisedInner?: Promise<NativeBatchIterator>;
   private inner?: NativeBatchIterator;
@@ -29,7 +29,7 @@ export class RecordBatchIterator implements AsyncIterator<RecordBatch> {
     this.promisedInner = promise;
   }
 
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  // biome-ignore lint/suspicious/noExplicitAny: skip
   async next(): Promise<IteratorResult<RecordBatch<any>>> {
     if (this.inner === undefined) {
       this.inner = await this.promisedInner;
@@ -56,7 +56,9 @@ export class QueryBase<
   QueryType,
 > implements AsyncIterable<RecordBatch>
 {
-  protected constructor(protected inner: NativeQueryType) {}
+  protected constructor(protected inner: NativeQueryType) {
+    // intentionally empty
+  }
 
   /**
    * A filter statement to be applied to this query.
@@ -150,7 +152,7 @@ export class QueryBase<
     return new RecordBatchIterator(this.nativeExecute());
   }
 
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  // biome-ignore lint/suspicious/noExplicitAny: skip
   [Symbol.asyncIterator](): AsyncIterator<RecordBatch<any>> {
     const promise = this.nativeExecute();
     return new RecordBatchIterator(promise);
@@ -368,7 +370,7 @@ export class Query extends QueryBase<NativeQuery, Query> {
    * a default `limit` of 10 will be used.  @see {@link Query#limit}
    */
   nearestTo(vector: unknown): VectorQuery {
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    // biome-ignore lint/suspicious/noExplicitAny: skip
     const vectorQuery = this.inner.nearestTo(Float32Array.from(vector as any));
     return new VectorQuery(vectorQuery);
   }

package/lancedb/sanitize.ts

@@ -21,60 +21,60 @@
 // and so we must sanitize the input to ensure that it is compatible.
 
 import {
-  Field,
-  Utf8,
-  FixedSizeBinary,
-  FixedSizeList,
-  Schema,
-  List,
-  Struct,
-  Float,
+  Binary,
   Bool,
+  DataType,
+  DateDay,
+  DateMillisecond,
+  type DateUnit,
   Date_,
   Decimal,
-  DataType,
+  DenseUnion,
   Dictionary,
-  Binary,
-  Float32,
-  Interval,
-  Map_,
   Duration,
-  Union,
-  Time,
-  Timestamp,
-  Type,
-  Null,
+  DurationMicrosecond,
+  DurationMillisecond,
+  DurationNanosecond,
+  DurationSecond,
+  Field,
+  FixedSizeBinary,
+  FixedSizeList,
+  Float,
+  Float16,
+  Float32,
+  Float64,
   Int,
-  type Precision,
-  type DateUnit,
   Int8,
   Int16,
   Int32,
   Int64,
-  Uint8,
-  Uint16,
-  Uint32,
-  Uint64,
-  Float16,
-  Float64,
-  DateDay,
-  DateMillisecond,
-  DenseUnion,
+  Interval,
+  IntervalDayTime,
+  IntervalYearMonth,
+  List,
+  Map_,
+  Null,
+  type Precision,
+  Schema,
   SparseUnion,
-  TimeNanosecond,
+  Struct,
+  Time,
   TimeMicrosecond,
   TimeMillisecond,
+  TimeNanosecond,
   TimeSecond,
-  TimestampNanosecond,
+  Timestamp,
   TimestampMicrosecond,
   TimestampMillisecond,
+  TimestampNanosecond,
   TimestampSecond,
-  IntervalDayTime,
-  IntervalYearMonth,
-  DurationNanosecond,
-  DurationMicrosecond,
-  DurationMillisecond,
-  DurationSecond,
+  Type,
+  Uint8,
+  Uint16,
+  Uint32,
+  Uint64,
+  Union,
+  Utf8,
 } from "apache-arrow";
 import type { IntBitWidth, TKeys, TimeBitWidth } from "apache-arrow/type";
 
@@ -228,7 +228,7 @@ function sanitizeUnion(typeLike: object) {
 
   return new Union(
     typeLike.mode,
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    // biome-ignore lint/suspicious/noExplicitAny: skip
     typeLike.typeIds as any,
     typeLike.children.map((child) => sanitizeField(child)),
   );
@@ -294,7 +294,7 @@ function sanitizeMap(typeLike: object) {
   }
 
   return new Map_(
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    // biome-ignore lint/suspicious/noExplicitAny: skip
     typeLike.children.map((field) => sanitizeField(field)) as any,
     typeLike.keysSorted,
   );
@@ -328,7 +328,7 @@ function sanitizeDictionary(typeLike: object) {
   );
 }
 
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
+// biome-ignore lint/suspicious/noExplicitAny: skip
 function sanitizeType(typeLike: unknown): DataType<any> {
   if (typeof typeLike !== "object" || typeLike === null) {
     throw Error("Expected a Type but object was null/undefined");

package/lancedb/table.ts

@@ -13,15 +13,16 @@
 // limitations under the License.
 
 import { Schema, tableFromIPC } from "apache-arrow";
+import { Data, fromDataToBuffer } from "./arrow";
+import { IndexOptions } from "./indices";
 import {
   AddColumnsSql,
   ColumnAlteration,
   IndexConfig,
+  OptimizeStats,
   Table as _NativeTable,
 } from "./native";
 import { Query, VectorQuery } from "./query";
-import { IndexOptions } from "./indices";
-import { Data, fromDataToBuffer } from "./arrow";
 
 export { IndexConfig } from "./native";
 /**
@@ -50,6 +51,23 @@ export interface UpdateOptions {
   where: string;
 }
 
+export interface OptimizeOptions {
+  /**
+   * If set then all versions older than the given date
+   * be removed.  The current version will never be removed.
+   * The default is 7 days
+   * @example
+   * // Delete all versions older than 1 day
+   * const olderThan = new Date();
+   * olderThan.setDate(olderThan.getDate() - 1));
+   * tbl.cleanupOlderVersions(olderThan);
+   *
+   * // Delete all versions except the current version
+   * tbl.cleanupOlderVersions(new Date());
+   */
+  cleanupOlderThan: Date;
+}
+
 /**
  * A Table is a collection of Records in a LanceDB Database.
  *
@@ -186,7 +204,7 @@ export class Table {
    */
   async createIndex(column: string, options?: Partial<IndexOptions>) {
     // Bit of a hack to get around the fact that TS has no package-scope.
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    // biome-ignore lint/suspicious/noExplicitAny: skip
     const nativeIndex = (options?.config as any)?.inner;
     await this.inner.createIndex(nativeIndex, column, options?.replace);
   }
@@ -352,6 +370,48 @@ export class Table {
     await this.inner.restore();
   }
 
+  /**
+   * Optimize the on-disk data and indices for better performance.
+   *
+   * Modeled after ``VACUUM`` in PostgreSQL.
+   *
+   *  Optimization covers three operations:
+   *
+   *  - Compaction: Merges small files into larger ones
+   *  - Prune: Removes old versions of the dataset
+   *  - Index: Optimizes the indices, adding new data to existing indices
+   *
+   *
+   *  Experimental API
+   *  ----------------
+   *
+   *  The optimization process is undergoing active development and may change.
+   *  Our goal with these changes is to improve the performance of optimization and
+   *  reduce the complexity.
+   *
+   *  That being said, it is essential today to run optimize if you want the best
+   *  performance.  It should be stable and safe to use in production, but it our
+   *  hope that the API may be simplified (or not even need to be called) in the
+   *  future.
+   *
+   *  The frequency an application shoudl call optimize is based on the frequency of
+   *  data modifications.  If data is frequently added, deleted, or updated then
+   *  optimize should be run frequently.  A good rule of thumb is to run optimize if
+   *  you have added or modified 100,000 or more records or run more than 20 data
+   *  modification operations.
+   */
+  async optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats> {
+    let cleanupOlderThanMs;
+    if (
+      options?.cleanupOlderThan !== undefined &&
+      options?.cleanupOlderThan !== null
+    ) {
+      cleanupOlderThanMs =
+        new Date().getTime() - options.cleanupOlderThan.getTime();
+    }
+    return await this.inner.optimize(cleanupOlderThanMs);
+  }
+
   /** List all indices that have been created with {@link Table.createIndex} */
   async listIndices(): Promise<IndexConfig[]> {
     return await this.inner.listIndices();

package/nodejs-artifacts/arrow.d.ts

@@ -1,5 +1,5 @@
 /// <reference types="node" />
-import { type Schema, Table as ArrowTable, type Float } from "apache-arrow";
+import { Table as ArrowTable, type Float, Schema } from "apache-arrow";
 import { type EmbeddingFunction } from "./embedding/embedding_function";
 /** Data type accepted by NodeJS SDK */
 export type Data = Record<string, unknown>[] | ArrowTable;
@@ -12,6 +12,7 @@ export declare class VectorColumnOptions {
 export declare class MakeArrowTableOptions {
     schema?: Schema;
     vectorColumns: Record<string, VectorColumnOptions>;
+    embeddings?: EmbeddingFunction<unknown>;
     /**
      * If true then string columns will be encoded with dictionary encoding
      *

package/nodejs-artifacts/arrow.js

@@ -60,6 +60,7 @@ class MakeArrowTableOptions {
     vectorColumns = {
         vector: new VectorColumnOptions(),
     };
+    embeddings;
     /**
      * If true then string columns will be encoded with dictionary encoding
      *
@@ -175,6 +176,7 @@ function makeArrowTable(data, options) {
     const opt = new MakeArrowTableOptions(options !== undefined ? options : {});
     if (opt.schema !== undefined && opt.schema !== null) {
         opt.schema = (0, sanitize_1.sanitizeSchema)(opt.schema);
+        opt.schema = validateSchemaEmbeddings(opt.schema, data, opt.embeddings);
     }
     const columns = {};
     // TODO: sample dataset to find missing columns
@@ -244,8 +246,9 @@ function makeArrowTable(data, options) {
         // then patch the schema of the batches so we can use
         // `new ArrowTable(schema, batches)` which does not do any schema inference
         const firstTable = new apache_arrow_1.Table(columns);
+        const batchesFixed = firstTable.batches.map(
         // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-        const batchesFixed = firstTable.batches.map((batch) => new apache_arrow_1.RecordBatch(opt.schema, batch.data));
+        (batch) => new apache_arrow_1.RecordBatch(opt.schema, batch.data));
         return new apache_arrow_1.Table(opt.schema, batchesFixed);
     }
     else {
@@ -269,7 +272,7 @@ function makeListVector(lists) {
         throw Error("Cannot infer list vector from empty array or empty list");
     }
     const sampleList = lists[0];
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    // biome-ignore lint/suspicious/noExplicitAny: skip
     let inferredType;
     try {
         const sampleVector = makeVector(sampleList);
@@ -537,3 +540,31 @@ function createEmptyTable(schema) {
     return new apache_arrow_1.Table((0, sanitize_1.sanitizeSchema)(schema));
 }
 exports.createEmptyTable = createEmptyTable;
+function validateSchemaEmbeddings(schema, data, embeddings) {
+    const fields = [];
+    const missingEmbeddingFields = [];
+    // First we check if the field is a `FixedSizeList`
+    // Then we check if the data contains the field
+    // if it does not, we add it to the list of missing embedding fields
+    // Finally, we check if those missing embedding fields are `this._embeddings`
+    // if they are not, we throw an error
+    for (const field of schema.fields) {
+        if (field.type instanceof apache_arrow_1.FixedSizeList) {
+            if (data.length !== 0 && data?.[0]?.[field.name] === undefined) {
+                missingEmbeddingFields.push(field);
+            }
+            else {
+                fields.push(field);
+            }
+        }
+        else {
+            fields.push(field);
+        }
+    }
+    if (missingEmbeddingFields.length > 0 && embeddings === undefined) {
+        throw new Error(`Table has embeddings: "${missingEmbeddingFields
+            .map((f) => f.name)
+            .join(",")}", but no embedding function was provided`);
+    }
+    return new apache_arrow_1.Schema(fields, schema.metadata);
+}

package/nodejs-artifacts/connection.d.ts

@@ -1,6 +1,6 @@
+import { Table as ArrowTable, Schema } from "apache-arrow";
 import { ConnectionOptions, Connection as LanceDbConnection } from "./native";
 import { Table } from "./table";
-import { Table as ArrowTable, Schema } from "apache-arrow";
 /**
  * Connect to a LanceDB instance at the given URI.
  *

package/nodejs-artifacts/connection.js

@@ -14,10 +14,10 @@
 // limitations under the License.
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Connection = exports.connect = void 0;
+const apache_arrow_1 = require("apache-arrow");
 const arrow_1 = require("./arrow");
 const native_1 = require("./native");
 const table_1 = require("./table");
-const apache_arrow_1 = require("apache-arrow");
 /**
  * Connect to a LanceDB instance at the given URI.
  *

package/nodejs-artifacts/native.d.ts

@@ -15,6 +15,31 @@ export interface IndexConfig {
    */
   columns: Array<string>
 }
+/** Statistics about a compaction operation. */
+export interface CompactionStats {
+  /** The number of fragments removed */
+  fragmentsRemoved: number
+  /** The number of new, compacted fragments added */
+  fragmentsAdded: number
+  /** The number of data files removed */
+  filesRemoved: number
+  /** The number of new, compacted data files added */
+  filesAdded: number
+}
+/** Statistics about a cleanup operation */
+export interface RemovalStats {
+  /** The number of bytes removed */
+  bytesRemoved: number
+  /** The number of old versions removed */
+  oldVersionsRemoved: number
+}
+/** Statistics about an optimize operation */
+export interface OptimizeStats {
+  /** Statistics about the compaction operation */
+  compaction: CompactionStats
+  /** Statistics about the removal operation */
+  prune: RemovalStats
+}
 /**
  *  A definition of a column alteration. The alteration changes the column at
  * `path` to have the new name `name`, to be nullable if `nullable` is true,
@@ -151,5 +176,6 @@ export class Table {
   checkout(version: number): Promise<void>
   checkoutLatest(): Promise<void>
   restore(): Promise<void>
+  optimize(olderThanMs?: number | undefined | null): Promise<OptimizeStats>
   listIndices(): Promise<Array<IndexConfig>>
 }

package/nodejs-artifacts/query.d.ts

@@ -1,4 +1,4 @@
-import { RecordBatch, Table as ArrowTable } from "apache-arrow";
+import { Table as ArrowTable, RecordBatch } from "apache-arrow";
 import { RecordBatchIterator as NativeBatchIterator, Query as NativeQuery, Table as NativeTable, VectorQuery as NativeVectorQuery } from "./native";
 export declare class RecordBatchIterator implements AsyncIterator<RecordBatch> {
     private promisedInner?;

package/nodejs-artifacts/query.js

@@ -22,7 +22,7 @@ class RecordBatchIterator {
         // TODO: check promise reliably so we dont need to pass two arguments.
         this.promisedInner = promise;
     }
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    // biome-ignore lint/suspicious/noExplicitAny: skip
     async next() {
         if (this.inner === undefined) {
             this.inner = await this.promisedInner;
@@ -48,6 +48,7 @@ class QueryBase {
     inner;
     constructor(inner) {
         this.inner = inner;
+        // intentionally empty
     }
     /**
      * A filter statement to be applied to this query.
@@ -136,7 +137,7 @@ class QueryBase {
     execute() {
         return new RecordBatchIterator(this.nativeExecute());
     }
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    // biome-ignore lint/suspicious/noExplicitAny: skip
     [Symbol.asyncIterator]() {
         const promise = this.nativeExecute();
         return new RecordBatchIterator(promise);
@@ -338,7 +339,7 @@ class Query extends QueryBase {
      * a default `limit` of 10 will be used.  @see {@link Query#limit}
      */
     nearestTo(vector) {
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        // biome-ignore lint/suspicious/noExplicitAny: skip
         const vectorQuery = this.inner.nearestTo(Float32Array.from(vector));
         return new VectorQuery(vectorQuery);
     }

package/nodejs-artifacts/sanitize.js

@@ -127,7 +127,7 @@ function sanitizeUnion(typeLike) {
         throw Error("Expected a Union type to have an array-like `children` property");
     }
     return new apache_arrow_1.Union(typeLike.mode, 
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    // biome-ignore lint/suspicious/noExplicitAny: skip
     typeLike.typeIds, typeLike.children.map((child) => sanitizeField(child)));
 }
 function sanitizeTypedUnion(typeLike, 
@@ -167,7 +167,7 @@ function sanitizeMap(typeLike) {
         throw Error("Expected a Map type to have a `keysSorted` property");
     }
     return new apache_arrow_1.Map_(
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    // biome-ignore lint/suspicious/noExplicitAny: skip
     typeLike.children.map((field) => sanitizeField(field)), typeLike.keysSorted);
 }
 function sanitizeDuration(typeLike) {
@@ -191,7 +191,7 @@ function sanitizeDictionary(typeLike) {
     }
     return new apache_arrow_1.Dictionary(sanitizeType(typeLike.dictionary), sanitizeType(typeLike.indices), typeLike.id, typeLike.isOrdered);
 }
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
+// biome-ignore lint/suspicious/noExplicitAny: skip
 function sanitizeType(typeLike) {
     if (typeof typeLike !== "object" || typeLike === null) {
         throw Error("Expected a Type but object was null/undefined");

package/nodejs-artifacts/table.d.ts

@@ -1,8 +1,8 @@
 import { Schema } from "apache-arrow";
-import { AddColumnsSql, ColumnAlteration, IndexConfig, Table as _NativeTable } from "./native";
-import { Query, VectorQuery } from "./query";
-import { IndexOptions } from "./indices";
 import { Data } from "./arrow";
+import { IndexOptions } from "./indices";
+import { AddColumnsSql, ColumnAlteration, IndexConfig, OptimizeStats, Table as _NativeTable } from "./native";
+import { Query, VectorQuery } from "./query";
 export { IndexConfig } from "./native";
 /**
  * Options for adding data to a table.
@@ -28,6 +28,22 @@ export interface UpdateOptions {
      */
     where: string;
 }
+export interface OptimizeOptions {
+    /**
+     * If set then all versions older than the given date
+     * be removed.  The current version will never be removed.
+     * The default is 7 days
+     * @example
+     * // Delete all versions older than 1 day
+     * const olderThan = new Date();
+     * olderThan.setDate(olderThan.getDate() - 1));
+     * tbl.cleanupOlderVersions(olderThan);
+     *
+     * // Delete all versions except the current version
+     * tbl.cleanupOlderVersions(new Date());
+     */
+    cleanupOlderThan: Date;
+}
 /**
  * A Table is a collection of Records in a LanceDB Database.
  *
@@ -253,6 +269,37 @@ export declare class Table {
      * out state and the read_consistency_interval, if any, will apply.
      */
     restore(): Promise<void>;
+    /**
+     * Optimize the on-disk data and indices for better performance.
+     *
+     * Modeled after ``VACUUM`` in PostgreSQL.
+     *
+     *  Optimization covers three operations:
+     *
+     *  - Compaction: Merges small files into larger ones
+     *  - Prune: Removes old versions of the dataset
+     *  - Index: Optimizes the indices, adding new data to existing indices
+     *
+     *
+     *  Experimental API
+     *  ----------------
+     *
+     *  The optimization process is undergoing active development and may change.
+     *  Our goal with these changes is to improve the performance of optimization and
+     *  reduce the complexity.
+     *
+     *  That being said, it is essential today to run optimize if you want the best
+     *  performance.  It should be stable and safe to use in production, but it our
+     *  hope that the API may be simplified (or not even need to be called) in the
+     *  future.
+     *
+     *  The frequency an application shoudl call optimize is based on the frequency of
+     *  data modifications.  If data is frequently added, deleted, or updated then
+     *  optimize should be run frequently.  A good rule of thumb is to run optimize if
+     *  you have added or modified 100,000 or more records or run more than 20 data
+     *  modification operations.
+     */
+    optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
     /** List all indices that have been created with {@link Table.createIndex} */
     listIndices(): Promise<IndexConfig[]>;
 }

package/nodejs-artifacts/table.js

@@ -15,8 +15,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Table = void 0;
 const apache_arrow_1 = require("apache-arrow");
-const query_1 = require("./query");
 const arrow_1 = require("./arrow");
+const query_1 = require("./query");
 /**
  * A Table is a collection of Records in a LanceDB Database.
  *
@@ -140,7 +140,7 @@ class Table {
      */
     async createIndex(column, options) {
         // Bit of a hack to get around the fact that TS has no package-scope.
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        // biome-ignore lint/suspicious/noExplicitAny: skip
         const nativeIndex = options?.config?.inner;
         await this.inner.createIndex(nativeIndex, column, options?.replace);
     }
@@ -296,6 +296,45 @@ class Table {
     async restore() {
         await this.inner.restore();
     }
+    /**
+     * Optimize the on-disk data and indices for better performance.
+     *
+     * Modeled after ``VACUUM`` in PostgreSQL.
+     *
+     *  Optimization covers three operations:
+     *
+     *  - Compaction: Merges small files into larger ones
+     *  - Prune: Removes old versions of the dataset
+     *  - Index: Optimizes the indices, adding new data to existing indices
+     *
+     *
+     *  Experimental API
+     *  ----------------
+     *
+     *  The optimization process is undergoing active development and may change.
+     *  Our goal with these changes is to improve the performance of optimization and
+     *  reduce the complexity.
+     *
+     *  That being said, it is essential today to run optimize if you want the best
+     *  performance.  It should be stable and safe to use in production, but it our
+     *  hope that the API may be simplified (or not even need to be called) in the
+     *  future.
+     *
+     *  The frequency an application shoudl call optimize is based on the frequency of
+     *  data modifications.  If data is frequently added, deleted, or updated then
+     *  optimize should be run frequently.  A good rule of thumb is to run optimize if
+     *  you have added or modified 100,000 or more records or run more than 20 data
+     *  modification operations.
+     */
+    async optimize(options) {
+        let cleanupOlderThanMs;
+        if (options?.cleanupOlderThan !== undefined &&
+            options?.cleanupOlderThan !== null) {
+            cleanupOlderThanMs =
+                new Date().getTime() - options.cleanupOlderThan.getTime();
+        }
+        return await this.inner.optimize(cleanupOlderThanMs);
+    }
     /** List all indices that have been created with {@link Table.createIndex} */
     async listIndices() {
         return await this.inner.listIndices();