@lancedb/lancedb 0.4.20 → 0.5.1

package/lancedb/sanitize.ts

@@ -20,65 +20,65 @@
 // comes from the exact same library instance.  This is not always the case
 // and so we must sanitize the input to ensure that it is compatible.
 
+import type { IntBitWidth, TKeys, TimeBitWidth } from "apache-arrow/type";
 import {
-  Field,
-  Utf8,
-  FixedSizeBinary,
-  FixedSizeList,
-  Schema,
-  List,
-  Struct,
-  Float,
+  Binary,
   Bool,
+  DataType,
+  DateDay,
+  DateMillisecond,
+  type DateUnit,
   Date_,
   Decimal,
-  DataType,
+  DenseUnion,
   Dictionary,
-  Binary,
-  Float32,
-  Interval,
-  Map_,
   Duration,
-  Union,
-  Time,
-  Timestamp,
-  Type,
-  Null,
+  DurationMicrosecond,
+  DurationMillisecond,
+  DurationNanosecond,
+  DurationSecond,
+  Field,
+  FixedSizeBinary,
+  FixedSizeList,
+  Float,
+  Float16,
+  Float32,
+  Float64,
   Int,
-  type Precision,
-  type DateUnit,
   Int8,
   Int16,
   Int32,
   Int64,
-  Uint8,
-  Uint16,
-  Uint32,
-  Uint64,
-  Float16,
-  Float64,
-  DateDay,
-  DateMillisecond,
-  DenseUnion,
+  Interval,
+  IntervalDayTime,
+  IntervalYearMonth,
+  List,
+  Map_,
+  Null,
+  type Precision,
+  Schema,
   SparseUnion,
-  TimeNanosecond,
+  Struct,
+  Time,
   TimeMicrosecond,
   TimeMillisecond,
+  TimeNanosecond,
   TimeSecond,
-  TimestampNanosecond,
+  Timestamp,
   TimestampMicrosecond,
   TimestampMillisecond,
+  TimestampNanosecond,
   TimestampSecond,
-  IntervalDayTime,
-  IntervalYearMonth,
-  DurationNanosecond,
-  DurationMicrosecond,
-  DurationMillisecond,
-  DurationSecond,
-} from "apache-arrow";
-import type { IntBitWidth, TKeys, TimeBitWidth } from "apache-arrow/type";
+  Type,
+  Uint8,
+  Uint16,
+  Uint32,
+  Uint64,
+  Union,
+  Utf8,
+} from "./arrow";
 
-function sanitizeMetadata(
+export function sanitizeMetadata(
   metadataLike?: unknown,
 ): Map<string, string> | undefined {
   if (metadataLike === undefined || metadataLike === null) {
@@ -97,7 +97,7 @@ function sanitizeMetadata(
   return metadataLike as Map<string, string>;
 }
 
-function sanitizeInt(typeLike: object) {
+export function sanitizeInt(typeLike: object) {
   if (
     !("bitWidth" in typeLike) ||
     typeof typeLike.bitWidth !== "number" ||
@@ -111,14 +111,14 @@ function sanitizeInt(typeLike: object) {
   return new Int(typeLike.isSigned, typeLike.bitWidth as IntBitWidth);
 }
 
-function sanitizeFloat(typeLike: object) {
+export function sanitizeFloat(typeLike: object) {
   if (!("precision" in typeLike) || typeof typeLike.precision !== "number") {
     throw Error("Expected a Float Type to have a `precision` property");
   }
   return new Float(typeLike.precision as Precision);
 }
 
-function sanitizeDecimal(typeLike: object) {
+export function sanitizeDecimal(typeLike: object) {
   if (
     !("scale" in typeLike) ||
     typeof typeLike.scale !== "number" ||
@@ -134,14 +134,14 @@ function sanitizeDecimal(typeLike: object) {
   return new Decimal(typeLike.scale, typeLike.precision, typeLike.bitWidth);
 }
 
-function sanitizeDate(typeLike: object) {
+export function sanitizeDate(typeLike: object) {
   if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
     throw Error("Expected a Date type to have a `unit` property");
   }
   return new Date_(typeLike.unit as DateUnit);
 }
 
-function sanitizeTime(typeLike: object) {
+export function sanitizeTime(typeLike: object) {
   if (
     !("unit" in typeLike) ||
     typeof typeLike.unit !== "number" ||
@@ -155,7 +155,7 @@ function sanitizeTime(typeLike: object) {
   return new Time(typeLike.unit, typeLike.bitWidth as TimeBitWidth);
 }
 
-function sanitizeTimestamp(typeLike: object) {
+export function sanitizeTimestamp(typeLike: object) {
   if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
     throw Error("Expected a Timestamp type to have a `unit` property");
   }
@@ -166,7 +166,7 @@ function sanitizeTimestamp(typeLike: object) {
   return new Timestamp(typeLike.unit, timezone);
 }
 
-function sanitizeTypedTimestamp(
+export function sanitizeTypedTimestamp(
   typeLike: object,
   // eslint-disable-next-line @typescript-eslint/naming-convention
   Datatype:
@@ -182,14 +182,14 @@ function sanitizeTypedTimestamp(
   return new Datatype(timezone);
 }
 
-function sanitizeInterval(typeLike: object) {
+export function sanitizeInterval(typeLike: object) {
   if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
     throw Error("Expected an Interval type to have a `unit` property");
   }
   return new Interval(typeLike.unit);
 }
 
-function sanitizeList(typeLike: object) {
+export function sanitizeList(typeLike: object) {
   if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
     throw Error(
       "Expected a List type to have an array-like `children` property",
@@ -201,7 +201,7 @@ function sanitizeList(typeLike: object) {
   return new List(sanitizeField(typeLike.children[0]));
 }
 
-function sanitizeStruct(typeLike: object) {
+export function sanitizeStruct(typeLike: object) {
   if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
     throw Error(
       "Expected a Struct type to have an array-like `children` property",
@@ -210,7 +210,7 @@ function sanitizeStruct(typeLike: object) {
   return new Struct(typeLike.children.map((child) => sanitizeField(child)));
 }
 
-function sanitizeUnion(typeLike: object) {
+export function sanitizeUnion(typeLike: object) {
   if (
     !("typeIds" in typeLike) ||
     !("mode" in typeLike) ||
@@ -228,13 +228,13 @@ function sanitizeUnion(typeLike: object) {
 
   return new Union(
     typeLike.mode,
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    // biome-ignore lint/suspicious/noExplicitAny: skip
     typeLike.typeIds as any,
     typeLike.children.map((child) => sanitizeField(child)),
   );
 }
 
-function sanitizeTypedUnion(
+export function sanitizeTypedUnion(
   typeLike: object,
   // eslint-disable-next-line @typescript-eslint/naming-convention
   UnionType: typeof DenseUnion | typeof SparseUnion,
@@ -256,7 +256,7 @@ function sanitizeTypedUnion(
   );
 }
 
-function sanitizeFixedSizeBinary(typeLike: object) {
+export function sanitizeFixedSizeBinary(typeLike: object) {
   if (!("byteWidth" in typeLike) || typeof typeLike.byteWidth !== "number") {
     throw Error(
       "Expected a FixedSizeBinary type to have a `byteWidth` property",
@@ -265,7 +265,7 @@ function sanitizeFixedSizeBinary(typeLike: object) {
   return new FixedSizeBinary(typeLike.byteWidth);
 }
 
-function sanitizeFixedSizeList(typeLike: object) {
+export function sanitizeFixedSizeList(typeLike: object) {
   if (!("listSize" in typeLike) || typeof typeLike.listSize !== "number") {
     throw Error("Expected a FixedSizeList type to have a `listSize` property");
   }
@@ -283,7 +283,7 @@ function sanitizeFixedSizeList(typeLike: object) {
   );
 }
 
-function sanitizeMap(typeLike: object) {
+export function sanitizeMap(typeLike: object) {
   if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
     throw Error(
       "Expected a Map type to have an array-like `children` property",
@@ -294,20 +294,20 @@ function sanitizeMap(typeLike: object) {
   }
 
   return new Map_(
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    // biome-ignore lint/suspicious/noExplicitAny: skip
     typeLike.children.map((field) => sanitizeField(field)) as any,
     typeLike.keysSorted,
   );
 }
 
-function sanitizeDuration(typeLike: object) {
+export function sanitizeDuration(typeLike: object) {
   if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
     throw Error("Expected a Duration type to have a `unit` property");
   }
   return new Duration(typeLike.unit);
 }
 
-function sanitizeDictionary(typeLike: object) {
+export function sanitizeDictionary(typeLike: object) {
   if (!("id" in typeLike) || typeof typeLike.id !== "number") {
     throw Error("Expected a Dictionary type to have an `id` property");
   }
@@ -328,8 +328,8 @@ function sanitizeDictionary(typeLike: object) {
   );
 }
 
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-function sanitizeType(typeLike: unknown): DataType<any> {
+// biome-ignore lint/suspicious/noExplicitAny: skip
+export function sanitizeType(typeLike: unknown): DataType<any> {
   if (typeof typeLike !== "object" || typeLike === null) {
     throw Error("Expected a Type but object was null/undefined");
   }
@@ -449,7 +449,7 @@ function sanitizeType(typeLike: unknown): DataType<any> {
   }
 }
 
-function sanitizeField(fieldLike: unknown): Field {
+export function sanitizeField(fieldLike: unknown): Field {
   if (fieldLike instanceof Field) {
     return fieldLike;
   }

package/lancedb/table.ts

@@ -12,16 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { Schema, tableFromIPC } from "apache-arrow";
+import { Data, Schema, fromDataToBuffer, tableFromIPC } from "./arrow";
+
+import { getRegistry } from "./embedding/registry";
+import { IndexOptions } from "./indices";
 import {
   AddColumnsSql,
   ColumnAlteration,
   IndexConfig,
+  OptimizeStats,
   Table as _NativeTable,
 } from "./native";
 import { Query, VectorQuery } from "./query";
-import { IndexOptions } from "./indices";
-import { Data, fromDataToBuffer } from "./arrow";
 
 export { IndexConfig } from "./native";
 /**
@@ -50,6 +52,23 @@ export interface UpdateOptions {
   where: string;
 }
 
+export interface OptimizeOptions {
+  /**
+   * If set then all versions older than the given date
+   * be removed.  The current version will never be removed.
+   * The default is 7 days
+   * @example
+   * // Delete all versions older than 1 day
+   * const olderThan = new Date();
+   * olderThan.setDate(olderThan.getDate() - 1));
+   * tbl.cleanupOlderVersions(olderThan);
+   *
+   * // Delete all versions except the current version
+   * tbl.cleanupOlderVersions(new Date());
+   */
+  cleanupOlderThan: Date;
+}
+
 /**
  * A Table is a collection of Records in a LanceDB Database.
  *
@@ -104,8 +123,14 @@ export class Table {
    */
   async add(data: Data, options?: Partial<AddDataOptions>): Promise<void> {
     const mode = options?.mode ?? "append";
+    const schema = await this.schema();
+    const registry = getRegistry();
+    const functions = registry.parseFunctions(schema.metadata);
 
-    const buffer = await fromDataToBuffer(data);
+    const buffer = await fromDataToBuffer(
+      data,
+      functions.values().next().value,
+    );
     await this.inner.add(buffer, mode);
   }
 
@@ -186,7 +211,7 @@ export class Table {
    */
   async createIndex(column: string, options?: Partial<IndexOptions>) {
     // Bit of a hack to get around the fact that TS has no package-scope.
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    // biome-ignore lint/suspicious/noExplicitAny: skip
     const nativeIndex = (options?.config as any)?.inner;
     await this.inner.createIndex(nativeIndex, column, options?.replace);
   }
@@ -352,6 +377,48 @@ export class Table {
     await this.inner.restore();
   }
 
+  /**
+   * Optimize the on-disk data and indices for better performance.
+   *
+   * Modeled after ``VACUUM`` in PostgreSQL.
+   *
+   *  Optimization covers three operations:
+   *
+   *  - Compaction: Merges small files into larger ones
+   *  - Prune: Removes old versions of the dataset
+   *  - Index: Optimizes the indices, adding new data to existing indices
+   *
+   *
+   *  Experimental API
+   *  ----------------
+   *
+   *  The optimization process is undergoing active development and may change.
+   *  Our goal with these changes is to improve the performance of optimization and
+   *  reduce the complexity.
+   *
+   *  That being said, it is essential today to run optimize if you want the best
+   *  performance.  It should be stable and safe to use in production, but it our
+   *  hope that the API may be simplified (or not even need to be called) in the
+   *  future.
+   *
+   *  The frequency an application shoudl call optimize is based on the frequency of
+   *  data modifications.  If data is frequently added, deleted, or updated then
+   *  optimize should be run frequently.  A good rule of thumb is to run optimize if
+   *  you have added or modified 100,000 or more records or run more than 20 data
+   *  modification operations.
+   */
+  async optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats> {
+    let cleanupOlderThanMs;
+    if (
+      options?.cleanupOlderThan !== undefined &&
+      options?.cleanupOlderThan !== null
+    ) {
+      cleanupOlderThanMs =
+        new Date().getTime() - options.cleanupOlderThan.getTime();
+    }
+    return await this.inner.optimize(cleanupOlderThanMs);
+  }
+
   /** List all indices that have been created with {@link Table.createIndex} */
   async listIndices(): Promise<IndexConfig[]> {
     return await this.inner.listIndices();

package/nodejs-artifacts/arrow.d.ts

@@ -1,6 +1,29 @@
 /// <reference types="node" />
-import { type Schema, Table as ArrowTable, type Float } from "apache-arrow";
+import { Table as ArrowTable, Binary, DataType, FixedSizeBinary, FixedSizeList, Float, Int, LargeBinary, List, Null, Schema, Struct, Utf8 } from "apache-arrow";
 import { type EmbeddingFunction } from "./embedding/embedding_function";
+import { EmbeddingFunctionConfig } from "./embedding/registry";
+export * from "apache-arrow";
+export declare function isArrowTable(value: object): value is ArrowTable;
+export declare function isDataType(value: unknown): value is DataType;
+export declare function isNull(value: unknown): value is Null;
+export declare function isInt(value: unknown): value is Int;
+export declare function isFloat(value: unknown): value is Float;
+export declare function isBinary(value: unknown): value is Binary;
+export declare function isLargeBinary(value: unknown): value is LargeBinary;
+export declare function isUtf8(value: unknown): value is Utf8;
+export declare function isLargeUtf8(value: unknown): value is Utf8;
+export declare function isBool(value: unknown): value is Utf8;
+export declare function isDecimal(value: unknown): value is Utf8;
+export declare function isDate(value: unknown): value is Utf8;
+export declare function isTime(value: unknown): value is Utf8;
+export declare function isTimestamp(value: unknown): value is Utf8;
+export declare function isInterval(value: unknown): value is Utf8;
+export declare function isDuration(value: unknown): value is Utf8;
+export declare function isList(value: unknown): value is List;
+export declare function isStruct(value: unknown): value is Struct;
+export declare function isUnion(value: unknown): value is Struct;
+export declare function isFixedSizeBinary(value: unknown): value is FixedSizeBinary;
+export declare function isFixedSizeList(value: unknown): value is FixedSizeList;
 /** Data type accepted by NodeJS SDK */
 export type Data = Record<string, unknown>[] | ArrowTable;
 export declare class VectorColumnOptions {
@@ -12,6 +35,7 @@ export declare class VectorColumnOptions {
 export declare class MakeArrowTableOptions {
     schema?: Schema;
     vectorColumns: Record<string, VectorColumnOptions>;
+    embeddings?: EmbeddingFunction<unknown>;
     /**
      * If true then string columns will be encoded with dictionary encoding
      *
@@ -116,11 +140,11 @@ export declare class MakeArrowTableOptions {
  * assert.deepEqual(table.schema, schema)
  * ```
  */
-export declare function makeArrowTable(data: Array<Record<string, unknown>>, options?: Partial<MakeArrowTableOptions>): ArrowTable;
+export declare function makeArrowTable(data: Array<Record<string, unknown>>, options?: Partial<MakeArrowTableOptions>, metadata?: Map<string, string>): ArrowTable;
 /**
  * Create an empty Arrow table with the provided schema
  */
-export declare function makeEmptyTable(schema: Schema): ArrowTable;
+export declare function makeEmptyTable(schema: Schema, metadata?: Map<string, string>): ArrowTable;
 /**
  * Convert an Array of records into an Arrow Table, optionally applying an
  * embeddings function to it.
@@ -139,7 +163,9 @@ export declare function makeEmptyTable(schema: Schema): ArrowTable;
  * embedding columns.  If no schema is provded then embedding columns will
  * be placed at the end of the table, after all of the input columns.
  */
-export declare function convertToTable<T>(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>, makeTableOptions?: Partial<MakeArrowTableOptions>): Promise<ArrowTable>;
+export declare function convertToTable(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunctionConfig, makeTableOptions?: Partial<MakeArrowTableOptions>): Promise<ArrowTable>;
+/** Creates the Arrow Type for a Vector column with dimension `dim` */
+export declare function newVectorType<T extends Float>(dim: number, innerType: T): FixedSizeList<T>;
 /**
  * Serialize an Array of records into a buffer using the Arrow IPC File serialization
  *
@@ -147,7 +173,7 @@ export declare function convertToTable<T>(data: Array<Record<string, unknown>>,
  *
  * `schema` is required if data is empty
  */
-export declare function fromRecordsToBuffer<T>(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>, schema?: Schema): Promise<Buffer>;
+export declare function fromRecordsToBuffer(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunctionConfig, schema?: Schema): Promise<Buffer>;
 /**
  * Serialize an Array of records into a buffer using the Arrow IPC Stream serialization
  *
@@ -155,7 +181,7 @@ export declare function fromRecordsToBuffer<T>(data: Array<Record<string, unknow
  *
  * `schema` is required if data is empty
  */
-export declare function fromRecordsToStreamBuffer<T>(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>, schema?: Schema): Promise<Buffer>;
+export declare function fromRecordsToStreamBuffer(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunctionConfig, schema?: Schema): Promise<Buffer>;
 /**
  * Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
  *
@@ -164,7 +190,7 @@ export declare function fromRecordsToStreamBuffer<T>(data: Array<Record<string,
  *
  * `schema` is required if the table is empty
  */
-export declare function fromTableToBuffer<T>(table: ArrowTable, embeddings?: EmbeddingFunction<T>, schema?: Schema): Promise<Buffer>;
+export declare function fromTableToBuffer(table: ArrowTable, embeddings?: EmbeddingFunctionConfig, schema?: Schema): Promise<Buffer>;
 /**
  * Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
  *
@@ -173,7 +199,7 @@ export declare function fromTableToBuffer<T>(table: ArrowTable, embeddings?: Emb
  *
  * `schema` is required if the table is empty
  */
-export declare function fromDataToBuffer<T>(data: Data, embeddings?: EmbeddingFunction<T>, schema?: Schema): Promise<Buffer>;
+export declare function fromDataToBuffer(data: Data, embeddings?: EmbeddingFunctionConfig, schema?: Schema): Promise<Buffer>;
 /**
  * Serialize an Arrow Table into a buffer using the Arrow IPC Stream serialization
  *
@@ -182,7 +208,7 @@ export declare function fromDataToBuffer<T>(data: Data, embeddings?: EmbeddingFu
  *
  * `schema` is required if the table is empty
  */
-export declare function fromTableToStreamBuffer<T>(table: ArrowTable, embeddings?: EmbeddingFunction<T>, schema?: Schema): Promise<Buffer>;
+export declare function fromTableToStreamBuffer(table: ArrowTable, embeddings?: EmbeddingFunctionConfig, schema?: Schema): Promise<Buffer>;
 /**
  * Create an empty table with the given schema
  */