@lancedb/lancedb 0.4.20 → 0.5.1

package/dist/sanitize.js

@@ -13,15 +13,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.sanitizeSchema = void 0;
-// The utilities in this file help sanitize data from the user's arrow
-// library into the types expected by vectordb's arrow library.  Node
-// generally allows for mulitple versions of the same library (and sometimes
-// even multiple copies of the same version) to be installed at the same
-// time.  However, arrow-js uses instanceof which expected that the input
-// comes from the exact same library instance.  This is not always the case
-// and so we must sanitize the input to ensure that it is compatible.
-const apache_arrow_1 = require("apache-arrow");
+exports.sanitizeSchema = exports.sanitizeField = exports.sanitizeType = exports.sanitizeDictionary = exports.sanitizeDuration = exports.sanitizeMap = exports.sanitizeFixedSizeList = exports.sanitizeFixedSizeBinary = exports.sanitizeTypedUnion = exports.sanitizeUnion = exports.sanitizeStruct = exports.sanitizeList = exports.sanitizeInterval = exports.sanitizeTypedTimestamp = exports.sanitizeTimestamp = exports.sanitizeTime = exports.sanitizeDate = exports.sanitizeDecimal = exports.sanitizeFloat = exports.sanitizeInt = exports.sanitizeMetadata = void 0;
+const arrow_1 = require("./arrow");
 function sanitizeMetadata(metadataLike) {
     if (metadataLike === undefined || metadataLike === null) {
         return undefined;
@@ -36,6 +29,7 @@ function sanitizeMetadata(metadataLike) {
     }
     return metadataLike;
 }
+exports.sanitizeMetadata = sanitizeMetadata;
 function sanitizeInt(typeLike) {
     if (!("bitWidth" in typeLike) ||
         typeof typeLike.bitWidth !== "number" ||
@@ -43,14 +37,16 @@ function sanitizeInt(typeLike) {
         typeof typeLike.isSigned !== "boolean") {
         throw Error("Expected an Int Type to have a `bitWidth` and `isSigned` property");
     }
-    return new apache_arrow_1.Int(typeLike.isSigned, typeLike.bitWidth);
+    return new arrow_1.Int(typeLike.isSigned, typeLike.bitWidth);
 }
+exports.sanitizeInt = sanitizeInt;
 function sanitizeFloat(typeLike) {
     if (!("precision" in typeLike) || typeof typeLike.precision !== "number") {
         throw Error("Expected a Float Type to have a `precision` property");
     }
-    return new apache_arrow_1.Float(typeLike.precision);
+    return new arrow_1.Float(typeLike.precision);
 }
+exports.sanitizeFloat = sanitizeFloat;
 function sanitizeDecimal(typeLike) {
     if (!("scale" in typeLike) ||
         typeof typeLike.scale !== "number" ||
@@ -60,14 +56,16 @@ function sanitizeDecimal(typeLike) {
         typeof typeLike.bitWidth !== "number") {
         throw Error("Expected a Decimal Type to have `scale`, `precision`, and `bitWidth` properties");
     }
-    return new apache_arrow_1.Decimal(typeLike.scale, typeLike.precision, typeLike.bitWidth);
+    return new arrow_1.Decimal(typeLike.scale, typeLike.precision, typeLike.bitWidth);
 }
+exports.sanitizeDecimal = sanitizeDecimal;
 function sanitizeDate(typeLike) {
     if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
         throw Error("Expected a Date type to have a `unit` property");
     }
-    return new apache_arrow_1.Date_(typeLike.unit);
+    return new arrow_1.Date_(typeLike.unit);
 }
+exports.sanitizeDate = sanitizeDate;
 function sanitizeTime(typeLike) {
     if (!("unit" in typeLike) ||
         typeof typeLike.unit !== "number" ||
@@ -75,8 +73,9 @@ function sanitizeTime(typeLike) {
         typeof typeLike.bitWidth !== "number") {
         throw Error("Expected a Time type to have `unit` and `bitWidth` properties");
     }
-    return new apache_arrow_1.Time(typeLike.unit, typeLike.bitWidth);
+    return new arrow_1.Time(typeLike.unit, typeLike.bitWidth);
 }
+exports.sanitizeTime = sanitizeTime;
 function sanitizeTimestamp(typeLike) {
     if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
         throw Error("Expected a Timestamp type to have a `unit` property");
@@ -85,8 +84,9 @@ function sanitizeTimestamp(typeLike) {
     if ("timezone" in typeLike && typeof typeLike.timezone === "string") {
         timezone = typeLike.timezone;
     }
-    return new apache_arrow_1.Timestamp(typeLike.unit, timezone);
+    return new arrow_1.Timestamp(typeLike.unit, timezone);
 }
+exports.sanitizeTimestamp = sanitizeTimestamp;
 function sanitizeTypedTimestamp(typeLike, 
 // eslint-disable-next-line @typescript-eslint/naming-convention
 Datatype) {
@@ -96,12 +96,14 @@ Datatype) {
     }
     return new Datatype(timezone);
 }
+exports.sanitizeTypedTimestamp = sanitizeTypedTimestamp;
 function sanitizeInterval(typeLike) {
     if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
         throw Error("Expected an Interval type to have a `unit` property");
     }
-    return new apache_arrow_1.Interval(typeLike.unit);
+    return new arrow_1.Interval(typeLike.unit);
 }
+exports.sanitizeInterval = sanitizeInterval;
 function sanitizeList(typeLike) {
     if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
         throw Error("Expected a List type to have an array-like `children` property");
@@ -109,14 +111,16 @@ function sanitizeList(typeLike) {
     if (typeLike.children.length !== 1) {
         throw Error("Expected a List type to have exactly one child");
     }
-    return new apache_arrow_1.List(sanitizeField(typeLike.children[0]));
+    return new arrow_1.List(sanitizeField(typeLike.children[0]));
 }
+exports.sanitizeList = sanitizeList;
 function sanitizeStruct(typeLike) {
     if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
         throw Error("Expected a Struct type to have an array-like `children` property");
     }
-    return new apache_arrow_1.Struct(typeLike.children.map((child) => sanitizeField(child)));
+    return new arrow_1.Struct(typeLike.children.map((child) => sanitizeField(child)));
 }
+exports.sanitizeStruct = sanitizeStruct;
 function sanitizeUnion(typeLike) {
     if (!("typeIds" in typeLike) ||
         !("mode" in typeLike) ||
@@ -126,10 +130,11 @@ function sanitizeUnion(typeLike) {
     if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
         throw Error("Expected a Union type to have an array-like `children` property");
     }
-    return new apache_arrow_1.Union(typeLike.mode, 
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    return new arrow_1.Union(typeLike.mode, 
+    // biome-ignore lint/suspicious/noExplicitAny: skip
     typeLike.typeIds, typeLike.children.map((child) => sanitizeField(child)));
 }
+exports.sanitizeUnion = sanitizeUnion;
 function sanitizeTypedUnion(typeLike, 
 // eslint-disable-next-line @typescript-eslint/naming-convention
 UnionType) {
@@ -141,12 +146,14 @@ UnionType) {
     }
     return new UnionType(typeLike.typeIds, typeLike.children.map((child) => sanitizeField(child)));
 }
+exports.sanitizeTypedUnion = sanitizeTypedUnion;
 function sanitizeFixedSizeBinary(typeLike) {
     if (!("byteWidth" in typeLike) || typeof typeLike.byteWidth !== "number") {
         throw Error("Expected a FixedSizeBinary type to have a `byteWidth` property");
     }
-    return new apache_arrow_1.FixedSizeBinary(typeLike.byteWidth);
+    return new arrow_1.FixedSizeBinary(typeLike.byteWidth);
 }
+exports.sanitizeFixedSizeBinary = sanitizeFixedSizeBinary;
 function sanitizeFixedSizeList(typeLike) {
     if (!("listSize" in typeLike) || typeof typeLike.listSize !== "number") {
         throw Error("Expected a FixedSizeList type to have a `listSize` property");
@@ -157,8 +164,9 @@ function sanitizeFixedSizeList(typeLike) {
     if (typeLike.children.length !== 1) {
         throw Error("Expected a FixedSizeList type to have exactly one child");
     }
-    return new apache_arrow_1.FixedSizeList(typeLike.listSize, sanitizeField(typeLike.children[0]));
+    return new arrow_1.FixedSizeList(typeLike.listSize, sanitizeField(typeLike.children[0]));
 }
+exports.sanitizeFixedSizeList = sanitizeFixedSizeList;
 function sanitizeMap(typeLike) {
     if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
         throw Error("Expected a Map type to have an array-like `children` property");
@@ -166,16 +174,18 @@ function sanitizeMap(typeLike) {
     if (!("keysSorted" in typeLike) || typeof typeLike.keysSorted !== "boolean") {
         throw Error("Expected a Map type to have a `keysSorted` property");
     }
-    return new apache_arrow_1.Map_(
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    return new arrow_1.Map_(
+    // biome-ignore lint/suspicious/noExplicitAny: skip
     typeLike.children.map((field) => sanitizeField(field)), typeLike.keysSorted);
 }
+exports.sanitizeMap = sanitizeMap;
 function sanitizeDuration(typeLike) {
     if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
         throw Error("Expected a Duration type to have a `unit` property");
     }
-    return new apache_arrow_1.Duration(typeLike.unit);
+    return new arrow_1.Duration(typeLike.unit);
 }
+exports.sanitizeDuration = sanitizeDuration;
 function sanitizeDictionary(typeLike) {
     if (!("id" in typeLike) || typeof typeLike.id !== "number") {
         throw Error("Expected a Dictionary type to have an `id` property");
@@ -189,9 +199,10 @@ function sanitizeDictionary(typeLike) {
     if (!("isOrdered" in typeLike) || typeof typeLike.isOrdered !== "boolean") {
         throw Error("Expected a Dictionary type to have an `isOrdered` property");
     }
-    return new apache_arrow_1.Dictionary(sanitizeType(typeLike.dictionary), sanitizeType(typeLike.indices), typeLike.id, typeLike.isOrdered);
+    return new arrow_1.Dictionary(sanitizeType(typeLike.dictionary), sanitizeType(typeLike.indices), typeLike.id, typeLike.isOrdered);
 }
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
+exports.sanitizeDictionary = sanitizeDictionary;
+// biome-ignore lint/suspicious/noExplicitAny: skip
 function sanitizeType(typeLike) {
     if (typeof typeLike !== "object" || typeLike === null) {
         throw Error("Expected a Type but object was null/undefined");
@@ -210,110 +221,111 @@ function sanitizeType(typeLike) {
         throw Error("Type's typeId property was not a function or number");
     }
     switch (typeId) {
-        case apache_arrow_1.Type.NONE:
+        case arrow_1.Type.NONE:
             throw Error("Received a Type with a typeId of NONE");
-        case apache_arrow_1.Type.Null:
-            return new apache_arrow_1.Null();
-        case apache_arrow_1.Type.Int:
+        case arrow_1.Type.Null:
+            return new arrow_1.Null();
+        case arrow_1.Type.Int:
             return sanitizeInt(typeLike);
-        case apache_arrow_1.Type.Float:
+        case arrow_1.Type.Float:
             return sanitizeFloat(typeLike);
-        case apache_arrow_1.Type.Binary:
-            return new apache_arrow_1.Binary();
-        case apache_arrow_1.Type.Utf8:
-            return new apache_arrow_1.Utf8();
-        case apache_arrow_1.Type.Bool:
-            return new apache_arrow_1.Bool();
-        case apache_arrow_1.Type.Decimal:
+        case arrow_1.Type.Binary:
+            return new arrow_1.Binary();
+        case arrow_1.Type.Utf8:
+            return new arrow_1.Utf8();
+        case arrow_1.Type.Bool:
+            return new arrow_1.Bool();
+        case arrow_1.Type.Decimal:
             return sanitizeDecimal(typeLike);
-        case apache_arrow_1.Type.Date:
+        case arrow_1.Type.Date:
             return sanitizeDate(typeLike);
-        case apache_arrow_1.Type.Time:
+        case arrow_1.Type.Time:
             return sanitizeTime(typeLike);
-        case apache_arrow_1.Type.Timestamp:
+        case arrow_1.Type.Timestamp:
             return sanitizeTimestamp(typeLike);
-        case apache_arrow_1.Type.Interval:
+        case arrow_1.Type.Interval:
             return sanitizeInterval(typeLike);
-        case apache_arrow_1.Type.List:
+        case arrow_1.Type.List:
             return sanitizeList(typeLike);
-        case apache_arrow_1.Type.Struct:
+        case arrow_1.Type.Struct:
             return sanitizeStruct(typeLike);
-        case apache_arrow_1.Type.Union:
+        case arrow_1.Type.Union:
             return sanitizeUnion(typeLike);
-        case apache_arrow_1.Type.FixedSizeBinary:
+        case arrow_1.Type.FixedSizeBinary:
             return sanitizeFixedSizeBinary(typeLike);
-        case apache_arrow_1.Type.FixedSizeList:
+        case arrow_1.Type.FixedSizeList:
             return sanitizeFixedSizeList(typeLike);
-        case apache_arrow_1.Type.Map:
+        case arrow_1.Type.Map:
             return sanitizeMap(typeLike);
-        case apache_arrow_1.Type.Duration:
+        case arrow_1.Type.Duration:
             return sanitizeDuration(typeLike);
-        case apache_arrow_1.Type.Dictionary:
+        case arrow_1.Type.Dictionary:
             return sanitizeDictionary(typeLike);
-        case apache_arrow_1.Type.Int8:
-            return new apache_arrow_1.Int8();
-        case apache_arrow_1.Type.Int16:
-            return new apache_arrow_1.Int16();
-        case apache_arrow_1.Type.Int32:
-            return new apache_arrow_1.Int32();
-        case apache_arrow_1.Type.Int64:
-            return new apache_arrow_1.Int64();
-        case apache_arrow_1.Type.Uint8:
-            return new apache_arrow_1.Uint8();
-        case apache_arrow_1.Type.Uint16:
-            return new apache_arrow_1.Uint16();
-        case apache_arrow_1.Type.Uint32:
-            return new apache_arrow_1.Uint32();
-        case apache_arrow_1.Type.Uint64:
-            return new apache_arrow_1.Uint64();
-        case apache_arrow_1.Type.Float16:
-            return new apache_arrow_1.Float16();
-        case apache_arrow_1.Type.Float32:
-            return new apache_arrow_1.Float32();
-        case apache_arrow_1.Type.Float64:
-            return new apache_arrow_1.Float64();
-        case apache_arrow_1.Type.DateMillisecond:
-            return new apache_arrow_1.DateMillisecond();
-        case apache_arrow_1.Type.DateDay:
-            return new apache_arrow_1.DateDay();
-        case apache_arrow_1.Type.TimeNanosecond:
-            return new apache_arrow_1.TimeNanosecond();
-        case apache_arrow_1.Type.TimeMicrosecond:
-            return new apache_arrow_1.TimeMicrosecond();
-        case apache_arrow_1.Type.TimeMillisecond:
-            return new apache_arrow_1.TimeMillisecond();
-        case apache_arrow_1.Type.TimeSecond:
-            return new apache_arrow_1.TimeSecond();
-        case apache_arrow_1.Type.TimestampNanosecond:
-            return sanitizeTypedTimestamp(typeLike, apache_arrow_1.TimestampNanosecond);
-        case apache_arrow_1.Type.TimestampMicrosecond:
-            return sanitizeTypedTimestamp(typeLike, apache_arrow_1.TimestampMicrosecond);
-        case apache_arrow_1.Type.TimestampMillisecond:
-            return sanitizeTypedTimestamp(typeLike, apache_arrow_1.TimestampMillisecond);
-        case apache_arrow_1.Type.TimestampSecond:
-            return sanitizeTypedTimestamp(typeLike, apache_arrow_1.TimestampSecond);
-        case apache_arrow_1.Type.DenseUnion:
-            return sanitizeTypedUnion(typeLike, apache_arrow_1.DenseUnion);
-        case apache_arrow_1.Type.SparseUnion:
-            return sanitizeTypedUnion(typeLike, apache_arrow_1.SparseUnion);
-        case apache_arrow_1.Type.IntervalDayTime:
-            return new apache_arrow_1.IntervalDayTime();
-        case apache_arrow_1.Type.IntervalYearMonth:
-            return new apache_arrow_1.IntervalYearMonth();
-        case apache_arrow_1.Type.DurationNanosecond:
-            return new apache_arrow_1.DurationNanosecond();
-        case apache_arrow_1.Type.DurationMicrosecond:
-            return new apache_arrow_1.DurationMicrosecond();
-        case apache_arrow_1.Type.DurationMillisecond:
-            return new apache_arrow_1.DurationMillisecond();
-        case apache_arrow_1.Type.DurationSecond:
-            return new apache_arrow_1.DurationSecond();
+        case arrow_1.Type.Int8:
+            return new arrow_1.Int8();
+        case arrow_1.Type.Int16:
+            return new arrow_1.Int16();
+        case arrow_1.Type.Int32:
+            return new arrow_1.Int32();
+        case arrow_1.Type.Int64:
+            return new arrow_1.Int64();
+        case arrow_1.Type.Uint8:
+            return new arrow_1.Uint8();
+        case arrow_1.Type.Uint16:
+            return new arrow_1.Uint16();
+        case arrow_1.Type.Uint32:
+            return new arrow_1.Uint32();
+        case arrow_1.Type.Uint64:
+            return new arrow_1.Uint64();
+        case arrow_1.Type.Float16:
+            return new arrow_1.Float16();
+        case arrow_1.Type.Float32:
+            return new arrow_1.Float32();
+        case arrow_1.Type.Float64:
+            return new arrow_1.Float64();
+        case arrow_1.Type.DateMillisecond:
+            return new arrow_1.DateMillisecond();
+        case arrow_1.Type.DateDay:
+            return new arrow_1.DateDay();
+        case arrow_1.Type.TimeNanosecond:
+            return new arrow_1.TimeNanosecond();
+        case arrow_1.Type.TimeMicrosecond:
+            return new arrow_1.TimeMicrosecond();
+        case arrow_1.Type.TimeMillisecond:
+            return new arrow_1.TimeMillisecond();
+        case arrow_1.Type.TimeSecond:
+            return new arrow_1.TimeSecond();
+        case arrow_1.Type.TimestampNanosecond:
+            return sanitizeTypedTimestamp(typeLike, arrow_1.TimestampNanosecond);
+        case arrow_1.Type.TimestampMicrosecond:
+            return sanitizeTypedTimestamp(typeLike, arrow_1.TimestampMicrosecond);
+        case arrow_1.Type.TimestampMillisecond:
+            return sanitizeTypedTimestamp(typeLike, arrow_1.TimestampMillisecond);
+        case arrow_1.Type.TimestampSecond:
+            return sanitizeTypedTimestamp(typeLike, arrow_1.TimestampSecond);
+        case arrow_1.Type.DenseUnion:
+            return sanitizeTypedUnion(typeLike, arrow_1.DenseUnion);
+        case arrow_1.Type.SparseUnion:
+            return sanitizeTypedUnion(typeLike, arrow_1.SparseUnion);
+        case arrow_1.Type.IntervalDayTime:
+            return new arrow_1.IntervalDayTime();
+        case arrow_1.Type.IntervalYearMonth:
+            return new arrow_1.IntervalYearMonth();
+        case arrow_1.Type.DurationNanosecond:
+            return new arrow_1.DurationNanosecond();
+        case arrow_1.Type.DurationMicrosecond:
+            return new arrow_1.DurationMicrosecond();
+        case arrow_1.Type.DurationMillisecond:
+            return new arrow_1.DurationMillisecond();
+        case arrow_1.Type.DurationSecond:
+            return new arrow_1.DurationSecond();
         default:
             throw new Error("Unrecoginized type id in schema: " + typeId);
     }
 }
+exports.sanitizeType = sanitizeType;
 function sanitizeField(fieldLike) {
-    if (fieldLike instanceof apache_arrow_1.Field) {
+    if (fieldLike instanceof arrow_1.Field) {
         return fieldLike;
     }
     if (typeof fieldLike !== "object" || fieldLike === null) {
@@ -337,8 +349,9 @@ function sanitizeField(fieldLike) {
     if ("metadata" in fieldLike) {
         metadata = sanitizeMetadata(fieldLike.metadata);
     }
-    return new apache_arrow_1.Field(name, type, nullable, metadata);
+    return new arrow_1.Field(name, type, nullable, metadata);
 }
+exports.sanitizeField = sanitizeField;
 /**
  * Convert something schemaLike into a Schema instance
  *
@@ -347,7 +360,7 @@ function sanitizeField(fieldLike) {
  * than lancedb is using.
  */
 function sanitizeSchema(schemaLike) {
-    if (schemaLike instanceof apache_arrow_1.Schema) {
+    if (schemaLike instanceof arrow_1.Schema) {
         return schemaLike;
     }
     if (typeof schemaLike !== "object" || schemaLike === null) {
@@ -364,6 +377,6 @@ function sanitizeSchema(schemaLike) {
         throw Error("The schema passed in had a 'fields' property but it was not an array");
     }
     const sanitizedFields = schemaLike.fields.map((field) => sanitizeField(field));
-    return new apache_arrow_1.Schema(sanitizedFields, metadata);
+    return new arrow_1.Schema(sanitizedFields, metadata);
 }
 exports.sanitizeSchema = sanitizeSchema;

package/dist/table.d.ts

@@ -1,8 +1,7 @@
-import { Schema } from "apache-arrow";
-import { AddColumnsSql, ColumnAlteration, IndexConfig, Table as _NativeTable } from "./native";
-import { Query, VectorQuery } from "./query";
+import { Data, Schema } from "./arrow";
 import { IndexOptions } from "./indices";
-import { Data } from "./arrow";
+import { AddColumnsSql, ColumnAlteration, IndexConfig, OptimizeStats, Table as _NativeTable } from "./native";
+import { Query, VectorQuery } from "./query";
 export { IndexConfig } from "./native";
 /**
  * Options for adding data to a table.
@@ -28,6 +27,22 @@ export interface UpdateOptions {
      */
     where: string;
 }
+export interface OptimizeOptions {
+    /**
+     * If set then all versions older than the given date
+     * be removed.  The current version will never be removed.
+     * The default is 7 days
+     * @example
+     * // Delete all versions older than 1 day
+     * const olderThan = new Date();
+     * olderThan.setDate(olderThan.getDate() - 1));
+     * tbl.cleanupOlderVersions(olderThan);
+     *
+     * // Delete all versions except the current version
+     * tbl.cleanupOlderVersions(new Date());
+     */
+    cleanupOlderThan: Date;
+}
 /**
  * A Table is a collection of Records in a LanceDB Database.
  *
@@ -253,6 +268,37 @@ export declare class Table {
      * out state and the read_consistency_interval, if any, will apply.
      */
     restore(): Promise<void>;
+    /**
+     * Optimize the on-disk data and indices for better performance.
+     *
+     * Modeled after ``VACUUM`` in PostgreSQL.
+     *
+     *  Optimization covers three operations:
+     *
+     *  - Compaction: Merges small files into larger ones
+     *  - Prune: Removes old versions of the dataset
+     *  - Index: Optimizes the indices, adding new data to existing indices
+     *
+     *
+     *  Experimental API
+     *  ----------------
+     *
+     *  The optimization process is undergoing active development and may change.
+     *  Our goal with these changes is to improve the performance of optimization and
+     *  reduce the complexity.
+     *
+     *  That being said, it is essential today to run optimize if you want the best
+     *  performance.  It should be stable and safe to use in production, but it our
+     *  hope that the API may be simplified (or not even need to be called) in the
+     *  future.
+     *
+     *  The frequency an application shoudl call optimize is based on the frequency of
+     *  data modifications.  If data is frequently added, deleted, or updated then
+     *  optimize should be run frequently.  A good rule of thumb is to run optimize if
+     *  you have added or modified 100,000 or more records or run more than 20 data
+     *  modification operations.
+     */
+    optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
     /** List all indices that have been created with {@link Table.createIndex} */
     listIndices(): Promise<IndexConfig[]>;
 }

package/dist/table.js

@@ -14,9 +14,9 @@
 // limitations under the License.
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Table = void 0;
-const apache_arrow_1 = require("apache-arrow");
-const query_1 = require("./query");
 const arrow_1 = require("./arrow");
+const registry_1 = require("./embedding/registry");
+const query_1 = require("./query");
 /**
  * A Table is a collection of Records in a LanceDB Database.
  *
@@ -56,7 +56,7 @@ class Table {
     /** Get the schema of the table. */
     async schema() {
         const schemaBuf = await this.inner.schema();
-        const tbl = (0, apache_arrow_1.tableFromIPC)(schemaBuf);
+        const tbl = (0, arrow_1.tableFromIPC)(schemaBuf);
         return tbl.schema;
     }
     /**
@@ -65,7 +65,10 @@ class Table {
      */
     async add(data, options) {
         const mode = options?.mode ?? "append";
-        const buffer = await (0, arrow_1.fromDataToBuffer)(data);
+        const schema = await this.schema();
+        const registry = (0, registry_1.getRegistry)();
+        const functions = registry.parseFunctions(schema.metadata);
+        const buffer = await (0, arrow_1.fromDataToBuffer)(data, functions.values().next().value);
         await this.inner.add(buffer, mode);
     }
     /**
@@ -140,7 +143,7 @@ class Table {
      */
     async createIndex(column, options) {
         // Bit of a hack to get around the fact that TS has no package-scope.
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        // biome-ignore lint/suspicious/noExplicitAny: skip
         const nativeIndex = options?.config?.inner;
         await this.inner.createIndex(nativeIndex, column, options?.replace);
     }
@@ -296,6 +299,45 @@ class Table {
     async restore() {
         await this.inner.restore();
     }
+    /**
+     * Optimize the on-disk data and indices for better performance.
+     *
+     * Modeled after ``VACUUM`` in PostgreSQL.
+     *
+     *  Optimization covers three operations:
+     *
+     *  - Compaction: Merges small files into larger ones
+     *  - Prune: Removes old versions of the dataset
+     *  - Index: Optimizes the indices, adding new data to existing indices
+     *
+     *
+     *  Experimental API
+     *  ----------------
+     *
+     *  The optimization process is undergoing active development and may change.
+     *  Our goal with these changes is to improve the performance of optimization and
+     *  reduce the complexity.
+     *
+     *  That being said, it is essential today to run optimize if you want the best
+     *  performance.  It should be stable and safe to use in production, but it our
+     *  hope that the API may be simplified (or not even need to be called) in the
+     *  future.
+     *
+     *  The frequency an application shoudl call optimize is based on the frequency of
+     *  data modifications.  If data is frequently added, deleted, or updated then
+     *  optimize should be run frequently.  A good rule of thumb is to run optimize if
+     *  you have added or modified 100,000 or more records or run more than 20 data
+     *  modification operations.
+     */
+    async optimize(options) {
+        let cleanupOlderThanMs;
+        if (options?.cleanupOlderThan !== undefined &&
+            options?.cleanupOlderThan !== null) {
+            cleanupOlderThanMs =
+                new Date().getTime() - options.cleanupOlderThan.getTime();
+        }
+        return await this.inner.optimize(cleanupOlderThanMs);
+    }
     /** List all indices that have been created with {@link Table.createIndex} */
     async listIndices() {
         return await this.inner.listIndices();