@lancedb/lancedb 0.4.20 → 0.5.1

package/dist/embedding/registry.js

@@ -0,0 +1,123 @@
+"use strict";
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getRegistry = exports.register = exports.EmbeddingFunctionRegistry = void 0;
+require("reflect-metadata");
+/**
+ * This is a singleton class used to register embedding functions
+ * and fetch them by name. It also handles serializing and deserializing.
+ * You can implement your own embedding function by subclassing EmbeddingFunction
+ * or TextEmbeddingFunction and registering it with the registry
+ */
+class EmbeddingFunctionRegistry {
+    #functions = new Map();
+    /**
+     * Register an embedding function
+     * @param name The name of the function
+     * @param func The function to register
+     */
+    register(alias) {
+        const self = this;
+        return function (ctor) {
+            if (!alias) {
+                alias = ctor.name;
+            }
+            if (self.#functions.has(alias)) {
+                throw new Error(`Embedding function with alias "${alias}" already exists`);
+            }
+            self.#functions.set(alias, ctor);
+            Reflect.defineMetadata("lancedb::embedding::name", alias, ctor);
+            return ctor;
+        };
+    }
+    /**
+     * Fetch an embedding function by name
+     * @param name The name of the function
+     */
+    get(name) {
+        const factory = this.#functions.get(name);
+        if (!factory) {
+            return undefined;
+        }
+        return {
+            create: function (options) {
+                return new factory(options);
+            },
+        };
+    }
+    /**
+     * reset the registry to the initial state
+     */
+    reset() {
+        this.#functions.clear();
+    }
+    parseFunctions(metadata) {
+        if (!metadata.has("embedding_functions")) {
+            return new Map();
+        }
+        else {
+            const functions = (JSON.parse(metadata.get("embedding_functions")));
+            return new Map(functions.map((f) => {
+                const fn = this.get(f.name);
+                if (!fn) {
+                    throw new Error(`Function "${f.name}" not found in registry`);
+                }
+                return [
+                    f.name,
+                    {
+                        sourceColumn: f.sourceColumn,
+                        vectorColumn: f.vectorColumn,
+                        function: this.get(f.name).create(f.model),
+                    },
+                ];
+            }));
+        }
+    }
+    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+    functionToMetadata(conf) {
+        // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+        const metadata = {};
+        const name = Reflect.getMetadata("lancedb::embedding::name", conf.function.constructor);
+        metadata["sourceColumn"] = conf.sourceColumn;
+        metadata["vectorColumn"] = conf.vectorColumn ?? "vector";
+        metadata["name"] = name ?? conf.function.constructor.name;
+        metadata["model"] = conf.function.toJSON();
+        return metadata;
+    }
+    getTableMetadata(functions) {
+        const metadata = new Map();
+        const jsonData = functions.map((conf) => this.functionToMetadata(conf));
+        metadata.set("embedding_functions", JSON.stringify(jsonData));
+        return metadata;
+    }
+}
+exports.EmbeddingFunctionRegistry = EmbeddingFunctionRegistry;
+const _REGISTRY = new EmbeddingFunctionRegistry();
+function register(name) {
+    return _REGISTRY.register(name);
+}
+exports.register = register;
+/**
+ * Utility function to get the global instance of the registry
+ * @returns `EmbeddingFunctionRegistry` The global instance of the registry
+ * @example
+ * ```ts
+ * const registry = getRegistry();
+ * const openai = registry.get("openai").create();
+ */
+function getRegistry() {
+    return _REGISTRY;
+}
+exports.getRegistry = getRegistry;

package/dist/native.d.ts

@@ -15,6 +15,31 @@ export interface IndexConfig {
    */
   columns: Array<string>
 }
+/** Statistics about a compaction operation. */
+export interface CompactionStats {
+  /** The number of fragments removed */
+  fragmentsRemoved: number
+  /** The number of new, compacted fragments added */
+  fragmentsAdded: number
+  /** The number of data files removed */
+  filesRemoved: number
+  /** The number of new, compacted data files added */
+  filesAdded: number
+}
+/** Statistics about a cleanup operation */
+export interface RemovalStats {
+  /** The number of bytes removed */
+  bytesRemoved: number
+  /** The number of old versions removed */
+  oldVersionsRemoved: number
+}
+/** Statistics about an optimize operation */
+export interface OptimizeStats {
+  /** Statistics about the compaction operation */
+  compaction: CompactionStats
+  /** Statistics about the removal operation */
+  prune: RemovalStats
+}
 /**
  *  A definition of a column alteration. The alteration changes the column at
  * `path` to have the new name `name`, to be nullable if `nullable` is true,
@@ -151,5 +176,6 @@ export class Table {
   checkout(version: number): Promise<void>
   checkoutLatest(): Promise<void>
   restore(): Promise<void>
+  optimize(olderThanMs?: number | undefined | null): Promise<OptimizeStats>
   listIndices(): Promise<Array<IndexConfig>>
 }

package/dist/query.d.ts

@@ -1,4 +1,4 @@
-import { RecordBatch, Table as ArrowTable } from "apache-arrow";
+import { Table as ArrowTable, RecordBatch } from "./arrow";
 import { RecordBatchIterator as NativeBatchIterator, Query as NativeQuery, Table as NativeTable, VectorQuery as NativeVectorQuery } from "./native";
 export declare class RecordBatchIterator implements AsyncIterator<RecordBatch> {
     private promisedInner?;

package/dist/query.js

@@ -14,7 +14,7 @@
 // limitations under the License.
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Query = exports.VectorQuery = exports.QueryBase = exports.RecordBatchIterator = void 0;
-const apache_arrow_1 = require("apache-arrow");
+const arrow_1 = require("./arrow");
 class RecordBatchIterator {
     promisedInner;
     inner;
@@ -22,7 +22,7 @@ class RecordBatchIterator {
         // TODO: check promise reliably so we dont need to pass two arguments.
         this.promisedInner = promise;
     }
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    // biome-ignore lint/suspicious/noExplicitAny: skip
     async next() {
         if (this.inner === undefined) {
             this.inner = await this.promisedInner;
@@ -34,7 +34,7 @@ class RecordBatchIterator {
         if (n == null) {
             return Promise.resolve({ done: true, value: null });
         }
-        const tbl = (0, apache_arrow_1.tableFromIPC)(n);
+        const tbl = (0, arrow_1.tableFromIPC)(n);
         if (tbl.batches.length != 1) {
             throw new Error("Expected only one batch");
         }
@@ -48,6 +48,7 @@ class QueryBase {
     inner;
     constructor(inner) {
         this.inner = inner;
+        // intentionally empty
     }
     /**
      * A filter statement to be applied to this query.
@@ -136,7 +137,7 @@ class QueryBase {
     execute() {
         return new RecordBatchIterator(this.nativeExecute());
     }
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    // biome-ignore lint/suspicious/noExplicitAny: skip
     [Symbol.asyncIterator]() {
         const promise = this.nativeExecute();
         return new RecordBatchIterator(promise);
@@ -147,7 +148,7 @@ class QueryBase {
         for await (const batch of this) {
             batches.push(batch);
         }
-        return new apache_arrow_1.Table(batches);
+        return new arrow_1.Table(batches);
     }
     /** Collect the results as an array of objects. */
     async toArray() {
@@ -338,7 +339,7 @@ class Query extends QueryBase {
      * a default `limit` of 10 will be used.  @see {@link Query#limit}
      */
     nearestTo(vector) {
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        // biome-ignore lint/suspicious/noExplicitAny: skip
         const vectorQuery = this.inner.nearestTo(Float32Array.from(vector));
         return new VectorQuery(vectorQuery);
     }

package/dist/sanitize.d.ts

@@ -1,4 +1,25 @@
-import { Schema } from "apache-arrow";
+import type { TKeys } from "apache-arrow/type";
+import { DataType, Date_, Decimal, DenseUnion, Dictionary, Duration, Field, FixedSizeBinary, FixedSizeList, Float, Int, Interval, List, Map_, Schema, SparseUnion, Struct, Time, Timestamp, TimestampMicrosecond, TimestampMillisecond, TimestampNanosecond, TimestampSecond, Type, Union } from "./arrow";
+export declare function sanitizeMetadata(metadataLike?: unknown): Map<string, string> | undefined;
+export declare function sanitizeInt(typeLike: object): Int<Type.Int | Type.Int8 | Type.Int16 | Type.Int32 | Type.Int64 | Type.Uint8 | Type.Uint16 | Type.Uint32 | Type.Uint64>;
+export declare function sanitizeFloat(typeLike: object): Float<Type.Float | Type.Float16 | Type.Float32 | Type.Float64>;
+export declare function sanitizeDecimal(typeLike: object): Decimal;
+export declare function sanitizeDate(typeLike: object): Date_<import("apache-arrow/type").Dates>;
+export declare function sanitizeTime(typeLike: object): Time<Type.Time | Type.TimeSecond | Type.TimeMillisecond | Type.TimeMicrosecond | Type.TimeNanosecond>;
+export declare function sanitizeTimestamp(typeLike: object): Timestamp<Type.Timestamp | Type.TimestampSecond | Type.TimestampMillisecond | Type.TimestampMicrosecond | Type.TimestampNanosecond>;
+export declare function sanitizeTypedTimestamp(typeLike: object, Datatype: typeof TimestampNanosecond | typeof TimestampMicrosecond | typeof TimestampMillisecond | typeof TimestampSecond): TimestampSecond | TimestampMillisecond | TimestampMicrosecond | TimestampNanosecond;
+export declare function sanitizeInterval(typeLike: object): Interval<Type.Interval | Type.IntervalDayTime | Type.IntervalYearMonth>;
+export declare function sanitizeList(typeLike: object): List<any>;
+export declare function sanitizeStruct(typeLike: object): Struct<any>;
+export declare function sanitizeUnion(typeLike: object): Union<Type.Union | Type.DenseUnion | Type.SparseUnion>;
+export declare function sanitizeTypedUnion(typeLike: object, UnionType: typeof DenseUnion | typeof SparseUnion): SparseUnion | DenseUnion;
+export declare function sanitizeFixedSizeBinary(typeLike: object): FixedSizeBinary;
+export declare function sanitizeFixedSizeList(typeLike: object): FixedSizeList<any>;
+export declare function sanitizeMap(typeLike: object): Map_<any, any>;
+export declare function sanitizeDuration(typeLike: object): Duration<Type.Duration | Type.DurationSecond | Type.DurationMillisecond | Type.DurationMicrosecond | Type.DurationNanosecond>;
+export declare function sanitizeDictionary(typeLike: object): Dictionary<DataType<any, any>, TKeys>;
+export declare function sanitizeType(typeLike: unknown): DataType<any>;
+export declare function sanitizeField(fieldLike: unknown): Field;
 /**
  * Convert something schemaLike into a Schema instance
  *