@lancedb/lancedb 0.4.20 → 0.5.1

package/lancedb/embedding/embedding_function.ts

@@ -1,4 +1,4 @@
-// Copyright 2023 Lance Developers.
+// Copyright 2024 Lance Developers.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,67 +12,151 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { type Float } from "apache-arrow";
+import "reflect-metadata";
+import {
+  DataType,
+  Field,
+  FixedSizeList,
+  Float,
+  Float32,
+  isDataType,
+  isFixedSizeList,
+  isFloat,
+  newVectorType,
+} from "../arrow";
+import { sanitizeType } from "../sanitize";
 
 /**
- * An embedding function that automatically creates vector representation for a given column.
+ * Options for a given embedding function
  */
-export interface EmbeddingFunction<T> {
-  /**
-   * The name of the column that will be used as input for the Embedding Function.
-   */
-  sourceColumn: string;
+export interface FunctionOptions {
+  // biome-ignore lint/suspicious/noExplicitAny: options can be anything
+  [key: string]: any;
+}
 
+/**
+ * An embedding function that automatically creates vector representation for a given column.
+ */
+export abstract class EmbeddingFunction<
+  // biome-ignore lint/suspicious/noExplicitAny: we don't know what the implementor will do
+  T = any,
+  M extends FunctionOptions = FunctionOptions,
+> {
   /**
-   * The data type of the embedding
+   * Convert the embedding function to a JSON object
+   * It is used to serialize the embedding function to the schema
+   * It's important that any object returned by this method contains all the necessary
+   * information to recreate the embedding function
    *
-   * The embedding function should return `number`.  This will be converted into
-   * an Arrow float array.  By default this will be Float32 but this property can
-   * be used to control the conversion.
-   */
-  embeddingDataType?: Float;
-
-  /**
-   * The dimension of the embedding
+   * It should return the same object that was passed to the constructor
+   * If it does not, the embedding function will not be able to be recreated, or could be recreated incorrectly
    *
-   * This is optional, normally this can be determined by looking at the results of
-   * `embed`.  If this is not specified, and there is an attempt to apply the embedding
-   * to an empty table, then that process will fail.
+   * @example
+   * ```ts
+   * class MyEmbeddingFunction extends EmbeddingFunction {
+   *   constructor(options: {model: string, timeout: number}) {
+   *     super();
+   *     this.model = options.model;
+   *     this.timeout = options.timeout;
+   *   }
+   *   toJSON() {
+   *     return {
+   *       model: this.model,
+   *       timeout: this.timeout,
+   *     };
+   * }
+   * ```
    */
-  embeddingDimension?: number;
+  abstract toJSON(): Partial<M>;
 
   /**
-   * The name of the column that will contain the embedding
+   * sourceField is used in combination with `LanceSchema` to provide a declarative data model
    *
-   * By default this is "vector"
+   * @param optionsOrDatatype - The options for the field or the datatype
+   *
+   * @see {@link lancedb.LanceSchema}
    */
-  destColumn?: string;
+  sourceField(
+    optionsOrDatatype: Partial<FieldOptions> | DataType,
+  ): [DataType, Map<string, EmbeddingFunction>] {
+    let datatype = isDataType(optionsOrDatatype)
+      ? optionsOrDatatype
+      : optionsOrDatatype?.datatype;
+    if (!datatype) {
+      throw new Error("Datatype is required");
+    }
+    datatype = sanitizeType(datatype);
+    const metadata = new Map<string, EmbeddingFunction>();
+    metadata.set("source_column_for", this);
+
+    return [datatype, metadata];
+  }
 
   /**
-   * Should the source column be excluded from the resulting table
+   * vectorField is used in combination with `LanceSchema` to provide a declarative data model
    *
-   * By default the source column is included.  Set this to true and
-   * only the embedding will be stored.
+   * @param options - The options for the field
+   *
+   * @see {@link lancedb.LanceSchema}
    */
-  excludeSource?: boolean;
+  vectorField(
+    options?: Partial<FieldOptions>,
+  ): [DataType, Map<string, EmbeddingFunction>] {
+    let dtype: DataType;
+    const dims = this.ndims() ?? options?.dims;
+    if (!options?.datatype) {
+      if (dims === undefined) {
+        throw new Error("ndims is required for vector field");
+      }
+      dtype = new FixedSizeList(dims, new Field("item", new Float32(), true));
+    } else {
+      if (isFixedSizeList(options.datatype)) {
+        dtype = options.datatype;
+      } else if (isFloat(options.datatype)) {
+        if (dims === undefined) {
+          throw new Error("ndims is required for vector field");
+        }
+        dtype = newVectorType(dims, options.datatype);
+      } else {
+        throw new Error(
+          "Expected FixedSizeList or Float as datatype for vector field",
+        );
+      }
+    }
+    const metadata = new Map<string, EmbeddingFunction>();
+    metadata.set("vector_column_for", this);
+
+    return [dtype, metadata];
+  }
+
+  /** The number of dimensions of the embeddings */
+  ndims(): number | undefined {
+    return undefined;
+  }
+
+  /** The datatype of the embeddings */
+  abstract embeddingDataType(): Float;
 
   /**
    * Creates a vector representation for the given values.
    */
-  embed: (data: T[]) => Promise<number[][]>;
-}
+  abstract computeSourceEmbeddings(
+    data: T[],
+  ): Promise<number[][] | Float32Array[] | Float64Array[]>;
 
-/** Test if the input seems to be an embedding function */
-export function isEmbeddingFunction<T>(
-  value: unknown,
-): value is EmbeddingFunction<T> {
-  if (typeof value !== "object" || value === null) {
-    return false;
-  }
-  if (!("sourceColumn" in value) || !("embed" in value)) {
-    return false;
+  /**
+  Compute the embeddings for a single query
+ */
+  async computeQueryEmbeddings(
+    data: T,
+  ): Promise<number[] | Float32Array | Float64Array> {
+    return this.computeSourceEmbeddings([data]).then(
+      (embeddings) => embeddings[0],
+    );
   }
-  return (
-    typeof value.sourceColumn === "string" && typeof value.embed === "function"
-  );
+}
+
+export interface FieldOptions<T extends DataType = DataType> {
+  datatype: T;
+  dims?: number;
 }

package/lancedb/embedding/index.ts

@@ -1,2 +1,113 @@
-export { EmbeddingFunction, isEmbeddingFunction } from "./embedding_function";
-export { OpenAIEmbeddingFunction } from "./openai";
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { DataType, Field, Schema } from "../arrow";
+import { isDataType } from "../arrow";
+import { sanitizeType } from "../sanitize";
+import { EmbeddingFunction } from "./embedding_function";
+import { EmbeddingFunctionConfig, getRegistry } from "./registry";
+
+export { EmbeddingFunction } from "./embedding_function";
+
+// We need to explicitly export '*' so that the `register` decorator actually registers the class.
+export * from "./openai";
+export * from "./registry";
+
+/**
+ * Create a schema with embedding functions.
+ *
+ * @param fields
+ * @returns Schema
+ * @example
+ * ```ts
+ * class MyEmbeddingFunction extends EmbeddingFunction {
+ * // ...
+ * }
+ * const func = new MyEmbeddingFunction();
+ * const schema = LanceSchema({
+ *   id: new Int32(),
+ *   text: func.sourceField(new Utf8()),
+ *   vector: func.vectorField(),
+ *   // optional: specify the datatype and/or dimensions
+ *   vector2: func.vectorField({ datatype: new Float32(), dims: 3}),
+ * });
+ *
+ * const table = await db.createTable("my_table", data, { schema });
+ * ```
+ */
+export function LanceSchema(
+  fields: Record<string, [object, Map<string, EmbeddingFunction>] | object>,
+): Schema {
+  const arrowFields: Field[] = [];
+
+  const embeddingFunctions = new Map<
+    EmbeddingFunction,
+    Partial<EmbeddingFunctionConfig>
+  >();
+  Object.entries(fields).forEach(([key, value]) => {
+    if (isDataType(value)) {
+      arrowFields.push(new Field(key, sanitizeType(value), true));
+    } else {
+      const [dtype, metadata] = value as [
+        object,
+        Map<string, EmbeddingFunction>,
+      ];
+      arrowFields.push(new Field(key, sanitizeType(dtype), true));
+      parseEmbeddingFunctions(embeddingFunctions, key, metadata);
+    }
+  });
+  const registry = getRegistry();
+  const metadata = registry.getTableMetadata(
+    Array.from(embeddingFunctions.values()) as EmbeddingFunctionConfig[],
+  );
+  const schema = new Schema(arrowFields, metadata);
+  return schema;
+}
+
+function parseEmbeddingFunctions(
+  embeddingFunctions: Map<EmbeddingFunction, Partial<EmbeddingFunctionConfig>>,
+  key: string,
+  metadata: Map<string, EmbeddingFunction>,
+): void {
+  if (metadata.has("source_column_for")) {
+    const embedFunction = metadata.get("source_column_for")!;
+    const current = embeddingFunctions.get(embedFunction);
+    if (current !== undefined) {
+      embeddingFunctions.set(embedFunction, {
+        ...current,
+        sourceColumn: key,
+      });
+    } else {
+      embeddingFunctions.set(embedFunction, {
+        sourceColumn: key,
+        function: embedFunction,
+      });
+    }
+  } else if (metadata.has("vector_column_for")) {
+    const embedFunction = metadata.get("vector_column_for")!;
+
+    const current = embeddingFunctions.get(embedFunction);
+    if (current !== undefined) {
+      embeddingFunctions.set(embedFunction, {
+        ...current,
+        vectorColumn: key,
+      });
+    } else {
+      embeddingFunctions.set(embedFunction, {
+        vectorColumn: key,
+        function: embedFunction,
+      });
+    }
+  }
+}

package/lancedb/embedding/openai.ts

@@ -12,18 +12,32 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { type EmbeddingFunction } from "./embedding_function";
 import type OpenAI from "openai";
+import { Float, Float32 } from "../arrow";
+import { EmbeddingFunction } from "./embedding_function";
+import { register } from "./registry";
 
-export class OpenAIEmbeddingFunction implements EmbeddingFunction<string> {
-  private readonly _openai: OpenAI;
-  private readonly _modelName: string;
+export type OpenAIOptions = {
+  apiKey?: string;
+  model?: string;
+};
+
+@register("openai")
+export class OpenAIEmbeddingFunction extends EmbeddingFunction<
+  string,
+  OpenAIOptions
+> {
+  #openai: OpenAI;
+  #modelName: string;
+
+  constructor(options: OpenAIOptions = { model: "text-embedding-ada-002" }) {
+    super();
+    const openAIKey = options?.apiKey ?? process.env.OPENAI_API_KEY;
+    if (!openAIKey) {
+      throw new Error("OpenAI API key is required");
+    }
+    const modelName = options?.model ?? "text-embedding-ada-002";
 
-  constructor(
-    sourceColumn: string,
-    openAIKey: string,
-    modelName: string = "text-embedding-ada-002",
-  ) {
     /**
      * @type {import("openai").default}
      */
@@ -36,18 +50,40 @@ export class OpenAIEmbeddingFunction implements EmbeddingFunction<string> {
       throw new Error("please install openai@^4.24.1 using npm install openai");
     }
 
-    this.sourceColumn = sourceColumn;
     const configuration = {
       apiKey: openAIKey,
     };
 
-    this._openai = new Openai(configuration);
-    this._modelName = modelName;
+    this.#openai = new Openai(configuration);
+    this.#modelName = modelName;
+  }
+
+  toJSON() {
+    return {
+      model: this.#modelName,
+    };
+  }
+
+  ndims(): number {
+    switch (this.#modelName) {
+      case "text-embedding-ada-002":
+        return 1536;
+      case "text-embedding-3-large":
+        return 3072;
+      case "text-embedding-3-small":
+        return 1536;
+      default:
+        return null as never;
+    }
+  }
+
+  embeddingDataType(): Float {
+    return new Float32();
   }
 
-  async embed(data: string[]): Promise<number[][]> {
-    const response = await this._openai.embeddings.create({
-      model: this._modelName,
+  async computeSourceEmbeddings(data: string[]): Promise<number[][]> {
+    const response = await this.#openai.embeddings.create({
+      model: this.#modelName,
       input: data,
     });
 
@@ -58,5 +94,15 @@ export class OpenAIEmbeddingFunction implements EmbeddingFunction<string> {
     return embeddings;
   }
 
-  sourceColumn: string;
+  async computeQueryEmbeddings(data: string): Promise<number[]> {
+    if (typeof data !== "string") {
+      throw new Error("Data must be a string");
+    }
+    const response = await this.#openai.embeddings.create({
+      model: this.#modelName,
+      input: data,
+    });
+
+    return response.data[0].embedding;
+  }
 }

package/lancedb/embedding/registry.ts

@@ -0,0 +1,172 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import type { EmbeddingFunction } from "./embedding_function";
+import "reflect-metadata";
+
+export interface EmbeddingFunctionOptions {
+  [key: string]: unknown;
+}
+
+export interface EmbeddingFunctionFactory<
+  T extends EmbeddingFunction = EmbeddingFunction,
+> {
+  new (modelOptions?: EmbeddingFunctionOptions): T;
+}
+
+interface EmbeddingFunctionCreate<T extends EmbeddingFunction> {
+  create(options?: EmbeddingFunctionOptions): T;
+}
+
+/**
+ * This is a singleton class used to register embedding functions
+ * and fetch them by name. It also handles serializing and deserializing.
+ * You can implement your own embedding function by subclassing EmbeddingFunction
+ * or TextEmbeddingFunction and registering it with the registry
+ */
+export class EmbeddingFunctionRegistry {
+  #functions: Map<string, EmbeddingFunctionFactory> = new Map();
+
+  /**
+   * Register an embedding function
+   * @param name The name of the function
+   * @param func The function to register
+   */
+  register<T extends EmbeddingFunctionFactory = EmbeddingFunctionFactory>(
+    this: EmbeddingFunctionRegistry,
+    alias?: string,
+    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+  ): (ctor: T) => any {
+    const self = this;
+    return function (ctor: T) {
+      if (!alias) {
+        alias = ctor.name;
+      }
+      if (self.#functions.has(alias)) {
+        throw new Error(
+          `Embedding function with alias "${alias}" already exists`,
+        );
+      }
+      self.#functions.set(alias, ctor);
+      Reflect.defineMetadata("lancedb::embedding::name", alias, ctor);
+      return ctor;
+    };
+  }
+
+  /**
+   * Fetch an embedding function by name
+   * @param name The name of the function
+   */
+  get<T extends EmbeddingFunction<unknown> = EmbeddingFunction>(
+    name: string,
+  ): EmbeddingFunctionCreate<T> | undefined {
+    const factory = this.#functions.get(name);
+    if (!factory) {
+      return undefined;
+    }
+    return {
+      create: function (options: EmbeddingFunctionOptions) {
+        return new factory(options) as unknown as T;
+      },
+    };
+  }
+
+  /**
+   * reset the registry to the initial state
+   */
+  reset(this: EmbeddingFunctionRegistry) {
+    this.#functions.clear();
+  }
+
+  parseFunctions(
+    this: EmbeddingFunctionRegistry,
+    metadata: Map<string, string>,
+  ): Map<string, EmbeddingFunctionConfig> {
+    if (!metadata.has("embedding_functions")) {
+      return new Map();
+    } else {
+      type FunctionConfig = {
+        name: string;
+        sourceColumn: string;
+        vectorColumn: string;
+        model: EmbeddingFunctionOptions;
+      };
+      const functions = <FunctionConfig[]>(
+        JSON.parse(metadata.get("embedding_functions")!)
+      );
+      return new Map(
+        functions.map((f) => {
+          const fn = this.get(f.name);
+          if (!fn) {
+            throw new Error(`Function "${f.name}" not found in registry`);
+          }
+          return [
+            f.name,
+            {
+              sourceColumn: f.sourceColumn,
+              vectorColumn: f.vectorColumn,
+              function: this.get(f.name)!.create(f.model),
+            },
+          ];
+        }),
+      );
+    }
+  }
+  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+  functionToMetadata(conf: EmbeddingFunctionConfig): Record<string, any> {
+    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+    const metadata: Record<string, any> = {};
+    const name = Reflect.getMetadata(
+      "lancedb::embedding::name",
+      conf.function.constructor,
+    );
+    metadata["sourceColumn"] = conf.sourceColumn;
+    metadata["vectorColumn"] = conf.vectorColumn ?? "vector";
+    metadata["name"] = name ?? conf.function.constructor.name;
+    metadata["model"] = conf.function.toJSON();
+    return metadata;
+  }
+
+  getTableMetadata(functions: EmbeddingFunctionConfig[]): Map<string, string> {
+    const metadata = new Map<string, string>();
+    const jsonData = functions.map((conf) => this.functionToMetadata(conf));
+    metadata.set("embedding_functions", JSON.stringify(jsonData));
+
+    return metadata;
+  }
+}
+
+const _REGISTRY = new EmbeddingFunctionRegistry();
+
+export function register(name?: string) {
+  return _REGISTRY.register(name);
+}
+
+/**
+ * Utility function to get the global instance of the registry
+ * @returns `EmbeddingFunctionRegistry` The global instance of the registry
+ * @example
+ * ```ts
+ * const registry = getRegistry();
+ * const openai = registry.get("openai").create();
+ */
+export function getRegistry(): EmbeddingFunctionRegistry {
+  return _REGISTRY;
+}
+
+export interface EmbeddingFunctionConfig {
+  sourceColumn: string;
+  vectorColumn?: string;
+  function: EmbeddingFunction;
+}

package/lancedb/query.ts

@@ -12,14 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { RecordBatch, tableFromIPC, Table as ArrowTable } from "apache-arrow";
+import { Table as ArrowTable, RecordBatch, tableFromIPC } from "./arrow";
+import { type IvfPqOptions } from "./indices";
 import {
   RecordBatchIterator as NativeBatchIterator,
   Query as NativeQuery,
   Table as NativeTable,
   VectorQuery as NativeVectorQuery,
 } from "./native";
-import { type IvfPqOptions } from "./indices";
 export class RecordBatchIterator implements AsyncIterator<RecordBatch> {
   private promisedInner?: Promise<NativeBatchIterator>;
   private inner?: NativeBatchIterator;
@@ -29,7 +29,7 @@ export class RecordBatchIterator implements AsyncIterator<RecordBatch> {
     this.promisedInner = promise;
   }
 
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  // biome-ignore lint/suspicious/noExplicitAny: skip
   async next(): Promise<IteratorResult<RecordBatch<any>>> {
     if (this.inner === undefined) {
       this.inner = await this.promisedInner;
@@ -56,7 +56,9 @@ export class QueryBase<
   QueryType,
 > implements AsyncIterable<RecordBatch>
 {
-  protected constructor(protected inner: NativeQueryType) {}
+  protected constructor(protected inner: NativeQueryType) {
+    // intentionally empty
+  }
 
   /**
    * A filter statement to be applied to this query.
@@ -150,7 +152,7 @@ export class QueryBase<
     return new RecordBatchIterator(this.nativeExecute());
   }
 
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  // biome-ignore lint/suspicious/noExplicitAny: skip
   [Symbol.asyncIterator](): AsyncIterator<RecordBatch<any>> {
     const promise = this.nativeExecute();
     return new RecordBatchIterator(promise);
@@ -168,6 +170,7 @@ export class QueryBase<
   /** Collect the results as an array of objects. */
   async toArray(): Promise<unknown[]> {
     const tbl = await this.toArrow();
+
     // eslint-disable-next-line @typescript-eslint/no-unsafe-return
     return tbl.toArray();
   }
@@ -368,7 +371,7 @@ export class Query extends QueryBase<NativeQuery, Query> {
    * a default `limit` of 10 will be used.  @see {@link Query#limit}
    */
   nearestTo(vector: unknown): VectorQuery {
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    // biome-ignore lint/suspicious/noExplicitAny: skip
     const vectorQuery = this.inner.nearestTo(Float32Array.from(vector as any));
     return new VectorQuery(vectorQuery);
   }