@lancedb/lancedb 0.7.1 → 0.9.0

package/lancedb/embedding/openai.ts

@@ -1,113 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import type OpenAI from "openai";
-import { type EmbeddingCreateParams } from "openai/resources";
-import { Float, Float32 } from "../arrow";
-import { EmbeddingFunction } from "./embedding_function";
-import { register } from "./registry";
-
-export type OpenAIOptions = {
-  apiKey: string;
-  model: EmbeddingCreateParams["model"];
-};
-
-@register("openai")
-export class OpenAIEmbeddingFunction extends EmbeddingFunction<
-  string,
-  Partial<OpenAIOptions>
-> {
-  #openai: OpenAI;
-  #modelName: OpenAIOptions["model"];
-
-  constructor(
-    options: Partial<OpenAIOptions> = {
-      model: "text-embedding-ada-002",
-    },
-  ) {
-    super();
-    const openAIKey = options?.apiKey ?? process.env.OPENAI_API_KEY;
-    if (!openAIKey) {
-      throw new Error("OpenAI API key is required");
-    }
-    const modelName = options?.model ?? "text-embedding-ada-002";
-
-    /**
-     * @type {import("openai").default}
-     */
-    // eslint-disable-next-line @typescript-eslint/naming-convention
-    let Openai;
-    try {
-      // eslint-disable-next-line @typescript-eslint/no-var-requires
-      Openai = require("openai");
-    } catch {
-      throw new Error("please install openai@^4.24.1 using npm install openai");
-    }
-
-    const configuration = {
-      apiKey: openAIKey,
-    };
-
-    this.#openai = new Openai(configuration);
-    this.#modelName = modelName;
-  }
-
-  toJSON() {
-    return {
-      model: this.#modelName,
-    };
-  }
-
-  ndims(): number {
-    switch (this.#modelName) {
-      case "text-embedding-ada-002":
-        return 1536;
-      case "text-embedding-3-large":
-        return 3072;
-      case "text-embedding-3-small":
-        return 1536;
-      default:
-        throw new Error(`Unknown model: ${this.#modelName}`);
-    }
-  }
-
-  embeddingDataType(): Float {
-    return new Float32();
-  }
-
-  async computeSourceEmbeddings(data: string[]): Promise<number[][]> {
-    const response = await this.#openai.embeddings.create({
-      model: this.#modelName,
-      input: data,
-    });
-
-    const embeddings: number[][] = [];
-    for (let i = 0; i < response.data.length; i++) {
-      embeddings.push(response.data[i].embedding);
-    }
-    return embeddings;
-  }
-
-  async computeQueryEmbeddings(data: string): Promise<number[]> {
-    if (typeof data !== "string") {
-      throw new Error("Data must be a string");
-    }
-    const response = await this.#openai.embeddings.create({
-      model: this.#modelName,
-      input: data,
-    });
-
-    return response.data[0].embedding;
-  }
-}

package/lancedb/embedding/registry.ts

@@ -1,188 +0,0 @@
-// Copyright 2024 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import {
-  type EmbeddingFunction,
-  type EmbeddingFunctionConstructor,
-} from "./embedding_function";
-import "reflect-metadata";
-import { OpenAIEmbeddingFunction } from "./openai";
-
-interface EmbeddingFunctionCreate<T extends EmbeddingFunction> {
-  create(options?: T["TOptions"]): T;
-}
-
-/**
- * This is a singleton class used to register embedding functions
- * and fetch them by name. It also handles serializing and deserializing.
- * You can implement your own embedding function by subclassing EmbeddingFunction
- * or TextEmbeddingFunction and registering it with the registry
- */
-export class EmbeddingFunctionRegistry {
-  #functions = new Map<string, EmbeddingFunctionConstructor>();
-
-  /**
-   * Register an embedding function
-   * @param name The name of the function
-   * @param func The function to register
-   * @throws Error if the function is already registered
-   */
-  register<
-    T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor,
-  >(
-    this: EmbeddingFunctionRegistry,
-    alias?: string,
-    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-  ): (ctor: T) => any {
-    const self = this;
-    return function (ctor: T) {
-      if (!alias) {
-        alias = ctor.name;
-      }
-      if (self.#functions.has(alias)) {
-        throw new Error(
-          `Embedding function with alias "${alias}" already exists`,
-        );
-      }
-      self.#functions.set(alias, ctor);
-      Reflect.defineMetadata("lancedb::embedding::name", alias, ctor);
-      return ctor;
-    };
-  }
-
-  /**
-   * Fetch an embedding function by name
-   * @param name The name of the function
-   */
-  get<T extends EmbeddingFunction<unknown>, Name extends string = "">(
-    name: Name extends "openai" ? "openai" : string,
-    //This makes it so that you can use string constants as "types", or use an explicitly supplied type
-    // ex:
-    // `registry.get("openai") -> EmbeddingFunctionCreate<OpenAIEmbeddingFunction>`
-    // `registry.get<MyCustomEmbeddingFunction>("my_func") -> EmbeddingFunctionCreate<MyCustomEmbeddingFunction> | undefined`
-    //
-    // the reason this is important is that we always know our built in functions are defined so the user isnt forced to do a non null/undefined
-    // ```ts
-    // const openai: OpenAIEmbeddingFunction = registry.get("openai").create()
-    // ```
-  ): Name extends "openai"
-    ? EmbeddingFunctionCreate<OpenAIEmbeddingFunction>
-    : EmbeddingFunctionCreate<T> | undefined {
-    type Output = Name extends "openai"
-      ? EmbeddingFunctionCreate<OpenAIEmbeddingFunction>
-      : EmbeddingFunctionCreate<T> | undefined;
-
-    const factory = this.#functions.get(name);
-    if (!factory) {
-      return undefined as Output;
-    }
-
-    return {
-      create: function (options?: T["TOptions"]) {
-        return new factory(options);
-      },
-    } as Output;
-  }
-
-  /**
-   * reset the registry to the initial state
-   */
-  reset(this: EmbeddingFunctionRegistry) {
-    this.#functions.clear();
-  }
-
-  /**
-   * @ignore
-   */
-  parseFunctions(
-    this: EmbeddingFunctionRegistry,
-    metadata: Map<string, string>,
-  ): Map<string, EmbeddingFunctionConfig> {
-    if (!metadata.has("embedding_functions")) {
-      return new Map();
-    } else {
-      type FunctionConfig = {
-        name: string;
-        sourceColumn: string;
-        vectorColumn: string;
-        model: EmbeddingFunction["TOptions"];
-      };
-      const functions = <FunctionConfig[]>(
-        JSON.parse(metadata.get("embedding_functions")!)
-      );
-      return new Map(
-        functions.map((f) => {
-          const fn = this.get(f.name);
-          if (!fn) {
-            throw new Error(`Function "${f.name}" not found in registry`);
-          }
-          return [
-            f.name,
-            {
-              sourceColumn: f.sourceColumn,
-              vectorColumn: f.vectorColumn,
-              function: this.get(f.name)!.create(f.model),
-            },
-          ];
-        }),
-      );
-    }
-  }
-  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-  functionToMetadata(conf: EmbeddingFunctionConfig): Record<string, any> {
-    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-    const metadata: Record<string, any> = {};
-    const name = Reflect.getMetadata(
-      "lancedb::embedding::name",
-      conf.function.constructor,
-    );
-    metadata["sourceColumn"] = conf.sourceColumn;
-    metadata["vectorColumn"] = conf.vectorColumn ?? "vector";
-    metadata["name"] = name ?? conf.function.constructor.name;
-    metadata["model"] = conf.function.toJSON();
-    return metadata;
-  }
-
-  getTableMetadata(functions: EmbeddingFunctionConfig[]): Map<string, string> {
-    const metadata = new Map<string, string>();
-    const jsonData = functions.map((conf) => this.functionToMetadata(conf));
-    metadata.set("embedding_functions", JSON.stringify(jsonData));
-
-    return metadata;
-  }
-}
-
-const _REGISTRY = new EmbeddingFunctionRegistry();
-
-export function register(name?: string) {
-  return _REGISTRY.register(name);
-}
-
-/**
- * Utility function to get the global instance of the registry
- * @returns `EmbeddingFunctionRegistry` The global instance of the registry
- * @example
- * ```ts
- * const registry = getRegistry();
- * const openai = registry.get("openai").create();
- */
-export function getRegistry(): EmbeddingFunctionRegistry {
-  return _REGISTRY;
-}
-
-export interface EmbeddingFunctionConfig {
-  sourceColumn: string;
-  vectorColumn?: string;
-  function: EmbeddingFunction;
-}

package/lancedb/index.ts

@@ -1,142 +0,0 @@
-// Copyright 2024 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import {
-  Connection,
-  LocalConnection,
-  cleanseStorageOptions,
-} from "./connection";
-
-import {
-  ConnectionOptions,
-  Connection as LanceDbConnection,
-} from "./native.js";
-
-import { RemoteConnection, RemoteConnectionOptions } from "./remote";
-
-export {
-  WriteOptions,
-  WriteMode,
-  AddColumnsSql,
-  ColumnAlteration,
-  ConnectionOptions,
-  IndexStatistics,
-  IndexMetadata,
-  IndexConfig,
-} from "./native.js";
-
-export {
-  makeArrowTable,
-  MakeArrowTableOptions,
-  Data,
-  VectorColumnOptions,
-} from "./arrow";
-
-export {
-  Connection,
-  CreateTableOptions,
-  TableNamesOptions,
-} from "./connection";
-
-export {
-  ExecutableQuery,
-  Query,
-  QueryBase,
-  VectorQuery,
-  RecordBatchIterator,
-} from "./query";
-
-export { Index, IndexOptions, IvfPqOptions } from "./indices";
-
-export { Table, AddDataOptions, UpdateOptions } from "./table";
-
-export * as embedding from "./embedding";
-
-/**
- * Connect to a LanceDB instance at the given URI.
- *
- * Accepted formats:
- *
- * - `/path/to/database` - local database
- * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
- * - `db://host:port` - remote database (LanceDB cloud)
- * @param {string} uri - The uri of the database. If the database uri starts
- * with `db://` then it connects to a remote database.
- * @see {@link ConnectionOptions} for more details on the URI format.
- * @example
- * ```ts
- * const conn = await connect("/path/to/database");
- * ```
- * @example
- * ```ts
- * const conn = await connect(
- *   "s3://bucket/path/to/database",
- *   {storageOptions: {timeout: "60s"}
- * });
- * ```
- */
-export async function connect(
-  uri: string,
-  opts?: Partial<ConnectionOptions | RemoteConnectionOptions>,
-): Promise<Connection>;
-/**
- * Connect to a LanceDB instance at the given URI.
- *
- * Accepted formats:
- *
- * - `/path/to/database` - local database
- * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
- * - `db://host:port` - remote database (LanceDB cloud)
- * @param  options - The options to use when connecting to the database
- * @see {@link ConnectionOptions} for more details on the URI format.
- * @example
- * ```ts
- * const conn = await connect({
- *   uri: "/path/to/database",
- *   storageOptions: {timeout: "60s"}
- * });
- * ```
- */
-export async function connect(
-  opts: Partial<RemoteConnectionOptions | ConnectionOptions> & { uri: string },
-): Promise<Connection>;
-export async function connect(
-  uriOrOptions:
-    | string
-    | (Partial<RemoteConnectionOptions | ConnectionOptions> & { uri: string }),
-  opts: Partial<ConnectionOptions | RemoteConnectionOptions> = {},
-): Promise<Connection> {
-  let uri: string | undefined;
-  if (typeof uriOrOptions !== "string") {
-    const { uri: uri_, ...options } = uriOrOptions;
-    uri = uri_;
-    opts = options;
-  } else {
-    uri = uriOrOptions;
-  }
-
-  if (!uri) {
-    throw new Error("uri is required");
-  }
-
-  if (uri?.startsWith("db://")) {
-    return new RemoteConnection(uri, opts as RemoteConnectionOptions);
-  }
-  opts = (opts as ConnectionOptions) ?? {};
-  (<ConnectionOptions>opts).storageOptions = cleanseStorageOptions(
-    (<ConnectionOptions>opts).storageOptions,
-  );
-  const nativeConn = await LanceDbConnection.new(uri, opts);
-  return new LocalConnection(nativeConn);
-}

package/lancedb/indices.ts

@@ -1,203 +0,0 @@
-// Copyright 2024 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { Index as LanceDbIndex } from "./native";
-
-/**
- * Options to create an `IVF_PQ` index
- */
-export interface IvfPqOptions {
-  /**
-   * The number of IVF partitions to create.
-   *
-   * This value should generally scale with the number of rows in the dataset.
-   * By default the number of partitions is the square root of the number of
-   * rows.
-   *
-   * If this value is too large then the first part of the search (picking the
-   * right partition) will be slow.  If this value is too small then the second
-   * part of the search (searching within a partition) will be slow.
-   */
-  numPartitions?: number;
-
-  /**
-   * Number of sub-vectors of PQ.
-   *
-   * This value controls how much the vector is compressed during the quantization step.
-   * The more sub vectors there are the less the vector is compressed.  The default is
-   * the dimension of the vector divided by 16.  If the dimension is not evenly divisible
-   * by 16 we use the dimension divded by 8.
-   *
-   * The above two cases are highly preferred.  Having 8 or 16 values per subvector allows
-   * us to use efficient SIMD instructions.
-   *
-   * If the dimension is not visible by 8 then we use 1 subvector.  This is not ideal and
-   * will likely result in poor performance.
-   */
-  numSubVectors?: number;
-
-  /**
-   * Distance type to use to build the index.
-   *
-   * Default value is "l2".
-   *
-   * This is used when training the index to calculate the IVF partitions
-   * (vectors are grouped in partitions with similar vectors according to this
-   * distance type) and to calculate a subvector's code during quantization.
-   *
-   * The distance type used to train an index MUST match the distance type used
-   * to search the index.  Failure to do so will yield inaccurate results.
-   *
-   * The following distance types are available:
-   *
-   * "l2" - Euclidean distance. This is a very common distance metric that
-   * accounts for both magnitude and direction when determining the distance
-   * between vectors. L2 distance has a range of [0, ∞).
-   *
-   * "cosine" - Cosine distance.  Cosine distance is a distance metric
-   * calculated from the cosine similarity between two vectors. Cosine
-   * similarity is a measure of similarity between two non-zero vectors of an
-   * inner product space. It is defined to equal the cosine of the angle
-   * between them.  Unlike L2, the cosine distance is not affected by the
-   * magnitude of the vectors.  Cosine distance has a range of [0, 2].
-   *
-   * Note: the cosine distance is undefined when one (or both) of the vectors
-   * are all zeros (there is no direction).  These vectors are invalid and may
-   * never be returned from a vector search.
-   *
-   * "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
-   * distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
-   * L2 norm is 1), then dot distance is equivalent to the cosine distance.
-   */
-  distanceType?: "l2" | "cosine" | "dot";
-
-  /**
-   * Max iteration to train IVF kmeans.
-   *
-   * When training an IVF PQ index we use kmeans to calculate the partitions.  This parameter
-   * controls how many iterations of kmeans to run.
-   *
-   * Increasing this might improve the quality of the index but in most cases these extra
-   * iterations have diminishing returns.
-   *
-   * The default value is 50.
-   */
-  maxIterations?: number;
-
-  /**
-   * The number of vectors, per partition, to sample when training IVF kmeans.
-   *
-   * When an IVF PQ index is trained, we need to calculate partitions.  These are groups
-   * of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
-   *
-   * Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
-   * random sample of the data.  This parameter controls the size of the sample.  The total
-   * number of vectors used to train the index is `sample_rate * num_partitions`.
-   *
-   * Increasing this value might improve the quality of the index but in most cases the
-   * default should be sufficient.
-   *
-   * The default value is 256.
-   */
-  sampleRate?: number;
-}
-
-export class Index {
-  private readonly inner: LanceDbIndex;
-  private constructor(inner: LanceDbIndex) {
-    this.inner = inner;
-  }
-
-  /**
-   * Create an IvfPq index
-   *
-   * This index stores a compressed (quantized) copy of every vector.  These vectors
-   * are grouped into partitions of similar vectors.  Each partition keeps track of
-   * a centroid which is the average value of all vectors in the group.
-   *
-   * During a query the centroids are compared with the query vector to find the closest
-   * partitions.  The compressed vectors in these partitions are then searched to find
-   * the closest vectors.
-   *
-   * The compression scheme is called product quantization.  Each vector is divided into
-   * subvectors and then each subvector is quantized into a small number of bits.  the
-   * parameters `num_bits` and `num_subvectors` control this process, providing a tradeoff
-   * between index size (and thus search speed) and index accuracy.
-   *
-   * The partitioning process is called IVF and the `num_partitions` parameter controls how
-   * many groups to create.
-   *
-   * Note that training an IVF PQ index on a large dataset is a slow operation and
-   * currently is also a memory intensive operation.
-   */
-  static ivfPq(options?: Partial<IvfPqOptions>) {
-    return new Index(
-      LanceDbIndex.ivfPq(
-        options?.distanceType,
-        options?.numPartitions,
-        options?.numSubVectors,
-        options?.maxIterations,
-        options?.sampleRate,
-      ),
-    );
-  }
-
-  /**
-   * Create a btree index
-   *
-   * A btree index is an index on a scalar columns.  The index stores a copy of the column
-   * in sorted order.  A header entry is created for each block of rows (currently the
-   * block size is fixed at 4096).  These header entries are stored in a separate
-   * cacheable structure (a btree).  To search for data the header is used to determine
-   * which blocks need to be read from disk.
-   *
-   * For example, a btree index in a table with 1Bi rows requires sizeof(Scalar) * 256Ki
-   * bytes of memory and will generally need to read sizeof(Scalar) * 4096 bytes to find
-   * the correct row ids.
-   *
-   * This index is good for scalar columns with mostly distinct values and does best when
-   * the query is highly selective.
-   *
-   * The btree index does not currently have any parameters though parameters such as the
-   * block size may be added in the future.
-   */
-  static btree() {
-    return new Index(LanceDbIndex.btree());
-  }
-}
-
-export interface IndexOptions {
-  /**
-   * Advanced index configuration
-   *
-   * This option allows you to specify a specfic index to create and also
-   * allows you to pass in configuration for training the index.
-   *
-   * See the static methods on Index for details on the various index types.
-   *
-   * If this is not supplied then column data type(s) and column statistics
-   * will be used to determine the most useful kind of index to create.
-   */
-  config?: Index;
-  /**
-   * Whether to replace the existing index
-   *
-   * If this is false, and another index already exists on the same columns
-   * and the same name, then an error will be returned.  This is true even if
-   * that index is out of date.
-   *
-   * The default is true
-   */
-  replace?: boolean;
-}