@lancedb/lancedb 0.7.1 → 0.8.0

package/lancedb/connection.ts

@@ -1,333 +0,0 @@
-// Copyright 2024 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { Data, Schema, SchemaLike, TableLike } from "./arrow";
-import { fromTableToBuffer, makeEmptyTable } from "./arrow";
-import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
-import { Connection as LanceDbConnection } from "./native";
-import { LocalTable, Table } from "./table";
-
-export interface CreateTableOptions {
-  /**
-   * The mode to use when creating the table.
-   *
-   * If this is set to "create" and the table already exists then either
-   * an error will be thrown or, if existOk is true, then nothing will
-   * happen.  Any provided data will be ignored.
-   *
-   * If this is set to "overwrite" then any existing table will be replaced.
-   */
-  mode: "create" | "overwrite";
-  /**
-   * If this is true and the table already exists and the mode is "create"
-   * then no error will be raised.
-   */
-  existOk: boolean;
-
-  /**
-   * Configuration for object storage.
-   *
-   * Options already set on the connection will be inherited by the table,
-   * but can be overridden here.
-   *
-   * The available options are described at https://lancedb.github.io/lancedb/guides/storage/
-   */
-  storageOptions?: Record<string, string>;
-  /**
-   * If true then data files will be written with the legacy format
-   *
-   * The default is true while the new format is in beta
-   */
-  useLegacyFormat?: boolean;
-  schema?: SchemaLike;
-  embeddingFunction?: EmbeddingFunctionConfig;
-}
-
-export interface OpenTableOptions {
-  /**
-   * Configuration for object storage.
-   *
-   * Options already set on the connection will be inherited by the table,
-   * but can be overridden here.
-   *
-   * The available options are described at https://lancedb.github.io/lancedb/guides/storage/
-   */
-  storageOptions?: Record<string, string>;
-  /**
-   * Set the size of the index cache, specified as a number of entries
-   *
-   * The exact meaning of an "entry" will depend on the type of index:
-   * - IVF: there is one entry for each IVF partition
-   * - BTREE: there is one entry for the entire index
-   *
-   * This cache applies to the entire opened table, across all indices.
-   * Setting this value higher will increase performance on larger datasets
-   * at the expense of more RAM
-   */
-  indexCacheSize?: number;
-}
-
-export interface TableNamesOptions {
-  /**
-   * If present, only return names that come lexicographically after the
-   * supplied value.
-   *
-   * This can be combined with limit to implement pagination by setting this to
-   * the last table name from the previous page.
-   */
-  startAfter?: string;
-  /** An optional limit to the number of results to return. */
-  limit?: number;
-}
-/**
- * A LanceDB Connection that allows you to open tables and create new ones.
- *
- * Connection could be local against filesystem or remote against a server.
- *
- * A Connection is intended to be a long lived object and may hold open
- * resources such as HTTP connection pools.  This is generally fine and
- * a single connection should be shared if it is going to be used many
- * times. However, if you are finished with a connection, you may call
- * close to eagerly free these resources.  Any call to a Connection
- * method after it has been closed will result in an error.
- *
- * Closing a connection is optional.  Connections will automatically
- * be closed when they are garbage collected.
- *
- * Any created tables are independent and will continue to work even if
- * the underlying connection has been closed.
- */
-export abstract class Connection {
-  [Symbol.for("nodejs.util.inspect.custom")](): string {
-    return this.display();
-  }
-
-  /**
-   * Return true if the connection has not been closed
-   */
-  abstract isOpen(): boolean;
-
-  /**
-   * Close the connection, releasing any underlying resources.
-   *
-   * It is safe to call this method multiple times.
-   *
-   * Any attempt to use the connection after it is closed will result in an error.
-   */
-  abstract close(): void;
-
-  /**
-   * Return a brief description of the connection
-   */
-  abstract display(): string;
-
-  /**
-   * List all the table names in this database.
-   *
-   * Tables will be returned in lexicographical order.
-   * @param {Partial<TableNamesOptions>} options - options to control the
-   * paging / start point
-   *
-   */
-  abstract tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
-
-  /**
-   * Open a table in the database.
-   * @param {string} name - The name of the table
-   */
-  abstract openTable(
-    name: string,
-    options?: Partial<OpenTableOptions>,
-  ): Promise<Table>;
-
-  /**
-   * Creates a new Table and initialize it with new data.
-   * @param {object} options - The options object.
-   * @param {string} options.name - The name of the table.
-   * @param {Data} options.data - Non-empty Array of Records to be inserted into the table
-   *
-   */
-  abstract createTable(
-    options: {
-      name: string;
-      data: Data;
-    } & Partial<CreateTableOptions>,
-  ): Promise<Table>;
-  /**
-   * Creates a new Table and initialize it with new data.
-   * @param {string} name - The name of the table.
-   * @param {Record<string, unknown>[] | TableLike} data - Non-empty Array of Records
-   * to be inserted into the table
-   */
-  abstract createTable(
-    name: string,
-    data: Record<string, unknown>[] | TableLike,
-    options?: Partial<CreateTableOptions>,
-  ): Promise<Table>;
-
-  /**
-   * Creates a new empty Table
-   * @param {string} name - The name of the table.
-   * @param {Schema} schema - The schema of the table
-   */
-  abstract createEmptyTable(
-    name: string,
-    schema: import("./arrow").SchemaLike,
-    options?: Partial<CreateTableOptions>,
-  ): Promise<Table>;
-
-  /**
-   * Drop an existing table.
-   * @param {string} name The name of the table to drop.
-   */
-  abstract dropTable(name: string): Promise<void>;
-}
-
-export class LocalConnection extends Connection {
-  readonly inner: LanceDbConnection;
-
-  constructor(inner: LanceDbConnection) {
-    super();
-    this.inner = inner;
-  }
-
-  isOpen(): boolean {
-    return this.inner.isOpen();
-  }
-
-  close(): void {
-    this.inner.close();
-  }
-
-  display(): string {
-    return this.inner.display();
-  }
-
-  async tableNames(options?: Partial<TableNamesOptions>): Promise<string[]> {
-    return this.inner.tableNames(options?.startAfter, options?.limit);
-  }
-
-  async openTable(
-    name: string,
-    options?: Partial<OpenTableOptions>,
-  ): Promise<Table> {
-    const innerTable = await this.inner.openTable(
-      name,
-      cleanseStorageOptions(options?.storageOptions),
-      options?.indexCacheSize,
-    );
-
-    return new LocalTable(innerTable);
-  }
-
-  async createTable(
-    nameOrOptions:
-      | string
-      | ({ name: string; data: Data } & Partial<CreateTableOptions>),
-    data?: Record<string, unknown>[] | TableLike,
-    options?: Partial<CreateTableOptions>,
-  ): Promise<Table> {
-    if (typeof nameOrOptions !== "string" && "name" in nameOrOptions) {
-      const { name, data, ...options } = nameOrOptions;
-      return this.createTable(name, data, options);
-    }
-    if (data === undefined) {
-      throw new Error("data is required");
-    }
-    const { buf, mode } = await Table.parseTableData(data, options);
-    const innerTable = await this.inner.createTable(
-      nameOrOptions,
-      buf,
-      mode,
-      cleanseStorageOptions(options?.storageOptions),
-      options?.useLegacyFormat,
-    );
-
-    return new LocalTable(innerTable);
-  }
-
-  async createEmptyTable(
-    name: string,
-    schema: import("./arrow").SchemaLike,
-    options?: Partial<CreateTableOptions>,
-  ): Promise<Table> {
-    let mode: string = options?.mode ?? "create";
-    const existOk = options?.existOk ?? false;
-
-    if (mode === "create" && existOk) {
-      mode = "exist_ok";
-    }
-    let metadata: Map<string, string> | undefined = undefined;
-    if (options?.embeddingFunction !== undefined) {
-      const embeddingFunction = options.embeddingFunction;
-      const registry = getRegistry();
-      metadata = registry.getTableMetadata([embeddingFunction]);
-    }
-
-    const table = makeEmptyTable(schema, metadata);
-    const buf = await fromTableToBuffer(table);
-    const innerTable = await this.inner.createEmptyTable(
-      name,
-      buf,
-      mode,
-      cleanseStorageOptions(options?.storageOptions),
-      options?.useLegacyFormat,
-    );
-    return new LocalTable(innerTable);
-  }
-
-  async dropTable(name: string): Promise<void> {
-    return this.inner.dropTable(name);
-  }
-}
-
-/**
- * Takes storage options and makes all the keys snake case.
- */
-export function cleanseStorageOptions(
-  options?: Record<string, string>,
-): Record<string, string> | undefined {
-  if (options === undefined) {
-    return undefined;
-  }
-  const result: Record<string, string> = {};
-  for (const [key, value] of Object.entries(options)) {
-    if (value !== undefined) {
-      const newKey = camelToSnakeCase(key);
-      result[newKey] = value;
-    }
-  }
-  return result;
-}
-
-/**
- * Convert a string to snake case. It might already be snake case, in which case it is
- * returned unchanged.
- */
-function camelToSnakeCase(camel: string): string {
-  if (camel.includes("_")) {
-    // Assume if there is at least one underscore, it is already snake case
-    return camel;
-  }
-  if (camel.toLocaleUpperCase() === camel) {
-    // Assume if the string is all uppercase, it is already snake case
-    return camel;
-  }
-
-  let result = camel.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
-  if (result.startsWith("_")) {
-    result = result.slice(1);
-  }
-  return result;
-}

package/lancedb/embedding/embedding_function.ts

@@ -1,194 +0,0 @@
-// Copyright 2024 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import "reflect-metadata";
-import {
-  DataType,
-  Field,
-  FixedSizeList,
-  Float,
-  Float32,
-  type IntoVector,
-  isDataType,
-  isFixedSizeList,
-  isFloat,
-  newVectorType,
-} from "../arrow";
-import { sanitizeType } from "../sanitize";
-
-/**
- * Options for a given embedding function
- */
-export interface FunctionOptions {
-  // biome-ignore lint/suspicious/noExplicitAny: options can be anything
-  [key: string]: any;
-}
-
-export interface EmbeddingFunctionConstructor<
-  T extends EmbeddingFunction = EmbeddingFunction,
-> {
-  new (modelOptions?: T["TOptions"]): T;
-}
-/**
- * An embedding function that automatically creates vector representation for a given column.
- */
-export abstract class EmbeddingFunction<
-  // biome-ignore lint/suspicious/noExplicitAny: we don't know what the implementor will do
-  T = any,
-  M extends FunctionOptions = FunctionOptions,
-> {
-  /**
-   * @ignore
-   *  This is only used for associating the options type with the class for type checking
-   */
-  // biome-ignore lint/style/useNamingConvention: we want to keep the name as it is
-  readonly TOptions!: M;
-  /**
-   * Convert the embedding function to a JSON object
-   * It is used to serialize the embedding function to the schema
-   * It's important that any object returned by this method contains all the necessary
-   * information to recreate the embedding function
-   *
-   * It should return the same object that was passed to the constructor
-   * If it does not, the embedding function will not be able to be recreated, or could be recreated incorrectly
-   *
-   * @example
-   * ```ts
-   * class MyEmbeddingFunction extends EmbeddingFunction {
-   *   constructor(options: {model: string, timeout: number}) {
-   *     super();
-   *     this.model = options.model;
-   *     this.timeout = options.timeout;
-   *   }
-   *   toJSON() {
-   *     return {
-   *       model: this.model,
-   *       timeout: this.timeout,
-   *     };
-   * }
-   * ```
-   */
-  abstract toJSON(): Partial<M>;
-
-  /**
-   * sourceField is used in combination with `LanceSchema` to provide a declarative data model
-   *
-   * @param optionsOrDatatype - The options for the field or the datatype
-   *
-   * @see {@link lancedb.LanceSchema}
-   */
-  sourceField(
-    optionsOrDatatype: Partial<FieldOptions> | DataType,
-  ): [DataType, Map<string, EmbeddingFunction>] {
-    let datatype = isDataType(optionsOrDatatype)
-      ? optionsOrDatatype
-      : optionsOrDatatype?.datatype;
-    if (!datatype) {
-      throw new Error("Datatype is required");
-    }
-    datatype = sanitizeType(datatype);
-    const metadata = new Map<string, EmbeddingFunction>();
-    metadata.set("source_column_for", this);
-
-    return [datatype, metadata];
-  }
-
-  /**
-   * vectorField is used in combination with `LanceSchema` to provide a declarative data model
-   *
-   * @param options - The options for the field
-   *
-   * @see {@link lancedb.LanceSchema}
-   */
-  vectorField(
-    optionsOrDatatype?: Partial<FieldOptions> | DataType,
-  ): [DataType, Map<string, EmbeddingFunction>] {
-    let dtype: DataType | undefined;
-    let vectorType: DataType;
-    let dims: number | undefined = this.ndims();
-
-    // `func.vectorField(new Float32())`
-    if (isDataType(optionsOrDatatype)) {
-      dtype = optionsOrDatatype;
-    } else {
-      // `func.vectorField({
-      //  datatype: new Float32(),
-      //  dims: 10
-      // })`
-      dims = dims ?? optionsOrDatatype?.dims;
-      dtype = optionsOrDatatype?.datatype;
-    }
-
-    if (dtype !== undefined) {
-      // `func.vectorField(new FixedSizeList(dims, new Field("item", new Float32(), true)))`
-      // or `func.vectorField({datatype: new FixedSizeList(dims, new Field("item", new Float32(), true))})`
-      if (isFixedSizeList(dtype)) {
-        vectorType = dtype;
-        // `func.vectorField(new Float32())`
-        // or `func.vectorField({datatype: new Float32()})`
-      } else if (isFloat(dtype)) {
-        // No `ndims` impl and no `{dims: n}` provided;
-        if (dims === undefined) {
-          throw new Error("ndims is required for vector field");
-        }
-        vectorType = newVectorType(dims, dtype);
-      } else {
-        throw new Error(
-          "Expected FixedSizeList or Float as datatype for vector field",
-        );
-      }
-    } else {
-      if (dims === undefined) {
-        throw new Error("ndims is required for vector field");
-      }
-      vectorType = new FixedSizeList(
-        dims,
-        new Field("item", new Float32(), true),
-      );
-    }
-    const metadata = new Map<string, EmbeddingFunction>();
-    metadata.set("vector_column_for", this);
-
-    return [vectorType, metadata];
-  }
-
-  /** The number of dimensions of the embeddings */
-  ndims(): number | undefined {
-    return undefined;
-  }
-
-  /** The datatype of the embeddings */
-  abstract embeddingDataType(): Float;
-
-  /**
-   * Creates a vector representation for the given values.
-   */
-  abstract computeSourceEmbeddings(
-    data: T[],
-  ): Promise<number[][] | Float32Array[] | Float64Array[]>;
-
-  /**
-  Compute the embeddings for a single query
- */
-  async computeQueryEmbeddings(data: T): Promise<Awaited<IntoVector>> {
-    return this.computeSourceEmbeddings([data]).then(
-      (embeddings) => embeddings[0],
-    );
-  }
-}
-
-export interface FieldOptions<T extends DataType = DataType> {
-  datatype: T;
-  dims?: number;
-}

package/lancedb/embedding/index.ts

@@ -1,113 +0,0 @@
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-import { DataType, Field, Schema } from "../arrow";
-import { isDataType } from "../arrow";
-import { sanitizeType } from "../sanitize";
-import { EmbeddingFunction } from "./embedding_function";
-import { EmbeddingFunctionConfig, getRegistry } from "./registry";
-
-export { EmbeddingFunction } from "./embedding_function";
-
-// We need to explicitly export '*' so that the `register` decorator actually registers the class.
-export * from "./openai";
-export * from "./registry";
-
-/**
- * Create a schema with embedding functions.
- *
- * @param fields
- * @returns Schema
- * @example
- * ```ts
- * class MyEmbeddingFunction extends EmbeddingFunction {
- * // ...
- * }
- * const func = new MyEmbeddingFunction();
- * const schema = LanceSchema({
- *   id: new Int32(),
- *   text: func.sourceField(new Utf8()),
- *   vector: func.vectorField(),
- *   // optional: specify the datatype and/or dimensions
- *   vector2: func.vectorField({ datatype: new Float32(), dims: 3}),
- * });
- *
- * const table = await db.createTable("my_table", data, { schema });
- * ```
- */
-export function LanceSchema(
-  fields: Record<string, [object, Map<string, EmbeddingFunction>] | object>,
-): Schema {
-  const arrowFields: Field[] = [];
-
-  const embeddingFunctions = new Map<
-    EmbeddingFunction,
-    Partial<EmbeddingFunctionConfig>
-  >();
-  Object.entries(fields).forEach(([key, value]) => {
-    if (isDataType(value)) {
-      arrowFields.push(new Field(key, sanitizeType(value), true));
-    } else {
-      const [dtype, metadata] = value as [
-        object,
-        Map<string, EmbeddingFunction>,
-      ];
-      arrowFields.push(new Field(key, sanitizeType(dtype), true));
-      parseEmbeddingFunctions(embeddingFunctions, key, metadata);
-    }
-  });
-  const registry = getRegistry();
-  const metadata = registry.getTableMetadata(
-    Array.from(embeddingFunctions.values()) as EmbeddingFunctionConfig[],
-  );
-  const schema = new Schema(arrowFields, metadata);
-  return schema;
-}
-
-function parseEmbeddingFunctions(
-  embeddingFunctions: Map<EmbeddingFunction, Partial<EmbeddingFunctionConfig>>,
-  key: string,
-  metadata: Map<string, EmbeddingFunction>,
-): void {
-  if (metadata.has("source_column_for")) {
-    const embedFunction = metadata.get("source_column_for")!;
-    const current = embeddingFunctions.get(embedFunction);
-    if (current !== undefined) {
-      embeddingFunctions.set(embedFunction, {
-        ...current,
-        sourceColumn: key,
-      });
-    } else {
-      embeddingFunctions.set(embedFunction, {
-        sourceColumn: key,
-        function: embedFunction,
-      });
-    }
-  } else if (metadata.has("vector_column_for")) {
-    const embedFunction = metadata.get("vector_column_for")!;
-
-    const current = embeddingFunctions.get(embedFunction);
-    if (current !== undefined) {
-      embeddingFunctions.set(embedFunction, {
-        ...current,
-        vectorColumn: key,
-      });
-    } else {
-      embeddingFunctions.set(embedFunction, {
-        vectorColumn: key,
-        function: embedFunction,
-      });
-    }
-  }
-}