@lancedb/lancedb 0.7.1 → 0.8.0

package/lancedb/util.ts

@@ -1,69 +0,0 @@
-export type IntoSql =
-  | string
-  | number
-  | boolean
-  | null
-  | Date
-  | ArrayBufferLike
-  | Buffer
-  | IntoSql[];
-
-export function toSQL(value: IntoSql): string {
-  if (typeof value === "string") {
-    return `'${value.replace(/'/g, "''")}'`;
-  } else if (typeof value === "number") {
-    return value.toString();
-  } else if (typeof value === "boolean") {
-    return value ? "TRUE" : "FALSE";
-  } else if (value === null) {
-    return "NULL";
-  } else if (value instanceof Date) {
-    return `'${value.toISOString()}'`;
-  } else if (Array.isArray(value)) {
-    return `[${value.map(toSQL).join(", ")}]`;
-  } else if (Buffer.isBuffer(value)) {
-    return `X'${value.toString("hex")}'`;
-  } else if (value instanceof ArrayBuffer) {
-    return `X'${Buffer.from(value).toString("hex")}'`;
-  } else {
-    throw new Error(
-      `Unsupported value type: ${typeof value} value: (${value})`,
-    );
-  }
-}
-
-export class TTLCache {
-  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-  private readonly cache: Map<string, { value: any; expires: number }>;
-
-  /**
-   * @param ttl Time to live in milliseconds
-   */
-  constructor(private readonly ttl: number) {
-    this.cache = new Map();
-  }
-
-  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-  get(key: string): any | undefined {
-    const entry = this.cache.get(key);
-    if (entry === undefined) {
-      return undefined;
-    }
-
-    if (entry.expires < Date.now()) {
-      this.cache.delete(key);
-      return undefined;
-    }
-
-    return entry.value;
-  }
-
-  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-  set(key: string, value: any): void {
-    this.cache.set(key, { value, expires: Date.now() + this.ttl });
-  }
-
-  delete(key: string): void {
-    this.cache.delete(key);
-  }
-}

package/native.d.ts

@@ -1,208 +0,0 @@
-/* tslint:disable */
-/* eslint-disable */
-
-/* auto-generated by NAPI-RS */
-
-/** A description of an index currently configured on a column */
-export interface IndexConfig {
-  /** The name of the index */
-  name: string
-  /** The type of the index */
-  indexType: string
-  /**
-   * The columns in the index
-   *
-   * Currently this is always an array of size 1. In the future there may
-   * be more columns to represent composite indices.
-   */
-  columns: Array<string>
-}
-/** Statistics about a compaction operation. */
-export interface CompactionStats {
-  /** The number of fragments removed */
-  fragmentsRemoved: number
-  /** The number of new, compacted fragments added */
-  fragmentsAdded: number
-  /** The number of data files removed */
-  filesRemoved: number
-  /** The number of new, compacted data files added */
-  filesAdded: number
-}
-/** Statistics about a cleanup operation */
-export interface RemovalStats {
-  /** The number of bytes removed */
-  bytesRemoved: number
-  /** The number of old versions removed */
-  oldVersionsRemoved: number
-}
-/** Statistics about an optimize operation */
-export interface OptimizeStats {
-  /** Statistics about the compaction operation */
-  compaction: CompactionStats
-  /** Statistics about the removal operation */
-  prune: RemovalStats
-}
-/**
- *  A definition of a column alteration. The alteration changes the column at
- * `path` to have the new name `name`, to be nullable if `nullable` is true,
- * and to have the data type `data_type`. At least one of `rename` or `nullable`
- * must be provided.
- */
-export interface ColumnAlteration {
-  /**
-   * The path to the column to alter. This is a dot-separated path to the column.
-   * If it is a top-level column then it is just the name of the column. If it is
-   * a nested column then it is the path to the column, e.g. "a.b.c" for a column
-   * `c` nested inside a column `b` nested inside a column `a`.
-   */
-  path: string
-  /**
-   * The new name of the column. If not provided then the name will not be changed.
-   * This must be distinct from the names of all other columns in the table.
-   */
-  rename?: string
-  /** Set the new nullability. Note that a nullable column cannot be made non-nullable. */
-  nullable?: boolean
-}
-/** A definition of a new column to add to a table. */
-export interface AddColumnsSql {
-  /** The name of the new column. */
-  name: string
-  /**
-   * The values to populate the new column with, as a SQL expression.
-   * The expression can reference other columns in the table.
-   */
-  valueSql: string
-}
-export interface IndexStatistics {
-  /** The number of rows indexed by the index */
-  numIndexedRows: number
-  /** The number of rows not indexed */
-  numUnindexedRows: number
-  /** The type of the index */
-  indexType?: string
-  /** The metadata for each index */
-  indices: Array<IndexMetadata>
-}
-export interface IndexMetadata {
-  metricType?: string
-  indexType?: string
-}
-export interface ConnectionOptions {
-  /**
-   * (For LanceDB OSS only): The interval, in seconds, at which to check for
-   * updates to the table from other processes. If None, then consistency is not
-   * checked. For performance reasons, this is the default. For strong
-   * consistency, set this to zero seconds. Then every read will check for
-   * updates from other processes. As a compromise, you can set this to a
-   * non-zero value for eventual consistency. If more than that interval
-   * has passed since the last check, then the table will be checked for updates.
-   * Note: this consistency only applies to read operations. Write operations are
-   * always consistent.
-   */
-  readConsistencyInterval?: number
-  /**
-   * (For LanceDB OSS only): configuration for object storage.
-   *
-   * The available options are described at https://lancedb.github.io/lancedb/guides/storage/
-   */
-  storageOptions?: Record<string, string>
-}
-/** Write mode for writing a table. */
-export const enum WriteMode {
-  Create = 'Create',
-  Append = 'Append',
-  Overwrite = 'Overwrite'
-}
-/** Write options when creating a Table. */
-export interface WriteOptions {
-  /** Write mode for writing to a table. */
-  mode?: WriteMode
-}
-export interface OpenTableOptions {
-  storageOptions?: Record<string, string>
-}
-export class Connection {
-  /** Create a new Connection instance from the given URI. */
-  static new(uri: string, options: ConnectionOptions): Promise<Connection>
-  display(): string
-  isOpen(): boolean
-  close(): void
-  /** List all tables in the dataset. */
-  tableNames(startAfter?: string | undefined | null, limit?: number | undefined | null): Promise<Array<string>>
-  /**
-   * Create table from a Apache Arrow IPC (file) buffer.
-   *
-   * Parameters:
-   * - name: The name of the table.
-   * - buf: The buffer containing the IPC file.
-   *
-   */
-  createTable(name: string, buf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, useLegacyFormat?: boolean | undefined | null): Promise<Table>
-  createEmptyTable(name: string, schemaBuf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, useLegacyFormat?: boolean | undefined | null): Promise<Table>
-  openTable(name: string, storageOptions?: Record<string, string> | undefined | null, indexCacheSize?: number | undefined | null): Promise<Table>
-  /** Drop table with the name. Or raise an error if the table does not exist. */
-  dropTable(name: string): Promise<void>
-}
-export class Index {
-  static ivfPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
-  static btree(): Index
-}
-/** Typescript-style Async Iterator over RecordBatches  */
-export class RecordBatchIterator {
-  next(): Promise<Buffer | null>
-}
-/** A builder used to create and run a merge insert operation */
-export class NativeMergeInsertBuilder {
-  whenMatchedUpdateAll(condition?: string | undefined | null): NativeMergeInsertBuilder
-  whenNotMatchedInsertAll(): NativeMergeInsertBuilder
-  whenNotMatchedBySourceDelete(filter?: string | undefined | null): NativeMergeInsertBuilder
-  execute(buf: Buffer): Promise<void>
-}
-export class Query {
-  onlyIf(predicate: string): void
-  select(columns: Array<[string, string]>): void
-  limit(limit: number): void
-  nearestTo(vector: Float32Array): VectorQuery
-  execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
-  explainPlan(verbose: boolean): Promise<string>
-}
-export class VectorQuery {
-  column(column: string): void
-  distanceType(distanceType: string): void
-  postfilter(): void
-  refineFactor(refineFactor: number): void
-  nprobes(nprobe: number): void
-  bypassVectorIndex(): void
-  onlyIf(predicate: string): void
-  select(columns: Array<[string, string]>): void
-  limit(limit: number): void
-  execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
-  explainPlan(verbose: boolean): Promise<string>
-}
-export class Table {
-  name: string
-  display(): string
-  isOpen(): boolean
-  close(): void
-  /** Return Schema as empty Arrow IPC file. */
-  schema(): Promise<Buffer>
-  add(buf: Buffer, mode: string): Promise<void>
-  countRows(filter?: string | undefined | null): Promise<number>
-  delete(predicate: string): Promise<void>
-  createIndex(index: Index | undefined | null, column: string, replace?: boolean | undefined | null): Promise<void>
-  update(onlyIf: string | undefined | null, columns: Array<[string, string]>): Promise<void>
-  query(): Query
-  vectorSearch(vector: Float32Array): VectorQuery
-  addColumns(transforms: Array<AddColumnsSql>): Promise<void>
-  alterColumns(alterations: Array<ColumnAlteration>): Promise<void>
-  dropColumns(columns: Array<string>): Promise<void>
-  version(): Promise<number>
-  checkout(version: number): Promise<void>
-  checkoutLatest(): Promise<void>
-  restore(): Promise<void>
-  optimize(olderThanMs?: number | undefined | null): Promise<OptimizeStats>
-  listIndices(): Promise<Array<IndexConfig>>
-  indexStats(indexName: string): Promise<IndexStatistics | null>
-  mergeInsert(on: Array<string>): NativeMergeInsertBuilder
-}

package/nodejs-artifacts/arrow.d.ts

@@ -1,250 +0,0 @@
-/// <reference types="node" />
-import { Table as ArrowTable, Binary, BufferType, DataType, Field, FixedSizeBinary, FixedSizeList, Float, Int, LargeBinary, List, Null, RecordBatch, Schema, Struct, Utf8 } from "apache-arrow";
-import { Buffers } from "apache-arrow/data";
-import { type EmbeddingFunction } from "./embedding/embedding_function";
-import { EmbeddingFunctionConfig } from "./embedding/registry";
-export * from "apache-arrow";
-export type SchemaLike = Schema | {
-    fields: FieldLike[];
-    metadata: Map<string, string>;
-    get names(): unknown[];
-};
-export type FieldLike = Field | {
-    type: string;
-    name: string;
-    nullable?: boolean;
-    metadata?: Map<string, string>;
-};
-export type DataLike = import("apache-arrow").Data<Struct<any>> | {
-    type: any;
-    length: number;
-    offset: number;
-    stride: number;
-    nullable: boolean;
-    children: DataLike[];
-    get nullCount(): number;
-    values: Buffers<any>[BufferType.DATA];
-    typeIds: Buffers<any>[BufferType.TYPE];
-    nullBitmap: Buffers<any>[BufferType.VALIDITY];
-    valueOffsets: Buffers<any>[BufferType.OFFSET];
-};
-export type RecordBatchLike = RecordBatch | {
-    schema: SchemaLike;
-    data: DataLike;
-};
-export type TableLike = ArrowTable | {
-    schema: SchemaLike;
-    batches: RecordBatchLike[];
-};
-export type IntoVector = Float32Array | Float64Array | number[] | Promise<Float32Array | Float64Array | number[]>;
-export declare function isArrowTable(value: object): value is TableLike;
-export declare function isDataType(value: unknown): value is DataType;
-export declare function isNull(value: unknown): value is Null;
-export declare function isInt(value: unknown): value is Int;
-export declare function isFloat(value: unknown): value is Float;
-export declare function isBinary(value: unknown): value is Binary;
-export declare function isLargeBinary(value: unknown): value is LargeBinary;
-export declare function isUtf8(value: unknown): value is Utf8;
-export declare function isLargeUtf8(value: unknown): value is Utf8;
-export declare function isBool(value: unknown): value is Utf8;
-export declare function isDecimal(value: unknown): value is Utf8;
-export declare function isDate(value: unknown): value is Utf8;
-export declare function isTime(value: unknown): value is Utf8;
-export declare function isTimestamp(value: unknown): value is Utf8;
-export declare function isInterval(value: unknown): value is Utf8;
-export declare function isDuration(value: unknown): value is Utf8;
-export declare function isList(value: unknown): value is List;
-export declare function isStruct(value: unknown): value is Struct;
-export declare function isUnion(value: unknown): value is Struct;
-export declare function isFixedSizeBinary(value: unknown): value is FixedSizeBinary;
-export declare function isFixedSizeList(value: unknown): value is FixedSizeList;
-/** Data type accepted by NodeJS SDK */
-export type Data = Record<string, unknown>[] | TableLike;
-export declare class VectorColumnOptions {
-    /** Vector column type. */
-    type: Float;
-    constructor(values?: Partial<VectorColumnOptions>);
-}
-/** Options to control the makeArrowTable call. */
-export declare class MakeArrowTableOptions {
-    schema?: SchemaLike;
-    vectorColumns: Record<string, VectorColumnOptions>;
-    embeddings?: EmbeddingFunction<unknown>;
-    embeddingFunction?: EmbeddingFunctionConfig;
-    /**
-     * If true then string columns will be encoded with dictionary encoding
-     *
-     * Set this to true if your string columns tend to repeat the same values
-     * often.  For more precise control use the `schema` property to specify the
-     * data type for individual columns.
-     *
-     * If `schema` is provided then this property is ignored.
-     */
-    dictionaryEncodeStrings: boolean;
-    constructor(values?: Partial<MakeArrowTableOptions>);
-}
-/**
- * An enhanced version of the {@link makeTable} function from Apache Arrow
- * that supports nested fields and embeddings columns.
- *
- * (typically you do not need to call this function.  It will be called automatically
- * when creating a table or adding data to it)
- *
- * This function converts an array of Record<String, any> (row-major JS objects)
- * to an Arrow Table (a columnar structure)
- *
- * Note that it currently does not support nulls.
- *
- * If a schema is provided then it will be used to determine the resulting array
- * types.  Fields will also be reordered to fit the order defined by the schema.
- *
- * If a schema is not provided then the types will be inferred and the field order
- * will be controlled by the order of properties in the first record.  If a type
- * is inferred it will always be nullable.
- *
- * If the input is empty then a schema must be provided to create an empty table.
- *
- * When a schema is not specified then data types will be inferred.  The inference
- * rules are as follows:
- *
- *  - boolean => Bool
- *  - number => Float64
- *  - String => Utf8
- *  - Buffer => Binary
- *  - Record<String, any> => Struct
- *  - Array<any> => List
- * @example
- * import { fromTableToBuffer, makeArrowTable } from "../arrow";
- * import { Field, FixedSizeList, Float16, Float32, Int32, Schema } from "apache-arrow";
- *
- * const schema = new Schema([
- *   new Field("a", new Int32()),
- *   new Field("b", new Float32()),
- *   new Field("c", new FixedSizeList(3, new Field("item", new Float16()))),
- *  ]);
- *  const table = makeArrowTable([
- *    { a: 1, b: 2, c: [1, 2, 3] },
- *    { a: 4, b: 5, c: [4, 5, 6] },
- *    { a: 7, b: 8, c: [7, 8, 9] },
- *  ], { schema });
- * ```
- *
- * By default it assumes that the column named `vector` is a vector column
- * and it will be converted into a fixed size list array of type float32.
- * The `vectorColumns` option can be used to support other vector column
- * names and data types.
- *
- * ```ts
- *
- * const schema = new Schema([
-    new Field("a", new Float64()),
-    new Field("b", new Float64()),
-    new Field(
-      "vector",
-      new FixedSizeList(3, new Field("item", new Float32()))
-    ),
-  ]);
-  const table = makeArrowTable([
-    { a: 1, b: 2, vector: [1, 2, 3] },
-    { a: 4, b: 5, vector: [4, 5, 6] },
-    { a: 7, b: 8, vector: [7, 8, 9] },
-  ]);
-  assert.deepEqual(table.schema, schema);
- * ```
- *
- * You can specify the vector column types and names using the options as well
- *
- * ```typescript
- *
- * const schema = new Schema([
-    new Field('a', new Float64()),
-    new Field('b', new Float64()),
-    new Field('vec1', new FixedSizeList(3, new Field('item', new Float16()))),
-    new Field('vec2', new FixedSizeList(3, new Field('item', new Float16())))
-  ]);
- * const table = makeArrowTable([
-    { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
-    { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
-    { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] }
-  ], {
-    vectorColumns: {
-      vec1: { type: new Float16() },
-      vec2: { type: new Float16() }
-    }
-  }
- * assert.deepEqual(table.schema, schema)
- * ```
- */
-export declare function makeArrowTable(data: Array<Record<string, unknown>>, options?: Partial<MakeArrowTableOptions>, metadata?: Map<string, string>): ArrowTable;
-/**
- * Create an empty Arrow table with the provided schema
- */
-export declare function makeEmptyTable(schema: SchemaLike, metadata?: Map<string, string>): ArrowTable;
-/**
- * Convert an Array of records into an Arrow Table, optionally applying an
- * embeddings function to it.
- *
- * This function calls `makeArrowTable` first to create the Arrow Table.
- * Any provided `makeTableOptions` (e.g. a schema) will be passed on to
- * that call.
- *
- * The embedding function will be passed a column of values (based on the
- * `sourceColumn` of the embedding function) and expects to receive back
- * number[][] which will be converted into a fixed size list column.  By
- * default this will be a fixed size list of Float32 but that can be
- * customized by the `embeddingDataType` property of the embedding function.
- *
- * If a schema is provided in `makeTableOptions` then it should include the
- * embedding columns.  If no schema is provded then embedding columns will
- * be placed at the end of the table, after all of the input columns.
- */
-export declare function convertToTable(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunctionConfig, makeTableOptions?: Partial<MakeArrowTableOptions>): Promise<ArrowTable>;
-/** Creates the Arrow Type for a Vector column with dimension `dim` */
-export declare function newVectorType<T extends Float>(dim: number, innerType: T): FixedSizeList<T>;
-/**
- * Serialize an Array of records into a buffer using the Arrow IPC File serialization
- *
- * This function will call `convertToTable` and pass on `embeddings` and `schema`
- *
- * `schema` is required if data is empty
- */
-export declare function fromRecordsToBuffer(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunctionConfig, schema?: Schema): Promise<Buffer>;
-/**
- * Serialize an Array of records into a buffer using the Arrow IPC Stream serialization
- *
- * This function will call `convertToTable` and pass on `embeddings` and `schema`
- *
- * `schema` is required if data is empty
- */
-export declare function fromRecordsToStreamBuffer(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunctionConfig, schema?: Schema): Promise<Buffer>;
-/**
- * Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
- *
- * This function will apply `embeddings` to the table in a manner similar to
- * `convertToTable`.
- *
- * `schema` is required if the table is empty
- */
-export declare function fromTableToBuffer(table: ArrowTable, embeddings?: EmbeddingFunctionConfig, schema?: SchemaLike): Promise<Buffer>;
-/**
- * Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
- *
- * This function will apply `embeddings` to the table in a manner similar to
- * `convertToTable`.
- *
- * `schema` is required if the table is empty
- */
-export declare function fromDataToBuffer(data: Data, embeddings?: EmbeddingFunctionConfig, schema?: Schema): Promise<Buffer>;
-/**
- * Serialize an Arrow Table into a buffer using the Arrow IPC Stream serialization
- *
- * This function will apply `embeddings` to the table in a manner similar to
- * `convertToTable`.
- *
- * `schema` is required if the table is empty
- */
-export declare function fromTableToStreamBuffer(table: ArrowTable, embeddings?: EmbeddingFunctionConfig, schema?: SchemaLike): Promise<Buffer>;
-/**
- * Create an empty table with the given schema
- */
-export declare function createEmptyTable(schema: Schema): ArrowTable;