@lancedb/lancedb 0.10.0-beta.0 → 0.10.0-beta.1

package/dist/native.d.ts

@@ -0,0 +1,215 @@
+/* tslint:disable */
+/* eslint-disable */
+
+/* auto-generated by NAPI-RS */
+
+/** A description of an index currently configured on a column */
+export interface IndexConfig {
+  /** The name of the index */
+  name: string
+  /** The type of the index */
+  indexType: string
+  /**
+   * The columns in the index
+   *
+   * Currently this is always an array of size 1. In the future there may
+   * be more columns to represent composite indices.
+   */
+  columns: Array<string>
+}
+/** Statistics about a compaction operation. */
+export interface CompactionStats {
+  /** The number of fragments removed */
+  fragmentsRemoved: number
+  /** The number of new, compacted fragments added */
+  fragmentsAdded: number
+  /** The number of data files removed */
+  filesRemoved: number
+  /** The number of new, compacted data files added */
+  filesAdded: number
+}
+/** Statistics about a cleanup operation */
+export interface RemovalStats {
+  /** The number of bytes removed */
+  bytesRemoved: number
+  /** The number of old versions removed */
+  oldVersionsRemoved: number
+}
+/** Statistics about an optimize operation */
+export interface OptimizeStats {
+  /** Statistics about the compaction operation */
+  compaction: CompactionStats
+  /** Statistics about the removal operation */
+  prune: RemovalStats
+}
+/**
+ *  A definition of a column alteration. The alteration changes the column at
+ * `path` to have the new name `name`, to be nullable if `nullable` is true,
+ * and to have the data type `data_type`. At least one of `rename` or `nullable`
+ * must be provided.
+ */
+export interface ColumnAlteration {
+  /**
+   * The path to the column to alter. This is a dot-separated path to the column.
+   * If it is a top-level column then it is just the name of the column. If it is
+   * a nested column then it is the path to the column, e.g. "a.b.c" for a column
+   * `c` nested inside a column `b` nested inside a column `a`.
+   */
+  path: string
+  /**
+   * The new name of the column. If not provided then the name will not be changed.
+   * This must be distinct from the names of all other columns in the table.
+   */
+  rename?: string
+  /** Set the new nullability. Note that a nullable column cannot be made non-nullable. */
+  nullable?: boolean
+}
+/** A definition of a new column to add to a table. */
+export interface AddColumnsSql {
+  /** The name of the new column. */
+  name: string
+  /**
+   * The values to populate the new column with, as a SQL expression.
+   * The expression can reference other columns in the table.
+   */
+  valueSql: string
+}
+export interface IndexStatistics {
+  /** The number of rows indexed by the index */
+  numIndexedRows: number
+  /** The number of rows not indexed */
+  numUnindexedRows: number
+  /** The type of the index */
+  indexType?: string
+  /** The metadata for each index */
+  indices: Array<IndexMetadata>
+}
+export interface IndexMetadata {
+  metricType?: string
+  indexType?: string
+}
+export interface ConnectionOptions {
+  /**
+   * (For LanceDB OSS only): The interval, in seconds, at which to check for
+   * updates to the table from other processes. If None, then consistency is not
+   * checked. For performance reasons, this is the default. For strong
+   * consistency, set this to zero seconds. Then every read will check for
+   * updates from other processes. As a compromise, you can set this to a
+   * non-zero value for eventual consistency. If more than that interval
+   * has passed since the last check, then the table will be checked for updates.
+   * Note: this consistency only applies to read operations. Write operations are
+   * always consistent.
+   */
+  readConsistencyInterval?: number
+  /**
+   * (For LanceDB OSS only): configuration for object storage.
+   *
+   * The available options are described at https://lancedb.github.io/lancedb/guides/storage/
+   */
+  storageOptions?: Record<string, string>
+}
+/** Write mode for writing a table. */
+export const enum WriteMode {
+  Create = 'Create',
+  Append = 'Append',
+  Overwrite = 'Overwrite'
+}
+/** Write options when creating a Table. */
+export interface WriteOptions {
+  /** Write mode for writing to a table. */
+  mode?: WriteMode
+}
+export interface OpenTableOptions {
+  storageOptions?: Record<string, string>
+}
+export class Connection {
+  /** Create a new Connection instance from the given URI. */
+  static new(uri: string, options: ConnectionOptions): Promise<Connection>
+  display(): string
+  isOpen(): boolean
+  close(): void
+  /** List all tables in the dataset. */
+  tableNames(startAfter?: string | undefined | null, limit?: number | undefined | null): Promise<Array<string>>
+  /**
+   * Create table from a Apache Arrow IPC (file) buffer.
+   *
+   * Parameters:
+   * - name: The name of the table.
+   * - buf: The buffer containing the IPC file.
+   *
+   */
+  createTable(name: string, buf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, dataStorageOptions?: string | undefined | null): Promise<Table>
+  createEmptyTable(name: string, schemaBuf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, dataStorageOptions?: string | undefined | null): Promise<Table>
+  openTable(name: string, storageOptions?: Record<string, string> | undefined | null, indexCacheSize?: number | undefined | null): Promise<Table>
+  /** Drop table with the name. Or raise an error if the table does not exist. */
+  dropTable(name: string): Promise<void>
+}
+export class Index {
+  static ivfPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
+  static btree(): Index
+  static bitmap(): Index
+  static labelList(): Index
+  static fts(): Index
+}
+/** Typescript-style Async Iterator over RecordBatches  */
+export class RecordBatchIterator {
+  next(): Promise<Buffer | null>
+}
+/** A builder used to create and run a merge insert operation */
+export class NativeMergeInsertBuilder {
+  whenMatchedUpdateAll(condition?: string | undefined | null): NativeMergeInsertBuilder
+  whenNotMatchedInsertAll(): NativeMergeInsertBuilder
+  whenNotMatchedBySourceDelete(filter?: string | undefined | null): NativeMergeInsertBuilder
+  execute(buf: Buffer): Promise<void>
+}
+export class Query {
+  onlyIf(predicate: string): void
+  fullTextSearch(query: string, columns?: Array<string> | undefined | null): void
+  select(columns: Array<[string, string]>): void
+  selectColumns(columns: Array<string>): void
+  limit(limit: number): void
+  nearestTo(vector: Float32Array): VectorQuery
+  execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
+  explainPlan(verbose: boolean): Promise<string>
+}
+export class VectorQuery {
+  column(column: string): void
+  distanceType(distanceType: string): void
+  postfilter(): void
+  refineFactor(refineFactor: number): void
+  nprobes(nprobe: number): void
+  bypassVectorIndex(): void
+  onlyIf(predicate: string): void
+  fullTextSearch(query: string, columns?: Array<string> | undefined | null): void
+  select(columns: Array<[string, string]>): void
+  selectColumns(columns: Array<string>): void
+  limit(limit: number): void
+  execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
+  explainPlan(verbose: boolean): Promise<string>
+}
+export class Table {
+  name: string
+  display(): string
+  isOpen(): boolean
+  close(): void
+  /** Return Schema as empty Arrow IPC file. */
+  schema(): Promise<Buffer>
+  add(buf: Buffer, mode: string): Promise<void>
+  countRows(filter?: string | undefined | null): Promise<number>
+  delete(predicate: string): Promise<void>
+  createIndex(index: Index | undefined | null, column: string, replace?: boolean | undefined | null): Promise<void>
+  update(onlyIf: string | undefined | null, columns: Array<[string, string]>): Promise<void>
+  query(): Query
+  vectorSearch(vector: Float32Array): VectorQuery
+  addColumns(transforms: Array<AddColumnsSql>): Promise<void>
+  alterColumns(alterations: Array<ColumnAlteration>): Promise<void>
+  dropColumns(columns: Array<string>): Promise<void>
+  version(): Promise<number>
+  checkout(version: number): Promise<void>
+  checkoutLatest(): Promise<void>
+  restore(): Promise<void>
+  optimize(olderThanMs?: number | undefined | null, deleteUnverified?: boolean | undefined | null): Promise<OptimizeStats>
+  listIndices(): Promise<Array<IndexConfig>>
+  indexStats(indexName: string): Promise<IndexStatistics | null>
+  mergeInsert(on: Array<string>): NativeMergeInsertBuilder
+}

package/dist/table.d.ts

@@ -45,6 +45,7 @@ export interface OptimizeOptions {
      * tbl.cleanupOlderVersions(new Date());
      */
     cleanupOlderThan: Date;
+    deleteUnverified: boolean;
 }
 /**
  * A Table is a collection of Records in a LanceDB Database.

package/dist/table.js

@@ -221,7 +221,7 @@ class LocalTable extends Table {
             cleanupOlderThanMs =
                 new Date().getTime() - options.cleanupOlderThan.getTime();
         }
-        return await this.inner.optimize(cleanupOlderThanMs);
+        return await this.inner.optimize(cleanupOlderThanMs, options?.deleteUnverified);
     }
     async listIndices() {
         return await this.inner.listIndices();

package/native.d.ts

@@ -0,0 +1,208 @@
+/* tslint:disable */
+/* eslint-disable */
+
+/* auto-generated by NAPI-RS */
+
+/** A description of an index currently configured on a column */
+export interface IndexConfig {
+  /** The name of the index */
+  name: string
+  /** The type of the index */
+  indexType: string
+  /**
+   * The columns in the index
+   *
+   * Currently this is always an array of size 1. In the future there may
+   * be more columns to represent composite indices.
+   */
+  columns: Array<string>
+}
+/** Statistics about a compaction operation. */
+export interface CompactionStats {
+  /** The number of fragments removed */
+  fragmentsRemoved: number
+  /** The number of new, compacted fragments added */
+  fragmentsAdded: number
+  /** The number of data files removed */
+  filesRemoved: number
+  /** The number of new, compacted data files added */
+  filesAdded: number
+}
+/** Statistics about a cleanup operation */
+export interface RemovalStats {
+  /** The number of bytes removed */
+  bytesRemoved: number
+  /** The number of old versions removed */
+  oldVersionsRemoved: number
+}
+/** Statistics about an optimize operation */
+export interface OptimizeStats {
+  /** Statistics about the compaction operation */
+  compaction: CompactionStats
+  /** Statistics about the removal operation */
+  prune: RemovalStats
+}
+/**
+ *  A definition of a column alteration. The alteration changes the column at
+ * `path` to have the new name `name`, to be nullable if `nullable` is true,
+ * and to have the data type `data_type`. At least one of `rename` or `nullable`
+ * must be provided.
+ */
+export interface ColumnAlteration {
+  /**
+   * The path to the column to alter. This is a dot-separated path to the column.
+   * If it is a top-level column then it is just the name of the column. If it is
+   * a nested column then it is the path to the column, e.g. "a.b.c" for a column
+   * `c` nested inside a column `b` nested inside a column `a`.
+   */
+  path: string
+  /**
+   * The new name of the column. If not provided then the name will not be changed.
+   * This must be distinct from the names of all other columns in the table.
+   */
+  rename?: string
+  /** Set the new nullability. Note that a nullable column cannot be made non-nullable. */
+  nullable?: boolean
+}
+/** A definition of a new column to add to a table. */
+export interface AddColumnsSql {
+  /** The name of the new column. */
+  name: string
+  /**
+   * The values to populate the new column with, as a SQL expression.
+   * The expression can reference other columns in the table.
+   */
+  valueSql: string
+}
+export interface IndexStatistics {
+  /** The number of rows indexed by the index */
+  numIndexedRows: number
+  /** The number of rows not indexed */
+  numUnindexedRows: number
+  /** The type of the index */
+  indexType?: string
+  /** The metadata for each index */
+  indices: Array<IndexMetadata>
+}
+export interface IndexMetadata {
+  metricType?: string
+  indexType?: string
+}
+export interface ConnectionOptions {
+  /**
+   * (For LanceDB OSS only): The interval, in seconds, at which to check for
+   * updates to the table from other processes. If None, then consistency is not
+   * checked. For performance reasons, this is the default. For strong
+   * consistency, set this to zero seconds. Then every read will check for
+   * updates from other processes. As a compromise, you can set this to a
+   * non-zero value for eventual consistency. If more than that interval
+   * has passed since the last check, then the table will be checked for updates.
+   * Note: this consistency only applies to read operations. Write operations are
+   * always consistent.
+   */
+  readConsistencyInterval?: number
+  /**
+   * (For LanceDB OSS only): configuration for object storage.
+   *
+   * The available options are described at https://lancedb.github.io/lancedb/guides/storage/
+   */
+  storageOptions?: Record<string, string>
+}
+/** Write mode for writing a table. */
+export const enum WriteMode {
+  Create = 'Create',
+  Append = 'Append',
+  Overwrite = 'Overwrite'
+}
+/** Write options when creating a Table. */
+export interface WriteOptions {
+  /** Write mode for writing to a table. */
+  mode?: WriteMode
+}
+export interface OpenTableOptions {
+  storageOptions?: Record<string, string>
+}
+export class Connection {
+  /** Create a new Connection instance from the given URI. */
+  static new(uri: string, options: ConnectionOptions): Promise<Connection>
+  display(): string
+  isOpen(): boolean
+  close(): void
+  /** List all tables in the dataset. */
+  tableNames(startAfter?: string | undefined | null, limit?: number | undefined | null): Promise<Array<string>>
+  /**
+   * Create table from a Apache Arrow IPC (file) buffer.
+   *
+   * Parameters:
+   * - name: The name of the table.
+   * - buf: The buffer containing the IPC file.
+   *
+   */
+  createTable(name: string, buf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, useLegacyFormat?: boolean | undefined | null): Promise<Table>
+  createEmptyTable(name: string, schemaBuf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, useLegacyFormat?: boolean | undefined | null): Promise<Table>
+  openTable(name: string, storageOptions?: Record<string, string> | undefined | null, indexCacheSize?: number | undefined | null): Promise<Table>
+  /** Drop table with the name. Or raise an error if the table does not exist. */
+  dropTable(name: string): Promise<void>
+}
+export class Index {
+  static ivfPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
+  static btree(): Index
+}
+/** Typescript-style Async Iterator over RecordBatches  */
+export class RecordBatchIterator {
+  next(): Promise<Buffer | null>
+}
+/** A builder used to create and run a merge insert operation */
+export class NativeMergeInsertBuilder {
+  whenMatchedUpdateAll(condition?: string | undefined | null): NativeMergeInsertBuilder
+  whenNotMatchedInsertAll(): NativeMergeInsertBuilder
+  whenNotMatchedBySourceDelete(filter?: string | undefined | null): NativeMergeInsertBuilder
+  execute(buf: Buffer): Promise<void>
+}
+export class Query {
+  onlyIf(predicate: string): void
+  select(columns: Array<[string, string]>): void
+  limit(limit: number): void
+  nearestTo(vector: Float32Array): VectorQuery
+  execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
+  explainPlan(verbose: boolean): Promise<string>
+}
+export class VectorQuery {
+  column(column: string): void
+  distanceType(distanceType: string): void
+  postfilter(): void
+  refineFactor(refineFactor: number): void
+  nprobes(nprobe: number): void
+  bypassVectorIndex(): void
+  onlyIf(predicate: string): void
+  select(columns: Array<[string, string]>): void
+  limit(limit: number): void
+  execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
+  explainPlan(verbose: boolean): Promise<string>
+}
+export class Table {
+  name: string
+  display(): string
+  isOpen(): boolean
+  close(): void
+  /** Return Schema as empty Arrow IPC file. */
+  schema(): Promise<Buffer>
+  add(buf: Buffer, mode: string): Promise<void>
+  countRows(filter?: string | undefined | null): Promise<number>
+  delete(predicate: string): Promise<void>
+  createIndex(index: Index | undefined | null, column: string, replace?: boolean | undefined | null): Promise<void>
+  update(onlyIf: string | undefined | null, columns: Array<[string, string]>): Promise<void>
+  query(): Query
+  vectorSearch(vector: Float32Array): VectorQuery
+  addColumns(transforms: Array<AddColumnsSql>): Promise<void>
+  alterColumns(alterations: Array<ColumnAlteration>): Promise<void>
+  dropColumns(columns: Array<string>): Promise<void>
+  version(): Promise<number>
+  checkout(version: number): Promise<void>
+  checkoutLatest(): Promise<void>
+  restore(): Promise<void>
+  optimize(olderThanMs?: number | undefined | null): Promise<OptimizeStats>
+  listIndices(): Promise<Array<IndexConfig>>
+  indexStats(indexName: string): Promise<IndexStatistics | null>
+  mergeInsert(on: Array<string>): NativeMergeInsertBuilder
+}

package/package.json

@@ -10,7 +10,7 @@
     "vector database",
     "ann"
   ],
-  "version": "0.10.0-beta.0",
+  "version": "0.10.0-beta.1",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -92,11 +92,11 @@
     "reflect-metadata": "^0.2.2"
   },
   "optionalDependencies": {
-    "@lancedb/lancedb-darwin-arm64": "0.10.0-beta.0",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.10.0-beta.0",
-    "@lancedb/lancedb-darwin-x64": "0.10.0-beta.0",
-    "@lancedb/lancedb-linux-x64-gnu": "0.10.0-beta.0",
-    "@lancedb/lancedb-win32-x64-msvc": "0.10.0-beta.0"
+    "@lancedb/lancedb-darwin-arm64": "0.10.0-beta.1",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.10.0-beta.1",
+    "@lancedb/lancedb-darwin-x64": "0.10.0-beta.1",
+    "@lancedb/lancedb-linux-x64-gnu": "0.10.0-beta.1",
+    "@lancedb/lancedb-win32-x64-msvc": "0.10.0-beta.1"
   },
   "peerDependencies": {
     "apache-arrow": ">=13.0.0 <=17.0.0"