@lancedb/lancedb 0.10.0 → 0.11.0-beta.1

package/dist/connection.d.ts

@@ -30,8 +30,8 @@ export interface CreateTableOptions {
     /**
      * The version of the data storage format to use.
      *
-     * The default is `legacy`, which is Lance format v1.
-     * `stable` is the new format, which is Lance format v2.
+     * The default is `stable`.
+     * Set to "legacy" to use the old format.
      */
     dataStorageVersion?: string;
     /**
@@ -45,9 +45,9 @@ export interface CreateTableOptions {
     /**
      * If true then data files will be written with the legacy format
      *
-     * The default is true while the new format is in beta
+     * The default is false.
      *
-     * Deprecated.
+     * Deprecated. Use data storage version instead.
      */
     useLegacyFormat?: boolean;
     schema?: SchemaLike;

package/dist/connection.js

@@ -73,7 +73,7 @@ class LocalConnection extends Connection {
             throw new Error("data is required");
         }
         const { buf, mode } = await table_1.Table.parseTableData(data, options);
-        let dataStorageVersion = "legacy";
+        let dataStorageVersion = "stable";
         if (options?.dataStorageVersion !== undefined) {
             dataStorageVersion = options.dataStorageVersion;
         }
@@ -95,7 +95,7 @@ class LocalConnection extends Connection {
             const registry = (0, registry_1.getRegistry)();
             metadata = registry.getTableMetadata([embeddingFunction]);
         }
-        let dataStorageVersion = "legacy";
+        let dataStorageVersion = "stable";
         if (options?.dataStorageVersion !== undefined) {
             dataStorageVersion = options.dataStorageVersion;
         }

package/dist/index.js

@@ -13,11 +13,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.embedding = exports.Table = exports.Index = exports.RecordBatchIterator = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Connection = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = void 0;
+exports.embedding = exports.Table = exports.Index = exports.RecordBatchIterator = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Connection = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = exports.WriteMode = void 0;
 exports.connect = connect;
 const connection_1 = require("./connection");
 const native_js_1 = require("./native.js");
 const remote_1 = require("./remote");
+var native_js_2 = require("./native.js");
+Object.defineProperty(exports, "WriteMode", { enumerable: true, get: function () { return native_js_2.WriteMode; } });
 var arrow_1 = require("./arrow");
 Object.defineProperty(exports, "makeArrowTable", { enumerable: true, get: function () { return arrow_1.makeArrowTable; } });
 Object.defineProperty(exports, "MakeArrowTableOptions", { enumerable: true, get: function () { return arrow_1.MakeArrowTableOptions; } });

package/dist/indices.d.ts

@@ -92,21 +92,206 @@ export interface IvfPqOptions {
      */
     sampleRate?: number;
 }
+/**
+ * Options to create an `HNSW_PQ` index
+ */
 export interface HnswPqOptions {
+    /**
+     * The distance metric used to train the index.
+     *
+     * Default value is "l2".
+     *
+     * The following distance types are available:
+     *
+     * "l2" - Euclidean distance. This is a very common distance metric that
+     * accounts for both magnitude and direction when determining the distance
+     * between vectors. L2 distance has a range of [0, ∞).
+     *
+     * "cosine" - Cosine distance.  Cosine distance is a distance metric
+     * calculated from the cosine similarity between two vectors. Cosine
+     * similarity is a measure of similarity between two non-zero vectors of an
+     * inner product space. It is defined to equal the cosine of the angle
+     * between them.  Unlike L2, the cosine distance is not affected by the
+     * magnitude of the vectors.  Cosine distance has a range of [0, 2].
+     *
+     * "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
+     * distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
+     * L2 norm is 1), then dot distance is equivalent to the cosine distance.
+     */
     distanceType?: "l2" | "cosine" | "dot";
+    /**
+     * The number of IVF partitions to create.
+     *
+     * For HNSW, we recommend a small number of partitions. Setting this to 1 works
+     * well for most tables. For very large tables, training just one HNSW graph
+     * will require too much memory. Each partition becomes its own HNSW graph, so
+     * setting this value higher reduces the peak memory use of training.
+     *
+     */
     numPartitions?: number;
+    /**
+     * Number of sub-vectors of PQ.
+     *
+     * This value controls how much the vector is compressed during the quantization step.
+     * The more sub vectors there are the less the vector is compressed.  The default is
+     * the dimension of the vector divided by 16.  If the dimension is not evenly divisible
+     * by 16 we use the dimension divded by 8.
+     *
+     * The above two cases are highly preferred.  Having 8 or 16 values per subvector allows
+     * us to use efficient SIMD instructions.
+     *
+     * If the dimension is not visible by 8 then we use 1 subvector.  This is not ideal and
+     * will likely result in poor performance.
+     *
+     */
     numSubVectors?: number;
+    /**
+     * Max iterations to train kmeans.
+     *
+     * The default value is 50.
+     *
+     * When training an IVF index we use kmeans to calculate the partitions.  This parameter
+     * controls how many iterations of kmeans to run.
+     *
+     * Increasing this might improve the quality of the index but in most cases the parameter
+     * is unused because kmeans will converge with fewer iterations.  The parameter is only
+     * used in cases where kmeans does not appear to converge.  In those cases it is unlikely
+     * that setting this larger will lead to the index converging anyways.
+     *
+     */
     maxIterations?: number;
+    /**
+     * The rate used to calculate the number of training vectors for kmeans.
+     *
+     * Default value is 256.
+     *
+     * When an IVF index is trained, we need to calculate partitions.  These are groups
+     * of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
+     *
+     * Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
+     * random sample of the data.  This parameter controls the size of the sample.  The total
+     * number of vectors used to train the index is `sample_rate * num_partitions`.
+     *
+     * Increasing this value might improve the quality of the index but in most cases the
+     * default should be sufficient.
+     *
+     */
     sampleRate?: number;
+    /**
+     * The number of neighbors to select for each vector in the HNSW graph.
+     *
+     * The default value is 20.
+     *
+     * This value controls the tradeoff between search speed and accuracy.
+     * The higher the value the more accurate the search but the slower it will be.
+     *
+     */
     m?: number;
+    /**
+     * The number of candidates to evaluate during the construction of the HNSW graph.
+     *
+     * The default value is 300.
+     *
+     * This value controls the tradeoff between build speed and accuracy.
+     * The higher the value the more accurate the build but the slower it will be.
+     * 150 to 300 is the typical range. 100 is a minimum for good quality search
+     * results. In most cases, there is no benefit to setting this higher than 500.
+     * This value should be set to a value that is not less than `ef` in the search phase.
+     *
+     */
     efConstruction?: number;
 }
+/**
+ * Options to create an `HNSW_SQ` index
+ */
 export interface HnswSqOptions {
+    /**
+     * The distance metric used to train the index.
+     *
+     * Default value is "l2".
+     *
+     * The following distance types are available:
+     *
+     * "l2" - Euclidean distance. This is a very common distance metric that
+     * accounts for both magnitude and direction when determining the distance
+     * between vectors. L2 distance has a range of [0, ∞).
+     *
+     * "cosine" - Cosine distance.  Cosine distance is a distance metric
+     * calculated from the cosine similarity between two vectors. Cosine
+     * similarity is a measure of similarity between two non-zero vectors of an
+     * inner product space. It is defined to equal the cosine of the angle
+     * between them.  Unlike L2, the cosine distance is not affected by the
+     * magnitude of the vectors.  Cosine distance has a range of [0, 2].
+     *
+     * "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
+     * distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
+     * L2 norm is 1), then dot distance is equivalent to the cosine distance.
+     */
     distanceType?: "l2" | "cosine" | "dot";
+    /**
+     * The number of IVF partitions to create.
+     *
+     * For HNSW, we recommend a small number of partitions. Setting this to 1 works
+     * well for most tables. For very large tables, training just one HNSW graph
+     * will require too much memory. Each partition becomes its own HNSW graph, so
+     * setting this value higher reduces the peak memory use of training.
+     *
+     */
     numPartitions?: number;
+    /**
+     * Max iterations to train kmeans.
+     *
+     * The default value is 50.
+     *
+     * When training an IVF index we use kmeans to calculate the partitions.  This parameter
+     * controls how many iterations of kmeans to run.
+     *
+     * Increasing this might improve the quality of the index but in most cases the parameter
+     * is unused because kmeans will converge with fewer iterations.  The parameter is only
+     * used in cases where kmeans does not appear to converge.  In those cases it is unlikely
+     * that setting this larger will lead to the index converging anyways.
+     *
+     */
     maxIterations?: number;
+    /**
+     * The rate used to calculate the number of training vectors for kmeans.
+     *
+     * Default value is 256.
+     *
+     * When an IVF index is trained, we need to calculate partitions.  These are groups
+     * of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
+     *
+     * Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
+     * random sample of the data.  This parameter controls the size of the sample.  The total
+     * number of vectors used to train the index is `sample_rate * num_partitions`.
+     *
+     * Increasing this value might improve the quality of the index but in most cases the
+     * default should be sufficient.
+     *
+     */
     sampleRate?: number;
+    /**
+     * The number of neighbors to select for each vector in the HNSW graph.
+     *
+     * The default value is 20.
+     *
+     * This value controls the tradeoff between search speed and accuracy.
+     * The higher the value the more accurate the search but the slower it will be.
+     *
+     */
     m?: number;
+    /**
+     * The number of candidates to evaluate during the construction of the HNSW graph.
+     *
+     * The default value is 300.
+     *
+     * This value controls the tradeoff between build speed and accuracy.
+     * The higher the value the more accurate the build but the slower it will be.
+     * 150 to 300 is the typical range. 100 is a minimum for good quality search
+     * results. In most cases, there is no benefit to setting this higher than 500.
+     * This value should be set to a value that is not less than `ef` in the search phase.
+     *
+     */
     efConstruction?: number;
 }
 /**
@@ -119,7 +304,7 @@ export interface FtsOptions {
      * If set to false, the index will not store the positions of the tokens in the text,
      * which will make the index smaller and faster to build, but will not support phrase queries.
      */
-    withPositions?: boolean;
+    withPosition?: boolean;
 }
 export declare class Index {
     private readonly inner;
@@ -199,13 +384,21 @@ export declare class Index {
     static fts(options?: Partial<FtsOptions>): Index;
     /**
      *
-     * Create a hnswpq index
+     * Create a hnswPq index
+     *
+     * HNSW-PQ stands for Hierarchical Navigable Small World - Product Quantization.
+     * It is a variant of the HNSW algorithm that uses product quantization to compress
+     * the vectors.
      *
      */
     static hnswPq(options?: Partial<HnswPqOptions>): Index;
     /**
      *
-     * Create a hnswsq index
+     * Create a hnswSq index
+     *
+     * HNSW-SQ stands for Hierarchical Navigable Small World - Scalar Quantization.
+     * It is a variant of the HNSW algorithm that uses scalar quantization to compress
+     * the vectors.
      *
      */
     static hnswSq(options?: Partial<HnswSqOptions>): Index;

package/dist/indices.js

@@ -101,11 +101,15 @@ class Index {
      * For now, the full text search index only supports English, and doesn't support phrase search.
      */
     static fts(options) {
-        return new Index(native_1.Index.fts(options?.withPositions));
+        return new Index(native_1.Index.fts(options?.withPosition));
     }
     /**
      *
-     * Create a hnswpq index
+     * Create a hnswPq index
+     *
+     * HNSW-PQ stands for Hierarchical Navigable Small World - Product Quantization.
+     * It is a variant of the HNSW algorithm that uses product quantization to compress
+     * the vectors.
      *
      */
     static hnswPq(options) {
@@ -113,7 +117,11 @@ class Index {
     }
     /**
      *
-     * Create a hnswsq index
+     * Create a hnswSq index
+     *
+     * HNSW-SQ stands for Hierarchical Navigable Small World - Scalar Quantization.
+     * It is a variant of the HNSW algorithm that uses scalar quantization to compress
+     * the vectors.
      *
      */
     static hnswSq(options) {

package/dist/native.d.ts

@@ -109,7 +109,7 @@ export interface ConnectionOptions {
   storageOptions?: Record<string, string>
 }
 /** Write mode for writing a table. */
-export const enum WriteMode {
+export enum WriteMode {
   Create = 'Create',
   Append = 'Append',
   Overwrite = 'Overwrite'
@@ -202,7 +202,7 @@ export class Table {
   countRows(filter?: string | undefined | null): Promise<number>
   delete(predicate: string): Promise<void>
   createIndex(index: Index | undefined | null, column: string, replace?: boolean | undefined | null): Promise<void>
-  update(onlyIf: string | undefined | null, columns: Array<[string, string]>): Promise<void>
+  update(onlyIf: string | undefined | null, columns: Array<[string, string]>): Promise<bigint>
   query(): Query
   vectorSearch(vector: Float32Array): VectorQuery
   addColumns(transforms: Array<AddColumnsSql>): Promise<void>

package/package.json

@@ -10,7 +10,7 @@
     "vector database",
     "ann"
   ],
-  "version": "0.10.0",
+  "version": "0.11.0-beta.1",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -73,8 +73,8 @@
   ],
   "scripts": {
     "artifacts": "napi artifacts",
-    "build:debug": "napi build --platform --dts ../lancedb/native.d.ts --js ../lancedb/native.js lancedb",
-    "build:release": "napi build --platform --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js dist/",
+    "build:debug": "napi build --platform --no-const-enum --dts ../lancedb/native.d.ts --js ../lancedb/native.js lancedb",
+    "build:release": "napi build --platform --no-const-enum --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js dist/",
     "build": "npm run build:debug && tsc -b && shx cp lancedb/native.d.ts dist/native.d.ts && shx cp lancedb/*.node dist/",
     "build-release": "npm run build:release && tsc -b && shx cp lancedb/native.d.ts dist/native.d.ts",
     "lint-ci": "biome ci .",
@@ -92,11 +92,11 @@
     "reflect-metadata": "^0.2.2"
   },
   "optionalDependencies": {
-    "@lancedb/lancedb-darwin-arm64": "0.10.0",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.10.0",
-    "@lancedb/lancedb-darwin-x64": "0.10.0",
-    "@lancedb/lancedb-linux-x64-gnu": "0.10.0",
-    "@lancedb/lancedb-win32-x64-msvc": "0.10.0"
+    "@lancedb/lancedb-darwin-arm64": "0.11.0-beta.1",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.11.0-beta.1",
+    "@lancedb/lancedb-darwin-x64": "0.11.0-beta.1",
+    "@lancedb/lancedb-linux-x64-gnu": "0.11.0-beta.1",
+    "@lancedb/lancedb-win32-x64-msvc": "0.11.0-beta.1"
   },
   "peerDependencies": {
     "apache-arrow": ">=13.0.0 <=17.0.0"

package/native.d.ts

@@ -1,208 +0,0 @@
-/* tslint:disable */
-/* eslint-disable */
-
-/* auto-generated by NAPI-RS */
-
-/** A description of an index currently configured on a column */
-export interface IndexConfig {
-  /** The name of the index */
-  name: string
-  /** The type of the index */
-  indexType: string
-  /**
-   * The columns in the index
-   *
-   * Currently this is always an array of size 1. In the future there may
-   * be more columns to represent composite indices.
-   */
-  columns: Array<string>
-}
-/** Statistics about a compaction operation. */
-export interface CompactionStats {
-  /** The number of fragments removed */
-  fragmentsRemoved: number
-  /** The number of new, compacted fragments added */
-  fragmentsAdded: number
-  /** The number of data files removed */
-  filesRemoved: number
-  /** The number of new, compacted data files added */
-  filesAdded: number
-}
-/** Statistics about a cleanup operation */
-export interface RemovalStats {
-  /** The number of bytes removed */
-  bytesRemoved: number
-  /** The number of old versions removed */
-  oldVersionsRemoved: number
-}
-/** Statistics about an optimize operation */
-export interface OptimizeStats {
-  /** Statistics about the compaction operation */
-  compaction: CompactionStats
-  /** Statistics about the removal operation */
-  prune: RemovalStats
-}
-/**
- *  A definition of a column alteration. The alteration changes the column at
- * `path` to have the new name `name`, to be nullable if `nullable` is true,
- * and to have the data type `data_type`. At least one of `rename` or `nullable`
- * must be provided.
- */
-export interface ColumnAlteration {
-  /**
-   * The path to the column to alter. This is a dot-separated path to the column.
-   * If it is a top-level column then it is just the name of the column. If it is
-   * a nested column then it is the path to the column, e.g. "a.b.c" for a column
-   * `c` nested inside a column `b` nested inside a column `a`.
-   */
-  path: string
-  /**
-   * The new name of the column. If not provided then the name will not be changed.
-   * This must be distinct from the names of all other columns in the table.
-   */
-  rename?: string
-  /** Set the new nullability. Note that a nullable column cannot be made non-nullable. */
-  nullable?: boolean
-}
-/** A definition of a new column to add to a table. */
-export interface AddColumnsSql {
-  /** The name of the new column. */
-  name: string
-  /**
-   * The values to populate the new column with, as a SQL expression.
-   * The expression can reference other columns in the table.
-   */
-  valueSql: string
-}
-export interface IndexStatistics {
-  /** The number of rows indexed by the index */
-  numIndexedRows: number
-  /** The number of rows not indexed */
-  numUnindexedRows: number
-  /** The type of the index */
-  indexType?: string
-  /** The metadata for each index */
-  indices: Array<IndexMetadata>
-}
-export interface IndexMetadata {
-  metricType?: string
-  indexType?: string
-}
-export interface ConnectionOptions {
-  /**
-   * (For LanceDB OSS only): The interval, in seconds, at which to check for
-   * updates to the table from other processes. If None, then consistency is not
-   * checked. For performance reasons, this is the default. For strong
-   * consistency, set this to zero seconds. Then every read will check for
-   * updates from other processes. As a compromise, you can set this to a
-   * non-zero value for eventual consistency. If more than that interval
-   * has passed since the last check, then the table will be checked for updates.
-   * Note: this consistency only applies to read operations. Write operations are
-   * always consistent.
-   */
-  readConsistencyInterval?: number
-  /**
-   * (For LanceDB OSS only): configuration for object storage.
-   *
-   * The available options are described at https://lancedb.github.io/lancedb/guides/storage/
-   */
-  storageOptions?: Record<string, string>
-}
-/** Write mode for writing a table. */
-export const enum WriteMode {
-  Create = 'Create',
-  Append = 'Append',
-  Overwrite = 'Overwrite'
-}
-/** Write options when creating a Table. */
-export interface WriteOptions {
-  /** Write mode for writing to a table. */
-  mode?: WriteMode
-}
-export interface OpenTableOptions {
-  storageOptions?: Record<string, string>
-}
-export class Connection {
-  /** Create a new Connection instance from the given URI. */
-  static new(uri: string, options: ConnectionOptions): Promise<Connection>
-  display(): string
-  isOpen(): boolean
-  close(): void
-  /** List all tables in the dataset. */
-  tableNames(startAfter?: string | undefined | null, limit?: number | undefined | null): Promise<Array<string>>
-  /**
-   * Create table from a Apache Arrow IPC (file) buffer.
-   *
-   * Parameters:
-   * - name: The name of the table.
-   * - buf: The buffer containing the IPC file.
-   *
-   */
-  createTable(name: string, buf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, useLegacyFormat?: boolean | undefined | null): Promise<Table>
-  createEmptyTable(name: string, schemaBuf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, useLegacyFormat?: boolean | undefined | null): Promise<Table>
-  openTable(name: string, storageOptions?: Record<string, string> | undefined | null, indexCacheSize?: number | undefined | null): Promise<Table>
-  /** Drop table with the name. Or raise an error if the table does not exist. */
-  dropTable(name: string): Promise<void>
-}
-export class Index {
-  static ivfPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
-  static btree(): Index
-}
-/** Typescript-style Async Iterator over RecordBatches  */
-export class RecordBatchIterator {
-  next(): Promise<Buffer | null>
-}
-/** A builder used to create and run a merge insert operation */
-export class NativeMergeInsertBuilder {
-  whenMatchedUpdateAll(condition?: string | undefined | null): NativeMergeInsertBuilder
-  whenNotMatchedInsertAll(): NativeMergeInsertBuilder
-  whenNotMatchedBySourceDelete(filter?: string | undefined | null): NativeMergeInsertBuilder
-  execute(buf: Buffer): Promise<void>
-}
-export class Query {
-  onlyIf(predicate: string): void
-  select(columns: Array<[string, string]>): void
-  limit(limit: number): void
-  nearestTo(vector: Float32Array): VectorQuery
-  execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
-  explainPlan(verbose: boolean): Promise<string>
-}
-export class VectorQuery {
-  column(column: string): void
-  distanceType(distanceType: string): void
-  postfilter(): void
-  refineFactor(refineFactor: number): void
-  nprobes(nprobe: number): void
-  bypassVectorIndex(): void
-  onlyIf(predicate: string): void
-  select(columns: Array<[string, string]>): void
-  limit(limit: number): void
-  execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
-  explainPlan(verbose: boolean): Promise<string>
-}
-export class Table {
-  name: string
-  display(): string
-  isOpen(): boolean
-  close(): void
-  /** Return Schema as empty Arrow IPC file. */
-  schema(): Promise<Buffer>
-  add(buf: Buffer, mode: string): Promise<void>
-  countRows(filter?: string | undefined | null): Promise<number>
-  delete(predicate: string): Promise<void>
-  createIndex(index: Index | undefined | null, column: string, replace?: boolean | undefined | null): Promise<void>
-  update(onlyIf: string | undefined | null, columns: Array<[string, string]>): Promise<void>
-  query(): Query
-  vectorSearch(vector: Float32Array): VectorQuery
-  addColumns(transforms: Array<AddColumnsSql>): Promise<void>
-  alterColumns(alterations: Array<ColumnAlteration>): Promise<void>
-  dropColumns(columns: Array<string>): Promise<void>
-  version(): Promise<number>
-  checkout(version: number): Promise<void>
-  checkoutLatest(): Promise<void>
-  restore(): Promise<void>
-  optimize(olderThanMs?: number | undefined | null): Promise<OptimizeStats>
-  listIndices(): Promise<Array<IndexConfig>>
-  indexStats(indexName: string): Promise<IndexStatistics | null>
-  mergeInsert(on: Array<string>): NativeMergeInsertBuilder
-}