@lancedb/lancedb 0.10.0-beta.1 → 0.11.0-beta.0

package/dist/connection.d.ts

@@ -30,16 +30,24 @@ export interface CreateTableOptions {
     /**
      * The version of the data storage format to use.
      *
-     * The default is `legacy`, which is Lance format v1.
-     * `stable` is the new format, which is Lance format v2.
+     * The default is `stable`.
+     * Set to "legacy" to use the old format.
      */
     dataStorageVersion?: string;
+    /**
+     * Use the new V2 manifest paths. These paths provide more efficient
+     * opening of datasets with many versions on object stores.  WARNING:
+     * turning this on will make the dataset unreadable for older versions
+     * of LanceDB (prior to 0.10.0). To migrate an existing dataset, instead
+     * use the {@link LocalTable#migrateManifestPathsV2} method.
+     */
+    enableV2ManifestPaths?: boolean;
     /**
      * If true then data files will be written with the legacy format
      *
-     * The default is true while the new format is in beta
+     * The default is false.
      *
-     * Deprecated.
+     * Deprecated. Use data storage version instead.
      */
     useLegacyFormat?: boolean;
     schema?: SchemaLike;

package/dist/connection.js

@@ -73,14 +73,14 @@ class LocalConnection extends Connection {
             throw new Error("data is required");
         }
         const { buf, mode } = await table_1.Table.parseTableData(data, options);
-        let dataStorageVersion = "legacy";
+        let dataStorageVersion = "stable";
         if (options?.dataStorageVersion !== undefined) {
             dataStorageVersion = options.dataStorageVersion;
         }
         else if (options?.useLegacyFormat !== undefined) {
             dataStorageVersion = options.useLegacyFormat ? "legacy" : "stable";
         }
-        const innerTable = await this.inner.createTable(nameOrOptions, buf, mode, cleanseStorageOptions(options?.storageOptions), dataStorageVersion);
+        const innerTable = await this.inner.createTable(nameOrOptions, buf, mode, cleanseStorageOptions(options?.storageOptions), dataStorageVersion, options?.enableV2ManifestPaths);
         return new table_1.LocalTable(innerTable);
     }
     async createEmptyTable(name, schema, options) {
@@ -95,7 +95,7 @@ class LocalConnection extends Connection {
             const registry = (0, registry_1.getRegistry)();
             metadata = registry.getTableMetadata([embeddingFunction]);
         }
-        let dataStorageVersion = "legacy";
+        let dataStorageVersion = "stable";
         if (options?.dataStorageVersion !== undefined) {
             dataStorageVersion = options.dataStorageVersion;
         }
@@ -104,7 +104,7 @@ class LocalConnection extends Connection {
         }
         const table = (0, arrow_1.makeEmptyTable)(schema, metadata);
         const buf = await (0, arrow_1.fromTableToBuffer)(table);
-        const innerTable = await this.inner.createEmptyTable(name, buf, mode, cleanseStorageOptions(options?.storageOptions), dataStorageVersion);
+        const innerTable = await this.inner.createEmptyTable(name, buf, mode, cleanseStorageOptions(options?.storageOptions), dataStorageVersion, options?.enableV2ManifestPaths);
         return new table_1.LocalTable(innerTable);
     }
     async dropTable(name) {

package/dist/indices.d.ts

@@ -92,6 +92,220 @@ export interface IvfPqOptions {
      */
     sampleRate?: number;
 }
+/**
+ * Options to create an `HNSW_PQ` index
+ */
+export interface HnswPqOptions {
+    /**
+     * The distance metric used to train the index.
+     *
+     * Default value is "l2".
+     *
+     * The following distance types are available:
+     *
+     * "l2" - Euclidean distance. This is a very common distance metric that
+     * accounts for both magnitude and direction when determining the distance
+     * between vectors. L2 distance has a range of [0, ∞).
+     *
+     * "cosine" - Cosine distance.  Cosine distance is a distance metric
+     * calculated from the cosine similarity between two vectors. Cosine
+     * similarity is a measure of similarity between two non-zero vectors of an
+     * inner product space. It is defined to equal the cosine of the angle
+     * between them.  Unlike L2, the cosine distance is not affected by the
+     * magnitude of the vectors.  Cosine distance has a range of [0, 2].
+     *
+     * "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
+     * distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
+     * L2 norm is 1), then dot distance is equivalent to the cosine distance.
+     */
+    distanceType?: "l2" | "cosine" | "dot";
+    /**
+     * The number of IVF partitions to create.
+     *
+     * For HNSW, we recommend a small number of partitions. Setting this to 1 works
+     * well for most tables. For very large tables, training just one HNSW graph
+     * will require too much memory. Each partition becomes its own HNSW graph, so
+     * setting this value higher reduces the peak memory use of training.
+     *
+     */
+    numPartitions?: number;
+    /**
+     * Number of sub-vectors of PQ.
+     *
+     * This value controls how much the vector is compressed during the quantization step.
+     * The more sub vectors there are the less the vector is compressed.  The default is
+     * the dimension of the vector divided by 16.  If the dimension is not evenly divisible
+     * by 16 we use the dimension divded by 8.
+     *
+     * The above two cases are highly preferred.  Having 8 or 16 values per subvector allows
+     * us to use efficient SIMD instructions.
+     *
+     * If the dimension is not visible by 8 then we use 1 subvector.  This is not ideal and
+     * will likely result in poor performance.
+     *
+     */
+    numSubVectors?: number;
+    /**
+     * Max iterations to train kmeans.
+     *
+     * The default value is 50.
+     *
+     * When training an IVF index we use kmeans to calculate the partitions.  This parameter
+     * controls how many iterations of kmeans to run.
+     *
+     * Increasing this might improve the quality of the index but in most cases the parameter
+     * is unused because kmeans will converge with fewer iterations.  The parameter is only
+     * used in cases where kmeans does not appear to converge.  In those cases it is unlikely
+     * that setting this larger will lead to the index converging anyways.
+     *
+     */
+    maxIterations?: number;
+    /**
+     * The rate used to calculate the number of training vectors for kmeans.
+     *
+     * Default value is 256.
+     *
+     * When an IVF index is trained, we need to calculate partitions.  These are groups
+     * of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
+     *
+     * Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
+     * random sample of the data.  This parameter controls the size of the sample.  The total
+     * number of vectors used to train the index is `sample_rate * num_partitions`.
+     *
+     * Increasing this value might improve the quality of the index but in most cases the
+     * default should be sufficient.
+     *
+     */
+    sampleRate?: number;
+    /**
+     * The number of neighbors to select for each vector in the HNSW graph.
+     *
+     * The default value is 20.
+     *
+     * This value controls the tradeoff between search speed and accuracy.
+     * The higher the value the more accurate the search but the slower it will be.
+     *
+     */
+    m?: number;
+    /**
+     * The number of candidates to evaluate during the construction of the HNSW graph.
+     *
+     * The default value is 300.
+     *
+     * This value controls the tradeoff between build speed and accuracy.
+     * The higher the value the more accurate the build but the slower it will be.
+     * 150 to 300 is the typical range. 100 is a minimum for good quality search
+     * results. In most cases, there is no benefit to setting this higher than 500.
+     * This value should be set to a value that is not less than `ef` in the search phase.
+     *
+     */
+    efConstruction?: number;
+}
+/**
+ * Options to create an `HNSW_SQ` index
+ */
+export interface HnswSqOptions {
+    /**
+     * The distance metric used to train the index.
+     *
+     * Default value is "l2".
+     *
+     * The following distance types are available:
+     *
+     * "l2" - Euclidean distance. This is a very common distance metric that
+     * accounts for both magnitude and direction when determining the distance
+     * between vectors. L2 distance has a range of [0, ∞).
+     *
+     * "cosine" - Cosine distance.  Cosine distance is a distance metric
+     * calculated from the cosine similarity between two vectors. Cosine
+     * similarity is a measure of similarity between two non-zero vectors of an
+     * inner product space. It is defined to equal the cosine of the angle
+     * between them.  Unlike L2, the cosine distance is not affected by the
+     * magnitude of the vectors.  Cosine distance has a range of [0, 2].
+     *
+     * "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
+     * distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
+     * L2 norm is 1), then dot distance is equivalent to the cosine distance.
+     */
+    distanceType?: "l2" | "cosine" | "dot";
+    /**
+     * The number of IVF partitions to create.
+     *
+     * For HNSW, we recommend a small number of partitions. Setting this to 1 works
+     * well for most tables. For very large tables, training just one HNSW graph
+     * will require too much memory. Each partition becomes its own HNSW graph, so
+     * setting this value higher reduces the peak memory use of training.
+     *
+     */
+    numPartitions?: number;
+    /**
+     * Max iterations to train kmeans.
+     *
+     * The default value is 50.
+     *
+     * When training an IVF index we use kmeans to calculate the partitions.  This parameter
+     * controls how many iterations of kmeans to run.
+     *
+     * Increasing this might improve the quality of the index but in most cases the parameter
+     * is unused because kmeans will converge with fewer iterations.  The parameter is only
+     * used in cases where kmeans does not appear to converge.  In those cases it is unlikely
+     * that setting this larger will lead to the index converging anyways.
+     *
+     */
+    maxIterations?: number;
+    /**
+     * The rate used to calculate the number of training vectors for kmeans.
+     *
+     * Default value is 256.
+     *
+     * When an IVF index is trained, we need to calculate partitions.  These are groups
+     * of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
+     *
+     * Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
+     * random sample of the data.  This parameter controls the size of the sample.  The total
+     * number of vectors used to train the index is `sample_rate * num_partitions`.
+     *
+     * Increasing this value might improve the quality of the index but in most cases the
+     * default should be sufficient.
+     *
+     */
+    sampleRate?: number;
+    /**
+     * The number of neighbors to select for each vector in the HNSW graph.
+     *
+     * The default value is 20.
+     *
+     * This value controls the tradeoff between search speed and accuracy.
+     * The higher the value the more accurate the search but the slower it will be.
+     *
+     */
+    m?: number;
+    /**
+     * The number of candidates to evaluate during the construction of the HNSW graph.
+     *
+     * The default value is 300.
+     *
+     * This value controls the tradeoff between build speed and accuracy.
+     * The higher the value the more accurate the build but the slower it will be.
+     * 150 to 300 is the typical range. 100 is a minimum for good quality search
+     * results. In most cases, there is no benefit to setting this higher than 500.
+     * This value should be set to a value that is not less than `ef` in the search phase.
+     *
+     */
+    efConstruction?: number;
+}
+/**
+ * Options to create a full text search index
+ */
+export interface FtsOptions {
+    /**
+     * Whether to build the index with positions.
+     * True by default.
+     * If set to false, the index will not store the positions of the tokens in the text,
+     * which will make the index smaller and faster to build, but will not support phrase queries.
+     */
+    withPosition?: boolean;
+}
 export declare class Index {
     private readonly inner;
     private constructor();
@@ -167,7 +381,27 @@ export declare class Index {
      *
      * For now, the full text search index only supports English, and doesn't support phrase search.
      */
-    static fts(): Index;
+    static fts(options?: Partial<FtsOptions>): Index;
+    /**
+     *
+     * Create a hnswPq index
+     *
+     * HNSW-PQ stands for Hierarchical Navigable Small World - Product Quantization.
+     * It is a variant of the HNSW algorithm that uses product quantization to compress
+     * the vectors.
+     *
+     */
+    static hnswPq(options?: Partial<HnswPqOptions>): Index;
+    /**
+     *
+     * Create a hnswSq index
+     *
+     * HNSW-SQ stands for Hierarchical Navigable Small World - Scalar Quantization.
+     * It is a variant of the HNSW algorithm that uses scalar quantization to compress
+     * the vectors.
+     *
+     */
+    static hnswSq(options?: Partial<HnswSqOptions>): Index;
 }
 export interface IndexOptions {
     /**

package/dist/indices.js

@@ -100,8 +100,32 @@ class Index {
      *
      * For now, the full text search index only supports English, and doesn't support phrase search.
      */
-    static fts() {
-        return new Index(native_1.Index.fts());
+    static fts(options) {
+        return new Index(native_1.Index.fts(options?.withPosition));
+    }
+    /**
+     *
+     * Create a hnswPq index
+     *
+     * HNSW-PQ stands for Hierarchical Navigable Small World - Product Quantization.
+     * It is a variant of the HNSW algorithm that uses product quantization to compress
+     * the vectors.
+     *
+     */
+    static hnswPq(options) {
+        return new Index(native_1.Index.hnswPq(options?.distanceType, options?.numPartitions, options?.numSubVectors, options?.maxIterations, options?.sampleRate, options?.m, options?.efConstruction));
+    }
+    /**
+     *
+     * Create a hnswSq index
+     *
+     * HNSW-SQ stands for Hierarchical Navigable Small World - Scalar Quantization.
+     * It is a variant of the HNSW algorithm that uses scalar quantization to compress
+     * the vectors.
+     *
+     */
+    static hnswSq(options) {
+        return new Index(native_1.Index.hnswSq(options?.distanceType, options?.numPartitions, options?.maxIterations, options?.sampleRate, options?.m, options?.efConstruction));
     }
 }
 exports.Index = Index;

package/dist/native.d.ts

@@ -138,8 +138,8 @@ export class Connection {
    * - buf: The buffer containing the IPC file.
    *
    */
-  createTable(name: string, buf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, dataStorageOptions?: string | undefined | null): Promise<Table>
-  createEmptyTable(name: string, schemaBuf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, dataStorageOptions?: string | undefined | null): Promise<Table>
+  createTable(name: string, buf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, dataStorageOptions?: string | undefined | null, enableV2ManifestPaths?: boolean | undefined | null): Promise<Table>
+  createEmptyTable(name: string, schemaBuf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, dataStorageOptions?: string | undefined | null, enableV2ManifestPaths?: boolean | undefined | null): Promise<Table>
   openTable(name: string, storageOptions?: Record<string, string> | undefined | null, indexCacheSize?: number | undefined | null): Promise<Table>
   /** Drop table with the name. Or raise an error if the table does not exist. */
   dropTable(name: string): Promise<void>
@@ -149,7 +149,9 @@ export class Index {
   static btree(): Index
   static bitmap(): Index
   static labelList(): Index
-  static fts(): Index
+  static fts(withPosition?: boolean | undefined | null): Index
+  static hnswPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null, m?: number | undefined | null, efConstruction?: number | undefined | null): Index
+  static hnswSq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null, m?: number | undefined | null, efConstruction?: number | undefined | null): Index
 }
 /** Typescript-style Async Iterator over RecordBatches  */
 export class RecordBatchIterator {
@@ -168,6 +170,7 @@ export class Query {
   select(columns: Array<[string, string]>): void
   selectColumns(columns: Array<string>): void
   limit(limit: number): void
+  offset(offset: number): void
   nearestTo(vector: Float32Array): VectorQuery
   execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
   explainPlan(verbose: boolean): Promise<string>
@@ -184,6 +187,7 @@ export class VectorQuery {
   select(columns: Array<[string, string]>): void
   selectColumns(columns: Array<string>): void
   limit(limit: number): void
+  offset(offset: number): void
   execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
   explainPlan(verbose: boolean): Promise<string>
 }
@@ -198,7 +202,7 @@ export class Table {
   countRows(filter?: string | undefined | null): Promise<number>
   delete(predicate: string): Promise<void>
   createIndex(index: Index | undefined | null, column: string, replace?: boolean | undefined | null): Promise<void>
-  update(onlyIf: string | undefined | null, columns: Array<[string, string]>): Promise<void>
+  update(onlyIf: string | undefined | null, columns: Array<[string, string]>): Promise<bigint>
   query(): Query
   vectorSearch(vector: Float32Array): VectorQuery
   addColumns(transforms: Array<AddColumnsSql>): Promise<void>
@@ -212,4 +216,6 @@ export class Table {
   listIndices(): Promise<Array<IndexConfig>>
   indexStats(indexName: string): Promise<IndexStatistics | null>
   mergeInsert(on: Array<string>): NativeMergeInsertBuilder
+  usesV2ManifestPaths(): Promise<boolean>
+  migrateManifestPathsV2(): Promise<void>
 }

package/dist/query.d.ts

@@ -94,6 +94,7 @@ export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVecto
      * called then every valid row from the table will be returned.
      */
     limit(limit: number): this;
+    offset(offset: number): this;
     protected nativeExecute(options?: Partial<QueryExecutionOptions>): Promise<NativeBatchIterator>;
     /**
      * Execute the query and return the results as an @see {@link AsyncIterator}

package/dist/query.js

@@ -176,6 +176,10 @@ class QueryBase {
         this.doCall((inner) => inner.limit(limit));
         return this;
     }
+    offset(offset) {
+        this.doCall((inner) => inner.offset(offset));
+        return this;
+    }
     nativeExecute(options) {
         if (this.inner instanceof Promise) {
             return this.inner.then((inner) => inner.execute(options?.maxBatchLength));

package/dist/table.d.ts

@@ -391,4 +391,25 @@ export declare class LocalTable extends Table {
     toArrow(): Promise<ArrowTable>;
     indexStats(name: string): Promise<IndexStatistics | undefined>;
     mergeInsert(on: string | string[]): MergeInsertBuilder;
+    /**
+     * Check if the table uses the new manifest path scheme.
+     *
+     * This function will return true if the table uses the V2 manifest
+     * path scheme.
+     */
+    usesV2ManifestPaths(): Promise<boolean>;
+    /**
+     * Migrate the table to use the new manifest path scheme.
+     *
+     * This function will rename all V1 manifests to V2 manifest paths.
+     * These paths provide more efficient opening of datasets with many versions
+     * on object stores.
+     *
+     * This function is idempotent, and can be run multiple times without
+     * changing the state of the object store.
+     *
+     * However, it should not be run while other concurrent operations are happening.
+     * And it should also run until completion before resuming other operations.
+     */
+    migrateManifestPathsV2(): Promise<void>;
 }

package/dist/table.js

@@ -240,5 +240,30 @@ class LocalTable extends Table {
         on = Array.isArray(on) ? on : [on];
         return new merge_1.MergeInsertBuilder(this.inner.mergeInsert(on));
     }
+    /**
+     * Check if the table uses the new manifest path scheme.
+     *
+     * This function will return true if the table uses the V2 manifest
+     * path scheme.
+     */
+    async usesV2ManifestPaths() {
+        return await this.inner.usesV2ManifestPaths();
+    }
+    /**
+     * Migrate the table to use the new manifest path scheme.
+     *
+     * This function will rename all V1 manifests to V2 manifest paths.
+     * These paths provide more efficient opening of datasets with many versions
+     * on object stores.
+     *
+     * This function is idempotent, and can be run multiple times without
+     * changing the state of the object store.
+     *
+     * However, it should not be run while other concurrent operations are happening.
+     * And it should also run until completion before resuming other operations.
+     */
+    async migrateManifestPathsV2() {
+        await this.inner.migrateManifestPathsV2();
+    }
 }
 exports.LocalTable = LocalTable;

package/package.json

@@ -10,7 +10,7 @@
     "vector database",
     "ann"
   ],
-  "version": "0.10.0-beta.1",
+  "version": "0.11.0-beta.0",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -92,11 +92,11 @@
     "reflect-metadata": "^0.2.2"
   },
   "optionalDependencies": {
-    "@lancedb/lancedb-darwin-arm64": "0.10.0-beta.1",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.10.0-beta.1",
-    "@lancedb/lancedb-darwin-x64": "0.10.0-beta.1",
-    "@lancedb/lancedb-linux-x64-gnu": "0.10.0-beta.1",
-    "@lancedb/lancedb-win32-x64-msvc": "0.10.0-beta.1"
+    "@lancedb/lancedb-darwin-arm64": "0.11.0-beta.0",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.11.0-beta.0",
+    "@lancedb/lancedb-darwin-x64": "0.11.0-beta.0",
+    "@lancedb/lancedb-linux-x64-gnu": "0.11.0-beta.0",
+    "@lancedb/lancedb-win32-x64-msvc": "0.11.0-beta.0"
   },
   "peerDependencies": {
     "apache-arrow": ">=13.0.0 <=17.0.0"