@lancedb/lancedb 0.10.0 → 0.11.0-beta.0

package/dist/connection.d.ts

@@ -30,8 +30,8 @@ export interface CreateTableOptions {
     /**
      * The version of the data storage format to use.
      *
-     * The default is `legacy`, which is Lance format v1.
-     * `stable` is the new format, which is Lance format v2.
+     * The default is `stable`.
+     * Set to "legacy" to use the old format.
      */
     dataStorageVersion?: string;
     /**
@@ -45,9 +45,9 @@ export interface CreateTableOptions {
     /**
      * If true then data files will be written with the legacy format
      *
-     * The default is true while the new format is in beta
+     * The default is false.
      *
-     * Deprecated.
+     * Deprecated. Use data storage version instead.
      */
     useLegacyFormat?: boolean;
     schema?: SchemaLike;

package/dist/connection.js

@@ -73,7 +73,7 @@ class LocalConnection extends Connection {
             throw new Error("data is required");
         }
         const { buf, mode } = await table_1.Table.parseTableData(data, options);
-        let dataStorageVersion = "legacy";
+        let dataStorageVersion = "stable";
         if (options?.dataStorageVersion !== undefined) {
             dataStorageVersion = options.dataStorageVersion;
         }
@@ -95,7 +95,7 @@ class LocalConnection extends Connection {
             const registry = (0, registry_1.getRegistry)();
             metadata = registry.getTableMetadata([embeddingFunction]);
         }
-        let dataStorageVersion = "legacy";
+        let dataStorageVersion = "stable";
         if (options?.dataStorageVersion !== undefined) {
             dataStorageVersion = options.dataStorageVersion;
         }

package/dist/indices.d.ts

@@ -92,21 +92,206 @@ export interface IvfPqOptions {
      */
     sampleRate?: number;
 }
+/**
+ * Options to create an `HNSW_PQ` index
+ */
 export interface HnswPqOptions {
+    /**
+     * The distance metric used to train the index.
+     *
+     * Default value is "l2".
+     *
+     * The following distance types are available:
+     *
+     * "l2" - Euclidean distance. This is a very common distance metric that
+     * accounts for both magnitude and direction when determining the distance
+     * between vectors. L2 distance has a range of [0, ∞).
+     *
+     * "cosine" - Cosine distance.  Cosine distance is a distance metric
+     * calculated from the cosine similarity between two vectors. Cosine
+     * similarity is a measure of similarity between two non-zero vectors of an
+     * inner product space. It is defined to equal the cosine of the angle
+     * between them.  Unlike L2, the cosine distance is not affected by the
+     * magnitude of the vectors.  Cosine distance has a range of [0, 2].
+     *
+     * "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
+     * distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
+     * L2 norm is 1), then dot distance is equivalent to the cosine distance.
+     */
     distanceType?: "l2" | "cosine" | "dot";
+    /**
+     * The number of IVF partitions to create.
+     *
+     * For HNSW, we recommend a small number of partitions. Setting this to 1 works
+     * well for most tables. For very large tables, training just one HNSW graph
+     * will require too much memory. Each partition becomes its own HNSW graph, so
+     * setting this value higher reduces the peak memory use of training.
+     *
+     */
     numPartitions?: number;
+    /**
+     * Number of sub-vectors of PQ.
+     *
+     * This value controls how much the vector is compressed during the quantization step.
+     * The more sub vectors there are the less the vector is compressed.  The default is
+     * the dimension of the vector divided by 16.  If the dimension is not evenly divisible
+     * by 16 we use the dimension divded by 8.
+     *
+     * The above two cases are highly preferred.  Having 8 or 16 values per subvector allows
+     * us to use efficient SIMD instructions.
+     *
+     * If the dimension is not visible by 8 then we use 1 subvector.  This is not ideal and
+     * will likely result in poor performance.
+     *
+     */
     numSubVectors?: number;
+    /**
+     * Max iterations to train kmeans.
+     *
+     * The default value is 50.
+     *
+     * When training an IVF index we use kmeans to calculate the partitions.  This parameter
+     * controls how many iterations of kmeans to run.
+     *
+     * Increasing this might improve the quality of the index but in most cases the parameter
+     * is unused because kmeans will converge with fewer iterations.  The parameter is only
+     * used in cases where kmeans does not appear to converge.  In those cases it is unlikely
+     * that setting this larger will lead to the index converging anyways.
+     *
+     */
     maxIterations?: number;
+    /**
+     * The rate used to calculate the number of training vectors for kmeans.
+     *
+     * Default value is 256.
+     *
+     * When an IVF index is trained, we need to calculate partitions.  These are groups
+     * of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
+     *
+     * Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
+     * random sample of the data.  This parameter controls the size of the sample.  The total
+     * number of vectors used to train the index is `sample_rate * num_partitions`.
+     *
+     * Increasing this value might improve the quality of the index but in most cases the
+     * default should be sufficient.
+     *
+     */
     sampleRate?: number;
+    /**
+     * The number of neighbors to select for each vector in the HNSW graph.
+     *
+     * The default value is 20.
+     *
+     * This value controls the tradeoff between search speed and accuracy.
+     * The higher the value the more accurate the search but the slower it will be.
+     *
+     */
     m?: number;
+    /**
+     * The number of candidates to evaluate during the construction of the HNSW graph.
+     *
+     * The default value is 300.
+     *
+     * This value controls the tradeoff between build speed and accuracy.
+     * The higher the value the more accurate the build but the slower it will be.
+     * 150 to 300 is the typical range. 100 is a minimum for good quality search
+     * results. In most cases, there is no benefit to setting this higher than 500.
+     * This value should be set to a value that is not less than `ef` in the search phase.
+     *
+     */
     efConstruction?: number;
 }
+/**
+ * Options to create an `HNSW_SQ` index
+ */
 export interface HnswSqOptions {
+    /**
+     * The distance metric used to train the index.
+     *
+     * Default value is "l2".
+     *
+     * The following distance types are available:
+     *
+     * "l2" - Euclidean distance. This is a very common distance metric that
+     * accounts for both magnitude and direction when determining the distance
+     * between vectors. L2 distance has a range of [0, ∞).
+     *
+     * "cosine" - Cosine distance.  Cosine distance is a distance metric
+     * calculated from the cosine similarity between two vectors. Cosine
+     * similarity is a measure of similarity between two non-zero vectors of an
+     * inner product space. It is defined to equal the cosine of the angle
+     * between them.  Unlike L2, the cosine distance is not affected by the
+     * magnitude of the vectors.  Cosine distance has a range of [0, 2].
+     *
+     * "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
+     * distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
+     * L2 norm is 1), then dot distance is equivalent to the cosine distance.
+     */
     distanceType?: "l2" | "cosine" | "dot";
+    /**
+     * The number of IVF partitions to create.
+     *
+     * For HNSW, we recommend a small number of partitions. Setting this to 1 works
+     * well for most tables. For very large tables, training just one HNSW graph
+     * will require too much memory. Each partition becomes its own HNSW graph, so
+     * setting this value higher reduces the peak memory use of training.
+     *
+     */
     numPartitions?: number;
+    /**
+     * Max iterations to train kmeans.
+     *
+     * The default value is 50.
+     *
+     * When training an IVF index we use kmeans to calculate the partitions.  This parameter
+     * controls how many iterations of kmeans to run.
+     *
+     * Increasing this might improve the quality of the index but in most cases the parameter
+     * is unused because kmeans will converge with fewer iterations.  The parameter is only
+     * used in cases where kmeans does not appear to converge.  In those cases it is unlikely
+     * that setting this larger will lead to the index converging anyways.
+     *
+     */
     maxIterations?: number;
+    /**
+     * The rate used to calculate the number of training vectors for kmeans.
+     *
+     * Default value is 256.
+     *
+     * When an IVF index is trained, we need to calculate partitions.  These are groups
+     * of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
+     *
+     * Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
+     * random sample of the data.  This parameter controls the size of the sample.  The total
+     * number of vectors used to train the index is `sample_rate * num_partitions`.
+     *
+     * Increasing this value might improve the quality of the index but in most cases the
+     * default should be sufficient.
+     *
+     */
     sampleRate?: number;
+    /**
+     * The number of neighbors to select for each vector in the HNSW graph.
+     *
+     * The default value is 20.
+     *
+     * This value controls the tradeoff between search speed and accuracy.
+     * The higher the value the more accurate the search but the slower it will be.
+     *
+     */
     m?: number;
+    /**
+     * The number of candidates to evaluate during the construction of the HNSW graph.
+     *
+     * The default value is 300.
+     *
+     * This value controls the tradeoff between build speed and accuracy.
+     * The higher the value the more accurate the build but the slower it will be.
+     * 150 to 300 is the typical range. 100 is a minimum for good quality search
+     * results. In most cases, there is no benefit to setting this higher than 500.
+     * This value should be set to a value that is not less than `ef` in the search phase.
+     *
+     */
     efConstruction?: number;
 }
 /**
@@ -119,7 +304,7 @@ export interface FtsOptions {
      * If set to false, the index will not store the positions of the tokens in the text,
      * which will make the index smaller and faster to build, but will not support phrase queries.
      */
-    withPositions?: boolean;
+    withPosition?: boolean;
 }
 export declare class Index {
     private readonly inner;
@@ -199,13 +384,21 @@ export declare class Index {
     static fts(options?: Partial<FtsOptions>): Index;
     /**
      *
-     * Create a hnswpq index
+     * Create a hnswPq index
+     *
+     * HNSW-PQ stands for Hierarchical Navigable Small World - Product Quantization.
+     * It is a variant of the HNSW algorithm that uses product quantization to compress
+     * the vectors.
      *
      */
     static hnswPq(options?: Partial<HnswPqOptions>): Index;
     /**
      *
-     * Create a hnswsq index
+     * Create a hnswSq index
+     *
+     * HNSW-SQ stands for Hierarchical Navigable Small World - Scalar Quantization.
+     * It is a variant of the HNSW algorithm that uses scalar quantization to compress
+     * the vectors.
      *
      */
     static hnswSq(options?: Partial<HnswSqOptions>): Index;

package/dist/indices.js

@@ -101,11 +101,15 @@ class Index {
      * For now, the full text search index only supports English, and doesn't support phrase search.
      */
     static fts(options) {
-        return new Index(native_1.Index.fts(options?.withPositions));
+        return new Index(native_1.Index.fts(options?.withPosition));
     }
     /**
      *
-     * Create a hnswpq index
+     * Create a hnswPq index
+     *
+     * HNSW-PQ stands for Hierarchical Navigable Small World - Product Quantization.
+     * It is a variant of the HNSW algorithm that uses product quantization to compress
+     * the vectors.
      *
      */
     static hnswPq(options) {
@@ -113,7 +117,11 @@ class Index {
     }
     /**
      *
-     * Create a hnswsq index
+     * Create a hnswSq index
+     *
+     * HNSW-SQ stands for Hierarchical Navigable Small World - Scalar Quantization.
+     * It is a variant of the HNSW algorithm that uses scalar quantization to compress
+     * the vectors.
      *
      */
     static hnswSq(options) {

package/dist/native.d.ts

@@ -202,7 +202,7 @@ export class Table {
   countRows(filter?: string | undefined | null): Promise<number>
   delete(predicate: string): Promise<void>
   createIndex(index: Index | undefined | null, column: string, replace?: boolean | undefined | null): Promise<void>
-  update(onlyIf: string | undefined | null, columns: Array<[string, string]>): Promise<void>
+  update(onlyIf: string | undefined | null, columns: Array<[string, string]>): Promise<bigint>
   query(): Query
   vectorSearch(vector: Float32Array): VectorQuery
   addColumns(transforms: Array<AddColumnsSql>): Promise<void>

package/package.json

@@ -10,7 +10,7 @@
     "vector database",
     "ann"
   ],
-  "version": "0.10.0",
+  "version": "0.11.0-beta.0",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -92,11 +92,11 @@
     "reflect-metadata": "^0.2.2"
   },
   "optionalDependencies": {
-    "@lancedb/lancedb-darwin-arm64": "0.10.0",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.10.0",
-    "@lancedb/lancedb-darwin-x64": "0.10.0",
-    "@lancedb/lancedb-linux-x64-gnu": "0.10.0",
-    "@lancedb/lancedb-win32-x64-msvc": "0.10.0"
+    "@lancedb/lancedb-darwin-arm64": "0.11.0-beta.0",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.11.0-beta.0",
+    "@lancedb/lancedb-darwin-x64": "0.11.0-beta.0",
+    "@lancedb/lancedb-linux-x64-gnu": "0.11.0-beta.0",
+    "@lancedb/lancedb-win32-x64-msvc": "0.11.0-beta.0"
   },
   "peerDependencies": {
     "apache-arrow": ">=13.0.0 <=17.0.0"