@lancedb/lancedb 0.10.0-beta.1 → 0.10.0

package/dist/connection.d.ts

@@ -34,6 +34,14 @@ export interface CreateTableOptions {
      * `stable` is the new format, which is Lance format v2.
      */
     dataStorageVersion?: string;
+    /**
+     * Use the new V2 manifest paths. These paths provide more efficient
+     * opening of datasets with many versions on object stores.  WARNING:
+     * turning this on will make the dataset unreadable for older versions
+     * of LanceDB (prior to 0.10.0). To migrate an existing dataset, instead
+     * use the {@link LocalTable#migrateManifestPathsV2} method.
+     */
+    enableV2ManifestPaths?: boolean;
     /**
      * If true then data files will be written with the legacy format
      *

package/dist/connection.js

@@ -80,7 +80,7 @@ class LocalConnection extends Connection {
         else if (options?.useLegacyFormat !== undefined) {
             dataStorageVersion = options.useLegacyFormat ? "legacy" : "stable";
         }
-        const innerTable = await this.inner.createTable(nameOrOptions, buf, mode, cleanseStorageOptions(options?.storageOptions), dataStorageVersion);
+        const innerTable = await this.inner.createTable(nameOrOptions, buf, mode, cleanseStorageOptions(options?.storageOptions), dataStorageVersion, options?.enableV2ManifestPaths);
         return new table_1.LocalTable(innerTable);
     }
     async createEmptyTable(name, schema, options) {
@@ -104,7 +104,7 @@ class LocalConnection extends Connection {
         }
         const table = (0, arrow_1.makeEmptyTable)(schema, metadata);
         const buf = await (0, arrow_1.fromTableToBuffer)(table);
-        const innerTable = await this.inner.createEmptyTable(name, buf, mode, cleanseStorageOptions(options?.storageOptions), dataStorageVersion);
+        const innerTable = await this.inner.createEmptyTable(name, buf, mode, cleanseStorageOptions(options?.storageOptions), dataStorageVersion, options?.enableV2ManifestPaths);
         return new table_1.LocalTable(innerTable);
     }
     async dropTable(name) {

package/dist/indices.d.ts

@@ -92,6 +92,35 @@ export interface IvfPqOptions {
      */
     sampleRate?: number;
 }
+export interface HnswPqOptions {
+    distanceType?: "l2" | "cosine" | "dot";
+    numPartitions?: number;
+    numSubVectors?: number;
+    maxIterations?: number;
+    sampleRate?: number;
+    m?: number;
+    efConstruction?: number;
+}
+export interface HnswSqOptions {
+    distanceType?: "l2" | "cosine" | "dot";
+    numPartitions?: number;
+    maxIterations?: number;
+    sampleRate?: number;
+    m?: number;
+    efConstruction?: number;
+}
+/**
+ * Options to create a full text search index
+ */
+export interface FtsOptions {
+    /**
+     * Whether to build the index with positions.
+     * True by default.
+     * If set to false, the index will not store the positions of the tokens in the text,
+     * which will make the index smaller and faster to build, but will not support phrase queries.
+     */
+    withPositions?: boolean;
+}
 export declare class Index {
     private readonly inner;
     private constructor();
@@ -167,7 +196,19 @@ export declare class Index {
      *
      * For now, the full text search index only supports English, and doesn't support phrase search.
      */
-    static fts(): Index;
+    static fts(options?: Partial<FtsOptions>): Index;
+    /**
+     *
+     * Create a hnswpq index
+     *
+     */
+    static hnswPq(options?: Partial<HnswPqOptions>): Index;
+    /**
+     *
+     * Create a hnswsq index
+     *
+     */
+    static hnswSq(options?: Partial<HnswSqOptions>): Index;
 }
 export interface IndexOptions {
     /**

package/dist/indices.js

@@ -100,8 +100,24 @@ class Index {
      *
      * For now, the full text search index only supports English, and doesn't support phrase search.
      */
-    static fts() {
-        return new Index(native_1.Index.fts());
+    static fts(options) {
+        return new Index(native_1.Index.fts(options?.withPositions));
+    }
+    /**
+     *
+     * Create a hnswpq index
+     *
+     */
+    static hnswPq(options) {
+        return new Index(native_1.Index.hnswPq(options?.distanceType, options?.numPartitions, options?.numSubVectors, options?.maxIterations, options?.sampleRate, options?.m, options?.efConstruction));
+    }
+    /**
+     *
+     * Create a hnswsq index
+     *
+     */
+    static hnswSq(options) {
+        return new Index(native_1.Index.hnswSq(options?.distanceType, options?.numPartitions, options?.maxIterations, options?.sampleRate, options?.m, options?.efConstruction));
     }
 }
 exports.Index = Index;

package/dist/native.d.ts

@@ -138,8 +138,8 @@ export class Connection {
    * - buf: The buffer containing the IPC file.
    *
    */
-  createTable(name: string, buf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, dataStorageOptions?: string | undefined | null): Promise<Table>
-  createEmptyTable(name: string, schemaBuf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, dataStorageOptions?: string | undefined | null): Promise<Table>
+  createTable(name: string, buf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, dataStorageOptions?: string | undefined | null, enableV2ManifestPaths?: boolean | undefined | null): Promise<Table>
+  createEmptyTable(name: string, schemaBuf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, dataStorageOptions?: string | undefined | null, enableV2ManifestPaths?: boolean | undefined | null): Promise<Table>
   openTable(name: string, storageOptions?: Record<string, string> | undefined | null, indexCacheSize?: number | undefined | null): Promise<Table>
   /** Drop table with the name. Or raise an error if the table does not exist. */
   dropTable(name: string): Promise<void>
@@ -149,7 +149,9 @@ export class Index {
   static btree(): Index
   static bitmap(): Index
   static labelList(): Index
-  static fts(): Index
+  static fts(withPosition?: boolean | undefined | null): Index
+  static hnswPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null, m?: number | undefined | null, efConstruction?: number | undefined | null): Index
+  static hnswSq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null, m?: number | undefined | null, efConstruction?: number | undefined | null): Index
 }
 /** Typescript-style Async Iterator over RecordBatches  */
 export class RecordBatchIterator {
@@ -168,6 +170,7 @@ export class Query {
   select(columns: Array<[string, string]>): void
   selectColumns(columns: Array<string>): void
   limit(limit: number): void
+  offset(offset: number): void
   nearestTo(vector: Float32Array): VectorQuery
   execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
   explainPlan(verbose: boolean): Promise<string>
@@ -184,6 +187,7 @@ export class VectorQuery {
   select(columns: Array<[string, string]>): void
   selectColumns(columns: Array<string>): void
   limit(limit: number): void
+  offset(offset: number): void
   execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
   explainPlan(verbose: boolean): Promise<string>
 }
@@ -212,4 +216,6 @@ export class Table {
   listIndices(): Promise<Array<IndexConfig>>
   indexStats(indexName: string): Promise<IndexStatistics | null>
   mergeInsert(on: Array<string>): NativeMergeInsertBuilder
+  usesV2ManifestPaths(): Promise<boolean>
+  migrateManifestPathsV2(): Promise<void>
 }

package/dist/query.d.ts

@@ -94,6 +94,7 @@ export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVecto
      * called then every valid row from the table will be returned.
      */
     limit(limit: number): this;
+    offset(offset: number): this;
     protected nativeExecute(options?: Partial<QueryExecutionOptions>): Promise<NativeBatchIterator>;
     /**
      * Execute the query and return the results as an @see {@link AsyncIterator}

package/dist/query.js

@@ -176,6 +176,10 @@ class QueryBase {
         this.doCall((inner) => inner.limit(limit));
         return this;
     }
+    offset(offset) {
+        this.doCall((inner) => inner.offset(offset));
+        return this;
+    }
     nativeExecute(options) {
         if (this.inner instanceof Promise) {
             return this.inner.then((inner) => inner.execute(options?.maxBatchLength));

package/dist/table.d.ts

@@ -391,4 +391,25 @@ export declare class LocalTable extends Table {
     toArrow(): Promise<ArrowTable>;
     indexStats(name: string): Promise<IndexStatistics | undefined>;
     mergeInsert(on: string | string[]): MergeInsertBuilder;
+    /**
+     * Check if the table uses the new manifest path scheme.
+     *
+     * This function will return true if the table uses the V2 manifest
+     * path scheme.
+     */
+    usesV2ManifestPaths(): Promise<boolean>;
+    /**
+     * Migrate the table to use the new manifest path scheme.
+     *
+     * This function will rename all V1 manifests to V2 manifest paths.
+     * These paths provide more efficient opening of datasets with many versions
+     * on object stores.
+     *
+     * This function is idempotent, and can be run multiple times without
+     * changing the state of the object store.
+     *
+     * However, it should not be run while other concurrent operations are happening.
+     * And it should also run until completion before resuming other operations.
+     */
+    migrateManifestPathsV2(): Promise<void>;
 }

package/dist/table.js

@@ -240,5 +240,30 @@ class LocalTable extends Table {
         on = Array.isArray(on) ? on : [on];
         return new merge_1.MergeInsertBuilder(this.inner.mergeInsert(on));
     }
+    /**
+     * Check if the table uses the new manifest path scheme.
+     *
+     * This function will return true if the table uses the V2 manifest
+     * path scheme.
+     */
+    async usesV2ManifestPaths() {
+        return await this.inner.usesV2ManifestPaths();
+    }
+    /**
+     * Migrate the table to use the new manifest path scheme.
+     *
+     * This function will rename all V1 manifests to V2 manifest paths.
+     * These paths provide more efficient opening of datasets with many versions
+     * on object stores.
+     *
+     * This function is idempotent, and can be run multiple times without
+     * changing the state of the object store.
+     *
+     * However, it should not be run while other concurrent operations are happening.
+     * And it should also run until completion before resuming other operations.
+     */
+    async migrateManifestPathsV2() {
+        await this.inner.migrateManifestPathsV2();
+    }
 }
 exports.LocalTable = LocalTable;

package/package.json

@@ -10,7 +10,7 @@
     "vector database",
     "ann"
   ],
-  "version": "0.10.0-beta.1",
+  "version": "0.10.0",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -92,11 +92,11 @@
     "reflect-metadata": "^0.2.2"
   },
   "optionalDependencies": {
-    "@lancedb/lancedb-darwin-arm64": "0.10.0-beta.1",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.10.0-beta.1",
-    "@lancedb/lancedb-darwin-x64": "0.10.0-beta.1",
-    "@lancedb/lancedb-linux-x64-gnu": "0.10.0-beta.1",
-    "@lancedb/lancedb-win32-x64-msvc": "0.10.0-beta.1"
+    "@lancedb/lancedb-darwin-arm64": "0.10.0",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.10.0",
+    "@lancedb/lancedb-darwin-x64": "0.10.0",
+    "@lancedb/lancedb-linux-x64-gnu": "0.10.0",
+    "@lancedb/lancedb-win32-x64-msvc": "0.10.0"
   },
   "peerDependencies": {
     "apache-arrow": ">=13.0.0 <=17.0.0"