@lancedb/lancedb 0.33.0 → 0.37.1-beta.1

package/CONTRIBUTING.md

@@ -1,7 +1,7 @@
 # Contributing to LanceDB Typescript
 
 This document outlines the process for contributing to LanceDB Typescript.
-For general contribution guidelines, see [CONTRIBUTING.md](../CONTRIBUTING.md).
+For general contribution guidelines, see [CONTRIBUTING.md](https://github.com/lancedb/lancedb/blob/main/CONTRIBUTING.md).
 
 ## Project layout
 

package/dist/connection.d.ts

@@ -1,7 +1,8 @@
 import { Data, SchemaLike, TableLike } from "./arrow";
+import { Table as ArrowTable } from "./arrow";
 import { EmbeddingFunctionConfig } from "./embedding/registry";
 import { Connection as LanceDbConnection } from "./native";
-import type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse, ListNamespacesResponse } from "./native";
+import type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse, Job, JobDescription, JobInfo, ListNamespacesResponse } from "./native";
 export type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse, ListNamespacesResponse, };
 import { Table } from "./table";
 export interface CreateTableOptions {
@@ -333,6 +334,35 @@ export declare abstract class Connection {
      *   `newNamespacePath` is omitted the table stays in `namespacePath`.
      */
     abstract renameTable(currentName: string, newName: string, options?: RenameTableOptions): Promise<void>;
+    /**
+     * A {@link Job} handle for a server-side job by id.
+     *
+     * The handle is constructed without a server round trip; an unknown id
+     * surfaces when the handle is used. Dropping the handle has no effect on
+     * the job itself.
+     */
+    abstract job(jobId: string): Job;
+    /** List server-side jobs across the database's tables. */
+    abstract listJobs(): Promise<JobInfo[]>;
+    /**
+     * Describe a single server-side job by id.
+     *
+     * Resolves to `null` when the server has no such job.
+     */
+    abstract getJob(jobId: string): Promise<JobDescription | null>;
+    /**
+     * Request cancellation of a server-side job by id.
+     *
+     * Resolves to true if the server accepted the cancellation, false if no
+     * such job exists. Cancelling an already-terminal job is a no-op success.
+     */
+    abstract cancelJob(jobId: string): Promise<boolean>;
+    /**
+     * The lifecycle event history of a server-side job, as an Arrow table.
+     *
+     * Lists history across all jobs when `jobId` is omitted.
+     */
+    abstract jobHistory(jobId?: string): Promise<ArrowTable>;
 }
 /** @hideconstructor */
 export declare class LocalConnection extends Connection {
@@ -364,6 +394,11 @@ export declare class LocalConnection extends Connection {
     createNamespace(namespacePath: string[], options?: Partial<CreateNamespaceOptions>): Promise<CreateNamespaceResponse>;
     dropNamespace(namespacePath: string[], options?: Partial<DropNamespaceOptions>): Promise<DropNamespaceResponse>;
     renameTable(currentName: string, newName: string, options?: RenameTableOptions): Promise<void>;
+    job(jobId: string): Job;
+    listJobs(): Promise<JobInfo[]>;
+    getJob(jobId: string): Promise<JobDescription | null>;
+    cancelJob(jobId: string): Promise<boolean>;
+    jobHistory(jobId?: string): Promise<ArrowTable>;
 }
 /**
  * Takes storage options and makes all the keys snake case.

package/dist/connection.js

@@ -4,6 +4,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LocalConnection = exports.Connection = void 0;
 exports.cleanseStorageOptions = cleanseStorageOptions;
+const apache_arrow_1 = require("apache-arrow");
 const arrow_1 = require("./arrow");
 const arrow_2 = require("./arrow");
 const registry_1 = require("./embedding/registry");
@@ -191,6 +192,25 @@ class LocalConnection extends Connection {
     async renameTable(currentName, newName, options) {
         return this.inner.renameTable(currentName, newName, options?.namespacePath ?? [], options?.newNamespacePath);
     }
+    job(jobId) {
+        return this.inner.job(jobId);
+    }
+    async listJobs() {
+        return this.inner.listJobs();
+    }
+    async getJob(jobId) {
+        return this.inner.getJob(jobId);
+    }
+    async cancelJob(jobId) {
+        return this.inner.cancelJob(jobId);
+    }
+    async jobHistory(jobId) {
+        const buf = await this.inner.jobHistory(jobId);
+        if (buf.length === 0) {
+            return new arrow_2.Table();
+        }
+        return (0, apache_arrow_1.tableFromIPC)(buf);
+    }
 }
 exports.LocalConnection = LocalConnection;
 /**

package/dist/index.d.ts

@@ -8,7 +8,7 @@ export { instrumentLanceDbMetrics } from "./otel";
 export { AddColumnsSql, ConnectionOptions, ConnectNamespaceOptions, IndexStatistics, IndexConfig, ClientConfig, TimeoutConfig, RetryConfig, TlsConfig, OptimizeStats, CompactionStats, RemovalStats, TableStatistics, FragmentStatistics, FragmentSummaryStats, Tags, TagContents, BranchContents, MergeResult, AddResult, AddColumnsResult, AlterColumnsResult, UpdateFieldMetadataResult, DeleteResult, DropColumnsResult, UpdateResult, SplitCalculatedOptions, SplitRandomOptions, SplitHashOptions, SplitSequentialOptions, ShuffleOptions, OAuthConfig as NativeOAuthConfig, } from "./native.js";
 export { makeArrowTable, MakeArrowTableOptions, Data, VectorColumnOptions, } from "./arrow";
 export { Connection, CreateTableOptions, TableNamesOptions, OpenTableOptions, ListNamespacesOptions, CreateNamespaceOptions, DropNamespaceOptions, ListNamespacesResponse, CreateNamespaceResponse, DropNamespaceResponse, DescribeNamespaceResponse, RenameTableOptions, } from "./connection";
-export { Session } from "./native.js";
+export { Job, JobDescription, JobFailureInfo, JobInfo, Session, } from "./native.js";
 export { ExecutableQuery, Query, QueryBase, VectorQuery, TakeQuery, AnalyzePlanDistributedMetrics, QueryExecutionOptions, ColumnOrdering, FullTextSearchOptions, RecordBatchIterator, FullTextQuery, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, FullTextQueryType, Operator, Occur, } from "./query";
 export { Index, IndexOptions, IvfPqOptions, IvfRqOptions, IvfFlatOptions, HnswPqOptions, HnswSqOptions, FtsOptions, BaseTokenizer, } from "./indices";
 export { Table, Branches, BranchColumnSummary, BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, MergeBlocker, BranchDiff, MergePreview, MergeBranchResult, AddDataOptions, UpdateOptions, OptimizeOptions, Version, WriteProgress, FtsToken, TokenizeTableOptions, LsmWriteSpec, ColumnAlteration, FieldMetadataUpdate, } from "./table";
@@ -39,6 +39,15 @@ export interface TokenizeOptions {
     stem?: boolean;
     /** Whether to remove stop words. */
     removeStopWords?: boolean;
+    /**
+     * Custom stop words that replace the built-in list for `language`.
+     *
+     * This option only affects tokenization when `removeStopWords` is true.
+     *
+     * `undefined` keeps the built-in language list. An empty array explicitly
+     * replaces it with no stop words.
+     */
+    customStopWords?: string[];
     /** Whether to fold ASCII characters. */
     asciiFolding?: boolean;
     /** N-gram minimum length. */

package/dist/index.js

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthFlowType = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Session = exports.Connection = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = exports.BranchContents = exports.TagContents = exports.Tags = exports.instrumentLanceDbMetrics = exports.NativeJsHeaderProvider = void 0;
+exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthFlowType = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Session = exports.Job = exports.Connection = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = exports.BranchContents = exports.TagContents = exports.Tags = exports.instrumentLanceDbMetrics = exports.NativeJsHeaderProvider = void 0;
 exports.tokenize = tokenize;
 exports.connect = connect;
 exports.connectNamespace = connectNamespace;
@@ -27,6 +27,7 @@ Object.defineProperty(exports, "VectorColumnOptions", { enumerable: true, get: f
 var connection_2 = require("./connection");
 Object.defineProperty(exports, "Connection", { enumerable: true, get: function () { return connection_2.Connection; } });
 var native_js_4 = require("./native.js");
+Object.defineProperty(exports, "Job", { enumerable: true, get: function () { return native_js_4.Job; } });
 Object.defineProperty(exports, "Session", { enumerable: true, get: function () { return native_js_4.Session; } });
 var query_1 = require("./query");
 Object.defineProperty(exports, "Query", { enumerable: true, get: function () { return query_1.Query; } });
@@ -71,7 +72,7 @@ Object.defineProperty(exports, "packBits", { enumerable: true, get: function ()
  * {@link Index.fts}.
  */
 async function tokenize(query, options) {
-    return await (0, native_js_1.tokenize)(query, options?.baseTokenizer, options?.language, options?.maxTokenLength, options?.lowercase, options?.stem, options?.removeStopWords, options?.asciiFolding, options?.ngramMinLength, options?.ngramMaxLength, options?.prefixOnly);
+    return await (0, native_js_1.tokenize)(query, options?.baseTokenizer, options?.language, options?.maxTokenLength, options?.lowercase, options?.stem, options?.removeStopWords, options?.customStopWords, options?.asciiFolding, options?.ngramMinLength, options?.ngramMaxLength, options?.prefixOnly);
 }
 async function connect(uriOrOptions, optionsOrSession, sessionOrHeaderProvider, headerProvider) {
     let uri;

package/dist/indices.d.ts

@@ -504,6 +504,15 @@ export interface FtsOptions {
      * whether to remove stop words
      */
     removeStopWords?: boolean;
+    /**
+     * Custom stop words that replace the built-in list for `language`.
+     *
+     * This option only affects tokenization when `removeStopWords` is true.
+     *
+     * `undefined` keeps the built-in language list. An empty array explicitly
+     * replaces it with no stop words.
+     */
+    customStopWords?: string[];
     /**
      * whether to remove punctuation
      */
@@ -520,6 +529,13 @@ export interface FtsOptions {
      * whether to only index the prefix of the token for ngram tokenizer
      */
     prefixOnly?: boolean;
+    /**
+     * Number of documents per compressed posting block.
+     *
+     * The default is 128. Supported values are 128 and 256. A value of 256 uses
+     * the experimental FTS V3 format and may introduce breaking changes.
+     */
+    blockSize?: 128 | 256;
 }
 export declare class Index {
     private readonly inner;

package/dist/indices.js

@@ -136,7 +136,7 @@ class Index {
      * You can combine filters with full text search.
      */
     static fts(options) {
-        return new Index(native_1.Index.fts(options?.withPosition, options?.baseTokenizer, options?.language, options?.maxTokenLength, options?.lowercase, options?.stem, options?.removeStopWords, options?.asciiFolding, options?.ngramMinLength, options?.ngramMaxLength, options?.prefixOnly));
+        return new Index(native_1.Index.fts(options?.withPosition, options?.baseTokenizer, options?.language, options?.maxTokenLength, options?.lowercase, options?.stem, options?.removeStopWords, options?.customStopWords, options?.asciiFolding, options?.ngramMinLength, options?.ngramMaxLength, options?.prefixOnly, options?.blockSize));
     }
     /**
      *

package/dist/merge.d.ts

@@ -57,17 +57,16 @@ export declare class MergeInsertBuilder {
      */
     useIndex(useIndex: boolean): MergeInsertBuilder;
     /**
-     * Controls whether the merge uses the MemWAL LSM write path.
+     * Control MemWAL routing for this merge.
      *
      * By default (unset), a `mergeInsert` on a table with an LSM write spec is
-     * routed through Lance's MemWAL shard writer, and a table without one uses
-     * the standard path. Pass `false` to force the standard path even when a
-     * spec is set. Pass `true` to require a spec — `mergeInsert` rejects if none
-     * is installed.
+     * routed through Lance's MemWAL shard writer, and a table without one uses the
+     * standard path.
      *
-     * @param useLsmWrite - Whether to use the LSM write path.
+     * @param enable - `true` forces MemWAL routing and errors if the table has no
+     * LSM write spec. `false` forces the standard write path even when a spec is set.
      */
-    useLsmWrite(useLsmWrite: boolean): MergeInsertBuilder;
+    useLsm(enable: boolean): MergeInsertBuilder;
     /**
      * Controls how an LSM merge checks that its input targets a single shard.
      *

package/dist/merge.js

@@ -69,18 +69,17 @@ class MergeInsertBuilder {
         return new MergeInsertBuilder(this.#native.useIndex(useIndex), this.#schema);
     }
     /**
-     * Controls whether the merge uses the MemWAL LSM write path.
+     * Control MemWAL routing for this merge.
      *
      * By default (unset), a `mergeInsert` on a table with an LSM write spec is
-     * routed through Lance's MemWAL shard writer, and a table without one uses
-     * the standard path. Pass `false` to force the standard path even when a
-     * spec is set. Pass `true` to require a spec — `mergeInsert` rejects if none
-     * is installed.
+     * routed through Lance's MemWAL shard writer, and a table without one uses the
+     * standard path.
      *
-     * @param useLsmWrite - Whether to use the LSM write path.
+     * @param enable - `true` forces MemWAL routing and errors if the table has no
+     * LSM write spec. `false` forces the standard write path even when a spec is set.
      */
-    useLsmWrite(useLsmWrite) {
-        return new MergeInsertBuilder(this.#native.useLsmWrite(useLsmWrite), this.#schema);
+    useLsm(enable) {
+        return new MergeInsertBuilder(this.#native.useLsm(enable), this.#schema);
     }
     /**
      * Controls how an LSM merge checks that its input targets a single shard.

package/dist/native.d.ts

@@ -39,6 +39,31 @@ export declare class Connection {
   /** Drop table with the name. Or raise an error if the table does not exist. */
   dropTable(name: string, namespacePath?: Array<string> | undefined | null): Promise<void>
   dropAllTables(namespacePath?: Array<string> | undefined | null): Promise<void>
+  /**
+   * A `Job` handle for a server-side job by id.
+   *
+   * The handle is constructed without a server round trip; an unknown id
+   * surfaces when the handle is used.
+   */
+  job(jobId: string): Job
+  /** List server-side jobs across the database's tables. */
+  listJobs(): Promise<Array<JobInfo>>
+  /**
+   * Describe a single server-side job by id. `null` when the server has
+   * no such job.
+   */
+  getJob(jobId: string): Promise<JobDescription | null>
+  /**
+   * Request cancellation of a server-side job by id. Returns true if the
+   * server accepted the cancellation, false if no such job exists.
+   */
+  cancelJob(jobId: string): Promise<boolean>
+  /**
+   * The lifecycle event history of a server-side job (all jobs when
+   * `job_id` is null), as an Arrow IPC stream buffer. Empty when there is
+   * no history.
+   */
+  jobHistory(jobId?: string | undefined | null): Promise<Buffer>
   /** Describe a namespace and return its properties. */
   describeNamespace(namespacePath: Array<string>): Promise<DescribeNamespaceResponse>
   /** List child namespaces under the given namespace path */
@@ -63,11 +88,33 @@ export declare class Index {
   static bitmap(): Index
   static labelList(): Index
   static fm(): Index
-  static fts(withPosition?: boolean | undefined | null, baseTokenizer?: string | undefined | null, language?: string | undefined | null, maxTokenLength?: number | undefined | null, lowerCase?: boolean | undefined | null, stem?: boolean | undefined | null, removeStopWords?: boolean | undefined | null, asciiFolding?: boolean | undefined | null, ngramMinLength?: number | undefined | null, ngramMaxLength?: number | undefined | null, prefixOnly?: boolean | undefined | null): Index
+  static fts(withPosition?: boolean | undefined | null, baseTokenizer?: string | undefined | null, language?: string | undefined | null, maxTokenLength?: number | undefined | null, lowerCase?: boolean | undefined | null, stem?: boolean | undefined | null, removeStopWords?: boolean | undefined | null, customStopWords?: Array<string> | undefined | null, asciiFolding?: boolean | undefined | null, ngramMinLength?: number | undefined | null, ngramMaxLength?: number | undefined | null, prefixOnly?: boolean | undefined | null, blockSize?: number | undefined | null): Index
   static hnswPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null, m?: number | undefined | null, efConstruction?: number | undefined | null): Index
   static hnswSq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null, m?: number | undefined | null, efConstruction?: number | undefined | null): Index
 }
 
+/** A handle to an operation that may still be running. */
+export declare class Job {
+  /**
+   * Identifies the operation on the server that is running it. Operations
+   * that run in this process have no server id. The value is opaque.
+   */
+  get id(): string | null
+  /**
+   * The operation's current lifecycle state: "running", "finished",
+   * "failed", or "cancelled".
+   *
+   * A point snapshot; unlike {@link Job.wait} it does not block or reject
+   * on a terminal failure state. States a newer server reports that this
+   * client version does not know pass through as-is.
+   */
+  status(): Promise<string>
+  /** Wait until the operation reaches a terminal state. */
+  wait(): Promise<void>
+  /** Request cancellation. Cancelling a finished operation is a no-op. */
+  cancel(): Promise<void>
+}
+
 export declare class JsFullTextQuery {
   static matchQuery(query: string, column: string, boost: number, fuzziness: number | undefined | null, maxExpansions: number, operator: string, prefixLength: number): JsFullTextQuery
   static phraseQuery(query: string, column: string, slop: number): JsFullTextQuery
@@ -118,7 +165,7 @@ export declare class NativeMergeInsertBuilder {
   whenNotMatchedBySourceDelete(filter?: string | undefined | null): NativeMergeInsertBuilder
   setTimeout(timeout: number): void
   useIndex(useIndex: boolean): NativeMergeInsertBuilder
-  useLsmWrite(useLsmWrite: boolean): NativeMergeInsertBuilder
+  useLsm(enable: boolean): NativeMergeInsertBuilder
   validateSingleShard(validateSingleShard: boolean): NativeMergeInsertBuilder
   execute(buf: Buffer): Promise<MergeResult>
 }
@@ -152,6 +199,7 @@ export declare class Query {
   nearestToRaw(data: Uint8Array, dtype: string): VectorQuery
   fastSearch(): void
   withRowId(): void
+  useLsm(enable: boolean): void
   orderBy(ordering?: Array<ColumnOrdering> | undefined | null): void
   outputSchema(): Promise<Buffer>
   execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null): Promise<RecordBatchIterator>
@@ -217,6 +265,7 @@ export declare class Table {
   countRows(filter?: string | undefined | null): Promise<number>
   delete(predicate: string): Promise<DeleteResult>
   createIndex(index: Index | undefined | null, column: string, replace?: boolean | undefined | null, waitTimeoutS?: number | undefined | null, name?: string | undefined | null, train?: boolean | undefined | null): Promise<void>
+  createIndexAsync(index: Index | undefined | null, column: string, replace?: boolean | undefined | null, waitTimeoutS?: number | undefined | null, name?: string | undefined | null, train?: boolean | undefined | null): Promise<Job>
   dropIndex(indexName: string): Promise<void>
   prewarmIndex(indexName: string): Promise<void>
   prewarmData(columns?: Array<string> | undefined | null): Promise<void>
@@ -275,6 +324,7 @@ export declare class TakeQuery {
   select(columns: Array<[string, string]>): void
   selectColumns(columns: Array<string>): void
   withRowId(): void
+  useLsm(enable: boolean): void
   outputSchema(): Promise<Buffer>
   execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null): Promise<RecordBatchIterator>
   explainPlan(verbose: boolean): Promise<string>
@@ -302,6 +352,7 @@ export declare class VectorQuery {
   offset(offset: number): void
   fastSearch(): void
   withRowId(): void
+  useLsm(enable: boolean): void
   rerank(rerankHybrid: (arg: RerankHybridCallbackArgs) => Promise<Buffer>): void
   orderBy(ordering?: Array<ColumnOrdering> | undefined | null): void
   outputSchema(): Promise<Buffer>
@@ -652,6 +703,46 @@ export interface IndexStatistics {
   numIndices?: number
 }
 
+/** A described job from `Connection.getJob`. */
+export interface JobDescription {
+  jobId: string
+  jobType: string
+  /** Lifecycle state: "running", "finished", "failed", or "cancelled". */
+  state: string
+  /** When the job was created, in milliseconds since the epoch. */
+  creationMs: number
+  /** The job-type-specific specification as a JSON string, when present. */
+  specJson?: string
+  /**
+   * Why the job failed, when the job is failed and the server reports a
+   * reason.
+   */
+  failure?: JobFailureInfo
+}
+
+/** The server's account of why a job failed. */
+export interface JobFailureInfo {
+  phase?: string
+  message?: string
+  retryable?: boolean
+}
+
+/** A row from `Connection.listJobs`: one server-side job. */
+export interface JobInfo {
+  /**
+   * The job id -- what `Connection.getJob` and `Connection.cancelJob`
+   * accept.
+   */
+  jobId: string
+  /** The table the job runs against, without URI or namespace. */
+  table: string
+  jobType: string
+  /** Lifecycle state: "running", "finished", "failed", or "cancelled". */
+  state: string
+  /** When the job was created, in milliseconds since the epoch. */
+  createdAtMillis: number
+}
+
 /** The catalog of described LanceDB metrics. Empty until the recorder is installed. */
 export declare function lancedbMetricsCatalog(): Array<MetricDescription>
 
@@ -675,7 +766,10 @@ export interface LsmWriteSpec {
   column?: string
   /** Bucket variant: the number of buckets, in `[1, 1024]`. */
   numBuckets?: number
-  /** Names of indexes the MemWAL should keep up to date during writes. */
+  /**
+   * Indexes the MemWAL keeps up to date. Omitted resolves every
+   * maintainable index on install; an empty array means none.
+   */
   maintainedIndexes?: Array<string>
   /** Default `ShardWriter` configuration recorded in the MemWAL index. */
   writerConfigDefaults?: Record<string, string>
@@ -936,7 +1030,7 @@ export interface TlsConfig {
   assertHostname?: boolean
 }
 
-export declare function tokenize(query: string, baseTokenizer?: string | undefined | null, language?: string | undefined | null, maxTokenLength?: number | undefined | null, lowerCase?: boolean | undefined | null, stem?: boolean | undefined | null, removeStopWords?: boolean | undefined | null, asciiFolding?: boolean | undefined | null, ngramMinLength?: number | undefined | null, ngramMaxLength?: number | undefined | null, prefixOnly?: boolean | undefined | null): Array<FtsToken>
+export declare function tokenize(query: string, baseTokenizer?: string | undefined | null, language?: string | undefined | null, maxTokenLength?: number | undefined | null, lowerCase?: boolean | undefined | null, stem?: boolean | undefined | null, removeStopWords?: boolean | undefined | null, customStopWords?: Array<string> | undefined | null, asciiFolding?: boolean | undefined | null, ngramMinLength?: number | undefined | null, ngramMaxLength?: number | undefined | null, prefixOnly?: boolean | undefined | null): Array<FtsToken>
 
 export interface UpdateFieldMetadataResult {
   version: number

package/dist/native.js

@@ -76,8 +76,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-android-arm64');
                 const bindingPackageVersion = require('@lancedb/lancedb-android-arm64/package.json').version;
-                if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -95,8 +95,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-android-arm-eabi');
                 const bindingPackageVersion = require('@lancedb/lancedb-android-arm-eabi/package.json').version;
-                if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -120,8 +120,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-win32-x64-gnu');
                     const bindingPackageVersion = require('@lancedb/lancedb-win32-x64-gnu/package.json').version;
-                    if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -139,8 +139,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-win32-x64-msvc');
                     const bindingPackageVersion = require('@lancedb/lancedb-win32-x64-msvc/package.json').version;
-                    if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -159,8 +159,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-win32-ia32-msvc');
                 const bindingPackageVersion = require('@lancedb/lancedb-win32-ia32-msvc/package.json').version;
-                if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -178,8 +178,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-win32-arm64-msvc');
                 const bindingPackageVersion = require('@lancedb/lancedb-win32-arm64-msvc/package.json').version;
-                if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -201,8 +201,8 @@ function requireNative() {
         try {
             const binding = require('@lancedb/lancedb-darwin-universal');
             const bindingPackageVersion = require('@lancedb/lancedb-darwin-universal/package.json').version;
-            if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+            if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
             }
             return binding;
         }
@@ -219,8 +219,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-darwin-x64');
                 const bindingPackageVersion = require('@lancedb/lancedb-darwin-x64/package.json').version;
-                if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -238,8 +238,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-darwin-arm64');
                 const bindingPackageVersion = require('@lancedb/lancedb-darwin-arm64/package.json').version;
-                if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -262,8 +262,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-freebsd-x64');
                 const bindingPackageVersion = require('@lancedb/lancedb-freebsd-x64/package.json').version;
-                if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -281,8 +281,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-freebsd-arm64');
                 const bindingPackageVersion = require('@lancedb/lancedb-freebsd-arm64/package.json').version;
-                if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -306,8 +306,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-x64-musl');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-x64-musl/package.json').version;
-                    if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -325,8 +325,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-x64-gnu');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-x64-gnu/package.json').version;
-                    if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -346,8 +346,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-arm64-musl');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-arm64-musl/package.json').version;
-                    if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -365,8 +365,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-arm64-gnu');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-arm64-gnu/package.json').version;
-                    if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -386,8 +386,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-arm-musleabihf');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-arm-musleabihf/package.json').version;
-                    if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -405,8 +405,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-arm-gnueabihf');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-arm-gnueabihf/package.json').version;
-                    if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -426,8 +426,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-loong64-musl');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-loong64-musl/package.json').version;
-                    if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -445,8 +445,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-loong64-gnu');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-loong64-gnu/package.json').version;
-                    if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -466,8 +466,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-riscv64-musl');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-riscv64-musl/package.json').version;
-                    if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -485,8 +485,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-riscv64-gnu');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-riscv64-gnu/package.json').version;
-                    if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -505,8 +505,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-linux-ppc64-gnu');
                 const bindingPackageVersion = require('@lancedb/lancedb-linux-ppc64-gnu/package.json').version;
-                if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -524,8 +524,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-linux-s390x-gnu');
                 const bindingPackageVersion = require('@lancedb/lancedb-linux-s390x-gnu/package.json').version;
-                if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -548,8 +548,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-openharmony-arm64');
                 const bindingPackageVersion = require('@lancedb/lancedb-openharmony-arm64/package.json').version;
-                if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -567,8 +567,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-openharmony-x64');
                 const bindingPackageVersion = require('@lancedb/lancedb-openharmony-x64/package.json').version;
-                if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -586,8 +586,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-openharmony-arm');
                 const bindingPackageVersion = require('@lancedb/lancedb-openharmony-arm/package.json').version;
-                if (bindingPackageVersion !== '0.33.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.33.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.37.1-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.37.1-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -665,6 +665,7 @@ module.exports.BranchContents = nativeBinding.BranchContents;
 module.exports.Branches = nativeBinding.Branches;
 module.exports.Connection = nativeBinding.Connection;
 module.exports.Index = nativeBinding.Index;
+module.exports.Job = nativeBinding.Job;
 module.exports.JsFullTextQuery = nativeBinding.JsFullTextQuery;
 module.exports.JsHeaderProvider = nativeBinding.JsHeaderProvider;
 module.exports.NapiScannable = nativeBinding.NapiScannable;

package/dist/query.d.ts

@@ -224,6 +224,26 @@ export declare class StandardQueryBase<NativeQueryType extends NativeQuery | Nat
      * Use {@link Table#optimize} to index all un-indexed data.
      */
     fastSearch(): this;
+    /**
+     * Control MemWAL read routing for this query.
+     *
+     * By default (unset), when the table carries a MemWAL write spec (see
+     * {@link Table#setLsmWriteSpec}), reads are routed through the LSM scanner so
+     * they also return data written via the `mergeInsert` LSM path that has not yet
+     * been compacted into the base table (the active/frozen in-memory memtables and
+     * the flushed generations), deduplicated by primary key; a table without a spec
+     * reads the base table.
+     *
+     * @param enable - `true` forces the LSM scanner and errors if the table has no
+     * MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
+     * even when a spec is present.
+     *
+     * Note: the LSM scanner does not support every query shape (e.g. reranking,
+     * hybrid search, `orderBy`). On a MemWAL table those shapes error unless
+     * `useLsm(false)` is set, because a base-only read would silently exclude
+     * un-compacted MemWAL data.
+     */
+    useLsm(enable: boolean): this;
 }
 /**
  * An interface for a query that can be executed
@@ -399,6 +419,16 @@ export declare class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
  */
 export declare class TakeQuery extends QueryBase<NativeTakeQuery> {
     constructor(inner: NativeTakeQuery);
+    /**
+     * Control MemWAL read routing for this take query.
+     *
+     * `false` bypasses the MemWAL and reads the base table only — the escape hatch,
+     * since take-by-row-id/offset is not supported on the LSM scanner and, on a
+     * MemWAL table, auto-routes to it and errors otherwise.
+     *
+     * @param enable - `false` reads the base table only.
+     */
+    useLsm(enable: boolean): this;
 }
 /** A builder for LanceDB queries.
  *

package/dist/query.js

@@ -362,6 +362,29 @@ class StandardQueryBase extends QueryBase {
         this.doCall((inner) => inner.fastSearch());
         return this;
     }
+    /**
+     * Control MemWAL read routing for this query.
+     *
+     * By default (unset), when the table carries a MemWAL write spec (see
+     * {@link Table#setLsmWriteSpec}), reads are routed through the LSM scanner so
+     * they also return data written via the `mergeInsert` LSM path that has not yet
+     * been compacted into the base table (the active/frozen in-memory memtables and
+     * the flushed generations), deduplicated by primary key; a table without a spec
+     * reads the base table.
+     *
+     * @param enable - `true` forces the LSM scanner and errors if the table has no
+     * MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
+     * even when a spec is present.
+     *
+     * Note: the LSM scanner does not support every query shape (e.g. reranking,
+     * hybrid search, `orderBy`). On a MemWAL table those shapes error unless
+     * `useLsm(false)` is set, because a base-only read would silently exclude
+     * un-compacted MemWAL data.
+     */
+    useLsm(enable) {
+        this.doCall((inner) => inner.useLsm(enable));
+        return this;
+    }
 }
 exports.StandardQueryBase = StandardQueryBase;
 /**
@@ -621,6 +644,19 @@ class TakeQuery extends QueryBase {
     constructor(inner) {
         super(inner);
     }
+    /**
+     * Control MemWAL read routing for this take query.
+     *
+     * `false` bypasses the MemWAL and reads the base table only — the escape hatch,
+     * since take-by-row-id/offset is not supported on the LSM scanner and, on a
+     * MemWAL table, auto-routes to it and errors otherwise.
+     *
+     * @param enable - `false` reads the base table only.
+     */
+    useLsm(enable) {
+        this.doCall((inner) => inner.useLsm(enable));
+        return this;
+    }
 }
 exports.TakeQuery = TakeQuery;
 /** A builder for LanceDB queries.

package/dist/sanitize.js

@@ -42,7 +42,7 @@ function sanitizeMetadata(metadataLike) {
         throw Error("Expected metadata, if present, to be a Map<string, string>");
     }
     for (const item of metadataLike) {
-        if (!(typeof item[0] === "string" || !(typeof item[1] === "string"))) {
+        if (typeof item[0] !== "string" || typeof item[1] !== "string") {
             throw Error("Expected metadata, if present, to be a Map<string, string> but it had non-string keys or values");
         }
     }

package/dist/table.d.ts

@@ -1,7 +1,7 @@
 import { Table as ArrowTable, Data, DataType, Field, IntoVector, MultiVector, Schema } from "./arrow";
 import { IndexOptions } from "./indices";
 import { MergeInsertBuilder } from "./merge";
-import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, IndexConfig, IndexStatistics, Branches as NativeBranches, OptimizeStats, TableStatistics, Tags, UpdateFieldMetadataResult, UpdateResult, Table as _NativeTable } from "./native";
+import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, IndexConfig, IndexStatistics, Job, Branches as NativeBranches, OptimizeStats, TableStatistics, Tags, UpdateFieldMetadataResult, UpdateResult, Table as _NativeTable } from "./native";
 import { FullTextQuery, Query, TakeQuery, VectorQuery } from "./query";
 import { IntoSql } from "./util";
 export { IndexConfig } from "./native";
@@ -142,7 +142,11 @@ export interface LsmWriteSpec {
     column?: string;
     /** Bucket variant: the number of buckets, in `[1, 1024]`. */
     numBuckets?: number;
-    /** Names of indexes the MemWAL should keep up to date during writes. */
+    /**
+     * Indexes the MemWAL keeps up to date. Omit to maintain every supported
+     * index, resolved on install — a snapshot, so indexes created later are not
+     * maintained. Pass `[]` for none.
+     */
     maintainedIndexes?: string[];
     /** Default `ShardWriter` configuration recorded in the MemWAL index. */
     writerConfigDefaults?: Record<string, string>;
@@ -284,6 +288,13 @@ export declare abstract class Table {
      * await table.createIndex("my_float_col");
      */
     abstract createIndex(column: string, options?: Partial<IndexOptions>): Promise<void>;
+    /**
+     * Create an index, returning a handle to the indexing job.
+     *
+     * The job may already be complete when returned; callers must not assume
+     * the index exists until {@link Job.wait} resolves.
+     */
+    abstract createIndexAsync(column: string, options?: Partial<IndexOptions>): Promise<Job>;
     /**
      * Drop an index from the table.
      *
@@ -486,6 +497,11 @@ export declare abstract class Table {
      * All variants require the table to have an unenforced primary key
      * ({@link Table#setUnenforcedPrimaryKey}); bucket sharding additionally
      * requires it to be the single column being bucketed.
+     *
+     * Omitting `maintainedIndexes` maintains every index on the table, resolved
+     * here, failing if one cannot be maintained — name them to install anyway.
+     * Naming them pins an exact set, and a still-building index is rejected
+     * rather than quietly omitted.
      * @param {LsmWriteSpec} spec The sharding spec to install.
      * @returns {Promise<void>}
      * @example
@@ -513,9 +529,10 @@ export declare abstract class Table {
      *
      * Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
      * spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}).
-     * The returned spec — including its `maintainedIndexes` and
-     * `writerConfigDefaults` — mirrors what was passed to
-     * {@link Table#setLsmWriteSpec}.
+     * The returned spec mirrors what was passed to
+     * {@link Table#setLsmWriteSpec}, except that `maintainedIndexes` always
+     * reports the concrete list resolved when the spec was set — `undefined`
+     * never round-trips.
      * @returns {Promise<LsmWriteSpec | undefined>}
      */
     abstract getLsmWriteSpec(): Promise<LsmWriteSpec | undefined>;
@@ -702,6 +719,7 @@ export declare class LocalTable extends Table {
     countRows(filter?: string): Promise<number>;
     delete(predicate: string): Promise<DeleteResult>;
     createIndex(column: string, options?: Partial<IndexOptions>): Promise<void>;
+    createIndexAsync(column: string, options?: Partial<IndexOptions>): Promise<Job>;
     dropIndex(name: string): Promise<void>;
     prewarmIndex(name: string): Promise<void>;
     prewarmData(columns?: string[]): Promise<void>;

package/dist/table.js

@@ -139,6 +139,11 @@ class LocalTable extends Table {
         const nativeIndex = options?.config?.inner;
         await this.inner.createIndex(nativeIndex, column, options?.replace, options?.waitTimeoutSeconds, options?.name, options?.train);
     }
+    async createIndexAsync(column, options) {
+        // biome-ignore lint/suspicious/noExplicitAny: skip
+        const nativeIndex = options?.config?.inner;
+        return await this.inner.createIndexAsync(nativeIndex, column, options?.replace, options?.waitTimeoutSeconds, options?.name, options?.train);
+    }
     async dropIndex(name) {
         await this.inner.dropIndex(name);
     }

package/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.33.0",
+  "version": "0.37.1-beta.1",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -106,15 +106,21 @@
   "optionalDependencies": {
     "@huggingface/transformers": "3.0.2",
     "openai": "4.29.2",
-    "@lancedb/lancedb-darwin-arm64": "0.33.0",
-    "@lancedb/lancedb-linux-x64-gnu": "0.33.0",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.33.0",
-    "@lancedb/lancedb-linux-x64-musl": "0.33.0",
-    "@lancedb/lancedb-linux-arm64-musl": "0.33.0",
-    "@lancedb/lancedb-win32-x64-msvc": "0.33.0",
-    "@lancedb/lancedb-win32-arm64-msvc": "0.33.0"
+    "@lancedb/lancedb-darwin-arm64": "0.37.1-beta.1",
+    "@lancedb/lancedb-linux-x64-gnu": "0.37.1-beta.1",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.37.1-beta.1",
+    "@lancedb/lancedb-linux-x64-musl": "0.37.1-beta.1",
+    "@lancedb/lancedb-linux-arm64-musl": "0.37.1-beta.1",
+    "@lancedb/lancedb-win32-x64-msvc": "0.37.1-beta.1",
+    "@lancedb/lancedb-win32-arm64-msvc": "0.37.1-beta.1"
   },
   "peerDependencies": {
+    "@types/node": ">=18",
     "apache-arrow": ">=15.0.0 <=18.1.0"
+  },
+  "peerDependenciesMeta": {
+    "@types/node": {
+      "optional": true
+    }
   }
 }