@lancedb/lancedb 0.31.0 → 0.32.0-beta.2

package/dist/index.d.ts

@@ -1,14 +1,17 @@
 import { Connection } from "./connection";
 import { ConnectNamespaceOptions, ConnectionOptions, Session } from "./native.js";
 import { HeaderProvider } from "./header";
+import type { BaseTokenizer } from "./indices";
+import type { FtsToken } from "./table";
 export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
+export { instrumentLanceDbMetrics } from "./otel";
 export { AddColumnsSql, ConnectionOptions, ConnectNamespaceOptions, IndexStatistics, IndexConfig, ClientConfig, TimeoutConfig, RetryConfig, TlsConfig, OptimizeStats, CompactionStats, RemovalStats, TableStatistics, FragmentStatistics, FragmentSummaryStats, Tags, TagContents, BranchContents, MergeResult, AddResult, AddColumnsResult, AlterColumnsResult, UpdateFieldMetadataResult, DeleteResult, DropColumnsResult, UpdateResult, SplitCalculatedOptions, SplitRandomOptions, SplitHashOptions, SplitSequentialOptions, ShuffleOptions, OAuthConfig as NativeOAuthConfig, } from "./native.js";
 export { makeArrowTable, MakeArrowTableOptions, Data, VectorColumnOptions, } from "./arrow";
 export { Connection, CreateTableOptions, TableNamesOptions, OpenTableOptions, ListNamespacesOptions, CreateNamespaceOptions, DropNamespaceOptions, ListNamespacesResponse, CreateNamespaceResponse, DropNamespaceResponse, DescribeNamespaceResponse, RenameTableOptions, } from "./connection";
 export { Session } from "./native.js";
 export { ExecutableQuery, Query, QueryBase, VectorQuery, TakeQuery, QueryExecutionOptions, ColumnOrdering, FullTextSearchOptions, RecordBatchIterator, FullTextQuery, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, FullTextQueryType, Operator, Occur, } from "./query";
-export { Index, IndexOptions, IvfPqOptions, IvfRqOptions, IvfFlatOptions, HnswPqOptions, HnswSqOptions, FtsOptions, } from "./indices";
-export { Table, Branches, AddDataOptions, UpdateOptions, OptimizeOptions, Version, WriteProgress, LsmWriteSpec, ColumnAlteration, FieldMetadataUpdate, } from "./table";
+export { Index, IndexOptions, IvfPqOptions, IvfRqOptions, IvfFlatOptions, HnswPqOptions, HnswSqOptions, FtsOptions, BaseTokenizer, } from "./indices";
+export { Table, Branches, AddDataOptions, UpdateOptions, OptimizeOptions, Version, WriteProgress, FtsToken, TokenizeTableOptions, LsmWriteSpec, ColumnAlteration, FieldMetadataUpdate, } from "./table";
 export { HeaderProvider, StaticHeaderProvider, OAuthHeaderProvider, TokenResponse, } from "./header";
 export { OAuthConfig, OAuthFlowType } from "./oauth";
 export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
@@ -18,6 +21,40 @@ export { Scannable, ScannableOptions } from "./scannable";
 export * as rerankers from "./rerankers";
 export { SchemaLike, TableLike, FieldLike, RecordBatchLike, DataLike, IntoVector, MultiVector, } from "./arrow";
 export { IntoSql, packBits } from "./util";
+/**
+ * Options for tokenizing a full-text search query without a table index.
+ */
+export interface TokenizeOptions {
+    /**
+     * The tokenizer to use. The default is "simple".
+     */
+    baseTokenizer?: BaseTokenizer;
+    /** Language for stemming and stop words. */
+    language?: string;
+    /** Maximum token length; tokens longer than this are ignored. */
+    maxTokenLength?: number;
+    /** Whether to lowercase tokens. */
+    lowercase?: boolean;
+    /** Whether to stem tokens. */
+    stem?: boolean;
+    /** Whether to remove stop words. */
+    removeStopWords?: boolean;
+    /** Whether to fold ASCII characters. */
+    asciiFolding?: boolean;
+    /** N-gram minimum length. */
+    ngramMinLength?: number;
+    /** N-gram maximum length. */
+    ngramMaxLength?: number;
+    /** Whether to only emit token prefixes for the n-gram tokenizer. */
+    prefixOnly?: boolean;
+}
+/**
+ * Tokenize a full-text search query using an explicit tokenizer.
+ *
+ * This does not require a table or FTS index. The tokenizer options match
+ * {@link Index.fts}.
+ */
+export declare function tokenize(query: string, options?: Partial<TokenizeOptions>): Promise<FtsToken[]>;
 /**
  * Connect to a LanceDB instance at the given URI.
  *

package/dist/index.js

@@ -2,7 +2,8 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthFlowType = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Session = exports.Connection = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = exports.BranchContents = exports.TagContents = exports.Tags = exports.NativeJsHeaderProvider = void 0;
+exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthFlowType = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Session = exports.Connection = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = exports.BranchContents = exports.TagContents = exports.Tags = exports.instrumentLanceDbMetrics = exports.NativeJsHeaderProvider = void 0;
+exports.tokenize = tokenize;
 exports.connect = connect;
 exports.connectNamespace = connectNamespace;
 const connection_1 = require("./connection");
@@ -10,6 +11,11 @@ const native_js_1 = require("./native.js");
 // Re-export native header provider for use with connectWithHeaderProvider
 var native_js_2 = require("./native.js");
 Object.defineProperty(exports, "NativeJsHeaderProvider", { enumerable: true, get: function () { return native_js_2.JsHeaderProvider; } });
+// OpenTelemetry metrics bridge. Only the high-level entry point is public; the
+// underlying recorder/catalog/snapshot functions remain internal plumbing that
+// `otel.ts` consumes from the native module.
+var otel_1 = require("./otel");
+Object.defineProperty(exports, "instrumentLanceDbMetrics", { enumerable: true, get: function () { return otel_1.instrumentLanceDbMetrics; } });
 var native_js_3 = require("./native.js");
 Object.defineProperty(exports, "Tags", { enumerable: true, get: function () { return native_js_3.Tags; } });
 Object.defineProperty(exports, "TagContents", { enumerable: true, get: function () { return native_js_3.TagContents; } });
@@ -58,6 +64,15 @@ Object.defineProperty(exports, "Scannable", { enumerable: true, get: function ()
 exports.rerankers = require("./rerankers");
 var util_1 = require("./util");
 Object.defineProperty(exports, "packBits", { enumerable: true, get: function () { return util_1.packBits; } });
+/**
+ * Tokenize a full-text search query using an explicit tokenizer.
+ *
+ * This does not require a table or FTS index. The tokenizer options match
+ * {@link Index.fts}.
+ */
+async function tokenize(query, options) {
+    return await (0, native_js_1.tokenize)(query, options?.baseTokenizer, options?.language, options?.maxTokenLength, options?.lowercase, options?.stem, options?.removeStopWords, options?.asciiFolding, options?.ngramMinLength, options?.ngramMaxLength, options?.prefixOnly);
+}
 async function connect(uriOrOptions, optionsOrSession, sessionOrHeaderProvider, headerProvider) {
     let uri;
     let finalOptions = {};

package/dist/indices.d.ts

@@ -453,6 +453,7 @@ export interface IvfFlatOptions {
      */
     sampleRate?: number;
 }
+export type BaseTokenizer = "simple" | "whitespace" | "raw" | "ngram" | "icu" | "icu/split" | `jieba/${string}` | `lindera/${string}`;
 /**
  * Options to create a full text search index
  */
@@ -475,8 +476,12 @@ export interface FtsOptions {
      * "whitespace" - Whitespace tokenizer. This tokenizer splits the text into tokens using whitespace as a delimiter.
      *
      * "raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
+     *
+     * "icu" - ICU dictionary-based word segmentation.
+     *
+     * "icu/split" - ICU segmentation with simple-style delimiter splitting.
      */
-    baseTokenizer?: "simple" | "whitespace" | "raw" | "ngram";
+    baseTokenizer?: BaseTokenizer;
     /**
      * language for stemming and stop words
      * this is only used when `stem` or `remove_stop_words` is true

package/dist/native.d.ts

@@ -235,6 +235,7 @@ export declare class Table {
   setUnenforcedPrimaryKey(columns: Array<string>): Promise<void>
   setLsmWriteSpec(spec: LsmWriteSpec): Promise<void>
   unsetLsmWriteSpec(): Promise<void>
+  getLsmWriteSpec(): Promise<LsmWriteSpec | null>
   closeLsmWriters(): Promise<void>
   version(): Promise<number>
   checkout(version: number): Promise<void>
@@ -248,6 +249,7 @@ export declare class Table {
   currentBranch(): string | null
   optimize(olderThanMs?: number | undefined | null, deleteUnverified?: boolean | undefined | null): Promise<OptimizeStats>
   listIndices(): Promise<Array<IndexConfig>>
+  tokenize(query: string, column?: string | undefined | null, indexName?: string | undefined | null): Promise<Array<FtsToken>>
   indexStats(indexName: string): Promise<IndexStatistics | null>
   mergeInsert(on: Array<string>): NativeMergeInsertBuilder
   usesV2ManifestPaths(): Promise<boolean>
@@ -553,6 +555,14 @@ export interface FragmentSummaryStats {
   p99: number
 }
 
+/** A token produced by the tokenizer configured on a full-text search index. */
+export interface FtsToken {
+  /** The token text after the index tokenizer has applied its filters. */
+  text: string
+  /** The token position used by full-text query matching. */
+  position: number
+}
+
 /** A description of an index currently configured on a column */
 export interface IndexConfig {
   /** The name of the index */
@@ -640,6 +650,9 @@ export interface IndexStatistics {
   numIndices?: number
 }
 
+/** The catalog of described LanceDB metrics. Empty until the recorder is installed. */
+export declare function lancedbMetricsCatalog(): Array<MetricDescription>
+
 export interface ListNamespacesResponse {
   namespaces: Array<string>
   pageToken?: string
@@ -675,6 +688,37 @@ export interface MergeResult {
   numRows: number
 }
 
+/** One cumulative histogram bucket: all samples with value `<= le`. */
+export interface MetricBucket {
+  /** The inclusive upper bound of the bucket, or `"+Inf"` for the final bucket. */
+  le: string
+  /** Cumulative number of samples less than or equal to `le`. */
+  cumulativeCount: number
+}
+
+/** A described metric, used by the JavaScript layer to create instruments up front. */
+export interface MetricDescription {
+  name: string
+  kind: string
+  unit?: string
+  description: string
+}
+
+/**
+ * One aggregated metric data point. For counters and gauges only `value` is
+ * set; for histograms `buckets` (cumulative `le` counts), `count`, and `sum`
+ * are set.
+ */
+export interface MetricPoint {
+  name: string
+  kind: string
+  attributes: Record<string, string>
+  value?: number
+  buckets?: Array<MetricBucket>
+  count?: number
+  sum?: number
+}
+
 /**
  * OAuth configuration for LanceDB authentication.
  *
@@ -725,6 +769,15 @@ export interface OptimizeStats {
 /** Create a permutation builder for the given table */
 export declare function permutationBuilder(table: Table): PermutationBuilder
 
+/**
+ * Install the LanceDB metrics recorder as the process-global `metrics` recorder.
+ *
+ * Returns `true` if the recorder is installed (now or previously). Returns
+ * `false` if a *different* recorder is already installed — `metrics` allows
+ * only one global recorder per process, so LanceDB cannot coexist with another.
+ */
+export declare function registerLancedbMetricsRecorder(): boolean
+
 /** Statistics about a cleanup operation */
 export interface RemovalStats {
   /** The number of bytes removed */
@@ -793,6 +846,12 @@ export interface ShuffleOptions {
   clumpSize?: number
 }
 
+/**
+ * A point-in-time snapshot of every recorded metric. Empty until the recorder
+ * is installed.
+ */
+export declare function snapshotLancedbMetrics(): Array<MetricPoint>
+
 export interface SplitCalculatedOptions {
   calculation: string
   splitNames?: Array<string>
@@ -810,6 +869,7 @@ export interface SplitRandomOptions {
   counts?: Array<number>
   fixed?: number
   seed?: number
+  clumpSize?: number
   splitNames?: Array<string>
 }
 
@@ -874,6 +934,8 @@ export interface TlsConfig {
   assertHostname?: boolean
 }
 
+export declare function tokenize(query: string, baseTokenizer?: string | undefined | null, language?: string | undefined | null, maxTokenLength?: number | undefined | null, lowerCase?: boolean | undefined | null, stem?: boolean | undefined | null, removeStopWords?: boolean | undefined | null, asciiFolding?: boolean | undefined | null, ngramMinLength?: number | undefined | null, ngramMaxLength?: number | undefined | null, prefixOnly?: boolean | undefined | null): Array<FtsToken>
+
 export interface UpdateFieldMetadataResult {
   version: number
 }

package/dist/native.js

@@ -76,8 +76,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-android-arm64');
                 const bindingPackageVersion = require('@lancedb/lancedb-android-arm64/package.json').version;
-                if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -95,8 +95,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-android-arm-eabi');
                 const bindingPackageVersion = require('@lancedb/lancedb-android-arm-eabi/package.json').version;
-                if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -120,8 +120,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-win32-x64-gnu');
                     const bindingPackageVersion = require('@lancedb/lancedb-win32-x64-gnu/package.json').version;
-                    if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -139,8 +139,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-win32-x64-msvc');
                     const bindingPackageVersion = require('@lancedb/lancedb-win32-x64-msvc/package.json').version;
-                    if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -159,8 +159,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-win32-ia32-msvc');
                 const bindingPackageVersion = require('@lancedb/lancedb-win32-ia32-msvc/package.json').version;
-                if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -178,8 +178,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-win32-arm64-msvc');
                 const bindingPackageVersion = require('@lancedb/lancedb-win32-arm64-msvc/package.json').version;
-                if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -201,8 +201,8 @@ function requireNative() {
         try {
             const binding = require('@lancedb/lancedb-darwin-universal');
             const bindingPackageVersion = require('@lancedb/lancedb-darwin-universal/package.json').version;
-            if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+            if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
             }
             return binding;
         }
@@ -219,8 +219,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-darwin-x64');
                 const bindingPackageVersion = require('@lancedb/lancedb-darwin-x64/package.json').version;
-                if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -238,8 +238,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-darwin-arm64');
                 const bindingPackageVersion = require('@lancedb/lancedb-darwin-arm64/package.json').version;
-                if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -262,8 +262,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-freebsd-x64');
                 const bindingPackageVersion = require('@lancedb/lancedb-freebsd-x64/package.json').version;
-                if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -281,8 +281,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-freebsd-arm64');
                 const bindingPackageVersion = require('@lancedb/lancedb-freebsd-arm64/package.json').version;
-                if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -306,8 +306,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-x64-musl');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-x64-musl/package.json').version;
-                    if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -325,8 +325,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-x64-gnu');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-x64-gnu/package.json').version;
-                    if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -346,8 +346,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-arm64-musl');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-arm64-musl/package.json').version;
-                    if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -365,8 +365,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-arm64-gnu');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-arm64-gnu/package.json').version;
-                    if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -386,8 +386,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-arm-musleabihf');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-arm-musleabihf/package.json').version;
-                    if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -405,8 +405,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-arm-gnueabihf');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-arm-gnueabihf/package.json').version;
-                    if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -426,8 +426,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-loong64-musl');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-loong64-musl/package.json').version;
-                    if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -445,8 +445,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-loong64-gnu');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-loong64-gnu/package.json').version;
-                    if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -466,8 +466,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-riscv64-musl');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-riscv64-musl/package.json').version;
-                    if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -485,8 +485,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-riscv64-gnu');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-riscv64-gnu/package.json').version;
-                    if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -505,8 +505,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-linux-ppc64-gnu');
                 const bindingPackageVersion = require('@lancedb/lancedb-linux-ppc64-gnu/package.json').version;
-                if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -524,8 +524,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-linux-s390x-gnu');
                 const bindingPackageVersion = require('@lancedb/lancedb-linux-s390x-gnu/package.json').version;
-                if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -548,8 +548,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-openharmony-arm64');
                 const bindingPackageVersion = require('@lancedb/lancedb-openharmony-arm64/package.json').version;
-                if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -567,8 +567,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-openharmony-x64');
                 const bindingPackageVersion = require('@lancedb/lancedb-openharmony-x64/package.json').version;
-                if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -586,8 +586,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-openharmony-arm');
                 const bindingPackageVersion = require('@lancedb/lancedb-openharmony-arm/package.json').version;
-                if (bindingPackageVersion !== '0.31.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.31.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -680,4 +680,8 @@ module.exports.TagContents = nativeBinding.TagContents;
 module.exports.Tags = nativeBinding.Tags;
 module.exports.TakeQuery = nativeBinding.TakeQuery;
 module.exports.VectorQuery = nativeBinding.VectorQuery;
+module.exports.lancedbMetricsCatalog = nativeBinding.lancedbMetricsCatalog;
 module.exports.permutationBuilder = nativeBinding.permutationBuilder;
+module.exports.registerLancedbMetricsRecorder = nativeBinding.registerLancedbMetricsRecorder;
+module.exports.snapshotLancedbMetrics = nativeBinding.snapshotLancedbMetrics;
+module.exports.tokenize = nativeBinding.tokenize;

package/dist/otel.d.ts

@@ -0,0 +1,26 @@
+import { type MeterProvider } from "@opentelemetry/api";
+/**
+ * Register LanceDB metrics as OpenTelemetry observable instruments.
+ *
+ * Installs a process-global metrics recorder and creates one observable
+ * instrument per LanceDB metric (currently object store request counts, bytes,
+ * latency, errors, and throttles) on the given (or global) `MeterProvider`. The
+ * configured `MetricReader` then collects them on its own schedule.
+ *
+ * Counters and gauges map directly to observable counters/gauges. Because
+ * OpenTelemetry has no asynchronous histogram instrument, each histogram is
+ * exported Prometheus-style as cumulative `le` bucket counts (`<name>_bucket`,
+ * with an `le` attribute) plus `<name>_count` and `<name>_sum`.
+ *
+ * Requires `@opentelemetry/api` (a dependency) and, to actually export, an
+ * OpenTelemetry SDK such as `@opentelemetry/sdk-metrics`.
+ *
+ * @param meterProvider The provider to register instruments on. Defaults to the
+ *   global provider from `@opentelemetry/api`.
+ * @returns `true` if the recorder is installed and instruments are registered.
+ *   `false` if a different `metrics` recorder is already installed in this
+ *   process (only one global recorder is permitted), in which case a warning is
+ *   emitted and no instruments are created. Calling this more than once is safe;
+ *   instruments are created only on the first successful call.
+ */
+export declare function instrumentLanceDbMetrics(meterProvider?: MeterProvider): boolean;

package/dist/otel.js

@@ -0,0 +1,114 @@
+"use strict";
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.instrumentLanceDbMetrics = instrumentLanceDbMetrics;
+const api_1 = require("@opentelemetry/api");
+const native_1 = require("./native");
+let instrumented = false;
+/**
+ * Register LanceDB metrics as OpenTelemetry observable instruments.
+ *
+ * Installs a process-global metrics recorder and creates one observable
+ * instrument per LanceDB metric (currently object store request counts, bytes,
+ * latency, errors, and throttles) on the given (or global) `MeterProvider`. The
+ * configured `MetricReader` then collects them on its own schedule.
+ *
+ * Counters and gauges map directly to observable counters/gauges. Because
+ * OpenTelemetry has no asynchronous histogram instrument, each histogram is
+ * exported Prometheus-style as cumulative `le` bucket counts (`<name>_bucket`,
+ * with an `le` attribute) plus `<name>_count` and `<name>_sum`.
+ *
+ * Requires `@opentelemetry/api` (a dependency) and, to actually export, an
+ * OpenTelemetry SDK such as `@opentelemetry/sdk-metrics`.
+ *
+ * @param meterProvider The provider to register instruments on. Defaults to the
+ *   global provider from `@opentelemetry/api`.
+ * @returns `true` if the recorder is installed and instruments are registered.
+ *   `false` if a different `metrics` recorder is already installed in this
+ *   process (only one global recorder is permitted), in which case a warning is
+ *   emitted and no instruments are created. Calling this more than once is safe;
+ *   instruments are created only on the first successful call.
+ */
+function instrumentLanceDbMetrics(meterProvider) {
+    if (!(0, native_1.registerLancedbMetricsRecorder)()) {
+        console.warn("Could not install the LanceDB metrics recorder: another `metrics` " +
+            "recorder is already installed in this process. LanceDB metrics will " +
+            "not be exported via OpenTelemetry.");
+        return false;
+    }
+    if (instrumented) {
+        return true;
+    }
+    const provider = meterProvider ?? api_1.metrics.getMeterProvider();
+    const meter = provider.getMeter("lancedb");
+    const scalarCallback = (metricName) => (result) => {
+        for (const point of (0, native_1.snapshotLancedbMetrics)()) {
+            if (point.name === metricName && point.value != null) {
+                result.observe(point.value, point.attributes);
+            }
+        }
+    };
+    const bucketCallback = (metricName) => (result) => {
+        for (const point of (0, native_1.snapshotLancedbMetrics)()) {
+            if (point.name !== metricName || point.buckets == null) {
+                continue;
+            }
+            for (const bucket of point.buckets) {
+                const attributes = {
+                    ...point.attributes,
+                    le: bucket.le,
+                };
+                result.observe(bucket.cumulativeCount, attributes);
+            }
+        }
+    };
+    const fieldCallback = (metricName, field) => (result) => {
+        for (const point of (0, native_1.snapshotLancedbMetrics)()) {
+            if (point.name !== metricName) {
+                continue;
+            }
+            const value = point[field];
+            if (value != null) {
+                result.observe(value, point.attributes);
+            }
+        }
+    };
+    for (const desc of (0, native_1.lancedbMetricsCatalog)()) {
+        const unit = desc.unit ?? "";
+        if (desc.kind === "counter") {
+            const counter = meter.createObservableCounter(desc.name, {
+                unit,
+                description: desc.description,
+            });
+            counter.addCallback(scalarCallback(desc.name));
+        }
+        else if (desc.kind === "gauge") {
+            const gauge = meter.createObservableGauge(desc.name, {
+                unit,
+                description: desc.description,
+            });
+            gauge.addCallback(scalarCallback(desc.name));
+        }
+        else if (desc.kind === "histogram") {
+            // `_bucket` and `_count` observe cumulative sample counts, not the
+            // histogram's measured quantity, so they are unitless; only `_sum`
+            // carries the histogram's unit.
+            const bucket = meter.createObservableCounter(`${desc.name}_bucket`, {
+                description: `${desc.description} (cumulative buckets)`,
+            });
+            bucket.addCallback(bucketCallback(desc.name));
+            const count = meter.createObservableCounter(`${desc.name}_count`, {
+                description: `${desc.description} (count)`,
+            });
+            count.addCallback(fieldCallback(desc.name, "count"));
+            const sum = meter.createObservableCounter(`${desc.name}_sum`, {
+                unit,
+                description: `${desc.description} (sum)`,
+            });
+            sum.addCallback(fieldCallback(desc.name, "sum"));
+        }
+    }
+    instrumented = true;
+    return true;
+}

package/dist/table.d.ts

@@ -108,6 +108,22 @@ export interface Version {
     timestamp: Date;
     metadata: Record<string, string>;
 }
+/** Token produced by the tokenizer configured on a full-text search index. */
+export interface FtsToken {
+    /** Token text after tokenizer filters have been applied. */
+    text: string;
+    /** Token position used by full-text query matching. */
+    position: number;
+}
+export type TokenizeTableOptions = {
+    /** FTS-indexed column whose tokenizer should be used. */
+    column: string;
+    indexName?: never;
+} | {
+    /** Name of the FTS index whose tokenizer should be used. */
+    indexName: string;
+    column?: never;
+};
 /**
  * Specification selecting Lance's MemWAL LSM-style write path for
  * `mergeInsert`.
@@ -492,6 +508,17 @@ export declare abstract class Table {
      * @returns {Promise<void>}
      */
     abstract unsetLsmWriteSpec(): Promise<void>;
+    /**
+     * Read the {@link LsmWriteSpec} currently installed on this table.
+     *
+     * Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
+     * spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}).
+     * The returned spec — including its `maintainedIndexes` and
+     * `writerConfigDefaults` — mirrors what was passed to
+     * {@link Table#setLsmWriteSpec}.
+     * @returns {Promise<LsmWriteSpec | undefined>}
+     */
+    abstract getLsmWriteSpec(): Promise<LsmWriteSpec | undefined>;
     /**
      * Drain and close any cached MemWAL shard writers held for this table.
      *
@@ -605,6 +632,16 @@ export declare abstract class Table {
     abstract optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
     /** List all indices that have been created with {@link Table.createIndex} */
     abstract listIndices(): Promise<IndexConfig[]>;
+    /**
+     * Tokenize a full-text search query using the tokenizer configured on an FTS index.
+     *
+     * Specify exactly one of `column` or `indexName`.
+     *
+     * Model-backed tokenizers such as `jieba/*` and `lindera/*` are rebuilt in
+     * the client process from index metadata. For remote tables, this means the
+     * same tokenizer model files must also exist locally.
+     */
+    abstract tokenize(query: string, options: TokenizeTableOptions): Promise<FtsToken[]>;
     /** Return the table as an arrow table */
     abstract toArrow(): Promise<ArrowTable>;
     abstract mergeInsert(on: string | string[]): MergeInsertBuilder;
@@ -681,6 +718,7 @@ export declare class LocalTable extends Table {
     setUnenforcedPrimaryKey(columns: string | string[]): Promise<void>;
     setLsmWriteSpec(spec: LsmWriteSpec): Promise<void>;
     unsetLsmWriteSpec(): Promise<void>;
+    getLsmWriteSpec(): Promise<LsmWriteSpec | undefined>;
     closeLsmWriters(): Promise<void>;
     version(): Promise<number>;
     checkout(version: number | string): Promise<void>;
@@ -692,6 +730,7 @@ export declare class LocalTable extends Table {
     currentBranch(): string | null;
     optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
     listIndices(): Promise<IndexConfig[]>;
+    tokenize(query: string, options: TokenizeTableOptions): Promise<FtsToken[]>;
     toArrow(): Promise<ArrowTable>;
     indexStats(name: string): Promise<IndexStatistics | undefined>;
     stats(): Promise<TableStatistics>;

package/dist/table.js

@@ -285,6 +285,12 @@ class LocalTable extends Table {
     async unsetLsmWriteSpec() {
         return await this.inner.unsetLsmWriteSpec();
     }
+    async getLsmWriteSpec() {
+        // The native binding types `specType` as a plain `string`; narrow it back
+        // to the public union. The Rust `From` impl only ever emits one of the
+        // three valid values, so the cast is safe.
+        return ((await this.inner.getLsmWriteSpec()) ?? undefined);
+    }
     async closeLsmWriters() {
         return await this.inner.closeLsmWriters();
     }
@@ -331,6 +337,9 @@ class LocalTable extends Table {
     async listIndices() {
         return await this.inner.listIndices();
     }
+    async tokenize(query, options) {
+        return await this.inner.tokenize(query, options?.column, options?.indexName);
+    }
     async toArrow() {
         return await this.query().toArrow();
     }

package/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.31.0",
+  "version": "0.32.0-beta.2",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -44,6 +44,7 @@
     "@biomejs/biome": "^1.7.3",
     "@jest/globals": "^29.7.0",
     "@napi-rs/cli": "3.7.0",
+    "@opentelemetry/sdk-metrics": "^1.30.0",
     "@types/axios": "^0.14.0",
     "@types/jest": "^29.1.2",
     "@types/node": "22.7.4",
@@ -99,18 +100,19 @@
     "version": "napi version"
   },
   "dependencies": {
+    "@opentelemetry/api": "^1.9.0",
     "reflect-metadata": "^0.2.2"
   },
   "optionalDependencies": {
     "@huggingface/transformers": "3.0.2",
     "openai": "4.29.2",
-    "@lancedb/lancedb-darwin-arm64": "0.31.0",
-    "@lancedb/lancedb-linux-x64-gnu": "0.31.0",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.31.0",
-    "@lancedb/lancedb-linux-x64-musl": "0.31.0",
-    "@lancedb/lancedb-linux-arm64-musl": "0.31.0",
-    "@lancedb/lancedb-win32-x64-msvc": "0.31.0",
-    "@lancedb/lancedb-win32-arm64-msvc": "0.31.0"
+    "@lancedb/lancedb-darwin-arm64": "0.32.0-beta.2",
+    "@lancedb/lancedb-linux-x64-gnu": "0.32.0-beta.2",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.32.0-beta.2",
+    "@lancedb/lancedb-linux-x64-musl": "0.32.0-beta.2",
+    "@lancedb/lancedb-linux-arm64-musl": "0.32.0-beta.2",
+    "@lancedb/lancedb-win32-x64-msvc": "0.32.0-beta.2",
+    "@lancedb/lancedb-win32-arm64-msvc": "0.32.0-beta.2"
   },
   "peerDependencies": {
     "apache-arrow": ">=15.0.0 <=18.1.0"