@lancedb/lancedb 0.32.0-beta.1 → 0.32.0-beta.2

package/dist/index.d.ts

@@ -1,6 +1,8 @@
 import { Connection } from "./connection";
 import { ConnectNamespaceOptions, ConnectionOptions, Session } from "./native.js";
 import { HeaderProvider } from "./header";
+import type { BaseTokenizer } from "./indices";
+import type { FtsToken } from "./table";
 export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
 export { instrumentLanceDbMetrics } from "./otel";
 export { AddColumnsSql, ConnectionOptions, ConnectNamespaceOptions, IndexStatistics, IndexConfig, ClientConfig, TimeoutConfig, RetryConfig, TlsConfig, OptimizeStats, CompactionStats, RemovalStats, TableStatistics, FragmentStatistics, FragmentSummaryStats, Tags, TagContents, BranchContents, MergeResult, AddResult, AddColumnsResult, AlterColumnsResult, UpdateFieldMetadataResult, DeleteResult, DropColumnsResult, UpdateResult, SplitCalculatedOptions, SplitRandomOptions, SplitHashOptions, SplitSequentialOptions, ShuffleOptions, OAuthConfig as NativeOAuthConfig, } from "./native.js";
@@ -8,8 +10,8 @@ export { makeArrowTable, MakeArrowTableOptions, Data, VectorColumnOptions, } fro
 export { Connection, CreateTableOptions, TableNamesOptions, OpenTableOptions, ListNamespacesOptions, CreateNamespaceOptions, DropNamespaceOptions, ListNamespacesResponse, CreateNamespaceResponse, DropNamespaceResponse, DescribeNamespaceResponse, RenameTableOptions, } from "./connection";
 export { Session } from "./native.js";
 export { ExecutableQuery, Query, QueryBase, VectorQuery, TakeQuery, QueryExecutionOptions, ColumnOrdering, FullTextSearchOptions, RecordBatchIterator, FullTextQuery, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, FullTextQueryType, Operator, Occur, } from "./query";
-export { Index, IndexOptions, IvfPqOptions, IvfRqOptions, IvfFlatOptions, HnswPqOptions, HnswSqOptions, FtsOptions, } from "./indices";
-export { Table, Branches, AddDataOptions, UpdateOptions, OptimizeOptions, Version, WriteProgress, LsmWriteSpec, ColumnAlteration, FieldMetadataUpdate, } from "./table";
+export { Index, IndexOptions, IvfPqOptions, IvfRqOptions, IvfFlatOptions, HnswPqOptions, HnswSqOptions, FtsOptions, BaseTokenizer, } from "./indices";
+export { Table, Branches, AddDataOptions, UpdateOptions, OptimizeOptions, Version, WriteProgress, FtsToken, TokenizeTableOptions, LsmWriteSpec, ColumnAlteration, FieldMetadataUpdate, } from "./table";
 export { HeaderProvider, StaticHeaderProvider, OAuthHeaderProvider, TokenResponse, } from "./header";
 export { OAuthConfig, OAuthFlowType } from "./oauth";
 export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
@@ -19,6 +21,40 @@ export { Scannable, ScannableOptions } from "./scannable";
 export * as rerankers from "./rerankers";
 export { SchemaLike, TableLike, FieldLike, RecordBatchLike, DataLike, IntoVector, MultiVector, } from "./arrow";
 export { IntoSql, packBits } from "./util";
+/**
+ * Options for tokenizing a full-text search query without a table index.
+ */
+export interface TokenizeOptions {
+    /**
+     * The tokenizer to use. The default is "simple".
+     */
+    baseTokenizer?: BaseTokenizer;
+    /** Language for stemming and stop words. */
+    language?: string;
+    /** Maximum token length; tokens longer than this are ignored. */
+    maxTokenLength?: number;
+    /** Whether to lowercase tokens. */
+    lowercase?: boolean;
+    /** Whether to stem tokens. */
+    stem?: boolean;
+    /** Whether to remove stop words. */
+    removeStopWords?: boolean;
+    /** Whether to fold ASCII characters. */
+    asciiFolding?: boolean;
+    /** N-gram minimum length. */
+    ngramMinLength?: number;
+    /** N-gram maximum length. */
+    ngramMaxLength?: number;
+    /** Whether to only emit token prefixes for the n-gram tokenizer. */
+    prefixOnly?: boolean;
+}
+/**
+ * Tokenize a full-text search query using an explicit tokenizer.
+ *
+ * This does not require a table or FTS index. The tokenizer options match
+ * {@link Index.fts}.
+ */
+export declare function tokenize(query: string, options?: Partial<TokenizeOptions>): Promise<FtsToken[]>;
 /**
  * Connect to a LanceDB instance at the given URI.
  *

package/dist/index.js

@@ -3,6 +3,7 @@
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthFlowType = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Session = exports.Connection = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = exports.BranchContents = exports.TagContents = exports.Tags = exports.instrumentLanceDbMetrics = exports.NativeJsHeaderProvider = void 0;
+exports.tokenize = tokenize;
 exports.connect = connect;
 exports.connectNamespace = connectNamespace;
 const connection_1 = require("./connection");
@@ -63,6 +64,15 @@ Object.defineProperty(exports, "Scannable", { enumerable: true, get: function ()
 exports.rerankers = require("./rerankers");
 var util_1 = require("./util");
 Object.defineProperty(exports, "packBits", { enumerable: true, get: function () { return util_1.packBits; } });
+/**
+ * Tokenize a full-text search query using an explicit tokenizer.
+ *
+ * This does not require a table or FTS index. The tokenizer options match
+ * {@link Index.fts}.
+ */
+async function tokenize(query, options) {
+    return await (0, native_js_1.tokenize)(query, options?.baseTokenizer, options?.language, options?.maxTokenLength, options?.lowercase, options?.stem, options?.removeStopWords, options?.asciiFolding, options?.ngramMinLength, options?.ngramMaxLength, options?.prefixOnly);
+}
 async function connect(uriOrOptions, optionsOrSession, sessionOrHeaderProvider, headerProvider) {
     let uri;
     let finalOptions = {};

package/dist/indices.d.ts

@@ -453,6 +453,7 @@ export interface IvfFlatOptions {
      */
     sampleRate?: number;
 }
+export type BaseTokenizer = "simple" | "whitespace" | "raw" | "ngram" | "icu" | "icu/split" | `jieba/${string}` | `lindera/${string}`;
 /**
  * Options to create a full text search index
  */
@@ -475,8 +476,12 @@ export interface FtsOptions {
      * "whitespace" - Whitespace tokenizer. This tokenizer splits the text into tokens using whitespace as a delimiter.
      *
      * "raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
+     *
+     * "icu" - ICU dictionary-based word segmentation.
+     *
+     * "icu/split" - ICU segmentation with simple-style delimiter splitting.
      */
-    baseTokenizer?: "simple" | "whitespace" | "raw" | "ngram";
+    baseTokenizer?: BaseTokenizer;
     /**
      * language for stemming and stop words
      * this is only used when `stem` or `remove_stop_words` is true

package/dist/native.d.ts

@@ -249,6 +249,7 @@ export declare class Table {
   currentBranch(): string | null
   optimize(olderThanMs?: number | undefined | null, deleteUnverified?: boolean | undefined | null): Promise<OptimizeStats>
   listIndices(): Promise<Array<IndexConfig>>
+  tokenize(query: string, column?: string | undefined | null, indexName?: string | undefined | null): Promise<Array<FtsToken>>
   indexStats(indexName: string): Promise<IndexStatistics | null>
   mergeInsert(on: Array<string>): NativeMergeInsertBuilder
   usesV2ManifestPaths(): Promise<boolean>
@@ -554,6 +555,14 @@ export interface FragmentSummaryStats {
   p99: number
 }
 
+/** A token produced by the tokenizer configured on a full-text search index. */
+export interface FtsToken {
+  /** The token text after the index tokenizer has applied its filters. */
+  text: string
+  /** The token position used by full-text query matching. */
+  position: number
+}
+
 /** A description of an index currently configured on a column */
 export interface IndexConfig {
   /** The name of the index */
@@ -925,6 +934,8 @@ export interface TlsConfig {
   assertHostname?: boolean
 }
 
+export declare function tokenize(query: string, baseTokenizer?: string | undefined | null, language?: string | undefined | null, maxTokenLength?: number | undefined | null, lowerCase?: boolean | undefined | null, stem?: boolean | undefined | null, removeStopWords?: boolean | undefined | null, asciiFolding?: boolean | undefined | null, ngramMinLength?: number | undefined | null, ngramMaxLength?: number | undefined | null, prefixOnly?: boolean | undefined | null): Array<FtsToken>
+
 export interface UpdateFieldMetadataResult {
   version: number
 }

package/dist/native.js

@@ -76,8 +76,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-android-arm64');
                 const bindingPackageVersion = require('@lancedb/lancedb-android-arm64/package.json').version;
-                if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -95,8 +95,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-android-arm-eabi');
                 const bindingPackageVersion = require('@lancedb/lancedb-android-arm-eabi/package.json').version;
-                if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -120,8 +120,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-win32-x64-gnu');
                     const bindingPackageVersion = require('@lancedb/lancedb-win32-x64-gnu/package.json').version;
-                    if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -139,8 +139,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-win32-x64-msvc');
                     const bindingPackageVersion = require('@lancedb/lancedb-win32-x64-msvc/package.json').version;
-                    if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -159,8 +159,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-win32-ia32-msvc');
                 const bindingPackageVersion = require('@lancedb/lancedb-win32-ia32-msvc/package.json').version;
-                if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -178,8 +178,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-win32-arm64-msvc');
                 const bindingPackageVersion = require('@lancedb/lancedb-win32-arm64-msvc/package.json').version;
-                if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -201,8 +201,8 @@ function requireNative() {
         try {
             const binding = require('@lancedb/lancedb-darwin-universal');
             const bindingPackageVersion = require('@lancedb/lancedb-darwin-universal/package.json').version;
-            if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+            if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
             }
             return binding;
         }
@@ -219,8 +219,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-darwin-x64');
                 const bindingPackageVersion = require('@lancedb/lancedb-darwin-x64/package.json').version;
-                if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -238,8 +238,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-darwin-arm64');
                 const bindingPackageVersion = require('@lancedb/lancedb-darwin-arm64/package.json').version;
-                if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -262,8 +262,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-freebsd-x64');
                 const bindingPackageVersion = require('@lancedb/lancedb-freebsd-x64/package.json').version;
-                if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -281,8 +281,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-freebsd-arm64');
                 const bindingPackageVersion = require('@lancedb/lancedb-freebsd-arm64/package.json').version;
-                if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -306,8 +306,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-x64-musl');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-x64-musl/package.json').version;
-                    if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -325,8 +325,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-x64-gnu');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-x64-gnu/package.json').version;
-                    if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -346,8 +346,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-arm64-musl');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-arm64-musl/package.json').version;
-                    if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -365,8 +365,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-arm64-gnu');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-arm64-gnu/package.json').version;
-                    if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -386,8 +386,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-arm-musleabihf');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-arm-musleabihf/package.json').version;
-                    if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -405,8 +405,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-arm-gnueabihf');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-arm-gnueabihf/package.json').version;
-                    if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -426,8 +426,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-loong64-musl');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-loong64-musl/package.json').version;
-                    if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -445,8 +445,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-loong64-gnu');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-loong64-gnu/package.json').version;
-                    if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -466,8 +466,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-riscv64-musl');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-riscv64-musl/package.json').version;
-                    if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -485,8 +485,8 @@ function requireNative() {
                 try {
                     const binding = require('@lancedb/lancedb-linux-riscv64-gnu');
                     const bindingPackageVersion = require('@lancedb/lancedb-linux-riscv64-gnu/package.json').version;
-                    if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                    if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                        throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                     }
                     return binding;
                 }
@@ -505,8 +505,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-linux-ppc64-gnu');
                 const bindingPackageVersion = require('@lancedb/lancedb-linux-ppc64-gnu/package.json').version;
-                if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -524,8 +524,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-linux-s390x-gnu');
                 const bindingPackageVersion = require('@lancedb/lancedb-linux-s390x-gnu/package.json').version;
-                if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -548,8 +548,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-openharmony-arm64');
                 const bindingPackageVersion = require('@lancedb/lancedb-openharmony-arm64/package.json').version;
-                if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -567,8 +567,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-openharmony-x64');
                 const bindingPackageVersion = require('@lancedb/lancedb-openharmony-x64/package.json').version;
-                if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -586,8 +586,8 @@ function requireNative() {
             try {
                 const binding = require('@lancedb/lancedb-openharmony-arm');
                 const bindingPackageVersion = require('@lancedb/lancedb-openharmony-arm/package.json').version;
-                if (bindingPackageVersion !== '0.32.0-beta.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
+                if (bindingPackageVersion !== '0.32.0-beta.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+                    throw new Error(`Native binding package version mismatch, expected 0.32.0-beta.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`);
                 }
                 return binding;
             }
@@ -684,3 +684,4 @@ module.exports.lancedbMetricsCatalog = nativeBinding.lancedbMetricsCatalog;
 module.exports.permutationBuilder = nativeBinding.permutationBuilder;
 module.exports.registerLancedbMetricsRecorder = nativeBinding.registerLancedbMetricsRecorder;
 module.exports.snapshotLancedbMetrics = nativeBinding.snapshotLancedbMetrics;
+module.exports.tokenize = nativeBinding.tokenize;

package/dist/table.d.ts

@@ -108,6 +108,22 @@ export interface Version {
     timestamp: Date;
     metadata: Record<string, string>;
 }
+/** Token produced by the tokenizer configured on a full-text search index. */
+export interface FtsToken {
+    /** Token text after tokenizer filters have been applied. */
+    text: string;
+    /** Token position used by full-text query matching. */
+    position: number;
+}
+export type TokenizeTableOptions = {
+    /** FTS-indexed column whose tokenizer should be used. */
+    column: string;
+    indexName?: never;
+} | {
+    /** Name of the FTS index whose tokenizer should be used. */
+    indexName: string;
+    column?: never;
+};
 /**
  * Specification selecting Lance's MemWAL LSM-style write path for
  * `mergeInsert`.
@@ -616,6 +632,16 @@ export declare abstract class Table {
     abstract optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
     /** List all indices that have been created with {@link Table.createIndex} */
     abstract listIndices(): Promise<IndexConfig[]>;
+    /**
+     * Tokenize a full-text search query using the tokenizer configured on an FTS index.
+     *
+     * Specify exactly one of `column` or `indexName`.
+     *
+     * Model-backed tokenizers such as `jieba/*` and `lindera/*` are rebuilt in
+     * the client process from index metadata. For remote tables, this means the
+     * same tokenizer model files must also exist locally.
+     */
+    abstract tokenize(query: string, options: TokenizeTableOptions): Promise<FtsToken[]>;
     /** Return the table as an arrow table */
     abstract toArrow(): Promise<ArrowTable>;
     abstract mergeInsert(on: string | string[]): MergeInsertBuilder;
@@ -704,6 +730,7 @@ export declare class LocalTable extends Table {
     currentBranch(): string | null;
     optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
     listIndices(): Promise<IndexConfig[]>;
+    tokenize(query: string, options: TokenizeTableOptions): Promise<FtsToken[]>;
     toArrow(): Promise<ArrowTable>;
     indexStats(name: string): Promise<IndexStatistics | undefined>;
     stats(): Promise<TableStatistics>;

package/dist/table.js

@@ -337,6 +337,9 @@ class LocalTable extends Table {
     async listIndices() {
         return await this.inner.listIndices();
     }
+    async tokenize(query, options) {
+        return await this.inner.tokenize(query, options?.column, options?.indexName);
+    }
     async toArrow() {
         return await this.query().toArrow();
     }

package/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.32.0-beta.1",
+  "version": "0.32.0-beta.2",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -106,13 +106,13 @@
   "optionalDependencies": {
     "@huggingface/transformers": "3.0.2",
     "openai": "4.29.2",
-    "@lancedb/lancedb-darwin-arm64": "0.32.0-beta.1",
-    "@lancedb/lancedb-linux-x64-gnu": "0.32.0-beta.1",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.32.0-beta.1",
-    "@lancedb/lancedb-linux-x64-musl": "0.32.0-beta.1",
-    "@lancedb/lancedb-linux-arm64-musl": "0.32.0-beta.1",
-    "@lancedb/lancedb-win32-x64-msvc": "0.32.0-beta.1",
-    "@lancedb/lancedb-win32-arm64-msvc": "0.32.0-beta.1"
+    "@lancedb/lancedb-darwin-arm64": "0.32.0-beta.2",
+    "@lancedb/lancedb-linux-x64-gnu": "0.32.0-beta.2",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.32.0-beta.2",
+    "@lancedb/lancedb-linux-x64-musl": "0.32.0-beta.2",
+    "@lancedb/lancedb-linux-arm64-musl": "0.32.0-beta.2",
+    "@lancedb/lancedb-win32-x64-msvc": "0.32.0-beta.2",
+    "@lancedb/lancedb-win32-arm64-msvc": "0.32.0-beta.2"
   },
   "peerDependencies": {
     "apache-arrow": ">=15.0.0 <=18.1.0"