@lancedb/lancedb 0.14.1-beta.2 → 0.14.1-beta.4

package/dist/indices.d.ts

@@ -314,6 +314,45 @@ export interface FtsOptions {
      * which will make the index smaller and faster to build, but will not support phrase queries.
      */
     withPosition?: boolean;
+    /**
+     * The tokenizer to use when building the index.
+     * The default is "simple".
+     *
+     * The following tokenizers are available:
+     *
+     * "simple" - Simple tokenizer. This tokenizer splits the text into tokens using whitespace and punctuation as a delimiter.
+     *
+     * "whitespace" - Whitespace tokenizer. This tokenizer splits the text into tokens using whitespace as a delimiter.
+     *
+     * "raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
+     */
+    baseTokenizer?: "simple" | "whitespace" | "raw";
+    /**
+     * language for stemming and stop words
+     * this is only used when `stem` or `remove_stop_words` is true
+     */
+    language?: string;
+    /**
+     * maximum token length
+     * tokens longer than this length will be ignored
+     */
+    maxTokenLength?: number;
+    /**
+     * whether to lowercase tokens
+     */
+    lowercase?: boolean;
+    /**
+     * whether to stem tokens
+     */
+    stem?: boolean;
+    /**
+     * whether to remove stop words
+     */
+    removeStopWords?: boolean;
+    /**
+     * whether to remove punctuation
+     */
+    asciiFolding?: boolean;
 }
 export declare class Index {
     private readonly inner;

package/dist/indices.js

@@ -101,7 +101,7 @@ class Index {
      * For now, the full text search index only supports English, and doesn't support phrase search.
      */
     static fts(options) {
-        return new Index(native_1.Index.fts(options?.withPosition));
+        return new Index(native_1.Index.fts(options?.withPosition, options?.baseTokenizer, options?.language, options?.maxTokenLength, options?.lowercase, options?.stem, options?.removeStopWords, options?.asciiFolding));
     }
     /**
      *

package/dist/native.d.ts

@@ -261,7 +261,7 @@ export class Index {
   static btree(): Index
   static bitmap(): Index
   static labelList(): Index
-  static fts(withPosition?: boolean | undefined | null): Index
+  static fts(withPosition?: boolean | undefined | null, baseTokenizer?: string | undefined | null, language?: string | undefined | null, maxTokenLength?: number | undefined | null, lowerCase?: boolean | undefined | null, stem?: boolean | undefined | null, removeStopWords?: boolean | undefined | null, asciiFolding?: boolean | undefined | null): Index
   static hnswPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null, m?: number | undefined | null, efConstruction?: number | undefined | null): Index
   static hnswSq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null, m?: number | undefined | null, efConstruction?: number | undefined | null): Index
 }

package/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.14.1-beta.2",
+  "version": "0.14.1-beta.4",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -98,14 +98,14 @@
     "reflect-metadata": "^0.2.2"
   },
   "optionalDependencies": {
-    "@lancedb/lancedb-darwin-x64": "0.14.1-beta.2",
-    "@lancedb/lancedb-darwin-arm64": "0.14.1-beta.2",
-    "@lancedb/lancedb-linux-x64-gnu": "0.14.1-beta.2",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.14.1-beta.2",
-    "@lancedb/lancedb-linux-x64-musl": "0.14.1-beta.2",
-    "@lancedb/lancedb-linux-arm64-musl": "0.14.1-beta.2",
-    "@lancedb/lancedb-win32-x64-msvc": "0.14.1-beta.2",
-    "@lancedb/lancedb-win32-arm64-msvc": "0.14.1-beta.2"
+    "@lancedb/lancedb-darwin-x64": "0.14.1-beta.4",
+    "@lancedb/lancedb-darwin-arm64": "0.14.1-beta.4",
+    "@lancedb/lancedb-linux-x64-gnu": "0.14.1-beta.4",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.14.1-beta.4",
+    "@lancedb/lancedb-linux-x64-musl": "0.14.1-beta.4",
+    "@lancedb/lancedb-linux-arm64-musl": "0.14.1-beta.4",
+    "@lancedb/lancedb-win32-x64-msvc": "0.14.1-beta.4",
+    "@lancedb/lancedb-win32-arm64-msvc": "0.14.1-beta.4"
   },
   "peerDependencies": {
     "apache-arrow": ">=15.0.0 <=18.1.0"