@elizaos/agent 0.25.9 → 2.0.0-alpha.151

package/packages/typescript/src/search.js

@@ -0,0 +1,1580 @@
+// Implementation of BM25 and Porter2 stemming
+// https://github.com/eilvelia/porter2.js
+// https://www.npmjs.com/package/fast-bm25
+// The MIT License
+// Copyright (c) 2024 eilvelia <hi@eilvelia.cat>
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+// MIT License
+// Copyright (c) 2024 Vivek Patel <me@patelvivek.dev>.
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+/**
+ * Checks if the character code represents a vowel (a, e, i, o, u, y).
+ * @param char - The character code.
+ * @returns True if the character is a vowel, false otherwise.
+ */
+const isV = (char) => {
+    switch (char) {
+        case 97:
+        case 101:
+        case 105:
+        case 111:
+        case 117:
+        case 121:
+            return true;
+        default:
+            return false;
+    }
+};
+/**
+ * Checks if the character code is 'w', 'x', 'y', or a vowel.
+ * Used in determining short syllables.
+ * @param char - The character code.
+ * @returns True if the character is w, x, y, or a vowel, false otherwise.
+ */
+const isWxy = (char) => {
+    switch (char) {
+        case 97:
+        case 101:
+        case 105:
+        case 111:
+        case 117:
+        case 121:
+        case 119:
+        case 120:
+        case 89:
+            return true;
+        default:
+            return false;
+    }
+};
+/**
+ * Checks if the character code is one of the valid endings for Step 1c ('li' rule).
+ * Valid endings: c, d, e, g, h, k, m, n, r, t.
+ * @param char - The character code.
+ * @returns True if the character is a valid 'li' ending, false otherwise.
+ */
+const isValidLi = (char) => {
+    switch (char) {
+        case 99:
+        case 100:
+        case 101:
+        case 103:
+        case 104:
+        case 107:
+        case 109:
+        case 110:
+        case 114:
+        case 116:
+            return true;
+        default:
+            return false;
+    }
+};
+/**
+ * Checks if the character code represents a "double" consonant sound
+ * (bb, dd, ff, gg, mm, nn, pp, rr, tt).
+ * @param char - The character code.
+ * @returns True if the character forms a double consonant, false otherwise.
+ */
+const isDouble = (char) => {
+    switch (char) {
+        case 98:
+        case 100:
+        case 102:
+        case 103:
+        case 109:
+        case 110:
+        case 112:
+        case 114:
+        case 116:
+            return true;
+        default:
+            return false;
+    }
+};
+/**
+ * Checks if a word ends in a short syllable.
+ * A short syllable is defined as:
+ * 1. A vowel followed by a non-vowel at the end of the word. (e.g., "hop")
+ * 2. A vowel followed by a non-vowel followed by a non-vowel ('w', 'x', 'y' excluded). (e.g., "trap")
+ * @param w - Array of character codes representing the word.
+ * @param len - The current effective length of the word being considered.
+ * @returns True if the word ends in a short syllable, false otherwise.
+ */
+const isShortV = (w, len) => {
+    // backwardmode: ( non-v_WXY v non-v ) or ( non-v v atlimit )
+    return (len >= 2 &&
+        isV(w[len - 2]) &&
+        ((len === 2 && !isV(w[len - 1])) ||
+            (len >= 3 && !isV(w[len - 3]) && !isWxy(w[len - 1]))));
+};
+// #endregion Porter2 Stemmer Helper Functions
+// #region Porter2 Stemmer Algorithm
+/**
+ * Stems a given word using the Porter2 (Snowball English) stemming algorithm.
+ *
+ * The algorithm works in several steps, applying suffix stripping rules based on
+ * regions R1 and R2 within the word.
+ * - R1: The region after the first non-vowel following a vowel.
+ * - R2: The region after the first non-vowel following a vowel in R1.
+ *
+ * The steps generally involve:
+ * 1. Handling plurals and past participles (-s, -es, -ed, -ing).
+ * 2. Turning terminal 'y' to 'i' if there is another vowel in the stem.
+ * 3. Mapping double suffixes to single ones (e.g., -ization to -ize).
+ * 4. Dealing with suffixes like -full, -ness, etc.
+ * 5. Removing suffixes like -ant, -ence, etc.
+ * 6. Removing a final -e.
+ *
+ * @param word - The word to be stemmed.
+ * @returns The stemmed version of the word.
+ */
+const stem = (word) => {
+    if (word.length < 3)
+        return word;
+    // exception1
+    if (word.length <= 6) {
+        switch (word) {
+            case "ski":
+                return "ski";
+            case "skies":
+                return "sky";
+            case "dying":
+                return "die";
+            case "lying":
+                return "lie";
+            case "tying":
+                return "tie";
+            // special -LY cases
+            case "idly":
+                return "idl";
+            case "gently":
+                return "gentl";
+            case "ugly":
+                return "ugli";
+            case "early":
+                return "earli";
+            case "only":
+                return "onli";
+            case "singly":
+                return "singl";
+            // invariant forms
+            case "sky":
+            case "news":
+            case "howe":
+            // not plural forms
+            case "atlas":
+            case "cosmos":
+            case "bias":
+            case "andes":
+                return word;
+        }
+    }
+    const initialOffset = word.charCodeAt(0) === 39 /* ' */ ? 1 : 0;
+    let l = word.length - initialOffset;
+    const w = new Array(l);
+    let yFound = false;
+    for (let i = 0; i < l; ++i) {
+        const ch = word.charCodeAt(i + initialOffset);
+        if (ch === 121 && (i === 0 || isV(w[i - 1]))) {
+            yFound = true;
+            w[i] = 89;
+            continue;
+        }
+        w[i] = ch;
+    }
+    if (w[l - 1] === 39 /* ' */)
+        --l;
+    if (l >= 2 && w[l - 2] === 39 /* ' */ && w[l - 1] === 115 /* s */)
+        l -= 2;
+    // mark_regions
+    let rv = 0;
+    // rv is the position after the first vowel
+    while (rv < l && !isV(w[rv]))
+        ++rv;
+    if (rv < l)
+        ++rv;
+    let r1 = rv;
+    if (l >= 5 &&
+        ((w[0] === 103 &&
+            w[1] === 101 &&
+            w[2] === 110 &&
+            w[3] === 101 &&
+            w[4] === 114) || // gener
+            (w[0] === 97 &&
+                w[1] === 114 &&
+                w[2] === 115 &&
+                w[3] === 101 &&
+                w[4] === 110)) // arsen
+    )
+        r1 = 5;
+    else if (l >= 6 &&
+        w[0] === 99 && // c
+        w[1] === 111 && // o
+        w[2] === 109 && // m
+        w[3] === 109 && // m
+        w[4] === 117 && // u
+        w[5] === 110 // n
+    )
+        // commun
+        r1 = 6;
+    else {
+        // > R1 is the region after the first non-vowel following a vowel,
+        // > or the end of the word if there is no such non-vowel.
+        while (r1 < l && isV(w[r1]))
+            ++r1;
+        if (r1 < l)
+            ++r1;
+    }
+    // > R2 is the region after the first non-vowel following a vowel in R1,
+    // > or the end of the word if there is no such non-vowel.
+    let r2 = r1;
+    while (r2 < l && !isV(w[r2]))
+        ++r2;
+    while (r2 < l && isV(w[r2]))
+        ++r2;
+    if (r2 < l)
+        ++r2;
+    // Step_1a
+    if (l >= 3) {
+        if (w[l - 1] === 115) {
+            // s
+            if (l >= 4 && w[l - 2] === 101 && w[l - 3] === 115 && w[l - 4] === 115)
+                // sses
+                l -= 2; // sses -> ss
+            else if (w[l - 2] === 101 && w[l - 3] === 105)
+                // ies
+                l -= l >= 5 ? 2 : 1; // ies
+            else if (w[l - 2] !== 117 && w[l - 2] !== 115 && rv < l - 1)
+                // us ss -> <nothing>; s -> "delete if the preceding word part
+                // contains a vowel not immediately before the s"
+                l -= 1;
+        }
+        else if (w[l - 1] === 100 && w[l - 2] === 101 && w[l - 3] === 105)
+            l -= l >= 5 ? 2 : 1; // ied
+    }
+    // exception2
+    if ((l === 6 &&
+        ((w[0] === 105 && // i
+            w[1] === 110 && // n
+            w[2] === 110 && // n
+            w[3] === 105 && // i
+            w[4] === 110 && // n
+            w[5] === 103) || // g (inning)
+            (w[0] === 111 && // o
+                w[1] === 117 && // u
+                w[2] === 116 && // t
+                w[3] === 105 && // i
+                w[4] === 110 && // n
+                w[5] === 103) || // g (outing)
+            (w[0] === 101 && // e
+                w[1] === 120 && // x
+                w[2] === 99 && // c
+                w[3] === 101 && // e
+                w[4] === 101 && // e
+                w[5] === 100))) || // d (exceed)
+        (l === 7 &&
+            ((w[0] === 99 && // c
+                w[1] === 97 && // a
+                w[2] === 110 && // n
+                w[3] === 110 && // n
+                w[4] === 105 && // i
+                w[5] === 110 && // n
+                w[6] === 103) || // g (canning)
+                (w[0] === 104 && // h
+                    w[1] === 101 && // e
+                    w[2] === 114 && // r
+                    w[3] === 114 && // r
+                    w[4] === 105 && // i
+                    w[5] === 110 && // n
+                    w[6] === 103) || // g (herring)
+                (w[0] === 101 && // e
+                    w[1] === 97 && // a
+                    w[2] === 114 && // r
+                    w[3] === 114 && // r
+                    w[4] === 105 && // i
+                    w[5] === 110 && // n
+                    w[6] === 103) || // g (earring)
+                (w[0] === 112 && // p
+                    w[1] === 114 && // r
+                    w[2] === 111 && // o
+                    w[3] === 99 && // c
+                    w[4] === 101 && // e
+                    w[5] === 101 && // e
+                    w[6] === 100) || // d (proceed)
+                (w[0] === 115 && // s
+                    w[1] === 117 && // u
+                    w[2] === 99 && // c
+                    w[3] === 99 && // c
+                    w[4] === 101 && // e
+                    w[5] === 101 && // e
+                    w[6] === 100))) // d (succeed)
+    ) {
+        let exp2Out = "";
+        for (let i = 0; i < l; ++i)
+            exp2Out += String.fromCharCode(w[i]);
+        return exp2Out;
+    }
+    // Step_1b
+    let ll = 
+    // l (length) without the -ly ending
+    l >= 2 && w[l - 1] === 121 && w[l - 2] === 108 ? l - 2 : l;
+    if (ll >= 3) {
+        if (w[ll - 3] === 101 && w[ll - 2] === 101 && w[ll - 1] === 100) {
+            // eed
+            if (ll >= r1 + 3)
+                l = ll - 1; // eed eedly -> ee (if in R1)
+        }
+        else {
+            // ll without: ed edly ing ingly (-1 if not found)
+            if (w[ll - 2] === 101 && w[ll - 1] === 100)
+                ll -= 2; // ed
+            else if (w[ll - 3] === 105 && w[ll - 2] === 110 && w[ll - 1] === 103)
+                ll -= 3; // ing
+            else
+                ll = -1;
+            if (ll >= 0 && rv <= ll) {
+                l = ll;
+                if (l >= 2) {
+                    if ((w[l - 1] === 116 && w[l - 2] === 97) || // at
+                        (w[l - 1] === 108 && w[l - 2] === 98) || // bl
+                        (w[l - 1] === 122 && w[l - 2] === 105) // iz
+                    ) {
+                        // at -> ate   bl -> ble   iz -> ize
+                        w[l] = 101;
+                        ++l;
+                    }
+                    else if (w[l - 2] === w[l - 1] && isDouble(w[l - 1])) {
+                        --l;
+                    }
+                    else if (r1 >= l && isShortV(w, l)) {
+                        // <shortv> -> e
+                        w[l] = 101;
+                        ++l;
+                    }
+                }
+            }
+        }
+    }
+    // Step_1c
+    if (l >= 3 && (w[l - 1] === 89 || w[l - 1] === 121) && !isV(w[l - 2]))
+        w[l - 1] = 105; // i
+    // Step_2
+    if (l >= r1 + 2) {
+        switch (w[l - 1]) {
+            case 108: // l
+                if (l >= r1 + 6 &&
+                    w[l - 2] === 97 && // a
+                    w[l - 3] === 110 && // n
+                    w[l - 4] === 111 && // o
+                    w[l - 5] === 105 && // i
+                    w[l - 6] === 116 // t (tional)
+                ) {
+                    if (l >= 7 && w[l - 7] === 97) {
+                        // a (ational)
+                        if (l >= r1 + 7) {
+                            // ational -> ate
+                            l -= 4;
+                            w[l - 1] = 101; // e
+                        }
+                    }
+                    else {
+                        l -= 2; // tional -> tion
+                    }
+                }
+                break;
+            case 110: // n
+                if (l >= r1 + 5 &&
+                    w[l - 2] === 111 && // o
+                    w[l - 3] === 105 && // i
+                    w[l - 4] === 116 && // t
+                    w[l - 5] === 97 // a (ation)
+                ) {
+                    if (l >= 7 && w[l - 6] === 122 && w[l - 7] === 105) {
+                        // iz (ization)
+                        if (l >= r1 + 7) {
+                            // ization -> ize
+                            l -= 4;
+                            w[l - 1] = 101; // e
+                        }
+                    }
+                    else {
+                        // ation -> ate
+                        l -= 2;
+                        w[l - 1] = 101; // e
+                    }
+                }
+                break;
+            case 114: // r
+                if (l >= r1 + 4) {
+                    if (w[l - 2] === 101) {
+                        // e (er)
+                        if (w[l - 3] === 122 && w[l - 4] === 105)
+                            --l; // izer -> ize
+                    }
+                    else if (w[l - 2] === 111) {
+                        // o (or)
+                        if (w[l - 3] === 116 && w[l - 4] === 97) {
+                            // ator
+                            --l;
+                            w[l - 1] = 101; // e
+                        }
+                    }
+                }
+                break;
+            case 115: // s
+                if (l >= r1 + 7 &&
+                    w[l - 2] === 115 && // s
+                    w[l - 3] === 101 && // e
+                    w[l - 4] === 110 && // n (ness)
+                    ((w[l - 5] === 108 && w[l - 6] === 117 && w[l - 7] === 102) || // fulness
+                        (w[l - 5] === 115 && w[l - 6] === 117 && w[l - 7] === 111) || // ousness
+                        (w[l - 5] === 101 && w[l - 6] === 118 && w[l - 7] === 105)) // iveness
+                ) {
+                    l -= 4; // fulness -> ful   ousness -> ous   iveness -> ive
+                }
+                break;
+            case 109: // m
+                if (l >= r1 + 5 &&
+                    w[l - 2] === 115 && // s
+                    w[l - 3] === 105 && // i
+                    w[l - 4] === 108 && // l
+                    w[l - 5] === 97 // a (alism)
+                )
+                    l -= 3; // alism -> al
+                break;
+            case 105: // i
+                if (w[l - 2] === 99) {
+                    // c (ic)
+                    if (l >= r1 + 4 &&
+                        (w[l - 4] === 101 || w[l - 4] === 97) &&
+                        w[l - 3] === 110) {
+                        // enci anci
+                        w[l - 1] = 101; // enci -> ence   anci -> ance
+                    }
+                }
+                else if (w[l - 2] === 103) {
+                    // g (gi)
+                    if (l >= r1 + 3 &&
+                        l >= 4 &&
+                        w[l - 2] === 103 &&
+                        w[l - 3] === 111 &&
+                        w[l - 4] === 108)
+                        // logi
+                        --l; // ogi -> og (if preceded by l)
+                }
+                else if (w[l - 2] === 116) {
+                    // t (ti)
+                    if (l >= r1 + 5 && w[l - 3] === 105) {
+                        // iti
+                        if (w[l - 4] === 108) {
+                            // liti
+                            if (l >= 6 && w[l - 5] === 105 && w[l - 6] === 98) {
+                                // biliti
+                                if (l >= r1 + 6) {
+                                    // biliti -> ble
+                                    l -= 3;
+                                    w[l - 2] = 108; // l
+                                    w[l - 1] = 101; // e
+                                }
+                            }
+                            else if (w[l - 4] === 108 && w[l - 5] === 97) {
+                                // aliti
+                                l -= 3; // aliti -> al
+                            }
+                        }
+                        else if (w[l - 4] === 118 && w[l - 5] === 105) {
+                            // iviti
+                            // iviti -> ive
+                            l -= 2;
+                            w[l - 1] = 101; // e
+                        }
+                    }
+                }
+                else if (w[l - 2] === 108 && l >= 3) {
+                    // l (li)
+                    if (w[l - 3] === 98) {
+                        // bli
+                        if (l >= 4 && w[l - 4] === 97) {
+                            // abli
+                            if (l >= r1 + 4)
+                                w[l - 1] = 101; // abli -> able
+                        }
+                        else if (l >= r1 + 3) {
+                            w[l - 1] = 101; // bli -> ble
+                        }
+                    }
+                    else {
+                        // Remove li
+                        if (w[l - 3] === 108) {
+                            // lli
+                            if (l >= 5 && w[l - 4] === 117 && w[l - 5] === 102) {
+                                // fulli
+                                if (l >= r1 + 5)
+                                    l -= 2; // fulli -> ful
+                            }
+                            else if (l >= r1 + 4 && w[l - 4] === 97) {
+                                // alli
+                                l -= 2; // alli -> al
+                            }
+                        }
+                        else if (w[l - 3] === 115) {
+                            // sli
+                            if (l >= 6 &&
+                                w[l - 4] === 115 &&
+                                w[l - 5] === 101 &&
+                                w[l - 6] === 108) {
+                                // lessli
+                                if (l >= r1 + 6)
+                                    l -= 2; // lessli -> less
+                            }
+                            else if (l >= r1 + 5 && w[l - 4] === 117 && w[l - 5] === 111) {
+                                // ousli
+                                l -= 2; // ousli -> ous
+                            }
+                        }
+                        else if (l >= 5 &&
+                            w[l - 3] === 116 &&
+                            w[l - 4] === 110 &&
+                            w[l - 5] === 101) {
+                            // entli
+                            if (l >= r1 + 5)
+                                l -= 2; // entli -> ent
+                        }
+                        else if (isValidLi(w[l - 3])) {
+                            l -= 2;
+                        }
+                    }
+                }
+        }
+    }
+    // Step_3
+    if (l >= r1 + 3) {
+        switch (w[l - 1]) {
+            case 108: // l
+                if (w[l - 3] === 99) {
+                    // cal
+                    if (l >= r1 + 4 && w[l - 4] === 105 && w[l - 2] === 97)
+                        l -= 2; // ical -> ic
+                }
+                else if (w[l - 3] === 102) {
+                    // ful
+                    if (w[l - 2] === 117)
+                        l -= 3; // ful -> <delete>
+                }
+                else if (w[l - 3] === 110) {
+                    // nal
+                    if (l >= r1 + 6 &&
+                        w[l - 2] === 97 && // a
+                        w[l - 4] === 111 && // o
+                        w[l - 5] === 105 && // i
+                        w[l - 6] === 116 // t (tional)
+                    ) {
+                        if (l >= 7 && w[l - 7] === 97) {
+                            // ational
+                            if (l >= r1 + 7) {
+                                // ational -> ate
+                                l -= 4;
+                                w[l - 1] = 101; // e
+                            }
+                        }
+                        else {
+                            l -= 2; // tional -> tion
+                        }
+                    }
+                }
+                break;
+            case 101: // e
+                if (w[l - 2] === 122) {
+                    // ze
+                    if (l >= r1 + 5 &&
+                        w[l - 3] === 105 &&
+                        w[l - 4] === 108 &&
+                        w[l - 5] === 97)
+                        l -= 3; // alize -> al
+                }
+                else if (w[l - 2] === 116) {
+                    // te
+                    if (l >= r1 + 5 &&
+                        w[l - 3] === 97 &&
+                        w[l - 4] === 99 &&
+                        w[l - 5] === 105)
+                        l -= 3; // icate -> ic
+                }
+                else if (w[l - 2] === 118) {
+                    // ve
+                    if (l >= r2 + 5 &&
+                        w[l - 3] === 105 &&
+                        w[l - 4] === 116 &&
+                        w[l - 5] === 97)
+                        l -= 5; // ative -> <delete> (if in R2)
+                }
+                break;
+            case 105: // i
+                if (l >= r1 + 5 &&
+                    w[l - 2] === 116 && // t
+                    w[l - 3] === 105 && // i
+                    w[l - 4] === 99 && // c
+                    w[l - 5] === 105 // i (iciti)
+                )
+                    l -= 3; // iciti -> ic
+                break;
+            case 115: // s
+                if (l >= r1 + 4 &&
+                    w[l - 2] === 115 &&
+                    w[l - 3] === 101 &&
+                    w[l - 4] === 110)
+                    l -= 4; // ness -> <delete>
+        }
+    }
+    // Step_4
+    if (l >= r2 + 2) {
+        switch (w[l - 1]) {
+            case 110: // n
+                if (l >= r2 + 3 &&
+                    w[l - 2] === 111 && // o
+                    w[l - 3] === 105 && // i (ion)
+                    (w[l - 4] === 115 || w[l - 4] === 116) // s or t
+                )
+                    l -= 3; // ion -> <delete> (if preceded by s or t)
+                break;
+            case 108: // l
+                if (w[l - 2] === 97)
+                    l -= 2; // al
+                break;
+            case 114: // r
+                if (w[l - 2] === 101)
+                    l -= 2; // er
+                break;
+            case 99: // c
+                if (w[l - 2] === 105)
+                    l -= 2; // ic
+                break;
+            case 109: // m
+                if (l >= r2 + 3 && w[l - 2] === 115 && w[l - 3] === 105)
+                    l -= 3; // ism
+                break;
+            case 105: // i
+                if (l >= r2 + 3 && w[l - 2] === 116 && w[l - 3] === 105)
+                    l -= 3; // iti
+                break;
+            case 115: // s
+                if (l >= r2 + 3 && w[l - 2] === 117 && w[l - 3] === 111)
+                    l -= 3; // ous
+                break;
+            case 116: // t
+                if (l >= r2 + 3 && w[l - 2] === 110) {
+                    // nt
+                    if (w[l - 3] === 97) {
+                        // ant
+                        l -= 3; // ant
+                    }
+                    else if (w[l - 3] === 101) {
+                        // ent
+                        if (l >= 4 && w[l - 4] === 109) {
+                            // ment
+                            if (l >= 5 && w[l - 5] === 101) {
+                                // ement
+                                if (l >= r2 + 5)
+                                    l -= 5; // ement
+                            }
+                            else if (l >= r2 + 4) {
+                                l -= 4; // ment
+                            }
+                        }
+                        else {
+                            l -= 3; // ent
+                        }
+                    }
+                }
+                break;
+            case 101: // e
+                if (w[l - 2] === 99) {
+                    // ce
+                    if (l >= r2 + 4 &&
+                        w[l - 3] === 110 &&
+                        (w[l - 4] === 97 || w[l - 4] === 101))
+                        l -= 4; // ance  ence
+                }
+                else if (w[l - 2] === 108) {
+                    // le
+                    if (l >= r2 + 4 &&
+                        w[l - 3] === 98 &&
+                        (w[l - 4] === 97 || w[l - 4] === 105))
+                        l -= 4; // able  ible
+                }
+                else if (w[l - 2] === 116) {
+                    // te
+                    if (l >= r2 + 3 && w[l - 3] === 97)
+                        l -= 3; // ate
+                }
+                else if (l >= r2 + 3 &&
+                    (w[l - 2] === 118 || w[l - 2] === 122) &&
+                    w[l - 3] === 105) {
+                    // ive ize
+                    l -= 3; // ive  ize
+                }
+        }
+    }
+    // Step_5
+    if (l >= r1 + 1 && // r1 is >= 1
+        ((l >= r2 + 1 && w[l - 1] === 108 && w[l - 2] === 108) || // ll
+            (w[l - 1] === 101 && (l >= r2 + 1 || !isShortV(w, l - 1)))) // e
+    )
+        --l;
+    let out = "";
+    if (yFound) {
+        for (let i = 0; i < l; ++i) {
+            out += String.fromCharCode(w[i] === 89 ? 121 : w[i]); // Y -> y
+        }
+    }
+    else {
+        for (let i = 0; i < l; ++i)
+            out += String.fromCharCode(w[i]);
+    }
+    return out;
+};
+// #endregion Porter2 Stemmer Algorithm
+// src/constants.ts
+const DEFAULT_OPTIONS = {
+    k1: 1.2,
+    b: 0.75,
+    minLength: 2,
+    stopWords: /* @__PURE__ */ new Set([
+        "a",
+        "an",
+        "and",
+        "are",
+        "as",
+        "at",
+        "be",
+        "by",
+        "for",
+        "from",
+        "has",
+        "he",
+        "in",
+        "is",
+        "it",
+        "its",
+        "of",
+        "on",
+        "that",
+        "the",
+        "to",
+        "was",
+        "were",
+        "will",
+        "with",
+    ]),
+    stemming: false,
+    stemWords: (word) => word,
+};
+/**
+ * Flexible text tokenizer with support for stop words, minimum token length,
+ * Unicode normalization, and optional Porter2 stemming with custom rules.
+ */
+export class Tokenizer {
+    /** Set of stop words to ignore. */
+    stopWords;
+    /** Minimum length of tokens to keep. */
+    minLength;
+    /** Flag indicating if stemming is enabled. */
+    stemming;
+    /** Custom stemming rules. */
+    stemmingRules;
+    /** Default options for the Tokenizer. */
+    static DEFAULT_OPTIONS = {
+        stopWords: /* @__PURE__ */ new Set(),
+        minLength: 2,
+        stemming: false,
+        stemmingRules: [],
+    };
+    /**
+     * Creates a new tokenizer instance.
+     * @param options - Tokenization options including stop words, min length, stemming, and custom rules.
+     */
+    constructor(options = {}) {
+        const mergedOptions = { ...Tokenizer.DEFAULT_OPTIONS, ...options };
+        this.stopWords = mergedOptions.stopWords;
+        this.minLength = mergedOptions.minLength;
+        this.stemming = mergedOptions.stemming;
+        // Ensure all rule patterns are RegExp objects
+        this.stemmingRules = mergedOptions.stemmingRules.map((rule) => ({
+            ...rule,
+            pattern: typeof rule.pattern === "string"
+                ? new RegExp(rule.pattern)
+                : rule.pattern,
+        }));
+    }
+    /**
+     * Tokenizes input text into an array of processed terms.
+     * Steps:
+     * 1. Cleans the text (lowercase, normalize, remove punctuation/symbols).
+     * 2. Splits the text into potential tokens.
+     * 3. Filters tokens based on `minLength` and `stopWords`.
+     * 4. Applies stemming if `stemming` is true (custom rules first, then Porter2).
+     * 5. Optionally calculates statistics.
+     *
+     * @param text - The input text string to tokenize.
+     * @param includeStats - If true, returns tokenization statistics along with tokens. Defaults to false.
+     * @returns A `TokenizationResult` object containing the array of tokens and optional stats.
+     * @throws {Error} If the input text is null, undefined, or empty.
+     */
+    tokenize(text, includeStats = false) {
+        if (!text) {
+            throw new Error("Input text cannot be null or empty");
+        }
+        const startTime = includeStats ? Date.now() : 0;
+        const cleaned = this.cleanText(text);
+        const tokens = cleaned
+            .split(/\s+/)
+            .filter((token) => this.isValidToken(token))
+            .map((token) => (this.stemming ? this.stemWord(token) : token));
+        let stats;
+        if (includeStats) {
+            const originalWords = text.split(/\s+/).filter((word) => word.length > 0);
+            stats = {
+                originalWordCount: originalWords.length,
+                stopWordsRemoved: originalWords.length - tokens.length, // This might be incorrect if stemming changes token count
+                stemmedWords: this.stemming ? tokens.length : 0,
+                processingTimeMs: Date.now() - startTime,
+            };
+        }
+        return { tokens, stats };
+    }
+    /**
+     * Cleans and normalizes text for tokenization.
+     * - Converts to lowercase.
+     * - Normalizes Unicode characters (NFKD).
+     * - Removes control characters and zero-width spaces.
+     * - Removes diacritical marks (accents).
+     * - Removes emojis and pictographs.
+     * - Removes common symbols (™, ®, ©, ℠, ‼).
+     * - Replaces Unicode punctuation with spaces.
+     * - Removes characters not matching basic Latin, CJK, Hangul, or whitespace.
+     * - Collapses multiple spaces into single spaces.
+     * - Trims leading/trailing whitespace.
+     *
+     * @param text - Input text to clean.
+     * @returns Cleaned and normalized text, ready for splitting into tokens.
+     *
+     * @example
+     * cleanText("Hello, World™!") // "hello world"
+     * cleanText("héllo 👋") // "hello"
+     * cleanText("Hello 世界!") // "hello 世界"
+     * cleanText("I'm don't") // "i'm don't" (apostrophes kept by replacing punctuation with space)
+     * cleanText("test©2023") // "test 2023"
+     */
+    cleanText(text) {
+        return (text
+            .toLowerCase()
+            .normalize("NFKD")
+            // biome-ignore lint/suspicious/noControlCharactersInRegex: Intentionally matching control characters to remove them
+            .replace(/[\u0000-\u001F\u007F-\u009F\u200B-\u200D\uFEFF]/g, "") // Control characters & zero-width spaces
+            .replace(/[\u0300-\u036f]/g, "") // Diacritical marks
+            .replace(/[\p{Emoji_Presentation}\p{Extended_Pictographic}]/gu, "") // Emojis and pictographs
+            .replace(/[™®©℠‼]/g, "") // Common symbols
+            .replace(/[\p{P}]/gu, " ") // Unicode punctuation to space
+            .replace(/[^a-z0-9\u3040-\u30FF\u3400-\u4DBF\u4E00-\u9FFF\uAC00-\uD7AF\s]/gu, " ") // Keep only latin, cjk, hangul, numbers, whitespace
+            .replace(/\s+/g, " ") // Collapse multiple spaces
+            .trim());
+    }
+    /**
+     * Checks if a token is valid (meets `minLength` criteria and is not a stop word).
+     * Numeric tokens are always considered valid regardless of length.
+     * @param token - The token string to validate.
+     * @returns `true` if the token is valid, `false` otherwise.
+     */
+    isValidToken(token) {
+        const isNumeric = /^\d+$/.test(token);
+        return ((token.length >= this.minLength || isNumeric) &&
+            !this.stopWords.has(token));
+    }
+    /**
+     * Applies stemming to a single word.
+     * First, tries to apply custom stemming rules defined in `stemmingRules`.
+     * If no custom rule matches, applies the default Porter2 stemming algorithm.
+     * Words shorter than 3 characters are not stemmed.
+     * @param word - The word to stem.
+     * @returns The stemmed word.
+     */
+    stemWord(word) {
+        if (word.length < 3)
+            return word;
+        let customRuleApplied = false;
+        let stemmed = word;
+        for (const rule of this.stemmingRules) {
+            const match = stemmed.match(rule.pattern);
+            if (match) {
+                if (!rule.minMeasure ||
+                    this.measure(stemmed.substring(0, match.index)) >= rule.minMeasure) {
+                    // Apply replacement
+                    if (typeof rule.replacement === "string") {
+                        stemmed = stemmed.replace(rule.pattern, rule.replacement);
+                    }
+                    else {
+                        // If replacement is a function, it might need more specific arguments based on its definition.
+                        // Assuming it takes the matched substring and potentially other match groups.
+                        stemmed = stemmed.replace(rule.pattern, (...args) => rule.replacement(...args));
+                    }
+                    customRuleApplied = true; // Mark that a custom rule was (potentially) applied
+                    // Depending on stemming strategy, might want to break or continue applying rules
+                }
+            }
+        }
+        // If a custom rule was applied and modified the word, return it.
+        // Otherwise, or if custom rules are meant to precede default stemming, apply Porter2.
+        if (customRuleApplied && stemmed !== word)
+            return stemmed; // Return if custom rule changed the word
+        // Fallback to Porter2 if no custom rule applied or if custom rules are pre-processing
+        return stem(stemmed); // Apply Porter2 to the (potentially already custom-stemmed) word
+    }
+    /**
+     * Checks if the character at a given index in a word is a consonant.
+     * Treats 'y' as a consonant if it's the first letter or follows a consonant.
+     * @param word - The word string.
+     * @param i - The index of the character to check.
+     * @returns `true` if the character is a consonant, `false` otherwise.
+     */
+    isConsonant(word, i) {
+        const char = word[i];
+        if ("aeiou".includes(char))
+            return false;
+        return char !== "y" || (i === 0 ? true : !this.isConsonant(word, i - 1));
+    }
+    /**
+     * Calculates the "measure" of a word stem (approximates syllable count).
+     * The measure (m) is the number of times a sequence of vowels is followed by a
+     * sequence of consonants (VC). Used in some stemming rules.
+     * Example: measure("tree") = 0, measure("trouble") = 1, measure("private") = 2
+     * @param word - The word (or stem) to measure.
+     * @returns The measure (m) of the word.
+     */
+    measure(word) {
+        let m = 0;
+        let vowelSeen = false;
+        for (let i = 0; i < word.length; i++) {
+            if (this.isConsonant(word, i)) {
+                if (vowelSeen) {
+                    m++;
+                    vowelSeen = false;
+                }
+            }
+            else {
+                vowelSeen = true;
+            }
+        }
+        return m;
+    }
+}
+/**
+ * Implements the Okapi BM25 (Best Matching 25) ranking function for information retrieval.
+ *
+ * BM25 ranks documents based on the query terms appearing in each document,
+ * considering term frequency (TF) and inverse document frequency (IDF).
+ * It improves upon basic TF-IDF by incorporating:
+ * - Term Frequency Saturation (k1): Prevents overly frequent terms from dominating the score.
+ * - Document Length Normalization (b): Penalizes documents that are longer than average,
+ *   assuming longer documents are more likely to contain query terms by chance.
+ *
+ * Key Components:
+ * - Tokenizer: Processes text into terms (words), handles stop words and stemming.
+ * - Document Indexing: Stores document lengths, term frequencies per document,
+ *   and overall document frequency for each term.
+ * - IDF Calculation: Measures the informativeness of a term based on how many documents contain it.
+ * - Scoring: Combines TF, IDF, document length, and parameters k1/b to calculate relevance.
+ */
+export class BM25 {
+    /** Term frequency saturation parameter (k1). */
+    termFrequencySaturation; // k1
+    /** Document length normalization factor (b). */
+    lengthNormalizationFactor; // b
+    /** Tokenizer instance used for processing text. */
+    tokenizer;
+    /** Array storing the length (number of tokens, adjusted by field boosts) of each document. */
+    documentLengths;
+    /** Average length of all documents in the index. */
+    averageDocLength;
+    /** Map from term (string) to its unique integer index. */
+    termToIndex;
+    /** Array storing the document frequency (number of docs containing the term) for each term index. */
+    documentFrequency; // DF for each term index
+    /** Map from term index to another map storing `docIndex: termFrequencyInDoc`. */
+    termFrequencies; // TermIndex -> { DocIndex -> TF }
+    /** Boost factors for different fields within documents. */
+    fieldBoosts;
+    /** Array storing the original documents added to the index. */
+    documents;
+    /**
+     * Creates a new BM25 search instance.
+     * @param docs - Optional array of initial documents (objects with string fields) to index.
+     * @param options - Configuration options for BM25 parameters (k1, b), tokenizer (stopWords, stemming, minLength), and field boosts.
+     */
+    constructor(docs, options = {}) {
+        const opts = { ...DEFAULT_OPTIONS, ...options };
+        this.termFrequencySaturation = opts.k1 ?? 1.2;
+        this.lengthNormalizationFactor = opts.b ?? 0.75;
+        this.tokenizer = new Tokenizer(opts);
+        this.fieldBoosts = opts.fieldBoosts || {};
+        // Initialize index structures
+        this.documents = [];
+        this.documentLengths = new Uint32Array(0);
+        this.termToIndex = new Map();
+        this.documentFrequency = new Uint32Array(0); // Will be sized later
+        this.averageDocLength = 0;
+        this.termFrequencies = new Map(); // TermIndex -> { DocIndex -> TF }
+        // Index initial documents if provided
+        if (docs && docs.length > 0) {
+            this.documents = [...docs]; // Store original documents
+            const { documentLengths, termToIndex, documentFrequency, averageDocLength, termFrequencies, } = this.processDocuments(docs);
+            // Assign processed data to instance properties
+            this.documentLengths = documentLengths;
+            this.termToIndex = termToIndex;
+            this.documentFrequency = documentFrequency;
+            this.averageDocLength = averageDocLength;
+            this.termFrequencies = termFrequencies;
+        }
+    }
+    /**
+     * Processes an array of documents to build the initial index structures.
+     * Calculates document lengths, term frequencies, document frequencies, and average document length.
+     * @param docs - Array of documents to process.
+     * @returns An object containing the calculated index data.
+     * @internal
+     */
+    processDocuments(docs) {
+        const numDocs = docs.length;
+        const documentLengths = new Uint32Array(numDocs);
+        const termToIndex = new Map();
+        const termDocs = new Map(); // Temp map: Term -> Set<DocIndex>
+        const termFrequencies = new Map(); // TermIndex -> { DocIndex -> TF }
+        let totalDocLength = 0;
+        let nextTermIndex = 0;
+        docs.forEach((doc, docIndex) => {
+            let currentDocLength = 0;
+            const docTermFrequencies = new Map(); // TermIndex -> TF for this doc
+            // Iterate through fields of the document
+            for (const field in doc) {
+                const content = doc[field];
+                if (typeof content !== "string")
+                    continue; // Skip non-string fields
+                const fieldBoost = this.fieldBoosts[field] || 1;
+                const { tokens } = this.tokenizer.tokenize(content);
+                const fieldLength = tokens.length * fieldBoost;
+                currentDocLength += fieldLength;
+                // Calculate term frequencies within this field/doc
+                tokens.forEach((term) => {
+                    // Assign index to new terms
+                    if (!termToIndex.has(term)) {
+                        termToIndex.set(term, nextTermIndex++);
+                    }
+                    const termIndexVal = termToIndex.get(term);
+                    // Track which documents contain the term
+                    let termDocsTerm = termDocs.get(term);
+                    if (!termDocsTerm) {
+                        termDocsTerm = new Set();
+                        termDocs.set(term, termDocsTerm);
+                    }
+                    termDocsTerm.add(docIndex);
+                    // Increment frequency for this term in this document
+                    const currentFreq = docTermFrequencies.get(termIndexVal) || 0;
+                    docTermFrequencies.set(termIndexVal, currentFreq + fieldBoost); // TF weighted by boost
+                });
+            }
+            // Store the calculated length for this document
+            documentLengths[docIndex] = currentDocLength;
+            totalDocLength += currentDocLength;
+            // Merge this document's term frequencies into the main structure
+            docTermFrequencies.forEach((freq, termIndexVal) => {
+                if (!termFrequencies.has(termIndexVal)) {
+                    termFrequencies.set(termIndexVal, new Map());
+                }
+                const termFrequenciesTermIndexVal = termFrequencies.get(termIndexVal);
+                if (termFrequenciesTermIndexVal) {
+                    termFrequenciesTermIndexVal.set(docIndex, freq);
+                }
+            });
+        });
+        // Calculate document frequency (DF) for each term
+        const documentFrequency = new Uint32Array(termToIndex.size);
+        termDocs.forEach((docsSet, term) => {
+            const termIndexVal = termToIndex.get(term);
+            if (termIndexVal !== undefined) {
+                documentFrequency[termIndexVal] = docsSet.size;
+            }
+        });
+        return {
+            documentLengths,
+            termToIndex,
+            documentFrequency,
+            averageDocLength: numDocs > 0 ? totalDocLength / numDocs : 0,
+            termFrequencies,
+        };
+    }
+    /**
+     * Recalculates the average document length based on the current `documentLengths`.
+     * @internal
+     */
+    recalculateAverageLength() {
+        if (this.documentLengths.length === 0) {
+            this.averageDocLength = 0;
+            return;
+        }
+        // Use the typed array's reduce method for type safety and performance
+        const totalLength = this.documentLengths.reduce((sum, len) => sum + len, 0);
+        this.averageDocLength = totalLength / this.documentLengths.length;
+    }
+    /**
+     * Searches the indexed documents for a given query string using the BM25 ranking formula.
+     *
+     * @param query - The search query text.
+     * @param topK - The maximum number of top-scoring results to return. Defaults to 10.
+     * @returns An array of `SearchResult` objects, sorted by descending BM25 score.
+     */
+    search(query, topK = 10) {
+        if (topK <= 0)
+            return [];
+        const { tokens: queryTokens } = this.tokenizer.tokenize(query); // Tokenize the query
+        const scores = new Float32Array(this.documentLengths.length).fill(0); // Initialize scores to 0
+        // Accumulate scores for each document based on query terms
+        queryTokens.forEach((term) => {
+            const termIndex = this.termToIndex.get(term);
+            // Ignore terms not found in the index
+            if (termIndex === undefined)
+                return;
+            const idf = this.calculateIdf(termIndex);
+            // Skip terms with non-positive IDF (e.g., term in all docs)
+            if (idf <= 0)
+                return;
+            const termFreqsInDocs = this.termFrequencies.get(termIndex); // Map<DocIndex, TF>
+            if (!termFreqsInDocs)
+                return; // Should not happen if termIndex exists, but check anyway
+            // Iterate over documents containing this term
+            termFreqsInDocs.forEach((tf, docIndex) => {
+                const docLength = this.documentLengths[docIndex];
+                // --- BM25 Term Score Calculation ---
+                // Normalizes TF based on document length and saturation parameters.
+                const numerator = tf * (this.termFrequencySaturation + 1);
+                const denominator = tf +
+                    this.termFrequencySaturation *
+                        (1 -
+                            this.lengthNormalizationFactor +
+                            (this.lengthNormalizationFactor * docLength) /
+                                this.averageDocLength);
+                // Add the weighted score (IDF * normalized TF) for this term to the document's total score
+                scores[docIndex] += idf * (numerator / denominator);
+            });
+        });
+        // --- Result Generation ---
+        // Keep only topK results without sorting the full set.
+        const results = [];
+        for (let i = 0; i < scores.length; i++) {
+            const score = scores[i];
+            if (score <= 0)
+                continue;
+            const result = { index: i, score };
+            if (results.length < topK) {
+                results.push(result);
+                if (results.length === topK) {
+                    results.sort((a, b) => b.score - a.score);
+                }
+                continue;
+            }
+            if (score <= results[results.length - 1].score)
+                continue;
+            let insertAt = results.length - 1;
+            while (insertAt > 0 && results[insertAt - 1].score < score) {
+                insertAt--;
+            }
+            results.splice(insertAt, 0, result);
+            results.pop();
+        }
+        return results;
+    }
+    /**
+     * Searches for an exact phrase within the indexed documents.
+     * Ranks documents containing the exact sequence of tokens higher.
+     * Basic implementation - proximity-based phrase search could improve results.
+     *
+     * @param phrase - The exact phrase to search for.
+     * @param topK - The maximum number of results to return. Defaults to 10.
+     * @returns An array of `SearchResult` objects, sorted by score, for documents containing the phrase.
+     */
+    searchPhrase(phrase, topK = 10) {
+        const { tokens: phraseTokens } = this.tokenizer.tokenize(phrase); // Tokenize the phrase
+        if (phraseTokens.length === 0)
+            return []; // Cannot search for empty phrase
+        // --- Find Candidate Documents ---
+        // Start with documents containing the *first* term, then intersect with subsequent terms.
+        let candidateDocs = null;
+        for (const term of phraseTokens) {
+            const termIndex = this.termToIndex.get(term);
+            if (termIndex === undefined)
+                return []; // Phrase cannot exist if any term is missing
+            const termFrequenciesTermIndex = this.termFrequencies.get(termIndex);
+            const docsContainingTermIter = termFrequenciesTermIndex?.keys();
+            if (!docsContainingTermIter)
+                return []; // Should not happen, but check
+            const currentTermDocs = new Set(docsContainingTermIter);
+            if (candidateDocs === null) {
+                // First term initializes the candidates
+                candidateDocs = currentTermDocs;
+            }
+            else {
+                // Intersect: Keep only documents present in both sets
+                candidateDocs = new Set([...candidateDocs].filter((docIdx) => currentTermDocs.has(docIdx)));
+            }
+            // If intersection becomes empty, the phrase cannot exist
+            if (candidateDocs.size === 0)
+                return [];
+        }
+        if (candidateDocs === null || candidateDocs.size === 0)
+            return []; // No candidates found
+        // --- Verify Phrase Occurrence and Score ---
+        const scores = new Map(); // Map<DocIndex, Score>
+        candidateDocs.forEach((docIndex) => {
+            const doc = this.getDocument(docIndex); // Get the original document content
+            let phraseFoundInDoc = false;
+            // Check each field for the phrase
+            for (const field in doc) {
+                const content = doc[field];
+                if (typeof content !== "string" || phraseFoundInDoc)
+                    continue; // Skip non-strings or if already found
+                const fieldBoost = this.fieldBoosts[field] || 1;
+                // Tokenize the field content using the same settings
+                const { tokens: docTokens } = this.tokenizer.tokenize(content);
+                // Simple sliding window check for the exact phrase sequence
+                for (let i = 0; i <= docTokens.length - phraseTokens.length; i++) {
+                    let match = true;
+                    for (let j = 0; j < phraseTokens.length; j++) {
+                        if (docTokens[i + j] !== phraseTokens[j]) {
+                            match = false;
+                            break;
+                        }
+                    }
+                    if (match) {
+                        // Phrase found! Calculate score for this document based on the phrase terms
+                        const phraseScoreVal = this.calculatePhraseScore(phraseTokens, docIndex) * fieldBoost;
+                        scores.set(docIndex, (scores.get(docIndex) || 0) + phraseScoreVal);
+                        phraseFoundInDoc = true; // Only score once per doc even if phrase repeats
+                        break; // Move to next document once found in this one
+                    }
+                }
+            }
+        });
+        // --- Format and Sort Results ---
+        return Array.from(scores.entries())
+            .map(([index, score]) => ({ index, score }))
+            .sort((a, b) => b.score - a.score) // Sort by score descending
+            .slice(0, topK); // Limit results
+    }
+    /**
+     * Calculates a BM25-like score for a sequence of phrase tokens within a specific document.
+     * Sums the individual BM25 scores of the terms in the phrase for that document.
+     * @param phraseTokens - The tokenized phrase.
+     * @param docIndex - The index of the document to score against.
+     * @returns The calculated phrase score for the document.
+     * @internal
+     */
+    calculatePhraseScore(phraseTokens, docIndex) {
+        return phraseTokens.reduce((currentScore, term) => {
+            const termIndex = this.termToIndex.get(term);
+            // Ignore terms not in index (shouldn't happen if candidate selection worked)
+            if (termIndex === undefined)
+                return currentScore;
+            const idf = this.calculateIdf(termIndex);
+            const termFrequenciesTermIndex = this.termFrequencies.get(termIndex);
+            const tf = termFrequenciesTermIndex?.get(docIndex) || 0;
+            const docLength = this.documentLengths[docIndex];
+            // Calculate the BM25 contribution of this single term
+            const numerator = tf * (this.termFrequencySaturation + 1);
+            const denominator = tf +
+                this.termFrequencySaturation *
+                    (1 -
+                        this.lengthNormalizationFactor +
+                        (this.lengthNormalizationFactor * docLength) /
+                            this.averageDocLength);
+            // Add IDF * normalized TF to the total phrase score
+            return currentScore + idf * (numerator / denominator);
+        }, 0); // Start score at 0
+    }
+    /**
+     * Adds a single new document to the index.
+     * Updates all internal index structures incrementally.
+     * For bulk additions, prefer `addDocumentsParallel` for better performance.
+     *
+     * @param doc - The document object (with string fields) to add.
+     * @throws {Error} If the document is null or undefined.
+     */
+    async addDocument(doc) {
+        if (!doc)
+            throw new Error("Document cannot be null");
+        const docIndex = this.documentLengths.length; // Index for the new document
+        // --- Update Document List and Lengths ---
+        this.documents.push(doc);
+        // Resize documentLengths array (simple append)
+        const newDocLengths = new Uint32Array(docIndex + 1);
+        newDocLengths.set(this.documentLengths, 0); // Copy old lengths
+        // Calculate length later...
+        this.documentLengths = newDocLengths; // Assign temporarily
+        let currentDocLength = 0;
+        const docTermFrequencies = new Map(); // TermIndex -> TF for this new doc
+        // --- Process Fields and Tokens ---
+        for (const field in doc) {
+            const content = doc[field];
+            if (typeof content !== "string")
+                continue;
+            const fieldBoost = this.fieldBoosts[field] || 1;
+            const { tokens } = this.tokenizer.tokenize(content);
+            currentDocLength += tokens.length * fieldBoost;
+            // Process each token in the field
+            tokens.forEach((term) => {
+                let termIndexVal;
+                // Check if term already exists in the index
+                const existingIndex = this.termToIndex.get(term);
+                if (existingIndex === undefined) {
+                    // Add term to index if new
+                    termIndexVal = this.termToIndex.size;
+                    this.termToIndex.set(term, termIndexVal);
+                    // Ensure documentFrequency array is large enough
+                    if (this.documentFrequency.length <= termIndexVal) {
+                        const oldDf = this.documentFrequency;
+                        // Grow exponentially, ensure it's at least termIndex + 1
+                        const newSize = Math.max(termIndexVal + 1, oldDf.length * 2 || 1); // Ensure newSize is at least 1 if oldDf.length is 0
+                        this.documentFrequency = new Uint32Array(newSize);
+                        this.documentFrequency.set(oldDf, 0);
+                    }
+                    // Initialize DF for new term (will be incremented below)
+                    this.documentFrequency[termIndexVal] = 0;
+                }
+                else {
+                    // Term exists in the map, use its existing index
+                    termIndexVal = existingIndex;
+                }
+                // Increment frequency for this term in this new document
+                const currentFreq = docTermFrequencies.get(termIndexVal) || 0;
+                docTermFrequencies.set(termIndexVal, currentFreq + fieldBoost); // Weighted TF
+            });
+        }
+        // --- Update Global Structures ---
+        // Set the calculated length for the new document
+        this.documentLengths[docIndex] = currentDocLength;
+        // Add this document's term frequencies to the main map and update DF
+        docTermFrequencies.forEach((freq, termIndexVal) => {
+            // Add TF entry
+            if (!this.termFrequencies.has(termIndexVal)) {
+                this.termFrequencies.set(termIndexVal, new Map());
+            }
+            const termFrequenciesTermIndexVal = this.termFrequencies.get(termIndexVal);
+            if (termFrequenciesTermIndexVal) {
+                termFrequenciesTermIndexVal.set(docIndex, freq);
+            }
+            // Increment document frequency for the term
+            // Ensure termIndexVal is within bounds of documentFrequency before incrementing
+            if (termIndexVal < this.documentFrequency.length) {
+                this.documentFrequency[termIndexVal]++;
+            }
+        });
+        // Recalculate average document length
+        this.recalculateAverageLength(); // Efficiently update average
+    }
+    /**
+     * Calculates the Inverse Document Frequency (IDF) for a given term index.
+     * Uses the BM25 IDF formula: log(1 + (N - n + 0.5) / (n + 0.5))
+     * where N is the total number of documents and n is the number of documents
+     * containing the term. The +1 smooths the logarithm.
+     *
+     * @param termIndex - The integer index of the term.
+     * @returns The IDF score for the term. Returns 0 if the term is not found or has 0 DF.
+     */
+    calculateIdf(termIndex) {
+        // Ensure termIndex is valid
+        if (termIndex < 0 || termIndex >= this.documentFrequency.length) {
+            return 0; // Term not in index or index out of bounds
+        }
+        const docFreq = this.documentFrequency[termIndex]; // n: number of docs containing the term
+        // If term appears in 0 documents or more docs than exist (error state), return 0 IDF.
+        if (docFreq <= 0 || docFreq > this.documentLengths.length) {
+            return 0;
+        }
+        const N = this.documentLengths.length; // Total number of documents
+        const numerator = N - docFreq + 0.5;
+        const denominator = docFreq + 0.5;
+        // Adding 1 inside the log ensures IDF is always non-negative.
+        return Math.log(1 + numerator / denominator);
+    }
+    /**
+     * Retrieves the term frequency (TF) for a specific term in a specific document.
+     * @param termIndex - The integer index of the term.
+     * @param docIndex - The index of the document.
+     * @returns The term frequency, or 0 if the term is not in the document or indices are invalid.
+     */
+    getTermFrequency(termIndex, docIndex) {
+        const termFrequenciesTermIndex = this.termFrequencies.get(termIndex);
+        return termFrequenciesTermIndex?.get(docIndex) || 0;
+    }
+    /**
+     * Retrieves the original document object stored at a given index.
+     * @param index - The index of the document to retrieve.
+     * @returns The document object.
+     * @throws {Error} If the index is out of bounds.
+     */
+    getDocument(index) {
+        // Consider using a generic <T>
+        if (index < 0 || index >= this.documents.length) {
+            throw new Error(`Document index ${index} out of bounds (0-${this.documents.length - 1})`);
+        }
+        return this.documents[index];
+    }
+    /**
+     * Clears all indexed documents and resets the BM25 instance to its initial state.
+     */
+    clearDocuments() {
+        this.documents = [];
+        this.documentLengths = new Uint32Array(0);
+        this.termToIndex.clear();
+        this.documentFrequency = new Uint32Array(0);
+        this.averageDocLength = 0;
+        this.termFrequencies.clear();
+    }
+    /**
+     * Gets the total number of documents currently indexed.
+     * @returns The document count.
+     */
+    getDocumentCount() {
+        return this.documents.length;
+    }
+    /**
+     * Adds multiple documents sequentially by calling `addDocument` for each.
+     * This method processes documents sequentially in the main thread.
+     * @param docs - An array of documents to add.
+     */
+    async addDocuments(docs) {
+        for (const doc of docs) {
+            await this.addDocument(doc);
+        }
+    }
+}
+/**
+ * Build an FTS (Full-Text Search) query from a raw search string.
+ * Extracts alphanumeric tokens and joins them with AND for strict matching.
+ *
+ * @param raw - The raw search query string.
+ * @returns The FTS query string, or null if no valid tokens found.
+ */
+export function buildFtsQuery(raw) {
+    const tokens = raw
+        .match(/[A-Za-z0-9_]+/g)
+        ?.map((t) => t.trim())
+        .filter(Boolean) ?? [];
+    if (tokens.length === 0)
+        return null;
+    const quoted = tokens.map((t) => `"${t.replaceAll('"', "")}"`);
+    return quoted.join(" AND ");
+}
+/**
+ * Convert BM25 rank to a normalized score between 0 and 1.
+ * Lower rank = higher score.
+ *
+ * @param rank - The BM25 rank value.
+ * @returns A normalized score where 1 is best and 0 is worst.
+ */
+export function bm25RankToScore(rank) {
+    const normalized = Number.isFinite(rank) ? Math.max(0, rank) : 999;
+    return 1 / (1 + normalized);
+}
+/**
+ * Merge vector similarity and keyword search results using weighted scoring.
+ *
+ * This implements a hybrid search approach where results from both vector
+ * similarity search and keyword (BM25) search are combined. Results that
+ * appear in both searches get boosted scores.
+ *
+ * @param params.vector - Results from vector similarity search.
+ * @param params.keyword - Results from keyword (BM25) search.
+ * @param params.vectorWeight - Weight for vector similarity scores (default: 0.7).
+ * @param params.textWeight - Weight for keyword/text scores (default: 0.3).
+ * @returns Merged and sorted results with combined scores.
+ */
+export function mergeHybridResults(params) {
+    const byId = new Map();
+    // Add vector search results
+    for (const r of params.vector) {
+        byId.set(r.id, {
+            id: r.id,
+            path: r.path,
+            startLine: r.startLine,
+            endLine: r.endLine,
+            source: r.source,
+            snippet: r.snippet,
+            vectorScore: r.vectorScore,
+            textScore: 0,
+        });
+    }
+    // Merge keyword search results
+    for (const r of params.keyword) {
+        const existing = byId.get(r.id);
+        if (existing) {
+            existing.textScore = r.textScore;
+            // Prefer keyword snippet if available (may have highlights)
+            if (r.snippet && r.snippet.length > 0) {
+                existing.snippet = r.snippet;
+            }
+        }
+        else {
+            byId.set(r.id, {
+                id: r.id,
+                path: r.path,
+                startLine: r.startLine,
+                endLine: r.endLine,
+                source: r.source,
+                snippet: r.snippet,
+                vectorScore: 0,
+                textScore: r.textScore,
+            });
+        }
+    }
+    // Calculate weighted scores and sort
+    const merged = Array.from(byId.values()).map((entry) => {
+        const score = params.vectorWeight * entry.vectorScore +
+            params.textWeight * entry.textScore;
+        return {
+            path: entry.path,
+            startLine: entry.startLine,
+            endLine: entry.endLine,
+            score,
+            snippet: entry.snippet,
+            source: entry.source,
+        };
+    });
+    return [...merged].sort((a, b) => b.score - a.score);
+}