cspell-trie-lib 8.6.1 → 8.8.0

package/dist/lib/Builder/TrieBuilder.d.ts

@@ -2,6 +2,16 @@ import type { PartialTrieOptions, TrieOptions } from '../trie.js';
 import type { TrieData } from '../TrieData.js';
 import type { BuilderCursor } from './BuilderCursor.js';
 export interface TrieBuilder<T extends TrieData> {
+    /**
+     * Use this method to convert a word into an array of characters.
+     * Since `[...word]` is not equal to `word.split('')` or `word[i]` in some cases,
+     * this method is used to ensure that the characters are split correctly.
+     * @see [String.codePointAt](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/codePointAt)
+     * @see [String.charCodeAt](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/charCodeAt)
+     * @param word - The word to convert into an array of characters.
+     * @returns An array of characters, one for each character in the word.
+     */
+    wordToCharacters(word: string): string[];
     getCursor(): BuilderCursor;
     build(): T;
     setOptions(options: Readonly<PartialTrieOptions>): Readonly<TrieOptions>;

package/dist/lib/Builder/cursor-util.d.ts

@@ -1,3 +1,4 @@
 import type { BuilderCursor } from './BuilderCursor.js';
 export declare function insertWordsAtCursor(cursor: BuilderCursor, words: Iterable<string>): void;
+export declare function commonStringPrefixLen(a: string, b: string): number;
 //# sourceMappingURL=cursor-util.d.ts.map

package/dist/lib/Builder/cursor-util.js

@@ -1,10 +1,11 @@
 export function insertWordsAtCursor(cursor, words) {
     let prevWord = '';
     for (const word of words) {
-        const pLen = commonPrefixLen(prevWord, word);
+        const pLen = commonStrPrefix(prevWord, word);
         const stepBack = prevWord.length - pLen;
         cursor.backStep(stepBack);
-        for (let i = pLen; i < word.length; ++i) {
+        const wLen = word.length;
+        for (let i = pLen; i < wLen; ++i) {
             cursor.insertChar(word[i]);
         }
         cursor.markEOW();
@@ -12,7 +13,22 @@ export function insertWordsAtCursor(cursor, words) {
     }
     cursor.backStep(prevWord.length);
 }
-function commonPrefixLen(a, b) {
+export function commonStringPrefixLen(a, b) {
+    let i = 0;
+    for (i = 0; i < a.length && a[i] === b[i]; ++i) {
+        /* empty */
+    }
+    if (i) {
+        // detect second half of a surrogate pair and backup.
+        // eslint-disable-next-line unicorn/prefer-code-point
+        const c = a.charCodeAt(i) & 0xffff;
+        if (c >= 0xdc00 && c <= 0xdfff) {
+            --i;
+        }
+    }
+    return i;
+}
+function commonStrPrefix(a, b) {
     let i = 0;
     for (i = 0; i < a.length && a[i] === b[i]; ++i) {
         /* empty */

package/dist/lib/ITrieNode/find.js

@@ -77,12 +77,15 @@ function _findWordNode(root, word, options) {
         return result;
     }
     switch (compoundMode) {
-        case 'none':
+        case 'none': {
             return options.matchCase ? __findExact() : __findCompound();
-        case 'compound':
+        }
+        case 'compound': {
             return __findCompound();
-        case 'legacy':
+        }
+        case 'legacy': {
             return findLegacyCompound(root, word, options);
+        }
     }
 }
 export function findLegacyCompound(root, word, options) {
@@ -120,7 +123,6 @@ export function findCompoundNode(root, word, compoundCharacter, ignoreCasePrefix
     let caseMatched = true;
     let i = 0;
     let node;
-    // eslint-disable-next-line no-constant-condition
     while (true) {
         const s = stack[i];
         const h = w[i++];
@@ -149,11 +151,9 @@ export function findCompoundNode(root, word, compoundCharacter, ignoreCasePrefix
                 if (!r.cr) {
                     break;
                 }
-                if (!i && !r.caseMatched) {
-                    if (w !== w.toLowerCase()) {
-                        // It is not going to be found.
-                        break;
-                    }
+                if (!i && !r.caseMatched && w !== w.toLowerCase()) {
+                    // It is not going to be found.
+                    break;
                 }
             }
             else {
@@ -186,10 +186,10 @@ export function isEndOfWordNode(n) {
     return !!n?.eow;
 }
 function walk(root, word) {
-    const w = word;
+    const w = [...word];
     let n = root;
     let i = 0;
-    while (n && i < word.length) {
+    while (n && i < w.length) {
         const h = w[i++];
         n = n.get(h);
     }
@@ -208,7 +208,6 @@ function findLegacyCompoundNode(roots, word, minCompoundLength) {
     let caseMatched = true;
     let i = 0;
     let node;
-    // eslint-disable-next-line no-constant-condition
     while (true) {
         const s = stack[i];
         const h = w[i++];

package/dist/lib/ITrieNode/walker/hintedWalker.js

@@ -20,11 +20,11 @@ function* hintedWalkerNext(root, ignoreCase, hint, compoundingMethod, emitWordSe
     const roots = rawRoots.map(filterRoot);
     const compoundRoots = rawRoots.map((r) => r.get(compoundCharacter)).filter(isDefined);
     const setOfCompoundRoots = new Set(compoundRoots);
-    const rootsForCompoundMethods = roots.concat(compoundRoots);
+    const rootsForCompoundMethods = [...roots, ...compoundRoots];
     const compoundMethodRoots = {
         [CompoundWordsMethod.NONE]: [],
-        [CompoundWordsMethod.JOIN_WORDS]: [...rootsForCompoundMethods.map((r) => [JOIN_SEPARATOR, r])],
-        [CompoundWordsMethod.SEPARATE_WORDS]: [...rootsForCompoundMethods.map((r) => [WORD_SEPARATOR, r])],
+        [CompoundWordsMethod.JOIN_WORDS]: rootsForCompoundMethods.map((r) => [JOIN_SEPARATOR, r]),
+        [CompoundWordsMethod.SEPARATE_WORDS]: rootsForCompoundMethods.map((r) => [WORD_SEPARATOR, r]),
     };
     function* children(n, hintOffset) {
         if (n.hasChildren()) {

package/dist/lib/ITrieNode/walker/walker.js

@@ -14,7 +14,7 @@ function* compoundWalker(root, compoundingMethod) {
     function children(n) {
         if (n.hasChildren()) {
             const c = n.keys().map((k, i) => [k, n.child(i)]);
-            return n.eow && rc ? c.concat(rc) : c;
+            return n.eow && rc ? [...c, ...rc] : c;
         }
         if (n.eow) {
             return roots[compoundingMethod];

package/dist/lib/SimpleDictionaryParser.js

@@ -52,18 +52,22 @@ export function createDictionaryLineParserMapper(options) {
                 .filter((a) => !!a);
             for (const flag of flags) {
                 switch (flag) {
-                    case 'split':
+                    case 'split': {
                         split = true;
                         break;
-                    case 'no-split':
+                    }
+                    case 'no-split': {
                         split = false;
                         break;
-                    case 'no-generate-alternatives':
+                    }
+                    case 'no-generate-alternatives': {
                         stripCaseAndAccents = false;
                         break;
-                    case 'generate-alternatives':
+                    }
+                    case 'generate-alternatives': {
                         stripCaseAndAccents = true;
                         break;
+                    }
                 }
             }
         }
@@ -101,7 +105,7 @@ export function createDictionaryLineParserMapper(options) {
         return w.startsWith(ignoreCase + ignoreCase) ? w.slice(1) : w;
     }
     function stripKeepCasePrefixAndQuotes(word) {
-        word = word.replace(/"(.*?)"/g, '$1');
+        word = word.replaceAll(/"(.*?)"/g, '$1');
         return word[0] === keepCase ? word.slice(1) : word;
     }
     function _normalize(word) {
@@ -121,11 +125,11 @@ export function createDictionaryLineParserMapper(options) {
     function* splitWords(lines) {
         for (const line of lines) {
             if (split) {
-                const lineEscaped = line.indexOf('"') >= 0
-                    ? line.replace(/".*?"/g, (quoted) => ' ' + quoted.replace(/(\s)/g, '\\$1') + ' ')
+                const lineEscaped = line.includes('"')
+                    ? line.replaceAll(/".*?"/g, (quoted) => ' ' + quoted.replaceAll(/(\s)/g, '\\$1') + ' ')
                     : line;
                 const words = splitLine(lineEscaped, splitSeparator);
-                yield* words.map((escaped) => escaped.replace(/\\/g, ''));
+                yield* words.map((escaped) => escaped.replaceAll('\\', ''));
                 if (!splitKeepBoth)
                     continue;
             }
@@ -189,10 +193,10 @@ function mergeOptions(base, ...partials) {
 const RegExpToEncode = /\\([\s,;])/g;
 const RegExpDecode = /<<(%[\da-f]{2})>>/gi;
 function encodeLine(line) {
-    return line.replace(RegExpToEncode, (_, v) => '<<' + encodeURIComponent(v) + '>>');
+    return line.replaceAll(RegExpToEncode, (_, v) => '<<' + encodeURIComponent(v) + '>>');
 }
 function decodeLine(line) {
-    return line.replace(RegExpDecode, (_, v) => '\\' + decodeURIComponent(v));
+    return line.replaceAll(RegExpDecode, (_, v) => '\\' + decodeURIComponent(v));
 }
 function splitLine(line, regExp) {
     return encodeLine(line)

package/dist/lib/TrieBlob/FastTrieBlob.d.ts

@@ -6,21 +6,29 @@ import { FastTrieBlobInternals } from './FastTrieBlobInternals.js';
 import { TrieBlob } from './TrieBlob.js';
 export declare class FastTrieBlob implements TrieData {
     private nodes;
-    private charIndex;
+    private _charIndex;
     readonly bitMasksInfo: FastTrieBlobBitMaskInfo;
-    private charToIndexMap;
+    private _charToIndexMap;
     private _readonly;
     private _forbidIdx;
     private _iTrieRoot;
+    wordToCharacters: (word: string) => readonly string[];
     readonly info: Readonly<TrieInfo>;
     private constructor();
-    private lookUpCharIndex;
+    private _lookUpCharIndex;
+    private wordToNodeCharIndexSequence;
+    private letterToNodeCharIndexSequence;
     has(word: string): boolean;
     private _has;
     words(): Iterable<string>;
     toTrieBlob(): TrieBlob;
     isReadonly(): boolean;
     freeze(): this;
+    toJSON(): {
+        info: Readonly<TrieInfo>;
+        nodes: NodeElement[];
+        charIndex: readonly string[];
+    };
     static create(data: FastTrieBlobInternals, options?: PartialTrieInfo): FastTrieBlob;
     static toITrieNodeRoot(trie: FastTrieBlob): ITrieNodeRoot;
     static NodeMaskEOW: number;
@@ -34,6 +42,20 @@ export declare class FastTrieBlob implements TrieData {
     hasForbiddenWords(): boolean;
     /** number of nodes */
     get size(): number;
-    private _lookupChar;
+    private _lookupCharIndexNode;
+    /** Search from nodeIdx for the node index representing the character. */
+    private _searchNodeForChar;
+    get charIndex(): readonly string[];
+    static fromTrieBlob(trie: TrieBlob): FastTrieBlob;
 }
+interface NodeElement {
+    id: number;
+    eow: boolean;
+    n: number;
+    c: {
+        c: number | string;
+        i: number;
+    }[];
+}
+export {};
 //# sourceMappingURL=FastTrieBlob.d.ts.map

package/dist/lib/TrieBlob/FastTrieBlob.js

@@ -3,26 +3,35 @@ import { mergeOptionalWithDefaults } from '../utils/mergeOptionalWithDefaults.js
 import { extractInfo } from './FastTrieBlobBitMaskInfo.js';
 import { FastTrieBlobInternals } from './FastTrieBlobInternals.js';
 import { FastTrieBlobIRoot } from './FastTrieBlobIRoot.js';
+import { NumberSequenceByteDecoderAccumulator } from './NumberSequenceByteDecoderAccumulator.js';
 import { TrieBlob } from './TrieBlob.js';
 export class FastTrieBlob {
     nodes;
-    charIndex;
+    _charIndex;
     bitMasksInfo;
-    charToIndexMap;
+    _charToIndexMap;
     _readonly = false;
     _forbidIdx;
     _iTrieRoot;
+    wordToCharacters;
     info;
-    constructor(nodes, charIndex, bitMasksInfo, options) {
+    constructor(nodes, _charIndex, bitMasksInfo, options) {
         this.nodes = nodes;
-        this.charIndex = charIndex;
+        this._charIndex = _charIndex;
         this.bitMasksInfo = bitMasksInfo;
         this.info = mergeOptionalWithDefaults(options);
-        this.charToIndexMap = createCharToIndexMap(charIndex);
-        this._forbidIdx = this._lookupChar(0, this.info.forbiddenWordPrefix);
+        this.wordToCharacters = (word) => [...word];
+        this._charToIndexMap = createCharToIndexMap(_charIndex);
+        this._forbidIdx = this._searchNodeForChar(0, this.info.forbiddenWordPrefix);
     }
-    lookUpCharIndex(char) {
-        return this.charToIndexMap[char] ?? -1;
+    _lookUpCharIndex(char) {
+        return this._charToIndexMap[char] ?? -1;
+    }
+    wordToNodeCharIndexSequence(word) {
+        return TrieBlob.charactersToCharIndexSequence(this.wordToCharacters(word), (c) => this._lookUpCharIndex(c));
+    }
+    letterToNodeCharIndexSequence(letter) {
+        return TrieBlob.toCharIndexSequence(this._lookUpCharIndex(letter));
     }
     has(word) {
         return this._has(0, word);
@@ -32,10 +41,11 @@ export class FastTrieBlob {
         const NodeChildRefShift = this.bitMasksInfo.NodeChildRefShift;
         const NodeMaskEOW = this.bitMasksInfo.NodeMaskEOW;
         const nodes = this.nodes;
-        const len = word.length;
+        const charIndexes = this.wordToNodeCharIndexSequence(word);
+        const len = charIndexes.length;
         let node = nodes[nodeIdx];
         for (let p = 0; p < len; ++p, node = nodes[nodeIdx]) {
-            const letterIdx = this.lookUpCharIndex(word[p]);
+            const letterIdx = charIndexes[p];
             const count = node.length;
             let i = count - 1;
             for (; i > 0; --i) {
@@ -56,10 +66,11 @@ export class FastTrieBlob {
         const NodeChildRefShift = this.bitMasksInfo.NodeChildRefShift;
         const NodeMaskEOW = this.bitMasksInfo.NodeMaskEOW;
         const nodes = this.nodes;
-        const stack = [{ nodeIdx: 0, pos: 0, word: '' }];
+        const accumulator = NumberSequenceByteDecoderAccumulator.create();
+        const stack = [{ nodeIdx: 0, pos: 0, word: '', accumulator }];
         let depth = 0;
         while (depth >= 0) {
-            const { nodeIdx, pos, word } = stack[depth];
+            const { nodeIdx, pos, word, accumulator } = stack[depth];
             const node = nodes[nodeIdx];
             if (!pos && node[0] & NodeMaskEOW) {
                 yield word;
@@ -71,12 +82,15 @@ export class FastTrieBlob {
             const nextPos = ++stack[depth].pos;
             const entry = node[nextPos];
             const charIdx = entry & NodeMaskChildCharIndex;
-            const letter = this.charIndex[charIdx];
+            const acc = accumulator.clone();
+            const letterIdx = acc.decode(charIdx);
+            const letter = (letterIdx && this._charIndex[letterIdx]) || '';
             ++depth;
             stack[depth] = {
                 nodeIdx: entry >>> NodeChildRefShift,
                 pos: 0,
                 word: word + letter,
+                accumulator: acc,
             };
         }
     }
@@ -111,7 +125,7 @@ export class FastTrieBlob {
                 binNodes[offset++] = (nodeToIndex[nodeRef] << refShift) | charIndex;
             }
         }
-        return new TrieBlob(binNodes, this.charIndex, this.info);
+        return new TrieBlob(binNodes, this._charIndex, this.info);
     }
     isReadonly() {
         return this._readonly;
@@ -120,11 +134,18 @@ export class FastTrieBlob {
         this._readonly = true;
         return this;
     }
+    toJSON() {
+        return {
+            info: this.info,
+            nodes: nodesToJson(this.nodes),
+            charIndex: this._charIndex,
+        };
+    }
     static create(data, options) {
         return new FastTrieBlob(data.nodes, data.charIndex, extractInfo(data), options);
     }
     static toITrieNodeRoot(trie) {
-        return new FastTrieBlobIRoot(new FastTrieBlobInternals(trie.nodes, trie.charIndex, trie.charToIndexMap, trie.bitMasksInfo), 0, trie.info);
+        return new FastTrieBlobIRoot(new FastTrieBlobInternals(trie.nodes, trie._charIndex, trie._charToIndexMap, trie.bitMasksInfo), 0, trie.info);
     }
     static NodeMaskEOW = TrieBlob.NodeMaskEOW;
     static NodeChildRefShift = TrieBlob.NodeChildRefShift;
@@ -153,12 +174,12 @@ export class FastTrieBlob {
     get size() {
         return this.nodes.length;
     }
-    _lookupChar(nodeIdx, char) {
+    _lookupCharIndexNode(nodeIdx, charIndex) {
         const NodeMaskChildCharIndex = this.bitMasksInfo.NodeMaskChildCharIndex;
         const NodeChildRefShift = this.bitMasksInfo.NodeChildRefShift;
         const nodes = this.nodes;
         const node = nodes[nodeIdx];
-        const letterIdx = this.lookUpCharIndex(char);
+        const letterIdx = charIndex;
         const count = node.length;
         let i = count - 1;
         for (; i > 0; --i) {
@@ -168,6 +189,55 @@ export class FastTrieBlob {
         }
         return 0;
     }
+    /** Search from nodeIdx for the node index representing the character. */
+    _searchNodeForChar(nodeIdx, char) {
+        const charIndexes = this.letterToNodeCharIndexSequence(char);
+        let idx = nodeIdx;
+        for (let i = 0; i < charIndexes.length; ++i) {
+            idx = this._lookupCharIndexNode(idx, charIndexes[i]);
+            if (!idx)
+                return 0;
+        }
+        return idx;
+    }
+    get charIndex() {
+        return [...this._charIndex];
+    }
+    static fromTrieBlob(trie) {
+        const bitMasksInfo = {
+            NodeMaskEOW: TrieBlob.NodeMaskEOW,
+            NodeMaskChildCharIndex: TrieBlob.NodeMaskChildCharIndex,
+            NodeChildRefShift: TrieBlob.NodeChildRefShift,
+        };
+        const trieNodesBin = TrieBlob.nodesView(trie);
+        const nodeOffsets = [];
+        for (let offset = 0; offset < trieNodesBin.length; offset += (trieNodesBin[offset] & TrieBlob.NodeMaskNumChildren) + 1) {
+            nodeOffsets.push(offset);
+        }
+        const offsetToNodeIndex = new Map(nodeOffsets.map((offset, i) => [offset, i]));
+        const nodes = Array.from({ length: nodeOffsets.length });
+        for (let i = 0; i < nodes.length; ++i) {
+            const offset = nodeOffsets[i];
+            const n = trieNodesBin[offset];
+            const eow = n & TrieBlob.NodeMaskEOW;
+            const count = n & TrieBlob.NodeMaskNumChildren;
+            // Preallocate the array to the correct size.
+            const node = Array.from({ length: count + 1 });
+            node[0] = eow;
+            nodes[i] = node;
+            for (let j = 1; j <= count; ++j) {
+                const n = trieNodesBin[offset + j];
+                const charIndex = n & TrieBlob.NodeMaskChildCharIndex;
+                const nodeIndex = n >>> TrieBlob.NodeChildRefShift;
+                const idx = offsetToNodeIndex.get(nodeIndex);
+                if (idx === undefined) {
+                    throw new Error(`Invalid node index ${nodeIndex}`);
+                }
+                node[j] = (idx << TrieBlob.NodeChildRefShift) | charIndex;
+            }
+        }
+        return new FastTrieBlob(nodes, trie.charIndex, bitMasksInfo, trie.info);
+    }
 }
 function createCharToIndexMap(charIndex) {
     const map = Object.create(null);
@@ -178,4 +248,16 @@ function createCharToIndexMap(charIndex) {
     }
     return map;
 }
+function nodesToJson(nodes) {
+    function nodeElement(node, index) {
+        const eow = !!(node[0] & TrieBlob.NodeMaskEOW);
+        const children = node.slice(1).map((n) => ({
+            c: ('00' + (n & TrieBlob.NodeMaskChildCharIndex).toString(16)).slice(-2),
+            i: n >>> TrieBlob.NodeChildRefShift,
+        }));
+        return { id: index, eow, n: node.length, c: children };
+    }
+    const elements = nodes.map((n, i) => nodeElement(n, i));
+    return elements;
+}
 //# sourceMappingURL=FastTrieBlob.js.map

package/dist/lib/TrieBlob/FastTrieBlobBuilder.d.ts

@@ -11,11 +11,14 @@ export declare class FastTrieBlobBuilder implements TrieBuilder<FastTrieBlob> {
     private IdxEOW;
     private _cursor;
     private _options;
+    wordToCharacters: (word: string) => string[];
     readonly bitMasksInfo: FastTrieBlobBitMaskInfo;
     constructor(options?: PartialTrieInfo, bitMasksInfo?: FastTrieBlobBitMaskInfo);
     setOptions(options: PartialTrieInfo): Readonly<TrieInfo>;
     get options(): Readonly<TrieInfo>;
     private getCharIndex;
+    private wordToNodeCharIndexSequence;
+    private letterToNodeCharIndexSequence;
     insert(word: string | Iterable<string> | string[]): this;
     getCursor(): BuilderCursor;
     private createCursor;
@@ -24,7 +27,7 @@ export declare class FastTrieBlobBuilder implements TrieBuilder<FastTrieBlob> {
     isReadonly(): boolean;
     freeze(): this;
     build(): FastTrieBlob;
-    static fromWordList(words: string[] | Iterable<string>, options?: PartialTrieInfo): FastTrieBlob;
+    static fromWordList(words: readonly string[] | Iterable<string>, options?: PartialTrieInfo): FastTrieBlob;
     static fromTrieRoot(root: TrieRoot): FastTrieBlob;
     static NodeMaskEOW: number;
     static NodeChildRefShift: number;