cspell-trie-lib 8.3.2 → 8.4.0

package/dist/lib/TrieBlob/FastTrieBlobIRoot.js

@@ -1,61 +1,56 @@
+import { NumberSequenceByteDecoderAccumulator, NumberSequenceByteEncoderDecoder, } from './NumberSequenceByteDecoderAccumulator.js';
+const SpecialCharIndexMask = NumberSequenceByteEncoderDecoder.SpecialCharIndexMask;
 const EmptyKeys = Object.freeze([]);
 const EmptyNodes = Object.freeze([]);
+const EmptyEntries = Object.freeze([]);
 class FastTrieBlobINode {
     trie;
     nodeIdx;
     id;
-    size;
     node;
     eow;
-    charToIdx;
     _keys;
+    _count;
+    _size;
+    _chained;
+    _nodesEntries;
+    _entries;
+    _values;
+    charToIdx;
     constructor(trie, nodeIdx) {
         this.trie = trie;
         this.nodeIdx = nodeIdx;
         const node = trie.nodes[nodeIdx];
         this.node = node;
-        const value = node[0];
-        this.eow = !!(value & trie.NodeMaskEOW);
-        this.size = node.length - 1;
+        this.eow = !!(node[0] & trie.NodeMaskEOW);
+        this._count = node.length - 1;
         this.id = nodeIdx;
     }
-    keys() {
-        return (this._keys ??= this.calcKeys());
-    }
     /** get keys to children */
-    calcKeys() {
-        if (!this.size)
+    keys() {
+        if (this._keys)
+            return this._keys;
+        if (!this._count)
             return EmptyKeys;
-        const NodeMaskChildCharIndex = this.trie.NodeMaskChildCharIndex;
-        const charIndex = this.trie.charIndex;
-        const keys = Array(this.size);
-        const len = this.size;
-        const node = this.trie.nodes[this.nodeIdx];
-        for (let i = 0; i < len; ++i) {
-            const entry = node[i + 1];
-            const charIdx = entry & NodeMaskChildCharIndex;
-            keys[i] = charIndex[charIdx];
-        }
-        return Object.freeze(keys);
+        this._keys = this.getNodesEntries().map(([key]) => key);
+        return this._keys;
     }
     values() {
-        if (!this.size)
+        if (!this._count)
             return EmptyNodes;
-        const nodes = Array(this.size);
-        for (let i = 0; i < this.size; ++i) {
-            nodes[i] = this.child(i);
-        }
-        return nodes;
+        if (this._values)
+            return this._values;
+        this._values = this.entries().map(([, value]) => value);
+        return this._values;
     }
     entries() {
-        const keys = this.keys();
-        const values = this.values();
-        const len = keys.length;
-        const entries = Array(len);
-        for (let i = 0; i < len; ++i) {
-            entries[i] = [keys[i], values[i]];
-        }
-        return entries;
+        if (this._entries)
+            return this._entries;
+        if (!this._count)
+            return EmptyEntries;
+        const entries = this.getNodesEntries();
+        this._entries = entries.map(([key, value]) => [key, new FastTrieBlobINode(this.trie, value)]);
+        return this._entries;
     }
     /** get child ITrieNode */
     get(char) {
@@ -69,12 +64,15 @@ class FastTrieBlobINode {
         return idx !== undefined;
     }
     hasChildren() {
-        return this.size > 0;
+        return this._count > 0;
     }
     child(keyIdx) {
-        const node = this.trie.nodes[this.nodeIdx];
-        const nodeIdx = node[keyIdx + 1] >>> this.trie.NodeChildRefShift;
-        return new FastTrieBlobINode(this.trie, nodeIdx);
+        if (!this._values && !this.containsChainedIndexes()) {
+            const n = this.node[keyIdx + 1];
+            const nodeIdx = n >>> this.trie.NodeChildRefShift;
+            return new FastTrieBlobINode(this.trie, nodeIdx);
+        }
+        return this.values()[keyIdx];
     }
     getCharToIdxMap() {
         const m = this.charToIdx;
@@ -88,6 +86,98 @@ class FastTrieBlobINode {
         this.charToIdx = map;
         return map;
     }
+    containsChainedIndexes() {
+        if (this._chained !== undefined)
+            return this._chained;
+        if (!this._count || !this.trie.isIndexDecoderNeeded) {
+            this._chained = false;
+            return false;
+        }
+        // scan the node to see if there are encoded entries.
+        let found = false;
+        const NodeMaskChildCharIndex = this.trie.NodeMaskChildCharIndex;
+        const len = this._count;
+        const node = this.node;
+        for (let i = 1; i <= len && !found; ++i) {
+            const entry = node[i];
+            const charIdx = entry & NodeMaskChildCharIndex;
+            found = (charIdx & SpecialCharIndexMask) === SpecialCharIndexMask;
+        }
+        this._chained = !!found;
+        return this._chained;
+    }
+    getNodesEntries() {
+        if (this._nodesEntries)
+            return this._nodesEntries;
+        if (!this.containsChainedIndexes()) {
+            const entries = Array(this._count);
+            const nodes = this.node;
+            const charIndex = this.trie.charIndex;
+            const NodeMaskChildCharIndex = this.trie.NodeMaskChildCharIndex;
+            const RefShift = this.trie.NodeChildRefShift;
+            for (let i = 0; i < this._count; ++i) {
+                const entry = nodes[i + 1];
+                const charIdx = entry & NodeMaskChildCharIndex;
+                entries[i] = [charIndex[charIdx], entry >>> RefShift];
+            }
+            this._nodesEntries = entries;
+            return entries;
+        }
+        this._nodesEntries = this.walkChainedIndexes();
+        return this._nodesEntries;
+    }
+    walkChainedIndexes() {
+        const NodeMaskChildCharIndex = this.trie.NodeMaskChildCharIndex;
+        const NodeChildRefShift = this.trie.NodeChildRefShift;
+        const nodes = this.trie.nodes;
+        const acc = NumberSequenceByteDecoderAccumulator.create();
+        const stack = [{ n: this.node, c: 1, acc }];
+        let depth = 0;
+        /** there is at least this._count number of entries, more if there are nested indexes. */
+        const entries = Array(this._count);
+        let eIdx = 0;
+        const charIndex = this.trie.charIndex;
+        while (depth >= 0) {
+            const s = stack[depth];
+            const { n: node, c: off } = s;
+            if (off >= node.length) {
+                --depth;
+                continue;
+            }
+            ++s.c;
+            const entry = node[off];
+            const charIdx = entry & NodeMaskChildCharIndex;
+            const acc = s.acc.clone();
+            const letterIdx = acc.decode(charIdx);
+            if (letterIdx !== undefined) {
+                const char = charIndex[letterIdx];
+                const nodeIdx = entry >>> NodeChildRefShift;
+                entries[eIdx++] = [char, nodeIdx];
+                continue;
+            }
+            const idx = entry >>> NodeChildRefShift;
+            const ss = stack[++depth];
+            if (ss) {
+                ss.n = nodes[idx];
+                ss.c = 1;
+                ss.acc = acc;
+            }
+            else {
+                stack[depth] = { n: nodes[idx], c: 1, acc };
+            }
+        }
+        return entries;
+    }
+    get size() {
+        if (this._size === undefined) {
+            if (!this.containsChainedIndexes()) {
+                this._size = this._count;
+                return this._size;
+            }
+            this._size = this.getNodesEntries().length;
+        }
+        return this._size;
+    }
 }
 export class FastTrieBlobIRoot extends FastTrieBlobINode {
     info;

package/dist/lib/TrieBlob/FastTrieBlobInternals.d.ts

@@ -1,11 +1,12 @@
 import type { FastTrieBlobBitMaskInfo } from './FastTrieBlobBitMaskInfo.js';
 export declare class FastTrieBlobInternals implements FastTrieBlobBitMaskInfo {
     readonly nodes: number[][];
-    readonly charIndex: string[];
+    readonly charIndex: readonly string[];
     readonly charToIndexMap: Readonly<Record<string, number>>;
     readonly NodeMaskEOW: number;
     readonly NodeMaskChildCharIndex: number;
     readonly NodeChildRefShift: number;
-    constructor(nodes: number[][], charIndex: string[], charToIndexMap: Readonly<Record<string, number>>, maskInfo: FastTrieBlobBitMaskInfo);
+    readonly isIndexDecoderNeeded: boolean;
+    constructor(nodes: number[][], charIndex: readonly string[], charToIndexMap: Readonly<Record<string, number>>, maskInfo: FastTrieBlobBitMaskInfo);
 }
 //# sourceMappingURL=FastTrieBlobInternals.d.ts.map

package/dist/lib/TrieBlob/FastTrieBlobInternals.js

@@ -1,3 +1,4 @@
+import { NumberSequenceByteEncoderDecoder } from './NumberSequenceByteDecoderAccumulator.js';
 export class FastTrieBlobInternals {
     nodes;
     charIndex;
@@ -5,6 +6,7 @@ export class FastTrieBlobInternals {
     NodeMaskEOW;
     NodeMaskChildCharIndex;
     NodeChildRefShift;
+    isIndexDecoderNeeded;
     constructor(nodes, charIndex, charToIndexMap, maskInfo) {
         this.nodes = nodes;
         this.charIndex = charIndex;
@@ -13,6 +15,7 @@ export class FastTrieBlobInternals {
         this.NodeMaskEOW = NodeMaskEOW;
         this.NodeMaskChildCharIndex = NodeMaskChildCharIndex;
         this.NodeChildRefShift = NodeChildRefShift;
+        this.isIndexDecoderNeeded = charIndex.length > NumberSequenceByteEncoderDecoder.MaxCharIndex;
     }
 }
 //# sourceMappingURL=FastTrieBlobInternals.js.map

package/dist/lib/TrieBlob/NumberSequenceByteDecoderAccumulator.d.ts

@@ -0,0 +1,34 @@
+export declare enum SpecialCharIndex {
+    Mask = 248,
+    MaxCharIndex = 247,
+    Index8bit = 249,
+    Index14bit = 250,
+    Index21bit = 251
+}
+export type EncodedSequence = [number] | [SpecialCharIndex.Index8bit, number] | [SpecialCharIndex.Index14bit, number, number] | [SpecialCharIndex.Index21bit, number, number, number];
+export declare class NumberSequenceByteEncoderDecoder {
+    static encode(n: number): EncodedSequence;
+    static decode(encodedSequence: EncodedSequence): number;
+    static encodeSequence(sequence: Iterable<number>): Iterable<number>;
+    static decodeSequence(sequence: Iterable<number>): number[];
+    static SpecialCharIndexMask: SpecialCharIndex.Mask;
+    static MaxCharIndex: SpecialCharIndex.MaxCharIndex;
+    /**
+     * SpecialCharIndex8bit is used to indicate a node chain. Where the final character index is 248 + the index found in the next node.
+     */
+    static SpecialCharIndex8bit: SpecialCharIndex.Index8bit;
+    static SpecialCharIndex16bit: SpecialCharIndex.Index14bit;
+    static SpecialCharIndex24bit: SpecialCharIndex.Index21bit;
+}
+export declare class NumberSequenceByteDecoderAccumulator {
+    private byteMode;
+    private accumulation;
+    protected constructor(byteMode?: number, accumulation?: number);
+    decodeSequence(sequence: Iterable<number>): Iterable<number>;
+    decode(idx: number): number | undefined;
+    reset(): void;
+    clone(): NumberSequenceByteDecoderAccumulator;
+    isPending(): boolean;
+    static create(): NumberSequenceByteDecoderAccumulator;
+}
+//# sourceMappingURL=NumberSequenceByteDecoderAccumulator.d.ts.map

package/dist/lib/TrieBlob/NumberSequenceByteDecoderAccumulator.js

@@ -0,0 +1,120 @@
+import { assert } from '../utils/assert.js';
+export var SpecialCharIndex;
+(function (SpecialCharIndex) {
+    SpecialCharIndex[SpecialCharIndex["Mask"] = 248] = "Mask";
+    SpecialCharIndex[SpecialCharIndex["MaxCharIndex"] = 247] = "MaxCharIndex";
+    SpecialCharIndex[SpecialCharIndex["Index8bit"] = 249] = "Index8bit";
+    SpecialCharIndex[SpecialCharIndex["Index14bit"] = 250] = "Index14bit";
+    SpecialCharIndex[SpecialCharIndex["Index21bit"] = 251] = "Index21bit";
+})(SpecialCharIndex || (SpecialCharIndex = {}));
+export class NumberSequenceByteEncoderDecoder {
+    static encode(n) {
+        if (n < this.SpecialCharIndexMask)
+            return [n];
+        if (n < this.SpecialCharIndexMask * 2) {
+            return [SpecialCharIndex.Index8bit, n - this.SpecialCharIndexMask];
+        }
+        if (n < 1 << 14)
+            return [SpecialCharIndex.Index14bit, n >>> 7, n & 0x7f];
+        assert(n < 1 << 21);
+        return [SpecialCharIndex.Index21bit, (n >>> 14) & 0x7f, (n >>> 7) & 0x7f, n & 0x7f];
+    }
+    static decode(encodedSequence) {
+        const [a, b, c, d] = encodedSequence;
+        switch (a) {
+            case SpecialCharIndex.Index8bit:
+                // assert(encodedSequence.length === 2);
+                return (b || 0) + this.SpecialCharIndexMask;
+            case SpecialCharIndex.Index14bit:
+                // assert(encodedSequence.length === 3);
+                return ((b || 0) << 7) + (c || 0);
+            case SpecialCharIndex.Index21bit:
+                // assert(encodedSequence.length === 4);
+                return ((b || 0) << 14) + ((c || 0) << 7) + (d || 0);
+            default:
+                // assert(a <= SpecialCharIndex.MaxCharIndex);
+                return a;
+        }
+    }
+    static *encodeSequence(sequence) {
+        for (const n of sequence) {
+            const encoded = this.encode(n);
+            yield* encoded;
+        }
+    }
+    static decodeSequence(sequence) {
+        const acc = NumberSequenceByteDecoderAccumulator.create();
+        return [...acc.decodeSequence(sequence)];
+    }
+    static SpecialCharIndexMask = SpecialCharIndex.Mask;
+    static MaxCharIndex = SpecialCharIndex.MaxCharIndex;
+    /**
+     * SpecialCharIndex8bit is used to indicate a node chain. Where the final character index is 248 + the index found in the next node.
+     */
+    static SpecialCharIndex8bit = SpecialCharIndex.Index8bit;
+    static SpecialCharIndex16bit = SpecialCharIndex.Index14bit;
+    static SpecialCharIndex24bit = SpecialCharIndex.Index21bit;
+}
+export class NumberSequenceByteDecoderAccumulator {
+    byteMode;
+    accumulation;
+    constructor(byteMode = 0, accumulation = 0) {
+        this.byteMode = byteMode;
+        this.accumulation = accumulation;
+    }
+    *decodeSequence(sequence) {
+        const accumulator = this.clone();
+        for (const idx of sequence) {
+            const r = accumulator.decode(idx);
+            if (r !== undefined) {
+                yield r;
+            }
+        }
+    }
+    decode(idx) {
+        if (!this.byteMode) {
+            if (idx < NumberSequenceByteEncoderDecoder.SpecialCharIndexMask) {
+                const v = idx + this.accumulation;
+                this.accumulation = 0;
+                return v;
+            }
+            switch (idx) {
+                case NumberSequenceByteEncoderDecoder.SpecialCharIndex8bit:
+                    this.accumulation += NumberSequenceByteEncoderDecoder.SpecialCharIndexMask;
+                    break;
+                case NumberSequenceByteEncoderDecoder.SpecialCharIndex16bit:
+                    this.byteMode = 2;
+                    break;
+                case NumberSequenceByteEncoderDecoder.SpecialCharIndex24bit:
+                    this.byteMode = 3;
+                    break;
+                default:
+                    throw new Error('Invalid SpecialCharIndex');
+            }
+            return undefined;
+        }
+        this.accumulation = (this.accumulation << 7) + idx;
+        --this.byteMode;
+        if (!this.byteMode) {
+            const r = this.accumulation;
+            this.accumulation = 0;
+            return r;
+        }
+        return undefined;
+    }
+    reset() {
+        this.byteMode = 0;
+        this.accumulation = 0;
+    }
+    clone() {
+        const n = new NumberSequenceByteDecoderAccumulator(this.byteMode, this.accumulation);
+        return n;
+    }
+    isPending() {
+        return !!(this.byteMode || this.accumulation);
+    }
+    static create() {
+        return new NumberSequenceByteDecoderAccumulator();
+    }
+}
+//# sourceMappingURL=NumberSequenceByteDecoderAccumulator.js.map

package/dist/lib/TrieBlob/TrieBlob.d.ts

@@ -3,13 +3,17 @@ import type { PartialTrieInfo, TrieInfo } from '../ITrieNode/TrieInfo.js';
 import type { TrieData } from '../TrieData.js';
 export declare class TrieBlob implements TrieData {
     protected nodes: Uint32Array;
-    protected charIndex: string[];
+    readonly charIndex: readonly string[];
     protected charToIndexMap: Record<string, number>;
     readonly info: Readonly<TrieInfo>;
     private _forbidIdx;
     private _size;
     private _iTrieRoot;
-    constructor(nodes: Uint32Array, charIndex: string[], info: PartialTrieInfo);
+    wordToCharacters: (word: string) => string[];
+    constructor(nodes: Uint32Array, charIndex: readonly string[], info: PartialTrieInfo);
+    private _lookUpCharIndex;
+    private wordToNodeCharIndexSequence;
+    private letterToNodeCharIndexSequence;
     has(word: string): boolean;
     isForbiddenWord(word: string): boolean;
     hasForbiddenWords(): boolean;
@@ -17,13 +21,26 @@ export declare class TrieBlob implements TrieData {
     private _getRoot;
     getNode(prefix: string): ITrieNode | undefined;
     private _has;
+    /**
+     * Find the node index for the given character.
+     * @param nodeIdx - node index to start the search
+     * @param char - character to look for
+     * @returns
+     */
     private _lookupNode;
+    /**
+     * Find the node index for the given character.
+     * @param nodeIdx - node index to start the search
+     * @param char - character to look for
+     * @returns
+     */
+    private _lookupNodeByCharIndexSeq;
     words(): Iterable<string>;
     get size(): number;
     toJSON(): {
-        charIndex: string[];
         options: Readonly<TrieInfo>;
-        nodes: string[];
+        nodes: NodeElement[];
+        charIndex: readonly string[];
     };
     encodeBin(): Uint8Array;
     static decodeBin(blob: Uint8Array): TrieBlob;
@@ -31,6 +48,42 @@ export declare class TrieBlob implements TrieData {
     static NodeMaskNumChildren: number;
     static NodeMaskNumChildrenShift: number;
     static NodeChildRefShift: number;
+    /**
+     * Only 8 bits are reserved for the character index.
+     * The max index is {@link TrieBlob.SpecialCharIndexMask} - 1.
+     * Node chaining is used to reference higher character indexes.
+     * - @see {@link TrieBlob.SpecialCharIndexMask}
+     * - @see {@link TrieBlob.MaxCharIndex}
+     */
     static NodeMaskChildCharIndex: number;
+    /** SpecialCharIndexMask is used to indicate a node chain */
+    static SpecialCharIndexMask: number;
+    static MaxCharIndex: number;
+    /**
+     * SpecialCharIndex8bit is used to indicate a node chain. Where the final character index is 248 + the index found in the next node.
+     */
+    static SpecialCharIndex8bit: number;
+    static SpecialCharIndex16bit: number;
+    static SpecialCharIndex24bit: number;
+    /**
+     * Since it is only possible to store single byte indexes, a multi-byte index is stored as a sequence of indexes chained between nodes.
+     * @param charIndex - character index to convert to a sequence of indexes
+     * @returns encoded index values.
+     */
+    static toCharIndexSequence(charIndex: number): number[];
+    static fromCharIndexSequence(charIndexes: Iterable<number>): Iterable<number>;
+    static charactersToCharIndexSequence(chars: readonly string[], charToIndexMap: Readonly<Record<string, number>> | ((char: string) => number)): number[];
+    static charIndexSequenceToCharacters(charIndexSequence: number[], charIndex: readonly string[] | Readonly<Record<number, string>>): readonly string[];
+    static nodesView(trie: TrieBlob): Readonly<Uint32Array>;
 }
+interface NodeElement {
+    id: number;
+    eow: boolean;
+    c: {
+        c: number | string;
+        o: number;
+    }[];
+    n: number;
+}
+export {};
 //# sourceMappingURL=TrieBlob.d.ts.map