cspell-dictionary 8.11.0 → 8.13.0

package/dist/SpellingDictionary/CachingDictionary.d.ts

@@ -21,6 +21,17 @@ export interface CachingDictionary {
     stats(): CallStats;
     getPreferredSuggestions(word: string): PreferredSuggestion[] | undefined;
 }
+interface LogEntryBase extends SearchOptions {
+    time: number;
+    method: 'has';
+    word: string;
+    value?: unknown;
+}
+interface LogEntryHas extends LogEntryBase {
+    method: 'has';
+    value: boolean;
+}
+export type LogEntry = LogEntryHas;
 /**
  * create a caching dictionary
  * @param dict - Dictionary to cache the search results.
@@ -28,5 +39,7 @@ export interface CachingDictionary {
  * @returns CachingDictionary
  */
 export declare function createCachingDictionary(dict: SpellingDictionary | SpellingDictionaryCollection, options: SearchOptions): CachingDictionary;
+export declare function enableLogging(enabled?: boolean): void;
+export declare function getLog(): LogEntryBase[];
 export {};
 //# sourceMappingURL=CachingDictionary.d.ts.map

package/dist/SpellingDictionary/CachingDictionary.js

@@ -2,6 +2,9 @@ import { autoCache, extractStats } from '../util/AutoCache.js';
 import { canonicalSearchOptions } from './SpellingDictionaryMethods.js';
 let dictionaryCounter = 0;
 const DefaultAutoCacheSize = 1000;
+let logRequests = false;
+const log = [];
+const startTime = performance.now();
 class CachedDict {
     dict;
     options;
@@ -13,7 +16,15 @@ class CachedDict {
         this.name = dict.name;
         // console.log(`CachedDict for ${this.name}`);
     }
-    has = autoCache((word) => this.dict.has(word, this.options), DefaultAutoCacheSize);
+    #has = autoCache((word) => this.dict.has(word, this.options), DefaultAutoCacheSize);
+    has = logRequests
+        ? (word) => {
+            const time = performance.now() - startTime;
+            const value = this.#has(word);
+            log.push({ time, method: 'has', word, value });
+            return value;
+        }
+        : this.#has;
     isNoSuggestWord = autoCache((word) => this.dict.isNoSuggestWord(word, this.options), DefaultAutoCacheSize);
     isForbidden = autoCache((word) => this.dict.isForbidden(word), DefaultAutoCacheSize);
     getPreferredSuggestions = autoCache((word) => this.dict.getPreferredSuggestions?.(word), DefaultAutoCacheSize);
@@ -21,7 +32,7 @@ class CachedDict {
         return {
             name: this.name,
             id: this.id,
-            has: extractStats(this.has),
+            has: extractStats(this.#has),
             isNoSuggestWord: extractStats(this.isNoSuggestWord),
             isForbidden: extractStats(this.isForbidden),
             getPreferredSuggestions: extractStats(this.getPreferredSuggestions),
@@ -49,4 +60,10 @@ export function createCachingDictionary(dict, options) {
     knownOptions.set(dict, cached);
     return cached;
 }
+export function enableLogging(enabled = !logRequests) {
+    logRequests = enabled;
+}
+export function getLog() {
+    return log;
+}
 //# sourceMappingURL=CachingDictionary.js.map

package/dist/SpellingDictionary/SpellingDictionaryCollection.js

@@ -1,5 +1,4 @@
 import { CASE_INSENSITIVE_PREFIX, CompoundWordsMethod } from 'cspell-trie-lib';
-import { genSequence } from 'gensequence';
 import { isDefined } from '../util/util.js';
 import * as Defaults from './defaults.js';
 import { defaultNumSuggestions, hasOptionToSearchOption, suggestionCollector } from './SpellingDictionaryMethods.js';
@@ -98,7 +97,7 @@ export function createCollection(dictionaries, name, source) {
     return new SpellingDictionaryCollectionImpl(dictionaries, name, source);
 }
 function isWordInAnyDictionary(dicts, word, options) {
-    return genSequence(dicts).first((dict) => dict.has(word, options));
+    return dicts.find((dict) => dict.has(word, options));
 }
 function findInAnyDictionary(dicts, word, options) {
     const found = dicts.map((dict) => dict.find(word, options)).filter(isDefined);
@@ -111,10 +110,10 @@ function findInAnyDictionary(dicts, word, options) {
     }));
 }
 function isNoSuggestWordInAnyDictionary(dicts, word, options) {
-    return genSequence(dicts).first((dict) => dict.isNoSuggestWord(word, options));
+    return dicts.find((dict) => dict.isNoSuggestWord(word, options));
 }
 function isWordForbiddenInAnyDictionary(dicts, word, ignoreCase) {
-    return genSequence(dicts).first((dict) => dict.isForbidden(word, ignoreCase));
+    return dicts.find((dict) => dict.isForbidden(word, ignoreCase));
 }
 export function isSpellingDictionaryCollection(dict) {
     return dict instanceof SpellingDictionaryCollectionImpl;

package/dist/SpellingDictionary/SpellingDictionaryFromTrie.d.ts

@@ -6,7 +6,6 @@ export declare class SpellingDictionaryFromTrie implements SpellingDictionary {
     readonly name: string;
     readonly options: SpellingDictionaryOptions;
     readonly source: string;
-    static readonly cachedWordsLimit = 50000;
     private _size;
     readonly knownWords: Set<string>;
     readonly unknownWords: Set<string>;
@@ -26,7 +25,6 @@ export declare class SpellingDictionaryFromTrie implements SpellingDictionary {
     private _findAnyForm;
     isNoSuggestWord(word: string, options?: HasOptions): boolean;
     isForbidden(word: string, _ignoreCaseAndAccents?: boolean): boolean;
-    private _isForbidden;
     suggest(word: string, suggestOptions?: SuggestOptions): SuggestionResult[];
     private _suggest;
     genSuggestions(collector: SuggestionCollector, suggestOptions: SuggestOptions): void;

package/dist/SpellingDictionary/SpellingDictionaryFromTrie.js

@@ -1,5 +1,4 @@
 import { CompoundWordsMethod, decodeTrie, suggestionCollector } from 'cspell-trie-lib';
-import { autoCache, createCache01 } from '../util/AutoCache.js';
 import { clean } from '../util/clean.js';
 import { createMapper, createRepMapper } from '../util/repMap.js';
 import * as Defaults from './defaults.js';
@@ -11,7 +10,6 @@ export class SpellingDictionaryFromTrie {
     name;
     options;
     source;
-    static cachedWordsLimit = 50_000;
     _size = 0;
     knownWords = new Set();
     unknownWords = new Set();
@@ -68,7 +66,7 @@ export class SpellingDictionaryFromTrie {
         const { useCompounds = this.options.useCompounds, ignoreCase = Defaults.ignoreCase } = hasOptionToSearchOption(hasOptions);
         return { useCompounds, ignoreCase };
     }
-    _find = findCache((word, useCompounds, ignoreCase) => this.findAnyForm(word, useCompounds, ignoreCase));
+    _find = (word, useCompounds, ignoreCase) => this.findAnyForm(word, useCompounds, ignoreCase);
     findAnyForm(word, useCompounds, ignoreCase) {
         const outerForms = outerWordForms(word, this.remapWord || ((word) => [this.mapWord(word)]));
         for (const form of outerForms) {
@@ -106,11 +104,8 @@ export class SpellingDictionaryFromTrie {
         return this.containsNoSuggestWords ? this.has(word, options) : false;
     }
     isForbidden(word, _ignoreCaseAndAccents) {
-        return this._isForbidden(word);
-    }
-    _isForbidden = autoCache((word) => {
         return this.trie.isForbiddenWord(word);
-    });
+    }
     suggest(word, suggestOptions = {}) {
         return this._suggest(word, suggestOptions);
     }
@@ -154,38 +149,29 @@ export function createSpellingDictionaryFromTrieFile(data, name, source, options
     const trie = decodeTrie(data);
     return new SpellingDictionaryFromTrie(trie, name, options, source);
 }
-function findCache(fn, size = 2000) {
-    const cache = createCache01(size);
-    function find(word, useCompounds, ignoreCase) {
-        const r = cache.get(word);
-        if (r !== undefined && r.useCompounds === useCompounds && r.ignoreCase === ignoreCase) {
-            return r.findResult;
-        }
-        const findResult = fn(word, useCompounds, ignoreCase);
-        cache.set(word, { useCompounds, ignoreCase, findResult });
-        return findResult;
-    }
-    return find;
-}
 function* outerWordForms(word, mapWord) {
     // Only generate the needed forms.
     const sent = new Set();
     let w = word;
+    const ww = w;
     yield w;
     sent.add(w);
     w = word.normalize('NFC');
-    if (!sent.has(w))
+    if (w !== ww) {
         yield w;
-    sent.add(w);
+        sent.add(w);
+    }
     w = word.normalize('NFD');
-    if (!sent.has(w))
+    if (w !== ww && !sent.has(w)) {
         yield w;
-    sent.add(w);
-    for (const f of [...sent]) {
+        sent.add(w);
+    }
+    for (const f of sent) {
         for (const m of mapWord(f)) {
-            if (!sent.has(m))
+            if (m !== ww && !sent.has(m)) {
                 yield m;
-            sent.add(m);
+                sent.add(m);
+            }
         }
     }
     return;

package/dist/index.d.ts

@@ -1,3 +1,11 @@
+import { enableLogging as cacheDictionaryEnableLogging, getLog as cacheDictionaryGetLog } from './SpellingDictionary/CachingDictionary.js';
 export type { CachingDictionary, FindOptions, FindResult, HasOptions, SearchOptions, SpellingDictionary, SpellingDictionaryCollection, SpellingDictionaryOptions, SuggestionCollector, SuggestionResult, SuggestOptions, } from './SpellingDictionary/index.js';
 export { createCachingDictionary, createCollection, createFailedToLoadDictionary, createFlagWordsDictionary, createForbiddenWordsDictionary, createIgnoreWordsDictionary, createInlineSpellingDictionary, createSpellingDictionary, createSpellingDictionaryFromTrieFile, createSuggestDictionary, createSuggestOptions, } from './SpellingDictionary/index.js';
+/**
+ * Debugging utilities.
+ */
+export declare const _debug: {
+    cacheDictionaryEnableLogging: typeof cacheDictionaryEnableLogging;
+    cacheDictionaryGetLog: typeof cacheDictionaryGetLog;
+};
 //# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -1,2 +1,10 @@
+import { enableLogging as cacheDictionaryEnableLogging, getLog as cacheDictionaryGetLog, } from './SpellingDictionary/CachingDictionary.js';
 export { createCachingDictionary, createCollection, createFailedToLoadDictionary, createFlagWordsDictionary, createForbiddenWordsDictionary, createIgnoreWordsDictionary, createInlineSpellingDictionary, createSpellingDictionary, createSpellingDictionaryFromTrieFile, createSuggestDictionary, createSuggestOptions, } from './SpellingDictionary/index.js';
+/**
+ * Debugging utilities.
+ */
+export const _debug = {
+    cacheDictionaryEnableLogging,
+    cacheDictionaryGetLog,
+};
 //# sourceMappingURL=index.js.map

package/dist/perf/has.perf.js

@@ -2,26 +2,41 @@ import assert from 'node:assert';
 import { buildITrieFromWords } from 'cspell-trie-lib';
 import { loremIpsum } from 'lorem-ipsum';
 import { suite } from 'perf-insight';
+import { createCachingDictionary } from '../SpellingDictionary/CachingDictionary.js';
 import { createSpellingDictionary } from '../SpellingDictionary/createSpellingDictionary.js';
 import { createCollection } from '../SpellingDictionary/SpellingDictionaryCollection.js';
 suite('dictionary has', async (test) => {
-    const words = genWords(10_000);
+    const words1 = genWords(10_000);
     const words2 = genWords(1000);
     const words3 = genWords(1000);
-    const iTrie = buildITrieFromWords(words);
-    const dict = createSpellingDictionary(words, 'test', import.meta.url);
+    const words = words1;
+    const iTrie = buildITrieFromWords(words1);
+    const dict = createSpellingDictionary(words1, 'test', import.meta.url);
     const dict2 = createSpellingDictionary(words2, 'test2', import.meta.url);
     const dict3 = createSpellingDictionary(words3, 'test3', import.meta.url);
     const dictCol = createCollection([dict, dict2, dict3], 'test-collection');
-    test('dictionary has 100k words', () => {
-        checkWords(dict, words);
+    const dictColRev = createCollection([dict3, dict2, dict], 'test-collection-reverse');
+    const cacheDictSingle = createCachingDictionary(dict, {});
+    const cacheDictCol = createCachingDictionary(dictCol, {});
+    const dictSet = new Set(words);
+    test('Set has 100k words', () => {
+        checkWords(dictSet, words);
     });
-    test('dictionary has 100k words (2nd time)', () => {
+    test('dictionary has 100k words', () => {
         checkWords(dict, words);
     });
     test('collection has 100k words', () => {
         checkWords(dictCol, words);
     });
+    test('collection reverse has 100k words', () => {
+        checkWords(dictColRev, words);
+    });
+    test('cache dictionary has 100k words', () => {
+        checkWords(cacheDictSingle, words);
+    });
+    test('cache collection has 100k words', () => {
+        checkWords(cacheDictCol, words);
+    });
     test('iTrie has 100k words', () => {
         checkWords(iTrie, words);
     });
@@ -43,24 +58,98 @@ suite('dictionary has Not', async (test) => {
     const dict2 = createSpellingDictionary(words2, 'test2', import.meta.url);
     const dict3 = createSpellingDictionary(words3, 'test3', import.meta.url);
     const dictCol = createCollection([dict, dict2, dict3], 'test-collection');
-    test('dictionary has 100k words', () => {
+    const dictSet = new Set(words);
+    test('Set has not 100k words', () => {
+        checkWords(dictSet, missingWords, false);
+    });
+    test('dictionary has not 100k words', () => {
         checkWords(dict, missingWords, false);
     });
-    test('dictionary has 100k words (2nd time)', () => {
+    test('collection has not 100k words', () => {
+        checkWords(dictCol, missingWords, false);
+    });
+    test('iTrie has not 100k words', () => {
+        checkWords(iTrie, missingWords, false);
+    });
+    test('iTrie.hasWord has not 100k words', () => {
+        const dict = { has: (word) => iTrie.hasWord(word, true) };
         checkWords(dict, missingWords, false);
     });
+    test('iTrie.data has not 100k words', () => {
+        checkWords(iTrie.data, missingWords, false);
+    });
+});
+suite('dictionary has sampling', async (test) => {
+    const words1 = genWords(10_000);
+    const words2 = genWords(1000);
+    const words3 = genWords(1000);
+    const sampleIdx = genSamples(100_000, words1.length);
+    const wordsSample = sampleIdx.map((i) => words1[i]);
+    const iTrie = buildITrieFromWords(words1);
+    const dict = createSpellingDictionary(words1, 'test', import.meta.url);
+    const dict2 = createSpellingDictionary(words2, 'test2', import.meta.url);
+    const dict3 = createSpellingDictionary(words3, 'test3', import.meta.url);
+    const dictCol = createCollection([dict, dict2, dict3], 'test-collection');
+    const dictColRev = createCollection([dict3, dict2, dict], 'test-collection-reverse');
+    const cacheDictSingle = createCachingDictionary(dict, {});
+    const cacheDictCol = createCachingDictionary(dictCol, {});
+    const dictSet = new Set(words1);
+    test('Set has 100k words', () => {
+        checkWords(dictSet, wordsSample);
+    });
+    test('dictionary has 100k words', () => {
+        checkWords(dict, wordsSample);
+    });
     test('collection has 100k words', () => {
-        checkWords(dictCol, missingWords, false);
+        checkWords(dictCol, wordsSample);
+    });
+    test('collection reverse has 100k words', () => {
+        checkWords(dictColRev, wordsSample);
+    });
+    test('cache dictionary has 100k words', () => {
+        checkWords(cacheDictSingle, wordsSample);
+    });
+    test('cache collection has 100k words', () => {
+        checkWords(cacheDictCol, wordsSample);
     });
     test('iTrie has 100k words', () => {
-        checkWords(iTrie, missingWords, false);
+        checkWords(iTrie, wordsSample);
     });
     test('iTrie.hasWord has 100k words', () => {
         const dict = { has: (word) => iTrie.hasWord(word, true) };
-        checkWords(dict, missingWords, false);
+        checkWords(dict, wordsSample);
     });
     test('iTrie.data has 100k words', () => {
-        checkWords(iTrie.data, missingWords, false);
+        checkWords(iTrie.data, wordsSample);
+    });
+});
+suite('dictionary isForbidden sampling', async (test) => {
+    const words1 = genWords(10_000);
+    const words2 = genWords(1000);
+    const words3 = genWords(1000);
+    const sampleIdx = genSamples(100_000, words1.length);
+    const wordsSample = sampleIdx.map((i) => words1[i]);
+    const dict = createSpellingDictionary(words1, 'test', import.meta.url);
+    const dict2 = createSpellingDictionary(words2, 'test2', import.meta.url);
+    const dict3 = createSpellingDictionary(words3, 'test3', import.meta.url);
+    const dictCol = createCollection([dict, dict2, dict3], 'test-collection');
+    const dictColRev = createCollection([dict3, dict2, dict], 'test-collection-reverse');
+    const cacheDictSingle = createCachingDictionary(dict, {});
+    const cacheDictCol = createCachingDictionary(dictCol, {});
+    test('dictionary isForbidden 100k words', () => {
+        checkForForbiddenWords(dict, wordsSample);
+    });
+    test('collection isForbidden 100k words', () => {
+        checkForForbiddenWords(dictCol, wordsSample);
+    });
+    test('collection reverse isForbidden 100k words', () => {
+        checkForForbiddenWords(dictColRev, wordsSample);
+    });
+    test('cache dictionary isForbidden 100k words', () => {
+        checkForForbiddenWords(cacheDictSingle, wordsSample);
+    });
+    test('cache collection isForbidden 100k words', () => {
+        checkForForbiddenWords(cacheDictCol, wordsSample);
     });
 });
 function checkWords(dict, words, expected = true, totalChecks = 100_000) {
@@ -76,6 +165,16 @@ function checkWords(dict, words, expected = true, totalChecks = 100_000) {
     }
     assert(has, 'All words should be found in the dictionary');
 }
+function checkForForbiddenWords(dict, words, totalChecks = 100_000) {
+    let result = true;
+    const len = words.length;
+    for (let i = 0; i < totalChecks; ++i) {
+        const word = words[i % len];
+        const r = !dict.isForbidden(word);
+        result = r && result;
+    }
+    assert(result, 'All words should not be forbidden');
+}
 function genWords(count, includeForbidden = true) {
     const setOfWords = new Set(loremIpsum({ count }).split(' '));
     if (includeForbidden) {
@@ -101,4 +200,15 @@ function genWords(count, includeForbidden = true) {
     }
     return [...setOfWords];
 }
+function genSamples(count, max, depth = 3) {
+    const r = Array(count);
+    for (let j = 0; j < count; ++j) {
+        let n = Math.random() * max;
+        for (let i = 1; i < depth; ++i) {
+            n = Math.random() * n;
+        }
+        r[j] = Math.floor(n);
+    }
+    return r;
+}
 //# sourceMappingURL=has.perf.js.map

package/dist/perf/misc.perf.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=misc.perf.d.ts.map

package/dist/perf/misc.perf.js

@@ -0,0 +1,48 @@
+import { genSequence } from 'gensequence';
+import { loremIpsum } from 'lorem-ipsum';
+import { suite } from 'perf-insight';
+suite('Array Primitives', async (test) => {
+    const words = genWords(20);
+    const toFind = [...words, 'not-a-word'];
+    const iterations = 1000;
+    test('Array.find', () => {
+        for (let i = 0; i < iterations; ++i) {
+            for (const word of toFind) {
+                words.find((w) => w === word);
+            }
+        }
+    });
+    test('genSequence.first', () => {
+        for (let i = 0; i < iterations; ++i) {
+            for (const word of toFind) {
+                genSequence(words).first((w) => w === word);
+            }
+        }
+    });
+});
+function genWords(count, includeForbidden = true) {
+    const setOfWords = new Set(loremIpsum({ count }).split(' '));
+    if (includeForbidden) {
+        setOfWords.add('!forbidden');
+        setOfWords.add('!bad-word');
+        setOfWords.add('!rejection');
+    }
+    while (setOfWords.size < count) {
+        const words = [...setOfWords];
+        for (const a of words) {
+            for (const b of words) {
+                if (a !== b) {
+                    setOfWords.add(a + b);
+                }
+                if (setOfWords.size >= count) {
+                    break;
+                }
+            }
+            if (setOfWords.size >= count) {
+                break;
+            }
+        }
+    }
+    return [...setOfWords];
+}
+//# sourceMappingURL=misc.perf.js.map

package/dist/util/AutoCache.d.ts

@@ -6,17 +6,14 @@ export interface CacheStats {
     misses: number;
     swaps: number;
 }
-declare class Cache01<R> implements CacheStats {
+declare abstract class Cache01<R> implements CacheStats {
     readonly maxSize: number;
-    private count;
-    private cache0;
-    private cache1;
     hits: number;
     misses: number;
     swaps: number;
     constructor(maxSize: number);
-    get(key: string): R | undefined;
-    set(key: string, value: R): this;
+    abstract get(key: string): R | undefined;
+    abstract set(key: string, value: R): this;
 }
 export declare function createCache01<R>(size: number): Cache01<R>;
 export declare function autoCache<R>(fn: (p: string) => R, size?: number): AutoCache<R>;

package/dist/util/AutoCache.js

@@ -1,46 +1,54 @@
 const CACHE_SIZE = 100;
 class Cache01 {
     maxSize;
-    count = 0;
-    cache0 = Object.create(null);
-    cache1 = Object.create(null);
     hits = 0;
     misses = 0;
     swaps = 0;
     constructor(maxSize) {
         this.maxSize = maxSize;
     }
+}
+class Cache01Map extends Cache01 {
+    count = 0;
+    cache0 = new Map();
+    cache1 = new Map();
+    constructor(maxSize) {
+        super(maxSize);
+    }
     get(key) {
         const cache0 = this.cache0;
         const cache1 = this.cache1;
-        if (key in cache0) {
+        let found = cache0.get(key);
+        if (found !== undefined) {
             ++this.hits;
-            return cache0[key];
+            return found;
         }
-        if (key in cache1) {
+        found = cache1.get(key);
+        if (found !== undefined) {
             ++this.hits;
             ++this.count;
-            const r = cache1[key];
-            cache0[key] = r;
-            return r;
+            cache0.set(key, found);
+            return found;
         }
         ++this.misses;
         return undefined;
     }
     set(key, value) {
         if (this.count >= this.maxSize) {
+            const c = this.cache1;
             this.cache1 = this.cache0;
-            this.cache0 = Object.create(null);
+            this.cache0 = c;
+            c.clear();
             this.swaps++;
             this.count = 0;
         }
         ++this.count;
-        this.cache0[key] = value;
+        this.cache0.set(key, value);
         return this;
     }
 }
 export function createCache01(size) {
-    return new Cache01(size);
+    return new Cache01Map(size);
 }
 export function autoCache(fn, size = CACHE_SIZE) {
     const cache = createCache01(size);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cspell-dictionary",
-  "version": "8.11.0",
+  "version": "8.13.0",
   "description": "A spelling dictionary library useful for checking words and getting suggestions.",
   "type": "module",
   "sideEffects": false,
@@ -50,14 +50,14 @@
     "node": ">=18"
   },
   "dependencies": {
-    "@cspell/cspell-pipe": "8.11.0",
-    "@cspell/cspell-types": "8.11.0",
-    "cspell-trie-lib": "8.11.0",
-    "fast-equals": "^5.0.1",
-    "gensequence": "^7.0.0"
+    "@cspell/cspell-pipe": "8.13.0",
+    "@cspell/cspell-types": "8.13.0",
+    "cspell-trie-lib": "8.13.0",
+    "fast-equals": "^5.0.1"
   },
   "devDependencies": {
+    "gensequence": "^7.0.0",
     "lorem-ipsum": "^2.0.8"
   },
-  "gitHead": "2b85b2b458b1117870a4f0aee18fb45ce991848d"
+  "gitHead": "2fd3fb430cc96a8a50543f57d96b288219a11923"
 }