cspell-dictionary 8.10.2 → 8.11.0

package/dist/SpellingDictionary/SpellingDictionaryFromTrie.d.ts

@@ -41,7 +41,7 @@ export declare class SpellingDictionaryFromTrie implements SpellingDictionary {
  * @returns SpellingDictionary
  */
 export declare function createSpellingDictionaryFromTrieFile(data: string | Buffer, name: string, source: string, options: SpellingDictionaryOptions): SpellingDictionary;
-declare function outerWordForms(word: string, mapWord: (word: string) => string[]): Set<string>;
+declare function outerWordForms(word: string, mapWord: (word: string) => string[]): Iterable<string>;
 export declare const __testing__: {
     outerWordForms: typeof outerWordForms;
 };

package/dist/SpellingDictionary/SpellingDictionaryFromTrie.js

@@ -1,4 +1,3 @@
-import { opConcatMap, pipe } from '@cspell/cspell-pipe/sync';
 import { CompoundWordsMethod, decodeTrie, suggestionCollector } from 'cspell-trie-lib';
 import { autoCache, createCache01 } from '../util/AutoCache.js';
 import { clean } from '../util/clean.js';
@@ -71,7 +70,7 @@ export class SpellingDictionaryFromTrie {
     }
     _find = findCache((word, useCompounds, ignoreCase) => this.findAnyForm(word, useCompounds, ignoreCase));
     findAnyForm(word, useCompounds, ignoreCase) {
-        const outerForms = outerWordForms(word, this.remapWord ? this.remapWord : (word) => [this.mapWord(word)]);
+        const outerForms = outerWordForms(word, this.remapWord || ((word) => [this.mapWord(word)]));
         for (const form of outerForms) {
             const r = this._findAnyForm(form, useCompounds, ignoreCase);
             if (r)
@@ -168,9 +167,28 @@ function findCache(fn, size = 2000) {
     }
     return find;
 }
-function outerWordForms(word, mapWord) {
-    const forms = pipe([word], opConcatMap((word) => [word, word.normalize('NFC'), word.normalize('NFD')]), opConcatMap((word) => [word, ...mapWord(word)]));
-    return new Set(forms);
+function* outerWordForms(word, mapWord) {
+    // Only generate the needed forms.
+    const sent = new Set();
+    let w = word;
+    yield w;
+    sent.add(w);
+    w = word.normalize('NFC');
+    if (!sent.has(w))
+        yield w;
+    sent.add(w);
+    w = word.normalize('NFD');
+    if (!sent.has(w))
+        yield w;
+    sent.add(w);
+    for (const f of [...sent]) {
+        for (const m of mapWord(f)) {
+            if (!sent.has(m))
+                yield m;
+            sent.add(m);
+        }
+    }
+    return;
 }
 export const __testing__ = { outerWordForms };
 //# sourceMappingURL=SpellingDictionaryFromTrie.js.map

package/dist/perf/has.perf.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=has.perf.d.ts.map

package/dist/perf/has.perf.js

@@ -0,0 +1,104 @@
+import assert from 'node:assert';
+import { buildITrieFromWords } from 'cspell-trie-lib';
+import { loremIpsum } from 'lorem-ipsum';
+import { suite } from 'perf-insight';
+import { createSpellingDictionary } from '../SpellingDictionary/createSpellingDictionary.js';
+import { createCollection } from '../SpellingDictionary/SpellingDictionaryCollection.js';
+suite('dictionary has', async (test) => {
+    const words = genWords(10_000);
+    const words2 = genWords(1000);
+    const words3 = genWords(1000);
+    const iTrie = buildITrieFromWords(words);
+    const dict = createSpellingDictionary(words, 'test', import.meta.url);
+    const dict2 = createSpellingDictionary(words2, 'test2', import.meta.url);
+    const dict3 = createSpellingDictionary(words3, 'test3', import.meta.url);
+    const dictCol = createCollection([dict, dict2, dict3], 'test-collection');
+    test('dictionary has 100k words', () => {
+        checkWords(dict, words);
+    });
+    test('dictionary has 100k words (2nd time)', () => {
+        checkWords(dict, words);
+    });
+    test('collection has 100k words', () => {
+        checkWords(dictCol, words);
+    });
+    test('iTrie has 100k words', () => {
+        checkWords(iTrie, words);
+    });
+    test('iTrie.hasWord has 100k words', () => {
+        const dict = { has: (word) => iTrie.hasWord(word, true) };
+        checkWords(dict, words);
+    });
+    test('iTrie.data has 100k words', () => {
+        checkWords(iTrie.data, words);
+    });
+});
+suite('dictionary has Not', async (test) => {
+    const words = genWords(10_000);
+    const words2 = genWords(1000);
+    const words3 = genWords(1000);
+    const missingWords = words.map((w) => w + '-x-');
+    const iTrie = buildITrieFromWords(words);
+    const dict = createSpellingDictionary(words, 'test', import.meta.url);
+    const dict2 = createSpellingDictionary(words2, 'test2', import.meta.url);
+    const dict3 = createSpellingDictionary(words3, 'test3', import.meta.url);
+    const dictCol = createCollection([dict, dict2, dict3], 'test-collection');
+    test('dictionary has 100k words', () => {
+        checkWords(dict, missingWords, false);
+    });
+    test('dictionary has 100k words (2nd time)', () => {
+        checkWords(dict, missingWords, false);
+    });
+    test('collection has 100k words', () => {
+        checkWords(dictCol, missingWords, false);
+    });
+    test('iTrie has 100k words', () => {
+        checkWords(iTrie, missingWords, false);
+    });
+    test('iTrie.hasWord has 100k words', () => {
+        const dict = { has: (word) => iTrie.hasWord(word, true) };
+        checkWords(dict, missingWords, false);
+    });
+    test('iTrie.data has 100k words', () => {
+        checkWords(iTrie.data, missingWords, false);
+    });
+});
+function checkWords(dict, words, expected = true, totalChecks = 100_000) {
+    let has = true;
+    const len = words.length;
+    for (let i = 0; i < totalChecks; ++i) {
+        const word = words[i % len];
+        const r = expected === dict.has(word);
+        if (!r) {
+            console.log(`Word ${expected ? 'not found' : 'found'}: ${word}`);
+        }
+        has = r && has;
+    }
+    assert(has, 'All words should be found in the dictionary');
+}
+function genWords(count, includeForbidden = true) {
+    const setOfWords = new Set(loremIpsum({ count }).split(' '));
+    if (includeForbidden) {
+        setOfWords.add('!forbidden');
+        setOfWords.add('!bad-word');
+        setOfWords.add('!rejection');
+    }
+    while (setOfWords.size < count) {
+        const words = [...setOfWords];
+        for (const a of words) {
+            for (const b of words) {
+                if (a !== b) {
+                    setOfWords.add(a + b);
+                }
+                if (setOfWords.size >= count) {
+                    break;
+                }
+            }
+            if (setOfWords.size >= count) {
+                break;
+            }
+        }
+    }
+    return [...setOfWords];
+}
+//# sourceMappingURL=has.perf.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cspell-dictionary",
-  "version": "8.10.2",
+  "version": "8.11.0",
   "description": "A spelling dictionary library useful for checking words and getting suggestions.",
   "type": "module",
   "sideEffects": false,
@@ -22,12 +22,15 @@
   ],
   "scripts": {
     "clean": "shx rm -rf dist temp coverage \"*.tsbuildInfo\"",
-    "build": "tsc -b . -f",
+    "build": "tsc -p .",
     "clean-build": "pnpm run clean && pnpm run build",
     "coverage": "vitest run --coverage",
     "test:watch": "vitest",
     "test": "vitest run",
-    "watch": "tsc -b . -w -f"
+    "test:perf": "NODE_ENV=production insight --register ts-node/esm --file \"**/*.perf.{mts,ts}\" -t 500",
+    "test:perf:js": "NODE_ENV=production insight -t 500",
+    "test:perf:prof": "NODE_ENV=production node --cpu-prof  ../../node_modules/perf-insight/bin.mjs",
+    "watch": "tsc -p . -w "
   },
   "repository": {
     "type": "git",
@@ -47,11 +50,14 @@
     "node": ">=18"
   },
   "dependencies": {
-    "@cspell/cspell-pipe": "8.10.2",
-    "@cspell/cspell-types": "8.10.2",
-    "cspell-trie-lib": "8.10.2",
+    "@cspell/cspell-pipe": "8.11.0",
+    "@cspell/cspell-types": "8.11.0",
+    "cspell-trie-lib": "8.11.0",
     "fast-equals": "^5.0.1",
     "gensequence": "^7.0.0"
   },
-  "gitHead": "1a1f15b593a0b25419c1e74eeb53d0b2839762cc"
+  "devDependencies": {
+    "lorem-ipsum": "^2.0.8"
+  },
+  "gitHead": "2b85b2b458b1117870a4f0aee18fb45ce991848d"
 }