@cspell/cspell-tools 8.15.4 → 8.15.6

package/cspell-tools.config.schema.json

@@ -38,7 +38,8 @@
               },
               "type": "array"
             }
-          ]
+          ],
+          "description": "Words in the `allowedSplitWords` are considered correct and can be used as a basis for splitting compound words.\n\nIf entries can be split so that all the words in the entry are allowed, then only the individual words are added, otherwise the entire entry is added. This is to prevent misspellings in CamelCase words from being introduced into the dictionary."
         },
         "keepRawCase": {
           "default": false,
@@ -52,6 +53,11 @@
           "description": "Maximum number of nested Hunspell Rules to apply. This is needed for recursive dictionaries like Hebrew.",
           "type": "number"
         },
+        "minCompoundLength": {
+          "default": 4,
+          "description": "Controls the minimum length of a compound word when storing words using `storeSplitWordsAsCompounds`. The compound words are prefixed / suffixed with `*`, to allow them to be combined with other compound words. If the length is too low, then the dictionary will consider many misspelled words as correct.",
+          "type": "number"
+        },
         "split": {
           "anyOf": [
             {
@@ -64,6 +70,11 @@
           ],
           "default": false,
           "description": "Split lines into words."
+        },
+        "storeSplitWordsAsCompounds": {
+          "default": false,
+          "description": "Camel case words that have been split using the `allowedSplitWords` are added to the dictionary as compoundable words. These words are prefixed / suffixed with `*`.",
+          "type": "boolean"
         }
       },
       "required": [
@@ -89,7 +100,8 @@
               },
               "type": "array"
             }
-          ]
+          ],
+          "description": "Words in the `allowedSplitWords` are considered correct and can be used as a basis for splitting compound words.\n\nIf entries can be split so that all the words in the entry are allowed, then only the individual words are added, otherwise the entire entry is added. This is to prevent misspellings in CamelCase words from being introduced into the dictionary."
         },
         "filename": {
           "$ref": "#/definitions/FilePath"
@@ -103,6 +115,11 @@
           "description": "Maximum number of nested Hunspell Rules to apply. This is needed for recursive dictionaries like Hebrew.",
           "type": "number"
         },
+        "minCompoundLength": {
+          "default": 4,
+          "description": "Controls the minimum length of a compound word when storing words using `storeSplitWordsAsCompounds`. The compound words are prefixed / suffixed with `*`, to allow them to be combined with other compound words. If the length is too low, then the dictionary will consider many misspelled words as correct.",
+          "type": "number"
+        },
         "split": {
           "anyOf": [
             {
@@ -115,6 +132,11 @@
           ],
           "default": false,
           "description": "Split lines into words."
+        },
+        "storeSplitWordsAsCompounds": {
+          "default": false,
+          "description": "Camel case words that have been split using the `allowedSplitWords` are added to the dictionary as compoundable words. These words are prefixed / suffixed with `*`.",
+          "type": "boolean"
         }
       },
       "required": [
@@ -171,6 +193,11 @@
           "description": "Name of target, used as the basis of target file name.",
           "type": "string"
         },
+        "removeDuplicates": {
+          "default": false,
+          "description": "Remove duplicate words, favor lower case words over mixed case words. Combine compound prefixes where possible.",
+          "type": "boolean"
+        },
         "sort": {
           "default": true,
           "description": "Sort the words in the resulting dictionary. Does not apply to `trie` based formats.",
@@ -249,6 +276,16 @@
       "description": "Maximum number of nested Hunspell Rules to apply. This is needed for recursive dictionaries like Hebrew.",
       "type": "number"
     },
+    "minCompoundLength": {
+      "default": 4,
+      "description": "Controls the minimum length of a compound word when storing words using `storeSplitWordsAsCompounds`. The compound words are prefixed / suffixed with `*`, to allow them to be combined with other compound words. If the length is too low, then the dictionary will consider many misspelled words as correct.",
+      "type": "number"
+    },
+    "removeDuplicates": {
+      "default": false,
+      "description": "Remove duplicate words, favor lower case words over mixed case words. Combine compound prefixes where possible.",
+      "type": "boolean"
+    },
     "rootDir": {
       "description": "Specify the directory where all relative paths will resolved against. By default, all relative paths are relative to the location of the config file.",
       "type": "string"
@@ -271,6 +308,11 @@
       "default": false,
       "description": "Split lines into words."
     },
+    "storeSplitWordsAsCompounds": {
+      "default": false,
+      "description": "Camel case words that have been split using the `allowedSplitWords` are added to the dictionary as compoundable words. These words are prefixed / suffixed with `*`.",
+      "type": "boolean"
+    },
     "targets": {
       "description": "Optional Target Dictionaries to create.",
       "items": {

package/dist/compiler/CompileOptions.d.ts

@@ -25,5 +25,11 @@ export interface CompileOptions {
      *
      */
     dictionaryDirectives?: string[] | undefined;
+    /**
+     * Remove duplicate words, favor lower case words over mixed case words.
+     * Combine compound prefixes where possible.
+     * @default false
+     */
+    removeDuplicates?: boolean;
 }
 //# sourceMappingURL=CompileOptions.d.ts.map

package/dist/compiler/SourceReader.d.ts

@@ -16,6 +16,8 @@ export interface SourceReaderOptions {
     legacy?: boolean;
     keepCase?: boolean;
     allowedSplitWords: AllowedSplitWordsCollection;
+    storeSplitWordsAsCompounds: boolean | undefined;
+    minCompoundLength?: number | undefined;
 }
 export type AnnotatedWord = string;
 export interface SourceReader {

package/dist/compiler/SourceReader.js

@@ -25,8 +25,17 @@ function splitLines(lines, options) {
     return split();
 }
 async function textFileReader(reader, options) {
-    const { legacy, splitWords: split, allowedSplitWords } = options;
-    const words = [...parseFileLines(reader, { legacy, split, allowedSplitWords })];
+    const { legacy, splitWords: split, allowedSplitWords, storeSplitWordsAsCompounds, minCompoundLength } = options;
+    const parseOptions = {
+        legacy,
+        split,
+        splitKeepBoth: undefined,
+        keepCase: undefined,
+        allowedSplitWords,
+        storeSplitWordsAsCompounds,
+        minCompoundLength,
+    };
+    const words = [...parseFileLines(reader.lines, parseOptions)];
     return {
         size: words.length,
         words,

package/dist/compiler/WordsCollection.d.ts

@@ -1,6 +1,6 @@
 export interface WordsCollection {
     size: number;
-    has(words: string): boolean;
+    has(words: string, caseSensitive: boolean): boolean;
     type?: string;
 }
 /**

package/dist/compiler/compile.js

@@ -18,6 +18,7 @@ export async function compile(request, options) {
     const targetOptions = {
         sort: request.sort,
         generateNonStrict: request.generateNonStrict,
+        removeDuplicates: request.removeDuplicates,
     };
     const conditional = options?.conditionalBuild || false;
     const checksumFile = resolveChecksumFile(request.checksumFile || conditional, rootDir);
@@ -57,6 +58,7 @@ export async function compileTarget(target, options, compileOptions) {
     const { format, sources, trieBase, sort = true, generateNonStrict = false, excludeWordsFrom } = target;
     const targetDirectory = path.resolve(rootDir, target.targetDirectory ?? cwd ?? process.cwd());
     const dictionaryDirectives = target.dictionaryDirectives ?? compileOptions.dictionaryDirectives;
+    const removeDuplicates = target.removeDuplicates ?? false;
     const excludeFilter = await createExcludeFilter(excludeWordsFrom);
     const generateNonStrictTrie = target.generateNonStrict ?? true;
     const name = normalizeTargetName(target.name);
@@ -69,6 +71,7 @@ export async function compileTarget(target, options, compileOptions) {
         generateNonStrict,
         filter: excludeFilter,
         dictionaryDirectives,
+        // removeDuplicates, // Add this in if we use it.
     });
     const checksumRoot = (checksumFile && path.dirname(checksumFile)) || rootDir;
     const deps = [...calculateDependencies(filename, filesToProcess, excludeWordsFrom, checksumRoot)];
@@ -88,10 +91,16 @@ export async function compileTarget(target, options, compileOptions) {
                 trie4: format === 'trie4',
                 generateNonStrict: generateNonStrictTrie,
                 dictionaryDirectives: undefined,
+                //   removeDuplicates, // Add this in if we use it.
             });
         }
         : async (words, dst) => {
-            return compileWordList(pipe(words, normalizer), dst, { sort, generateNonStrict, dictionaryDirectives });
+            return compileWordList(pipe(words, normalizer), dst, {
+                sort,
+                generateNonStrict,
+                dictionaryDirectives,
+                removeDuplicates,
+            });
         };
     await processFiles(action, filesToProcess, filename);
     logWithTimestamp(`Done compile: ${target.name}`);
@@ -166,7 +175,7 @@ async function readFileList(fileList) {
         .filter((a) => !!a);
 }
 async function readFileSource(fileSource, sourceOptions) {
-    const { filename, keepRawCase = sourceOptions.keepRawCase || false, split = sourceOptions.split || false, maxDepth, } = fileSource;
+    const { filename, keepRawCase = sourceOptions.keepRawCase || false, split = sourceOptions.split || false, maxDepth, storeSplitWordsAsCompounds, minCompoundLength, } = fileSource;
     const legacy = split === 'legacy';
     const splitWords = legacy ? false : split;
     // console.warn('fileSource: %o,\n targetOptions %o, \n opt: %o', fileSource, targetOptions, opt);
@@ -177,6 +186,8 @@ async function readFileSource(fileSource, sourceOptions) {
         splitWords,
         keepCase: keepRawCase,
         allowedSplitWords,
+        storeSplitWordsAsCompounds,
+        minCompoundLength,
     };
     logWithTimestamp(`Reading ${path.basename(filename)}`);
     const stream = await streamSourceWordsFromFile(filename, readerOptions);
@@ -208,6 +219,6 @@ async function createExcludeFilter(excludeWordsFrom) {
     if (!excludeWordsFrom || !excludeWordsFrom.length)
         return () => true;
     const excludeWords = await createWordsCollectionFromFiles(excludeWordsFrom);
-    return (word) => !excludeWords.has(word);
+    return (word) => !excludeWords.has(word, word.toUpperCase() !== word);
 }
 //# sourceMappingURL=compile.js.map

package/dist/compiler/createWordsCollection.js

@@ -1,14 +1,15 @@
+import { parseDictionary } from 'cspell-trie-lib';
 import { createReader } from './Reader.js';
 import { defaultAllowedSplitWords, defaultExcludeWordsCollection } from './WordsCollection.js';
 class AllowedSplitWordsImpl {
-    words;
+    collection;
     size;
     constructor(collection) {
-        this.words = collection;
+        this.collection = collection;
         this.size = collection.size;
     }
-    has(word) {
-        return !this.size || this.words.has(word);
+    has(word, caseSensitive) {
+        return !this.size || this.collection.has(word, caseSensitive);
     }
 }
 export async function createAllowedSplitWordsFromFiles(files) {
@@ -22,9 +23,30 @@ export function createAllowedSplitWords(words) {
         return defaultAllowedSplitWords;
     return new AllowedSplitWordsImpl(createWordsCollection(words));
 }
+function buildHasFn(dict) {
+    function has(word, caseSensitive) {
+        const r = dict.hasWord(word, true);
+        if (r || caseSensitive)
+            return r;
+        const lc = word.toLowerCase();
+        if (lc == word)
+            return false;
+        return dict.hasWord(lc, true);
+    }
+    return has;
+}
 async function readFile(filename) {
-    const reader = await createReader(filename, {});
-    return [...reader];
+    return await createReader(filename, {});
+}
+function readersToCollection(readers) {
+    const dictReaders = readers.filter(isDictionaryReader).map(dictReaderToCollection);
+    const nonDictCollection = lineReadersToCollection(readers.filter((a) => !isDictionaryReader(a)));
+    const collections = [...dictReaders, nonDictCollection];
+    const collection = {
+        size: collections.reduce((s, a) => s + a.size, 0),
+        has: (word, caseSensitive) => collections.some((a) => a.has(word, caseSensitive)),
+    };
+    return collection;
 }
 const cache = new WeakMap();
 export async function createWordsCollectionFromFiles(files) {
@@ -33,7 +55,7 @@ export async function createWordsCollectionFromFiles(files) {
     if (cached)
         return cached;
     const sources = await Promise.all(files.map((file) => readFile(file)));
-    const collection = createWordsCollection(sources.flat());
+    const collection = readersToCollection(sources);
     cache.set(files, collection);
     return collection;
 }
@@ -44,17 +66,19 @@ export function createWordsCollection(words) {
         .map((a) => a.trim())
         .filter((a) => !!a)
         .filter((a) => !a.startsWith('#'));
-    return new Set(arrWords);
+    const setOfWords = new Set(arrWords);
+    const has = buildHasFn({ hasWord: (word) => setOfWords.has(word) });
+    return { size: setOfWords.size, has };
 }
 class ExcludeWordsCollectionImpl {
-    words;
+    collection;
     size;
     constructor(collection) {
-        this.words = collection;
+        this.collection = collection;
         this.size = collection.size;
     }
-    has(word) {
-        return this.words.has(word);
+    has(word, caseSensitive) {
+        return this.collection.has(word, caseSensitive);
     }
 }
 export async function createExcludeWordsCollectionFromFiles(files) {
@@ -66,4 +90,19 @@ export async function createExcludeWordsCollectionFromFiles(files) {
 export function createExcludeWordsCollection(words) {
     return new ExcludeWordsCollectionImpl(words ? createWordsCollection(words) : new Set());
 }
+function isDictionaryReader(reader) {
+    return 'hasWord' in reader && !!reader.hasWord;
+}
+function dictReaderToCollection(reader) {
+    return { size: reader.size, has: buildHasFn(reader) };
+}
+function lineReadersToCollection(readers) {
+    function* words() {
+        for (const reader of readers) {
+            yield* reader.lines;
+        }
+    }
+    const dict = parseDictionary(words(), { stripCaseAndAccents: false });
+    return { size: dict.size, has: buildHasFn(dict) };
+}
 //# sourceMappingURL=createWordsCollection.js.map

package/dist/compiler/legacyLineToWords.js

@@ -1,12 +1,15 @@
 import { opConcatMap, opFilter, opMap, pipe } from '@cspell/cspell-pipe/sync';
+import { defaultCompileSourceOptions } from '../config/configDefaults.js';
 import { regExpSpaceOrDash, splitCamelCaseIfAllowed } from './splitCamelCaseIfAllowed.js';
 const regNonWord = /[^\p{L}\p{M}' _\d]+/giu;
 const regExpRepeatChars = /(.)\1{5}/i;
+const minCompoundLength = defaultCompileSourceOptions.minCompoundLength;
 export function legacyLineToWords(line, keepCase, allowedSplitWords) {
     // Remove punctuation and non-letters.
     const filteredLine = line.replaceAll(regNonWord, '|');
     const wordGroups = filteredLine.split('|');
-    const words = pipe(wordGroups, opConcatMap((a) => a.split(regExpSpaceOrDash)), opConcatMap((a) => splitCamelCaseIfAllowed(a, allowedSplitWords, keepCase)), opMap((a) => a.trim()), opFilter((a) => !!a), opFilter((s) => !regExpRepeatChars.test(s)));
+    const _minCompoundLength = minCompoundLength;
+    const words = pipe(wordGroups, opConcatMap((a) => a.split(regExpSpaceOrDash)), opConcatMap((a) => splitCamelCaseIfAllowed(a, allowedSplitWords, keepCase, '', _minCompoundLength)), opMap((a) => a.trim()), opFilter((a) => !!a), opFilter((s) => !regExpRepeatChars.test(s)));
     return words;
 }
 export function* legacyLinesToWords(lines, keepCase, allowedSplitWords) {

package/dist/compiler/readers/ReaderOptions.d.ts

@@ -9,7 +9,11 @@ export interface BaseReader {
     size: number;
     type: 'Hunspell' | 'TextFile' | 'Trie';
     lines: Iterable<AnnotatedWord>;
+    readonly hasWord?: (word: string, caseSensitive: boolean) => boolean;
 }
-export interface Reader extends BaseReader, Iterable<string> {
+export interface Reader extends BaseReader {
+}
+export interface DictionaryReader extends BaseReader {
+    readonly hasWord: (word: string, caseSensitive: boolean) => boolean;
 }
 //# sourceMappingURL=ReaderOptions.d.ts.map

package/dist/compiler/readers/trieFileReader.d.ts

@@ -1,3 +1,3 @@
-import type { BaseReader } from './ReaderOptions.js';
-export declare function trieFileReader(filename: string): Promise<BaseReader>;
+import type { DictionaryReader } from './ReaderOptions.js';
+export declare function trieFileReader(filename: string): Promise<DictionaryReader>;
 //# sourceMappingURL=trieFileReader.d.ts.map

package/dist/compiler/readers/trieFileReader.js

@@ -10,6 +10,7 @@ export async function trieFileReader(filename) {
             return trie.size();
         },
         lines: words,
+        hasWord: (word, caseSensitive) => trie.hasWord(word, caseSensitive),
     };
 }
 //# sourceMappingURL=trieFileReader.js.map

package/dist/compiler/splitCamelCaseIfAllowed.d.ts

@@ -1,5 +1,5 @@
 import type { AllowedSplitWordsCollection } from './WordsCollection.js';
 export declare const regExpSpaceOrDash: RegExp;
 export declare const regExpIsNumber: RegExp;
-export declare function splitCamelCaseIfAllowed(word: string, allowedWords: AllowedSplitWordsCollection, keepCase: boolean): string[];
+export declare function splitCamelCaseIfAllowed(word: string, allowedWords: AllowedSplitWordsCollection, keepCase: boolean, compoundPrefix: string, minCompoundLength: number): string[];
 //# sourceMappingURL=splitCamelCaseIfAllowed.d.ts.map

package/dist/compiler/splitCamelCaseIfAllowed.js

@@ -1,21 +1,34 @@
-import * as Text from './text.js';
+import { isSingleLetter, splitCamelCaseWord } from './text.js';
 export const regExpSpaceOrDash = /[- ]+/g;
 export const regExpIsNumber = /^\d+$/;
-export function splitCamelCaseIfAllowed(word, allowedWords, keepCase) {
+export function splitCamelCaseIfAllowed(word, allowedWords, keepCase, compoundPrefix, minCompoundLength) {
     const split = [...splitCamelCase(word)];
     if (split.length == 1)
         return adjustCases(split, allowedWords, keepCase);
-    const missing = split.find((w) => isUnknown(w, allowedWords));
-    if (missing !== undefined)
+    const missing = split.some((w) => isUnknown(w, allowedWords));
+    if (missing)
         return [word];
-    return adjustCases(split, allowedWords, keepCase);
+    const wordIndexes = calcWordIndex(word, split);
+    const adjusted = adjustCases(split, allowedWords, keepCase);
+    return !compoundPrefix
+        ? adjusted
+        : adjusted.map((w, i) => {
+            const { px, sx } = wordIndexes[i];
+            const canCompound = w.length >= minCompoundLength;
+            const lc = w.toLowerCase();
+            const p = canCompound && isSingleLetter(px) ? compoundPrefix : '';
+            const s = canCompound && isSingleLetter(sx) ? compoundPrefix : '';
+            if (lc.length < 4 || allowedWords.has(w, true))
+                return p + w + s;
+            return p + lc + s;
+        });
 }
 function adjustCases(words, allowedWords, keepCase) {
     return words.map((w) => adjustCase(w, allowedWords, keepCase));
 }
 function adjustCase(word, allowedWords, keepCase) {
     const lc = word.toLowerCase();
-    if (!allowedWords.has(lc))
+    if (!allowedWords.has(lc, true))
         return word;
     if (lc === word)
         return word;
@@ -26,14 +39,27 @@ function adjustCase(word, allowedWords, keepCase) {
     return word;
 }
 function isUnknown(word, allowedWords) {
-    return !allowedWords.has(word) && !allowedWords.has(word.toLowerCase());
+    if (word === 'ERROR') {
+        return !allowedWords.has(word, false);
+    }
+    return !allowedWords.has(word, false);
 }
 function splitCamelCase(word) {
-    const splitWords = Text.splitCamelCaseWord(word, false).filter((word) => !regExpIsNumber.test(word));
+    const splitWords = splitCamelCaseWord(word).filter((word) => !regExpIsNumber.test(word));
     // We only want to preserve this: "New York" and not "Namespace DNSLookup"
     if (splitWords.length > 1 && regExpSpaceOrDash.test(word)) {
         return splitWords.flatMap((w) => w.split(regExpSpaceOrDash));
     }
     return splitWords;
 }
+function calcWordIndex(word, words) {
+    let i = 0;
+    return words.map((w) => {
+        const j = word.indexOf(w, i);
+        const k = j + w.length;
+        const wIndex = { word: w, i: j, px: word[j - 1] || '', sx: word[k] || '' };
+        i = k;
+        return wIndex;
+    });
+}
 //# sourceMappingURL=splitCamelCaseIfAllowed.js.map

package/dist/compiler/text.d.ts

@@ -1,5 +1,10 @@
 /**
  * Split camelCase words into an array of strings.
  */
-export declare function splitCamelCaseWord(word: string, autoStem?: boolean): string[];
+export declare function splitCamelCaseWord(word: string): string[];
+/**
+ * Split camelCase words into an array of strings, try to fix English words.
+ */
+export declare function splitCamelCaseWordAutoStem(word: string): string[];
+export declare function isSingleLetter(c: string): boolean;
 //# sourceMappingURL=text.d.ts.map

package/dist/compiler/text.js

@@ -2,14 +2,27 @@
 const regExUpperSOrIng = /(\p{Lu}+'?(?:s|ing|ies|es|ings|ed|ning))(?!\p{Ll})/gu;
 const regExSplitWords = /([\p{Ll}])([\p{Lu}])/gu;
 const regExSplitWords2 = /(\p{Lu})(\p{Lu}\p{Ll})/gu;
+const regExpIsLetter = /^\p{L}\p{M}{0,2}$/u;
 /**
  * Split camelCase words into an array of strings.
  */
-export function splitCamelCaseWord(word, autoStem = true) {
-    const wPrime = autoStem ? word.replaceAll(regExUpperSOrIng, (s) => s[0] + s.slice(1).toLowerCase()) : word;
-    const pass1 = wPrime.replaceAll(regExSplitWords, '$1|$2');
+export function splitCamelCaseWord(word) {
+    const pass1 = word.replaceAll(regExSplitWords, '$1|$2');
     const pass2 = pass1.replaceAll(regExSplitWords2, '$1|$2');
     const pass3 = pass2.replaceAll(/[\d_]+/g, '|');
     return pass3.split('|').filter((a) => !!a);
 }
+/**
+ * Split camelCase words into an array of strings, try to fix English words.
+ */
+export function splitCamelCaseWordAutoStem(word) {
+    return splitCamelCaseWord(word.replaceAll(regExUpperSOrIng, tailToLowerCase));
+}
+function tailToLowerCase(word) {
+    const letters = [...word];
+    return letters[0] + letters.slice(1).join('').toLowerCase();
+}
+export function isSingleLetter(c) {
+    return regExpIsLetter.test(c);
+}
 //# sourceMappingURL=text.js.map

package/dist/compiler/wordListCompiler.d.ts

@@ -1,5 +1,6 @@
 import type { CompileOptions } from './CompileOptions.js';
 export declare function compileWordList(lines: Iterable<string>, destFilename: string, options: CompileOptions): Promise<void>;
+declare function removeDuplicates(words: Iterable<string>): Iterable<string>;
 export interface TrieOptions {
     base?: number;
     trie3?: boolean;
@@ -10,5 +11,7 @@ export interface CompileTrieOptions extends CompileOptions, TrieOptions {
 export declare function compileTrie(words: Iterable<string>, destFilename: string, options: CompileTrieOptions): Promise<void>;
 export declare const __testing__: {
     wordListHeader: string;
+    removeDuplicates: typeof removeDuplicates;
 };
+export {};
 //# sourceMappingURL=wordListCompiler.d.ts.map

package/dist/compiler/wordListCompiler.js

@@ -21,12 +21,140 @@ export async function compileWordList(lines, destFilename, options) {
 }
 function normalize(lines, options) {
     const filter = normalizeTargetWords(options);
-    const iter = pipe(lines, filter);
+    const cleanLines = options.removeDuplicates ? removeDuplicates(lines) : lines;
+    const iter = pipe(cleanLines, filter);
     if (!options.sort)
         return iter;
     const result = new Set(iter);
     return [...result].sort();
 }
+function stripCompoundAFix(word) {
+    return word.replaceAll('*', '').replaceAll('+', '');
+}
+function* removeDuplicates(words) {
+    const wordSet = new Set(words);
+    const wordForms = new Map();
+    for (const word of wordSet) {
+        const lc = stripCompoundAFix(word.toLowerCase());
+        const forms = wordForms.get(lc) ?? [];
+        forms.push(word);
+        wordForms.set(lc, forms);
+    }
+    for (const forms of wordForms.values()) {
+        if (forms.length <= 1) {
+            yield* forms;
+            continue;
+        }
+        const mForms = removeDuplicateForms(forms);
+        // if (forms.some((a) => /^[*+]?col[*+]?$/.test(a))) {
+        //     console.warn('Found col %o', { forms, mForms });
+        // }
+        if (mForms.size <= 1) {
+            for (const form of mForms.values()) {
+                yield* form;
+            }
+            continue;
+        }
+        // Handle upper / lower mix.
+        const words = [...mForms.keys()];
+        const lc = words[0].toLowerCase();
+        const lcForm = mForms.get(lc);
+        if (!lcForm) {
+            for (const form of mForms.values()) {
+                yield* form;
+            }
+            continue;
+        }
+        mForms.delete(lc);
+        const sLcForms = new Set(lcForm);
+        yield* lcForm;
+        if (sLcForms.has('*' + lc + '*'))
+            continue;
+        for (const forms of mForms.values()) {
+            for (const form of forms) {
+                if (sLcForms.has(form.toLowerCase()))
+                    continue;
+                yield form;
+            }
+        }
+    }
+}
+/**
+ * solo
+ * optional_prefix*
+ * optional_suffix*
+ * required_prefix+
+ * required_suffix+
+ */
+var Flags;
+(function (Flags) {
+    Flags[Flags["base"] = 0] = "base";
+    Flags[Flags["none"] = 1] = "none";
+    Flags[Flags["both"] = 2] = "both";
+    Flags[Flags["pfx"] = 4] = "pfx";
+    Flags[Flags["sfx"] = 8] = "sfx";
+    Flags[Flags["all"] = 15] = "all";
+})(Flags || (Flags = {}));
+function applyFlags(word, flags) {
+    if (flags === Flags.none)
+        return [word];
+    if (flags === Flags.all)
+        return ['*' + word + '*'];
+    if (flags === Flags.both)
+        return ['+' + word + '+'];
+    if (flags === Flags.pfx)
+        return [word + '+'];
+    if (flags === Flags.sfx)
+        return ['+' + word];
+    if (flags === (Flags.none | Flags.sfx))
+        return ['*' + word];
+    if (flags === (Flags.none | Flags.pfx))
+        return [word + '*'];
+    if (flags === (Flags.none | Flags.pfx | Flags.sfx))
+        return [word + '*', '*' + word];
+    if (flags === (Flags.none | Flags.both))
+        return [word, '+' + word + '+'];
+    if (flags === (Flags.none | Flags.both | Flags.sfx))
+        return [word, '+' + word + '*'];
+    if (flags === (Flags.none | Flags.both | Flags.pfx))
+        return [word, '*' + word + '+'];
+    if (flags === (Flags.both | Flags.pfx))
+        return ['*' + word + '+'];
+    if (flags === (Flags.both | Flags.sfx))
+        return ['+' + word + '*'];
+    if (flags === (Flags.both | Flags.pfx | Flags.sfx))
+        return ['+' + word + '*', '*' + word + '+'];
+    return ['+' + word, word + '+'];
+}
+function removeDuplicateForms(forms) {
+    function flags(word, flag = 0) {
+        const canBePrefix = word.endsWith('*');
+        const mustBePrefix = !canBePrefix && word.endsWith('+');
+        const isPrefix = canBePrefix || mustBePrefix;
+        const canBeSuffix = word.startsWith('*');
+        const mustBeSuffix = !canBeSuffix && word.startsWith('+');
+        const isSuffix = canBeSuffix || mustBeSuffix;
+        if (canBePrefix && canBeSuffix)
+            return flag | Flags.all;
+        if (mustBePrefix && mustBeSuffix)
+            return flag | Flags.both;
+        if (!isPrefix && !isSuffix)
+            return flag | Flags.none;
+        flag |= isPrefix && !isSuffix ? Flags.pfx : 0;
+        flag |= isSuffix && !isPrefix ? Flags.sfx : 0;
+        flag |= canBePrefix && !mustBeSuffix ? Flags.none : 0;
+        flag |= canBeSuffix && !mustBePrefix ? Flags.none : 0;
+        return flag;
+    }
+    const m = new Map();
+    for (const form of forms) {
+        const k = stripCompoundAFix(form);
+        m.set(k, flags(form, m.get(k)));
+    }
+    return new Map([...m.entries()].map(([form, flag]) => {
+        return [form, applyFlags(form, flag)];
+    }));
+}
 function createWordListTarget(destFilename) {
     const target = createTarget(destFilename);
     return (seq) => target(pipe(seq, opMap((a) => a + '\n')));
@@ -63,5 +191,6 @@ function createTrieTarget(destFilename, options) {
 }
 export const __testing__ = {
     wordListHeader,
+    removeDuplicates,
 };
 //# sourceMappingURL=wordListCompiler.js.map

package/dist/compiler/wordListParser.d.ts

@@ -24,6 +24,19 @@ export interface ParseFileOptions {
      */
     legacy?: boolean;
     allowedSplitWords: AllowedSplitWordsCollection;
+    /**
+     * Words that have been split using the `allowedSplitWords` are added to the dictionary as compoundable words.
+     * These words are prefixed / suffixed with `*`.
+     * @default undefined
+     */
+    storeSplitWordsAsCompounds: boolean | undefined;
+    /**
+     * Controls the minimum length of a compound word when storing words using `storeSplitWordsAsCompounds`.
+     * The compound words are prefixed / suffixed with `*`, to allow them to be combined with other compound words.
+     * If the length is too low, then the dictionary will consider many misspelled words as correct.
+     * @default 4
+     */
+    minCompoundLength: number | undefined;
 }
 type ParseFileOptionsRequired = Required<ParseFileOptions>;
 export declare const defaultParseDictionaryOptions: ParseFileOptionsRequired;
@@ -43,6 +56,6 @@ export declare function createParseFileLineMapper(options?: Partial<ParseFileOpt
  * @param _options - defines prefixes used when parsing lines.
  * @returns words that have been normalized.
  */
-export declare function parseFileLines(lines: Iterable<string> | string, options: Partial<ParseFileOptions>): Iterable<string>;
+export declare function parseFileLines(lines: Iterable<string> | string, options: ParseFileOptions): Iterable<string>;
 export {};
 //# sourceMappingURL=wordListParser.d.ts.map

package/dist/compiler/wordListParser.js

@@ -1,12 +1,14 @@
 import { opCombine, opCombine as opPipe, opFilter, opMap } from '@cspell/cspell-pipe/sync';
 import { createDictionaryLineParser } from 'cspell-trie-lib';
 import { uniqueFilter } from 'hunspell-reader';
+import { defaultCompileSourceOptions } from '../config/configDefaults.js';
 import { legacyLineToWords } from './legacyLineToWords.js';
 import { splitCamelCaseIfAllowed } from './splitCamelCaseIfAllowed.js';
 export function normalizeTargetWords(options) {
     const lineParser = createDictionaryLineParser({
         stripCaseAndAccents: options.generateNonStrict,
         stripCaseAndAccentsOnForbidden: true,
+        keepOptionalCompoundCharacter: true,
     });
     const operations = [
         opFilter((a) => !!a),
@@ -44,6 +46,8 @@ const _defaultOptions = {
     splitKeepBoth: false,
     // splitSeparator: regExpSplit,
     allowedSplitWords: { has: () => true, size: 0 },
+    storeSplitWordsAsCompounds: defaultCompileSourceOptions.storeSplitWordsAsCompounds,
+    minCompoundLength: defaultCompileSourceOptions.minCompoundLength,
 };
 export const defaultParseDictionaryOptions = Object.freeze(_defaultOptions);
 export const cSpellToolDirective = 'cspell-tools:';
@@ -56,9 +60,10 @@ export const setOfCSpellDirectiveFlags = ['no-split', 'split', 'keep-case', 'no-
  */
 export function createParseFileLineMapper(options) {
     const _options = options || _defaultOptions;
-    const { splitKeepBoth = _defaultOptions.splitKeepBoth, allowedSplitWords = _defaultOptions.allowedSplitWords } = _options;
+    const { splitKeepBoth = _defaultOptions.splitKeepBoth, allowedSplitWords = _defaultOptions.allowedSplitWords, storeSplitWordsAsCompounds, minCompoundLength = _defaultOptions.minCompoundLength, } = _options;
     let { legacy = _defaultOptions.legacy } = _options;
     let { split = _defaultOptions.split, keepCase = legacy ? false : _defaultOptions.keepCase } = _options;
+    const compoundFix = storeSplitWordsAsCompounds ? '+' : '';
     function isString(line) {
         return typeof line === 'string';
     }
@@ -131,6 +136,9 @@ export function createParseFileLineMapper(options) {
             .filter((a) => !/^0[xo][0-9A-F]+$/i.test(a)); // c-style hex/octal digits
         return lines;
     }
+    function splitWordIntoWords(word) {
+        return splitCamelCaseIfAllowed(word, allowedSplitWords, keepCase, compoundFix, minCompoundLength);
+    }
     function* splitWords(lines) {
         for (const line of lines) {
             if (legacy) {
@@ -139,9 +147,7 @@ export function createParseFileLineMapper(options) {
             }
             if (split) {
                 const words = splitLine(line);
-                yield* !allowedSplitWords.size
-                    ? words
-                    : words.flatMap((word) => splitCamelCaseIfAllowed(word, allowedSplitWords, keepCase));
+                yield* !allowedSplitWords.size ? words : words.flatMap((word) => splitWordIntoWords(word));
                 if (!splitKeepBoth)
                     continue;
             }

package/dist/config/config.d.ts

@@ -76,6 +76,12 @@ export interface CompileTargetOptions {
      * ```
      */
     dictionaryDirectives?: string[] | undefined;
+    /**
+     * Remove duplicate words, favor lower case words over mixed case words.
+     * Combine compound prefixes where possible.
+     * @default false
+     */
+    removeDuplicates?: boolean | undefined;
 }
 export interface Target extends CompileTargetOptions {
     /**
@@ -144,7 +150,29 @@ export interface CompileSourceOptions {
      * @default false
      */
     keepRawCase?: boolean | undefined;
+    /**
+     * Words in the `allowedSplitWords` are considered correct and can be used
+     * as a basis for splitting compound words.
+     *
+     * If entries can be split so that all the words in the entry are allowed,
+     * then only the individual words are added, otherwise the entire entry is added.
+     * This is to prevent misspellings in CamelCase words from being introduced into the
+     * dictionary.
+     */
     allowedSplitWords?: FilePath | FilePath[] | undefined;
+    /**
+     * Camel case words that have been split using the `allowedSplitWords` are added to the dictionary as compoundable words.
+     * These words are prefixed / suffixed with `*`.
+     * @default false
+     */
+    storeSplitWordsAsCompounds?: boolean | undefined;
+    /**
+     * Controls the minimum length of a compound word when storing words using `storeSplitWordsAsCompounds`.
+     * The compound words are prefixed / suffixed with `*`, to allow them to be combined with other compound words.
+     * If the length is too low, then the dictionary will consider many misspelled words as correct.
+     * @default 4
+     */
+    minCompoundLength?: number | undefined;
 }
 export declare const configFileSchemaURL = "https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-tools/cspell-tools.config.schema.json";
 //# sourceMappingURL=config.d.ts.map

package/dist/config/configDefaults.d.ts

@@ -0,0 +1,9 @@
+export declare const defaultCompileSourceOptions: {
+    readonly maxDepth: undefined;
+    readonly split: false;
+    readonly keepRawCase: false;
+    readonly allowedSplitWords: undefined;
+    readonly storeSplitWordsAsCompounds: false;
+    readonly minCompoundLength: 4;
+};
+//# sourceMappingURL=configDefaults.d.ts.map

package/dist/config/configDefaults.js

@@ -0,0 +1,9 @@
+export const defaultCompileSourceOptions = {
+    maxDepth: undefined,
+    split: false,
+    keepRawCase: false,
+    allowedSplitWords: undefined,
+    storeSplitWordsAsCompounds: false,
+    minCompoundLength: 4,
+};
+//# sourceMappingURL=configDefaults.js.map

package/dist/types.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Make all properties in T required, but keep the original optionality of the properties.
+ */
+export type RequireFields<T> = {
+    [P in keyof Required<T>]: T[P];
+};
+//# sourceMappingURL=types.d.ts.map

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cspell/cspell-tools",
-  "version": "8.15.4",
+  "version": "8.15.6",
   "description": "Tools to assist with the development of cSpell",
   "publishConfig": {
     "access": "public",
@@ -51,12 +51,12 @@
   },
   "homepage": "https://github.com/streetsidesoftware/cspell/tree/main/packages/cspell-tools#readme",
   "dependencies": {
-    "@cspell/cspell-pipe": "8.15.4",
+    "@cspell/cspell-pipe": "8.15.6",
     "commander": "^12.1.0",
     "cosmiconfig": "9.0.0",
-    "cspell-trie-lib": "8.15.4",
+    "cspell-trie-lib": "8.15.6",
     "glob": "^10.4.5",
-    "hunspell-reader": "8.15.4",
+    "hunspell-reader": "8.15.6",
     "yaml": "^2.6.0"
   },
   "engines": {
@@ -67,5 +67,5 @@
     "ts-json-schema-generator": "^2.3.0"
   },
   "module": "bin.mjs",
-  "gitHead": "27882ec49769126e3725b2cf180fee137c9a6ebe"
+  "gitHead": "fa9d2dff7ea196b9b59b9089c8511c538433c054"
 }