@cspell/cspell-tools 6.28.0 → 6.29.1

package/cspell-tools.config.schema.json

@@ -27,6 +27,19 @@
     "FileListSource": {
       "additionalProperties": false,
       "properties": {
+        "allowedSplitWords": {
+          "anyOf": [
+            {
+              "$ref": "#/definitions/FilePath"
+            },
+            {
+              "items": {
+                "$ref": "#/definitions/FilePath"
+              },
+              "type": "array"
+            }
+          ]
+        },
         "keepRawCase": {
           "default": false,
           "description": "Do not generate lower case / accent free versions of words.",
@@ -65,6 +78,19 @@
     "FileSource": {
       "additionalProperties": false,
       "properties": {
+        "allowedSplitWords": {
+          "anyOf": [
+            {
+              "$ref": "#/definitions/FilePath"
+            },
+            {
+              "items": {
+                "$ref": "#/definitions/FilePath"
+              },
+              "type": "array"
+            }
+          ]
+        },
         "filename": {
           "$ref": "#/definitions/FilePath"
         },
@@ -99,6 +125,20 @@
     "Target": {
       "additionalProperties": false,
       "properties": {
+        "allowedSplitWords": {
+          "anyOf": [
+            {
+              "$ref": "#/definitions/FilePath"
+            },
+            {
+              "items": {
+                "$ref": "#/definitions/FilePath"
+              },
+              "type": "array"
+            }
+          ],
+          "description": "Words in the `allowedSplitWords` are considered correct and can be used as a basis for splitting compound words.\n\nIf entries can be split so that all the words in the entry are allowed, then only the individual words are added, otherwise the entire entry is added. This is to prevent misspellings in CamelCase words from being introduced into the dictionary."
+        },
         "compress": {
           "default": ": false",
           "description": "gzip the file?",
@@ -155,6 +195,20 @@
     }
   },
   "properties": {
+    "allowedSplitWords": {
+      "anyOf": [
+        {
+          "$ref": "#/definitions/FilePath"
+        },
+        {
+          "items": {
+            "$ref": "#/definitions/FilePath"
+          },
+          "type": "array"
+        }
+      ],
+      "description": "Words in the `allowedSplitWords` are considered correct and can be used as a basis for splitting compound words.\n\nIf entries can be split so that all the words in the entry are allowed, then only the individual words are added, otherwise the entire entry is added. This is to prevent misspellings in CamelCase words from being introduced into the dictionary."
+    },
     "generateNonStrict": {
       "default": true,
       "description": "Generate lower case / accent free versions of words.",

package/dist/compiler/Reader.d.ts

@@ -1,28 +1,4 @@
-export interface ReaderOptions {
-    /**
-     * Max Hunspell recursive depth.
-     */
-    maxDepth?: number;
-    /**
-     * split words if necessary.
-     */
-    splitWords: boolean;
-    /**
-     * Indicate that it is an unformatted file and needs to be cleaned
-     * before processing. Applies only to text file sources.
-     * @default false
-     */
-    legacy?: boolean;
-    keepCase?: boolean;
-}
-export type AnnotatedWord = string;
-interface BaseReader {
-    size: number;
-    words: Iterable<AnnotatedWord>;
-}
-export interface Reader extends BaseReader, Iterable<string> {
-}
+import type { Reader, ReaderOptions } from './readers/ReaderOptions';
+export declare const regHunspellFile: RegExp;
 export declare function createReader(filename: string, options: ReaderOptions): Promise<Reader>;
-export declare function readHunspellFiles(filename: string, options: ReaderOptions): Promise<BaseReader>;
-export {};
 //# sourceMappingURL=Reader.d.ts.map

package/dist/compiler/Reader.js

@@ -1,41 +1,14 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.readHunspellFiles = exports.createReader = void 0;
-const sync_1 = require("@cspell/cspell-pipe/sync");
-const cspell_trie_lib_1 = require("cspell-trie-lib");
-const HR = __importStar(require("hunspell-reader"));
-const readTextFile_1 = require("./readTextFile");
-const wordListParser_1 = require("./wordListParser");
-const regHunspellFile = /\.(dic|aff)$/i;
-// cspell:word dedupe
-const DEDUPE_SIZE = 1000;
+exports.createReader = exports.regHunspellFile = void 0;
+const readHunspellFiles_1 = require("./readers/readHunspellFiles");
+const textFileReader_1 = require("./readers/textFileReader");
+const trieFileReader_1 = require("./readers/trieFileReader");
+exports.regHunspellFile = /\.(dic|aff)$/i;
 // Readers first match wins
 const readers = [
-    { test: /\.trie\b/, method: trieFileReader },
-    { test: regHunspellFile, method: readHunspellFiles },
+    { test: /\.trie\b/, method: trieFileReader_1.trieFileReader },
+    { test: exports.regHunspellFile, method: readHunspellFiles_1.readHunspellFiles },
 ];
 function findMatchingReader(filename, options) {
     for (const reader of readers) {
@@ -43,94 +16,13 @@ function findMatchingReader(filename, options) {
             return reader.method(filename, options);
         }
     }
-    return textFileReader(filename, options);
+    return (0, textFileReader_1.textFileReader)(filename);
 }
 async function createReader(filename, options) {
     const baseReader = await findMatchingReader(filename, options);
     return Object.assign(baseReader, {
-        [Symbol.iterator]: () => baseReader.words[Symbol.iterator](),
+        [Symbol.iterator]: () => baseReader.lines[Symbol.iterator](),
     });
 }
 exports.createReader = createReader;
-async function readHunspellFiles(filename, options) {
-    const dicFile = filename.replace(regHunspellFile, '.dic');
-    const affFile = filename.replace(regHunspellFile, '.aff');
-    const reader = await HR.IterableHunspellReader.createFromFiles(affFile, dicFile);
-    reader.maxDepth = options.maxDepth !== undefined ? options.maxDepth : reader.maxDepth;
-    const words = (0, sync_1.pipe)(reader.seqAffWords(), _mapAffWords, dedupeAndSort);
-    return {
-        size: reader.dic.length,
-        words,
-    };
-}
-exports.readHunspellFiles = readHunspellFiles;
-async function trieFileReader(filename) {
-    const trieRoot = (0, cspell_trie_lib_1.importTrie)(await (0, readTextFile_1.readTextFileLines)(filename));
-    const trie = new cspell_trie_lib_1.Trie(trieRoot);
-    const words = trie.words();
-    return {
-        get size() {
-            return trie.size();
-        },
-        words,
-    };
-}
-async function textFileReader(filename, options) {
-    const content = await (0, readTextFile_1.readTextFile)(filename);
-    const words = [...(0, wordListParser_1.parseFileLines)(content, { legacy: options.legacy, split: options.splitWords })];
-    return {
-        size: words.length,
-        words,
-    };
-}
-// function* _stripCaseAndAccents(words: Iterable<AnnotatedWord>): Iterable<AnnotatedWord> {
-//     for (const word of words) {
-//         // Words are normalized to the compact format: e + ` => è
-//         yield word.normalize();
-//         // covert to lower case and strip accents.
-//         const n = word.toLowerCase().normalize('NFD').replace(/\p{M}/gu, '');
-//         // All words are added for case-insensitive searches.
-//         // It is a space / speed trade-off. In this case, speed is more important.
-//         yield CASE_INSENSITIVE_PREFIX + n;
-//     }
-// }
-function* dedupeAndSort(words) {
-    const buffer = new Set();
-    function flush() {
-        const result = [...buffer].sort();
-        buffer.clear();
-        return result;
-    }
-    for (const word of words) {
-        buffer.add(word);
-        if (buffer.size >= DEDUPE_SIZE) {
-            yield* flush();
-        }
-    }
-    yield* flush();
-}
-function* _mapAffWords(affWords) {
-    const hasSpecial = /[~+!]/;
-    for (const affWord of affWords) {
-        const { word, flags } = affWord;
-        // For now do not include words with special characters.
-        if (hasSpecial.test(word))
-            continue;
-        const compound = flags.isCompoundForbidden ? '' : cspell_trie_lib_1.COMPOUND_FIX;
-        const forbid = flags.isForbiddenWord ? cspell_trie_lib_1.FORBID_PREFIX : '';
-        if (!forbid) {
-            if (flags.canBeCompoundBegin || flags.isCompoundPermitted)
-                yield word + compound;
-            if (flags.canBeCompoundEnd || flags.isCompoundPermitted)
-                yield compound + word;
-            if (flags.canBeCompoundMiddle || flags.isCompoundPermitted)
-                yield compound + word + compound;
-            if (!flags.isOnlyAllowedInCompound)
-                yield word;
-        }
-        else {
-            yield forbid + word;
-        }
-    }
-}
 //# sourceMappingURL=Reader.js.map

package/dist/compiler/SourceReader.d.ts

@@ -0,0 +1,26 @@
+import type { AllowedSplitWordsCollection } from './WordsCollection';
+export interface SourceReaderOptions {
+    /**
+     * Max Hunspell recursive depth.
+     */
+    maxDepth?: number;
+    /**
+     * split words if necessary.
+     */
+    splitWords: boolean;
+    /**
+     * Indicate that it is an unformatted file and needs to be cleaned
+     * before processing. Applies only to text file sources.
+     * @default false
+     */
+    legacy?: boolean;
+    keepCase?: boolean;
+    allowedSplitWords: AllowedSplitWordsCollection;
+}
+export type AnnotatedWord = string;
+export interface SourceReader {
+    size: number;
+    words: Iterable<AnnotatedWord>;
+}
+export declare function createSourceReader(filename: string, options: SourceReaderOptions): Promise<SourceReader>;
+//# sourceMappingURL=SourceReader.d.ts.map

package/dist/compiler/SourceReader.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createSourceReader = void 0;
+const Reader_1 = require("./Reader");
+const wordListParser_1 = require("./wordListParser");
+async function createSourceReader(filename, options) {
+    const reader = await (0, Reader_1.createReader)(filename, options);
+    if (reader.type !== 'TextFile') {
+        return {
+            words: reader.lines,
+            get size() {
+                return reader.size;
+            },
+        };
+    }
+    return textFileReader(reader, options);
+}
+exports.createSourceReader = createSourceReader;
+async function textFileReader(reader, options) {
+    const { legacy, splitWords: split, allowedSplitWords } = options;
+    const words = [...(0, wordListParser_1.parseFileLines)(reader, { legacy, split, allowedSplitWords })];
+    return {
+        size: words.length,
+        words,
+    };
+}
+//# sourceMappingURL=SourceReader.js.map

package/dist/compiler/WordsCollection.d.ts

@@ -0,0 +1,20 @@
+export interface WordsCollection {
+    size: number;
+    has(words: string): boolean;
+    type?: string;
+}
+/**
+ * Collection of words to be allowed after splitting.
+ */
+export interface AllowedSplitWordsCollection extends WordsCollection {
+    type?: 'AllowedSplitWordsCollection';
+}
+export declare const defaultAllowedSplitWords: AllowedSplitWordsCollection;
+/**
+ * Collection of words to be excluded.
+ */
+export interface ExcludeWordsCollection extends WordsCollection {
+    type?: 'ExcludeWordsCollection';
+}
+export declare const defaultExcludeWordsCollection: ExcludeWordsCollection;
+//# sourceMappingURL=WordsCollection.d.ts.map

package/dist/compiler/WordsCollection.js

@@ -0,0 +1,6 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.defaultExcludeWordsCollection = exports.defaultAllowedSplitWords = void 0;
+exports.defaultAllowedSplitWords = Object.freeze({ size: 0, has: () => true });
+exports.defaultExcludeWordsCollection = Object.freeze({ size: 0, has: () => false });
+//# sourceMappingURL=WordsCollection.js.map

package/dist/compiler/compile.js

@@ -29,9 +29,10 @@ const operators_1 = require("@cspell/cspell-pipe/operators");
 const sync_1 = require("@cspell/cspell-pipe/sync");
 const path = __importStar(require("path"));
 const config_1 = require("../config");
-const iterateWordsFromFile_1 = require("./iterateWordsFromFile");
+const createWordsCollection_1 = require("./createWordsCollection");
 const logWithTimestamp_1 = require("./logWithTimestamp");
-const readTextFile_1 = require("./readTextFile");
+const readTextFile_1 = require("./readers/readTextFile");
+const streamSourceWordsFromFile_1 = require("./streamSourceWordsFromFile");
 const wordListCompiler_1 = require("./wordListCompiler");
 const wordListParser_1 = require("./wordListParser");
 async function compile(request, options) {
@@ -54,7 +55,6 @@ async function compile(request, options) {
 exports.compile = compile;
 async function compileTarget(target, options, rootDir) {
     (0, logWithTimestamp_1.logWithTimestamp)(`Start compile: ${target.name}`);
-    // console.log('Target: %o', target);
     const { format, sources, trieBase, sort = true, generateNonStrict = false } = target;
     const targetDirectory = path.resolve(rootDir, target.targetDirectory ?? process.cwd());
     const generateNonStrictTrie = target.generateNonStrict ?? true;
@@ -141,9 +141,16 @@ async function readFileSource(fileSource, sourceOptions) {
     const legacy = split === 'legacy';
     const splitWords = legacy ? false : split;
     // console.warn('fileSource: %o,\n targetOptions %o, \n opt: %o', fileSource, targetOptions, opt);
-    const readerOptions = { maxDepth, legacy, splitWords, keepCase: keepRawCase };
+    const allowedSplitWords = await (0, createWordsCollection_1.createAllowedSplitWordsFromFiles)(fileSource.allowedSplitWords || sourceOptions.allowedSplitWords);
+    const readerOptions = {
+        maxDepth,
+        legacy,
+        splitWords,
+        keepCase: keepRawCase,
+        allowedSplitWords,
+    };
     (0, logWithTimestamp_1.logWithTimestamp)(`Reading ${path.basename(filename)}`);
-    const stream = await (0, iterateWordsFromFile_1.streamWordsFromFile)(filename, readerOptions);
+    const stream = await (0, streamSourceWordsFromFile_1.streamSourceWordsFromFile)(filename, readerOptions);
     (0, logWithTimestamp_1.logWithTimestamp)(`Done reading ${path.basename(filename)}`);
     const f = {
         src: filename,

package/dist/compiler/createWordsCollection.d.ts

@@ -0,0 +1,9 @@
+import type { FilePath } from '../config/config';
+import type { AllowedSplitWordsCollection, ExcludeWordsCollection, WordsCollection } from './WordsCollection';
+export declare function createAllowedSplitWordsFromFiles(files: FilePath | FilePath[] | undefined): Promise<AllowedSplitWordsCollection>;
+export declare function createAllowedSplitWords(words: Iterable<string> | undefined): AllowedSplitWordsCollection;
+export declare function createWordsCollectionFromFiles(files: FilePath | FilePath[]): Promise<WordsCollection>;
+export declare function createWordsCollection(words: Iterable<string>): WordsCollection;
+export declare function createExcludeWordsCollectionFromFiles(files: FilePath | FilePath[] | undefined): Promise<ExcludeWordsCollection>;
+export declare function createExcludeWordsCollection(words: Iterable<string> | undefined): ExcludeWordsCollection;
+//# sourceMappingURL=createWordsCollection.d.ts.map

package/dist/compiler/createWordsCollection.js

@@ -0,0 +1,74 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createExcludeWordsCollection = exports.createExcludeWordsCollectionFromFiles = exports.createWordsCollection = exports.createWordsCollectionFromFiles = exports.createAllowedSplitWords = exports.createAllowedSplitWordsFromFiles = void 0;
+const Reader_1 = require("./Reader");
+const WordsCollection_1 = require("./WordsCollection");
+class AllowedSplitWordsImpl {
+    constructor(collection) {
+        this.words = collection;
+        this.size = collection.size;
+    }
+    has(word) {
+        return !this.size || this.words.has(word);
+    }
+}
+async function createAllowedSplitWordsFromFiles(files) {
+    if (!files || !files.length)
+        return WordsCollection_1.defaultAllowedSplitWords;
+    const collection = await createWordsCollectionFromFiles(files);
+    return new AllowedSplitWordsImpl(collection);
+}
+exports.createAllowedSplitWordsFromFiles = createAllowedSplitWordsFromFiles;
+function createAllowedSplitWords(words) {
+    if (!words)
+        return WordsCollection_1.defaultAllowedSplitWords;
+    return new AllowedSplitWordsImpl(createWordsCollection(words));
+}
+exports.createAllowedSplitWords = createAllowedSplitWords;
+async function readFile(filename) {
+    const reader = await (0, Reader_1.createReader)(filename, {});
+    return [...reader];
+}
+const cache = new WeakMap();
+async function createWordsCollectionFromFiles(files) {
+    files = Array.isArray(files) ? files : [files];
+    const cached = cache.get(files);
+    if (cached)
+        return cached;
+    const sources = await Promise.all(files.map((file) => readFile(file)));
+    const collection = createWordsCollection(sources.flatMap((a) => a));
+    cache.set(files, collection);
+    return collection;
+}
+exports.createWordsCollectionFromFiles = createWordsCollectionFromFiles;
+function createWordsCollection(words) {
+    if (words instanceof Set)
+        return words;
+    const arrWords = (Array.isArray(words) ? words : [...words])
+        .map((a) => a.trim())
+        .filter((a) => !!a)
+        .filter((a) => !a.startsWith('#'));
+    return new Set(arrWords);
+}
+exports.createWordsCollection = createWordsCollection;
+class ExcludeWordsCollectionImpl {
+    constructor(collection) {
+        this.words = collection;
+        this.size = collection.size;
+    }
+    has(word) {
+        return this.words.has(word);
+    }
+}
+async function createExcludeWordsCollectionFromFiles(files) {
+    if (!files || !files.length)
+        return WordsCollection_1.defaultExcludeWordsCollection;
+    const collection = await createWordsCollectionFromFiles(files);
+    return new ExcludeWordsCollectionImpl(collection);
+}
+exports.createExcludeWordsCollectionFromFiles = createExcludeWordsCollectionFromFiles;
+function createExcludeWordsCollection(words) {
+    return new ExcludeWordsCollectionImpl(words ? createWordsCollection(words) : new Set());
+}
+exports.createExcludeWordsCollection = createExcludeWordsCollection;
+//# sourceMappingURL=createWordsCollection.js.map

package/dist/compiler/legacyLineToWords.d.ts

@@ -1,3 +1,4 @@
-export declare function legacyLineToWords(line: string, keepCase: boolean): Iterable<string>;
-export declare function legacyLinesToWords(lines: Iterable<string>, keepCase?: boolean): Iterable<string>;
+import type { AllowedSplitWordsCollection } from './WordsCollection';
+export declare function legacyLineToWords(line: string, keepCase: boolean, allowedSplitWords: AllowedSplitWordsCollection): Iterable<string>;
+export declare function legacyLinesToWords(lines: Iterable<string>, keepCase: boolean, allowedSplitWords: AllowedSplitWordsCollection): Iterable<string>;
 //# sourceMappingURL=legacyLineToWords.d.ts.map

package/dist/compiler/legacyLineToWords.js

@@ -1,54 +1,22 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.legacyLinesToWords = exports.legacyLineToWords = void 0;
 const sync_1 = require("@cspell/cspell-pipe/sync");
-const Text = __importStar(require("./text"));
-const regNonWord = /[^\p{L}\p{M}' ]+/giu;
-const regExpSpaceOrDash = /[- ]+/g;
-const regExpRepeatChars = /(.)\1{4,}/i;
-function legacyLineToWords(line, keepCase) {
+const splitCamelCaseIfAllowed_1 = require("./splitCamelCaseIfAllowed");
+const regNonWord = /[^\p{L}\p{M}' \d]+/giu;
+const regExpRepeatChars = /(.)\1{5}/i;
+function legacyLineToWords(line, keepCase, allowedSplitWords) {
     // Remove punctuation and non-letters.
     const filteredLine = line.replace(regNonWord, '|');
     const wordGroups = filteredLine.split('|');
-    const words = (0, sync_1.pipe)(wordGroups, (0, sync_1.opConcatMap)((a) => [a, ...a.split(regExpSpaceOrDash)]), (0, sync_1.opConcatMap)((a) => splitCamelCase(a)), (0, sync_1.opMap)((a) => a.trim()), (0, sync_1.opFilter)((a) => !!a), (0, sync_1.opFilter)((s) => !regExpRepeatChars.test(s)), (0, sync_1.opMap)((a) => (keepCase ? a : a.toLowerCase())));
+    const words = (0, sync_1.pipe)(wordGroups, (0, sync_1.opConcatMap)((a) => [...a.split(splitCamelCaseIfAllowed_1.regExpSpaceOrDash)]), (0, sync_1.opConcatMap)((a) => (0, splitCamelCaseIfAllowed_1.splitCamelCaseIfAllowed)(a, allowedSplitWords, keepCase)), (0, sync_1.opMap)((a) => a.trim()), (0, sync_1.opFilter)((a) => !!a), (0, sync_1.opFilter)((s) => !regExpRepeatChars.test(s)));
     return words;
 }
 exports.legacyLineToWords = legacyLineToWords;
-function* legacyLinesToWords(lines, keepCase = false) {
+function* legacyLinesToWords(lines, keepCase, allowedSplitWords) {
     for (const line of lines) {
-        yield* legacyLineToWords(line, keepCase);
+        yield* legacyLineToWords(line, keepCase, allowedSplitWords);
     }
 }
 exports.legacyLinesToWords = legacyLinesToWords;
-function splitCamelCase(word) {
-    const splitWords = Text.splitCamelCaseWord(word);
-    // We only want to preserve this: "New York" and not "Namespace DNSLookup"
-    if (splitWords.length > 1 && regExpSpaceOrDash.test(word)) {
-        return (0, sync_1.pipe)(splitWords, (0, sync_1.opConcatMap)((w) => w.split(regExpSpaceOrDash)));
-    }
-    return splitWords;
-}
 //# sourceMappingURL=legacyLineToWords.js.map

package/dist/compiler/readers/ReaderOptions.d.ts

@@ -0,0 +1,15 @@
+export interface ReaderOptions {
+    /**
+     * Max Hunspell recursive depth.
+     */
+    maxDepth?: number;
+}
+export type AnnotatedWord = string;
+export interface BaseReader {
+    size: number;
+    type: 'Hunspell' | 'TextFile' | 'Trie';
+    lines: Iterable<AnnotatedWord>;
+}
+export interface Reader extends BaseReader, Iterable<string> {
+}
+//# sourceMappingURL=ReaderOptions.d.ts.map

package/dist/compiler/readers/ReaderOptions.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=ReaderOptions.js.map

package/dist/compiler/readers/readHunspellFiles.d.ts

@@ -0,0 +1,3 @@
+import type { BaseReader, ReaderOptions } from './ReaderOptions';
+export declare function readHunspellFiles(filename: string, options: ReaderOptions): Promise<BaseReader>;
+//# sourceMappingURL=readHunspellFiles.d.ts.map

package/dist/compiler/readers/readHunspellFiles.js

@@ -0,0 +1,84 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readHunspellFiles = void 0;
+const sync_1 = require("@cspell/cspell-pipe/sync");
+const cspell_trie_lib_1 = require("cspell-trie-lib");
+const HR = __importStar(require("hunspell-reader"));
+const Reader_1 = require("../Reader");
+const DEDUPE_SIZE = 1000;
+async function readHunspellFiles(filename, options) {
+    const dicFile = filename.replace(Reader_1.regHunspellFile, '.dic');
+    const affFile = filename.replace(Reader_1.regHunspellFile, '.aff');
+    const reader = await HR.IterableHunspellReader.createFromFiles(affFile, dicFile);
+    reader.maxDepth = options.maxDepth !== undefined ? options.maxDepth : reader.maxDepth;
+    const words = (0, sync_1.pipe)(reader.seqAffWords(), _mapAffWords, dedupeAndSort);
+    return {
+        type: 'Hunspell',
+        size: reader.dic.length,
+        lines: words,
+    };
+}
+exports.readHunspellFiles = readHunspellFiles;
+function* dedupeAndSort(words) {
+    const buffer = new Set();
+    function flush() {
+        const result = [...buffer].sort();
+        buffer.clear();
+        return result;
+    }
+    for (const word of words) {
+        buffer.add(word);
+        if (buffer.size >= DEDUPE_SIZE) {
+            yield* flush();
+        }
+    }
+    yield* flush();
+}
+function* _mapAffWords(affWords) {
+    const hasSpecial = /[~+!]/;
+    for (const affWord of affWords) {
+        const { word, flags } = affWord;
+        // For now do not include words with special characters.
+        if (hasSpecial.test(word))
+            continue;
+        const compound = flags.isCompoundForbidden ? '' : cspell_trie_lib_1.COMPOUND_FIX;
+        const forbid = flags.isForbiddenWord ? cspell_trie_lib_1.FORBID_PREFIX : '';
+        if (!forbid) {
+            if (flags.canBeCompoundBegin || flags.isCompoundPermitted)
+                yield word + compound;
+            if (flags.canBeCompoundEnd || flags.isCompoundPermitted)
+                yield compound + word;
+            if (flags.canBeCompoundMiddle || flags.isCompoundPermitted)
+                yield compound + word + compound;
+            if (!flags.isOnlyAllowedInCompound)
+                yield word;
+        }
+        else {
+            yield forbid + word;
+        }
+    }
+}
+//# sourceMappingURL=readHunspellFiles.js.map

package/dist/compiler/readers/textFileReader.d.ts

@@ -0,0 +1,3 @@
+import type { BaseReader } from './ReaderOptions';
+export declare function textFileReader(filename: string): Promise<BaseReader>;
+//# sourceMappingURL=textFileReader.d.ts.map

package/dist/compiler/readers/textFileReader.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.textFileReader = void 0;
+const readTextFile_1 = require("./readTextFile");
+async function textFileReader(filename) {
+    const content = await (0, readTextFile_1.readTextFile)(filename);
+    const words = content.split('\n').map((s) => s.trim());
+    return {
+        type: 'TextFile',
+        size: words.length,
+        lines: words,
+    };
+}
+exports.textFileReader = textFileReader;
+//# sourceMappingURL=textFileReader.js.map

package/dist/compiler/readers/trieFileReader.d.ts

@@ -0,0 +1,3 @@
+import type { BaseReader } from './ReaderOptions';
+export declare function trieFileReader(filename: string): Promise<BaseReader>;
+//# sourceMappingURL=trieFileReader.d.ts.map

package/dist/compiler/readers/trieFileReader.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.trieFileReader = void 0;
+const cspell_trie_lib_1 = require("cspell-trie-lib");
+const readTextFile_1 = require("./readTextFile");
+async function trieFileReader(filename) {
+    const trieRoot = (0, cspell_trie_lib_1.importTrie)(await (0, readTextFile_1.readTextFileLines)(filename));
+    const trie = new cspell_trie_lib_1.Trie(trieRoot);
+    const words = trie.words();
+    return {
+        type: 'Trie',
+        get size() {
+            return trie.size();
+        },
+        lines: words,
+    };
+}
+exports.trieFileReader = trieFileReader;
+//# sourceMappingURL=trieFileReader.js.map

package/dist/compiler/splitCamelCaseIfAllowed.d.ts

@@ -0,0 +1,5 @@
+import type { AllowedSplitWordsCollection } from './WordsCollection';
+export declare const regExpSpaceOrDash: RegExp;
+export declare const regExpIsNumber: RegExp;
+export declare function splitCamelCaseIfAllowed(word: string, allowedWords: AllowedSplitWordsCollection, keepCase: boolean): string[];
+//# sourceMappingURL=splitCamelCaseIfAllowed.d.ts.map

package/dist/compiler/splitCamelCaseIfAllowed.js

@@ -0,0 +1,66 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.splitCamelCaseIfAllowed = exports.regExpIsNumber = exports.regExpSpaceOrDash = void 0;
+const Text = __importStar(require("./text"));
+exports.regExpSpaceOrDash = /[- ]+/g;
+exports.regExpIsNumber = /^\d+$/;
+function splitCamelCaseIfAllowed(word, allowedWords, keepCase) {
+    const split = [...splitCamelCase(word)];
+    if (split.length == 1)
+        return adjustCases(split, allowedWords, keepCase);
+    const missing = split.find((w) => isUnknown(w, allowedWords));
+    if (missing !== undefined)
+        return [word];
+    return adjustCases(split, allowedWords, keepCase);
+}
+exports.splitCamelCaseIfAllowed = splitCamelCaseIfAllowed;
+function adjustCases(words, allowedWords, keepCase) {
+    return words.map((w) => adjustCase(w, allowedWords, keepCase));
+}
+function adjustCase(word, allowedWords, keepCase) {
+    const lc = word.toLowerCase();
+    if (!allowedWords.has(lc))
+        return word;
+    if (lc === word)
+        return word;
+    if (word.slice(1).toLowerCase() === word.slice(1))
+        return lc;
+    if (!keepCase && word.toUpperCase() === word)
+        return word.toLowerCase();
+    return word;
+}
+function isUnknown(word, allowedWords) {
+    return !allowedWords.has(word) && !allowedWords.has(word.toLowerCase());
+}
+function splitCamelCase(word) {
+    const splitWords = Text.splitCamelCaseWord(word).filter((word) => !exports.regExpIsNumber.test(word));
+    // We only want to preserve this: "New York" and not "Namespace DNSLookup"
+    if (splitWords.length > 1 && exports.regExpSpaceOrDash.test(word)) {
+        return splitWords.flatMap((w) => w.split(exports.regExpSpaceOrDash));
+    }
+    return splitWords;
+}
+//# sourceMappingURL=splitCamelCaseIfAllowed.js.map

package/dist/compiler/streamSourceWordsFromFile.d.ts

@@ -0,0 +1,3 @@
+import type { SourceReaderOptions } from './SourceReader';
+export declare function streamSourceWordsFromFile(filename: string, options: SourceReaderOptions): Promise<Iterable<string>>;
+//# sourceMappingURL=streamSourceWordsFromFile.d.ts.map

package/dist/compiler/streamSourceWordsFromFile.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.streamSourceWordsFromFile = void 0;
+const SourceReader_1 = require("./SourceReader");
+async function streamSourceWordsFromFile(filename, options) {
+    const reader = await (0, SourceReader_1.createSourceReader)(filename, options);
+    return reader.words;
+}
+exports.streamSourceWordsFromFile = streamSourceWordsFromFile;
+//# sourceMappingURL=streamSourceWordsFromFile.js.map

package/dist/compiler/text.js

@@ -3,17 +3,17 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.splitCamelCaseWord = void 0;
 const regExUpperSOrIng = /(\p{Lu}+'?(?:s|ing|ies|es|ings|ed|ning))(?!\p{Ll})/gu;
-const regExSplitWords = /(\p{Ll})(\p{Lu})/gu;
+const regExSplitWords = /([\p{Ll}])([\p{Lu}])/gu;
 const regExSplitWords2 = /(\p{Lu})(\p{Lu}\p{Ll})/gu;
 /**
  * Split camelCase words into an array of strings.
  */
 function splitCamelCaseWord(word) {
-    const wPrime = word.replace(regExUpperSOrIng, (s) => s[0] + s.substr(1).toLowerCase());
-    const separator = '_<^*_*^>_';
-    const pass1 = wPrime.replace(regExSplitWords, '$1' + separator + '$2');
-    const pass2 = pass1.replace(regExSplitWords2, '$1' + separator + '$2');
-    return pass2.split(separator);
+    const wPrime = word.replace(regExUpperSOrIng, (s) => s[0] + s.slice(1).toLowerCase());
+    const pass1 = wPrime.replace(regExSplitWords, '$1|$2');
+    const pass2 = pass1.replace(regExSplitWords2, '$1|$2');
+    const pass3 = pass2.replace(/[\d_]+/g, '|');
+    return pass3.split('|').filter((a) => !!a);
 }
 exports.splitCamelCaseWord = splitCamelCaseWord;
 //# sourceMappingURL=text.js.map

package/dist/compiler/wordListParser.d.ts

@@ -1,5 +1,6 @@
 import { type Operator } from '@cspell/cspell-pipe/sync';
 import type { CompileOptions } from './CompileOptions';
+import type { AllowedSplitWordsCollection } from './WordsCollection';
 export declare function normalizeTargetWords(options: CompileOptions): Operator<string>;
 export interface ParseFileOptions {
     /**
@@ -22,6 +23,7 @@ export interface ParseFileOptions {
      * @default false
      */
     legacy?: boolean;
+    allowedSplitWords: AllowedSplitWordsCollection;
 }
 type ParseFileOptionsRequired = Required<ParseFileOptions>;
 export declare const defaultParseDictionaryOptions: ParseFileOptionsRequired;
@@ -41,6 +43,6 @@ export declare function createParseFileLineMapper(options?: Partial<ParseFileOpt
  * @param _options - defines prefixes used when parsing lines.
  * @returns words that have been normalized.
  */
-export declare function parseFileLines(lines: Iterable<string> | string, options?: Partial<ParseFileOptions>): Iterable<string>;
+export declare function parseFileLines(lines: Iterable<string> | string, options: Partial<ParseFileOptions>): Iterable<string>;
 export {};
 //# sourceMappingURL=wordListParser.d.ts.map

package/dist/compiler/wordListParser.js

@@ -5,6 +5,7 @@ const sync_1 = require("@cspell/cspell-pipe/sync");
 const cspell_trie_lib_1 = require("cspell-trie-lib");
 const util_1 = require("hunspell-reader/dist/util");
 const legacyLineToWords_1 = require("./legacyLineToWords");
+const splitCamelCaseIfAllowed_1 = require("./splitCamelCaseIfAllowed");
 function normalizeTargetWords(options) {
     const lineParser = (0, cspell_trie_lib_1.createDictionaryLineParser)({
         stripCaseAndAccents: options.generateNonStrict,
@@ -45,6 +46,7 @@ const _defaultOptions = {
     split: false,
     splitKeepBoth: false,
     // splitSeparator: regExpSplit,
+    allowedSplitWords: { has: () => true, size: 0 },
 };
 exports.defaultParseDictionaryOptions = Object.freeze(_defaultOptions);
 exports.cSpellToolDirective = 'cspell-tools:';
@@ -57,10 +59,9 @@ exports.setOfCSpellDirectiveFlags = ['no-split', 'split', 'keep-case', 'no-keep-
  */
 function createParseFileLineMapper(options) {
     const _options = options || _defaultOptions;
-    const { splitKeepBoth = _defaultOptions.splitKeepBoth } = _options;
+    const { splitKeepBoth = _defaultOptions.splitKeepBoth, allowedSplitWords = _defaultOptions.allowedSplitWords } = _options;
     let { legacy = _defaultOptions.legacy } = _options;
     let { split = _defaultOptions.split, keepCase = legacy ? false : _defaultOptions.keepCase } = _options;
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
     function isString(line) {
         return typeof line === 'string';
     }
@@ -131,11 +132,17 @@ function createParseFileLineMapper(options) {
     function* splitWords(lines) {
         for (const line of lines) {
             if (legacy) {
-                yield* (0, legacyLineToWords_1.legacyLineToWords)(line, keepCase);
+                yield* (0, legacyLineToWords_1.legacyLineToWords)(line, keepCase, allowedSplitWords);
                 continue;
             }
             if (split) {
-                yield* splitLine(line);
+                const words = splitLine(line);
+                if (!allowedSplitWords.size) {
+                    yield* words;
+                }
+                else {
+                    yield* words.flatMap((word) => (0, splitCamelCaseIfAllowed_1.splitCamelCaseIfAllowed)(word, allowedSplitWords, keepCase));
+                }
                 if (!splitKeepBoth)
                     continue;
             }

package/dist/config/config.d.ts

@@ -39,6 +39,16 @@ export interface CompileTargetOptions {
      * @default: true
      */
     sort?: boolean | undefined;
+    /**
+     * Words in the `allowedSplitWords` are considered correct and can be used
+     * as a basis for splitting compound words.
+     *
+     * If entries can be split so that all the words in the entry are allowed,
+     * then only the individual words are added, otherwise the entire entry is added.
+     * This is to prevent misspellings in CamelCase words from being introduced into the
+     * dictionary.
+     */
+    allowedSplitWords?: FilePath | FilePath[] | undefined;
 }
 export interface Target extends CompileTargetOptions {
     /**
@@ -66,6 +76,8 @@ export interface Target extends CompileTargetOptions {
     /**
      * Words from the sources that are found in `excludeWordsFrom` files
      * will not be added to the dictionary.
+     *
+     * @version TBD
      */
     excludeWordsFrom?: FilePath[] | undefined;
     /**
@@ -105,5 +117,6 @@ export interface CompileSourceOptions {
      * @default false
      */
     keepRawCase?: boolean | undefined;
+    allowedSplitWords?: FilePath | FilePath[] | undefined;
 }
 //# sourceMappingURL=config.d.ts.map

package/dist/test/TestHelper.d.ts

@@ -39,4 +39,5 @@ export interface TestHelper {
     fileExists(path: string): Promise<boolean>;
 }
 export declare function createTestHelper(testFilename?: string): TestHelper;
+export declare function resolvePathToFixture(...segments: string[]): string;
 //# sourceMappingURL=TestHelper.d.ts.map

package/dist/test/TestHelper.js

@@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.createTestHelper = void 0;
+exports.resolvePathToFixture = exports.createTestHelper = void 0;
 const fs_1 = require("fs");
 const path = __importStar(require("path"));
 const shell = __importStar(require("shelljs"));
@@ -34,6 +34,7 @@ function createTestHelper(testFilename) {
     return new TestHelperImpl(testFilename || expect.getState().testPath || 'test');
 }
 exports.createTestHelper = createTestHelper;
+const fixtureDir = path.join(packageRoot, 'fixtures');
 class TestHelperImpl {
     constructor(testFilename) {
         this.packageRoot = packageRoot;
@@ -41,7 +42,7 @@ class TestHelperImpl {
         this.testCounter = new Map();
         this.createTempDir = this.mkdir;
         this.tempDir = path.join(tempDirBase, path.relative(packageRoot, testFilename));
-        this.fixtureDir = path.join(packageRoot, 'fixtures');
+        this.fixtureDir = fixtureDir;
     }
     beginTest() {
         const currentTestName = this.getRawTestName();
@@ -128,4 +129,8 @@ class TestHelperImpl {
         }
     }
 }
+function resolvePathToFixture(...segments) {
+    return path.resolve(fixtureDir, ...segments);
+}
+exports.resolvePathToFixture = resolvePathToFixture;
 //# sourceMappingURL=TestHelper.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cspell/cspell-tools",
-  "version": "6.28.0",
+  "version": "6.29.1",
   "description": "Tools to assist with the development of cSpell",
   "typings": "dist/index.d.ts",
   "publishConfig": {
@@ -48,13 +48,13 @@
   },
   "homepage": "https://github.com/streetsidesoftware/cspell#readme",
   "dependencies": {
-    "@cspell/cspell-pipe": "6.28.0",
+    "@cspell/cspell-pipe": "6.29.1",
     "commander": "^10.0.0",
     "cosmiconfig": "^8.1.0",
-    "cspell-trie-lib": "6.28.0",
+    "cspell-trie-lib": "6.29.1",
     "gensequence": "^5.0.2",
     "glob": "^8.1.0",
-    "hunspell-reader": "6.28.0"
+    "hunspell-reader": "6.29.1"
   },
   "engines": {
     "node": ">=14"
@@ -63,11 +63,11 @@
     "@types/glob": "^8.1.0",
     "@types/jest": "^29.4.0",
     "@types/shelljs": "^0.8.11",
-    "jest": "^29.4.3",
+    "jest": "^29.5.0",
     "lorem-ipsum": "^2.0.8",
     "shelljs": "^0.8.5",
     "ts-json-schema-generator": "^1.2.0"
   },
   "main": "bin.js",
-  "gitHead": "1c314413e76908e5fbf61fd2555726112b177c0e"
+  "gitHead": "e524c611f3529b22a7e8ae3449a5c9a01332d44f"
 }

package/dist/compiler/iterateWordsFromFile.d.ts

@@ -1,3 +0,0 @@
-import type { ReaderOptions } from './Reader';
-export declare function streamWordsFromFile(filename: string, options: ReaderOptions): Promise<Iterable<string>>;
-//# sourceMappingURL=iterateWordsFromFile.d.ts.map

package/dist/compiler/iterateWordsFromFile.js

@@ -1,10 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.streamWordsFromFile = void 0;
-const Reader_1 = require("./Reader");
-async function streamWordsFromFile(filename, options) {
-    const reader = await (0, Reader_1.createReader)(filename, options);
-    return reader;
-}
-exports.streamWordsFromFile = streamWordsFromFile;
-//# sourceMappingURL=iterateWordsFromFile.js.map