@cspell/cspell-tools 8.19.3 → 8.19.4

package/cspell-tools.config.schema.json

@@ -174,7 +174,21 @@
           "type": "array"
         },
         "excludeWordsFrom": {
-          "description": "Words from the sources that are found in `excludeWordsFrom` files will not be added to the dictionary.",
+          "description": "Words from the sources that are found in `excludeWordsFrom` files will NOT be added to the dictionary.",
+          "items": {
+            "$ref": "#/definitions/FilePath"
+          },
+          "type": "array"
+        },
+        "excludeWordsMatchingRegex": {
+          "description": "Words from the sources that match the regex in `excludeWordsMatchingRegex` will NOT be added to the dictionary.\n\nNote: The regex must be a valid JavaScript literal regex expression including the `/` delimiters.",
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "excludeWordsNotFoundIn": {
+          "description": "Words from the sources that are NOT found in `excludeWordsNotFoundIn` files will NOT be added to the dictionary.",
           "items": {
             "$ref": "#/definitions/FilePath"
           },

package/dist/compiler/compile.js

@@ -4,6 +4,7 @@ import { opAwaitAsync, opMapAsync } from '@cspell/cspell-pipe/operators';
 import { opConcatMap, opMap, pipe } from '@cspell/cspell-pipe/sync';
 import { isFileListSource, isFilePath, isFileSource } from '../config/index.js';
 import { checkShasumFile, updateChecksumForFiles } from '../shasum/index.js';
+import { stringToRegExp } from '../util/textRegex.js';
 import { createAllowedSplitWordsFromFiles, createWordsCollectionFromFiles } from './createWordsCollection.js';
 import { logWithTimestamp } from './logWithTimestamp.js';
 import { readTextFile } from './readers/readTextFile.js';
@@ -55,11 +56,16 @@ function resolveChecksumFile(checksumFile, root) {
 export async function compileTarget(target, options, compileOptions) {
     logWithTimestamp(`Start compile: ${target.name}`);
     const { rootDir, cwd, checksumFile, conditional } = compileOptions;
-    const { format, sources, trieBase, sort = true, generateNonStrict = false, excludeWordsFrom } = target;
+    const { format, sources, trieBase, sort = true, generateNonStrict = false, excludeWordsFrom = [], excludeWordsNotFoundIn = [], excludeWordsMatchingRegex, } = target;
     const targetDirectory = path.resolve(rootDir, target.targetDirectory ?? cwd ?? process.cwd());
     const dictionaryDirectives = target.dictionaryDirectives ?? compileOptions.dictionaryDirectives;
     const removeDuplicates = target.removeDuplicates ?? false;
-    const excludeFilter = await createExcludeFilter(excludeWordsFrom);
+    const excludeFromFilter = await createExcludeFilter(excludeWordsFrom);
+    const includeFromFilter = await createIncludeFilter(excludeWordsNotFoundIn);
+    const excludeRegexFilter = createExcludeRegexFilter(excludeWordsMatchingRegex);
+    const excludeFilter = (word) => {
+        return excludeFromFilter(word) && includeFromFilter(word) && excludeRegexFilter(word);
+    };
     const generateNonStrictTrie = target.generateNonStrict ?? true;
     const name = normalizeTargetName(target.name);
     const useTrie = format.startsWith('trie');
@@ -74,7 +80,9 @@ export async function compileTarget(target, options, compileOptions) {
         // removeDuplicates, // Add this in if we use it.
     });
     const checksumRoot = (checksumFile && path.dirname(checksumFile)) || rootDir;
-    const deps = [...calculateDependencies(filename, filesToProcess, excludeWordsFrom, checksumRoot)];
+    const deps = [
+        ...calculateDependencies(filename, filesToProcess, [...excludeWordsFrom, ...excludeWordsNotFoundIn], checksumRoot),
+    ];
     if (conditional && checksumFile) {
         const check = await checkShasumFile(checksumFile, deps, checksumRoot).catch(() => undefined);
         if (check?.passed) {
@@ -215,10 +223,41 @@ function logProgress(freq = 100_000) {
     }
     return logProgress;
 }
+/**
+ * @param excludeWordsFrom - List of files to read words from.
+ * @returns a function that returns true if the word is not in the exclude list.
+ */
 async function createExcludeFilter(excludeWordsFrom) {
     if (!excludeWordsFrom || !excludeWordsFrom.length)
         return () => true;
     const excludeWords = await createWordsCollectionFromFiles(excludeWordsFrom);
     return (word) => !excludeWords.has(word, word.toUpperCase() !== word);
 }
+/**
+ * @param includeWordsFrom - List of files to read words from.
+ * @returns a function that returns true if the word is in the include list.
+ */
+async function createIncludeFilter(includeWordsFrom) {
+    if (!includeWordsFrom || !includeWordsFrom.length)
+        return () => true;
+    const excludeWords = await createWordsCollectionFromFiles(includeWordsFrom);
+    return (word) => excludeWords.has(word, word.toUpperCase() !== word);
+}
+/**
+ * @param excludeWordsMatchingRegex - List of regex patterns to exclude.
+ * @returns a function that returns true if the word does not match any of the regex patterns.
+ */
+function createExcludeRegexFilter(excludeWordsMatchingRegex) {
+    if (!excludeWordsMatchingRegex || !excludeWordsMatchingRegex.length)
+        return () => true;
+    const regexes = excludeWordsMatchingRegex
+        .map((a) => stringToRegExp(a))
+        .filter((a, i) => {
+        if (a)
+            return true;
+        console.warn('Invalid regex: "%s"', excludeWordsMatchingRegex[i]);
+        return false;
+    });
+    return (word) => !regexes.some((r) => r.test(word));
+}
 //# sourceMappingURL=compile.js.map

package/dist/config/config.d.ts

@@ -108,11 +108,27 @@ export interface Target extends CompileTargetOptions {
     sources: DictionarySource[];
     /**
      * Words from the sources that are found in `excludeWordsFrom` files
-     * will not be added to the dictionary.
+     * will NOT be added to the dictionary.
      *
      * @since 8.3.2
      */
     excludeWordsFrom?: FilePath[] | undefined;
+    /**
+     * Words from the sources that are NOT found in `excludeWordsNotFoundIn` files
+     * will NOT be added to the dictionary.
+     *
+     * @since 8.19.4
+     */
+    excludeWordsNotFoundIn?: FilePath[] | undefined;
+    /**
+     * Words from the sources that match the regex in `excludeWordsMatchingRegex`
+     * will NOT be added to the dictionary.
+     *
+     * Note: The regex must be a valid JavaScript literal regex expression including the `/` delimiters.
+     *
+     * @since 8.19.4
+     */
+    excludeWordsMatchingRegex?: string[] | undefined;
     /**
      * Advanced: Set the trie base number. A value between 10 and 36
      * Set numeric base to use.

package/dist/util/textRegex.d.ts

@@ -0,0 +1,3 @@
+export declare const regExMatchRegExParts: RegExp;
+export declare function stringToRegExp(pattern: string | RegExp, defaultFlags?: string, forceFlags?: string): RegExp | undefined;
+//# sourceMappingURL=textRegex.d.ts.map

package/dist/util/textRegex.js

@@ -0,0 +1,109 @@
+// cspell:ignore gimuxy
+export const regExMatchRegExParts = /^\s*\/([\s\S]*?)\/([gimuxy]*)\s*$/;
+export function stringToRegExp(pattern, defaultFlags = '', forceFlags = '') {
+    if (pattern instanceof RegExp) {
+        return pattern;
+    }
+    try {
+        const [, pat, flag] = [
+            ...(pattern.match(regExMatchRegExParts) || ['', pattern.trim(), defaultFlags]),
+            forceFlags,
+        ];
+        if (pat) {
+            const regPattern = flag.includes('x') ? removeVerboseFromRegExp(pat) : pat;
+            // Make sure the flags are unique.
+            const flags = [...new Set(forceFlags + flag)].join('').replaceAll(/[^gimuy]/g, '');
+            const regex = new RegExp(regPattern, flags);
+            return regex;
+        }
+    }
+    catch {
+        /* empty */
+    }
+    return undefined;
+}
+const SPACES = {
+    ' ': true,
+    '\n': true,
+    '\r': true,
+    '\t': true,
+};
+/**
+ * Remove all whitespace and comments from a regexp string. The format follows Pythons Verbose.
+ * Note: this is a best attempt. Special cases for comments: `#` and spaces should be proceeded with a `\`
+ *
+ * All space must be proceeded by a `\` or in a character class `[]`
+ *
+ * @param pattern - the pattern to clean
+ */
+function removeVerboseFromRegExp(pattern) {
+    function escape(acc) {
+        const char = pattern[acc.idx];
+        if (char !== '\\')
+            return undefined;
+        const next = pattern[++acc.idx];
+        acc.idx++;
+        if (next === '#') {
+            acc.result += '#';
+            return acc;
+        }
+        if (!(next in SPACES)) {
+            acc.result += '\\' + next;
+            return acc;
+        }
+        acc.result += next;
+        if (next === '\r' && pattern[acc.idx] === '\n') {
+            acc.result += '\n';
+            acc.idx++;
+        }
+        return acc;
+    }
+    function braces(acc) {
+        const char = pattern[acc.idx];
+        if (char !== '[')
+            return undefined;
+        acc.result += char;
+        acc.idx++;
+        let escCount = 0;
+        while (acc.idx < pattern.length) {
+            const char = pattern[acc.idx];
+            acc.result += char;
+            acc.idx++;
+            if (char === ']' && !(escCount & 1))
+                break;
+            escCount = char === '\\' ? escCount + 1 : 0;
+        }
+        return acc;
+    }
+    function spaces(acc) {
+        const char = pattern[acc.idx];
+        if (!(char in SPACES))
+            return undefined;
+        acc.idx++;
+        return acc;
+    }
+    function comments(acc) {
+        const char = pattern[acc.idx];
+        if (char !== '#')
+            return undefined;
+        while (acc.idx < pattern.length && pattern[acc.idx] !== '\n') {
+            acc.idx++;
+        }
+        return acc;
+    }
+    function copy(acc) {
+        const char = pattern[acc.idx++];
+        acc.result += char;
+        return acc;
+    }
+    const reducers = [escape, braces, spaces, comments, copy];
+    const result = { idx: 0, result: '' };
+    while (result.idx < pattern.length) {
+        for (const r of reducers) {
+            if (r(result))
+                break;
+        }
+    }
+    return result.result;
+}
+//# sourceMappingURL=textRegex.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cspell/cspell-tools",
-  "version": "8.19.3",
+  "version": "8.19.4",
   "description": "Tools to assist with the development of cSpell",
   "publishConfig": {
     "access": "public",
@@ -63,12 +63,12 @@
   },
   "homepage": "https://github.com/streetsidesoftware/cspell/tree/main/packages/cspell-tools#readme",
   "dependencies": {
-    "@cspell/cspell-pipe": "8.19.3",
+    "@cspell/cspell-pipe": "8.19.4",
     "commander": "^13.1.0",
     "cosmiconfig": "9.0.0",
-    "cspell-trie-lib": "8.19.3",
+    "cspell-trie-lib": "8.19.4",
     "glob": "^10.4.5",
-    "hunspell-reader": "8.19.3",
+    "hunspell-reader": "8.19.4",
     "yaml": "^2.7.1"
   },
   "engines": {
@@ -79,5 +79,5 @@
     "ts-json-schema-generator": "^2.4.0"
   },
   "module": "bin.mjs",
-  "gitHead": "cb6946b709da590d0c6175f9962799d18aa43e93"
+  "gitHead": "1bee5f5aa4429a1b1ae0e88934b093c5440b44dc"
 }