cspell-lib 10.2.1 → 10.3.0-alpha.0

package/dist/lib/Settings/internal/DictionarySettings.js

@@ -107,7 +107,7 @@ class _DictionaryDefinitionInternalWithSource {
         this.__source = sourceURL.href;
         // this bit of assignment is to have the compiler help use if any new fields are added.
         const defAll = def;
-        const { path: relPath = '', file = '', btrie, addWords, description, dictionaryInformation, type, repMap, noSuggest, ignoreForbiddenWords, scope, supportNonStrictSearches, useCompounds, } = defAll;
+        const { path: relPath = '', file = '', btrie, addWords, description, dictionaryInformation, type, repMap, noSuggest, kind, ignoreForbiddenWords, scope, supportNonStrictSearches, useCompounds, } = defAll;
         const defaultPath = sourceURL;
         const filePath = fixDicPath(relPath, file);
         const name = determineName(filePath, def);
@@ -125,6 +125,7 @@ class _DictionaryDefinitionInternalWithSource {
             type,
             repMap,
             noSuggest,
+            kind,
             ignoreForbiddenWords,
             supportNonStrictSearches,
             scope,

package/dist/lib/SpellingDictionary/DictionaryController/DictionaryLoader.js

@@ -52,7 +52,7 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
 });
 import { opConcatMap, opFilter, opMap, pipe } from '@cspell/cspell-pipe/sync';
 import { StrongWeakMap } from '@cspell/strong-weak-map';
-import { createFailedToLoadDictionary, createInlineSpellingDictionary, createSpellingDictionary, createSpellingDictionaryFromTrieFile, } from 'cspell-dictionary';
+import { createFailedToLoadDictionary, createFlagWordsDictionary, createFlagWordsDictionaryFromTrieFile, createInlineSpellingDictionary, createSpellingDictionary, createSpellingDictionaryFromTrieFile, } from 'cspell-dictionary';
 import { compareStats, toFileURL, urlBasename } from 'cspell-io';
 import { measurePerf } from '../../perf/index.js';
 import { isDictionaryDefinitionInlineInternal, isDictionaryFileDefinitionInternal } from '../../Settings/index.js';
@@ -221,7 +221,13 @@ function toReader(fs) {
         readLines: async (filename) => toLines(await readText(filename)),
     };
 }
-const importantOptionKeys = ['name', 'noSuggest', 'useCompounds', 'type'];
+const importantOptionKeys = [
+    'name',
+    'noSuggest',
+    'kind',
+    'useCompounds',
+    'type',
+];
 function isError(e) {
     const err = e;
     return !!err.message;
@@ -248,7 +254,10 @@ async function legacyWordList(reader, filename, options) {
         opMap((line) => line.replaceAll(/#.*/g, '')), 
         // Split on everything else
         opConcatMap((line) => line.split(/[^\w\p{L}\p{M}'’]+/gu)), opFilter((word) => !!word));
-        return createSpellingDictionary(words, options.name, filename.toString(), options, true);
+        if (options.kind === 'flag-words') {
+            return createFlagWordsDictionary([...words], options.name, filename.toString());
+        }
+        return createSpellingDictionary(words, options.name, filename.toString(), applyKind(options), true);
     }
     catch (e_1) {
         env_1.error = e_1;
@@ -265,14 +274,20 @@ async function wordsPerLineWordList(reader, filename, options) {
     opMap((line) => line.replaceAll(/#.*/g, '')), 
     // Split on everything else
     opConcatMap((line) => line.split(/\s+/gu)), opFilter((word) => !!word));
-    return createSpellingDictionary(words, options.name, filename.href, options, true);
+    if (options.kind === 'flag-words') {
+        return createFlagWordsDictionary([...words], options.name, filename.href);
+    }
+    return createSpellingDictionary(words, options.name, filename.href, applyKind(options), true);
 }
 async function loadSimpleWordList(reader, filename, options) {
     const env_2 = { stack: [], error: void 0, hasError: false };
     try {
         const lines = await reader.readLines(filename);
         const _ = __addDisposableResource(env_2, measurePerf('loadSimpleWordList'), false);
-        return createSpellingDictionary(lines, options.name, filename.href, options);
+        if (options.kind === 'flag-words') {
+            return createFlagWordsDictionary(lines, options.name, filename.href);
+        }
+        return createSpellingDictionary(lines, options.name, filename.href, applyKind(options));
     }
     catch (e_2) {
         env_2.error = e_2;
@@ -287,7 +302,10 @@ async function loadTrie(reader, filename, options) {
     try {
         const content = await reader.read(filename);
         const _ = __addDisposableResource(env_3, measurePerf('loadTrie'), false);
-        return createSpellingDictionaryFromTrieFile(content, options.name, filename.href, options);
+        if (options.kind === 'flag-words') {
+            return createFlagWordsDictionaryFromTrieFile(content, options.name, filename.href);
+        }
+        return createSpellingDictionaryFromTrieFile(content, options.name, filename.href, applyKind(options));
     }
     catch (e_3) {
         env_3.error = e_3;
@@ -297,6 +315,13 @@ async function loadTrie(reader, filename, options) {
         __disposeResources(env_3);
     }
 }
+/**
+ * Apply the effect of `kind` on the options passed to a normal `words` dictionary loader.
+ * `flag-words` is handled by the callers directly since it uses a different dictionary implementation.
+ */
+function applyKind(options) {
+    return options.kind === 'ignore-words' && !options.noSuggest ? { ...options, noSuggest: true } : options;
+}
 function toLines(content) {
     return content.split(/\n|\r\n|\r/);
 }

package/dist/lib/textValidation/lineValidatorFactory.js

@@ -329,10 +329,16 @@ export function lineValidatorFactory(sDict, options) {
                 });
                 // Ignore 1 or 2 letter segments.
                 const minWordLen = Math.min(3, minWordLength);
+                const keepIssue = (w) => w.isFlagged ||
+                    w.text.length >= minWordLen ||
+                    // eslint-disable-next-line unicorn/prefer-code-point
+                    (w.text.charCodeAt(0) & 0xff80) !== 0 || // Keep if Non-ASCII
+                    // eslint-disable-next-line unicorn/prefer-code-point
+                    (w.text.charCodeAt(1) & 0xff80) !== 0; // Keep if Non-ASCII
                 const filtered = filterExcludedTextOffsets(nonMatching
                     .map((w) => ({ ...w, line: lineSegment.line }))
                     .map(annotateIsFlagged)
-                    .filter((w) => w.isFlagged || w.text.length >= minWordLen), hexSequences);
+                    .filter(keepIssue), hexSequences);
                 if (filtered.length < mismatches.length) {
                     return filtered;
                 }

package/dist/lib/util/wordSplitter/wordSplitter.js

@@ -1,8 +1,62 @@
+var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
+    if (value !== null && value !== void 0) {
+        if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
+        var dispose, inner;
+        if (async) {
+            if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
+            dispose = value[Symbol.asyncDispose];
+        }
+        if (dispose === void 0) {
+            if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
+            dispose = value[Symbol.dispose];
+            if (async) inner = dispose;
+        }
+        if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
+        if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
+        env.stack.push({ value: value, dispose: dispose, async: async });
+    }
+    else if (async) {
+        env.stack.push({ async: true });
+    }
+    return value;
+};
+var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
+    return function (env) {
+        function fail(e) {
+            env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
+            env.hasError = true;
+        }
+        var r, s = 0;
+        function next() {
+            while (r = env.stack.pop()) {
+                try {
+                    if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
+                    if (r.dispose) {
+                        var result = r.dispose.call(r.value);
+                        if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
+                    }
+                    else s |= 1;
+                }
+                catch (e) {
+                    fail(e);
+                }
+            }
+            if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
+            if (env.hasError) throw env.error;
+        }
+        return next();
+    };
+})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
+    var e = new Error(message);
+    return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
+});
+import { counter, measurePerf } from '@cspell/cspell-performance-monitor';
 import { PairingHeap } from '../PairingHeap.js';
 import { regExFirstUpper, regExNumericLiteral, regExWordsAndDigits } from '../textRegex.js';
 import { generateWordBreaks, softHyphen } from './generateWordBreaks.js';
 const ignoreBreak = Object.freeze([]);
-const maxSkippedBreaks = 8; // Maximum number of breaks that can be skipped during word splitting.
+const maxSkippedBreaks = 16; // Maximum number of breaks that can be skipped during word splitting.
+const maxAttempts = 64_000; // Maximum number of attempts to split words.
 // Quote/backtick characters that `regExWordsAndDigits` treats as word characters, so a quoted
 // or templated string literal in source code gets scanned with its delimiters attached.
 const regExLeadingTrimCharacters = /^['’`\\-]+/;
@@ -16,98 +70,110 @@ const regExTrailingTrimCharacters = /['’`\\-]+$/;
  * @returns SplitResult
  */
 export function split(line, offset, isValidWord, options = {}) {
-    const relWordToSplit = findNextWordText({ text: line.text, offset: offset - line.offset });
-    const hasSoftHyphen = relWordToSplit.text.includes(softHyphen);
-    const lineOffset = line.offset;
-    const requested = new Map();
-    const regExpIgnoreSegment = /^[-.+\d_eE'`\\\s\u00AD]+$/;
-    if (!relWordToSplit.text) {
-        const text = rebaseTextOffsetToAbsolute(relWordToSplit);
-        return {
-            line,
-            offset,
-            text: text,
-            words: [],
-            endOffset: text.offset + (text.length ?? text.text.length),
-        };
-    }
-    const lineSegment = {
-        line,
-        relStart: relWordToSplit.offset,
-        relEnd: relWordToSplit.offset + relWordToSplit.text.length,
-    };
-    const possibleBreaks = generateWordBreaks(lineSegment, options);
-    if (!possibleBreaks.length) {
-        const text = rebaseTextOffsetToAbsolute(relWordToSplit);
-        return {
+    const env_1 = { stack: [], error: void 0, hasError: false };
+    try {
+        const _x = __addDisposableResource(env_1, measurePerf('wordSplitter.split'), false);
+        counter('wordSplitter.split').inc();
+        const relWordToSplit = findNextWordText({ text: line.text, offset: offset - line.offset });
+        const hasSoftHyphen = relWordToSplit.text.includes(softHyphen);
+        const lineOffset = line.offset;
+        const requested = new Map();
+        const regExpIgnoreSegment = /^[-.+\d_eE'`\\\s\u00AD]+$/;
+        if (!relWordToSplit.text) {
+            const text = rebaseTextOffsetToAbsolute(relWordToSplit);
+            return {
+                line,
+                offset,
+                text: text,
+                words: [],
+                endOffset: text.offset + (text.length ?? text.text.length),
+            };
+        }
+        const lineSegment = {
             line,
-            offset,
-            text: text,
-            words: [{ ...text, length: text.length || text.text.length, isFound: isValidWord(text) }],
-            endOffset: text.offset + (text.length ?? text.text.length),
-        };
-    }
-    function rebaseTextOffsetToAbsolute(relText) {
-        relText.offset += lineOffset;
-        return relText;
-    }
-    function rebaseTextOffsetToRelative(relText) {
-        relText.offset -= lineOffset;
-        return relText;
-    }
-    function createTextOffsetWithValid(start, end, isFound) {
-        const word = {
-            offset: start,
-            length: end - start,
-            text: line.text.slice(start, end),
-            isFound,
+            relStart: relWordToSplit.offset,
+            relEnd: relWordToSplit.offset + relWordToSplit.text.length,
         };
-        return word;
-    }
-    function calcKey(start, end) {
-        const i = start;
-        const j = end - start;
-        const v = i + (j << 20);
-        return i < 1 << 20 && j < 1 << 11 ? v : -1;
-    }
-    function checkTextRange(start, end) {
-        const v = calcKey(start, end);
-        const f = v >= 0 ? requested.get(v) : undefined;
-        if (f)
-            return f;
-        const validated = createTextOffsetWithValid(start, end, false);
-        if (regExpIgnoreSegment.test(validated.text)) {
-            validated.isFound = true;
+        const possibleBreaks = generateWordBreaks(lineSegment, options);
+        if (!possibleBreaks.length) {
+            const text = rebaseTextOffsetToAbsolute(relWordToSplit);
+            return {
+                line,
+                offset,
+                text: text,
+                words: [{ ...text, length: text.length || text.text.length, isFound: isValidWord(text) }],
+                endOffset: text.offset + (text.length ?? text.text.length),
+            };
+        }
+        function rebaseTextOffsetToAbsolute(relText) {
+            relText.offset += lineOffset;
+            return relText;
+        }
+        function rebaseTextOffsetToRelative(relText) {
+            relText.offset -= lineOffset;
+            return relText;
+        }
+        function createTextOffsetWithValid(start, end, isFound) {
+            const word = {
+                offset: start,
+                length: end - start,
+                text: line.text.slice(start, end),
+                isFound,
+            };
+            return word;
+        }
+        function calcKey(start, end) {
+            const i = start;
+            const j = end - start;
+            const v = i + (j << 20);
+            return i < 1 << 20 && j < 1 << 11 ? v : -1;
+        }
+        function checkTextRange(start, end) {
+            const v = calcKey(start, end);
+            const f = v >= 0 ? requested.get(v) : undefined;
+            if (f)
+                return f;
+            const validated = createTextOffsetWithValid(start, end, false);
+            if (regExpIgnoreSegment.test(validated.text)) {
+                validated.isFound = true;
+                if (v >= 0) {
+                    requested.set(v, validated);
+                }
+                return validated;
+            }
+            rebaseTextOffsetToAbsolute(validated);
+            validated.isFound = isValidWord(validated);
+            if (hasSoftHyphen && !validated.isFound && validated.text.includes(softHyphen)) {
+                removeSoftHyphen(validated);
+                validated.isFound = isValidWord(validated);
+            }
+            rebaseTextOffsetToRelative(validated);
             if (v >= 0) {
                 requested.set(v, validated);
             }
             return validated;
         }
-        rebaseTextOffsetToAbsolute(validated);
-        validated.isFound = isValidWord(validated);
-        if (hasSoftHyphen && !validated.isFound && validated.text.includes(softHyphen)) {
-            removeSoftHyphen(validated);
-            validated.isFound = isValidWord(validated);
-        }
-        rebaseTextOffsetToRelative(validated);
-        if (v >= 0) {
-            requested.set(v, validated);
-        }
-        return validated;
+        // Add a dummy break at the end to avoid needing to check for last break.
+        possibleBreaks.push({
+            offset: lineSegment.relEnd,
+            breaks: [ignoreBreak],
+        });
+        const result = {
+            line,
+            offset,
+            text: rebaseTextOffsetToAbsolute(relWordToSplit),
+            words: splitIntoWords(lineSegment, possibleBreaks, checkTextRange).map(rebaseTextOffsetToAbsolute),
+            endOffset: lineOffset + lineSegment.relEnd,
+        };
+        return result;
+    }
+    catch (e_1) {
+        env_1.error = e_1;
+        env_1.hasError = true;
+    }
+    finally {
+        __disposeResources(env_1);
     }
-    // Add a dummy break at the end to avoid needing to check for last break.
-    possibleBreaks.push({
-        offset: lineSegment.relEnd,
-        breaks: [ignoreBreak],
-    });
-    const result = {
-        line,
-        offset,
-        text: rebaseTextOffsetToAbsolute(relWordToSplit),
-        words: splitIntoWords(lineSegment, possibleBreaks, checkTextRange).map(rebaseTextOffsetToAbsolute),
-        endOffset: lineOffset + lineSegment.relEnd,
-    };
-    return result;
 }
 function removeSoftHyphen(word) {
     word.text = word.text.replaceAll(softHyphen, '');
@@ -132,7 +198,6 @@ function findNextWordText({ text, offset }) {
 }
 function splitIntoWords(lineSeg, breaks, checkTextRange) {
     const maxIndex = lineSeg.relEnd;
-    const maxAttempts = 1000;
     /**
      * Map of known paths by their starting index in the text.
      * This helps in quickly finding previously computed paths to avoid redundant calculations.
@@ -182,6 +247,7 @@ function splitIntoWords(lineSeg, breaks, checkTextRange) {
      * @param currentCost - current cost accrued
      */
     function makeCandidates(p, i, bi, bs, currentCost) {
+        counter('wordSplitter.makeCandidates').inc();
         const len = maxIndex;
         const nBi = findNearestBreakIndex(i, bi);
         const nBs = findNearestBreakIndex(i, bs);
@@ -195,6 +261,7 @@ function splitIntoWords(lineSeg, breaks, checkTextRange) {
         }
         const br = breaks[bi];
         function calcBreakCost(bp) {
+            counter('wordSplitter.calcBreakCost').inc();
             if (bp.length < 2) {
                 // We are skipping this break pair.
                 return { p, i, j: len, bi, bs, bp, c: currentCost, ec: currentCost + len - i, text: undefined };

package/package.json

@@ -4,7 +4,7 @@
     "access": "public",
     "provenance": true
   },
-  "version": "10.2.1",
+  "version": "10.3.0-alpha.0",
   "description": "A library of useful functions used across various cspell tools.",
   "type": "module",
   "sideEffects": false,
@@ -75,22 +75,22 @@
   },
   "homepage": "https://github.com/streetsidesoftware/cspell/tree/main/packages/cspell-lib#readme",
   "dependencies": {
-    "@cspell/cspell-bundled-dicts": "10.2.1",
-    "@cspell/cspell-performance-monitor": "10.2.1",
-    "@cspell/cspell-pipe": "10.2.1",
-    "@cspell/cspell-resolver": "10.2.1",
-    "@cspell/cspell-types": "10.2.1",
-    "@cspell/dynamic-import": "10.2.1",
-    "@cspell/filetypes": "10.2.1",
-    "@cspell/rpc": "10.2.1",
-    "@cspell/strong-weak-map": "10.2.1",
-    "@cspell/url": "10.2.1",
-    "cspell-config-lib": "10.2.1",
-    "cspell-dictionary": "10.2.1",
-    "cspell-glob": "10.2.1",
-    "cspell-grammar": "10.2.1",
-    "cspell-io": "10.2.1",
-    "cspell-trie-lib": "10.2.1",
+    "@cspell/cspell-bundled-dicts": "10.3.0-alpha.0",
+    "@cspell/cspell-performance-monitor": "10.3.0-alpha.0",
+    "@cspell/cspell-pipe": "10.3.0-alpha.0",
+    "@cspell/cspell-resolver": "10.3.0-alpha.0",
+    "@cspell/cspell-types": "10.3.0-alpha.0",
+    "@cspell/dynamic-import": "10.3.0-alpha.0",
+    "@cspell/filetypes": "10.3.0-alpha.0",
+    "@cspell/rpc": "10.3.0-alpha.0",
+    "@cspell/strong-weak-map": "10.3.0-alpha.0",
+    "@cspell/url": "10.3.0-alpha.0",
+    "cspell-config-lib": "10.3.0-alpha.0",
+    "cspell-dictionary": "10.3.0-alpha.0",
+    "cspell-glob": "10.3.0-alpha.0",
+    "cspell-grammar": "10.3.0-alpha.0",
+    "cspell-io": "10.3.0-alpha.0",
+    "cspell-trie-lib": "10.3.0-alpha.0",
     "env-paths": "^4.0.0",
     "gensequence": "^8.0.8",
     "import-fresh": "^4.0.0",
@@ -103,7 +103,7 @@
     "node": ">=22.18.0"
   },
   "devDependencies": {
-    "@cspell/cspell-tools": "10.2.1",
+    "@cspell/cspell-tools": "10.3.0-alpha.0",
     "@cspell/dict-cpp": "^7.1.0",
     "@cspell/dict-csharp": "^4.0.8",
     "@cspell/dict-css": "^4.1.2",
@@ -123,5 +123,5 @@
     "lorem-ipsum": "^3.0.0",
     "perf-insight": "^2.0.1"
   },
-  "gitHead": "67a64ad0f3f4dcb681f9b9deac60c9758619ffbc"
+  "gitHead": "39297075c7a4946feb1a4f0a509e329cfb0b4f11"
 }