cspell-lib 8.13.0 → 8.13.2

package/dist/{esm → lib}/Settings/Controller/configLoader/configLoader.js

@@ -4,10 +4,10 @@ import { fileURLToPath, pathToFileURL } from 'node:url';
 import { createReaderWriter, CSpellConfigFileInMemory } from 'cspell-config-lib';
 import { isUrlLike, toFileURL } from 'cspell-io';
 import { URI, Utils as UriUtils } from 'vscode-uri';
-import { srcDirectory } from '../../../../lib-cjs/index.cjs';
 import { onClearCache } from '../../../events/index.js';
 import { getVirtualFS } from '../../../fileSystem.js';
 import { createCSpellSettingsInternal as csi } from '../../../Models/CSpellSettingsInternalDef.js';
+import { srcDirectory } from '../../../pkg-info.mjs';
 import { autoResolve, AutoResolveCache, autoResolveWeak } from '../../../util/AutoResolve.js';
 import { logError, logWarning } from '../../../util/logger.js';
 import { FileResolver } from '../../../util/resolveFile.js';

package/dist/{esm → lib}/Settings/DefaultSettings.js

@@ -1,7 +1,7 @@
 import { parsers } from 'cspell-grammar';
-import { srcDirectory } from '../../lib-cjs/index.cjs';
 import { createCSpellSettingsInternal } from '../Models/CSpellSettingsInternalDef.js';
 import { PatternRegExp } from '../Models/PatternRegExp.js';
+import { srcDirectory } from '../pkg-info.mjs';
 import { resolveFile } from '../util/resolveFile.js';
 import { defaultConfigFileModuleRef } from './constants.js';
 import { readSettings } from './Controller/configLoader/index.js';

package/dist/{lib-cjs/pkg-info.d.cts → lib/pkg-info.d.mts}

@@ -1,2 +1,2 @@
 export declare const srcDirectory: string;
-//# sourceMappingURL=pkg-info.d.cts.map
+//# sourceMappingURL=pkg-info.d.mts.map

package/dist/lib/pkg-info.mjs

@@ -0,0 +1,15 @@
+import { fileURLToPath } from 'node:url';
+/**
+ * This is the url of the current file, but it might be undefined if the environment does not support it.
+ */
+const url = import.meta.url;
+function calcSrcDirectory() {
+    try {
+        return __dirname;
+    }
+    catch {
+        return url ? fileURLToPath(new URL('./', url)) : process.cwd();
+    }
+}
+export const srcDirectory = calcSrcDirectory();
+//# sourceMappingURL=pkg-info.mjs.map

package/dist/{esm → lib}/textValidation/ValidationTypes.d.ts

@@ -28,7 +28,9 @@ export interface WordRangeAcc {
 export type ValidationIssueRO = Readonly<ValidationIssue>;
 export type LineValidatorFn = (line: LineSegment) => Iterable<ValidationIssue>;
 export interface LineSegment {
+    /** A line from the document, the offset is relative to the beginning of the document. */
     line: TextOffsetRO;
+    /** A segment of text from the line, the offset is relative to the beginning of the document. */
     segment: TextOffsetRO;
 }
 export interface MappedTextValidationResult extends MappedText {

package/dist/lib/textValidation/isWordValid.d.ts

@@ -0,0 +1,11 @@
+import type { TextOffsetRO } from './ValidationTypes.js';
+interface Dict {
+    has(word: string): boolean;
+}
+declare function hasWordCheck(dict: Dict, word: string): boolean;
+export declare function isWordValidWithEscapeRetry(dict: Dict, wo: TextOffsetRO, line: TextOffsetRO): boolean;
+export declare const __testing__: {
+    hasWordCheck: typeof hasWordCheck;
+};
+export {};
+//# sourceMappingURL=isWordValid.d.ts.map

package/dist/lib/textValidation/lineValidatorFactory.js

@@ -0,0 +1,271 @@
+import assert from 'node:assert';
+import { opConcatMap, opFilter, pipe } from '@cspell/cspell-pipe/sync';
+import { createCachingDictionary } from 'cspell-dictionary';
+import * as RxPat from '../Settings/RegExpPatterns.js';
+import { extractPossibleWordsFromTextOffset, extractText, extractWordsFromCodeTextOffset, extractWordsFromTextOffset, } from '../util/text.js';
+import { split } from '../util/wordSplitter.js';
+import { defaultMinWordLength } from './defaultConstants.js';
+import { isWordValidWithEscapeRetry } from './isWordValid.js';
+import { mapRangeBackToOriginalPos } from './parsedText.js';
+export function lineValidatorFactory(sDict, options) {
+    const { minWordLength = defaultMinWordLength, flagWords = [], allowCompoundWords = false, ignoreCase = true, } = options;
+    const hasWordOptions = {
+        ignoreCase,
+        useCompounds: allowCompoundWords || undefined, // let the dictionaries decide on useCompounds if allow is false
+    };
+    const dictCol = createCachingDictionary(sDict, hasWordOptions);
+    const knownWords = new Map();
+    const setOfFlagWords = new Set(flagWords);
+    const setOfKnownIssues = new Map();
+    const setOfKnownSuccessfulWords = new Set();
+    const rememberFilter = (fn) => (v) => {
+        const keep = fn(v);
+        if (!keep) {
+            setOfKnownSuccessfulWords.add(v.text);
+        }
+        return keep;
+    };
+    const filterAlreadyChecked = (wo) => {
+        return !setOfKnownSuccessfulWords.has(wo.text);
+    };
+    const hasDict = {
+        has(word) {
+            const info = getWordInfo(word);
+            if (info.isFound !== undefined)
+                return info.isFound;
+            if (info.isFlagged)
+                return true;
+            if (info.isFlagged)
+                return false;
+            info.isFound = dictCol.has(word);
+            return info.isFound;
+        },
+    };
+    function calcIgnored(info) {
+        info.isIgnored ??= dictCol.isNoSuggestWord(info.word);
+        return info.isIgnored;
+    }
+    function calcFlagged(info) {
+        if (info.isFlagged !== undefined)
+            return info.isFlagged;
+        const word = info.word;
+        info.isFlagged =
+            (setOfFlagWords.has(word) || setOfFlagWords.has(word.toLowerCase()) || dictCol.isForbidden(word)) &&
+                !calcIgnored(info);
+        return info.isFlagged;
+    }
+    function isWordIgnored(word) {
+        return calcIgnored(getWordInfo(word));
+    }
+    function getSuggestions(word) {
+        return dictCol.getPreferredSuggestions(word);
+    }
+    function isWordFlagged(wo) {
+        return calcFlagged(getWordInfo(wo.text));
+    }
+    function annotateIsFlagged(word) {
+        word.isFlagged = isWordFlagged(word);
+        return word;
+    }
+    function annotateIssue(issue) {
+        const sugs = getSuggestions(issue.text);
+        if (sugs && sugs.length) {
+            issue.suggestionsEx = sugs;
+        }
+        return issue;
+    }
+    const isFlaggedOrMinLength = (wo) => wo.text.length >= minWordLength || !!wo.isFlagged;
+    const isFlaggedOrNotFound = rememberFilter((wo) => wo.isFlagged || !wo.isFound);
+    const isNotRepeatingChar = rememberFilter((wo) => !RxPat.regExRepeatedChar.test(wo.text));
+    function checkWord(issue) {
+        const info = getWordInfo(issue.text);
+        if (info.fin) {
+            const { isFlagged: isForbidden, isFound, isIgnored } = info;
+            const isFlagged = issue.isFlagged ?? (!isIgnored && isForbidden);
+            issue.isFlagged = isFlagged;
+            issue.isFound = isFlagged ? undefined : isFound;
+            return issue;
+        }
+        const isIgnored = calcIgnored(info);
+        const isFlagged = issue.isFlagged ?? calcFlagged(info);
+        info.isFound ??= isFlagged ? false : isIgnored || isWordValidWithEscapeRetry(hasDict, issue, issue.line);
+        info.isFlagged = !!isFlagged;
+        info.fin = true;
+        issue.isFlagged = isFlagged;
+        issue.isFound = isFlagged ? undefined : info.isFound;
+        return issue;
+    }
+    const regExUpperCaseWithTrailingCommonEnglishSuffix = /^([\p{Lu}\p{M}]{2,})['’]?(?:s|ing|ies|es|ings|ize|ed|ning)$/u; // cspell:disable-line
+    const regExpIsLetter = /\p{L}/u;
+    const fn = (lineSegment) => {
+        const line = lineSegment.line;
+        function isWordTooShort(word, ignoreSuffix = false) {
+            if (word.text.length >= minWordLength)
+                return false;
+            const offset = word.offset - line.offset;
+            assert.equal(line.text.slice(offset, offset + word.text.length), word.text);
+            const prefix = [...line.text.slice(Math.max(0, offset - 2), offset)];
+            const hasLetterPrefix = !!prefix.length && regExpIsLetter.test(prefix[prefix.length - 1]);
+            if (hasLetterPrefix)
+                return false;
+            if (ignoreSuffix)
+                return true;
+            const suffix = [...line.text.slice(offset + word.text.length, offset + word.text.length + 2)];
+            const hasLetterSuffix = !!suffix.length && regExpIsLetter.test(suffix[0]);
+            return !hasLetterSuffix;
+        }
+        function splitterIsValid(word) {
+            if (setOfKnownSuccessfulWords.has(word.text))
+                return true;
+            if (isWordFlagged(word))
+                return false;
+            if (isWordValidWithEscapeRetry(hasDict, word, lineSegment.line))
+                return true;
+            if (isWordTooShort(word))
+                return true;
+            return isAllCapsWithTrailingCommonEnglishSuffixOk(word);
+        }
+        function isAllCapsWithTrailingCommonEnglishSuffixOk(tWord) {
+            if (!regExUpperCaseWithTrailingCommonEnglishSuffix.test(tWord.text))
+                return false;
+            const m = tWord.text.match(regExUpperCaseWithTrailingCommonEnglishSuffix);
+            if (!m)
+                return false;
+            const offset = tWord.offset;
+            const v = { offset, text: m[1], line };
+            const check = checkWord(v);
+            if (check.isFlagged)
+                return false;
+            if (check.isFound)
+                return true;
+            if (isWordTooShort(v, true))
+                return true;
+            return false;
+        }
+        function checkFullWord(vr) {
+            if (vr.isFlagged) {
+                return [vr];
+            }
+            // English exceptions :-(
+            if (isAllCapsWithTrailingCommonEnglishSuffixOk(vr))
+                return [];
+            const codeWordResults = [];
+            for (const wo of extractWordsFromCodeTextOffset(vr)) {
+                if (setOfKnownSuccessfulWords.has(wo.text))
+                    continue;
+                const issue = wo;
+                issue.line = vr.line;
+                issue.isFlagged = undefined;
+                issue.isFound = undefined;
+                annotateIsFlagged(issue);
+                if (!isFlaggedOrMinLength(issue))
+                    continue;
+                checkWord(issue);
+                if (!isFlaggedOrNotFound(issue) || !isNotRepeatingChar(issue))
+                    continue;
+                issue.text = extractText(lineSegment.segment, issue.offset, issue.offset + issue.text.length);
+                codeWordResults.push(issue);
+            }
+            if (!codeWordResults.length || isWordIgnored(vr.text) || checkWord(vr).isFound) {
+                rememberFilter((_) => false)(vr);
+                return [];
+            }
+            return codeWordResults;
+        }
+        function rebaseKnownIssues(possibleWord, known) {
+            const { issues } = known;
+            const adjOffset = possibleWord.offset - known.possibleWord.offset;
+            return issues.map((issue) => {
+                issue = { ...issue };
+                issue.offset += adjOffset;
+                issue.line = lineSegment.line;
+                return issue;
+            });
+        }
+        function checkPossibleWords(possibleWord) {
+            const known = setOfKnownIssues.get(possibleWord.text);
+            if (known) {
+                if (!known.issues.length)
+                    return known.issues;
+                const adjusted = rebaseKnownIssues(possibleWord, known);
+                return adjusted;
+            }
+            const issues = _checkPossibleWords(possibleWord).map(annotateIssue);
+            setOfKnownIssues.set(possibleWord.text, { possibleWord, issues });
+            return issues;
+        }
+        function _checkPossibleWords(possibleWord) {
+            if (isWordFlagged(possibleWord)) {
+                const vr = {
+                    ...possibleWord,
+                    line: lineSegment.line,
+                    isFlagged: true,
+                };
+                return [vr];
+            }
+            const mismatches = [];
+            for (const wo of extractWordsFromTextOffset(possibleWord)) {
+                if (setOfKnownSuccessfulWords.has(wo.text))
+                    continue;
+                const issue = wo;
+                issue.line = lineSegment.line;
+                annotateIsFlagged(issue);
+                if (!isFlaggedOrMinLength(issue))
+                    continue;
+                for (const w of checkFullWord(issue)) {
+                    mismatches.push(w);
+                }
+            }
+            if (mismatches.length) {
+                // Try the more expensive word splitter
+                const splitResult = split(lineSegment.segment, possibleWord.offset, splitterIsValid);
+                const nonMatching = splitResult.words
+                    .filter((w) => !w.isFound)
+                    .filter((w) => {
+                    const m = w.text.match(regExUpperCaseWithTrailingCommonEnglishSuffix);
+                    if (!m)
+                        return true;
+                    const v = checkWord({ ...w, text: m[1], line: lineSegment.line });
+                    return v.isFlagged || !v.isFound;
+                });
+                if (nonMatching.length < mismatches.length) {
+                    return nonMatching.map((w) => ({ ...w, line: lineSegment.line })).map(annotateIsFlagged);
+                }
+            }
+            return mismatches;
+        }
+        const checkedPossibleWords = pipe(extractPossibleWordsFromTextOffset(lineSegment.segment), opFilter(filterAlreadyChecked), opConcatMap(checkPossibleWords));
+        return checkedPossibleWords;
+    };
+    function getWordInfo(word) {
+        const info = knownWords.get(word);
+        if (info)
+            return info;
+        const result = { word, isFound: undefined, isFlagged: undefined, isIgnored: undefined, fin: false };
+        knownWords.set(word, result);
+        return result;
+    }
+    return { fn, dict: dictCol };
+}
+export function textValidatorFactory(dict, options) {
+    const lineValidator = lineValidatorFactory(dict, options);
+    const lineValidatorFn = lineValidator.fn;
+    function validate(pText) {
+        const { text, range: srcRange, map } = pText;
+        const srcOffset = srcRange[0];
+        const segment = { text, offset: 0 };
+        const lineSegment = { line: segment, segment };
+        function mapBackToOriginSimple(vr) {
+            const { text, offset, isFlagged, isFound, suggestionsEx } = vr;
+            const r = mapRangeBackToOriginalPos([offset, offset + text.length], map);
+            const range = [r[0] + srcOffset, r[1] + srcOffset];
+            return { text, range, isFlagged, isFound, suggestionsEx };
+        }
+        return [...lineValidatorFn(lineSegment)].map(mapBackToOriginSimple);
+    }
+    return {
+        validate,
+        lineValidator,
+    };
+}
+//# sourceMappingURL=lineValidatorFactory.js.map

package/dist/{esm → lib}/util/TextRange.d.ts

@@ -13,20 +13,15 @@ export interface MatchRangeOptionalText extends MatchRange {
     text?: string;
 }
 export declare function findMatchingRanges(pattern: RegExp, text: string): MatchRangeOptionalText[];
-export declare function unionRanges(ranges: MatchRange[]): MatchRange[];
+export declare function unionRanges(ranges: MatchRange[]): SortedMatchRangeArray;
 export declare function findMatchingRangesForPatterns(patterns: RegExp[], text: string): MatchRange[];
 /**
  * Create a new set of positions that have the excluded position ranges removed.
  */
 export declare function excludeRanges(includeRanges: MatchRange[], excludeRanges: MatchRange[]): MatchRange[];
 export declare function extractRangeText(text: string, ranges: MatchRange[]): MatchRangeWithText[];
-declare const SymSortedMatchRangeArray: unique symbol;
-interface SortedMatchRangeArray extends Array<MatchRange> {
-    [SymSortedMatchRangeArray]: true;
+interface SortedMatchRangeArray {
+    values: MatchRange[];
 }
-declare function makeSortedMatchRangeArray(sortedValues: MatchRange[]): SortedMatchRangeArray;
-export declare const __testing__: {
-    makeSortedMatchRangeArray: typeof makeSortedMatchRangeArray;
-};
 export {};
 //# sourceMappingURL=TextRange.d.ts.map

package/dist/{esm → lib}/util/TextRange.js

@@ -1,4 +1,3 @@
-import * as GS from 'gensequence';
 function toMatchRangeWithText(m) {
     const index = m.index || 0;
     const _text = m[0];
@@ -25,16 +24,17 @@ function compareRanges(a, b) {
     return a.startPos - b.startPos || a.endPos - b.endPos;
 }
 export function unionRanges(ranges) {
-    return makeSortedMatchRangeArray([..._unionRanges(ranges)]);
-}
-function* _unionRanges(ranges) {
     const sortedRanges = sortMatchRangeArray(ranges);
-    if (!sortedRanges.length)
-        return;
-    let { startPos, endPos } = sortedRanges[0];
-    for (const r of ranges) {
+    ranges = sortedRanges.values;
+    if (!ranges.length)
+        return sortedRanges;
+    let i = 0;
+    let j = 0;
+    let { startPos, endPos } = ranges[i++];
+    for (; i < ranges.length; ++i) {
+        const r = ranges[i];
         if (r.startPos > endPos) {
-            yield { startPos, endPos };
+            ranges[j++] = { startPos, endPos };
             startPos = r.startPos;
             endPos = r.endPos;
             continue;
@@ -42,26 +42,32 @@ function* _unionRanges(ranges) {
         endPos = Math.max(endPos, r.endPos);
     }
     if (startPos < endPos) {
-        yield { startPos, endPos };
+        ranges[j++] = { startPos, endPos };
     }
+    ranges.length = j;
+    return sortedRanges;
 }
 export function findMatchingRangesForPatterns(patterns, text) {
-    const matchedPatterns = GS.genSequence(patterns).concatMap((pattern) => findMatchingRanges(pattern, text));
-    return unionRanges(matchedPatterns.toArray());
+    const nested = patterns.map((pattern) => findMatchingRanges(pattern, text));
+    return unionRanges(flatten(nested)).values;
 }
 /**
  * Create a new set of positions that have the excluded position ranges removed.
  */
 export function excludeRanges(includeRanges, excludeRanges) {
-    return [..._excludeRanges(sortMatchRangeArray(includeRanges), sortMatchRangeArray(excludeRanges))];
+    return _excludeRanges(sortMatchRangeArray(includeRanges), sortMatchRangeArray(excludeRanges));
 }
-function* _excludeRanges(includeRanges, excludeRanges) {
+function _excludeRanges(sortedIncludeRanges, sortedExcludeRanges) {
+    const includeRanges = sortedIncludeRanges.values;
+    const excludeRanges = sortedExcludeRanges.values;
     if (!includeRanges.length)
-        return;
+        return includeRanges;
     if (!excludeRanges.length) {
-        yield* includeRanges;
-        return;
+        return includeRanges;
     }
+    const ranges = [];
+    ranges.length = includeRanges.length + excludeRanges.length + 1;
+    let i = 0;
     let exIndex = 0;
     const limit = excludeRanges.length;
     for (const incRange of includeRanges) {
@@ -74,16 +80,18 @@ function* _excludeRanges(includeRanges, excludeRanges) {
             if (ex.endPos <= startPos)
                 continue;
             if (ex.startPos > startPos) {
-                yield { startPos, endPos: ex.startPos };
+                ranges[i++] = { startPos, endPos: ex.startPos };
             }
             startPos = ex.endPos;
             if (startPos >= endPos)
                 break;
         }
         if (startPos < endPos) {
-            yield { startPos, endPos };
+            ranges[i++] = { startPos, endPos };
         }
     }
+    ranges.length = i;
+    return ranges;
 }
 export function extractRangeText(text, ranges) {
     return ranges.map(({ startPos, endPos }) => ({
@@ -92,22 +100,23 @@ export function extractRangeText(text, ranges) {
         text: text.slice(startPos, endPos),
     }));
 }
-const SymSortedMatchRangeArray = Symbol('SortedMatchRangeArray');
 function sortMatchRangeArray(values) {
-    if (isSortedMatchRangeArray(values))
-        return values;
-    return makeSortedMatchRangeArray(values.sort(compareRanges));
-}
-function isSortedMatchRangeArray(a) {
-    return a[SymSortedMatchRangeArray] === true;
+    values.sort(compareRanges);
+    return { values };
 }
-function makeSortedMatchRangeArray(sortedValues) {
-    const sorted = sortedValues;
-    sorted[SymSortedMatchRangeArray] = true;
-    Object.freeze(sorted);
-    return sorted;
+function flatten(data) {
+    let size = 0;
+    for (let i = data.length - 1; i >= 0; --i) {
+        size += data[i].length;
+    }
+    const result = new Array(size);
+    let k = 0;
+    for (let i = 0; i < data.length; ++i) {
+        const d = data[i];
+        for (let j = 0; j < d.length; ++j) {
+            result[k++] = d[j];
+        }
+    }
+    return result;
 }
-export const __testing__ = {
-    makeSortedMatchRangeArray,
-};
 //# sourceMappingURL=TextRange.js.map

package/dist/{esm → lib}/util/resolveFile.js

@@ -6,8 +6,8 @@ import { fileURLToPath } from 'node:url';
 import { resolveGlobal } from '@cspell/cspell-resolver';
 import { importResolveModuleName } from '@cspell/dynamic-import';
 import resolveFrom from 'resolve-from';
-import { srcDirectory } from '../../lib-cjs/pkg-info.cjs';
 import { getFileSystem } from '../fileSystem.js';
+import { srcDirectory } from '../pkg-info.mjs';
 import { envToTemplateVars, replaceTemplate } from './templates.js';
 import { fileURLOrPathToPath, isDataURL, isFileURL, isURLLike, resolveFileWithURL, toFilePathOrHref, toFileUrl, toURL, } from './url.js';
 const regExpStartsWidthNodeModules = /^node_modules[/\\]/;

package/dist/{esm → lib}/util/text.d.ts

@@ -11,7 +11,7 @@ export declare function splitCamelCaseWord(word: string): string[];
  */
 export declare function match(reg: RegExp, text: string): Iterable<RegExpExecArray>;
 export declare function matchStringToTextOffset(reg: RegExp, text: string): Iterable<TextOffset>;
-export declare function matchToTextOffset(reg: RegExp, text: TextOffset): Iterable<TextOffset>;
+export declare function matchToTextOffset(reg: RegExp, t: TextOffset): Iterable<TextOffset>;
 export declare function extractLinesOfText(text: string): Iterable<TextOffset>;
 /**
  * Extract out whole words from a string of text.
@@ -21,6 +21,11 @@ export declare function extractWordsFromText(text: string): Iterable<TextOffset>
  * Extract out whole words from a string of text.
  */
 export declare function extractWordsFromTextOffset(text: TextOffset): Iterable<TextOffset>;
+/**
+ * Remove Hiragana, Han, Katakana, Hangul characters from the text.
+ * @param text
+ * @returns the text with the characters removed.
+ */
 export declare function cleanText(text: string): string;
 export declare function cleanTextOffset(text: TextOffset): TextOffset;
 /**

package/dist/{esm → lib}/util/text.js

@@ -1,6 +1,6 @@
 import { opConcatMap, opMap, pipe } from '@cspell/cspell-pipe/sync';
 import { binarySearch } from './search.js';
-import { regExAccents, regExAllLower, regExAllUpper, regExFirstUpper, regExIgnoreCharacters, regExSplitWords, regExSplitWords2, regExUpperSOrIng, regExWords, regExWordsAndDigits, } from './textRegex.js';
+import { regExAccents, regExAllLower, regExAllUpper, regExFirstUpper, regExIgnoreCharacters, regExpSplitWordBreaks, regExWords, regExWordsAndDigits, } from './textRegex.js';
 import { toUri } from './Uri.js';
 import { scanMap } from './util.js';
 export { stringToRegExp } from './textRegex.js';
@@ -15,11 +15,7 @@ export function splitCamelCaseWordWithOffset(wo) {
  * Split camelCase words into an array of strings.
  */
 export function splitCamelCaseWord(word) {
-    const wPrime = word.replace(regExUpperSOrIng, (s) => s[0] + s.slice(1).toLowerCase());
-    const separator = '_<^*_*^>_';
-    const pass1 = wPrime.replace(regExSplitWords, '$1' + separator + '$2');
-    const pass2 = pass1.replace(regExSplitWords2, '$1' + separator + '$2');
-    return pass2.split(separator);
+    return word.split(regExpSplitWordBreaks);
 }
 /**
  * This function lets you iterate over regular expression matches.
@@ -33,11 +29,11 @@ export function match(reg, text) {
 export function matchStringToTextOffset(reg, text) {
     return matchToTextOffset(reg, { text, offset: 0 });
 }
-export function matchToTextOffset(reg, text) {
-    const textOffset = text;
-    const fnOffsetMap = offsetMap(textOffset.offset);
-    textOffset.text.matchAll(reg);
-    return pipe(match(reg, textOffset.text), opMap((m) => fnOffsetMap({ text: m[0], offset: m.index || 0 })));
+export function matchToTextOffset(reg, t) {
+    const text = t.text;
+    const offset = t.offset;
+    // return opMap((m: RegExpExecArray) => ({ text: m[0], offset: offset + m.index }))(match(reg, text));
+    return pipe(match(reg, text), opMap((m) => ({ text: m[0], offset: offset + m.index })));
 }
 export function* extractLinesOfText(text) {
     let i = 0;
@@ -61,11 +57,23 @@ export function extractWordsFromTextOffset(text) {
     const reg = new RegExp(regExWords);
     return matchToTextOffset(reg, cleanTextOffset(text));
 }
+/**
+ * Remove Hiragana, Han, Katakana, Hangul characters from the text.
+ * @param text
+ * @returns the text with the characters removed.
+ */
 export function cleanText(text) {
+    regExIgnoreCharacters.lastIndex = 0;
+    if (!regExIgnoreCharacters.test(text))
+        return text;
     text = text.replace(regExIgnoreCharacters, (match) => ' '.repeat(match.length));
     return text;
 }
 export function cleanTextOffset(text) {
+    // Do not make a new object if the text is already clean.
+    regExIgnoreCharacters.lastIndex = 0;
+    if (!regExIgnoreCharacters.test(text.text))
+        return text;
     return {
         text: cleanText(text.text),
         offset: text.offset,
@@ -135,9 +143,6 @@ export function extractText(textOffset, startPos, endPos) {
     const b = Math.max(endPos - orig, 0);
     return text.slice(a, b);
 }
-function offsetMap(offset) {
-    return (xo) => ({ ...xo, offset: xo.offset + offset });
-}
 export function calculateTextDocumentOffsets(uri, doc, wordOffsets) {
     const lines = [
         -1,

package/dist/{esm → lib}/util/textRegex.d.ts

@@ -1,6 +1,8 @@
 export declare const regExUpperSOrIng: RegExp;
 export declare const regExSplitWords: RegExp;
 export declare const regExSplitWords2: RegExp;
+export declare const regExpSplitWordBreaks: RegExp;
+export declare const regExpAllPossibleWordBreaks: RegExp;
 export declare const regExWords: RegExp;
 export declare const regExWordsAndDigits: RegExp;
 export declare const regExIgnoreCharacters: RegExp;

package/dist/{esm → lib}/util/textRegex.js

@@ -1,7 +1,9 @@
 // cspell:ignore ings ning gimuy anrvtbf gimuxy
-export const regExUpperSOrIng = /([\p{Lu}\p{M}]+\\?['’]?(?:s|ing|ies|es|ings|ed|ning))(?!\p{Ll})/gu;
+export const regExUpperSOrIng = /([\p{Lu}\p{M}]+(?:\\?['’])?(?:s|ing|ies|es|ings|ed|ning))(?!\p{Ll})/gu;
 export const regExSplitWords = /(\p{Ll}\p{M}?)(\p{Lu})/gu;
 export const regExSplitWords2 = /(\p{Lu}\p{M}?)(\p{Lu}\p{M}?\p{Ll})/gu;
+export const regExpSplitWordBreaks = /(?<=\p{Ll}\p{M}?)(?=\p{Lu})|(?<=\p{Lu}\p{M}?)(?=\p{Lu}\p{M}?\p{Ll})(?!\p{Lu}\p{M}?(?:s|ing|ies|es|ings|ed|ning)(?!\p{Ll}))/gu;
+export const regExpAllPossibleWordBreaks = /(?<=\p{Ll}\p{M}?)(?=\p{Lu})|(?<=\p{Lu}\p{M}?)(?=\p{Lu}\p{M}?\p{Ll})|(?<=\p{Lu}\p{M}?\p{Lu}\p{M}?)(?=\p{Ll})|(?<=\p{L}\p{M}?)(?=\P{L})|(?<=\P{L})(?=\p{L})/gu;
 export const regExWords = /\p{L}\p{M}?(?:(?:\\?['’])?\p{L}\p{M}?)*/gu;
 // Words can be made of letters, numbers, period, underscore, dash, plus, and single quote
 export const regExWordsAndDigits = /[\p{L}\w'’`.+-](?:(?:\\(?=[']))?[\p{L}\p{M}\w'’`.+-])*/gu;

package/dist/{esm → lib}/util/url.js

@@ -1,7 +1,7 @@
 import path from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { toFilePathOrHref, toFileURL } from '@cspell/url';
-import { srcDirectory } from '../../lib-cjs/pkg-info.cjs';
+import { srcDirectory } from '../pkg-info.mjs';
 export { addTrailingSlash, isDataURL, isFileURL, isUrlLike as isURLLike, toFileURL as resolveFileWithURL, toFileDirURL, toFilePathOrHref, toURL, } from '@cspell/url';
 /**
  * This is a URL that can be used for searching for modules.

package/dist/{esm → lib}/util/wordSplitter.d.ts

@@ -24,7 +24,7 @@ export interface SplitOptions extends WordBreakOptions {
 }
 export declare function split(line: TextOffset, offset: number, isValidWord: IsValidWordFn, options?: SplitOptions): SplitResult;
 declare function findNextWordText({ text, offset }: TextOffset): TextOffset;
-type BreakPairs = readonly number[];
+type BreakPairs = readonly [number, number];
 interface PossibleWordBreak {
     /** offset from the start of the string */
     offset: number;