cspell-lib 10.2.0 → 10.2.2

package/dist/lib/Transform/IntlSegmentTextTransformer.d.ts

@@ -1,5 +1,6 @@
 import type { MappedText } from '@cspell/cspell-types';
 import type { TextTransformer } from './Transformer.js';
+export declare const softHyphen = "\u00AD";
 export declare class IntlSegmentTextTransformer implements TextTransformer {
     #private;
     constructor(locale: string | string[]);

package/dist/lib/Transform/IntlSegmentTextTransformer.js

@@ -1,6 +1,7 @@
 import { normalizeLocaleIntl } from '../Settings/LanguageSettings.js';
 import { applyEditsToMappedText } from './TextMapEdit.js';
 import { toMappedText } from './Transformer.js';
+export const softHyphen = '\u00AD';
 const regExSplitWordCharacters = /(?<=[\p{L}\p{M}])[\p{L}\p{M}]/uy;
 export class IntlSegmentTextTransformer {
     #locale;
@@ -23,11 +24,11 @@ function segmentText(mText, segmenter) {
     let lastIndex = -1;
     const reg = new RegExp(regExSplitWordCharacters);
     const text = mText.text;
-    for (const { segment, index } of segmenter.segment(mText.text)) {
+    for (const { segment, index, isWordLike } of segmenter.segment(mText.text)) {
         const endIndex = index + segment.length;
         reg.lastIndex = index;
-        if (index === lastIndex && reg.test(text)) {
-            const edit = { text: ' ' + segment, range: [index, endIndex] };
+        if (isWordLike && index === lastIndex && reg.test(text)) {
+            const edit = { text: softHyphen + segment, range: [index, endIndex] };
             edits.push(edit);
         }
         lastIndex = endIndex;

package/dist/lib/textValidation/isRandomString.js

@@ -1,3 +1,4 @@
+import { softHyphen } from '../util/wordSplitter/index.js';
 const maxRadio = 0.5;
 /**
  * Try to detect if a string is a random string of characters or is it camel case / snake case words.
@@ -5,6 +6,9 @@ const maxRadio = 0.5;
  * @returns true if the string is considered random;
  */
 export function isRandomString(s, maxNoiseToLengthRatio = maxRadio) {
+    // Soft hyphen indicates a word break, so it's likely not a random string.
+    if (s.includes(softHyphen))
+        return false;
     return scoreRandomString(s) >= maxNoiseToLengthRatio;
 }
 /**
@@ -28,9 +32,6 @@ function sumWordLengths(s) {
     }
     return total;
 }
-function _nonLetterRatio(s) {
-    return s.replaceAll(/\p{L}/gu, '').length / s.length;
-}
 export function categorizeString(s) {
     const n = s
         .replaceAll(/\d+/g, '0')

package/dist/lib/textValidation/lineValidatorFactory.js

@@ -7,7 +7,7 @@ import { mapRangeBackToOriginalPos } from '../Transform/index.js';
 import { autoResolve } from '../util/AutoResolve.js';
 import { extractPossibleWordsFromTextOffset, extractText, extractWordsFromTextOffset, splitWordWithOffset, } from '../util/text.js';
 import { regExpCamelCaseWordBreaksWithEnglishSuffix } from '../util/textRegex.js';
-import { split } from '../util/wordSplitter.js';
+import { split } from '../util/wordSplitter/index.js';
 import { defaultMinWordLength } from './defaultConstants.js';
 import { extractHexSequences, isRandomString } from './isRandomString.js';
 import { isWordValidWithEscapeRetry } from './isWordValid.js';
@@ -327,7 +327,12 @@ export function lineValidatorFactory(sDict, options) {
                     const v = checkWord({ ...w, text: m[1], line: lineSegment.line });
                     return v.isFlagged || !v.isFound;
                 });
-                const filtered = filterExcludedTextOffsets(nonMatching.map((w) => ({ ...w, line: lineSegment.line })).map(annotateIsFlagged), hexSequences);
+                // Ignore 1 or 2 letter segments.
+                const minWordLen = Math.min(3, minWordLength);
+                const filtered = filterExcludedTextOffsets(nonMatching
+                    .map((w) => ({ ...w, line: lineSegment.line }))
+                    .map(annotateIsFlagged)
+                    .filter((w) => w.isFlagged || w.text.length >= minWordLen), hexSequences);
                 if (filtered.length < mismatches.length) {
                     return filtered;
                 }

package/dist/lib/textValidation/traceWord.js

@@ -2,7 +2,7 @@ import { getSources } from '../Settings/index.js';
 import { createCollection, getInlineConfigDictionaries, mapSpecialDictionaryNamesToSettings, } from '../SpellingDictionary/index.js';
 import { toFileUrl } from '../util/url.js';
 import { uniqueFn } from '../util/util.js';
-import { split } from '../util/wordSplitter.js';
+import { split } from '../util/wordSplitter/index.js';
 export function traceWord(word, dictCollection, config) {
     const opts = {
         ignoreCase: config.ignoreCase ?? true,

package/dist/lib/util/textRegex.js

@@ -7,7 +7,7 @@ export const regExpCamelCaseWordBreaks_NOT_USED = /(?<=\p{Ll}\p{M}?)(?=\p{Lu})|(
 export const regExpAllPossibleWordBreaks_NOT_USED = /(?<=\p{Ll}\p{M}?)(?=\p{Lu})|(?<=\p{Lu}\p{M}?)(?=\p{Lu}\p{M}?\p{Ll})|(?<=\p{Lu}\p{M}?\p{Lu}\p{M}?)(?=\p{Ll})|(?<=\p{L}\p{M}?)(?=\P{L})|(?<=\P{L})(?=\p{L})/gu;
 export const regExWords = /\p{L}\p{M}?(?:(?:\\?['’])?\p{L}\p{M}?)*/gu;
 // Words can be made of letters, numbers, period, underscore, dash, plus, and single quote
-export const regExWordsAndDigits = /[\p{L}\w'’`.+-](?:(?:\\(?=[']))?[\p{L}\p{M}\w'’`.+-])*/gu;
+export const regExWordsAndDigits = /[\p{L}\w'’`.+\u00AD-](?:(?:\\(?=[']))?[\p{L}\p{M}\w'’`.+\u00AD-])*/gu;
 export const regExIgnoreCharacters = /[\p{sc=Hiragana}\p{sc=Han}\p{sc=Katakana}\u30A0-\u30FF\p{sc=Hangul}]/gu;
 export const regExFirstUpper = /^\p{Lu}\p{M}?\p{Ll}+$/u;
 export const regExAllUpper = /^(?:\p{Lu}\p{M}?)+$/u;

package/dist/lib/util/wordSplitter/generateWordBreaks.d.ts

@@ -0,0 +1,32 @@
+import type { TextOffset } from '@cspell/cspell-types';
+export declare const softHyphen = "\u00AD";
+export interface LineSegment {
+    line: TextOffset;
+    /**
+     * The starting offset within line.text.
+     * The absolute offset can be obtained by adding line.offset.
+     */
+    relStart: number;
+    /**
+     * The ending offset within line.text.
+     * The absolute offset can be obtained by adding line.offset.
+     */
+    relEnd: number;
+}
+export type BreakPairs = readonly [number, number];
+export interface PossibleWordBreak {
+    /** offset from the start of the string */
+    offset: number;
+    /**
+     * break pairs (start, end)
+     * (the characters between the start and end are removed)
+     * With a pure break, start === end.
+     */
+    breaks: BreakPairs[];
+}
+export type SortedBreaks = PossibleWordBreak[];
+export interface WordBreakOptions {
+    optionalWordBreakCharacters?: string;
+}
+export declare function generateWordBreaks(line: LineSegment, options: WordBreakOptions): SortedBreaks;
+//# sourceMappingURL=generateWordBreaks.d.ts.map

package/dist/lib/util/wordSplitter/generateWordBreaks.js

@@ -0,0 +1,120 @@
+import { escapeRegEx } from '../regexHelper.js';
+import { regExDanglingQuote, regExEscapeCharacters, regExPossibleWordBreaks, regExSplitWords, regExSplitWords2, regExTrailingEndings, } from '../textRegex.js';
+export const softHyphen = '\u00AD';
+const ignoreBreak = Object.freeze([]);
+export function generateWordBreaks(line, options) {
+    const camelBreaks = genWordBreakCamel(line);
+    const symbolBreaks = genSymbolBreaks(line);
+    const optionalBreaks = genOptionalWordBreaks(line, options.optionalWordBreakCharacters);
+    return mergeSortedBreaks(...camelBreaks, ...symbolBreaks, ...optionalBreaks);
+}
+function offsetRegEx(reg, offset) {
+    const r = new RegExp(reg);
+    r.lastIndex = offset;
+    return r;
+}
+function genWordBreakCamel(line) {
+    const breaksCamel1 = [];
+    const text = line.line.text.slice(0, line.relEnd);
+    // lower,Upper: camelCase -> camel|Case
+    for (const m of text.matchAll(offsetRegEx(regExSplitWords, line.relStart))) {
+        if (m.index === undefined)
+            break;
+        const i = m.index + m[1].length;
+        breaksCamel1.push({
+            offset: m.index,
+            breaks: [[i, i], ignoreBreak],
+        });
+    }
+    const breaksCamel2 = [];
+    // cspell:ignore ERRORC
+    // Upper,Upper,lower: ERRORCodes -> ERROR|Codes, ERRORC|odes
+    for (const m of text.matchAll(offsetRegEx(regExSplitWords2, line.relStart))) {
+        if (m.index === undefined)
+            break;
+        const i = m.index + m[1].length;
+        const j = i + m[3].length;
+        breaksCamel2.push({
+            offset: m.index,
+            breaks: [[i, i], [j, j], ignoreBreak],
+        });
+    }
+    return [breaksCamel1, breaksCamel2];
+}
+function calcBreaksForSubString(line, subStr, calcBreak) {
+    const sb = [];
+    const text = line.line.text;
+    const inc = subStr.length;
+    for (let pos = text.indexOf(subStr, line.relStart); pos >= 0 && pos < line.relEnd; pos = text.indexOf(subStr, pos + inc)) {
+        const b = calcBreak(pos, pos + inc);
+        if (b) {
+            sb.push(b);
+        }
+    }
+    return sb;
+}
+function calcBreaksForRegEx(line, reg, calcBreak) {
+    const sb = [];
+    const text = line.line.text.slice(0, line.relEnd);
+    for (const m of text.matchAll(offsetRegEx(reg, line.relStart))) {
+        const b = calcBreak(m);
+        if (b) {
+            sb.push(b);
+        }
+    }
+    return sb;
+}
+function calcOptionalWordBreak(i, j) {
+    return {
+        offset: i,
+        breaks: [
+            [i, j], // Remove the characters
+            ignoreBreak,
+        ],
+    };
+}
+function genOptionalWordBreaks(line, optionalBreakCharacters) {
+    function calcBreaks(m) {
+        const i = m.index;
+        if (i === undefined)
+            return;
+        const j = i + m[0].length;
+        return calcOptionalWordBreak(i, j);
+    }
+    const breaks = [
+        calcBreaksForSubString(line, softHyphen, calcOptionalWordBreak),
+        calcBreaksForRegEx(line, regExDanglingQuote, calcBreaks),
+        calcBreaksForRegEx(line, regExTrailingEndings, calcBreaks),
+    ];
+    if (optionalBreakCharacters) {
+        const regex = new RegExp(`[${escapeRegEx(optionalBreakCharacters)}]`, 'gu');
+        breaks.push(calcBreaksForRegEx(line, regex, calcBreaks));
+    }
+    return breaks;
+}
+function genSymbolBreaks(line) {
+    function calcBreaks(m) {
+        const i = m.index;
+        if (i === undefined)
+            return;
+        const j = i + m[0].length;
+        return {
+            offset: i,
+            breaks: [
+                [i, j], // Remove the characters
+                [i, i], // keep characters with word to right
+                [j, j], // keep characters with word to left
+                ignoreBreak,
+            ],
+        };
+    }
+    return [
+        calcBreaksForRegEx(line, regExPossibleWordBreaks, calcBreaks),
+        calcBreaksForRegEx(line, /\d+/g, calcBreaks),
+        calcBreaksForRegEx(line, regExEscapeCharacters, calcBreaks),
+    ];
+}
+function mergeSortedBreaks(...maps) {
+    return maps.flat().sort((a, b) => a.offset - b.offset);
+}
+//# sourceMappingURL=generateWordBreaks.js.map

package/dist/lib/util/wordSplitter/index.d.ts

@@ -0,0 +1,4 @@
+export { softHyphen } from './generateWordBreaks.js';
+export type { SplitOptions, SplitResult } from './wordSplitter.js';
+export { split } from './wordSplitter.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/lib/util/wordSplitter/index.js

@@ -0,0 +1,3 @@
+export { softHyphen } from './generateWordBreaks.js';
+export { split } from './wordSplitter.js';
+//# sourceMappingURL=index.js.map

package/dist/lib/util/{wordSplitter.d.ts → wordSplitter/wordSplitter.d.ts}

@@ -1,4 +1,5 @@
 import type { TextOffset } from '@cspell/cspell-types';
+import type { WordBreakOptions } from './generateWordBreaks.js';
 export type IsValidWordFn = (word: TextOffset) => boolean;
 export interface SplitResult {
     /** Original line passed to the split function */
@@ -12,36 +13,26 @@ export interface SplitResult {
     /** the offset at which the split stopped */
     endOffset: number;
 }
-export interface LineSegment {
-    line: TextOffset;
-    relStart: number;
-    relEnd: number;
+export interface TextOffsetWithOptionalValid extends TextOffset {
+    length: number;
+    isFound?: boolean;
 }
-export interface TextOffsetWithValid extends TextOffset {
+export interface TextOffsetWithValid extends TextOffsetWithOptionalValid {
     isFound: boolean;
 }
 export interface SplitOptions extends WordBreakOptions {
 }
+/**
+ *
+ * @param line - the line of text
+ * @param offset - absolute offset, comparable to line.offset.
+ * @param isValidWord - predicate function to test if a word is valid.
+ * @param options - SplitOptions
+ * @returns SplitResult
+ */
 export declare function split(line: TextOffset, offset: number, isValidWord: IsValidWordFn, options?: SplitOptions): SplitResult;
 declare function findNextWordText({ text, offset }: TextOffset): TextOffset;
-type BreakPairs = readonly [number, number];
-interface PossibleWordBreak {
-    /** offset from the start of the string */
-    offset: number;
-    /**
-     * break pairs (start, end)
-     * (the characters between the start and end are removed)
-     * With a pure break, start === end.
-     */
-    breaks: BreakPairs[];
-}
-export type SortedBreaks = PossibleWordBreak[];
-interface WordBreakOptions {
-    optionalWordBreakCharacters?: string;
-}
-declare function generateWordBreaks(line: LineSegment, options: WordBreakOptions): SortedBreaks;
 export declare const __testing__: {
-    generateWordBreaks: typeof generateWordBreaks;
     findNextWordText: typeof findNextWordText;
 };
 export {};

package/dist/lib/util/wordSplitter/wordSplitter.js

@@ -0,0 +1,338 @@
+import { PairingHeap } from '../PairingHeap.js';
+import { regExFirstUpper, regExNumericLiteral, regExWordsAndDigits } from '../textRegex.js';
+import { generateWordBreaks, softHyphen } from './generateWordBreaks.js';
+const ignoreBreak = Object.freeze([]);
+const maxSkippedBreaks = 16; // Maximum number of breaks that can be skipped during word splitting.
+const maxAttempts = 64_000; // Maximum number of attempts to split words.
+// Quote/backtick characters that `regExWordsAndDigits` treats as word characters, so a quoted
+// or templated string literal in source code gets scanned with its delimiters attached.
+const regExLeadingTrimCharacters = /^['’`\\-]+/;
+const regExTrailingTrimCharacters = /['’`\\-]+$/;
+/**
+ *
+ * @param line - the line of text
+ * @param offset - absolute offset, comparable to line.offset.
+ * @param isValidWord - predicate function to test if a word is valid.
+ * @param options - SplitOptions
+ * @returns SplitResult
+ */
+export function split(line, offset, isValidWord, options = {}) {
+    const relWordToSplit = findNextWordText({ text: line.text, offset: offset - line.offset });
+    const hasSoftHyphen = relWordToSplit.text.includes(softHyphen);
+    const lineOffset = line.offset;
+    const requested = new Map();
+    const regExpIgnoreSegment = /^[-.+\d_eE'`\\\s\u00AD]+$/;
+    if (!relWordToSplit.text) {
+        const text = rebaseTextOffsetToAbsolute(relWordToSplit);
+        return {
+            line,
+            offset,
+            text: text,
+            words: [],
+            endOffset: text.offset + (text.length ?? text.text.length),
+        };
+    }
+    const lineSegment = {
+        line,
+        relStart: relWordToSplit.offset,
+        relEnd: relWordToSplit.offset + relWordToSplit.text.length,
+    };
+    const possibleBreaks = generateWordBreaks(lineSegment, options);
+    if (!possibleBreaks.length) {
+        const text = rebaseTextOffsetToAbsolute(relWordToSplit);
+        return {
+            line,
+            offset,
+            text: text,
+            words: [{ ...text, length: text.length || text.text.length, isFound: isValidWord(text) }],
+            endOffset: text.offset + (text.length ?? text.text.length),
+        };
+    }
+    function rebaseTextOffsetToAbsolute(relText) {
+        relText.offset += lineOffset;
+        return relText;
+    }
+    function rebaseTextOffsetToRelative(relText) {
+        relText.offset -= lineOffset;
+        return relText;
+    }
+    function createTextOffsetWithValid(start, end, isFound) {
+        const word = {
+            offset: start,
+            length: end - start,
+            text: line.text.slice(start, end),
+            isFound,
+        };
+        return word;
+    }
+    function calcKey(start, end) {
+        const i = start;
+        const j = end - start;
+        const v = i + (j << 20);
+        return i < 1 << 20 && j < 1 << 11 ? v : -1;
+    }
+    function checkTextRange(start, end) {
+        const v = calcKey(start, end);
+        const f = v >= 0 ? requested.get(v) : undefined;
+        if (f)
+            return f;
+        const validated = createTextOffsetWithValid(start, end, false);
+        if (regExpIgnoreSegment.test(validated.text)) {
+            validated.isFound = true;
+            if (v >= 0) {
+                requested.set(v, validated);
+            }
+            return validated;
+        }
+        rebaseTextOffsetToAbsolute(validated);
+        validated.isFound = isValidWord(validated);
+        if (hasSoftHyphen && !validated.isFound && validated.text.includes(softHyphen)) {
+            removeSoftHyphen(validated);
+            validated.isFound = isValidWord(validated);
+        }
+        rebaseTextOffsetToRelative(validated);
+        if (v >= 0) {
+            requested.set(v, validated);
+        }
+        return validated;
+    }
+    // Add a dummy break at the end to avoid needing to check for last break.
+    possibleBreaks.push({
+        offset: lineSegment.relEnd,
+        breaks: [ignoreBreak],
+    });
+    const result = {
+        line,
+        offset,
+        text: rebaseTextOffsetToAbsolute(relWordToSplit),
+        words: splitIntoWords(lineSegment, possibleBreaks, checkTextRange).map(rebaseTextOffsetToAbsolute),
+        endOffset: lineOffset + lineSegment.relEnd,
+    };
+    return result;
+}
+function removeSoftHyphen(word) {
+    word.text = word.text.replaceAll(softHyphen, '');
+    return word;
+}
+function findNextWordText({ text, offset }) {
+    const reg = new RegExp(regExWordsAndDigits);
+    reg.lastIndex = offset;
+    const m = reg.exec(text);
+    if (!m) {
+        // No more words found, return an empty text with offset at the end of the string within the document.
+        return { text: '', offset: offset + text.length };
+    }
+    // Skip numeric literals.
+    if (regExNumericLiteral.test(m[0])) {
+        return findNextWordText({ text, offset: offset + m[0].length });
+    }
+    return {
+        text: m[0],
+        offset: m.index,
+    };
+}
+function splitIntoWords(lineSeg, breaks, checkTextRange) {
+    const maxIndex = lineSeg.relEnd;
+    /**
+     * Map of known paths by their starting index in the text.
+     * This helps in quickly finding previously computed paths to avoid redundant calculations.
+     */
+    const knownPathsByIndex = new Map();
+    const terminalNode = { n: undefined, i: maxIndex, j: maxIndex, c: 0, nc: 0, text: undefined };
+    knownPathsByIndex.set(maxIndex, terminalNode);
+    // Seed the search with the whole word treated as a single candidate. This guarantees a
+    // correct fallback result (e.g. a whole word matched via ignoreWords/the dictionary, or
+    // reporting the word as a single misspelling) even when the segmented search below never
+    // finds a strictly cheaper split.
+    let wholeWord = checkTextRange(lineSeg.relStart, maxIndex);
+    if (!wholeWord.isFound) {
+        // `regExWordsAndDigits` treats quote/backtick characters as word characters so that
+        // things like `Tom's` or `n'cpp` are scanned as a single word; but it also means a
+        // quoted/templated string literal in source code (e.g. `'ignoredWord'`) is scanned
+        // including its delimiters. Retry against the span with that framing peeled off so a
+        // whole word that is otherwise valid/ignored is still recognized as such.
+        const raw = lineSeg.line.text.slice(lineSeg.relStart, maxIndex);
+        const lead = raw.match(regExLeadingTrimCharacters)?.[0].length ?? 0;
+        const tail = raw.slice(lead).match(regExTrailingTrimCharacters)?.[0].length ?? 0;
+        if ((lead || tail) && lead + tail < raw.length) {
+            wholeWord = checkTextRange(lineSeg.relStart + lead, maxIndex - tail);
+        }
+    }
+    const wholeWordCost = wholeWord.isFound ? 0 : wholeWord.text.length;
+    const wholeWordNode = {
+        n: terminalNode,
+        i: lineSeg.relStart,
+        j: maxIndex,
+        c: wholeWordCost,
+        nc: wholeWordCost,
+        text: wholeWord,
+    };
+    knownPathsByIndex.set(lineSeg.relStart, wholeWordNode);
+    function findNearestBreakIndex(offset, bi) {
+        while (bi < breaks.length && breaks[bi].offset < offset) {
+            bi += 1;
+        }
+        return bi < breaks.length ? bi : undefined;
+    }
+    /**
+     * Create a set of possible candidate to consider
+     * @param p - prev candidate that lead to this one
+     * @param i - offset within the string
+     * @param bi - current index into the set of breaks
+     * @param currentCost - current cost accrued
+     */
+    function makeCandidates(p, i, bi, bs, currentCost) {
+        const len = maxIndex;
+        const nBi = findNearestBreakIndex(i, bi);
+        const nBs = findNearestBreakIndex(i, bs);
+        if (nBi === undefined || nBs === undefined) {
+            return [];
+        }
+        bi = nBi;
+        bs = nBs;
+        if (bi - bs > maxSkippedBreaks) {
+            return [];
+        }
+        const br = breaks[bi];
+        function calcBreakCost(bp) {
+            if (bp.length < 2) {
+                // We are skipping this break pair.
+                return { p, i, j: len, bi, bs, bp, c: currentCost, ec: currentCost + len - i, text: undefined };
+            }
+            const startOfBreak = bp[0];
+            const endOfBreak = bp[1];
+            // cost from the end of the segment to the end of the text
+            // const costToEnd = knownPathsByIndex.get(endOfBreak)?.c ?? len - endOfBreak;
+            const costToEnd = len - endOfBreak;
+            // Guess the cost of the word.
+            const costOfWord = (startOfBreak - i) * 0.5;
+            const d = costOfWord + costToEnd;
+            const ec = currentCost + d;
+            return { p, i, j: endOfBreak, bi, bs, bp, c: currentCost, ec, text: undefined };
+        }
+        return br.breaks.map(calcBreakCost);
+    }
+    function compare(a, b) {
+        return a.bi - a.bs - (b.bi - b.bs) || a.ec - b.ec || b.i - a.i || a.j - b.j;
+    }
+    function pathToWords(node) {
+        const results = [];
+        for (let p = node; p; p = p.n) {
+            if (p.text) {
+                results.push(p.text);
+            }
+        }
+        return results;
+    }
+    function addToKnownPaths(candidate) {
+        let path = knownPathsByIndex.get(candidate.j);
+        for (let can = candidate; can !== undefined; can = can.p) {
+            const t = can.text;
+            const i = can.i;
+            const j = can.j;
+            const currentNode = knownPathsByIndex.get(i);
+            const exitingPath = knownPathsByIndex.get(j);
+            // If a better-or-equal path is already known, reuse it and keep propagating
+            // the (possibly cheaper) suffix back toward the start instead of abandoning it.
+            if (exitingPath && path && exitingPath.c <= path.c) {
+                path = exitingPath;
+            }
+            const nc = !t || t.isFound ? 0 : t.length;
+            const cost = nc + (path?.c || 0);
+            const node = { n: path, i, j, c: cost, nc, text: t };
+            if (!currentNode || currentNode.c > cost) {
+                knownPathsByIndex.set(i, node);
+            }
+            path = node;
+        }
+        return path;
+    }
+    // A candidate that only ties the whole-word fallback is still preferred over it, since it
+    // represents a more granular split; ties between two genuine split candidates keep whichever
+    // was found first. Allow the search to explore up to (and including) a tying cost by giving
+    // maxCost one extra unit of slack above the seeded whole-word cost.
+    function preferCandidate(candidate, current) {
+        if (candidate.c < current.c)
+            return true;
+        if (candidate.c > current.c)
+            return false;
+        return current === wholeWordNode && candidate !== wholeWordNode;
+    }
+    let maxCost = wholeWordCost + 1;
+    const candidates = new PairingHeap(compare);
+    candidates.append(makeCandidates(undefined, lineSeg.relStart, 0, 0, 0));
+    let attempts = 0;
+    let bestPath = wholeWordNode;
+    while (maxCost && candidates.length && attempts++ < maxAttempts) {
+        /** Best Candidate Index */
+        const best = candidates.dequeue();
+        if (!best || best.c >= maxCost) {
+            continue;
+        }
+        // Does it have a split?
+        if (best.bp.length) {
+            // yes
+            const i = best.bp[0];
+            const j = best.bp[1];
+            const parentPath = knownPathsByIndex.get(j);
+            const parentCost = parentPath?.c || 0;
+            if (parentCost + best.c >= maxCost) {
+                continue;
+            }
+            const t = i > best.i ? checkTextRange(best.i, i) : undefined;
+            const cost = !t || t.isFound ? 0 : t.length;
+            const mc = maxIndex - j;
+            best.c += cost;
+            best.ec = best.c + mc;
+            best.text = t;
+            if (best.c >= maxCost) {
+                continue;
+            }
+            if (parentPath) {
+                // We found a known apply to candidate
+                const f = addToKnownPaths(best);
+                bestPath = f && preferCandidate(f, bestPath) ? f : bestPath;
+            }
+            const c = makeCandidates(t ? best : best.p, j, best.bi + 1, best.bs, best.c);
+            candidates.append(c);
+        }
+        else {
+            // It is a pass through
+            const c = makeCandidates(best.p, best.i, best.bi + 1, best.bs, best.c);
+            candidates.append(c);
+            if (!c.length) {
+                const j = maxIndex;
+                const t = j > best.i ? checkTextRange(best.i, j) : undefined;
+                const cost = !t || t.isFound ? 0 : t.length;
+                best.c += cost;
+                best.ec = best.c;
+                best.text = t;
+                if (best.c >= maxCost) {
+                    continue;
+                }
+                const segText = t || best.p?.text || checkTextRange(best.i, best.i);
+                const can = t ? { ...best, text: segText } : { ...best, ...best.p, text: segText };
+                can.j = j;
+                const f = addToKnownPaths(can);
+                bestPath = f && preferCandidate(f, bestPath) ? f : bestPath;
+            }
+        }
+        // While the whole-word fallback is still the best known answer, keep one unit of
+        // slack above its cost so a genuine split that only ties it can still be explored
+        // to completion and preferred by preferCandidate. Once a real split has taken over,
+        // tighten exactly like before.
+        const slack = bestPath === wholeWordNode ? 1 : 0;
+        if (bestPath.c + slack < maxCost) {
+            maxCost = bestPath.c + slack;
+        }
+    }
+    const result = pathToWords(bestPath);
+    if (result.length === 2 && result.every((r) => !r.isFound) && regExFirstUpper.test(wholeWord.text)) {
+        return [wholeWord];
+    }
+    return result;
+}
+export const __testing__ = {
+    findNextWordText,
+};
+//# sourceMappingURL=wordSplitter.js.map

package/dist/lib/util/{wordSplitter.perf.js → wordSplitter/wordSplitter.perf.js}

@@ -1,8 +1,8 @@
 import { readFile } from 'node:fs/promises';
 import { extname } from 'node:path/posix';
 import { suite } from 'perf-insight';
-import { extractPossibleWordsFromTextOffset, extractWordsFromCode, extractWordsFromText, matchStringToTextOffset, splitCamelCaseWordWithOffset, textOffset, } from './text.js';
-import { regExWordsAndDigits } from './textRegex.js';
+import { extractPossibleWordsFromTextOffset, extractWordsFromCode, extractWordsFromText, matchStringToTextOffset, splitCamelCaseWordWithOffset, textOffset, } from '../text.js';
+import { regExWordsAndDigits } from '../textRegex.js';
 const regExpWord = /\b[\w\p{L}\p{M}]+\b/gu;
 suite('wordSplitter', async (test) => {
     const lines = await sampleLines();

package/package.json

@@ -4,7 +4,7 @@
     "access": "public",
     "provenance": true
   },
-  "version": "10.2.0",
+  "version": "10.2.2",
   "description": "A library of useful functions used across various cspell tools.",
   "type": "module",
   "sideEffects": false,
@@ -75,22 +75,22 @@
   },
   "homepage": "https://github.com/streetsidesoftware/cspell/tree/main/packages/cspell-lib#readme",
   "dependencies": {
-    "@cspell/cspell-bundled-dicts": "10.2.0",
-    "@cspell/cspell-performance-monitor": "10.2.0",
-    "@cspell/cspell-pipe": "10.2.0",
-    "@cspell/cspell-resolver": "10.2.0",
-    "@cspell/cspell-types": "10.2.0",
-    "@cspell/dynamic-import": "10.2.0",
-    "@cspell/filetypes": "10.2.0",
-    "@cspell/rpc": "10.2.0",
-    "@cspell/strong-weak-map": "10.2.0",
-    "@cspell/url": "10.2.0",
-    "cspell-config-lib": "10.2.0",
-    "cspell-dictionary": "10.2.0",
-    "cspell-glob": "10.2.0",
-    "cspell-grammar": "10.2.0",
-    "cspell-io": "10.2.0",
-    "cspell-trie-lib": "10.2.0",
+    "@cspell/cspell-bundled-dicts": "10.2.2",
+    "@cspell/cspell-performance-monitor": "10.2.2",
+    "@cspell/cspell-pipe": "10.2.2",
+    "@cspell/cspell-resolver": "10.2.2",
+    "@cspell/cspell-types": "10.2.2",
+    "@cspell/dynamic-import": "10.2.2",
+    "@cspell/filetypes": "10.2.2",
+    "@cspell/rpc": "10.2.2",
+    "@cspell/strong-weak-map": "10.2.2",
+    "@cspell/url": "10.2.2",
+    "cspell-config-lib": "10.2.2",
+    "cspell-dictionary": "10.2.2",
+    "cspell-glob": "10.2.2",
+    "cspell-grammar": "10.2.2",
+    "cspell-io": "10.2.2",
+    "cspell-trie-lib": "10.2.2",
     "env-paths": "^4.0.0",
     "gensequence": "^8.0.8",
     "import-fresh": "^4.0.0",
@@ -103,7 +103,7 @@
     "node": ">=22.18.0"
   },
   "devDependencies": {
-    "@cspell/cspell-tools": "10.2.0",
+    "@cspell/cspell-tools": "10.2.2",
     "@cspell/dict-cpp": "^7.1.0",
     "@cspell/dict-csharp": "^4.0.8",
     "@cspell/dict-css": "^4.1.2",
@@ -113,6 +113,7 @@
     "@cspell/dict-html": "^4.0.16",
     "@cspell/dict-nl-nl": "^2.4.2",
     "@cspell/dict-python": "^4.5.0",
+    "@cspell/dict-th-th": "^1.0.0",
     "@cspell/dictionary-bundler-plugin": "",
     "@types/configstore": "^6.0.2",
     "comment-json": "^5.0.0",
@@ -122,5 +123,5 @@
     "lorem-ipsum": "^3.0.0",
     "perf-insight": "^2.0.1"
   },
-  "gitHead": "2d310afc22fa500b1be07e3c2e2d3897ec00daad"
+  "gitHead": "22fae7ac63d4330c52fe97eebee743e3541f2e36"
 }

package/dist/lib/util/wordSplitter.js

@@ -1,331 +0,0 @@
-import { PairingHeap } from './PairingHeap.js';
-import { escapeRegEx } from './regexHelper.js';
-import { regExDanglingQuote, regExEscapeCharacters, regExNumericLiteral, regExPossibleWordBreaks, regExSplitWords, regExSplitWords2, regExTrailingEndings, regExWordsAndDigits, } from './textRegex.js';
-const ignoreBreak = Object.freeze([]);
-export function split(line, offset, isValidWord, options = {}) {
-    const relWordToSplit = findNextWordText({ text: line.text, offset: offset - line.offset });
-    const lineOffset = line.offset;
-    const requested = new Map();
-    const regExpIgnoreSegment = /^[-.+\d_eE'`\\\s]+$/;
-    if (!relWordToSplit.text) {
-        const text = rebaseTextOffset(relWordToSplit);
-        return {
-            line,
-            offset,
-            text: text,
-            words: [],
-            endOffset: text.offset + text.text.length,
-        };
-    }
-    const lineSegment = {
-        line,
-        relStart: relWordToSplit.offset,
-        relEnd: relWordToSplit.offset + relWordToSplit.text.length,
-    };
-    const possibleBreaks = generateWordBreaks(lineSegment, options);
-    if (!possibleBreaks.length) {
-        const text = rebaseTextOffset(relWordToSplit);
-        return {
-            line,
-            offset,
-            text: text,
-            words: [{ ...text, isFound: isValidWord(text) }],
-            endOffset: text.offset + text.text.length,
-        };
-    }
-    function rebaseTextOffset(relText) {
-        return {
-            ...relText,
-            offset: relText.offset + lineOffset,
-        };
-    }
-    function has(word) {
-        if (regExpIgnoreSegment.test(word.text)) {
-            return true;
-        }
-        const i = word.offset;
-        const j = word.text.length;
-        let v = i + (j << 20);
-        if (i < 1 << 20 && j < 1 << 11) {
-            const b = requested.get(v);
-            if (b !== undefined)
-                return b;
-        }
-        else {
-            v = -1;
-        }
-        const r = isValidWord(rebaseTextOffset(word));
-        if (v >= 0) {
-            requested.set(v, r);
-        }
-        return r;
-    }
-    // Add a dummy break at the end to avoid needing to check for last break.
-    possibleBreaks.push({
-        offset: lineSegment.relEnd,
-        breaks: [ignoreBreak],
-    });
-    const result = {
-        line,
-        offset,
-        text: rebaseTextOffset(relWordToSplit),
-        words: splitIntoWords(lineSegment, possibleBreaks, has).map(rebaseTextOffset),
-        endOffset: lineOffset + lineSegment.relEnd,
-    };
-    return result;
-}
-function findNextWordText({ text, offset }) {
-    const reg = new RegExp(regExWordsAndDigits);
-    reg.lastIndex = offset;
-    const m = reg.exec(text);
-    if (!m) {
-        return {
-            text: '',
-            offset: offset + text.length,
-        };
-    }
-    // Skip numeric literals.
-    if (regExNumericLiteral.test(m[0])) {
-        return findNextWordText({ text, offset: offset + m[0].length });
-    }
-    return {
-        text: m[0],
-        offset: m.index,
-    };
-}
-function generateWordBreaks(line, options) {
-    const camelBreaks = genWordBreakCamel(line);
-    const symbolBreaks = genSymbolBreaks(line);
-    const optionalBreaks = genOptionalWordBreaks(line, options.optionalWordBreakCharacters);
-    return mergeSortedBreaks(...camelBreaks, ...symbolBreaks, ...optionalBreaks);
-}
-function offsetRegEx(reg, offset) {
-    const r = new RegExp(reg);
-    r.lastIndex = offset;
-    return r;
-}
-function genWordBreakCamel(line) {
-    const breaksCamel1 = [];
-    const text = line.line.text.slice(0, line.relEnd);
-    // lower,Upper: camelCase -> camel|Case
-    for (const m of text.matchAll(offsetRegEx(regExSplitWords, line.relStart))) {
-        if (m.index === undefined)
-            break;
-        const i = m.index + m[1].length;
-        breaksCamel1.push({
-            offset: m.index,
-            breaks: [[i, i], ignoreBreak],
-        });
-    }
-    const breaksCamel2 = [];
-    // cspell:ignore ERRORC
-    // Upper,Upper,lower: ERRORCodes -> ERROR|Codes, ERRORC|odes
-    for (const m of text.matchAll(offsetRegEx(regExSplitWords2, line.relStart))) {
-        if (m.index === undefined)
-            break;
-        const i = m.index + m[1].length;
-        const j = i + m[3].length;
-        breaksCamel2.push({
-            offset: m.index,
-            breaks: [[i, i], [j, j], ignoreBreak],
-        });
-    }
-    return [breaksCamel1, breaksCamel2];
-}
-function calcBreaksForRegEx(line, reg, calcBreak) {
-    const sb = [];
-    const text = line.line.text.slice(0, line.relEnd);
-    for (const m of text.matchAll(offsetRegEx(reg, line.relStart))) {
-        const b = calcBreak(m);
-        if (b) {
-            sb.push(b);
-        }
-    }
-    return sb;
-}
-function genOptionalWordBreaks(line, optionalBreakCharacters) {
-    function calcBreaks(m) {
-        const i = m.index;
-        if (i === undefined)
-            return;
-        const j = i + m[0].length;
-        return {
-            offset: i,
-            breaks: [
-                [i, j], // Remove the characters
-                ignoreBreak,
-            ],
-        };
-    }
-    const breaks = [
-        calcBreaksForRegEx(line, regExDanglingQuote, calcBreaks),
-        calcBreaksForRegEx(line, regExTrailingEndings, calcBreaks),
-    ];
-    if (optionalBreakCharacters) {
-        const regex = new RegExp(`[${escapeRegEx(optionalBreakCharacters)}]`, 'gu');
-        breaks.push(calcBreaksForRegEx(line, regex, calcBreaks));
-    }
-    return breaks;
-}
-function genSymbolBreaks(line) {
-    function calcBreaks(m) {
-        const i = m.index;
-        if (i === undefined)
-            return;
-        const j = i + m[0].length;
-        return {
-            offset: i,
-            breaks: [
-                [i, j], // Remove the characters
-                [i, i], // keep characters with word to right
-                [j, j], // keep characters with word to left
-                ignoreBreak,
-            ],
-        };
-    }
-    return [
-        calcBreaksForRegEx(line, regExPossibleWordBreaks, calcBreaks),
-        calcBreaksForRegEx(line, /\d+/g, calcBreaks),
-        calcBreaksForRegEx(line, regExEscapeCharacters, calcBreaks),
-    ];
-}
-function splitIntoWords(lineSeg, breaks, has) {
-    const maxIndex = lineSeg.relEnd;
-    const maxAttempts = 1000;
-    const knownPathsByIndex = new Map();
-    /**
-     * Create a set of possible candidate to consider
-     * @param p - prev candidate that lead to this one
-     * @param i - offset within the string
-     * @param bi - current index into the set of breaks
-     * @param currentCost - current cost accrued
-     */
-    function makeCandidates(p, i, bi, currentCost) {
-        const len = maxIndex;
-        while (bi < breaks.length && breaks[bi].offset < i) {
-            bi += 1;
-        }
-        if (bi >= breaks.length) {
-            return [];
-        }
-        const br = breaks[bi];
-        function c(bp) {
-            const d = bp.length < 2 ? len - i : (bp[0] - i) * 0.5 + len - bp[1];
-            const ec = currentCost + d;
-            return {
-                p,
-                i,
-                bi,
-                bp,
-                c: currentCost,
-                ec,
-                text: undefined,
-            };
-        }
-        return br.breaks.map(c);
-    }
-    function checkTextOffset(text, offset) {
-        const valid = has({ text, offset });
-        return {
-            text,
-            offset,
-            isFound: valid,
-        };
-    }
-    function compare(a, b) {
-        return a.ec - b.ec || b.i - a.i;
-    }
-    function pathToWords(node) {
-        const results = [];
-        for (let p = node; p; p = p.n) {
-            if (p.text) {
-                results.push(p.text);
-            }
-        }
-        return results;
-    }
-    function addToKnownPaths(candidate, path) {
-        for (let can = candidate; can !== undefined; can = can.p) {
-            const t = can.text;
-            const i = can.i;
-            const cost = (!t || t.isFound ? 0 : t.text.length) + (path?.c ?? 0);
-            const exitingPath = knownPathsByIndex.get(i);
-            // Keep going only if this is a better candidate than the existing path
-            if (exitingPath && exitingPath.c <= cost) {
-                return undefined;
-            }
-            const node = {
-                n: path,
-                i,
-                c: cost,
-                text: t,
-            };
-            knownPathsByIndex.set(i, node);
-            path = node;
-        }
-        return path;
-    }
-    let maxCost = lineSeg.relEnd - lineSeg.relStart;
-    const candidates = new PairingHeap(compare);
-    const text = lineSeg.line.text;
-    candidates.append(makeCandidates(undefined, lineSeg.relStart, 0, 0));
-    let attempts = 0;
-    let bestPath;
-    while (maxCost && candidates.length && attempts++ < maxAttempts) {
-        /** Best Candidate Index */
-        const best = candidates.dequeue();
-        if (!best || best.c >= maxCost) {
-            continue;
-        }
-        // Does it have a split?
-        if (best.bp.length) {
-            // yes
-            const i = best.bp[0];
-            const j = best.bp[1];
-            const t = i > best.i ? checkTextOffset(text.slice(best.i, i), best.i) : undefined;
-            const cost = !t || t.isFound ? 0 : t.text.length;
-            const mc = maxIndex - j;
-            best.c += cost;
-            best.ec = best.c + mc;
-            best.text = t;
-            const possiblePath = knownPathsByIndex.get(j);
-            if (possiblePath) {
-                // We found a known apply to candidate
-                const f = addToKnownPaths(best, possiblePath);
-                bestPath = !bestPath || (f && f.c < bestPath.c) ? f : bestPath;
-            }
-            else if (best.c < maxCost) {
-                const c = makeCandidates(t ? best : best.p, j, best.bi + 1, best.c);
-                candidates.append(c);
-            }
-        }
-        else {
-            // It is a pass through
-            const c = makeCandidates(best.p, best.i, best.bi + 1, best.c);
-            candidates.append(c);
-            if (!c.length) {
-                const t = maxIndex > best.i ? checkTextOffset(text.slice(best.i, maxIndex), best.i) : undefined;
-                const cost = !t || t.isFound ? 0 : t.text.length;
-                best.c += cost;
-                best.ec = best.c;
-                best.text = t;
-                const segText = t || best.p?.text || checkTextOffset('', best.i);
-                const can = t ? { ...best, text: segText } : { ...best, ...best.p, text: segText };
-                const f = addToKnownPaths(can, undefined);
-                bestPath = !bestPath || (f && f.c < bestPath.c) ? f : bestPath;
-            }
-        }
-        if (bestPath && bestPath.c < maxCost) {
-            maxCost = bestPath.c;
-        }
-    }
-    return pathToWords(bestPath);
-}
-function mergeSortedBreaks(...maps) {
-    return maps.flat().sort((a, b) => a.offset - b.offset);
-}
-export const __testing__ = {
-    generateWordBreaks,
-    findNextWordText,
-};
-//# sourceMappingURL=wordSplitter.js.map