cspell-lib 10.2.0-alpha.7 → 10.2.1

package/dist/lib/Settings/LanguageSettings.js

@@ -38,9 +38,25 @@ export function normalizeLocale(locale) {
     return _normalizeLocale(locale);
 }
 export function normalizeLocaleIntl(locale) {
-    const values = [...normalizeLocale(locale)].map((locale) => locale.replace(/^([a-z]{2})-?([a-z]{2})$/, (_, lang, locale) => locale ? `${lang}-${locale.toUpperCase()}` : lang));
+    const values = [...normalizeLocale(locale)].map((locale) => locale.replace(/^([a-z]{2})(?:-?([a-z]{4}))?-?([a-z]{2})$/, (_, lang, script, region) => toLocaleIntl(lang, script, region)));
     return new Set(values);
 }
+/**
+ * See: https://developer.mozilla.org/en-US/docs/Glossary/BCP_47_language_tag
+ * @returns locale normalized to BCP_47
+ */
+function toLocaleIntl(lang, script, region) {
+    const parts = [lang];
+    if (script)
+        parts.push(firstUpper(script.toLowerCase()));
+    if (region)
+        parts.push(region.toUpperCase());
+    return parts.join('-');
+}
+function firstUpper(str) {
+    const [first, ...rest] = [...str];
+    return [first.toUpperCase(), ...rest].join('');
+}
 export function isLocaleInSet(locale, setOfLocals) {
     const locales = normalizeLocale(locale);
     return doSetsIntersect(locales, setOfLocals);

package/dist/lib/Settings/sanitizeSettings.js

@@ -68,6 +68,7 @@ const handlers = {
     suggestWords: copy1,
     unknownWords: copy1,
     useGitignore: copy1,
+    useIntlWordSegmentation: copy1,
     usePnP: skip,
     userWords: copy1,
     validateDirectives: copy1,
@@ -183,6 +184,7 @@ const LanguageSettingsHandlers = {
     substitutions: copy1,
     suggestWords: copy1,
     unknownWords: copy1,
+    useIntlWordSegmentation: copy1,
     words: copy1,
     // Experimental
     parser: skip,
@@ -245,6 +247,7 @@ const OverridesHandlers = {
     suggestionsTimeout: copy1,
     suggestWords: copy1,
     unknownWords: copy1,
+    useIntlWordSegmentation: copy1,
     usePnP: skip,
     words: copy1,
     parser: skip,

package/dist/lib/Transform/IntlSegmentTextTransformer.d.ts

@@ -0,0 +1,11 @@
+import type { MappedText } from '@cspell/cspell-types';
+import type { TextTransformer } from './Transformer.js';
+export declare const softHyphen = "\u00AD";
+export declare class IntlSegmentTextTransformer implements TextTransformer {
+    #private;
+    constructor(locale: string | string[]);
+    transform(text: string | MappedText): MappedText;
+    transformAll(src: Iterable<string | MappedText>): Iterable<MappedText>;
+}
+export declare function createIntlSegmentTextTransformer(locale: string | string[]): IntlSegmentTextTransformer;
+//# sourceMappingURL=IntlSegmentTextTransformer.d.ts.map

package/dist/lib/Transform/IntlSegmentTextTransformer.js

@@ -0,0 +1,41 @@
+import { normalizeLocaleIntl } from '../Settings/LanguageSettings.js';
+import { applyEditsToMappedText } from './TextMapEdit.js';
+import { toMappedText } from './Transformer.js';
+export const softHyphen = '\u00AD';
+const regExSplitWordCharacters = /(?<=[\p{L}\p{M}])[\p{L}\p{M}]/uy;
+export class IntlSegmentTextTransformer {
+    #locale;
+    constructor(locale) {
+        this.#locale = [...normalizeLocaleIntl(locale)];
+    }
+    transform(text) {
+        // Implement the transformation logic here
+        text = toMappedText(text);
+        return segmentText(text, new Intl.Segmenter(this.#locale, { granularity: 'word' }));
+    }
+    *transformAll(src) {
+        for (const item of src) {
+            yield this.transform(item);
+        }
+    }
+}
+function segmentText(mText, segmenter) {
+    const edits = [];
+    let lastIndex = -1;
+    const reg = new RegExp(regExSplitWordCharacters);
+    const text = mText.text;
+    for (const { segment, index, isWordLike } of segmenter.segment(mText.text)) {
+        const endIndex = index + segment.length;
+        reg.lastIndex = index;
+        if (isWordLike && index === lastIndex && reg.test(text)) {
+            const edit = { text: softHyphen + segment, range: [index, endIndex] };
+            edits.push(edit);
+        }
+        lastIndex = endIndex;
+    }
+    return applyEditsToMappedText(mText, edits);
+}
+export function createIntlSegmentTextTransformer(locale) {
+    return new IntlSegmentTextTransformer(locale);
+}
+//# sourceMappingURL=IntlSegmentTextTransformer.js.map

package/dist/lib/Transform/SubstitutionTransformer.d.ts

@@ -1,4 +1,5 @@
 import type { MappedText, SubstitutionDefinitions, Substitutions } from '@cspell/cspell-types';
+import type { TextTransformer } from './Transformer.js';
 type DeepReadonly<T> = T extends object ? {
     readonly [K in keyof T]: DeepReadonly<T[K]>;
 } : T;
@@ -7,10 +8,11 @@ export interface SubstitutionInfo {
     substitutionDefinitions?: SubstitutionDefinitions | undefined;
 }
 export type ReadonlySubstitutionInfo = DeepReadonly<SubstitutionInfo>;
-export declare class SubstitutionTransformer {
+export declare class SubstitutionTransformer implements TextTransformer {
     #private;
     constructor(subMap: Map<string, string> | undefined);
     transform(text: string | MappedText): MappedText;
+    transformAll(src: Iterable<string | MappedText>): Iterable<MappedText>;
     transformString(text: string): MappedText;
 }
 export interface CreateSubstitutionTransformerResult {

package/dist/lib/Transform/SubstitutionTransformer.js

@@ -1,5 +1,7 @@
 import { GTrie } from 'cspell-trie-lib';
-import { mapOffsetPairsToSourceMap, mergeSourceMaps } from './SourceMap.js';
+import { mergeSourceMaps } from './SourceMap.js';
+import { applyEditsToText } from './TextMapEdit.js';
+import { toMappedText } from './Transformer.js';
 export class SubstitutionTransformer {
     #trie;
     constructor(subMap) {
@@ -21,36 +23,16 @@ export class SubstitutionTransformer {
         };
         return result;
     }
+    *transformAll(src) {
+        for (const item of src) {
+            yield this.transform(item);
+        }
+    }
     transformString(text) {
         if (!this.#trie) {
-            return { text, range: [0, text.length], rawText: text };
-        }
-        const map = [0, 0];
-        let repText = '';
-        let lastEnd = 0;
-        for (const edit of calcEdits(text, this.#trie)) {
-            if (edit.range[0] > lastEnd) {
-                repText += text.slice(lastEnd, edit.range[0]);
-                map.push(edit.range[0], repText.length);
-            }
-            repText += edit.text;
-            map.push(edit.range[1], repText.length);
-            lastEnd = edit.range[1];
-        }
-        if (lastEnd === 0) {
-            return { text, range: [0, text.length], rawText: text };
+            return toMappedText(text);
         }
-        if (lastEnd < text.length) {
-            repText += text.slice(lastEnd);
-            map.push(text.length, repText.length);
-        }
-        const result = {
-            text: repText,
-            range: [0, text.length],
-            map: mapOffsetPairsToSourceMap(map),
-            rawText: text,
-        };
-        return result;
+        return applyEditsToText(text, calcEdits(text, this.#trie));
     }
 }
 function* calcEdits(text, subTrie) {

package/dist/lib/Transform/TextMapEdit.d.ts

@@ -0,0 +1,5 @@
+import type { MappedText } from '@cspell/cspell-types';
+import type { Edit } from './Transformer.js';
+export declare function applyEditsToMappedText(text: MappedText, edits: Iterable<Edit>): MappedText;
+export declare function applyEditsToText(text: string, edits: Iterable<Edit>): MappedText;
+//# sourceMappingURL=TextMapEdit.d.ts.map

package/dist/lib/Transform/TextMapEdit.js

@@ -0,0 +1,42 @@
+import { mapOffsetPairsToSourceMap, mergeSourceMaps } from './SourceMap.js';
+import { toMappedText } from './Transformer.js';
+export function applyEditsToMappedText(text, edits) {
+    const transformed = applyEditsToText(text.text, edits);
+    const rawText = text.rawText ?? text.text;
+    const result = {
+        ...text,
+        text: transformed.text,
+        map: mergeSourceMaps(text.map, transformed.map),
+        rawText,
+    };
+    return result;
+}
+export function applyEditsToText(text, edits) {
+    const map = [0, 0];
+    let repText = '';
+    let lastEnd = 0;
+    for (const edit of edits) {
+        if (edit.range[0] > lastEnd) {
+            repText += text.slice(lastEnd, edit.range[0]);
+            map.push(edit.range[0], repText.length);
+        }
+        repText += edit.text;
+        map.push(edit.range[1], repText.length);
+        lastEnd = edit.range[1];
+    }
+    if (lastEnd === 0) {
+        return toMappedText(text);
+    }
+    if (lastEnd < text.length) {
+        repText += text.slice(lastEnd);
+        map.push(text.length, repText.length);
+    }
+    const result = {
+        text: repText,
+        range: [0, text.length],
+        map: mapOffsetPairsToSourceMap(map),
+        rawText: text,
+    };
+    return result;
+}
+//# sourceMappingURL=TextMapEdit.js.map

package/dist/lib/Transform/Transformer.d.ts

@@ -0,0 +1,25 @@
+import type { MappedText } from '@cspell/cspell-types';
+import type { Range } from '@cspell/cspell-types/Parser';
+export interface TextTransformer {
+    transform(text: string | MappedText): MappedText;
+    transformAll(src: Iterable<string | MappedText>): Iterable<MappedText>;
+}
+export interface Edit {
+    /**
+     * The replacement text for the edit. This is the text that will replace the original text in the string.
+     */
+    text: string;
+    /**
+     * The range of the text to be replaced. This is a tuple of the form `[start, end]`,
+     * where start is the index of the first character to be replaced,
+     * and end is the index of the first character after the last character to be replaced.
+     */
+    range: Range;
+}
+export declare function toMappedText(text: string | MappedText): MappedText;
+export declare function chainTransformers(...transformers: TextTransformer[]): TextTransformer;
+export declare class IdentityTextTransformer implements TextTransformer {
+    transform(text: string | MappedText): MappedText;
+    transformAll(src: Iterable<string | MappedText>): Iterable<MappedText>;
+}
+//# sourceMappingURL=Transformer.d.ts.map

package/dist/lib/Transform/Transformer.js

@@ -0,0 +1,27 @@
+export function toMappedText(text) {
+    if (typeof text === 'string') {
+        return { text, range: [0, text.length], rawText: text };
+    }
+    return text;
+}
+export function chainTransformers(...transformers) {
+    return {
+        transform(text) {
+            return transformers.reduce((acc, transformer) => transformer.transform(acc), toMappedText(text));
+        },
+        transformAll(src) {
+            return Array.from(src, (text) => transformers.reduce((acc, transformer) => transformer.transform(acc), toMappedText(text)));
+        },
+    };
+}
+export class IdentityTextTransformer {
+    transform(text) {
+        return toMappedText(text);
+    }
+    *transformAll(src) {
+        for (const item of src) {
+            yield this.transform(item);
+        }
+    }
+}
+//# sourceMappingURL=Transformer.js.map

package/dist/lib/Transform/index.d.ts

@@ -1,3 +1,4 @@
+export { createIntlSegmentTextTransformer } from './IntlSegmentTextTransformer.js';
 export { createMappedTextSegmenter, mapRangeBackToOriginalPos } from './parsedText.js';
 export type { SourceMapCursor } from './SourceMap.js';
 export { createSourceMapCursor } from './SourceMap.js';
@@ -5,5 +6,7 @@ export type { SubstitutionInfo } from './SubstitutionTransformer.js';
 export { createSubstitutionTransformer, SubstitutionTransformer } from './SubstitutionTransformer.js';
 export type { MatchRange } from './TextRange.js';
 export { excludeRanges, extractRangeText, findMatchingRanges, findMatchingRangesForPatterns } from './TextRange.js';
+export type { TextTransformer } from './Transformer.js';
+export { chainTransformers } from './Transformer.js';
 export type { Offset, SimpleRange } from './types.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/lib/Transform/index.js

@@ -1,5 +1,7 @@
+export { createIntlSegmentTextTransformer } from './IntlSegmentTextTransformer.js';
 export { createMappedTextSegmenter, mapRangeBackToOriginalPos } from './parsedText.js';
 export { createSourceMapCursor } from './SourceMap.js';
 export { createSubstitutionTransformer, SubstitutionTransformer } from './SubstitutionTransformer.js';
 export { excludeRanges, extractRangeText, findMatchingRanges, findMatchingRangesForPatterns } from './TextRange.js';
+export { chainTransformers } from './Transformer.js';
 //# sourceMappingURL=index.js.map

package/dist/lib/textValidation/docValidator.d.ts

@@ -6,7 +6,7 @@ import type { ValidationIssue } from '../Models/ValidationIssue.js';
 import type { CSpellSettingsInternal, CSpellSettingsInternalFinalized, ImportFileRefWithError } from '../Settings/index.js';
 import type { SpellingDictionaryCollection } from '../SpellingDictionary/index.js';
 import type { WordSuggestion } from '../suggestions.js';
-import type { MatchRange, SimpleRange, SubstitutionTransformer } from '../Transform/index.js';
+import type { MatchRange, SimpleRange, TextTransformer } from '../Transform/index.js';
 import type { TextValidationFactoryOptions, TextValidator } from './lineValidatorFactory.js';
 import type { TraceResult } from './traceWord.js';
 import type { ValidateTextOptions } from './ValidateTextOptions.js';
@@ -37,6 +37,7 @@ export interface DocumentValidatorOptions extends ValidateTextOptions {
 }
 type PerfTimings = Record<string, number>;
 export declare class DocumentValidator {
+    #private;
     readonly settings: CSpellUserSettings;
     private _document;
     private _ready;
@@ -133,7 +134,7 @@ interface Preparations {
     validateOptions: TextValidationFactoryOptions;
     localConfig: CSpellUserSettings | undefined;
     localConfigFilepath: string | undefined;
-    subTransformer: SubstitutionTransformer;
+    transformer: TextTransformer;
 }
 interface ShouldCheckDocumentResult {
     /** possible errors found while loading configuration. */

package/dist/lib/textValidation/docValidator.js

@@ -10,7 +10,7 @@ import { extractImportErrors, finalizeSettings, loadConfig, mergeSettings, resol
 import { validateInDocumentSettings } from '../Settings/InDocSettings.js';
 import { getDictionaryInternal } from '../SpellingDictionary/index.js';
 import { calcSuggestionAdjustedToToMatchCase } from '../suggestions.js';
-import { createMappedTextSegmenter, createSubstitutionTransformer } from '../Transform/index.js';
+import { chainTransformers, createIntlSegmentTextTransformer, createMappedTextSegmenter, createSubstitutionTransformer, } from '../Transform/index.js';
 import { catchPromiseError, toError } from '../util/errors.js';
 import { AutoCache } from '../util/simpleCache.js';
 import { uriToFilePath } from '../util/Uri.js';
@@ -101,15 +101,7 @@ export class DocumentValidator {
         recShouldCheckTime();
         const recFinalizeTime = recordPerfTime(this.perfTiming, '_finalizeSettings');
         const finalSettings = finalizeSettings(docSettings);
-        const sub = createSubstitutionTransformer(finalSettings);
-        if (sub.missing) {
-            this.addPossibleError(`Missing substitutions: ${sub.missing.join(', ')}`);
-        }
-        const validateOptions = { ...settingsToValidateOptions(finalSettings), transformer: sub.transformer };
-        const rangeTransformer = finalSettings.substitutions?.length ? sub.transformer : undefined;
-        const includeRanges = calcTextInclusionRanges(this._document.text, validateOptions, rangeTransformer);
-        const segmenter = createMappedTextSegmenter(includeRanges);
-        const textValidator = textValidatorFactory(dict, validateOptions);
+        const { transformer, validateOptions, includeRanges, segmenter, textValidator } = this.#calcSharedPrep(finalSettings, dict);
         recFinalizeTime();
         this._preparations = {
             config,
@@ -123,7 +115,7 @@ export class DocumentValidator {
             textValidator,
             localConfig,
             localConfigFilepath: localConfig?.__importRef?.filename,
-            subTransformer: sub.transformer,
+            transformer,
         };
         this._ready = true;
         this._preparationTime = timer.elapsed;
@@ -139,28 +131,38 @@ export class DocumentValidator {
         const stopMeasure = measurePerf('DocumentValidator._updatePrep');
         const shouldCheck = docSettings.enabled ?? true;
         const finalSettings = finalizeSettings(docSettings);
-        const sub = createSubstitutionTransformer(finalSettings);
-        if (sub.missing) {
-            this.addPossibleError(`Missing substitutions: ${sub.missing.join(', ')}`);
-        }
-        const validateOptions = { ...settingsToValidateOptions(finalSettings), transformer: sub.transformer };
-        const rangeTransformer = finalSettings.substitutions?.length ? sub.transformer : undefined;
-        const includeRanges = calcTextInclusionRanges(this._document.text, validateOptions, rangeTransformer);
-        const segmenter = createMappedTextSegmenter(includeRanges);
-        const textValidator = textValidatorFactory(dict, validateOptions);
+        const { transformer, validateOptions, includeRanges, segmenter, textValidator } = this.#calcSharedPrep(finalSettings, dict);
         this._preparations = {
             ...prep,
             dictionary: dict,
             docSettings,
+            finalSettings,
             shouldCheck,
             validateOptions,
             includeRanges,
             segmenter,
             textValidator,
+            transformer,
         };
         this._preparationTime = timer.elapsed;
         stopMeasure();
     }
+    #calcSharedPrep(finalSettings, dict) {
+        const sub = createSubstitutionTransformer(finalSettings);
+        if (sub.missing) {
+            this.addPossibleError(`Missing substitutions: ${sub.missing.join(', ')}`);
+        }
+        const transformer = finalSettings.useIntlWordSegmentation
+            ? chainTransformers(sub.transformer, createIntlSegmentTextTransformer(finalSettings.language || ''))
+            : sub.transformer;
+        const validateOptions = { ...settingsToValidateOptions(finalSettings), transformer };
+        // Avoid excluding text ranges that have a full substitution applied.
+        const rangeTransformer = finalSettings.substitutions?.length ? sub.transformer : undefined;
+        const includeRanges = calcTextInclusionRanges(this._document.text, validateOptions, rangeTransformer);
+        const segmenter = createMappedTextSegmenter(includeRanges);
+        const textValidator = textValidatorFactory(dict, validateOptions);
+        return { segmenter, textValidator, includeRanges, validateOptions, transformer };
+    }
     /**
      * The amount of time in ms to prepare for validation.
      */

package/dist/lib/textValidation/isRandomString.js

@@ -1,3 +1,4 @@
+import { softHyphen } from '../util/wordSplitter/index.js';
 const maxRadio = 0.5;
 /**
  * Try to detect if a string is a random string of characters or is it camel case / snake case words.
@@ -5,6 +6,9 @@ const maxRadio = 0.5;
  * @returns true if the string is considered random;
  */
 export function isRandomString(s, maxNoiseToLengthRatio = maxRadio) {
+    // Soft hyphen indicates a word break, so it's likely not a random string.
+    if (s.includes(softHyphen))
+        return false;
     return scoreRandomString(s) >= maxNoiseToLengthRatio;
 }
 /**
@@ -28,9 +32,6 @@ function sumWordLengths(s) {
     }
     return total;
 }
-function _nonLetterRatio(s) {
-    return s.replaceAll(/\p{L}/gu, '').length / s.length;
-}
 export function categorizeString(s) {
     const n = s
         .replaceAll(/\d+/g, '0')

package/dist/lib/textValidation/lineValidatorFactory.d.ts

@@ -1,5 +1,5 @@
 import type { CachingDictionary, SpellingDictionary } from 'cspell-dictionary';
-import type { SubstitutionTransformer } from '../Transform/index.js';
+import type { TextTransformer } from '../Transform/index.js';
 import type { LineValidatorFn, TextValidatorFn, ValidationOptions } from './ValidationTypes.js';
 interface LineValidator {
     fn: LineValidatorFn;
@@ -11,7 +11,7 @@ export interface TextValidator {
     lineValidator: LineValidator;
 }
 export interface TextValidationFactoryOptions extends ValidationOptions {
-    transformer: SubstitutionTransformer | undefined;
+    transformer: TextTransformer | undefined;
 }
 export declare function textValidatorFactory(dict: SpellingDictionary, options: TextValidationFactoryOptions): TextValidator;
 export {};

package/dist/lib/textValidation/lineValidatorFactory.js

@@ -7,7 +7,7 @@ import { mapRangeBackToOriginalPos } from '../Transform/index.js';
 import { autoResolve } from '../util/AutoResolve.js';
 import { extractPossibleWordsFromTextOffset, extractText, extractWordsFromTextOffset, splitWordWithOffset, } from '../util/text.js';
 import { regExpCamelCaseWordBreaksWithEnglishSuffix } from '../util/textRegex.js';
-import { split } from '../util/wordSplitter.js';
+import { split } from '../util/wordSplitter/index.js';
 import { defaultMinWordLength } from './defaultConstants.js';
 import { extractHexSequences, isRandomString } from './isRandomString.js';
 import { isWordValidWithEscapeRetry } from './isWordValid.js';
@@ -327,7 +327,12 @@ export function lineValidatorFactory(sDict, options) {
                     const v = checkWord({ ...w, text: m[1], line: lineSegment.line });
                     return v.isFlagged || !v.isFound;
                 });
-                const filtered = filterExcludedTextOffsets(nonMatching.map((w) => ({ ...w, line: lineSegment.line })).map(annotateIsFlagged), hexSequences);
+                // Ignore 1 or 2 letter segments.
+                const minWordLen = Math.min(3, minWordLength);
+                const filtered = filterExcludedTextOffsets(nonMatching
+                    .map((w) => ({ ...w, line: lineSegment.line }))
+                    .map(annotateIsFlagged)
+                    .filter((w) => w.isFlagged || w.text.length >= minWordLen), hexSequences);
                 if (filtered.length < mismatches.length) {
                     return filtered;
                 }

package/dist/lib/textValidation/textValidator.d.ts

@@ -1,6 +1,6 @@
 import type { SpellingDictionary } from 'cspell-dictionary';
 import type { ValidationResult } from '../Models/ValidationResult.js';
-import type { SubstitutionTransformer } from '../Transform/index.js';
+import type { TextTransformer } from '../Transform/index.js';
 import * as TextRange from '../Transform/index.js';
 import type { IncludeExcludeOptions, LineSegment, ValidationOptions } from './ValidationTypes.js';
 /**
@@ -8,7 +8,7 @@ import type { IncludeExcludeOptions, LineSegment, ValidationOptions } from './Va
  * @deprecation Use spellCheckDocument
  */
 export declare function validateText(text: string, dict: SpellingDictionary, options: ValidationOptions): Iterable<ValidationResult>;
-export declare function calcTextInclusionRanges(text: string, options: IncludeExcludeOptions, transformer?: SubstitutionTransformer): TextRange.MatchRange[];
+export declare function calcTextInclusionRanges(text: string, options: IncludeExcludeOptions, transformer?: TextTransformer): TextRange.MatchRange[];
 /**
  * Returns a mapper function that will segment a TextOffset based upon the includeRanges.
  * This function is optimized for forward scanning. It will perform poorly for randomly ordered offsets.

package/dist/lib/textValidation/traceWord.js

@@ -2,7 +2,7 @@ import { getSources } from '../Settings/index.js';
 import { createCollection, getInlineConfigDictionaries, mapSpecialDictionaryNamesToSettings, } from '../SpellingDictionary/index.js';
 import { toFileUrl } from '../util/url.js';
 import { uniqueFn } from '../util/util.js';
-import { split } from '../util/wordSplitter.js';
+import { split } from '../util/wordSplitter/index.js';
 export function traceWord(word, dictCollection, config) {
     const opts = {
         ignoreCase: config.ignoreCase ?? true,

package/dist/lib/util/textRegex.js

@@ -7,7 +7,7 @@ export const regExpCamelCaseWordBreaks_NOT_USED = /(?<=\p{Ll}\p{M}?)(?=\p{Lu})|(
 export const regExpAllPossibleWordBreaks_NOT_USED = /(?<=\p{Ll}\p{M}?)(?=\p{Lu})|(?<=\p{Lu}\p{M}?)(?=\p{Lu}\p{M}?\p{Ll})|(?<=\p{Lu}\p{M}?\p{Lu}\p{M}?)(?=\p{Ll})|(?<=\p{L}\p{M}?)(?=\P{L})|(?<=\P{L})(?=\p{L})/gu;
 export const regExWords = /\p{L}\p{M}?(?:(?:\\?['’])?\p{L}\p{M}?)*/gu;
 // Words can be made of letters, numbers, period, underscore, dash, plus, and single quote
-export const regExWordsAndDigits = /[\p{L}\w'’`.+-](?:(?:\\(?=[']))?[\p{L}\p{M}\w'’`.+-])*/gu;
+export const regExWordsAndDigits = /[\p{L}\w'’`.+\u00AD-](?:(?:\\(?=[']))?[\p{L}\p{M}\w'’`.+\u00AD-])*/gu;
 export const regExIgnoreCharacters = /[\p{sc=Hiragana}\p{sc=Han}\p{sc=Katakana}\u30A0-\u30FF\p{sc=Hangul}]/gu;
 export const regExFirstUpper = /^\p{Lu}\p{M}?\p{Ll}+$/u;
 export const regExAllUpper = /^(?:\p{Lu}\p{M}?)+$/u;

package/dist/lib/util/wordSplitter/generateWordBreaks.d.ts

@@ -0,0 +1,32 @@
+import type { TextOffset } from '@cspell/cspell-types';
+export declare const softHyphen = "\u00AD";
+export interface LineSegment {
+    line: TextOffset;
+    /**
+     * The starting offset within line.text.
+     * The absolute offset can be obtained by adding line.offset.
+     */
+    relStart: number;
+    /**
+     * The ending offset within line.text.
+     * The absolute offset can be obtained by adding line.offset.
+     */
+    relEnd: number;
+}
+export type BreakPairs = readonly [number, number];
+export interface PossibleWordBreak {
+    /** offset from the start of the string */
+    offset: number;
+    /**
+     * break pairs (start, end)
+     * (the characters between the start and end are removed)
+     * With a pure break, start === end.
+     */
+    breaks: BreakPairs[];
+}
+export type SortedBreaks = PossibleWordBreak[];
+export interface WordBreakOptions {
+    optionalWordBreakCharacters?: string;
+}
+export declare function generateWordBreaks(line: LineSegment, options: WordBreakOptions): SortedBreaks;
+//# sourceMappingURL=generateWordBreaks.d.ts.map