cspell-lib 10.2.0-alpha.7 → 10.2.0

package/dist/lib/Settings/LanguageSettings.js

@@ -38,9 +38,25 @@ export function normalizeLocale(locale) {
     return _normalizeLocale(locale);
 }
 export function normalizeLocaleIntl(locale) {
-    const values = [...normalizeLocale(locale)].map((locale) => locale.replace(/^([a-z]{2})-?([a-z]{2})$/, (_, lang, locale) => locale ? `${lang}-${locale.toUpperCase()}` : lang));
+    const values = [...normalizeLocale(locale)].map((locale) => locale.replace(/^([a-z]{2})(?:-?([a-z]{4}))?-?([a-z]{2})$/, (_, lang, script, region) => toLocaleIntl(lang, script, region)));
     return new Set(values);
 }
+/**
+ * See: https://developer.mozilla.org/en-US/docs/Glossary/BCP_47_language_tag
+ * @returns locale normalized to BCP_47
+ */
+function toLocaleIntl(lang, script, region) {
+    const parts = [lang];
+    if (script)
+        parts.push(firstUpper(script.toLowerCase()));
+    if (region)
+        parts.push(region.toUpperCase());
+    return parts.join('-');
+}
+function firstUpper(str) {
+    const [first, ...rest] = [...str];
+    return [first.toUpperCase(), ...rest].join('');
+}
 export function isLocaleInSet(locale, setOfLocals) {
     const locales = normalizeLocale(locale);
     return doSetsIntersect(locales, setOfLocals);

package/dist/lib/Settings/sanitizeSettings.js

@@ -68,6 +68,7 @@ const handlers = {
     suggestWords: copy1,
     unknownWords: copy1,
     useGitignore: copy1,
+    useIntlWordSegmentation: copy1,
     usePnP: skip,
     userWords: copy1,
     validateDirectives: copy1,
@@ -183,6 +184,7 @@ const LanguageSettingsHandlers = {
     substitutions: copy1,
     suggestWords: copy1,
     unknownWords: copy1,
+    useIntlWordSegmentation: copy1,
     words: copy1,
     // Experimental
     parser: skip,
@@ -245,6 +247,7 @@ const OverridesHandlers = {
     suggestionsTimeout: copy1,
     suggestWords: copy1,
     unknownWords: copy1,
+    useIntlWordSegmentation: copy1,
     usePnP: skip,
     words: copy1,
     parser: skip,

package/dist/lib/Transform/IntlSegmentTextTransformer.d.ts

@@ -0,0 +1,10 @@
+import type { MappedText } from '@cspell/cspell-types';
+import type { TextTransformer } from './Transformer.js';
+export declare class IntlSegmentTextTransformer implements TextTransformer {
+    #private;
+    constructor(locale: string | string[]);
+    transform(text: string | MappedText): MappedText;
+    transformAll(src: Iterable<string | MappedText>): Iterable<MappedText>;
+}
+export declare function createIntlSegmentTextTransformer(locale: string | string[]): IntlSegmentTextTransformer;
+//# sourceMappingURL=IntlSegmentTextTransformer.d.ts.map

package/dist/lib/Transform/IntlSegmentTextTransformer.js

@@ -0,0 +1,40 @@
+import { normalizeLocaleIntl } from '../Settings/LanguageSettings.js';
+import { applyEditsToMappedText } from './TextMapEdit.js';
+import { toMappedText } from './Transformer.js';
+const regExSplitWordCharacters = /(?<=[\p{L}\p{M}])[\p{L}\p{M}]/uy;
+export class IntlSegmentTextTransformer {
+    #locale;
+    constructor(locale) {
+        this.#locale = [...normalizeLocaleIntl(locale)];
+    }
+    transform(text) {
+        // Implement the transformation logic here
+        text = toMappedText(text);
+        return segmentText(text, new Intl.Segmenter(this.#locale, { granularity: 'word' }));
+    }
+    *transformAll(src) {
+        for (const item of src) {
+            yield this.transform(item);
+        }
+    }
+}
+function segmentText(mText, segmenter) {
+    const edits = [];
+    let lastIndex = -1;
+    const reg = new RegExp(regExSplitWordCharacters);
+    const text = mText.text;
+    for (const { segment, index } of segmenter.segment(mText.text)) {
+        const endIndex = index + segment.length;
+        reg.lastIndex = index;
+        if (index === lastIndex && reg.test(text)) {
+            const edit = { text: ' ' + segment, range: [index, endIndex] };
+            edits.push(edit);
+        }
+        lastIndex = endIndex;
+    }
+    return applyEditsToMappedText(mText, edits);
+}
+export function createIntlSegmentTextTransformer(locale) {
+    return new IntlSegmentTextTransformer(locale);
+}
+//# sourceMappingURL=IntlSegmentTextTransformer.js.map

package/dist/lib/Transform/SubstitutionTransformer.d.ts

@@ -1,4 +1,5 @@
 import type { MappedText, SubstitutionDefinitions, Substitutions } from '@cspell/cspell-types';
+import type { TextTransformer } from './Transformer.js';
 type DeepReadonly<T> = T extends object ? {
     readonly [K in keyof T]: DeepReadonly<T[K]>;
 } : T;
@@ -7,10 +8,11 @@ export interface SubstitutionInfo {
     substitutionDefinitions?: SubstitutionDefinitions | undefined;
 }
 export type ReadonlySubstitutionInfo = DeepReadonly<SubstitutionInfo>;
-export declare class SubstitutionTransformer {
+export declare class SubstitutionTransformer implements TextTransformer {
     #private;
     constructor(subMap: Map<string, string> | undefined);
     transform(text: string | MappedText): MappedText;
+    transformAll(src: Iterable<string | MappedText>): Iterable<MappedText>;
     transformString(text: string): MappedText;
 }
 export interface CreateSubstitutionTransformerResult {

package/dist/lib/Transform/SubstitutionTransformer.js

@@ -1,5 +1,7 @@
 import { GTrie } from 'cspell-trie-lib';
-import { mapOffsetPairsToSourceMap, mergeSourceMaps } from './SourceMap.js';
+import { mergeSourceMaps } from './SourceMap.js';
+import { applyEditsToText } from './TextMapEdit.js';
+import { toMappedText } from './Transformer.js';
 export class SubstitutionTransformer {
     #trie;
     constructor(subMap) {
@@ -21,36 +23,16 @@ export class SubstitutionTransformer {
         };
         return result;
     }
+    *transformAll(src) {
+        for (const item of src) {
+            yield this.transform(item);
+        }
+    }
     transformString(text) {
         if (!this.#trie) {
-            return { text, range: [0, text.length], rawText: text };
-        }
-        const map = [0, 0];
-        let repText = '';
-        let lastEnd = 0;
-        for (const edit of calcEdits(text, this.#trie)) {
-            if (edit.range[0] > lastEnd) {
-                repText += text.slice(lastEnd, edit.range[0]);
-                map.push(edit.range[0], repText.length);
-            }
-            repText += edit.text;
-            map.push(edit.range[1], repText.length);
-            lastEnd = edit.range[1];
-        }
-        if (lastEnd === 0) {
-            return { text, range: [0, text.length], rawText: text };
+            return toMappedText(text);
         }
-        if (lastEnd < text.length) {
-            repText += text.slice(lastEnd);
-            map.push(text.length, repText.length);
-        }
-        const result = {
-            text: repText,
-            range: [0, text.length],
-            map: mapOffsetPairsToSourceMap(map),
-            rawText: text,
-        };
-        return result;
+        return applyEditsToText(text, calcEdits(text, this.#trie));
     }
 }
 function* calcEdits(text, subTrie) {

package/dist/lib/Transform/TextMapEdit.d.ts

@@ -0,0 +1,5 @@
+import type { MappedText } from '@cspell/cspell-types';
+import type { Edit } from './Transformer.js';
+export declare function applyEditsToMappedText(text: MappedText, edits: Iterable<Edit>): MappedText;
+export declare function applyEditsToText(text: string, edits: Iterable<Edit>): MappedText;
+//# sourceMappingURL=TextMapEdit.d.ts.map

package/dist/lib/Transform/TextMapEdit.js

@@ -0,0 +1,42 @@
+import { mapOffsetPairsToSourceMap, mergeSourceMaps } from './SourceMap.js';
+import { toMappedText } from './Transformer.js';
+export function applyEditsToMappedText(text, edits) {
+    const transformed = applyEditsToText(text.text, edits);
+    const rawText = text.rawText ?? text.text;
+    const result = {
+        ...text,
+        text: transformed.text,
+        map: mergeSourceMaps(text.map, transformed.map),
+        rawText,
+    };
+    return result;
+}
+export function applyEditsToText(text, edits) {
+    const map = [0, 0];
+    let repText = '';
+    let lastEnd = 0;
+    for (const edit of edits) {
+        if (edit.range[0] > lastEnd) {
+            repText += text.slice(lastEnd, edit.range[0]);
+            map.push(edit.range[0], repText.length);
+        }
+        repText += edit.text;
+        map.push(edit.range[1], repText.length);
+        lastEnd = edit.range[1];
+    }
+    if (lastEnd === 0) {
+        return toMappedText(text);
+    }
+    if (lastEnd < text.length) {
+        repText += text.slice(lastEnd);
+        map.push(text.length, repText.length);
+    }
+    const result = {
+        text: repText,
+        range: [0, text.length],
+        map: mapOffsetPairsToSourceMap(map),
+        rawText: text,
+    };
+    return result;
+}
+//# sourceMappingURL=TextMapEdit.js.map

package/dist/lib/Transform/Transformer.d.ts

@@ -0,0 +1,25 @@
+import type { MappedText } from '@cspell/cspell-types';
+import type { Range } from '@cspell/cspell-types/Parser';
+export interface TextTransformer {
+    transform(text: string | MappedText): MappedText;
+    transformAll(src: Iterable<string | MappedText>): Iterable<MappedText>;
+}
+export interface Edit {
+    /**
+     * The replacement text for the edit. This is the text that will replace the original text in the string.
+     */
+    text: string;
+    /**
+     * The range of the text to be replaced. This is a tuple of the form `[start, end]`,
+     * where start is the index of the first character to be replaced,
+     * and end is the index of the first character after the last character to be replaced.
+     */
+    range: Range;
+}
+export declare function toMappedText(text: string | MappedText): MappedText;
+export declare function chainTransformers(...transformers: TextTransformer[]): TextTransformer;
+export declare class IdentityTextTransformer implements TextTransformer {
+    transform(text: string | MappedText): MappedText;
+    transformAll(src: Iterable<string | MappedText>): Iterable<MappedText>;
+}
+//# sourceMappingURL=Transformer.d.ts.map

package/dist/lib/Transform/Transformer.js

@@ -0,0 +1,27 @@
+export function toMappedText(text) {
+    if (typeof text === 'string') {
+        return { text, range: [0, text.length], rawText: text };
+    }
+    return text;
+}
+export function chainTransformers(...transformers) {
+    return {
+        transform(text) {
+            return transformers.reduce((acc, transformer) => transformer.transform(acc), toMappedText(text));
+        },
+        transformAll(src) {
+            return Array.from(src, (text) => transformers.reduce((acc, transformer) => transformer.transform(acc), toMappedText(text)));
+        },
+    };
+}
+export class IdentityTextTransformer {
+    transform(text) {
+        return toMappedText(text);
+    }
+    *transformAll(src) {
+        for (const item of src) {
+            yield this.transform(item);
+        }
+    }
+}
+//# sourceMappingURL=Transformer.js.map

package/dist/lib/Transform/index.d.ts

@@ -1,3 +1,4 @@
+export { createIntlSegmentTextTransformer } from './IntlSegmentTextTransformer.js';
 export { createMappedTextSegmenter, mapRangeBackToOriginalPos } from './parsedText.js';
 export type { SourceMapCursor } from './SourceMap.js';
 export { createSourceMapCursor } from './SourceMap.js';
@@ -5,5 +6,7 @@ export type { SubstitutionInfo } from './SubstitutionTransformer.js';
 export { createSubstitutionTransformer, SubstitutionTransformer } from './SubstitutionTransformer.js';
 export type { MatchRange } from './TextRange.js';
 export { excludeRanges, extractRangeText, findMatchingRanges, findMatchingRangesForPatterns } from './TextRange.js';
+export type { TextTransformer } from './Transformer.js';
+export { chainTransformers } from './Transformer.js';
 export type { Offset, SimpleRange } from './types.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/lib/Transform/index.js

@@ -1,5 +1,7 @@
+export { createIntlSegmentTextTransformer } from './IntlSegmentTextTransformer.js';
 export { createMappedTextSegmenter, mapRangeBackToOriginalPos } from './parsedText.js';
 export { createSourceMapCursor } from './SourceMap.js';
 export { createSubstitutionTransformer, SubstitutionTransformer } from './SubstitutionTransformer.js';
 export { excludeRanges, extractRangeText, findMatchingRanges, findMatchingRangesForPatterns } from './TextRange.js';
+export { chainTransformers } from './Transformer.js';
 //# sourceMappingURL=index.js.map

package/dist/lib/textValidation/docValidator.d.ts

@@ -6,7 +6,7 @@ import type { ValidationIssue } from '../Models/ValidationIssue.js';
 import type { CSpellSettingsInternal, CSpellSettingsInternalFinalized, ImportFileRefWithError } from '../Settings/index.js';
 import type { SpellingDictionaryCollection } from '../SpellingDictionary/index.js';
 import type { WordSuggestion } from '../suggestions.js';
-import type { MatchRange, SimpleRange, SubstitutionTransformer } from '../Transform/index.js';
+import type { MatchRange, SimpleRange, TextTransformer } from '../Transform/index.js';
 import type { TextValidationFactoryOptions, TextValidator } from './lineValidatorFactory.js';
 import type { TraceResult } from './traceWord.js';
 import type { ValidateTextOptions } from './ValidateTextOptions.js';
@@ -37,6 +37,7 @@ export interface DocumentValidatorOptions extends ValidateTextOptions {
 }
 type PerfTimings = Record<string, number>;
 export declare class DocumentValidator {
+    #private;
     readonly settings: CSpellUserSettings;
     private _document;
     private _ready;
@@ -133,7 +134,7 @@ interface Preparations {
     validateOptions: TextValidationFactoryOptions;
     localConfig: CSpellUserSettings | undefined;
     localConfigFilepath: string | undefined;
-    subTransformer: SubstitutionTransformer;
+    transformer: TextTransformer;
 }
 interface ShouldCheckDocumentResult {
     /** possible errors found while loading configuration. */

package/dist/lib/textValidation/docValidator.js

@@ -10,7 +10,7 @@ import { extractImportErrors, finalizeSettings, loadConfig, mergeSettings, resol
 import { validateInDocumentSettings } from '../Settings/InDocSettings.js';
 import { getDictionaryInternal } from '../SpellingDictionary/index.js';
 import { calcSuggestionAdjustedToToMatchCase } from '../suggestions.js';
-import { createMappedTextSegmenter, createSubstitutionTransformer } from '../Transform/index.js';
+import { chainTransformers, createIntlSegmentTextTransformer, createMappedTextSegmenter, createSubstitutionTransformer, } from '../Transform/index.js';
 import { catchPromiseError, toError } from '../util/errors.js';
 import { AutoCache } from '../util/simpleCache.js';
 import { uriToFilePath } from '../util/Uri.js';
@@ -101,15 +101,7 @@ export class DocumentValidator {
         recShouldCheckTime();
         const recFinalizeTime = recordPerfTime(this.perfTiming, '_finalizeSettings');
         const finalSettings = finalizeSettings(docSettings);
-        const sub = createSubstitutionTransformer(finalSettings);
-        if (sub.missing) {
-            this.addPossibleError(`Missing substitutions: ${sub.missing.join(', ')}`);
-        }
-        const validateOptions = { ...settingsToValidateOptions(finalSettings), transformer: sub.transformer };
-        const rangeTransformer = finalSettings.substitutions?.length ? sub.transformer : undefined;
-        const includeRanges = calcTextInclusionRanges(this._document.text, validateOptions, rangeTransformer);
-        const segmenter = createMappedTextSegmenter(includeRanges);
-        const textValidator = textValidatorFactory(dict, validateOptions);
+        const { transformer, validateOptions, includeRanges, segmenter, textValidator } = this.#calcSharedPrep(finalSettings, dict);
         recFinalizeTime();
         this._preparations = {
             config,
@@ -123,7 +115,7 @@ export class DocumentValidator {
             textValidator,
             localConfig,
             localConfigFilepath: localConfig?.__importRef?.filename,
-            subTransformer: sub.transformer,
+            transformer,
         };
         this._ready = true;
         this._preparationTime = timer.elapsed;
@@ -139,28 +131,38 @@ export class DocumentValidator {
         const stopMeasure = measurePerf('DocumentValidator._updatePrep');
         const shouldCheck = docSettings.enabled ?? true;
         const finalSettings = finalizeSettings(docSettings);
-        const sub = createSubstitutionTransformer(finalSettings);
-        if (sub.missing) {
-            this.addPossibleError(`Missing substitutions: ${sub.missing.join(', ')}`);
-        }
-        const validateOptions = { ...settingsToValidateOptions(finalSettings), transformer: sub.transformer };
-        const rangeTransformer = finalSettings.substitutions?.length ? sub.transformer : undefined;
-        const includeRanges = calcTextInclusionRanges(this._document.text, validateOptions, rangeTransformer);
-        const segmenter = createMappedTextSegmenter(includeRanges);
-        const textValidator = textValidatorFactory(dict, validateOptions);
+        const { transformer, validateOptions, includeRanges, segmenter, textValidator } = this.#calcSharedPrep(finalSettings, dict);
         this._preparations = {
             ...prep,
             dictionary: dict,
             docSettings,
+            finalSettings,
             shouldCheck,
             validateOptions,
             includeRanges,
             segmenter,
             textValidator,
+            transformer,
         };
         this._preparationTime = timer.elapsed;
         stopMeasure();
     }
+    #calcSharedPrep(finalSettings, dict) {
+        const sub = createSubstitutionTransformer(finalSettings);
+        if (sub.missing) {
+            this.addPossibleError(`Missing substitutions: ${sub.missing.join(', ')}`);
+        }
+        const transformer = finalSettings.useIntlWordSegmentation
+            ? chainTransformers(sub.transformer, createIntlSegmentTextTransformer(finalSettings.language || ''))
+            : sub.transformer;
+        const validateOptions = { ...settingsToValidateOptions(finalSettings), transformer };
+        // Avoid excluding text ranges that have a full substitution applied.
+        const rangeTransformer = finalSettings.substitutions?.length ? sub.transformer : undefined;
+        const includeRanges = calcTextInclusionRanges(this._document.text, validateOptions, rangeTransformer);
+        const segmenter = createMappedTextSegmenter(includeRanges);
+        const textValidator = textValidatorFactory(dict, validateOptions);
+        return { segmenter, textValidator, includeRanges, validateOptions, transformer };
+    }
     /**
      * The amount of time in ms to prepare for validation.
      */

package/dist/lib/textValidation/lineValidatorFactory.d.ts

@@ -1,5 +1,5 @@
 import type { CachingDictionary, SpellingDictionary } from 'cspell-dictionary';
-import type { SubstitutionTransformer } from '../Transform/index.js';
+import type { TextTransformer } from '../Transform/index.js';
 import type { LineValidatorFn, TextValidatorFn, ValidationOptions } from './ValidationTypes.js';
 interface LineValidator {
     fn: LineValidatorFn;
@@ -11,7 +11,7 @@ export interface TextValidator {
     lineValidator: LineValidator;
 }
 export interface TextValidationFactoryOptions extends ValidationOptions {
-    transformer: SubstitutionTransformer | undefined;
+    transformer: TextTransformer | undefined;
 }
 export declare function textValidatorFactory(dict: SpellingDictionary, options: TextValidationFactoryOptions): TextValidator;
 export {};

package/dist/lib/textValidation/textValidator.d.ts

@@ -1,6 +1,6 @@
 import type { SpellingDictionary } from 'cspell-dictionary';
 import type { ValidationResult } from '../Models/ValidationResult.js';
-import type { SubstitutionTransformer } from '../Transform/index.js';
+import type { TextTransformer } from '../Transform/index.js';
 import * as TextRange from '../Transform/index.js';
 import type { IncludeExcludeOptions, LineSegment, ValidationOptions } from './ValidationTypes.js';
 /**
@@ -8,7 +8,7 @@ import type { IncludeExcludeOptions, LineSegment, ValidationOptions } from './Va
  * @deprecation Use spellCheckDocument
  */
 export declare function validateText(text: string, dict: SpellingDictionary, options: ValidationOptions): Iterable<ValidationResult>;
-export declare function calcTextInclusionRanges(text: string, options: IncludeExcludeOptions, transformer?: SubstitutionTransformer): TextRange.MatchRange[];
+export declare function calcTextInclusionRanges(text: string, options: IncludeExcludeOptions, transformer?: TextTransformer): TextRange.MatchRange[];
 /**
  * Returns a mapper function that will segment a TextOffset based upon the includeRanges.
  * This function is optimized for forward scanning. It will perform poorly for randomly ordered offsets.

package/package.json

@@ -4,7 +4,7 @@
     "access": "public",
     "provenance": true
   },
-  "version": "10.2.0-alpha.7",
+  "version": "10.2.0",
   "description": "A library of useful functions used across various cspell tools.",
   "type": "module",
   "sideEffects": false,
@@ -75,22 +75,22 @@
   },
   "homepage": "https://github.com/streetsidesoftware/cspell/tree/main/packages/cspell-lib#readme",
   "dependencies": {
-    "@cspell/cspell-bundled-dicts": "10.2.0-alpha.7",
-    "@cspell/cspell-performance-monitor": "10.2.0-alpha.7",
-    "@cspell/cspell-pipe": "10.2.0-alpha.7",
-    "@cspell/cspell-resolver": "10.2.0-alpha.7",
-    "@cspell/cspell-types": "10.2.0-alpha.7",
-    "@cspell/dynamic-import": "10.2.0-alpha.7",
-    "@cspell/filetypes": "10.2.0-alpha.7",
-    "@cspell/rpc": "10.2.0-alpha.7",
-    "@cspell/strong-weak-map": "10.2.0-alpha.7",
-    "@cspell/url": "10.2.0-alpha.7",
-    "cspell-config-lib": "10.2.0-alpha.7",
-    "cspell-dictionary": "10.2.0-alpha.7",
-    "cspell-glob": "10.2.0-alpha.7",
-    "cspell-grammar": "10.2.0-alpha.7",
-    "cspell-io": "10.2.0-alpha.7",
-    "cspell-trie-lib": "10.2.0-alpha.7",
+    "@cspell/cspell-bundled-dicts": "10.2.0",
+    "@cspell/cspell-performance-monitor": "10.2.0",
+    "@cspell/cspell-pipe": "10.2.0",
+    "@cspell/cspell-resolver": "10.2.0",
+    "@cspell/cspell-types": "10.2.0",
+    "@cspell/dynamic-import": "10.2.0",
+    "@cspell/filetypes": "10.2.0",
+    "@cspell/rpc": "10.2.0",
+    "@cspell/strong-weak-map": "10.2.0",
+    "@cspell/url": "10.2.0",
+    "cspell-config-lib": "10.2.0",
+    "cspell-dictionary": "10.2.0",
+    "cspell-glob": "10.2.0",
+    "cspell-grammar": "10.2.0",
+    "cspell-io": "10.2.0",
+    "cspell-trie-lib": "10.2.0",
     "env-paths": "^4.0.0",
     "gensequence": "^8.0.8",
     "import-fresh": "^4.0.0",
@@ -103,7 +103,7 @@
     "node": ">=22.18.0"
   },
   "devDependencies": {
-    "@cspell/cspell-tools": "10.2.0-alpha.7",
+    "@cspell/cspell-tools": "10.2.0",
     "@cspell/dict-cpp": "^7.1.0",
     "@cspell/dict-csharp": "^4.0.8",
     "@cspell/dict-css": "^4.1.2",
@@ -122,5 +122,5 @@
     "lorem-ipsum": "^3.0.0",
     "perf-insight": "^2.0.1"
   },
-  "gitHead": "308c7ea1e38518476e0ef060bfb561ca43e24892"
+  "gitHead": "2d310afc22fa500b1be07e3c2e2d3897ec00daad"
 }