cspell-lib 10.4.0-alpha.1 → 10.4.1

package/README.md

@@ -10,7 +10,7 @@ If our spell checkers save you time, please consider supporting their developmen
 
 Please show your support through one of the following sites:
 
-[![GitHub Sponsor](https://cspell.org/img/sponsor/github-sponsor-176px.png)](https://github.com/sponsors/streetsidesoftware) [![PayPal](https://cspell.org/img/sponsor/paypal-176px.png)](https://www.paypal.com/donate/?hosted_button_id=26LNBP2Q6MKCY) [![Open Collective](https://cspell.org/img/sponsor/open-collective-176px.png)](https://opencollective.com/cspell) [![CSpell](https://cspell.org/img/sponsor/cspell-176px.png)](https://streetsidesoftware.com/sponsor/)
+[![GitHub Sponsor](https://cspell.org/img/sponsor/github-sponsor-176px.png)](https://github.com/sponsors/streetsidesoftware) [![PayPal](https://cspell.org/img/sponsor/paypal-176px.png)](https://www.paypal.com/donate/?hosted_button_id=26LNBP2Q6MKCY) [![Open Collective](https://cspell.org/img/sponsor/open-collective-176px.png)](https://opencollective.com/cspell) [![CSpell](https://cspell.org/img/sponsor/cspell-176px.png)](https://streetsidesoftware.com/sponsor/)
 
 <!--- @@inject-end: ../../static/sponsor.md --->
 

package/dist/lib/Settings/Controller/configLoader/normalizeRawSettings.js

@@ -56,7 +56,7 @@ export async function normalizeReporters(settings, pathToSettingsFile) {
             return resolve(s);
         }
         if (!Array.isArray(s) || typeof s[0] !== 'string')
-            throw new Error('Invalid Reporter');
+            throw new TypeError('Invalid Reporter');
         // Preserve the shape of Reporter Setting while resolving the reporter file.
         const [r, ...rest] = s;
         return [await resolve(r), ...rest];

package/dist/lib/Settings/InDocSettings.js

@@ -213,9 +213,20 @@ function reduceWordList(acc, match, key) {
 function parseWords(acc, match) {
     return reduceWordList(acc, match, 'words');
 }
+// a non-perfect but reasonable regex to match locale-like strings
+// must match "lorem-ipsum" and similar fake locale-like strings
+const localeLikeRegEx = /^[a-z][a-z0-9_-]{0,16}[a-z0-9]$/i;
 function parseLocale(acc, match) {
-    const parts = match.trim().split(/[\s,]+/);
-    const language = parts.slice(1).join(',');
+    // Remove the closing marker of an HTML or block comment, it is not a locale.
+    const parts = match
+        .replace(/[^a-z0-9]+$/i, '')
+        .split(/[\s,]+/)
+        .slice(1);
+    const stop = parts.findIndex((locale) => !localeLikeRegEx.test(locale));
+    if (stop !== -1) {
+        parts.splice(stop);
+    }
+    const language = parts.join(',');
     if (language) {
         acc.language = language;
     }

package/dist/lib/Settings/LanguageSettings.d.ts

@@ -3,15 +3,18 @@ export type LanguageSettings = LanguageSetting[];
 export declare function getDefaultLanguageSettings(): LanguageSettings;
 export declare function normalizeLanguageId(langId: LanguageId | LanguageId[]): Set<LanguageId>;
 export declare function normalizeLocale(locale: LocaleId | LocaleId[]): Set<LocaleId>;
+/**
+ * Normalize locales to the form `Intl` uses, for example `en_us` becomes `en-US`.
+ * Locales that `Intl` rejects are dropped.
+ */
 export declare function normalizeLocaleIntl(locale: LocaleId | LocaleId[]): Set<LocaleId>;
 export declare function isLocaleInSet(locale: LocaleId | LocaleId[], setOfLocals: Set<LocaleId>): boolean;
 /**
- * Test if a locale should be ok with Intl
- * @param locale - locale string
- * @param strict - case must match
- * @returns true if it matches the standard 2 letter or 4 letter forms.
+ * Normalize a locale for use with `Intl` and `toLocaleLowerCase`. Locales that `Intl` rejects are dropped.
+ * @param locale - a locale or a list of locales, for example `en,tr_TR`.
+ * @returns the valid locales, or `undefined` if there are none.
  */
-export declare function isValidLocaleIntlFormat(locale: LocaleId | LocaleId[], strict?: boolean): boolean;
+export declare function normalizeLocaleForIntl(locale: LocaleId | LocaleId[] | undefined): string | string[] | undefined;
 export declare function calcSettingsForLanguage(languageSettings: LanguageSettings, languageId: LanguageId, locale: LocaleId): BaseSetting;
 export declare function calcUserSettingsForLanguage(settings: CSpellUserSettings, languageId: string): CSpellUserSettings;
 export declare function calcSettingsForLanguageId(baseSettings: CSpellUserSettings, languageId: LanguageId[] | LanguageId): CSpellUserSettings;

package/dist/lib/Settings/LanguageSettings.js

@@ -37,46 +37,49 @@ export function normalizeLocale(locale) {
     locale = typeof locale === 'string' ? locale : locale.join(',');
     return _normalizeLocale(locale);
 }
-export function normalizeLocaleIntl(locale) {
-    const values = [...normalizeLocale(locale)].map((locale) => locale.replace(/^([a-z]{2})(?:-?([a-z]{4}))?-?([a-z]{2})$/, (_, lang, script, region) => toLocaleIntl(lang, script, region)));
-    return new Set(values);
-}
 /**
- * See: https://developer.mozilla.org/en-US/docs/Glossary/BCP_47_language_tag
- * @returns locale normalized to BCP_47
+ * Normalize locales to the form `Intl` uses, for example `en_us` becomes `en-US`.
+ * Locales that `Intl` rejects are dropped.
  */
-function toLocaleIntl(lang, script, region) {
-    const parts = [lang];
-    if (script)
-        parts.push(firstUpper(script.toLowerCase()));
-    if (region)
-        parts.push(region.toUpperCase());
-    return parts.join('-');
-}
-function firstUpper(str) {
-    const [first, ...rest] = [...str];
-    return [first.toUpperCase(), ...rest].join('');
+export function normalizeLocaleIntl(locale) {
+    return _normalizeLocaleIntl(typeof locale === 'string' ? locale : locale.join(','));
+}
+const _normalizeLocaleIntl = memoize(__normalizeLocaleIntl);
+function __normalizeLocaleIntl(locale) {
+    const locales = localesToList(locale)
+        .map((locale) => toLocaleIntl(locale.replaceAll('_', '-')))
+        .filter((locale) => locale !== undefined);
+    return new Set(locales);
+}
+// A locale written without separators, for example `enUS` or `zhHantTW`.
+const regExpLocaleWithoutSeparators = /^([a-z]{2})([a-z]{4})?([a-z]{2})$/i;
+function toLocaleIntl(locale) {
+    locale = locale.replace(regExpLocaleWithoutSeparators, (_, lang, script, region) => script ? `${lang}-${script}-${region}` : `${lang}-${region}`);
+    try {
+        return Intl.getCanonicalLocales(locale)[0];
+    }
+    catch {
+        return undefined;
+    }
 }
 export function isLocaleInSet(locale, setOfLocals) {
     const locales = normalizeLocale(locale);
     return doSetsIntersect(locales, setOfLocals);
 }
-const regExpValidIntlLocaleStrict = /^[a-z]{2}(-[A-Z]{2})?$/;
-const regExpValidIntlLocale = new RegExp(regExpValidIntlLocaleStrict, 'i');
 /**
- * Test if a locale should be ok with Intl
- * @param locale - locale string
- * @param strict - case must match
- * @returns true if it matches the standard 2 letter or 4 letter forms.
+ * Normalize a locale for use with `Intl` and `toLocaleLowerCase`. Locales that `Intl` rejects are dropped.
+ * @param locale - a locale or a list of locales, for example `en,tr_TR`.
+ * @returns the valid locales, or `undefined` if there are none.
  */
-export function isValidLocaleIntlFormat(locale, strict = false) {
-    if (typeof locale === 'string')
-        return strict ? regExpValidIntlLocaleStrict.test(locale) : regExpValidIntlLocale.test(locale);
-    for (const item of locale) {
-        if (!isValidLocaleIntlFormat(item, strict))
-            return false;
-    }
-    return locale.length > 0;
+export function normalizeLocaleForIntl(locale) {
+    if (!locale)
+        return undefined;
+    const locales = [...normalizeLocaleIntl(locale)];
+    if (!locales.length)
+        return undefined;
+    if (locales.length === 1)
+        return locales[0];
+    return locales;
 }
 const cacheCalcSettingsForLanguage = createAutoResolveWeakCache();
 export function calcSettingsForLanguage(languageSettings, languageId, locale) {

package/dist/lib/SpellingDictionary/DictionaryController/DictionaryLoader.js

@@ -56,6 +56,7 @@ import { createFailedToLoadDictionary, createFlagWordsDictionary, createFlagWord
 import { compareStats, toFileURL, urlBasename } from 'cspell-io';
 import { measurePerf } from '../../perf/index.js';
 import { isDictionaryDefinitionInlineInternal, isDictionaryFileDefinitionInternal } from '../../Settings/index.js';
+import { normalizeLocaleForIntl } from '../../Settings/LanguageSettings.js';
 import { AutoResolveWeakCache, AutoResolveWeakWeakCache } from '../../util/AutoResolve.js';
 import { toError } from '../../util/errors.js';
 import { SimpleCache } from '../../util/simpleCache.js';
@@ -260,7 +261,7 @@ async function legacyWordList(reader, filename, options) {
         if (options.kind === 'suggest-words') {
             return createSuggestDictionary([...words], options.name, filename.toString());
         }
-        return createSpellingDictionary(words, options.name, filename.toString(), applyKind(options), true);
+        return createSpellingDictionary(words, options.name, filename.toString(), toSpellingDictionaryOptions(options), true);
     }
     catch (e_1) {
         env_1.error = e_1;
@@ -283,7 +284,7 @@ async function wordsPerLineWordList(reader, filename, options) {
     if (options.kind === 'suggest-words') {
         return createSuggestDictionary([...words], options.name, filename.href);
     }
-    return createSpellingDictionary(words, options.name, filename.href, applyKind(options), true);
+    return createSpellingDictionary(words, options.name, filename.href, toSpellingDictionaryOptions(options), true);
 }
 async function loadSimpleWordList(reader, filename, options) {
     const env_2 = { stack: [], error: void 0, hasError: false };
@@ -296,7 +297,7 @@ async function loadSimpleWordList(reader, filename, options) {
         if (options.kind === 'suggest-words') {
             return createSuggestDictionary(lines, options.name, filename.href);
         }
-        return createSpellingDictionary(lines, options.name, filename.href, applyKind(options));
+        return createSpellingDictionary(lines, options.name, filename.href, toSpellingDictionaryOptions(options));
     }
     catch (e_2) {
         env_2.error = e_2;
@@ -317,7 +318,7 @@ async function loadTrie(reader, filename, options) {
         if (options.kind === 'suggest-words') {
             return createSuggestDictionaryFromTrieFile(content, options.name, filename.href);
         }
-        return createSpellingDictionaryFromTrieFile(content, options.name, filename.href, applyKind(options));
+        return createSpellingDictionaryFromTrieFile(content, options.name, filename.href, toSpellingDictionaryOptions(options));
     }
     catch (e_3) {
         env_3.error = e_3;
@@ -327,6 +328,21 @@ async function loadTrie(reader, filename, options) {
         __disposeResources(env_3);
     }
 }
+/**
+ * The options for a spelling dictionary: the effect of `kind`, and the dictionary's locale, normalized once here so
+ * lookups don't have to. A locale that isn't valid is dropped.
+ */
+function toSpellingDictionaryOptions(options) {
+    const opts = applyKind(options);
+    if (!isDictionaryFileDefinitionInternal(opts))
+        return opts;
+    const locales = normalizeLocaleForIntl(opts.dictionaryInformation?.locale);
+    const locale = Array.isArray(locales) ? locales[0] : locales;
+    if (!locale)
+        return opts;
+    // `weightMap` is a getter on some definitions, so copy it explicitly.
+    return { ...opts, weightMap: opts.weightMap, locale };
+}
 /**
  * Apply the effect of `kind` on the options passed to a normal `words` dictionary loader.
  * `flag-words` and `suggest-words` are handled by the callers directly since they use a different

package/dist/lib/Transform/IntlSegmentTextTransformer.js

@@ -1,11 +1,11 @@
-import { normalizeLocaleIntl } from '../Settings/LanguageSettings.js';
+import { normalizeLocaleForIntl } from '../Settings/LanguageSettings.js';
 import { applyEditsToMappedText } from './TextMapEdit.js';
 import { softHyphen, toMappedText } from './Transformer.js';
 const regExSplitWordCharacters = /(?<=[\p{L}\p{M}])[\p{L}\p{M}]/uy;
 export class IntlSegmentTextTransformer {
     #locale;
     constructor(locale) {
-        this.#locale = [...normalizeLocaleIntl(locale)];
+        this.#locale = normalizeLocaleForIntl(locale);
     }
     transform(text) {
         // Implement the transformation logic here

package/dist/lib/suggestions.js

@@ -1,7 +1,7 @@
 import assert from 'node:assert';
 import { satisfiesCSpellConfigFile } from 'cspell-config-lib';
 import { finalizeSettings, getDefaultSettings, getGlobalSettingsAsync, mergeSettings, resolveConfigFileImports, } from './Settings/index.js';
-import { calcSettingsForLanguageId, isValidLocaleIntlFormat, normalizeLocaleIntl, } from './Settings/LanguageSettings.js';
+import { calcSettingsForLanguageId, normalizeLocaleForIntl } from './Settings/LanguageSettings.js';
 import { getDictionaryInternal, refreshDictionaryCache } from './SpellingDictionary/index.js';
 import { createAutoResolveCache } from './util/AutoResolve.js';
 import { memoizeLastCall } from './util/memoizeLastCall.js';
@@ -58,7 +58,7 @@ async function _suggestionsForWordAsync(word, options, settings, dictionaryColle
         : settings;
     const opts = { ignoreCase, numChanges, numSuggestions, includeTies };
     const suggestionsByDictionary = dictionaryCollection.dictionaries.flatMap((dict) => dict.suggest(word, opts).map((r) => ({ ...r, dictName: dict.name })));
-    const locale = adjustLocale(language || config.language || undefined);
+    const locale = normalizeLocaleForIntl(language || config.language || undefined);
     const collator = Intl.Collator(locale);
     const combined = limitResults(combine(suggestionsByDictionary.sort((a, b) => a.cost - b.cost || collator.compare(a.word, b.word))), numSuggestions, includeTies);
     const sugsAdjusted = calcSuggestionAdjustedToToMatchCase(word, combined, locale, ignoreCase, extendsDictionaryCollection);
@@ -87,18 +87,8 @@ function combine(suggestions) {
     }
     return [...words.values()];
 }
-function adjustLocale(locale) {
-    if (!locale)
-        return undefined;
-    const locales = [...normalizeLocaleIntl(locale)].filter((locale) => isValidLocaleIntlFormat(locale));
-    if (!locales.length)
-        return undefined;
-    if (locales.length === 1)
-        return locales[0];
-    return locales;
-}
 export function calcSuggestionAdjustedToToMatchCase(originalWord, sugs, locale, ignoreCase, dict) {
-    locale = adjustLocale(locale);
+    locale = normalizeLocaleForIntl(locale);
     const knownSugs = new Set(sugs.map((sug) => sug.word));
     const matchStyle = { ...analyzeCase(originalWord), locale, ignoreCase };
     /* Add adjusted words */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cspell-lib",
-  "version": "10.4.0-alpha.1",
+  "version": "10.4.1",
   "description": "A library of useful functions used across various cspell tools.",
   "publishConfig": {
     "access": "public",
@@ -75,46 +75,43 @@
     "watch": "tsc -p . --watch"
   },
   "dependencies": {
-    "@cspell/cspell-bundled-dicts": "10.4.0-alpha.1",
-    "@cspell/cspell-performance-monitor": "10.4.0-alpha.1",
-    "@cspell/cspell-pipe": "10.4.0-alpha.1",
-    "@cspell/cspell-resolver": "10.4.0-alpha.1",
-    "@cspell/cspell-types": "10.4.0-alpha.1",
-    "@cspell/dynamic-import": "10.4.0-alpha.1",
-    "@cspell/filetypes": "10.4.0-alpha.1",
-    "@cspell/rpc": "10.4.0-alpha.1",
-    "@cspell/strong-weak-map": "10.4.0-alpha.1",
-    "@cspell/url": "10.4.0-alpha.1",
-    "cspell-config-lib": "10.4.0-alpha.1",
-    "cspell-dictionary": "10.4.0-alpha.1",
-    "cspell-glob": "10.4.0-alpha.1",
-    "cspell-grammar": "10.4.0-alpha.1",
-    "cspell-io": "10.4.0-alpha.1",
-    "cspell-trie-lib": "10.4.0-alpha.1",
+    "@cspell/cspell-bundled-dicts": "10.4.1",
+    "@cspell/cspell-performance-monitor": "10.4.1",
+    "@cspell/cspell-pipe": "10.4.1",
+    "@cspell/cspell-resolver": "10.4.1",
+    "@cspell/cspell-types": "10.4.1",
+    "@cspell/dynamic-import": "10.4.1",
+    "@cspell/filetypes": "10.4.1",
+    "@cspell/rpc": "10.4.1",
+    "@cspell/strong-weak-map": "10.4.1",
+    "@cspell/url": "10.4.1",
+    "cspell-config-lib": "10.4.1",
+    "cspell-dictionary": "10.4.1",
+    "cspell-glob": "10.4.1",
+    "cspell-grammar": "10.4.1",
+    "cspell-io": "10.4.1",
+    "cspell-trie-lib": "10.4.1",
     "env-paths": "^4.0.0",
     "gensequence": "^8.0.8",
     "import-fresh": "^4.0.1",
     "resolve-from": "^5.0.0",
-    "vscode-languageserver-textdocument": "^1.0.15",
+    "vscode-languageserver-textdocument": "^1.0.16",
     "vscode-uri": "^3.2.0",
     "xdg-basedir": "^5.1.0"
   },
   "devDependencies": {
-    "@cspell/dict-cpp": "^7.1.2",
+    "@cspell/dict-cpp": "^7.2.0",
     "@cspell/dict-csharp": "^4.0.8",
     "@cspell/dict-css": "^4.1.2",
-    "@cspell/dict-en_us": "^4.4.40",
-    "@cspell/dict-fa-ir": "^4.0.5",
+    "@cspell/dict-en_us": "^4.4.43",
+    "@cspell/dict-fa-ir": "^4.0.6",
     "@cspell/dict-fr-fr": "^2.3.2",
     "@cspell/dict-html": "^4.0.16",
     "@cspell/dict-nl-nl": "^2.4.2",
-    "@cspell/dict-python": "^4.5.2",
-    "@cspell/dict-th-th": "^1.0.1",
+    "@cspell/dict-python": "^4.5.3",
+    "@cspell/dict-th-th": "^1.0.4",
     "@cspell/dictionary-bundler-plugin": "",
-    "@types/configstore": "^6.0.2",
     "comment-json": "^5.0.0",
-    "configstore": "^8.0.0",
-    "cspell-dict-nl-nl": "^1.1.2",
     "leaked-handles": "^5.2.0",
     "lorem-ipsum": "^3.0.0",
     "perf-insight": "^2.0.1"
@@ -122,5 +119,5 @@
   "engines": {
     "node": ">=22.18.0"
   },
-  "gitHead": "628768a6e62b792e628189b41e63e12e5d3d8fe3"
+  "gitHead": "ddb1699a4bfc2766675c2c97412a813d775772eb"
 }