@cspell/cspell-tools 10.4.0-alpha.1 → 10.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -270,6 +270,11 @@
270
270
  "markdownDescription": "If `compress` is true, setting this value to true will also keep an uncompressed version of the dictionary.",
271
271
  "type": "boolean"
272
272
  },
273
+ "locale": {
274
+ "description": "The locale of the dictionary's words: a language and an optional region, such as `tr` or `tr-TR`. Words are lower-cased with this locale's case rules, for example, when generating the case insensitive forms with `generateNonStrict`. Without it, the default case rules are used.\n\nSet `dictionaryInformation.locale` in the dictionary definition to the same locale, so cspell searches the dictionary with the case rules it was built with. If the definition lists more than one locale, cspell uses the first valid one for case rules, so put this locale first.",
275
+ "markdownDescription": "The locale of the dictionary's words: a language and an optional region, such as `tr` or `tr-TR`.\nWords are lower-cased with this locale's case rules, for example, when generating the case insensitive\nforms with `generateNonStrict`. Without it, the default case rules are used.\n\nSet `dictionaryInformation.locale` in the dictionary definition to the same locale, so cspell searches the\ndictionary with the case rules it was built with. If the definition lists more than one locale, cspell uses the\nfirst valid one for case rules, so put this locale first.",
276
+ "type": "string"
277
+ },
273
278
  "name": {
274
279
  "description": "Name of target, used as the basis of target file name.",
275
280
  "markdownDescription": "Name of target, used as the basis of target file name.",
@@ -370,6 +375,11 @@
370
375
  "markdownDescription": "Do not generate lower case / accent free versions of words.",
371
376
  "type": "boolean"
372
377
  },
378
+ "locale": {
379
+ "description": "The locale of the dictionary's words: a language and an optional region, such as `tr` or `tr-TR`. Words are lower-cased with this locale's case rules, for example, when generating the case insensitive forms with `generateNonStrict`. Without it, the default case rules are used.\n\nSet `dictionaryInformation.locale` in the dictionary definition to the same locale, so cspell searches the dictionary with the case rules it was built with. If the definition lists more than one locale, cspell uses the first valid one for case rules, so put this locale first.",
380
+ "markdownDescription": "The locale of the dictionary's words: a language and an optional region, such as `tr` or `tr-TR`.\nWords are lower-cased with this locale's case rules, for example, when generating the case insensitive\nforms with `generateNonStrict`. Without it, the default case rules are used.\n\nSet `dictionaryInformation.locale` in the dictionary definition to the same locale, so cspell searches the\ndictionary with the case rules it was built with. If the definition lists more than one locale, cspell uses the\nfirst valid one for case rules, so put this locale first.",
381
+ "type": "string"
382
+ },
373
383
  "maxDepth": {
374
384
  "description": "Maximum number of nested Hunspell Rules to apply. This is needed for recursive dictionaries like Hebrew.",
375
385
  "markdownDescription": "Maximum number of nested Hunspell Rules to apply.\nThis is needed for recursive dictionaries like Hebrew.",
package/dist/app.d.mts CHANGED
@@ -28,7 +28,7 @@ declare class FeatureFlags {
28
28
  }
29
29
  //#endregion
30
30
  //#region src/app.d.ts
31
- declare function run(program: Command, argv: string[], flags?: FeatureFlags): Promise<void>;
31
+ export declare function run(program: Command, argv: string[], flags?: FeatureFlags): Promise<void>;
32
32
  //#endregion
33
- export { run };
33
+ export {};
34
34
  //# sourceMappingURL=app.d.mts.map
package/dist/app.mjs CHANGED
@@ -1,4 +1,4 @@
1
- import { a as reportCheckChecksumFile, c as toError, i as logWithTimestamp, l as generateBTrie, n as compile, o as reportChecksumForFiles, r as setLogger, s as updateChecksumForFiles, t as build, u as compressFile } from "./build-Fod7lDHd.mjs";
1
+ import { a as reportCheckChecksumFile, c as toError, i as logWithTimestamp, l as generateBTrie, n as compile, o as reportChecksumForFiles, r as setLogger, s as updateChecksumForFiles, t as build, u as compressFile } from "./build-DitWkAJE.mjs";
2
2
  import { readFileSync } from "node:fs";
3
3
  import { CommanderError, Option } from "commander";
4
4
  import { writeFile } from "node:fs/promises";
@@ -550,6 +550,29 @@ function removeVerboseFromRegExp(pattern) {
550
550
  return result.result;
551
551
  }
552
552
  //#endregion
553
+ //#region src/compiler/text.ts
554
+ const regExSplitWords = /([\p{Ll}])([\p{Lu}])/gu;
555
+ const regExSplitWords2 = /(\p{Lu})(\p{Lu}\p{Ll})/gu;
556
+ const regExpIsLetter = /^\p{L}\p{M}{0,2}$/u;
557
+ /**
558
+ * Split camelCase words into an array of strings.
559
+ */
560
+ function splitCamelCaseWord(word) {
561
+ return word.replaceAll(regExSplitWords, "$1|$2").replaceAll(regExSplitWords2, "$1|$2").replaceAll(/[\d_]+/g, "|").split("|").filter((a) => !!a);
562
+ }
563
+ /**
564
+ * Lower-case a word.
565
+ * @param word - the word to lower-case.
566
+ * @param locale - the locale whose case rules are used. Default rules are used if not given.
567
+ * @returns the lower-case word.
568
+ */
569
+ function lowerCase(word, locale) {
570
+ return locale ? word.toLocaleLowerCase(locale) : word.toLowerCase();
571
+ }
572
+ function isSingleLetter(c) {
573
+ return regExpIsLetter.test(c);
574
+ }
575
+ //#endregion
553
576
  //#region src/compiler/WordsCollection.ts
554
577
  const defaultAllowedSplitWords = Object.freeze({
555
578
  size: 0,
@@ -572,15 +595,15 @@ var AllowedSplitWordsImpl = class {
572
595
  return !this.size || this.collection.has(word, caseSensitive);
573
596
  }
574
597
  };
575
- async function createAllowedSplitWordsFromFiles(files) {
598
+ async function createAllowedSplitWordsFromFiles(files, locale) {
576
599
  if (!files || !files.length) return defaultAllowedSplitWords;
577
- return new AllowedSplitWordsImpl(await createWordsCollectionFromFiles(files));
600
+ return new AllowedSplitWordsImpl(await createWordsCollectionFromFiles(files, locale));
578
601
  }
579
- function buildHasFn(dict) {
602
+ function buildHasFn(dict, locale) {
580
603
  function has(word, caseSensitive) {
581
604
  const r = dict.hasWord(word, true);
582
605
  if (r || caseSensitive) return r;
583
- const lc = word.toLowerCase();
606
+ const lc = lowerCase(word, locale);
584
607
  if (lc === word) return false;
585
608
  return dict.hasWord(lc, true);
586
609
  }
@@ -589,9 +612,9 @@ function buildHasFn(dict) {
589
612
  async function readFile$1(filename) {
590
613
  return await createReader(filename, {});
591
614
  }
592
- function readersToCollection(readers) {
593
- const dictReaders = readers.filter(isDictionaryReader).map(dictReaderToCollection);
594
- const nonDictCollection = lineReadersToCollection(readers.filter((a) => !isDictionaryReader(a)));
615
+ function readersToCollection(readers, locale) {
616
+ const dictReaders = readers.filter(isDictionaryReader).map((reader) => dictReaderToCollection(reader, locale));
617
+ const nonDictCollection = lineReadersToCollection(readers.filter((a) => !isDictionaryReader(a)), locale);
595
618
  const collections = [...dictReaders, nonDictCollection];
596
619
  return {
597
620
  size: collections.reduce((s, a) => s + a.size, 0),
@@ -599,35 +622,49 @@ function readersToCollection(readers) {
599
622
  };
600
623
  }
601
624
  const cache = /* @__PURE__ */ new WeakMap();
602
- async function createWordsCollectionFromFiles(files) {
625
+ const filePaths = /* @__PURE__ */ new Map();
626
+ function createWordsCollectionFromFiles(files, locale) {
603
627
  files = toFilePathArray(files);
604
- const cached = cache.get(files);
628
+ let byLocale = cache.get(files);
629
+ if (!byLocale) {
630
+ byLocale = /* @__PURE__ */ new Map();
631
+ cache.set(files, byLocale);
632
+ }
633
+ const cached = byLocale.get(locale);
605
634
  if (cached) return cached;
606
- const collection = readersToCollection(await Promise.all(files.map((file) => readFile$1(file))));
607
- cache.set(files, collection);
635
+ const collection = _createWordsCollectionFromFiles(files, locale);
636
+ byLocale.set(locale, collection);
608
637
  return collection;
609
638
  }
639
+ async function _createWordsCollectionFromFiles(files, locale) {
640
+ return readersToCollection(await Promise.all(files.map((file) => readFile$1(file))), locale);
641
+ }
610
642
  function isDictionaryReader(reader) {
611
643
  return "hasWord" in reader && !!reader.hasWord;
612
644
  }
613
- function dictReaderToCollection(reader) {
645
+ function dictReaderToCollection(reader, locale) {
614
646
  return {
615
647
  size: reader.size,
616
- has: buildHasFn(reader)
648
+ has: buildHasFn(reader, locale)
617
649
  };
618
650
  }
619
- function lineReadersToCollection(readers) {
651
+ function lineReadersToCollection(readers, locale) {
620
652
  function* words() {
621
653
  for (const reader of readers) yield* reader.lines;
622
654
  }
623
655
  const dict = parseDictionary(words(), { stripCaseAndAccents: false });
624
656
  return {
625
657
  size: dict.size,
626
- has: buildHasFn(dict)
658
+ has: buildHasFn(dict, locale)
627
659
  };
628
660
  }
629
661
  function toFilePathArray(filePathOrArray) {
630
- return Array.isArray(filePathOrArray) ? filePathOrArray : [filePathOrArray];
662
+ if (Array.isArray(filePathOrArray)) return filePathOrArray;
663
+ const found = filePaths.get(filePathOrArray);
664
+ if (found) return found;
665
+ const paths = [filePathOrArray];
666
+ filePaths.set(filePathOrArray, paths);
667
+ return paths;
631
668
  }
632
669
  //#endregion
633
670
  //#region src/compiler/logWithTimestamp.ts
@@ -645,48 +682,34 @@ const defaultCompileSourceOptions = {
645
682
  minCompoundLength: 4
646
683
  };
647
684
  //#endregion
648
- //#region src/compiler/text.ts
649
- const regExSplitWords = /([\p{Ll}])([\p{Lu}])/gu;
650
- const regExSplitWords2 = /(\p{Lu})(\p{Lu}\p{Ll})/gu;
651
- const regExpIsLetter = /^\p{L}\p{M}{0,2}$/u;
652
- /**
653
- * Split camelCase words into an array of strings.
654
- */
655
- function splitCamelCaseWord(word) {
656
- return word.replaceAll(regExSplitWords, "$1|$2").replaceAll(regExSplitWords2, "$1|$2").replaceAll(/[\d_]+/g, "|").split("|").filter((a) => !!a);
657
- }
658
- function isSingleLetter(c) {
659
- return regExpIsLetter.test(c);
660
- }
661
- //#endregion
662
685
  //#region src/compiler/splitCamelCaseIfAllowed.ts
663
686
  const regExpSpaceOrDash = /[- ]+/g;
664
687
  const regExpIsNumber = /^\d+$/;
665
- function splitCamelCaseIfAllowed(word, allowedWords, keepCase, compoundPrefix, minCompoundLength) {
688
+ function splitCamelCaseIfAllowed(word, allowedWords, keepCase, compoundPrefix, minCompoundLength, locale) {
666
689
  const split = [...splitCamelCase(word)];
667
- if (split.length === 1) return adjustCases(split, allowedWords, keepCase);
690
+ if (split.length === 1) return adjustCases(split, allowedWords, keepCase, locale);
668
691
  if (split.some((w) => isUnknown(w, allowedWords))) return [word];
669
692
  const wordIndexes = calcWordIndex(word, split);
670
- const adjusted = adjustCases(split, allowedWords, keepCase);
693
+ const adjusted = adjustCases(split, allowedWords, keepCase, locale);
671
694
  return !compoundPrefix ? adjusted : adjusted.map((w, i) => {
672
695
  const { px, sx } = wordIndexes[i];
673
696
  const canCompound = w.length >= minCompoundLength;
674
- const lc = w.toLowerCase();
697
+ const lc = lowerCase(w, locale);
675
698
  const p = canCompound && isSingleLetter(px) ? compoundPrefix : "";
676
699
  const s = canCompound && isSingleLetter(sx) ? compoundPrefix : "";
677
700
  if (lc.length < 4 || allowedWords.has(w, true)) return p + w + s;
678
701
  return p + lc + s;
679
702
  });
680
703
  }
681
- function adjustCases(words, allowedWords, keepCase) {
682
- return words.map((w) => adjustCase(w, allowedWords, keepCase));
704
+ function adjustCases(words, allowedWords, keepCase, locale) {
705
+ return words.map((w) => adjustCase(w, allowedWords, keepCase, locale));
683
706
  }
684
- function adjustCase(word, allowedWords, keepCase) {
685
- const lc = word.toLowerCase();
707
+ function adjustCase(word, allowedWords, keepCase, locale) {
708
+ const lc = lowerCase(word, locale);
686
709
  if (!allowedWords.has(lc, true)) return word;
687
710
  if (lc === word) return word;
688
711
  if (word.slice(1).toLowerCase() === word.slice(1)) return lc;
689
- if (!keepCase && word.toUpperCase() === word) return word.toLowerCase();
712
+ if (!keepCase && word.toUpperCase() === word) return lc;
690
713
  return word;
691
714
  }
692
715
  function isUnknown(word, allowedWords) {
@@ -718,16 +741,17 @@ function calcWordIndex(word, words) {
718
741
  const regNonWord = /[^\p{L}\p{M}' _\d]+/giu;
719
742
  const regExpRepeatChars = /(.)\1{5}/i;
720
743
  const minCompoundLength = defaultCompileSourceOptions.minCompoundLength;
721
- function legacyLineToWords(line, keepCase, allowedSplitWords) {
744
+ function legacyLineToWords(line, keepCase, allowedSplitWords, locale) {
722
745
  const wordGroups = line.replaceAll(regNonWord, "|").split("|");
723
746
  const _minCompoundLength = minCompoundLength;
724
- return pipe(wordGroups, opConcatMap((a) => a.split(regExpSpaceOrDash)), opConcatMap((a) => splitCamelCaseIfAllowed(a, allowedSplitWords, keepCase, "", _minCompoundLength)), opMap((a) => a.trim()), opFilter((a) => !!a), opFilter((s) => !regExpRepeatChars.test(s)));
747
+ return pipe(wordGroups, opConcatMap((a) => a.split(regExpSpaceOrDash)), opConcatMap((a) => splitCamelCaseIfAllowed(a, allowedSplitWords, keepCase, "", _minCompoundLength, locale)), opMap((a) => a.trim()), opFilter((a) => !!a), opFilter((s) => !regExpRepeatChars.test(s)));
725
748
  }
726
749
  //#endregion
727
750
  //#region src/compiler/wordListParser.ts
728
751
  function normalizeTargetWords(options) {
729
752
  const lineParser = createDictionaryLineParser({
730
753
  stripCaseAndAccents: options.generateNonStrict,
754
+ locale: options.locale,
731
755
  stripCaseAndAccentsOnForbidden: true,
732
756
  keepOptionalCompoundCharacter: true
733
757
  });
@@ -771,7 +795,8 @@ const _defaultOptions = {
771
795
  size: 0
772
796
  },
773
797
  storeSplitWordsAsCompounds: defaultCompileSourceOptions.storeSplitWordsAsCompounds,
774
- minCompoundLength: defaultCompileSourceOptions.minCompoundLength
798
+ minCompoundLength: defaultCompileSourceOptions.minCompoundLength,
799
+ locale: void 0
775
800
  };
776
801
  Object.freeze(_defaultOptions);
777
802
  const cSpellToolDirective = "cspell-tools:";
@@ -783,7 +808,7 @@ const cSpellToolDirective = "cspell-tools:";
783
808
  */
784
809
  function createParseFileLineMapper(options) {
785
810
  const _options = options || _defaultOptions;
786
- const { splitKeepBoth = _defaultOptions.splitKeepBoth, allowedSplitWords = _defaultOptions.allowedSplitWords, storeSplitWordsAsCompounds, minCompoundLength = _defaultOptions.minCompoundLength } = _options;
811
+ const { splitKeepBoth = _defaultOptions.splitKeepBoth, allowedSplitWords = _defaultOptions.allowedSplitWords, storeSplitWordsAsCompounds, minCompoundLength = _defaultOptions.minCompoundLength, locale } = _options;
787
812
  let { legacy = _defaultOptions.legacy } = _options;
788
813
  let { split = _defaultOptions.split, keepCase = legacy ? false : _defaultOptions.keepCase } = _options;
789
814
  const compoundFix = storeSplitWordsAsCompounds ? "+" : "";
@@ -841,12 +866,12 @@ function createParseFileLineMapper(options) {
841
866
  return line.split("|").map((a) => a.trim()).filter((a) => !!a).filter((a) => !/^[0-9_-]+$/.test(a)).filter((a) => !/^0[xo][0-9A-F]+$/i.test(a));
842
867
  }
843
868
  function splitWordIntoWords(word) {
844
- return splitCamelCaseIfAllowed(word, allowedSplitWords, keepCase, compoundFix, minCompoundLength);
869
+ return splitCamelCaseIfAllowed(word, allowedSplitWords, keepCase, compoundFix, minCompoundLength, locale);
845
870
  }
846
871
  function* splitWords(lines) {
847
872
  for (const line of lines) {
848
873
  if (legacy) {
849
- yield* legacyLineToWords(line, keepCase, allowedSplitWords);
874
+ yield* legacyLineToWords(line, keepCase, allowedSplitWords, locale);
850
875
  continue;
851
876
  }
852
877
  if (split) {
@@ -911,7 +936,7 @@ function splitLines(lines, options) {
911
936
  return split();
912
937
  }
913
938
  async function textFileReader(reader, options) {
914
- const { legacy, splitWords: split, allowedSplitWords, storeSplitWordsAsCompounds, minCompoundLength } = options;
939
+ const { legacy, splitWords: split, allowedSplitWords, storeSplitWordsAsCompounds, minCompoundLength, locale } = options;
915
940
  const parseOptions = {
916
941
  legacy,
917
942
  split,
@@ -919,7 +944,8 @@ async function textFileReader(reader, options) {
919
944
  keepCase: void 0,
920
945
  allowedSplitWords,
921
946
  storeSplitWordsAsCompounds,
922
- minCompoundLength
947
+ minCompoundLength,
948
+ locale
923
949
  };
924
950
  const words = [...parseFileLines(reader.lines, parseOptions)];
925
951
  return {
@@ -974,7 +1000,7 @@ function createWordListCompiler(options) {
974
1000
  }
975
1001
  function normalize(lines, options) {
976
1002
  const filter = normalizeTargetWords(options);
977
- const cleanLines = options.removeDuplicates ? removeDuplicates(lines) : lines;
1003
+ const cleanLines = options.removeDuplicates ? removeDuplicates(lines, options.locale) : lines;
978
1004
  const iter = pipe(cleanLines, filter);
979
1005
  if (!options.sort) return iter;
980
1006
  return [...new Set(iter)].sort();
@@ -982,11 +1008,11 @@ function normalize(lines, options) {
982
1008
  function stripCompoundAFix(word) {
983
1009
  return word.replaceAll("*", "").replaceAll("+", "");
984
1010
  }
985
- function* removeDuplicates(words) {
1011
+ function* removeDuplicates(words, locale) {
986
1012
  const wordSet = new Set(words);
987
1013
  const wordForms = /* @__PURE__ */ new Map();
988
1014
  for (const word of wordSet) {
989
- const lc = stripCompoundAFix(word.toLowerCase());
1015
+ const lc = stripCompoundAFix(lowerCase(word, locale));
990
1016
  const forms = wordForms.get(lc) ?? [];
991
1017
  forms.push(word);
992
1018
  wordForms.set(lc, forms);
@@ -1001,7 +1027,7 @@ function* removeDuplicates(words) {
1001
1027
  for (const form of mForms.values()) yield* form;
1002
1028
  continue;
1003
1029
  }
1004
- const lc = [...mForms.keys()][0].toLowerCase();
1030
+ const lc = lowerCase([...mForms.keys()][0], locale);
1005
1031
  const lcForm = mForms.get(lc);
1006
1032
  if (!lcForm) {
1007
1033
  for (const form of mForms.values()) yield* form;
@@ -1012,7 +1038,7 @@ function* removeDuplicates(words) {
1012
1038
  yield* lcForm;
1013
1039
  if (sLcForms.has("*" + lc + "*")) continue;
1014
1040
  for (const forms of mForms.values()) for (const form of forms) {
1015
- if (sLcForms.has(form.toLowerCase())) continue;
1041
+ if (sLcForms.has(lowerCase(form, locale))) continue;
1016
1042
  yield form;
1017
1043
  }
1018
1044
  }
@@ -1093,6 +1119,7 @@ async function compile(request, options) {
1093
1119
  const targetOptions = {
1094
1120
  sort: request.sort,
1095
1121
  generateNonStrict: request.generateNonStrict,
1122
+ locale: request.locale,
1096
1123
  removeDuplicates: request.removeDuplicates,
1097
1124
  replacements: void 0
1098
1125
  };
@@ -1157,18 +1184,20 @@ async function buildTargetDictionary(target, options, compileOptions, buildOptio
1157
1184
  const { filename, useTrie, generateOnlyCompressedDictionary, checksumRoot } = buildOptions;
1158
1185
  const dictionaryDirectives = target.dictionaryDirectives ?? compileOptions.dictionaryDirectives;
1159
1186
  const removeDuplicates = target.removeDuplicates ?? false;
1160
- const excludeFromFilter = await createExcludeFilter(excludeWordsFrom);
1161
- const includeFromFilter = await createIncludeFilter(excludeWordsNotFoundIn);
1187
+ const locale = canonicalLocale(target);
1188
+ const excludeFromFilter = await createExcludeFilter(excludeWordsFrom, locale);
1189
+ const includeFromFilter = await createIncludeFilter(excludeWordsNotFoundIn, locale);
1162
1190
  const excludeRegexFilter = createExcludeRegexFilter(excludeWordsMatchingRegex);
1163
1191
  const excludeFilter = (word) => {
1164
1192
  return excludeFromFilter(word) && includeFromFilter(word) && excludeRegexFilter(word);
1165
1193
  };
1166
1194
  const compress = generateOnlyCompressedDictionary;
1167
- const filesToProcessAsync = pipeAsync(readSourceList(sources, rootDir), opMapAsync((src) => readFileSource(src, options)), opAwaitAsync());
1195
+ const filesToProcessAsync = pipeAsync(readSourceList(sources, rootDir), opMapAsync((src) => readFileSource(src, options, locale)), opAwaitAsync());
1168
1196
  const filesToProcess = await toArray(filesToProcessAsync);
1169
1197
  const normalizer = normalizeTargetWords({
1170
1198
  sort: useTrie || sort,
1171
1199
  generateNonStrict,
1200
+ locale,
1172
1201
  filter: excludeFilter,
1173
1202
  dictionaryDirectives,
1174
1203
  replacements
@@ -1188,6 +1217,7 @@ async function buildTargetDictionary(target, options, compileOptions, buildOptio
1188
1217
  }) : createWordListCompiler({
1189
1218
  sort,
1190
1219
  generateNonStrict,
1220
+ locale,
1191
1221
  dictionaryDirectives,
1192
1222
  removeDuplicates,
1193
1223
  replacements
@@ -1284,7 +1314,7 @@ function readSourceList(sources, rootDir) {
1284
1314
  async function readFileList(fileList) {
1285
1315
  return (await readTextFile(fileList)).split("\n").map((a) => a.trim()).filter((a) => !!a);
1286
1316
  }
1287
- async function readFileSource(fileSource, sourceOptions) {
1317
+ async function readFileSource(fileSource, sourceOptions, locale) {
1288
1318
  const { filename, keepRawCase = sourceOptions.keepRawCase || false, split = sourceOptions.split || false, maxDepth, storeSplitWordsAsCompounds, minCompoundLength } = fileSource;
1289
1319
  const legacy = split === "legacy";
1290
1320
  const readerOptions = {
@@ -1292,9 +1322,10 @@ async function readFileSource(fileSource, sourceOptions) {
1292
1322
  legacy,
1293
1323
  splitWords: legacy ? false : split,
1294
1324
  keepCase: keepRawCase,
1295
- allowedSplitWords: await createAllowedSplitWordsFromFiles(fileSource.allowedSplitWords || sourceOptions.allowedSplitWords),
1325
+ allowedSplitWords: await createAllowedSplitWordsFromFiles(fileSource.allowedSplitWords || sourceOptions.allowedSplitWords, locale),
1296
1326
  storeSplitWordsAsCompounds,
1297
- minCompoundLength
1327
+ minCompoundLength,
1328
+ locale
1298
1329
  };
1299
1330
  logWithTimestamp(`Reading ${path.basename(filename)}`);
1300
1331
  const stream = await streamSourceWordsFromFile(filename, readerOptions);
@@ -1304,6 +1335,19 @@ async function readFileSource(fileSource, sourceOptions) {
1304
1335
  words: stream
1305
1336
  };
1306
1337
  }
1338
+ const regExpLocale = /^([a-z]{2})(?:[-_]([a-z]{2}))?$/i;
1339
+ /**
1340
+ * @returns the target's locale in its canonical form, such as `tr-TR` for `tr_tr`.
1341
+ * @throws if the locale isn't a language and an optional region.
1342
+ */
1343
+ function canonicalLocale(target) {
1344
+ const { locale } = target;
1345
+ if (locale === void 0) return void 0;
1346
+ const m = regExpLocale.exec(locale);
1347
+ if (!m) throw new Error(`Invalid locale "${locale}" in target "${target.name}". Use a language and an optional region, such as "tr" or "tr-TR".`);
1348
+ const [, language, region] = m;
1349
+ return region ? `${language.toLowerCase()}-${region.toUpperCase()}` : language.toLowerCase();
1350
+ }
1307
1351
  function normalizeTargetName(name) {
1308
1352
  return name.replace(/((\.txt|\.dic|\.aff|\.trie)(\.gz)?)?$/, "").replaceAll(/[^\p{L}\p{M}.\w\\/-]/gu, "_");
1309
1353
  }
@@ -1321,20 +1365,22 @@ function logProgress(freq = 1e5) {
1321
1365
  }
1322
1366
  /**
1323
1367
  * @param excludeWordsFrom - List of files to read words from.
1368
+ * @param locale - the locale used to lower-case words when matching them.
1324
1369
  * @returns a function that returns true if the word is not in the exclude list.
1325
1370
  */
1326
- async function createExcludeFilter(excludeWordsFrom) {
1371
+ async function createExcludeFilter(excludeWordsFrom, locale) {
1327
1372
  if (!excludeWordsFrom || !excludeWordsFrom.length) return () => true;
1328
- const excludeWords = await createWordsCollectionFromFiles(excludeWordsFrom);
1373
+ const excludeWords = await createWordsCollectionFromFiles(excludeWordsFrom, locale);
1329
1374
  return (word) => !excludeWords.has(word, word.toUpperCase() !== word);
1330
1375
  }
1331
1376
  /**
1332
1377
  * @param includeWordsFrom - List of files to read words from.
1378
+ * @param locale - the locale used to lower-case words when matching them.
1333
1379
  * @returns a function that returns true if the word is in the include list.
1334
1380
  */
1335
- async function createIncludeFilter(includeWordsFrom) {
1381
+ async function createIncludeFilter(includeWordsFrom, locale) {
1336
1382
  if (!includeWordsFrom || !includeWordsFrom.length) return () => true;
1337
- const excludeWords = await createWordsCollectionFromFiles(includeWordsFrom);
1383
+ const excludeWords = await createWordsCollectionFromFiles(includeWordsFrom, locale);
1338
1384
  return (word) => excludeWords.has(word, word.toUpperCase() !== word);
1339
1385
  }
1340
1386
  /**
@@ -1399,4 +1445,4 @@ function normalizeRequest(buildInfo, root) {
1399
1445
  //#endregion
1400
1446
  export { reportCheckChecksumFile as a, toError as c, logWithTimestamp as i, generateBTrie as l, compile as n, reportChecksumForFiles as o, setLogger as r, updateChecksumForFiles as s, build as t, compressFile as u };
1401
1447
 
1402
- //# sourceMappingURL=build-Fod7lDHd.mjs.map
1448
+ //# sourceMappingURL=build-DitWkAJE.mjs.map
package/dist/index.d.mts CHANGED
@@ -19,7 +19,7 @@ interface GenerateBTrieOptions extends BTrieOptions {
19
19
  }
20
20
  //#endregion
21
21
  //#region src/bTrie.d.ts
22
- declare function generateBTrie(files: string[], options: GenerateBTrieOptions): Promise<void>;
22
+ export declare function generateBTrie(files: string[], options: GenerateBTrieOptions): Promise<void>;
23
23
  //#endregion
24
24
  //#region src/build.d.ts
25
25
  interface BuildOptions {
@@ -32,7 +32,7 @@ interface BuildOptions {
32
32
  /** Conditional build based upon the targets matching the `checksum.txt` file. */
33
33
  conditional?: boolean;
34
34
  }
35
- declare function build(targets: string[] | undefined, options: BuildOptions): Promise<void>;
35
+ export declare function build(targets: string[] | undefined, options: BuildOptions): Promise<void>;
36
36
  //#endregion
37
- export { build, generateBTrie };
37
+ export {};
38
38
  //# sourceMappingURL=index.d.mts.map
package/dist/index.mjs CHANGED
@@ -1,2 +1 @@
1
- import { l as generateBTrie, t as build } from "./build-Fod7lDHd.mjs";
2
- export { build, generateBTrie };
1
+ export { t as build, l as generateBTrie } from "./build-DitWkAJE.mjs";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cspell/cspell-tools",
3
- "version": "10.4.0-alpha.1",
3
+ "version": "10.4.0",
4
4
  "description": "Tools to assist with the development of cSpell",
5
5
  "publishConfig": {
6
6
  "access": "public",
@@ -65,12 +65,12 @@
65
65
  "watch": "tsdown -w"
66
66
  },
67
67
  "dependencies": {
68
- "@cspell/cspell-pipe": "10.4.0-alpha.1",
68
+ "@cspell/cspell-pipe": "10.4.0",
69
69
  "commander": "^15.0.0",
70
70
  "cosmiconfig": "10.0.1",
71
- "cspell-trie-lib": "10.4.0-alpha.1",
71
+ "cspell-trie-lib": "10.4.0",
72
72
  "glob": "^13.0.6",
73
- "hunspell-reader": "10.4.0-alpha.1",
73
+ "hunspell-reader": "10.4.0",
74
74
  "yaml": "^2.9.1"
75
75
  },
76
76
  "devDependencies": {
@@ -80,5 +80,5 @@
80
80
  "engines": {
81
81
  "node": ">=22.18.0"
82
82
  },
83
- "gitHead": "628768a6e62b792e628189b41e63e12e5d3d8fe3"
83
+ "gitHead": "7e2bbb5e866c827ac5d2cf6b84bf3e1a016d0823"
84
84
  }