@cspell/cspell-tools 10.4.0-alpha.1 → 10.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cspell-tools.config.schema.json +10 -0
- package/dist/app.d.mts +2 -2
- package/dist/app.mjs +1 -1
- package/dist/{build-Fod7lDHd.mjs → build-DitWkAJE.mjs} +109 -63
- package/dist/index.d.mts +3 -3
- package/dist/index.mjs +1 -2
- package/package.json +5 -5
|
@@ -270,6 +270,11 @@
|
|
|
270
270
|
"markdownDescription": "If `compress` is true, setting this value to true will also keep an uncompressed version of the dictionary.",
|
|
271
271
|
"type": "boolean"
|
|
272
272
|
},
|
|
273
|
+
"locale": {
|
|
274
|
+
"description": "The locale of the dictionary's words: a language and an optional region, such as `tr` or `tr-TR`. Words are lower-cased with this locale's case rules, for example, when generating the case insensitive forms with `generateNonStrict`. Without it, the default case rules are used.\n\nSet `dictionaryInformation.locale` in the dictionary definition to the same locale, so cspell searches the dictionary with the case rules it was built with. If the definition lists more than one locale, cspell uses the first valid one for case rules, so put this locale first.",
|
|
275
|
+
"markdownDescription": "The locale of the dictionary's words: a language and an optional region, such as `tr` or `tr-TR`.\nWords are lower-cased with this locale's case rules, for example, when generating the case insensitive\nforms with `generateNonStrict`. Without it, the default case rules are used.\n\nSet `dictionaryInformation.locale` in the dictionary definition to the same locale, so cspell searches the\ndictionary with the case rules it was built with. If the definition lists more than one locale, cspell uses the\nfirst valid one for case rules, so put this locale first.",
|
|
276
|
+
"type": "string"
|
|
277
|
+
},
|
|
273
278
|
"name": {
|
|
274
279
|
"description": "Name of target, used as the basis of target file name.",
|
|
275
280
|
"markdownDescription": "Name of target, used as the basis of target file name.",
|
|
@@ -370,6 +375,11 @@
|
|
|
370
375
|
"markdownDescription": "Do not generate lower case / accent free versions of words.",
|
|
371
376
|
"type": "boolean"
|
|
372
377
|
},
|
|
378
|
+
"locale": {
|
|
379
|
+
"description": "The locale of the dictionary's words: a language and an optional region, such as `tr` or `tr-TR`. Words are lower-cased with this locale's case rules, for example, when generating the case insensitive forms with `generateNonStrict`. Without it, the default case rules are used.\n\nSet `dictionaryInformation.locale` in the dictionary definition to the same locale, so cspell searches the dictionary with the case rules it was built with. If the definition lists more than one locale, cspell uses the first valid one for case rules, so put this locale first.",
|
|
380
|
+
"markdownDescription": "The locale of the dictionary's words: a language and an optional region, such as `tr` or `tr-TR`.\nWords are lower-cased with this locale's case rules, for example, when generating the case insensitive\nforms with `generateNonStrict`. Without it, the default case rules are used.\n\nSet `dictionaryInformation.locale` in the dictionary definition to the same locale, so cspell searches the\ndictionary with the case rules it was built with. If the definition lists more than one locale, cspell uses the\nfirst valid one for case rules, so put this locale first.",
|
|
381
|
+
"type": "string"
|
|
382
|
+
},
|
|
373
383
|
"maxDepth": {
|
|
374
384
|
"description": "Maximum number of nested Hunspell Rules to apply. This is needed for recursive dictionaries like Hebrew.",
|
|
375
385
|
"markdownDescription": "Maximum number of nested Hunspell Rules to apply.\nThis is needed for recursive dictionaries like Hebrew.",
|
package/dist/app.d.mts
CHANGED
|
@@ -28,7 +28,7 @@ declare class FeatureFlags {
|
|
|
28
28
|
}
|
|
29
29
|
//#endregion
|
|
30
30
|
//#region src/app.d.ts
|
|
31
|
-
declare function run(program: Command, argv: string[], flags?: FeatureFlags): Promise<void>;
|
|
31
|
+
export declare function run(program: Command, argv: string[], flags?: FeatureFlags): Promise<void>;
|
|
32
32
|
//#endregion
|
|
33
|
-
export {
|
|
33
|
+
export {};
|
|
34
34
|
//# sourceMappingURL=app.d.mts.map
|
package/dist/app.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { a as reportCheckChecksumFile, c as toError, i as logWithTimestamp, l as generateBTrie, n as compile, o as reportChecksumForFiles, r as setLogger, s as updateChecksumForFiles, t as build, u as compressFile } from "./build-
|
|
1
|
+
import { a as reportCheckChecksumFile, c as toError, i as logWithTimestamp, l as generateBTrie, n as compile, o as reportChecksumForFiles, r as setLogger, s as updateChecksumForFiles, t as build, u as compressFile } from "./build-DitWkAJE.mjs";
|
|
2
2
|
import { readFileSync } from "node:fs";
|
|
3
3
|
import { CommanderError, Option } from "commander";
|
|
4
4
|
import { writeFile } from "node:fs/promises";
|
|
@@ -550,6 +550,29 @@ function removeVerboseFromRegExp(pattern) {
|
|
|
550
550
|
return result.result;
|
|
551
551
|
}
|
|
552
552
|
//#endregion
|
|
553
|
+
//#region src/compiler/text.ts
|
|
554
|
+
const regExSplitWords = /([\p{Ll}])([\p{Lu}])/gu;
|
|
555
|
+
const regExSplitWords2 = /(\p{Lu})(\p{Lu}\p{Ll})/gu;
|
|
556
|
+
const regExpIsLetter = /^\p{L}\p{M}{0,2}$/u;
|
|
557
|
+
/**
|
|
558
|
+
* Split camelCase words into an array of strings.
|
|
559
|
+
*/
|
|
560
|
+
function splitCamelCaseWord(word) {
|
|
561
|
+
return word.replaceAll(regExSplitWords, "$1|$2").replaceAll(regExSplitWords2, "$1|$2").replaceAll(/[\d_]+/g, "|").split("|").filter((a) => !!a);
|
|
562
|
+
}
|
|
563
|
+
/**
|
|
564
|
+
* Lower-case a word.
|
|
565
|
+
* @param word - the word to lower-case.
|
|
566
|
+
* @param locale - the locale whose case rules are used. Default rules are used if not given.
|
|
567
|
+
* @returns the lower-case word.
|
|
568
|
+
*/
|
|
569
|
+
function lowerCase(word, locale) {
|
|
570
|
+
return locale ? word.toLocaleLowerCase(locale) : word.toLowerCase();
|
|
571
|
+
}
|
|
572
|
+
function isSingleLetter(c) {
|
|
573
|
+
return regExpIsLetter.test(c);
|
|
574
|
+
}
|
|
575
|
+
//#endregion
|
|
553
576
|
//#region src/compiler/WordsCollection.ts
|
|
554
577
|
const defaultAllowedSplitWords = Object.freeze({
|
|
555
578
|
size: 0,
|
|
@@ -572,15 +595,15 @@ var AllowedSplitWordsImpl = class {
|
|
|
572
595
|
return !this.size || this.collection.has(word, caseSensitive);
|
|
573
596
|
}
|
|
574
597
|
};
|
|
575
|
-
async function createAllowedSplitWordsFromFiles(files) {
|
|
598
|
+
async function createAllowedSplitWordsFromFiles(files, locale) {
|
|
576
599
|
if (!files || !files.length) return defaultAllowedSplitWords;
|
|
577
|
-
return new AllowedSplitWordsImpl(await createWordsCollectionFromFiles(files));
|
|
600
|
+
return new AllowedSplitWordsImpl(await createWordsCollectionFromFiles(files, locale));
|
|
578
601
|
}
|
|
579
|
-
function buildHasFn(dict) {
|
|
602
|
+
function buildHasFn(dict, locale) {
|
|
580
603
|
function has(word, caseSensitive) {
|
|
581
604
|
const r = dict.hasWord(word, true);
|
|
582
605
|
if (r || caseSensitive) return r;
|
|
583
|
-
const lc = word
|
|
606
|
+
const lc = lowerCase(word, locale);
|
|
584
607
|
if (lc === word) return false;
|
|
585
608
|
return dict.hasWord(lc, true);
|
|
586
609
|
}
|
|
@@ -589,9 +612,9 @@ function buildHasFn(dict) {
|
|
|
589
612
|
async function readFile$1(filename) {
|
|
590
613
|
return await createReader(filename, {});
|
|
591
614
|
}
|
|
592
|
-
function readersToCollection(readers) {
|
|
593
|
-
const dictReaders = readers.filter(isDictionaryReader).map(dictReaderToCollection);
|
|
594
|
-
const nonDictCollection = lineReadersToCollection(readers.filter((a) => !isDictionaryReader(a)));
|
|
615
|
+
function readersToCollection(readers, locale) {
|
|
616
|
+
const dictReaders = readers.filter(isDictionaryReader).map((reader) => dictReaderToCollection(reader, locale));
|
|
617
|
+
const nonDictCollection = lineReadersToCollection(readers.filter((a) => !isDictionaryReader(a)), locale);
|
|
595
618
|
const collections = [...dictReaders, nonDictCollection];
|
|
596
619
|
return {
|
|
597
620
|
size: collections.reduce((s, a) => s + a.size, 0),
|
|
@@ -599,35 +622,49 @@ function readersToCollection(readers) {
|
|
|
599
622
|
};
|
|
600
623
|
}
|
|
601
624
|
const cache = /* @__PURE__ */ new WeakMap();
|
|
602
|
-
|
|
625
|
+
const filePaths = /* @__PURE__ */ new Map();
|
|
626
|
+
function createWordsCollectionFromFiles(files, locale) {
|
|
603
627
|
files = toFilePathArray(files);
|
|
604
|
-
|
|
628
|
+
let byLocale = cache.get(files);
|
|
629
|
+
if (!byLocale) {
|
|
630
|
+
byLocale = /* @__PURE__ */ new Map();
|
|
631
|
+
cache.set(files, byLocale);
|
|
632
|
+
}
|
|
633
|
+
const cached = byLocale.get(locale);
|
|
605
634
|
if (cached) return cached;
|
|
606
|
-
const collection =
|
|
607
|
-
|
|
635
|
+
const collection = _createWordsCollectionFromFiles(files, locale);
|
|
636
|
+
byLocale.set(locale, collection);
|
|
608
637
|
return collection;
|
|
609
638
|
}
|
|
639
|
+
async function _createWordsCollectionFromFiles(files, locale) {
|
|
640
|
+
return readersToCollection(await Promise.all(files.map((file) => readFile$1(file))), locale);
|
|
641
|
+
}
|
|
610
642
|
function isDictionaryReader(reader) {
|
|
611
643
|
return "hasWord" in reader && !!reader.hasWord;
|
|
612
644
|
}
|
|
613
|
-
function dictReaderToCollection(reader) {
|
|
645
|
+
function dictReaderToCollection(reader, locale) {
|
|
614
646
|
return {
|
|
615
647
|
size: reader.size,
|
|
616
|
-
has: buildHasFn(reader)
|
|
648
|
+
has: buildHasFn(reader, locale)
|
|
617
649
|
};
|
|
618
650
|
}
|
|
619
|
-
function lineReadersToCollection(readers) {
|
|
651
|
+
function lineReadersToCollection(readers, locale) {
|
|
620
652
|
function* words() {
|
|
621
653
|
for (const reader of readers) yield* reader.lines;
|
|
622
654
|
}
|
|
623
655
|
const dict = parseDictionary(words(), { stripCaseAndAccents: false });
|
|
624
656
|
return {
|
|
625
657
|
size: dict.size,
|
|
626
|
-
has: buildHasFn(dict)
|
|
658
|
+
has: buildHasFn(dict, locale)
|
|
627
659
|
};
|
|
628
660
|
}
|
|
629
661
|
function toFilePathArray(filePathOrArray) {
|
|
630
|
-
|
|
662
|
+
if (Array.isArray(filePathOrArray)) return filePathOrArray;
|
|
663
|
+
const found = filePaths.get(filePathOrArray);
|
|
664
|
+
if (found) return found;
|
|
665
|
+
const paths = [filePathOrArray];
|
|
666
|
+
filePaths.set(filePathOrArray, paths);
|
|
667
|
+
return paths;
|
|
631
668
|
}
|
|
632
669
|
//#endregion
|
|
633
670
|
//#region src/compiler/logWithTimestamp.ts
|
|
@@ -645,48 +682,34 @@ const defaultCompileSourceOptions = {
|
|
|
645
682
|
minCompoundLength: 4
|
|
646
683
|
};
|
|
647
684
|
//#endregion
|
|
648
|
-
//#region src/compiler/text.ts
|
|
649
|
-
const regExSplitWords = /([\p{Ll}])([\p{Lu}])/gu;
|
|
650
|
-
const regExSplitWords2 = /(\p{Lu})(\p{Lu}\p{Ll})/gu;
|
|
651
|
-
const regExpIsLetter = /^\p{L}\p{M}{0,2}$/u;
|
|
652
|
-
/**
|
|
653
|
-
* Split camelCase words into an array of strings.
|
|
654
|
-
*/
|
|
655
|
-
function splitCamelCaseWord(word) {
|
|
656
|
-
return word.replaceAll(regExSplitWords, "$1|$2").replaceAll(regExSplitWords2, "$1|$2").replaceAll(/[\d_]+/g, "|").split("|").filter((a) => !!a);
|
|
657
|
-
}
|
|
658
|
-
function isSingleLetter(c) {
|
|
659
|
-
return regExpIsLetter.test(c);
|
|
660
|
-
}
|
|
661
|
-
//#endregion
|
|
662
685
|
//#region src/compiler/splitCamelCaseIfAllowed.ts
|
|
663
686
|
const regExpSpaceOrDash = /[- ]+/g;
|
|
664
687
|
const regExpIsNumber = /^\d+$/;
|
|
665
|
-
function splitCamelCaseIfAllowed(word, allowedWords, keepCase, compoundPrefix, minCompoundLength) {
|
|
688
|
+
function splitCamelCaseIfAllowed(word, allowedWords, keepCase, compoundPrefix, minCompoundLength, locale) {
|
|
666
689
|
const split = [...splitCamelCase(word)];
|
|
667
|
-
if (split.length === 1) return adjustCases(split, allowedWords, keepCase);
|
|
690
|
+
if (split.length === 1) return adjustCases(split, allowedWords, keepCase, locale);
|
|
668
691
|
if (split.some((w) => isUnknown(w, allowedWords))) return [word];
|
|
669
692
|
const wordIndexes = calcWordIndex(word, split);
|
|
670
|
-
const adjusted = adjustCases(split, allowedWords, keepCase);
|
|
693
|
+
const adjusted = adjustCases(split, allowedWords, keepCase, locale);
|
|
671
694
|
return !compoundPrefix ? adjusted : adjusted.map((w, i) => {
|
|
672
695
|
const { px, sx } = wordIndexes[i];
|
|
673
696
|
const canCompound = w.length >= minCompoundLength;
|
|
674
|
-
const lc = w
|
|
697
|
+
const lc = lowerCase(w, locale);
|
|
675
698
|
const p = canCompound && isSingleLetter(px) ? compoundPrefix : "";
|
|
676
699
|
const s = canCompound && isSingleLetter(sx) ? compoundPrefix : "";
|
|
677
700
|
if (lc.length < 4 || allowedWords.has(w, true)) return p + w + s;
|
|
678
701
|
return p + lc + s;
|
|
679
702
|
});
|
|
680
703
|
}
|
|
681
|
-
function adjustCases(words, allowedWords, keepCase) {
|
|
682
|
-
return words.map((w) => adjustCase(w, allowedWords, keepCase));
|
|
704
|
+
function adjustCases(words, allowedWords, keepCase, locale) {
|
|
705
|
+
return words.map((w) => adjustCase(w, allowedWords, keepCase, locale));
|
|
683
706
|
}
|
|
684
|
-
function adjustCase(word, allowedWords, keepCase) {
|
|
685
|
-
const lc = word
|
|
707
|
+
function adjustCase(word, allowedWords, keepCase, locale) {
|
|
708
|
+
const lc = lowerCase(word, locale);
|
|
686
709
|
if (!allowedWords.has(lc, true)) return word;
|
|
687
710
|
if (lc === word) return word;
|
|
688
711
|
if (word.slice(1).toLowerCase() === word.slice(1)) return lc;
|
|
689
|
-
if (!keepCase && word.toUpperCase() === word) return
|
|
712
|
+
if (!keepCase && word.toUpperCase() === word) return lc;
|
|
690
713
|
return word;
|
|
691
714
|
}
|
|
692
715
|
function isUnknown(word, allowedWords) {
|
|
@@ -718,16 +741,17 @@ function calcWordIndex(word, words) {
|
|
|
718
741
|
const regNonWord = /[^\p{L}\p{M}' _\d]+/giu;
|
|
719
742
|
const regExpRepeatChars = /(.)\1{5}/i;
|
|
720
743
|
const minCompoundLength = defaultCompileSourceOptions.minCompoundLength;
|
|
721
|
-
function legacyLineToWords(line, keepCase, allowedSplitWords) {
|
|
744
|
+
function legacyLineToWords(line, keepCase, allowedSplitWords, locale) {
|
|
722
745
|
const wordGroups = line.replaceAll(regNonWord, "|").split("|");
|
|
723
746
|
const _minCompoundLength = minCompoundLength;
|
|
724
|
-
return pipe(wordGroups, opConcatMap((a) => a.split(regExpSpaceOrDash)), opConcatMap((a) => splitCamelCaseIfAllowed(a, allowedSplitWords, keepCase, "", _minCompoundLength)), opMap((a) => a.trim()), opFilter((a) => !!a), opFilter((s) => !regExpRepeatChars.test(s)));
|
|
747
|
+
return pipe(wordGroups, opConcatMap((a) => a.split(regExpSpaceOrDash)), opConcatMap((a) => splitCamelCaseIfAllowed(a, allowedSplitWords, keepCase, "", _minCompoundLength, locale)), opMap((a) => a.trim()), opFilter((a) => !!a), opFilter((s) => !regExpRepeatChars.test(s)));
|
|
725
748
|
}
|
|
726
749
|
//#endregion
|
|
727
750
|
//#region src/compiler/wordListParser.ts
|
|
728
751
|
function normalizeTargetWords(options) {
|
|
729
752
|
const lineParser = createDictionaryLineParser({
|
|
730
753
|
stripCaseAndAccents: options.generateNonStrict,
|
|
754
|
+
locale: options.locale,
|
|
731
755
|
stripCaseAndAccentsOnForbidden: true,
|
|
732
756
|
keepOptionalCompoundCharacter: true
|
|
733
757
|
});
|
|
@@ -771,7 +795,8 @@ const _defaultOptions = {
|
|
|
771
795
|
size: 0
|
|
772
796
|
},
|
|
773
797
|
storeSplitWordsAsCompounds: defaultCompileSourceOptions.storeSplitWordsAsCompounds,
|
|
774
|
-
minCompoundLength: defaultCompileSourceOptions.minCompoundLength
|
|
798
|
+
minCompoundLength: defaultCompileSourceOptions.minCompoundLength,
|
|
799
|
+
locale: void 0
|
|
775
800
|
};
|
|
776
801
|
Object.freeze(_defaultOptions);
|
|
777
802
|
const cSpellToolDirective = "cspell-tools:";
|
|
@@ -783,7 +808,7 @@ const cSpellToolDirective = "cspell-tools:";
|
|
|
783
808
|
*/
|
|
784
809
|
function createParseFileLineMapper(options) {
|
|
785
810
|
const _options = options || _defaultOptions;
|
|
786
|
-
const { splitKeepBoth = _defaultOptions.splitKeepBoth, allowedSplitWords = _defaultOptions.allowedSplitWords, storeSplitWordsAsCompounds, minCompoundLength = _defaultOptions.minCompoundLength } = _options;
|
|
811
|
+
const { splitKeepBoth = _defaultOptions.splitKeepBoth, allowedSplitWords = _defaultOptions.allowedSplitWords, storeSplitWordsAsCompounds, minCompoundLength = _defaultOptions.minCompoundLength, locale } = _options;
|
|
787
812
|
let { legacy = _defaultOptions.legacy } = _options;
|
|
788
813
|
let { split = _defaultOptions.split, keepCase = legacy ? false : _defaultOptions.keepCase } = _options;
|
|
789
814
|
const compoundFix = storeSplitWordsAsCompounds ? "+" : "";
|
|
@@ -841,12 +866,12 @@ function createParseFileLineMapper(options) {
|
|
|
841
866
|
return line.split("|").map((a) => a.trim()).filter((a) => !!a).filter((a) => !/^[0-9_-]+$/.test(a)).filter((a) => !/^0[xo][0-9A-F]+$/i.test(a));
|
|
842
867
|
}
|
|
843
868
|
function splitWordIntoWords(word) {
|
|
844
|
-
return splitCamelCaseIfAllowed(word, allowedSplitWords, keepCase, compoundFix, minCompoundLength);
|
|
869
|
+
return splitCamelCaseIfAllowed(word, allowedSplitWords, keepCase, compoundFix, minCompoundLength, locale);
|
|
845
870
|
}
|
|
846
871
|
function* splitWords(lines) {
|
|
847
872
|
for (const line of lines) {
|
|
848
873
|
if (legacy) {
|
|
849
|
-
yield* legacyLineToWords(line, keepCase, allowedSplitWords);
|
|
874
|
+
yield* legacyLineToWords(line, keepCase, allowedSplitWords, locale);
|
|
850
875
|
continue;
|
|
851
876
|
}
|
|
852
877
|
if (split) {
|
|
@@ -911,7 +936,7 @@ function splitLines(lines, options) {
|
|
|
911
936
|
return split();
|
|
912
937
|
}
|
|
913
938
|
async function textFileReader(reader, options) {
|
|
914
|
-
const { legacy, splitWords: split, allowedSplitWords, storeSplitWordsAsCompounds, minCompoundLength } = options;
|
|
939
|
+
const { legacy, splitWords: split, allowedSplitWords, storeSplitWordsAsCompounds, minCompoundLength, locale } = options;
|
|
915
940
|
const parseOptions = {
|
|
916
941
|
legacy,
|
|
917
942
|
split,
|
|
@@ -919,7 +944,8 @@ async function textFileReader(reader, options) {
|
|
|
919
944
|
keepCase: void 0,
|
|
920
945
|
allowedSplitWords,
|
|
921
946
|
storeSplitWordsAsCompounds,
|
|
922
|
-
minCompoundLength
|
|
947
|
+
minCompoundLength,
|
|
948
|
+
locale
|
|
923
949
|
};
|
|
924
950
|
const words = [...parseFileLines(reader.lines, parseOptions)];
|
|
925
951
|
return {
|
|
@@ -974,7 +1000,7 @@ function createWordListCompiler(options) {
|
|
|
974
1000
|
}
|
|
975
1001
|
function normalize(lines, options) {
|
|
976
1002
|
const filter = normalizeTargetWords(options);
|
|
977
|
-
const cleanLines = options.removeDuplicates ? removeDuplicates(lines) : lines;
|
|
1003
|
+
const cleanLines = options.removeDuplicates ? removeDuplicates(lines, options.locale) : lines;
|
|
978
1004
|
const iter = pipe(cleanLines, filter);
|
|
979
1005
|
if (!options.sort) return iter;
|
|
980
1006
|
return [...new Set(iter)].sort();
|
|
@@ -982,11 +1008,11 @@ function normalize(lines, options) {
|
|
|
982
1008
|
function stripCompoundAFix(word) {
|
|
983
1009
|
return word.replaceAll("*", "").replaceAll("+", "");
|
|
984
1010
|
}
|
|
985
|
-
function* removeDuplicates(words) {
|
|
1011
|
+
function* removeDuplicates(words, locale) {
|
|
986
1012
|
const wordSet = new Set(words);
|
|
987
1013
|
const wordForms = /* @__PURE__ */ new Map();
|
|
988
1014
|
for (const word of wordSet) {
|
|
989
|
-
const lc = stripCompoundAFix(word
|
|
1015
|
+
const lc = stripCompoundAFix(lowerCase(word, locale));
|
|
990
1016
|
const forms = wordForms.get(lc) ?? [];
|
|
991
1017
|
forms.push(word);
|
|
992
1018
|
wordForms.set(lc, forms);
|
|
@@ -1001,7 +1027,7 @@ function* removeDuplicates(words) {
|
|
|
1001
1027
|
for (const form of mForms.values()) yield* form;
|
|
1002
1028
|
continue;
|
|
1003
1029
|
}
|
|
1004
|
-
const lc = [...mForms.keys()][0]
|
|
1030
|
+
const lc = lowerCase([...mForms.keys()][0], locale);
|
|
1005
1031
|
const lcForm = mForms.get(lc);
|
|
1006
1032
|
if (!lcForm) {
|
|
1007
1033
|
for (const form of mForms.values()) yield* form;
|
|
@@ -1012,7 +1038,7 @@ function* removeDuplicates(words) {
|
|
|
1012
1038
|
yield* lcForm;
|
|
1013
1039
|
if (sLcForms.has("*" + lc + "*")) continue;
|
|
1014
1040
|
for (const forms of mForms.values()) for (const form of forms) {
|
|
1015
|
-
if (sLcForms.has(form
|
|
1041
|
+
if (sLcForms.has(lowerCase(form, locale))) continue;
|
|
1016
1042
|
yield form;
|
|
1017
1043
|
}
|
|
1018
1044
|
}
|
|
@@ -1093,6 +1119,7 @@ async function compile(request, options) {
|
|
|
1093
1119
|
const targetOptions = {
|
|
1094
1120
|
sort: request.sort,
|
|
1095
1121
|
generateNonStrict: request.generateNonStrict,
|
|
1122
|
+
locale: request.locale,
|
|
1096
1123
|
removeDuplicates: request.removeDuplicates,
|
|
1097
1124
|
replacements: void 0
|
|
1098
1125
|
};
|
|
@@ -1157,18 +1184,20 @@ async function buildTargetDictionary(target, options, compileOptions, buildOptio
|
|
|
1157
1184
|
const { filename, useTrie, generateOnlyCompressedDictionary, checksumRoot } = buildOptions;
|
|
1158
1185
|
const dictionaryDirectives = target.dictionaryDirectives ?? compileOptions.dictionaryDirectives;
|
|
1159
1186
|
const removeDuplicates = target.removeDuplicates ?? false;
|
|
1160
|
-
const
|
|
1161
|
-
const
|
|
1187
|
+
const locale = canonicalLocale(target);
|
|
1188
|
+
const excludeFromFilter = await createExcludeFilter(excludeWordsFrom, locale);
|
|
1189
|
+
const includeFromFilter = await createIncludeFilter(excludeWordsNotFoundIn, locale);
|
|
1162
1190
|
const excludeRegexFilter = createExcludeRegexFilter(excludeWordsMatchingRegex);
|
|
1163
1191
|
const excludeFilter = (word) => {
|
|
1164
1192
|
return excludeFromFilter(word) && includeFromFilter(word) && excludeRegexFilter(word);
|
|
1165
1193
|
};
|
|
1166
1194
|
const compress = generateOnlyCompressedDictionary;
|
|
1167
|
-
const filesToProcessAsync = pipeAsync(readSourceList(sources, rootDir), opMapAsync((src) => readFileSource(src, options)), opAwaitAsync());
|
|
1195
|
+
const filesToProcessAsync = pipeAsync(readSourceList(sources, rootDir), opMapAsync((src) => readFileSource(src, options, locale)), opAwaitAsync());
|
|
1168
1196
|
const filesToProcess = await toArray(filesToProcessAsync);
|
|
1169
1197
|
const normalizer = normalizeTargetWords({
|
|
1170
1198
|
sort: useTrie || sort,
|
|
1171
1199
|
generateNonStrict,
|
|
1200
|
+
locale,
|
|
1172
1201
|
filter: excludeFilter,
|
|
1173
1202
|
dictionaryDirectives,
|
|
1174
1203
|
replacements
|
|
@@ -1188,6 +1217,7 @@ async function buildTargetDictionary(target, options, compileOptions, buildOptio
|
|
|
1188
1217
|
}) : createWordListCompiler({
|
|
1189
1218
|
sort,
|
|
1190
1219
|
generateNonStrict,
|
|
1220
|
+
locale,
|
|
1191
1221
|
dictionaryDirectives,
|
|
1192
1222
|
removeDuplicates,
|
|
1193
1223
|
replacements
|
|
@@ -1284,7 +1314,7 @@ function readSourceList(sources, rootDir) {
|
|
|
1284
1314
|
async function readFileList(fileList) {
|
|
1285
1315
|
return (await readTextFile(fileList)).split("\n").map((a) => a.trim()).filter((a) => !!a);
|
|
1286
1316
|
}
|
|
1287
|
-
async function readFileSource(fileSource, sourceOptions) {
|
|
1317
|
+
async function readFileSource(fileSource, sourceOptions, locale) {
|
|
1288
1318
|
const { filename, keepRawCase = sourceOptions.keepRawCase || false, split = sourceOptions.split || false, maxDepth, storeSplitWordsAsCompounds, minCompoundLength } = fileSource;
|
|
1289
1319
|
const legacy = split === "legacy";
|
|
1290
1320
|
const readerOptions = {
|
|
@@ -1292,9 +1322,10 @@ async function readFileSource(fileSource, sourceOptions) {
|
|
|
1292
1322
|
legacy,
|
|
1293
1323
|
splitWords: legacy ? false : split,
|
|
1294
1324
|
keepCase: keepRawCase,
|
|
1295
|
-
allowedSplitWords: await createAllowedSplitWordsFromFiles(fileSource.allowedSplitWords || sourceOptions.allowedSplitWords),
|
|
1325
|
+
allowedSplitWords: await createAllowedSplitWordsFromFiles(fileSource.allowedSplitWords || sourceOptions.allowedSplitWords, locale),
|
|
1296
1326
|
storeSplitWordsAsCompounds,
|
|
1297
|
-
minCompoundLength
|
|
1327
|
+
minCompoundLength,
|
|
1328
|
+
locale
|
|
1298
1329
|
};
|
|
1299
1330
|
logWithTimestamp(`Reading ${path.basename(filename)}`);
|
|
1300
1331
|
const stream = await streamSourceWordsFromFile(filename, readerOptions);
|
|
@@ -1304,6 +1335,19 @@ async function readFileSource(fileSource, sourceOptions) {
|
|
|
1304
1335
|
words: stream
|
|
1305
1336
|
};
|
|
1306
1337
|
}
|
|
1338
|
+
const regExpLocale = /^([a-z]{2})(?:[-_]([a-z]{2}))?$/i;
|
|
1339
|
+
/**
|
|
1340
|
+
* @returns the target's locale in its canonical form, such as `tr-TR` for `tr_tr`.
|
|
1341
|
+
* @throws if the locale isn't a language and an optional region.
|
|
1342
|
+
*/
|
|
1343
|
+
function canonicalLocale(target) {
|
|
1344
|
+
const { locale } = target;
|
|
1345
|
+
if (locale === void 0) return void 0;
|
|
1346
|
+
const m = regExpLocale.exec(locale);
|
|
1347
|
+
if (!m) throw new Error(`Invalid locale "${locale}" in target "${target.name}". Use a language and an optional region, such as "tr" or "tr-TR".`);
|
|
1348
|
+
const [, language, region] = m;
|
|
1349
|
+
return region ? `${language.toLowerCase()}-${region.toUpperCase()}` : language.toLowerCase();
|
|
1350
|
+
}
|
|
1307
1351
|
function normalizeTargetName(name) {
|
|
1308
1352
|
return name.replace(/((\.txt|\.dic|\.aff|\.trie)(\.gz)?)?$/, "").replaceAll(/[^\p{L}\p{M}.\w\\/-]/gu, "_");
|
|
1309
1353
|
}
|
|
@@ -1321,20 +1365,22 @@ function logProgress(freq = 1e5) {
|
|
|
1321
1365
|
}
|
|
1322
1366
|
/**
|
|
1323
1367
|
* @param excludeWordsFrom - List of files to read words from.
|
|
1368
|
+
* @param locale - the locale used to lower-case words when matching them.
|
|
1324
1369
|
* @returns a function that returns true if the word is not in the exclude list.
|
|
1325
1370
|
*/
|
|
1326
|
-
async function createExcludeFilter(excludeWordsFrom) {
|
|
1371
|
+
async function createExcludeFilter(excludeWordsFrom, locale) {
|
|
1327
1372
|
if (!excludeWordsFrom || !excludeWordsFrom.length) return () => true;
|
|
1328
|
-
const excludeWords = await createWordsCollectionFromFiles(excludeWordsFrom);
|
|
1373
|
+
const excludeWords = await createWordsCollectionFromFiles(excludeWordsFrom, locale);
|
|
1329
1374
|
return (word) => !excludeWords.has(word, word.toUpperCase() !== word);
|
|
1330
1375
|
}
|
|
1331
1376
|
/**
|
|
1332
1377
|
* @param includeWordsFrom - List of files to read words from.
|
|
1378
|
+
* @param locale - the locale used to lower-case words when matching them.
|
|
1333
1379
|
* @returns a function that returns true if the word is in the include list.
|
|
1334
1380
|
*/
|
|
1335
|
-
async function createIncludeFilter(includeWordsFrom) {
|
|
1381
|
+
async function createIncludeFilter(includeWordsFrom, locale) {
|
|
1336
1382
|
if (!includeWordsFrom || !includeWordsFrom.length) return () => true;
|
|
1337
|
-
const excludeWords = await createWordsCollectionFromFiles(includeWordsFrom);
|
|
1383
|
+
const excludeWords = await createWordsCollectionFromFiles(includeWordsFrom, locale);
|
|
1338
1384
|
return (word) => excludeWords.has(word, word.toUpperCase() !== word);
|
|
1339
1385
|
}
|
|
1340
1386
|
/**
|
|
@@ -1399,4 +1445,4 @@ function normalizeRequest(buildInfo, root) {
|
|
|
1399
1445
|
//#endregion
|
|
1400
1446
|
export { reportCheckChecksumFile as a, toError as c, logWithTimestamp as i, generateBTrie as l, compile as n, reportChecksumForFiles as o, setLogger as r, updateChecksumForFiles as s, build as t, compressFile as u };
|
|
1401
1447
|
|
|
1402
|
-
//# sourceMappingURL=build-
|
|
1448
|
+
//# sourceMappingURL=build-DitWkAJE.mjs.map
|
package/dist/index.d.mts
CHANGED
|
@@ -19,7 +19,7 @@ interface GenerateBTrieOptions extends BTrieOptions {
|
|
|
19
19
|
}
|
|
20
20
|
//#endregion
|
|
21
21
|
//#region src/bTrie.d.ts
|
|
22
|
-
declare function generateBTrie(files: string[], options: GenerateBTrieOptions): Promise<void>;
|
|
22
|
+
export declare function generateBTrie(files: string[], options: GenerateBTrieOptions): Promise<void>;
|
|
23
23
|
//#endregion
|
|
24
24
|
//#region src/build.d.ts
|
|
25
25
|
interface BuildOptions {
|
|
@@ -32,7 +32,7 @@ interface BuildOptions {
|
|
|
32
32
|
/** Conditional build based upon the targets matching the `checksum.txt` file. */
|
|
33
33
|
conditional?: boolean;
|
|
34
34
|
}
|
|
35
|
-
declare function build(targets: string[] | undefined, options: BuildOptions): Promise<void>;
|
|
35
|
+
export declare function build(targets: string[] | undefined, options: BuildOptions): Promise<void>;
|
|
36
36
|
//#endregion
|
|
37
|
-
export {
|
|
37
|
+
export {};
|
|
38
38
|
//# sourceMappingURL=index.d.mts.map
|
package/dist/index.mjs
CHANGED
|
@@ -1,2 +1 @@
|
|
|
1
|
-
|
|
2
|
-
export { build, generateBTrie };
|
|
1
|
+
export { t as build, l as generateBTrie } from "./build-DitWkAJE.mjs";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cspell/cspell-tools",
|
|
3
|
-
"version": "10.4.0
|
|
3
|
+
"version": "10.4.0",
|
|
4
4
|
"description": "Tools to assist with the development of cSpell",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public",
|
|
@@ -65,12 +65,12 @@
|
|
|
65
65
|
"watch": "tsdown -w"
|
|
66
66
|
},
|
|
67
67
|
"dependencies": {
|
|
68
|
-
"@cspell/cspell-pipe": "10.4.0
|
|
68
|
+
"@cspell/cspell-pipe": "10.4.0",
|
|
69
69
|
"commander": "^15.0.0",
|
|
70
70
|
"cosmiconfig": "10.0.1",
|
|
71
|
-
"cspell-trie-lib": "10.4.0
|
|
71
|
+
"cspell-trie-lib": "10.4.0",
|
|
72
72
|
"glob": "^13.0.6",
|
|
73
|
-
"hunspell-reader": "10.4.0
|
|
73
|
+
"hunspell-reader": "10.4.0",
|
|
74
74
|
"yaml": "^2.9.1"
|
|
75
75
|
},
|
|
76
76
|
"devDependencies": {
|
|
@@ -80,5 +80,5 @@
|
|
|
80
80
|
"engines": {
|
|
81
81
|
"node": ">=22.18.0"
|
|
82
82
|
},
|
|
83
|
-
"gitHead": "
|
|
83
|
+
"gitHead": "7e2bbb5e866c827ac5d2cf6b84bf3e1a016d0823"
|
|
84
84
|
}
|