cspell-lib 8.13.1 → 8.13.3

package/dist/lib/index.d.ts

@@ -25,7 +25,7 @@ export type { TraceOptions, TraceResult, TraceWordResult } from './trace.js';
 export { traceWords, traceWordsAsync } from './trace.js';
 export { getLogger, Logger, setLogger } from './util/logger.js';
 export { resolveFile } from './util/resolveFile.js';
-export * as Text from './util/text.js';
+export * as Text from './util/textApi.js';
 export { checkText, checkTextDocument, CheckTextInfo, IncludeExcludeFlag, IncludeExcludeOptions, TextInfoItem, validateText, ValidationIssue, } from './validator.js';
 export * from '@cspell/cspell-types';
 export { asyncIterableToArray, readFileText as readFile, readFileTextSync as readFileSync, writeToFile, writeToFileIterable, writeToFileIterableP, } from 'cspell-io';

package/dist/lib/index.js

@@ -18,7 +18,7 @@ export { DocumentValidator, shouldCheckDocument } from './textValidation/index.j
 export { traceWords, traceWordsAsync } from './trace.js';
 export { getLogger, setLogger } from './util/logger.js';
 export { resolveFile } from './util/resolveFile.js';
-export * as Text from './util/text.js';
+export * as Text from './util/textApi.js';
 export { checkText, checkTextDocument, IncludeExcludeFlag, validateText, } from './validator.js';
 export * from '@cspell/cspell-types';
 export { asyncIterableToArray, readFileText as readFile, readFileTextSync as readFileSync, writeToFile, writeToFileIterable, writeToFileIterableP, } from 'cspell-io';

package/dist/lib/textValidation/ValidationTypes.d.ts

@@ -28,7 +28,9 @@ export interface WordRangeAcc {
 export type ValidationIssueRO = Readonly<ValidationIssue>;
 export type LineValidatorFn = (line: LineSegment) => Iterable<ValidationIssue>;
 export interface LineSegment {
+    /** A line from the document, the offset is relative to the beginning of the document. */
     line: TextOffsetRO;
+    /** A segment of text from the line, the offset is relative to the beginning of the document. */
     segment: TextOffsetRO;
 }
 export interface MappedTextValidationResult extends MappedText {

package/dist/lib/textValidation/lineValidatorFactory.js

@@ -1,7 +1,9 @@
+import assert from 'node:assert';
 import { opConcatMap, opFilter, pipe } from '@cspell/cspell-pipe/sync';
 import { createCachingDictionary } from 'cspell-dictionary';
 import * as RxPat from '../Settings/RegExpPatterns.js';
-import { extractPossibleWordsFromTextOffset, extractText, extractWordsFromCodeTextOffset, extractWordsFromTextOffset, } from '../util/text.js';
+import { extractPossibleWordsFromTextOffset, extractText, extractWordsFromTextOffset, splitWordWithOffset, } from '../util/text.js';
+import { regExpCamelCaseWordBreaksWithEnglishSuffix } from '../util/textRegex.js';
 import { split } from '../util/wordSplitter.js';
 import { defaultMinWordLength } from './defaultConstants.js';
 import { isWordValidWithEscapeRetry } from './isWordValid.js';
@@ -73,7 +75,7 @@ export function lineValidatorFactory(sDict, options) {
         }
         return issue;
     }
-    const isFlaggedOrMinLength = rememberFilter((wo) => wo.text.length >= minWordLength || !!wo.isFlagged);
+    const isFlaggedOrMinLength = (wo) => wo.text.length >= minWordLength || !!wo.isFlagged;
     const isFlaggedOrNotFound = rememberFilter((wo) => wo.isFlagged || !wo.isFound);
     const isNotRepeatingChar = rememberFilter((wo) => !RxPat.regExRepeatedChar.test(wo.text));
     function checkWord(issue) {
@@ -94,17 +96,100 @@ export function lineValidatorFactory(sDict, options) {
         issue.isFound = isFlagged ? undefined : info.isFound;
         return issue;
     }
+    const regExUpperCaseWithTrailingCommonEnglishSuffix = /^([\p{Lu}\p{M}]{2,})['’]?(?:s|ing|ies|es|ings|ize|ed|ning)$/u; // cspell:disable-line
+    const regExpIsLetter = /\p{L}/u;
     const fn = (lineSegment) => {
+        const line = lineSegment.line;
+        function isWordTooShort(word, ignoreSuffix = false) {
+            if (word.text.length >= minWordLength)
+                return false;
+            const offset = word.offset - line.offset;
+            assert.equal(line.text.slice(offset, offset + word.text.length), word.text);
+            const prefix = [...line.text.slice(Math.max(0, offset - 2), offset)];
+            const hasLetterPrefix = !!prefix.length && regExpIsLetter.test(prefix[prefix.length - 1]);
+            if (hasLetterPrefix)
+                return false;
+            if (ignoreSuffix)
+                return true;
+            const suffix = [...line.text.slice(offset + word.text.length, offset + word.text.length + 2)];
+            const hasLetterSuffix = !!suffix.length && regExpIsLetter.test(suffix[0]);
+            return !hasLetterSuffix;
+        }
         function splitterIsValid(word) {
-            return (setOfKnownSuccessfulWords.has(word.text) ||
-                (!isWordFlagged(word) && isWordValidWithEscapeRetry(hasDict, word, lineSegment.line)));
+            if (setOfKnownSuccessfulWords.has(word.text))
+                return true;
+            if (isWordFlagged(word))
+                return false;
+            if (isWordValidWithEscapeRetry(hasDict, word, lineSegment.line))
+                return true;
+            if (isWordTooShort(word))
+                return true;
+            return isAllCapsWithTrailingCommonEnglishSuffixOk(word);
+        }
+        function isAllCapsWithTrailingCommonEnglishSuffixOk(tWord) {
+            if (!regExUpperCaseWithTrailingCommonEnglishSuffix.test(tWord.text))
+                return false;
+            const m = tWord.text.match(regExUpperCaseWithTrailingCommonEnglishSuffix);
+            if (!m)
+                return false;
+            const offset = tWord.offset;
+            const v = { offset, text: m[1], line };
+            const check = checkWord(v);
+            if (check.isFlagged)
+                return false;
+            if (check.isFound)
+                return true;
+            if (isWordTooShort(v, true))
+                return true;
+            return false;
         }
         function checkFullWord(vr) {
             if (vr.isFlagged) {
                 return [vr];
             }
+            // English exceptions :-(
+            if (isAllCapsWithTrailingCommonEnglishSuffixOk(vr))
+                return [];
+            if (isWordIgnored(vr.text) || checkWord(vr).isFound) {
+                rememberFilter((_) => false)(vr);
+                return [];
+            }
+            if (vr.isFlagged)
+                return [vr];
+            const codeWordResults = checkCamelCaseWord(vr);
+            if (!codeWordResults.length) {
+                rememberFilter((_) => false)(vr);
+                return [];
+            }
+            return codeWordResults;
+        }
+        /**
+         * Break a camel case word into its parts and check each part.
+         *
+         * There are two word break patterns:
+         * - `regExpCamelCaseWordBreaks`
+         * - `regExpCamelCaseWordBreaksWithEnglishSuffix` is the default pattern with English suffixes on ALL CAPS words.
+         *
+         * Note: See [#6066](https://github.com/streetsidesoftware/cspell/pull/6066)
+         * Using just `regExpCamelCaseWordBreaks` misses unknown 4-letter words.
+         *
+         * The code below was tried, but it missed words.
+         * - `LSTM` was caught. // cspell:disable-line
+         * - `LSTMs` was missed because it becomes `LST` and `Ms`. // cspell:disable-line
+         *
+         * ```ts
+         * const results = _checkCamelCaseWord(vr, regExpCamelCaseWordBreaks);
+         * if (!results.length) return results;
+         * const resultsEnglishBreaks = _checkCamelCaseWord(vr, regExpCamelCaseWordBreaksWithEnglishSuffix);
+         * return results.length < resultsEnglishBreaks.length ? results : resultsEnglishBreaks;
+         * ```
+         */
+        function checkCamelCaseWord(vr) {
+            return _checkCamelCaseWord(vr, regExpCamelCaseWordBreaksWithEnglishSuffix);
+        }
+        function _checkCamelCaseWord(vr, regExpWordBreaks) {
             const codeWordResults = [];
-            for (const wo of extractWordsFromCodeTextOffset(vr)) {
+            for (const wo of splitWordWithOffset(vr, regExpWordBreaks)) {
                 if (setOfKnownSuccessfulWords.has(wo.text))
                     continue;
                 const issue = wo;
@@ -120,13 +205,8 @@ export function lineValidatorFactory(sDict, options) {
                 issue.text = extractText(lineSegment.segment, issue.offset, issue.offset + issue.text.length);
                 codeWordResults.push(issue);
             }
-            if (!codeWordResults.length || isWordIgnored(vr.text) || checkWord(vr).isFound) {
-                rememberFilter((_) => false)(vr);
-                return [];
-            }
             return codeWordResults;
         }
-        const useKnownIssues = false;
         function rebaseKnownIssues(possibleWord, known) {
             const { issues } = known;
             const adjOffset = possibleWord.offset - known.possibleWord.offset;
@@ -139,9 +219,9 @@ export function lineValidatorFactory(sDict, options) {
         }
         function checkPossibleWords(possibleWord) {
             const known = setOfKnownIssues.get(possibleWord.text);
-            if (known && !known.issues.length)
-                return known.issues;
-            if (known && useKnownIssues) {
+            if (known) {
+                if (!known.issues.length)
+                    return known.issues;
                 const adjusted = rebaseKnownIssues(possibleWord, known);
                 return adjusted;
             }
@@ -174,7 +254,15 @@ export function lineValidatorFactory(sDict, options) {
             if (mismatches.length) {
                 // Try the more expensive word splitter
                 const splitResult = split(lineSegment.segment, possibleWord.offset, splitterIsValid);
-                const nonMatching = splitResult.words.filter((w) => !w.isFound);
+                const nonMatching = splitResult.words
+                    .filter((w) => !w.isFound)
+                    .filter((w) => {
+                    const m = w.text.match(regExUpperCaseWithTrailingCommonEnglishSuffix);
+                    if (!m)
+                        return true;
+                    const v = checkWord({ ...w, text: m[1], line: lineSegment.line });
+                    return v.isFlagged || !v.isFound;
+                });
                 if (nonMatching.length < mismatches.length) {
                     return nonMatching.map((w) => ({ ...w, line: lineSegment.line })).map(annotateIsFlagged);
                 }

package/dist/lib/util/text.d.ts

@@ -1,11 +1,16 @@
 import type { TextDocumentOffset, TextOffset } from '@cspell/cspell-types';
 import type { Uri } from './Uri.js';
 export { stringToRegExp } from './textRegex.js';
-export declare function splitCamelCaseWordWithOffset(wo: TextOffset): Array<TextOffset>;
+export declare function splitCamelCaseWordWithOffset(wo: TextOffset): TextOffset[];
 /**
  * Split camelCase words into an array of strings.
  */
 export declare function splitCamelCaseWord(word: string): string[];
+export declare function splitWordWithOffset(wo: TextOffset, regExpWordBreaks: RegExp): TextOffset[];
+/**
+ * Split camelCase words into an array of strings.
+ */
+export declare function splitWord(word: string, regExpWordBreaks: RegExp): string[];
 /**
  * This function lets you iterate over regular expression matches.
  */

package/dist/lib/util/text.js

@@ -1,6 +1,6 @@
 import { opConcatMap, opMap, pipe } from '@cspell/cspell-pipe/sync';
 import { binarySearch } from './search.js';
-import { regExAccents, regExAllLower, regExAllUpper, regExFirstUpper, regExIgnoreCharacters, regExpSplitWordBreaks, regExWords, regExWordsAndDigits, } from './textRegex.js';
+import { regExAccents, regExAllLower, regExAllUpper, regExFirstUpper, regExIgnoreCharacters, regExpCamelCaseWordBreaksWithEnglishSuffix, regExWords, regExWordsAndDigits, } from './textRegex.js';
 import { toUri } from './Uri.js';
 import { scanMap } from './util.js';
 export { stringToRegExp } from './textRegex.js';
@@ -15,7 +15,19 @@ export function splitCamelCaseWordWithOffset(wo) {
  * Split camelCase words into an array of strings.
  */
 export function splitCamelCaseWord(word) {
-    return word.split(regExpSplitWordBreaks);
+    return splitWord(word, regExpCamelCaseWordBreaksWithEnglishSuffix);
+}
+export function splitWordWithOffset(wo, regExpWordBreaks) {
+    return splitWord(wo.text, regExpWordBreaks).map(scanMap((last, text) => ({ text, offset: last.offset + last.text.length }), {
+        text: '',
+        offset: wo.offset,
+    }));
+}
+/**
+ * Split camelCase words into an array of strings.
+ */
+export function splitWord(word, regExpWordBreaks) {
+    return word.split(regExpWordBreaks);
 }
 /**
  * This function lets you iterate over regular expression matches.

package/dist/lib/util/textApi.d.ts

@@ -0,0 +1,2 @@
+export { calculateTextDocumentOffsets, camelToSnake, cleanText, cleanTextOffset, extractLinesOfText, extractPossibleWordsFromTextOffset, extractText, extractWordsFromCode, extractWordsFromCodeTextOffset, extractWordsFromText, extractWordsFromTextOffset, isFirstCharacterLower, isFirstCharacterUpper, isLowerCase, isUpperCase, lcFirst, match, matchCase, matchStringToTextOffset, matchToTextOffset, removeAccents, snakeToCamel, splitCamelCaseWord, splitCamelCaseWordWithOffset, stringToRegExp, textOffset, ucFirst, } from './text.js';
+//# sourceMappingURL=textApi.d.ts.map

package/dist/lib/util/textApi.js

@@ -0,0 +1,2 @@
+export { calculateTextDocumentOffsets, camelToSnake, cleanText, cleanTextOffset, extractLinesOfText, extractPossibleWordsFromTextOffset, extractText, extractWordsFromCode, extractWordsFromCodeTextOffset, extractWordsFromText, extractWordsFromTextOffset, isFirstCharacterLower, isFirstCharacterUpper, isLowerCase, isUpperCase, lcFirst, match, matchCase, matchStringToTextOffset, matchToTextOffset, removeAccents, snakeToCamel, splitCamelCaseWord, splitCamelCaseWordWithOffset, stringToRegExp, textOffset, ucFirst, } from './text.js';
+//# sourceMappingURL=textApi.js.map

package/dist/lib/util/textRegex.d.ts

@@ -1,7 +1,8 @@
 export declare const regExUpperSOrIng: RegExp;
 export declare const regExSplitWords: RegExp;
 export declare const regExSplitWords2: RegExp;
-export declare const regExpSplitWordBreaks: RegExp;
+export declare const regExpCamelCaseWordBreaksWithEnglishSuffix: RegExp;
+export declare const regExpCamelCaseWordBreaks: RegExp;
 export declare const regExpAllPossibleWordBreaks: RegExp;
 export declare const regExWords: RegExp;
 export declare const regExWordsAndDigits: RegExp;

package/dist/lib/util/textRegex.js

@@ -1,8 +1,9 @@
 // cspell:ignore ings ning gimuy anrvtbf gimuxy
 export const regExUpperSOrIng = /([\p{Lu}\p{M}]+(?:\\?['’])?(?:s|ing|ies|es|ings|ed|ning))(?!\p{Ll})/gu;
 export const regExSplitWords = /(\p{Ll}\p{M}?)(\p{Lu})/gu;
-export const regExSplitWords2 = /(\p{Lu}\p{M}?)(\p{Lu}\p{M}?\p{Ll})/gu;
-export const regExpSplitWordBreaks = /(?<=\p{Ll}\p{M}?)(?=\p{Lu})|(?<=\p{Lu}\p{M}?)(?=\p{Lu}\p{M}?\p{Ll})(?!\p{Lu}\p{M}?(?:s|ing|ies|es|ings|ed|ning)(?!\p{Ll}))/gu;
+export const regExSplitWords2 = /(\p{Lu}\p{M}?)((\p{Lu}\p{M}?)\p{Ll})/gu;
+export const regExpCamelCaseWordBreaksWithEnglishSuffix = /(?<=\p{Ll}\p{M}?)(?=\p{Lu})|(?<=\p{Lu}\p{M}?)(?=\p{Lu}\p{M}?\p{Ll})(?!\p{Lu}\p{M}?(?:s|ing|ies|es|ings|ed|ning)(?!\p{Ll}))/gu;
+export const regExpCamelCaseWordBreaks = /(?<=\p{Ll}\p{M}?)(?=\p{Lu})|(?<=\p{Lu}\p{M}?)(?=\p{Lu}\p{M}?\p{Ll})/gu;
 export const regExpAllPossibleWordBreaks = /(?<=\p{Ll}\p{M}?)(?=\p{Lu})|(?<=\p{Lu}\p{M}?)(?=\p{Lu}\p{M}?\p{Ll})|(?<=\p{Lu}\p{M}?\p{Lu}\p{M}?)(?=\p{Ll})|(?<=\p{L}\p{M}?)(?=\P{L})|(?<=\P{L})(?=\p{L})/gu;
 export const regExWords = /\p{L}\p{M}?(?:(?:\\?['’])?\p{L}\p{M}?)*/gu;
 // Words can be made of letters, numbers, period, underscore, dash, plus, and single quote

package/dist/lib/util/wordSplitter.js

@@ -111,7 +111,7 @@ function genWordBreakCamel(line) {
     for (const m of text.matchAll(offsetRegEx(regExSplitWords, line.relStart))) {
         if (m.index === undefined)
             break;
-        const i = m.index + 1;
+        const i = m.index + m[1].length;
         breaksCamel1.push({
             offset: m.index,
             breaks: [[i, i], ignoreBreak],
@@ -124,7 +124,7 @@ function genWordBreakCamel(line) {
         if (m.index === undefined)
             break;
         const i = m.index + m[1].length;
-        const j = i + 1;
+        const j = i + m[3].length;
         breaksCamel2.push({
             offset: m.index,
             breaks: [[i, i], [j, j], ignoreBreak],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cspell-lib",
-  "version": "8.13.1",
+  "version": "8.13.3",
   "description": "A library of useful functions used across various cspell tools.",
   "type": "module",
   "sideEffects": false,
@@ -58,21 +58,21 @@
   },
   "homepage": "https://github.com/streetsidesoftware/cspell#readme",
   "dependencies": {
-    "@cspell/cspell-bundled-dicts": "8.13.1",
-    "@cspell/cspell-pipe": "8.13.1",
-    "@cspell/cspell-resolver": "8.13.1",
-    "@cspell/cspell-types": "8.13.1",
-    "@cspell/dynamic-import": "8.13.1",
-    "@cspell/strong-weak-map": "8.13.1",
-    "@cspell/url": "8.13.1",
+    "@cspell/cspell-bundled-dicts": "8.13.3",
+    "@cspell/cspell-pipe": "8.13.3",
+    "@cspell/cspell-resolver": "8.13.3",
+    "@cspell/cspell-types": "8.13.3",
+    "@cspell/dynamic-import": "8.13.3",
+    "@cspell/strong-weak-map": "8.13.3",
+    "@cspell/url": "8.13.3",
     "clear-module": "^4.1.2",
-    "comment-json": "^4.2.4",
-    "cspell-config-lib": "8.13.1",
-    "cspell-dictionary": "8.13.1",
-    "cspell-glob": "8.13.1",
-    "cspell-grammar": "8.13.1",
-    "cspell-io": "8.13.1",
-    "cspell-trie-lib": "8.13.1",
+    "comment-json": "^4.2.5",
+    "cspell-config-lib": "8.13.3",
+    "cspell-dictionary": "8.13.3",
+    "cspell-glob": "8.13.3",
+    "cspell-grammar": "8.13.3",
+    "cspell-io": "8.13.3",
+    "cspell-trie-lib": "8.13.3",
     "env-paths": "^3.0.0",
     "fast-equals": "^5.0.1",
     "gensequence": "^7.0.0",
@@ -93,7 +93,7 @@
     "@cspell/dict-fr-fr": "^2.2.2",
     "@cspell/dict-html": "^4.0.5",
     "@cspell/dict-nl-nl": "^2.3.0",
-    "@cspell/dict-python": "^4.2.3",
+    "@cspell/dict-python": "^4.2.4",
     "@types/configstore": "^6.0.2",
     "configstore": "^7.0.0",
     "cspell-dict-nl-nl": "^1.1.2",
@@ -101,5 +101,5 @@
     "lorem-ipsum": "^2.0.8",
     "perf-insight": "^1.2.0"
   },
-  "gitHead": "99cdb4e3e6579c57de1014b0cd3c168188b9c1f5"
+  "gitHead": "e017775a1d181b20abce3c6325f2527a7554a3a9"
 }