cspell-lib 8.13.0 → 8.13.2

package/dist/{esm → lib}/util/wordSplitter.js

@@ -224,7 +224,7 @@ function splitIntoWords(lineSeg, breaks, has) {
         }
         return br.breaks.map(c);
     }
-    function toTextOffset(text, offset) {
+    function checkTextOffset(text, offset) {
         const valid = has({ text, offset });
         return {
             text,
@@ -282,7 +282,7 @@ function splitIntoWords(lineSeg, breaks, has) {
             // yes
             const i = best.bp[0];
             const j = best.bp[1];
-            const t = i > best.i ? toTextOffset(text.slice(best.i, i), best.i) : undefined;
+            const t = i > best.i ? checkTextOffset(text.slice(best.i, i), best.i) : undefined;
             const cost = !t || t.isFound ? 0 : t.text.length;
             const mc = maxIndex - j;
             best.c += cost;
@@ -304,12 +304,12 @@ function splitIntoWords(lineSeg, breaks, has) {
             const c = makeCandidates(best.p, best.i, best.bi + 1, best.c);
             candidates.append(c);
             if (!c.length) {
-                const t = maxIndex > best.i ? toTextOffset(text.slice(best.i, maxIndex), best.i) : undefined;
+                const t = maxIndex > best.i ? checkTextOffset(text.slice(best.i, maxIndex), best.i) : undefined;
                 const cost = !t || t.isFound ? 0 : t.text.length;
                 best.c += cost;
                 best.ec = best.c;
                 best.text = t;
-                const segText = t || best.p?.text || toTextOffset('', best.i);
+                const segText = t || best.p?.text || checkTextOffset('', best.i);
                 const can = t ? { ...best, text: segText } : { ...best, ...best.p, text: segText };
                 const f = addToKnownPaths(can, undefined);
                 bestPath = !bestPath || (f && f.c < bestPath.c) ? f : bestPath;

package/dist/lib/util/wordSplitter.perf.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=wordSplitter.perf.d.ts.map

package/dist/lib/util/wordSplitter.perf.js

@@ -0,0 +1,125 @@
+import { readFile } from 'node:fs/promises';
+import { extname } from 'node:path/posix';
+import { suite } from 'perf-insight';
+import { extractPossibleWordsFromTextOffset, extractWordsFromCode, extractWordsFromText, matchStringToTextOffset, splitCamelCaseWordWithOffset, textOffset, } from './text.js';
+import { regExWordsAndDigits } from './textRegex.js';
+const regExpWord = /\b[\w\p{L}\p{M}]+\b/gu;
+suite('wordSplitter', async (test) => {
+    const lines = await sampleLines();
+    const iterations = 1;
+    test('baseline: matchAll /[\\w\\p{L}\\p{M}]+/gu', () => {
+        const s = [];
+        const _regExpWord = regExpWord;
+        for (let i = iterations; i > 0; --i) {
+            for (const line of lines) {
+                for (const m of line.matchAll(_regExpWord)) {
+                    s.push({ text: m[0], offset: m.index });
+                }
+            }
+        }
+        return s;
+    });
+    test('baseline: matchAll non-space /\\S+/g', () => {
+        const s = [];
+        const regExp = /\S+/g;
+        for (let i = iterations; i > 0; --i) {
+            for (const line of lines) {
+                s.push(...[...line.matchAll(regExp)].map((a) => ({ text: a[0], offset: a.index })));
+            }
+        }
+        return s;
+    });
+    test('baseline: matchAll non-special', () => {
+        const s = [];
+        const regExp = /[^\s();:{}[\]*&^%$#@!~"?/\\,<>+=]+/g;
+        for (let i = iterations; i > 0; --i) {
+            for (const line of lines) {
+                s.push(...[...line.matchAll(regExp)].map((a) => ({ text: a[0], offset: a.index })));
+            }
+        }
+        return s;
+    });
+    test('baseline: matchAll regExWordsAndDigits', () => {
+        const s = [];
+        const regExp = regExWordsAndDigits;
+        for (let i = iterations; i > 0; --i) {
+            for (const line of lines) {
+                s.push(...[...line.matchAll(regExp)].map((a) => ({ text: a[0], offset: a.index })));
+            }
+        }
+        return s;
+    });
+    test('baseline: matchAll regExWordsAndDigits matchStringToTextOffset', () => {
+        const s = [];
+        const regExp = regExWordsAndDigits;
+        for (let i = iterations; i > 0; --i) {
+            for (const line of lines) {
+                s.push(...matchStringToTextOffset(regExp, line));
+            }
+        }
+        return s;
+    });
+    test('matchAll into possible words', () => {
+        const s = [];
+        const regExpMaybeWord = /[^\s();:{}[\]*&^%$#@!~"?/\\,<>+=]+/g;
+        const _regExpWord = regExpWord;
+        for (let i = iterations; i > 0; --i) {
+            for (const line of lines) {
+                for (const m of line.matchAll(regExpMaybeWord)) {
+                    const index = m.index;
+                    for (const words of m[0].matchAll(_regExpWord)) {
+                        s.push({ text: words[0], offset: index + words.index });
+                    }
+                }
+            }
+        }
+        return s;
+    });
+    test('extractWordsFromText', () => {
+        const s = [];
+        for (let i = iterations; i > 0; --i) {
+            for (const line of lines) {
+                s.push(...extractWordsFromText(line));
+            }
+        }
+        return s;
+    });
+    test('extractPossibleWordsFromTextOffset', () => {
+        const s = [];
+        for (let i = iterations; i > 0; --i) {
+            for (const line of lines) {
+                s.push(...extractPossibleWordsFromTextOffset(textOffset(line)));
+            }
+        }
+        return s;
+    });
+    test('extractPossibleWordsFromTextOffset splitCamelCaseWordWithOffset', () => {
+        const s = [];
+        for (let i = iterations; i > 0; --i) {
+            for (const line of lines) {
+                for (const wo of extractPossibleWordsFromTextOffset(textOffset(line))) {
+                    for (const cWord of splitCamelCaseWordWithOffset(wo)) {
+                        s.push(cWord);
+                    }
+                }
+            }
+        }
+        return s;
+    });
+    test('extractWordsFromCode', () => {
+        const s = [];
+        for (let i = iterations; i > 0; --i) {
+            for (const line of lines) {
+                s.push(...extractWordsFromCode(line));
+            }
+        }
+        return s;
+    });
+});
+async function sampleLines() {
+    const ext = extname(new URL(import.meta.url).pathname);
+    const url = new URL('wordSplitter.test' + ext, import.meta.url);
+    const context = await readFile(url, 'utf8');
+    return context.replaceAll('\r\n', '\n').replaceAll('\r', '\n').split('\n');
+}
+//# sourceMappingURL=wordSplitter.perf.js.map

package/package.json

@@ -1,14 +1,14 @@
 {
   "name": "cspell-lib",
-  "version": "8.13.0",
+  "version": "8.13.2",
   "description": "A library of useful functions used across various cspell tools.",
   "type": "module",
   "sideEffects": false,
-  "types": "dist/esm/index.d.ts",
-  "module": "dist/esm/index.js",
+  "types": "dist/lib/index.d.ts",
+  "module": "dist/lib/index.js",
   "exports": {
     ".": {
-      "import": "./dist/esm/index.js"
+      "import": "./dist/lib/index.js"
     }
   },
   "files": [
@@ -24,17 +24,16 @@
   "scripts": {
     "clean": "shx rm -rf dist temp coverage \"*.tsbuildInfo\"",
     "clean-build": "pnpm clean && pnpm build",
-    "build": "tsc -b . -f && pnpm run build:api",
+    "build": "tsc -p . && pnpm run build:api",
     "build:api": "rollup -c api/rollup.config.mjs",
-    "build:esm": "tsc -b tsconfig.esm.json -f",
-    "build:lib": "tsc -b src/lib/tsconfig.json -f",
-    "watch": "tsc -b . --watch -f",
+    "watch": "tsc -p . --watch",
     "coverage": "vitest run --coverage --pool=forks",
     "test-watch": "vitest",
     "prepublishOnly": "pnpm run clean-build",
     "#test": "vitest run --reporter=hanging-process --reporter=default",
     "test": "vitest run --pool=forks",
-    "test:update-snapshot": "vitest run -u"
+    "test:update-snapshot": "vitest run -u",
+    "test:perf": "NODE_ENV=production insight --register ts-node/esm --file \"**/*.perf.{mts,ts}\""
   },
   "repository": {
     "type": "git",
@@ -59,27 +58,27 @@
   },
   "homepage": "https://github.com/streetsidesoftware/cspell#readme",
   "dependencies": {
-    "@cspell/cspell-bundled-dicts": "8.13.0",
-    "@cspell/cspell-pipe": "8.13.0",
-    "@cspell/cspell-resolver": "8.13.0",
-    "@cspell/cspell-types": "8.13.0",
-    "@cspell/dynamic-import": "8.13.0",
-    "@cspell/strong-weak-map": "8.13.0",
-    "@cspell/url": "8.13.0",
+    "@cspell/cspell-bundled-dicts": "8.13.2",
+    "@cspell/cspell-pipe": "8.13.2",
+    "@cspell/cspell-resolver": "8.13.2",
+    "@cspell/cspell-types": "8.13.2",
+    "@cspell/dynamic-import": "8.13.2",
+    "@cspell/strong-weak-map": "8.13.2",
+    "@cspell/url": "8.13.2",
     "clear-module": "^4.1.2",
     "comment-json": "^4.2.4",
-    "cspell-config-lib": "8.13.0",
-    "cspell-dictionary": "8.13.0",
-    "cspell-glob": "8.13.0",
-    "cspell-grammar": "8.13.0",
-    "cspell-io": "8.13.0",
-    "cspell-trie-lib": "8.13.0",
+    "cspell-config-lib": "8.13.2",
+    "cspell-dictionary": "8.13.2",
+    "cspell-glob": "8.13.2",
+    "cspell-grammar": "8.13.2",
+    "cspell-io": "8.13.2",
+    "cspell-trie-lib": "8.13.2",
     "env-paths": "^3.0.0",
     "fast-equals": "^5.0.1",
     "gensequence": "^7.0.0",
     "import-fresh": "^3.3.0",
     "resolve-from": "^5.0.0",
-    "vscode-languageserver-textdocument": "^1.0.11",
+    "vscode-languageserver-textdocument": "^1.0.12",
     "vscode-uri": "^3.0.8",
     "xdg-basedir": "^5.1.0"
   },
@@ -94,12 +93,13 @@
     "@cspell/dict-fr-fr": "^2.2.2",
     "@cspell/dict-html": "^4.0.5",
     "@cspell/dict-nl-nl": "^2.3.0",
-    "@cspell/dict-python": "^4.2.3",
+    "@cspell/dict-python": "^4.2.4",
     "@types/configstore": "^6.0.2",
     "configstore": "^7.0.0",
     "cspell-dict-nl-nl": "^1.1.2",
     "leaked-handles": "^5.2.0",
-    "lorem-ipsum": "^2.0.8"
+    "lorem-ipsum": "^2.0.8",
+    "perf-insight": "^1.2.0"
   },
-  "gitHead": "2fd3fb430cc96a8a50543f57d96b288219a11923"
+  "gitHead": "5c9aaf8829991e7d6d37355acb9b9685088bba14"
 }

package/dist/esm/textValidation/isWordValid.d.ts

@@ -1,9 +0,0 @@
-import type { CachingDictionary } from 'cspell-dictionary';
-import type { TextOffsetRO } from './ValidationTypes.js';
-declare function hasWordCheck(dict: CachingDictionary, word: string): boolean;
-export declare function isWordValidWithEscapeRetry(dict: CachingDictionary, wo: TextOffsetRO, line: TextOffsetRO): boolean;
-export declare const __testing__: {
-    hasWordCheck: typeof hasWordCheck;
-};
-export {};
-//# sourceMappingURL=isWordValid.d.ts.map

package/dist/esm/textValidation/lineValidatorFactory.js

@@ -1,128 +0,0 @@
-import { opConcatMap, opFilter, opMap, pipe, toArray } from '@cspell/cspell-pipe/sync';
-import { createCachingDictionary } from 'cspell-dictionary';
-import * as RxPat from '../Settings/RegExpPatterns.js';
-import * as Text from '../util/text.js';
-import { clean } from '../util/util.js';
-import { split } from '../util/wordSplitter.js';
-import { defaultMinWordLength } from './defaultConstants.js';
-import { isWordValidWithEscapeRetry } from './isWordValid.js';
-import { mapRangeBackToOriginalPos } from './parsedText.js';
-export function lineValidatorFactory(sDict, options) {
-    const { minWordLength = defaultMinWordLength, flagWords = [], allowCompoundWords = false, ignoreCase = true, } = options;
-    const hasWordOptions = {
-        ignoreCase,
-        useCompounds: allowCompoundWords || undefined, // let the dictionaries decide on useCompounds if allow is false
-    };
-    const dictCol = createCachingDictionary(sDict, hasWordOptions);
-    const setOfFlagWords = new Set(flagWords);
-    const setOfKnownSuccessfulWords = new Set();
-    const rememberFilter = (fn) => (v) => {
-        const keep = fn(v);
-        if (!keep) {
-            setOfKnownSuccessfulWords.add(v.text);
-        }
-        return keep;
-    };
-    const filterAlreadyChecked = (wo) => {
-        return !setOfKnownSuccessfulWords.has(wo.text);
-    };
-    function testForFlaggedWord(wo) {
-        const text = wo.text;
-        return setOfFlagWords.has(text) || setOfFlagWords.has(text.toLowerCase()) || dictCol.isForbidden(text);
-    }
-    function isWordIgnored(word) {
-        return dictCol.isNoSuggestWord(word);
-    }
-    function getSuggestions(word) {
-        return dictCol.getPreferredSuggestions(word);
-    }
-    function isWordFlagged(word) {
-        const isIgnored = isWordIgnored(word.text);
-        const isFlagged = !isIgnored && testForFlaggedWord(word);
-        return isFlagged;
-    }
-    function annotateIsFlagged(word) {
-        word.isFlagged = isWordFlagged(word);
-        return word;
-    }
-    function annotateIssue(issue) {
-        const sugs = getSuggestions(issue.text);
-        if (sugs && sugs.length) {
-            issue.suggestionsEx = sugs;
-        }
-        return issue;
-    }
-    function checkWord(word) {
-        const isIgnored = isWordIgnored(word.text);
-        const { isFlagged = !isIgnored && testForFlaggedWord(word) } = word;
-        const isFound = isFlagged ? undefined : isIgnored || isWordValidWithEscapeRetry(dictCol, word, word.line);
-        return clean({ ...word, isFlagged, isFound });
-    }
-    const fn = (lineSegment) => {
-        function splitterIsValid(word) {
-            return (setOfKnownSuccessfulWords.has(word.text) ||
-                (!testForFlaggedWord(word) && isWordValidWithEscapeRetry(dictCol, word, lineSegment.line)));
-        }
-        function checkFullWord(vr) {
-            if (vr.isFlagged) {
-                return [vr];
-            }
-            const codeWordResults = toArray(pipe(Text.extractWordsFromCodeTextOffset(vr), opFilter(filterAlreadyChecked), opMap((t) => ({ ...t, line: vr.line })), opMap(annotateIsFlagged), opFilter(rememberFilter((wo) => wo.text.length >= minWordLength || !!wo.isFlagged)), opMap((wo) => (wo.isFlagged ? wo : checkWord(wo))), opFilter(rememberFilter((wo) => wo.isFlagged || !wo.isFound)), opFilter(rememberFilter((wo) => !RxPat.regExRepeatedChar.test(wo.text))), 
-            // get back the original text.
-            opMap((wo) => ({
-                ...wo,
-                text: Text.extractText(lineSegment.segment, wo.offset, wo.offset + wo.text.length),
-            }))));
-            if (!codeWordResults.length || isWordIgnored(vr.text) || checkWord(vr).isFound) {
-                rememberFilter((_) => false)(vr);
-                return [];
-            }
-            return codeWordResults;
-        }
-        function checkPossibleWords(possibleWord) {
-            if (isWordFlagged(possibleWord)) {
-                const vr = {
-                    ...possibleWord,
-                    line: lineSegment.line,
-                    isFlagged: true,
-                };
-                return [vr];
-            }
-            const mismatches = toArray(pipe(Text.extractWordsFromTextOffset(possibleWord), opFilter((wo) => filterAlreadyChecked(wo)), opMap((wo) => ((wo.line = lineSegment.line), wo)), opMap(annotateIsFlagged), opFilter(rememberFilter((wo) => wo.text.length >= minWordLength || !!wo.isFlagged)), opConcatMap(checkFullWord)));
-            if (mismatches.length) {
-                // Try the more expensive word splitter
-                const splitResult = split(lineSegment.segment, possibleWord.offset, splitterIsValid);
-                const nonMatching = splitResult.words.filter((w) => !w.isFound);
-                if (nonMatching.length < mismatches.length) {
-                    return nonMatching.map((w) => ({ ...w, line: lineSegment.line })).map(annotateIsFlagged);
-                }
-            }
-            return mismatches;
-        }
-        const checkedPossibleWords = pipe(Text.extractPossibleWordsFromTextOffset(lineSegment.segment), opFilter(filterAlreadyChecked), opConcatMap(checkPossibleWords), opMap(annotateIssue));
-        return checkedPossibleWords;
-    };
-    return { fn, dict: dictCol };
-}
-export function textValidatorFactory(dict, options) {
-    const lineValidator = lineValidatorFactory(dict, options);
-    const lineValidatorFn = lineValidator.fn;
-    function validate(pText) {
-        const { text, range: srcRange, map } = pText;
-        const srcOffset = srcRange[0];
-        const segment = { text, offset: 0 };
-        const lineSegment = { line: segment, segment };
-        function mapBackToOriginSimple(vr) {
-            const { text, offset, isFlagged, isFound, suggestionsEx } = vr;
-            const r = mapRangeBackToOriginalPos([offset, offset + text.length], map);
-            const range = [r[0] + srcOffset, r[1] + srcOffset];
-            return { text, range, isFlagged, isFound, suggestionsEx };
-        }
-        return [...lineValidatorFn(lineSegment)].map(mapBackToOriginSimple);
-    }
-    return {
-        validate,
-        lineValidator,
-    };
-}
-//# sourceMappingURL=lineValidatorFactory.js.map

package/dist/lib-cjs/index.cjs

@@ -1,10 +0,0 @@
-"use strict";
-/**
- * This module contains CJS only files.
- * It includes files that use 3rd part libs that can only be CJS due to their exports.
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.srcDirectory = void 0;
-var pkg_info_cjs_1 = require("./pkg-info.cjs");
-Object.defineProperty(exports, "srcDirectory", { enumerable: true, get: function () { return pkg_info_cjs_1.srcDirectory; } });
-//# sourceMappingURL=index.cjs.map

package/dist/lib-cjs/index.d.cts

@@ -1,6 +0,0 @@
-/**
- * This module contains CJS only files.
- * It includes files that use 3rd part libs that can only be CJS due to their exports.
- */
-export { srcDirectory } from './pkg-info.cjs';
-//# sourceMappingURL=index.d.cts.map

package/dist/lib-cjs/pkg-info.cjs

@@ -1,7 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.srcDirectory = void 0;
-// import { join } from 'path';
-// export const srcDirectory = join(__dirname, '/');
-exports.srcDirectory = __dirname;
-//# sourceMappingURL=pkg-info.cjs.map