@cspell/cspell-tools 9.4.0 → 9.5.0

package/cspell-tools.config.schema.json

@@ -28,17 +28,7 @@
       "additionalProperties": false,
       "properties": {
         "allowedSplitWords": {
-          "anyOf": [
-            {
-              "$ref": "#/definitions/FilePath"
-            },
-            {
-              "items": {
-                "$ref": "#/definitions/FilePath"
-              },
-              "type": "array"
-            }
-          ],
+          "$ref": "#/definitions/FilePathOrFilePathArray",
           "description": "Words in the `allowedSplitWords` are considered correct and can be used as a basis for splitting compound words.\n\nIf entries can be split so that all the words in the entry are allowed, then only the individual words are added, otherwise the entire entry is added. This is to prevent misspellings in CamelCase words from being introduced into the dictionary."
         },
         "keepRawCase": {
@@ -86,21 +76,24 @@
       "description": "Note: All relative paths are relative to the config file location.",
       "type": "string"
     },
+    "FilePathOrFilePathArray": {
+      "anyOf": [
+        {
+          "$ref": "#/definitions/FilePath"
+        },
+        {
+          "items": {
+            "$ref": "#/definitions/FilePath"
+          },
+          "type": "array"
+        }
+      ]
+    },
     "FileSource": {
       "additionalProperties": false,
       "properties": {
         "allowedSplitWords": {
-          "anyOf": [
-            {
-              "$ref": "#/definitions/FilePath"
-            },
-            {
-              "items": {
-                "$ref": "#/definitions/FilePath"
-              },
-              "type": "array"
-            }
-          ],
+          "$ref": "#/definitions/FilePathOrFilePathArray",
           "description": "Words in the `allowedSplitWords` are considered correct and can be used as a basis for splitting compound words.\n\nIf entries can be split so that all the words in the entry are allowed, then only the individual words are added, otherwise the entire entry is added. This is to prevent misspellings in CamelCase words from being introduced into the dictionary."
         },
         "filename": {
@@ -162,8 +155,8 @@
           "description": "Words in the `allowedSplitWords` are considered correct and can be used as a basis for splitting compound words.\n\nIf entries can be split so that all the words in the entry are allowed, then only the individual words are added, otherwise the entire entry is added. This is to prevent misspellings in CamelCase words from being introduced into the dictionary."
         },
         "compress": {
-          "default": ": false",
-          "description": "gzip the file?",
+          "default": false,
+          "description": "Setting this value to true will create a `.gz` dictionary file. Use `keepUncompressed` to also keep an uncompressed version.",
           "type": "boolean"
         },
         "dictionaryDirectives": {
@@ -203,6 +196,10 @@
           "description": "Generate lower case / accent free versions of words.",
           "type": "boolean"
         },
+        "keepUncompressed": {
+          "description": "If `compress` is true, setting this value to true will also keep an uncompressed version of the dictionary.",
+          "type": "boolean"
+        },
         "name": {
           "description": "Name of target, used as the basis of target file name.",
           "type": "string"

package/dist/compiler/CompilerDefinitions.d.ts

@@ -0,0 +1,2 @@
+export type WordListCompiler = (srcWords: Iterable<string>) => Iterable<string>;
+//# sourceMappingURL=CompilerDefinitions.d.ts.map

package/dist/compiler/CompilerDefinitions.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=CompilerDefinitions.js.map

package/dist/compiler/compile.js

@@ -9,7 +9,7 @@ import { createAllowedSplitWordsFromFiles, createWordsCollectionFromFiles } from
 import { logWithTimestamp } from './logWithTimestamp.js';
 import { readTextFile } from './readers/readTextFile.js';
 import { streamSourceWordsFromFile } from './streamSourceWordsFromFile.js';
-import { compileTrie, compileWordList } from './wordListCompiler.js';
+import { createTargetFile, createTrieCompiler, createWordListCompiler } from './wordListCompiler.js';
 import { normalizeTargetWords } from './wordListParser.js';
 export async function compile(request, options) {
     const { targets } = request;
@@ -66,10 +66,16 @@ export async function compileTarget(target, options, compileOptions) {
     const excludeFilter = (word) => {
         return excludeFromFilter(word) && includeFromFilter(word) && excludeRegexFilter(word);
     };
-    const generateNonStrictTrie = target.generateNonStrict ?? true;
     const name = normalizeTargetName(target.name);
     const useTrie = format.startsWith('trie');
-    const filename = resolveTarget(name, targetDirectory, useTrie, target.compress ?? false);
+    const generateCompressed = target.compress ?? false;
+    const generateUncompressed = target.keepUncompressed ?? false;
+    const genSet = new Set();
+    genSet.add(generateCompressed);
+    if (generateUncompressed) {
+        genSet.add(false);
+    }
+    const filename = resolveTarget(name, targetDirectory, useTrie);
     const filesToProcessAsync = pipeAsync(readSourceList(sources, rootDir), opMapAsync((src) => readFileSource(src, options)), opAwaitAsync());
     const filesToProcess = await toArray(filesToProcessAsync);
     const normalizer = normalizeTargetWords({
@@ -81,7 +87,7 @@ export async function compileTarget(target, options, compileOptions) {
     });
     const checksumRoot = (checksumFile && path.dirname(checksumFile)) || rootDir;
     const deps = [
-        ...calculateDependencies(filename, filesToProcess, [...excludeWordsFrom, ...excludeWordsNotFoundIn], checksumRoot),
+        ...calculateDependencies(filename + (generateCompressed ? '.gz' : ''), filesToProcess, [...excludeWordsFrom, ...excludeWordsNotFoundIn], checksumRoot),
     ];
     if (conditional && checksumFile) {
         const check = await checkShasumFile(checksumFile, deps, checksumRoot).catch(() => undefined);
@@ -90,34 +96,32 @@ export async function compileTarget(target, options, compileOptions) {
             return [];
         }
     }
-    const action = useTrie
-        ? async (words, dst) => {
-            return compileTrie(pipe(words, normalizer), dst, {
+    async function action(words, dst) {
+        const compiler = useTrie
+            ? createTrieCompiler({
                 base: trieBase,
-                sort: false,
                 trie3: format === 'trie3',
                 trie4: format === 'trie4',
-                generateNonStrict: generateNonStrictTrie,
-                dictionaryDirectives: undefined,
-                //   removeDuplicates, // Add this in if we use it.
-            });
-        }
-        : async (words, dst) => {
-            return compileWordList(pipe(words, normalizer), dst, {
+            })
+            : createWordListCompiler({
                 sort,
                 generateNonStrict,
                 dictionaryDirectives,
                 removeDuplicates,
             });
-        };
-    await processFiles(action, filesToProcess, filename);
+        const data = iterableToString(pipe(words, normalizer, compiler));
+        for (const compress of genSet) {
+            await createTargetFile(dst, data, compress);
+        }
+    }
+    await processFiles({ action, filesToProcess, mergeTarget: filename });
     logWithTimestamp(`Done compile: ${target.name}`);
     return deps;
 }
-function calculateDependencies(targetFile, filesToProcess, excludeFiles, rootDir) {
+function calculateDependencies(targetFile, filesToProcess, extraDependencyFiles, rootDir) {
     const dependencies = new Set();
     addDependency(targetFile);
-    excludeFiles?.forEach((f) => addDependency(f));
+    extraDependencyFiles?.forEach((f) => addDependency(f));
     filesToProcess.forEach((f) => addDependency(f.src));
     return dependencies;
     function addDependency(filename) {
@@ -130,7 +134,7 @@ function calculateDependencies(targetFile, filesToProcess, excludeFiles, rootDir
 function rel(filePath) {
     return path.relative(process.cwd(), filePath);
 }
-async function processFiles(action, filesToProcess, mergeTarget) {
+async function processFiles({ action, filesToProcess, mergeTarget }) {
     const toProcess = filesToProcess;
     const dst = mergeTarget;
     const words = pipe(toProcess, opMap((ftp) => {
@@ -146,8 +150,8 @@ async function processFiles(action, filesToProcess, mergeTarget) {
     await action(words, dst);
     logWithTimestamp('Done "%s"', rel(dst));
 }
-function resolveTarget(name, directory, useTrie, useGzCompress) {
-    const ext = ((useTrie && '.trie') || '.txt') + ((useGzCompress && '.gz') || '');
+function resolveTarget(name, directory, useTrie) {
+    const ext = (useTrie && '.trie') || '.txt';
     const filename = name + ext;
     return path.resolve(directory, filename);
 }
@@ -260,4 +264,7 @@ function createExcludeRegexFilter(excludeWordsMatchingRegex) {
     });
     return (word) => !regexes.some((r) => r.test(word));
 }
+function iterableToString(iter) {
+    return Array.isArray(iter) ? iter.join('') : [...iter].join('');
+}
 //# sourceMappingURL=compile.js.map

package/dist/compiler/createWordsCollection.d.ts

@@ -1,9 +1,10 @@
-import type { FilePath } from '../config/config.js';
+import type { FilePath, FilePathOrFilePathArray } from '../config/config.js';
 import type { AllowedSplitWordsCollection, ExcludeWordsCollection, WordsCollection } from './WordsCollection.js';
-export declare function createAllowedSplitWordsFromFiles(files: FilePath | FilePath[] | undefined): Promise<AllowedSplitWordsCollection>;
+export declare function createAllowedSplitWordsFromFiles(files: FilePathOrFilePathArray | undefined): Promise<AllowedSplitWordsCollection>;
 export declare function createAllowedSplitWords(words: Iterable<string> | undefined): AllowedSplitWordsCollection;
-export declare function createWordsCollectionFromFiles(files: FilePath | FilePath[]): Promise<WordsCollection>;
+export declare function createWordsCollectionFromFiles(files: FilePathOrFilePathArray): Promise<WordsCollection>;
 export declare function createWordsCollection(words: Iterable<string>): WordsCollection;
-export declare function createExcludeWordsCollectionFromFiles(files: FilePath | FilePath[] | undefined): Promise<ExcludeWordsCollection>;
+export declare function createExcludeWordsCollectionFromFiles(files: FilePathOrFilePathArray | undefined): Promise<ExcludeWordsCollection>;
 export declare function createExcludeWordsCollection(words: Iterable<string> | undefined): ExcludeWordsCollection;
+export declare function toFilePathArray(filePathOrArray: FilePathOrFilePathArray): FilePath[];
 //# sourceMappingURL=createWordsCollection.d.ts.map

package/dist/compiler/createWordsCollection.js

@@ -50,7 +50,7 @@ function readersToCollection(readers) {
 }
 const cache = new WeakMap();
 export async function createWordsCollectionFromFiles(files) {
-    files = Array.isArray(files) ? files : [files];
+    files = toFilePathArray(files);
     const cached = cache.get(files);
     if (cached)
         return cached;
@@ -105,4 +105,7 @@ function lineReadersToCollection(readers) {
     const dict = parseDictionary(words(), { stripCaseAndAccents: false });
     return { size: dict.size, has: buildHasFn(dict) };
 }
+export function toFilePathArray(filePathOrArray) {
+    return Array.isArray(filePathOrArray) ? filePathOrArray : [filePathOrArray];
+}
 //# sourceMappingURL=createWordsCollection.js.map

package/dist/compiler/index.d.ts

@@ -1,5 +1,4 @@
 export type { CompileRequest, CompileTargetOptions, RunConfig } from '../config/index.js';
 export { compile, compileTarget } from './compile.js';
 export { type Logger, setLogger } from './logger.js';
-export { compileTrie, compileWordList } from './wordListCompiler.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/compiler/index.js

@@ -1,4 +1,3 @@
 export { compile, compileTarget } from './compile.js';
 export { setLogger } from './logger.js';
-export { compileTrie, compileWordList } from './wordListCompiler.js';
 //# sourceMappingURL=index.js.map

package/dist/compiler/wordListCompiler.d.ts

@@ -1,6 +1,9 @@
 import type { CompileOptions } from './CompileOptions.js';
-export declare function compileWordList(lines: Iterable<string>, destFilename: string, options: CompileOptions): Promise<void>;
+import { WordListCompiler } from './CompilerDefinitions.js';
+export declare function compileWordListToTarget(lines: Iterable<string>, destFilename: string, options: CompileOptions): Promise<void>;
+export declare function createWordListCompiler(options: CompileOptions): WordListCompiler;
 declare function removeDuplicates(words: Iterable<string>): Iterable<string>;
+export declare function createTargetFile(destFilename: string, seq: Iterable<string> | string, compress?: boolean): Promise<void>;
 export interface TrieOptions {
     base?: number | undefined;
     trie3?: boolean | undefined;
@@ -8,7 +11,8 @@ export interface TrieOptions {
 }
 export interface CompileTrieOptions extends CompileOptions, TrieOptions {
 }
-export declare function compileTrie(words: Iterable<string>, destFilename: string, options: CompileTrieOptions): Promise<void>;
+export declare function compileTrieToTarget(words: Iterable<string>, destFilename: string, options: CompileTrieOptions): Promise<void>;
+export declare function createTrieCompiler(options: TrieOptions): WordListCompiler;
 export declare const __testing__: {
     wordListHeader: string;
     removeDuplicates: typeof removeDuplicates;

package/dist/compiler/wordListCompiler.js

@@ -2,9 +2,9 @@ import { mkdir } from 'node:fs/promises';
 import * as path from 'node:path';
 import { opAppend, opMap, pipe } from '@cspell/cspell-pipe/sync';
 import * as Trie from 'cspell-trie-lib';
-import { writeSeqToFile } from './fileWriter.js';
 import { getLogger } from './logger.js';
 import { normalizeTargetWords } from './wordListParser.js';
+import { writeTextToFile } from './writeTextToFile.js';
 const mkdirp = async (p) => {
     await mkdir(p, { recursive: true });
 };
@@ -12,12 +12,17 @@ const mkdirp = async (p) => {
 const wordListHeader = `
 # cspell-tools: keep-case no-split`;
 const wordListHeaderLines = wordListHeader.split('\n').map((a) => a.trim());
-export async function compileWordList(lines, destFilename, options) {
-    const finalLines = normalize(lines, options);
-    const directives = options.dictionaryDirectives ?? [];
-    const directivesLines = directives.map((a) => `# cspell-dictionary: ${a}`);
-    const finalSeq = pipe([...wordListHeaderLines, ...directivesLines, ''], opAppend(finalLines));
-    return createWordListTarget(destFilename)(finalSeq);
+export async function compileWordListToTarget(lines, destFilename, options) {
+    const compiler = createWordListCompiler(options);
+    return createTargetFile(destFilename, compiler(lines));
+}
+export function createWordListCompiler(options) {
+    return (lines) => {
+        const finalLines = normalize(lines, options);
+        const directives = options.dictionaryDirectives ?? [];
+        const directivesLines = directives.map((a) => `# cspell-dictionary: ${a}`);
+        return pipe([...wordListHeaderLines, ...directivesLines, ''], opAppend(finalLines), opMap((a) => a + '\n'));
+    };
 }
 function normalize(lines, options) {
     const filter = normalizeTargetWords(options);
@@ -158,24 +163,26 @@ function removeDuplicateForms(forms) {
         return [form, applyFlags(form, flag)];
     }));
 }
-function createWordListTarget(destFilename) {
-    const target = createTarget(destFilename);
-    return (seq) => target(pipe(seq, opMap((a) => a + '\n')));
-}
-function createTarget(destFilename) {
+export async function createTargetFile(destFilename, seq, compress) {
+    const rel = path.relative(process.cwd(), destFilename).replaceAll(path.sep, '/');
+    const log = getLogger();
+    log(`Writing to file ${rel}${compress ? '.gz' : ''}`);
     const destDir = path.dirname(destFilename);
-    const pDir = mkdirp(destDir);
-    return async (seq) => {
-        await pDir;
-        await writeSeqToFile(seq, destFilename);
-    };
+    await mkdirp(destDir);
+    await writeTextToFile(destFilename, seq, compress);
 }
-export async function compileTrie(words, destFilename, options) {
+export async function compileTrieToTarget(words, destFilename, options) {
     await createTrieTarget(destFilename, options)(words);
 }
 function createTrieTarget(destFilename, options) {
-    const target = createTarget(destFilename);
     return async (words) => {
+        await createTargetFile(destFilename, createTrieCompiler(options)(words));
+        const log = getLogger();
+        log(`Done writing to file ${path.basename(destFilename)}`);
+    };
+}
+export function createTrieCompiler(options) {
+    return (words) => {
         const log = getLogger();
         log('Reading Words into Trie');
         const base = options.base ?? 32;
@@ -183,13 +190,12 @@ function createTrieTarget(destFilename, options) {
         const root = Trie.buildTrie(words).root;
         log('Reduce duplicate word endings');
         const trie = Trie.consolidate(root);
-        log(`Writing to file ${path.basename(destFilename)}`);
-        await target(Trie.serializeTrie(trie, {
+        log('Trie compilation complete');
+        return Trie.serializeTrie(trie, {
             base,
             comment: 'Built by cspell-tools.',
             version,
-        }));
-        log(`Done writing to file ${path.basename(destFilename)}`);
+        });
     };
 }
 export const __testing__ = {

package/dist/compiler/writeTextToFile.d.ts

@@ -1,3 +1,2 @@
-export declare function writeTextToFile(filename: string, data: string): Promise<void>;
-export declare function writeTextLinesToFile(filename: string, lines: Iterable<string>): Promise<void>;
+export declare function writeTextToFile(filename: string, data: string | Iterable<string>, useGzCompress?: boolean): Promise<void>;
 //# sourceMappingURL=writeTextToFile.d.ts.map

package/dist/compiler/writeTextToFile.js

@@ -2,14 +2,15 @@ import { Buffer } from 'node:buffer';
 import { promises as fs } from 'node:fs';
 import { compress } from '../gzip/index.js';
 const isGzFile = /\.gz$/;
-export async function writeTextToFile(filename, data) {
-    const useGz = isGzFile.test(filename);
-    const buf = Buffer.from(data, 'utf8');
+export async function writeTextToFile(filename, data, useGzCompress) {
+    const dataStr = typeof data === 'string' ? data : Array.isArray(data) ? data.join('') : [...data].join('');
+    const hasGzExt = isGzFile.test(filename);
+    const useGz = useGzCompress ?? hasGzExt;
+    if (useGz && !hasGzExt) {
+        filename += '.gz';
+    }
+    const buf = Buffer.from(dataStr, 'utf8');
     const buffer = useGz ? await compress(buf) : buf;
     await fs.writeFile(filename, buffer);
 }
-export function writeTextLinesToFile(filename, lines) {
-    const data = Array.isArray(lines) ? lines.join('') : [...lines].join('');
-    return writeTextToFile(filename, data);
-}
 //# sourceMappingURL=writeTextToFile.js.map

package/dist/config/config.d.ts

@@ -94,10 +94,15 @@ export interface Target extends CompileTargetOptions {
      */
     targetDirectory?: FilePath | undefined;
     /**
-     * gzip the file?
-     * @default: false
+     * Setting this value to true will create a `.gz` dictionary file.
+     * Use `keepUncompressed` to also keep an uncompressed version.
+     * @default false
      */
     compress?: boolean | undefined;
+    /**
+     * If `compress` is true, setting this value to true will also keep an uncompressed version of the dictionary.
+     */
+    keepUncompressed?: boolean | undefined;
     /**
      * Format of the dictionary.
      */
@@ -143,6 +148,7 @@ export type DictionaryFormats = 'plaintext' | 'trie' | 'trie3' | 'trie4';
  * Note: All relative paths are relative to the config file location.
  */
 export type FilePath = string;
+export type FilePathOrFilePathArray = FilePath | FilePath[];
 export type DictionarySource = FilePath | FileSource | FileListSource;
 export interface FileSource extends CompileSourceOptions {
     filename: FilePath;
@@ -175,7 +181,7 @@ export interface CompileSourceOptions {
      * This is to prevent misspellings in CamelCase words from being introduced into the
      * dictionary.
      */
-    allowedSplitWords?: FilePath | FilePath[] | undefined;
+    allowedSplitWords?: FilePathOrFilePathArray | undefined;
     /**
      * Camel case words that have been split using the `allowedSplitWords` are added to the dictionary as compoundable words.
      * These words are prefixed / suffixed with `*`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cspell/cspell-tools",
-  "version": "9.4.0",
+  "version": "9.5.0",
   "description": "Tools to assist with the development of cSpell",
   "publishConfig": {
     "access": "public",
@@ -63,12 +63,12 @@
   },
   "homepage": "https://github.com/streetsidesoftware/cspell/tree/main/packages/cspell-tools#readme",
   "dependencies": {
-    "@cspell/cspell-pipe": "9.4.0",
+    "@cspell/cspell-pipe": "9.5.0",
     "commander": "^14.0.2",
     "cosmiconfig": "9.0.0",
-    "cspell-trie-lib": "9.4.0",
+    "cspell-trie-lib": "9.5.0",
     "glob": "^13.0.0",
-    "hunspell-reader": "9.4.0",
+    "hunspell-reader": "9.5.0",
     "yaml": "^2.8.2"
   },
   "engines": {
@@ -79,5 +79,5 @@
     "ts-json-schema-generator": "^2.4.0"
   },
   "module": "bin.mjs",
-  "gitHead": "12dba3d8b880384d1401c765cb2186647f5a266f"
+  "gitHead": "4407eed11650481d9037e5a1d53488a93ff4b66d"
 }

package/dist/compiler/fileWriter.d.ts

@@ -1,3 +0,0 @@
-export { writeTextLinesToFile as writeToFileIterableP } from './writeTextToFile.js';
-export declare function writeSeqToFile(seq: Iterable<string>, outFile: string): Promise<void>;
-//# sourceMappingURL=fileWriter.d.ts.map

package/dist/compiler/fileWriter.js

@@ -1,6 +0,0 @@
-import { writeTextLinesToFile } from './writeTextToFile.js';
-export { writeTextLinesToFile as writeToFileIterableP } from './writeTextToFile.js';
-export function writeSeqToFile(seq, outFile) {
-    return writeTextLinesToFile(outFile, seq);
-}
-//# sourceMappingURL=fileWriter.js.map