@cspell/cspell-tools 7.0.0-alpha.2 → 7.0.0

package/bin.js

@@ -6,6 +6,8 @@ import { run } from './dist/app.js';
 run(program, process.argv).catch((e) => {
     if (!(e instanceof CommanderError)) {
         console.log(e);
+    } else {
+        console.log(e.message);
     }
     process.exitCode = 1;
 });

package/cspell-tools.config.schema.json

@@ -195,6 +195,11 @@
     }
   },
   "properties": {
+    "$schema": {
+      "default": "https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-tools/cspell-tools.config.schema.json",
+      "description": "Url to JSON Schema",
+      "type": "string"
+    },
     "allowedSplitWords": {
       "anyOf": [
         {
@@ -209,6 +214,13 @@
       ],
       "description": "Words in the `allowedSplitWords` are considered correct and can be used as a basis for splitting compound words.\n\nIf entries can be split so that all the words in the entry are allowed, then only the individual words are added, otherwise the entire entry is added. This is to prevent misspellings in CamelCase words from being introduced into the dictionary."
     },
+    "checksumFile": {
+      "description": "Path to checksum file. `true` - defaults to `./checksum.txt`.",
+      "type": [
+        "string",
+        "boolean"
+      ]
+    },
     "generateNonStrict": {
       "default": true,
       "description": "Generate lower case / accent free versions of words.",

package/dist/AppOptions.d.ts

@@ -7,7 +7,7 @@ export interface CompileCommonAppOptions {
     max_depth?: string;
     maxDepth?: string;
     merge?: string;
-    experimental: string[];
+    experimental?: string[];
     split?: boolean;
     sort?: boolean;
     keepRawCase?: boolean;
@@ -15,7 +15,10 @@ export interface CompileCommonAppOptions {
     trie3?: boolean;
     trie4?: boolean;
     trieBase?: string;
+    listFile?: string[];
     useLegacySplitter?: boolean;
+    /** Indicate that a config file should be created instead of building. */
+    init?: boolean;
 }
 export interface CompileAppOptions extends CompileCommonAppOptions {
     sort: boolean;

package/dist/app.js

@@ -1,9 +1,13 @@
 // For large dictionaries, it is necessary to increase the memory limit.
+import { CommanderError, Option } from 'commander';
 import { readFileSync } from 'fs';
 import { build } from './build.js';
 import { processCompileAction } from './compile.js';
 import * as compiler from './compiler/index.js';
 import { logWithTimestamp } from './compiler/logWithTimestamp.js';
+import { gzip } from './gzip/index.js';
+import { reportCheckChecksumFile, reportChecksumForFiles, updateChecksumForFiles } from './shasum/shasum.js';
+import { toError } from './util/errors.js';
 const npmPackageRaw = readFileSync(new URL('../package.json', import.meta.url), 'utf8');
 const npmPackage = JSON.parse(npmPackageRaw);
 compiler.setLogger(logWithTimestamp);
@@ -23,28 +27,60 @@ function addCompileOptions(compileCommand) {
         .option('-x, --experimental <flag>', 'Experimental flags, used for testing new concepts. Flags: compound', collect, [])
         .option('--trie3', 'Use file format trie3', false)
         .option('--trie4', 'Use file format trie4', false)
-        .option('--trie-base <number>', 'Advanced: Set the trie base number. A value between 10 and 36');
+        .option('--trie-base <number>', 'Advanced: Set the trie base number. A value between 10 and 36')
+        .option('--list-file <filename...>', 'Path to a file that contains the list of files to compile. A list file contains one file per line.')
+        .option('--init', 'Create a build command `cspell-tools.config.yaml` file based upon the options given instead of building.');
 }
 export async function run(program, argv, flags) {
+    async function handleGzip(files) {
+        try {
+            await gzip(files);
+        }
+        catch (error) {
+            const err = toError(error);
+            program.error(err.message);
+        }
+    }
+    async function shasum(files, options) {
+        const report = options.check
+            ? await reportCheckChecksumFile(options.check, files, options)
+            : options.update
+                ? await updateChecksumForFiles(options.update, files, options)
+                : await reportChecksumForFiles(files, options);
+        console.log('%s', report.report);
+        if (!report.passed) {
+            throw new CommanderError(1, 'Failed Checksum', 'One or more files had issues.');
+        }
+    }
     program.exitOverride();
     program.version(npmPackage.version);
-    addCompileOptions(program.command('compile <src...>').description('Compile words into a cspell dictionary files.'))
+    addCompileOptions(program.command('compile [src...]').description('Compile words into a cspell dictionary files.'))
         .option('--trie', 'Compile into a trie file.', false)
         .option('--no-sort', 'Do not sort the result')
         .action((src, options) => {
         return processCompileAction(src, options, flags);
     });
     addCompileOptions(program
-        .command('compile-trie <src...>')
+        .command('compile-trie [src...]')
         .description('Compile words lists or Hunspell dictionary into trie files used by cspell.\nAlias of `compile --trie`')).action((src, options) => {
         return processCompileAction(src, { ...options, trie: true }, flags);
     });
     program
-        .command('build [targets]')
+        .command('build [targets...]')
         .description('Build the targets defined in the run configuration.')
         .option('-c, --config <path to run configuration>', 'Specify the run configuration file.')
+        .option('--conditional', 'Conditional build.')
         .option('-r, --root <directory>', 'Specify the run directory')
         .action(build);
+    program.command('gzip <files...>').description('GZip files while keeping the original.').action(handleGzip);
+    program
+        .command('shasum [files...]')
+        .description('Calculate the checksum for files.')
+        .option('--list-file <list-file.txt...>', 'Specify one or more files that contain paths of files to check.')
+        .option('-c, --check <checksum.txt>', 'Verify the checksum of files against those stored in the checksum.txt file.')
+        .addOption(new Option('-u, --update <checksum.txt>', 'Update checksums found in the file.').conflicts('--check'))
+        .option('-r, --root <root>', 'Specify the root to use for relative paths. The current working directory is used by default.')
+        .action(shasum);
     await program.parseAsync(argv);
 }
 //# sourceMappingURL=app.js.map

package/dist/build.d.ts

@@ -5,6 +5,8 @@ export interface BuildOptions {
     root?: string | undefined;
     /** Current working directory */
     cwd?: string | undefined;
+    /** Conditional build based upon the targets matching the `checksum.txt` file. */
+    conditional?: boolean;
 }
 export declare function build(targets: string[] | undefined, options: BuildOptions): Promise<void>;
 //# sourceMappingURL=build.d.ts.map

package/dist/build.js

@@ -12,7 +12,7 @@ const searchPlaces = [
 export async function build(targets, options) {
     const allowedTargets = new Set(targets || []);
     function filter(target) {
-        return !targets || allowedTargets.has(target.name);
+        return !allowedTargets.size || allowedTargets.has(target.name);
     }
     const searchDir = path.resolve(options.root || options.cwd || '.');
     const explorer = cosmiconfig(moduleName, {
@@ -25,11 +25,12 @@ export async function build(targets, options) {
         console.error('root: %s', options.root);
         throw 'cspell-tools.config not found.';
     }
-    const buildInfo = normalizeRequest(config.config, options.root || path.dirname(config.filepath));
-    await compile(buildInfo, { filter, cwd: options.cwd });
+    const configDir = path.dirname(config.filepath);
+    const buildInfo = normalizeRequest(config.config, path.resolve(configDir, options.root || configDir));
+    await compile(buildInfo, { filter, cwd: options.cwd, conditionalBuild: options.conditional || false });
 }
 function normalizeRequest(buildInfo, root) {
-    const { rootDir = root, targets = [] } = buildInfo;
-    return { rootDir, targets };
+    const { rootDir = root, targets = [], checksumFile } = buildInfo;
+    return { rootDir: path.resolve(rootDir), targets, checksumFile };
 }
 //# sourceMappingURL=build.js.map

package/dist/compile.d.ts

@@ -1,4 +1,5 @@
 import type { CompileCommonAppOptions } from './AppOptions.js';
 import type { FeatureFlags } from './FeatureFlags/index.js';
+export declare const configFileHeader: string;
 export declare function processCompileAction(src: string[], options: CompileCommonAppOptions, featureFlags: FeatureFlags | undefined): Promise<void>;
 //# sourceMappingURL=compile.d.ts.map

package/dist/compile.js

@@ -1,20 +1,41 @@
+import { writeFile } from 'node:fs/promises';
 import { opConcatMap, pipe } from '@cspell/cspell-pipe/sync';
+import YAML from 'yaml';
 import { compile } from './compiler/compile.js';
 import { createCompileRequest } from './compiler/createCompileRequest.js';
-import { globP } from './compiler/globP.js';
+import { configFileSchemaURL } from './config/config.js';
 import { getSystemFeatureFlags, parseFlags } from './FeatureFlags/index.js';
+import { globP } from './util/globP.js';
 getSystemFeatureFlags().register('compound', 'Enable compound dictionary sources.');
+const defaultConfigFile = 'cspell-tools.config.yaml';
+export const configFileHeader = `# yaml-language-server: $schema=${configFileSchemaURL}\n\n`;
 export async function processCompileAction(src, options, featureFlags) {
     const ff = featureFlags || getSystemFeatureFlags();
-    parseFlags(ff, options.experimental);
+    parseFlags(ff, options.experimental || []);
     return useCompile(src, options);
 }
 async function useCompile(src, options) {
-    console.log('Compile:\n output: %s\n compress: %s\n files:\n  %s \n\n', options.output || 'default', options.compress ? 'true' : 'false', src.join('\n  '));
+    console.log('Compile:\n output: %s\n compress: %s\n files:\n  %s ', options.output || 'default', options.compress ? 'true' : 'false', src.join('\n  '));
+    if (options.listFile && options.listFile.length) {
+        console.log(' list files:\n  %s', options.listFile.join('\n  '));
+    }
+    console.log('\n\n');
     const globResults = await Promise.all(src.map((s) => globP(s)));
     const sources = [
         ...pipe(globResults, opConcatMap((a) => a)),
     ];
-    return compile(createCompileRequest(sources, options));
+    const request = createCompileRequest(sources, options);
+    return options.init ? initConfig(request) : compile(request);
+}
+async function initConfig(runConfig) {
+    const { $schema = configFileSchemaURL, ...cfg } = runConfig;
+    const config = { $schema, ...cfg };
+    const content = configFileHeader + YAML.stringify(config, null, 2);
+    console.log('Writing config file: %s', defaultConfigFile);
+    await writeFile(defaultConfigFile, content);
+    console.log(`Init complete.
+To build, use:
+  cspell-tools-cli build
+`);
 }
 //# sourceMappingURL=compile.js.map

package/dist/compiler/compile.d.ts

@@ -1,4 +1,4 @@
-import type { CompileRequest, CompileSourceOptions, Target } from '../config/index.js';
+import type { CompileRequest, CompileSourceOptions as CompileSourceConfig, Target } from '../config/index.js';
 interface CompileOptions {
     /**
      * Optional filter function to filter targets.
@@ -8,8 +8,18 @@ interface CompileOptions {
      * The current working directory. Defaults to process.cwd()
      */
     cwd?: string;
+    /**
+     * `true` - only build if files do not match checksum.
+     */
+    conditionalBuild: boolean;
 }
 export declare function compile(request: CompileRequest, options?: CompileOptions): Promise<void>;
-export declare function compileTarget(target: Target, options: CompileSourceOptions, rootDir: string, cwd?: string): Promise<void>;
+interface CompileTargetOptions {
+    rootDir: string;
+    cwd: string | undefined;
+    conditional: boolean;
+    checksumFile: string | undefined;
+}
+export declare function compileTarget(target: Target, options: CompileSourceConfig, compileOptions: CompileTargetOptions): Promise<string[]>;
 export {};
 //# sourceMappingURL=compile.d.ts.map

package/dist/compiler/compile.js

@@ -3,6 +3,7 @@ import { opAwaitAsync, opMapAsync } from '@cspell/cspell-pipe/operators';
 import { opConcatMap, opMap, pipe } from '@cspell/cspell-pipe/sync';
 import * as path from 'path';
 import { isFileListSource, isFilePath, isFileSource } from '../config/index.js';
+import { checkShasumFile, updateChecksumForFiles } from '../shasum/index.js';
 import { createAllowedSplitWordsFromFiles } from './createWordsCollection.js';
 import { logWithTimestamp } from './logWithTimestamp.js';
 import { readTextFile } from './readers/readTextFile.js';
@@ -18,17 +19,33 @@ export async function compile(request, options) {
         sort: request.sort,
         generateNonStrict: request.generateNonStrict,
     };
+    const conditional = options?.conditionalBuild || false;
+    const checksumFile = resolveChecksumFile(request.checksumFile || conditional, rootDir);
+    const dependencies = new Set();
     for (const target of targets) {
         const keep = options?.filter?.(target) ?? true;
         if (!keep)
             continue;
         const adjustedTarget = { ...targetOptions, ...target };
-        await compileTarget(adjustedTarget, request, rootDir, cwd);
+        const deps = await compileTarget(adjustedTarget, request, { rootDir, cwd, conditional, checksumFile });
+        deps.forEach((dep) => dependencies.add(dep));
+    }
+    if (checksumFile && dependencies.size) {
+        logWithTimestamp('%s', `Update checksum: ${checksumFile}`);
+        await updateChecksumForFiles(checksumFile, [...dependencies], { root: path.dirname(checksumFile) });
     }
     logWithTimestamp(`Complete.`);
+    return;
+}
+function resolveChecksumFile(checksumFile, root) {
+    const cFilename = (typeof checksumFile === 'string' && checksumFile) || (checksumFile && './checksum.txt') || undefined;
+    const file = cFilename && path.resolve(root, cFilename);
+    // console.warn('%o', { checksumFile, cFilename, file });
+    return file;
 }
-export async function compileTarget(target, options, rootDir, cwd) {
+export async function compileTarget(target, options, compileOptions) {
     logWithTimestamp(`Start compile: ${target.name}`);
+    const { rootDir, cwd, checksumFile, conditional } = compileOptions;
     const { format, sources, trieBase, sort = true, generateNonStrict = false } = target;
     const targetDirectory = path.resolve(rootDir, target.targetDirectory ?? cwd ?? process.cwd());
     const generateNonStrictTrie = target.generateNonStrict ?? true;
@@ -38,6 +55,15 @@ export async function compileTarget(target, options, rootDir, cwd) {
     const filesToProcessAsync = pipeAsync(readSourceList(sources, rootDir), opMapAsync((src) => readFileSource(src, options)), opAwaitAsync());
     const filesToProcess = await toArray(filesToProcessAsync);
     const normalizer = normalizeTargetWords({ sort: useTrie || sort, generateNonStrict });
+    const checksumRoot = (checksumFile && path.dirname(checksumFile)) || rootDir;
+    const deps = [...calculateDependencies(filename, filesToProcess, checksumRoot)];
+    if (conditional && checksumFile) {
+        const check = await checkShasumFile(checksumFile, deps, checksumRoot).catch(() => undefined);
+        if (check?.passed) {
+            logWithTimestamp(`Skip ${target.name}, nothing changed.`);
+            return [];
+        }
+    }
     const action = useTrie
         ? async (words, dst) => {
             return compileTrie(pipe(words, normalizer), dst, {
@@ -53,6 +79,19 @@ export async function compileTarget(target, options, rootDir, cwd) {
         };
     await processFiles(action, filesToProcess, filename);
     logWithTimestamp(`Done compile: ${target.name}`);
+    return deps;
+}
+function calculateDependencies(targetFile, filesToProcess, rootDir) {
+    const dependencies = new Set();
+    addDependency(targetFile);
+    filesToProcess.forEach((f) => addDependency(f.src));
+    return dependencies;
+    function addDependency(filename) {
+        const rel = path.relative(rootDir, filename);
+        dependencies.add(rel);
+        dependencies.add(rel.replace(/\.aff$/, '.dic'));
+        dependencies.add(rel.replace(/\.dic$/, '.aff'));
+    }
 }
 function rel(filePath) {
     return path.relative(process.cwd(), filePath);
@@ -67,9 +106,9 @@ async function processFiles(action, filesToProcess, mergeTarget) {
     }), opConcatMap(function* (ftp) {
         yield* ftp.words;
         logWithTimestamp('Done processing %s', rel(ftp.src));
-    })
+    }), 
     // opMap((a) => (console.warn(a), a))
-    );
+    logProgress());
     await action(words, dst);
     logWithTimestamp('Done "%s"', rel(dst));
 }
@@ -134,4 +173,18 @@ async function readFileSource(fileSource, sourceOptions) {
 function normalizeTargetName(name) {
     return name.replace(/((\.txt|\.dic|\.aff|\.trie)(\.gz)?)?$/, '').replace(/[^\p{L}\p{M}.\w\\/-]/gu, '_');
 }
+function logProgress(freq = 100000) {
+    function* logProgress(iter) {
+        const _freq = freq;
+        let count = 0;
+        for (const v of iter) {
+            ++count;
+            if (!(count % _freq)) {
+                logWithTimestamp('Progress: Words Processed - %s', count.toLocaleString());
+            }
+            yield v;
+        }
+    }
+    return logProgress;
+}
 //# sourceMappingURL=compile.js.map

package/dist/compiler/createCompileRequest.d.ts

@@ -1,4 +1,4 @@
 import type { CompileCommonAppOptions } from '../AppOptions.js';
 import type { CompileRequest } from '../config/index.js';
-export declare function createCompileRequest(sources: string[], options: CompileCommonAppOptions): CompileRequest;
+export declare function createCompileRequest(sourceFiles: string[], options: CompileCommonAppOptions): CompileRequest;
 //# sourceMappingURL=createCompileRequest.d.ts.map

package/dist/compiler/createCompileRequest.js

@@ -1,12 +1,13 @@
 import * as path from 'path';
-export function createCompileRequest(sources, options) {
-    const { max_depth, maxDepth, experimental = [], split, keepRawCase, useLegacySplitter } = options;
+export function createCompileRequest(sourceFiles, options) {
+    options = { ...options };
+    options.maxDepth ??= options.max_depth;
+    const { maxDepth, split, keepRawCase, useLegacySplitter } = options;
+    const sources = [...sourceFiles, ...(options.listFile || []).map((listFile) => ({ listFile }))];
     const targets = calcTargets(sources, options);
-    const generateNonStrict = experimental.includes('compound') || undefined;
     const req = {
         targets,
-        generateNonStrict,
-        maxDepth: parseNumber(maxDepth) ?? parseNumber(max_depth),
+        maxDepth: parseNumber(maxDepth),
         split: useLegacySplitter ? 'legacy' : split,
         /**
          * Do not generate lower case / accent free versions of words.
@@ -17,30 +18,35 @@ export function createCompileRequest(sources, options) {
     return req;
 }
 function calcTargets(sources, options) {
-    const { merge, output = '.' } = options;
+    const { merge, output = '.', experimental = [] } = options;
+    const generateNonStrict = experimental.includes('compound') || undefined;
+    // console.log('%o', sources);
     const format = calcFormat(options);
+    const sort = (format === 'plaintext' && options.sort) || undefined;
     if (merge) {
         const target = {
             name: merge,
             targetDirectory: output,
             compress: options.compress,
             format,
-            sources,
-            sort: options.sort,
+            sources: sources.map(normalizeSource),
+            sort,
             trieBase: parseNumber(options.trieBase),
+            generateNonStrict,
         };
         return [target];
     }
     const targets = sources.map((source) => {
-        const name = toTargetName(path.basename(source));
+        const name = toTargetName(baseNameOfSource(source));
         const target = {
             name,
             targetDirectory: output,
             compress: options.compress,
             format,
-            sources: [source],
+            sources: [normalizeSource(source)],
             sort: options.sort,
             trieBase: parseNumber(options.trieBase),
+            generateNonStrict,
         };
         return target;
     });
@@ -56,4 +62,23 @@ function parseNumber(s) {
     const n = parseInt(s ?? '');
     return isNaN(n) ? undefined : n;
 }
+function baseNameOfSource(source) {
+    return typeof source === 'string' ? source : isFileSource(source) ? source.filename : source.listFile;
+}
+function isFileSource(source) {
+    return typeof source !== 'string' && source.filename !== undefined;
+}
+function normalizeSource(source) {
+    if (typeof source === 'string') {
+        return normalizeSourcePath(source);
+    }
+    if (isFileSource(source))
+        return { ...source, filename: normalizeSourcePath(source.filename) };
+    return { ...source, listFile: normalizeSourcePath(source.listFile) };
+}
+function normalizeSourcePath(source) {
+    const cwd = process.cwd();
+    const rel = path.relative(cwd, source);
+    return rel.split('\\').join('/');
+}
 //# sourceMappingURL=createCompileRequest.js.map

package/dist/config/config.d.ts

@@ -1,4 +1,9 @@
 export interface RunConfig extends Partial<CompileRequest> {
+    /**
+     * Url to JSON Schema
+     * @default "https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-tools/cspell-tools.config.schema.json"
+     */
+    $schema?: string;
     /**
      * Optional Target Dictionaries to create.
      */
@@ -20,6 +25,10 @@ export interface CompileRequest extends CompileTargetOptions, CompileSourceOptio
      * Target Dictionaries to create.
      */
     targets: Target[];
+    /**
+     * Path to checksum file. `true` - defaults to `./checksum.txt`.
+     */
+    checksumFile?: string | boolean;
 }
 export interface Experimental {
     /**
@@ -119,4 +128,5 @@ export interface CompileSourceOptions {
     keepRawCase?: boolean | undefined;
     allowedSplitWords?: FilePath | FilePath[] | undefined;
 }
+export declare const configFileSchemaURL = "https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-tools/cspell-tools.config.schema.json";
 //# sourceMappingURL=config.d.ts.map

package/dist/config/config.js

@@ -1,2 +1,2 @@
-export {};
+export const configFileSchemaURL = 'https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-tools/cspell-tools.config.schema.json';
 //# sourceMappingURL=config.js.map

package/dist/gzip/compressFiles.d.ts

@@ -0,0 +1,2 @@
+export declare function compressFile(file: string): Promise<string>;
+//# sourceMappingURL=compressFiles.d.ts.map

package/dist/gzip/compressFiles.js

@@ -0,0 +1,14 @@
+import { readFile, writeFile } from 'node:fs/promises';
+import { promisify } from 'node:util';
+import { gzip as gz } from 'node:zlib';
+const gzip = promisify(gz);
+export async function compressFile(file) {
+    if (file.endsWith('.gz'))
+        return file;
+    const targetFile = file + '.gz';
+    const buf = await readFile(file);
+    const zBuf = await gzip(buf);
+    await writeFile(targetFile, zBuf);
+    return targetFile;
+}
+//# sourceMappingURL=compressFiles.js.map

package/dist/gzip/gzip.d.ts

@@ -0,0 +1,2 @@
+export declare function gzip(globs: string[]): Promise<void>;
+//# sourceMappingURL=gzip.d.ts.map

package/dist/gzip/gzip.js

@@ -0,0 +1,10 @@
+import { globP } from '../util/globP.js';
+import { compressFile } from './compressFiles.js';
+// cspell:ignore nodir
+export async function gzip(globs) {
+    const files = await globP(globs, { nodir: true });
+    for (const fileName of files) {
+        await compressFile(fileName);
+    }
+}
+//# sourceMappingURL=gzip.js.map

package/dist/gzip/index.d.ts

@@ -0,0 +1,3 @@
+export { compressFile } from './compressFiles.js';
+export { gzip } from './gzip.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/gzip/index.js

@@ -0,0 +1,3 @@
+export { compressFile } from './compressFiles.js';
+export { gzip } from './gzip.js';
+//# sourceMappingURL=index.js.map

package/dist/shasum/checksum.d.ts

@@ -0,0 +1,8 @@
+/// <reference types="node" resolution-mode="require"/>
+type HashAlgorithm = 'SHA1';
+export declare function calcChecksum(buf: Buffer, alg?: HashAlgorithm): string;
+export declare function checkChecksum(checksum: string, buf: Buffer, alg?: HashAlgorithm): boolean;
+export declare function calcFileChecksum(filename: string, alg?: HashAlgorithm): Promise<string>;
+export declare function checkFile(checksum: string, filename: string, alg?: HashAlgorithm): Promise<boolean>;
+export {};
+//# sourceMappingURL=checksum.d.ts.map

package/dist/shasum/checksum.js

@@ -0,0 +1,19 @@
+import { createHash } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+export function calcChecksum(buf, alg = 'SHA1') {
+    const hash = createHash(alg);
+    hash.update(buf);
+    return hash.digest('hex');
+}
+export function checkChecksum(checksum, buf, alg) {
+    const value = calcChecksum(buf, alg);
+    return value === checksum;
+}
+export async function calcFileChecksum(filename, alg) {
+    const buf = await readFile(filename);
+    return calcChecksum(buf, alg);
+}
+export async function checkFile(checksum, filename, alg) {
+    return (await calcFileChecksum(filename, alg)) === checksum;
+}
+//# sourceMappingURL=checksum.js.map

package/dist/shasum/index.d.ts

@@ -0,0 +1,3 @@
+export { calcFileChecksum } from './checksum.js';
+export { checkShasumFile, updateChecksumForFiles } from './shasum.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/shasum/index.js

@@ -0,0 +1,3 @@
+export { calcFileChecksum } from './checksum.js';
+export { checkShasumFile, updateChecksumForFiles } from './shasum.js';
+//# sourceMappingURL=index.js.map

package/dist/shasum/shasum.d.ts

@@ -0,0 +1,38 @@
+export interface CheckShasumFileResult {
+    passed: boolean;
+    results: CheckFileResult[];
+}
+export interface CheckFileResult {
+    filename: string;
+    passed: boolean;
+    error?: Error;
+}
+export declare function shasumFile(filename: string, root: string | undefined): Promise<string>;
+/**
+ *
+ * @param filename - name of checksum file
+ * @param files - optional list of files to check
+ * @param root - optional root, default cwd.
+ */
+export declare function checkShasumFile(filename: string, files: string[] | undefined, root?: string): Promise<CheckShasumFileResult>;
+export interface ChecksumEntry {
+    filename: string;
+    checksum: string;
+    lineNumber: number;
+}
+export declare function readAndParseShasumFile(filename: string): Promise<ChecksumEntry[]>;
+export declare function parseShasumFile(content: string): ChecksumEntry[];
+interface ReportResult {
+    report: string;
+    passed: boolean;
+}
+interface ReportOptions {
+    root?: string | undefined;
+    listFile?: string[];
+}
+export declare function reportChecksumForFiles(files: string[], options: ReportOptions): Promise<ReportResult>;
+export declare function reportCheckChecksumFile(filename: string, files: string[] | undefined, options: ReportOptions): Promise<ReportResult>;
+export declare function calcUpdateChecksumForFiles(filename: string, files: string[], options: ReportOptions): Promise<string>;
+export declare function updateChecksumForFiles(filename: string, files: string[], options: ReportOptions): Promise<ReportResult>;
+export {};
+//# sourceMappingURL=shasum.d.ts.map

package/dist/shasum/shasum.js

@@ -0,0 +1,148 @@
+import { readFile, writeFile } from 'node:fs/promises';
+import { resolve, sep as pathSep } from 'node:path';
+import { toError } from '../util/errors.js';
+import { isDefined } from '../util/index.js';
+import { calcFileChecksum, checkFile } from './checksum.js';
+export async function shasumFile(filename, root) {
+    try {
+        const file = resolve(root || '.', filename);
+        const checksum = await calcFileChecksum(file);
+        return `${checksum}  ${filename}`;
+    }
+    catch (error) {
+        // const err = toError(error);
+        return Promise.reject(`shasum: ${filename}: Unable to read file.`);
+    }
+}
+/**
+ *
+ * @param filename - name of checksum file
+ * @param files - optional list of files to check
+ * @param root - optional root, default cwd.
+ */
+export async function checkShasumFile(filename, files, root) {
+    files = !files ? files : files.length ? files : undefined;
+    const shaFiles = await readAndParseShasumFile(filename);
+    const filesToCheck = !files ? shaFiles.map(({ filename }) => filename) : files;
+    const mapNameToChecksum = new Map(shaFiles.map((r) => [normalizeFilename(r.filename), r.checksum]));
+    const resolvedRoot = resolve(root || '.');
+    const results = await Promise.all(filesToCheck.map(normalizeFilename).map((filename) => {
+        return tryToCheckFile(filename, resolvedRoot, mapNameToChecksum.get(filename));
+    }));
+    const passed = !results.find((v) => !v.passed);
+    return { passed, results };
+}
+async function tryToCheckFile(filename, root, checksum) {
+    if (!checksum) {
+        return { filename, passed: false, error: Error('Missing Checksum.') };
+    }
+    const file = resolve(root, filename);
+    try {
+        const passed = await checkFile(checksum, file);
+        return { filename, passed };
+    }
+    catch (error) {
+        return { filename, passed: false, error: Error('Failed to read file.') };
+    }
+}
+const regLine = /([a-f0-9]{40,}) {2}(.*)/;
+export async function readAndParseShasumFile(filename) {
+    const content = await readFile(resolve(filename), 'utf8');
+    const shaFiles = parseShasumFile(content);
+    return shaFiles;
+}
+export function parseShasumFile(content) {
+    const lines = content.split(/\r?\n|\r/g);
+    return lines.map(parseLine).filter(isDefined);
+    function parseLine(line, index) {
+        const m = line.match(regLine);
+        const lineNumber = index + 1;
+        if (!m) {
+            if (line.trim()) {
+                throw new Error(`Failed to parse line ${lineNumber} of checksum file.`);
+            }
+            return undefined;
+        }
+        const checksum = m[1];
+        const filename = m[2];
+        return { checksum, filename, lineNumber };
+    }
+}
+export async function reportChecksumForFiles(files, options) {
+    const root = options.root;
+    const filesToCheck = await resolveFileList(files, options.listFile);
+    let numFailed = 0;
+    const result = await Promise.all(filesToCheck.map((file) => shasumFile(file, root).catch((e) => {
+        ++numFailed;
+        if (typeof e !== 'string')
+            throw e;
+        return e;
+    })));
+    const report = result.join('\n');
+    const passed = !numFailed;
+    return { report, passed };
+}
+export async function reportCheckChecksumFile(filename, files, options) {
+    const root = options.root;
+    const filesToCheck = await resolveFileList(files, options.listFile);
+    const checkResult = await checkShasumFile(filename, filesToCheck, root);
+    const results = checkResult.results;
+    const lines = results.map(({ filename, passed, error }) => `${filename}: ${passed ? 'OK' : 'FAILED'} ${error ? '- ' + error.message : ''}`.trim());
+    const withErrors = results.filter((a) => !a.passed);
+    const passed = !withErrors.length;
+    if (!passed) {
+        lines.push(`shasum: WARNING: ${withErrors.length} computed checksum${withErrors.length > 1 ? 's' : ''} did NOT match`);
+    }
+    return { report: lines.join('\n'), passed };
+}
+async function resolveFileList(files, listFile) {
+    files = files || [];
+    listFile = listFile || [];
+    const setOfFiles = new Set(files);
+    const pending = listFile.map((filename) => readFile(filename, 'utf8'));
+    for await (const content of pending) {
+        content
+            .split('\n')
+            .map((a) => a.trim())
+            .filter((a) => a)
+            .forEach((file) => setOfFiles.add(file));
+    }
+    return [...setOfFiles].map(normalizeFilename);
+}
+export async function calcUpdateChecksumForFiles(filename, files, options) {
+    const root = options.root || '.';
+    const filesToCheck = await resolveFileList(files, options.listFile);
+    const currentEntries = (await readAndParseShasumFile(filename).catch((err) => {
+        const e = toError(err);
+        if (e.code !== 'ENOENT')
+            throw e;
+        return [];
+    })).map((entry) => ({ ...entry, filename: normalizeFilename(entry.filename) }));
+    const entriesToUpdate = new Set([...filesToCheck, ...currentEntries.map((e) => e.filename)]);
+    const mustExist = new Set(filesToCheck);
+    const checksumMap = new Map(currentEntries.map(({ filename, checksum }) => [filename, checksum]));
+    for (const file of entriesToUpdate) {
+        try {
+            const checksum = await calcFileChecksum(resolve(root, file));
+            checksumMap.set(file, checksum);
+        }
+        catch (e) {
+            if (mustExist.has(file) || toError(e).code !== 'ENOENT')
+                throw e;
+            checksumMap.delete(file);
+        }
+    }
+    const updatedEntries = [...checksumMap]
+        .map(([filename, checksum]) => ({ filename, checksum }))
+        .sort((a, b) => (a.filename < b.filename ? -1 : 1));
+    return updatedEntries.map((e) => `${e.checksum}  ${e.filename}`).join('\n') + '\n';
+}
+export async function updateChecksumForFiles(filename, files, options) {
+    const content = await calcUpdateChecksumForFiles(filename, files, options);
+    await writeFile(filename, content);
+    return { passed: true, report: content };
+}
+function normalizeFilename(filename) {
+    return filename.split(pathSep).join('/');
+}
+//# sourceMappingURL=shasum.js.map

package/dist/util/errors.d.ts

@@ -0,0 +1,6 @@
+export interface NodeError extends Error {
+    code?: string;
+}
+export declare function toError(err: unknown): NodeError;
+export declare function isError(err: unknown): err is Error;
+//# sourceMappingURL=errors.d.ts.map

package/dist/util/errors.js

@@ -0,0 +1,11 @@
+export function toError(err) {
+    if (isError(err))
+        return err;
+    return new Error(`${err}`);
+}
+export function isError(err) {
+    if (err instanceof Error)
+        return true;
+    return false;
+}
+//# sourceMappingURL=errors.js.map

package/dist/util/globP.d.ts

@@ -0,0 +1,7 @@
+interface Options {
+    cwd?: string | URL;
+    nodir?: boolean;
+}
+export declare function globP(pattern: string | string[], options?: Options): Promise<string[]>;
+export {};
+//# sourceMappingURL=globP.d.ts.map

package/dist/util/globP.js

@@ -0,0 +1,7 @@
+import { glob } from 'glob';
+export function globP(pattern, options) {
+    // Convert windows separators.
+    const globs = (Array.isArray(pattern) ? pattern : [pattern]).map((pattern) => pattern.replace(/\\/g, '/'));
+    return glob(globs, options);
+}
+//# sourceMappingURL=globP.js.map

package/dist/util/index.d.ts

@@ -0,0 +1,2 @@
+export declare function isDefined<T>(a: T | undefined | null): a is T;
+//# sourceMappingURL=index.d.ts.map

package/dist/util/index.js

@@ -0,0 +1,4 @@
+export function isDefined(a) {
+    return !!a || (a !== null && a !== undefined);
+}
+//# sourceMappingURL=index.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cspell/cspell-tools",
-  "version": "7.0.0-alpha.2",
+  "version": "7.0.0",
   "description": "Tools to assist with the development of cSpell",
   "publishConfig": {
     "access": "public"
@@ -16,12 +16,11 @@
     "watch": "tsc -p . -w",
     "clean-build": "pnpm run clean && pnpm run build",
     "clean": "shx rm -rf dist temp coverage \"*.tsbuildInfo\"",
-    "coverage": "pnpm coverage:vitest && pnpm coverage:fix",
+    "coverage": "pnpm coverage:vitest",
     "coverage:vitest": "vitest run --coverage",
-    "coverage:fix": "nyc report --temp-dir \"$(pwd)/coverage\" --reporter lcov --report-dir \"$(pwd)/coverage\" --cwd ../..",
-    "test-watch": "jest --watch",
+    "test:watch": "vitest",
     "test": "vitest run",
-    "update-snapshot": "jest --updateSnapshot"
+    "update-snapshot": "vitest run -u"
   },
   "repository": {
     "type": "git",
@@ -50,26 +49,25 @@
   },
   "homepage": "https://github.com/streetsidesoftware/cspell#readme",
   "dependencies": {
-    "@cspell/cspell-pipe": "7.0.0-alpha.2",
-    "commander": "^10.0.1",
+    "@cspell/cspell-pipe": "7.0.0",
+    "commander": "^11.0.0",
     "cosmiconfig": "8.0.0",
-    "cspell-trie-lib": "7.0.0-alpha.2",
+    "cspell-trie-lib": "7.0.0",
     "gensequence": "^5.0.2",
-    "glob": "^8.1.0",
-    "hunspell-reader": "7.0.0-alpha.2"
+    "glob": "^10.3.3",
+    "hunspell-reader": "7.0.0",
+    "yaml": "^2.3.1"
   },
   "engines": {
     "node": ">=16"
   },
   "devDependencies": {
     "@types/glob": "^8.1.0",
-    "@types/jest": "^29.5.1",
     "@types/shelljs": "^0.8.12",
-    "jest": "^29.5.0",
     "lorem-ipsum": "^2.0.8",
     "shelljs": "^0.8.5",
     "ts-json-schema-generator": "^1.2.0"
   },
   "main": "bin.js",
-  "gitHead": "a1b7c5daeef5afdb14d6444318f450b9fd9c035a"
+  "gitHead": "52960d5ed75655978f9b633f44fd106937a63cd7"
 }

package/dist/compiler/globP.d.ts

@@ -1,2 +0,0 @@
-export declare function globP(pattern: string): Promise<string[]>;
-//# sourceMappingURL=globP.d.ts.map

package/dist/compiler/globP.js

@@ -1,9 +0,0 @@
-import glob from 'glob';
-import { promisify } from 'util';
-const pGlob = promisify(glob);
-export function globP(pattern) {
-    // Convert windows separators.
-    pattern = pattern.replace(/\\/g, '/');
-    return pGlob(pattern);
-}
-//# sourceMappingURL=globP.js.map

package/dist/compiler/inlineSettings.d.ts

@@ -1,7 +0,0 @@
-export interface InlineSettings {
-    split?: boolean;
-    keepRawCase?: boolean;
-    caseSensitive?: boolean;
-}
-export declare function extractInlineSettings(line: string): InlineSettings | undefined;
-//# sourceMappingURL=inlineSettings.d.ts.map

package/dist/compiler/inlineSettings.js

@@ -1,35 +0,0 @@
-const allowedSettings = ['split', 'no-split', 'keep-case', 'no-keep-case', 'case-sensitive'];
-export function extractInlineSettings(line) {
-    const m = line.match(/cspell-tools:(.*)/);
-    if (!m)
-        return undefined;
-    const flags = m[1].split(/\s+/g).filter((a) => !!a);
-    const settings = {};
-    for (const flag of flags) {
-        switch (flag) {
-            case 'split':
-                settings.split = true;
-                break;
-            case 'no-split':
-                settings.split = false;
-                break;
-            case 'keep-case':
-                settings.keepRawCase = true;
-                break;
-            case 'no-keep-case':
-                settings.keepRawCase = false;
-                break;
-            case 'case-sensitive':
-                settings.caseSensitive = true;
-                break;
-            case 'flag':
-            case 'flags':
-                // Ignore flags
-                break;
-            default:
-                throw new Error(`Unknown inline setting: "${flag}" allowed values are ${allowedSettings.join(', ')}`);
-        }
-    }
-    return settings;
-}
-//# sourceMappingURL=inlineSettings.js.map