@cspell/cspell-tools 7.0.1-alpha.5 → 7.0.1-alpha.7

package/bin.js

@@ -6,6 +6,8 @@ import { run } from './dist/app.js';
 run(program, process.argv).catch((e) => {
     if (!(e instanceof CommanderError)) {
         console.log(e);
+    } else {
+        console.log(e.message);
     }
     process.exitCode = 1;
 });

package/dist/app.js

@@ -1,12 +1,12 @@
 // For large dictionaries, it is necessary to increase the memory limit.
-import { CommanderError } from 'commander';
+import { CommanderError, Option } from 'commander';
 import { readFileSync } from 'fs';
 import { build } from './build.js';
 import { processCompileAction } from './compile.js';
 import * as compiler from './compiler/index.js';
 import { logWithTimestamp } from './compiler/logWithTimestamp.js';
 import { gzip } from './gzip/index.js';
-import { reportCheckChecksumFile, reportChecksumForFiles } from './shasum/shasum.js';
+import { reportCheckChecksumFile, reportChecksumForFiles, updateChecksumForFiles } from './shasum/shasum.js';
 import { toError } from './util/errors.js';
 const npmPackageRaw = readFileSync(new URL('../package.json', import.meta.url), 'utf8');
 const npmPackage = JSON.parse(npmPackageRaw);
@@ -43,8 +43,10 @@ export async function run(program, argv, flags) {
     }
     async function shasum(files, options) {
         const report = options.check
-            ? await reportCheckChecksumFile(options.check, files, options.root)
-            : await reportChecksumForFiles(files, options.root);
+            ? await reportCheckChecksumFile(options.check, files, options)
+            : options.update
+                ? await updateChecksumForFiles(options.update, files, options)
+                : await reportChecksumForFiles(files, options);
         console.log('%s', report.report);
         if (!report.passed) {
             throw new CommanderError(1, 'Failed Checksum', 'One or more files had issues.');
@@ -73,7 +75,9 @@ export async function run(program, argv, flags) {
     program
         .command('shasum [files...]')
         .description('Calculate the checksum for files.')
+        .option('--list-file <list-file.txt...>', 'Specify one or more files that contain paths of files to check.')
         .option('-c, --check <checksum.txt>', 'Verify the checksum of files against those stored in the checksum.txt file.')
+        .addOption(new Option('-u, --update <checksum.txt>', 'Update checksums found in the file.').conflicts('--check'))
         .option('-r, --root <root>', 'Specify the root to use for relative paths. The current working directory is used by default.')
         .action(shasum);
     await program.parseAsync(argv);

package/dist/compile.d.ts

@@ -1,4 +1,5 @@
 import type { CompileCommonAppOptions } from './AppOptions.js';
 import type { FeatureFlags } from './FeatureFlags/index.js';
+export declare const configFileHeader = "# yaml-language-server: $schema=https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-tools/cspell-tools.config.schema.json\n\n";
 export declare function processCompileAction(src: string[], options: CompileCommonAppOptions, featureFlags: FeatureFlags | undefined): Promise<void>;
 //# sourceMappingURL=compile.d.ts.map

package/dist/compile.js

@@ -7,6 +7,7 @@ import { getSystemFeatureFlags, parseFlags } from './FeatureFlags/index.js';
 import { globP } from './util/globP.js';
 getSystemFeatureFlags().register('compound', 'Enable compound dictionary sources.');
 const defaultConfigFile = 'cspell-tools.config.yaml';
+export const configFileHeader = '# yaml-language-server: $schema=https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-tools/cspell-tools.config.schema.json\n\n';
 export async function processCompileAction(src, options, featureFlags) {
     const ff = featureFlags || getSystemFeatureFlags();
     parseFlags(ff, options.experimental || []);
@@ -26,7 +27,7 @@ async function useCompile(src, options) {
     return options.init ? initConfig(request) : compile(request);
 }
 async function initConfig(request) {
-    const content = YAML.stringify(request, null, 2);
+    const content = configFileHeader + YAML.stringify(request, null, 2);
     console.log('Writing config file: %s', defaultConfigFile);
     await writeFile(defaultConfigFile, content);
     console.log(`Init complete.

package/dist/compiler/compile.js

@@ -67,9 +67,9 @@ async function processFiles(action, filesToProcess, mergeTarget) {
     }), opConcatMap(function* (ftp) {
         yield* ftp.words;
         logWithTimestamp('Done processing %s', rel(ftp.src));
-    })
+    }), 
     // opMap((a) => (console.warn(a), a))
-    );
+    logProgress());
     await action(words, dst);
     logWithTimestamp('Done "%s"', rel(dst));
 }
@@ -134,4 +134,18 @@ async function readFileSource(fileSource, sourceOptions) {
 function normalizeTargetName(name) {
     return name.replace(/((\.txt|\.dic|\.aff|\.trie)(\.gz)?)?$/, '').replace(/[^\p{L}\p{M}.\w\\/-]/gu, '_');
 }
+function logProgress(freq = 100000) {
+    function* logProgress(iter) {
+        const _freq = freq;
+        let count = 0;
+        for (const v of iter) {
+            ++count;
+            if (!(count % _freq)) {
+                logWithTimestamp('Progress: Words Processed - %s', count.toLocaleString());
+            }
+            yield v;
+        }
+    }
+    return logProgress;
+}
 //# sourceMappingURL=compile.js.map

package/dist/compiler/createCompileRequest.js

@@ -1,13 +1,13 @@
 import * as path from 'path';
 export function createCompileRequest(sourceFiles, options) {
-    const { max_depth, maxDepth, experimental = [], split, keepRawCase, useLegacySplitter } = options;
+    options = { ...options };
+    options.maxDepth ??= options.max_depth;
+    const { maxDepth, split, keepRawCase, useLegacySplitter } = options;
     const sources = [...sourceFiles, ...(options.listFile || []).map((listFile) => ({ listFile }))];
     const targets = calcTargets(sources, options);
-    const generateNonStrict = experimental.includes('compound') || undefined;
     const req = {
         targets,
-        generateNonStrict,
-        maxDepth: parseNumber(maxDepth) ?? parseNumber(max_depth),
+        maxDepth: parseNumber(maxDepth),
         split: useLegacySplitter ? 'legacy' : split,
         /**
          * Do not generate lower case / accent free versions of words.
@@ -18,17 +18,21 @@ export function createCompileRequest(sourceFiles, options) {
     return req;
 }
 function calcTargets(sources, options) {
-    const { merge, output = '.' } = options;
+    const { merge, output = '.', experimental = [] } = options;
+    const generateNonStrict = experimental.includes('compound') || undefined;
+    // console.log('%o', sources);
     const format = calcFormat(options);
+    const sort = (format === 'plaintext' && options.sort) || undefined;
     if (merge) {
         const target = {
             name: merge,
             targetDirectory: output,
             compress: options.compress,
             format,
-            sources,
-            sort: options.sort,
+            sources: sources.map(normalizeSource),
+            sort,
             trieBase: parseNumber(options.trieBase),
+            generateNonStrict,
         };
         return [target];
     }
@@ -39,9 +43,10 @@ function calcTargets(sources, options) {
             targetDirectory: output,
             compress: options.compress,
             format,
-            sources: [source],
+            sources: [normalizeSource(source)],
             sort: options.sort,
             trieBase: parseNumber(options.trieBase),
+            generateNonStrict,
         };
         return target;
     });
@@ -63,4 +68,17 @@ function baseNameOfSource(source) {
 function isFileSource(source) {
     return typeof source !== 'string' && source.filename !== undefined;
 }
+function normalizeSource(source) {
+    if (typeof source === 'string') {
+        return normalizeSourcePath(source);
+    }
+    if (isFileSource(source))
+        return { ...source, filename: normalizeSourcePath(source.filename) };
+    return { ...source, listFile: normalizeSourcePath(source.listFile) };
+}
+function normalizeSourcePath(source) {
+    const cwd = process.cwd();
+    const rel = path.relative(cwd, source);
+    return rel.split('\\').join('/');
+}
 //# sourceMappingURL=createCompileRequest.js.map

package/dist/shasum/shasum.d.ts

@@ -22,7 +22,13 @@ interface ReportResult {
     report: string;
     passed: boolean;
 }
-export declare function reportChecksumForFiles(files: string[], root: string | undefined): Promise<ReportResult>;
-export declare function reportCheckChecksumFile(filename: string, files: string[] | undefined, root: string | undefined): Promise<ReportResult>;
+interface ReportOptions {
+    root?: string | undefined;
+    listFile?: string[];
+}
+export declare function reportChecksumForFiles(files: string[], options: ReportOptions): Promise<ReportResult>;
+export declare function reportCheckChecksumFile(filename: string, files: string[] | undefined, options: ReportOptions): Promise<ReportResult>;
+export declare function calcUpdateChecksumForFiles(filename: string, files: string[], options: ReportOptions): Promise<string>;
+export declare function updateChecksumForFiles(filename: string, files: string[], options: ReportOptions): Promise<ReportResult>;
 export {};
 //# sourceMappingURL=shasum.d.ts.map

package/dist/shasum/shasum.js

@@ -1,5 +1,6 @@
-import { readFile } from 'node:fs/promises';
+import { readFile, writeFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
+import { toError } from '../util/errors.js';
 import { isDefined } from '../util/index.js';
 import { calcFileChecksum, checkFile } from './checksum.js';
 export async function shasumFile(filename, root) {
@@ -66,9 +67,11 @@ export function parseShasumFile(content) {
         return { checksum, filename, lineNumber };
     }
 }
-export async function reportChecksumForFiles(files, root) {
+export async function reportChecksumForFiles(files, options) {
+    const root = options.root;
+    const filesToCheck = await resolveFileList(files, options.listFile);
     let numFailed = 0;
-    const result = await Promise.all(files.map((file) => shasumFile(file, root).catch((e) => {
+    const result = await Promise.all(filesToCheck.map((file) => shasumFile(file, root).catch((e) => {
         ++numFailed;
         if (typeof e !== 'string')
             throw e;
@@ -78,8 +81,10 @@ export async function reportChecksumForFiles(files, root) {
     const passed = !numFailed;
     return { report, passed };
 }
-export async function reportCheckChecksumFile(filename, files, root) {
-    const result = await checkShasumFile(filename, files, root);
+export async function reportCheckChecksumFile(filename, files, options) {
+    const root = options.root;
+    const filesToCheck = await resolveFileList(files, options.listFile);
+    const result = await checkShasumFile(filename, filesToCheck, root);
     const lines = result.map(({ filename, passed, error }) => `${filename}: ${passed ? 'OK' : 'FAILED'} ${error ? '- ' + error.message : ''}`.trim());
     const withErrors = result.filter((a) => !a.passed);
     const passed = !withErrors.length;
@@ -88,4 +93,51 @@ export async function reportCheckChecksumFile(filename, files, root) {
     }
     return { report: lines.join('\n'), passed };
 }
+async function resolveFileList(files, listFile) {
+    files = files || [];
+    listFile = listFile || [];
+    const setOfFiles = new Set(files);
+    const pending = listFile.map((filename) => readFile(filename, 'utf8'));
+    for await (const content of pending) {
+        content
+            .split('\n')
+            .map((a) => a.trim())
+            .filter((a) => a)
+            .forEach((file) => setOfFiles.add(file));
+    }
+    return [...setOfFiles];
+}
+export async function calcUpdateChecksumForFiles(filename, files, options) {
+    const root = options.root || '.';
+    const filesToCheck = await resolveFileList(files, options.listFile);
+    const currentEntries = await readAndParseShasumFile(filename).catch((err) => {
+        const e = toError(err);
+        if (e.code !== 'ENOENT')
+            throw e;
+        return [];
+    });
+    const entriesToUpdate = new Set([...filesToCheck, ...currentEntries.map((e) => e.filename)]);
+    const mustExist = new Set(filesToCheck);
+    const checksumMap = new Map(currentEntries.map(({ filename, checksum }) => [filename, checksum]));
+    for (const file of entriesToUpdate) {
+        try {
+            const checksum = await calcFileChecksum(resolve(root, file));
+            checksumMap.set(file, checksum);
+        }
+        catch (e) {
+            if (mustExist.has(file) || toError(e).code !== 'ENOENT')
+                throw e;
+            checksumMap.delete(file);
+        }
+    }
+    const updatedEntries = [...checksumMap]
+        .map(([filename, checksum]) => ({ filename, checksum }))
+        .sort((a, b) => (a.filename < b.filename ? -1 : 1));
+    return updatedEntries.map((e) => `${e.checksum}  ${e.filename}`).join('\n') + '\n';
+}
+export async function updateChecksumForFiles(filename, files, options) {
+    const content = await calcUpdateChecksumForFiles(filename, files, options);
+    await writeFile(filename, content);
+    return { passed: true, report: content };
+}
 //# sourceMappingURL=shasum.js.map

package/dist/util/errors.d.ts

@@ -1,3 +1,6 @@
-export declare function toError(err: unknown): Error;
+export interface NodeError extends Error {
+    code?: string;
+}
+export declare function toError(err: unknown): NodeError;
 export declare function isError(err: unknown): err is Error;
 //# sourceMappingURL=errors.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cspell/cspell-tools",
-  "version": "7.0.1-alpha.5",
+  "version": "7.0.1-alpha.7",
   "description": "Tools to assist with the development of cSpell",
   "publishConfig": {
     "access": "public"
@@ -50,13 +50,13 @@
   },
   "homepage": "https://github.com/streetsidesoftware/cspell#readme",
   "dependencies": {
-    "@cspell/cspell-pipe": "7.0.1-alpha.5",
+    "@cspell/cspell-pipe": "7.0.1-alpha.7",
     "commander": "^11.0.0",
     "cosmiconfig": "8.0.0",
-    "cspell-trie-lib": "7.0.1-alpha.5",
+    "cspell-trie-lib": "7.0.1-alpha.7",
     "gensequence": "^5.0.2",
     "glob": "^10.3.3",
-    "hunspell-reader": "7.0.1-alpha.5",
+    "hunspell-reader": "7.0.1-alpha.7",
     "yaml": "^2.3.1"
   },
   "engines": {
@@ -72,5 +72,5 @@
     "ts-json-schema-generator": "^1.2.0"
   },
   "main": "bin.js",
-  "gitHead": "5de6ec67ac1f77c2c994cb956317598a9034e923"
+  "gitHead": "719c2f9f55c79d35074f8493a37de6f40077325d"
 }