@cspell/cspell-tools 7.0.1-alpha.6 → 7.0.1-alpha.8

package/cspell-tools.config.schema.json

@@ -209,6 +209,13 @@
       ],
       "description": "Words in the `allowedSplitWords` are considered correct and can be used as a basis for splitting compound words.\n\nIf entries can be split so that all the words in the entry are allowed, then only the individual words are added, otherwise the entire entry is added. This is to prevent misspellings in CamelCase words from being introduced into the dictionary."
     },
+    "checksumFile": {
+      "description": "Path to checksum file. `true` - defaults to `./checksum.txt`.",
+      "type": [
+        "string",
+        "boolean"
+      ]
+    },
     "generateNonStrict": {
       "default": true,
       "description": "Generate lower case / accent free versions of words.",

package/dist/app.js

@@ -69,6 +69,7 @@ export async function run(program, argv, flags) {
         .command('build [targets...]')
         .description('Build the targets defined in the run configuration.')
         .option('-c, --config <path to run configuration>', 'Specify the run configuration file.')
+        .option('--conditional', 'Conditional build.')
         .option('-r, --root <directory>', 'Specify the run directory')
         .action(build);
     program.command('gzip <files...>').description('GZip files while keeping the original.').action(handleGzip);

package/dist/build.d.ts

@@ -5,6 +5,8 @@ export interface BuildOptions {
     root?: string | undefined;
     /** Current working directory */
     cwd?: string | undefined;
+    /** Conditional build based upon the targets matching the `checksum.txt` file. */
+    conditional?: boolean;
 }
 export declare function build(targets: string[] | undefined, options: BuildOptions): Promise<void>;
 //# sourceMappingURL=build.d.ts.map

package/dist/build.js

@@ -25,11 +25,12 @@ export async function build(targets, options) {
         console.error('root: %s', options.root);
         throw 'cspell-tools.config not found.';
     }
-    const buildInfo = normalizeRequest(config.config, options.root || path.dirname(config.filepath));
-    await compile(buildInfo, { filter, cwd: options.cwd });
+    const configDir = path.dirname(config.filepath);
+    const buildInfo = normalizeRequest(config.config, path.resolve(configDir, options.root || configDir));
+    await compile(buildInfo, { filter, cwd: options.cwd, conditionalBuild: options.conditional || false });
 }
 function normalizeRequest(buildInfo, root) {
-    const { rootDir = root, targets = [] } = buildInfo;
-    return { rootDir, targets };
+    const { rootDir = root, targets = [], checksumFile } = buildInfo;
+    return { rootDir: path.resolve(rootDir), targets, checksumFile };
 }
 //# sourceMappingURL=build.js.map

package/dist/compile.d.ts

@@ -1,4 +1,5 @@
 import type { CompileCommonAppOptions } from './AppOptions.js';
 import type { FeatureFlags } from './FeatureFlags/index.js';
+export declare const configFileHeader = "# yaml-language-server: $schema=https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-tools/cspell-tools.config.schema.json\n\n";
 export declare function processCompileAction(src: string[], options: CompileCommonAppOptions, featureFlags: FeatureFlags | undefined): Promise<void>;
 //# sourceMappingURL=compile.d.ts.map

package/dist/compile.js

@@ -7,6 +7,7 @@ import { getSystemFeatureFlags, parseFlags } from './FeatureFlags/index.js';
 import { globP } from './util/globP.js';
 getSystemFeatureFlags().register('compound', 'Enable compound dictionary sources.');
 const defaultConfigFile = 'cspell-tools.config.yaml';
+export const configFileHeader = '# yaml-language-server: $schema=https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-tools/cspell-tools.config.schema.json\n\n';
 export async function processCompileAction(src, options, featureFlags) {
     const ff = featureFlags || getSystemFeatureFlags();
     parseFlags(ff, options.experimental || []);
@@ -26,7 +27,7 @@ async function useCompile(src, options) {
     return options.init ? initConfig(request) : compile(request);
 }
 async function initConfig(request) {
-    const content = YAML.stringify(request, null, 2);
+    const content = configFileHeader + YAML.stringify(request, null, 2);
     console.log('Writing config file: %s', defaultConfigFile);
     await writeFile(defaultConfigFile, content);
     console.log(`Init complete.

package/dist/compiler/compile.d.ts

@@ -1,4 +1,4 @@
-import type { CompileRequest, CompileSourceOptions, Target } from '../config/index.js';
+import type { CompileRequest, CompileSourceOptions as CompileSourceConfig, Target } from '../config/index.js';
 interface CompileOptions {
     /**
      * Optional filter function to filter targets.
@@ -8,8 +8,18 @@ interface CompileOptions {
      * The current working directory. Defaults to process.cwd()
      */
     cwd?: string;
+    /**
+     * `true` - only build if files do not match checksum.
+     */
+    conditionalBuild: boolean;
 }
 export declare function compile(request: CompileRequest, options?: CompileOptions): Promise<void>;
-export declare function compileTarget(target: Target, options: CompileSourceOptions, rootDir: string, cwd?: string): Promise<void>;
+interface CompileTargetOptions {
+    rootDir: string;
+    cwd: string | undefined;
+    conditional: boolean;
+    checksumFile: string | undefined;
+}
+export declare function compileTarget(target: Target, options: CompileSourceConfig, compileOptions: CompileTargetOptions): Promise<string[]>;
 export {};
 //# sourceMappingURL=compile.d.ts.map

package/dist/compiler/compile.js

@@ -3,6 +3,7 @@ import { opAwaitAsync, opMapAsync } from '@cspell/cspell-pipe/operators';
 import { opConcatMap, opMap, pipe } from '@cspell/cspell-pipe/sync';
 import * as path from 'path';
 import { isFileListSource, isFilePath, isFileSource } from '../config/index.js';
+import { checkShasumFile, updateChecksumForFiles } from '../shasum/index.js';
 import { createAllowedSplitWordsFromFiles } from './createWordsCollection.js';
 import { logWithTimestamp } from './logWithTimestamp.js';
 import { readTextFile } from './readers/readTextFile.js';
@@ -18,17 +19,33 @@ export async function compile(request, options) {
         sort: request.sort,
         generateNonStrict: request.generateNonStrict,
     };
+    const conditional = options?.conditionalBuild || false;
+    const checksumFile = resolveChecksumFile(request.checksumFile || conditional, rootDir);
+    const dependencies = new Set();
     for (const target of targets) {
         const keep = options?.filter?.(target) ?? true;
         if (!keep)
             continue;
         const adjustedTarget = { ...targetOptions, ...target };
-        await compileTarget(adjustedTarget, request, rootDir, cwd);
+        const deps = await compileTarget(adjustedTarget, request, { rootDir, cwd, conditional, checksumFile });
+        deps.forEach((dep) => dependencies.add(dep));
+    }
+    if (checksumFile && dependencies.size) {
+        logWithTimestamp('%s', `Update checksum: ${checksumFile}`);
+        await updateChecksumForFiles(checksumFile, [...dependencies], { root: path.dirname(checksumFile) });
     }
     logWithTimestamp(`Complete.`);
+    return;
+}
+function resolveChecksumFile(checksumFile, root) {
+    const cFilename = (typeof checksumFile === 'string' && checksumFile) || (checksumFile && './checksum.txt') || undefined;
+    const file = cFilename && path.resolve(root, cFilename);
+    // console.warn('%o', { checksumFile, cFilename, file });
+    return file;
 }
-export async function compileTarget(target, options, rootDir, cwd) {
+export async function compileTarget(target, options, compileOptions) {
     logWithTimestamp(`Start compile: ${target.name}`);
+    const { rootDir, cwd, checksumFile, conditional } = compileOptions;
     const { format, sources, trieBase, sort = true, generateNonStrict = false } = target;
     const targetDirectory = path.resolve(rootDir, target.targetDirectory ?? cwd ?? process.cwd());
     const generateNonStrictTrie = target.generateNonStrict ?? true;
@@ -38,6 +55,15 @@ export async function compileTarget(target, options, rootDir, cwd) {
     const filesToProcessAsync = pipeAsync(readSourceList(sources, rootDir), opMapAsync((src) => readFileSource(src, options)), opAwaitAsync());
     const filesToProcess = await toArray(filesToProcessAsync);
     const normalizer = normalizeTargetWords({ sort: useTrie || sort, generateNonStrict });
+    const checksumRoot = (checksumFile && path.dirname(checksumFile)) || rootDir;
+    const deps = [...calculateDependencies(filename, filesToProcess, checksumRoot)];
+    if (conditional && checksumFile) {
+        const check = await checkShasumFile(checksumFile, deps, checksumRoot).catch(() => undefined);
+        if (check?.passed) {
+            logWithTimestamp(`Skip ${target.name}, nothing changed.`);
+            return [];
+        }
+    }
     const action = useTrie
         ? async (words, dst) => {
             return compileTrie(pipe(words, normalizer), dst, {
@@ -53,6 +79,19 @@ export async function compileTarget(target, options, rootDir, cwd) {
         };
     await processFiles(action, filesToProcess, filename);
     logWithTimestamp(`Done compile: ${target.name}`);
+    return deps;
+}
+function calculateDependencies(targetFile, filesToProcess, rootDir) {
+    const dependencies = new Set();
+    addDependency(targetFile);
+    filesToProcess.forEach((f) => addDependency(f.src));
+    return dependencies;
+    function addDependency(filename) {
+        const rel = path.relative(rootDir, filename);
+        dependencies.add(rel);
+        dependencies.add(rel.replace(/\.aff$/, '.dic'));
+        dependencies.add(rel.replace(/\.dic$/, '.aff'));
+    }
 }
 function rel(filePath) {
     return path.relative(process.cwd(), filePath);
@@ -67,9 +106,9 @@ async function processFiles(action, filesToProcess, mergeTarget) {
     }), opConcatMap(function* (ftp) {
         yield* ftp.words;
         logWithTimestamp('Done processing %s', rel(ftp.src));
-    })
+    }), 
     // opMap((a) => (console.warn(a), a))
-    );
+    logProgress());
     await action(words, dst);
     logWithTimestamp('Done "%s"', rel(dst));
 }
@@ -134,4 +173,18 @@ async function readFileSource(fileSource, sourceOptions) {
 function normalizeTargetName(name) {
     return name.replace(/((\.txt|\.dic|\.aff|\.trie)(\.gz)?)?$/, '').replace(/[^\p{L}\p{M}.\w\\/-]/gu, '_');
 }
+function logProgress(freq = 100000) {
+    function* logProgress(iter) {
+        const _freq = freq;
+        let count = 0;
+        for (const v of iter) {
+            ++count;
+            if (!(count % _freq)) {
+                logWithTimestamp('Progress: Words Processed - %s', count.toLocaleString());
+            }
+            yield v;
+        }
+    }
+    return logProgress;
+}
 //# sourceMappingURL=compile.js.map

package/dist/compiler/createCompileRequest.js

@@ -1,13 +1,13 @@
 import * as path from 'path';
 export function createCompileRequest(sourceFiles, options) {
-    const { max_depth, maxDepth, experimental = [], split, keepRawCase, useLegacySplitter } = options;
+    options = { ...options };
+    options.maxDepth ??= options.max_depth;
+    const { maxDepth, split, keepRawCase, useLegacySplitter } = options;
     const sources = [...sourceFiles, ...(options.listFile || []).map((listFile) => ({ listFile }))];
     const targets = calcTargets(sources, options);
-    const generateNonStrict = experimental.includes('compound') || undefined;
     const req = {
         targets,
-        generateNonStrict,
-        maxDepth: parseNumber(maxDepth) ?? parseNumber(max_depth),
+        maxDepth: parseNumber(maxDepth),
         split: useLegacySplitter ? 'legacy' : split,
         /**
          * Do not generate lower case / accent free versions of words.
@@ -18,17 +18,21 @@ export function createCompileRequest(sourceFiles, options) {
     return req;
 }
 function calcTargets(sources, options) {
-    const { merge, output = '.' } = options;
+    const { merge, output = '.', experimental = [] } = options;
+    const generateNonStrict = experimental.includes('compound') || undefined;
+    // console.log('%o', sources);
     const format = calcFormat(options);
+    const sort = (format === 'plaintext' && options.sort) || undefined;
     if (merge) {
         const target = {
             name: merge,
             targetDirectory: output,
             compress: options.compress,
             format,
-            sources,
-            sort: options.sort,
+            sources: sources.map(normalizeSource),
+            sort,
             trieBase: parseNumber(options.trieBase),
+            generateNonStrict,
         };
         return [target];
     }
@@ -39,9 +43,10 @@ function calcTargets(sources, options) {
             targetDirectory: output,
             compress: options.compress,
             format,
-            sources: [source],
+            sources: [normalizeSource(source)],
             sort: options.sort,
             trieBase: parseNumber(options.trieBase),
+            generateNonStrict,
         };
         return target;
     });
@@ -63,4 +68,17 @@ function baseNameOfSource(source) {
 function isFileSource(source) {
     return typeof source !== 'string' && source.filename !== undefined;
 }
+function normalizeSource(source) {
+    if (typeof source === 'string') {
+        return normalizeSourcePath(source);
+    }
+    if (isFileSource(source))
+        return { ...source, filename: normalizeSourcePath(source.filename) };
+    return { ...source, listFile: normalizeSourcePath(source.listFile) };
+}
+function normalizeSourcePath(source) {
+    const cwd = process.cwd();
+    const rel = path.relative(cwd, source);
+    return rel.split('\\').join('/');
+}
 //# sourceMappingURL=createCompileRequest.js.map

package/dist/config/config.d.ts

@@ -20,6 +20,10 @@ export interface CompileRequest extends CompileTargetOptions, CompileSourceOptio
      * Target Dictionaries to create.
      */
     targets: Target[];
+    /**
+     * Path to checksum file. `true` - defaults to `./checksum.txt`.
+     */
+    checksumFile?: string | boolean;
 }
 export interface Experimental {
     /**

package/dist/shasum/index.d.ts

@@ -1,3 +1,3 @@
 export { calcFileChecksum } from './checksum.js';
-export { checkShasumFile } from './shasum.js';
+export { checkShasumFile, updateChecksumForFiles } from './shasum.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/shasum/index.js

@@ -1,3 +1,3 @@
 export { calcFileChecksum } from './checksum.js';
-export { checkShasumFile } from './shasum.js';
+export { checkShasumFile, updateChecksumForFiles } from './shasum.js';
 //# sourceMappingURL=index.js.map

package/dist/shasum/shasum.d.ts

@@ -1,4 +1,8 @@
 export interface CheckShasumFileResult {
+    passed: boolean;
+    results: CheckFileResult[];
+}
+export interface CheckFileResult {
     filename: string;
     passed: boolean;
     error?: Error;
@@ -10,7 +14,7 @@ export declare function shasumFile(filename: string, root: string | undefined):
  * @param files - optional list of files to check
  * @param root - optional root, default cwd.
  */
-export declare function checkShasumFile(filename: string, files: string[] | undefined, root?: string): Promise<CheckShasumFileResult[]>;
+export declare function checkShasumFile(filename: string, files: string[] | undefined, root?: string): Promise<CheckShasumFileResult>;
 export interface ChecksumEntry {
     filename: string;
     checksum: string;

package/dist/shasum/shasum.js

@@ -1,5 +1,5 @@
 import { readFile, writeFile } from 'node:fs/promises';
-import { resolve } from 'node:path';
+import { resolve, sep as pathSep } from 'node:path';
 import { toError } from '../util/errors.js';
 import { isDefined } from '../util/index.js';
 import { calcFileChecksum, checkFile } from './checksum.js';
@@ -24,12 +24,13 @@ export async function checkShasumFile(filename, files, root) {
     files = !files ? files : files.length ? files : undefined;
     const shaFiles = await readAndParseShasumFile(filename);
     const filesToCheck = !files ? shaFiles.map(({ filename }) => filename) : files;
-    const mapNameToChecksum = new Map(shaFiles.map((r) => [r.filename, r.checksum]));
+    const mapNameToChecksum = new Map(shaFiles.map((r) => [normalizeFilename(r.filename), r.checksum]));
     const resolvedRoot = resolve(root || '.');
-    const results = await Promise.all(filesToCheck.map((filename) => {
+    const results = await Promise.all(filesToCheck.map(normalizeFilename).map((filename) => {
         return tryToCheckFile(filename, resolvedRoot, mapNameToChecksum.get(filename));
     }));
-    return results;
+    const passed = !results.find((v) => !v.passed);
+    return { passed, results };
 }
 async function tryToCheckFile(filename, root, checksum) {
     if (!checksum) {
@@ -84,9 +85,10 @@ export async function reportChecksumForFiles(files, options) {
 export async function reportCheckChecksumFile(filename, files, options) {
     const root = options.root;
     const filesToCheck = await resolveFileList(files, options.listFile);
-    const result = await checkShasumFile(filename, filesToCheck, root);
-    const lines = result.map(({ filename, passed, error }) => `${filename}: ${passed ? 'OK' : 'FAILED'} ${error ? '- ' + error.message : ''}`.trim());
-    const withErrors = result.filter((a) => !a.passed);
+    const checkResult = await checkShasumFile(filename, filesToCheck, root);
+    const results = checkResult.results;
+    const lines = results.map(({ filename, passed, error }) => `${filename}: ${passed ? 'OK' : 'FAILED'} ${error ? '- ' + error.message : ''}`.trim());
+    const withErrors = results.filter((a) => !a.passed);
     const passed = !withErrors.length;
     if (!passed) {
         lines.push(`shasum: WARNING: ${withErrors.length} computed checksum${withErrors.length > 1 ? 's' : ''} did NOT match`);
@@ -105,17 +107,17 @@ async function resolveFileList(files, listFile) {
             .filter((a) => a)
             .forEach((file) => setOfFiles.add(file));
     }
-    return [...setOfFiles];
+    return [...setOfFiles].map(normalizeFilename);
 }
 export async function calcUpdateChecksumForFiles(filename, files, options) {
     const root = options.root || '.';
     const filesToCheck = await resolveFileList(files, options.listFile);
-    const currentEntries = await readAndParseShasumFile(filename).catch((err) => {
+    const currentEntries = (await readAndParseShasumFile(filename).catch((err) => {
         const e = toError(err);
         if (e.code !== 'ENOENT')
             throw e;
         return [];
-    });
+    })).map((entry) => ({ ...entry, filename: normalizeFilename(entry.filename) }));
     const entriesToUpdate = new Set([...filesToCheck, ...currentEntries.map((e) => e.filename)]);
     const mustExist = new Set(filesToCheck);
     const checksumMap = new Map(currentEntries.map(({ filename, checksum }) => [filename, checksum]));
@@ -140,4 +142,7 @@ export async function updateChecksumForFiles(filename, files, options) {
     await writeFile(filename, content);
     return { passed: true, report: content };
 }
+function normalizeFilename(filename) {
+    return filename.split(pathSep).join('/');
+}
 //# sourceMappingURL=shasum.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cspell/cspell-tools",
-  "version": "7.0.1-alpha.6",
+  "version": "7.0.1-alpha.8",
   "description": "Tools to assist with the development of cSpell",
   "publishConfig": {
     "access": "public"
@@ -16,12 +16,11 @@
     "watch": "tsc -p . -w",
     "clean-build": "pnpm run clean && pnpm run build",
     "clean": "shx rm -rf dist temp coverage \"*.tsbuildInfo\"",
-    "coverage": "pnpm coverage:vitest && pnpm coverage:fix",
+    "coverage": "pnpm coverage:vitest",
     "coverage:vitest": "vitest run --coverage",
-    "coverage:fix": "nyc report --temp-dir \"$(pwd)/coverage\" --reporter lcov --report-dir \"$(pwd)/coverage\" --cwd ../..",
-    "test-watch": "jest --watch",
+    "test:watch": "vitest",
     "test": "vitest run",
-    "update-snapshot": "jest --updateSnapshot"
+    "update-snapshot": "vitest run -u"
   },
   "repository": {
     "type": "git",
@@ -50,13 +49,13 @@
   },
   "homepage": "https://github.com/streetsidesoftware/cspell#readme",
   "dependencies": {
-    "@cspell/cspell-pipe": "7.0.1-alpha.6",
+    "@cspell/cspell-pipe": "7.0.1-alpha.8",
     "commander": "^11.0.0",
     "cosmiconfig": "8.0.0",
-    "cspell-trie-lib": "7.0.1-alpha.6",
+    "cspell-trie-lib": "7.0.1-alpha.8",
     "gensequence": "^5.0.2",
     "glob": "^10.3.3",
-    "hunspell-reader": "7.0.1-alpha.6",
+    "hunspell-reader": "7.0.1-alpha.8",
     "yaml": "^2.3.1"
   },
   "engines": {
@@ -72,5 +71,5 @@
     "ts-json-schema-generator": "^1.2.0"
   },
   "main": "bin.js",
-  "gitHead": "42a31c7216a9af068ae4b7c3921f8edd327f756f"
+  "gitHead": "2c2d4db95da9d1cb7868d5de68e7c43776de06e7"
 }

package/dist/compiler/inlineSettings.d.ts

@@ -1,7 +0,0 @@
-export interface InlineSettings {
-    split?: boolean;
-    keepRawCase?: boolean;
-    caseSensitive?: boolean;
-}
-export declare function extractInlineSettings(line: string): InlineSettings | undefined;
-//# sourceMappingURL=inlineSettings.d.ts.map

package/dist/compiler/inlineSettings.js

@@ -1,35 +0,0 @@
-const allowedSettings = ['split', 'no-split', 'keep-case', 'no-keep-case', 'case-sensitive'];
-export function extractInlineSettings(line) {
-    const m = line.match(/cspell-tools:(.*)/);
-    if (!m)
-        return undefined;
-    const flags = m[1].split(/\s+/g).filter((a) => !!a);
-    const settings = {};
-    for (const flag of flags) {
-        switch (flag) {
-            case 'split':
-                settings.split = true;
-                break;
-            case 'no-split':
-                settings.split = false;
-                break;
-            case 'keep-case':
-                settings.keepRawCase = true;
-                break;
-            case 'no-keep-case':
-                settings.keepRawCase = false;
-                break;
-            case 'case-sensitive':
-                settings.caseSensitive = true;
-                break;
-            case 'flag':
-            case 'flags':
-                // Ignore flags
-                break;
-            default:
-                throw new Error(`Unknown inline setting: "${flag}" allowed values are ${allowedSettings.join(', ')}`);
-        }
-    }
-    return settings;
-}
-//# sourceMappingURL=inlineSettings.js.map