@cspell/cspell-tools 6.12.0 → 6.13.1

package/cspell-tools.config.schema.json

@@ -0,0 +1,203 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "additionalProperties": false,
+  "definitions": {
+    "DictionaryFormats": {
+      "enum": [
+        "plaintext",
+        "trie",
+        "trie3",
+        "trie4"
+      ],
+      "type": "string"
+    },
+    "DictionarySource": {
+      "anyOf": [
+        {
+          "$ref": "#/definitions/FilePath"
+        },
+        {
+          "$ref": "#/definitions/FileSource"
+        },
+        {
+          "$ref": "#/definitions/FileListSource"
+        }
+      ]
+    },
+    "FileListSource": {
+      "additionalProperties": false,
+      "properties": {
+        "keepRawCase": {
+          "default": false,
+          "description": "Do not generate lower case / accent free versions of words.",
+          "type": "boolean"
+        },
+        "listFile": {
+          "$ref": "#/definitions/FilePath"
+        },
+        "maxDepth": {
+          "description": "Maximum number of nested Hunspell Rules to apply. This is needed for recursive dictionaries like Hebrew.",
+          "type": "number"
+        },
+        "split": {
+          "anyOf": [
+            {
+              "type": "boolean"
+            },
+            {
+              "const": "legacy",
+              "type": "string"
+            }
+          ],
+          "default": false,
+          "description": "Split lines into words."
+        }
+      },
+      "required": [
+        "listFile"
+      ],
+      "type": "object"
+    },
+    "FilePath": {
+      "description": "Note: All relative paths are relative to the config file location.",
+      "type": "string"
+    },
+    "FileSource": {
+      "additionalProperties": false,
+      "properties": {
+        "filename": {
+          "$ref": "#/definitions/FilePath"
+        },
+        "keepRawCase": {
+          "default": false,
+          "description": "Do not generate lower case / accent free versions of words.",
+          "type": "boolean"
+        },
+        "maxDepth": {
+          "description": "Maximum number of nested Hunspell Rules to apply. This is needed for recursive dictionaries like Hebrew.",
+          "type": "number"
+        },
+        "split": {
+          "anyOf": [
+            {
+              "type": "boolean"
+            },
+            {
+              "const": "legacy",
+              "type": "string"
+            }
+          ],
+          "default": false,
+          "description": "Split lines into words."
+        }
+      },
+      "required": [
+        "filename"
+      ],
+      "type": "object"
+    },
+    "Target": {
+      "additionalProperties": false,
+      "properties": {
+        "compress": {
+          "default": ": false",
+          "description": "gzip the file?",
+          "type": "boolean"
+        },
+        "excludeWordsFrom": {
+          "description": "Words from the sources that are found in `excludeWordsFrom` files will not be added to the dictionary.",
+          "items": {
+            "$ref": "#/definitions/FilePath"
+          },
+          "type": "array"
+        },
+        "format": {
+          "$ref": "#/definitions/DictionaryFormats",
+          "description": "Format of the dictionary."
+        },
+        "generateNonStrict": {
+          "default": true,
+          "description": "Generate lower case / accent free versions of words.",
+          "type": "boolean"
+        },
+        "name": {
+          "description": "Name of target, used as the basis of target file name.",
+          "type": "string"
+        },
+        "sort": {
+          "default": ": true",
+          "description": "Sort the words in the resulting dictionary. Does not apply to `trie` based formats.",
+          "type": "boolean"
+        },
+        "sources": {
+          "description": "File sources used to build the dictionary.",
+          "items": {
+            "$ref": "#/definitions/DictionarySource"
+          },
+          "type": "array"
+        },
+        "targetDirectory": {
+          "$ref": "#/definitions/FilePath",
+          "default": "current directory",
+          "description": "The target directory"
+        },
+        "trieBase": {
+          "description": "Advanced: Set the trie base number. A value between 10 and 36 Set numeric base to use. 10 is the easiest to read. 16 is common hex format. 36 is the most compact.",
+          "type": "number"
+        }
+      },
+      "required": [
+        "name",
+        "format",
+        "sources"
+      ],
+      "type": "object"
+    }
+  },
+  "properties": {
+    "generateNonStrict": {
+      "default": true,
+      "description": "Generate lower case / accent free versions of words.",
+      "type": "boolean"
+    },
+    "keepRawCase": {
+      "default": false,
+      "description": "Do not generate lower case / accent free versions of words.",
+      "type": "boolean"
+    },
+    "maxDepth": {
+      "description": "Maximum number of nested Hunspell Rules to apply. This is needed for recursive dictionaries like Hebrew.",
+      "type": "number"
+    },
+    "rootDir": {
+      "description": "Specify the directory where all relative paths will resolved against. By default, all relative paths are relative to the location of the config file.",
+      "type": "string"
+    },
+    "sort": {
+      "default": ": true",
+      "description": "Sort the words in the resulting dictionary. Does not apply to `trie` based formats.",
+      "type": "boolean"
+    },
+    "split": {
+      "anyOf": [
+        {
+          "type": "boolean"
+        },
+        {
+          "const": "legacy",
+          "type": "string"
+        }
+      ],
+      "default": false,
+      "description": "Split lines into words."
+    },
+    "targets": {
+      "description": "Optional Target Dictionaries to create.",
+      "items": {
+        "$ref": "#/definitions/Target"
+      },
+      "type": "array"
+    }
+  },
+  "type": "object"
+}

package/dist/AppOptions.d.ts

@@ -0,0 +1,28 @@
+export interface CompileCommonAppOptions {
+    output?: string;
+    compress: boolean;
+    /**
+     * @deprecated Use maxDepth
+     */
+    max_depth?: string;
+    maxDepth?: string;
+    merge?: string;
+    experimental: string[];
+    split?: boolean;
+    sort?: boolean;
+    keepRawCase?: boolean;
+    trie?: boolean;
+    trie3?: boolean;
+    trie4?: boolean;
+    trieBase?: string;
+    useLegacySplitter?: boolean;
+}
+export interface CompileAppOptions extends CompileCommonAppOptions {
+    sort: boolean;
+    keepRawCase: boolean;
+}
+export interface CompileTrieAppOptions extends CompileCommonAppOptions {
+    trie3: boolean;
+    trie4: boolean;
+}
+//# sourceMappingURL=AppOptions.d.ts.map

package/dist/AppOptions.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=AppOptions.js.map

package/dist/FeatureFlags/FeatureFlags.d.ts

@@ -0,0 +1,34 @@
+export interface FeatureFlag {
+    name: string;
+    description: string;
+}
+declare type FlagTypes = string | boolean;
+/**
+ * Feature Flags are used to turn on/off features.
+ * These are primarily used before a feature has been fully released.
+ */
+export declare class FeatureFlags {
+    private flags;
+    private flagValues;
+    constructor(flags?: FeatureFlag[]);
+    register(flag: FeatureFlag): this;
+    register(name: string, description: string): this;
+    registerFeatures(flags: FeatureFlag[]): this;
+    getFlag(flag: string): FlagTypes | undefined;
+    getFlagBool(flag: string): boolean | undefined;
+    setFlag(flag: string, value?: FlagTypes): this;
+    getFlagInfo(flag: string): FeatureFlag | undefined;
+    getFlags(): FeatureFlag[];
+    getFlagValues(): Map<string, FlagTypes>;
+    reset(): this;
+    help(): string;
+    fork(): FeatureFlags;
+}
+export declare class UnknownFeatureFlagError extends Error {
+    readonly flag: string;
+    constructor(flag: string);
+}
+export declare function getSystemFeatureFlags(): FeatureFlags;
+export declare function createFeatureFlags(): FeatureFlags;
+export {};
+//# sourceMappingURL=FeatureFlags.d.ts.map

package/dist/FeatureFlags/FeatureFlags.js

@@ -0,0 +1,99 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createFeatureFlags = exports.getSystemFeatureFlags = exports.UnknownFeatureFlagError = exports.FeatureFlags = void 0;
+let systemFeatureFlags;
+/**
+ * Feature Flags are used to turn on/off features.
+ * These are primarily used before a feature has been fully released.
+ */
+class FeatureFlags {
+    constructor(flags = []) {
+        this.flagValues = new Map();
+        this.flags = new Map(flags.map((f) => [f.name, f]));
+    }
+    register(flagOrName, description) {
+        if (typeof flagOrName === 'string') {
+            return this.register({ name: flagOrName, description: description || '' });
+        }
+        this.flags.set(flagOrName.name, flagOrName);
+        return this;
+    }
+    registerFeatures(flags) {
+        flags.forEach((flag) => this.register(flag));
+        return this;
+    }
+    getFlag(flag) {
+        return this.flagValues.get(flag);
+    }
+    getFlagBool(flag) {
+        return toBool(this.getFlag(flag));
+    }
+    setFlag(flag, value = true) {
+        if (!this.flags.has(flag)) {
+            throw new UnknownFeatureFlagError(flag);
+        }
+        this.flagValues.set(flag, toBool(value) ?? value);
+        return this;
+    }
+    getFlagInfo(flag) {
+        return this.flags.get(flag);
+    }
+    getFlags() {
+        return [...this.flags.values()];
+    }
+    getFlagValues() {
+        return new Map(this.flagValues);
+    }
+    reset() {
+        this.flagValues.clear();
+        return this;
+    }
+    help() {
+        const flags = [{ name: 'Name', description: 'Description' }, ...this.flags.values()].sort((a, b) => a.name < b.name ? -1 : 1);
+        const nameColWidth = flags.map((f) => f.name.length).reduce((a, b) => Math.max(a, b), 0) + 1;
+        const entries = flags.map((f) => `- ${f.name}${' '.repeat(nameColWidth - f.name.length)} ${f.description}`);
+        const text = `Valid Flags:\n${entries.join('\n')}`;
+        return text;
+    }
+    fork() {
+        const fork = new FeatureFlags([...this.flags.values()]);
+        for (const [key, value] of this.flagValues) {
+            fork.flagValues.set(key, value);
+        }
+        return fork;
+    }
+}
+exports.FeatureFlags = FeatureFlags;
+class UnknownFeatureFlagError extends Error {
+    constructor(flag) {
+        super(`Unknown feature flag: ${flag}`);
+        this.flag = flag;
+    }
+}
+exports.UnknownFeatureFlagError = UnknownFeatureFlagError;
+function getSystemFeatureFlags() {
+    return systemFeatureFlags || (systemFeatureFlags = createFeatureFlags());
+}
+exports.getSystemFeatureFlags = getSystemFeatureFlags;
+function createFeatureFlags() {
+    return new FeatureFlags();
+}
+exports.createFeatureFlags = createFeatureFlags;
+const boolValues = {
+    0: false,
+    1: true,
+    f: false,
+    false: false,
+    n: false,
+    no: false,
+    t: true,
+    true: true,
+    y: true,
+    yes: true,
+};
+function toBool(value) {
+    if (typeof value !== 'string')
+        return value;
+    return boolValues[value.toLowerCase()];
+}
+//# sourceMappingURL=FeatureFlags.js.map

package/dist/FeatureFlags/index.d.ts

@@ -0,0 +1,3 @@
+export { FeatureFlag, FeatureFlags, getSystemFeatureFlags, UnknownFeatureFlagError } from './FeatureFlags';
+export { parseFlags } from './parseFlags';
+//# sourceMappingURL=index.d.ts.map

package/dist/FeatureFlags/index.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseFlags = exports.UnknownFeatureFlagError = exports.getSystemFeatureFlags = exports.FeatureFlags = void 0;
+var FeatureFlags_1 = require("./FeatureFlags");
+Object.defineProperty(exports, "FeatureFlags", { enumerable: true, get: function () { return FeatureFlags_1.FeatureFlags; } });
+Object.defineProperty(exports, "getSystemFeatureFlags", { enumerable: true, get: function () { return FeatureFlags_1.getSystemFeatureFlags; } });
+Object.defineProperty(exports, "UnknownFeatureFlagError", { enumerable: true, get: function () { return FeatureFlags_1.UnknownFeatureFlagError; } });
+var parseFlags_1 = require("./parseFlags");
+Object.defineProperty(exports, "parseFlags", { enumerable: true, get: function () { return parseFlags_1.parseFlags; } });
+//# sourceMappingURL=index.js.map

package/dist/FeatureFlags/parseFlags.d.ts

@@ -0,0 +1,3 @@
+import { FeatureFlags } from './FeatureFlags';
+export declare function parseFlags(ff: FeatureFlags, flags: string[]): FeatureFlags;
+//# sourceMappingURL=parseFlags.d.ts.map

package/dist/FeatureFlags/parseFlags.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseFlags = void 0;
+const FeatureFlags_1 = require("./FeatureFlags");
+const splitFlag = /[:=]/;
+const leadingEql = /^=/;
+function parseFlags(ff, flags) {
+    for (const flag of flags) {
+        const [name, value] = flag.replace(leadingEql, '').split(splitFlag, 2);
+        try {
+            ff.setFlag(name, value);
+        }
+        catch (e) {
+            if (e instanceof FeatureFlags_1.UnknownFeatureFlagError) {
+                console.error(e.message);
+                console.error(ff.help());
+            }
+            throw e;
+        }
+    }
+    return ff;
+}
+exports.parseFlags = parseFlags;
+//# sourceMappingURL=parseFlags.js.map

package/dist/app.d.ts

@@ -1,23 +1,4 @@
 import * as program from 'commander';
-interface CompileCommonOptions {
-    output?: string;
-    compress: boolean;
-    max_depth?: string;
-    merge?: string;
-    experimental: string[];
-    split?: boolean;
-    sort?: boolean;
-    keepRawCase?: boolean;
-    trie?: boolean;
-    trie3?: boolean;
-    trie4?: boolean;
-    trieBase?: string;
-    useLegacySplitter?: boolean;
-}
-export declare function run(program: program.Command, argv: string[]): Promise<void>;
-declare function processAction(src: string[], options: CompileCommonOptions): Promise<void>;
-export declare const __testing__: {
-    processAction: typeof processAction;
-};
-export {};
+import { FeatureFlags } from './FeatureFlags';
+export declare function run(program: program.Command, argv: string[], flags?: FeatureFlags): Promise<void>;
 //# sourceMappingURL=app.d.ts.map

package/dist/app.js

@@ -23,33 +23,16 @@ var __importStar = (this && this.__importStar) || function (mod) {
     __setModuleDefault(result, mod);
     return result;
 };
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.__testing__ = exports.run = void 0;
-const compiler_1 = require("./compiler");
-const compiler = __importStar(require("./compiler"));
+exports.run = void 0;
 const path = __importStar(require("path"));
-const glob_1 = __importDefault(require("glob"));
-const gensequence_1 = require("gensequence");
-const iterateWordsFromFile_1 = require("./compiler/iterateWordsFromFile");
+const compiler = __importStar(require("./compiler"));
+const logWithTimestamp_1 = require("./compiler/logWithTimestamp");
+const compile_1 = require("./compile");
+const build_1 = require("./build");
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const npmPackage = require(path.join(__dirname, '..', 'package.json'));
-function globP(pattern) {
-    // Convert windows separators.
-    pattern = pattern.replace(/\\/g, '/');
-    return new Promise((resolve, reject) => {
-        (0, glob_1.default)(pattern, (err, result) => {
-            err ? reject(err) : resolve(result);
-        });
-    });
-}
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-const log = (message, ...optionalParams) => {
-    console.log(`${new Date().toISOString()} ${message}`, ...optionalParams);
-};
-compiler.setLogger(log);
+compiler.setLogger(logWithTimestamp_1.logWithTimestamp);
 function collect(value, previous) {
     return previous.concat([value]);
 }
@@ -68,129 +51,27 @@ function addCompileOptions(compileCommand) {
         .option('--trie4', 'Use file format trie4', false)
         .option('--trie-base <number>', 'Advanced: Set the trie base number. A value between 10 and 36');
 }
-function run(program, argv) {
+async function run(program, argv, flags) {
     program.exitOverride();
-    return new Promise((resolve, reject) => {
-        program.version(npmPackage.version);
-        addCompileOptions(program.command('compile <src...>').description('Compile words into a cspell dictionary files.'))
-            .option('--trie', 'Compile into a trie file.', false)
-            .option('--no-sort', 'Do not sort the result')
-            .action((src, options) => {
-            const result = processAction(src, options);
-            resolve(result);
-        });
-        addCompileOptions(program
-            .command('compile-trie <src...>')
-            .description('Compile words lists or Hunspell dictionary into trie files used by cspell.\nAlias of `compile --trie`')).action((src, options) => {
-            const result = processAction(src, { ...options, trie: true });
-            resolve(result);
-        });
-        try {
-            program.parse(argv);
-            if (!argv.slice(2).length) {
-                program.help();
-            }
-        }
-        catch (e) {
-            reject(e);
-        }
-        resolve();
-    });
-}
-exports.run = run;
-function parseNumber(s) {
-    const n = parseInt(s ?? '');
-    return isNaN(n) ? undefined : n;
-}
-async function processAction(src, options) {
-    const useTrie = options.trie || options.trie3 || options.trie4 || false;
-    const fileExt = useTrie ? '.trie' : '.txt';
-    console.log('Compile:\n output: %s\n compress: %s\n files:\n  %s \n\n', options.output || 'default', options.compress ? 'true' : 'false', src.join('\n  '));
-    const experimental = new Set(options.experimental);
-    const skipNormalization = experimental.has('compound');
-    const { keepRawCase = false, split: splitWords = false, sort = true, useLegacySplitter: legacy } = options;
-    const action = useTrie
-        ? async (words, dst) => {
-            return (0, compiler_1.compileTrie)(words, dst, {
-                ...options,
-                skipNormalization,
-                splitWords,
-                keepRawCase,
-                legacy,
-                base: parseNumber(options.trieBase),
-                sort: false,
-            });
-        }
-        : async (src, dst) => {
-            return (0, compiler_1.compileWordList)(src, dst, {
-                splitWords,
-                sort,
-                skipNormalization,
-                keepRawCase,
-                legacy,
-            }).then(() => src);
-        };
-    const ext = fileExt + (options.compress ? '.gz' : '');
-    const maxDepth = parseNumber(options.max_depth);
-    const useAnnotation = experimental.has('compound');
-    const readerOptions = { maxDepth, useAnnotation };
-    const globResults = await Promise.all(src.map((s) => globP(s)));
-    const filesToProcess = (0, gensequence_1.genSequence)(globResults)
-        .concatMap((files) => files)
-        .map(async (filename) => {
-        log(`Reading ${path.basename(filename)}`);
-        const words = await (0, iterateWordsFromFile_1.streamWordsFromFile)(filename, readerOptions);
-        log(`Done reading ${path.basename(filename)}`);
-        const f = {
-            src: filename,
-            words,
-        };
-        return f;
+    program.version(npmPackage.version);
+    addCompileOptions(program.command('compile <src...>').description('Compile words into a cspell dictionary files.'))
+        .option('--trie', 'Compile into a trie file.', false)
+        .option('--no-sort', 'Do not sort the result')
+        .action((src, options) => {
+        return (0, compile_1.processCompileAction)(src, options, flags);
     });
-    const r = options.merge
-        ? processFiles(action, filesToProcess, toMergeTargetFile(options.merge, options.output, ext))
-        : processFilesIndividually(action, filesToProcess, (s) => toTargetFile(s, options.output, ext));
-    await r;
-    log(`Complete.`);
-}
-function toFilename(name, ext) {
-    return path.basename(name).replace(/((\.txt|\.dic|\.aff|\.trie)(\.gz)?)?$/, '') + ext;
-}
-function toTargetFile(filename, destination, ext) {
-    const outFileName = toFilename(filename, ext);
-    const dir = destination ?? path.dirname(filename);
-    return path.join(dir, outFileName);
-}
-function toMergeTargetFile(filename, destination, ext) {
-    const outFileName = path.join(path.dirname(filename), toFilename(filename, ext));
-    return path.resolve(destination ?? '.', outFileName);
-}
-async function processFilesIndividually(action, filesToProcess, srcToTarget) {
-    const toProcess = filesToProcess.map(async (pFtp) => {
-        const { src, words } = await pFtp;
-        const dst = srcToTarget(src);
-        log('Process "%s" to "%s"', src, dst);
-        await action(words, dst);
-        log('Done "%s" to "%s"', src, dst);
+    addCompileOptions(program
+        .command('compile-trie <src...>')
+        .description('Compile words lists or Hunspell dictionary into trie files used by cspell.\nAlias of `compile --trie`')).action((src, options) => {
+        return (0, compile_1.processCompileAction)(src, { ...options, trie: true }, flags);
     });
-    for (const p of toProcess) {
-        await p;
-    }
-}
-async function processFiles(action, filesToProcess, mergeTarget) {
-    const toProcess = await Promise.all([...filesToProcess]);
-    const dst = mergeTarget;
-    const words = (0, gensequence_1.genSequence)(toProcess)
-        .map((ftp) => {
-        const { src } = ftp;
-        log('Process "%s" to "%s"', src, dst);
-        return ftp;
-    })
-        .concatMap((ftp) => ftp.words);
-    await action(words, dst);
-    log('Done "%s"', dst);
+    program
+        .command('build [targets]')
+        .description('Build the targets defined in the run configuration.')
+        .option('-c, --config <path to run configuration>', 'Specify the run configuration file.')
+        .option('-r, --root <directory>', 'Specify the run directory')
+        .action(build_1.build);
+    await program.parseAsync(argv);
 }
-exports.__testing__ = {
-    processAction,
-};
+exports.run = run;
 //# sourceMappingURL=app.js.map

package/dist/build.d.ts

@@ -0,0 +1,7 @@
+export interface BuildOptions {
+    /** Optional path to config file */
+    config?: string;
+    root?: string;
+}
+export declare function build(targets: string[] | undefined, options: BuildOptions): Promise<void>;
+//# sourceMappingURL=build.d.ts.map