@cspell/cspell-tools 9.4.0 → 9.6.0

package/bin.mjs

@@ -2,7 +2,7 @@
 
 import { CommanderError, program } from 'commander';
 
-import { run } from './dist/app.js';
+import { run } from './dist/app.mjs';
 
 run(program, process.argv).catch((e) => {
     if (!(e instanceof CommanderError)) {

package/cspell-tools.config.schema.json

@@ -2,6 +2,24 @@
   "$schema": "http://json-schema.org/draft-07/schema#",
   "additionalProperties": false,
   "definitions": {
+    "BTrieOptions": {
+      "additionalProperties": false,
+      "properties": {
+        "compress": {
+          "description": "compress the resulting file",
+          "type": "boolean"
+        },
+        "optimize": {
+          "description": "optimize the trie into a DAWG",
+          "type": "boolean"
+        },
+        "useStringTable": {
+          "description": "use a string table to reduce size",
+          "type": "boolean"
+        }
+      },
+      "type": "object"
+    },
     "DictionaryFormats": {
       "enum": [
         "plaintext",
@@ -28,17 +46,7 @@
       "additionalProperties": false,
       "properties": {
         "allowedSplitWords": {
-          "anyOf": [
-            {
-              "$ref": "#/definitions/FilePath"
-            },
-            {
-              "items": {
-                "$ref": "#/definitions/FilePath"
-              },
-              "type": "array"
-            }
-          ],
+          "$ref": "#/definitions/FilePathOrFilePathArray",
           "description": "Words in the `allowedSplitWords` are considered correct and can be used as a basis for splitting compound words.\n\nIf entries can be split so that all the words in the entry are allowed, then only the individual words are added, otherwise the entire entry is added. This is to prevent misspellings in CamelCase words from being introduced into the dictionary."
         },
         "keepRawCase": {
@@ -86,21 +94,24 @@
       "description": "Note: All relative paths are relative to the config file location.",
       "type": "string"
     },
+    "FilePathOrFilePathArray": {
+      "anyOf": [
+        {
+          "$ref": "#/definitions/FilePath"
+        },
+        {
+          "items": {
+            "$ref": "#/definitions/FilePath"
+          },
+          "type": "array"
+        }
+      ]
+    },
     "FileSource": {
       "additionalProperties": false,
       "properties": {
         "allowedSplitWords": {
-          "anyOf": [
-            {
-              "$ref": "#/definitions/FilePath"
-            },
-            {
-              "items": {
-                "$ref": "#/definitions/FilePath"
-              },
-              "type": "array"
-            }
-          ],
+          "$ref": "#/definitions/FilePathOrFilePathArray",
           "description": "Words in the `allowedSplitWords` are considered correct and can be used as a basis for splitting compound words.\n\nIf entries can be split so that all the words in the entry are allowed, then only the individual words are added, otherwise the entire entry is added. This is to prevent misspellings in CamelCase words from being introduced into the dictionary."
         },
         "filename": {
@@ -161,9 +172,20 @@
           ],
           "description": "Words in the `allowedSplitWords` are considered correct and can be used as a basis for splitting compound words.\n\nIf entries can be split so that all the words in the entry are allowed, then only the individual words are added, otherwise the entire entry is added. This is to prevent misspellings in CamelCase words from being introduced into the dictionary."
         },
+        "bTrie": {
+          "anyOf": [
+            {
+              "type": "boolean"
+            },
+            {
+              "$ref": "#/definitions/BTrieOptions"
+            }
+          ],
+          "description": "Generate a `.btrie.gz` for the target."
+        },
         "compress": {
-          "default": ": false",
-          "description": "gzip the file?",
+          "default": false,
+          "description": "Setting this value to true will create a `.gz` dictionary file. Use `keepUncompressed` to also keep an uncompressed version.",
           "type": "boolean"
         },
         "dictionaryDirectives": {
@@ -203,6 +225,10 @@
           "description": "Generate lower case / accent free versions of words.",
           "type": "boolean"
         },
+        "keepUncompressed": {
+          "description": "If `compress` is true, setting this value to true will also keep an uncompressed version of the dictionary.",
+          "type": "boolean"
+        },
         "name": {
           "description": "Name of target, used as the basis of target file name.",
           "type": "string"

package/dist/app.d.mts

@@ -0,0 +1,35 @@
+import { Command } from "commander";
+
+//#region src/FeatureFlags/FeatureFlags.d.ts
+interface FeatureFlag {
+  name: string;
+  description: string;
+}
+type FlagTypes = string | boolean;
+/**
+* Feature Flags are used to turn on/off features.
+* These are primarily used before a feature has been fully released.
+*/
+declare class FeatureFlags {
+  private flags;
+  private flagValues;
+  constructor(flags?: FeatureFlag[]);
+  register(flag: FeatureFlag): this;
+  register(name: string, description: string): this;
+  registerFeatures(flags: FeatureFlag[]): this;
+  getFlag(flag: string): FlagTypes | undefined;
+  getFlagBool(flag: string): boolean | undefined;
+  setFlag(flag: string, value?: FlagTypes): this;
+  getFlagInfo(flag: string): FeatureFlag | undefined;
+  getFlags(): FeatureFlag[];
+  getFlagValues(): Map<string, FlagTypes>;
+  reset(): this;
+  help(): string;
+  fork(): FeatureFlags;
+}
+//#endregion
+//#region src/app.d.ts
+declare function run(program: Command, argv: string[], flags?: FeatureFlags): Promise<void>;
+//#endregion
+export { run };
+//# sourceMappingURL=app.d.mts.map

package/dist/app.mjs

@@ -0,0 +1,292 @@
+import { a as reportCheckChecksumFile, c as toError, d as compressFile, i as logWithTimestamp, l as generateBTrie, n as compile, o as reportChecksumForFiles, r as setLogger, s as updateChecksumForFiles, t as build, u as OSFlags } from "./build-OgMPaXPZ.mjs";
+import { readFileSync } from "node:fs";
+import { CommanderError, Option } from "commander";
+import { writeFile } from "node:fs/promises";
+import * as path from "node:path";
+import { glob } from "glob";
+import { opConcatMap, pipe } from "@cspell/cspell-pipe/sync";
+import YAML from "yaml";
+
+//#region src/util/globP.ts
+function globP(pattern, options) {
+	return glob((Array.isArray(pattern) ? pattern : [pattern]).map((pattern$1) => pattern$1.replaceAll("\\", "/")), options);
+}
+
+//#endregion
+//#region src/gzip/gzip.ts
+/**
+* GZip files matching the given globs.
+* @param globs - array of globs to gzip
+* @param os - optional OS flag for the gzip file
+*/
+async function gzipFiles(globs, os) {
+	const files = await globP(globs, { nodir: true });
+	for (const fileName of files) await compressFile(fileName, os);
+}
+
+//#endregion
+//#region src/compiler/createCompileRequest.ts
+function createCompileRequest(sourceFiles, options) {
+	options = { ...options };
+	options.maxDepth ??= options.max_depth;
+	const { maxDepth, split, keepRawCase, useLegacySplitter } = options;
+	return {
+		targets: calcTargets([...sourceFiles, ...(options.listFile || []).map((listFile) => ({ listFile }))], options),
+		maxDepth: parseNumber(maxDepth),
+		split: useLegacySplitter ? "legacy" : split,
+		keepRawCase
+	};
+}
+function calcTargets(sources, options) {
+	const { merge, output = ".", experimental = [] } = options;
+	const generateNonStrict = experimental.includes("compound") || void 0;
+	const format = calcFormat(options);
+	const sort = format === "plaintext" && options.sort || void 0;
+	if (merge) return [{
+		name: merge,
+		targetDirectory: output,
+		compress: options.compress,
+		format,
+		sources: sources.map(normalizeSource),
+		sort,
+		trieBase: parseNumber(options.trieBase),
+		generateNonStrict
+	}];
+	return sources.map((source) => {
+		return {
+			name: toTargetName(baseNameOfSource(source)),
+			targetDirectory: output,
+			compress: options.compress,
+			format,
+			sources: [normalizeSource(source)],
+			sort: options.sort,
+			trieBase: parseNumber(options.trieBase),
+			generateNonStrict
+		};
+	});
+}
+function calcFormat(options) {
+	return options.trie4 && "trie4" || options.trie3 && "trie3" || options.trie && "trie" || "plaintext";
+}
+function toTargetName(sourceFile) {
+	return path.basename(sourceFile).replace(/((\.txt|\.dic|\.aff|\.trie)(\.gz)?)?$/, "");
+}
+function parseNumber(s) {
+	const n = Number.parseInt(s ?? "");
+	return Number.isNaN(n) ? void 0 : n;
+}
+function baseNameOfSource(source) {
+	return typeof source === "string" ? source : isFileSource(source) ? source.filename : source.listFile;
+}
+function isFileSource(source) {
+	return typeof source !== "string" && source.filename !== void 0;
+}
+function normalizeSource(source) {
+	if (typeof source === "string") return normalizeSourcePath(source);
+	if (isFileSource(source)) return {
+		...source,
+		filename: normalizeSourcePath(source.filename)
+	};
+	return {
+		...source,
+		listFile: normalizeSourcePath(source.listFile)
+	};
+}
+function normalizeSourcePath(source) {
+	const cwd = process.cwd();
+	return path.relative(cwd, source).split("\\").join("/");
+}
+
+//#endregion
+//#region src/config/config.ts
+const configFileSchemaURL = "https://raw.githubusercontent.com/streetsidesoftware/cspell/main/packages/cspell-tools/cspell-tools.config.schema.json";
+
+//#endregion
+//#region src/FeatureFlags/FeatureFlags.ts
+let systemFeatureFlags;
+/**
+* Feature Flags are used to turn on/off features.
+* These are primarily used before a feature has been fully released.
+*/
+var FeatureFlags = class FeatureFlags {
+	flags;
+	flagValues = /* @__PURE__ */ new Map();
+	constructor(flags = []) {
+		this.flags = new Map(flags.map((f) => [f.name, f]));
+	}
+	register(flagOrName, description) {
+		if (typeof flagOrName === "string") return this.register({
+			name: flagOrName,
+			description: description || ""
+		});
+		this.flags.set(flagOrName.name, flagOrName);
+		return this;
+	}
+	registerFeatures(flags) {
+		flags.forEach((flag) => this.register(flag));
+		return this;
+	}
+	getFlag(flag) {
+		return this.flagValues.get(flag);
+	}
+	getFlagBool(flag) {
+		return toBool(this.getFlag(flag));
+	}
+	setFlag(flag, value = true) {
+		if (!this.flags.has(flag)) throw new UnknownFeatureFlagError(flag);
+		this.flagValues.set(flag, toBool(value) ?? value);
+		return this;
+	}
+	getFlagInfo(flag) {
+		return this.flags.get(flag);
+	}
+	getFlags() {
+		return [...this.flags.values()];
+	}
+	getFlagValues() {
+		return new Map(this.flagValues);
+	}
+	reset() {
+		this.flagValues.clear();
+		return this;
+	}
+	help() {
+		const flags = [{
+			name: "Name",
+			description: "Description"
+		}, ...this.flags.values()].sort((a, b) => a.name < b.name ? -1 : 1);
+		const nameColWidth = flags.map((f) => f.name.length).reduce((a, b) => Math.max(a, b), 0) + 1;
+		return `Valid Flags:\n${flags.map((f) => `- ${f.name}${" ".repeat(nameColWidth - f.name.length)} ${f.description}`).join("\n")}`;
+	}
+	fork() {
+		const fork = new FeatureFlags([...this.flags.values()]);
+		for (const [key, value] of this.flagValues) fork.flagValues.set(key, value);
+		return fork;
+	}
+};
+var UnknownFeatureFlagError = class extends Error {
+	constructor(flag) {
+		super(`Unknown feature flag: ${flag}`);
+		this.flag = flag;
+	}
+};
+function getSystemFeatureFlags() {
+	return systemFeatureFlags || (systemFeatureFlags = createFeatureFlags());
+}
+function createFeatureFlags() {
+	return new FeatureFlags();
+}
+const boolValues = {
+	0: false,
+	1: true,
+	f: false,
+	false: false,
+	n: false,
+	no: false,
+	t: true,
+	true: true,
+	y: true,
+	yes: true
+};
+function toBool(value) {
+	if (typeof value !== "string") return value;
+	return boolValues[value.toLowerCase()];
+}
+
+//#endregion
+//#region src/FeatureFlags/parseFlags.ts
+const splitFlag = /[:=]/;
+const leadingEql = /^=/;
+function parseFlags(ff, flags) {
+	for (const flag of flags) {
+		const [name, value] = flag.replace(leadingEql, "").split(splitFlag, 2);
+		try {
+			ff.setFlag(name, value);
+		} catch (e) {
+			if (e instanceof UnknownFeatureFlagError) {
+				console.error(e.message);
+				console.error(ff.help());
+			}
+			throw e;
+		}
+	}
+	return ff;
+}
+
+//#endregion
+//#region src/compile.ts
+getSystemFeatureFlags().register("compound", "Enable compound dictionary sources.");
+const defaultConfigFile = "cspell-tools.config.yaml";
+const configFileHeader = `# yaml-language-server: $schema=${configFileSchemaURL}\n\n`;
+async function processCompileAction(src, options, featureFlags) {
+	parseFlags(featureFlags || getSystemFeatureFlags(), options.experimental || []);
+	return useCompile(src, options);
+}
+async function useCompile(src, options) {
+	console.log("Compile:\n output: %s\n compress: %s\n files:\n  %s", options.output || "default", options.compress ? "true" : "false", src.join("\n  "));
+	if (options.listFile && options.listFile.length) console.log(" list files:\n  %s", options.listFile.join("\n  "));
+	console.log("\n\n");
+	const request = createCompileRequest([...pipe(await Promise.all(src.map((s) => globP(s))), opConcatMap((a) => a))], options);
+	return options.init ? initConfig(request) : compile(request);
+}
+async function initConfig(runConfig) {
+	const { $schema = configFileSchemaURL, ...cfg } = runConfig;
+	const config = {
+		$schema,
+		...cfg
+	};
+	const content = configFileHeader + YAML.stringify(config, void 0, 2);
+	console.log("Writing config file: %s", defaultConfigFile);
+	await writeFile(defaultConfigFile, content);
+	console.log(`Init complete.
+To build, use:
+  cspell-tools-cli build
+`);
+}
+
+//#endregion
+//#region src/app.ts
+const npmPackageRaw = readFileSync(new URL("../package.json", import.meta.url), "utf8");
+const npmPackage = JSON.parse(npmPackageRaw);
+setLogger(logWithTimestamp);
+function collect(value, previous) {
+	return [...previous, value];
+}
+function addCompileOptions(compileCommand) {
+	return compileCommand.option("-o, --output <path>", "Specify the output directory, otherwise files are written back to the same location.").option("-n, --no-compress", "By default the files are GZipped, this will turn off GZ compression.").option("-m, --max_depth <limit>", "Maximum depth to apply suffix rules.").option("-M, --merge <target>", "Merge all files into a single target file (extensions are applied)").option("--split", "Split each line", void 0).option("--no-split", "Treat each line as a dictionary entry, do not split").option("--use-legacy-splitter", "Do not use legacy line splitter logic.").option("--keep-raw-case", "Do not normalize words before adding them to dictionary.").option("-x, --experimental <flag>", "Experimental flags, used for testing new concepts. Flags: compound", collect, []).option("--trie3", "Use file format trie3", false).option("--trie4", "Use file format trie4", false).option("--trie-base <number>", "Advanced: Set the trie base number. A value between 10 and 36").option("--list-file <filename...>", "Path to a file that contains the list of files to compile. A list file contains one file per line.").option("--init", "Create a build command `cspell-tools.config.yaml` file based upon the options given instead of building.");
+}
+async function run(program, argv, flags) {
+	async function handleGzip(files) {
+		try {
+			await gzipFiles(files, OSFlags.Unix);
+		} catch (error) {
+			const err = toError(error);
+			program.error(err.message);
+		}
+	}
+	async function shasum(files, options) {
+		const report = options.check ? await reportCheckChecksumFile(options.check, files, options) : options.update ? await updateChecksumForFiles(options.update, files, options) : await reportChecksumForFiles(files, options);
+		console.log("%s", report.report);
+		if (!report.passed) throw new CommanderError(1, "Failed Checksum", "One or more files had issues.");
+	}
+	program.exitOverride();
+	program.version(npmPackage.version);
+	addCompileOptions(program.command("compile [src...]").description("Compile words into a cspell dictionary files.")).option("--trie", "Compile into a trie file.", false).option("--no-sort", "Do not sort the result").action((src, options) => {
+		return processCompileAction(src, options, flags);
+	});
+	addCompileOptions(program.command("compile-trie [src...]").description("Compile words lists or Hunspell dictionary into trie files used by cspell.\nAlias of `compile --trie`")).action((src, options) => {
+		return processCompileAction(src, {
+			...options,
+			trie: true
+		}, flags);
+	});
+	program.command("build [targets...]").description("Build the targets defined in the run configuration.").option("-c, --config <path to run configuration>", "Specify the run configuration file.").option("--conditional", "Conditional build.").option("-r, --root <directory>", "Specify the run directory").action(build);
+	program.command("gzip <files...>").description("GZip files while keeping the original.").action(handleGzip);
+	program.command("btrie [files...]").description("Generate BTrie files from word list files.").option("-n, --no-compress", "By default the files are GZipped, this will turn off GZ compression.").option("--no-optimize", "Do not try to optimize.").option("--no-use-string-table", "Do not use a string table in the BTrie.").option("-o, --output <path>", "Specify the output directory, otherwise files are written back to the same location.").action(generateBTrie);
+	program.command("shasum [files...]").description("Calculate the checksum for files.").option("--list-file <list-file.txt...>", "Specify one or more files that contain paths of files to check.").option("-c, --check <checksum.txt>", "Verify the checksum of files against those stored in the checksum.txt file.").addOption(new Option("-u, --update <checksum.txt>", "Update checksums found in the file.").conflicts("--check")).option("-r, --root <root>", "Specify the root to use for relative paths. The current working directory is used by default.").action(shasum);
+	await program.parseAsync(argv);
+}
+
+//#endregion
+export { run };
+//# sourceMappingURL=app.mjs.map