ilib-lint 1.0.0

package/src/formatters/AnsiConsoleFormatter.js

@@ -0,0 +1,78 @@
+/*
+ * AnsiConsoleFormatter.js - Formats result output
+ *
+ * Copyright © 2022 JEDLSoft
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import log4js from 'log4js';
+
+import Formatter from '../Formatter.js';
+
+var logger = log4js.getLogger("ilib-lint.formatters.AnsiConsoleFormatter");
+
+/**
+ * @class Represent an output formatter for an ANSI console/terminal
+ * @abstract
+ */
+class AnsiConsoleFormatter extends Formatter {
+    /**
+     * Construct an formatter instance.
+     */
+    constructor() {
+        super();
+        this.name = "ansi-console-formatter";
+    }
+
+    /**
+     * Return a general description of the formatter for use in help output.
+     *
+     * @returns {String} a general description of the formatter
+     */
+    getDescription() {
+        return "Formats results for an ANSI terminal/console with colors.";
+    }
+
+    /**
+     * Format the given result with the current formatter and return the
+     * formatted string.
+     *
+     * @abstract
+     * @param {Result} result the result to format
+     * @returns {String} the formatted result
+     */
+    format(result) {
+        if (!result) return;
+        let output = "";
+        const startColor = (result.severity === "error" ? "\u001B[91m" : "\u001B[33m");
+        output += `${result.pathName}${typeof(result.lineNumber) === "number" ? ('(' + result.lineNumber + ')') : ""}:
+  ${startColor}${result.description}\u001B[0m\n`;
+        if (result.id) {
+            output += `  Key: ${result.id}\n`;
+        }
+        if (result.source) {
+            output += `  Source: ${result.source}\n`;
+        }
+        output += `  ${result.highlight}
+  Rule (${result.rule.getName()}): ${result.rule.getDescription()}
+`;
+        // output ascii terminal escape sequences
+        output = output.replace(/<e\d><\/e\d>/g, "\u001B[91m \u001B[0m");
+        output = output.replace(/<e\d>/g, "\u001B[91m");
+        output = output.replace(/<\/e\d>/g, "\u001B[0m");
+        return output;
+    }
+}
+
+export default AnsiConsoleFormatter;

package/src/index.js

@@ -0,0 +1,213 @@
+/*
+ * index.js - main program of ilib-lint
+ *
+ * Copyright © 2022 JEDLSoft
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs';
+import path from 'path';
+
+import OptionsParser from 'options-parser';
+import Locale from 'ilib-locale';
+import { JSUtils, Utils, Path } from 'ilib-common';
+import json5 from 'json5';
+import log4js from 'log4js';
+
+import walk from './walk.js';
+import ResourceICUPlurals from './rules/ResourceICUPlurals.js';
+import ResourceQuoteStyle from './rules/ResourceQuoteStyle.js';
+import ResourceRegExpChecker from './rules/ResourceRegExpChecker.js';
+import ResourceUniqueKeys from './rules/ResourceUniqueKeys.js';
+import FormatterFactory from './FormatterFactory.js';
+import RuleSet from './RuleSet.js';
+
+const __dirname = Path.dirname(Path.fileUriToPath(import.meta.url));
+log4js.configure(path.join(__dirname, '..', 'log4js.json'));
+
+var logger = log4js.getLogger("ilib-lint.root");
+
+const optionConfig = {
+    help: {
+        short: "h",
+        help: "This help message",
+        showHelp: {
+            banner: 'Usage: ilib-lint [-h] [options] path [path ...]',
+            output: logger.info
+        }
+    },
+    config: {
+        short: "c",
+        help: "Give an explicit path to a configuration file instead of trying to find it in the current directory."
+    },
+    errorsOnly: {
+        short: "e",
+        flag: true,
+        "default": false,
+        help: "Only return errors and supress warnings"
+    },
+    locales: {
+        short: "l",
+        "default": "en-AU,en-CA,en-GB,en-IN,en-NG,en-PH,en-PK,en-US,en-ZA,de-DE,fr-CA,fr-FR,es-AR,es-ES,es-MX,id-ID,it-IT,ja-JP,ko-KR,pt-BR,ru-RU,tr-TR,vi-VN,zxx-XX,zh-Hans-CN,zh-Hant-HK,zh-Hant-TW,zh-Hans-SG",
+        help: "Locales you want your app to support. Value is a comma-separated list of BCP-47 style locale tags. Default: the top 20 locales on the internet by traffic."
+    },
+    sourceLocale: {
+        short: "s",
+        "default": "en-US",
+        help: "Default locale used to interpret the strings in the source code or the source strings in resource files."
+    },
+    quiet: {
+        short: "q",
+        flag: true,
+        help: "Produce no progress output during the run, except for error messages. Instead exit with a return value. Zero indicates no errors, and a positive exit value indicates errors."
+    }
+};
+
+const options = OptionsParser.parse(optionConfig);
+
+/*
+if (options.args.length < 1) {
+    logger.info("Error: missing path parameter");
+    OptionsParser.help(optionConfig, {
+        banner: 'Usage: iilib-lint [-h] [options] path [path ...]',
+        output: logger.info
+    });
+    process.exit(1);
+}
+*/
+
+if (!options.opt.quiet) logger.info("ilib-lint - Copyright (c) 2022 JEDLsoft, All rights reserved.");
+
+let paths = options.args;
+if (paths.length === 0) {
+    paths.push(".");
+}
+
+if (options.opt.locales) {
+    options.opt.locales = options.opt.locales.split(/,/g);
+}
+// normalize the locale specs
+options.opt.locales = options.opt.locales.map(spec => {
+    let loc = new Locale(spec);
+    if (!loc.getLanguage()) {
+        loc = new Locale("und", loc.getRegion(), loc.getVariant(), loc.getScript());
+    }
+    return loc.getSpec();
+});
+
+// used if no explicit config is found or given
+const defaultConfig = {
+    "name": "ilib-lint",
+    "locales": [
+        "en-US",
+        "de-DE",
+        "ja-JP",
+        "ko-KR"
+    ],
+    "paths": {
+        "**/*.json": {
+            "locales": [
+                "en-US",
+                "de-DE",
+                "ja-JP"
+            ]
+        },
+        "**/*.xliff": {
+            "rules": {
+                "resource-icu-plurals": true,
+                "resource-quote-style": true,
+                "resource-url-match": true,
+                "resource-named-params": "localeOnly"
+            }
+        }
+    },
+    "excludes": [
+        "**/.git",
+        "**/node_modules",
+        "**/.svn",
+        "package.json",
+        "package-lock.json"
+    ]
+};
+
+let config = {};
+if (options.opt.config) {
+    if (!fs.existsSync(options.opt.config)) {
+        logger.warn(`Config file ${options.opt.config} does not exist. Aborting...`);
+        process.exit(2);
+    }
+    const data = fs.readFileSync(options.opt.config, "utf-8");
+    config = json5.parse(data);
+} else {
+    config = defaultConfig;
+}
+
+if (!options.opt.quiet) logger.debug(`Scanning input paths: ${JSON.stringify(paths)}`);
+
+let files = [];
+
+paths.forEach(pathName => {
+    files = files.concat(walk(pathName, {
+        quiet: options.opt.quiet,
+        config
+    }));
+});
+
+const rules = {
+    url: {
+        name: "resource-url-match",
+        description: "Ensure that URLs that appear in the source string are also used in the translated string",
+        note: "URL '{matchString}' from the source string does not appear in the target string",
+        regexps: [ "((https?|github|ftps?|mailto|file|data|irc):\\/\\/)([\\da-zA-Z\\.-]+)\\.([a-zA-Z\\.]{2,6})([\\/\w\\.-]*)*\\/?" ]
+    },
+    namedParams: {
+        name: "resource-named-params",
+        description: "Ensure that named parameters that appear in the source string are also used in the translated string",
+        note: "The named parameter '{matchString}' from the source string does not appear in the target string",
+        regexps: [ "\\{\\w+\\}" ]
+    }
+};
+
+const defaultRules = new RuleSet([
+    new ResourceICUPlurals(),
+    new ResourceQuoteStyle(),
+    new ResourceRegExpChecker(rules.url),
+    new ResourceRegExpChecker(rules.namedParams),
+    new ResourceUniqueKeys()
+]);
+const fmt = FormatterFactory(options.opt);
+
+// main loop
+let exitValue = 0;
+
+files.forEach(file => {
+    logger.trace(`Examining ${file.filePath}`);
+    file.parse();
+    const issues = file.findIssues(defaultRules, options.opt.locales);
+    issues.forEach(issue => {
+        const str = fmt.format(issue);
+        if (str) {
+            if (issue.severity === "error") {
+                logger.error(str);
+                exitValue = 2;
+            } else if (!options.opt.errorsOnly) {
+                logger.warn(str);
+                exitValue = Math.max(exitValue, 1);
+            }
+        }
+    });
+});
+
+process.exit(exitValue);

package/src/plugins/XliffParser.js

@@ -0,0 +1,60 @@
+/*
+ * XliffParser.js - common SPI for parser plugins
+ *
+ * Copyright © 2022 JEDLSoft
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'node:fs';
+import { ResourceXliff } from 'ilib-tools-common';
+
+import Parser from '../Parser.js';
+
+/**
+ * @class common SPI for parser plugins
+ * @abstract
+ */
+class XliffParser extends Parser {
+    /**
+     * Construct a new plugin.
+     */
+    constructor(options) {
+        super(options);
+        this.path = options.filePath;
+        this.xliff = new ResourceXliff({
+            path: options.filePath
+        });
+    }
+
+    /**
+     * Parse the current file into an intermediate representation.
+     */
+    parse() {
+        const data = fs.readFileSync(this.path, "utf-8");
+        this.xliff.parse(data);
+    }
+
+    /**
+     * For a "resource" type of plugin, this returns a list of Resource instances
+     * that result from parsing the file.
+     *
+     * @returns {Array.<Resource>} list of Resource instances in this file
+     */
+    getResources() {
+        return this.xliff.getResources();
+    }
+};
+
+export default XliffParser;

package/src/plugins/XliffPlugin.js

@@ -0,0 +1,60 @@
+/*
+ * XliffPlugin.js - plugin that can parse an Xliff file
+ *
+ * Copyright © 2022 JEDLSoft
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Plugin from '../Plugin.js';
+import XliffParser from './XliffParser.js';
+
+/**
+ * @class Plugin that can parse XLIFF files
+ */
+class XliffPlugin extends Plugin {
+    /**
+     *
+     */
+    constructor(options) {
+        super(options);
+    }
+
+    /**
+     * @override
+     */
+    getType() {
+        return "parser";
+    }
+
+    /**
+     * @override
+     */
+    getExtensions() {
+        return ["xliff", "xlif", "xlf"];
+    }
+
+    /**
+     * For a "parser" type of plugin, this returns a list of Parser classes
+     * that this plugin implements.
+     *
+     * @returns {Array.<Parser>} list of Parser classes implemented by this
+     * plugin
+     */
+    getParsers() {
+        return [XliffParser];
+    }
+};
+
+export default XliffPlugin;

package/src/rules/ResourceICUPlurals.js

@@ -0,0 +1,229 @@
+/*
+ * ResourceICUPlurals.js - rule to check formatjs/ICU style plurals in the target string
+ *
+ * Copyright © 2022 JEDLSoft
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { IntlMessageFormat } from 'intl-messageformat';
+import Locale from 'ilib-locale';
+
+import Rule from '../Rule.js';
+import Result from '../Result.js';
+
+// all the plural categories from CLDR
+const allCategories = ["zero", "one", "two", "few", "many", "other"];
+
+// Map the language to the set of plural categories that the language
+// uses. If the language is not listed below, it uses the default
+// list of plurals: "one" and "other"
+const categoriesForLang = {
+    "ja": [ "other" ],
+    "zh": [ "other" ],
+    "ko": [ "other" ],
+    "th": [ "other" ],
+    "lv": [ "zero", "one", "other" ],
+    "ga": [ "one", "two", "other" ],
+    "ro": [ "one", "few", "other" ],
+    "lt": [ "one", "few", "other" ],
+    "ru": [ "one", "few", "other" ],
+    "uk": [ "one", "few", "other" ],
+    "be": [ "one", "few", "other" ],
+    "sr": [ "one", "few", "other" ],
+    "hr": [ "one", "few", "other" ],
+    "cs": [ "one", "few", "other" ],
+    "sk": [ "one", "few", "other" ],
+    "pl": [ "one", "few", "other" ],
+    "sl": [ "one", "two", "few", "other" ],
+    "ar": [ "zero", "one", "two", "few", "many", "other" ]
+}
+
+/**
+ * @class Represent an ilib-lint rule.
+ */
+class ResourceICUPlurals extends Rule {
+    constructor(options) {
+        super(options);
+        this.name = "resource-icu-plurals";
+        this.description = "Ensure that plurals in translated resources have the correct syntax";
+        this.sourceLocale = (options && options.sourceLocale) || "en-US";
+    }
+
+    getRuleType() {
+        return "resource";
+    }
+
+    /**
+     * @private
+     */
+    checkPluralCategories(ast, neededCategories, isSource, stringToCheck, key, pathName, source) {
+        let value = [];
+        for (let i = 0; i < ast.length; i++) {
+            const opts = ast[i].options;
+            if (opts) {
+                // check if any of the needed categories are missing
+                const missing = neededCategories.filter(category => {
+                    return typeof(opts[category]) === 'undefined';
+                });
+                if ( missing && missing.length ) {
+                    let opts = {
+                        severity: "error",
+                        rule: this,
+                        description: `Missing plural categories in ${isSource ? "source" : "target"} string: ${missing.join(", ")}. Expecting these: ${neededCategories.join(", ")}`,
+                        id: key,
+                        highlight: `${isSource ? "Source" : "Target"}: ${stringToCheck}<e0></e0>`,
+                        pathName,
+                        source
+                    };
+                    value.push(new Result(opts));
+                }
+                for (let category in opts) {
+                    if ( opts[category] && Array.isArray(opts[category].value) ) {
+                        value = value.concat(this.checkPluralCategories(opts[category].value, neededCategories, isSource, stringToCheck, key, pathName, source));
+                    }
+                }
+                // now check the other way around. That is, if the categories that exist are not needed.
+                if (!isSource) {
+                    const extras = Object.keys(opts).filter(category => {
+                        return neededCategories.indexOf(category) < 0;
+                    });
+                    if (extras && extras.length) {
+                        let opts = {
+                            severity: "warning",
+                            rule: this,
+                            description: `Extra plural categories in ${isSource ? "source" : "target"} string: ${extras.join(", ")}. Expecting only these: ${neededCategories.join(", ")}`,
+                            id: key,
+                            highlight: `${isSource ? "Source" : "Target"}: ${stringToCheck}<e0></e0>`,
+                            pathName,
+                            source
+                        };
+                        value.push(new Result(opts));
+                    }
+                }
+            }
+        }
+        return value;
+    }
+
+    checkString(src, tar, file, resource, sourceLocale, targetLocale, lineNumber) {
+        const sLoc = new Locale(sourceLocale);
+        const tLoc = new Locale(targetLocale);
+        let results;
+        let problems = [];
+        let sourceCategories = [];
+        try {
+            const imf = new IntlMessageFormat(src, sourceLocale);
+            let categories = categoriesForLang[sLoc.getLanguage()] || [ "one", "other" ];
+            // look in the abstract syntax tree for the categories that were parsed out and make
+            // sure the required ones are there
+            const ast = imf.getAst();
+            problems = problems.concat(this.checkPluralCategories(ast, categories, true, src, resource.getKey(), file, resource.getSource()));
+            if ( ast[0] && ast[0].options ) {
+                sourceCategories = Object.keys(ast[0].options).filter(category => {
+                    // if it is not one of the standard categories, it is a special one, so search for it
+                    // in the target too
+                    return allCategories.indexOf(category) < 0;
+                });
+            }
+        } catch (e) {
+            let value = {
+                pathName: file,
+                severity: "error",
+                rule: this,
+                description: `Incorrect plural or select syntax in source string: ${e}`,
+                id: resource.getKey(),
+                highlight: `Source: ${src.substring(0, e.location.end.offset)}<e0>${src.substring(e.location.end.offset)}</e0>`,
+                pathName: file
+            };
+            if (typeof(lineNumber) !== 'undefined') {
+                value.lineNumber = lineNumber + e.location.end.line - 1;
+            }
+            problems.push(new Result(value));
+        }
+        try {
+            const imf = new IntlMessageFormat(tar, targetLocale);
+            let categories = categoriesForLang[tLoc.getLanguage()] || [ "one", "other" ];
+            if (sourceCategories.length) {
+                categories = categories.concat(sourceCategories);
+            }
+            // look in the abstract syntax tree for the categories that were parsed out and make
+            // sure the required ones are there
+            const ast = imf.getAst();
+            problems = problems.concat(this.checkPluralCategories(ast, categories, false, tar, resource.getKey(), file, resource.getSource()));
+        } catch (e) {
+            let value = {
+                severity: "error",
+                description: `Incorrect plural or select syntax in target string: ${e}`,
+                rule: this,
+                id: resource.getKey(),
+                source: src,
+                highlight: `Target: ${tar.substring(0, e.location.end.offset)}<e0>${tar.substring(e.location.end.offset)}</e0>`,
+                pathName: file
+            };
+            if (typeof(lineNumber) !== 'undefined') {
+                value.lineNumber = lineNumber + e.location.end.line - 1;
+            }
+            problems.push(new Result(value));
+        }
+        return problems.length < 2 ? problems[0] : problems;
+    }
+
+    /**
+     * @override
+     */
+    match(options) {
+        const { resource, file } = options;
+        const sourceLocale = this.sourceLocale;
+        let problems = [];
+
+        switch (resource.getType()) {
+            case 'string':
+                const tarString = resource.getTarget();
+                if (tarString) {
+                    return this.checkString(resource.getSource(), tarString, file, resource, sourceLocale, options.locale, options.lineNumber);
+                }
+                break;
+
+            case 'array':
+                const srcArray = resource.getSource();
+                const tarArray = resource.getTarget();
+                if (tarArray) {
+                    return srcArray.map((item, i) => {
+                        if (i < tarArray.length && tarArray[i]) {
+                            return this.checkString(srcArray[i], tarArray[i], file, resource, sourceLocale, options.locale, options.lineNumber);
+                        }
+                    }).filter(element => {
+                        return element;
+                    });
+                }
+                break;
+
+            case 'plural':
+                const srcPlural = resource.getSource();
+                const tarPlural = resource.getTarget();
+                if (tarPlural) {
+                    return categories.map(category => {
+                        return this.checkString(srcPlural.other, tarPlural[category], file, resource, sourceLocale, options.locale, options.lineNumber);
+                    });
+                }
+                break;
+        }
+    }
+
+    // no match
+    return;
+}
+
+export default ResourceICUPlurals;