ilib-lint 1.0.0

package/src/rules/ResourceQuoteStyle.js

@@ -0,0 +1,170 @@
+/*
+ * ResourceQuoteStyle.js - rule to check quotes in the target string
+ *
+ * Copyright © 2022 JEDLSoft
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import LocaleInfo from 'ilib-localeinfo';
+
+import Rule from '../Rule.js';
+import Result from '../Result.js';
+
+let LICache = {};
+
+// superset of all the start and end chars used in CLDR
+const quoteChars = "«»‘“”„「」’‚‹›『』\"\'";
+
+/**
+ * @class Represent an ilib-lint rule.
+ */
+class ResourceQuoteStyle extends Rule {
+    constructor(options) {
+        super(options);
+        this.name = "resource-quote-style";
+        this.description = "Ensure that the proper quote characters are used in translated resources";
+        this.sourceLocale = (options && options.sourceLocale) || "en-US";
+    }
+
+    getRuleType() {
+        return "resource";
+    }
+
+    /**
+     * @override
+     */
+    match(options) {
+        const { locale, resource, file } = options || {};
+        let li = LICache[locale];
+        const _this = this;
+
+        if (!li) {
+            li = new LocaleInfo(locale);
+            LICache[locale] = li;
+        }
+
+        let sourceLI = LICache[this.sourceLocale];
+        if (!sourceLI) {
+            sourceLI = new LocaleInfo(this.sourceLocale);
+            LICache[this.sourceLocale] = sourceLI;
+        }
+
+        const srcQuoteStart = sourceLI.getDelimiterQuotationStart();
+        const srcAltQuoteStart = sourceLI.info.delimiter.alternateQuotationStart;
+
+        const srcQuoteEnd = sourceLI.getDelimiterQuotationEnd();
+        const srcAltQuoteEnd = sourceLI.info.delimiter.alternateQuotationEnd;
+
+        const tarQuoteStart = li.getDelimiterQuotationStart();
+        const tarAltQuoteStart = li.info.delimiter.alternateQuotationStart;
+
+        const tarQuoteEnd = li.getDelimiterQuotationEnd();
+        const tarAltQuoteEnd = li.info.delimiter.alternateQuotationEnd;
+
+        // if the source uses ASCII quotes, then the target could have ASCII or native quotes
+        const srcQuotesAscii = new RegExp(`((^|\\W)['"]\\p{Letter}|\\p{Letter}['"](\\W|$))`, "gu");
+        const srcQuotesNative = new RegExp(`((^|\\W)[${srcQuoteStart}${srcAltQuoteStart}]\\p{Letter}|\\p{Letter}[${srcQuoteEnd}${srcAltQuoteEnd}](\\W|$))`, "gu");
+
+        // if the source contains native quotes, then the target should also have native quotes
+        const tarQuotesAll = new RegExp(`((^|\\W)[${tarQuoteStart}${tarAltQuoteStart}'"]\\p{Letter}|\\p{Letter}[${tarQuoteEnd}${tarAltQuoteEnd}'"](\\W|$))`, "gu");
+        const tarQuotesNative = new RegExp(`((^|\\W)[${tarQuoteStart}${tarAltQuoteStart}]\\p{Letter}|\\p{Letter}[${tarQuoteEnd}${tarAltQuoteEnd}](\\W|$))`, "gu");
+
+        const nonQuoteChars = `([${
+            quoteChars.
+                replace(srcQuoteStart, "").
+                replace(srcAltQuoteStart, "").
+                replace(tarQuoteStart, "").
+                replace(tarAltQuoteStart, "").
+                replace(srcQuoteEnd, "").
+                replace(srcAltQuoteEnd, "").
+                replace(tarQuoteEnd, "").
+                replace(tarAltQuoteEnd, "")}])`;
+        const re = new RegExp(nonQuoteChars, "g");
+
+        /**
+         * @private
+         */
+        function checkString(src, tar) {
+            srcQuotesAscii.lastIndex = 0;
+            tarQuotesAll.lastIndex = 0;
+            if ((src.match(srcQuotesAscii) && !tar.match(tarQuotesAll)) ||
+                (src.match(srcQuotesNative) && !tar.match(tarQuotesNative))) {
+                const matches = re.exec(tar);
+                let value = {
+                    severity: "warning",
+                    id: resource.getKey(),
+                    source: src,
+                    rule: _this,
+                    pathName: file
+                };
+                if (matches) {
+                    value.highlight = `Target: ${tar.replace(re, "<e0>$1</e0>")}`;
+                    value.description = `Quote style for the the locale ${locale} should be ${tarQuoteStart}text${tarQuoteEnd}`;
+                } else {
+                    value.highlight = `Target: ${tar}<e0></e0>`;
+                    value.description = `Quotes are missing in the target. Quote style for the the locale ${locale} should be ${tarQuoteStart}text${tarQuoteEnd}`;
+                }
+                if (typeof(options.lineNumber) !== 'undefined') {
+                    value.lineNumber = options.lineNumber;
+                }
+                return new Result(value);
+            }
+        }
+
+        switch (resource.getType()) {
+            case 'string':
+                const tarString = resource.getTarget();
+                if (tarString) {
+                    return checkString(resource.getSource(), tarString);
+                }
+                break;
+
+            case 'array':
+                const srcArray = resource.getSource();
+                const tarArray = resource.getTarget();
+                if (tarArray) {
+                    return srcArray.map((item, i) => {
+                        if (i < tarArray.length && tarArray[i]) {
+                            return checkString(srcArray[i], tarArray[i]);
+                        }
+                    }).filter(element => {
+                        return element;
+                    });
+                }
+                break;
+
+            case 'plural':
+                const srcPlural = resource.getSource();
+                const tarPlural = resource.getTarget();
+                if (tarPlural) {
+                    const hasQuotes = categories.find(category => {
+                        return (srcPlural[category] && srcPlural[category].contains(srcQuote));
+                    });
+
+                    if (hasQuotes) {
+                        return categories.map(category => {
+                            return checkString(srcPlural.other, tarPlural[category]);
+                        });
+                    }
+                }
+                break;
+        }
+    }
+
+    // no match
+    return;
+}
+
+export default ResourceQuoteStyle;

package/src/rules/ResourceRegExpChecker.js

@@ -0,0 +1,179 @@
+/*
+ * ResourceRegExpChecker.js - rule to check if URLs in the source string also
+ * appear in the target string
+ *
+ * Copyright © 2022 JEDLSoft
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Rule from '../Rule.js';
+import Result from '../Result.js';
+import { stripPlurals } from './utils.js';
+
+function findMissing(source, target) {
+    let missing = [];
+    for (var i = 0; i < source.length; i++) {
+        if (target.indexOf(source[i]) < 0) {
+            missing.push(source[i]);
+        }
+    }
+    return missing;
+}
+
+/**
+ * @class Resource checker class that checks that any regular expressions
+ * that matches in the source also appears in the translation.
+ */
+class ResourceRegExpChecker extends Rule {
+    /**
+     * Construct a new regular expression-based resource checker.
+     *
+     * The options must contain the following required properties:
+     *
+     * - name - a unique name for this rule
+     * - description - a one-line description of what this rule checks for.
+     *   Example: "Check that URLs in the source also appear in the target"
+     * - note - a one-line note that will be printed on screen when the
+     *   check fails. Example: "The URL {matchString} did not appear in the
+     *   the target." (Currently, matchString is the only replacement
+     *   param that is supported.)
+     * - regexps - an array of strings that encode regular expressions to
+     *   look for
+     */
+    constructor(options) {
+        super(options);
+
+        if (!options || !options.name || !options.description || !options.note || !options.regexps) {
+            throw "Missing required options for the ResourceRegExpChecker constructor";
+        }
+        ["name", "description", "regexps", "note", "sourceLocale"].forEach(prop => {
+            this[prop] = options[prop];
+        });
+        this.sourceLocale = this.sourceLocale || "en-US";
+
+        // this may throw if you got to the syntax wrong:
+        this.re = this.regexps.map(regexp => new RegExp(regexp, "g"));
+    }
+
+    getRuleType() {
+        return "resource";
+    }
+
+    /**
+     * @override
+     */
+    match(options) {
+        const { locale, resource, file } = options || {};
+        const _this = this;
+
+        /**
+         * @private
+         */
+        function checkString(re, src, tar) {
+            re.lastIndex = 0;
+            let sourceMatches = [];
+            const strippedSrc = stripPlurals(src);
+            const strippedTar = stripPlurals(tar);
+
+            let match = re.exec(strippedSrc);
+            while (match) {
+                sourceMatches.push(match[0]);
+                match = re.exec(strippedSrc);
+            }
+
+            if (sourceMatches.length > 0) {
+                // contains URLs, so check the target
+                re.lastIndex = 0;
+                let targetMatches = [];
+                match = re.exec(strippedTar);
+                while (match) {
+                    targetMatches.push(match[0]);
+                    match = re.exec(strippedTar);
+                }
+                const missing = findMissing(sourceMatches, targetMatches);
+                if (missing.length > 0) {
+                    return missing.map(missing => {
+                        let value = {
+                            severity: "error",
+                            id: resource.getKey(),
+                            source: src,
+                            rule: _this,
+                            pathName: file,
+                            highlight:`Target: ${tar}<e0></e0>`,
+                            description: _this.note.replace(/\{matchString\}/g, missing)
+                        };
+                        if (typeof(options.lineNumber) !== 'undefined') {
+                            value.lineNumber = options.lineNumber;
+                        }
+                        return new Result(value);
+                    });
+                }
+            }
+        }
+
+        function checkRegExps(src, tar) {
+            let results = [];
+            _this.re.forEach(re => {
+                results = results.concat(checkString(re, src, tar));
+            });
+            results = results.filter(result => result);
+            return results && results.length ? results : undefined;
+        }
+
+        switch (resource.getType()) {
+            case 'string':
+                const tarString = resource.getTarget();
+                if (tarString) {
+                    return checkRegExps(resource.getSource(), tarString);
+                }
+                break;
+
+            case 'array':
+                const srcArray = resource.getSource();
+                const tarArray = resource.getTarget();
+                if (tarArray) {
+                    return srcArray.map((item, i) => {
+                        if (i < tarArray.length && tarArray[i]) {
+                            return checkRegExps(srcArray[i], tarArray[i]);
+                        }
+                    }).filter(element => {
+                        return element;
+                    });
+                }
+                break;
+
+            case 'plural':
+                const srcPlural = resource.getSource();
+                const tarPlural = resource.getTarget();
+                if (tarPlural) {
+                    const hasQuotes = categories.find(category => {
+                        return (srcPlural[category] && srcPlural[category].contains(srcQuote));
+                    });
+
+                    if (hasQuotes) {
+                        return categories.map(category => {
+                            return checkRegExps(srcPlural.other, tarPlural[category]);
+                        });
+                    }
+                }
+                break;
+        }
+    }
+
+    // no match
+    return;
+}
+
+export default ResourceRegExpChecker;

package/src/rules/ResourceUniqueKeys.js

@@ -0,0 +1,76 @@
+/*
+ * ResourceUniqueKeys.js - rule to check quotes in the target string
+ *
+ * Copyright © 2022 JEDLSoft
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import LocaleInfo from 'ilib-localeinfo';
+import { TranslationSet } from 'ilib-tools-common';
+
+import Rule from '../Rule.js';
+import Result from '../Result.js';
+
+/**
+ * @class Represent an ilib-lint rule.
+ */
+class ResourceUniqueKeys extends Rule {
+    constructor(options) {
+        super(options);
+        this.name = "resource-unique-keys";
+        this.description = "Ensure that the keys are unique within a locale across all resource files";
+        this.sourceLocale = (options && options.sourceLocale) || "en-US";
+
+        this.ts = new TranslationSet();
+    }
+
+    getRuleType() {
+        return "resource";
+    }
+
+    /**
+     * @override
+     */
+    match(options) {
+        const { locale, resource, file } = options || {};
+
+        const hash = resource.hashKey();
+        const other = this.ts.get(hash);
+
+        if (other) {
+            let value = {
+                severity: "error",
+                id: resource.getKey(),
+                rule: this,
+                pathName: file,
+                highlight: `Also defined in this file: ${other.resfile}`,
+                description: `Key is not unique within locale ${locale}.`,
+                locale
+            };
+            if (typeof(options.lineNumber) !== 'undefined') {
+                value.lineNumber = options.lineNumber;
+            }
+            return new Result(value);
+        }
+
+        resource.resfile = file;
+        this.ts.add(resource);
+
+        // no result
+        return;
+    }
+}
+
+export default ResourceUniqueKeys;

package/src/rules/utils.js

@@ -0,0 +1,78 @@
+/*
+ * utils.js - utility functions for the rules
+ *
+ * Copyright © 2022 JEDLSoft
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { IntlMessageFormat } from 'intl-messageformat';
+
+function processNode(node) {
+    let text = "";
+    switch (node.type) {
+        case 0:
+            text = node.value;
+            break;
+
+        // Actual parameter
+        case 1:
+            text = `{${node.value}}`;
+            break;
+
+        case 6:
+            if (node.options) {
+                text = Object.keys(node.options).map(category => {
+                    return concatText(node.options[category].value);
+                }).join(" ");
+            }
+            break;
+    }
+
+    if (node.children) {
+        text += concatText(node.children);
+    }
+
+    return text;
+}
+
+function concatText(ast) {
+    if (!ast) return "";
+
+    let result = "";
+
+    if (Array.isArray(ast)) {
+        result += ast.map(node => {
+            return processNode(node);
+        }).join(" ");
+    } else if (typeof(ast) === "Object") {
+        result = processNode(node);
+    } else if (typeof(ast) === "string") {
+        result = ast;
+    } // else just ignore
+
+    return result;
+}
+
+export function stripPlurals(str, locale) {
+    try {
+        const imf = new IntlMessageFormat(str, locale);
+        const ast = imf.getAst();
+
+        return concatText(ast).replace(/\s+/g, " ");
+    } catch (e) {
+        // punt
+        return str;
+    }
+}

package/src/walk.js

@@ -0,0 +1,146 @@
+/*
+ * walk.js - walk a directory tree
+ *
+ * Copyright © 2022 JEDLSoft
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import log4js from 'log4js';
+import mm from 'micromatch';
+
+import SourceFile from './SourceFile.js';
+
+const logger = log4js.getLogger("ilib-lint.walk");
+
+/**
+ * Recursively walk a directory and return a list of files and directories
+ * within that directory. The walk is controlled via a list of exclude and
+ * include patterns. Each pattern should be a micromatch pattern like this:
+ *
+ * <code>
+ * "*.json"
+ * </code>
+ *
+ * The full path to every file and directory in the top-level directory will
+ * be included, unless it matches an exclude pattern, it which case, it will be
+ * excluded from the output. However, if the path
+ * also matches an include pattern, it will still be included nonetheless. The
+ * idea is that you can exclude a whole category of files (like all json files),
+ * but include specific ones. For example, you may exclude all json files, but
+ * still want to include the "config.json" file.<p>
+
+ * The options parameter may include any of the following optional properties:
+ *
+ * <ul>
+ * <li><i>quiet</i> (boolean) - whether or not to give output while walking
+ * the directory tree
+ * <li><i>excludes</i> (Array of strings) - A list of micromatch patterns to
+ * exclude from the output. If a pattern matches a directory, that directory
+ * will not be recursively searched.
+ * <li><i>includes</i> (Array of strings) - A list of micromatch patterns to
+ * include in the walk. If a pattern matches both an exclude and an include, the
+ * include will override the exclude.
+ * </ul>
+ *
+ * @param {String} root Directory to walk
+ * @param {Object} options Options controlling how this walk happens. (See
+ * the description for more details.)
+ * @returns {Array.<SourceFile>} an array of file names in the directory, filtered
+ * by the the excludes and includes list
+ */
+function walk(root, options) {
+    let results = [], projectRoot = false, newProject, list;
+
+    if (typeof(root) !== "string") {
+        return results;
+    }
+
+    const { config, quiet = false } = options || {};
+    const includes = config && config.paths ? Object.keys(config.paths) : ["**"];
+    let excludes = config && config.excludes;
+    let pathName, relPath, included, stat, glob;
+
+    try {
+        if (fs.existsSync(root)) {
+            stat = fs.statSync(root);
+            if (stat && stat.isDirectory()) {
+                list = fs.readdirSync(root);
+                if (!quiet) logger.trace("Searching " + root);
+
+                if (list && list.length !== 0) {
+                    list.sort().forEach((file) => {
+                        if (file === "." || file === "..") {
+                            return;
+                        }
+
+                        pathName = path.join(root, file);
+                        included = true;
+
+                        if (excludes) {
+                            if (!quiet) logger.trace(`There are excludes. Relpath is ${pathName}`);
+                            included = !mm.isMatch(pathName, excludes);
+                        }
+
+                        if (included) {
+                            results = results.concat(walk(pathName, options));
+                        }
+                    });
+                }
+            } else {
+                // file
+                included = false;
+
+                if (includes) {
+                    if (!quiet) logger.trace(`There are includes.`);
+                    mm.match(root, includes, {
+                        onMatch: (params) => {
+                            if (!glob && params.isMatch) {
+                                glob = params.glob;
+                                excludes = config && ((config.paths && config.paths[glob] && config.paths[glob].excludes) || excludes);
+                                included = excludes ? !mm.isMatch(root, excludes) : true;
+                            }
+                        }
+                    });
+                }
+
+                if (included) {
+                    if (!quiet) logger.trace(`${pathName} ... included`);
+                    glob = glob || "**";
+                    results.push(new SourceFile({
+                        filePath: root,
+                        settings: config.paths && config.paths[glob]
+                    }));
+                } else {
+                    if (!quiet) logger.trace(`${pathName} ... excluded`);
+                }
+            }
+        } else {
+            if (!quiet) logger.warn(`File ${pathName} does not exist.`);
+        }
+    } catch (e) {
+        // if the readdirSync did not work, it's maybe a file?
+        if (fs.existsSync(root)) {
+            return [new SourceFile({
+                filePath: root
+            })];
+        }
+    }
+
+    return results;
+}
+
+export default walk;