@lingual/i18n-check 0.9.0 → 0.9.1

package/dist/index.js

@@ -0,0 +1,345 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.checkUndefinedKeys = exports.checkUnusedKeys = exports.checkTranslations = exports.checkMissingTranslations = exports.checkInvalidTranslations = void 0;
+const findMissingKeys_1 = require("./utils/findMissingKeys");
+const findInvalidTranslations_1 = require("./utils/findInvalidTranslations");
+const findInvalidI18NextTranslations_1 = require("./utils/findInvalidI18NextTranslations");
+const cli_lib_1 = require("@formatjs/cli-lib");
+const i18NextSrcParser_1 = require("./utils/i18NextSrcParser");
+const nextIntlSrcParser_1 = require("./utils/nextIntlSrcParser");
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const constants_1 = require("./utils/constants");
+const ParseFormats = ['react-intl', 'i18next', 'next-intl'];
+const checkInvalidTranslations = (source, targets, options = { format: 'icu' }) => {
+    return options.format === 'i18next'
+        ? (0, findInvalidI18NextTranslations_1.findInvalidI18NextTranslations)(source, targets)
+        : (0, findInvalidTranslations_1.findInvalidTranslations)(source, targets);
+};
+exports.checkInvalidTranslations = checkInvalidTranslations;
+const checkMissingTranslations = (source, targets, options) => {
+    return (0, findMissingKeys_1.findMissingKeys)(source, targets, options);
+};
+exports.checkMissingTranslations = checkMissingTranslations;
+const checkTranslations = (source, targets, options = { format: 'icu', checks: ['invalidKeys', 'missingKeys'] }) => {
+    const { checks = ['invalidKeys', 'missingKeys'] } = options;
+    const missingKeys = {};
+    const invalidKeys = {};
+    const hasMissingKeysCheck = checks.includes('missingKeys');
+    const hasInvalidKeysCheck = checks.includes('invalidKeys');
+    source.forEach(({ name, content }) => {
+        const files = Object.fromEntries(targets
+            .filter(({ reference }) => reference === name)
+            .map(({ name, content }) => [name, content]));
+        const filteredContent = filterKeys(content, options.ignore ?? []);
+        if (hasMissingKeysCheck) {
+            merge(missingKeys, (0, exports.checkMissingTranslations)(filteredContent, files, options));
+        }
+        if (hasInvalidKeysCheck) {
+            merge(invalidKeys, (0, exports.checkInvalidTranslations)(filteredContent, files, options));
+        }
+    });
+    return {
+        missingKeys: hasMissingKeysCheck ? missingKeys : undefined,
+        invalidKeys: hasInvalidKeysCheck ? invalidKeys : undefined,
+    };
+};
+exports.checkTranslations = checkTranslations;
+function merge(left, right) {
+    for (const [k, v] of Object.entries(right)) {
+        left[k] = (left?.[k] ?? []).concat(v);
+    }
+}
+const checkUnusedKeys = async (translationFiles, filesToParse, options = {
+    format: 'react-intl',
+    checks: [],
+}, componentFunctions = []) => {
+    if (!options.format || !ParseFormats.includes(options.format)) {
+        return undefined;
+    }
+    if (!options.checks || !options.checks.includes('unused')) {
+        return undefined;
+    }
+    const filteredTranslationFiles = translationFiles.map(({ content, ...rest }) => ({
+        ...rest,
+        content: filterKeys(content, options.ignore),
+    }));
+    if (options.format === 'react-intl') {
+        return findUnusedReactIntlTranslations(filteredTranslationFiles, filesToParse);
+    }
+    else if (options.format === 'i18next') {
+        return findUnusedI18NextTranslations(filteredTranslationFiles, filesToParse, componentFunctions);
+    }
+    else if (options.format === 'next-intl') {
+        return findUnusedNextIntlTranslations(filteredTranslationFiles, filesToParse);
+    }
+};
+exports.checkUnusedKeys = checkUnusedKeys;
+const findUnusedReactIntlTranslations = async (translationFiles, filesToParse) => {
+    const unusedKeys = {};
+    const extracted = await (0, cli_lib_1.extract)(filesToParse, {});
+    const extractedResultSet = new Set(Object.keys(JSON.parse(extracted)));
+    translationFiles.forEach(({ name, content }) => {
+        const keysInSource = Object.keys(content);
+        const found = [];
+        for (const keyInSource of keysInSource) {
+            if (!extractedResultSet.has(keyInSource)) {
+                found.push(keyInSource);
+            }
+        }
+        unusedKeys[name] = found;
+    });
+    return unusedKeys;
+};
+const findUnusedI18NextTranslations = async (source, filesToParse, componentFunctions = []) => {
+    const unusedKeys = {};
+    const { extractedResult, skippableKeys } = await getI18NextKeysInCode(filesToParse, componentFunctions);
+    const extractedResultSet = new Set(extractedResult.map(({ key, namespace }) => namespace ? `${namespace}.${key}` : key));
+    source.forEach(({ name, content }) => {
+        const keysInSource = Object.keys(content)
+            // Ensure that any plural definitiions like key_one, key_other etc.
+            // are flatted into a single key
+            .map((key) => {
+            const pluralSuffix = constants_1.I18NEXT_PLURAL_SUFFIX.find((suffix) => {
+                return key.endsWith(suffix);
+            });
+            return pluralSuffix ? key.replace(pluralSuffix, '') : key;
+        });
+        const found = [];
+        for (const keyInSource of keysInSource) {
+            const isSkippable = skippableKeys.find((skippableKey) => {
+                return keyInSource.includes(skippableKey);
+            });
+            if (isSkippable !== undefined) {
+                continue;
+            }
+            // find the file name
+            const [fileName] = (name.split(path_1.default.sep).pop() ?? '').split('.');
+            if (!extractedResultSet.has(`${fileName}.${keyInSource}`) &&
+                !extractedResultSet.has(keyInSource)) {
+                found.push(keyInSource);
+            }
+        }
+        unusedKeys[name] = found;
+    });
+    return unusedKeys;
+};
+const findUnusedNextIntlTranslations = async (translationFiles, filesToParse) => {
+    const unusedKeys = {};
+    const extracted = (0, nextIntlSrcParser_1.extract)(filesToParse);
+    const dynamicNamespaces = extracted.flatMap((namespace) => {
+        if (namespace.meta.dynamic) {
+            return [namespace.key];
+        }
+        return [];
+    });
+    const extractedResultSet = new Set(extracted.flatMap((namespace) => {
+        if (!namespace.meta.dynamic) {
+            return [namespace.key];
+        }
+        return [];
+    }));
+    translationFiles.forEach(({ name, content }) => {
+        const keysInSource = Object.keys(content);
+        const found = [];
+        for (const keyInSource of keysInSource) {
+            // Check if key is part of a dynamic namespace
+            // Skip the key if it is part of the dynamic namespace
+            const isDynamicNamespace = dynamicNamespaces.find((dynamicNamespace) => {
+                const keyInSourceNamespaces = keyInSource.split('.');
+                return dynamicNamespace.split('.').every((namePart, index) => {
+                    return namePart === keyInSourceNamespaces[index];
+                });
+            });
+            if (isDynamicNamespace) {
+                continue;
+            }
+            if (!extractedResultSet.has(keyInSource)) {
+                found.push(keyInSource);
+            }
+        }
+        unusedKeys[name] = found;
+    });
+    return unusedKeys;
+};
+const checkUndefinedKeys = async (source, filesToParse, options = {
+    format: 'react-intl',
+    checks: [],
+}, componentFunctions = []) => {
+    if (!options.format || !ParseFormats.includes(options.format)) {
+        return undefined;
+    }
+    if (!options.checks || !options.checks.includes('undefined')) {
+        return undefined;
+    }
+    if (options.format === 'react-intl') {
+        return findUndefinedReactIntlKeys(source, filesToParse, options);
+    }
+    else if (options.format === 'i18next') {
+        return findUndefinedI18NextKeys(source, filesToParse, options, componentFunctions);
+    }
+    else if (options.format === 'next-intl') {
+        return findUndefinedNextIntlKeys(source, filesToParse, options);
+    }
+};
+exports.checkUndefinedKeys = checkUndefinedKeys;
+const findUndefinedReactIntlKeys = async (translationFiles, filesToParse, options = {
+    ignore: [],
+}) => {
+    const sourceKeys = new Set(translationFiles.flatMap(({ content }) => {
+        return Object.keys(content);
+    }));
+    const extractedResult = await (0, cli_lib_1.extract)(filesToParse, {
+        extractSourceLocation: true,
+    });
+    const undefinedKeys = {};
+    Object.entries(JSON.parse(extractedResult)).forEach(([key, meta]) => {
+        if (!sourceKeys.has(key) && !isIgnoredKey(options.ignore ?? [], key)) {
+            const data = meta;
+            if (!('file' in data) || typeof data.file !== 'string') {
+                return;
+            }
+            const file = path_1.default.normalize(data.file);
+            if (!undefinedKeys[file]) {
+                undefinedKeys[file] = [];
+            }
+            undefinedKeys[file].push(key);
+        }
+    });
+    return undefinedKeys;
+};
+const findUndefinedI18NextKeys = async (source, filesToParse, options = {
+    ignore: [],
+}, componentFunctions = []) => {
+    const { extractedResult, skippableKeys } = await getI18NextKeysInCode(filesToParse, componentFunctions);
+    const sourceKeys = new Set(source
+        .flatMap(({ content }) => {
+        return Object.keys(content);
+    })
+        // Ensure that any plural definitiions like key_one, key_other etc.
+        // are flatted into a single key
+        .map((key) => {
+        const pluralSuffix = constants_1.I18NEXT_PLURAL_SUFFIX.find((suffix) => {
+            return key.endsWith(suffix);
+        });
+        return pluralSuffix ? key.replace(pluralSuffix, '') : key;
+    }));
+    const undefinedKeys = {};
+    extractedResult.forEach(({ file, key }) => {
+        const isSkippable = skippableKeys.find((skippableKey) => {
+            return key.includes(skippableKey);
+        });
+        if (isSkippable === undefined &&
+            !sourceKeys.has(key) &&
+            !isIgnoredKey(options.ignore ?? [], key)) {
+            if (!undefinedKeys[file]) {
+                undefinedKeys[file] = [];
+            }
+            undefinedKeys[file].push(key);
+        }
+    });
+    return undefinedKeys;
+};
+const findUndefinedNextIntlKeys = async (translationFiles, filesToParse, options = {
+    ignore: [],
+}) => {
+    const sourceKeys = new Set(translationFiles.flatMap(({ content }) => {
+        return Object.keys(content);
+    }));
+    const extractedResult = (0, nextIntlSrcParser_1.extract)(filesToParse);
+    const undefinedKeys = {};
+    extractedResult.forEach(({ key, meta }) => {
+        if (!meta.dynamic &&
+            !sourceKeys.has(key) &&
+            !isIgnoredKey(options.ignore ?? [], key)) {
+            const file = meta.file;
+            if (!undefinedKeys[file]) {
+                undefinedKeys[file] = [];
+            }
+            undefinedKeys[file].push(key);
+        }
+    });
+    return undefinedKeys;
+};
+const isRecord = (data) => {
+    return (typeof data === 'object' &&
+        !Array.isArray(data) &&
+        data !== null &&
+        data !== undefined);
+};
+const getI18NextKeysInCode = async (filesToParse, componentFunctions = []) => {
+    // Skip any parsed keys that have the `returnObjects` property set to true
+    // As these are used dynamically, they will be skipped to prevent
+    // these keys from being marked as unused.
+    const extractedResult = [];
+    const skippableKeys = [];
+    filesToParse.forEach((file) => {
+        const rawContent = fs_1.default.readFileSync(file, 'utf-8');
+        const entries = (0, i18NextSrcParser_1.getKeys)(file, {
+            componentFunctions: componentFunctions.concat(['Trans']),
+        }, rawContent);
+        // Intermediate solution to retrieve all keys from the parser.
+        // This will be built out to also include the namespace and check
+        // the key against the namespace corresponding file.
+        // The current implementation considers the key as used no matter the namespace.
+        for (const entry of entries) {
+            // check for namespace, i.e. `namespace:some.key`
+            const [namespace, ...keyParts] = entry.key.split(':');
+            // If there is a namespace make sure to assign the namespace
+            // and update the key name
+            // Ensure that the assumed key is not the default value
+            if (keyParts.length > 0 && entry.key !== entry.defaultValue) {
+                entry.namespace = namespace;
+                // rebuild the key without the namespace
+                entry.key = keyParts.join(':');
+            }
+            if (entry.returnObjects) {
+                skippableKeys.push(entry.key);
+            }
+            else {
+                extractedResult.push({
+                    file,
+                    key: entry.key,
+                    namespace: entry.namespace,
+                });
+            }
+        }
+    });
+    return { extractedResult, skippableKeys };
+};
+const filterKeys = (content, keysToIgnore = []) => {
+    if (keysToIgnore.length > 0) {
+        return Object.entries(content).reduce((acc, [key, value]) => {
+            if (isIgnoredKey(keysToIgnore, key)) {
+                return acc;
+            }
+            acc[key] = value;
+            return acc;
+        }, {});
+    }
+    return content;
+};
+const isIgnoredKey = (keysToIgnore, key) => {
+    return (keysToIgnore.find((ignoreKey) => {
+        if (ignoreKey.endsWith('*')) {
+            return key.includes(ignoreKey.slice(0, ignoreKey.length - 1));
+        }
+        return ignoreKey === key;
+    }) !== undefined);
+};
+function _flatten(object, prefix = null, result = {}) {
+    for (const key in object) {
+        const propName = prefix ? `${prefix}.${key}` : key;
+        const data = object[key];
+        if (isRecord(data)) {
+            _flatten(data, propName, result);
+        }
+        else {
+            result[propName] = data;
+        }
+    }
+    return result;
+}

package/dist/types.d.ts

@@ -0,0 +1,23 @@
+import { Context } from './errorReporters';
+export type Translation = Record<string, unknown>;
+export type CheckResult = Record<string, string[]>;
+export type InvalidTranslationEntry = {
+    key: string;
+    msg: string;
+};
+export type InvalidTranslationsResult = Record<string, InvalidTranslationEntry[]>;
+export type TranslationFile = {
+    reference: string | null;
+    name: string;
+    content: Translation;
+};
+export type FileInfo = {
+    file: string;
+    name: string;
+    path: string[];
+};
+export type Options = {
+    format?: 'icu' | 'i18next' | 'react-intl' | 'next-intl';
+    checks?: Context[];
+    ignore?: string[];
+};

package/dist/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/utils/constants.d.ts

@@ -0,0 +1 @@
+export declare const I18NEXT_PLURAL_SUFFIX: string[];

package/dist/utils/constants.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.I18NEXT_PLURAL_SUFFIX = void 0;
+exports.I18NEXT_PLURAL_SUFFIX = [
+    '_zero',
+    '_one',
+    '_two',
+    '_few',
+    '_many',
+    '_other',
+    '_interval',
+];

package/dist/utils/findInvalidI18NextTranslations.d.ts

@@ -0,0 +1,11 @@
+/**
+ *
+ * i18next specific invalid translations check
+ *
+ *
+ */
+import { MessageFormatElement } from './i18NextParser';
+import { InvalidTranslationEntry, InvalidTranslationsResult, Translation } from '../types';
+export declare const findInvalidI18NextTranslations: (source: Translation, targets: Record<string, Translation>) => InvalidTranslationsResult;
+export declare const compareTranslationFiles: (a: Translation, b: Translation) => InvalidTranslationEntry[];
+export declare const hasDiff: (a: MessageFormatElement[], b: MessageFormatElement[]) => boolean;

package/dist/utils/findInvalidI18NextTranslations.js

@@ -0,0 +1,192 @@
+"use strict";
+/**
+ *
+ * i18next specific invalid translations check
+ *
+ *
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.hasDiff = exports.compareTranslationFiles = exports.findInvalidI18NextTranslations = void 0;
+const i18NextParser_1 = require("./i18NextParser");
+const findInvalidI18NextTranslations = (source, targets) => {
+    const differences = {};
+    if (Object.keys(targets).length === 0) {
+        return differences;
+    }
+    for (const [lang, file] of Object.entries(targets)) {
+        const result = (0, exports.compareTranslationFiles)(source, file);
+        if (result.length > 0) {
+            differences[lang] = result;
+        }
+    }
+    return differences;
+};
+exports.findInvalidI18NextTranslations = findInvalidI18NextTranslations;
+const compareTranslationFiles = (a, b) => {
+    const diffs = [];
+    for (const key in a) {
+        if (b[key] === undefined) {
+            continue;
+        }
+        const parsedTranslationA = (0, i18NextParser_1.parse)(String(a[key]));
+        const parsedTranslationB = (0, i18NextParser_1.parse)(String(b[key]));
+        if ((0, exports.hasDiff)(parsedTranslationA, parsedTranslationB)) {
+            const msg = getErrorMessage(parsedTranslationA, parsedTranslationB);
+            diffs.push({ key, msg });
+        }
+    }
+    return diffs;
+};
+exports.compareTranslationFiles = compareTranslationFiles;
+const lookUp = {
+    text: 0,
+    interpolation: 1,
+    interpolation_unescaped: 2,
+    nesting: 3,
+    plural: 4,
+    tag: 5,
+};
+const sortParsedKeys = (a, b) => {
+    if (a.type === b.type && a.type !== 'tag' && b.type !== 'tag') {
+        return a.content < b.content ? -1 : 1;
+    }
+    if (a.type === 'tag' && b.type === 'tag') {
+        return a.raw < b.raw ? -1 : 1;
+    }
+    return lookUp[a.type] - lookUp[b.type];
+};
+const hasDiff = (a, b) => {
+    const compA = a
+        .filter((element) => element.type !== 'text')
+        .sort(sortParsedKeys);
+    const compB = b
+        .filter((element) => element.type !== 'text')
+        .sort(sortParsedKeys);
+    if (compA.length !== compB.length) {
+        return true;
+    }
+    const hasErrors = compA.some((formatElementA, index) => {
+        const formatElementB = compB[index];
+        if (formatElementA.type !== formatElementB.type) {
+            return true;
+        }
+        if (formatElementA.type === 'tag' && formatElementB.type === 'tag') {
+            return (formatElementA.raw !== formatElementB.raw ||
+                formatElementA.voidElement !== formatElementB.voidElement);
+        }
+        if ((formatElementA.type === 'interpolation' &&
+            formatElementB.type === 'interpolation') ||
+            (formatElementA.type === 'interpolation_unescaped' &&
+                formatElementB.type === 'interpolation_unescaped') ||
+            (formatElementA.type === 'nesting' &&
+                formatElementB.type === 'nesting') ||
+            (formatElementA.type === 'plural' && formatElementB.type === 'plural')) {
+            const optionsA = formatElementA.variable
+                .split(',')
+                .map((value) => value.trim())
+                .sort()
+                .join('-')
+                .trim();
+            const optionsB = formatElementB.variable
+                .split(',')
+                .map((value) => value.trim())
+                .sort()
+                .join('-')
+                .trim();
+            if (optionsA !== optionsB) {
+                return true;
+            }
+            if (formatElementA.prefix !== formatElementA.prefix) {
+                return true;
+            }
+            if (formatElementA.suffix !== formatElementA.suffix) {
+                return true;
+            }
+        }
+        return false;
+    });
+    return hasErrors;
+};
+exports.hasDiff = hasDiff;
+const getErrorMessage = (a, b) => {
+    const compA = a
+        .filter((element) => element.type !== 'text')
+        .sort(sortParsedKeys);
+    const compB = b
+        .filter((element) => element.type !== 'text')
+        .sort(sortParsedKeys);
+    const errors = compA.reduce((acc, formatElementA, index) => {
+        const formatElementB = compB[index];
+        if (!formatElementB) {
+            acc.push(`Missing element ${formatElementA.type}`);
+            return acc;
+        }
+        if (formatElementA.type !== formatElementB.type) {
+            acc.push(`Expected element of type "${formatElementA.type}" but received "${formatElementB.type}"`);
+            return acc;
+        }
+        if (formatElementA.type === 'tag' && formatElementB.type === 'tag') {
+            if (formatElementA.raw !== formatElementB.raw) {
+                acc.push(`Expected tag "${formatElementA.raw}" but received "${formatElementB.raw}"`);
+            }
+            else if (formatElementA.voidElement !== formatElementB.voidElement &&
+                formatElementA.voidElement === true) {
+                acc.push(`Expected a self-closing "${formatElementB.raw}" tag`);
+                return acc;
+            }
+            else if (formatElementA.voidElement !== formatElementB.voidElement &&
+                formatElementA.voidElement === false) {
+                acc.push(`Non expected self-closing "${formatElementB.raw}" tag`);
+                return acc;
+            }
+        }
+        if ((formatElementA.type === 'interpolation' &&
+            formatElementB.type === 'interpolation') ||
+            (formatElementA.type === 'interpolation_unescaped' &&
+                formatElementB.type === 'interpolation_unescaped') ||
+            (formatElementA.type === 'nesting' &&
+                formatElementB.type === 'nesting') ||
+            (formatElementA.type === 'plural' && formatElementB.type === 'plural')) {
+            if (formatElementA.prefix !== formatElementA.prefix) {
+                acc.push(`Error in ${formatElementA.type}: Expected prefix "${formatElementA.prefix}" but received "${formatElementB.prefix}"`);
+                return acc;
+            }
+            if (formatElementA.suffix !== formatElementA.suffix) {
+                acc.push(`Error in ${formatElementA.type}: Expected suffix "${formatElementA.suffix}" but received "${formatElementB.suffix}"`);
+                return acc;
+            }
+            const optionsA = formatElementA.variable
+                .split(',')
+                .map((value) => value.trim())
+                .sort();
+            const optionsB = formatElementB.variable
+                .split(',')
+                .map((value) => value.trim())
+                .sort();
+            const elementErrors = [];
+            optionsA.forEach((key, index) => {
+                if (key !== optionsB[index]) {
+                    elementErrors.push(`Expected ${key} but received ${optionsB[index]}`);
+                }
+            });
+            if (elementErrors.length > 0) {
+                acc.push(`Error in ${formatElementA.type}: ${elementErrors
+                    .flatMap((elementError) => elementError)
+                    .join(', ')}`);
+            }
+            return acc;
+        }
+        return acc;
+    }, []);
+    if (compA.length < compB.length) {
+        const unexpectedElements = compB
+            .slice(compA.length)
+            .reduce((acc, formatElementB) => {
+            acc.push(`Unexpected ${formatElementB.type} element`);
+            return acc;
+        }, [])
+            .join(', ');
+        return [...errors, unexpectedElements].join(', ');
+    }
+    return errors.join(', ');
+};

package/dist/utils/findInvalidTranslations.d.ts

@@ -0,0 +1,16 @@
+import { MessageFormatElement } from '@formatjs/icu-messageformat-parser';
+import { InvalidTranslationEntry, InvalidTranslationsResult, Translation } from '../types';
+type Location = {
+    start: {
+        line: number;
+        column: number;
+    };
+};
+export declare class CheckError extends Error {
+    location?: Location;
+    originalMessage?: string;
+}
+export declare const findInvalidTranslations: (source: Translation, files: Record<string, Translation>) => InvalidTranslationsResult;
+export declare const compareTranslationFiles: (a: Translation, b: Translation) => InvalidTranslationEntry[];
+export declare const hasDiff: (a: MessageFormatElement[], b: MessageFormatElement[]) => boolean;
+export {};