@tolgee/cli 1.0.0-prerelease.1

package/dist/extractor/machines/shared/comments.js

@@ -0,0 +1,82 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const json5_1 = require("json5");
+function isValidKeyOverride(data) {
+    if (!('key' in data)) {
+        return false;
+    }
+    if (typeof data.key !== 'string') {
+        return false;
+    }
+    if ('ns' in data && typeof data.ns !== 'string') {
+        return false;
+    }
+    if ('defaultValue' in data && typeof data.defaultValue !== 'string') {
+        return false;
+    }
+    return true;
+}
+// This service is responsible for emitting events when magic comments are encountered
+function default_1(callback, onReceive) {
+    onReceive((evt) => {
+        const comment = evt.data.trim();
+        if (comment.startsWith('@tolgee-ignore')) {
+            return callback({
+                type: 'MAGIC_COMMENT',
+                kind: 'ignore',
+                line: evt.line,
+            });
+        }
+        if (comment.startsWith('@tolgee-key')) {
+            const data = comment.slice(11).trim();
+            // Data is escaped; extract all as string
+            if (data.startsWith('\\')) {
+                return callback({
+                    type: 'MAGIC_COMMENT',
+                    kind: 'key',
+                    keyName: data.slice(1),
+                    line: evt.line,
+                });
+            }
+            // Data is a json5 struct
+            if (data.startsWith('{')) {
+                try {
+                    const key = (0, json5_1.parse)(data);
+                    if (!isValidKeyOverride(key)) {
+                        // No key in the struct; invalid override
+                        callback({
+                            type: 'WARNING',
+                            kind: 'W_INVALID_KEY_OVERRIDE',
+                            line: evt.line,
+                        });
+                    }
+                    else {
+                        callback({
+                            type: 'MAGIC_COMMENT',
+                            kind: 'key',
+                            keyName: key.key,
+                            namespace: key.ns,
+                            defaultValue: key.defaultValue,
+                            line: evt.line,
+                        });
+                    }
+                }
+                catch {
+                    callback({
+                        type: 'WARNING',
+                        kind: 'W_MALFORMED_KEY_OVERRIDE',
+                        line: evt.line,
+                    });
+                }
+                return;
+            }
+            callback({
+                type: 'MAGIC_COMMENT',
+                kind: 'key',
+                keyName: data,
+                line: evt.line,
+            });
+        }
+    });
+}
+exports.default = default_1;

package/dist/extractor/machines/shared/properties.js

@@ -0,0 +1,226 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const xstate_1 = require("xstate");
+// This state machine is responsible for extracting translation key properties from an object/props
+exports.default = (0, xstate_1.createMachine)({
+    predictableActionArguments: true,
+    id: 'properties',
+    initial: 'idle',
+    context: {
+        property: null,
+        depth: 0,
+        static: false,
+        keyName: null,
+        defaultValue: null,
+        namespace: null,
+    },
+    states: {
+        idle: {
+            on: {
+                // JS/TS
+                'meta.brace.square.ts': {
+                    target: 'complex_key',
+                    cond: 'isOpenSquare',
+                },
+                'meta.object-literal.key.ts': {
+                    actions: 'storePropertyType',
+                    cond: 'isRootLevel',
+                },
+                'punctuation.separator.key-value.ts': {
+                    target: 'value',
+                    cond: 'isRootLevel',
+                },
+                'variable.other.readwrite.ts': {
+                    actions: 'markImmediatePropertyAsDynamic',
+                    cond: (_ctx, evt) => ['key', 'keyName', 'ns', 'defaultValue'].includes(evt.token),
+                },
+                // JSX/TSX
+                'entity.other.attribute-name.tsx': {
+                    actions: ['markPropertyAsDynamic', 'storePropertyType'],
+                },
+                'keyword.operator.assignment.ts': {
+                    target: 'value',
+                    actions: 'markAsStatic',
+                },
+            },
+        },
+        complex_key: {
+            on: {
+                // On string
+                'punctuation.definition.string.begin.ts': 'complex_key_string',
+                'punctuation.definition.string.template.begin.ts': 'complex_key_string',
+                // Key end
+                'meta.brace.square.ts': {
+                    target: 'idle',
+                    cond: 'isCloseSquare',
+                },
+            },
+        },
+        complex_key_string: {
+            on: {
+                '*': {
+                    target: 'idle',
+                    actions: 'storePropertyType',
+                },
+            },
+        },
+        value: {
+            on: {
+                // Extract strings
+                'punctuation.definition.string.begin.ts': 'value_string',
+                'punctuation.definition.string.template.begin.ts': 'value_string',
+                // Variable
+                'variable.other.readwrite.ts': {
+                    target: 'idle',
+                    actions: [
+                        'unmarkAsStatic',
+                        'markPropertyAsDynamic',
+                        'clearPropertyType',
+                    ],
+                },
+                // JSX Expression
+                'punctuation.section.embedded.begin.tsx': {
+                    actions: 'unmarkAsStatic',
+                },
+                // Value end
+                'punctuation.separator.comma.ts': {
+                    target: 'idle',
+                    actions: [
+                        'unmarkAsStatic',
+                        'markPropertyAsDynamic',
+                        'clearPropertyType',
+                    ],
+                },
+                'punctuation.definition.block.ts': {
+                    target: 'idle',
+                    // Replay the event to let depth update itself if necessary
+                    actions: [
+                        'unmarkAsStatic',
+                        'markPropertyAsDynamic',
+                        'clearPropertyType',
+                        (0, xstate_1.send)((_ctx, evt) => evt),
+                    ],
+                },
+            },
+        },
+        value_string: {
+            on: {
+                '*': [
+                    {
+                        target: 'idle',
+                        actions: [
+                            'storePropertyValue',
+                            'clearPropertyType',
+                            'unmarkAsStatic',
+                        ],
+                        cond: (ctx) => ctx.static,
+                    },
+                    {
+                        target: 'string_end',
+                        actions: 'storePropertyValue',
+                    },
+                ],
+            },
+        },
+        string_end: {
+            on: {
+                'punctuation.separator.comma.ts': {
+                    target: 'idle',
+                    actions: 'clearPropertyType',
+                },
+                'punctuation.definition.template-expression.begin.ts': {
+                    target: 'idle',
+                    actions: ['markPropertyAsDynamic', 'clearPropertyType'],
+                },
+                'keyword.operator.arithmetic.ts': {
+                    target: 'idle',
+                    actions: ['markPropertyAsDynamic', 'clearPropertyType'],
+                },
+                // JSX
+                'punctuation.section.embedded.end.tsx': {
+                    target: 'idle',
+                    actions: 'clearPropertyType',
+                },
+            },
+        },
+        end: {
+            type: 'final',
+            data: (ctx, evt) => ({
+                keyName: ctx.keyName,
+                defaultValue: ctx.defaultValue,
+                namespace: ctx.namespace,
+                lastEvent: evt,
+            }),
+        },
+    },
+    on: {
+        'punctuation.definition.block.ts': [
+            {
+                actions: 'incrementDepth',
+                cond: 'isOpenCurly',
+            },
+            {
+                target: 'end',
+                cond: 'isFinalCloseCurly',
+            },
+            {
+                actions: 'decrementDepth',
+                cond: 'isCloseCurly',
+            },
+        ],
+        'punctuation.definition.tag.end.tsx': {
+            target: 'end',
+            actions: 'markPropertyAsDynamic',
+        },
+    },
+}, {
+    guards: {
+        isOpenCurly: (_ctx, evt) => evt.token === '{' &&
+            !evt.scopes.includes('meta.embedded.expression.tsx'),
+        isCloseCurly: (_ctx, evt) => evt.token === '}' &&
+            !evt.scopes.includes('meta.embedded.expression.tsx'),
+        isFinalCloseCurly: (ctx, evt) => evt.token === '}' && ctx.depth === 1,
+        isOpenSquare: (_ctx, evt) => evt.token === '[',
+        isCloseSquare: (_ctx, evt) => evt.token === ']',
+        isRootLevel: (ctx) => ctx.depth === 1,
+    },
+    actions: {
+        storePropertyType: (0, xstate_1.assign)({
+            property: (_ctx, evt) => evt.token,
+        }),
+        clearPropertyType: (0, xstate_1.assign)({
+            property: (_ctx, _evt) => null,
+        }),
+        storePropertyValue: (0, xstate_1.assign)({
+            keyName: (ctx, evt) => ctx.property === 'key' || ctx.property === 'keyName'
+                ? evt.token
+                : ctx.keyName,
+            defaultValue: (ctx, evt) => ctx.property === 'defaultValue' ? evt.token : ctx.defaultValue,
+            namespace: (ctx, evt) => ctx.property === 'ns' ? evt.token : ctx.namespace,
+        }),
+        markPropertyAsDynamic: (0, xstate_1.assign)({
+            keyName: (ctx, _evt) => ctx.property === 'key' || ctx.property === 'keyName'
+                ? false
+                : ctx.keyName,
+            defaultValue: (ctx, _evt) => ctx.property === 'defaultValue' ? false : ctx.defaultValue,
+            namespace: (ctx, _evt) => ctx.property === 'ns' ? false : ctx.namespace,
+        }),
+        markImmediatePropertyAsDynamic: (0, xstate_1.assign)({
+            keyName: (ctx, evt) => evt.token === 'key' || evt.token === 'keyName' ? false : ctx.keyName,
+            defaultValue: (ctx, evt) => evt.token === 'defaultValue' ? false : ctx.defaultValue,
+            namespace: (ctx, evt) => (evt.token === 'ns' ? false : ctx.namespace),
+        }),
+        incrementDepth: (0, xstate_1.assign)({
+            depth: (ctx, _evt) => ctx.depth + 1,
+        }),
+        decrementDepth: (0, xstate_1.assign)({
+            depth: (ctx, _evt) => ctx.depth - 1,
+        }),
+        markAsStatic: (0, xstate_1.assign)({
+            static: (_ctx, _evt) => true,
+        }),
+        unmarkAsStatic: (0, xstate_1.assign)({
+            static: (_ctx, _evt) => false,
+        }),
+    },
+});

package/dist/extractor/presets/react.js

@@ -0,0 +1,29 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const path_1 = require("path");
+const xstate_1 = require("xstate");
+const react_1 = __importDefault(require("../machines/react"));
+const tokenizer_1 = __importDefault(require("../tokenizer"));
+const EXTS = ['.js', '.mjs', '.cjs', '.ts', '.mts', '.cts', '.jsx', '.tsx'];
+const TOLGEE_REACT_PKG_RE = /["'`]@tolgee\/react["'`]|@tolgee-key/;
+async function default_1(code, fileName) {
+    if (!EXTS.includes((0, path_1.extname)(fileName)) || !TOLGEE_REACT_PKG_RE.test(code)) {
+        // File is not a file that contains keys
+        return { warnings: [], keys: [] };
+    }
+    const tokens = await (0, tokenizer_1.default)(code, fileName);
+    const machine = (0, xstate_1.interpret)(react_1.default);
+    machine.start();
+    for (const token of tokens) {
+        machine.send(token);
+    }
+    const snapshot = machine.getSnapshot();
+    return {
+        warnings: snapshot.context.warnings,
+        keys: snapshot.context.keys,
+    };
+}
+exports.default = default_1;

package/dist/extractor/runner.js

@@ -0,0 +1,39 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.filterExtractionResult = exports.extractKeysOfFiles = exports.extractKeysFromFile = exports.NullNamespace = void 0;
+const glob_1 = require("glob");
+const worker_1 = require("./worker");
+const util_1 = require("util");
+const glob = (0, util_1.promisify)(glob_1.glob);
+exports.NullNamespace = Symbol('namespace.null');
+async function extractKeysFromFile(file, extractor) {
+    return (0, worker_1.callWorker)({
+        extractor: extractor,
+        file: file,
+    });
+}
+exports.extractKeysFromFile = extractKeysFromFile;
+async function extractKeysOfFiles(filesPattern, extractor) {
+    const files = await glob(filesPattern, { nodir: true });
+    const result = new Map();
+    await Promise.all(files.map(async (file) => {
+        const keys = await extractKeysFromFile(file, extractor);
+        result.set(file, keys);
+    }));
+    return result;
+}
+exports.extractKeysOfFiles = extractKeysOfFiles;
+function filterExtractionResult(data) {
+    const result = Object.create(null);
+    for (const { keys } of data.values()) {
+        for (const key of keys) {
+            const namespace = key.namespace || exports.NullNamespace;
+            if (!(namespace in result)) {
+                result[namespace] = new Map();
+            }
+            result[namespace].set(key.keyName, key.defaultValue || null);
+        }
+    }
+    return result;
+}
+exports.filterExtractionResult = filterExtractionResult;

package/dist/extractor/tokenizer.js

@@ -0,0 +1,102 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const path_1 = require("path");
+const promises_1 = require("fs/promises");
+const vscode_textmate_1 = require("vscode-textmate");
+const vscode_oniguruma_1 = require("vscode-oniguruma");
+const GRAMMAR_PATH = (0, path_1.join)(__dirname, '..', '..', 'textmate');
+const GrammarFiles = {
+    ["source.ts" /* Grammar.TYPESCRIPT */]: (0, path_1.join)(GRAMMAR_PATH, 'TypeScript.tmLanguage'),
+    ["source.tsx" /* Grammar.TYPESCRIPT_TSX */]: (0, path_1.join)(GRAMMAR_PATH, 'TypeScriptReact.tmLanguage'),
+};
+let oniguruma;
+let registry;
+async function initializeOniguruma() {
+    const wasmBlobPath = require
+        .resolve('vscode-oniguruma')
+        .replace('main.js', 'onig.wasm');
+    const wasmBlob = await (0, promises_1.readFile)(wasmBlobPath);
+    await (0, vscode_oniguruma_1.loadWASM)(wasmBlob);
+    return {
+        createOnigScanner: (patterns) => new vscode_oniguruma_1.OnigScanner(patterns),
+        createOnigString: (s) => new vscode_oniguruma_1.OnigString(s),
+    };
+}
+async function loadGrammar(scope) {
+    const file = GrammarFiles[scope];
+    if (!file)
+        return null;
+    const grammar = await (0, promises_1.readFile)(file, 'utf8');
+    return (0, vscode_textmate_1.parseRawGrammar)(grammar);
+}
+function extnameToGrammar(extname) {
+    switch (extname) {
+        case '.js':
+        case '.mjs':
+        case '.cjs':
+        case '.ts':
+        case '.mts':
+        case '.cts':
+            return "source.ts" /* Grammar.TYPESCRIPT */;
+        case '.jsx':
+        case '.tsx':
+            return "source.tsx" /* Grammar.TYPESCRIPT_TSX */;
+    }
+}
+function tokenize(code, grammar) {
+    let stack = vscode_textmate_1.INITIAL;
+    let linePtr = 0;
+    const lines = code.split('\n');
+    const tokens = [];
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        const res = grammar.tokenizeLine(line, stack);
+        for (const token of res.tokens) {
+            // Opinionated take: if a token is scope-less, chances are we don't care about it.
+            // Ditching it allows us to reduce complexity from the state machine's POV.
+            if (token.scopes.length !== 1) {
+                const codeToken = code.slice(linePtr + token.startIndex, linePtr + token.endIndex);
+                tokens.push({
+                    type: token.scopes[token.scopes.length - 1],
+                    token: codeToken,
+                    line: i + 1,
+                    startIndex: linePtr + token.startIndex,
+                    endIndex: linePtr + token.endIndex,
+                    scopes: token.scopes,
+                });
+            }
+        }
+        tokens.push({
+            type: 'newline',
+            token: '\n',
+            line: i + 1,
+            startIndex: linePtr + line.length,
+            endIndex: linePtr + line.length + 1,
+            scopes: [],
+        });
+        linePtr += line.length + 1;
+        stack = res.ruleStack;
+    }
+    return tokens;
+}
+async function default_1(code, fileName) {
+    if (!oniguruma) {
+        // Lazily initialize the WebAssembly runtime
+        oniguruma = initializeOniguruma();
+        registry = new vscode_textmate_1.Registry({
+            onigLib: oniguruma,
+            loadGrammar: loadGrammar,
+        });
+    }
+    const fileType = (0, path_1.extname)(fileName);
+    const grammarName = extnameToGrammar(fileType);
+    if (!grammarName) {
+        throw new Error(`Cannot find grammar for file type ${fileType}`);
+    }
+    const grammar = await registry.loadGrammar(grammarName);
+    if (!grammar) {
+        throw new Error(`Cannot load grammar ${grammarName}`);
+    }
+    return tokenize(code, grammar);
+}
+exports.default = default_1;

package/dist/extractor/warnings.js

@@ -0,0 +1,89 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.emitGitHubWarning = exports.dumpWarnings = exports.WarningMessages = void 0;
+const path_1 = require("path");
+exports.WarningMessages = {
+    W_DYNAMIC_KEY: {
+        name: 'Dynamic key',
+        description: 'The key not static and cannot be extracted automatically. This key will be ignored.\n\nUse @tolgee-key or @tolgee-ignore to suppress this warning.',
+    },
+    W_DYNAMIC_NAMESPACE: {
+        name: 'Dynamic namespace',
+        description: 'The namespace for this key is not static and cannot be extracted automatically. This key will be ignored.\n\nUse @tolgee-key or @tolgee-ignore to suppress this warning.',
+    },
+    W_DYNAMIC_DEFAULT_VALUE: {
+        name: 'Dynamic default value',
+        description: 'The default value for this key is not static and cannot be extracted automatically.\n\nUse @tolgee-key or @tolgee-ignore to suppress this warning.',
+    },
+    W_DYNAMIC_OPTIONS: {
+        name: 'Dynamic options',
+        description: 'The options for this key are not static and cannot be extracted automatically. This key will be ignored.\n\nUse @tolgee-key or @tolgee-ignore to suppress this warning.',
+    },
+    W_UNRESOLVABLE_NAMESPACE: {
+        name: 'Cannot resolve namespace',
+        description: 'The namespace for this key cannot be resolved. This key will be ignored.\n\nUse @tolgee-key or @tolgee-ignore to suppress this warning.',
+    },
+    W_UNUSED_IGNORE: {
+        name: 'Unused ignore directive',
+        description: 'This directive is unused.',
+    },
+    W_MALFORMED_KEY_OVERRIDE: {
+        name: 'Malformed key override',
+        description: "Couldn't parse the JSON5 object.",
+    },
+    W_INVALID_KEY_OVERRIDE: {
+        name: 'Invalid key override',
+        description: 'The `key` must be present, and the `key`, `ns`, and `defaultValue` must be strings.',
+    },
+};
+/**
+ * Dumps warnings emitted during an extraction to stdout, with GitHub integration, and counts them.
+ *
+ * @param extractionResult Extraction result to dump warnings from.
+ * @returns Count of emitted warnings in the extraction.
+ */
+function dumpWarnings(extractionResult) {
+    let warningCount = 0;
+    for (const [file, { warnings }] of extractionResult.entries()) {
+        if (warnings.length) {
+            if (!warningCount) {
+                console.error('Warnings were emitted during extraction.');
+            }
+            console.error(file);
+            for (const warning of warnings) {
+                const warnText = warning.warning in exports.WarningMessages
+                    ? exports.WarningMessages[warning.warning].name
+                    : warning.warning;
+                console.error('\tline %d: %s', warning.line, warnText);
+                emitGitHubWarning(warning.warning, file, warning.line);
+            }
+            warningCount += warnings.length;
+        }
+    }
+    if (warningCount) {
+        console.error('Total: %d warning%s', warningCount, warningCount === 1 ? '' : 's');
+    }
+    return warningCount;
+}
+exports.dumpWarnings = dumpWarnings;
+// TODO: Revisit this function and turn it into an actual GitHub Action Annotation?
+// https://docs.github.com/en/rest/checks/runs?apiVersion=2022-11-28#update-a-check-run
+function emitGitHubWarning(warning, file, line) {
+    if (!process.env.GITHUB_ACTIONS)
+        return;
+    file = (0, path_1.relative)(process.env.GITHUB_WORKSPACE ?? process.cwd(), file);
+    if (warning in exports.WarningMessages) {
+        const { name, description } = exports.WarningMessages[warning];
+        const encodedDescription = description
+            .replaceAll('%', '%25')
+            .replaceAll('\r', '%0D')
+            .replaceAll('\n', '%0A')
+            .replaceAll(':', '%3A')
+            .replaceAll(',', '%2C')
+            .replaceAll('@', '@​'); // This has a ZWS; preventing mentions
+        console.log(`::warning file=${file},line=${line},title=${name}::${encodedDescription}`);
+        return;
+    }
+    console.log(`::warning file=${file},line=${line}::${warning}`);
+}
+exports.emitGitHubWarning = emitGitHubWarning;

package/dist/extractor/worker.js

@@ -0,0 +1,82 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.callWorker = void 0;
+const path_1 = require("path");
+const worker_threads_1 = require("worker_threads");
+const promises_1 = require("fs/promises");
+// TODO: this solution won't handle new integrations and it will need a slight tweaking before adding new ones
+const react_1 = __importDefault(require("./presets/react"));
+const moduleLoader_1 = require("../utils/moduleLoader");
+const deferred_1 = require("../utils/deferred");
+const IS_TS_NODE = (0, path_1.extname)(__filename) === '.ts';
+// --- Worker functions
+let loadedExtractor = Symbol('unloaded');
+let extractor;
+async function handleJob(args) {
+    if (loadedExtractor !== args.extractor) {
+        loadedExtractor = args.extractor;
+        extractor = args.extractor
+            ? await (0, moduleLoader_1.loadModule)(args.extractor).then((mdl) => mdl.default)
+            : react_1.default;
+    }
+    const file = (0, path_1.resolve)(args.file);
+    const code = await (0, promises_1.readFile)(file, 'utf8');
+    return extractor(code, file);
+}
+async function workerInit() {
+    worker_threads_1.parentPort.on('message', (params) => {
+        handleJob(params)
+            .then((res) => worker_threads_1.parentPort.postMessage({ data: res }))
+            .catch((e) => worker_threads_1.parentPort.postMessage({ err: e }));
+    });
+}
+if (!worker_threads_1.isMainThread)
+    workerInit();
+// --- Main thread functions
+let worker;
+const jobQueue = [];
+function createWorker() {
+    const worker = new worker_threads_1.Worker(__filename, {
+        // ts-node workaround
+        execArgv: IS_TS_NODE ? ['--require', 'ts-node/register'] : undefined,
+    });
+    let timeout;
+    let currentDeferred;
+    function workOrDie() {
+        const job = jobQueue.shift();
+        if (!job) {
+            worker.terminate();
+            return;
+        }
+        worker.postMessage(job[0]);
+        currentDeferred = job[1];
+        timeout = setTimeout(() => {
+            worker.terminate();
+            currentDeferred.reject(new Error('aborted'));
+        }, 10e3);
+    }
+    worker.on('message', (msg) => {
+        if ('data' in msg) {
+            currentDeferred.resolve(msg.data);
+        }
+        else {
+            currentDeferred.reject(msg.err);
+        }
+        clearTimeout(timeout);
+        workOrDie();
+    });
+    workOrDie();
+    return worker;
+}
+async function callWorker(params) {
+    const deferred = (0, deferred_1.createDeferred)();
+    jobQueue.push([params, deferred]);
+    if (!worker) {
+        worker = createWorker();
+    }
+    return deferred.promise;
+}
+exports.callWorker = callWorker;