@tolgee/cli 1.1.2 → 1.3.0

package/dist/extractor/machines/vue/extract.js

@@ -0,0 +1,491 @@
+import { createMachine, assign, send, forwardTo } from 'xstate';
+import translateCallMachine from '../shared/translateCall.js';
+import propertiesMachine from '../shared/properties.js';
+import commentsService from '../shared/comments.js';
+const VOID_KEY = { keyName: '', line: -1 };
+export default createMachine({
+    predictableActionArguments: true,
+    id: 'vueExtractor',
+    type: 'parallel',
+    context: {
+        line: 0,
+        key: VOID_KEY,
+        useTranslate: null,
+        ignore: null,
+        currentState: 0 /* State.EXTERNAL */,
+        keys: [],
+        warnings: [],
+    },
+    states: {
+        comments: {
+            invoke: {
+                id: 'comments',
+                src: () => commentsService,
+            },
+            on: {
+                // Service messages
+                MAGIC_COMMENT: [
+                    {
+                        actions: 'ignoreNextLine',
+                        cond: (_ctx, evt) => evt.kind === 'ignore',
+                    },
+                    {
+                        actions: 'pushImmediateKey',
+                        cond: (_ctx, evt) => evt.kind === 'key',
+                    },
+                ],
+                WARNING: {
+                    actions: 'pushWarning',
+                },
+                // Code messages
+                'comment.line.double-slash.ts': {
+                    actions: send((_ctx, evt) => ({
+                        type: 'COMMENT',
+                        data: evt.token,
+                        line: evt.line,
+                    }), { to: 'comments' }),
+                },
+                'comment.block.ts': {
+                    actions: send((_ctx, evt) => ({
+                        type: 'COMMENT',
+                        data: evt.token,
+                        line: evt.line,
+                    }), { to: 'comments' }),
+                },
+                'comment.block.html': {
+                    actions: send((_ctx, evt) => ({
+                        type: 'COMMENT',
+                        data: evt.token,
+                        line: evt.line,
+                    }), { to: 'comments' }),
+                },
+                newline: {
+                    actions: 'warnUnusedIgnore',
+                    cond: (ctx, evt) => ctx.ignore?.type === 'ignore' && ctx.ignore.line === evt.line,
+                },
+            },
+        },
+        useTranslate: {
+            initial: 'idle',
+            states: {
+                idle: {
+                    on: {
+                        'entity.name.function.ts': {
+                            target: 'func',
+                            actions: 'storeLine',
+                            cond: (ctx, evt) => (ctx.currentState === 1 /* State.SETUP */ ||
+                                ctx.currentState === 0 /* State.EXTERNAL */) &&
+                                evt.token === 'useTranslate',
+                        },
+                    },
+                },
+                func: {
+                    on: {
+                        '*': 'idle',
+                        newline: undefined,
+                        'meta.block.ts': undefined,
+                        'meta.var.expr.ts': undefined,
+                        'meta.brace.round.ts': [
+                            {
+                                target: 'idle',
+                                actions: 'consumeIgnoredLine',
+                                cond: (ctx, evt) => ctx.ignore?.line === ctx.line &&
+                                    ctx.ignore.type === 'ignore' &&
+                                    evt.token === '(',
+                            },
+                            {
+                                target: 'call',
+                                cond: (_ctx, evt) => evt.token === '(',
+                            },
+                        ],
+                    },
+                },
+                call: {
+                    on: {
+                        'punctuation.definition.string.begin.ts': 'namespace',
+                        'punctuation.definition.string.template.begin.ts': 'namespace',
+                        'variable.other.readwrite.ts': {
+                            target: 'idle',
+                            actions: ['storeUseTranslate', 'markUseTranslateAsDynamic'],
+                        },
+                        'meta.brace.round.ts': {
+                            target: 'idle',
+                            cond: (_ctx, evt) => evt.token === ')',
+                            actions: 'storeUseTranslate',
+                        },
+                    },
+                },
+                namespace: {
+                    on: {
+                        '*': {
+                            target: 'namespace_end',
+                            actions: 'storeNamespacedUseTranslate',
+                        },
+                    },
+                },
+                namespace_end: {
+                    on: {
+                        'punctuation.separator.comma.ts': 'idle',
+                        'meta.brace.round.ts': 'idle',
+                        'punctuation.definition.template-expression.begin.ts': {
+                            target: 'idle',
+                            actions: 'markUseTranslateAsDynamic',
+                        },
+                        'keyword.operator.arithmetic.ts': {
+                            target: 'idle',
+                            actions: 'markUseTranslateAsDynamic',
+                        },
+                    },
+                },
+                done: { type: 'final' },
+            },
+        },
+        t: {
+            initial: 'idle',
+            states: {
+                idle: {
+                    on: {
+                        'variable.other.object.ts': {
+                            target: 'tRef',
+                            actions: 'storeLine',
+                            cond: (ctx, evt) => ctx.currentState !== 2 /* State.SCRIPT */ &&
+                                ctx.currentState !== 3 /* State.TEMPLATE */ &&
+                                ctx.useTranslate !== null &&
+                                evt.token === 't',
+                        },
+                        'entity.name.function.ts': [
+                            {
+                                target: 'func',
+                                actions: 'storeLine',
+                                cond: (ctx, evt) => ctx.currentState === 3 /* State.TEMPLATE */ &&
+                                    ctx.useTranslate !== null &&
+                                    evt.token === 't',
+                            },
+                            {
+                                target: 'func',
+                                actions: 'storeLine',
+                                cond: (ctx, evt) => ctx.currentState === 3 /* State.TEMPLATE */ && evt.token === '$t',
+                            },
+                        ],
+                        'variable.language.this.ts': {
+                            cond: (ctx) => ctx.currentState !== 3 /* State.TEMPLATE */ &&
+                                ctx.currentState !== 0 /* State.EXTERNAL */,
+                            actions: 'storeLine',
+                            target: 'this',
+                        },
+                    },
+                },
+                this: {
+                    on: {
+                        'meta.function-call.ts': undefined,
+                        'punctuation.accessor.ts': 'thisDot',
+                        '*': 'idle',
+                    },
+                },
+                thisDot: {
+                    on: {
+                        'meta.function-call.ts': undefined,
+                        'entity.name.function.ts': {
+                            cond: (_, evt) => evt.token === '$t',
+                            target: 'func',
+                        },
+                    },
+                },
+                tRef: {
+                    on: {
+                        'meta.function-call.ts': undefined,
+                        'punctuation.accessor.ts': 'tRefDot',
+                        '*': 'idle',
+                    },
+                },
+                tRefDot: {
+                    on: {
+                        'meta.function-call.ts': undefined,
+                        'entity.name.function.ts': {
+                            cond: (_, evt) => evt.token === 'value',
+                            target: 'func',
+                        },
+                    },
+                },
+                func: {
+                    on: {
+                        '*': 'idle',
+                        newline: undefined,
+                        'source.ts': undefined,
+                        'source.ts.embedded.html.vue': undefined,
+                        'meta.brace.round.ts': [
+                            {
+                                target: 'idle',
+                                actions: 'consumeIgnoredLine',
+                                cond: (ctx, evt) => ctx.ignore?.line === ctx.line && evt.token === '(',
+                            },
+                            {
+                                target: 'call',
+                                cond: (_ctx, evt) => evt.token === '(',
+                            },
+                        ],
+                    },
+                },
+                call: {
+                    invoke: {
+                        id: 'tCall',
+                        src: translateCallMachine,
+                        onDone: [
+                            {
+                                target: 'idle',
+                                actions: 'dynamicKeyName',
+                                cond: (_, evt) => evt.data.keyName === false,
+                            },
+                            {
+                                target: 'idle',
+                                actions: 'dynamicNamespace',
+                                cond: (_, evt) => evt.data.namespace === false,
+                            },
+                            {
+                                target: 'idle',
+                                actions: 'dynamicOptions',
+                                cond: (_, evt) => evt.data.dynamicOptions,
+                            },
+                            {
+                                target: 'idle',
+                                cond: (_, evt) => !evt.data.keyName,
+                            },
+                            {
+                                target: 'idle',
+                                actions: 'consumeTranslateCall',
+                            },
+                        ],
+                    },
+                    on: {
+                        '*': {
+                            actions: forwardTo('tCall'),
+                        },
+                    },
+                },
+            },
+        },
+        component: {
+            initial: 'idle',
+            states: {
+                idle: {
+                    on: {
+                        'punctuation.definition.tag.begin.html': {
+                            target: 'tag',
+                            actions: 'storeLine',
+                        },
+                    },
+                },
+                tag: {
+                    on: {
+                        '*': 'idle',
+                        newline: undefined,
+                        'text.html.derivative': undefined,
+                        'entity.name.tag.T.html.vue': [
+                            {
+                                target: 'idle',
+                                actions: 'consumeIgnoredLine',
+                                cond: (ctx) => ctx.ignore?.line === ctx.line,
+                            },
+                            {
+                                target: 'props',
+                            },
+                        ],
+                    },
+                },
+                props: {
+                    invoke: {
+                        id: 'propertiesMachine',
+                        src: propertiesMachine,
+                        onDone: [
+                            {
+                                target: 'idle',
+                                actions: 'emitWarningFromParameters',
+                                cond: 'isPropertiesDataDynamic',
+                            },
+                            {
+                                target: 'idle',
+                                actions: ['consumeParameters', 'pushKey'],
+                                cond: (ctx, evt) => evt.data.lastEvent.token !== '/>' &&
+                                    ((!ctx.key.keyName && !evt.data.keyName) ||
+                                        (!ctx.key.defaultValue && !evt.data.defaultValue)),
+                            },
+                            {
+                                target: 'idle',
+                                actions: ['consumeParameters', 'pushKey'],
+                            },
+                        ],
+                    },
+                    on: {
+                        '*': {
+                            actions: forwardTo('propertiesMachine'),
+                        },
+                    },
+                },
+            },
+        },
+    },
+    on: {
+        SETUP: { actions: 'markAsInSetup' },
+        SCRIPT: { actions: 'markAsInScript' },
+        TEMPLATE: { actions: 'markAsInTemplate' },
+    },
+}, {
+    guards: {
+        isPropertiesDataDynamic: (_ctx, evt) => evt.data.keyName === false || evt.data.namespace === false,
+    },
+    actions: {
+        storeLine: assign({
+            line: (_ctx, evt) => evt.line,
+        }),
+        ignoreNextLine: assign({
+            ignore: (_ctx, evt) => ({ type: 'ignore', line: evt.line + 1 }),
+        }),
+        consumeIgnoredLine: assign({
+            ignore: (_ctx, _evt) => null,
+        }),
+        warnUnusedIgnore: assign({
+            warnings: (ctx, evt) => [
+                ...ctx.warnings,
+                { warning: 'W_UNUSED_IGNORE', line: evt.line - 1 },
+            ],
+        }),
+        storeUseTranslate: assign({
+            useTranslate: (_ctx, _evt) => '',
+        }),
+        storeNamespacedUseTranslate: assign({
+            useTranslate: (_ctx, evt) => evt.token,
+        }),
+        markUseTranslateAsDynamic: assign({
+            useTranslate: (_ctx, _evt) => false,
+            warnings: (ctx, _evt) => [
+                ...ctx.warnings,
+                { warning: 'W_DYNAMIC_NAMESPACE', line: ctx.line },
+            ],
+        }),
+        consumeTranslateCall: assign({
+            warnings: (ctx, evt) => {
+                if (!evt.data.namespace && ctx.useTranslate === false) {
+                    return [
+                        ...ctx.warnings,
+                        { warning: 'W_UNRESOLVABLE_NAMESPACE', line: ctx.line },
+                    ];
+                }
+                if (evt.data.defaultValue === false) {
+                    return [
+                        ...ctx.warnings,
+                        { warning: 'W_DYNAMIC_DEFAULT_VALUE', line: ctx.line },
+                    ];
+                }
+                return ctx.warnings;
+            },
+            keys: (ctx, evt) => {
+                const ns = evt.data.namespace === null ? ctx.useTranslate : evt.data.namespace;
+                if (!evt.data.keyName || ns === false)
+                    return ctx.keys;
+                return [
+                    ...ctx.keys,
+                    {
+                        keyName: evt.data.keyName,
+                        namespace: ns || undefined,
+                        defaultValue: evt.data.defaultValue || undefined,
+                        line: ctx.line,
+                    },
+                ];
+            },
+        }),
+        consumeParameters: assign({
+            key: (ctx, evt) => ({
+                keyName: ctx.key.keyName || evt.data.keyName,
+                defaultValue: ctx.key.defaultValue || evt.data.defaultValue || undefined,
+                namespace: evt.data.namespace ?? ctx.key.namespace,
+                line: ctx.line,
+            }),
+            warnings: (ctx, evt) => {
+                if (evt.data.defaultValue !== false)
+                    return ctx.warnings;
+                return [
+                    ...ctx.warnings,
+                    { warning: 'W_DYNAMIC_DEFAULT_VALUE', line: ctx.line },
+                ];
+            },
+        }),
+        emitWarningFromParameters: assign({
+            warnings: (ctx, evt) => [
+                ...ctx.warnings,
+                {
+                    warning: evt.data.keyName === false
+                        ? 'W_DYNAMIC_KEY'
+                        : 'W_DYNAMIC_NAMESPACE',
+                    line: ctx.line,
+                },
+            ],
+            key: VOID_KEY,
+        }),
+        dynamicKeyName: assign({
+            warnings: (ctx, _evt) => [
+                ...ctx.warnings,
+                { warning: 'W_DYNAMIC_KEY', line: ctx.line },
+            ],
+            key: VOID_KEY,
+        }),
+        dynamicNamespace: assign({
+            warnings: (ctx, _evt) => [
+                ...ctx.warnings,
+                { warning: 'W_DYNAMIC_NAMESPACE', line: ctx.line },
+            ],
+            key: VOID_KEY,
+        }),
+        dynamicOptions: assign({
+            key: VOID_KEY,
+            warnings: (ctx, _evt) => [
+                ...ctx.warnings,
+                { warning: 'W_DYNAMIC_OPTIONS', line: ctx.line },
+            ],
+        }),
+        pushKey: assign({
+            keys: (ctx, _evt) => {
+                if (!ctx.key.keyName || ctx.key.namespace === false)
+                    return ctx.keys;
+                return [
+                    ...ctx.keys,
+                    {
+                        keyName: ctx.key.keyName.trim(),
+                        namespace: ctx.key.namespace === ''
+                            ? undefined
+                            : ctx.key.namespace?.trim(),
+                        defaultValue: ctx.key.defaultValue?.trim().replace(/\s+/g, ' '),
+                        line: ctx.line,
+                    },
+                ];
+            },
+            key: (_ctx, _evt) => ({ keyName: '', line: 0 }),
+        }),
+        pushImmediateKey: assign({
+            ignore: (_ctx, evt) => ({ type: 'key', line: evt.line + 1 }),
+            keys: (ctx, evt) => [
+                ...ctx.keys,
+                {
+                    keyName: evt.keyName,
+                    namespace: evt.namespace,
+                    defaultValue: evt.defaultValue,
+                    line: evt.line,
+                },
+            ],
+        }),
+        pushWarning: assign({
+            warnings: (ctx, evt) => [
+                ...ctx.warnings,
+                { warning: evt.kind, line: evt.line },
+            ],
+        }),
+        markAsInSetup: assign({
+            currentState: 1 /* State.SETUP */,
+        }),
+        markAsInScript: assign({
+            currentState: 2 /* State.SCRIPT */,
+        }),
+        markAsInTemplate: assign({
+            currentState: 3 /* State.TEMPLATE */,
+        }),
+    },
+});

package/dist/extractor/processors/vueSfc.js

@@ -0,0 +1,55 @@
+// Vue SFC are a bit of a nightmare. :D
+//
+// 1: We need to extract the script setup, script and template blocks.
+// 2: We need to resolve them if necessary (script src="" is allowed).
+// 3: We need to define uses of `useTranslate` in setup script.
+// 4: We finally can go through the template file.
+import { interpret } from 'xstate';
+import tokenizer from '../tokenizer.js';
+import vueSfcDecoderMachine from '../machines/vue/decoder.js';
+import vueSfcExtractorMachine from '../machines/vue/extract.js';
+function extractSfcTokens(tokens) {
+    const machine = interpret(vueSfcDecoderMachine);
+    machine.start();
+    for (let i = 0; i < tokens.length; i++) {
+        machine.send(tokens[i]);
+    }
+    const snapshot = machine.getSnapshot();
+    return snapshot.context;
+}
+function sort(a, b) {
+    if (a.line === b.line)
+        return 0;
+    if (a.line > b.line)
+        return 1;
+    return -1;
+}
+export default async function handleVueSfc(code, fileName) {
+    const tokens = await tokenizer(code, fileName);
+    const decoded = extractSfcTokens(tokens);
+    const machine = interpret(vueSfcExtractorMachine);
+    machine.start();
+    machine.send({ type: 'SETUP' });
+    for (let i = 0; i < decoded.setup.length; i++) {
+        machine.send(decoded.setup[i]);
+    }
+    machine.send({ type: 'SCRIPT' });
+    for (let i = 0; i < decoded.script.length; i++) {
+        machine.send(decoded.script[i]);
+    }
+    machine.send({ type: 'TEMPLATE' });
+    for (let i = 0; i < decoded.template.length; i++) {
+        machine.send(decoded.template[i]);
+    }
+    const snapshot = machine.getSnapshot();
+    const warnings = decoded.invalidSetup
+        ? [
+            ...snapshot.context.warnings,
+            { warning: 'W_VUE_SETUP_IS_A_REFERENCE', line: decoded.invalidSetup },
+        ]
+        : snapshot.context.warnings;
+    return {
+        warnings: warnings.sort(sort),
+        keys: snapshot.context.keys.sort(sort),
+    };
+}

package/dist/extractor/tokenizer.js

@@ -8,6 +8,9 @@ const GrammarFiles = {
     ["source.ts" /* Grammar.TYPESCRIPT */]: new URL('TypeScript.tmLanguage', GRAMMAR_PATH),
     ["source.tsx" /* Grammar.TYPESCRIPT_TSX */]: new URL('TypeScriptReact.tmLanguage', GRAMMAR_PATH),
     ["source.svelte" /* Grammar.SVELTE */]: new URL('Svelte.tmLanguage', GRAMMAR_PATH),
+    ["source.vue" /* Grammar.VUE */]: new URL('Vue.tmLanguage', GRAMMAR_PATH),
+    ["text.html.basic" /* Grammar.HTML */]: new URL('HTML.tmLanguage', GRAMMAR_PATH),
+    ["text.html.derivative" /* Grammar.HTML_D */]: new URL('HTML.tmLanguage', GRAMMAR_PATH),
 };
 let oniguruma;
 let registry;
@@ -28,9 +31,7 @@ async function loadGrammar(scope) {
     if (!file)
         return null;
     const grammar = await readFile(file, 'utf8');
-    return grammar.startsWith('{')
-        ? JSON.parse(grammar)
-        : TextMate.parseRawGrammar(grammar);
+    return JSON.parse(grammar);
 }
 function extnameToGrammar(extname) {
     switch (extname) {
@@ -44,8 +45,12 @@ function extnameToGrammar(extname) {
         case '.jsx':
         case '.tsx':
             return "source.tsx" /* Grammar.TYPESCRIPT_TSX */;
+        case '.vue':
+            return "source.vue" /* Grammar.VUE */;
         case '.svelte':
             return "source.svelte" /* Grammar.SVELTE */;
+        case '.html':
+            return "text.html.basic" /* Grammar.HTML */;
     }
 }
 function tokenize(code, grammar) {
@@ -57,10 +62,10 @@ function tokenize(code, grammar) {
         const line = lines[i];
         const res = grammar.tokenizeLine(line, stack);
         for (const token of res.tokens) {
-            // Opinionated take: if a token is scope-less, chances are we don't care about it.
+            const codeToken = code.slice(linePtr + token.startIndex, linePtr + token.endIndex);
+            // Opinionated take: if a token is scope-less and void, chances are we don't care about it.
             // Ditching it allows us to reduce complexity from the state machine's POV.
-            if (token.scopes.length !== 1) {
-                const codeToken = code.slice(linePtr + token.startIndex, linePtr + token.endIndex);
+            if (token.scopes.length !== 1 || codeToken.trim()) {
                 tokens.push({
                     type: token.scopes[token.scopes.length - 1],
                     token: codeToken,

package/dist/extractor/warnings.js

@@ -32,6 +32,10 @@ export const WarningMessages = {
         name: 'Invalid key override',
         description: 'The `key` must be present, and the `key`, `ns`, and `defaultValue` must be strings.',
     },
+    W_VUE_SETUP_IS_A_REFERENCE: {
+        name: 'Vue setup function is a reference',
+        description: 'The setup function must be directly defined on-site, and not be a reference to a previously defined function.',
+    },
 };
 /**
  * Dumps warnings emitted during an extraction to stdout, with GitHub integration, and counts them.

package/dist/extractor/worker.js

@@ -1,3 +1,4 @@
+import { fileURLToPath } from 'url';
 import { resolve, extname } from 'path';
 import { Worker, isMainThread, parentPort } from 'worker_threads';
 import { readFile } from 'fs/promises';
@@ -33,11 +34,11 @@ let worker;
 const jobQueue = [];
 function createWorker() {
     const worker = IS_TS_NODE
-        ? new Worker(new URL(import.meta.url).pathname.replace('.ts', '.js'), {
+        ? new Worker(fileURLToPath(new URL(import.meta.url)).replace('.ts', '.js'), {
             // ts-node workaround
             execArgv: ['--require', 'ts-node/register'],
         })
-        : new Worker(new URL(import.meta.url).pathname);
+        : new Worker(fileURLToPath(new URL(import.meta.url)));
     let timeout;
     let currentDeferred;
     function workOrDie() {

package/extractor.d.ts

@@ -5,12 +5,14 @@ export type Key = {
 };
 export type ExtractedKey = Key & {
     line: number;
+    /** Specified when the file differs from the file being processed (sub-file) */
+    file?: string;
 };
 export type Warning = {
     warning: string;
     line: number;
 };
-export type Extractor = (fileContents: string, fileName: string) => string[];
+export type Extractor = (fileContents: string, fileName: string) => ExtractionResult[];
 export type ExtractionResult = {
     keys: ExtractedKey[];
     warnings: Warning[];