@tolgee/cli 2.0.4 → 2.1.0

package/dist/extractor/runner.js

@@ -1,17 +1,65 @@
 import { glob } from 'glob';
+import { extname } from 'path';
 import { callWorker } from './worker.js';
+import { exitWithError } from '../utils/logger.js';
 export const NullNamespace = Symbol('namespace.null');
-export async function extractKeysFromFile(file, extractor) {
+function parseVerbose(v) {
+    return Array.isArray(v) ? v : v ? [] : undefined;
+}
+export async function extractKeysFromFile(file, parserType, options, extractor) {
     return callWorker({
         extractor: extractor,
+        parserType,
         file: file,
+        options,
     });
 }
-export async function extractKeysOfFiles(filesPatterns, extractor) {
-    const files = await glob(filesPatterns, { nodir: true });
+export function findPossibleFrameworks(fileNames) {
+    const possibleFrameworks = [];
+    const extensions = new Set(fileNames.map((name) => extname(name)));
+    if (extensions.has('.jsx') || extensions.has('.tsx')) {
+        possibleFrameworks.push('react');
+    }
+    if (extensions.has('.vue')) {
+        possibleFrameworks.push('vue');
+    }
+    if (extensions.has('.svelte')) {
+        possibleFrameworks.push('svelte');
+    }
+    return possibleFrameworks;
+}
+function detectParserType(fileNames) {
+    const possibleFrameworks = findPossibleFrameworks(fileNames);
+    if (possibleFrameworks.length === 1) {
+        return possibleFrameworks[0];
+    }
+    else if (possibleFrameworks.length === 0) {
+        exitWithError("Couldn't detect which framework is used, use '--parser' or 'config.parser' option");
+    }
+    else {
+        exitWithError(`Detected multiple possible frameworks used (${possibleFrameworks.join(', ')}), use '--parser' or 'config.parser' options`);
+    }
+}
+export async function extractKeysOfFiles(opts) {
+    if (!opts.patterns?.length) {
+        exitWithError("Missing '--patterns' or 'config.patterns' option");
+    }
+    const files = await glob(opts.patterns, { nodir: true });
+    if (files.length === 0) {
+        exitWithError('No files were matched for extraction');
+    }
+    let parserType = opts.parser;
+    if (!parserType) {
+        parserType = detectParserType(files);
+    }
     const result = new Map();
+    const options = {
+        strictNamespace: Boolean(opts.strictNamespace),
+        defaultNamespace: opts.defaultNamespace,
+        verbose: parseVerbose(opts.verbose),
+    };
     await Promise.all(files.map(async (file) => {
-        const keys = await extractKeysFromFile(file, extractor);
+        const keys = await extractKeysFromFile(file, parserType, options, opts.extractor);
         result.set(file, keys);
     }));
     return result;

package/dist/extractor/visualizers/printTokens.js

@@ -0,0 +1,7 @@
+export const tokensList = (tokens) => {
+    const result = [];
+    for (const t of tokens) {
+        result.push(`${JSON.stringify(t.token).padEnd(30)} ${t.type.padEnd(40)} ${t.customType ?? ''}`);
+    }
+    return result.join('\n');
+};

package/dist/extractor/visualizers/tokensToString.js

@@ -0,0 +1,28 @@
+import ansi from 'ansi-colors';
+export function tokensToString(tokens, originalString, getColorizer) {
+    const result = [];
+    let lastIndex = 0;
+    let lastColorizer;
+    for (const token of tokens) {
+        const colorizer = getColorizer(token);
+        if (lastIndex < token.startIndex) {
+            if (colorizer === lastColorizer) {
+                result.push(colorizer(originalString.substring(lastIndex, token.startIndex)));
+            }
+            else {
+                result.push(ansi.grey(originalString.substring(lastIndex, token.startIndex)));
+            }
+        }
+        lastIndex = token.endIndex;
+        const fullText = originalString.substring(token.startIndex, token.endIndex);
+        result.push(getColorizer(token)(fullText));
+    }
+    if (lastIndex < originalString.length) {
+        result.push(originalString.substring(lastIndex, originalString.length));
+    }
+    return result
+        .join('')
+        .split('\n')
+        .map((line, index) => `${ansi.gray(String(index + 1).padEnd(3))}${line}`)
+        .join('\n');
+}

package/dist/extractor/visualizers/visualizeRules.js

@@ -0,0 +1,40 @@
+import { tokensToString } from './tokensToString.js';
+import ansi from 'ansi-colors';
+const palette = [
+    ansi.blue,
+    ansi.red,
+    ansi.yellow,
+    ansi.bgBlue,
+    ansi.bgRed,
+    ansi.bgYellow,
+];
+export const visualizeRules = (tokens, originalString) => {
+    let paletteIndex = 0;
+    const colors = {};
+    function getColor(rule) {
+        if (!colors[rule]) {
+            colors[rule] = palette[paletteIndex % palette.length];
+            paletteIndex += 1;
+        }
+        return colors[rule];
+    }
+    const colorized = tokensToString(tokens, originalString, (token) => {
+        if (token.customType) {
+            return getColor(token.customType);
+        }
+        else {
+            return ansi.white;
+        }
+    });
+    return (colorized +
+        '\n\n' +
+        Object.entries({
+            ...colors,
+            ...{
+                unknown: ansi.white,
+                ignored: ansi.grey,
+            },
+        })
+            .map(([rule, paint]) => paint(rule))
+            .join('\n'));
+};

package/dist/extractor/warnings.js

@@ -36,6 +36,10 @@ export const WarningMessages = {
         name: 'Vue setup function is a reference',
         description: 'The setup function must be directly defined on-site, and not be a reference to a previously defined function.',
     },
+    W_MISSING_T_SOURCE: {
+        name: 'Expected source of `t` function (useTranslate or getTranslate)',
+        description: "When used like this namespace can't be reliably detected so the warning is emitted, use `--no-strict-namespace` or `config.strictNamespace: false` if you don't use namespaces",
+    },
 };
 /**
  * Dumps warnings emitted during an extraction to stdout, with GitHub integration, and counts them.

package/dist/extractor/worker.js

@@ -10,15 +10,18 @@ const IS_TS_NODE = extname(import.meta.url) === '.ts';
 let loadedExtractor = Symbol('unloaded');
 let extractor;
 async function handleJob(args) {
-    if (loadedExtractor !== args.extractor) {
-        loadedExtractor = args.extractor;
-        extractor = args.extractor
-            ? await loadModule(args.extractor).then((mdl) => mdl.default)
-            : internalExtractor;
-    }
     const file = resolve(args.file);
     const code = await readFile(file, 'utf8');
-    return extractor(code, file);
+    if (args.extractor) {
+        if (args.extractor !== loadedExtractor) {
+            loadedExtractor = args.extractor;
+            extractor = await loadModule(args.extractor).then((mdl) => mdl.default);
+        }
+        return extractor(code, file, args.options);
+    }
+    else {
+        return internalExtractor(code, file, args.parserType, args.options);
+    }
 }
 async function workerInit() {
     parentPort.on('message', (params) => {

package/dist/options.js

@@ -56,3 +56,8 @@ export const FORMAT_OPT = new Option('--format <format>', 'Localization files fo
     'XLIFF_RUBY',
 ]);
 export const FILE_PATTERNS = new Option('-pt, --patterns <patterns...>', 'File glob patterns to include (hint: make sure to escape it in quotes, or your shell might attempt to unroll some tokens like *)');
+export const STRICT_NAMESPACE = new Option('--strict-namespace').hideHelp(true);
+export const STRICT_NAMESPACE_NEGATION = new Option('--no-strict-namespace', "No require namespace to be reachable, use if you don't use namespaces.");
+export const DEFAULT_NAMESPACE = new Option('--default-namespace <namespace>', 'Default namespace used in extraction if not specified otherwise.');
+export const PARSER = new Option('--parser <parser>', 'Override parser detection.').choices(['react', 'vue', 'svelte']);
+export const VERBOSE = new Option('-v, --verbose [rules...]', 'Enable verbose logging. If you want more info to be logged pass an option.').choices(['extractor']);

package/extractor.d.ts

@@ -12,7 +12,14 @@ export type Warning = {
     warning: string;
     line: number;
 };
-export type Extractor = (fileContents: string, fileName: string) => ExtractionResult[];
+export type VerboseOption = 'extractor';
+export type ExtractOptions = {
+    verbose?: VerboseOption[];
+    strictNamespace: boolean;
+    defaultNamespace: string | undefined;
+};
+export type ParserType = 'react' | 'vue' | 'svelte';
+export type Extractor = (fileContents: string, fileName: string, options: ExtractOptions) => ExtractionResult;
 export type ExtractionResult = {
     keys: ExtractedKey[];
     warnings: Warning[];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tolgee/cli",
-  "version": "2.0.4",
+  "version": "2.1.0",
   "type": "module",
   "description": "A tool to interact with the Tolgee Platform through CLI",
   "repository": {
@@ -18,7 +18,7 @@
     "test:package": "node scripts/validatePackage.js",
     "tolgee:start": "node scripts/startDocker.js",
     "tolgee:stop": "docker stop tolgee_cli_e2e",
-    "lint": "eslint --ext .ts --ext .js --ext .cjs ./src ./scripts vitest.config.ts",
+    "eslint": "eslint --max-warnings 0 --ext .ts --ext .js --ext .cjs ./src ./scripts vitest.config.ts",
     "prettier": "prettier --write ./src ./scripts vitest.config.ts",
     "run-dev": "cross-env NODE_OPTIONS=\"--import=./scripts/registerTsNode.js\" node ./src/cli.ts",
     "schema": "openapi-typescript http://localhost:22222/v3/api-docs/All%20Internal%20-%20for%20Tolgee%20Web%20application --output src/client/internal/schema.generated.ts",
@@ -37,10 +37,8 @@
     "json5": "^2.2.3",
     "jsonschema": "^1.4.1",
     "openapi-fetch": "^0.9.7",
-    "undici": "^5.22.1",
     "vscode-oniguruma": "^1.7.0",
     "vscode-textmate": "^9.0.0",
-    "xstate": "^4.38.1",
     "yauzl": "^2.10.0"
   },
   "devDependencies": {

package/schema.json

@@ -5,14 +5,17 @@
       "description": "Project ID. Only required when using a Personal Access Token.",
       "type": ["number", "string"]
     },
-    "extractor": {
-      "description": "A path to a custom extractor to use instead of the default one.",
-      "type": "string"
-    },
     "apiUrl": {
       "description": "The url of Tolgee API.",
       "type": "string"
     },
+    "format": {
+      "$ref": "#/$defs/format"
+    },
+    "extractor": {
+      "description": "A path to a custom extractor to use instead of the default one.",
+      "type": "string"
+    },
     "patterns": {
       "description": "File glob patterns to your source code, used for keys extraction.",
       "type": "array",
@@ -20,8 +23,17 @@
         "type": "string"
       }
     },
-    "format": {
-      "$ref": "#/$defs/format"
+    "strictNamespace": {
+      "description": "Require namespace to be reachable, turn off if you don't use namespaces. (Default: true)",
+      "type": "boolean"
+    },
+    "defaultNamespace": {
+      "description": "Default namespace used in extraction if not specified otherwise.",
+      "type": "string"
+    },
+    "parser": {
+      "description": "Override parser detection.",
+      "enum": ["react", "vue", "svelte"]
     },
     "push": {
       "type": "object",

package/dist/client/internal/requester.js

@@ -1,130 +0,0 @@
-import { STATUS_CODES } from 'http';
-import { request } from 'undici';
-import FormData from 'form-data';
-import { debug } from '../../utils/logger.js';
-import { USER_AGENT } from '../../constants.js';
-export default class Requester {
-    params;
-    constructor(params) {
-        this.params = params;
-    }
-    get projectUrl() {
-        return `/v2/projects/${this.params.projectId}`;
-    }
-    /**
-     * Performs an HTTP request to the API
-     *
-     * @param req Request data
-     * @returns The response
-     */
-    async request(req) {
-        const url = new URL(req.path, this.params.apiUrl);
-        if (req.query) {
-            for (const param in req.query) {
-                if (param in req.query) {
-                    const val = req.query[param];
-                    if (val !== undefined) {
-                        if (Array.isArray(val)) {
-                            for (const v of val) {
-                                url.searchParams.append(param, String(v));
-                            }
-                        }
-                        else {
-                            url.searchParams.set(param, String(val));
-                        }
-                    }
-                }
-            }
-        }
-        const headers = {
-            ...(req.headers || {}),
-            'user-agent': USER_AGENT,
-            'x-api-key': this.params.apiKey,
-        };
-        let body = undefined;
-        if (req.body) {
-            if (req.body instanceof FormData) {
-                const header = `multipart/form-data; boundary=${req.body.getBoundary()}`;
-                headers['content-type'] = header;
-                body = req.body.getBuffer();
-            }
-            else {
-                headers['content-type'] = 'application/json';
-                body = JSON.stringify(req.body);
-            }
-        }
-        debug(`[HTTP] Requesting: ${req.method} ${url}`);
-        const response = await request(url, {
-            method: req.method,
-            headers: headers,
-            body: body,
-            headersTimeout: req.headersTimeout ?? 300_000,
-            bodyTimeout: req.bodyTimeout ?? 300_000,
-        });
-        debug(`[HTTP] ${req.method} ${url} -> ${response.statusCode} ${STATUS_CODES[response.statusCode]}`);
-        return response;
-    }
-    /**
-     * Performs an HTTP request to the API and returns the result as a JSON object
-     *
-     * @param req Request data
-     * @returns The response data
-     */
-    async requestJson(req) {
-        return this.request(req).then((r) => r.body.json());
-    }
-    /**
-     * Performs an HTTP request to the API and returns the result as a Blob
-     *
-     * @param req Request data
-     * @returns The response blob
-     */
-    async requestBlob(req) {
-        return this.request(req).then((r) => r.body.blob());
-    }
-    /**
-     * Performs an HTTP request to the API.
-     */
-    async requestVoid(req) {
-        await this.request(req);
-    }
-    /**
-     * Performs an HTTP request to the API to a resource which is paginated.
-     * The returned result is a view with helpers to get next (or previous) data.
-     *
-     * @param req Request data
-     */
-    async requestPaginatedResource(req) {
-        const res = await this.requestJson(req);
-        const _this = this;
-        const view = {
-            data: res._embedded,
-            page: res.page,
-            hasNext() {
-                return !!res._links.next?.href;
-            },
-            hasPrevious() {
-                return !!res._links.prev?.href;
-            },
-            async next() {
-                if (!this.hasNext())
-                    return null;
-                return _this.requestPaginatedResource({
-                    ...req,
-                    path: res._links.next.href,
-                    query: undefined,
-                });
-            },
-            async previous() {
-                if (!this.hasPrevious())
-                    return null;
-                return _this.requestPaginatedResource({
-                    ...req,
-                    path: res._links.prev.href,
-                    query: undefined,
-                });
-            },
-        };
-        return view;
-    }
-}

package/dist/extractor/machines/comments.js

@@ -1,78 +0,0 @@
-import { createMachine, assign, send } from 'xstate';
-import commentsService from './shared/comments.js';
-export default createMachine({
-    predictableActionArguments: true,
-    id: 'commentsExtractor',
-    context: {
-        keys: [],
-        warnings: [],
-    },
-    invoke: {
-        id: 'comments',
-        src: () => commentsService,
-    },
-    on: {
-        // Service messages
-        MAGIC_COMMENT: [
-            {
-                actions: 'warnUnusedIgnore',
-                cond: (_ctx, evt) => evt.kind === 'ignore',
-            },
-            {
-                actions: 'pushKey',
-                cond: (_ctx, evt) => evt.kind === 'key',
-            },
-        ],
-        WARNING: {
-            actions: 'pushWarning',
-        },
-        // Code messages
-        'comment.line.double-slash.ts': {
-            actions: send((_ctx, evt) => ({
-                type: 'COMMENT',
-                data: evt.token,
-                line: evt.line,
-            }), { to: 'comments' }),
-        },
-        'comment.block.ts': {
-            actions: send((_ctx, evt) => ({
-                type: 'COMMENT',
-                data: evt.token,
-                line: evt.line,
-            }), { to: 'comments' }),
-        },
-        'comment.block.svelte': {
-            actions: send((_ctx, evt) => ({
-                type: 'COMMENT',
-                data: evt.token,
-                line: evt.line,
-            }), { to: 'comments' }),
-        },
-    },
-}, {
-    actions: {
-        warnUnusedIgnore: assign({
-            warnings: (ctx, evt) => [
-                ...ctx.warnings,
-                { warning: 'W_UNUSED_IGNORE', line: evt.line },
-            ],
-        }),
-        pushKey: assign({
-            keys: (ctx, evt) => [
-                ...ctx.keys,
-                {
-                    keyName: evt.keyName,
-                    namespace: evt.namespace,
-                    defaultValue: evt.defaultValue,
-                    line: evt.line,
-                },
-            ],
-        }),
-        pushWarning: assign({
-            warnings: (ctx, evt) => [
-                ...ctx.warnings,
-                { warning: evt.kind, line: evt.line },
-            ],
-        }),
-    },
-});