@tolgee/cli 2.0.3 → 2.1.0

package/dist/extractor/parser/iterator.js

@@ -0,0 +1,37 @@
+export const createIterator = (items) => {
+    const iterator = items[Symbol.iterator]();
+    let currentItem;
+    let nextItem = iterator.next();
+    const listeners = [];
+    let currentContext = undefined;
+    const self = {
+        getLineNumber() {
+            return currentItem?.line ?? 0;
+        },
+        current() {
+            return currentItem;
+        },
+        peek() {
+            return nextItem.done ? undefined : nextItem.value;
+        },
+        onAccept(listener) {
+            listeners.push(listener);
+        },
+        setLabel(context) {
+            currentContext = context;
+        },
+        getLabel() {
+            return currentContext;
+        },
+        next() {
+            const value = self.peek();
+            if (currentItem) {
+                listeners.forEach((cb) => cb(currentItem, currentContext));
+            }
+            currentItem = nextItem.done ? undefined : nextItem.value;
+            nextItem = iterator.next();
+            return value;
+        },
+    };
+    return self;
+};

package/dist/extractor/parser/mergerMachine.js

@@ -0,0 +1,66 @@
+function defaultResultToken(matched) {
+    return matched.map((t) => t.token).join('');
+}
+const MERGE_ALL = Symbol('MERGE_ALL');
+const MERGE_WITHOUT_LAST = Symbol('MERGE_WITHOUT_LAST');
+export const endOptions = {
+    MERGE_ALL,
+    MERGE_WITHOUT_LAST,
+};
+function createNewToken(tokens, customType, merger) {
+    return {
+        customType,
+        type: 'custom',
+        startIndex: tokens[0].startIndex,
+        endIndex: tokens[tokens.length - 1].endIndex,
+        scopes: [],
+        line: tokens[0].line,
+        token: merger(tokens),
+    };
+}
+export function createMachine(machine) {
+    function* generator(tokens) {
+        let state = machine.initial;
+        let stack = [];
+        for (const token of tokens) {
+            const newState = machine.step(state, token, endOptions);
+            stack.push(token);
+            if (newState === undefined) {
+                state = machine.initial;
+                for (const result of stack) {
+                    yield result;
+                }
+                stack = [];
+                continue;
+            }
+            else if (newState === endOptions.MERGE_ALL ||
+                newState === endOptions.MERGE_WITHOUT_LAST) {
+                const lastToken = newState === endOptions.MERGE_WITHOUT_LAST ? stack.pop() : undefined;
+                const newToken = createNewToken(stack, machine.customType, machine.resultToken ?? defaultResultToken);
+                state = machine.initial;
+                stack = [];
+                yield newToken;
+                if (lastToken) {
+                    yield lastToken;
+                }
+                continue;
+            }
+            else {
+                state = newState;
+                continue;
+            }
+        }
+    }
+    return generator;
+}
+export function pipeMachines(definitions) {
+    const machines = definitions.map((def) => createMachine(def));
+    function generator(tokens) {
+        let items = tokens;
+        for (const machine of machines) {
+            items = machine(items);
+        }
+        return items;
+    }
+    return generator;
+}

package/dist/extractor/parser/nodeUtils.js

@@ -0,0 +1,21 @@
+export function isPrimitive(node) {
+    return node.type === 'primitive';
+}
+export function isString(node) {
+    return node.type === 'primitive' && typeof node.value === 'string';
+}
+export function isEmptyExpr(node) {
+    return node.type === 'expr' && node.values.length === 0;
+}
+export function extractString(node) {
+    return node && isString(node) ? node.value : undefined;
+}
+export function extractValue(node) {
+    return node && node.type === 'primitive' ? node.value : undefined;
+}
+export function simplifyNode(node) {
+    if (node?.type === 'expr' && node.values.length === 1) {
+        return node.values[0];
+    }
+    return node;
+}

package/dist/extractor/parser/parser.js

@@ -0,0 +1,129 @@
+import { extractComment } from './extractComment.js';
+import { createIterator, } from './iterator.js';
+import { generateReport } from './generateReport.js';
+import { parseGeneral } from './tree/parseGeneral.js';
+import { commentsMerger } from './tokenMergers/commentsMerger.js';
+import { createMachine } from './mergerMachine.js';
+import { stringMerger } from './tokenMergers/stringMerger.js';
+import { templateStringMerger } from './tokenMergers/templateStringMerger.js';
+import { closingTagMerger } from './tokenMergers/closingTagMerger.js';
+import { typesAsMergerer } from './tokenMergers/typesAsMergerer.js';
+import { typesCastMergerer } from './tokenMergers/typesCastMerger.js';
+export const DEFAULT_BLOCKS = {
+    'block.begin': ['block.end'],
+    'expression.begin': ['expression.end'],
+    'expression.template.begin': ['expression.template.end'],
+    'tag.regular.begin': [
+        'tag.closing',
+        'tag.self-closing.end',
+        'tag.closing.begin',
+    ],
+};
+export const DEFAULT_MERGERERS = [
+    stringMerger,
+    templateStringMerger,
+    closingTagMerger,
+    typesAsMergerer,
+    typesCastMergerer,
+];
+export const Parser = ({ mappers, blocks, rules, merger, treeTransform, }) => {
+    const ruleMap = new Map();
+    rules.forEach(({ trigger, call }) => {
+        ruleMap.set(trigger, call);
+    });
+    let iterator;
+    function getCurrentLine() {
+        return iterator.getLineNumber();
+    }
+    return {
+        parse({ tokens, onAccept, options }) {
+            for (const t of tokens) {
+                // use first mapper, which gives some result
+                const type = mappers.find((mapper) => mapper(t))?.(t);
+                t.customType = type;
+            }
+            const mergedComments = [...createMachine(commentsMerger)(tokens)];
+            const comments = mergedComments
+                .filter((c) => c.customType === 'comment')
+                .map((token) => extractComment(token))
+                .filter(Boolean);
+            // remove whitespaces and comments
+            const filteredWhitespaces = mergedComments.filter((t) => {
+                if (t.customType === 'comment') {
+                    return false;
+                }
+                if (t.customType === 'ignore') {
+                    return false;
+                }
+                if (!t.customType && t.token.match(/^[\s]+$/gmu)) {
+                    return false;
+                }
+                return true;
+            });
+            const mergedTokens = [...merger(filteredWhitespaces)];
+            // remove ignored after merge
+            const filteredIgnored = mergedTokens.filter((t) => {
+                if (t.customType === 'ignore') {
+                    return false;
+                }
+                return true;
+            });
+            iterator = createIterator(filteredIgnored);
+            const context = {
+                tokens: iterator,
+                getCurrentLine,
+                withLabel,
+                ruleMap,
+                blocks,
+            };
+            let depth = 0;
+            function withLabel(fn) {
+                return (...args) => {
+                    let label;
+                    const currentTokenName = context.tokens.current()?.customType || '';
+                    const isTrigger = currentTokenName.startsWith('trigger.');
+                    if (!isTrigger) {
+                        depth += 1;
+                        label = `depth.${depth}`;
+                    }
+                    else {
+                        label = currentTokenName;
+                    }
+                    const previous = iterator.getLabel();
+                    iterator.setLabel(label);
+                    const result = fn(...args);
+                    iterator.setLabel(previous);
+                    if (!isTrigger) {
+                        depth -= 1;
+                    }
+                    return result;
+                };
+            }
+            iterator.next();
+            if (onAccept) {
+                iterator.onAccept(onAccept);
+            }
+            let rootNode = parseGeneral(context, {
+                end: [],
+                keepNested: true,
+            });
+            const keys = [];
+            const warnings = [];
+            if (treeTransform) {
+                const { tree, report } = treeTransform(rootNode);
+                rootNode = tree;
+                if (report) {
+                    keys.push(...report.keys);
+                    warnings.push(...report.warnings);
+                }
+            }
+            const report = generateReport({ node: rootNode, comments, options });
+            keys.push(...report.keys);
+            warnings.push(...report.warnings);
+            return {
+                tree: rootNode,
+                report: { keys, warnings },
+            };
+        },
+    };
+};

package/dist/extractor/parser/rules/tComponentGeneral.js

@@ -0,0 +1,45 @@
+import { parseTag } from '../tree/parseTag.js';
+// <T keyName="test">Default</T>
+// ^^-----------------------^^^^
+export const tComponentGeneral = (context) => {
+    const { getCurrentLine } = context;
+    const line = getCurrentLine();
+    const { props, child } = parseTag(context);
+    const result = {
+        type: 'keyInfo',
+        line,
+        dependsOnContext: false,
+        values: [],
+    };
+    // read props
+    if (props.type === 'dict') {
+        for (const [key, value] of Object.entries(props.value)) {
+            switch (key) {
+                case 'keyName':
+                    result.keyName = value;
+                    break;
+                case 'defaultValue':
+                    result.defaultValue = value;
+                    break;
+                case 'ns':
+                    result.namespace = value;
+                    break;
+                default:
+                    result.values.push(value);
+            }
+        }
+    }
+    // read children
+    if (child) {
+        if (!result.keyName) {
+            result.keyName = child;
+        }
+        else if (!result.defaultValue) {
+            result.defaultValue = child;
+        }
+        else {
+            result.values.push(child);
+        }
+    }
+    return result;
+};

package/dist/extractor/parser/rules/tFunctionGeneral.js

@@ -0,0 +1,41 @@
+import { getTranslateProps } from '../tree/getTranslateProps.js';
+import { parseList } from '../tree/parseList.js';
+// t('key_name', 'default_value', { ns: 'namespace' })
+// ^^------------------------------------------------^
+export const tFunctionGeneral = (context, dependsOnContext) => {
+    const line = context.getCurrentLine();
+    const args = parseList(context, 'expression.end');
+    if (args.type !== 'array') {
+        // invalid arguments
+        return args;
+    }
+    const { result: props, optionsDynamic } = getTranslateProps(args);
+    if (props.type !== 'dict') {
+        return props;
+    }
+    const result = {
+        type: 'keyInfo',
+        line,
+        dependsOnContext,
+        values: [],
+        optionsDynamic,
+    };
+    // read props
+    for (const [key, value] of Object.entries(props.value)) {
+        switch (key) {
+            case 'key':
+                result.keyName = value;
+                break;
+            case 'defaultValue':
+                result.defaultValue = value;
+                break;
+            case 'ns':
+                result.namespace = value;
+                break;
+            default:
+                // unknown parameter
+                result.values.push(value);
+        }
+    }
+    return result;
+};

package/dist/extractor/parser/rules/tNsSourceGeneral.js

@@ -0,0 +1,36 @@
+import { parseList } from '../tree/parseList.js';
+/**
+ * useTranslate('namespace')
+ * ^^^^^^^^^^^^^-----------^
+ *
+ * or
+ *
+ * useTranslate(['namespace', ...])
+ * ^^^^^^^^^^^^^------------------^
+ */
+export const tNsSourceGeneral = (context) => {
+    const line = context.getCurrentLine();
+    const args = parseList(context, 'expression.end');
+    if (args.type !== 'array') {
+        // invalid arguments
+        return args;
+    }
+    const result = {
+        type: 'nsInfo',
+        line,
+        values: [],
+    };
+    const [firstArg, ...otherArgs] = args.values;
+    if (firstArg?.type === 'array') {
+        // useTranslate(['namespace', ...])
+        const [firstItem, ...otherItems] = firstArg.values;
+        result.name = firstItem;
+        otherItems.forEach((item) => result.values.push(item));
+    }
+    else {
+        // useTranslate('namespace')
+        result.name = firstArg;
+    }
+    otherArgs.forEach((arg) => result.values.push(arg));
+    return result;
+};

package/dist/extractor/parser/tokenMergers/closingTagMerger.js

@@ -0,0 +1,27 @@
+export const closingTagMerger = {
+    initial: 0 /* S.Idle */,
+    step: (state, token, end) => {
+        const type = token.customType;
+        switch (state) {
+            case 0 /* S.Idle */:
+                if (type === 'tag.closing.begin') {
+                    return 1 /* S.ExpectName */;
+                }
+                break;
+            case 1 /* S.ExpectName */:
+                if (type === 'tag.name') {
+                    return 2 /* S.ExpectEnd */;
+                }
+                break;
+            case 2 /* S.ExpectEnd */:
+                if (type === 'tag.regular.end') {
+                    return end.MERGE_ALL;
+                }
+                break;
+        }
+    },
+    customType: 'tag.closing',
+    resultToken: (matched) => {
+        return matched[1].token;
+    },
+};

package/dist/extractor/parser/tokenMergers/commentsMerger.js

@@ -0,0 +1,36 @@
+export const commentsMerger = {
+    initial: 0 /* S.Idle */,
+    step: (state, token, end) => {
+        const type = token.customType;
+        switch (state) {
+            case 0 /* S.Idle */:
+                if (type === 'comment.definition') {
+                    return 1 /* S.CommentStart */;
+                }
+                break;
+            case 1 /* S.CommentStart */:
+                if (type === 'comment.line') {
+                    return end.MERGE_ALL;
+                }
+                else if (type === 'comment.block') {
+                    return 2 /* S.CommentBlock */;
+                }
+                break;
+            case 2 /* S.CommentBlock */:
+                if (type === 'comment.block' || token.token === '\n') {
+                    return 2 /* S.CommentBlock */;
+                }
+                else if (type === 'comment.definition') {
+                    return end.MERGE_ALL;
+                }
+                break;
+        }
+    },
+    resultToken: (matched) => {
+        return matched
+            .filter((t) => t.customType && t.customType !== 'comment.definition')
+            .map((t) => t.token)
+            .join('');
+    },
+    customType: 'comment',
+};

package/dist/extractor/parser/tokenMergers/stringMerger.js

@@ -0,0 +1,50 @@
+export const stringMerger = {
+    initial: 0 /* S.Idle */,
+    step: (state, token, end) => {
+        const type = token.customType;
+        switch (state) {
+            case 0 /* S.Idle */:
+                if (type === 'string.begin') {
+                    return 1 /* S.RegularString */;
+                }
+                else if (type === 'string.teplate.begin') {
+                    return 3 /* S.TemplateString */;
+                }
+                break;
+            case 1 /* S.RegularString */:
+                if (type === 'string.body') {
+                    return 2 /* S.RegularStringEnd */;
+                }
+                else if (type === 'string.end') {
+                    return end.MERGE_ALL;
+                }
+                break;
+            case 2 /* S.RegularStringEnd */:
+                if (type === 'string.end') {
+                    return end.MERGE_ALL;
+                }
+                break;
+            case 3 /* S.TemplateString */:
+                if (type === 'string.template.body') {
+                    return 4 /* S.TemplateStringEnd */;
+                }
+                else if (type === 'string.template.end') {
+                    return end.MERGE_ALL;
+                }
+                break;
+            case 4 /* S.TemplateStringEnd */:
+                if (type === 'string.template.end') {
+                    return end.MERGE_ALL;
+                }
+        }
+    },
+    customType: 'string',
+    resultToken: (matched) => {
+        if (matched.length === 3) {
+            return matched[1].token;
+        }
+        else {
+            return '';
+        }
+    },
+};

package/dist/extractor/parser/tokenMergers/templateStringMerger.js

@@ -0,0 +1,55 @@
+export const templateStringMerger = {
+    initial: 0 /* S.Idle */,
+    step: (state, token, end) => {
+        const type = token.customType;
+        switch (state) {
+            case 0 /* S.Idle */:
+                if (type === 'string.template') {
+                    return 1 /* S.Merging */;
+                }
+                else if (type === 'expression.template.begin') {
+                    return 2 /* S.ExpectingExpression */;
+                }
+                break;
+            case 1 /* S.Merging */:
+                if (type === 'string.template') {
+                    return 1 /* S.Merging */;
+                }
+                else if (type === 'expression.template.begin') {
+                    return 2 /* S.ExpectingExpression */;
+                }
+                else {
+                    return end.MERGE_WITHOUT_LAST;
+                }
+            case 2 /* S.ExpectingExpression */:
+                if (type === 'string') {
+                    return 3 /* S.ExpectingEndBrace */;
+                }
+                break;
+            case 3 /* S.ExpectingEndBrace */:
+                if (type === 'expression.template.end') {
+                    return 1 /* S.Merging */;
+                }
+                break;
+        }
+    },
+    customType: 'string',
+    resultToken: (matched) => {
+        const result = [];
+        for (const token of matched) {
+            if (token.customType === 'string.template') {
+                // Match and remove leading whitespace except for newlines
+                let value = token.token.replace(/^[^\S\n]+/gm, '');
+                // Match and remove trailing whitespace except for newlines
+                value = value.replace(/[^\S\n]+$/gm, '');
+                // replace multiple newlines with one space
+                value = value.replace(/[\n]+/gm, ' ');
+                result.push(value);
+            }
+            else if (token.customType === 'string') {
+                result.push(token.token);
+            }
+        }
+        return result.join('').trim();
+    },
+};

package/dist/extractor/parser/tokenMergers/typesAsMergerer.js

@@ -0,0 +1,40 @@
+const INITIAL_DEPTH = -1;
+// in ts files we want to ignore stuff after 'as' `[count as number]`
+export const typesAsMergerer = {
+    initial: INITIAL_DEPTH,
+    step: (depth, token, end) => {
+        const type = token.customType;
+        if (depth === INITIAL_DEPTH) {
+            if (type === 'typescript.as') {
+                return 0;
+            }
+        }
+        else if (depth === 0) {
+            // anything unexpected will end the ignoring
+            switch (type) {
+                case 'typescript.expr.begin':
+                case 'expression.begin':
+                    return depth + 1;
+                case 'typescript.type.primitive':
+                case 'typescript.operator':
+                    return depth;
+                default:
+                    return end.MERGE_WITHOUT_LAST;
+            }
+        }
+        else if (depth > 0) {
+            // we are deeply nested, ignore everything
+            switch (type) {
+                case 'typescript.expr.begin':
+                case 'expression.begin':
+                    return depth + 1;
+                case 'typescript.expr.end':
+                case 'expression.end':
+                    return depth - 1;
+                default:
+                    return depth;
+            }
+        }
+    },
+    customType: 'ignore',
+};

package/dist/extractor/parser/tokenMergers/typesCastMerger.js

@@ -0,0 +1,28 @@
+const INITIAL_DEPTH = 0;
+// in ts files we want to ignore type casting (<number> count)
+export const typesCastMergerer = {
+    initial: INITIAL_DEPTH,
+    step: (depth, token, end) => {
+        const type = token.customType;
+        if (depth === INITIAL_DEPTH) {
+            if (type === 'typescript.expr.begin') {
+                return 1;
+            }
+        }
+        else if (depth > 0) {
+            if (type === 'typescript.expr.end' && depth === 1) {
+                return end.MERGE_ALL;
+            }
+            else if (type === 'typescript.expr.end') {
+                return depth - 1;
+            }
+            else if (type === 'typescript.expr.begin') {
+                return depth + 1;
+            }
+            else {
+                return depth;
+            }
+        }
+    },
+    customType: 'ignore',
+};

package/dist/extractor/parser/tree/getTranslateProps.js

@@ -0,0 +1,54 @@
+function getCombinedOptions({ ns, noWrap, orEmpty, params, language, ...rest }, line) {
+    const options = {
+        ns: ns,
+        noWrap: noWrap,
+        orEmpty: orEmpty,
+        language: language,
+    };
+    return {
+        ...options,
+        params: {
+            type: 'dict',
+            line,
+            value: {
+                ...rest,
+            },
+            unknown: [],
+        },
+    };
+}
+export const getTranslateProps = (node) => {
+    const [keyOrProps, ...params] = node.values;
+    let result = {
+        type: 'dict',
+        line: node.line,
+        value: {},
+        unknown: [],
+    };
+    let options = undefined;
+    if (keyOrProps?.type === 'dict') {
+        result = keyOrProps;
+    }
+    else {
+        result.value.key = keyOrProps;
+        if (params[0]?.type === 'primitive' || params.length >= 2) {
+            result.value.defaultValue = params[0];
+            options = params[1];
+        }
+        else {
+            options = params[0];
+        }
+    }
+    let optionsDynamic = false;
+    if (options?.type === 'dict') {
+        result.value = {
+            ...getCombinedOptions(options.value, options.line),
+            ...result.value,
+        };
+        result.unknown.push(...options.unknown);
+    }
+    else if (options) {
+        optionsDynamic = true;
+    }
+    return { result, optionsDynamic };
+};