@babelize/engine 0.0.0-stage → 0.1.2

package/dist/utils/icu-message.js

@@ -0,0 +1,346 @@
+"use strict";
+/**
+ * Minimal ICU MessageFormat parser and validator for translated messages.
+ *
+ * ICU messages can't be tokenized like simple placeholders: a translation
+ * legitimately changes their structure (Polish needs `few`/`many` plural
+ * branches English doesn't have). So instead of protecting them, we let the
+ * model rewrite the whole message and then check the result is still a valid
+ * message with the same arguments and selector branches.
+ *
+ * Covers what UI strings use: simple and formatted arguments, plural,
+ * selectordinal (with offset and =n keys), select, `#`, and apostrophe
+ * quoting. Not a full ICU implementation (no nested formatting skeletons).
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IcuParseError = void 0;
+exports.parseIcu = parseIcu;
+exports.isIcuMessage = isIcuMessage;
+exports.compareIcuMessages = compareIcuMessages;
+exports.isCountLikeName = isCountLikeName;
+exports.countArguments = countArguments;
+exports.extractSimpleArguments = extractSimpleArguments;
+exports.requiredPluralCategories = requiredPluralCategories;
+class IcuParseError extends Error {
+}
+exports.IcuParseError = IcuParseError;
+function parseIcu(message) {
+    let pos = 0;
+    const fail = (msg) => {
+        throw new IcuParseError(`${msg} at position ${pos}`);
+    };
+    const skipSpace = () => {
+        while (pos < message.length && /\s/.test(message[pos]))
+            pos++;
+    };
+    const readIdentifier = () => {
+        skipSpace();
+        const start = pos;
+        while (pos < message.length && /[^\s,{}#]/.test(message[pos]))
+            pos++;
+        if (pos === start)
+            fail('Expected identifier');
+        return message.slice(start, pos);
+    };
+    const parseMessage = (inPlural, depth) => {
+        const nodes = [];
+        let text = '';
+        const flush = () => {
+            if (text)
+                nodes.push({ type: 'text', value: text });
+            text = '';
+        };
+        while (pos < message.length) {
+            const ch = message[pos];
+            if (ch === "'") {
+                // '' is a literal apostrophe; '{…}' / '#…' quotes syntax characters.
+                if (message[pos + 1] === "'") {
+                    text += "'";
+                    pos += 2;
+                    continue;
+                }
+                const next = message[pos + 1];
+                if (next === '{' || next === '}' || (inPlural && next === '#')) {
+                    const end = message.indexOf("'", pos + 1);
+                    if (end === -1)
+                        fail('Unterminated quote');
+                    text += message.slice(pos + 1, end);
+                    pos = end + 1;
+                    continue;
+                }
+                text += ch;
+                pos++;
+                continue;
+            }
+            if (ch === '#' && inPlural) {
+                flush();
+                nodes.push({ type: 'pound' });
+                pos++;
+                continue;
+            }
+            if (ch === '}') {
+                if (depth === 0)
+                    fail('Unbalanced }');
+                break;
+            }
+            if (ch === '{') {
+                flush();
+                nodes.push(parseArgument(depth));
+                continue;
+            }
+            text += ch;
+            pos++;
+        }
+        flush();
+        return nodes;
+    };
+    const parseArgument = (depth) => {
+        pos++; // {
+        const name = readIdentifier();
+        skipSpace();
+        if (message[pos] === '}') {
+            pos++;
+            return { type: 'argument', name };
+        }
+        if (message[pos] !== ',')
+            fail('Expected , or }');
+        pos++;
+        const kind = readIdentifier();
+        skipSpace();
+        if (kind === 'plural' || kind === 'selectordinal' || kind === 'select') {
+            if (message[pos] !== ',')
+                fail(`Expected , after ${kind}`);
+            pos++;
+            const options = {};
+            skipSpace();
+            if (kind !== 'select' && message.startsWith('offset:', pos)) {
+                pos += 'offset:'.length;
+                readIdentifier();
+            }
+            while (true) {
+                skipSpace();
+                if (message[pos] === '}') {
+                    pos++;
+                    break;
+                }
+                if (pos >= message.length)
+                    fail(`Unterminated ${kind}`);
+                const key = readIdentifier();
+                skipSpace();
+                if (message[pos] !== '{')
+                    fail(`Expected { after option "${key}"`);
+                if (key in options)
+                    fail(`Duplicate option "${key}"`);
+                pos++;
+                options[key] = parseMessage(kind !== 'select', depth + 1);
+                if (message[pos] !== '}')
+                    fail(`Unterminated option "${key}"`);
+                pos++;
+            }
+            if (Object.keys(options).length === 0)
+                fail(`${kind} has no options`);
+            return { type: kind, name, options };
+        }
+        // Formatted argument: {price, number} / {d, date, short} — keep the rest verbatim.
+        let format = kind;
+        if (message[pos] === ',') {
+            const end = message.indexOf('}', pos);
+            if (end === -1)
+                fail('Unterminated argument');
+            format += message.slice(pos, end);
+            pos = end;
+        }
+        skipSpace();
+        if (message[pos] !== '}')
+            fail('Expected }');
+        pos++;
+        return { type: 'argument', name, format };
+    };
+    const nodes = parseMessage(false, 0);
+    if (pos < message.length)
+        fail('Unexpected }');
+    return nodes;
+}
+/** True when the message uses plural/select/selectordinal syntax. */
+function isIcuMessage(message) {
+    return /\{\s*[\w.]+\s*,\s*(?:plural|select|selectordinal)\s*,/.test(message);
+}
+function collect(nodes, args, selectors, texts) {
+    for (const node of nodes) {
+        if (node.type === 'text')
+            texts.push(node.value);
+        else if (node.type === 'argument')
+            args.add(node.name);
+        else if (node.type !== 'pound') {
+            args.add(node.name);
+            selectors.push({ kind: node.type, name: node.name, keys: Object.keys(node.options), branches: Object.values(node.options) });
+            for (const branch of Object.values(node.options))
+                collect(branch, args, selectors, texts);
+        }
+    }
+}
+const CLDR_ORDER = ['zero', 'one', 'two', 'few', 'many', 'other'];
+/** Plural categories for a locale, always in CLDR order (runtimes differ in ordering). */
+function pluralCategories(locale, type) {
+    try {
+        const found = new Set(new Intl.PluralRules(locale, { type }).resolvedOptions().pluralCategories);
+        return CLDR_ORDER.filter((c) => found.has(c));
+    }
+    catch {
+        return [...CLDR_ORDER];
+    }
+}
+/**
+ * Compare a translated ICU message to its source. Errors mean the message is
+ * broken at runtime (unparseable, arguments renamed, select branches changed,
+ * no `other`). Missing target plural categories are only a warning: ICU
+ * falls back to `other` for them.
+ */
+function compareIcuMessages(source, translated, targetLocale, options = {}) {
+    let sourceAst;
+    try {
+        sourceAst = parseIcu(source);
+    }
+    catch {
+        // Source isn't valid ICU itself; nothing meaningful to compare.
+        return [];
+    }
+    let targetAst;
+    try {
+        targetAst = parseIcu(translated);
+    }
+    catch (err) {
+        return [{ severity: 'error', code: 'icu_invalid', message: `Translation is not a valid ICU message: ${err.message}` }];
+    }
+    const issues = [];
+    const srcArgs = new Set();
+    const tgtArgs = new Set();
+    const srcSelectors = [];
+    const tgtSelectors = [];
+    collect(sourceAst, srcArgs, srcSelectors, []);
+    collect(targetAst, tgtArgs, tgtSelectors, []);
+    const missingArgs = [...srcArgs].filter((a) => !tgtArgs.has(a));
+    const extraArgs = [...tgtArgs].filter((a) => !srcArgs.has(a));
+    if (missingArgs.length || extraArgs.length) {
+        issues.push({
+            severity: 'error',
+            code: 'icu_arguments_changed',
+            message: [
+                missingArgs.length ? `missing {${missingArgs.join('}, {')}}` : '',
+                extraArgs.length ? `unknown {${extraArgs.join('}, {')}}` : '',
+            ].filter(Boolean).join('; '),
+        });
+    }
+    // A plural/select dropped from the translation (e.g. "{count} fichiers")
+    // still renders, but wrongly for every count but one.
+    for (const src of srcSelectors) {
+        if (!tgtSelectors.some((t) => t.name === src.name && t.kind === src.kind)) {
+            issues.push({
+                severity: 'error',
+                code: 'icu_selector_removed',
+                message: `{${src.name}, ${src.kind}} was removed from the translation`,
+            });
+        }
+    }
+    // A translation may add a plural over a simple argument of the source:
+    // "{files} files" → Polish "{files, plural, one {# plik} few {# pliki} …}".
+    // Only a cardinal plural, and every branch must still show the number.
+    for (const target of tgtSelectors) {
+        if (srcSelectors.some((s) => s.name === target.name && s.kind === target.kind))
+            continue;
+        if (target.kind !== 'plural') {
+            issues.push({ severity: 'error', code: 'icu_selector_added', message: `{${target.name}, ${target.kind}} is not in the source; only a plural may be added` });
+            continue;
+        }
+        if (!(options.pluralizable ?? []).includes(target.name)) {
+            issues.push({ severity: 'error', code: 'icu_selector_added', message: `{${target.name}} is not a number; don't add a plural over it` });
+            continue;
+        }
+        const lost = target.keys.filter((k, i) => !mentions(target.branches[i], target.name));
+        if (lost.length) {
+            issues.push({
+                severity: 'error',
+                code: 'icu_branch_lost_argument',
+                message: `{${target.name}, plural} branches [${lost.join(', ')}] don't show the number (use # or {${target.name}})`,
+            });
+        }
+    }
+    for (const target of tgtSelectors) {
+        if (!target.keys.includes('other')) {
+            issues.push({ severity: 'error', code: 'icu_missing_other', message: `{${target.name}, ${target.kind}} has no "other" branch` });
+        }
+        if (target.kind === 'select') {
+            const source = srcSelectors.find((s) => s.kind === 'select' && s.name === target.name);
+            if (source && [...source.keys].sort().join('|') !== [...target.keys].sort().join('|')) {
+                issues.push({
+                    severity: 'error',
+                    code: 'icu_select_keys_changed',
+                    message: `{${target.name}, select} branches changed from [${source.keys.join(', ')}] to [${target.keys.join(', ')}]`,
+                });
+            }
+            continue;
+        }
+        const allowed = pluralCategories(targetLocale, target.kind === 'selectordinal' ? 'ordinal' : 'cardinal');
+        const invalid = target.keys.filter((k) => !/^=\d+$/.test(k) && !allowed.includes(k));
+        if (invalid.length) {
+            issues.push({
+                severity: 'error',
+                code: 'icu_plural_category_invalid',
+                message: `{${target.name}, ${target.kind}} uses categories [${invalid.join(', ')}] that ${targetLocale} doesn't have (valid: ${allowed.join(', ')})`,
+            });
+        }
+        const missing = allowed.filter((c) => !target.keys.includes(c));
+        if (missing.length) {
+            issues.push({
+                severity: 'warning',
+                code: 'icu_plural_category_missing',
+                message: `{${target.name}, ${target.kind}} has no [${missing.join(', ')}] branch for ${targetLocale}; "other" will be used`,
+            });
+        }
+    }
+    return issues;
+}
+/** Whether a plural branch shows its number: `#`, or `{name}` (also inside nested selectors). */
+function mentions(nodes, name) {
+    return nodes.some((n) => n.type === 'pound' ||
+        (n.type === 'argument' && n.name === name) ||
+        ((n.type === 'plural' || n.type === 'select' || n.type === 'selectordinal') && Object.values(n.options).some((b) => mentions(b, name))));
+}
+/** Argument names that read like counts (`count`, `total`, `itemCount`, `items.length` → itemsLength). */
+const COUNT_NAME = /^(?:count|total|num|n|length)$|(?:Count|Length)$/;
+/** Argument names that are a plural noun, i.e. how many of them (`{files}`, `{unreadMessages}`). */
+const COUNTED_NOUNS = new Set([
+    'attempts', 'comments', 'credits', 'days', 'documents', 'entries', 'errors', 'events', 'files', 'followers',
+    'hours', 'images', 'issues', 'items', 'keys', 'licenses', 'likes', 'members', 'messages', 'minutes',
+    'notifications', 'orders', 'pages', 'photos', 'points', 'products', 'projects', 'records', 'results', 'rows',
+    'seats', 'seconds', 'steps', 'tasks', 'users', 'views', 'warnings', 'weeks', 'months', 'years', 'people',
+]);
+function isCountLikeName(name) {
+    if (COUNT_NAME.test(name))
+        return true;
+    const last = name.split(/(?=[A-Z])|_/).pop().toLowerCase();
+    return COUNTED_NOUNS.has(last);
+}
+/**
+ * Simple `{name}` arguments of a non-ICU message that look like counts. A
+ * translation may turn such an argument into a plural for languages that
+ * inflect words by number, without the source having to.
+ */
+function countArguments(message) {
+    // Only ICU-style messages: i18next `{{count}}`, printf and template
+    // placeholders keep token protection and its checks.
+    if (isIcuMessage(message) || /\{\{|\}\}|%(?:\(\w+\))?[sd]|\$\{/.test(message))
+        return [];
+    return extractSimpleArguments(message).filter(isCountLikeName);
+}
+/** Single-brace `{name}` arguments (not `{{name}}`). */
+function extractSimpleArguments(message) {
+    const names = new Set();
+    for (const m of message.matchAll(/(?<!\{)\{\s*([A-Za-z_]\w*)\s*\}(?!\})/g))
+        names.add(m[1]);
+    return [...names];
+}
+/** CLDR plural categories a translator should produce for this locale. */
+function requiredPluralCategories(locale) {
+    return { cardinal: pluralCategories(locale, 'cardinal'), ordinal: pluralCategories(locale, 'ordinal') };
+}

package/dist/utils/icu-spans.d.ts

@@ -0,0 +1,10 @@
+/** Span differences of `target` per branch combination (empty when fine). */
+export declare function branchSpanProblems(source: string, target: string, neverTranslate?: string[]): BranchSpanProblem[];
+export interface BranchSpanProblem {
+    /** Which branch combination, e.g. "count=other" ("message" when the target has no selectors). */
+    branch: string;
+    lost: string[];
+    added: string[];
+}
+/** "count=other: lost <b>, </b>" */
+export declare function describeBranchSpanProblem(p: BranchSpanProblem): string;

package/dist/utils/icu-spans.js

@@ -0,0 +1,126 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.branchSpanProblems = branchSpanProblems;
+exports.describeBranchSpanProblem = describeBranchSpanProblem;
+/**
+ * Per-branch check of the protected spans (tags, URLs, emails, code,
+ * never-translate terms) of an ICU translation. Shared by the translate
+ * phase and locale QA.
+ *
+ * Each combination of plural/select branches the translation can render
+ * (a "variant") must carry exactly the spans of the matching source variant:
+ * same branch keys where the source has them, the source's `other` branch for
+ * categories only the target language has, and the whole source when a plain
+ * source was turned into a plural. So `<b>` dropped from one branch, or kept
+ * in only one branch of an added plural, is caught — not just a span missing
+ * from every branch.
+ */
+const icu_message_1 = require("./icu-message");
+const placeholder_protection_1 = require("./placeholder-protection");
+/** Variants beyond this are not expanded (deeply nested selects); the check is skipped. */
+const MAX_VARIANTS = 256;
+class TooManyVariants extends Error {
+}
+function expand(nodes) {
+    let variants = [{ path: new Map(), text: '' }];
+    for (const node of nodes) {
+        if (node.type === 'text') {
+            for (const v of variants)
+                v.text += node.value;
+        }
+        else if (node.type === 'pound') {
+            for (const v of variants)
+                v.text += '#';
+        }
+        else if (node.type === 'argument') {
+            for (const v of variants)
+                v.text += `{${node.name}}`;
+        }
+        else {
+            const next = [];
+            for (const v of variants) {
+                for (const [key, branch] of Object.entries(node.options)) {
+                    for (const b of expand(branch)) {
+                        const path = new Map([...v.path, [node.name, key], ...b.path]);
+                        next.push({ path, text: v.text + b.text });
+                        if (next.length > MAX_VARIANTS)
+                            throw new TooManyVariants();
+                    }
+                }
+            }
+            variants = next;
+        }
+    }
+    return variants;
+}
+function spans(text, neverTranslate) {
+    const counts = new Map();
+    for (const t of (0, placeholder_protection_1.protectText)(text, neverTranslate, { icu: true }).tokens) {
+        const k = t.toLowerCase();
+        counts.set(k, (counts.get(k) ?? 0) + 1);
+    }
+    return counts;
+}
+/** The source variant a target variant is checked against. */
+function matchSource(source, target) {
+    const score = (s) => {
+        let exact = 0;
+        for (const [name, key] of s.path) {
+            const want = target.path.get(name);
+            if (want === key)
+                exact += 2;
+            else if (key !== 'other')
+                return -1;
+        }
+        return exact;
+    };
+    // Prefer the variant with the most matching branch keys; unmatched selectors fall back to `other`.
+    let best = source[0];
+    let bestScore = -Infinity;
+    for (const s of source) {
+        const sc = score(s);
+        if (sc > bestScore) {
+            best = s;
+            bestScore = sc;
+        }
+    }
+    return best;
+}
+function label(v) {
+    return v.path.size ? [...v.path].map(([n, k]) => `${n}=${k}`).join(', ') : 'message';
+}
+/** Span differences of `target` per branch combination (empty when fine). */
+function branchSpanProblems(source, target, neverTranslate = []) {
+    let src;
+    let tgt;
+    try {
+        src = expand((0, icu_message_1.parseIcu)(source));
+        tgt = expand((0, icu_message_1.parseIcu)(target));
+    }
+    catch {
+        // Unparseable (reported by the ICU checks) or too many branch combinations to expand.
+        return [];
+    }
+    const problems = new Map();
+    for (const t of tgt) {
+        const s = matchSource(src, t);
+        const want = spans(s.text, neverTranslate);
+        const have = spans(t.text, neverTranslate);
+        const lost = [];
+        const added = [];
+        for (const [span, n] of want)
+            if ((have.get(span) ?? 0) < n)
+                lost.push(span);
+        for (const [span, n] of have)
+            if ((want.get(span) ?? 0) < n)
+                added.push(span);
+        if (lost.length || added.length)
+            problems.set(label(t), { branch: label(t), lost, added });
+    }
+    return [...problems.values()];
+}
+/** "count=other: lost <b>, </b>" */
+function describeBranchSpanProblem(p) {
+    const parts = [p.lost.length ? `lost ${p.lost.join(', ')}` : '', p.added.length ? `added ${p.added.join(', ')}` : ''].filter(Boolean);
+    return `${p.branch}: ${parts.join('; ')}`;
+}

package/dist/utils/jsx-message.d.ts

@@ -0,0 +1,93 @@
+import * as t from '@babel/types';
+/**
+ * JSX sentences as one message.
+ *
+ * `<h1>Welcome back, {user}</h1>` used to be extracted as the fragment
+ * "Welcome back," and injected as `{t('…')} {user}`. A translation can't move
+ * words around a variable it never sees, and a count can't be pluralized. This
+ * module turns an element's mixed children — text, `{expressions}` and inline
+ * elements — into a single ICU message:
+ *
+ *   <h1>Welcome back, {user}</h1>                 → "Welcome back, {user}"
+ *   <p>Hi {user.name}</p>                         → "Hi {userName}"
+ *   <p>Total: {format(sum)} today</p>             → "Total: {0} today"
+ *   <p>Read the <Link href="/t">terms</Link></p>  → "Read the <link>terms</link>"
+ *   <p>You have {count} items</p>                 → "You have {count, plural, one {# item} other {# items}}"
+ *
+ * The extractor and the injector both call {@link buildJsxMessage} on the same
+ * untouched source, so they agree on the text (and so on the extraction id).
+ */
+export interface MessagePlaceholder {
+    kind: 'placeholder';
+    /** Argument name used in the message and in the values object. */
+    name: string;
+    expression: t.Expression;
+}
+export interface MessageTag {
+    kind: 'tag';
+    /** Tag name used in the message (`<link>…</link>`) and as the rich-text key. */
+    name: string;
+    element: t.JSXElement;
+}
+export interface JsxMessage {
+    /** The ICU message, e.g. "Welcome back, {user}". */
+    text: string;
+    /** Placeholders and tags in order of first appearance (each name once). */
+    entries: Array<MessagePlaceholder | MessageTag>;
+    /** Source nodes folded into the message (text, literal expressions, inline elements). */
+    consumed: t.Node[];
+    /** Name of the placeholder turned into an ICU plural, if any. */
+    plural?: string;
+}
+export type JsxMessageResult = {
+    kind: 'message';
+    message: JsxMessage;
+}
+/** Not a sentence: plain text, only expressions, block-level children, … */
+ | {
+    kind: 'none';
+}
+/** Reads like a sentence but can't be moved safely; extract fragments as before. */
+ | {
+    kind: 'unsafe';
+    reason: string;
+};
+/**
+ * Names that suggest a React node rather than text or a number — such a value
+ * can't be a message argument. Matched case-insensitively against the last
+ * name segment (`user.avatar`, `statusIcon`, `renderBadge()`). Extendable.
+ */
+export declare const ELEMENT_LIKE_NAMES: RegExp[];
+/** Rendered text of a JSXText node, with JSX's whitespace rules (same as Babel's cleanJSXElementLiteralChild). */
+export declare function cleanJsxText(value: string): string;
+/**
+ * Whether the expression is very likely a string or number. Message values
+ * must be: next-intl throws on a React element, i18next prints [object Object].
+ */
+export declare function isLikelyPrimitive(expr: t.Node): boolean;
+/**
+ * Build the single message for `node`'s children, or say why there is none.
+ * Pure: reads the AST, never changes it.
+ */
+export declare function buildJsxMessage(node: t.JSXElement | t.JSXFragment, options?: {
+    /**
+     * Turn "{count} items" into an ICU plural. Only when an LLM translator is
+     * configured: the NLLB service can't translate ICU plural messages.
+     */
+    plurals?: boolean;
+}): JsxMessageResult;
+/**
+ * Text used to name the key: the message without arguments and tags, with a
+ * plural's `other` branch standing in for it. "Welcome back, {user}" →
+ * "Welcome back,"; "{count, plural, one {# item} other {# items}} left" → "items left".
+ */
+export declare function messageKeyText(message: string): string;
+/** `{ user, userName: user.name, 0: format(x) }` — placeholders only. */
+export declare function placeholderValues(message: JsxMessage): t.ObjectProperty[];
+/** next-intl rich values: `b: (chunks) => <b>{chunks}</b>`. Moves the element into the callback. */
+export declare function richTagValue(tag: MessageTag): t.ObjectProperty;
+/** react-i18next Trans components: `b: <b />`. */
+export declare function transComponent(tag: MessageTag): t.ObjectProperty;
+export declare function hasTags(message: JsxMessage): boolean;
+/** `t("key")` or `t("key", { …values })`. */
+export declare function createMessageTCall(key: string, message: JsxMessage, callee?: t.Expression): t.CallExpression;