@babelize/engine 0.0.0-stage → 0.1.0

package/dist/utils/py-extractor.js

@@ -0,0 +1,285 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractStringsFromPython = extractStringsFromPython;
+const string_utils_1 = require("./string-utils");
+// Function calls that indicate translatable strings
+const GETTEXT_CALLS = [
+    '_',
+    'gettext',
+    'gettext_lazy',
+    'ngettext',
+    'ngettext_lazy',
+    'ugettext',
+    'ugettext_lazy',
+    'pgettext',
+    'pgettext_lazy',
+    'npgettext',
+    'npgettext_lazy',
+];
+// Keyword argument names that Django/Python treat as translatable
+const TRANSlatable_KWARGS = [
+    'verbose_name',
+    'verbose_name_plural',
+    'help_text',
+    'label',
+];
+/**
+ * Extract translatable strings from Python source code.
+ *
+ * Matches: _(), gettext(), gettext_lazy(), ugettext*(), ngettext()
+ * Also opt-in: verbose_name=, help_text=, label= kwargs
+ *
+ * Uses a character-position lexer approach (not a full AST parser)
+ * to handle single/double/triple quotes, raw strings, f-strings, and escape sequences.
+ */
+function extractStringsFromPython(content, filePath) {
+    const ctx = {
+        filePath,
+        content,
+        strings: [],
+        seenTexts: new Set(),
+        warnings: [],
+    };
+    let i = 0;
+    const len = content.length;
+    while (i < len) {
+        // Skip lines that are comments
+        if (content[i] === '#') {
+            i = skipToNextLine(content, i);
+            continue;
+        }
+        // Skip string literals on their own (not preceded by a function call or kwarg)
+        // We only extract strings that are arguments to gettext functions or Django kwargs
+        // Check for gettext function calls: _( "..." ), gettext( "..." ), etc.
+        for (const callName of GETTEXT_CALLS) {
+            const callPattern = new RegExp(`\\b${callName.replace(/_/g, '\\_')}\\s*\\(`);
+            const match = callPattern.exec(content.slice(i));
+            if (match && match.index === 0) {
+                const parenStart = i + match[0].length;
+                const args = collectCallArgs(content, parenStart);
+                if (args) {
+                    for (const arg of args) {
+                        const str = extractStringFromArg(content, arg);
+                        if (str !== null) {
+                            addString(ctx, str.text, arg.startLine, 0, str.context);
+                        }
+                    }
+                }
+                i = args ? (args[args.length - 1]?.end || parenStart) : parenStart;
+                break;
+            }
+        }
+        // Check for Django keyword arguments: verbose_name="...", help_text="..."
+        for (const kwarg of TRANSlatable_KWARGS) {
+            const kwargPattern = new RegExp(`\\b${kwarg}\\s*=\\s*`);
+            const match = kwargPattern.exec(content.slice(i));
+            if (match && match.index === 0) {
+                const valueStart = i + match[0].length;
+                const str = extractStringLiteral(content, valueStart);
+                if (str !== null) {
+                    addString(ctx, str.text, findLineNumber(content, valueStart), 0, str.context);
+                }
+                i = valueStart + (str ? str.rawLength : 0);
+                break;
+            }
+        }
+        i++;
+    }
+    return { strings: ctx.strings, warnings: ctx.warnings };
+}
+/**
+ * Collect argument ranges for a function call starting after the opening paren.
+ * Returns an array of {start, end, startLine} for each argument position.
+ */
+function collectCallArgs(content, start) {
+    let depth = 1;
+    let i = start;
+    const args = [];
+    let argStart = -1;
+    let parenDepth = 0;
+    while (i < content.length && depth > 0) {
+        const ch = content[i];
+        if (ch === '(') {
+            parenDepth++;
+        }
+        else if (ch === ')') {
+            parenDepth--;
+            if (parenDepth === 0 && depth === 1) {
+                if (argStart !== -1) {
+                    args.push({
+                        start: argStart,
+                        end: i,
+                        startLine: findLineNumber(content, argStart),
+                    });
+                }
+                depth--;
+                i++;
+                continue;
+            }
+        }
+        else if (ch === ',' && parenDepth === 1) {
+            if (argStart !== -1) {
+                args.push({
+                    start: argStart,
+                    end: i,
+                    startLine: findLineNumber(content, argStart),
+                });
+            }
+            argStart = -1;
+        }
+        else if (ch === '"' || ch === "'" || ch === 'f') {
+            // String literal — skip over it
+            const strEnd = skipString(content, i);
+            if (strEnd !== i) {
+                if (argStart === -1)
+                    argStart = i;
+                i = strEnd;
+                continue;
+            }
+        }
+        // Track first non-whitespace as arg start
+        if (argStart === -1 && !/\s/.test(ch) && ch !== ',') {
+            // Check if this is a keyword arg or string
+            if (ch !== ')') {
+                argStart = i;
+            }
+        }
+        i++;
+    }
+    return args.length > 0 ? args : null;
+}
+/**
+ * Extract the string value from an argument position.
+ * Handles named arguments (_("text"), _("text", count)).
+ * Only extracts the first string argument (singular form).
+ */
+function extractStringFromArg(content, arg) {
+    // Skip keyword name if present (e.g., msgid="text")
+    let i = arg.start;
+    const end = arg.end;
+    while (i < end && content[i] !== '"' && content[i] !== "'" && content[i] !== 'f') {
+        i++;
+    }
+    if (i >= end)
+        return null;
+    const str = extractStringLiteral(content, i);
+    if (!str)
+        return null;
+    return {
+        text: str.text,
+        rawLength: str.rawLength,
+        context: str.context,
+    };
+}
+/**
+ * Extract a string literal from content starting at the given position.
+ * Handles: "...", '...', """...""", '''...''', f"...", r"...", b"..."
+ */
+function extractStringLiteral(content, start) {
+    let i = start;
+    const len = content.length;
+    // Skip prefix characters: r, f, b, rb, fr, etc.
+    while (i < len && /[a-zA-Z]/.test(content[i]) && content[i] !== '"' && content[i] !== "'") {
+        i++;
+    }
+    if (i >= len)
+        return null;
+    const quote = content[i];
+    if (quote !== '"' && quote !== "'")
+        return null;
+    // Check for triple-quote
+    const isTriple = content.slice(i, i + 3) === quote.repeat(3);
+    const openQuote = isTriple ? quote.repeat(3) : quote;
+    const closeQuote = openQuote;
+    const valueStart = i + openQuote.length;
+    let j = valueStart;
+    let text = '';
+    let context = 'string-literal';
+    // Detect f-string
+    const prefixStart = start;
+    const prefix = content.slice(prefixStart, i).toLowerCase();
+    if (prefix.includes('f')) {
+        context = 'template-literal';
+    }
+    while (j < len) {
+        if (content[j] === '\\') {
+            // Escape sequence — skip next char
+            j += 2;
+            continue;
+        }
+        if (content.slice(j, j + closeQuote.length) === closeQuote) {
+            // Found closing quote
+            const rawLength = j + closeQuote.length - start;
+            return { text, rawLength, context };
+        }
+        text += content[j];
+        j++;
+    }
+    return null;
+}
+/**
+ * Skip over a string literal starting at the given position.
+ * Returns the index after the closing quote.
+ */
+function skipString(content, start) {
+    let i = start;
+    const len = content.length;
+    // Skip prefix characters
+    while (i < len && /[a-zA-Z]/.test(content[i]) && content[i] !== '"' && content[i] !== "'") {
+        i++;
+    }
+    if (i >= len)
+        return start;
+    const quote = content[i];
+    if (quote !== '"' && quote !== "'")
+        return start;
+    const isTriple = content.slice(i, i + 3) === quote.repeat(3);
+    const closeQuote = isTriple ? quote.repeat(3) : quote;
+    i += closeQuote.length;
+    while (i < len) {
+        if (content[i] === '\\') {
+            i += 2;
+            continue;
+        }
+        if (content.slice(i, i + closeQuote.length) === closeQuote) {
+            return i + closeQuote.length;
+        }
+        i++;
+    }
+    return start; // Unterminated string
+}
+/**
+ * Add a string to the extraction context, deduplicating by content hash.
+ */
+function addString(ctx, text, line, column, context) {
+    const trimmed = text.trim();
+    if (!trimmed || trimmed.length < 2)
+        return;
+    if (!(0, string_utils_1.isTranslatableText)(trimmed))
+        return;
+    const contentKey = `${ctx.filePath}:${(0, string_utils_1.hashString)(trimmed)}`;
+    if (ctx.seenTexts.has(contentKey))
+        return;
+    const id = `${ctx.filePath}:${line}:${(0, string_utils_1.hashString)(trimmed)}`;
+    ctx.seenTexts.add(contentKey);
+    ctx.strings.push({
+        text: trimmed,
+        file: ctx.filePath,
+        line,
+        column,
+        context,
+        id,
+    });
+}
+function skipToNextLine(content, pos) {
+    const newline = content.indexOf('\n', pos);
+    return newline === -1 ? content.length : newline + 1;
+}
+function findLineNumber(content, pos) {
+    let line = 1;
+    for (let i = 0; i < pos && i < content.length; i++) {
+        if (content[i] === '\n')
+            line++;
+    }
+    return line;
+}

package/dist/utils/string-utils.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Engine v2 string utilities.
+ *
+ * Copied from v1 to preserve keygen/normalization behavior.
+ */
+export declare function normalizeString(text: string): string;
+export declare function toCamelCase(text: string): string;
+export declare function inferNamespace(filePath: string): string;
+export declare function generateUniqueKey(baseKey: string, existingKeys: Set<string>): string;
+export declare function truncateForKey(text: string, maxWords?: number): string;
+export declare function isTranslatableText(text: string): boolean;
+export declare function extractPlaceholders(text: string): string[];
+/**
+ * Protect placeholders by replacing them with AI-safe tokens before translation
+ */
+export declare function protectPlaceholders(text: string): {
+    protectedText: string;
+    vars: string[];
+};
+export interface RestorationResult {
+    restoredText: string;
+    restoredCount: number;
+    expectedCount: number;
+    missingTokens: string[];
+}
+/**
+ * Restore placeholders from AI-safe tokens after translation
+ * Returns a diagnostic result to detect dropped variables
+ */
+export declare function restorePlaceholders(text: string, vars: string[]): RestorationResult;
+/**
+ * Check if text text and associated variables likely represent a pluralized string
+ * (e.g. "count files", "Total {length}")
+ */
+export declare function isLikelyPlural(text: string, variables?: string[]): boolean;
+export declare function hasICUSyntax(text: string): boolean;
+export declare function escapeRegex(text: string): string;
+export declare function hashString(text: string): string;
+export declare function matchesCurrencyPattern(text: string): boolean;
+export declare function matchesDatePattern(text: string): boolean;
+export declare function containsPII(text: string): boolean;
+export declare const SKIP_TAGS: Set<string>;
+/**
+ * Extract content between balanced opening and closing tags, handling nested tags.
+ * Finds the first `<tagName>` and its matching `</tagName>`, correctly counting
+ * nested `<tagName ...>` elements.
+ */
+export declare function extractBalancedTag(content: string, tagName: string, allowAttrs?: boolean): {
+    fullMatch: string;
+    body: string;
+    attrs: string;
+} | null;

package/dist/utils/string-utils.js

@@ -0,0 +1,270 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SKIP_TAGS = void 0;
+exports.normalizeString = normalizeString;
+exports.toCamelCase = toCamelCase;
+exports.inferNamespace = inferNamespace;
+exports.generateUniqueKey = generateUniqueKey;
+exports.truncateForKey = truncateForKey;
+exports.isTranslatableText = isTranslatableText;
+exports.extractPlaceholders = extractPlaceholders;
+exports.protectPlaceholders = protectPlaceholders;
+exports.restorePlaceholders = restorePlaceholders;
+exports.isLikelyPlural = isLikelyPlural;
+exports.hasICUSyntax = hasICUSyntax;
+exports.escapeRegex = escapeRegex;
+exports.hashString = hashString;
+exports.matchesCurrencyPattern = matchesCurrencyPattern;
+exports.matchesDatePattern = matchesDatePattern;
+exports.containsPII = containsPII;
+exports.extractBalancedTag = extractBalancedTag;
+const node_crypto_1 = require("node:crypto");
+/**
+ * Engine v2 string utilities.
+ *
+ * Copied from v1 to preserve keygen/normalization behavior.
+ */
+function normalizeString(text) {
+    return text
+        .toLowerCase()
+        .replace(/[^\p{L}\p{N}\s']/gu, ' ')
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+function toCamelCase(text) {
+    const normalized = normalizeString(text);
+    const words = normalized.split(' ').filter(Boolean);
+    if (words.length === 0)
+        return 'untitled';
+    const result = words
+        .map((word, index) => {
+        word = word.replace(/'/g, '');
+        if (index === 0)
+            return word.toLowerCase();
+        return word.charAt(0).toUpperCase() + word.slice(1).toLowerCase();
+    })
+        .join('');
+    if (!/[a-zA-Z]/.test(result))
+        return 'untitled';
+    return result;
+}
+function inferNamespace(filePath) {
+    const normalizedPath = filePath.replace(/\\/g, '/');
+    const pathWithoutExt = normalizedPath.replace(/\.(ts|tsx|js|jsx)$/, '');
+    const parts = pathWithoutExt.split('/').filter(Boolean);
+    const prefixesToRemove = ['src', 'app', 'pages', 'components', 'features', 'modules'];
+    while (parts.length > 0 && prefixesToRemove.includes(parts[0].toLowerCase())) {
+        parts.shift();
+    }
+    const suffixesToRemove = ['index', 'page', 'component', 'container', 'view'];
+    while (parts.length > 1 && suffixesToRemove.includes(parts[parts.length - 1].toLowerCase())) {
+        parts.pop();
+    }
+    if (parts.length === 0)
+        return 'common';
+    const namespace = parts[parts.length - 1]
+        .replace(/([A-Z])/g, (match, p1, offset) => (offset > 0 ? '-' + p1 : p1))
+        .toLowerCase()
+        .replace(/^-/, '');
+    return namespace.replace(/[^a-z0-9-]/g, '') || 'common';
+}
+function generateUniqueKey(baseKey, existingKeys) {
+    if (!existingKeys.has(baseKey))
+        return baseKey;
+    let counter = 2;
+    let uniqueKey = `${baseKey}_${counter}`;
+    while (existingKeys.has(uniqueKey)) {
+        counter++;
+        uniqueKey = `${baseKey}_${counter}`;
+    }
+    return uniqueKey;
+}
+function truncateForKey(text, maxWords = 5) {
+    const words = text.split(/\s+/).filter(Boolean);
+    if (words.length <= maxWords)
+        return text;
+    return words.slice(0, maxWords).join(' ');
+}
+function isTranslatableText(text) {
+    const trimmed = text.trim();
+    if (!trimmed)
+        return false;
+    if (/^\s*$/.test(trimmed))
+        return false;
+    if (/^\d+$/.test(trimmed))
+        return false;
+    if (/^[^\p{L}\p{N}\s]+$/u.test(trimmed))
+        return false;
+    if (trimmed.length < 2)
+        return false;
+    if (/^\{.*\}$/.test(trimmed))
+        return false;
+    if (/^(https?:\/\/|\/|\.\/|\.\.\/)/.test(trimmed))
+        return false;
+    if (/^[.#][a-zA-Z]/.test(trimmed))
+        return false;
+    if (/^\p{Ll}[\p{L}\p{N}_]*$/u.test(trimmed) && trimmed.length > 20)
+        return false;
+    return /[\p{L}]/u.test(trimmed);
+}
+function extractPlaceholders(text) {
+    // Use a more specific regex that only matches simple variable names to avoid greedy ICU matching
+    // or use non-greedy matching if we expect alphanumeric+underscore names.
+    const matches = text.match(/\{(\w+)\}/g);
+    if (!matches)
+        return [];
+    return matches.map((m) => m.slice(1, -1));
+}
+/**
+ * Protect placeholders by replacing them with AI-safe tokens before translation
+ */
+function protectPlaceholders(text) {
+    // If it's complex ICU syntax, we don't protect simple placeholders as they might be part of the ICU structure
+    // This avoids destroying nested braces like {count, plural, one {# item} other {# items}}
+    if (hasICUSyntax(text)) {
+        return { protectedText: text, vars: [] };
+    }
+    const vars = [];
+    let protectedText = text;
+    // Refined regex to only match simple alphanumeric variable names in braces
+    const regex = /\{(\w+)\}/g;
+    let i = 0;
+    protectedText = text.replace(regex, (match, varName) => {
+        vars.push(varName);
+        const token = `[#VAR_${i}#]`;
+        i++;
+        return token;
+    });
+    return { protectedText, vars };
+}
+/**
+ * Restore placeholders from AI-safe tokens after translation
+ * Returns a diagnostic result to detect dropped variables
+ */
+function restorePlaceholders(text, vars) {
+    let restored = text;
+    let restoredCount = 0;
+    const missingTokens = [];
+    for (let i = 0; i < vars.length; i++) {
+        const regex = new RegExp(`\\[\\s*#\\s*VAR_${i}\\s*#\\s*\\]`, 'gi');
+        const before = restored;
+        restored = restored.replace(regex, `{${vars[i]}}`);
+        if (restored !== before) {
+            restoredCount++;
+        }
+        else {
+            missingTokens.push(`[#VAR_${i}#]`);
+        }
+    }
+    return {
+        restoredText: restored,
+        restoredCount,
+        expectedCount: vars.length,
+        missingTokens
+    };
+}
+/**
+ * Check if text text and associated variables likely represent a pluralized string
+ * (e.g. "count files", "Total {length}")
+ */
+function isLikelyPlural(text, variables = []) {
+    const lowerText = text.toLowerCase().trim();
+    // Exclude question strings — plural generation produces nonsensical variants
+    // for interrogative sentences (e.g. "Do you support tracked shipping?").
+    if (/^(do|does|is|are|can|will|would|have|has|did|should|could|was|were)\s/i.test(lowerText))
+        return false;
+    // Exclude single-brand terms (e.g. "GIFTIFY")
+    if (/^[A-Z]{3,}$/.test(text.trim()))
+        return false;
+    const pluralVars = ['count', 'length', 'total', 'num', 'size'];
+    if (variables.some((v) => pluralVars.includes(v.toLowerCase())))
+        return true;
+    if (/\{\s*(count|length|total|num|size)\s*\}/.test(lowerText))
+        return true;
+    // Digit + plural noun pattern — exclude longer title-like phrases
+    // (5+ words, e.g. "15 reasons naive automation fails") to reduce false positives
+    if (/\b\d+\s+[a-z]+s\b/.test(lowerText)) {
+        const wordCount = lowerText.split(/\s+/).length;
+        if (wordCount >= 5)
+            return false;
+        return true;
+    }
+    return false;
+}
+function hasICUSyntax(text) {
+    return /\{[^}]+,\s*(plural|select|selectordinal)\s*,/.test(text);
+}
+function escapeRegex(text) {
+    return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+function hashString(text) {
+    return (0, node_crypto_1.createHash)('sha256').update(text, 'utf8').digest('hex').slice(0, 16);
+}
+function matchesCurrencyPattern(text) {
+    return /[€$£¥]/.test(text) || /\d[.,]\d{2}\b/.test(text) || /\b\d{1,3}(,\d{3})+\b/.test(text);
+}
+function matchesDatePattern(text) {
+    return (/(0[1-9]|1[012])[- /.](0[1-9]|[12][0-9]|3[01])[- /.](19|20)\d\d/.test(text) ||
+        /\b(19|20)\d\d[- /.](0[1-9]|1[012])[- /.](0[1-9]|[12][0-9]|3[01])\b/.test(text));
+}
+function containsPII(text) {
+    const PATTERNS = {
+        email: /[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*/,
+        ipv4: /\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b/,
+        creditCard: /\b(?:\d[ -]*?){13,16}\b/,
+        secretToken: /\b(?:sk|pk|key|secret|token|auth|password)[-_=]\w{12,}\b/i,
+        hexKey: /\b[a-f0-9]{32,64}\b/i,
+        phone: /\b(?:\+\d{1,4}[- ]?)?\(?\d{2,5}\)?[- ]?\d{3,5}[- ]?\d{3,5}\b/,
+    };
+    return Object.values(PATTERNS).some((pattern) => pattern.test(text));
+}
+exports.SKIP_TAGS = new Set(['script', 'style', 'noscript', 'code', 'pre', 'svg']);
+/**
+ * Extract content between balanced opening and closing tags, handling nested tags.
+ * Finds the first `<tagName>` and its matching `</tagName>`, correctly counting
+ * nested `<tagName ...>` elements.
+ */
+function extractBalancedTag(content, tagName, allowAttrs = false) {
+    const openPattern = allowAttrs ? `<${tagName}` : `<${tagName}>`;
+    const closeTag = `</${tagName}>`;
+    const startIdx = content.indexOf(openPattern);
+    if (startIdx === -1)
+        return null;
+    // For allowAttrs=true, find the end of the opening tag
+    let afterOpen;
+    let attrs = '';
+    if (allowAttrs) {
+        const gtIdx = content.indexOf('>', startIdx);
+        if (gtIdx === -1)
+            return null;
+        afterOpen = gtIdx + 1;
+        attrs = content.slice(startIdx + tagName.length + 1, gtIdx).trim();
+    }
+    else {
+        afterOpen = startIdx + openPattern.length;
+    }
+    let depth = 1;
+    let i = afterOpen;
+    while (i < content.length && depth > 0) {
+        const nextOpen = content.indexOf(`<${tagName}`, i);
+        const nextClose = content.indexOf(closeTag, i);
+        if (nextClose === -1)
+            return null;
+        if (nextOpen !== -1 && nextOpen < nextClose) {
+            depth++;
+            i = nextOpen + tagName.length + 1;
+        }
+        else {
+            depth--;
+            if (depth === 0) {
+                return {
+                    fullMatch: content.slice(startIdx, nextClose + closeTag.length),
+                    body: content.slice(afterOpen, nextClose),
+                    attrs,
+                };
+            }
+            i = nextClose + closeTag.length;
+        }
+    }
+    return null;
+}

package/dist/utils/translation-quality.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Translation Quality Validation
+ *
+ * Validates translation quality and detects potential issues
+ */
+export interface QualityIssue {
+    type: 'missing_placeholder' | 'extra_placeholder' | 'empty_translation' | 'suspicious_length' | 'untranslated_text' | 'html_tag_mismatch' | 'placeholder_order_mismatch';
+    severity: 'error' | 'warning';
+    message: string;
+    original: string;
+    translated: string;
+}
+export interface QualityResult {
+    isValid: boolean;
+    score: number;
+    issues: QualityIssue[];
+}
+/**
+ * Validate translation quality
+ */
+export declare function validateTranslationQuality(original: string, translated: string): QualityResult;
+/**
+ * Validate batch of translations
+ */
+export declare function validateBatchQuality(entries: Array<{
+    key: string;
+    original: string;
+    translated: string;
+}>): {
+    valid: number;
+    invalid: number;
+    averageScore: number;
+    issues: QualityIssue[];
+};