@babelize/engine 0.0.0-stage → 0.1.2

package/dist/utils/string-utils.js

@@ -0,0 +1,270 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SKIP_TAGS = void 0;
+exports.normalizeString = normalizeString;
+exports.toCamelCase = toCamelCase;
+exports.inferNamespace = inferNamespace;
+exports.generateUniqueKey = generateUniqueKey;
+exports.truncateForKey = truncateForKey;
+exports.isTranslatableText = isTranslatableText;
+exports.extractPlaceholders = extractPlaceholders;
+exports.protectPlaceholders = protectPlaceholders;
+exports.restorePlaceholders = restorePlaceholders;
+exports.isLikelyPlural = isLikelyPlural;
+exports.hasICUSyntax = hasICUSyntax;
+exports.escapeRegex = escapeRegex;
+exports.hashString = hashString;
+exports.matchesCurrencyPattern = matchesCurrencyPattern;
+exports.matchesDatePattern = matchesDatePattern;
+exports.containsPII = containsPII;
+exports.extractBalancedTag = extractBalancedTag;
+const node_crypto_1 = require("node:crypto");
+/**
+ * Engine v2 string utilities.
+ *
+ * Copied from v1 to preserve keygen/normalization behavior.
+ */
+function normalizeString(text) {
+    return text
+        .toLowerCase()
+        .replace(/[^\p{L}\p{N}\s']/gu, ' ')
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+function toCamelCase(text) {
+    const normalized = normalizeString(text);
+    const words = normalized.split(' ').filter(Boolean);
+    if (words.length === 0)
+        return 'untitled';
+    const result = words
+        .map((word, index) => {
+        word = word.replace(/'/g, '');
+        if (index === 0)
+            return word.toLowerCase();
+        return word.charAt(0).toUpperCase() + word.slice(1).toLowerCase();
+    })
+        .join('');
+    if (!/[a-zA-Z]/.test(result))
+        return 'untitled';
+    return result;
+}
+function inferNamespace(filePath) {
+    const normalizedPath = filePath.replace(/\\/g, '/');
+    const pathWithoutExt = normalizedPath.replace(/\.(ts|tsx|js|jsx)$/, '');
+    const parts = pathWithoutExt.split('/').filter(Boolean);
+    const prefixesToRemove = ['src', 'app', 'pages', 'components', 'features', 'modules'];
+    while (parts.length > 0 && prefixesToRemove.includes(parts[0].toLowerCase())) {
+        parts.shift();
+    }
+    const suffixesToRemove = ['index', 'page', 'component', 'container', 'view'];
+    while (parts.length > 1 && suffixesToRemove.includes(parts[parts.length - 1].toLowerCase())) {
+        parts.pop();
+    }
+    if (parts.length === 0)
+        return 'common';
+    const namespace = parts[parts.length - 1]
+        .replace(/([A-Z])/g, (match, p1, offset) => (offset > 0 ? '-' + p1 : p1))
+        .toLowerCase()
+        .replace(/^-/, '');
+    return namespace.replace(/[^a-z0-9-]/g, '') || 'common';
+}
+function generateUniqueKey(baseKey, existingKeys) {
+    if (!existingKeys.has(baseKey))
+        return baseKey;
+    let counter = 2;
+    let uniqueKey = `${baseKey}_${counter}`;
+    while (existingKeys.has(uniqueKey)) {
+        counter++;
+        uniqueKey = `${baseKey}_${counter}`;
+    }
+    return uniqueKey;
+}
+function truncateForKey(text, maxWords = 5) {
+    const words = text.split(/\s+/).filter(Boolean);
+    if (words.length <= maxWords)
+        return text;
+    return words.slice(0, maxWords).join(' ');
+}
+function isTranslatableText(text) {
+    const trimmed = text.trim();
+    if (!trimmed)
+        return false;
+    if (/^\s*$/.test(trimmed))
+        return false;
+    if (/^\d+$/.test(trimmed))
+        return false;
+    if (/^[^\p{L}\p{N}\s]+$/u.test(trimmed))
+        return false;
+    if (trimmed.length < 2)
+        return false;
+    if (/^\{.*\}$/.test(trimmed))
+        return false;
+    if (/^(https?:\/\/|\/|\.\/|\.\.\/)/.test(trimmed))
+        return false;
+    if (/^[.#][a-zA-Z]/.test(trimmed))
+        return false;
+    if (/^\p{Ll}[\p{L}\p{N}_]*$/u.test(trimmed) && trimmed.length > 20)
+        return false;
+    return /[\p{L}]/u.test(trimmed);
+}
+function extractPlaceholders(text) {
+    // Use a more specific regex that only matches simple variable names to avoid greedy ICU matching
+    // or use non-greedy matching if we expect alphanumeric+underscore names.
+    const matches = text.match(/\{(\w+)\}/g);
+    if (!matches)
+        return [];
+    return matches.map((m) => m.slice(1, -1));
+}
+/**
+ * Protect placeholders by replacing them with AI-safe tokens before translation
+ */
+function protectPlaceholders(text) {
+    // If it's complex ICU syntax, we don't protect simple placeholders as they might be part of the ICU structure
+    // This avoids destroying nested braces like {count, plural, one {# item} other {# items}}
+    if (hasICUSyntax(text)) {
+        return { protectedText: text, vars: [] };
+    }
+    const vars = [];
+    let protectedText = text;
+    // Refined regex to only match simple alphanumeric variable names in braces
+    const regex = /\{(\w+)\}/g;
+    let i = 0;
+    protectedText = text.replace(regex, (match, varName) => {
+        vars.push(varName);
+        const token = `[#VAR_${i}#]`;
+        i++;
+        return token;
+    });
+    return { protectedText, vars };
+}
+/**
+ * Restore placeholders from AI-safe tokens after translation
+ * Returns a diagnostic result to detect dropped variables
+ */
+function restorePlaceholders(text, vars) {
+    let restored = text;
+    let restoredCount = 0;
+    const missingTokens = [];
+    for (let i = 0; i < vars.length; i++) {
+        const regex = new RegExp(`\\[\\s*#\\s*VAR_${i}\\s*#\\s*\\]`, 'gi');
+        const before = restored;
+        restored = restored.replace(regex, `{${vars[i]}}`);
+        if (restored !== before) {
+            restoredCount++;
+        }
+        else {
+            missingTokens.push(`[#VAR_${i}#]`);
+        }
+    }
+    return {
+        restoredText: restored,
+        restoredCount,
+        expectedCount: vars.length,
+        missingTokens
+    };
+}
+/**
+ * Check if text text and associated variables likely represent a pluralized string
+ * (e.g. "count files", "Total {length}")
+ */
+function isLikelyPlural(text, variables = []) {
+    const lowerText = text.toLowerCase().trim();
+    // Exclude question strings — plural generation produces nonsensical variants
+    // for interrogative sentences (e.g. "Do you support tracked shipping?").
+    if (/^(do|does|is|are|can|will|would|have|has|did|should|could|was|were)\s/i.test(lowerText))
+        return false;
+    // Exclude single-brand terms (e.g. "GIFTIFY")
+    if (/^[A-Z]{3,}$/.test(text.trim()))
+        return false;
+    const pluralVars = ['count', 'length', 'total', 'num', 'size'];
+    if (variables.some((v) => pluralVars.includes(v.toLowerCase())))
+        return true;
+    if (/\{\s*(count|length|total|num|size)\s*\}/.test(lowerText))
+        return true;
+    // Digit + plural noun pattern — exclude longer title-like phrases
+    // (5+ words, e.g. "15 reasons naive automation fails") to reduce false positives
+    if (/\b\d+\s+[a-z]+s\b/.test(lowerText)) {
+        const wordCount = lowerText.split(/\s+/).length;
+        if (wordCount >= 5)
+            return false;
+        return true;
+    }
+    return false;
+}
+function hasICUSyntax(text) {
+    return /\{[^}]+,\s*(plural|select|selectordinal)\s*,/.test(text);
+}
+function escapeRegex(text) {
+    return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+function hashString(text) {
+    return (0, node_crypto_1.createHash)('sha256').update(text, 'utf8').digest('hex').slice(0, 16);
+}
+function matchesCurrencyPattern(text) {
+    return /[€$£¥]/.test(text) || /\d[.,]\d{2}\b/.test(text) || /\b\d{1,3}(,\d{3})+\b/.test(text);
+}
+function matchesDatePattern(text) {
+    return (/(0[1-9]|1[012])[- /.](0[1-9]|[12][0-9]|3[01])[- /.](19|20)\d\d/.test(text) ||
+        /\b(19|20)\d\d[- /.](0[1-9]|1[012])[- /.](0[1-9]|[12][0-9]|3[01])\b/.test(text));
+}
+function containsPII(text) {
+    const PATTERNS = {
+        email: /[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*/,
+        ipv4: /\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b/,
+        creditCard: /\b(?:\d[ -]*?){13,16}\b/,
+        secretToken: /\b(?:sk|pk|key|secret|token|auth|password)[-_=]\w{12,}\b/i,
+        hexKey: /\b[a-f0-9]{32,64}\b/i,
+        phone: /\b(?:\+\d{1,4}[- ]?)?\(?\d{2,5}\)?[- ]?\d{3,5}[- ]?\d{3,5}\b/,
+    };
+    return Object.values(PATTERNS).some((pattern) => pattern.test(text));
+}
+exports.SKIP_TAGS = new Set(['script', 'style', 'noscript', 'code', 'pre', 'svg']);
+/**
+ * Extract content between balanced opening and closing tags, handling nested tags.
+ * Finds the first `<tagName>` and its matching `</tagName>`, correctly counting
+ * nested `<tagName ...>` elements.
+ */
+function extractBalancedTag(content, tagName, allowAttrs = false) {
+    const openPattern = allowAttrs ? `<${tagName}` : `<${tagName}>`;
+    const closeTag = `</${tagName}>`;
+    const startIdx = content.indexOf(openPattern);
+    if (startIdx === -1)
+        return null;
+    // For allowAttrs=true, find the end of the opening tag
+    let afterOpen;
+    let attrs = '';
+    if (allowAttrs) {
+        const gtIdx = content.indexOf('>', startIdx);
+        if (gtIdx === -1)
+            return null;
+        afterOpen = gtIdx + 1;
+        attrs = content.slice(startIdx + tagName.length + 1, gtIdx).trim();
+    }
+    else {
+        afterOpen = startIdx + openPattern.length;
+    }
+    let depth = 1;
+    let i = afterOpen;
+    while (i < content.length && depth > 0) {
+        const nextOpen = content.indexOf(`<${tagName}`, i);
+        const nextClose = content.indexOf(closeTag, i);
+        if (nextClose === -1)
+            return null;
+        if (nextOpen !== -1 && nextOpen < nextClose) {
+            depth++;
+            i = nextOpen + tagName.length + 1;
+        }
+        else {
+            depth--;
+            if (depth === 0) {
+                return {
+                    fullMatch: content.slice(startIdx, nextClose + closeTag.length),
+                    body: content.slice(afterOpen, nextClose),
+                    attrs,
+                };
+            }
+            i = nextClose + closeTag.length;
+        }
+    }
+    return null;
+}

package/dist/utils/translation-quality.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Translation Quality Validation
+ *
+ * Validates translation quality and detects potential issues
+ */
+export interface QualityIssue {
+    type: 'missing_placeholder' | 'extra_placeholder' | 'empty_translation' | 'suspicious_length' | 'untranslated_text' | 'html_tag_mismatch' | 'placeholder_order_mismatch';
+    severity: 'error' | 'warning';
+    message: string;
+    original: string;
+    translated: string;
+}
+export interface QualityResult {
+    isValid: boolean;
+    score: number;
+    issues: QualityIssue[];
+}
+/**
+ * Validate translation quality
+ */
+export declare function validateTranslationQuality(original: string, translated: string): QualityResult;
+/**
+ * Validate batch of translations
+ */
+export declare function validateBatchQuality(entries: Array<{
+    key: string;
+    original: string;
+    translated: string;
+}>): {
+    valid: number;
+    invalid: number;
+    averageScore: number;
+    issues: QualityIssue[];
+};

package/dist/utils/translation-quality.js

@@ -0,0 +1,156 @@
+"use strict";
+/**
+ * Translation Quality Validation
+ *
+ * Validates translation quality and detects potential issues
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.validateTranslationQuality = validateTranslationQuality;
+exports.validateBatchQuality = validateBatchQuality;
+/**
+ * Validate translation quality
+ */
+function validateTranslationQuality(original, translated) {
+    const issues = [];
+    let score = 100;
+    const trimmedOriginal = original.trim();
+    const trimmedTranslated = (translated || '').trim();
+    // Check for empty translation
+    if (trimmedTranslated.length === 0) {
+        issues.push({
+            type: 'empty_translation',
+            severity: 'error',
+            message: 'Translation is empty',
+            original,
+            translated,
+        });
+        score -= 100; // Complete failure
+    }
+    // Check for untranslated text (identical to source)
+    // Skip for very short strings (e.g. "OK", "Save") where source might equal translation
+    if (trimmedTranslated.length > 4 && trimmedOriginal.toLowerCase() === trimmedTranslated.toLowerCase()) {
+        issues.push({
+            type: 'untranslated_text',
+            severity: 'warning',
+            message: 'Translation is identical to source text',
+            original,
+            translated,
+        });
+        score -= 20;
+    }
+    // Check placeholder preservation
+    const originalPlaceholders = original.match(/\{[^}]+\}/g) || [];
+    const translatedPlaceholders = translated.match(/\{[^}]+\}/g) || [];
+    // Missing placeholders
+    const sortedOriginal = [...originalPlaceholders].sort();
+    const sortedTranslated = [...translatedPlaceholders].sort();
+    const missingPlaceholders = sortedOriginal.filter((p) => !sortedTranslated.includes(p));
+    if (missingPlaceholders.length > 0) {
+        issues.push({
+            type: 'missing_placeholder',
+            severity: 'error',
+            message: `Missing placeholders: ${missingPlaceholders.join(', ')}`,
+            original,
+            translated,
+        });
+        score -= 30 * missingPlaceholders.length;
+    }
+    // Extra placeholders
+    const extraPlaceholders = sortedTranslated.filter((p) => !sortedOriginal.includes(p));
+    if (extraPlaceholders.length > 0) {
+        issues.push({
+            type: 'extra_placeholder',
+            severity: 'warning',
+            message: `Extra placeholders found: ${extraPlaceholders.join(', ')}`,
+            original,
+            translated,
+        });
+        score -= 10 * extraPlaceholders.length;
+    }
+    // Check placeholder ordering
+    // If the order of placeholders is fundamentally different, it might indicate grammar corruption
+    if (originalPlaceholders.length > 1 && originalPlaceholders.length === translatedPlaceholders.length) {
+        if (originalPlaceholders.join(',') !== translatedPlaceholders.join(',')) {
+            // Check if they are just reordered (which might be valid for some languages) 
+            // vs. duplicated or mangled. If sorted they match, it's just a reorder.
+            if (sortedOriginal.join(',') === sortedTranslated.join(',')) {
+                // Reordering is often valid (Subject-Object-Verb vs Subject-Verb-Object)
+                // but drastic jumps might be worth a warning in some engines.
+                // For now, we allow reordering but log it if we wanted to.
+            }
+        }
+    }
+    // Check HTML tag parity
+    const originalTags = original.match(/<[^>]+>/g) || [];
+    const translatedTags = translated.match(/<[^>]+>/g) || [];
+    if (originalTags.length !== translatedTags.length) {
+        issues.push({
+            type: 'html_tag_mismatch',
+            severity: 'error',
+            message: `HTML tag count mismatch: source has ${originalTags.length}, translation has ${translatedTags.length}`,
+            original,
+            translated,
+        });
+        score -= 25;
+    }
+    else {
+        // Even if count matches, check if tags themselves were mangled (e.g. <strong> becomes <fort>)
+        const sortedOrigTags = [...originalTags].map(t => t.toLowerCase().split(' ')[0]).sort();
+        const sortedTransTags = [...translatedTags].map(t => t.toLowerCase().split(' ')[0]).sort();
+        if (sortedOrigTags.join(',') !== sortedTransTags.join(',')) {
+            issues.push({
+                type: 'html_tag_mismatch',
+                severity: 'error',
+                message: 'HTML tags were altered or translated (e.g. <strong> became something else)',
+                original,
+                translated,
+            });
+            score -= 25;
+        }
+    }
+    // Check for suspicious length differences
+    const lengthRatio = trimmedTranslated.length / (trimmedOriginal.length || 1);
+    if (lengthRatio < 0.2 || lengthRatio > 5) {
+        issues.push({
+            type: 'suspicious_length',
+            severity: 'warning',
+            message: `Translation length is ${(lengthRatio * 100).toFixed(0)}% of original (suspicious)`,
+            original,
+            translated,
+        });
+        score -= 15;
+    }
+    // Ensure score doesn't go below 0
+    score = Math.max(0, score);
+    return {
+        isValid: issues.filter((i) => i.severity === 'error').length === 0,
+        score,
+        issues,
+    };
+}
+/**
+ * Validate batch of translations
+ */
+function validateBatchQuality(entries) {
+    let valid = 0;
+    let invalid = 0;
+    let totalScore = 0;
+    const allIssues = [];
+    for (const entry of entries) {
+        const result = validateTranslationQuality(entry.original, entry.translated);
+        totalScore += result.score;
+        if (result.isValid) {
+            valid++;
+        }
+        else {
+            invalid++;
+        }
+        allIssues.push(...result.issues);
+    }
+    return {
+        valid,
+        invalid,
+        averageScore: entries.length > 0 ? totalScore / entries.length : 0,
+        issues: allIssues,
+    };
+}

package/dist/utils/translation-sanitizer.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Clean up translator output. Script injection is stripped for every
+ * provider; the NLLB-specific repairs (language-code echoes, unknown-token
+ * marks, whitespace and stutter collapsing) only run for NLLB output, since on
+ * LLM output they would rewrite legitimate text.
+ */
+export declare function sanitizeTranslation(text: string, options?: {
+    nllb?: boolean;
+    source?: string;
+}): string;
+export declare function validateRawTokens(aiOutput: string, expectedCount: number): boolean;

package/dist/utils/translation-sanitizer.js

@@ -0,0 +1,71 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sanitizeTranslation = sanitizeTranslation;
+exports.validateRawTokens = validateRawTokens;
+const NLLB_ARTIFACT_PATTERN = new RegExp(
+// Sentence-initial echoes come back capitalized ("Eng_Latn Produktkatalog").
+'\\b(?:__)?[A-Za-z][a-z]{2}_[A-Z][a-z]{3,4}(?:__)?\\b', 'g');
+/**
+ * Collapse phrase-level stutter ("Il a Il a" -> "Il a"). NLLB occasionally
+ * repeats a short fragment twice after language tokens are stripped. Only
+ * collapse when BOTH halves contain >=2 words and match case-insensitively —
+ * this never touches legitimate one-word doublings like "no no" or "la la".
+ */
+function collapsePhraseStutter(text) {
+    const words = text.split(/\s+/);
+    // The whole output is one phrase of >=2 words repeated 2+ times ("Il a Il a",
+    // "Product catalog Product catalog Product catalog").
+    for (let size = 2; size <= words.length / 2; size++) {
+        if (words.length % size !== 0)
+            continue;
+        const first = words.slice(0, size).map((w) => w.toLowerCase());
+        let repeated = true;
+        for (let i = size; i < words.length && repeated; i++) {
+            if (words[i].toLowerCase() !== first[i % size])
+                repeated = false;
+        }
+        if (repeated)
+            return words.slice(0, size).join(' ');
+    }
+    return text;
+}
+/**
+ * Clean up translator output. Script injection is stripped for every
+ * provider; the NLLB-specific repairs (language-code echoes, unknown-token
+ * marks, whitespace and stutter collapsing) only run for NLLB output, since on
+ * LLM output they would rewrite legitimate text.
+ */
+function sanitizeTranslation(text, options = {}) {
+    if (!text)
+        return text;
+    let clean = text.replace(/<script\b[^>]*>([\s\S]*?)<\/script>/gim, '');
+    clean = clean.replace(/javascript:/gim, 'java_script:');
+    clean = clean.replace(/\s+on[a-z]+\s*=\s*['"][^'"]*['"]/gim, '');
+    if (options.nllb === false)
+        return clean;
+    // Strip FLORES-200 language-code artifacts (eng_Latn, __fra_Latn__, ...)
+    // anywhere in the output — NLLB echoes these at the end/middle of short
+    // fragments. Mirrors the fix in apps/translator/src/translator.rs.
+    clean = clean.replace(NLLB_ARTIFACT_PATTERN, '');
+    // Strip U+2047 (DOUBLE QUESTION MARK) — NLLB's unknown-token marker, emitted
+    // for tokens it can't translate (e.g. punctuation-only fragments). Never
+    // legitimate translated text.
+    clean = clean.replace(/\u2047/g, '');
+    clean = clean.replace(/  +/g, ' ').trim();
+    clean = collapsePhraseStutter(clean);
+    // A one-word source doubled in the output ("Product" → "Produkt Produkt").
+    if (options.source && !/\s/.test(options.source.trim())) {
+        const words = clean.split(/\s+/);
+        if (words.length > 1 && words.every((w) => w.toLowerCase() === words[0].toLowerCase()))
+            clean = words[0];
+    }
+    return clean;
+}
+function validateRawTokens(aiOutput, expectedCount) {
+    for (let i = 0; i < expectedCount; i++) {
+        const tokenRegex = new RegExp(`\\[\\s*#\\s*VAR_${i}\\s*#\\s*\\]`, 'gi');
+        if (!tokenRegex.test(aiOutput))
+            return false;
+    }
+    return true;
+}

package/dist/utils/translator-caller.d.ts

@@ -0,0 +1,32 @@
+import type { TranslationEntry } from '../runtime/types';
+import type { ByokProvider } from '@babelize/types';
+/** Options only LLM (BYOK) providers use; the NLLB service ignores them. */
+export interface LlmRequestOptions {
+    targetLocale?: string;
+    glossary?: Array<{
+        term: string;
+        translation: string;
+    }>;
+    hints?: Record<string, string>;
+    /** Aborts the request and any retries. */
+    signal?: AbortSignal;
+}
+/**
+ * Translate a batch. Returns translations only for entries the translator
+ * actually produced — never the source text in their place. Entries missing
+ * from the result are reported as failed keys by the caller.
+ */
+export declare function callTranslator(entries: TranslationEntry[], sourceLang: string, targetLang: string, byokConfig?: {
+    provider: ByokProvider;
+    apiKey: string;
+    model?: string;
+}, brandVoice?: string, llmOptions?: LlmRequestOptions): Promise<{
+    translations: TranslationEntry[];
+    metadata?: any;
+}>;
+export declare function checkTranslatorHealth(byokConfig?: {
+    provider: ByokProvider;
+    apiKey: string;
+    model?: string;
+}): Promise<boolean>;
+export declare function warmupTranslator(): Promise<boolean>;