@babelize/engine 0.0.0-stage → 0.1.0

package/dist/utils/translation-quality.js

@@ -0,0 +1,156 @@
+"use strict";
+/**
+ * Translation Quality Validation
+ *
+ * Validates translation quality and detects potential issues
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.validateTranslationQuality = validateTranslationQuality;
+exports.validateBatchQuality = validateBatchQuality;
+/**
+ * Validate translation quality
+ */
+function validateTranslationQuality(original, translated) {
+    const issues = [];
+    let score = 100;
+    const trimmedOriginal = original.trim();
+    const trimmedTranslated = (translated || '').trim();
+    // Check for empty translation
+    if (trimmedTranslated.length === 0) {
+        issues.push({
+            type: 'empty_translation',
+            severity: 'error',
+            message: 'Translation is empty',
+            original,
+            translated,
+        });
+        score -= 100; // Complete failure
+    }
+    // Check for untranslated text (identical to source)
+    // Skip for very short strings (e.g. "OK", "Save") where source might equal translation
+    if (trimmedTranslated.length > 4 && trimmedOriginal.toLowerCase() === trimmedTranslated.toLowerCase()) {
+        issues.push({
+            type: 'untranslated_text',
+            severity: 'warning',
+            message: 'Translation is identical to source text',
+            original,
+            translated,
+        });
+        score -= 20;
+    }
+    // Check placeholder preservation
+    const originalPlaceholders = original.match(/\{[^}]+\}/g) || [];
+    const translatedPlaceholders = translated.match(/\{[^}]+\}/g) || [];
+    // Missing placeholders
+    const sortedOriginal = [...originalPlaceholders].sort();
+    const sortedTranslated = [...translatedPlaceholders].sort();
+    const missingPlaceholders = sortedOriginal.filter((p) => !sortedTranslated.includes(p));
+    if (missingPlaceholders.length > 0) {
+        issues.push({
+            type: 'missing_placeholder',
+            severity: 'error',
+            message: `Missing placeholders: ${missingPlaceholders.join(', ')}`,
+            original,
+            translated,
+        });
+        score -= 30 * missingPlaceholders.length;
+    }
+    // Extra placeholders
+    const extraPlaceholders = sortedTranslated.filter((p) => !sortedOriginal.includes(p));
+    if (extraPlaceholders.length > 0) {
+        issues.push({
+            type: 'extra_placeholder',
+            severity: 'warning',
+            message: `Extra placeholders found: ${extraPlaceholders.join(', ')}`,
+            original,
+            translated,
+        });
+        score -= 10 * extraPlaceholders.length;
+    }
+    // Check placeholder ordering
+    // If the order of placeholders is fundamentally different, it might indicate grammar corruption
+    if (originalPlaceholders.length > 1 && originalPlaceholders.length === translatedPlaceholders.length) {
+        if (originalPlaceholders.join(',') !== translatedPlaceholders.join(',')) {
+            // Check if they are just reordered (which might be valid for some languages) 
+            // vs. duplicated or mangled. If sorted they match, it's just a reorder.
+            if (sortedOriginal.join(',') === sortedTranslated.join(',')) {
+                // Reordering is often valid (Subject-Object-Verb vs Subject-Verb-Object)
+                // but drastic jumps might be worth a warning in some engines.
+                // For now, we allow reordering but log it if we wanted to.
+            }
+        }
+    }
+    // Check HTML tag parity
+    const originalTags = original.match(/<[^>]+>/g) || [];
+    const translatedTags = translated.match(/<[^>]+>/g) || [];
+    if (originalTags.length !== translatedTags.length) {
+        issues.push({
+            type: 'html_tag_mismatch',
+            severity: 'error',
+            message: `HTML tag count mismatch: source has ${originalTags.length}, translation has ${translatedTags.length}`,
+            original,
+            translated,
+        });
+        score -= 25;
+    }
+    else {
+        // Even if count matches, check if tags themselves were mangled (e.g. <strong> becomes <fort>)
+        const sortedOrigTags = [...originalTags].map(t => t.toLowerCase().split(' ')[0]).sort();
+        const sortedTransTags = [...translatedTags].map(t => t.toLowerCase().split(' ')[0]).sort();
+        if (sortedOrigTags.join(',') !== sortedTransTags.join(',')) {
+            issues.push({
+                type: 'html_tag_mismatch',
+                severity: 'error',
+                message: 'HTML tags were altered or translated (e.g. <strong> became something else)',
+                original,
+                translated,
+            });
+            score -= 25;
+        }
+    }
+    // Check for suspicious length differences
+    const lengthRatio = trimmedTranslated.length / (trimmedOriginal.length || 1);
+    if (lengthRatio < 0.2 || lengthRatio > 5) {
+        issues.push({
+            type: 'suspicious_length',
+            severity: 'warning',
+            message: `Translation length is ${(lengthRatio * 100).toFixed(0)}% of original (suspicious)`,
+            original,
+            translated,
+        });
+        score -= 15;
+    }
+    // Ensure score doesn't go below 0
+    score = Math.max(0, score);
+    return {
+        isValid: issues.filter((i) => i.severity === 'error').length === 0,
+        score,
+        issues,
+    };
+}
+/**
+ * Validate batch of translations
+ */
+function validateBatchQuality(entries) {
+    let valid = 0;
+    let invalid = 0;
+    let totalScore = 0;
+    const allIssues = [];
+    for (const entry of entries) {
+        const result = validateTranslationQuality(entry.original, entry.translated);
+        totalScore += result.score;
+        if (result.isValid) {
+            valid++;
+        }
+        else {
+            invalid++;
+        }
+        allIssues.push(...result.issues);
+    }
+    return {
+        valid,
+        invalid,
+        averageScore: entries.length > 0 ? totalScore / entries.length : 0,
+        issues: allIssues,
+    };
+}

package/dist/utils/translation-sanitizer.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Clean up translator output. Script injection is stripped for every
+ * provider; the NLLB-specific repairs (language-code echoes, unknown-token
+ * marks, whitespace and stutter collapsing) only run for NLLB output, since on
+ * LLM output they would rewrite legitimate text.
+ */
+export declare function sanitizeTranslation(text: string, options?: {
+    nllb?: boolean;
+}): string;
+export declare function validateRawTokens(aiOutput: string, expectedCount: number): boolean;

package/dist/utils/translation-sanitizer.js

@@ -0,0 +1,57 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sanitizeTranslation = sanitizeTranslation;
+exports.validateRawTokens = validateRawTokens;
+const NLLB_ARTIFACT_PATTERN = new RegExp('\\b(?:__)?[a-z]{3}_[A-Z][a-z]{3,4}(?:__)?\\b', 'g');
+/**
+ * Collapse phrase-level stutter ("Il a Il a" -> "Il a"). NLLB occasionally
+ * repeats a short fragment twice after language tokens are stripped. Only
+ * collapse when BOTH halves contain >=2 words and match case-insensitively —
+ * this never touches legitimate one-word doublings like "no no" or "la la".
+ */
+function collapsePhraseStutter(text) {
+    const words = text.split(/\s+/);
+    if (words.length >= 4 && words.length % 2 === 0) {
+        const half = words.length / 2;
+        const first = words.slice(0, half);
+        const second = words.slice(half);
+        if (first.length >= 2 &&
+            first.every((w, i) => w.toLowerCase() === second[i].toLowerCase())) {
+            return first.join(' ');
+        }
+    }
+    return text;
+}
+/**
+ * Clean up translator output. Script injection is stripped for every
+ * provider; the NLLB-specific repairs (language-code echoes, unknown-token
+ * marks, whitespace and stutter collapsing) only run for NLLB output, since on
+ * LLM output they would rewrite legitimate text.
+ */
+function sanitizeTranslation(text, options = {}) {
+    if (!text)
+        return text;
+    let clean = text.replace(/<script\b[^>]*>([\s\S]*?)<\/script>/gim, '');
+    clean = clean.replace(/javascript:/gim, 'java_script:');
+    clean = clean.replace(/\s+on[a-z]+\s*=\s*['"][^'"]*['"]/gim, '');
+    if (options.nllb === false)
+        return clean;
+    // Strip FLORES-200 language-code artifacts (eng_Latn, __fra_Latn__, ...)
+    // anywhere in the output — NLLB echoes these at the end/middle of short
+    // fragments. Mirrors the fix in apps/translator/src/translator.rs.
+    clean = clean.replace(NLLB_ARTIFACT_PATTERN, '');
+    // Strip U+2047 (DOUBLE QUESTION MARK) — NLLB's unknown-token marker, emitted
+    // for tokens it can't translate (e.g. punctuation-only fragments). Never
+    // legitimate translated text.
+    clean = clean.replace(/\u2047/g, '');
+    clean = clean.replace(/  +/g, ' ').trim();
+    return collapsePhraseStutter(clean);
+}
+function validateRawTokens(aiOutput, expectedCount) {
+    for (let i = 0; i < expectedCount; i++) {
+        const tokenRegex = new RegExp(`\\[\\s*#\\s*VAR_${i}\\s*#\\s*\\]`, 'gi');
+        if (!tokenRegex.test(aiOutput))
+            return false;
+    }
+    return true;
+}

package/dist/utils/translator-caller.d.ts

@@ -0,0 +1,32 @@
+import type { TranslationEntry } from '../runtime/types';
+import type { ByokProvider } from '@babelize/types';
+/** Options only LLM (BYOK) providers use; the NLLB service ignores them. */
+export interface LlmRequestOptions {
+    targetLocale?: string;
+    glossary?: Array<{
+        term: string;
+        translation: string;
+    }>;
+    hints?: Record<string, string>;
+    /** Aborts the request and any retries. */
+    signal?: AbortSignal;
+}
+/**
+ * Translate a batch. Returns translations only for entries the translator
+ * actually produced — never the source text in their place. Entries missing
+ * from the result are reported as failed keys by the caller.
+ */
+export declare function callTranslator(entries: TranslationEntry[], sourceLang: string, targetLang: string, byokConfig?: {
+    provider: ByokProvider;
+    apiKey: string;
+    model?: string;
+}, brandVoice?: string, llmOptions?: LlmRequestOptions): Promise<{
+    translations: TranslationEntry[];
+    metadata?: any;
+}>;
+export declare function checkTranslatorHealth(byokConfig?: {
+    provider: ByokProvider;
+    apiKey: string;
+    model?: string;
+}): Promise<boolean>;
+export declare function warmupTranslator(): Promise<boolean>;

package/dist/utils/translator-caller.js

@@ -0,0 +1,217 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.callTranslator = callTranslator;
+exports.checkTranslatorHealth = checkTranslatorHealth;
+exports.warmupTranslator = warmupTranslator;
+const logger_1 = require("./logger");
+const translation_quality_1 = require("./translation-quality");
+const byok_provider_1 = require("../modules/translator/client/byok.provider");
+/**
+ * Translate a batch. Returns translations only for entries the translator
+ * actually produced — never the source text in their place. Entries missing
+ * from the result are reported as failed keys by the caller.
+ */
+async function callTranslator(entries, sourceLang, targetLang, byokConfig, brandVoice, llmOptions = {}) {
+    if (byokConfig) {
+        return callByokTranslator(entries, sourceLang, targetLang, byokConfig, brandVoice, llmOptions);
+    }
+    const outerSignal = llmOptions.signal;
+    const { translatorLoadBalancer } = await Promise.resolve().then(() => __importStar(require('./translator-load-balancer')));
+    let rustResult = null;
+    try {
+        rustResult = await translatorLoadBalancer.executeWithFailover(async (translatorUrl) => {
+            const response = await fetch(`${translatorUrl}/translate`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ sourceLang, targetLang, entries }),
+                signal: outerSignal ? AbortSignal.any([AbortSignal.timeout(300000), outerSignal]) : AbortSignal.timeout(300000),
+            });
+            if (!response.ok) {
+                throw new Error(`Translation API error: ${response.status} ${response.statusText}`);
+            }
+            const data = await response.json();
+            if (data.metadata) {
+                logger_1.engineLogger.info('Translation metadata', {
+                    dictVersion: data.metadata.dictVersion,
+                    processingTimeMs: data.metadata.processingTimeMs,
+                    dictionaryHits: data.metadata.dictionaryHits,
+                    modelTranslations: data.metadata.modelTranslations,
+                });
+            }
+            return { translations: data.translations || [], metadata: data.metadata };
+        });
+    }
+    catch (err) {
+        logger_1.engineLogger.warn('Rust translator unavailable, will try HF Inference API fallback', {
+            error: err instanceof Error ? err.message : String(err),
+        });
+    }
+    const needsFallback = rustResult === null || (rustResult.translations.length > 0 &&
+        rustResult.translations.every((t) => {
+            const orig = entries.find((e) => e.key === t.key)?.value || '';
+            if (orig.length <= 4)
+                return false;
+            const val = t.value || '';
+            if (orig.toLowerCase() === val.toLowerCase())
+                return true;
+            return /^\s*(?:[a-z]{3}_[A-Za-z]{4})\s+/.test(val);
+        }));
+    if (needsFallback) {
+        logger_1.engineLogger.warn('Rust translator returned untargeted translations or unavailable, falling back to HF Inference API');
+        try {
+            const hfResults = await callHuggingFaceNLLB(entries, sourceLang, targetLang);
+            if (hfResults.translations.length > 0) {
+                return { translations: hfResults.translations, metadata: { ...rustResult?.metadata, fallback: 'hf-inference-api' } };
+            }
+        }
+        catch (hfErr) {
+            logger_1.engineLogger.error('HF Inference API fallback also failed', { error: hfErr });
+        }
+    }
+    if (rustResult !== null && rustResult.translations.length > 0) {
+        return rustResult;
+    }
+    // Never pass the source text off as a translation: an empty result makes
+    // every key a reported failure, and the next run retries them.
+    logger_1.engineLogger.error(`Translator unavailable for ${sourceLang} → ${targetLang}; no translations produced`);
+    return { translations: [], metadata: { translatorUnavailable: true } };
+}
+async function callByokTranslator(entries, sourceLang, targetLang, byokConfig, brandVoice, llmOptions = {}) {
+    const client = new byok_provider_1.BYOKTranslatorClient(byokConfig);
+    const { signal, ...promptOptions } = llmOptions;
+    const hfRequest = {
+        entries: entries.map((e) => ({ key: e.key, value: e.value, context: e.parentContext, extractionContext: e.extractionContext })),
+        sourceLang,
+        targetLang,
+        brandVoice,
+        ...promptOptions,
+    };
+    const result = await client.translate(hfRequest, signal);
+    const byKey = new Map(result.translations.map((t) => [t.key, t]));
+    const translations = [];
+    for (const entry of entries) {
+        const t = byKey.get(entry.key);
+        if (!t)
+            continue;
+        const quality = (0, translation_quality_1.validateTranslationQuality)(entry.value, t.value);
+        translations.push({
+            key: entry.key,
+            value: t.value,
+            confidence: quality.isValid ? Math.max(0.5, quality.score / 100) : 0.3,
+            engine: byokConfig.provider,
+            warnings: quality.issues?.map((i) => `${i.type}:${i.severity}`) || [],
+        });
+    }
+    return { translations, metadata: result.metadata };
+}
+async function callHuggingFaceNLLB(entries, sourceLang, targetLang) {
+    const hfToken = process.env.HF_TOKEN || process.env.HF_API_TOKEN;
+    const model = 'facebook/nllb-200-distilled-600M';
+    const baseUrl = `https://api-inference.huggingface.co/models/${model}`;
+    const translations = [];
+    for (const entry of entries) {
+        const response = await fetch(baseUrl, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                ...(hfToken ? { Authorization: `Bearer ${hfToken}` } : {}),
+            },
+            body: JSON.stringify({
+                inputs: entry.value,
+                parameters: { src_lang: sourceLang, tgt_lang: targetLang },
+            }),
+            signal: AbortSignal.timeout(60000),
+        });
+        if (!response.ok) {
+            const body = await response.text().catch(() => '');
+            throw new Error(`HF Inference API error: ${response.status} — ${body.slice(0, 200)}`);
+        }
+        const data = await response.json();
+        const translatedText = data?.[0]?.translation_text;
+        if (!translatedText)
+            continue;
+        translations.push({
+            key: entry.key,
+            value: translatedText,
+            confidence: 0.9,
+            engine: 'hf-nllb-200',
+        });
+    }
+    return { translations };
+}
+async function checkTranslatorHealth(byokConfig) {
+    try {
+        if (byokConfig) {
+            const client = new byok_provider_1.BYOKTranslatorClient(byokConfig);
+            return await client.health();
+        }
+        const { DirectTranslatorClient } = await Promise.resolve().then(() => __importStar(require('../modules/translator/client/direct-translator.client')));
+        const client = new DirectTranslatorClient();
+        return await client.health();
+    }
+    catch {
+        return false;
+    }
+}
+async function warmupTranslator() {
+    logger_1.engineLogger.info('[Translator] Starting warmup...');
+    const MAX_ATTEMPTS = 3;
+    const RETRY_DELAY_MS = 15000;
+    for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+        try {
+            logger_1.engineLogger.info(`[Translator] Warmup attempt ${attempt}/${MAX_ATTEMPTS}...`);
+            const { DirectTranslatorClient } = await Promise.resolve().then(() => __importStar(require('../modules/translator/client/direct-translator.client')));
+            const client = new DirectTranslatorClient();
+            const ok = await client.warmup();
+            if (ok) {
+                logger_1.engineLogger.info('[Translator] Warmup successful');
+                return true;
+            }
+            logger_1.engineLogger.warn(`[Translator] Warmup attempt ${attempt} failed`);
+        }
+        catch (err) {
+            logger_1.engineLogger.warn(`[Translator] Warmup attempt ${attempt} failed:`, {
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+        if (attempt < MAX_ATTEMPTS) {
+            logger_1.engineLogger.info(`[Translator] Retrying warmup in ${RETRY_DELAY_MS / 1000}s...`);
+            await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY_MS));
+        }
+    }
+    logger_1.engineLogger.warn('[Translator] All warmup attempts failed — running cold');
+    return false;
+}

package/dist/utils/translator-load-balancer.d.ts

@@ -0,0 +1,73 @@
+/**
+ * Translator Load Balancer
+ *
+ * Manages multiple translator endpoints with:
+ * - Health checking
+ * - Round-robin load balancing
+ * - Automatic failover on errors
+ * - Circuit breaker per endpoint
+ */
+export interface TranslatorEndpoint {
+    url: string;
+    name: string;
+    priority: number;
+    isHealthy: boolean;
+    lastHealthCheck: Date | null;
+    consecutiveFailures: number;
+}
+declare class TranslatorLoadBalancer {
+    private config;
+    private circuitBreakers;
+    private healthCheckInterval;
+    private currentEndpointIndex;
+    constructor();
+    /**
+     * Initialize endpoints from environment variable
+     * Format: url1|name1,url2|name2 or just url1,url2
+     */
+    private initializeEndpoints;
+    /**
+     * Get the next healthy endpoint using weighted round-robin
+     */
+    getNextEndpoint(): TranslatorEndpoint | null;
+    /**
+     * Mark an endpoint as failed
+     */
+    markEndpointFailed(endpoint: TranslatorEndpoint): void;
+    /**
+     * Mark an endpoint as successful
+     */
+    markEndpointSuccess(endpoint: TranslatorEndpoint): void;
+    /**
+     * Execute a translation request with automatic failover and exponential backoff
+     */
+    executeWithFailover<T>(requestFn: (url: string) => Promise<T>): Promise<T>;
+    /**
+     * Sleep utility
+     */
+    private sleep;
+    /**
+     * Perform health check on all endpoints
+     */
+    healthCheck(): Promise<Record<string, boolean>>;
+    /**
+     * Start periodic health checks
+     */
+    startHealthChecks(): void;
+    /**
+     * Stop periodic health checks
+     */
+    stopHealthChecks(): void;
+    /**
+     * Get current status of all endpoints
+     */
+    getStatus(): Array<{
+        name: string;
+        url: string;
+        isHealthy: boolean;
+        failures: number;
+    }>;
+}
+export declare function getTranslatorLoadBalancer(): TranslatorLoadBalancer;
+export declare const translatorLoadBalancer: TranslatorLoadBalancer;
+export {};