@babelize/engine 0.0.0-stage → 0.1.0

package/dist/modules/translator/translate.js

@@ -0,0 +1,932 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FALLBACK_QA_WARNING = void 0;
+exports.translateLocales = translateLocales;
+exports.backfillTranslationMemory = backfillTranslationMemory;
+const path = __importStar(require("path"));
+const cache_keys_1 = require("../../utils/cache-keys");
+const logger_1 = require("../../utils/logger");
+const translation_quality_1 = require("../../utils/translation-quality");
+const file_utils_1 = require("../../utils/file-utils");
+const placeholder_protection_1 = require("../../utils/placeholder-protection");
+const icu_message_1 = require("../../utils/icu-message");
+const keygen_1 = require("../keygen");
+const concurrent_map_1 = require("../../utils/concurrent-map");
+const language_codes_1 = require("./language-codes");
+const glossary_1 = require("../glossary");
+const gemini_provider_1 = require("./client/gemini.provider");
+const memory_matcher_1 = require("./memory/memory-matcher");
+const default_translation_provider_1 = require("../../providers/default-translation-provider");
+const translation_sanitizer_1 = require("../../utils/translation-sanitizer");
+const glossary_match_1 = require("../../utils/glossary-match");
+const icu_spans_1 = require("../../utils/icu-spans");
+const BATCH_SIZE = 15;
+/** QA note on keys the backup (NLLB) translator produced. */
+exports.FALLBACK_QA_WARNING = 'translated with the backup translator; will be retried with the AI translator next run';
+/** Keys re-sent after failing validation go in small batches with per-key notes. */
+const REPAIR_BATCH_SIZE = 5;
+const TRANSLATE_CONCURRENCY = 3; // Max concurrent locales/batches
+/** A plain source whose stored translation is ICU only fits a project whose runtime formats ICU. */
+function reusableTranslation(source, translated, targetPlurals) {
+    return targetPlurals || (0, icu_message_1.isIcuMessage)(source) || !(0, icu_message_1.isIcuMessage)(translated);
+}
+/** Per-entry instruction: pluralize a count argument if the target language needs it. */
+function pluralHint(args, targetLocale) {
+    const list = args.map((a) => `{${a}}`).join(', ');
+    return `${list} ${args.length === 1 ? 'is a number' : 'are numbers'}. If ${targetLocale} changes the words around it by number, return an ICU plural over it, e.g. {${args[0]}, plural, one {…} other {…}} with every ${targetLocale} plural category and "#" in each branch; otherwise keep ${list} as plain text. Keep every {argument} name and tag exactly.`;
+}
+function chunk(items, size) {
+    const out = [];
+    for (let i = 0; i < items.length; i += size)
+        out.push(items.slice(i, i + size));
+    return out;
+}
+/**
+ * A full sentence (3+ real words) that came back identical to the source was
+ * not translated. Short strings ("OK", "Email", a product name) legitimately
+ * stay the same in many languages. Both arguments are the tokenized text, so
+ * placeholders and terms neither count as words nor hide a difference.
+ */
+function isUnchangedSentence(sentText, returnedText) {
+    const norm = (t) => t.replace(/\[\s*#\s*VAR_\d+\s*#\s*\]/gi, ' ').replace(/\s+/g, ' ').trim().toLowerCase();
+    const words = norm(sentText).split(' ').filter((w) => /\p{L}{2,}/u.test(w));
+    return words.length >= 3 && norm(sentText) === norm(returnedText);
+}
+function escapeTerm(term) {
+    return term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+function containsTerm(text, term) {
+    return new RegExp(`(?<![\\w])${escapeTerm(term)}(?![\\w])`, 'i').test(text);
+}
+/** Tags, URLs, code and never-translate terms present in the source but absent from the output. */
+function missingSpans(source, translated, neverTranslateTerms) {
+    const have = new Set((0, placeholder_protection_1.protectText)(translated, neverTranslateTerms, { icu: true }).tokens.map((t) => t.toLowerCase()));
+    const want = new Set((0, placeholder_protection_1.protectText)(source, neverTranslateTerms, { icu: true }).tokens);
+    return [...want].filter((t) => !have.has(t.toLowerCase()));
+}
+/** Glossary terms in the source whose required translation is missing from the output. */
+function glossaryWarnings(source, translated, glossary) {
+    return glossary
+        .filter((g) => containsTerm(source, g.term) && !(0, glossary_match_1.containsGlossaryTranslation)(translated, g.translation))
+        .map((g) => `Glossary: "${g.term}" should be translated as "${g.translation}"`);
+}
+const SCRIPT_RANGES = [
+    [0x0000, 0x024F], // Latin (Basic, Latin-1, Extended-A/B) — one script
+    [0x0370, 0x03FF], // Greek
+    [0x0400, 0x04FF], // Cyrillic
+    [0x0590, 0x05FF], // Hebrew
+    [0x0600, 0x06FF], // Arabic
+    [0x0900, 0x097F], // Devanagari
+    [0x0980, 0x09FF], // Bengali
+    [0x0E00, 0x0E7F], // Thai
+    [0x1100, 0x11FF], // Hangul Jamo
+    [0x3040, 0x309F], // Hiragana
+    [0x30A0, 0x30FF], // Katakana
+    [0x4E00, 0x9FFF], // CJK Unified
+    [0xAC00, 0xD7AF], // Hangul Syllables
+];
+function detectScript(ch) {
+    const code = ch.codePointAt(0);
+    for (let i = 0; i < SCRIPT_RANGES.length; i++) {
+        if (code >= SCRIPT_RANGES[i][0] && code <= SCRIPT_RANGES[i][1])
+            return i;
+    }
+    return -1;
+}
+/**
+ * True when the text's letters come from more than one writing system
+ * (e.g. "Hello 世界"). Only letters count: "Café", "© 2024", "10 °C" and
+ * non-breaking spaces are plain Latin text.
+ */
+function hasMixedScripts(text) {
+    let seen = -1;
+    for (const ch of text) {
+        if (!/\p{L}/u.test(ch))
+            continue;
+        const script = detectScript(ch);
+        if (script === -1)
+            continue;
+        if (seen === -1) {
+            seen = script;
+        }
+        else if (script !== seen) {
+            return true;
+        }
+    }
+    return false;
+}
+/**
+ * Load a project's glossary and brand voice. Never throws: a failure means
+ * translating without that rule, as before. The two lookups are independent,
+ * so a glossary failure doesn't also drop the brand voice. `complete` is false
+ * if either failed — callers that copy text into shared TM must then assume
+ * project-specific wording may be baked in.
+ */
+async function loadProjectTranslationRules(projectId, rules) {
+    let neverTranslateTerms = [];
+    let alwaysTranslateAsEntries = [];
+    let brandVoice = null;
+    let complete = true;
+    if (!rules)
+        return { neverTranslateTerms, alwaysTranslateAsEntries, brandVoice, complete };
+    try {
+        const entries = await rules.getGlossary(projectId);
+        if (entries?.length) {
+            neverTranslateTerms = entries
+                .filter((e) => e.type === 'never_translate')
+                .map((e) => e.term);
+            alwaysTranslateAsEntries = entries
+                .filter((e) => e.type === 'always_translate_as')
+                .map((e) => ({
+                term: e.term,
+                type: 'always_translate_as',
+                translations: e.translations || {},
+            }));
+            logger_1.engineLogger.info('Loaded project glossary', {
+                neverTranslate: neverTranslateTerms.length,
+                alwaysTranslateAs: alwaysTranslateAsEntries.length,
+            });
+        }
+    }
+    catch (err) {
+        complete = false;
+        logger_1.engineLogger.warn('Failed to load glossary entries', { error: err });
+    }
+    try {
+        brandVoice = await rules.getBrandVoice(projectId);
+        if (brandVoice) {
+            logger_1.engineLogger.info('Brand voice configured', {
+                length: brandVoice.length,
+                hash: (0, cache_keys_1.hashText)(brandVoice),
+            });
+        }
+    }
+    catch (err) {
+        complete = false;
+        logger_1.engineLogger.warn('Failed to load brand voice', { error: err });
+    }
+    return { neverTranslateTerms, alwaysTranslateAsEntries, brandVoice, complete };
+}
+/**
+ * Whether a source string is subject to one of the project's "always
+ * translate as" rules for this locale, i.e. its stored translation has
+ * project-specific wording baked in.
+ */
+function hasAlwaysTranslateAsTerm(sourceText, entries, targetLocale) {
+    return entries.some((e) => !!e.translations[targetLocale] &&
+        new RegExp(`\\b${e.term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'i').test(sourceText));
+}
+async function translateLocales(workspace, sourceLocale, targetLocales, forceTranslation = false, dryRun = false, projectId, onProgress, keysToTranslateByLocale, cacheProvider, byokConfig, contextMap, extractionContextMap, providers = {}, options = {}) {
+    const locales = [];
+    const errors = [];
+    const failures = [];
+    let tmHitsTotal = 0;
+    const perLocaleMetadata = {};
+    // Ensure locale output directories exist for all target locales
+    if (!dryRun) {
+        for (const locale of targetLocales) {
+            const { dir } = (0, file_utils_1.localeFilePath)(workspace.path, workspace.publicDir, workspace.localeFormat, locale);
+            await (0, file_utils_1.ensureDir)(dir);
+        }
+    }
+    const flatSource = (0, keygen_1.flattenLocaleData)(sourceLocale.data);
+    // Strip _meta.* entries from flatSource so meta fields are never sent to the
+    // translation engine, never stored in the lockfile, and never overwrite the
+    // target locale's own computed _meta values.
+    for (const key of Object.keys(flatSource)) {
+        if (key === '_meta' || key.startsWith('_meta.')) {
+            delete flatSource[key];
+        }
+    }
+    // Fetch glossary entries for this project
+    let neverTranslateTerms = [];
+    let alwaysTranslateAsEntries = [];
+    let brandVoice = null;
+    let rulesComplete = true;
+    if (projectId) {
+        const rules = await loadProjectTranslationRules(projectId, providers.rules);
+        neverTranslateTerms = rules.neverTranslateTerms;
+        alwaysTranslateAsEntries = rules.alwaysTranslateAsEntries;
+        brandVoice = rules.brandVoice;
+        rulesComplete = rules.complete;
+    }
+    // Fall back to workspace config for CLI/MCP runs (no projectId)
+    if (!projectId) {
+        try {
+            const babelizeConfigPath = path.join(workspace.path, '.babelize', 'config.json');
+            const fileUtils = await Promise.resolve().then(() => __importStar(require('../../utils/file-utils')));
+            const { readJsonFile } = fileUtils;
+            if (await fileUtils.pathExists(babelizeConfigPath)) {
+                const config = await readJsonFile(babelizeConfigPath);
+                if (config?.customTranslations) {
+                    for (const [term, entry] of Object.entries(config.customTranslations)) {
+                        if (entry.neverTranslate) {
+                            neverTranslateTerms.push(term);
+                        }
+                        if (entry.translations) {
+                            alwaysTranslateAsEntries.push({
+                                term,
+                                type: 'always_translate_as',
+                                translations: entry.translations,
+                            });
+                        }
+                    }
+                    logger_1.engineLogger.info('Loaded glossary from workspace config', {
+                        neverTranslate: neverTranslateTerms.length,
+                        alwaysTranslateAs: alwaysTranslateAsEntries.length,
+                    });
+                }
+            }
+        }
+        catch (err) {
+            logger_1.engineLogger.warn('Failed to load glossary from workspace config', { error: err });
+        }
+    }
+    // Check if translator is configured before processing locales
+    if (!byokConfig && !providers.translator && !process.env.TRANSLATOR_URL && targetLocales.some(l => l !== sourceLocale.locale)) {
+        logger_1.engineLogger.error('TRANSLATOR_URL environment variable is not set and no BYOK config provided. Translations will fall back to source text.');
+    }
+    const localeTasks = targetLocales
+        .filter((l) => l !== sourceLocale.locale && (0, language_codes_1.isSupported)(l));
+    const { results: localeResults, errors: localeErrors } = await (0, concurrent_map_1.concurrentMap)(localeTasks, async (targetLocale) => {
+        const targetLangCode = (0, language_codes_1.toFlores)(targetLocale);
+        const localePaths = (0, file_utils_1.localeFilePath)(workspace.path, workspace.publicDir, workspace.localeFormat, targetLocale);
+        const targetPath = localePaths.file;
+        const relativePath = localePaths.relativeFile;
+        let entriesToTranslate = flatSource;
+        if (keysToTranslateByLocale && keysToTranslateByLocale[targetLocale]) {
+            entriesToTranslate = {};
+            for (const key of keysToTranslateByLocale[targetLocale]) {
+                if (flatSource[key]) {
+                    entriesToTranslate[key] = flatSource[key];
+                }
+            }
+        }
+        if (Object.keys(entriesToTranslate).length === 0) {
+            return {
+                locale: targetLocale,
+                file: { locale: targetLocale, path: relativePath, data: {}, isNew: true },
+                tmHits: 0,
+                metadata: {},
+                failures: {},
+                error: null,
+            };
+        }
+        const sourceLangCode = (0, language_codes_1.toFlores)(sourceLocale.locale);
+        // LLM translators (BYOK, or an injected one such as Babelize-hosted)
+        // take the brand voice in the prompt; NLLB output is polished after.
+        const hasBrandVoiceInPrompt = !!((byokConfig || providers.translator) && brandVoice);
+        const translationResult = await translateEntries(entriesToTranslate, sourceLangCode, targetLangCode, projectId, onProgress, neverTranslateTerms, alwaysTranslateAsEntries, targetLocale, cacheProvider, byokConfig, contextMap, extractionContextMap, hasBrandVoiceInPrompt ? (brandVoice || undefined) : undefined, dryRun, forceTranslation, providers, rulesComplete, options);
+        const entriesResult = translationResult.result;
+        const batchStats = translationResult.stats;
+        const batchMetadata = translationResult.metadata;
+        const keyFailures = translationResult.failures;
+        if (Object.keys(entriesResult).length === 0 && Object.keys(entriesToTranslate).length > 0) {
+            // Nothing translated: write no file. Every key is in keyFailures with
+            // its reason, so the caller can report exactly what went wrong.
+            logger_1.engineLogger.error(`No strings could be translated for locale "${targetLocale}"`, {
+                locale: targetLocale, sourceKeys: Object.keys(entriesToTranslate).length,
+            });
+            return { locale: targetLocale, file: null, tmHits: batchStats.tmHits, metadata: batchMetadata, failures: keyFailures, error: null };
+        }
+        let translatedEntries = entriesResult;
+        // Post-hoc brand voice polish only for the NLLB service; LLM
+        // translators already had the brand voice in the prompt.
+        if (brandVoice && !hasBrandVoiceInPrompt) {
+            // TM hits are approved/previously-accepted text: polishing them again
+            // would rewrite reviewed translations nondeterministically every run.
+            const toPolish = {};
+            for (const [k, v] of Object.entries(translatedEntries)) {
+                if (!batchMetadata[k]?.fromMemory)
+                    toPolish[k] = v;
+            }
+            if (Object.keys(toPolish).length > 0) {
+                const polished = await (0, gemini_provider_1.polishWithBrandVoice)(toPolish, brandVoice, targetLocale);
+                if (polished !== toPolish) {
+                    // The polish is a free rewrite of already-validated text: keep a
+                    // rewrite only if it still has exactly the same placeholders, tags
+                    // and terms (and, for ICU, is still a valid equivalent message).
+                    const accepted = {};
+                    let rejected = 0;
+                    for (const [k, v] of Object.entries(polished)) {
+                        const before = toPolish[k];
+                        if (before === undefined || typeof v !== 'string' || !v.trim())
+                            continue;
+                        // ICU (in the source, or a plural the translation added) must
+                        // stay an equivalent message of the validated translation.
+                        const icu = (0, icu_message_1.isIcuMessage)(entriesToTranslate[k] ?? '') || (0, icu_message_1.isIcuMessage)(before) || (0, icu_message_1.isIcuMessage)(v);
+                        const safe = (0, placeholder_protection_1.sameProtectedSpans)(before, v, neverTranslateTerms) &&
+                            (!icu || !(0, icu_message_1.compareIcuMessages)(before, v, targetLocale).some((i) => i.severity === 'error'));
+                        if (safe)
+                            accepted[k] = v;
+                        else
+                            rejected++;
+                    }
+                    logger_1.engineLogger.info('Brand voice polish applied', { locale: targetLocale, accepted: Object.keys(accepted).length, rejected });
+                    translatedEntries = { ...translatedEntries, ...accepted };
+                }
+            }
+        }
+        let translatedData = (0, keygen_1.unflattenLocaleData)(translatedEntries);
+        const existingTranslatedMeta = typeof translatedData._meta === 'object' && translatedData._meta !== null
+            ? translatedData._meta
+            : {};
+        translatedData = {
+            ...translatedData,
+            _meta: { ...existingTranslatedMeta, dir: (0, language_codes_1.isRTL)(targetLocale) ? 'rtl' : 'ltr', language: targetLocale },
+        };
+        if (!dryRun)
+            await (0, file_utils_1.writeJsonFile)(targetPath, translatedData, true);
+        return {
+            locale: targetLocale,
+            file: {
+                locale: targetLocale,
+                path: relativePath,
+                data: translatedData,
+                isNew: true,
+                isRTL: (0, language_codes_1.isRTL)(targetLocale),
+            },
+            tmHits: batchStats.tmHits,
+            metadata: batchMetadata,
+            failures: keyFailures,
+            error: null,
+        };
+    }, TRANSLATE_CONCURRENCY);
+    for (const r of localeResults) {
+        if (!r)
+            continue;
+        for (const [key, f] of Object.entries(r.failures ?? {})) {
+            failures.push({ locale: r.locale, key, source: flatSource[key] ?? '', ...f });
+        }
+        if (r.error) {
+            errors.push({ locale: r.locale, error: r.error });
+        }
+        else if (r.file) {
+            locales.push(r.file);
+            tmHitsTotal += r.tmHits;
+            perLocaleMetadata[r.locale] = r.metadata;
+        }
+    }
+    // Collect errors from unsupported locales
+    for (const targetLocale of targetLocales) {
+        if (targetLocale === sourceLocale.locale)
+            continue;
+        if (!(0, language_codes_1.isSupported)(targetLocale)) {
+            errors.push({ locale: targetLocale, error: `Unsupported language code: ${targetLocale}` });
+        }
+    }
+    // Fail immediately when BYOK translation errors occurred
+    if (byokConfig && (errors.length > 0 || localeErrors.length > 0)) {
+        throw localeErrors.length > 0 ? localeErrors[0] : new Error(errors[0].error);
+    }
+    return {
+        translation: { locales, errors, failures },
+        stats: { tmHits: tmHitsTotal },
+        metadata: perLocaleMetadata
+    };
+}
+async function translateEntries(entries, sourceLang, targetLang, projectId, onProgress, neverTranslateTerms = [], alwaysTranslateAsEntries = [], targetLocale = '', cacheProvider, byokConfig, contextMap, extractionContextMap, brandVoice, dryRun = false, forceTranslation = false, providers = {}, rulesComplete = true, options = {}) {
+    const result = {};
+    const metadata = {};
+    const keys = Object.keys(entries);
+    // Cache and TM key: the FLORES code, plus the region for regional variants
+    // (pt-BR vs pt-PT share por_Latn but must not share translations).
+    const storeLang = targetLocale && (0, language_codes_1.isSupported)(targetLocale) ? (0, language_codes_1.storageLangCode)(targetLocale) : targetLang;
+    let uncachedEntries = [];
+    let cacheHits = 0;
+    let tmHits = 0;
+    const cacheBackfill = [];
+    const markInTm = (keys) => {
+        for (const k of keys)
+            metadata[k] = { ...(metadata[k] ?? {}), inTm: true };
+    };
+    for (const key of keys) {
+        const text = entries[key];
+        const entryContext = contextMap?.[key];
+        const textForHash = entryContext ? `${text}:::${entryContext}` : text;
+        const textHash = (0, cache_keys_1.hashText)(textForHash);
+        const cacheKey = (0, cache_keys_1.translationCacheKey)(sourceLang, storeLang, textHash);
+        const cached = cacheProvider && !forceTranslation ? await cacheProvider.get(cacheKey) : { hit: false, data: null };
+        if (cached.hit && cached.data !== null && reusableTranslation(text, cached.data, !!options.targetPlurals)) {
+            result[key] = applyGlossaryPostProcessing(text, cached.data, neverTranslateTerms, alwaysTranslateAsEntries, targetLocale);
+            cacheHits++;
+            // Reused, not translated this run (billing counts only fresh output).
+            metadata[key] = { ...(metadata[key] ?? {}), fromCache: true };
+            // Keep out of shared TM: sentence fragments (context-dependent), and —
+            // since cache entries written before TM stored pre-glossary text may
+            // have them baked in — strings with project glossary rules or an
+            // in-prompt brand voice, which must not leak into other projects.
+            if (rulesComplete && !entryContext && !brandVoice && !hasAlwaysTranslateAsTerm(text, alwaysTranslateAsEntries, targetLocale)) {
+                cacheBackfill.push({ key, sourceText: text, translatedText: restoreNeverTranslate(text, cached.data, neverTranslateTerms) });
+            }
+        }
+        else {
+            uncachedEntries.push({ key, value: text, hash: textHash, context: entryContext });
+        }
+    }
+    const memory = providers.memory;
+    const translator = providers.translator ?? (0, default_translation_provider_1.createDefaultTranslationProvider)(byokConfig);
+    let tmContext = null;
+    if (projectId && memory) {
+        try {
+            tmContext = await memory.getProjectContext(projectId);
+        }
+        catch (err) {
+            logger_1.engineLogger.error('Failed to resolve TM owner; TM disabled for this locale', { projectId, error: err });
+        }
+    }
+    const ownerId = memory ? tmContext?.ownerId ?? null : null;
+    if (cacheBackfill.length > 0 && projectId && tmContext && !dryRun) {
+        // Redis hits never reach the translator, so without this a re-run would
+        // never populate TM for strings translated before TM writes worked.
+        if (await persistToTM(memory, cacheBackfill, projectId, tmContext, sourceLang, storeLang, { overwrite: false })) {
+            markInTm(cacheBackfill.map((i) => i.key));
+        }
+    }
+    // When the TM lookup fails we can't tell which strings already have a
+    // (possibly human-edited) TM entry, so later writes must not overwrite.
+    let tmLookupFailed = false;
+    // Source texts whose TM entry a human approved: a forced re-translate still
+    // uses fresh output for this run but must not overwrite those entries.
+    const approvedSourceTexts = new Set();
+    if (forceTranslation && ownerId && projectId && !dryRun && uncachedEntries.length > 0) {
+        try {
+            const existing = await memory.getAllForOwner(ownerId, sourceLang, storeLang);
+            for (const m of existing)
+                if (m.approvedAt)
+                    approvedSourceTexts.add(m.sourceText);
+        }
+        catch (err) {
+            tmLookupFailed = true;
+            logger_1.engineLogger.error('Failed to read Translation Memory before forced re-translate; TM writes will not overwrite', { error: err });
+        }
+    }
+    if (uncachedEntries.length > 0 && ownerId && !forceTranslation) {
+        try {
+            logger_1.engineLogger.info('Checking Memory for entries', {
+                count: uncachedEntries.length,
+                ownerId,
+                targetLocale,
+            });
+            // 1. Fetch all TM for owner (for fuzzy matching)
+            // TM rows store FLORES codes (eng_Latn → fra_Latn), so look up by the
+            // FLORES codes, not the ISO locale.
+            const allMemory = await memory.getAllForOwner(ownerId, sourceLang, storeLang);
+            const stillUncached = [];
+            for (const entry of uncachedEntries) {
+                // TM is context-free; a sentence fragment's translation depends on
+                // the sentence around it, so never auto-apply TM to one.
+                if (entry.context) {
+                    stillUncached.push(entry);
+                    continue;
+                }
+                const lookup = (0, memory_matcher_1.matchFromMemory)(allMemory, entry.value, storeLang);
+                if (lookup.hit && lookup.translatedText && reusableTranslation(entry.value, lookup.translatedText, !!options.targetPlurals)) {
+                    result[entry.key] = applyGlossaryPostProcessing(entry.value, lookup.translatedText, neverTranslateTerms, alwaysTranslateAsEntries, targetLocale);
+                    tmHits++;
+                    metadata[entry.key] = { fromMemory: true, inTm: true };
+                    // Warm Redis cache (stored pre-glossary, like every other cache write)
+                    const cacheKey = (0, cache_keys_1.translationCacheKey)(sourceLang, storeLang, entry.hash);
+                    if (cacheProvider) {
+                        cacheProvider.set(cacheKey, lookup.translatedText, cache_keys_1.TRANSLATION_CACHE_TTL_SECONDS).catch((err) => {
+                            logger_1.engineLogger.error('Failed to warm Redis cache from TM', { cacheKey, error: err });
+                        });
+                    }
+                }
+                else if (lookup.fuzzyMatches && lookup.fuzzyMatches.length > 0) {
+                    // Store fuzzy matches as metadata so they can be surfaced as suggestions in dashboard
+                    metadata[entry.key] = { fuzzyMatches: lookup.fuzzyMatches };
+                    stillUncached.push(entry);
+                }
+                else {
+                    stillUncached.push(entry);
+                }
+            }
+            uncachedEntries = stillUncached;
+        }
+        catch (err) {
+            tmLookupFailed = true;
+            logger_1.engineLogger.error('Failed to query Translation Memory; new TM writes will not overwrite existing entries', { error: err });
+        }
+    }
+    if (keys.length > 0) {
+        const hitRate = Math.round(((cacheHits + tmHits) / keys.length) * 100);
+        logger_1.engineLogger.info('Translation cache stats', {
+            cacheHits,
+            tmHits,
+            total: keys.length,
+            hitRate,
+            toTranslate: uncachedEntries.length,
+        });
+    }
+    const failures = {};
+    // LLM providers get structure-aware prompts and repair attempts; the NLLB
+    // service can't handle ICU or mixed-script text, so those fail up front.
+    const isLlm = !!byokConfig || !!providers.translator;
+    // Scripts FLORES-200 lacks (sr-Latn, uz-Cyrl) only work with an LLM.
+    const nllbTarget = !targetLocale || !(0, language_codes_1.isSupported)(targetLocale) || (0, language_codes_1.isNllbSupported)(targetLocale);
+    if (uncachedEntries.length > 0) {
+        const glossaryForLocale = alwaysTranslateAsEntries
+            .filter((e) => e.translations[targetLocale])
+            .map((e) => ({ term: e.term, translation: e.translations[targetLocale] }));
+        // 1. Protect placeholders, tags and never-translate terms. Strings with
+        //    nothing translatable left (e.g. "{count}", a brand name) are their own
+        //    translation and never reach the translator.
+        const prepared = [];
+        for (const entry of uncachedEntries) {
+            const icu = (0, icu_message_1.isIcuMessage)(entry.value);
+            if (!isLlm && (icu || hasMixedScripts(entry.value) || !nllbTarget)) {
+                failures[entry.key] = {
+                    reason: 'unsupported_by_provider',
+                    retryable: false,
+                    message: !nllbTarget
+                        ? `${targetLocale} needs an LLM translation provider`
+                        : icu
+                            ? 'ICU plural/select messages need an LLM translation provider'
+                            : 'Text mixing writing systems needs an LLM translation provider',
+                };
+                continue;
+            }
+            // ICU branches legitimately repeat or drop spans (Polish adds branches,
+            // Japanese keeps only "other"), so ICU text is sent untokenized and its
+            // tags/URLs/terms are checked as a set after translation instead. The
+            // same goes for a plain message with a count ("{files} files were
+            // uploaded"): an LLM may return a plural for a language that needs one.
+            const countArgs = isLlm && options.targetPlurals && !icu ? (0, icu_message_1.countArguments)(entry.value) : [];
+            const tokenized = (0, placeholder_protection_1.protectText)(entry.value, neverTranslateTerms);
+            const sendRaw = icu || countArgs.length > 0;
+            const prot = sendRaw ? { text: entry.value, tokens: [] } : tokenized;
+            if ((0, placeholder_protection_1.hasNothingToTranslate)(tokenized.text)) {
+                result[entry.key] = entry.value;
+                // Copied through (placeholders/codes only); the translator never saw it.
+                metadata[entry.key] = { ...(metadata[entry.key] ?? {}), passthrough: true };
+                continue;
+            }
+            prepared.push({
+                ...entry,
+                sendText: prot.text,
+                tokens: prot.tokens,
+                icu: sendRaw,
+                countArgs: countArgs.length ? countArgs : undefined,
+                plain: countArgs.length ? { sendText: tokenized.text, tokens: tokenized.tokens } : undefined,
+            });
+        }
+        // 2. Translate in batches; judge every key independently.
+        const outcomes = new Map();
+        const judge = (entry, keyResult, nllb = !isLlm) => {
+            const fail = (reason, retryable, message) => ({ ok: false, reason, retryable, message });
+            if (!keyResult)
+                return fail('missing_from_response', true, 'The translator returned no translation for this string');
+            if (keyResult.status === 'failed') {
+                return fail(keyResult.reason, keyResult.retryable, keyResult.message ?? `Translator reported ${keyResult.reason}`);
+            }
+            if (!keyResult.value.trim())
+                return fail('empty_value', true, 'The translator returned an empty string');
+            // A plain source must stay plain unless a plural was allowed for it.
+            if (!(0, icu_message_1.isIcuMessage)(entry.value) && !entry.countArgs && (0, icu_message_1.isIcuMessage)(keyResult.value)) {
+                const args = (0, icu_message_1.extractSimpleArguments)(entry.value);
+                return fail('icu_invalid', true, `The source is plain text: return plain text, not ICU plural/select syntax${args.length ? `, and keep ${args.map((a) => `{${a}}`).join(', ')} as is` : ''}`);
+            }
+            const tokenCheck = (0, placeholder_protection_1.checkTokens)(keyResult.value, entry.tokens.length);
+            if (!tokenCheck.ok)
+                return fail('placeholder_lost', true, `Placeholder problem: ${(0, placeholder_protection_1.describeTokenProblem)(tokenCheck, entry.tokens)}`);
+            const value = (0, placeholder_protection_1.restoreText)((0, translation_sanitizer_1.sanitizeTranslation)(keyResult.value, { nllb }), entry.tokens);
+            const warnings = [];
+            // A plural added for a count placeholder is validated as ICU too.
+            if (entry.icu || (0, icu_message_1.isIcuMessage)(value)) {
+                const issues = (0, icu_message_1.compareIcuMessages)(entry.value, value, targetLocale, { pluralizable: entry.countArgs ?? [] });
+                const error = issues.find((i) => i.severity === 'error');
+                if (error)
+                    return fail('icu_invalid', true, error.message);
+                const lost = missingSpans(entry.value, value, neverTranslateTerms);
+                if (lost.length > 0)
+                    return fail('placeholder_lost', true, `Placeholder problem: dropped ${lost.join(', ')}`);
+                // Every branch must keep its own tags/URLs/terms, not just one of them.
+                const branch = (0, icu_spans_1.branchSpanProblems)(entry.value, value, neverTranslateTerms);
+                if (branch.length > 0) {
+                    const reason = branch.some((p) => p.lost.length) ? 'placeholder_lost' : 'markup_mismatch';
+                    return fail(reason, true, `Placeholder problem in ${branch.map(icu_spans_1.describeBranchSpanProblem).join('; ')} (each branch needs the source's tags and links)`);
+                }
+                warnings.push(...issues.map((i) => i.message));
+            }
+            if (isUnchangedSentence(entry.sendText, keyResult.value)) {
+                return fail('untranslated', true, `Returned unchanged instead of translated into ${targetLocale}`);
+            }
+            const quality = (0, translation_quality_1.validateTranslationQuality)(entry.value, value);
+            // Tag counts legitimately change with an ICU message's branch count;
+            // its tags were already checked as a set above.
+            if (!entry.icu && !(0, icu_message_1.isIcuMessage)(value) && quality.issues?.some((i) => i.type === 'html_tag_mismatch')) {
+                return fail('markup_mismatch', true, 'HTML/JSX tags differ from the source');
+            }
+            if (quality.issues?.some((i) => i.type === 'suspicious_length')) {
+                warnings.push('Length differs a lot from the source; check it fits the UI');
+            }
+            return { ok: true, value, warnings };
+        };
+        const translateBatch = async (batch, hints) => {
+            const batchGlossary = [
+                ...glossaryForLocale.filter((g) => batch.some((b) => containsTerm(b.value, g.term))),
+                // Untokenized ICU text: keep never-translate terms as-is via the glossary.
+                ...neverTranslateTerms
+                    .filter((t) => batch.some((b) => b.icu && containsTerm(b.value, t)))
+                    .map((t) => ({ term: t, translation: t })),
+            ];
+            const keyResults = await translator.translateBatch({
+                entries: batch.map((b) => ({
+                    key: b.key,
+                    value: b.sendText,
+                    parentContext: contextMap?.[b.key],
+                    extractionContext: extractionContextMap?.[b.key],
+                })),
+                sourceLang,
+                targetLang,
+                brandVoice,
+                targetLocale,
+                glossary: batchGlossary.length > 0 ? batchGlossary : undefined,
+                hints,
+            });
+            const byKey = new Map(keyResults.map((r) => [r.key, r]));
+            for (const b of batch)
+                outcomes.set(b.key, judge(b, byKey.get(b.key)));
+        };
+        // A failed request fails only its own keys — except BYOK, where it means
+        // the customer's key/quota is the problem and the whole run should stop
+        // with that error rather than produce a partial result.
+        const runBatches = async (batches, hintsFor) => {
+            let byokError = null;
+            await (0, concurrent_map_1.concurrentMap)(batches, async (batch, i) => {
+                onProgress?.(`Translating batch ${i + 1}/${batches.length} for ${targetLang}...`);
+                const hints = hintsFor
+                    ? Object.fromEntries(batch.map((b) => [b.key, hintsFor(b)]).filter(([, h]) => !!h))
+                    : undefined;
+                try {
+                    await translateBatch(batch, hints);
+                }
+                catch (error) {
+                    const message = error instanceof Error ? error.message : String(error);
+                    logger_1.engineLogger.error('Translation batch failed', { error: message, sourceLang, targetLang });
+                    if (byokConfig)
+                        byokError ?? (byokError = error instanceof Error ? error : new Error(message));
+                    for (const b of batch) {
+                        outcomes.set(b.key, { ok: false, reason: 'provider_error', retryable: true, message });
+                    }
+                }
+            }, TRANSLATE_CONCURRENCY);
+            if (byokError)
+                throw byokError;
+        };
+        const firstPassHint = (b) => (b.countArgs ? pluralHint(b.countArgs, targetLocale) : undefined);
+        await runBatches(chunk(prepared, BATCH_SIZE), isLlm ? firstPassHint : undefined);
+        // 3. One targeted repair attempt for keys that failed validation, telling
+        //    the model exactly what was wrong. NLLB is deterministic, so for it
+        //    only keys it skipped are worth asking for again.
+        const toRepair = prepared.filter((b) => {
+            const o = outcomes.get(b.key);
+            return o && !o.ok && o.retryable && o.reason !== 'provider_error' && (isLlm || o.reason === 'missing_from_response');
+        });
+        if (toRepair.length > 0) {
+            logger_1.engineLogger.info('Retrying keys that failed validation', { count: toRepair.length, targetLang });
+            onProgress?.(`Retrying ${toRepair.length} string(s) that failed checks for ${targetLang}...`);
+            const firstAttempt = new Map(toRepair.map((b) => [b.key, outcomes.get(b.key)]));
+            await runBatches(chunk(toRepair, REPAIR_BATCH_SIZE), isLlm
+                ? (b) => {
+                    const o = firstAttempt.get(b.key);
+                    if (!o || o.ok)
+                        return undefined;
+                    const fix = `A previous translation was rejected — ${o.message}. Fix that, and copy every [#VAR_n#] token exactly once.`;
+                    return b.countArgs ? `${fix} ${pluralHint(b.countArgs, targetLocale)}` : fix;
+                }
+                : undefined);
+        }
+        // 3b. Keys the LLM couldn't be reached for (rate limit, outage) get one
+        //     attempt on the fallback translator, if there is one. It's NLLB-class,
+        //     so ICU and mixed-script strings stay failed.
+        const fellBack = new Set();
+        const fallback = providers.fallbackTranslator;
+        if (fallback && isLlm && nllbTarget) {
+            const toFallback = prepared.filter((b) => {
+                const o = outcomes.get(b.key);
+                // A count entry is a plain message: the fallback translates its tokenized form (no plural).
+                return o && !o.ok && o.reason === 'provider_error' && (!b.icu || !!b.plain) && !hasMixedScripts(b.value);
+            }).map((b) => (b.plain ? { ...b, ...b.plain, icu: false, countArgs: undefined, plain: undefined } : b));
+            if (toFallback.length > 0) {
+                logger_1.engineLogger.warn('LLM translator unavailable; trying the fallback translator', { count: toFallback.length, targetLang });
+                for (const batch of chunk(toFallback, BATCH_SIZE)) {
+                    try {
+                        const keyResults = await fallback.translateBatch({
+                            entries: batch.map((b) => ({ key: b.key, value: b.sendText, parentContext: contextMap?.[b.key] })),
+                            sourceLang,
+                            targetLang,
+                            targetLocale,
+                        });
+                        const byKey = new Map(keyResults.map((r) => [r.key, r]));
+                        for (const b of batch) {
+                            const o = judge(b, byKey.get(b.key), true);
+                            if (o.ok) {
+                                outcomes.set(b.key, o);
+                                fellBack.add(b.key);
+                            }
+                        }
+                    }
+                    catch (error) {
+                        logger_1.engineLogger.error('Fallback translation batch failed', { error: error instanceof Error ? error.message : String(error), targetLang });
+                    }
+                }
+            }
+        }
+        // 4. Record results. Failed keys get no value at all: the locale file
+        //    omits them (the app shows the source string, honestly), the lockfile
+        //    doesn't mark them done, and the next run retries them.
+        const tmInserts = [];
+        for (const entry of prepared) {
+            const outcome = outcomes.get(entry.key);
+            if (!outcome || !outcome.ok) {
+                failures[entry.key] = outcome && !outcome.ok
+                    ? { reason: outcome.reason, retryable: outcome.retryable, message: outcome.message }
+                    : { reason: 'missing_from_response', retryable: true };
+                continue;
+            }
+            const finalValue = applyGlossaryPostProcessing(entry.value, outcome.value, [], alwaysTranslateAsEntries, targetLocale);
+            result[entry.key] = finalValue;
+            const qa = [...outcome.warnings, ...glossaryWarnings(entry.value, finalValue, glossaryForLocale)];
+            if (qa.length > 0)
+                metadata[entry.key] = { ...(metadata[entry.key] ?? {}), qa };
+            // Fallback output is a stopgap: keep it out of the shared cache and TM
+            // so they only ever hold the primary translator's work.
+            if (fellBack.has(entry.key)) {
+                const prev = metadata[entry.key] ?? {};
+                metadata[entry.key] = {
+                    ...prev,
+                    fallback: true,
+                    qa: [...(prev.qa ?? []), exports.FALLBACK_QA_WARNING],
+                };
+                continue;
+            }
+            // Cache and TM store the translation *before* project glossary rules
+            // ("always translate as") are applied; those rules are re-applied on
+            // every hit, so glossary edits reach cached strings and one project's
+            // glossary never leaks into the owner's other projects through TM.
+            const cacheKey = (0, cache_keys_1.translationCacheKey)(sourceLang, storeLang, entry.hash);
+            if (cacheProvider) {
+                cacheProvider.set(cacheKey, outcome.value, cache_keys_1.TRANSLATION_CACHE_TTL_SECONDS).catch((err) => {
+                    logger_1.engineLogger.error('Failed to cache translation', { cacheKey, error: err });
+                });
+            }
+            if (projectId && !contextMap?.[entry.key] && !approvedSourceTexts.has(entry.value)) {
+                tmInserts.push({ key: entry.key, sourceText: entry.value, translatedText: outcome.value });
+            }
+        }
+        if (tmInserts.length > 0 && projectId && tmContext && !dryRun) {
+            // Normally the translator only sees strings TM had no entry for, so
+            // overwriting is safe; a forced re-translate overwrites on purpose.
+            // If the lookup failed, existing entries may be human edits — keep them.
+            const saved = await persistToTM(memory, tmInserts, projectId, tmContext, sourceLang, storeLang, { overwrite: !tmLookupFailed });
+            if (saved)
+                markInTm(tmInserts.map((i) => i.key));
+        }
+    }
+    const failureCount = Object.keys(failures).length;
+    if (failureCount > 0) {
+        logger_1.engineLogger.warn('Some strings could not be translated', { targetLang, failed: failureCount, of: keys.length });
+    }
+    // Every key this run produced gets metadata (possibly empty) so the lockfile
+    // replaces the previous run's — cache hits, fallbacks and mixed-script keys
+    // included — instead of keeping a stale fromMemory/fuzzyMatches.
+    for (const key of Object.keys(result)) {
+        if (!metadata[key])
+            metadata[key] = {};
+    }
+    return { result, stats: { tmHits }, metadata, failures };
+}
+/** Put back never-translate terms that were masked before translation. */
+function restoreNeverTranslate(originalText, translatedText, neverTranslateTerms) {
+    if (neverTranslateTerms.length === 0)
+        return translatedText;
+    const { terms: usedTerms } = (0, glossary_1.applyGlossaryMask)(originalText, neverTranslateTerms);
+    return (0, glossary_1.restoreGlossaryMask)(translatedText, usedTerms);
+}
+function applyGlossaryPostProcessing(originalText, translatedText, neverTranslateTerms, alwaysTranslateAsEntries, targetLocale) {
+    let processed = restoreNeverTranslate(originalText, translatedText, neverTranslateTerms);
+    if (alwaysTranslateAsEntries.length > 0 && targetLocale) {
+        processed = (0, glossary_1.applyAlwaysTranslateAs)(processed, alwaysTranslateAsEntries, targetLocale);
+    }
+    return processed;
+}
+/**
+ * Persist translations to the project owner's TM. Never throws: a TM failure
+ * must not fail the run, but it is logged at error level rather than skipped.
+ * `sourceLang`/`targetLang` are FLORES codes, matching what TM lookups use.
+ */
+async function persistToTM(memory, inserts, projectId, tmContext, sourceLang, targetLang, options = {}) {
+    if (!tmContext.ownerId) {
+        logger_1.engineLogger.error('TM persistence skipped: project has no resolvable owner', {
+            projectId,
+            count: inserts.length,
+        });
+        return false;
+    }
+    try {
+        await memory.saveBatch(tmContext.ownerId, projectId, inserts.map((insert) => ({
+            sourceText: insert.sourceText,
+            sourceLocale: sourceLang,
+            targetLocale: targetLang,
+            translatedText: insert.translatedText,
+            isoKey: insert.key,
+            projectName: tmContext.projectName ?? undefined,
+        })), options);
+        logger_1.engineLogger.info('TM persistence complete', { count: inserts.length, projectId, overwrite: options.overwrite ?? true });
+        return true;
+    }
+    catch (err) {
+        logger_1.engineLogger.error('TM persistence failed', {
+            count: inserts.length,
+            projectId,
+            error: err instanceof Error ? err.message : String(err),
+        });
+        return false;
+    }
+}
+/**
+ * Backfill TM from translations that were reused without calling the
+ * translator (lockfile hits). Existing TM rows are left untouched.
+ * Locales are ISO codes here; identity translations (source === target) are
+ * skipped since they mean a previous run had no working translator.
+ */
+async function backfillTranslationMemory(projectId, sourceLocale, flatSource, translationsByLocale, excludeKeys = new Set(), providers = {}) {
+    const memory = providers.memory;
+    if (!memory)
+        return {};
+    const tmContext = await memory.getProjectContext(projectId);
+    const sourceLang = (0, language_codes_1.toFlores)(sourceLocale);
+    // Lockfile values are final output: project glossary substitutions and
+    // brand-voice rewording are baked in and can't be separated out. Shared TM
+    // must not carry them into the owner's other projects, so skip those.
+    const rules = await loadProjectTranslationRules(projectId, providers.rules);
+    if (rules.brandVoice) {
+        logger_1.engineLogger.info('Skipping lockfile TM backfill: project has a brand voice', { projectId });
+        return {};
+    }
+    if (!rules.complete) {
+        // Can't tell whether glossary/brand-voice wording is baked into these
+        // values; retry on a later run rather than risk leaking it into shared TM.
+        logger_1.engineLogger.warn('Skipping lockfile TM backfill: project rules failed to load', { projectId });
+        return {};
+    }
+    // Keys whose TM state is settled for a locale: written, or deliberately
+    // skipped. The caller records these so later runs don't redo them.
+    const settled = {};
+    for (const [locale, translations] of Object.entries(translationsByLocale)) {
+        if (locale === sourceLocale || !(0, language_codes_1.isSupported)(locale))
+            continue;
+        const inserts = [];
+        const skipped = [];
+        for (const [key, translatedText] of Object.entries(translations)) {
+            const sourceText = flatSource[key];
+            if (excludeKeys.has(key) ||
+                hasAlwaysTranslateAsTerm(sourceText ?? '', rules.alwaysTranslateAsEntries, locale) ||
+                !sourceText || !translatedText || sourceText === translatedText) {
+                skipped.push(key);
+                continue;
+            }
+            inserts.push({ key, sourceText, translatedText });
+        }
+        const saved = inserts.length === 0 ||
+            await persistToTM(memory, inserts, projectId, tmContext, sourceLang, (0, language_codes_1.storageLangCode)(locale), { overwrite: false });
+        if (saved)
+            settled[locale] = [...skipped, ...inserts.map((i) => i.key)];
+    }
+    return settled;
+}