@purposeinplay/payload-ai-translate 0.1.0

package/dist/translate.js

@@ -0,0 +1,911 @@
+import { DEFAULT_RETRY, REENTRY_FLAG } from './defaults.js';
+import { checkPerCallLimit } from './lib/cost-guards.js';
+import { emitAlert, emitEvent } from './lib/events.js';
+import { isFieldEmpty } from './lib/field-empty.js';
+import { mergeTranslationsIntoTargetDoc } from './lib/locale-merge.js';
+import { checkLocaleRowExists } from './lib/locale-row-check.js';
+import { createScopedLogger } from './lib/logger.js';
+import { applyHashSkip, hashFieldValue, loadFieldHashes, pickManualEditedFields, recordHashesAfterWrite } from './lib/manual-edit-guard.js';
+import { validateTranslation } from './lib/output-validation.js';
+import { findByIdNoFallback } from './lib/payload-read.js';
+import { truncateSourceValue } from './lib/truncate-source-value.js';
+import { DEFAULT_MANUAL_EDIT_COLLECTION_SLUG } from './manual-edit-collection.js';
+export async function translateForLocale(params) {
+    const { payload, config, sourceDoc, documentId, collection, targetLocale, fields, allLocalizedTopFields, writeMode, targetPolicy, signal, req, force } = params;
+    let { units } = params;
+    const succeeded = [];
+    const preserved = [];
+    const failed = [];
+    const totalUsage = {
+        inputTokens: 0,
+        outputTokens: 0,
+        estimatedCostUsd: 0
+    };
+    let providerLatencyMs = 0;
+    // Same source and target locale → no-op
+    if (targetLocale === config.sourceLocale) {
+        console.warn(`[ai-translate] Source and target locale are the same ("${targetLocale}") — skipping.`);
+        return {
+            succeeded,
+            preserved,
+            failed,
+            usage: totalUsage,
+            providerLatencyMs
+        };
+    }
+    // Preserve policy: skip fields that already have content in the target locale
+    if (targetPolicy === 'preserve') {
+        units = await filterPreservedFields(payload, collection, documentId, targetLocale, units, fields);
+    }
+    if (units.length === 0) {
+        return {
+            succeeded,
+            preserved,
+            failed,
+            usage: totalUsage,
+            providerLatencyMs
+        };
+    }
+    // BUG-21 fix: hash-skip on unchanged-content re-translate.
+    //
+    // If every translatable top-level field in this surface has stored
+    // source+target hashes matching the current source+target, the
+    // editor clicked Translate but nothing changed since the last write.
+    // Sending those units to the LLM would just spend tokens to produce
+    // identical output. Skip the LLM call AND report the units as
+    // 'skipped' with reason 'unchanged since last translate' so the
+    // editor sees clearly that nothing was re-translated.
+    //
+    // Requires `preserveManualEdits` — that's where the hashes are
+    // stored (and where they're recorded post-write). Without it, the
+    // plugin has no per-field hash history to compare against.
+    //
+    // `force` bypass: when the caller signals that the editor explicitly
+    // asked for a re-translate regardless of cached state (bulk-translate
+    // `mode: 'force'`), we skip the hash-skip path entirely so every
+    // translatable field hits the LLM. Hash bookkeeping after the write
+    // still runs so the next non-force call benefits from fresh hashes.
+    if (config.preserveManualEdits && !force) {
+        const metaSlug = config.manualEditCollectionSlug ?? DEFAULT_MANUAL_EDIT_COLLECTION_SLUG;
+        const skip = await applyHashSkip({
+            payload,
+            metaCollection: metaSlug,
+            surfaceSlug: collection,
+            documentId,
+            targetLocale,
+            units,
+            sourceDoc,
+            loadTargetDoc: ()=>findByIdNoFallback(payload, collection, documentId, targetLocale, {
+                    draft: true
+                })
+        });
+        units = skip.remainingUnits;
+        succeeded.push(...skip.hashSkipped);
+        if (units.length === 0) {
+            return {
+                succeeded,
+                preserved,
+                failed,
+                usage: totalUsage,
+                providerLatencyMs
+            };
+        }
+    }
+    // Batch units by perCallCharLimit + perCallItemLimit
+    const batches = batchUnits(units, config.costLimits.perCallCharLimit, config.costLimits.perCallItemLimit);
+    const retryConfig = {
+        ...DEFAULT_RETRY,
+        ...config.retry ?? {}
+    };
+    // Translate each batch
+    const translatedMap = new Map();
+    for (const batch of batches){
+        if (signal?.aborted) {
+            for (const unit of batch){
+                failed.push({
+                    fieldPath: unit.fieldPath,
+                    locale: targetLocale,
+                    status: 'failed',
+                    error: 'Aborted'
+                });
+            }
+            continue;
+        }
+        const batchResult = await translateBatch({
+            batch,
+            config,
+            collection,
+            targetLocale,
+            signal,
+            retryConfig,
+            payload
+        });
+        // Merge usage
+        totalUsage.inputTokens += batchResult.usage.inputTokens;
+        totalUsage.outputTokens += batchResult.usage.outputTokens;
+        totalUsage.estimatedCostUsd = (totalUsage.estimatedCostUsd ?? 0) + (batchResult.usage.estimatedCostUsd ?? 0);
+        totalUsage.model = totalUsage.model ?? batchResult.usage.model;
+        providerLatencyMs += batchResult.latencyMs;
+        // Process results
+        for (const unit of batch){
+            const translated = batchResult.translations.get(unit.id);
+            if (translated !== undefined) {
+                translatedMap.set(unit.id, translated);
+                succeeded.push({
+                    fieldPath: unit.fieldPath,
+                    locale: targetLocale,
+                    status: 'success',
+                    characterCount: translated.length,
+                    durationMs: batchResult.latencyMs,
+                    // Only return raw text for `plain` units — for inline/block the
+                    // structured patch (richText tree, blocks array) can't be safely
+                    // applied via a flat setValue on the client.
+                    ...unit.kind === 'plain' ? {
+                        translatedText: translated
+                    } : {}
+                });
+            } else if (batchResult.skipped.has(unit.id)) {
+                // Verbatim echo or other soft validation. Don't write the value
+                // (we never overwrite target with source) but mark as skipped so
+                // the caller doesn't surface it as a real failure. Capture the
+                // source text (truncated) so the Hub can show editors WHAT was
+                // kept instead of just a field path.
+                failed.push({
+                    fieldPath: unit.fieldPath,
+                    locale: targetLocale,
+                    status: 'skipped',
+                    error: batchResult.skipped.get(unit.id),
+                    errorCode: batchResult.skippedCodes.get(unit.id),
+                    sourceValue: truncateSourceValue(unit.text)
+                });
+            } else {
+                const err = batchResult.errors.get(unit.id);
+                failed.push({
+                    fieldPath: unit.fieldPath,
+                    locale: targetLocale,
+                    status: 'failed',
+                    error: err ?? 'No translation returned',
+                    errorCode: batchResult.errorCodes.get(unit.id) ?? 'bulk.transient.provider'
+                });
+            }
+        }
+    }
+    // Read the existing target-locale doc (without source fallback). We use
+    // it as the merge base so required-localized siblings the user hasn't
+    // asked us to translate stay populated, and Payload-internal columns
+    // never round-trip from a source-locale read into the update body.
+    let targetDoc = null;
+    try {
+        // Read draft state for target locale too — if the user has staged
+        // partial translations as a draft, we want to preserve those.
+        targetDoc = await findByIdNoFallback(payload, collection, documentId, targetLocale, {
+            draft: true
+        });
+    } catch  {
+        targetDoc = null;
+    }
+    // 2026-06-04 fix: Payload's `findByIdNoFallback` returns source-locale
+    // values for SHARED block-parent tables when the target locale has no
+    // row in `<collection>_locales` yet. The merge then treats those
+    // shared values as "real target data" and writes them back with
+    // their source-locale ids — which collides with the source's own
+    // rows in `<collection>_blocks_*` (PK violation). Fix: probe the
+    // per-locale row directly via the pg pool. If the locale row
+    // doesn't exist, force targetDoc to null so the merge falls back to
+    // the source-clone branch, where `stripArrayItemIds` strips the
+    // source ids and lets Payload generate fresh ones.
+    if (targetDoc !== null) {
+        const localeRowExists = await checkLocaleRowExists(payload, collection, documentId, targetLocale);
+        if (!localeRowExists) {
+            payload.logger?.info?.(`[ai-translate] No ${collection}_locales row for ${String(documentId)}/${targetLocale} yet — forcing source-fallback merge so Payload generates fresh per-locale ids on the first write.`);
+            targetDoc = null;
+        }
+    }
+    // BUG-28 fix: detect structural drift between source and target on any
+    // localized array field (layout blocks, nested arrays). If the editor
+    // reordered, inserted, or deleted blocks in source WITHOUT changing
+    // any text, `translatedMap` is empty — the legacy gate skipped the
+    // merge+write entirely and target locales retained the old structure
+    // forever. We force the structural-sync write when drift is detected.
+    const structuralDrift = hasStructuralDrift(sourceDoc, targetDoc, fields);
+    // Patch translated content and write when EITHER we have translations
+    // to apply, OR structural sync is needed to align target shape to source.
+    if (translatedMap.size > 0 || structuralDrift) {
+        const translatedTopFields = new Set();
+        for (const unit of units){
+            if (translatedMap.has(unit.id)) {
+                translatedTopFields.add(unit.fieldPath.split('.')[0]);
+            }
+        }
+        const writeData = mergeTranslationsIntoTargetDoc({
+            sourceDoc,
+            targetDoc,
+            translatedMap,
+            fields,
+            translatedTopFields,
+            allLocalizedTopFields,
+            writeMode
+        });
+        // `preserveManualEdits` opt-in: drop any top-level fields whose
+        // current target value diverges from the hash the plugin recorded
+        // on its last write. Detection is per-field on the top-level path
+        // so a translator can manually fix `de.title` while still letting
+        // automated re-translation refresh `de.content` on the next
+        // publish. Storage lives in the auto-registered
+        // `ai-translate-meta` sidecar collection — see
+        // `lib/manual-edit-guard.ts`.
+        //
+        // `force` bypass: when the caller signals an intentional override
+        // (bulk-translate `mode: 'force'`), we skip this guard and let the
+        // new translation overwrite whatever's in the target — that's the
+        // contract the editor agreed to by checking "Force re-translate
+        // (includes docs already up to date)" in the UI. Without this
+        // bypass, force-mode bulk runs silently preserve every prior
+        // translation and pay for tokens without writing anything.
+        let manualEditSkipReasons = null;
+        if (config.preserveManualEdits && targetDoc && !force) {
+            const metaSlug = config.manualEditCollectionSlug ?? DEFAULT_MANUAL_EDIT_COLLECTION_SLUG;
+            const storedHashes = await loadFieldHashes(payload, metaSlug, collection, documentId, targetLocale);
+            const { fieldsToDrop, skipReasons } = pickManualEditedFields(writeData, targetDoc, storedHashes);
+            if (fieldsToDrop.length > 0) {
+                manualEditSkipReasons = skipReasons;
+                for (const field of fieldsToDrop){
+                    delete writeData[field];
+                    payload.logger?.info?.(`[ai-translate] Skipping ${collection}/${String(documentId)} locale=${targetLocale} field=${field}: ${skipReasons.get(field)}`);
+                    // BUG-22 fix: move the corresponding `succeeded` entry to
+                    // `preserved` so callers can tell "wrote" from "kept editor's
+                    // hand-edit." The translation itself succeeded; the WRITE
+                    // didn't happen. Pre-1.2.0 these stayed in `succeeded` and
+                    // misled downstream automation (audit logs, change feeds,
+                    // editor notifications).
+                    for(let i = succeeded.length - 1; i >= 0; i--){
+                        const entry = succeeded[i];
+                        if (!entry) continue;
+                        // Match by top-level field path. Inline / block units use
+                        // dotted paths so we compare on the top segment.
+                        const entryTop = entry.fieldPath.split('.')[0];
+                        if (entryTop === field) {
+                            preserved.push({
+                                ...entry,
+                                status: 'skipped',
+                                error: skipReasons.get(field)
+                            });
+                            succeeded.splice(i, 1);
+                        }
+                    }
+                }
+            }
+        }
+        // BUG-27 fix: concurrent-edit stale-source check.
+        // Compare a HASH of the translated source-locale fields between
+        // dispatch and just-before-write. If those values changed mid-flight
+        // (editor's autosave landed during our LLM call), abort and let the
+        // next save fire a fresh pass — otherwise the target gets a stale
+        // translation the editor has already moved past.
+        //
+        // Previous implementation compared the parent doc's `updatedAt`, but
+        // that field is bumped by EVERY locale write — including SIBLING
+        // locales translated concurrently in the same job. When two locales
+        // ran in parallel, whichever wrote first bumped `updatedAt`, and the
+        // second one then falsely detected "source advanced" and aborted.
+        // Hashing the source-locale field values sidesteps that: a sibling
+        // locale's write doesn't change the source-locale row's content, only
+        // the parent doc's timestamp.
+        const stalenessCheckFields = new Set();
+        for (const u of units){
+            const top = u.fieldPath.split('.')[0];
+            if (top) stalenessCheckFields.add(top);
+        }
+        const computeSourceHash = (doc)=>{
+            const sub = {};
+            for (const k of stalenessCheckFields)sub[k] = doc[k];
+            return hashFieldValue(sub);
+        };
+        const sourceHashAtDispatch = stalenessCheckFields.size > 0 ? computeSourceHash(sourceDoc) : null;
+        if (sourceHashAtDispatch !== null) {
+            try {
+                const freshSource = await findByIdNoFallback(payload, collection, documentId, config.sourceLocale, {
+                    draft: true
+                });
+                const freshHash = freshSource ? computeSourceHash(freshSource) : null;
+                if (freshHash !== null && freshHash !== sourceHashAtDispatch) {
+                    payload.logger?.warn?.(`[ai-translate] Source content advanced during translation of ${collection}/${String(documentId)} locale=${targetLocale}. Aborting this write — the next save will fire a fresh pass.`);
+                    for (const result of succeeded){
+                        failed.push({
+                            ...result,
+                            status: 'skipped',
+                            error: 'Source advanced during translation; write aborted to avoid stale-source target content.'
+                        });
+                    }
+                    succeeded.length = 0;
+                    return {
+                        succeeded,
+                        preserved,
+                        failed,
+                        usage: totalUsage,
+                        providerLatencyMs
+                    };
+                }
+            } catch  {
+            // If we can't read the fresh source, fall through to write —
+            // staleness check is best-effort; transient DB errors shouldn't
+            // block a translation that otherwise succeeded.
+            }
+        }
+        // BUG-25 fix: skip the DB write entirely when manual-edit-guard
+        // filtering left writeData empty. Pre-fix, calling `payload.update`
+        // with `data: {}` (after every field was preserved as manual-edit)
+        // tripped required-field validation on collections that have a
+        // required top-level field, producing noisy 'failed' Jobs entries
+        // even though every field was correctly preserved. An empty write
+        // also touches `updated_at` and causes a downstream cache flush for
+        // no semantic change.
+        const writeKeys = Object.keys(writeData);
+        if (writeKeys.length === 0) {
+            payload.logger?.info?.(`[ai-translate] No fields to write for ${collection}/${String(documentId)} locale=${targetLocale} — all fields were preserved as manual edits. Skipping payload.update.`);
+        // Skip the write but keep the manualEditSkipReasons / succeeded
+        // tally so the per-locale response shape stays consistent.
+        } else {
+            try {
+                // Shallow-copy req with `transactionID: undefined` so each locale
+                // write gets its own postgres transaction. Sharing the parent
+                // request's transactionID across N concurrent locale writes
+                // serialises Payload's update flow on a single txn — N parallel
+                // delete-then-insert plans on the same `<col>_blocks_*` junction
+                // tables collide and Payload bails with errors like
+                // `delete from "<col>_blocks_p_ban" where "_parent_id" = $1` or
+                // `field is invalid: _locale, _parent_id`. Issuing a fresh txn per
+                // write isolates them. We keep `req.user` (and the rest) so the
+                // audit-log afterChange hook can still attribute the write to the
+                // human editor who triggered the publish.
+                const writeReq = req ? {
+                    ...req,
+                    transactionID: undefined
+                } : undefined;
+                // Allowlist context forward: a naive `...writeReq.context` would
+                // propagate audit-log's AUDIT_CONTEXT_KEY (and any future
+                // request-scoped flags) into the locale write, causing
+                // cascade-guard to mis-attribute or suppress audit entries for
+                // the translated row. Only known-safe keys flow through.
+                const forwardedContext = {};
+                if (writeReq?.context && 'disableRevalidate' in writeReq.context) {
+                    forwardedContext.disableRevalidate = writeReq.context.disableRevalidate;
+                }
+                await payload.update({
+                    collection: collection,
+                    id: documentId,
+                    locale: targetLocale,
+                    data: writeData,
+                    context: {
+                        ...forwardedContext,
+                        [REENTRY_FLAG]: true
+                    },
+                    overrideAccess: true,
+                    draft: false,
+                    ...writeReq ? {
+                        req: writeReq
+                    } : {}
+                });
+                // Persist per-field source+target hashes for next-translation
+                // hash-skip. Best-effort — failure here doesn't roll back the
+                // successful translation write above (helper logs internally).
+                if (config.preserveManualEdits) {
+                    const metaSlug = config.manualEditCollectionSlug ?? DEFAULT_MANUAL_EDIT_COLLECTION_SLUG;
+                    const { written, failed: failedHashes } = await recordHashesAfterWrite({
+                        payload,
+                        metaCollection: metaSlug,
+                        surfaceSlug: collection,
+                        documentId,
+                        targetLocale,
+                        writeData,
+                        sourceDoc
+                    });
+                    if (failedHashes > 0) {
+                        const localeLog = createScopedLogger(payload, {
+                            component: 'translation',
+                            collection,
+                            documentId: String(documentId),
+                            locale: targetLocale
+                        });
+                        localeLog.event('warn', 'translation.hash-record.partial', {
+                            written,
+                            failed: failedHashes
+                        });
+                    }
+                }
+            } catch (error) {
+                const localeLog = createScopedLogger(payload, {
+                    component: 'translation',
+                    collection,
+                    documentId: String(documentId),
+                    locale: targetLocale
+                });
+                localeLog.event('error', 'translation.locale-write.failed', {
+                    err: error
+                });
+                const errMsg = error instanceof Error ? error.message : 'Unknown write error';
+                // Move all succeeded to failed
+                for (const result of succeeded){
+                    failed.push({
+                        ...result,
+                        status: 'failed',
+                        error: `Write failed: ${errMsg}`
+                    });
+                }
+                succeeded.length = 0;
+            }
+        } // end else (write was non-empty)
+    }
+    const isCanary = targetLocale === config.quality?.canaryLocale;
+    // Emit per-field events for debugging
+    for (const result of [
+        ...succeeded,
+        ...failed
+    ]){
+        emitEvent(config, {
+            type: 'translation.field',
+            documentId,
+            collection,
+            sourceLocale: config.sourceLocale,
+            targetLocales: [
+                targetLocale
+            ],
+            timestamp: new Date(),
+            fields: [
+                result
+            ],
+            canary: isCanary || undefined
+        });
+    }
+    // Random sampling: log source + translated pairs for quality spot-checks
+    if (config.quality?.sampling && translatedMap.size > 0) {
+        const { rate, onSample } = config.quality.sampling;
+        if (onSample && Math.random() < rate) {
+            for (const unit of units){
+                const translated = translatedMap.get(unit.id);
+                if (translated) {
+                    try {
+                        const result = onSample({
+                            documentId,
+                            collection,
+                            fieldPath: unit.fieldPath,
+                            sourceLocale: config.sourceLocale,
+                            targetLocale,
+                            sourceText: unit.text,
+                            translatedText: translated,
+                            model: 'unknown',
+                            timestamp: new Date()
+                        });
+                        if (result && typeof result.catch === 'function') {
+                            result.catch(()=>{});
+                        }
+                    } catch  {
+                    // Sampling errors never break the main flow
+                    }
+                }
+            }
+        }
+    }
+    return {
+        succeeded,
+        preserved,
+        failed,
+        usage: totalUsage,
+        providerLatencyMs
+    };
+}
+export async function translateBatch(params) {
+    const { batch, config, collection, targetLocale, signal, retryConfig, payload } = params;
+    const translations = new Map();
+    const errors = new Map();
+    const errorCodes = new Map();
+    const skipped = new Map();
+    const skippedCodes = new Map();
+    const usage = {
+        inputTokens: 0,
+        outputTokens: 0,
+        estimatedCostUsd: 0
+    };
+    let latencyMs = 0;
+    // Enforce per-call character limit
+    const batchChars = batch.reduce((sum, u)=>sum + u.text.length, 0);
+    try {
+        checkPerCallLimit(batchChars, config.costLimits.perCallCharLimit);
+    } catch  {
+        // Oversized batch — mark all items as failed
+        for (const unit of batch){
+            errors.set(unit.id, `Batch exceeds per-call character limit (${batchChars} > ${config.costLimits.perCallCharLimit})`);
+            errorCodes.set(unit.id, 'cost-guard.per-call');
+        }
+        return {
+            translations,
+            errors,
+            errorCodes,
+            skipped,
+            skippedCodes,
+            usage,
+            latencyMs
+        };
+    }
+    // BUG-30 fix: dedup identical (text, kind) units before sending to the
+    // provider. Real-world content (CTAs, taglines, section headers, repeated
+    // disclaimers) often repeats the same source string across multiple
+    // fields. Pre-fix, every duplicate was a separate provider call → wasted
+    // tokens. Now we send each unique (text, kind) once, then fan the response
+    // back to every original unit id.
+    //
+    // Why key on (text, kind): the kind ('plain' vs 'inline' vs 'block') changes
+    // how the provider treats the source (markup-aware vs not). Don't dedup
+    // across kinds — a 'plain' "Hello" and an 'inline' "<b>Hello</b>" are
+    // different units even if the raw text matches after tag-stripping.
+    const dedupKey = (u)=>`${u.kind}\u0000${u.text}`;
+    const representativeByKey = new Map(); // dedupKey -> representative unit.id
+    const aliasMap = new Map(); // alias unit.id -> representative unit.id
+    const items = [];
+    for (const unit of batch){
+        const key = dedupKey(unit);
+        const existing = representativeByKey.get(key);
+        if (existing) {
+            aliasMap.set(unit.id, existing);
+            continue;
+        }
+        representativeByKey.set(key, unit.id);
+        items.push({
+            id: unit.id,
+            text: unit.text,
+            kind: unit.kind
+        });
+    }
+    const request = {
+        items,
+        sourceLocale: config.sourceLocale,
+        targetLocale,
+        context: {
+            collectionSlug: collection,
+            fieldPath: batch[0]?.fieldPath ?? ''
+        },
+        signal
+    };
+    let attempts = 0;
+    let lastError;
+    while(attempts <= retryConfig.attempts){
+        try {
+            // Rate limit before calling provider
+            if (config._rateLimiter) {
+                await config._rateLimiter.acquire();
+            }
+            const response = await config.provider.translate(request);
+            usage.inputTokens += response.usage.inputTokens;
+            usage.outputTokens += response.usage.outputTokens;
+            usage.estimatedCostUsd = (usage.estimatedCostUsd ?? 0) + (response.usage.estimatedCostUsd ?? 0);
+            // Capture the model on first response. Provider can switch on retry
+            // (rare), but the analytics value is "what billed the work" — first
+            // value is canonical for this batch.
+            usage.model = usage.model ?? response.model;
+            latencyMs = response.latencyMs;
+            // Handle item count mismatches
+            const responseItems = response.items.length > items.length ? response.items.slice(0, items.length) // trim extras
+             : response.items;
+            if (responseItems.length < items.length && attempts < retryConfig.attempts) {
+                // Retry — provider returned fewer items
+                attempts++;
+                lastError = `Provider returned ${responseItems.length} items, expected ${items.length}`;
+                await sleep(retryConfig.backoffMs * attempts);
+                continue;
+            }
+            // Validate each translated item
+            const retryUnits = [];
+            for (const item of responseItems){
+                const sourceUnit = batch.find((u)=>u.id === item.id);
+                if (!sourceUnit) {
+                    continue;
+                }
+                const validation = validateTranslation(sourceUnit.text, item.text, config.quality?.validation, {
+                    sourceLocale: config.sourceLocale,
+                    targetLocale
+                });
+                if (validation.valid) {
+                    translations.set(item.id, item.text);
+                } else if (validation.soft) {
+                    // Soft fail (verbatim echo) — don't retry, don't error. Surface
+                    // as `skipped` to the caller so the UI shows it as info-level.
+                    skipped.set(item.id, validation.reason ?? 'Skipped');
+                    if (validation.code) skippedCodes.set(item.id, validation.code);
+                } else if (attempts < retryConfig.attempts) {
+                    retryUnits.push(sourceUnit);
+                } else {
+                    errors.set(item.id, validation.reason ?? 'Validation failed');
+                    // Hard validation failure after retries — fold into the soft-skip
+                    // code family. UI treats both as "AI output couldn't be saved",
+                    // varying only in whether retries were attempted.
+                    if (validation.code) errorCodes.set(item.id, validation.code);
+                }
+            }
+            // Mark items not in response as errors.
+            //
+            // The `!skipped.has(item.id)` guard is load-bearing (1.2.8): when a
+            // batch contains a mix of soft-skip (echo / validator soft fail)
+            // and hard-fail items, a retry sends only the hard-fails — the
+            // soft-skipped ids are absent from the next response by design.
+            // Without the guard, this loop saw the missing soft-skipped id
+            // and either re-pushed it to retryUnits (wasting a retry on a
+            // unit we'd already decided to skip) or, on retry-exhaustion,
+            // promoted it to `errors` while ALSO leaving it in `skipped`.
+            // The downstream effects were both visible in the Hub:
+            //   - The `persistent-failure` alert (line 959 — fires when
+            //     `errors.size > 0`) counted those soft-skipped ids as
+            //     hard failures, producing red "Translation failed
+            //     repeatedly · N fields couldn't be translated to <locale>"
+            //     banners.
+            //   - api.ts:910-940 (translateGlobalForLocale) routes via
+            //     `skipped.has(unit.id)` BEFORE `errors.has(unit.id)`, so the
+            //     same units landed in `failed[]` with status:'skipped' →
+            //     `realFailed.length === 0` → `failedCount === 0` on the
+            //     usage row → `deriveStatus` returned `'needs-review'` (yellow)
+            //     per `UsageTable.helpers.ts:111-113`.
+            // Net: a red "failed repeatedly" alert above the same row that
+            // showed yellow "needs review", for the same fields, from the
+            // same run. Excluding already-skipped ids from this loop is the
+            // minimal correct fix: skip status is final, don't churn it.
+            for (const item of items){
+                if (!translations.has(item.id) && !skipped.has(item.id) && !errors.has(item.id) && !retryUnits.find((u)=>u.id === item.id)) {
+                    if (attempts < retryConfig.attempts) {
+                        const unit = batch.find((u)=>u.id === item.id);
+                        if (unit) retryUnits.push(unit);
+                    } else {
+                        errors.set(item.id, lastError ?? 'No translation returned');
+                        // Missing-response items fall through to provider error
+                        // bucket — UI shows generic "Translation service" copy.
+                        errorCodes.set(item.id, 'bulk.transient.provider');
+                    }
+                }
+            }
+            if (retryUnits.length === 0) {
+                break; // All items handled
+            }
+            // Retry with the failed units
+            attempts++;
+            request.items = retryUnits.map((u)=>({
+                    id: u.id,
+                    text: u.text,
+                    kind: u.kind
+                }));
+            await sleep(retryConfig.backoffMs * attempts);
+        } catch (error) {
+            lastError = error instanceof Error ? error.message : 'Provider error';
+            const batchLog = payload ? createScopedLogger(payload, {
+                component: 'translation.batch',
+                collection,
+                locale: targetLocale
+            }) : undefined;
+            if (attempts < retryConfig.attempts) {
+                batchLog?.event('warn', 'translation.batch.retry', {
+                    err: error,
+                    attempt: attempts + 1,
+                    maxAttempts: retryConfig.attempts,
+                    batchSize: items.length,
+                    locale: targetLocale
+                });
+                attempts++;
+                await sleep(retryConfig.backoffMs * attempts);
+            } else {
+                batchLog?.event('error', 'translation.batch.exhausted', {
+                    err: error,
+                    attempts,
+                    batchSize: items.length,
+                    locale: targetLocale
+                });
+                // All retries exhausted — classify the provider error so the UI
+                // shows the right friendly message (rate-limited vs config vs
+                // schema vs generic transient).
+                const exhaustedCode = classifyProviderErrorToEditorCode(error);
+                for (const item of items){
+                    if (!translations.has(item.id)) {
+                        errors.set(item.id, lastError);
+                        errorCodes.set(item.id, exhaustedCode);
+                    }
+                }
+                break;
+            }
+        }
+    }
+    // BUG-30 fan-out: replicate each representative's result to every alias
+    // id so downstream patching sees a complete translation map.
+    for (const [aliasId, representativeId] of aliasMap.entries()){
+        if (translations.has(representativeId)) {
+            translations.set(aliasId, translations.get(representativeId));
+        } else if (skipped.has(representativeId)) {
+            skipped.set(aliasId, skipped.get(representativeId));
+            const code = skippedCodes.get(representativeId);
+            if (code) skippedCodes.set(aliasId, code);
+        } else if (errors.has(representativeId)) {
+            errors.set(aliasId, errors.get(representativeId));
+            const code = errorCodes.get(representativeId);
+            if (code) errorCodes.set(aliasId, code);
+        }
+    }
+    // Fire alert if too many persistent failures
+    if (errors.size > 0) {
+        emitAlert(config, {
+            type: 'translation.persistent-failure',
+            code: 'alert.persistent-failure',
+            context: {
+                count: errors.size,
+                locale: targetLocale,
+                maxAttempts: retryConfig.attempts
+            },
+            message: `${errors.size} translation(s) failed after ${retryConfig.attempts} attempt${retryConfig.attempts === 1 ? '' : 's'} for locale "${targetLocale}"`,
+            collection,
+            timestamp: new Date(),
+            metadata: {
+                errorCount: errors.size,
+                locale: targetLocale
+            }
+        });
+    }
+    return {
+        translations,
+        errors,
+        errorCodes,
+        skipped,
+        skippedCodes,
+        usage,
+        latencyMs
+    };
+}
+/**
+ * Map a provider exception (network error, AI SDK throw, etc.) to an
+ * editor-facing code. Stays in sync with bulk-translate-doc-task's
+ * `classifyError` heuristics so the same provider failure surfaces
+ * the same friendly copy whether it's hit in a single-doc translate
+ * or a bulk worker.
+ */ function classifyProviderErrorToEditorCode(err) {
+    if (err == null) return 'bulk.unknown';
+    const message = err instanceof Error ? err.message : String(err);
+    const name = err instanceof Error ? err.name : '';
+    const codeProp = err?.code;
+    const codeStr = typeof codeProp === 'string' ? codeProp : '';
+    if (codeStr === 'PER_DOC_CEILING' || codeStr === 'PER_CALL_LIMIT') {
+        return 'bulk.permanent.too_large';
+    }
+    if (name === 'NoObjectGeneratedError' || /noobjectgeneratederror/i.test(message)) {
+        return 'bulk.permanent.schema';
+    }
+    if (/\b401\b|\b403\b|unauthorized|forbidden|api[_-]?key/i.test(message)) {
+        return 'bulk.permanent.config';
+    }
+    if (/\b429\b|rate[_-]?limit/i.test(message)) {
+        return 'bulk.transient.rate_limited';
+    }
+    return 'bulk.transient.provider';
+}
+// ---------------------------------------------------------------------------
+// Preserve policy: filter out non-empty target fields
+// ---------------------------------------------------------------------------
+async function filterPreservedFields(payload, collection, documentId, targetLocale, units, fields) {
+    try {
+        const targetDoc = await findByIdNoFallback(payload, collection, documentId, targetLocale);
+        return units.filter((unit)=>{
+            // Find the matching field for this unit
+            const field = fields.find((f)=>unit.fieldPath === f.path || unit.fieldPath.startsWith(`${f.path}.`));
+            if (!field) return true; // Can't determine — keep it
+            const targetValue = resolvePathValue(targetDoc, field.path);
+            return isFieldEmpty(targetValue, field.type);
+        });
+    } catch  {
+        // If we can't read the target doc, proceed with all units
+        return units;
+    }
+}
+function resolvePathValue(obj, path) {
+    const segments = path.split('.');
+    let current = obj;
+    for (const segment of segments){
+        if (current == null || typeof current !== 'object') return undefined;
+        current = current[segment];
+    }
+    return current;
+}
+/**
+ * Returns `true` when any localized top-level array field shape in
+ * `sourceDoc` diverges from `targetDoc`. Used by the BUG-28 fix to
+ * decide whether to force a structural-sync write even when no text
+ * units changed (block reorder / insert / delete with unchanged text).
+ *
+ * Detection is shallow at the top level — `alignBaseShapeToSource` in
+ * `locale-merge.ts` handles nested-array drift once we decide to write.
+ * The check covers:
+ *   - length mismatch (insert / delete)
+ *   - per-index `blockType` mismatch (reorder of different block types)
+ *   - per-index nested-array length mismatch (e.g. block's `columns` array changed)
+ */ function hasStructuralDrift(sourceDoc, targetDoc, fields) {
+    if (!targetDoc) return false; // Fresh target → first-write path covers it
+    const topArrayFields = new Set();
+    for (const f of fields){
+        if (!f.localized) continue;
+        const top = f.path.split('.')[0];
+        if (!top) continue;
+        if (Array.isArray(sourceDoc[top])) {
+            topArrayFields.add(top);
+        }
+    }
+    for (const top of topArrayFields){
+        const src = sourceDoc[top];
+        const tgt = targetDoc[top];
+        if (!Array.isArray(src) || !Array.isArray(tgt)) continue;
+        if (src.length !== tgt.length) return true;
+        for(let i = 0; i < src.length; i++){
+            const s = src[i];
+            const t = tgt[i];
+            if (typeof s === 'object' && s !== null && typeof t === 'object' && t !== null) {
+                const sb = s.blockType;
+                const tb = t.blockType;
+                if (sb !== undefined && tb !== undefined && sb !== tb) return true;
+            } else if (typeof s !== typeof t || Array.isArray(s) !== Array.isArray(t)) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+// ---------------------------------------------------------------------------
+// Batching
+// ---------------------------------------------------------------------------
+/**
+ * Default items-per-batch cap. Independent of `perCallCharLimit`. Exists
+ * because some structured-output upstreams (Gemini 2.5 Flash via OpenRouter
+ * is the canonical case, surfaced 2026-06-04 on wild-payload-cms) silently
+ * stall on response generation when the response items array grows past
+ * a few hundred entries — HTTP 200 + headers arrive in <2s, then the body
+ * never finishes streaming and the per-call timeout aborts the whole
+ * batch. The char cap doesn't catch this because short strings (nav
+ * labels, JSON keys) pack many items into modest byte counts.
+ *
+ * 250 is chosen empirically: a `loyalty` JSON sub-object on
+ * wild-payload-cms (286 short keys, ~25k chars) completed in ~30s and
+ * 100% success; a 500-item batch on the same provider timed out roughly
+ * 3 of every 4 attempts. 250 puts us under the observed stall threshold
+ * with margin. Consumers running larger-context models (Claude Sonnet,
+ * GPT-4o) can override with a higher value via `costLimits.perCallItemLimit`.
+ */ export const DEFAULT_PER_CALL_ITEM_LIMIT = 250;
+export function batchUnits(units, perCallCharLimit, perCallItemLimit = DEFAULT_PER_CALL_ITEM_LIMIT) {
+    const batches = [];
+    let currentBatch = [];
+    let currentChars = 0;
+    for (const unit of units){
+        const unitChars = unit.text.length;
+        // Single unit exceeds char limit — send it solo
+        if (unitChars > perCallCharLimit) {
+            if (currentBatch.length > 0) {
+                batches.push(currentBatch);
+                currentBatch = [];
+                currentChars = 0;
+            }
+            batches.push([
+                unit
+            ]);
+            console.warn(`[ai-translate] Single translation unit exceeds perCallCharLimit (${unitChars} > ${perCallCharLimit}). Sending solo.`);
+            continue;
+        }
+        // Would exceed either cap — close the current batch and start a new one
+        const wouldExceedChars = currentChars + unitChars > perCallCharLimit;
+        const wouldExceedItems = currentBatch.length >= perCallItemLimit;
+        if (wouldExceedChars || wouldExceedItems) {
+            batches.push(currentBatch);
+            currentBatch = [];
+            currentChars = 0;
+        }
+        currentBatch.push(unit);
+        currentChars += unitChars;
+    }
+    if (currentBatch.length > 0) {
+        batches.push(currentBatch);
+    }
+    return batches;
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function sleep(ms) {
+    return new Promise((resolve)=>setTimeout(resolve, ms));
+}