@purposeinplay/payload-ai-translate 0.1.0

package/dist/tasks/bulk-translate-doc-task.js

@@ -0,0 +1,1000 @@
+import { DEFAULT_BULK_TRANSLATE_BATCHES_COLLECTION_SLUG } from '../bulk-translate-batches-collection.js';
+import { DEFAULT_BULK_TRANSLATE_UNITS_COLLECTION_SLUG } from '../bulk-translate-units-collection.js';
+import { resolveTranslatableFields } from '../lib/field-resolver.js';
+import { createScopedLogger } from '../lib/logger.js';
+import { findByIdNoFallback, findGlobalNoFallback } from '../lib/payload-read.js';
+import { tryClaimAsLock } from '../lib/per-doc-claim.js';
+import { captureLocalizedSnapshot } from '../lib/snapshot-select.js';
+export const BULK_TRANSLATE_DOC_TASK_SLUG = 'bulk-translate-doc';
+// ---------------------------------------------------------------------------
+// Task config
+// ---------------------------------------------------------------------------
+const taskInputSchema = [
+    {
+        name: 'unitId',
+        type: 'text',
+        required: true
+    }
+];
+export function buildBulkTranslateDocTask(options = {}) {
+    const unitsSlug = options.unitsCollectionSlug ?? DEFAULT_BULK_TRANSLATE_UNITS_COLLECTION_SLUG;
+    const batchesSlug = options.batchesCollectionSlug ?? DEFAULT_BULK_TRANSLATE_BATCHES_COLLECTION_SLUG;
+    const maxAttempts = options.maxAttempts ?? 3;
+    const callbacks = options.callbacks;
+    const allowedCollections = options.allowedCollections ?? [];
+    const allowedGlobals = options.allowedGlobals ?? [];
+    return {
+        slug: BULK_TRANSLATE_DOC_TASK_SLUG,
+        label: 'Bulk translate — per-doc worker',
+        inputSchema: taskInputSchema,
+        // We manage our own retry semantics on the unit row. Payload's
+        // built-in retry would double-count attempts.
+        retries: 0,
+        handler: async ({ input, req })=>{
+            const typed = input;
+            const translateApi = options.translateApi ?? await loadDefaultTranslateApi();
+            const result = await runWorkerForUnit({
+                payload: req.payload,
+                unitId: typed.unitId,
+                unitsSlug,
+                batchesSlug,
+                maxAttempts,
+                translateApi,
+                callbacks,
+                allowedCollections,
+                allowedGlobals,
+                // Forward the originating request so the locale write carries
+                // `req.context = { disableRevalidate: true }` — same shape as
+                // the per-doc retry endpoint sets.
+                req: {
+                    ...req,
+                    context: {
+                        ...req.context ?? {},
+                        disableRevalidate: true
+                    }
+                }
+            });
+            return {
+                output: {
+                    ok: true,
+                    status: result.status
+                }
+            };
+        }
+    };
+}
+export async function runWorkerForUnit(params) {
+    const { payload, unitId, unitsSlug, batchesSlug, maxAttempts, translateApi, callbacks } = params;
+    const allowedCollections = new Set(params.allowedCollections ?? []);
+    const allowedGlobals = new Set(params.allowedGlobals ?? []);
+    const now = params.now ?? (()=>new Date());
+    // -------------------------------------------------------------------
+    // 1. Load the unit row.
+    // -------------------------------------------------------------------
+    const unit = await payload.findByID({
+        collection: unitsSlug,
+        id: unitId,
+        overrideAccess: true,
+        depth: 0
+    });
+    if (!unit) {
+        return {
+            status: 'failed',
+            failureCode: 'permanent.deleted',
+            failureMessage: 'unit not found'
+        };
+    }
+    // Terminal-state guards. Re-running a finished unit must be a no-op
+    // — Payload re-fires the task if it sees ambiguous state. `reverted`
+    // surfaces as `skipped` to the caller since both mean "do not retry".
+    if (unit.status === 'success') return {
+        status: 'success'
+    };
+    if (unit.status === 'skipped' || unit.status === 'reverted') {
+        return {
+            status: 'skipped'
+        };
+    }
+    // -------------------------------------------------------------------
+    // 1c. Load the parent batch row to read `mode`. Threaded into the
+    //     translate call as `force: batchMode === 'force'` so the UI's
+    //     "Force re-translate (includes docs already up to date)" checkbox
+    //     actually bypasses the field-level hash-skip + manual-edit guards
+    //     in translate.ts. Without this, force-mode bulk runs silently
+    //     preserve every prior translation while still paying for tokens.
+    //
+    //     Tolerant on miss: if the batch row can't be loaded (deleted /
+    //     orphan unit / db hiccup), fall through with `force: false` and
+    //     let the per-locale guards take their default conservative path.
+    //     That's the same behaviour as pre-v1.2.5 and never makes things
+    //     worse.
+    // -------------------------------------------------------------------
+    let batchMode;
+    let batchStatus;
+    try {
+        const batch = await payload.findByID({
+            collection: batchesSlug,
+            id: unit.batchId,
+            overrideAccess: true,
+            depth: 0
+        });
+        batchMode = batch?.scope?.mode ?? batch?.mode;
+        batchStatus = batch?.status;
+    } catch  {
+        batchMode = undefined;
+        batchStatus = undefined;
+    }
+    const isForceMode = batchMode === 'force';
+    // -------------------------------------------------------------------
+    // 1d. Cancel gate. The cancel endpoint sweeps `pending` units exactly
+    //     once; units whose job was already queued (or claimed between the
+    //     sweep's find and its updates) would otherwise still translate —
+    //     and bill — after the admin pressed Cancel. Checking the batch
+    //     status at worker entry closes that window. Also covers terminal
+    //     batches: a janitor-reset unit whose batch already ended must not
+    //     run again.
+    // -------------------------------------------------------------------
+    if (batchStatus === 'cancelling' || batchStatus === 'cancelled' || batchStatus === 'reverted') {
+        // Only rewrite units that are still open — a terminal unit (e.g.
+        // `failed`) re-fired by a duplicate job must keep its real outcome.
+        if (unit.status === 'pending' || unit.status === 'running') {
+            await payload.update({
+                collection: unitsSlug,
+                id: unitId,
+                data: {
+                    status: 'skipped',
+                    failureMessage: 'cancelled_by_admin',
+                    completedAt: now().toISOString()
+                },
+                overrideAccess: true
+            });
+            await bumpBatchCounter(payload, batchesSlug, unit.batchId, 'skippedUnits');
+        }
+        // The skip we just wrote may have been the last open unit — give the
+        // batch its chance to leave `cancelling`.
+        await maybeTransitionBatch(payload, batchesSlug, unitsSlug, unit.batchId, callbacks);
+        return {
+            status: 'skipped',
+            failureMessage: 'cancelled_by_admin'
+        };
+    }
+    // -------------------------------------------------------------------
+    // 1b. Security allowlist (Decision #11 / N5). A `bulk-translate-units`
+    //     row's `collection` field is free-text. Without this guard, an
+    //     attacker with write access to that collection could craft a
+    //     unit targeting `users` and the worker would call translateDocument
+    //     with overrideAccess: true. The plugin auto-wires the allowlist
+    //     from `options.collections` + `options.globals`.
+    // -------------------------------------------------------------------
+    const isGlobalForGate = isGlobalSlug(payload, unit.collection);
+    if (!isGlobalForGate && allowedCollections.size > 0 && !allowedCollections.has(unit.collection)) {
+        await payload.update({
+            collection: unitsSlug,
+            id: unitId,
+            data: {
+                status: 'failed',
+                failureCode: 'permanent.config',
+                failureMessage: `Collection '${unit.collection}' not in plugin allowlist`,
+                completedAt: now().toISOString()
+            },
+            overrideAccess: true
+        });
+        return {
+            status: 'failed',
+            failureCode: 'permanent.config',
+            failureMessage: `Collection '${unit.collection}' not in plugin allowlist`
+        };
+    }
+    if (isGlobalForGate && allowedGlobals.size > 0 && !allowedGlobals.has(unit.collection)) {
+        await payload.update({
+            collection: unitsSlug,
+            id: unitId,
+            data: {
+                status: 'failed',
+                failureCode: 'permanent.config',
+                failureMessage: `Global '${unit.collection}' not in plugin allowlist`,
+                completedAt: now().toISOString()
+            },
+            overrideAccess: true
+        });
+        return {
+            status: 'failed',
+            failureCode: 'permanent.config',
+            failureMessage: `Global '${unit.collection}' not in plugin allowlist`
+        };
+    }
+    // -------------------------------------------------------------------
+    // 1d. Per-doc serialization via atomic UPDATE (2026-06-05 rewrite).
+    //
+    // Previously we used `pg_try_advisory_lock` + polling. That held one
+    // pool connection per acquired lock for the full LLM round-trip,
+    // starving Payload's connection pool (default 10) when bulk runs
+    // touched many docs concurrently — admin UI + status endpoints
+    // stalled because every pool slot was tied up by a lock holder.
+    //
+    // New design: a single atomic UPDATE that conditionally transitions
+    // `pending → running` IFF no sibling unit for the same
+    // `(collection, documentId)` is already `running`. Winners proceed
+    // with no held connection. Losers see `0 rowsAffected` → we
+    // re-enqueue this unit's Payload job so a later cron tick retries.
+    //
+    // `concurrency.perDocument: 1` in plugin config only gates the
+    // in-process `translateDocument` API and does NOT cover the bulk-
+    // translate worker pool — this claim is what actually serialises
+    // per-doc work for bulk runs.
+    //
+    // On non-Postgres adapters the claim is a no-op (returns `'noop'`)
+    // and we fall through to the legacy mark-running path below.
+    // -------------------------------------------------------------------
+    const earlyLog = createScopedLogger(payload, {
+        component: 'bulk.worker',
+        batchId: unit.batchId,
+        unitId,
+        collection: unit.collection,
+        documentId: unit.documentId,
+        locale: unit.locale
+    });
+    const claim = await tryClaimAsLock({
+        payload,
+        unitsCollectionSlug: unitsSlug,
+        unitId,
+        collection: unit.collection,
+        documentId: unit.documentId
+    });
+    if (claim.acquired === false) {
+        // Couldn't claim — sibling unit for the same doc is running, OR
+        // the unit was already moved out of `pending` by a competing fire
+        // of the same job. Re-enqueue this unit so a later cron tick
+        // retries; don't bump attempts (deferral isn't a translation
+        // failure).
+        try {
+            await payload.jobs.queue({
+                task: BULK_TRANSLATE_DOC_TASK_SLUG,
+                input: {
+                    unitId
+                }
+            });
+        } catch (enqueueErr) {
+            payload.logger?.warn?.(`[ai-translate] per-doc claim deferred but re-enqueue failed for ${unit.collection}/${String(unit.documentId)} locale=${unit.locale}: ${String(enqueueErr)}`);
+        }
+        earlyLog.event('info', 'bulk.worker.unit.per-doc-busy-deferred', {});
+        return {
+            status: 'pending'
+        };
+    }
+    const releaseLock = claim.release;
+    try {
+        // -------------------------------------------------------------------
+        // 2. Mark running + increment attempts.
+        //
+        // On Postgres the atomic claim above ALREADY transitioned the row
+        // to `running` and bumped `attempts`, so this block is a no-op for
+        // that path. It runs as the fallback when the claim returned
+        // `'noop'` (non-Postgres adapter) — preserving the legacy code path
+        // for those callers.
+        // -------------------------------------------------------------------
+        const isPgClaim = claim.acquired === true;
+        const nextAttempts = isPgClaim ? claim.attempts : (unit.attempts ?? 0) + 1;
+        const log = earlyLog;
+        const unitStartedAtMs = Date.now();
+        if (!isPgClaim) {
+            await payload.update({
+                collection: unitsSlug,
+                id: unitId,
+                data: {
+                    status: 'running',
+                    attempts: nextAttempts,
+                    startedAt: now().toISOString()
+                },
+                overrideAccess: true
+            });
+        }
+        log.event('info', 'bulk.worker.unit.started', {
+            collection: unit.collection,
+            documentId: unit.documentId,
+            locale: unit.locale,
+            attempt: nextAttempts
+        });
+        // -------------------------------------------------------------------
+        // 3. F-DA-TOCTOU scenario B: another batch's unit may be racing this
+        // one. Round-5 pr-reviewer caught the dual-skip bug — naive "skip if
+        // any other pending/running" caused BOTH racing workers to mark
+        // themselves running, query, see each other, and skip. Zero
+        // translation for the doc.
+        //
+        // Fix: deterministic tiebreaker on `batchId`. Among all concurrently-
+        // active units (this one + any others found), the LOWEST batchId
+        // wins. The losers skip. Guarantees exactly one winner per
+        // (collection, doc, locale) tuple under any number of concurrent
+        // batches.
+        // -------------------------------------------------------------------
+        const concurrent = await payload.find({
+            collection: unitsSlug,
+            where: {
+                and: [
+                    {
+                        collection: {
+                            equals: unit.collection
+                        }
+                    },
+                    {
+                        documentId: {
+                            equals: unit.documentId
+                        }
+                    },
+                    {
+                        locale: {
+                            equals: unit.locale
+                        }
+                    },
+                    {
+                        status: {
+                            in: [
+                                'pending',
+                                'running'
+                            ]
+                        }
+                    },
+                    {
+                        id: {
+                            not_equals: unitId
+                        }
+                    }
+                ]
+            },
+            limit: 100,
+            overrideAccess: true,
+            depth: 0
+        });
+        if (concurrent.totalDocs > 0) {
+            const competingBatchIds = concurrent.docs.map((d)=>d.batchId).filter((b)=>typeof b === 'string' && b.length > 0);
+            const allBatchIds = [
+                unit.batchId,
+                ...competingBatchIds
+            ].sort();
+            if (allBatchIds[0] !== unit.batchId) {
+                // A different batch has the lower id — they win. We skip.
+                await payload.update({
+                    collection: unitsSlug,
+                    id: unitId,
+                    data: {
+                        status: 'skipped',
+                        failureMessage: 'concurrent_batch',
+                        completedAt: now().toISOString()
+                    },
+                    overrideAccess: true
+                });
+                await bumpBatchCounter(payload, batchesSlug, unit.batchId, 'skippedUnits');
+                await maybeTransitionBatch(payload, batchesSlug, unitsSlug, unit.batchId, callbacks, log);
+                return {
+                    status: 'skipped',
+                    failureMessage: 'concurrent_batch'
+                };
+            }
+            // We have the lowest batchId — we win. Continue.
+            payload.logger?.debug?.(`[ai-translate] worker: ${concurrent.totalDocs} concurrent unit(s) lost the tiebreaker to batch ${unit.batchId}; proceeding`);
+        }
+        // -------------------------------------------------------------------
+        // 4. Dispatch to the translate API. Globals route through
+        //    `translateGlobal`, collections through `translateDocument`.
+        //    Detection: presence of the slug in `payload.config.globals`.
+        // -------------------------------------------------------------------
+        const isGlobal = isGlobalSlug(payload, unit.collection);
+        const targetLocales = [
+            unit.locale
+        ];
+        // -------------------------------------------------------------------
+        // 3c. Per-locale pre-write snapshot (v1.2.12). Captures the TARGET
+        //     locale's current localized-translatable values onto the unit
+        //     row BEFORE any write, so Revert restores exactly what this run
+        //     overwrote. Replaces the coordinator-era snapshot, which was
+        //     captured from the SOURCE-locale enumeration read and reused for
+        //     every locale — replaying it stamped source-language content
+        //     over translated locales (2026-06-11 prod incident).
+        //
+        //     First capture wins: a retry inside the same batch must not
+        //     overwrite the true pre-batch state with the previous attempt's
+        //     machine output. Absent paths are stored as explicit nulls so a
+        //     revert of a fresh locale clears the run's writes back to empty.
+        //     Best-effort: on capture failure the unit has no locale-tagged
+        //     snapshot and Revert refuses it (safe default) while translation
+        //     proceeds normally.
+        // -------------------------------------------------------------------
+        if (unit.snapshotLocale !== unit.locale) {
+            try {
+                const surfaceFields = isGlobal ? (payload.config.globals ?? []).find((g)=>g.slug === unit.collection)?.fields : (payload.config.collections ?? []).find((c)=>c.slug === unit.collection)?.fields;
+                if (surfaceFields) {
+                    const topLevelPaths = Array.from(new Set(resolveTranslatableFields(surfaceFields).filter((f)=>f.localized).map((f)=>f.path.split('.')[0])));
+                    const targetDoc = isGlobal ? await findGlobalNoFallback(payload, unit.collection, unit.locale) : await findByIdNoFallback(payload, unit.collection, unit.documentId, unit.locale);
+                    const snapshot = captureLocalizedSnapshot(targetDoc ?? {}, topLevelPaths);
+                    for (const path of topLevelPaths){
+                        if (!(path in snapshot)) snapshot[path] = null;
+                    }
+                    await payload.update({
+                        collection: unitsSlug,
+                        id: unitId,
+                        data: {
+                            preRunSnapshot: snapshot,
+                            snapshotLocale: unit.locale
+                        },
+                        overrideAccess: true
+                    });
+                }
+            } catch (err) {
+                payload.logger?.warn?.(`[ai-translate] worker: pre-write snapshot capture failed for ${unit.collection}/${unit.documentId} locale=${unit.locale} — unit will not be revertable: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+        try {
+            let costUsd;
+            // `providerLatencyMs` is the SUM of per-batch LLM round-trip times
+            // reported by the provider — measured around the SDK call, so it
+            // excludes token-bucket wait + provider rate-limit backoff. That's
+            // the "how long did the AI actually take" signal the editor wants.
+            //
+            // The delta between this unit's `startedAt` and `completedAt`
+            // includes throttle wait and is wallclock-in-worker — still useful
+            // for engineering (it answers "how long was this unit blocked by
+            // the rate limiter?") but not what the editor needs.
+            let providerLatencyMs = 0;
+            let outcome;
+            if (isGlobal) {
+                outcome = await translateApi.translateGlobal(payload, {
+                    global: unit.collection,
+                    targetLocales,
+                    req: params.req,
+                    force: isForceMode
+                });
+            } else {
+                outcome = await translateApi.translateDocument(payload, {
+                    collection: unit.collection,
+                    id: unit.documentId,
+                    targetLocales,
+                    req: params.req,
+                    force: isForceMode
+                });
+            }
+            costUsd = requireCostUsd(outcome);
+            providerLatencyMs = outcome?.providerLatencyMs ?? 0;
+            // ---------------------------------------------------------------------
+            // Unit-status truth check (v1.2.5 + v1.2.7 refinement).
+            //
+            // The translate API returns `{ succeeded, preserved, failed }` where
+            // `failed` mixes two outcome classes:
+            //   - `status: 'failed'` — HARD failure (provider error, no response,
+            //     schema rejection). These are real failures the editor must see.
+            //   - `status: 'skipped'` — SOFT skip (verbatim echo, too-short ratio,
+            //     placeholder mismatch). The output validator rejected the value
+            //     but it wasn't a hardware error — the model gave us something we
+            //     deliberately won't write. Common case: a 1-word `alt` field
+            //     containing a brand name the model correctly returns unchanged.
+            //
+            // v1.2.5 lumped both into `failedCount` and stamped every all-rejected
+            // unit as `permanent.schema` with an "AI response was unreadable"
+            // editor message — which lies when the actual cause was an
+            // intentional echo on a proper noun. v1.2.7 splits them:
+            //
+            //   - succeeded > 0  → 'success' (at least one write landed)
+            //   - hardFailed > 0 && succeeded === 0 && preserved === 0
+            //                    → 'failed' / 'permanent.schema'
+            //                      (genuine output-validation failure)
+            //   - softSkipped > 0 && hardFailed === 0 && succeeded === 0
+            //                    && preserved === 0
+            //                    → 'skipped' / `soft-skip.no-translation-needed`
+            //                      (model returned source unchanged for every
+            //                      field — typically proper nouns / short labels)
+            //   - preserved > 0  → 'success' (manual-edit guard kept everything)
+            //   - all zero       → 'success' (empty source, no-op)
+            // ---------------------------------------------------------------------
+            const succeededCount = outcome?.succeeded?.length ?? 0;
+            const preservedCount = outcome?.preserved?.length ?? 0;
+            const failedFields = outcome?.failed ?? [];
+            const hardFailed = failedFields.filter((f)=>f.status === 'failed');
+            const softSkipped = failedFields.filter((f)=>f.status === 'skipped');
+            if (succeededCount === 0 && preservedCount === 0 && hardFailed.length === 0 && softSkipped.length > 0) {
+                const paths = softSkipped.map((f)=>f.fieldPath ?? '?').slice(0, 5).join(', ');
+                const message = `${softSkipped.length} field(s) returned unchanged from source${paths ? `: ${paths}` : ''} — kept as-is`;
+                await payload.update({
+                    collection: unitsSlug,
+                    id: unitId,
+                    data: {
+                        status: 'skipped',
+                        failureMessage: message,
+                        processingDurationMs: providerLatencyMs,
+                        completedAt: now().toISOString()
+                    },
+                    overrideAccess: true
+                });
+                await bumpBatchCounter(payload, batchesSlug, unit.batchId, 'skippedUnits');
+                await maybeTransitionBatch(payload, batchesSlug, unitsSlug, unit.batchId, callbacks, log);
+                log.event('info', 'bulk.worker.unit.all-soft-skipped', {
+                    durationMs: Date.now() - unitStartedAtMs,
+                    softSkippedCount: softSkipped.length,
+                    attempt: nextAttempts
+                });
+                return {
+                    status: 'skipped',
+                    failureMessage: message
+                };
+            }
+            if (succeededCount === 0 && preservedCount === 0 && hardFailed.length > 0) {
+                const paths = hardFailed.map((f)=>f.fieldPath ?? '?').slice(0, 5).join(', ');
+                const message = `${hardFailed.length} field(s) failed validation${paths ? `: ${paths}` : ''}`;
+                await payload.update({
+                    collection: unitsSlug,
+                    id: unitId,
+                    data: {
+                        status: 'failed',
+                        failureCode: 'permanent.schema',
+                        failureMessage: message,
+                        processingDurationMs: providerLatencyMs,
+                        completedAt: now().toISOString()
+                    },
+                    overrideAccess: true
+                });
+                await bumpBatchCounter(payload, batchesSlug, unit.batchId, 'failedUnits');
+                await maybeTransitionBatch(payload, batchesSlug, unitsSlug, unit.batchId, callbacks, log);
+                log.event('warn', 'bulk.worker.unit.all-fields-failed', {
+                    durationMs: Date.now() - unitStartedAtMs,
+                    failedCount: hardFailed.length,
+                    attempt: nextAttempts,
+                    failureCode: 'permanent.schema'
+                });
+                return {
+                    status: 'failed',
+                    failureCode: 'permanent.schema',
+                    failureMessage: message
+                };
+            }
+            await payload.update({
+                collection: unitsSlug,
+                id: unitId,
+                data: {
+                    status: 'success',
+                    costUsd,
+                    processingDurationMs: providerLatencyMs,
+                    completedAt: now().toISOString()
+                },
+                overrideAccess: true
+            });
+            await bumpBatchCounter(payload, batchesSlug, unit.batchId, 'completedUnits', costUsd);
+            await maybeTransitionBatch(payload, batchesSlug, unitsSlug, unit.batchId, callbacks, log);
+            log.event('info', 'bulk.worker.unit.succeeded', {
+                durationMs: Date.now() - unitStartedAtMs,
+                costUsd,
+                attempt: nextAttempts
+            });
+            return {
+                status: 'success',
+                costUsd
+            };
+        } catch (err) {
+            const code = classifyError(err);
+            const message = err instanceof Error ? err.message : String(err);
+            // Transient errors: send back to `pending` for the cron tick to
+            // retry, UNLESS we've exhausted maxAttempts. Past that, mark
+            // failed terminally with `transient.crashed`.
+            const isTransient = code === 'transient.rate_limited' || code === 'transient.provider' || code === 'transient.crashed';
+            if (isTransient && nextAttempts < maxAttempts) {
+                await payload.update({
+                    collection: unitsSlug,
+                    id: unitId,
+                    data: {
+                        status: 'pending',
+                        failureCode: code,
+                        failureMessage: message
+                    },
+                    overrideAccess: true
+                });
+                log.event('warn', 'bulk.worker.unit.transient', {
+                    err,
+                    attempt: nextAttempts,
+                    maxAttempts,
+                    failureCode: code,
+                    durationMs: Date.now() - unitStartedAtMs
+                });
+                return {
+                    status: 'pending',
+                    failureCode: code,
+                    failureMessage: message
+                };
+            }
+            const terminalCode = isTransient ? 'transient.crashed' : code;
+            await payload.update({
+                collection: unitsSlug,
+                id: unitId,
+                data: {
+                    status: 'failed',
+                    failureCode: terminalCode,
+                    failureMessage: message,
+                    completedAt: now().toISOString()
+                },
+                overrideAccess: true
+            });
+            await bumpBatchCounter(payload, batchesSlug, unit.batchId, 'failedUnits');
+            await maybeTransitionBatch(payload, batchesSlug, unitsSlug, unit.batchId, callbacks, log);
+            log.event('error', 'bulk.worker.unit.failed', {
+                err,
+                attempt: nextAttempts,
+                maxAttempts,
+                failureCode: terminalCode,
+                durationMs: Date.now() - unitStartedAtMs
+            });
+            return {
+                status: 'failed',
+                failureCode: terminalCode,
+                failureMessage: message
+            };
+        }
+    } finally{
+        // Per-doc lock acquired above the try/catch. Release in every exit
+        // path (success / failed / pending) so the next sibling locale for
+        // this doc can proceed. No-op on non-Postgres adapters.
+        await releaseLock();
+    }
+}
+// ---------------------------------------------------------------------------
+// Error classification
+// ---------------------------------------------------------------------------
+/**
+ * Maps a thrown error to a `failureCode`. Order matters — more
+ * specific patterns win. Defaults to `unknown` for anything we can't
+ * recognise so the row stays visible in the admin drill-down rather
+ * than disappearing into a generic 'failed'.
+ *
+ * Classes per design doc Decision §worker §error-classification:
+ *   - transient.rate_limited: 429 / explicit rate-limit
+ *   - transient.provider:     5xx / fetch timeout
+ *   - permanent.schema:       NoObjectGeneratedError (Vercel AI SDK
+ *                             throws this when the model returns
+ *                             malformed structured output)
+ *   - permanent.too_large:    cost-guard PER_DOC_CEILING/PER_CALL_LIMIT
+ *   - permanent.config:       401 / 403 / auth
+ *   - permanent.deleted:      "Document not found" from findByIdNoFallback
+ *   - unknown:                default
+ */ export function classifyError(err) {
+    if (err === null || err === undefined) return 'unknown';
+    const message = err instanceof Error ? err.message : String(err);
+    const lower = message.toLowerCase();
+    const name = err instanceof Error ? err.name : '';
+    const codeProp = err?.code;
+    const codeStr = typeof codeProp === 'string' ? codeProp : '';
+    // Cost-guard: check first — its error messages contain '429' fragments
+    // in some surfaces.
+    if (codeStr === 'PER_DOC_CEILING' || codeStr === 'PER_CALL_LIMIT') {
+        return 'permanent.too_large';
+    }
+    if (lower.includes('per_doc_ceiling') || lower.includes('per_call_limit')) {
+        return 'permanent.too_large';
+    }
+    // Decision #29 / NEW-12 (v1.2.6): undefined / non-finite cost is a
+    // provider-flake signal — we hard-throw COST_UNDEFINED inside the
+    // worker so the bad cost row never persists as $0 spend. In practice
+    // (batches #5, #6 on blog-wild prod) 124 / 168 + 129 / 168 units in
+    // the same batch succeeded with the same model + same config, so
+    // "config broken" is the wrong story. Classify as transient.provider
+    // so the cron retries it on the next tick and the UI doesn't
+    // misdirect editors to the settings page. If retries exhaust, the
+    // worker still escalates to `transient.crashed` (terminal) — the
+    // unit doesn't silently disappear.
+    if (codeStr === 'COST_UNDEFINED') {
+        return 'transient.provider';
+    }
+    // Defense in depth: when COST_UNDEFINED is rethrown by an outer
+    // wrapper that drops the `.code` property, fall back to message
+    // matching. The literal text is set in `throwCostUndefined` below
+    // and is stable across versions.
+    if (lower.includes('provider returned undefined / non-finite estimatedcostusd') || lower.includes('cost instrumentation broken')) {
+        return 'transient.provider';
+    }
+    // Schema-invalid (AI SDK throws NoObjectGeneratedError when the
+    // model produces malformed JSON against the requested schema).
+    if (name === 'NoObjectGeneratedError' || lower.includes('noobjectgeneratederror')) {
+        return 'permanent.schema';
+    }
+    // Document-not-found from the plugin's `findByIdNoFallback` path.
+    // NEW-12 (v1.2.6): also catch the bare "Not Found" payload throws when
+    // a doc is hard-deleted between enqueue and worker pickup — observed
+    // on prod (batches #5/6 had `failure_message='Not Found'` rows that
+    // were silently classified as `unknown` and rendered as "Translation
+    // failed unexpectedly" instead of the more accurate "document was
+    // deleted" copy.
+    if (/document not found|global not found|not found in payload/.test(lower) || /^not found$/i.test(message.trim())) {
+        return 'permanent.deleted';
+    }
+    // Auth / config — distinct from transient 5xx.
+    if (/\b401\b|\b403\b|unauthorized|forbidden|api[_-]?key/i.test(message)) {
+        return 'permanent.config';
+    }
+    // Rate-limit BEFORE provider — both can include '429' in the
+    // message but rate-limit is the more specific reason.
+    if (/\b429\b|rate[_-]?limit/i.test(message)) {
+        return 'transient.rate_limited';
+    }
+    // Provider 5xx / fetch timeout / network.
+    if (/\b5\d\d\b|timeout|timed out|fetch failed|econnreset|enotfound|network/i.test(message) || name === 'AbortError' || name === 'FetchError') {
+        return 'transient.provider';
+    }
+    return 'unknown';
+}
+async function bumpBatchCounter(payload, batchesSlug, batchId, counter, costUsd = 0) {
+    const log = createScopedLogger(payload, {
+        component: 'bulk.worker',
+        batchId
+    });
+    // Atomic increment via raw SQL — `payload.update()` with a fetched
+    // value would race under concurrent workers (read-then-write).
+    // Postgres `column = column + N` is the standard fix and lets us
+    // bump the counter + actualCostUsd in a single statement.
+    //
+    // The column-name map mirrors Payload-Drizzle's camelCase →
+    // snake_case convention. Hard-coded rather than computed so a
+    // future column rename surfaces as a build/test failure here
+    // rather than a silent drop.
+    const columnByCounter = {
+        completedUnits: 'completed_units',
+        failedUnits: 'failed_units',
+        skippedUnits: 'skipped_units'
+    };
+    const column = columnByCounter[counter];
+    const numericBatchId = Number.parseInt(batchId, 10);
+    if (!Number.isFinite(numericBatchId)) {
+        log.event('warn', 'bulk.counter-bump.invalid-batch-id', {
+            counter,
+            batchId
+        });
+        return;
+    }
+    const drizzle = payload.db.drizzle;
+    if (!drizzle) {
+        // Non-postgres adapter — fall back to the read-modify-write path.
+        // Concurrency-sensitive counters will drift but the system stays
+        // functional. The maybeTransitionBatch reconciler re-derives the
+        // terminal status from the source-of-truth units count, so a
+        // skewed counter doesn't permanently block batch completion.
+        try {
+            const row = await payload.findByID({
+                collection: batchesSlug,
+                id: batchId,
+                overrideAccess: true,
+                depth: 0
+            });
+            if (!row) return;
+            const current = Number(row[counter] ?? 0);
+            const data = {
+                [counter]: current + 1
+            };
+            if (costUsd > 0) {
+                data.actualCostUsd = Number(row.actualCostUsd ?? 0) + costUsd;
+            }
+            await payload.update({
+                collection: batchesSlug,
+                id: batchId,
+                data,
+                overrideAccess: true
+            });
+        } catch (err) {
+            log.event('warn', 'bulk.counter-bump.failed', {
+                err,
+                counter,
+                batchId,
+                path: 'fallback'
+            });
+        }
+        return;
+    }
+    let sqlTag;
+    try {
+        const drizzleMod = await import('drizzle-orm');
+        sqlTag = drizzleMod.sql;
+    } catch  {
+        sqlTag = undefined;
+    }
+    if (!sqlTag) {
+        log.event('warn', 'bulk.counter-bump.drizzle-unavailable', {
+            counter,
+            batchId
+        });
+        return;
+    }
+    try {
+        const columnExpr = sqlTag.raw(`"${column}"`);
+        if (costUsd > 0) {
+            await drizzle.execute(sqlTag`UPDATE "bulk_translate_batches"
+               SET ${columnExpr} = COALESCE(${columnExpr}, 0) + 1,
+                   "actual_cost_usd" = COALESCE("actual_cost_usd", 0) + ${costUsd},
+                   "updated_at" = now()
+               WHERE "id" = ${numericBatchId}`);
+        } else {
+            await drizzle.execute(sqlTag`UPDATE "bulk_translate_batches"
+               SET ${columnExpr} = COALESCE(${columnExpr}, 0) + 1,
+                   "updated_at" = now()
+               WHERE "id" = ${numericBatchId}`);
+        }
+    } catch (err) {
+        log.event('warn', 'bulk.counter-bump.failed', {
+            err,
+            counter,
+            batchId
+        });
+    }
+}
+/**
+ * Check whether every unit in the batch is in a terminal state. If
+ * yes, transition the batch and fire the callback.
+ */ export async function maybeTransitionBatch(payload, batchesSlug, unitsSlug, batchId, callbacks, parentLog) {
+    const log = parentLog ?? createScopedLogger(payload, {
+        component: 'bulk.worker',
+        batchId
+    });
+    const row = await payload.findByID({
+        collection: batchesSlug,
+        id: batchId,
+        overrideAccess: true,
+        depth: 0
+    });
+    if (!row) return;
+    // v1.2.5: also handle `cancelling`. Pre-1.2.5 the cancel endpoint
+    // flipped the batch to `cancelling` and relied on workers finishing
+    // in-flight units to call maybeTransitionBatch; this function then
+    // bailed because status was no longer `running` / `queued`. Result:
+    // the batch sat in `cancelling` forever and only `force-reset`
+    // could unstick it. Allowing `cancelling` here lets the last worker
+    // to finish flip the batch to `cancelled`.
+    if (row.status !== 'running' && row.status !== 'queued' && row.status !== 'cancelling') return;
+    // Re-derive counts from the source of truth (units collection) so
+    // we don't trust the eventually-consistent counters on the batch
+    // row.
+    const totalsByStatus = await aggregateUnitStatuses(payload, unitsSlug, batchId);
+    const total = row.totalUnits ?? totalsByStatus.total;
+    if (total === 0) return; // coordinator hasn't stamped totals yet
+    const terminal = totalsByStatus.success + totalsByStatus.failed + totalsByStatus.skipped + totalsByStatus.reverted;
+    if (terminal < total) return;
+    // Determine terminal status. Vocabulary: 'success' (not 'completed')
+    // — aligned across schema + types + readers in v1.2.4.
+    //
+    // v1.2.5: when transitioning out of `cancelling`, the destination is
+    // always `cancelled` regardless of per-unit outcomes. The admin
+    // explicitly asked to stop; reporting `success` / `partial` /
+    // `failed` would erase the intent.
+    let nextStatus;
+    if (row.status === 'cancelling') {
+        nextStatus = 'cancelled';
+    } else if (totalsByStatus.failed === 0 && totalsByStatus.success > 0) {
+        nextStatus = 'success';
+    } else if (totalsByStatus.success === 0 && totalsByStatus.failed > 0) {
+        nextStatus = 'failed';
+    } else {
+        nextStatus = 'partial';
+    }
+    await payload.update({
+        collection: batchesSlug,
+        id: batchId,
+        data: {
+            status: nextStatus,
+            completedAt: new Date().toISOString()
+        },
+        overrideAccess: true
+    });
+    log.event('info', 'bulk.batch.transition', {
+        batchId,
+        nextStatus,
+        totalsByStatus
+    });
+    await fireBatchCallback(payload, callbacks, row, nextStatus, totalsByStatus, log);
+}
+async function aggregateUnitStatuses(payload, unitsSlug, batchId) {
+    const statuses = [
+        'success',
+        'failed',
+        'skipped',
+        'reverted',
+        'pending',
+        'running'
+    ];
+    const out = {
+        total: 0,
+        success: 0,
+        failed: 0,
+        skipped: 0,
+        reverted: 0,
+        pending: 0,
+        running: 0
+    };
+    for (const status of statuses){
+        const result = await payload.count({
+            collection: unitsSlug,
+            where: {
+                and: [
+                    {
+                        batchId: {
+                            equals: batchId
+                        }
+                    },
+                    {
+                        status: {
+                            equals: status
+                        }
+                    }
+                ]
+            },
+            overrideAccess: true
+        });
+        out[status] = result.totalDocs;
+        out.total += result.totalDocs;
+    }
+    return out;
+}
+async function fireBatchCallback(payload, callbacks, batch, status, totals, log) {
+    if (!callbacks) return;
+    const event = {
+        batchId: String(batch.id),
+        status,
+        scope: {
+            collections: batch.scope?.collections,
+            globals: batch.scope?.globals,
+            locales: batch.scope?.locales,
+            mode: batch.scope?.mode ?? 'changed'
+        },
+        counts: {
+            total: totals.total,
+            completed: totals.success,
+            failed: totals.failed,
+            skipped: totals.skipped
+        },
+        costUsd: {
+            estimated: Number(batch.estimatedCostUsd ?? 0),
+            actual: Number(batch.actualCostUsd ?? 0)
+        },
+        triggeredByUserId: String(batch.triggeredByUserId ?? ''),
+        triggeredByEmail: batch.triggeredByEmail,
+        durationMs: batch.startedAt ? Date.now() - new Date(batch.startedAt).getTime() : 0
+    };
+    try {
+        await callbacks.onBatchComplete?.(event);
+        if (status === 'failed') {
+            await callbacks.onBatchFailed?.(event);
+        }
+    } catch (err) {
+        log.event('error', 'bulk.batch.callback.failed', {
+            err,
+            batchId: String(batch.id),
+            status
+        });
+    }
+}
+function isGlobalSlug(payload, slug) {
+    const globals = payload.config?.globals ?? [];
+    return globals.some((g)=>g.slug === slug);
+}
+/**
+ * Decision #29 + round-5 pr-reviewer blocker #3 — `undefined`
+ * `estimatedCostUsd` is the canonical signal that cost instrumentation
+ * is broken upstream. Silently coercing it to `0` (the previous
+ * behavior) corrupted `actualCostUsd` aggregates on every batch using
+ * the OpenRouter provider, hiding real spend from the cap + reporting.
+ *
+ * Hard-throw with a classified error so `classifyError` routes it to
+ * `permanent.config`, surfacing the broken instrumentation in the
+ * admin drill-down rather than silently aggregating $0.
+ */ function requireCostUsd(result) {
+    const raw = result?.usage?.estimatedCostUsd;
+    if (typeof raw !== 'number' || !Number.isFinite(raw) || raw < 0) {
+        throw Object.assign(new Error('Provider returned undefined / non-finite estimatedCostUsd — cost instrumentation broken; refusing to record $0'), {
+            code: 'COST_UNDEFINED'
+        });
+    }
+    return raw;
+}
+/**
+ * Lazy-load the plugin's translate API for the default builder. Tests
+ * inject their own implementation via `options.translateApi`.
+ */ async function loadDefaultTranslateApi() {
+    const mod = await import('../api.js');
+    // Single narrow-cast (NOT `as unknown as`) — the real api signatures
+    // are a strict subset of `TranslateApiFns` (they accept a stricter
+    // `req?: PayloadRequest`, we expose the looser `req?: unknown`).
+    return {
+        translateDocument: mod.translateDocument,
+        translateGlobal: mod.translateGlobal
+    };
+}