@purposeinplay/payload-ai-translate 0.1.0

package/dist/tasks/bulk-translate-janitor.d.ts

@@ -0,0 +1,87 @@
+import type { Payload, TaskConfig } from 'payload';
+import type { BulkTranslateConfig } from '../types.js';
+export declare const BULK_TRANSLATE_JANITOR_SLUG = "bulk-translate-janitor";
+/**
+ * Janitor task — sweeps stale `running` units back into `pending` so
+ * the cron picks them up again. Cron-driven (recommended cadence
+ * `*\/5 * * * *`), but the task is also idempotent for ad-hoc admin
+ * triggering.
+ *
+ * Threshold: `max(2 × p99-of-recent-LLM-latency, 10 min)` per Decision
+ * #27 + F-DA-JANITOR-P99. P99 is computed from the
+ * `translation-usage.durationMs` column over the last 1000 rows.
+ * Self-tuning — if real translations start taking 4 minutes the
+ * threshold drifts up automatically.
+ *
+ * The P99 read is cached for 5 minutes per process. Janitor's
+ * threshold can lag actual P99 by up to 5 min; acceptable for a
+ * stale-row reset task.
+ */
+export interface BulkJanitorOptions {
+    /** Slug override for the units collection. */
+    unitsCollectionSlug?: string;
+    /** Slug override for the batches collection. */
+    batchesCollectionSlug?: string;
+    /** Slug override for the usage collection used to compute P99. */
+    usageCollectionSlug?: string;
+    /** Batch lifecycle callbacks, forwarded to `maybeTransitionBatch`. */
+    callbacks?: Pick<BulkTranslateConfig, 'onBatchComplete' | 'onBatchFailed'>;
+    /**
+     * Floor for the staleness threshold. Even if recent P99 is fast,
+     * never reset rows newer than this. Default 10 * 60_000 (10 min).
+     */
+    minThresholdMs?: number;
+    /**
+     * Multiplier applied to recent P99 to compute the threshold.
+     * Default 2 — gives slow translations a full 2× headroom over
+     * historical P99 before the janitor steps in.
+     */
+    p99Multiplier?: number;
+    /**
+     * Maximum attempts before a stale `running` row is marked
+     * permanently failed (rather than re-queued). Default 3.
+     */
+    maxAttempts?: number;
+    /**
+     * P99 cache TTL. Default 5 minutes. Per-process cache; multi-process
+     * deployments compute independently which is fine — the threshold
+     * change is gradual.
+     */
+    p99CacheTtlMs?: number;
+}
+type JanitorTaskInput = Record<string, never>;
+export declare function buildBulkTranslateJanitor(options?: BulkJanitorOptions): TaskConfig<{
+    input: JanitorTaskInput;
+    output: {
+        ok: true;
+        reset: number;
+        failed: number;
+        requeued: number;
+        thresholdMs: number;
+    };
+}>;
+export type JanitorSweepParams = {
+    payload: Payload;
+    unitsSlug: string;
+    usageSlug: string;
+    /** Slug of the batches collection — used to unstick finished batches. */
+    batchesSlug?: string;
+    minThresholdMs: number;
+    p99Multiplier: number;
+    maxAttempts: number;
+    p99CacheTtlMs: number;
+    /** Batch lifecycle callbacks, forwarded to `maybeTransitionBatch`. */
+    callbacks?: Pick<BulkTranslateConfig, 'onBatchComplete' | 'onBatchFailed'>;
+    /** Test override for the current time. */
+    now?: () => number;
+};
+export type JanitorSweepResult = {
+    reset: number;
+    failed: number;
+    requeued: number;
+    thresholdMs: number;
+};
+/** Reset for tests. */
+export declare function resetJanitorP99Cache(payload?: Payload): void;
+export declare function runJanitorSweep(params: JanitorSweepParams): Promise<JanitorSweepResult>;
+export {};

package/dist/tasks/bulk-translate-janitor.js

@@ -0,0 +1,311 @@
+import { DEFAULT_BULK_TRANSLATE_BATCHES_COLLECTION_SLUG } from '../bulk-translate-batches-collection.js';
+import { DEFAULT_BULK_TRANSLATE_UNITS_COLLECTION_SLUG } from '../bulk-translate-units-collection.js';
+import { DEFAULT_USAGE_COLLECTION_SLUG } from '../defaults.js';
+import { BULK_TRANSLATE_DOC_TASK_SLUG, maybeTransitionBatch } from './bulk-translate-doc-task.js';
+export const BULK_TRANSLATE_JANITOR_SLUG = 'bulk-translate-janitor';
+// ---------------------------------------------------------------------------
+// Task config
+// ---------------------------------------------------------------------------
+const taskInputSchema = [];
+export function buildBulkTranslateJanitor(options = {}) {
+    const unitsSlug = options.unitsCollectionSlug ?? DEFAULT_BULK_TRANSLATE_UNITS_COLLECTION_SLUG;
+    const batchesSlug = options.batchesCollectionSlug ?? DEFAULT_BULK_TRANSLATE_BATCHES_COLLECTION_SLUG;
+    const usageSlug = options.usageCollectionSlug ?? DEFAULT_USAGE_COLLECTION_SLUG;
+    const minThresholdMs = options.minThresholdMs ?? 10 * 60_000;
+    const p99Multiplier = options.p99Multiplier ?? 2;
+    const maxAttempts = options.maxAttempts ?? 3;
+    const p99CacheTtlMs = options.p99CacheTtlMs ?? 5 * 60_000;
+    return {
+        slug: BULK_TRANSLATE_JANITOR_SLUG,
+        label: 'Bulk translate — janitor',
+        inputSchema: taskInputSchema,
+        retries: 0,
+        handler: async ({ req })=>{
+            const result = await runJanitorSweep({
+                payload: req.payload,
+                unitsSlug,
+                batchesSlug,
+                usageSlug,
+                minThresholdMs,
+                p99Multiplier,
+                maxAttempts,
+                p99CacheTtlMs,
+                callbacks: options.callbacks
+            });
+            return {
+                output: {
+                    ok: true,
+                    ...result
+                }
+            };
+        }
+    };
+}
+/**
+ * Module-level P99 cache. WeakMap keyed by the Payload instance so a
+ * test that creates two payload mocks doesn't see cache pollution.
+ */ const P99_CACHE = new WeakMap();
+/** Reset for tests. */ export function resetJanitorP99Cache(payload) {
+    if (payload) {
+        P99_CACHE.delete(payload);
+        return;
+    }
+// WeakMap has no .clear() — tests can pass a payload reference to
+// evict its entry. Skipping payload deletes nothing; callers wanting
+// a fresh cache for a brand-new mock just rebuild the mock.
+}
+export async function runJanitorSweep(params) {
+    const { payload, unitsSlug, usageSlug, minThresholdMs, p99Multiplier, maxAttempts, p99CacheTtlMs } = params;
+    const now = params.now ?? (()=>Date.now());
+    const p99Ms = await readP99Cached(payload, usageSlug, now, p99CacheTtlMs);
+    const thresholdMs = Math.max(minThresholdMs, p99Multiplier * p99Ms);
+    const cutoffIso = new Date(now() - thresholdMs).toISOString();
+    const result = await payload.find({
+        collection: unitsSlug,
+        where: {
+            and: [
+                {
+                    status: {
+                        equals: 'running'
+                    }
+                },
+                {
+                    startedAt: {
+                        less_than: cutoffIso
+                    }
+                }
+            ]
+        },
+        // Bound the per-tick work. 200 stale rows per sweep is plenty —
+        // the next cron tick picks up anything left.
+        limit: 200,
+        depth: 0,
+        overrideAccess: true
+    });
+    const stale = result.docs;
+    const batchesSlug = params.batchesSlug ?? DEFAULT_BULK_TRANSLATE_BATCHES_COLLECTION_SLUG;
+    const affectedBatchIds = new Set();
+    const noteBatch = (u)=>{
+        const raw = typeof u.batchId === 'object' && u.batchId !== null ? u.batchId.id : u.batchId;
+        if (raw !== undefined && raw !== null && String(raw).length > 0) {
+            affectedBatchIds.add(String(raw));
+        }
+    };
+    // A unit's worker job is queued exactly once (at enumeration, or by the
+    // worker's own deferral re-queue). The worker task has `retries: 0`, and
+    // the coordinator never re-runs after enumeration finishes — so a reset
+    // WITHOUT a replacement job leaves the unit `pending` forever and its
+    // batch stuck in `running`/`cancelling` for eternity. Every reset must
+    // therefore queue a fresh job. Cancelled batches are safe to re-queue
+    // into: the worker's cancel gate marks their units `skipped` on entry.
+    const requeueUnit = async (unitId)=>{
+        await payload.jobs.queue({
+            task: BULK_TRANSLATE_DOC_TASK_SLUG,
+            input: {
+                unitId: String(unitId)
+            }
+        });
+    };
+    let reset = 0;
+    let failed = 0;
+    for (const u of stale){
+        const attempts = u.attempts ?? 0;
+        try {
+            if (attempts < maxAttempts) {
+                await payload.update({
+                    collection: unitsSlug,
+                    id: u.id,
+                    data: {
+                        status: 'pending',
+                        failureCode: 'transient.crashed',
+                        failureMessage: `janitor reset: running > ${Math.round(thresholdMs / 1000)}s`,
+                        // Cleared so the orphan-recovery pass below can tell "fresh
+                        // pending awaiting its queued job" (startedAt null) apart from
+                        // "previously claimed, job chain dead" (startedAt set).
+                        startedAt: null
+                    },
+                    overrideAccess: true
+                });
+                await requeueUnit(u.id);
+                reset += 1;
+            } else {
+                await payload.update({
+                    collection: unitsSlug,
+                    id: u.id,
+                    data: {
+                        status: 'failed',
+                        failureCode: 'transient.crashed',
+                        failureMessage: `janitor reset: exceeded ${maxAttempts} attempts`,
+                        completedAt: new Date(now()).toISOString()
+                    },
+                    overrideAccess: true
+                });
+                failed += 1;
+            }
+            noteBatch(u);
+        } catch (err) {
+            payload.logger?.warn?.(`[ai-translate] janitor: failed to update unit ${u.id}: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    // ----- Orphan-recovery pass -----------------------------------------
+    // Units whose job chain is dead and that nothing will ever pick up:
+    //   (a) `pending` with a stale `startedAt` (janitor-reset rows that
+    //       kept their last claim's timestamp), and
+    //   (b) `pending` with NO `startedAt` but a stale `updatedAt` — a
+    //       previous recovery cleared the marker and queued a job that
+    //       died before claiming (or the queue write itself failed). The
+    //       original one-shot predicate matched only (a), so a single
+    //       failed recovery made the unit permanently invisible
+    //       (observed: 4 units stuck `pending` >25 min on blog-wild prod,
+    //       2026-06-11). `updatedAt` refreshes on every touch, so retries
+    //       self-pace at one attempt per threshold window.
+    // Re-queue FIRST, then clear the marker — if the queue write throws,
+    // the unit stays matchable by branch (a) instead of burning its
+    // marker. Duplicate jobs are harmless either way — the worker's
+    // atomic claim turns a second fire into a no-op.
+    let requeued = 0;
+    try {
+        const orphans = await payload.find({
+            collection: unitsSlug,
+            where: {
+                and: [
+                    {
+                        status: {
+                            equals: 'pending'
+                        }
+                    },
+                    {
+                        or: [
+                            {
+                                startedAt: {
+                                    less_than: cutoffIso
+                                }
+                            },
+                            {
+                                and: [
+                                    {
+                                        startedAt: {
+                                            exists: false
+                                        }
+                                    },
+                                    {
+                                        updatedAt: {
+                                            less_than: cutoffIso
+                                        }
+                                    }
+                                ]
+                            }
+                        ]
+                    }
+                ]
+            },
+            limit: 200,
+            depth: 0,
+            overrideAccess: true
+        });
+        for (const u of orphans.docs){
+            try {
+                await requeueUnit(u.id);
+                await payload.update({
+                    collection: unitsSlug,
+                    id: u.id,
+                    data: {
+                        startedAt: null
+                    },
+                    overrideAccess: true
+                });
+                requeued += 1;
+                noteBatch(u);
+            } catch (err) {
+                payload.logger?.warn?.(`[ai-translate] janitor: orphan re-queue failed for unit ${u.id}: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+    } catch (err) {
+        payload.logger?.warn?.(`[ai-translate] janitor: orphan-recovery pass failed: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    // ----- Stuck-batch unsticker -----------------------------------------
+    // `maybeTransitionBatch` only ever ran from a finishing worker, so a
+    // batch whose last open unit was resolved by anything else (cancel
+    // sweep races, janitor `failed` transitions above, crashed workers)
+    // stayed `running`/`cancelling` forever. Re-evaluate every batch this
+    // sweep touched — plus every non-terminal batch, so batches that were
+    // already stuck before this sweep (all units terminal, transition never
+    // fired) get unstuck even when the unit passes above found nothing.
+    try {
+        const open = await payload.find({
+            collection: batchesSlug,
+            where: {
+                status: {
+                    in: [
+                        'running',
+                        'cancelling'
+                    ]
+                }
+            },
+            limit: 20,
+            depth: 0,
+            overrideAccess: true
+        });
+        for (const b of open.docs){
+            affectedBatchIds.add(String(b.id));
+        }
+    } catch (err) {
+        payload.logger?.warn?.(`[ai-translate] janitor: open-batch scan failed: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    for (const batchId of affectedBatchIds){
+        try {
+            await maybeTransitionBatch(payload, batchesSlug, unitsSlug, batchId, params.callbacks);
+        } catch (err) {
+            payload.logger?.warn?.(`[ai-translate] janitor: batch transition check failed for ${batchId}: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    return {
+        reset,
+        failed,
+        requeued,
+        thresholdMs
+    };
+}
+// ---------------------------------------------------------------------------
+// P99 read
+// ---------------------------------------------------------------------------
+async function readP99Cached(payload, usageSlug, now, ttlMs) {
+    const cached = P99_CACHE.get(payload);
+    if (cached && now() - cached.computedAt < ttlMs) {
+        return cached.p99Ms;
+    }
+    const p99Ms = await computeRecentP99(payload, usageSlug);
+    P99_CACHE.set(payload, {
+        p99Ms,
+        computedAt: now()
+    });
+    return p99Ms;
+}
+/**
+ * Read the most recent 1000 `translation-usage` rows and return the
+ * P99 of their `durationMs` values. Falls back to 0 (which lets the
+ * threshold floor dominate) when no data is available — fresh
+ * installs and test fixtures.
+ */ async function computeRecentP99(payload, usageSlug) {
+    try {
+        const result = await payload.find({
+            collection: usageSlug,
+            where: {
+                durationMs: {
+                    exists: true
+                }
+            },
+            limit: 1000,
+            sort: '-createdAt',
+            depth: 0,
+            overrideAccess: true
+        });
+        const rows = result.docs;
+        const durations = rows.map((r)=>r.durationMs ?? 0).filter((n)=>typeof n === 'number' && n > 0).sort((a, b)=>a - b);
+        if (durations.length === 0) return 0;
+        const idx = Math.min(durations.length - 1, Math.floor(durations.length * 0.99));
+        return durations[idx];
+    } catch  {
+        return 0;
+    }
+}

package/dist/tasks/translate-job-task.d.ts

@@ -0,0 +1,51 @@
+import type { TaskConfig } from 'payload';
+import type { TargetPolicy } from '../types.js';
+export declare const TRANSLATE_DOCUMENT_TASK_SLUG = "ai-translate-document";
+export declare const TRANSLATE_GLOBAL_TASK_SLUG = "ai-translate-global";
+type TranslateDocumentTaskInput = {
+    collection: string;
+    documentId: string;
+    jobId?: string;
+    targetLocales?: string[] | null;
+    previousDoc?: Record<string, unknown> | null;
+    draft?: boolean;
+    targetPolicy?: TargetPolicy;
+};
+type TranslateGlobalTaskInput = {
+    global: string;
+    jobId?: string;
+    targetLocales?: string[] | null;
+    previousDoc?: Record<string, unknown> | null;
+    targetPolicy?: TargetPolicy;
+};
+/**
+ * Builds the Payload task config for translating a collection
+ * document. Consumers don't see this — the plugin auto-registers it
+ * when `persistJobs: true`.
+ *
+ * `allowedCollections` is the plugin's configured `collections` list.
+ * Task inputs are queryable via Payload's REST API, so without this
+ * allowlist an attacker with write access to `payload-jobs` could
+ * enqueue a task targeting any collection (e.g. `users`) and the
+ * worker would execute it with `overrideAccess: true`. Empty list
+ * disables the check for back-compat with consumers wiring tasks
+ * manually.
+ */
+export declare function buildTranslateDocumentTask(allowedCollections?: readonly string[]): TaskConfig<{
+    input: TranslateDocumentTaskInput;
+    output: {
+        ok: true;
+    };
+}>;
+/**
+ * Sibling task config for translating a global. Same persistence
+ * contract — survives server restart. Same allowlist rationale as
+ * `buildTranslateDocumentTask`.
+ */
+export declare function buildTranslateGlobalTask(allowedGlobals?: readonly string[]): TaskConfig<{
+    input: TranslateGlobalTaskInput;
+    output: {
+        ok: true;
+    };
+}>;
+export {};

package/dist/tasks/translate-job-task.js

@@ -0,0 +1,154 @@
+import { translateDocument, translateGlobal } from '../api.js';
+export const TRANSLATE_DOCUMENT_TASK_SLUG = 'ai-translate-document';
+export const TRANSLATE_GLOBAL_TASK_SLUG = 'ai-translate-global';
+/**
+ * Payload jobs task that runs `translateDocument` from a durable queue
+ * entry rather than the in-memory coalescing closure. Surviving a
+ * server restart works because the task's input lives in the
+ * `payload-jobs` collection — when the process restarts, Payload's
+ * cron picks up un-completed jobs and re-runs the handler.
+ *
+ * The original after-change closure captured `req`, `doc`, and
+ * `previousDoc` directly. We can't serialize those across a restart,
+ * so the task input carries (collection, documentId, previousDoc-as-
+ * JSON, targetLocales, jobId, draft). The handler reads the live doc
+ * via the plugin's existing `findByIdNoFallback` inside
+ * `translateDocument`, so it always operates on current state.
+ */ const taskInputSchema = [
+    {
+        name: 'collection',
+        type: 'text',
+        required: true
+    },
+    {
+        name: 'documentId',
+        type: 'text',
+        required: true
+    },
+    {
+        name: 'jobId',
+        type: 'text'
+    },
+    /**
+   * Serialized as JSON so it round-trips through the jobs collection
+   * cleanly. May be `null` when the after-change hook detected a
+   * publish-transition and asked for a full re-translate.
+   */ {
+        name: 'targetLocales',
+        type: 'json'
+    },
+    {
+        name: 'previousDoc',
+        type: 'json'
+    },
+    {
+        name: 'draft',
+        type: 'checkbox',
+        defaultValue: true
+    },
+    {
+        name: 'targetPolicy',
+        type: 'text'
+    }
+];
+const taskInputSchemaGlobal = [
+    {
+        name: 'global',
+        type: 'text',
+        required: true
+    },
+    {
+        name: 'jobId',
+        type: 'text'
+    },
+    {
+        name: 'targetLocales',
+        type: 'json'
+    },
+    {
+        name: 'previousDoc',
+        type: 'json'
+    },
+    {
+        name: 'targetPolicy',
+        type: 'text'
+    }
+];
+/**
+ * Builds the Payload task config for translating a collection
+ * document. Consumers don't see this — the plugin auto-registers it
+ * when `persistJobs: true`.
+ *
+ * `allowedCollections` is the plugin's configured `collections` list.
+ * Task inputs are queryable via Payload's REST API, so without this
+ * allowlist an attacker with write access to `payload-jobs` could
+ * enqueue a task targeting any collection (e.g. `users`) and the
+ * worker would execute it with `overrideAccess: true`. Empty list
+ * disables the check for back-compat with consumers wiring tasks
+ * manually.
+ */ export function buildTranslateDocumentTask(allowedCollections = []) {
+    const allowed = new Set(allowedCollections);
+    return {
+        slug: TRANSLATE_DOCUMENT_TASK_SLUG,
+        label: 'AI translate — collection document',
+        inputSchema: taskInputSchema,
+        // No retries: the per-locale work inside translateDocument already
+        // applies the configured `retry` config to provider calls. A second
+        // outer attempt would double-bill on transient errors that already
+        // got their inner retries.
+        retries: 0,
+        handler: async ({ input, req })=>{
+            const typed = input;
+            if (allowed.size > 0 && !allowed.has(typed.collection)) {
+                throw new Error(`[ai-translate] Task input rejected: collection '${typed.collection}' is not in the plugin's allowed collections list.`);
+            }
+            await translateDocument(req.payload, {
+                collection: typed.collection,
+                id: typed.documentId,
+                jobId: typed.jobId,
+                targetLocales: typed.targetLocales ?? undefined,
+                previousDoc: typed.previousDoc ?? undefined,
+                draft: typed.draft ?? true,
+                targetPolicy: typed.targetPolicy,
+                req
+            });
+            return {
+                output: {
+                    ok: true
+                }
+            };
+        }
+    };
+}
+/**
+ * Sibling task config for translating a global. Same persistence
+ * contract — survives server restart. Same allowlist rationale as
+ * `buildTranslateDocumentTask`.
+ */ export function buildTranslateGlobalTask(allowedGlobals = []) {
+    const allowed = new Set(allowedGlobals);
+    return {
+        slug: TRANSLATE_GLOBAL_TASK_SLUG,
+        label: 'AI translate — global',
+        inputSchema: taskInputSchemaGlobal,
+        retries: 0,
+        handler: async ({ input, req })=>{
+            const typed = input;
+            if (allowed.size > 0 && !allowed.has(typed.global)) {
+                throw new Error(`[ai-translate] Task input rejected: global '${typed.global}' is not in the plugin's allowed globals list.`);
+            }
+            await translateGlobal(req.payload, {
+                global: typed.global,
+                jobId: typed.jobId,
+                targetLocales: typed.targetLocales ?? undefined,
+                previousDoc: typed.previousDoc ?? undefined,
+                targetPolicy: typed.targetPolicy,
+                req
+            });
+            return {
+                output: {
+                    ok: true
+                }
+            };
+        }
+    };
+}