@purposeinplay/payload-ai-translate 0.1.0

package/dist/lexical/serializer.js

@@ -0,0 +1,233 @@
+import { classifyNode } from './classifier.js';
+import { wrapFormat, wrapLink } from './placeholders.js';
+/**
+ * Walks an embedded Payload block (`type: 'block'` Lexical node, surfaced
+ * by `BlocksFeature`) in a deterministic order, emitting translation
+ * units for strings and recursing into nested Lexical roots / arrays /
+ * objects. Walk order MUST match `applyTranslatedBlockData` in the
+ * deserializer so blockId sequence stays in sync between serialize and
+ * deserialize passes.
+ *
+ * No full schema awareness yet — the BlocksFeature config isn't
+ * reachable from the extractor at runtime. As a defensive heuristic the
+ * walker skips strings that look like select-option values (short,
+ * lowercase identifiers like `info`, `warning`, `oneThird`) so the LLM
+ * doesn't translate them into invalid values that Payload's locale
+ * write validator would reject. The top-level blocks-field path
+ * (handled by `extractBlockFields` in `lib/content-extractor.ts`) uses
+ * full schema-aware gating.
+ */ export function serializeBlockData(data, units, nextBlockId, innerWalkNodes) {
+    for (const [key, value] of Object.entries(data)){
+        if (key === 'id' || key === 'blockType' || key === 'blockName') continue;
+        if (typeof value === 'string') {
+            if (value.trim().length > 0 && !isLikelySelectOption(value)) {
+                units.push({
+                    blockId: nextBlockId(),
+                    text: value,
+                    sourceNode: {
+                        type: 'block-field',
+                        text: value
+                    }
+                });
+            }
+            continue;
+        }
+        if (isLexicalRootValue(value)) {
+            // Nested richText inside the block — recurse with the existing walker
+            // so its inline formatting placeholders stay consistent.
+            innerWalkNodes(value.root.children ?? []);
+            continue;
+        }
+        if (Array.isArray(value)) {
+            for (const item of value){
+                if (item && typeof item === 'object' && !Array.isArray(item)) {
+                    serializeBlockData(item, units, nextBlockId, innerWalkNodes);
+                } else if (typeof item === 'string' && item.trim().length > 0) {
+                    units.push({
+                        blockId: nextBlockId(),
+                        text: item,
+                        sourceNode: {
+                            type: 'block-field-array',
+                            text: item
+                        }
+                    });
+                }
+            }
+            continue;
+        }
+        if (value && typeof value === 'object') {
+            serializeBlockData(value, units, nextBlockId, innerWalkNodes);
+        }
+    }
+}
+function isLexicalRootValue(value) {
+    if (value === null || typeof value !== 'object') return false;
+    const obj = value;
+    return 'root' in obj && obj.root !== null && typeof obj.root === 'object';
+}
+/**
+ * Heuristic: a short identifier-shaped string is almost certainly a
+ * `select` / `radio` option value (e.g. `info`, `warning`, `oneThird`,
+ * `primary`, `dark-mode`) — translating it produces a value Payload's
+ * locale-write validator will reject. We can't see the block schema
+ * from the embedded-Lexical context so this filter is the next best
+ * thing.
+ *
+ * Tradeoff: a real translatable word like `info` (English `info` →
+ * Spanish `info`/`información`) stops being translated inside embedded
+ * Lexical blocks. Acceptable: nobody writes prose into select-typed
+ * fields, and consumers who DO want their short labels translated can
+ * still call the manual translate API or use a top-level blocks field
+ * (which has full schema-aware gating).
+ *
+ * Matches:
+ *   - `^[a-z][a-z0-9_-]{0,20}$` — lowercase identifier 1-21 chars
+ *   - `^[a-z]+(?:[A-Z][a-z]+)+$` — camelCase 6-30 chars (e.g. oneThird)
+ */ function isLikelySelectOption(text) {
+    const t = text.trim();
+    if (t.length === 0 || t.length > 30) return false;
+    // No spaces allowed in select option values.
+    if (/\s/.test(t)) return false;
+    // Lowercase identifier
+    if (/^[a-z][a-z0-9_-]{0,20}$/.test(t)) return true;
+    // camelCase identifier
+    if (/^[a-z]+(?:[A-Z][a-z0-9]+)+$/.test(t) && t.length <= 30) return true;
+    return false;
+}
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+export function serializeLexicalTree(root, registeredNodes) {
+    const units = [];
+    let counter = 0;
+    function nextBlockId() {
+        return `blk_${counter++}`;
+    }
+    function walkNodes(nodes) {
+        for (const node of nodes){
+            const classified = classifyNode(node, registeredNodes);
+            switch(classified.classification){
+                case 'recurse':
+                    {
+                        // Block-level nodes that contain inline children
+                        if (node.type === 'paragraph' || node.type === 'heading' || node.type === 'quote' || node.type === 'listitem') {
+                            const inlineText = serializeInlineChildren(node.children ?? []);
+                            if (inlineText.trim().length > 0) {
+                                units.push({
+                                    blockId: nextBlockId(),
+                                    text: inlineText,
+                                    sourceNode: node
+                                });
+                            }
+                            // Also walk children for nested lists inside listitems
+                            if (node.type === 'listitem' && node.children) {
+                                for (const child of node.children){
+                                    if (child.type === 'list') {
+                                        walkNodes(child.children ?? []);
+                                    }
+                                }
+                            }
+                        } else if (node.type === 'list') {
+                            // Lists themselves recurse into their children (listitems)
+                            walkNodes(node.children ?? []);
+                        } else {
+                            // Other recurse nodes (e.g. link called at top level, autolink)
+                            walkNodes(node.children ?? []);
+                        }
+                        break;
+                    }
+                case 'translate-attr':
+                    {
+                        if (node.type === 'upload') {
+                            const alt = getNodeAlt(node);
+                            if (alt) {
+                                units.push({
+                                    blockId: nextBlockId(),
+                                    text: alt,
+                                    sourceNode: node
+                                });
+                            }
+                        } else if (classified.translatableAttributes) {
+                            // Custom registered nodes
+                            for (const attr of classified.translatableAttributes){
+                                const value = getNestedAttr(node, attr);
+                                if (typeof value === 'string' && value.trim().length > 0) {
+                                    units.push({
+                                        blockId: nextBlockId(),
+                                        text: value,
+                                        sourceNode: node
+                                    });
+                                }
+                            }
+                        }
+                        break;
+                    }
+                case 'block':
+                    {
+                        // Lexical's BlocksFeature embeds Payload blocks inline. Walk
+                        // the block's `fields` and emit translation units the same way
+                        // we'd handle them at the top-level blocks-array path. Inner
+                        // Lexical roots (e.g. `banner.content` richText) recurse back
+                        // through `walkNodes` so their inline placeholders stay in
+                        // sync with the rest of the document's blockId sequence.
+                        const blockFields = node.fields ?? {};
+                        serializeBlockData(blockFields, units, nextBlockId, walkNodes);
+                        break;
+                    }
+                case 'skip':
+                    break;
+            }
+        }
+    }
+    walkNodes(root.root.children ?? []);
+    return units;
+}
+// ---------------------------------------------------------------------------
+// Inline serialization
+// ---------------------------------------------------------------------------
+function serializeInlineChildren(children) {
+    return children.map((child)=>serializeInlineNode(child)).join('');
+}
+function serializeInlineNode(node) {
+    switch(node.type){
+        case 'text':
+            {
+                const text = node.text ?? '';
+                const format = node.format ?? 0;
+                return format > 0 ? wrapFormat(text, format) : text;
+            }
+        case 'link':
+            {
+                const linkFields = node.fields ?? {};
+                const innerText = serializeInlineChildren(node.children ?? []);
+                return wrapLink(innerText, linkFields);
+            }
+        case 'linebreak':
+            return '\n';
+        case 'list':
+            // Nested lists inside listitems are handled at the block walk level
+            return '';
+        default:
+            // For any unknown inline node, try to serialize its children
+            if (node.children) {
+                return serializeInlineChildren(node.children);
+            }
+            return '';
+    }
+}
+// ---------------------------------------------------------------------------
+// Attribute helpers
+// ---------------------------------------------------------------------------
+function getNodeAlt(node) {
+    const fromFields = node.fields?.alt;
+    const fromNode = node.alt;
+    const alt = fromFields ?? fromNode;
+    return typeof alt === 'string' ? alt : undefined;
+}
+function getNestedAttr(node, attr) {
+    // Check fields first, then the node itself
+    if (node.fields && attr in node.fields) {
+        return node.fields[attr];
+    }
+    return node[attr];
+}

package/dist/lexical/types.d.ts

@@ -0,0 +1,32 @@
+export type LexicalNode = {
+    type: string;
+    children?: LexicalNode[];
+    text?: string;
+    format?: number;
+    tag?: string;
+    listType?: string;
+    fields?: Record<string, unknown>;
+    [key: string]: unknown;
+};
+export type LexicalRoot = {
+    root: LexicalNode;
+};
+export type NodeClassification = 'recurse' | 'translate-attr' | 'skip'
+/**
+ * Lexical's `BlocksFeature` embeds Payload blocks inline (`type: 'block'`,
+ * `fields: { blockType, ... }`). Treated as a separate classification so
+ * the serializer/deserializer can walk the block's fields the same way
+ * the top-level `extractBlockFields` walker does, instead of treating
+ * the embedded block as opaque.
+ */
+ | 'block';
+export type ClassifiedNode = {
+    node: LexicalNode;
+    classification: NodeClassification;
+    translatableAttributes?: string[];
+};
+export type SerializedUnit = {
+    blockId: string;
+    text: string;
+    sourceNode: LexicalNode;
+};

package/dist/lexical/types.js

@@ -0,0 +1 @@
+export { };

package/dist/lib/auth-diagnostics.d.ts

@@ -0,0 +1,14 @@
+import type { PayloadRequest } from 'payload';
+/**
+ * Build a one-line diagnostic explaining *why* a request was rejected with
+ * 401 by `if (!req.user) return 401` gates. Payload's `extractJWT.cookie`
+ * silently returns `null` when the request lacks an allowlisted `Origin`
+ * AND lacks an acceptable `Sec-Fetch-Site` header — most commonly because
+ * the page was opened via Next's `Network:` URL (`127.0.x.x`) instead of
+ * `localhost`, which the consumer's `csrf` allowlist doesn't include.
+ *
+ * Surfacing the cookie/Origin/Sec-Fetch-Site triple in the response body
+ * lets the next dev land on the root cause without trawling Payload core.
+ * Kept terse — adds maybe 80 bytes to a 401 response.
+ */
+export declare function describeAuthRejection(req: PayloadRequest): string;

package/dist/lib/auth-diagnostics.js

@@ -0,0 +1,19 @@
+/**
+ * Build a one-line diagnostic explaining *why* a request was rejected with
+ * 401 by `if (!req.user) return 401` gates. Payload's `extractJWT.cookie`
+ * silently returns `null` when the request lacks an allowlisted `Origin`
+ * AND lacks an acceptable `Sec-Fetch-Site` header — most commonly because
+ * the page was opened via Next's `Network:` URL (`127.0.x.x`) instead of
+ * `localhost`, which the consumer's `csrf` allowlist doesn't include.
+ *
+ * Surfacing the cookie/Origin/Sec-Fetch-Site triple in the response body
+ * lets the next dev land on the root cause without trawling Payload core.
+ * Kept terse — adds maybe 80 bytes to a 401 response.
+ */ export function describeAuthRejection(req) {
+    const cookie = req.headers?.get?.('cookie') ?? '';
+    const hasCookie = cookie.includes('payload-token');
+    const origin = req.headers?.get?.('origin') ?? null;
+    const sfs = req.headers?.get?.('sec-fetch-site') ?? null;
+    if (!hasCookie) return 'No payload-token cookie sent.';
+    return `Cookie present but rejected. Origin=${origin ?? '(none)'} Sec-Fetch-Site=${sfs ?? '(none)'}. Add Origin to payload.config.csrf or open the admin via an allowlisted host (commonly the "Network:" URL printed by Next dev is the trigger).`;
+}

package/dist/lib/batch-counts.d.ts

@@ -0,0 +1,58 @@
+import type { Payload } from 'payload';
+/**
+ * Source-of-truth counts for one or more bulk-translate batches.
+ *
+ * The plugin used to read `batch.completedUnits` / `failedUnits` /
+ * `skippedUnits` straight off the batch row — populated by atomic SQL
+ * bumps in the worker (`completed_units = completed_units + 1` on each
+ * unit transition). That worked for the happy path but drifted on:
+ *   - Retries (`retry-failed` flips `failed → pending` without
+ *     decrementing the previous failure's contribution).
+ *   - Admin SDK interventions (manually marking a `success` unit as
+ *     `failed`, then retrying — the original success increment stays
+ *     and a new one fires when the retry succeeds, so `completedUnits`
+ *     can exceed `totalUnits`).
+ *   - Cancel paths (`pending → skipped` writes the unit row but
+ *     skipping bumps the counter via a separate path that can race).
+ *
+ * Real fix per editor feedback: API endpoints never trust the cached
+ * counters. They always recompute from the underlying
+ * `bulk_translate_units` table. The cached counters stay as an
+ * internal optimization for the worker's "is this batch done?"
+ * transition check, but they're no longer authoritative.
+ *
+ * One query per call regardless of how many batches: a single `find`
+ * with `where { batchId: { in: [...] } }` and a narrow `select`
+ * projects (`batchId`, `status`) only. JS aggregates the counts.
+ * Realistic plugin batches (≤1000 units each, ~10 batches per list
+ * page) finish well under 100ms.
+ *
+ * For very large batches (10k+ units each), this approach still works
+ * but a switch to `drizzle.execute(sql\`SELECT batch_id_id, status,
+ * COUNT(*) ... GROUP BY 1, 2\`)` would be O(1) on the wire. We don't
+ * ship that today because drizzle is an optional peer dep.
+ */
+export type LiveBatchCounts = {
+    total: number;
+    pending: number;
+    running: number;
+    /** Worker-`success` count. Exposed to the API as `completed`. */
+    completed: number;
+    failed: number;
+    skipped: number;
+    reverted: number;
+};
+/**
+ * Compute fresh status counts for every batch id passed in.
+ *
+ * Returns a Map keyed by stringified batch id. Batches with no units
+ * (newly enqueued or anomalously empty) map to the zero record so
+ * callers don't have to null-check.
+ */
+export declare function computeLiveBatchCounts(payload: Payload, unitsSlug: string, batchIds: ReadonlyArray<string | number>): Promise<Map<string, LiveBatchCounts>>;
+/**
+ * Convenience wrapper for the single-batch case (status endpoint,
+ * worker transition check). Returns the zero record if the batch has
+ * no units (anomalous but should never crash).
+ */
+export declare function computeLiveBatchCount(payload: Payload, unitsSlug: string, batchId: string): Promise<LiveBatchCounts>;

package/dist/lib/batch-counts.js

@@ -0,0 +1,105 @@
+import { getDrizzle, slugToTable } from './bulk-translate-migrations.js';
+const ZERO_COUNTS = {
+    total: 0,
+    pending: 0,
+    running: 0,
+    completed: 0,
+    failed: 0,
+    skipped: 0,
+    reverted: 0
+};
+/**
+ * Compute fresh status counts for every batch id passed in.
+ *
+ * Returns a Map keyed by stringified batch id. Batches with no units
+ * (newly enqueued or anomalously empty) map to the zero record so
+ * callers don't have to null-check.
+ */ export async function computeLiveBatchCounts(payload, unitsSlug, batchIds) {
+    const out = new Map();
+    if (batchIds.length === 0) return out;
+    // Seed every requested batchId so the caller can lookup without
+    // checking for undefined.
+    for (const id of batchIds){
+        out.set(String(id), {
+            ...ZERO_COUNTS
+        });
+    }
+    const tally = (bid, status, n)=>{
+        const counts = out.get(bid);
+        if (!counts) return;
+        counts.total += n;
+        if (status === 'pending') counts.pending += n;
+        else if (status === 'running') counts.running += n;
+        else if (status === 'success') counts.completed += n;
+        else if (status === 'failed') counts.failed += n;
+        else if (status === 'skipped') counts.skipped += n;
+        else if (status === 'reverted') counts.reverted += n;
+    };
+    // Fast path: one GROUP BY on Postgres — O(1) on the wire no matter
+    // how many units the batches hold. This matters far more than the
+    // original comment assumed: `depth: 0` does NOT drop the
+    // `pre_run_snapshot` jsonb column, so the row-fetch path was moving
+    // the entire snapshot payload (tens of KB × every unit) through
+    // Postgres TOAST + JSON.parse on EVERY Hub poll. On an 11.5k-unit
+    // batch that measured 7-12s per poll and was a driver of the
+    // 2026-06-10 prod outage.
+    if (batchIds.every((id)=>/^\d+$/.test(String(id)))) {
+        const db = getDrizzle(payload);
+        if (db) {
+            try {
+                const res = await db.execute(`SELECT batch_id_id AS bid, status, count(*) AS n
+             FROM ${slugToTable(unitsSlug)}
+            WHERE batch_id_id IN (${batchIds.map((id)=>String(id)).join(', ')})
+            GROUP BY 1, 2`);
+                for (const row of res?.rows ?? []){
+                    tally(String(row.bid), String(row.status), Number(row.n ?? 0));
+                }
+                return out;
+            } catch (err) {
+                payload.logger?.warn?.(`[ai-translate] computeLiveBatchCounts: grouped-count SQL failed, falling back to row scan: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+    }
+    // Fallback (non-Postgres adapters / non-numeric ids): paginated row
+    // scan, projected to just (batchId, status) so the snapshot jsonb
+    // never leaves the database.
+    let page = 1;
+    const limit = 5000;
+    // eslint-disable-next-line no-constant-condition
+    while(true){
+        const result = await payload.find({
+            collection: unitsSlug,
+            where: {
+                batchId: {
+                    in: batchIds.map((id)=>String(id))
+                }
+            },
+            page,
+            limit,
+            depth: 0,
+            select: {
+                batchId: true,
+                status: true
+            },
+            overrideAccess: true
+        });
+        for (const doc of result.docs){
+            tally(String(doc.batchId ?? ''), String(doc.status ?? ''), 1);
+        }
+        if (!result.hasNextPage) break;
+        page += 1;
+    }
+    return out;
+}
+/**
+ * Convenience wrapper for the single-batch case (status endpoint,
+ * worker transition check). Returns the zero record if the batch has
+ * no units (anomalous but should never crash).
+ */ export async function computeLiveBatchCount(payload, unitsSlug, batchId) {
+    const map = await computeLiveBatchCounts(payload, unitsSlug, [
+        batchId
+    ]);
+    return map.get(batchId) ?? {
+        ...ZERO_COUNTS
+    };
+}

package/dist/lib/bulk-translate-migrations.d.ts

@@ -0,0 +1,92 @@
+import type { Payload } from 'payload';
+/**
+ * Idempotent schema-shaping helpers for the bulk-translate engine
+ * (PR2 / v1.2.0). Payload's auto-migration generator handles plain
+ * column / index changes from collection definitions, but the bulk
+ * engine needs:
+ *
+ *   1. A PARTIAL UNIQUE INDEX on
+ *      `bulk_translate_units (collection, document_id, locale)
+ *      WHERE status IN ('pending','running')`. This enforces the
+ *      F-DA-TOCTOU dedup invariant at the DB level (a second worker
+ *      cannot insert / upsert a unit for an already-active locale).
+ *      Payload's index generator does not emit `WHERE`-clauses, so
+ *      this is hand-written SQL.
+ *
+ *   2. A composite UNIQUE constraint on
+ *      `translation_daily_spend (date, consumer_key)` to back the
+ *      cap utility's atomic upsert. Payload's index generator can
+ *      emit this from the collection definition, but on existing
+ *      installs the rename may not propagate — calling this helper
+ *      ensures it.
+ *
+ * Both run via `payload.db.drizzle.execute(sql\`...\`)`. SQL uses
+ * `CREATE ... IF NOT EXISTS` so the same call across many boots is
+ * a no-op after the first success.
+ *
+ * Two ways to use:
+ *   - Add to consumer's migration file (recommended for clean
+ *     migration history): import and call from inside an `up`
+ *     function.
+ *   - Call from the plugin's `onInit` (when `bulk.enabled` is true) —
+ *     idempotent boot-time self-healing. Trade-off: errors at boot
+ *     surface as plugin failures rather than as migration failures.
+ */
+export interface EnsureBulkTranslateSchemaOptions {
+    /**
+     * Override the collection slugs if the consumer customised them
+     * via plugin config. Defaults pull from the collection modules.
+     */
+    bulkTranslateUnitsSlug?: string;
+    translationDailySpendSlug?: string;
+}
+export declare function ensureBulkTranslateSchema(payload: Payload, opts?: EnsureBulkTranslateSchemaOptions): Promise<void>;
+/**
+ * One-off data migration: prefix existing `ai-translate-meta`
+ * source-hash rows with the `v1:` version tag (R2 / F-ENG-07).
+ *
+ * Without this, the first bulk run after a plugin upgrade would treat
+ * every previously-translated field as "source changed" — burning
+ * tokens to re-translate everything that was already in sync.
+ *
+ * Idempotent: rows that already start with `v1:` are skipped. Safe
+ * to run repeatedly. Consumer should run ONCE per deploy that bumps
+ * past 1.1.16. Returns a count of rows updated.
+ */
+export declare function migrateHashesToV1Prefix(payload: Payload, opts?: {
+    metaSlug?: string;
+    dryRun?: boolean;
+}): Promise<{
+    scanned: number;
+    updated: number;
+    alreadyMigrated: number;
+}>;
+/**
+ * v1.2.4 data migration: align `bulk_translate_batches.status` vocabulary
+ * on `'success'` (matching the units collection + the
+ * `BulkTranslateBatchStatus` TS union). Pre-1.2.4 the schema declared
+ * `'completed'` as the terminal-success value while the type union used
+ * `'success'`, so badges, filter chips, and the Hub's "recently
+ * terminal" query were all silently dead for successful runs. Worker
+ * + schema now write `'success'`; this helper backfills existing rows.
+ *
+ * Idempotent: only updates rows where status = 'completed'. Safe to run
+ * repeatedly; subsequent calls are no-ops.
+ *
+ * Consumer wiring:
+ *   - Add to a forward migration's up() OR
+ *   - Call from plugin onInit() once per deploy (boot-time self-heal).
+ */
+export declare function migrateBatchStatusVocabulary(payload: Payload, opts?: {
+    batchesSlug?: string;
+    dryRun?: boolean;
+}): Promise<{
+    scanned: number;
+    updated: number;
+    alreadyAligned: number;
+}>;
+export declare function slugToTable(slug: string): string;
+export type Drizzle = {
+    execute(sql: string): Promise<unknown>;
+};
+export declare function getDrizzle(payload: Payload): Drizzle | null;