@purposeinplay/payload-ai-translate 0.1.0

package/dist/endpoints/estimate.js

@@ -0,0 +1,237 @@
+import { describeAuthRejection } from '../lib/auth-diagnostics.js';
+import { collectBlocksConfig, extractTranslationUnits } from '../lib/content-extractor.js';
+import { getEffectiveExcludePatternsForSurface, getExcludedFieldPaths, isPathExcluded } from '../lib/effective-locales.js';
+import { resolveTranslatableFields } from '../lib/field-resolver.js';
+import { findByIdNoFallback } from '../lib/payload-read.js';
+/**
+ * Heuristic: would the validator mark this string as a verbatim echo if the
+ * LLM returned it unchanged? Used to subtract "likely skipped" chars from
+ * the billable estimate so the editor sees an accurate cost.
+ *
+ * Conservative — false positives would just make the estimate look smaller
+ * than reality, but we'd rather over-estimate than under-estimate. The
+ * actual validator runs at translation time; this is a pre-flight guess.
+ */ function isLikelyVerbatim(text) {
+    const trimmed = text.trim();
+    if (trimmed.length < 3) return true; // too short to translate meaningfully
+    if (/^https?:\/\//i.test(trimmed)) return true; // URLs
+    if (/^[\d.\s,-]+$/.test(trimmed)) return true; // pure numeric / dates
+    if (/^[A-Z0-9_-]+$/.test(trimmed) && trimmed.length < 12) return true; // SKU / code
+    return false;
+}
+function resolveContext(arg, url) {
+    if (typeof arg === 'object' && arg !== null) return arg;
+    try {
+        const segments = new URL(url).pathname.split('/').filter(Boolean);
+        const apiIndex = segments.indexOf('api');
+        if (apiIndex >= 0) {
+            if (segments[apiIndex + 1] === 'globals' && segments[apiIndex + 2]) {
+                return {
+                    kind: 'global',
+                    slug: segments[apiIndex + 2]
+                };
+            }
+            if (segments[apiIndex + 1]) {
+                return {
+                    kind: 'collection',
+                    slug: arg ?? segments[apiIndex + 1]
+                };
+            }
+        }
+    } catch  {
+    // fall through
+    }
+    return {
+        kind: 'collection',
+        slug: typeof arg === 'string' ? arg : ''
+    };
+}
+export const getEstimateHandler = (arg)=>async (req)=>{
+        try {
+            if (!req.user) {
+                return Response.json({
+                    error: 'Unauthorized',
+                    diagnostic: describeAuthRejection(req)
+                }, {
+                    status: 401
+                });
+            }
+            const config = req.payload.config?.custom?.aiTranslate;
+            if (config.access?.translate) {
+                const allowed = await config.access.translate({
+                    req
+                });
+                if (!allowed) {
+                    return Response.json({
+                        error: 'Forbidden'
+                    }, {
+                        status: 403
+                    });
+                }
+            }
+            let body;
+            try {
+                body = await req.json();
+            } catch  {
+                return Response.json({
+                    error: 'Invalid JSON body'
+                }, {
+                    status: 400
+                });
+            }
+            const { ids, targetLocales: bodyLocales, draft: bodyDraft } = body;
+            const targetLocales = bodyLocales ?? config.targetLocales;
+            const draft = bodyDraft ?? true;
+            const ctx = resolveContext(arg, req.url ?? '');
+            let totalCharacters = 0;
+            const allItems = [];
+            let documentCount = 0;
+            const excludePatterns = await getEffectiveExcludePatternsForSurface(req.payload, config, ctx.slug);
+            // Honour admin-configured exclusions so the estimate matches what
+            // the translate pipeline will actually send. Without this, the
+            // editor sees an inflated cost and may bail unnecessarily.
+            const adminExcludedPaths = await getExcludedFieldPaths(req.payload, config, ctx.slug);
+            if (ctx.kind === 'global') {
+                const globals = req.payload.config?.globals ?? [];
+                const globalConfig = globals.find((g)=>g.slug === ctx.slug);
+                if (!globalConfig) {
+                    return Response.json({
+                        error: `Global "${ctx.slug}" not found`
+                    }, {
+                        status: 404
+                    });
+                }
+                // Globals don't have drafts in Payload's model, so `draft` is a no-op
+                // here. Read the source-locale doc with no fallback so we estimate
+                // against the actual source content.
+                const doc = await req.payload.findGlobal({
+                    slug: ctx.slug,
+                    locale: config.sourceLocale,
+                    fallbackLocale: null
+                });
+                if (doc) {
+                    let translatableFields = resolveTranslatableFields(globalConfig.fields);
+                    if (adminExcludedPaths.size > 0) {
+                        translatableFields = translatableFields.filter((f)=>!isPathExcluded(f.path, adminExcludedPaths));
+                    }
+                    const blocksConfig = collectBlocksConfig(globalConfig.fields);
+                    const units = extractTranslationUnits(doc, translatableFields, excludePatterns, blocksConfig);
+                    for (const unit of units){
+                        totalCharacters += unit.text.length;
+                        allItems.push({
+                            id: unit.id,
+                            text: unit.text,
+                            kind: unit.kind
+                        });
+                    }
+                    documentCount = 1;
+                }
+            } else {
+                if (!ids || !Array.isArray(ids) || ids.length === 0) {
+                    return Response.json({
+                        error: '"ids" must be a non-empty array'
+                    }, {
+                        status: 400
+                    });
+                }
+                const collections = req.payload.config?.collections ?? [];
+                const collectionConfig = collections.find((c)=>c.slug === ctx.slug);
+                if (!collectionConfig) {
+                    return Response.json({
+                        error: `Collection "${ctx.slug}" not found`
+                    }, {
+                        status: 404
+                    });
+                }
+                const collectionFields = collectionConfig.fields;
+                let translatableFields = resolveTranslatableFields(collectionFields);
+                if (adminExcludedPaths.size > 0) {
+                    translatableFields = translatableFields.filter((f)=>!isPathExcluded(f.path, adminExcludedPaths));
+                }
+                const blocksConfig = collectBlocksConfig(collectionFields);
+                for (const docId of ids){
+                    const doc = await findByIdNoFallback(req.payload, ctx.slug, docId, config.sourceLocale, {
+                        draft
+                    });
+                    if (!doc) continue;
+                    const units = extractTranslationUnits(doc, translatableFields, excludePatterns, blocksConfig);
+                    for (const unit of units){
+                        totalCharacters += unit.text.length;
+                        allItems.push({
+                            id: unit.id,
+                            text: unit.text,
+                            kind: unit.kind
+                        });
+                    }
+                }
+                documentCount = ids.length;
+            }
+            // Tally how many characters we expect the validator to mark as
+            // `skipped` (verbatim echo). The estimate stays honest by separating
+            // billable from skipped, so the editor isn't surprised when "152 chars"
+            // turns into a smaller actual LLM call.
+            let likelySkippedCharacters = 0;
+            for (const item of allItems){
+                if (isLikelyVerbatim(item.text)) {
+                    likelySkippedCharacters += item.text.length;
+                }
+            }
+            const billableCharacters = totalCharacters - likelySkippedCharacters;
+            // Token estimation. The previous formula (`billable / 3.5`) under-
+            // counted by ~7-30× because it ignored:
+            // 1) System prompt + few-shot overhead — the prompt template adds
+            //    ~SYSTEM_PROMPT_TOKENS regardless of input size.
+            // 2) Output tokens — most LLMs return responses of length similar to
+            //    or slightly larger than the input (translated text is ~1.0-1.2×
+            //    source for most language pairs).
+            // 3) Per-batch repetition — the prompt is sent once per locale.
+            //
+            // Result: a more realistic projection that lines up within ~2× of
+            // billed cost for the typical case rather than being 30× off.
+            const SYSTEM_PROMPT_TOKENS_PER_LOCALE_PASS = 500;
+            const OUTPUT_TO_INPUT_RATIO = 1.1; // tend to be slightly longer
+            const estimatedInputTokens = Math.ceil(billableCharacters / 3.5);
+            const estimatedOutputTokens = Math.ceil(estimatedInputTokens * OUTPUT_TO_INPUT_RATIO);
+            const overheadTokens = SYSTEM_PROMPT_TOKENS_PER_LOCALE_PASS * targetLocales.length;
+            const estimatedTokens = (estimatedInputTokens + estimatedOutputTokens) * targetLocales.length + overheadTokens;
+            let estimatedCostUsd;
+            if (config.provider.estimate) {
+                // Cost estimate is based only on billable items — skipped strings
+                // never reach the provider so they don't bill.
+                const billableItems = allItems.filter((i)=>!isLikelyVerbatim(i.text));
+                const mockRequest = {
+                    items: billableItems,
+                    sourceLocale: config.sourceLocale,
+                    targetLocale: targetLocales[0] ?? '',
+                    context: {
+                        collectionSlug: ctx.slug,
+                        fieldPath: '*'
+                    }
+                };
+                const estimate = await config.provider.estimate(mockRequest);
+                estimatedCostUsd = estimate.estimatedCostUsd;
+            }
+            // Multiply by locale count (one translation pass per target locale)
+            if (estimatedCostUsd !== undefined) {
+                estimatedCostUsd = estimatedCostUsd * targetLocales.length;
+            }
+            return Response.json({
+                totalCharacters,
+                billableCharacters,
+                likelySkippedCharacters,
+                estimatedTokens,
+                estimatedInputTokens,
+                estimatedOutputTokens,
+                estimatedCostUsd,
+                documentCount,
+                localeCount: targetLocales.length
+            });
+        } catch (error) {
+            const message = error instanceof Error ? error.message : 'Internal server error';
+            return Response.json({
+                error: message
+            }, {
+                status: 500
+            });
+        }
+    };

package/dist/endpoints/progress.d.ts

@@ -0,0 +1,2 @@
+import type { PayloadHandler } from 'payload';
+export declare const getProgressHandler: (collectionSlug?: string) => PayloadHandler;

package/dist/endpoints/progress.js

@@ -0,0 +1,314 @@
+import { describeAuthRejection } from '../lib/auth-diagnostics.js';
+import { getActiveJobForDoc, getJob, subscribe, subscribeToDoc } from '../lib/progress-store.js';
+const SSE_HEADERS = {
+    'Content-Type': 'text/event-stream',
+    'Cache-Control': 'no-cache',
+    Connection: 'keep-alive'
+};
+/**
+ * Long-lived stream lifetime for the `?docId=X` "wait for any job" mode.
+ * After this, the stream closes and the browser auto-reconnects (the
+ * `retry: 30000` directive paces idle reconnects). 5 minutes balances:
+ * - Don't expire mid-edit-session.
+ * - Don't pile up forever per editor when nothing's happening.
+ */ const DOC_STREAM_TTL_MS = 5 * 60_000;
+/**
+ * Heartbeat keeps the SSE connection alive across proxies/load balancers
+ * that idle-time TCP. Sent as an SSE comment (`:hb`), which clients
+ * ignore but counts as activity.
+ */ const HEARTBEAT_MS = 25_000;
+/**
+ * Hold-open lifetime for an active-job stream (`?jobId=X` or `?docId=X`
+ * that resolved to a job). Jobs typically finish in <30s; 2min is a
+ * generous ceiling for long batch translations.
+ */ const JOB_STREAM_TTL_MS = 120_000;
+function jobToEvent(job) {
+    return {
+        jobId: job.jobId,
+        completed: [
+            ...job.completedLocales
+        ],
+        failed: [
+            ...job.failedLocales
+        ],
+        total: job.totalLocales,
+        status: job.status
+    };
+}
+export const getProgressHandler = (collectionSlug)=>(req)=>{
+        if (!req.user) {
+            return Response.json({
+                error: 'Unauthorized',
+                diagnostic: describeAuthRejection(req)
+            }, {
+                status: 401
+            });
+        }
+        const url = new URL(req.url);
+        const jobIdParam = url.searchParams.get('jobId');
+        const docIdParam = url.searchParams.get('docId');
+        // Fast path: explicit jobId — stream that job's events directly.
+        if (jobIdParam) {
+            const job = getJob(jobIdParam);
+            if (job) {
+                return new Response(buildJobStream(job.jobId, req.signal), {
+                    headers: SSE_HEADERS
+                });
+            }
+            // Unknown jobId — fall through to idle close (client will reconnect
+            // and may resolve a different job by then).
+            return idleResponse();
+        }
+        // docId path: subscribe to "any active or future job for this doc".
+        if (docIdParam) {
+            const slug = collectionSlug ?? resolveSlugFromUrl(req.url ?? '');
+            if (!slug) return idleResponse();
+            const existing = getActiveJobForDoc(slug, docIdParam);
+            if (existing) {
+                // Existing in-flight job — stream it.
+                return new Response(buildJobStream(existing.jobId, req.signal), {
+                    headers: SSE_HEADERS
+                });
+            }
+            // No active job. Hold the stream open and wait for `createJob` to
+            // fire for this (slug, docId). When it does, push the event and
+            // hand off to the per-job stream pattern (replay + subscribe to
+            // future updates).
+            return new Response(buildDocStream(slug, docIdParam, req.signal), {
+                headers: SSE_HEADERS
+            });
+        }
+        // Neither param — return idle and close. Same shape as before to
+        // keep the client's parser happy.
+        return idleResponse();
+    };
+/**
+ * Idle-close response. Used when the request lacks both `jobId` and
+ * `docId`, or when an explicit `jobId` references an unknown job.
+ * `retry: 30000` paces the browser's auto-reconnect.
+ */ function idleResponse() {
+    const body = `retry: 30000\nevent: progress\ndata: ${JSON.stringify({
+        jobId: '',
+        completed: [],
+        failed: [],
+        total: 0,
+        status: 'idle'
+    })}\n\n`;
+    return new Response(body, {
+        headers: {
+            'Content-Type': 'text/event-stream',
+            'Cache-Control': 'no-cache'
+        }
+    });
+}
+/**
+ * Build a stream that subscribes to an existing job and pushes its
+ * events until completion (+ a 2s grace period so the client sees the
+ * final state before the connection drops).
+ */ function buildJobStream(jobId, reqSignal) {
+    let unsubscribe;
+    let timeout;
+    let heartbeat;
+    return new ReadableStream({
+        start (controller) {
+            const encoder = new TextEncoder();
+            controller.enqueue(encoder.encode('retry: 30000\n\n'));
+            const tryClose = ()=>{
+                try {
+                    controller.close();
+                } catch  {
+                // already closed
+                }
+            };
+            const send = (event)=>{
+                // Only `completed` and `failed` are terminal — `idle` means
+                // "waiting for a job", `running` means "in flight". Don't close
+                // the stream on idle or running events.
+                const isTerminal = event.status === 'completed' || event.status === 'failed';
+                const eventType = isTerminal ? 'complete' : 'progress';
+                try {
+                    controller.enqueue(encoder.encode(`event: ${eventType}\ndata: ${JSON.stringify(event)}\n\n`));
+                } catch  {
+                    // controller closed mid-write
+                    return;
+                }
+                if (isTerminal) {
+                    cleanup();
+                    // Brief grace so the client paints the final state before close.
+                    setTimeout(tryClose, 2000);
+                }
+            };
+            const cleanup = ()=>{
+                if (unsubscribe) {
+                    unsubscribe();
+                    unsubscribe = undefined;
+                }
+                if (timeout) {
+                    clearTimeout(timeout);
+                    timeout = undefined;
+                }
+                if (heartbeat) {
+                    clearInterval(heartbeat);
+                    heartbeat = undefined;
+                }
+            };
+            // Replay current state immediately.
+            const currentJob = getJob(jobId);
+            if (currentJob) {
+                send(jobToEvent(currentJob));
+                if (currentJob.status !== 'running') return;
+            }
+            unsubscribe = subscribe(jobId, send);
+            heartbeat = setInterval(()=>{
+                try {
+                    controller.enqueue(encoder.encode(':hb\n\n'));
+                } catch  {
+                    cleanup();
+                }
+            }, HEARTBEAT_MS);
+            timeout = setTimeout(()=>{
+                cleanup();
+                tryClose();
+            }, JOB_STREAM_TTL_MS);
+            reqSignal?.addEventListener('abort', ()=>{
+                cleanup();
+                tryClose();
+            });
+        },
+        cancel () {
+            if (unsubscribe) unsubscribe();
+            if (timeout) clearTimeout(timeout);
+            if (heartbeat) clearInterval(heartbeat);
+        }
+    });
+}
+/**
+ * Doc-level stream. Holds open, waits for a `createJob` matching
+ * (slug, docId), then transitions into the regular per-job streaming
+ * pattern in-place. Closes after `DOC_STREAM_TTL_MS` so the connection
+ * doesn't pile up forever on idle editors.
+ *
+ * This is the architectural piece that drops on-publish bar-appearance
+ * latency from 9-11s to ~50ms. Polling can't beat this — there's no
+ * client-side timer fast enough to race the server.
+ */ function buildDocStream(slug, documentId, reqSignal) {
+    let unsubscribeDoc;
+    let unsubscribeJob;
+    let timeout;
+    let heartbeat;
+    return new ReadableStream({
+        start (controller) {
+            const encoder = new TextEncoder();
+            controller.enqueue(encoder.encode('retry: 30000\n\n'));
+            // Send an initial idle event so the client knows the connection is
+            // healthy and waiting. Without this, browsers may consider a
+            // never-emitting stream "stalled".
+            controller.enqueue(encoder.encode(`event: progress\ndata: ${JSON.stringify({
+                jobId: '',
+                completed: [],
+                failed: [],
+                total: 0,
+                status: 'idle'
+            })}\n\n`));
+            const tryClose = ()=>{
+                try {
+                    controller.close();
+                } catch  {
+                // already closed
+                }
+            };
+            const cleanup = ()=>{
+                if (unsubscribeDoc) {
+                    unsubscribeDoc();
+                    unsubscribeDoc = undefined;
+                }
+                if (unsubscribeJob) {
+                    unsubscribeJob();
+                    unsubscribeJob = undefined;
+                }
+                if (timeout) {
+                    clearTimeout(timeout);
+                    timeout = undefined;
+                }
+                if (heartbeat) {
+                    clearInterval(heartbeat);
+                    heartbeat = undefined;
+                }
+            };
+            const send = (event)=>{
+                const eventType = event.status === 'running' ? 'progress' : 'complete';
+                try {
+                    controller.enqueue(encoder.encode(`event: ${eventType}\ndata: ${JSON.stringify(event)}\n\n`));
+                } catch  {
+                    return;
+                }
+                if (event.status !== 'running') {
+                    cleanup();
+                    setTimeout(tryClose, 2000);
+                }
+            };
+            // Subscribe FIRST, then re-check for an existing job. This order
+            // closes a tiny race window: between the outer handler's
+            // `getActiveJobForDoc` (where we decided to take the doc-stream
+            // branch) and this point, an after-change hook from a concurrent
+            // request could have fired `createJob`. Subscribing first
+            // guarantees we'll receive any event that fires from now on; the
+            // re-check handles the case where the job already exists.
+            unsubscribeDoc = subscribeToDoc(slug, documentId, (initial)=>{
+                send(initial);
+                if (initial.status === 'running' && !unsubscribeJob) {
+                    unsubscribeJob = subscribe(initial.jobId, send);
+                }
+            // If status was already terminal at first event (rare), `send`
+            // already triggered cleanup + close above.
+            });
+            const raceCheck = getActiveJobForDoc(slug, documentId);
+            if (raceCheck) {
+                send(jobToEvent(raceCheck));
+                if (raceCheck.status === 'running' && !unsubscribeJob) {
+                    unsubscribeJob = subscribe(raceCheck.jobId, send);
+                }
+            }
+            heartbeat = setInterval(()=>{
+                try {
+                    controller.enqueue(encoder.encode(':hb\n\n'));
+                } catch  {
+                    cleanup();
+                }
+            }, HEARTBEAT_MS);
+            timeout = setTimeout(()=>{
+                cleanup();
+                tryClose();
+            }, DOC_STREAM_TTL_MS);
+            reqSignal?.addEventListener('abort', ()=>{
+                cleanup();
+                tryClose();
+            });
+        },
+        cancel () {
+            if (unsubscribeDoc) unsubscribeDoc();
+            if (unsubscribeJob) unsubscribeJob();
+            if (timeout) clearTimeout(timeout);
+            if (heartbeat) clearInterval(heartbeat);
+        }
+    });
+}
+function resolveSlugFromUrl(url) {
+    try {
+        const parsed = new URL(url);
+        const segments = parsed.pathname.split('/').filter(Boolean);
+        const apiIndex = segments.indexOf('api');
+        if (apiIndex < 0) return '';
+        // Globals pattern: /api/globals/{slug}/ai-translate/progress
+        if (segments[apiIndex + 1] === 'globals' && segments[apiIndex + 2]) {
+            return segments[apiIndex + 2];
+        }
+        // Collections pattern: /api/{slug}/ai-translate/progress
+        if (segments[apiIndex + 1]) {
+            return segments[apiIndex + 1];
+        }
+    } catch  {
+    // fall through
+    }
+    return '';
+}

package/dist/endpoints/translate.d.ts

@@ -0,0 +1,11 @@
+import type { PayloadHandler } from 'payload';
+export type HandlerContext = {
+    kind: 'collection';
+    slug: string;
+} | {
+    kind: 'global';
+    slug: string;
+};
+type HandlerArg = HandlerContext | string | undefined;
+export declare const getTranslateHandler: (arg?: HandlerArg) => PayloadHandler;
+export {};