@purposeinplay/payload-ai-translate 0.1.0

package/dist/endpoints/translation-hub/status.js

@@ -0,0 +1,391 @@
+import { DEFAULT_BULK_TRANSLATE_BATCHES_COLLECTION_SLUG } from '../../bulk-translate-batches-collection.js';
+import { DEFAULT_BULK_TRANSLATE_UNITS_COLLECTION_SLUG } from '../../bulk-translate-units-collection.js';
+import { computeLiveBatchCount } from '../../lib/batch-counts.js';
+import { createScopedLogger } from '../../lib/logger.js';
+import { decodeCursor, encodeCursor, errorResponse, isEditorOrAdmin, unauthorizedResponse } from './_helpers.js';
+// ---------------------------------------------------------------------------
+// Handler
+// ---------------------------------------------------------------------------
+/**
+ * `GET /api/translation-hub/bulk-translate/:id/status`
+ *
+ * Returns batch metadata + aggregate counts by unit status + a
+ * paginated drill-down list of units (filterable by status).
+ *
+ * Query params:
+ *   - `cursor` — opaque cursor returned by the previous page
+ *   - `limit` — page size (default 20, max 100)
+ *   - `status` — filter to one unit status (`failed` is the typical
+ *     value for the failure drill-down UI)
+ */ export const getBulkTranslateStatusHandler = (options = {})=>async (req)=>{
+        const batchesSlug = options.batchesCollectionSlug ?? DEFAULT_BULK_TRANSLATE_BATCHES_COLLECTION_SLUG;
+        const unitsSlug = options.unitsCollectionSlug ?? DEFAULT_BULK_TRANSLATE_UNITS_COLLECTION_SLUG;
+        if (!req.user) {
+            return unauthorizedResponse(req);
+        }
+        if (!isEditorOrAdmin(req.user)) {
+            return errorResponse('forbidden', "You don't have permission to view bulk-translation status. Contact an admin.", 403);
+        }
+        const batchId = extractBatchId(req.url ?? '');
+        if (!batchId) {
+            return errorResponse('invalid_batch_id', 'Batch ID could not be parsed from the request URL.', 400);
+        }
+        let batch;
+        try {
+            batch = await req.payload.findByID({
+                collection: batchesSlug,
+                id: batchId,
+                overrideAccess: true,
+                depth: 0
+            });
+        } catch (err) {
+            const log = createScopedLogger(req.payload, {
+                component: 'hub.status',
+                batchId
+            });
+            log.event('warn', 'hub.status.batch-read.failed', {
+                err,
+                endpoint: 'hub.status',
+                batchId
+            });
+            batch = undefined;
+        }
+        if (!batch) {
+            return errorResponse('not_found', 'This translation run no longer exists. Refresh the page.', 404);
+        }
+        const url = new URL(req.url ?? 'http://localhost');
+        const cursor = url.searchParams.get('cursor') ?? undefined;
+        const limitRaw = url.searchParams.get('limit');
+        const statusFilter = url.searchParams.get('status') ?? undefined;
+        const collectionFilter = url.searchParams.get('collection') ?? undefined;
+        let limit = limitRaw ? Number.parseInt(limitRaw, 10) : 20;
+        if (!Number.isFinite(limit) || limit <= 0) limit = 20;
+        if (limit > 100) limit = 100;
+        const offset = decodeCursor(cursor);
+        // Live counts from the units table — bypasses cached counters that
+        // drift on retries / cancels / admin SDK interventions. See
+        // `lib/batch-counts.ts` for the architecture note.
+        const liveCounts = await computeLiveBatchCount(req.payload, unitsSlug, batchId);
+        const counts = {
+            total: liveCounts.total,
+            pending: liveCounts.pending,
+            running: liveCounts.running,
+            completed: liveCounts.completed,
+            failed: liveCounts.failed,
+            skipped: liveCounts.skipped,
+            reverted: liveCounts.reverted
+        };
+        const countsByCollection = await aggregateUnitCountsByCollection(req.payload, unitsSlug, batchId);
+        const jobs = await fetchUnitPage(req.payload, unitsSlug, batchId, statusFilter, offset, limit, collectionFilter);
+        const nextCursor = jobs.length === limit ? encodeCursor(offset + limit) : null;
+        return Response.json({
+            data: {
+                batchId,
+                status: batch.status,
+                mode: batch.mode,
+                canaryLimit: batch.canaryLimit ?? null,
+                estimatedCostUsd: batch.estimatedCostUsd ?? null,
+                actualCostUsd: batch.actualCostUsd ?? 0,
+                counts,
+                countsByCollection,
+                jobs,
+                nextCursor,
+                snapshot: {
+                    providerKey: batch.snapshotProviderKey ?? null,
+                    modelId: batch.snapshotModelId ?? null,
+                    sourceLocale: batch.snapshotSourceLocale ?? null
+                },
+                timestamps: {
+                    enqueuedAt: batch.enqueuedAt ?? null,
+                    startedAt: batch.startedAt ?? null,
+                    completedAt: batch.completedAt ?? null,
+                    cancelledAt: batch.cancelledAt ?? null,
+                    revertedAt: batch.revertedAt ?? null
+                },
+                triggeredByUserId: batch.triggeredByUserId ?? null,
+                triggeredByEmail: batch.triggeredByEmail ?? null,
+                failures: batch.failures ?? null
+            }
+        });
+    };
+/**
+ * Extracts the batch id from a URL of the shape
+ * `/api/translation-hub/bulk-translate/<id>/status`. Defensive across
+ * URL variants Payload may give us — we look for the segment between
+ * `bulk-translate` and `status`.
+ */ export function extractBatchId(url) {
+    try {
+        const segments = new URL(url, 'http://placeholder').pathname.split('/').filter(Boolean);
+        const anchorIndex = segments.indexOf('bulk-translate');
+        if (anchorIndex === -1) return undefined;
+        const candidate = segments[anchorIndex + 1];
+        // Reject the literal endpoint paths (no id present).
+        if (!candidate || candidate === 'status' || candidate === 'enqueue') {
+            return undefined;
+        }
+        return candidate;
+    } catch  {
+        return undefined;
+    }
+}
+/**
+ * Per-collection aggregate counts + top-2 failure codes for a batch.
+ * Drives the tree DrillDown's bucket-header counts — without this, the
+ * UI would have to compute bucket totals from the paginated `jobs` page
+ * and the numbers would only reflect "how many of the loaded units are
+ * in this bucket", not the actual total.
+ *
+ * Strategy: one `payload.find` with `limit: 5000`, `depth: 0`, and
+ * `select` narrowed to (collection, documentId, status, failureCode).
+ * Aggregate in JS — provider-agnostic, no raw SQL. For batches above
+ * 5000 units we'd switch to drizzle GROUP BY but realistic batches
+ * stay well under that ceiling.
+ */ async function aggregateUnitCountsByCollection(payload, unitsSlug, batchId) {
+    const out = {};
+    let page = 1;
+    // eslint-disable-next-line no-constant-condition
+    while(true){
+        const result = await payload.find({
+            collection: unitsSlug,
+            where: {
+                batchId: {
+                    equals: batchId
+                }
+            },
+            page,
+            limit: 1000,
+            depth: 0,
+            // Everything the aggregation reads — and nothing else. The
+            // unselected `preRunSnapshot` jsonb is the heavy column (tens of
+            // KB per unit); without this projection every status-page load
+            // dragged the full snapshot payload of all units through
+            // Postgres TOAST + JSON.parse.
+            select: {
+                collection: true,
+                documentId: true,
+                status: true,
+                failureCode: true,
+                startedAt: true,
+                completedAt: true,
+                processingDurationMs: true
+            },
+            overrideAccess: true
+        });
+        for (const doc of result.docs){
+            const collection = String(doc.collection ?? '');
+            const documentId = String(doc.documentId ?? '');
+            const status = String(doc.status ?? '');
+            const failureCode = typeof doc.failureCode === 'string' ? doc.failureCode : null;
+            let bucket = out[collection];
+            if (!bucket) {
+                bucket = {
+                    // For globals, the worker writes documentId === collection (the
+                    // slug doubles as the id).
+                    isGlobal: documentId === collection,
+                    total: 0,
+                    pending: 0,
+                    running: 0,
+                    succeeded: 0,
+                    failed: 0,
+                    skipped: 0,
+                    reverted: 0,
+                    _failureCodes: new Map(),
+                    _docIds: new Set(),
+                    _docIdsByPending: new Set(),
+                    _docIdsByRunning: new Set(),
+                    _docIdsBySucceeded: new Set(),
+                    _docIdsByFailed: new Set(),
+                    _docIdsBySkipped: new Set(),
+                    _docIdsByReverted: new Set(),
+                    _minStartedAtMs: null,
+                    _maxCompletedAtMs: null,
+                    _aiActiveMs: 0,
+                    _aiActiveSeen: false
+                };
+                out[collection] = bucket;
+            }
+            bucket._docIds.add(documentId);
+            bucket.total += 1;
+            // Per-collection duration aggregates. min(startedAt) anchors the
+            // "when did we start translating this collection" moment;
+            // max(completedAt) is the last unit finish.
+            const startedAtRaw = doc.startedAt;
+            if (typeof startedAtRaw === 'string') {
+                const ms = new Date(startedAtRaw).getTime();
+                if (Number.isFinite(ms)) {
+                    if (bucket._minStartedAtMs === null || ms < bucket._minStartedAtMs) {
+                        bucket._minStartedAtMs = ms;
+                    }
+                }
+            }
+            const completedAtRaw = doc.completedAt;
+            if (typeof completedAtRaw === 'string') {
+                const ms = new Date(completedAtRaw).getTime();
+                if (Number.isFinite(ms)) {
+                    if (bucket._maxCompletedAtMs === null || ms > bucket._maxCompletedAtMs) {
+                        bucket._maxCompletedAtMs = ms;
+                    }
+                }
+            }
+            if (typeof doc.processingDurationMs === 'number' && doc.processingDurationMs > 0) {
+                bucket._aiActiveMs += doc.processingDurationMs;
+                bucket._aiActiveSeen = true;
+            }
+            if (status === 'success') {
+                bucket.succeeded += 1;
+                bucket._docIdsBySucceeded.add(documentId);
+            } else if (status === 'failed') {
+                bucket.failed += 1;
+                bucket._docIdsByFailed.add(documentId);
+                if (failureCode) {
+                    bucket._failureCodes.set(failureCode, (bucket._failureCodes.get(failureCode) ?? 0) + 1);
+                }
+            } else if (status === 'pending') {
+                bucket.pending += 1;
+                bucket._docIdsByPending.add(documentId);
+            } else if (status === 'running') {
+                bucket.running += 1;
+                bucket._docIdsByRunning.add(documentId);
+            } else if (status === 'skipped') {
+                bucket.skipped += 1;
+                bucket._docIdsBySkipped.add(documentId);
+            } else if (status === 'reverted') {
+                bucket.reverted += 1;
+                bucket._docIdsByReverted.add(documentId);
+            }
+        }
+        if (!result.hasNextPage) break;
+        page += 1;
+    }
+    // Flatten + compute top-2 failure codes per bucket.
+    const flat = {};
+    for (const [collection, bucket] of Object.entries(out)){
+        const codeEntries = [
+            ...bucket._failureCodes.entries()
+        ];
+        const collectionSpanMs = bucket._minStartedAtMs !== null && bucket._maxCompletedAtMs !== null ? Math.max(0, bucket._maxCompletedAtMs - bucket._minStartedAtMs) : null;
+        const aiActiveMs = bucket._aiActiveSeen ? bucket._aiActiveMs : null;
+        // Queue wait is the gap between the editor-perceived wallclock and
+        // the actual AI compute time. Clamp to >=0 because aiActiveMs can
+        // exceed span when units ran in parallel across multiple workers.
+        const queueWaitMs = collectionSpanMs !== null && aiActiveMs !== null ? Math.max(0, collectionSpanMs - aiActiveMs) : null;
+        flat[collection] = {
+            isGlobal: bucket.isGlobal,
+            total: bucket.total,
+            pending: bucket.pending,
+            running: bucket.running,
+            succeeded: bucket.succeeded,
+            failed: bucket.failed,
+            skipped: bucket.skipped,
+            reverted: bucket.reverted,
+            docCountsByStatus: {
+                all: bucket._docIds.size,
+                pending: bucket._docIdsByPending.size,
+                running: bucket._docIdsByRunning.size,
+                completed: bucket._docIdsBySucceeded.size,
+                failed: bucket._docIdsByFailed.size,
+                skipped: bucket._docIdsBySkipped.size,
+                reverted: bucket._docIdsByReverted.size
+            },
+            totalDocs: bucket._docIds.size,
+            collectionSpanMs,
+            aiActiveMs,
+            queueWaitMs,
+            distinctFailureCodes: codeEntries.length,
+            topFailureCodes: codeEntries.sort((a, b)=>b[1] - a[1]).slice(0, 2).map(([code, count])=>({
+                    code,
+                    count
+                }))
+        };
+    }
+    return flat;
+}
+/**
+ * Type alias used by the aggregator above. Inlined here so the
+ * function signature stays self-documenting without a named export.
+ */ function aggregateUnitCountsByCollectionShape() {
+    return {
+        isGlobal: false,
+        total: 0,
+        pending: 0,
+        running: 0,
+        succeeded: 0,
+        failed: 0,
+        skipped: 0,
+        reverted: 0,
+        docCountsByStatus: {
+            all: 0,
+            pending: 0,
+            running: 0,
+            completed: 0,
+            failed: 0,
+            skipped: 0,
+            reverted: 0
+        },
+        totalDocs: 0,
+        collectionSpanMs: null,
+        aiActiveMs: null,
+        queueWaitMs: null,
+        distinctFailureCodes: 0,
+        topFailureCodes: []
+    };
+}
+async function fetchUnitPage(payload, unitsSlug, batchId, statusFilter, offset, limit, collectionFilter) {
+    const andClauses = [
+        {
+            batchId: {
+                equals: batchId
+            }
+        }
+    ];
+    if (statusFilter) {
+        // Map the public `completed` alias back onto storage's `success`.
+        const storageStatus = statusFilter === 'completed' ? 'success' : statusFilter;
+        andClauses.push({
+            status: {
+                equals: storageStatus
+            }
+        });
+    }
+    if (collectionFilter) {
+        // Used by the tree DrillDown's bucket-on-expand fetch to load just
+        // that collection's units rather than paging through every unit in
+        // the batch. The slug doubles as the global id, so this works for
+        // both collection and global buckets.
+        andClauses.push({
+            collection: {
+                equals: collectionFilter
+            }
+        });
+    }
+    // Payload doesn't expose an `offset`-based pagination today, so we
+    // compute the page number from offset+limit. This is correct so
+    // long as `limit` stays stable between requests — which is the
+    // contract of cursor-based pagination.
+    const page = Math.floor(offset / limit) + 1;
+    const result = await payload.find({
+        collection: unitsSlug,
+        where: {
+            and: andClauses
+        },
+        sort: 'createdAt',
+        page,
+        limit,
+        depth: 0,
+        overrideAccess: true
+    });
+    return result.docs.map((u)=>({
+            unitId: String(u.id),
+            collection: String(u.collection ?? ''),
+            documentId: String(u.documentId ?? ''),
+            locale: String(u.locale ?? ''),
+            status: String(u.status ?? ''),
+            attempts: Number(u.attempts ?? 0),
+            failureCode: typeof u.failureCode === 'string' ? u.failureCode : null,
+            failureMessage: typeof u.failureMessage === 'string' ? u.failureMessage : null,
+            costUsd: Number(u.costUsd ?? 0),
+            startedAt: typeof u.startedAt === 'string' ? u.startedAt : null,
+            completedAt: typeof u.completedAt === 'string' ? u.completedAt : null,
+            processingDurationMs: typeof u.processingDurationMs === 'number' ? u.processingDurationMs : null
+        }));
+}

package/dist/endpoints/translation-hub/usage-summary.d.ts

@@ -0,0 +1,114 @@
+import type { PayloadHandler } from 'payload';
+export type UsageSummaryRange = {
+    kind: '7d' | '30d' | '90d';
+    since: string;
+} | {
+    kind: 'all';
+} | {
+    kind: 'custom';
+    from?: string;
+    to?: string;
+};
+export interface UsageSummaryTotals {
+    runs: number;
+    /**
+     * ROUND2-2 (v1.2.6, fix round): canonical "did this run write any
+     * fields" definition — `status = 'succeeded' AND fieldsTranslated > 0`.
+     * Counts only runs that actually called the LLM and persisted at
+     * least one field. Manual-edit-preserve-only runs (status =
+     * 'succeeded' but fieldsTranslated = 0) are NOT counted here; they
+     * surface as `preserved` instead.
+     *
+     * Before this fix, Hub Overview counted bare `status = 'succeeded'`
+     * (which includes preserved-only no-ops) while Audit & Cost counted
+     * `fieldsTranslated > 0`. Same KPI named the same way reported two
+     * different numbers — the editor-trust failure mode NEW-1 was filed
+     * to prevent.
+     *
+     * Invariant: `runs === succeeded + failed + preserved`.
+     */
+    succeeded: number;
+    failed: number;
+    /**
+     * ROUND2-2: status = 'succeeded' AND fieldsTranslated <= 0. Manual-
+     * edit-guard preserves, hash-skip-only re-runs, etc. Surfaced as a
+     * separate count so editors can see "your translation didn't do
+     * nothing — it just had nothing new to write." Falls under "not
+     * failed" but isn't a write either.
+     */
+    preserved: number;
+    inputTokens: number;
+    outputTokens: number;
+    costUsd: number;
+    totalMatching: number;
+    /**
+     * Always `false` from this endpoint — the SQL aggregation always
+     * covers every row matching the window. The flag is preserved on the
+     * response so the UI's existing truncation chip code can keep
+     * checking it without branching. Client-side aggregation is gone.
+     */
+    truncated: boolean;
+}
+export interface UsageSummaryCollectionBucket {
+    slug: string;
+    kind: 'collection' | 'global';
+    runs: number;
+    tokens: number;
+    costUsd: number;
+}
+export interface UsageSummaryModelBucket {
+    /** Provider/model slug; may be the literal `'__unresolved__'` for
+     * rows that failed before model selection (see NEW-2). */
+    model: string;
+    runs: number;
+    tokens: number;
+    costUsd: number;
+}
+export interface UsageSummaryLocaleBucket {
+    locale: string;
+    runs: number;
+    failed: number;
+}
+export interface UsageSummarySampleRow {
+    id: number | string;
+    createdAt: string;
+    slug: string;
+    kind: 'collection' | 'global';
+    documentId?: string | null;
+    status: 'succeeded' | 'failed';
+    model?: string | null;
+    inputTokens: number;
+    outputTokens: number;
+    estimatedCostUsd?: number | null;
+    durationMs?: number | null;
+    error?: string | null;
+    failedCount: number;
+    succeededCount: number;
+    targetLocales?: Array<{
+        locale: string;
+        status?: 'succeeded' | 'failed' | null;
+    }> | null;
+}
+export interface UsageSummaryResponse {
+    range: UsageSummaryRange;
+    totals: UsageSummaryTotals;
+    byCollection: UsageSummaryCollectionBucket[];
+    byModel: UsageSummaryModelBucket[];
+    byLocale: UsageSummaryLocaleBucket[];
+    samples: UsageSummarySampleRow[];
+}
+export interface UsageSummaryHandlerOptions {
+    /** Override the usage collection slug. Defaults to `translation-usage`. */
+    usageCollectionSlug?: string;
+}
+/**
+ * Parse the `range`/`from`/`to` query params. Exported for testing —
+ * mirrors the StatusStrip/AuditPanel resolution rules so the UI and
+ * the endpoint agree on what "Last 7 days" means.
+ */
+export declare function parseRangeParams(params: URLSearchParams): {
+    range: UsageSummaryRange;
+    since: string | null;
+    until: string | null;
+};
+export declare const getTranslationHubUsageSummaryHandler: (options?: UsageSummaryHandlerOptions) => PayloadHandler;