@captiveorigins/payload-plugin-polyglot 0.0.0-stage → 0.2.0

package/dist/translate.js

@@ -0,0 +1,337 @@
+import { fingerprint, memoryKey, textsOf, visitPaths } from './text.js';
+import { clearStop, loadSettings, recordStop, recordUsage } from './settings.js';
+import { QuotaExceededError } from './types.js';
+export const polyglotOf = (payload) => payload.config.custom?.polyglot;
+const rulesOf = (o) => ({ skip: o.skip, skipKeys: new Set(o.skipKeys) });
+const topFields = (o, slug) => [...new Set((o.collections[slug] ?? []).map((p) => p.split('.')[0]))];
+const stateKey = (collection, id, locale) => `${collection}:${id}:${locale}`;
+const engineFor = (o, locale) => o.translatorFor?.(locale) ?? o.translator;
+/** Is this document one Polyglot translates? */
+export function included(o, collection, doc) {
+    if (!o.collections[collection])
+        return false;
+    if (o.include)
+        return o.include({ collection, doc });
+    return doc._status === undefined || doc._status === 'published';
+}
+async function loadSource(payload, o, slug, id) {
+    const doc = (await payload.findByID({
+        collection: slug,
+        id,
+        locale: o.defaultLocale,
+        depth: 0,
+        draft: false,
+        overrideAccess: true,
+        disableErrors: true,
+    }));
+    if (!doc || !included(o, slug, doc))
+        return null;
+    const source = Object.fromEntries(topFields(o, slug).filter((k) => doc[k] !== undefined && doc[k] !== null).map((k) => [k, structuredClone(doc[k])]));
+    const texts = textsOf(source, o.collections[slug], rulesOf(o));
+    const reviewed = new Set((doc[o.statusField]?.reviewed ?? []));
+    const { docs } = await payload.find({
+        collection: o.stateSlug,
+        where: { and: [{ collection: { equals: slug } }, { doc: { equals: id } }] },
+        pagination: false,
+        depth: 0,
+        overrideAccess: true,
+    });
+    const state = new Map(docs.map((r) => [r.locale, r]));
+    return { doc, source, texts, hash: fingerprint(texts), state, reviewed };
+}
+async function saveState(payload, o, collection, id, locale, hash, stale, existing) {
+    const data = { key: stateKey(collection, id, locale), collection, doc: id, locale, hash, stale, at: new Date().toISOString() };
+    if (existing)
+        await payload.update({ collection: o.stateSlug, id: existing.id, data: data, overrideAccess: true });
+    else
+        await payload.create({ collection: o.stateSlug, data: data, overrideAccess: true });
+}
+/** Translations known in memory for these texts in one language. */
+async function recall(payload, o, locale, texts) {
+    const known = new Map();
+    if (!texts.length)
+        return known;
+    const { docs } = await payload.find({
+        collection: o.memorySlug,
+        where: { key: { in: texts.map((t) => memoryKey(locale, t)) } },
+        pagination: false,
+        depth: 0,
+        overrideAccess: true,
+        select: { key: true, text: true },
+    });
+    const byKey = new Map(docs.map((d) => [d.key, d.text]));
+    for (const t of texts) {
+        const hit = byKey.get(memoryKey(locale, t));
+        if (hit)
+            known.set(t, hit);
+    }
+    return known;
+}
+async function remember(payload, o, locale, pairs) {
+    for (const [source, text] of pairs)
+        await payload
+            .create({ collection: o.memorySlug, data: { key: memoryKey(locale, source), locale, source, text }, overrideAccess: true })
+            .catch(() => undefined); // already remembered (unique key)
+}
+const CHUNK = 80;
+const chunks = (list) => Array.from({ length: Math.ceil(list.length / CHUNK) }, (_, i) => list.slice(i * CHUNK, (i + 1) * CHUNK));
+const argsFor = (o, s, locale, items, context) => ({
+    locale,
+    items,
+    context,
+    sourceLocale: o.defaultLocale,
+    keep: [...new Set([...o.keep, ...s.keep])],
+    instructions: s.instructions,
+    glossary: s.glossary.filter((g) => g.locale === locale).map(({ term, translation }) => ({ term, translation })),
+});
+const localesOf = (o, s, wanted) => (wanted ?? o.locales).filter((l) => !s.skipLocales.has(l));
+/** A per-run character budget (free tiers). */
+export class Budget {
+    max;
+    used = 0;
+    constructor(max) {
+        this.max = max;
+    }
+    allows(chars) {
+        return this.max === undefined || this.used + chars <= this.max;
+    }
+    spend(chars) {
+        this.used += chars;
+    }
+}
+/**
+ * Bring one document up to date in every language (or `locales`). Unchanged
+ * source costs nothing (the fingerprint matches); known sentences come from
+ * memory; only new ones reach the translator. Reviewed languages are never
+ * overwritten: they are marked stale. Languages are written one at a time.
+ */
+export async function translateDocument(payload, collection, id, opts = {}) {
+    const o = polyglotOf(payload);
+    const settings = opts.settings ?? (await loadSettings(payload, o));
+    const result = { translated: [], upToDate: [], reviewedStale: [], failed: [], pending: [] };
+    const src = await loadSource(payload, o, collection, id);
+    if (!src)
+        return result;
+    const todo = [];
+    for (const locale of localesOf(o, settings, opts.locales)) {
+        const row = src.state.get(locale);
+        if (row?.hash === src.hash)
+            result.upToDate.push(locale);
+        else if (src.reviewed.has(locale)) {
+            result.reviewedStale.push(locale);
+            if (!opts.dryRun && row && !row.stale)
+                await saveState(payload, o, collection, id, locale, row.hash, true, row);
+        }
+        else
+            todo.push(locale);
+    }
+    if (opts.dryRun)
+        return { ...result, pending: todo };
+    const unique = [...new Set(src.texts)];
+    const context = { ...(o.writeContext?.({ inRequest: Boolean(opts.inRequest) }) ?? {}), polyglot: true };
+    let sent = 0;
+    let recovered = !settings.stop;
+    try {
+        for (const [n, locale] of todo.entries()) {
+            try {
+                let known = await recall(payload, o, locale, unique);
+                const missing = unique.filter((t) => !known.has(t));
+                if (missing.length && !opts.memoryOnly) {
+                    const chars = missing.reduce((s, t) => s + t.length, 0);
+                    if (opts.budget && !opts.budget.allows(chars)) {
+                        result.pending.push(...todo.slice(n));
+                        break;
+                    }
+                    for (const part of chunks(missing)) {
+                        const items = part.map((text, i) => ({ key: String(i), text }));
+                        const out = await engineFor(o, locale).translate(argsFor(o, settings, locale, items, o.describe?.(collection)));
+                        const chars = part.reduce((s, t) => s + t.length, 0);
+                        opts.budget?.spend(chars);
+                        sent += chars;
+                        if (!recovered && out.size) {
+                            recovered = true;
+                            await clearStop(payload, o);
+                        }
+                        await remember(payload, o, locale, items.flatMap((it) => (out.get(it.key) ? [[it.text, out.get(it.key)]] : [])));
+                    }
+                    known = await recall(payload, o, locale, unique);
+                }
+                if (unique.some((t) => !known.has(t))) {
+                    ;
+                    (opts.memoryOnly ? result.pending : result.failed).push(locale);
+                    continue;
+                }
+                const data = structuredClone(src.source);
+                visitPaths(data, o.collections[collection], rulesOf(o), (t, set) => set(known.get(t) ?? t));
+                await payload.update({
+                    collection: collection,
+                    id,
+                    locale: locale,
+                    data: { ...data, ...(src.doc._status ? { _status: 'published' } : {}) },
+                    overrideAccess: true,
+                    context,
+                });
+                await saveState(payload, o, collection, id, locale, src.hash, false, src.state.get(locale));
+                result.translated.push(locale);
+            }
+            catch (error) {
+                if (error instanceof QuotaExceededError) {
+                    result.pending.push(...todo.slice(n));
+                    await recordStop(payload, o, error).catch((err) => payload.logger.warn({ err }, 'Polyglot: could not record the stop'));
+                    throw Object.assign(error, { result });
+                }
+                payload.logger.error({ err: error, collection, id, locale }, 'Polyglot: translation failed');
+                result.failed.push(locale);
+            }
+        }
+    }
+    finally {
+        await recordUsage(payload, o, sent).catch((err) => payload.logger.warn({ err }, 'Polyglot: could not record usage'));
+    }
+    return result;
+}
+/**
+ * Find all content and bring it up to date (the sweep). With a translator that
+ * has `batch` and `useBatch`, every missing sentence of every language goes in
+ * one batch first (cheaper), then documents are written from memory. Stops
+ * cleanly at `maxCharactersPerRun` or when the engine's quota is used up; the
+ * next run continues. `dryRun` only reports what is pending. Respects the
+ * Translation settings (paused, skipped languages, cap); `force` runs even when
+ * paused (Translate now).
+ */
+export async function syncAll(payload, opts = {}) {
+    const o = polyglotOf(payload);
+    const report = { documents: 0, upToDate: 0, translated: 0, pending: 0, reviewedStale: 0, failed: 0, characters: 0, stopped: 'done' };
+    const settings = await loadSettings(payload, o);
+    if (settings.paused && !opts.force && !opts.dryRun)
+        return { ...report, stopped: 'paused' };
+    const budget = new Budget(settings.maxCharactersPerRun ?? o.maxCharactersPerRun);
+    const targets = [];
+    for (const collection of Object.keys(o.collections).filter((c) => !opts.collections || opts.collections.includes(c))) {
+        const { docs } = await payload.find({ collection: collection, locale: o.defaultLocale, depth: 0, pagination: false, overrideAccess: true, draft: false });
+        for (const d of docs)
+            if (included(o, collection, d))
+                targets.push({ collection, id: String(d.id) });
+    }
+    const locales = localesOf(o, settings, opts.locales);
+    const work = opts.limit ? targets.slice(0, opts.limit) : targets;
+    const batching = Boolean(opts.useBatch && o.translator.batch && !o.translatorFor && !opts.dryRun);
+    if (batching) {
+        const missing = new Map();
+        for (const t of work) {
+            const src = await loadSource(payload, o, t.collection, t.id);
+            if (!src)
+                continue;
+            for (const locale of locales) {
+                if (src.state.get(locale)?.hash === src.hash || src.reviewed.has(locale))
+                    continue;
+                const set = missing.get(locale) ?? new Set();
+                for (const text of src.texts)
+                    set.add(text);
+                missing.set(locale, set);
+            }
+        }
+        const requests = [];
+        for (const [locale, set] of missing) {
+            const known = await recall(payload, o, locale, [...set]);
+            for (const part of chunks([...set].filter((t) => !known.has(t)))) {
+                const chars = part.reduce((s, t) => s + t.length, 0);
+                if (!budget.allows(chars)) {
+                    report.stopped = 'budget';
+                    break;
+                }
+                budget.spend(chars);
+                requests.push(argsFor(o, settings, locale, part.map((text, i) => ({ key: String(i), text }))));
+            }
+        }
+        if (requests.length) {
+            let results;
+            try {
+                results = await o.translator.batch(requests);
+            }
+            catch (error) {
+                if (!(error instanceof QuotaExceededError))
+                    throw error;
+                await recordStop(payload, o, error);
+                return { ...report, stopped: 'quota', stopReason: error.reason, pending: requests.length };
+            }
+            await recordUsage(payload, o, budget.used);
+            if (settings.stop && results.some((m) => m.size))
+                await clearStop(payload, o);
+            for (const [i, r] of requests.entries())
+                await remember(payload, o, r.locale, r.items.flatMap((it) => (results[i]?.get(it.key) ? [[it.text, results[i].get(it.key)]] : [])));
+        }
+    }
+    for (const t of work) {
+        let r;
+        try {
+            r = await translateDocument(payload, t.collection, t.id, { locales, dryRun: opts.dryRun, memoryOnly: batching, budget: batching ? undefined : budget, settings });
+        }
+        catch (error) {
+            if (!(error instanceof QuotaExceededError))
+                throw error;
+            r = error.result;
+            report.stopped = 'quota';
+            report.stopReason = error.reason;
+        }
+        report.documents++;
+        report.upToDate += r.upToDate.length;
+        report.translated += r.translated.length;
+        report.pending += r.pending.length;
+        report.reviewedStale += r.reviewedStale.length;
+        report.failed += r.failed.length;
+        if (report.stopped === 'quota')
+            break;
+        if (!batching && r.pending.length && !opts.dryRun)
+            report.stopped = 'budget';
+    }
+    report.characters = budget.used;
+    return report;
+}
+/** Translation status of one document in one language, for notices: source, reviewed, stale, machine or pending. */
+export async function languageStatus(payload, collection, doc, locale) {
+    const o = polyglotOf(payload);
+    if (locale === o.defaultLocale)
+        return 'source';
+    const reviewed = (doc[o.statusField]?.reviewed ?? []).includes(locale);
+    const { docs } = await payload.find({
+        collection: o.stateSlug,
+        where: { key: { equals: stateKey(collection, String(doc.id), locale) } },
+        limit: 1,
+        depth: 0,
+        overrideAccess: true,
+    });
+    const row = docs[0];
+    if (reviewed)
+        return row?.stale ? 'stale' : 'reviewed';
+    return row ? 'machine' : 'pending';
+}
+/** Translation coverage per collection and language (for the admin view and reports). */
+export async function coverage(payload, opts = {}) {
+    const o = polyglotOf(payload);
+    const rows = [];
+    for (const collection of Object.keys(o.collections).filter((c) => !opts.collections || opts.collections.includes(c))) {
+        const row = { collection, documents: 0, locales: Object.fromEntries(o.locales.map((l) => [l, { translated: 0, pending: 0, reviewed: 0, stale: 0 }])) };
+        const { docs } = await payload.find({ collection: collection, locale: o.defaultLocale, depth: 0, pagination: false, overrideAccess: true, draft: false });
+        for (const d of docs) {
+            if (!included(o, collection, d))
+                continue;
+            const src = await loadSource(payload, o, collection, String(d.id));
+            if (!src)
+                continue;
+            row.documents++;
+            for (const locale of o.locales) {
+                const r = src.state.get(locale);
+                const c = row.locales[locale];
+                if (src.reviewed.has(locale))
+                    r && r.hash !== src.hash ? c.stale++ : c.reviewed++;
+                else if (r?.hash === src.hash)
+                    c.translated++;
+                else
+                    c.pending++;
+            }
+        }
+        rows.push(row);
+    }
+    return rows;
+}

package/dist/types.d.ts

@@ -0,0 +1,119 @@
+import type { Payload, PayloadRequest, RequestContext } from 'payload';
+/** One piece of text to translate, with a caller-chosen key. */
+export type Item = {
+    key: string;
+    text: string;
+};
+/**
+ * A translation engine. `translate` returns translations by key (missing keys
+ * stay untranslated). `batch` (optional) translates many requests at once,
+ * usually at a lower price and more slowly; the sweep prefers it.
+ */
+export type TranslateArgs = {
+    locale: string;
+    items: Item[];
+    context?: string;
+    sourceLocale: string;
+    keep: string[];
+    /** Editors' notes (tone, audience); AI engines use them, others may ignore them. */
+    instructions?: string;
+    /** Preferred translations for this language; AI engines use them, others may ignore them. */
+    glossary?: {
+        term: string;
+        translation: string;
+    }[];
+};
+export type Translator = {
+    translate: (args: TranslateArgs) => Promise<Map<string, string>>;
+    batch?: (requests: TranslateArgs[]) => Promise<Map<string, string>[]>;
+};
+/** Why an engine stopped: out of credits (paid), a free quota used up, or the key refused. */
+export type StopReason = 'credits' | 'quota' | 'auth';
+/**
+ * Thrown by a translator when it cannot continue until someone acts (credits,
+ * quota, key): the run stops cleanly, admins are told why, and the next run
+ * after the fix continues.
+ */
+export declare class QuotaExceededError extends Error {
+    readonly reason: StopReason;
+    constructor(message?: string, reason?: StopReason);
+}
+/** A recorded stop, shown to admins until a translation succeeds again. */
+export type EngineStop = {
+    reason: StopReason;
+    message: string;
+    at: string;
+};
+export type PolyglotOptions = {
+    /** Fields to keep in every language, per collection. Dotted paths reach into arrays/groups: `sections.heading`. */
+    collections: Record<string, string[]>;
+    translator: Translator;
+    /** May this request/run call the translator? Default: always. Use it for keys, budgets or environments. */
+    enabled?: (args: {
+        payload: Payload;
+        req?: PayloadRequest;
+    }) => boolean | Promise<boolean>;
+    /** Which documents to translate. Default: published ones (or all, without drafts). */
+    include?: (args: {
+        collection: string;
+        doc: Record<string, unknown>;
+    }) => boolean;
+    /** Terms never translated: product names, brands, scientific names… */
+    keep?: string[];
+    /** Engine per language (e.g. DeepL for European languages, another for the rest). Default: `translator`. */
+    translatorFor?: (locale: string) => Translator | undefined;
+    /** Stop a sweep after sending this many characters (stay inside a free tier). Default: no cap. */
+    maxCharactersPerRun?: number;
+    /** Text never sent (placeholders). Default: lorem ipsum and friends. */
+    skip?: RegExp;
+    /** Keys whose string values are never text (urls, ids, option values…). Added to the defaults. */
+    skipKeys?: string[];
+    /** Fields that must never change when another language is saved (computed from the default language). */
+    identityFields?: string[];
+    /** Slug of the translation memory collection. Default `polyglot-memory`. */
+    memorySlug?: string;
+    /** Slug of the per-language fingerprint collection. Default `polyglot-state`. */
+    stateSlug?: string;
+    /** Name of the per-document status group (reviewed languages + fingerprints). Default `translations`. */
+    statusField?: string;
+    /** Context for writes of translated languages, e.g. `{ disableRevalidate: true }` outside a web request. */
+    writeContext?: (args: {
+        inRequest: boolean;
+    }) => RequestContext;
+    /** Short description for the translator ("Content of a page on a reptile-care website"). */
+    describe?: (collection: string) => string;
+    /** Who may see coverage and start translations in the admin. Default: any signed-in user. */
+    canManage?: (args: {
+        req: PayloadRequest;
+    }) => boolean | Promise<boolean>;
+    /** Who may change Translation settings (pause, languages, glossary…). Default: `canManage`. */
+    canConfigure?: (args: {
+        req: PayloadRequest;
+    }) => boolean | Promise<boolean>;
+    /** Add the Translation settings global (admin-editable pause, skipped languages, glossary, usage). Default true. */
+    settings?: boolean;
+    /** Slug of the settings global. Default `polyglot-settings`. */
+    settingsSlug?: string;
+    /** Admin group of the settings global. Default `Settings`. */
+    settingsGroup?: string;
+    /** Add the Polyglot admin view (coverage per collection and language, Translate now). Default true. */
+    adminView?: boolean;
+    /** Translate after a document is published (in Next.js, after the response). Default true. */
+    onPublish?: boolean;
+    /** Called once when translation stops (credits, quota, key), e.g. to email admins. Not called again until it has recovered. */
+    onStop?: (args: {
+        payload: Payload;
+        stop: EngineStop;
+    }) => void | Promise<void>;
+};
+export type ResolvedPolyglot = Required<Omit<PolyglotOptions, 'enabled' | 'include' | 'writeContext' | 'describe' | 'translatorFor' | 'maxCharactersPerRun' | 'canManage' | 'canConfigure' | 'onStop'>> & Pick<PolyglotOptions, 'enabled' | 'include' | 'writeContext' | 'describe' | 'translatorFor' | 'maxCharactersPerRun' | 'canManage' | 'canConfigure' | 'onStop'> & {
+    defaultLocale: string;
+    locales: string[];
+};
+export type DocumentResult = {
+    translated: string[];
+    upToDate: string[];
+    reviewedStale: string[];
+    failed: string[];
+    pending: string[];
+};

package/dist/types.js

@@ -0,0 +1,13 @@
+/**
+ * Thrown by a translator when it cannot continue until someone acts (credits,
+ * quota, key): the run stops cleanly, admins are told why, and the next run
+ * after the fix continues.
+ */
+export class QuotaExceededError extends Error {
+    reason;
+    constructor(message = 'Translation quota used up', reason = 'quota') {
+        super(message);
+        this.reason = reason;
+        this.name = 'QuotaExceededError';
+    }
+}

package/package.json

@@ -1,6 +1,88 @@
 {
   "name": "@captiveorigins/payload-plugin-polyglot",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.2.0",
+  "description": "Keeps all your Payload content translated into every language you have, a little at a time: finds content on its own, translates on publish and in a sweep, remembers every sentence, skips placeholder text, and never overwrites a human-reviewed translation.",
+  "keywords": [
+    "payload",
+    "payload-plugin",
+    "payloadcms",
+    "i18n",
+    "localization",
+    "translation",
+    "machine-translation",
+    "claude"
+  ],
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/mgilardi/payload-plugin-polyglot.git"
+  },
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./claude": {
+      "types": "./dist/claude.d.ts",
+      "import": "./dist/claude.js"
+    },
+    "./rsc": {
+      "types": "./dist/rsc/index.d.ts",
+      "import": "./dist/rsc/index.js"
+    },
+    "./client": {
+      "types": "./dist/client/index.d.ts",
+      "import": "./dist/client/index.js"
+    }
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "engines": {
+    "node": ">=20.9"
+  },
+  "scripts": {
+    "build": "rm -rf dist && tsc -p tsconfig.build.json",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run",
+    "prepack": "npm run build",
+    "prepublishOnly": "npm run typecheck && npm test"
+  },
+  "peerDependencies": {
+    "@anthropic-ai/sdk": ">=1.0.0",
+    "@payloadcms/next": "^3.80.0",
+    "@payloadcms/ui": "^3.80.0",
+    "next": ">=15",
+    "payload": "^3.80.0",
+    "react": ">=19"
+  },
+  "peerDependenciesMeta": {
+    "@anthropic-ai/sdk": {
+      "optional": true
+    },
+    "next": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@anthropic-ai/sdk": "1.3.0",
+    "@payloadcms/db-mongodb": "3.92.1",
+    "@payloadcms/next": "3.92.1",
+    "@payloadcms/ui": "3.92.1",
+    "@types/node": "^24.0.0",
+    "@types/react": "^19.3.0",
+    "graphql": "^16.14.2",
+    "mongodb-memory-server": "^11.3.0",
+    "next": "16.3.3",
+    "payload": "3.92.1",
+    "react": "^19.3.1",
+    "react-dom": "^19.3.1",
+    "typescript": "^5.9.0",
+    "vitest": "^3.2.0"
+  }
+}