@elytracms/studio 0.0.102 → 0.0.104

package/dist/lib/ai/translate-document.js

@@ -2,21 +2,30 @@
  * `translateDocument` — the full-document AI translation engine (EC-332).
  *
  * ONE implementation behind two surfaces (the detail-editor "Translate document" header
- * action + the chat tool). It walks a document's LOCALIZED fields, computes the units
- * still **connected to base** (EC-318 — never overwriting a manually diverged value),
- * translates them with bounded client-side concurrency over the EC-331 provider, and
- * returns a NEW draft with the translations filled in. The caller writes that draft
- * through the operations layer with the user's session (provenance-tagged) — the engine
- * itself has no data access, so the deterministic FakeProvider drives the whole flow.
+ * action + the chat tool). It walks a document's LOCALIZED fields, plans the field ×
+ * locale values to write ({@link planTranslation} — the ONE pre-provider judge, EC-752),
+ * translates their units with bounded client-side concurrency over the EC-331 provider,
+ * and returns a NEW draft with the translations in. In the default mode (`fill-empty`)
+ * only values still **connected to base** are filled (EC-318 — a value with its own
+ * translation is never overwritten); `retranslate-all` (EC-752, the dialog's "Re-translate
+ * everything", behind a confirm that lists what it replaces) also REPLACES a value with its
+ * own translation, rebuilt from the base. The caller writes that draft through the
+ * operations layer with the user's session (provenance-tagged) — the engine itself has no
+ * data access, so the deterministic FakeProvider drives the whole flow.
  *
- * Unit-walk semantics (fill-empty-only, the default):
+ * Unit-walk semantics:
  *  - target locales = configured locales minus the document's base locale;
  *  - only LOCALIZED, writer-unlocked fields participate (a non-localized field is shared
  *    across locales — there is nothing per-locale to fill; EC-716: an `ai.exclude` or
  *    `form.readOnly` position, at any depth, is no unit — and no derived slug either);
- *  - a field/locale is translated only when it is CONNECTED (no per-locale key yet) and
- *    the base value is non-empty — so writing the translation DIVERGES it, which is
- *    exactly what makes a re-run an idempotent no-op;
+ *  - `fill-empty`: a field/locale is translated only when it is CONNECTED (no per-locale
+ *    key yet) and the base value is non-empty — so writing the translation DIVERGES it,
+ *    which is exactly what makes a re-run an idempotent no-op. `retranslate-all`: also when
+ *    it holds its own value — and then a field/locale is written only WHOLE, when every
+ *    PLANNED unit of it landed (a dropped unit, a rejected block, a cancel or an unreadable
+ *    position leaves the target's value as it was, reported); its locked content is kept
+ *    (EC-738). A text position the base leaves empty is no unit and no part of that judgement
+ *    (EC-759): the rebuilt value takes the base's state there, as it does for a number;
  *  - EVERY text position of the base value is a unit (EC-593, ratified 4e): the walk is
  *    core's ONE value-position traversal, so a `text` position is one unit and a
  *    `richText` / `blocks` sequence one unit per translatable block (structure preserved
@@ -28,10 +37,14 @@
  *  - the target bucket is the BASE value with its text translated — every other value
  *    (an image, a select, a number) carried from it, the displayed-value rule (2c);
  *  - whatever cannot be translated is REPORTED (`skipped`), never silently left: a
- *    position whose value is unreadable, a unit the provider did not return, a rich-text
- *    block whose translation did not decode token-faithfully (its original is kept);
- *  - SLUG fields are never model-translated: after the text fills land, each empty
- *    per-locale slug is derived (EC-315 `autoDeriveSlug`) from the translated source.
+ *    position whose value is unreadable (EC-752: text inside a value the traversal cannot
+ *    read too), a unit the provider did not return, a rich-text block whose translation did
+ *    not decode token-faithfully (fill-empty keeps the source's block in the fill;
+ *    retranslate-all leaves the whole target value as it was);
+ *  - SLUG fields are never model-translated: after the text fills land, each per-locale
+ *    slug that still FOLLOWS the base is derived (EC-315 `autoDeriveSlug`) from the
+ *    translated source — in both modes: a slug the editor set is never touched (EC-752 — a
+ *    changed slug moves the page's address, and nothing redirects it).
  *
  * Batching (improves on the Sanity precedent's context-blind per-block calls): units are
  * grouped per target locale and chunked into batches; each batch is ONE `generate` call
@@ -39,7 +52,7 @@
  * translates with cross-block context. Bounded concurrency (default 6) keeps a realistic
  * document well under the route's per-minute rate limit.
  */
-import { autoDeriveSlug, createCollectionRegistry, fieldValuePosition, isFieldLocalized, isWriterLocked, resolveField, slugFieldOf, validateDocument, valuesInLocale, writeField, } from '@elytracms/core/cms-core';
+import { autoDeriveSlug, childPositions, createCollectionRegistry, declarationMayHold, fieldValuePosition, isFieldLocalized, isOpaquePosition, isWriterLocked, resolveField, slugFieldOf, validateDocument, valuesInLocale, writeField, } from '@elytracms/core/cms-core';
 import { introducedErrors, warningsOf } from './proposal-validation.js';
 import { fieldWritePatch } from '../cms/field-patch.js';
 import { fieldConnection, hasLocaleVariant } from '../locale/index.js';
@@ -88,6 +101,34 @@ function positionId(locale, field, position) {
     const key = positionKey(position);
     return key === '' ? `${locale}|${field}` : `${locale}|${field}|${key}`;
 }
+/** A declaration whose value is text a writer may translate (a unit, or a sequence of them). */
+function isTranslatableDeclaration(field) {
+    return (field.type === 'text' || field.type === 'richText' || field.type === 'blocks') && !isWriterLocked(field);
+}
+/**
+ * EC-752 (the EC-748 traversal contract, read by translate): every value at or below
+ * `root` the traversal cannot read (an {@link OpaquePosition} — a holder stored in a shape
+ * its declaration no longer reads) whose declaration may hold translatable text. Its text
+ * is no unit — it cannot be found — so the caller REPORTS it (`unreadable`), never leaves
+ * it untranslated with no word. A writer-locked position is not descended (no unit, never
+ * a skip — EC-716), exactly as {@link textPositionsOf} does.
+ */
+function unreadableTextUnder(root, ctx) {
+    const out = [];
+    const visit = (position) => {
+        if (isWriterLocked(position.field) || position.field.type === 'text')
+            return;
+        for (const child of childPositions(position, ctx)) {
+            if (!isOpaquePosition(child))
+                visit(child);
+            else if (!isWriterLocked(child.declaration) && declarationMayHold(child.declaration, isTranslatableDeclaration)) {
+                out.push(child);
+            }
+        }
+    };
+    visit(root);
+    return out;
+}
 /** A minimal bounded-concurrency limiter (the precedent's `createLimiter`). */
 function createLimiter(max) {
     let active = 0;
@@ -196,31 +237,39 @@ function chunk(items, size) {
         out.push(items.slice(i, i + size));
     return out;
 }
-export async function translateDocument(input) {
-    const { provider, collection, document, localeConfig } = input;
+/** A skip entry — the component it sits in, when it does. */
+function skipOf(field, locale, site, reason, at = site.at) {
+    return { field, locale, at, ...(site.component ? { component: { ...site.component } } : {}), reason };
+}
+/**
+ * EC-752 — THE pre-provider judge: which field × locale values a translation of `input`
+ * writes when every unit lands (`fill` / `replace`), and what it already knows it will not
+ * translate (`locked`, `unreadable`). Pure: no provider, no store, no clock. The studio's
+ * confirm lists it; {@link translateDocument} runs exactly it — one judge, never a second
+ * count. See {@link TranslatePlan} for why it is an upper bound.
+ */
+export function planTranslation(input) {
+    return planRun(input).plan;
+}
+function planRun(input) {
+    const { collection, document, localeConfig } = input;
     const mode = input.mode ?? 'fill-empty';
+    // EC-752 rule 2: a re-translation writes a field × locale WHOLE or not at all — it
+    // replaces a value someone made, so a part of it must never become the base's text.
+    const whole = mode === 'retranslate-all';
     // EC-580: the base is the project's default locale — the only default there is,
     // and the one the translate dialog names (MODE #12: the engine used to translate
     // FROM a per-document default while the dialog said "from EN").
     const base = localeConfig.default;
     const targets = (input.targetLocales ?? localeConfig.locales).filter((l) => l !== base);
-    const provenance = aiProvenance(provider, input.now ?? new Date().toISOString());
     // EC-593: the traversal reaches component props through the validator's own lookup.
     const lookupManifest = input.validateOptions?.lookupManifest;
     const positionCtx = lookupManifest ? { lookupManifest } : {};
-    // 1. Walk every text position of each connected bucket's base value → units + plans.
+    // Walk every text position of each eligible bucket's base value → units + plans.
     const units = [];
-    const plans = new Map();
+    const richPlans = new Map();
     const pending = [];
     const skipped = [];
-    const slugFields = collection.fields.filter((f) => isSlugField(f, collection) && !isWriterLocked(f) && isFieldLocalized(collection, f.name));
-    const skip = (field, locale, position, reason, at = position.at) => skipped.push({
-        field,
-        locale,
-        at,
-        ...(position.component ? { component: { ...position.component } } : {}),
-        reason,
-    });
     for (const field of collection.fields) {
         if (isWriterLocked(field))
             continue;
@@ -246,35 +295,38 @@ export async function translateDocument(input) {
             const kept = withLockedFrom(root, current, positionCtx);
             const changes = lockedChanges(fieldValuePosition(field, current, { locale }), kept, positionCtx);
             if (changes.lost.length > 0 || changes.created.length > 0) {
-                skip(field.name, locale, root, 'locked');
+                skipped.push(skipOf(field.name, locale, root, 'locked'));
                 continue;
             }
-            const before = units.length;
+            const bucketUnits = [];
+            const bucketPlans = new Map();
+            const bucketSkips = [];
+            const unreadable = (site) => bucketSkips.push(skipOf(field.name, locale, site, 'unreadable'));
             // EC-593: every text position of the base value — the converter module's fold
             // over core's ONE value-position traversal.
-            for (const position of textPositionsOf(fieldValuePosition(field, conn.baseValue, { locale: base }), positionCtx)) {
+            for (const position of textPositionsOf(root, positionCtx)) {
                 const { value } = position;
                 if (value === undefined || value === null)
                     continue;
                 if (position.field.type === 'text') {
                     if (typeof value !== 'string')
-                        skip(field.name, locale, position, 'unreadable');
+                        unreadable(position);
                     else if (value.trim().length > 0) {
-                        units.push({ kind: 'text', id: positionId(locale, field.name, position), locale, field: field.name, text: value });
+                        bucketUnits.push({ kind: 'text', id: positionId(locale, field.name, position), locale, field: field.name, text: value });
                     }
                     continue;
                 }
                 const extracted = extractRichTextUnits(value);
                 if (!extracted.readable) {
-                    skip(field.name, locale, position, 'unreadable');
+                    unreadable(position);
                     continue;
                 }
                 if (extracted.units.length === 0)
                     continue;
                 const planId = positionId(locale, field.name, position);
-                plans.set(planId, extracted.plan);
+                bucketPlans.set(planId, extracted.plan);
                 for (const block of extracted.units) {
-                    units.push({
+                    bucketUnits.push({
                         kind: 'richText',
                         id: `${planId}|${block.id}`,
                         locale,
@@ -285,10 +337,53 @@ export async function translateDocument(input) {
                     });
                 }
             }
-            if (units.length > before)
-                pending.push({ field, locale, base: conn.baseValue, current });
+            // EC-752 (8b): text inside a value the traversal cannot read is no unit — reported, not
+            // silent (once per location: an unreadable sequence was named by the walk above).
+            for (const opaque of unreadableTextUnder(root, positionCtx)) {
+                const at = JSON.stringify(opaque.at);
+                if (!bucketSkips.some((entry) => JSON.stringify(entry.at) === at))
+                    unreadable(opaque);
+            }
+            skipped.push(...bucketSkips);
+            // Rule 2, known before the provider: a value with text that cannot be translated is
+            // never re-translated in part — its whole bucket stays as it was (and spends no call).
+            if (whole && bucketSkips.length > 0)
+                continue;
+            if (bucketUnits.length === 0)
+                continue;
+            units.push(...bucketUnits);
+            for (const [id, plan] of bucketPlans)
+                richPlans.set(id, plan);
+            pending.push({ field, locale, base: conn.baseValue, current, kind: conn.connected ? 'fill' : 'replace' });
         }
     }
+    return {
+        plan: {
+            mode,
+            base,
+            targets,
+            buckets: pending.map(({ field, locale, kind }) => ({ field: field.name, locale, kind })),
+            skipped,
+            total: units.length,
+        },
+        units,
+        richPlans,
+        pending,
+        positionCtx,
+    };
+}
+export async function translateDocument(input) {
+    const { provider, collection, document, localeConfig } = input;
+    // EC-752 (rule 4): the run executes the ONE pre-provider plan — the list the studio's
+    // confirm showed is the list the units come from.
+    const { plan, units, richPlans: plans, pending, positionCtx } = planRun(input);
+    const { mode, base, targets } = plan;
+    const whole = mode === 'retranslate-all';
+    const provenance = aiProvenance(provider, input.now ?? new Date().toISOString());
+    const lookupManifest = input.validateOptions?.lookupManifest;
+    const skipped = [...plan.skipped];
+    const skip = (field, locale, position, reason, at = position.at) => skipped.push(skipOf(field, locale, position, reason, at));
+    const slugFields = collection.fields.filter((f) => isSlugField(f, collection) && !isWriterLocked(f) && isFieldLocalized(collection, f.name));
     const total = units.length;
     // 2. Batch per locale, translate with bounded concurrency, honour the abort signal.
     const batchSize = Math.max(1, input.batchSize ?? DEFAULT_TRANSLATE_BATCH_SIZE);
@@ -331,11 +426,12 @@ export async function translateDocument(input) {
     //    text position replaced (the traversal's writing fold), then slug derivation.
     //    `translated` / `perLocale` count applied UNITS (comparable to `total`).
     let next = document;
-    // EC-595 A2: every bucket this run writes, emitted as its patch at the end.
+    // EC-595 A2: every bucket this run writes, emitted as its patch at the end (EC-752:
+    // with what the write was — the outcome the studio reports).
     const written = new Map();
-    const write = (field, locale, value) => {
+    const write = (field, locale, value, kind) => {
         next = writeField(collection, next, field, locale, value, lookupManifest ? { lookupManifest } : {});
-        written.set(`${locale}|${field}`, { field, locale });
+        written.set(`${locale}|${field}`, { field, locale, kind });
     };
     const perLocale = {};
     let translated = 0;
@@ -343,10 +439,15 @@ export async function translateDocument(input) {
         perLocale[locale] = (perLocale[locale] ?? 0) + count;
         translated += count;
     };
+    // The plan's units by the position they translate — the rebuild's ONE reader of what was
+    // planned (EC-759): a text unit by its position id, a rich-text position's blocks by its plan.
+    const textUnitAt = new Map();
     const blocksOf = new Map();
     for (const unit of units) {
-        if (unit.kind !== 'richText')
+        if (unit.kind === 'text') {
+            textUnitAt.set(unit.id, unit);
             continue;
+        }
         const list = blocksOf.get(unit.plan) ?? new Map();
         list.set(unit.blockId, unit);
         blocksOf.set(unit.plan, list);
@@ -357,18 +458,35 @@ export async function translateDocument(input) {
         'content',
         ...blockId.split('.').map(Number),
     ];
-    for (const { field, locale, base: baseValue, current } of pending) {
+    for (const { field, locale, base: baseValue, current, kind } of pending) {
         let applied = 0;
+        // EC-752 rule 2: whether every PLANNED unit of the bucket landed — a planned unit never
+        // sent (a cancel), not returned, or a block whose tokens did not survive makes it
+        // incomplete. A position the plan made no unit of is no part of it (EC-759).
+        let complete = true;
         // EC-593: the base value with each translated text position replaced — the
         // converter module's writing fold; every other value is carried from the base.
         const value = rebuildTextPositions(fieldValuePosition(field, baseValue, { locale: base }), positionCtx, (position) => {
             const id = positionId(locale, field.name, position);
             if (position.field.type === 'text') {
-                if (!attempted.has(id))
+                // EC-759 (Codex r10 F48): the PLANNED unit at this position, read as the rich-text
+                // branch reads its plan. None — the base leaves the position empty (absent, `null`,
+                // `''`, whitespace), or its value is unreadable and reported by the plan — means
+                // the base's value is carried, as every non-text value is, and completeness is
+                // untouched. Judging "every text position" by the units whose batch ran read a
+                // position never planned as a cancel, and the whole-bucket rule then dropped a
+                // listed replacement with no skip.
+                const unit = textUnitAt.get(id);
+                if (!unit)
+                    return position.value;
+                if (!attempted.has(unit.id)) {
+                    complete = false;
                     return position.value;
-                const text = returned(id);
+                }
+                const text = returned(unit.id);
                 if (text === undefined) {
                     skip(field.name, locale, position, 'not-returned');
+                    complete = false;
                     return position.value;
                 }
                 applied += 1;
@@ -379,11 +497,15 @@ export async function translateDocument(input) {
                 return position.value;
             const blockTranslations = new Map();
             for (const [blockId, unit] of blocksOf.get(id) ?? []) {
-                if (!attempted.has(unit.id))
+                if (!attempted.has(unit.id)) {
+                    complete = false;
                     continue;
+                }
                 const text = returned(unit.id);
-                if (text === undefined)
+                if (text === undefined) {
                     skip(field.name, locale, position, 'not-returned', blockAt(position, blockId));
+                    complete = false;
+                }
                 else
                     blockTranslations.set(blockId, text);
             }
@@ -392,6 +514,7 @@ export async function translateDocument(input) {
             const rebuilt = rebuildRichTextReport(plan, blockTranslations);
             for (const blockId of rebuilt.rejected) {
                 skip(field.name, locale, position, 'structure-changed', blockAt(position, blockId));
+                complete = false;
             }
             if (rebuilt.applied.length === 0)
                 return position.value;
@@ -399,15 +522,22 @@ export async function translateDocument(input) {
             return rebuilt.value;
         });
         if (applied === 0)
-            continue; // nothing landed — the bucket stays connected to base
-        // EC-738: the target's locked positions, by key (judged in step 1).
-        write(field.name, locale, withLockedFrom(fieldValuePosition(field, value, { locale }), current, positionCtx));
+            continue; // nothing landed — the bucket stays as it was
+        // EC-752 rule 2: a re-translation writes a field × locale only WHOLE. A part of it is
+        // never replaced by the base's text: the target keeps its value, the skips say why.
+        if (whole && !complete)
+            continue;
+        // EC-738: the target's locked positions, by key (judged in the plan).
+        write(field.name, locale, withLockedFrom(fieldValuePosition(field, value, { locale }), current, positionCtx), kind);
         bump(locale, applied);
     }
-    // 4. Slugs: derive each empty per-locale slug from the (now translated) source. Never
-    //    model-translated; only for a still-connected slug whose SOURCE field this run
-    //    actually translated (so the derived slug differs from the base — no pointless
-    //    divergence, and an aborted run with no translations derives nothing).
+    // 4. Slugs: derive each per-locale slug that still follows the base from the (now
+    //    translated) source. Never model-translated; only for a still-connected slug whose
+    //    SOURCE field holds its own value in that locale (so the derived slug differs from the
+    //    base — no pointless divergence, and an aborted run with no translations derives
+    //    nothing). EC-752 rule 1 — in BOTH modes: a slug the editor set is never touched. A
+    //    changed slug moves the page's address and nothing redirects the old one, so a
+    //    re-translation silently changing the URL was the measured ship-stopper.
     let slugsDerived = 0;
     for (const field of slugFields) {
         const sourceField = slugSourceField(field, collection);
@@ -415,7 +545,7 @@ export async function translateDocument(input) {
             const conn = fieldConnection(collection, next, field.name, locale, localeConfig);
             if (!conn)
                 continue;
-            if (mode !== 'retranslate-all' && !conn.connected)
+            if (!conn.connected)
                 continue;
             const sourceTranslated = sourceField
                 ? hasLocaleVariant(next, sourceField, locale)
@@ -426,7 +556,7 @@ export async function translateDocument(input) {
             const derived = autoDeriveSlug(field.slug, collection, valuesInLocale(collection, next, locale, localeConfig));
             if (!derived || derived === conn.baseValue)
                 continue;
-            write(field.name, locale, derived);
+            write(field.name, locale, derived, 'slug');
             slugsDerived += 1;
         }
     }
@@ -442,9 +572,17 @@ export async function translateDocument(input) {
         ...(input.validateOptions ?? {}),
     };
     const producedIssues = validateDocument(registry, next, validateOptions);
+    const writes = [...written.values()].map(({ field, locale, kind }) => ({
+        field,
+        locale,
+        kind,
+        patch: fieldWritePatch(collection, next, field, locale),
+    }));
     return {
         document: next,
-        patches: [...written.values()].map(({ field, locale }) => fieldWritePatch(collection, next, field, locale)),
+        plan,
+        writes,
+        patches: writes.map((entry) => entry.patch),
         total,
         translated,
         cancelled,