champollion 0.3.4 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/README.md +41 -26
  2. package/bin/cli.js +53 -5
  3. package/index.js +63 -2
  4. package/lib/api-key.js +17 -4
  5. package/lib/autofix.js +82 -37
  6. package/lib/bridge/method_bridge.py +15 -3
  7. package/lib/cards/reader.js +34 -0
  8. package/lib/cards/remote.js +15 -0
  9. package/lib/cards/search-names.js +178 -0
  10. package/lib/command-help.js +286 -85
  11. package/lib/commands/audit.js +10 -3
  12. package/lib/commands/card.js +583 -226
  13. package/lib/commands/doctor.js +54 -18
  14. package/lib/commands/help.js +37 -32
  15. package/lib/commands/init.js +1709 -101
  16. package/lib/commands/integrity.js +127 -40
  17. package/lib/commands/leaderboard.js +187 -67
  18. package/lib/commands/models.js +9 -2
  19. package/lib/commands/provenance.js +7 -2
  20. package/lib/commands/recommend.js +43 -14
  21. package/lib/commands/register-corpus.js +632 -125
  22. package/lib/commands/seal-corpus.js +1 -1
  23. package/lib/commands/status.js +564 -27
  24. package/lib/commands/submit.js +17 -12
  25. package/lib/commands/sync.js +31 -7
  26. package/lib/commands/tm.js +15 -9
  27. package/lib/commands/verify.js +27 -3
  28. package/lib/commands/wrap.js +63 -5
  29. package/lib/commands/xliff.js +135 -64
  30. package/lib/commercial-eligibility.js +1 -1
  31. package/lib/config.js +230 -17
  32. package/lib/content-estimate.js +96 -0
  33. package/lib/content-refusals.js +284 -0
  34. package/lib/content-review.js +372 -0
  35. package/lib/content-sync.js +1136 -345
  36. package/lib/content.js +94 -7
  37. package/lib/corpus-registration.mjs +194 -35
  38. package/lib/cost-label.js +29 -0
  39. package/lib/cost-report.js +727 -79
  40. package/lib/diff.js +38 -4
  41. package/lib/docusaurus-sync.js +975 -262
  42. package/lib/edit-distance.js +31 -0
  43. package/lib/fallback.js +1102 -0
  44. package/lib/file-scope.js +106 -0
  45. package/lib/flatten.js +80 -3
  46. package/lib/flutter-locales.js +124 -0
  47. package/lib/format.js +266 -12
  48. package/lib/hash.js +146 -21
  49. package/lib/icu-structure.js +929 -0
  50. package/lib/integrity.js +223 -75
  51. package/lib/language-pair.js +157 -0
  52. package/lib/lint.js +78 -16
  53. package/lib/local-only-marks.js +106 -0
  54. package/lib/locale-layout.js +1103 -0
  55. package/lib/locale-state.js +575 -0
  56. package/lib/methods/anthropic.js +5 -0
  57. package/lib/methods/apertium.js +6 -3
  58. package/lib/methods/api.js +138 -25
  59. package/lib/methods/base.js +17 -0
  60. package/lib/methods/coaching-data.js +153 -0
  61. package/lib/methods/content-separator.js +43 -0
  62. package/lib/methods/deepl.js +1 -1
  63. package/lib/methods/direct-llm.js +257 -103
  64. package/lib/methods/external.js +146 -63
  65. package/lib/methods/gemini.js +1 -0
  66. package/lib/methods/google-translate.js +1 -0
  67. package/lib/methods/http-utils.js +41 -0
  68. package/lib/methods/libretranslate.js +7 -2
  69. package/lib/methods/llm-coached.js +71 -129
  70. package/lib/methods/llm.js +87 -33
  71. package/lib/methods/local.js +93 -10
  72. package/lib/methods/microsoft-translator.js +1 -2
  73. package/lib/methods/openai.js +4 -2
  74. package/lib/methods/openrouter-client.js +20 -19
  75. package/lib/methods/openrouter-pricing.js +150 -13
  76. package/lib/methods/prompt-methods.js +20 -0
  77. package/lib/methods/provider-pricing.js +42 -1
  78. package/lib/methods/request-capture.js +104 -0
  79. package/lib/methods/tilde.js +1 -1
  80. package/lib/methods/translated.js +1 -2
  81. package/lib/missing-key.js +93 -0
  82. package/lib/models.js +102 -55
  83. package/lib/name-rules.js +32 -0
  84. package/lib/named-keys.js +172 -0
  85. package/lib/no-translate.js +4 -3
  86. package/lib/output.js +160 -19
  87. package/lib/pairs.js +586 -30
  88. package/lib/placeholders.js +394 -0
  89. package/lib/plugins.js +8 -0
  90. package/lib/plural-gap-redo.js +109 -0
  91. package/lib/plurals.js +323 -0
  92. package/lib/po.js +1187 -0
  93. package/lib/public-catalogue.js +74 -0
  94. package/lib/recommend.js +527 -32
  95. package/lib/redo.js +95 -0
  96. package/lib/refusal-category.js +44 -0
  97. package/lib/registers.js +255 -11
  98. package/lib/repair-script.js +20 -13
  99. package/lib/scripts.js +6 -1
  100. package/lib/seal.mjs +4 -3
  101. package/lib/sealed-qualifier.mjs +1 -1
  102. package/lib/segment.js +24 -8
  103. package/lib/seo.js +19 -9
  104. package/lib/serve.js +43 -6
  105. package/lib/shared-output-seed.js +164 -0
  106. package/lib/source-contexts.js +39 -0
  107. package/lib/submit.mjs +57 -5
  108. package/lib/sync.js +2923 -474
  109. package/lib/terminology.js +13 -4
  110. package/lib/tm-evict.js +179 -0
  111. package/lib/tm-seed.js +5 -2
  112. package/lib/tm.js +818 -36
  113. package/lib/translate-pair.js +639 -34
  114. package/lib/translate.js +78 -5
  115. package/lib/types.js +22 -3
  116. package/lib/validate.js +996 -21
  117. package/lib/verify.js +1332 -104
  118. package/lib/watch.js +32 -13
  119. package/lib/xliff.js +44 -3
  120. package/package.json +1 -1
  121. package/shared/CORPORA-CARDS.md +2 -0
  122. package/shared/cards-fallback.json +1 -1
  123. package/shared/curated-orthography-conventions.json +26 -8
  124. package/shared/gettext-plural-forms.json +45 -0
  125. package/shared/licenses.json +99 -3
  126. package/shared/method-registry.json +2 -0
  127. package/shared/metric-registry.json +96 -18
  128. package/shared/retired-model-aliases.json +9 -0
  129. package/shared/schemas/champollion-plugin.schema.json +4 -0
  130. package/shared/schemas/corpora-card.schema.json +8 -2
  131. package/shared/schemas/human-services.schema.json +1 -1
  132. package/shared/schemas/method-index-record.schema.json +67 -0
  133. package/shared/schemas/method-registry.schema.json +4 -0
  134. package/shared/schemas/metric-registry.schema.json +55 -1
  135. package/shared/schemas/retired-model-aliases.schema.json +32 -0
  136. package/shared/docent/corpus.json +0 -11739
  137. package/shared/model-aliases.json +0 -7
  138. package/shared/schemas/model-aliases.schema.json +0 -27
@@ -0,0 +1,1102 @@
1
+ /**
2
+ * fallback.js — the per-pair fallback method: shared machinery for every lane.
3
+ *
4
+ * A pair may name a second method (`"fallback": { "method": … }`, resolved by
5
+ * lib/pairs.js resolveFallbackForPair into a full pair config). The pair's own
6
+ * method runs first; what it could not translate safely — keys the quality
7
+ * gate refused or that came back empty, Markdown blocks it dropped or damaged,
8
+ * front-matter fields it hollowed — goes to the fallback ONCE, through the
9
+ * same gate. What the fallback translates is cached under its own TM key, so
10
+ * the cache always says which method produced a value. What both fail stays
11
+ * failed exactly as without a fallback ('[EN] ' remains the content lanes'
12
+ * last resort).
13
+ *
14
+ * The TM ladder, everywhere: the pair's cache → the fallback's cache → the
15
+ * pair's method → the fallback's method. Text the fallback already paid for
16
+ * is reused, not re-sent to the primary on every re-run (re-ask the primary
17
+ * with --fresh or --retranslate).
18
+ *
19
+ * This module holds what the key-value, Hugo-content and Docusaurus lanes
20
+ * share: the --max-cost guard for fallback batches, the per-pair tally and its
21
+ * [FALLBACK] report line, and the content-lane helpers (front-matter fields,
22
+ * Markdown blocks, whole pages). The key-value pipeline itself is
23
+ * translateWithFallback in lib/translate-pair.js.
24
+ */
25
+
26
+ import { estimateCost } from './pairs.js';
27
+ import { lookupTM, lookupTMValidated, tmMethodKey } from './tm.js';
28
+ import { createTMEvictor } from './tm-evict.js';
29
+ import { translatableBlockSources } from './content-estimate.js';
30
+ import { checkContentPreservation, contentGateFault, sharedOutputReason, isKeepAsWrittenFault } from './validate.js';
31
+ import { restoreBlocks, hasOrphanedPlaceholders, PLACEHOLDER_PREFIX, PLACEHOLDER_SUFFIX } from './content.js';
32
+ import { contentKeyUnits } from './config.js';
33
+ import { output } from './output.js';
34
+ import { refusalCategory, notePrimaryReason } from './refusal-category.js';
35
+
36
+ export { refusalCategory, notePrimaryReason };
37
+
38
+ // ── TM keys ──────────────────────────────────────────────────────────
39
+
40
+ /**
41
+ * The TM method keys whose entries a pair's output may have come from: its
42
+ * own, then its fallback's. Echo confirmation (lib/diff.js) and verify must
43
+ * consult both — a value the fallback produced is cached under the fallback.
44
+ *
45
+ * @param {object} pairConfig
46
+ * @returns {string[]}
47
+ */
48
+ export function tmKeysForPair(pairConfig) {
49
+ const keys = [tmMethodKey(pairConfig)];
50
+ if (pairConfig?.fallback) keys.push(tmMethodKey(pairConfig.fallback));
51
+ return keys;
52
+ }
53
+
54
+ /**
55
+ * True when the TM holds `value` for `text` under any of `tmKeys` — the
56
+ * "the pipeline produced this" proof the echo checks need.
57
+ *
58
+ * @param {object} tm
59
+ * @param {string} text - Source text as cached (tmSourceText)
60
+ * @param {string} locale
61
+ * @param {string[]} tmKeys - From tmKeysForPair
62
+ * @param {string} value
63
+ * @returns {boolean}
64
+ */
65
+ export function tmHoldsValue(tm, text, locale, tmKeys, value) {
66
+ return tmKeys.some(k => lookupTM(tm, text, locale, k) === value);
67
+ }
68
+
69
+ // ── Cost guard ───────────────────────────────────────────────────────
70
+
71
+ /**
72
+ * The --max-cost guard for fallback batches.
73
+ *
74
+ * The pre-run estimate cannot know which keys or blocks the primary will
75
+ * fail, so fallback spend is not in it. Each fallback batch is priced just
76
+ * before it runs, with the same estimator the run's estimate used
77
+ * (pairs.js estimateCost → the method's own estimateCost). Under a cap, a
78
+ * batch runs only if the spend already committed — the pre-run estimate the
79
+ * cap admitted, plus every fallback batch approved so far — plus this one
80
+ * stays within the cap. An unpriced fallback is refused: unknown is not
81
+ * free, the same rule the pre-run gate applies to the primary.
82
+ *
83
+ * Concurrency: approvals from parallel locales await the estimate, then
84
+ * check-and-commit synchronously, so two batches cannot both slip under the
85
+ * cap.
86
+ *
87
+ * @param {{ maxCost?: number|null, committed?: number }} [opts]
88
+ * @returns {{ maxCost: number|null, readonly committed: number,
89
+ * approve: (units: number, fallbackConfig: object) => Promise<{ ok: boolean, estimate: number|null, reason?: string }> }}
90
+ */
91
+ export function createFallbackBudget({ maxCost = null, committed = 0, cwd = null } = {}) {
92
+ let spent = Number.isFinite(committed) ? committed : 0;
93
+ return {
94
+ maxCost,
95
+ get committed() { return spent; },
96
+ async approve(units, fallbackConfig) {
97
+ if (maxCost === null || units <= 0) return { ok: true, estimate: null };
98
+ // cwd: the project, where a local fallback's endpoint may be set (.env).
99
+ const estimate = await estimateCost(units, fallbackConfig, cwd ? { cwd } : {});
100
+ const cost = estimate?.estimatedCost ?? null;
101
+ if (cost === null) {
102
+ return {
103
+ ok: false,
104
+ estimate: null,
105
+ reason: `its cost cannot be estimated (${estimate?.source || 'unknown pricing'}) and --max-cost is set — unknown is not free`,
106
+ };
107
+ }
108
+ if (spent + cost > maxCost) {
109
+ return {
110
+ ok: false,
111
+ estimate: cost,
112
+ reason: `~$${cost.toFixed(4)} on top of the ~$${spent.toFixed(4)} already committed would pass --max-cost $${maxCost.toFixed(4)}`,
113
+ };
114
+ }
115
+ spent += cost;
116
+ return { ok: true, estimate: cost };
117
+ },
118
+ };
119
+ }
120
+
121
+ /**
122
+ * Key-equivalents for a run of source characters — the unit the content
123
+ * lanes' estimates price (lib/cost-report.js priceContentChars).
124
+ *
125
+ * @param {number} chars
126
+ * @returns {number}
127
+ */
128
+ export function charsToKeyUnits(chars, pairConfig = {}) {
129
+ return contentKeyUnits(chars, pairConfig);
130
+ }
131
+
132
+ // ── Reporting ────────────────────────────────────────────────────────
133
+
134
+ /**
135
+ * A per-call fallback report (one file, one batch). Callers add these into
136
+ * a per-pair tally (addToTally) and print one [FALLBACK] line per pair.
137
+ *
138
+ * @param {object} fallbackConfig
139
+ * @returns {FallbackReport}
140
+ *
141
+ * @typedef {object} FallbackReport
142
+ * @property {string} method - The fallback's method
143
+ * @property {string|null} model
144
+ * @property {number} attempted - Units the primary failed and the fallback was asked to translate
145
+ * @property {number} accepted - Of those, how many the fallback translated and the gate accepted
146
+ * @property {number} cached - Units served from the fallback's cache (the primary was not re-asked)
147
+ * @property {boolean} primaryFailedWholesale - The primary returned nothing at all
148
+ * @property {{ reason: string, items: string[] }|null} skipped - Set when the
149
+ * fallback did not run (--max-cost); items are the keys/segments left failing
150
+ * @property {number} primaryAccepted - Units the pair's OWN method answered
151
+ * this run and the gate accepted (cache hits not counted) — with
152
+ * `accepted`, the run's fresh translations for the pair
153
+ * @property {Object<string, number>} primaryReasons - Why the units sent to
154
+ * the fallback were not the primary's: a refusal in a few words
155
+ * (refusalCategory) → how many
156
+ */
157
+ export function newFallbackReport(fallbackConfig) {
158
+ return {
159
+ method: fallbackConfig.method,
160
+ model: fallbackConfig.model || null,
161
+ attempted: 0,
162
+ accepted: 0,
163
+ cached: 0,
164
+ primaryFailedWholesale: false,
165
+ skipped: null,
166
+ // What the fallback produced (keys, or pages for the content lanes) —
167
+ // named in the [FALLBACK] line, so its output is findable in the files.
168
+ produced: [],
169
+ primaryAccepted: 0,
170
+ primaryReasons: {},
171
+ };
172
+ }
173
+
174
+ /**
175
+ * Above this share of a run's fresh translations for a pair, the fallback
176
+ * wrote most of it, and sync says so in one warning (warnFallbackMajority) —
177
+ * and `status` beside the share of the files it holds. A fallback is the
178
+ * second opinion for what the pair's method cannot translate safely; when it
179
+ * writes most of the output, what ships is the fallback's work (Round 11,
180
+ * school persona: every app string and 5 of 7 newsletter segments came from
181
+ * the weakest method measured, and nothing said so in context).
182
+ */
183
+ export const FALLBACK_MAJORITY_SHARE = 0.5;
184
+
185
+
186
+ /**
187
+ * Fold one call's report into a pair's running tally (mutates `tally`).
188
+ *
189
+ * @param {FallbackReport} tally
190
+ * @param {FallbackReport|null} report
191
+ * @returns {FallbackReport}
192
+ */
193
+ export function addToTally(tally, report) {
194
+ if (!report) return tally;
195
+ tally.attempted += report.attempted;
196
+ tally.accepted += report.accepted;
197
+ tally.cached += report.cached;
198
+ tally.primaryAccepted = (tally.primaryAccepted || 0) + (report.primaryAccepted || 0);
199
+ for (const [why, n] of Object.entries(report.primaryReasons || {})) {
200
+ tally.primaryReasons = tally.primaryReasons || {};
201
+ tally.primaryReasons[why] = (tally.primaryReasons[why] || 0) + n;
202
+ }
203
+ if (Array.isArray(report.produced) && report.produced.length > 0) {
204
+ tally.produced = tally.produced || [];
205
+ for (const k of report.produced) if (!tally.produced.includes(k)) tally.produced.push(k);
206
+ }
207
+ if (report.primaryFailedWholesale) tally.primaryFailedWholesale = true;
208
+ if (report.skipped) {
209
+ tally.skipped = tally.skipped || { reason: report.skipped.reason, items: [] };
210
+ tally.skipped.items.push(...report.skipped.items);
211
+ }
212
+ return tally;
213
+ }
214
+
215
+ /**
216
+ * The machine-readable form of a tally (the --json summary's per-locale
217
+ * `fallback` object).
218
+ *
219
+ * @param {FallbackReport} tally
220
+ * @returns {object}
221
+ */
222
+ export function fallbackSummary(tally) {
223
+ return {
224
+ method: tally.method,
225
+ model: tally.model,
226
+ attempted: tally.attempted,
227
+ accepted: tally.accepted,
228
+ failed: tally.attempted - tally.accepted,
229
+ cached: tally.cached,
230
+ // The pair's own method's accepted answers this run (with `accepted`:
231
+ // this run's fresh translations), and why the rest went to the fallback.
232
+ primaryAccepted: tally.primaryAccepted || 0,
233
+ ...(tally.primaryReasons && Object.keys(tally.primaryReasons).length > 0 && { primaryReasons: { ...tally.primaryReasons } }),
234
+ primaryFailedWholesale: tally.primaryFailedWholesale,
235
+ // Keys (or pages) whose text the fallback produced this run.
236
+ ...(Array.isArray(tally.produced) && tally.produced.length > 0 && { produced: [...tally.produced] }),
237
+ ...(tally.skipped && { skipped: { reason: tally.skipped.reason, items: [...tally.skipped.items] } }),
238
+ };
239
+ }
240
+
241
+ /**
242
+ * Print a pair's [FALLBACK] line(s). Silent when the fallback had nothing
243
+ * to do — a pair whose own method translated everything says nothing here.
244
+ *
245
+ * [FALLBACK] en:crk — 6 key(s) the primary (api) could not translate
246
+ * safely → translated by llm-coached (4 accepted, 2 still failing)
247
+ *
248
+ * @param {string} pairKey
249
+ * @param {string} primaryMethod
250
+ * @param {FallbackReport} tally
251
+ * @param {{ unit?: string }} [opts] - "key(s)" or "content segment(s)"
252
+ */
253
+ export function printFallbackReport(pairKey, primaryMethod, tally, { unit = 'key(s)', producedLabel = 'keys' } = {}) {
254
+ if (!tally) return;
255
+ const still = tally.attempted - tally.accepted;
256
+ // Which ones: the files do not say what the fallback wrote (Round 8 school
257
+ // + hospital personas), so the line names them, and status counts them.
258
+ const produced = Array.isArray(tally.produced) ? tally.produced : [];
259
+ const named = produced.length > 0
260
+ ? ` — ${producedLabel}: ${produced.slice(0, 8).join(', ')}${produced.length > 8 ? `, +${produced.length - 8} more` : ''}`
261
+ + ' (`champollion status` counts what each locale holds from the fallback)'
262
+ : '';
263
+ if (tally.skipped) {
264
+ const items = tally.skipped.items;
265
+ const shown = items.slice(0, 10).join(', ') + (items.length > 10 ? `, +${items.length - 10} more` : '');
266
+ output.warn(
267
+ `[FALLBACK] ${pairKey} — ${items.length} ${unit} the primary (${primaryMethod}) could not translate safely; `
268
+ + `the fallback (${tally.method}) was skipped: ${tally.skipped.reason}. Still untranslated: ${shown}`
269
+ );
270
+ }
271
+ if (tally.attempted > 0) {
272
+ const line = `[FALLBACK] ${pairKey} — ${tally.attempted} ${unit} the primary (${primaryMethod}) could not `
273
+ + `translate safely → translated by ${tally.method} (${tally.accepted} accepted, ${still} still failing)`
274
+ + (tally.cached > 0 ? `; ${tally.cached} more served from its cache` : '')
275
+ + named;
276
+ if (still > 0) output.warn(line);
277
+ else output.info(line);
278
+ } else if (tally.cached > 0) {
279
+ output.info(
280
+ `[FALLBACK] ${pairKey} — ${tally.cached} ${unit} served from the fallback's cache (${tally.method}); `
281
+ + `the primary (${primaryMethod}) was not asked again${named}`
282
+ );
283
+ }
284
+ }
285
+
286
+ /**
287
+ * When the fallback wrote most of a run's fresh translations for a pair
288
+ * (more than FALLBACK_MAJORITY_SHARE of them — the primary's accepted
289
+ * answers plus the fallback's), say it once, plainly: how many of how many,
290
+ * by which method and model, why the primary's answers were not used
291
+ * (counted), and what to consider. Silent otherwise, and for a run that
292
+ * translated nothing fresh (everything from the cache: `status` carries the
293
+ * standing share of the files).
294
+ *
295
+ * @param {string} pairKey
296
+ * @param {string} primaryMethod
297
+ * @param {FallbackReport} tally
298
+ * @param {{ unit?: string }} [opts] - "key(s)" or "content segment(s)"
299
+ * @returns {{ fromFallback: number, translated: number, share: number }|null} what was said (null: nothing)
300
+ */
301
+ export function warnFallbackMajority(pairKey, primaryMethod, tally, { unit = 'key(s)' } = {}) {
302
+ if (!tally) return null;
303
+ const fromFallback = tally.accepted || 0;
304
+ const translated = (tally.primaryAccepted || 0) + fromFallback;
305
+ if (translated === 0 || fromFallback / translated <= FALLBACK_MAJORITY_SHARE) return null;
306
+ const who = `${tally.method}${tally.model ? `, model ${tally.model}` : ''}`;
307
+ const reasons = Object.entries(tally.primaryReasons || {}).sort((a, b) => b[1] - a[1])
308
+ .map(([why, n]) => `${why} (${n})`).join('; ');
309
+ output.warn(
310
+ `[FALLBACK] ${pairKey} — most of this run's translations came from the fallback: ${fromFallback} of ${translated} ${unit} `
311
+ + `were written by ${who}, not by the pair's method (${primaryMethod})`
312
+ + (reasons ? `. Why ${primaryMethod}'s answers were not used, for the ${tally.attempted} ${unit} sent to the fallback: ${reasons}` : '')
313
+ + `. What ships is the fallback's work. Consider: ${primaryMethod} may not suit these strings; check what was written `
314
+ + '(`champollion verify` checks structure — a speaker checks meaning); or a stronger fallback ("fallback" in the pair\'s "pairs" entry).',
315
+ );
316
+ return { fromFallback, translated, share: fromFallback / translated };
317
+ }
318
+
319
+ // ── Content lanes ────────────────────────────────────────────────────
320
+
321
+ /**
322
+ * Read-time validation shared by every content TM read (see lookupTMValidated):
323
+ * the gate a fresh answer passes (lib/validate.js contentGateFault), so a
324
+ * cached block the gate refuses today — a heading turned into a sentence,
325
+ * cached before the lane checked length — is evicted, not served.
326
+ */
327
+ const passesGate = (pairConfig) => (src, cached) => !contentGateFault(src, cached, pairConfig);
328
+
329
+ /**
330
+ * TM ladder for one content segment (a block, a whole body): the pair's own
331
+ * cache, then its fallback's. Each read is validated; a failing entry is
332
+ * evicted and reported as a miss.
333
+ *
334
+ * @returns {{ text: string, fromFallback: boolean }|null}
335
+ */
336
+ export function lookupContentTM(tm, source, code, pairConfig, { countCarry = true, gate = true } = {}) {
337
+ // gate false: a look for the repeat check, which names what it refuses —
338
+ // the gate still applies when the entry is served.
339
+ const check = (cfg) => (gate ? passesGate(cfg) : () => true);
340
+ const own = lookupTMValidated(tm, source, code, tmMethodKey(pairConfig), check(pairConfig), { countCarry });
341
+ if (own !== null) return { text: own, fromFallback: false };
342
+ if (!pairConfig.fallback) return null;
343
+ const fb = lookupTMValidated(tm, source, code, tmMethodKey(pairConfig.fallback), check(pairConfig.fallback), { countCarry });
344
+ return fb !== null ? { text: fb, fromFallback: true } : null;
345
+ }
346
+
347
+ // ── Cached content and the repeat check ──────────────────────────────
348
+
349
+ /**
350
+ * Items through the locale's different-inputs-same-output index; a suspect
351
+ * is evicted where it lives — every entry, under the pair's or its
352
+ * fallback's key, that holds that text for that source (lib/tm-evict.js).
353
+ *
354
+ * @param {Array<{ key: string, source: string, value: string }>} items
355
+ * @returns {Set<string>} Keys refused (and evicted)
356
+ */
357
+ function refuseRepeatedCached(items, { sharedOutputs, tm, code, pairConfig }) {
358
+ const refused = new Set();
359
+ if (items.length === 0) return refused;
360
+ const suspects = sharedOutputs.suspects(items);
361
+ if (suspects.size === 0) return refused;
362
+ const evictor = createTMEvictor(tm);
363
+ const keys = tmKeysForPair(pairConfig);
364
+ for (const it of items) {
365
+ if (!suspects.has(it.key)) continue;
366
+ refused.add(it.key);
367
+ evictor.evictProducing(it.source, code, it.value, keys);
368
+ }
369
+ return refused;
370
+ }
371
+
372
+ /**
373
+ * A page's cached content — its front-matter fields, its blocks, its whole
374
+ * body — through the repeat check (lib/validate.js SharedOutputIndex) in ONE
375
+ * batch, before the lane serves any of it: the way a key-value file's cached
376
+ * keys are checked together (lib/translate-pair.js). Cache hits used to skip
377
+ * the check: one memorized sentence cached for three different paragraphs
378
+ * was re-served by every sync, and only `verify` caught it on disk.
379
+ *
380
+ * A repeat is evicted where it lives, so the lane's own cache lookups miss
381
+ * it and it is translated again with the misses — through the same check.
382
+ * A whole-body entry holding a repeated block is evicted too (the body is
383
+ * then translated block by block, where only the repeats are asked again).
384
+ * What passes is recorded in the index, so later answers are compared with
385
+ * it. Keys are named as `verify` and the fresh path name them:
386
+ * "<label> front matter:<field>" and "<label>#<n>" (n = place among the
387
+ * page's translatable blocks). A whole body whose block count differs from
388
+ * its source's cannot be compared block by block (verify skips it too).
389
+ *
390
+ * @param {object} p
391
+ * @param {string} p.label - "content:<file>"
392
+ * @param {Record<string, string>} p.fields - field → source text: the fields the lane looks up
393
+ * @param {string} p.body - Source body ('' = none)
394
+ * @param {Set<number>} [p.skipBlocks] - Block places the lane does not serve from the cache (kept by hand)
395
+ * @param {boolean} [p.wholeBody=true] - false when the lane does not serve a whole-body hit
396
+ * @param {object} p.tm
397
+ * @param {string} p.code - Target locale
398
+ * @param {object} p.pairConfig
399
+ * @param {import('./validate.js').SharedOutputIndex|null} p.sharedOutputs
400
+ * @returns {string[]} What was refused, for the lane's warning ("title", "paragraph 2")
401
+ */
402
+ export function refuseRepeatedCachedPage({
403
+ label, fields = {}, body = '', skipBlocks = new Set(), wholeBody = true, tm, code, pairConfig, sharedOutputs,
404
+ }) {
405
+ if (!sharedOutputs || !tm) return [];
406
+ // A look, not a serve: a carried (other-model) hit is counted when served.
407
+ const peek = (source) => lookupContentTM(tm, source, code, pairConfig, { countCarry: false, gate: false });
408
+ const items = [];
409
+ for (const [field, source] of Object.entries(fields)) {
410
+ if (typeof source !== 'string') continue;
411
+ const hit = peek(source);
412
+ if (hit) items.push({ key: `${label} front matter:${field}`, source, value: hit.text, name: field });
413
+ }
414
+ const sources = typeof body === 'string' && body.trim() ? translatableBlockSources(body) : [];
415
+ const blockKeys = new Set();
416
+ sources.forEach((source, i) => {
417
+ if (skipBlocks.has(i)) return;
418
+ const hit = peek(source);
419
+ if (!hit) return;
420
+ items.push({ key: `${label}#${i + 1}`, source, value: hit.text, name: `paragraph ${i + 1}` });
421
+ blockKeys.add(`${label}#${i + 1}`);
422
+ });
423
+ let whole = null;
424
+ const wholeKeys = [];
425
+ if (wholeBody && sources.length > 0) {
426
+ const hit = peek(body);
427
+ const outs = hit ? translatableBlockSources(hit.text) : [];
428
+ if (hit && outs.length === sources.length) {
429
+ whole = hit;
430
+ sources.forEach((source, i) => {
431
+ const key = `${label}#${i + 1}`;
432
+ wholeKeys.push(key);
433
+ // The block's own entry, when there is one, speaks for it.
434
+ if (!blockKeys.has(key)) items.push({ key, source, value: outs[i], name: `paragraph ${i + 1}` });
435
+ });
436
+ }
437
+ }
438
+ const ctx = { sharedOutputs, tm, code, pairConfig };
439
+ const refused = refuseRepeatedCached(items, ctx);
440
+ if (whole && wholeKeys.some(k => refused.has(k))) {
441
+ createTMEvictor(tm).evictProducing(body, code, whole.text, tmKeysForPair(pairConfig));
442
+ }
443
+ sharedOutputs.add(items.filter(it => !refused.has(it.key)));
444
+ return items.filter(it => refused.has(it.key)).map(it => it.name);
445
+ }
446
+
447
+ /**
448
+ * Serve front-matter fields the pair's own cache missed from its fallback's
449
+ * cache (validated). No-op without a fallback.
450
+ *
451
+ * @param {object} tm
452
+ * @param {Record<string,string>} fields - field → source text
453
+ * @param {string[]} misses - Fields the pair's own cache did not hold
454
+ * @param {string} code
455
+ * @param {object} pairConfig
456
+ * @returns {{ hits: Record<string,string>, misses: string[] }}
457
+ */
458
+ export function serveFieldsFromFallbackCache(tm, fields, misses, code, pairConfig) {
459
+ if (!pairConfig.fallback || misses.length === 0) return { hits: {}, misses };
460
+ const fbKey = tmMethodKey(pairConfig.fallback);
461
+ const hits = {};
462
+ const rest = [];
463
+ for (const field of misses) {
464
+ const cached = lookupTMValidated(tm, fields[field], code, fbKey, passesGate(pairConfig.fallback));
465
+ if (cached !== null) hits[field] = cached;
466
+ else rest.push(field);
467
+ }
468
+ return { hits, misses: rest };
469
+ }
470
+
471
+ /**
472
+ * Translate the front-matter fields the TM did not hold: the pair's method,
473
+ * then — for every field it returned nothing for or hollowed — its fallback.
474
+ *
475
+ * NOTHING IS CACHED HERE. The lanes promise "nothing was written or cached"
476
+ * when a field fails, so the TM stores come back in `stores` and the caller
477
+ * commits them only once no field is left failing.
478
+ *
479
+ * Without a fallback this behaves exactly as the lanes always did: a null
480
+ * response is `noResults`, the first hollowed field is reported, a field
481
+ * missing from the response is left as it was.
482
+ *
483
+ * @param {object} p
484
+ * @param {Record<string,string>} p.fields - field → source text
485
+ * @param {string[]} p.misses - Fields to translate
486
+ * @param {object} p.pairConfig
487
+ * @param {(fields: string[], config: object) => Promise<object|null>} p.translate
488
+ * - The lane's translateBatch call for a given pair config
489
+ * @param {object|null} [p.budget] - createFallbackBudget(); null = no cap
490
+ * @param {Set<string>} [p.fallbackOnly] - Fields the pair's method refused
491
+ * before (lib/content-refusals.js): not sent to it, only to the fallback
492
+ * @returns {Promise<{ translated: Record<string,string>, stores: Array<{text: string, value: string, tmKey: string}>,
493
+ * hollowed: Array<{field: string, reason: string, value: string, fallbackReason?: string}>,
494
+ * noResults: boolean, report: FallbackReport|null, refusedBy: Record<string, string[]> }>}
495
+ * refusedBy: field → the method keys whose answer the gate refused, for the
496
+ * fields left untranslated (the lane remembers them)
497
+ */
498
+ export async function translateFieldsWithFallback({
499
+ fields, misses, pairConfig, translate, budget = null, sharedOutputs = null, label = 'front matter',
500
+ fallbackOnly = new Set(),
501
+ }) {
502
+ const fb = pairConfig.fallback || null;
503
+ const primaryKey = tmMethodKey(pairConfig);
504
+ const translated = {};
505
+ const stores = [];
506
+ /** @type {Map<string, {reason: string|null, value?: string}>} field → why it failed */
507
+ const failed = new Map();
508
+ const fieldKey = (field) => `${label}:${field}`;
509
+ // A field answered with the text the model gave for other source strings
510
+ // (lib/validate.js SharedOutputIndex) is refused like a hollowed one.
511
+ const sharedCheck = (candidates) => {
512
+ if (!sharedOutputs || candidates.length === 0) return new Map();
513
+ const suspects = sharedOutputs.suspects(candidates.map(([field, value]) => ({ key: fieldKey(field), source: fields[field], value })));
514
+ return new Map(candidates.filter(([field]) => suspects.has(fieldKey(field)))
515
+ .map(([field]) => [field, sharedOutputReason(suspects.get(fieldKey(field)))]));
516
+ };
517
+ const remember = (pairs) => {
518
+ if (sharedOutputs) sharedOutputs.add(pairs.map(([field, value]) => ({ key: fieldKey(field), source: fields[field], value })));
519
+ };
520
+
521
+ // Fields the pair's method refused before go to the fallback only.
522
+ const primaryAsked = fb ? misses.filter(f => !fallbackOnly.has(f)) : misses;
523
+ const out = primaryAsked.length > 0 ? await translate(primaryAsked, pairConfig) : {};
524
+ for (const field of misses) if (!primaryAsked.includes(field)) failed.set(field, { reason: null, fallbackOnly: true });
525
+ if (out) {
526
+ const passing = [];
527
+ for (const field of primaryAsked) {
528
+ const value = out[field];
529
+ if (typeof value !== 'string') {
530
+ // Missing from the response: left untranslated, as always — unless
531
+ // there is a fallback to ask.
532
+ if (fb) failed.set(field, { reason: null });
533
+ continue;
534
+ }
535
+ const refused = contentGateFault(fields[field], value, pairConfig);
536
+ if (refused) {
537
+ failed.set(field, { reason: refused, value });
538
+ continue;
539
+ }
540
+ passing.push([field, value]);
541
+ }
542
+ const shared = sharedCheck(passing);
543
+ for (const [field, value] of passing) {
544
+ if (shared.has(field)) { failed.set(field, { reason: shared.get(field), value, sharedOutput: true }); continue; }
545
+ translated[field] = value;
546
+ stores.push({ text: fields[field], value, tmKey: primaryKey });
547
+ }
548
+ remember(passing.filter(([field]) => !shared.has(field)));
549
+
550
+ // Refused: asked once more, with the reason (the key-value lane's
551
+ // feedback retry, and retryRefusedBlocks for blocks). A keep-as-written
552
+ // refusal answered the same way twice is taken at its word.
553
+ const refusedFields = [...failed].filter(([, f]) => f.reason && !f.fallbackOnly).map(([field]) => field);
554
+ if (refusedFields.length > 0) {
555
+ const descriptions = {};
556
+ for (const field of refusedFields) {
557
+ const f = failed.get(field);
558
+ descriptions[field] = `RETRY: a previous translation ("${String(f.value ?? '').slice(0, 60)}") was refused by an automatic quality check: ${f.reason}. `
559
+ + `Translate this ${field} into ${pairConfig.name || pairConfig.target}. If it is correct exactly as written — a name, title, acronym or code — return it unchanged.`;
560
+ }
561
+ const chars = refusedFields.reduce((n, f) => n + fields[f].length, 0);
562
+ const deaf = pairConfig.acceptsInstructions === false;
563
+ const verdict = deaf ? { ok: false } : budget ? await budget.approve(charsToKeyUnits(chars, pairConfig), pairConfig) : { ok: true };
564
+ // Declared deaf: not asked again; its first answer is judged as a second one.
565
+ let again = deaf ? Object.fromEntries(refusedFields.map(f => [f, failed.get(f).value])) : null;
566
+ if (verdict.ok) {
567
+ try { again = await translate(refusedFields, pairConfig, { descriptions }); } catch { again = null; }
568
+ }
569
+ const againPassing = [];
570
+ for (const field of refusedFields) {
571
+ const value = again?.[field];
572
+ if (typeof value !== 'string') continue;
573
+ const f = failed.get(field);
574
+ const fault = contentGateFault(fields[field], value, pairConfig);
575
+ const insisted = isKeepAsWrittenFault(fault) && isKeepAsWrittenFault(f.reason) && value === f.value;
576
+ if (!fault || insisted) againPassing.push([field, value]);
577
+ else f.reason = `${f.reason}; asked again with that reason: ${fault}`;
578
+ }
579
+ const againShared = sharedCheck(againPassing);
580
+ for (const [field, value] of againPassing) {
581
+ if (againShared.has(field)) continue;
582
+ translated[field] = value;
583
+ stores.push({ text: fields[field], value, tmKey: primaryKey });
584
+ failed.delete(field);
585
+ }
586
+ remember(againPassing.filter(([field]) => !againShared.has(field)));
587
+ }
588
+ } else if (fb) {
589
+ for (const field of primaryAsked) failed.set(field, { reason: null });
590
+ }
591
+
592
+ // With a fallback, a report even when nothing goes to it: the primary's
593
+ // accepted fields count toward the run's share (warnFallbackMajority).
594
+ let report = fb ? newFallbackReport(fb) : null;
595
+ if (report) report.primaryAccepted = Object.keys(translated).length;
596
+ let fallbackOut = null;
597
+ if (fb && failed.size > 0) {
598
+ report.primaryFailedWholesale = !out;
599
+ const asked = [...failed.keys()];
600
+ for (const field of asked) {
601
+ const f = failed.get(field);
602
+ notePrimaryReason(report, refusalCategory(f.reason, { sharedOutput: !!f.sharedOutput, heldBefore: !!f.fallbackOnly, noAnswer: !f.reason }));
603
+ }
604
+ const chars = asked.reduce((n, f) => n + fields[f].length, 0);
605
+ const verdict = budget ? await budget.approve(charsToKeyUnits(chars, fb), fb) : { ok: true };
606
+ if (!verdict.ok) {
607
+ report.skipped = { reason: verdict.reason, items: asked.map(f => `front matter "${f}"`) };
608
+ } else {
609
+ report.attempted = asked.length;
610
+ fallbackOut = await translate(asked, fb);
611
+ const fbKey = tmMethodKey(fb);
612
+ const fbPassing = [];
613
+ for (const field of asked) {
614
+ const value = fallbackOut?.[field];
615
+ if (typeof value !== 'string') continue;
616
+ const refused = contentGateFault(fields[field], value, fb);
617
+ if (refused) {
618
+ failed.get(field).fallbackReason = refused;
619
+ if (failed.get(field).fallbackOnly) failed.get(field).value = value;
620
+ continue;
621
+ }
622
+ fbPassing.push([field, value]);
623
+ }
624
+ const fbShared = sharedCheck(fbPassing);
625
+ for (const [field, value] of fbPassing) {
626
+ if (fbShared.has(field)) {
627
+ failed.get(field).fallbackReason = fbShared.get(field);
628
+ if (failed.get(field).fallbackOnly) failed.get(field).value = value;
629
+ continue;
630
+ }
631
+ translated[field] = value;
632
+ stores.push({ text: fields[field], value, tmKey: fbKey });
633
+ failed.delete(field);
634
+ report.accepted++;
635
+ }
636
+ remember(fbPassing.filter(([field]) => !fbShared.has(field)));
637
+ }
638
+ }
639
+
640
+ // A field still hollowed (by the primary, and by the fallback when there
641
+ // is one) fails the file, as always; the caller names the first. A field
642
+ // only the fallback was asked for fails it when the fallback's answer is
643
+ // refused too.
644
+ const hollowed = [];
645
+ const refusedBy = {};
646
+ const fbKey = fb ? tmMethodKey(fb) : null;
647
+ for (const [field, f] of failed) {
648
+ const methods = [...(f.reason ? [primaryKey] : []), ...(f.fallbackReason ? [fbKey] : [])];
649
+ if (methods.length > 0) refusedBy[field] = methods;
650
+ if (f.reason || (f.fallbackOnly && f.fallbackReason)) {
651
+ hollowed.push({
652
+ field, reason: f.reason || `refused before: the gate refused ${pairConfig.method}'s translation of it on an earlier sync`, value: f.value,
653
+ ...(f.sharedOutput && { sharedOutput: true }),
654
+ ...(f.fallbackReason && { fallbackReason: f.fallbackReason }),
655
+ });
656
+ }
657
+ }
658
+
659
+ return {
660
+ translated,
661
+ stores,
662
+ hollowed,
663
+ noResults: !out && Object.keys(translated).length === 0,
664
+ report,
665
+ refusedBy,
666
+ };
667
+ }
668
+
669
+ /** ⟦PROTECTED_N⟧ tokens in a protected text. */
670
+ function protectedTokens(text) {
671
+ const re = new RegExp(`${PLACEHOLDER_PREFIX}\\d+${PLACEHOLDER_SUFFIX}`, 'g');
672
+ return text.match(re) || [];
673
+ }
674
+
675
+ /**
676
+ * Why a translated Markdown block cannot be used, or null. The same gates
677
+ * the lanes apply to a whole body — orphaned placeholders, lost content —
678
+ * applied per block so the failing block (not the whole file) goes to the
679
+ * fallback, plus a check the whole-body gate cannot make: every protected
680
+ * element of the source block (code, link target, shortcode, HTML) must
681
+ * still be there.
682
+ *
683
+ * Used only for pairs with a fallback: without one there is no second
684
+ * method to route a block to, and the whole-body gates decide as before.
685
+ *
686
+ * @param {string} protectedSource - The block as sent (placeholders in)
687
+ * @param {string} protectedOut - The model's block (placeholders in)
688
+ * @param {string} restoredOut - protectedOut with placeholders restored
689
+ * @param {string} restoredSource - The block's source text, restored
690
+ * @returns {string|null}
691
+ */
692
+ export function blockFault(protectedSource, protectedOut, restoredOut, restoredSource, pairConfig = {}) {
693
+ if (hasOrphanedPlaceholders(restoredOut)) return 'a protected element was damaged';
694
+ const dropped = protectedTokens(protectedSource).filter(t => !protectedOut.includes(t));
695
+ if (dropped.length > 0) return `${dropped.length} protected element(s) (code, link, markup) missing`;
696
+ // The key-value gate's checks, per block (lib/validate.js contentGateFault):
697
+ // hollowing, echo, length inflation, truncation, repetition, script.
698
+ return contentGateFault(restoredSource, restoredOut, pairConfig);
699
+ }
700
+
701
+ /**
702
+ * Ask the pair's method once more for the blocks the gate refused, telling it
703
+ * why. WHY: a refusal used to be final for the run, and the block was left in
704
+ * the source language — while a good share of refusals are the model keeping
705
+ * text that is correct as written (a name heading, a citation, a table of
706
+ * codes), which no fixed rule can tell from a missed translation (dogfood
707
+ * 2026-10-05: 0.5% of 40,352 accepted blocks of champollion.dev). Told the
708
+ * reason, the model either translates it or returns it unchanged again — and
709
+ * a keep-as-written refusal (isKeepAsWrittenFault) answered the same way
710
+ * twice is taken at its word, as the key-value lane takes a name. Every other
711
+ * fault must clear the gate outright. One ask, never a loop; under --max-cost
712
+ * it is billed like any other call (budget.approve).
713
+ *
714
+ * @param {object} p
715
+ * @param {number[]} p.idx - Indices (into `missed`) the gate refused
716
+ * @param {string[]} p.texts - The protected block texts
717
+ * @param {Array<{source: string}>} p.missed
718
+ * @param {Map<string,string>} p.blocks - protectBlocks() map
719
+ * @param {object} p.pairConfig
720
+ * @param {Function} p.runBatch
721
+ * @param {Map<number,string>} p.firstOut - The refused (restored) answers
722
+ * @param {Map<number,string>} p.reasons - Why each was refused; a block asked
723
+ * again and refused again gets the second reason appended
724
+ * @param {object|null} [p.budget]
725
+ * @returns {Promise<Map<number,string>>} index → accepted (restored) answer
726
+ */
727
+ async function retryRefusedBlocks({ idx, texts, missed, blocks, pairConfig, runBatch, firstOut, reasons, budget = null }) {
728
+ const accepted = new Map();
729
+ if (idx.length === 0) return accepted;
730
+ // An endpoint that declares it follows no instructions (a trained NMT
731
+ // model) would answer the same: not asked again — its first answers are
732
+ // judged as second answers are (the key-value lane's rule).
733
+ if (pairConfig.acceptsInstructions === false) {
734
+ for (const i of idx) if (isKeepAsWrittenFault(reasons.get(i))) accepted.set(i, firstOut.get(i));
735
+ return accepted;
736
+ }
737
+ if (budget) {
738
+ const chars = idx.reduce((n, i) => n + missed[i].source.length, 0);
739
+ const verdict = await budget.approve(charsToKeyUnits(chars, pairConfig), pairConfig);
740
+ if (!verdict.ok) return accepted;
741
+ }
742
+ const retryNotes = new Map(idx.map(i => [texts[i], reasons.get(i)]));
743
+ let res = null;
744
+ try {
745
+ res = await runBatch(idx.map(i => texts[i]), { ...pairConfig, retryNotes });
746
+ } catch {
747
+ return accepted;
748
+ }
749
+ const fell = new Set(res?.fellBack || []);
750
+ idx.forEach((i, j) => {
751
+ if (!res || fell.has(j) || typeof res.blocks[j] !== 'string') return;
752
+ const restored = restoreBlocks(res.blocks[j], blocks);
753
+ const fault = blockFault(texts[i], res.blocks[j], restored, missed[i].source, pairConfig);
754
+ const insisted = isKeepAsWrittenFault(fault) && isKeepAsWrittenFault(reasons.get(i)) && restored === firstOut.get(i);
755
+ if (!fault || insisted) accepted.set(i, restored);
756
+ else reasons.set(i, `${reasons.get(i)}; asked again with that reason: ${fault}`);
757
+ });
758
+ return accepted;
759
+ }
760
+
761
+ /**
762
+ * Translate the Markdown blocks the TM ladder did not hold: one batch through
763
+ * the pair's method, then one batch through its fallback for every block the
764
+ * primary dropped or damaged. A block both fail is written as
765
+ * its source text, unmarked (never cached; the caller records the refusal,
766
+ * does not advance the file's lock, and reports it). A refused block is first
767
+ * asked again once, with the reason (retryRefusedBlocks).
768
+ *
769
+ * Without a fallback this is exactly the lanes' old block path: the primary's
770
+ * output stands, its dropped blocks carry the prefix, and a structural
771
+ * failure (no response, duplicate markers) throws.
772
+ *
773
+ * With a fallback, a structural primary failure sends every block to the
774
+ * fallback; only when the fallback ALSO produces nothing does the primary's
775
+ * error fail the file. When --max-cost skips the fallback, the primary's
776
+ * output stands exactly as it would without one.
777
+ *
778
+ * @param {object} p
779
+ * @param {Array<{seg: {text: string}, source: string}>} p.missed - Blocks to translate
780
+ * @param {Map<string,string>} p.blocks - protectBlocks() placeholder map
781
+ * @param {object} p.pairConfig
782
+ * @param {(texts: string[], config: object) => Promise<{blocks: string[], fellBack: number[]}>} p.runBatch
783
+ * - The lane's translateBlockBatchResilient call for a given pair config
784
+ * @param {string} p.fallbackPrefix
785
+ * @param {object|null} [p.budget]
786
+ * @param {import('./validate.js').SharedOutputIndex|null} [p.sharedOutputs] - The
787
+ * locale's different-inputs-same-output index: a block answered with the
788
+ * text the model gave for other source strings is refused like a damaged
789
+ * one (the fallback gets it; without one, the source text stands)
790
+ * @param {string} [p.label] - Names the blocks in that index ("posts/a.md")
791
+ * @param {Set<number>} [p.fallbackOnly] - Indices (into `missed`) of blocks the
792
+ * pair's method refused before (lib/content-refusals.js): not sent to it,
793
+ * only to the fallback (the source text stands when the fallback does not translate them)
794
+ * @returns {Promise<{ outs: string[], stores: Array<{source: string, translation: string, tmKey: string}>,
795
+ * fellBack: number[], fromFallback: number, report: FallbackReport|null, sharedOutput: number[],
796
+ * refused: Array<{ i: number, block: number, reason: string }>, refusedBy: string[][] }>} refused: blocks written as the
797
+ * source text because the gate refused them (block = place among the page's blocks, from 1);
798
+ * refusedBy[i]: the method keys whose answer for block i the gate refused, for blocks left in the source language
799
+ * (empty for the rest) — the lane remembers them
800
+ */
801
+ export async function translateBlocksWithFallback({
802
+ missed, blocks, pairConfig, runBatch, fallbackPrefix, budget = null, sharedOutputs = null, label = 'block',
803
+ fallbackOnly = new Set(),
804
+ }) {
805
+ const texts = missed.map(r => r.seg.text);
806
+ const primaryKey = tmMethodKey(pairConfig);
807
+ const fb = pairConfig.fallback || null;
808
+ // Whose answers the gate refused, per block (the primary's, the fallback's).
809
+ const primaryRefused = new Set();
810
+ const fallbackRefused = new Set();
811
+ const refusedByOf = (fellBack) => {
812
+ const fell = new Set(fellBack);
813
+ return texts.map((_, i) => (fell.has(i)
814
+ ? [...(primaryRefused.has(i) ? [primaryKey] : []), ...(fallbackRefused.has(i) ? [tmMethodKey(fb)] : [])]
815
+ : []));
816
+ };
817
+ // Named by the block's place among the page's translatable blocks (as
818
+ // verify and the cached-block check name it), not its place in `missed`.
819
+ const blockKey = (i) => `${label}#${(Number.isInteger(missed[i].pos) ? missed[i].pos : i) + 1}`;
820
+ // Blocks whose output repeats what the model answered for other sources.
821
+ const sharedFault = (candidates) => {
822
+ if (!sharedOutputs) return new Set();
823
+ const suspects = sharedOutputs.suspects(candidates.map(({ i, out }) => ({ key: blockKey(i), source: missed[i].source, value: out })));
824
+ return new Set(candidates.filter(({ i }) => suspects.has(blockKey(i))).map(({ i }) => i));
825
+ };
826
+ const remember = (accepted) => {
827
+ if (sharedOutputs) sharedOutputs.add(accepted.map(({ i, out }) => ({ key: blockKey(i), source: missed[i].source, value: out })));
828
+ };
829
+ // Why the gate refused a block (index → reason), for the lane's warning.
830
+ const refusedReason = new Map();
831
+ /** Blocks left as their source text because the gate refused them. */
832
+ const refusedList = (fellBack) => fellBack.filter(i => refusedReason.has(i))
833
+ .map(i => ({ i, block: (Number.isInteger(missed[i].pos) ? missed[i].pos : i) + 1, reason: refusedReason.get(i) }));
834
+
835
+ if (!fb) {
836
+ const { blocks: translatedBlocks, fellBack } = await runBatch(texts, pairConfig);
837
+ const fellSet = new Set(fellBack);
838
+ const outs = translatedBlocks.map(t => restoreBlocks(t, blocks));
839
+ // The key-value gate's checks, per block: a short heading turned into a
840
+ // sentence is refused here as an app key is (Round 7, school persona).
841
+ for (let i = 0; i < outs.length; i++) {
842
+ if (fellSet.has(i)) continue;
843
+ const fault = contentGateFault(missed[i].source, outs[i], pairConfig);
844
+ if (fault) refusedReason.set(i, fault);
845
+ }
846
+ const shared = sharedFault(outs.map((out, i) => ({ i, out })).filter(({ i }) => !fellSet.has(i) && !refusedReason.has(i)));
847
+ for (const i of shared) refusedReason.set(i, 'the same text the model gave for other, different source strings');
848
+ // Refused: asked once more, with the reason.
849
+ const again = await retryRefusedBlocks({
850
+ idx: [...refusedReason.keys()], texts, missed, blocks, pairConfig, runBatch,
851
+ firstOut: new Map([...refusedReason.keys()].map(i => [i, outs[i]])), reasons: refusedReason, budget,
852
+ });
853
+ const againShared = sharedFault([...again].map(([i, out]) => ({ i, out })));
854
+ for (const [i, out] of again) {
855
+ if (againShared.has(i)) continue;
856
+ outs[i] = out;
857
+ refusedReason.delete(i);
858
+ shared.delete(i);
859
+ }
860
+ // A memorized answer is reported as one (sharedOutput), not as a refusal.
861
+ for (const i of shared) refusedReason.delete(i);
862
+ const stores = [];
863
+ const accepted = [];
864
+ outs.forEach((out, i) => {
865
+ if (shared.has(i) || refusedReason.has(i)) {
866
+ // Refused twice: the source text stands, never cached.
867
+ outs[i] = missed[i].source;
868
+ fellBack.push(i);
869
+ primaryRefused.add(i);
870
+ return;
871
+ }
872
+ if (!fellSet.has(i)) {
873
+ stores.push({ source: missed[i].source, translation: out, tmKey: primaryKey });
874
+ accepted.push({ i, out });
875
+ }
876
+ });
877
+ remember(accepted);
878
+ return {
879
+ outs, stores, fellBack, fromFallback: 0, report: null, sharedOutput: [...shared], refused: refusedList(fellBack),
880
+ refusedBy: refusedByOf(fellBack),
881
+ };
882
+ }
883
+
884
+ // Blocks the pair's method refused before are not sent to it again.
885
+ const primaryIdx = texts.map((_, i) => i).filter(i => !fallbackOnly.has(i));
886
+ let primary = null;
887
+ let primaryError = null;
888
+ try {
889
+ if (primaryIdx.length === texts.length) {
890
+ primary = await runBatch(texts, pairConfig);
891
+ } else if (primaryIdx.length > 0) {
892
+ // Back to the page's own block indices.
893
+ const res = await runBatch(primaryIdx.map(i => texts[i]), pairConfig);
894
+ const all = new Array(texts.length);
895
+ primaryIdx.forEach((i, j) => { all[i] = res.blocks[j]; });
896
+ primary = { blocks: all, fellBack: res.fellBack.map(j => primaryIdx[j]) };
897
+ } else {
898
+ primary = { blocks: new Array(texts.length), fellBack: [] };
899
+ }
900
+ } catch (err) {
901
+ primaryError = err;
902
+ }
903
+
904
+ const outs = new Array(texts.length);
905
+ const stores = [];
906
+ const failed = [];
907
+ const primaryFell = new Set(primary ? primary.fellBack : []);
908
+ const sharedIdx = new Set();
909
+ if (primary) {
910
+ const good = [];
911
+ for (let i = 0; i < texts.length; i++) {
912
+ if (primaryFell.has(i) || fallbackOnly.has(i)) { failed.push(i); continue; }
913
+ const restored = restoreBlocks(primary.blocks[i], blocks);
914
+ const fault = blockFault(texts[i], primary.blocks[i], restored, missed[i].source, pairConfig);
915
+ if (fault) {
916
+ refusedReason.set(i, fault);
917
+ primaryRefused.add(i);
918
+ failed.push(i);
919
+ continue;
920
+ }
921
+ good.push({ i, out: restored });
922
+ }
923
+ const shared = sharedFault(good);
924
+ for (const { i, out } of good) {
925
+ if (shared.has(i)) { failed.push(i); sharedIdx.add(i); primaryRefused.add(i); continue; }
926
+ outs[i] = out;
927
+ stores.push({ source: missed[i].source, translation: out, tmKey: primaryKey });
928
+ }
929
+ remember(good.filter(({ i }) => !shared.has(i)));
930
+ // Refused (not dropped, not held before): asked once more, with the reason.
931
+ const firstOut = new Map();
932
+ for (const i of failed) {
933
+ if (sharedIdx.has(i)) refusedReason.set(i, 'the same text the model gave for other, different source strings');
934
+ if (refusedReason.has(i)) firstOut.set(i, restoreBlocks(primary.blocks[i], blocks));
935
+ }
936
+ const again = await retryRefusedBlocks({
937
+ idx: [...firstOut.keys()], texts, missed, blocks, pairConfig, runBatch, firstOut, reasons: refusedReason, budget,
938
+ });
939
+ const againShared = sharedFault([...again].map(([i, out]) => ({ i, out })));
940
+ const recovered = new Set();
941
+ for (const [i, out] of again) {
942
+ if (againShared.has(i)) continue;
943
+ outs[i] = out;
944
+ stores.push({ source: missed[i].source, translation: out, tmKey: primaryKey });
945
+ refusedReason.delete(i);
946
+ sharedIdx.delete(i);
947
+ primaryRefused.delete(i);
948
+ recovered.add(i);
949
+ }
950
+ remember([...again].filter(([i]) => recovered.has(i)).map(([i, out]) => ({ i, out })));
951
+ if (recovered.size > 0) failed.splice(0, failed.length, ...failed.filter(i => !recovered.has(i)));
952
+ for (const i of sharedIdx) refusedReason.delete(i);
953
+ } else {
954
+ for (let i = 0; i < texts.length; i++) failed.push(i);
955
+ }
956
+
957
+ const report = newFallbackReport(fb);
958
+ report.primaryFailedWholesale = !primary;
959
+ // The primary's accepted blocks, and why the rest go to the fallback
960
+ // (lib/fallback.js warnFallbackMajority).
961
+ report.primaryAccepted = outs.filter(o => o !== undefined).length;
962
+ for (const i of failed) {
963
+ notePrimaryReason(report, refusalCategory(refusedReason.get(i) || null, {
964
+ sharedOutput: sharedIdx.has(i), heldBefore: fallbackOnly.has(i), noAnswer: !refusedReason.has(i) && !sharedIdx.has(i),
965
+ }));
966
+ }
967
+ const fellBack = [];
968
+ failed.sort((a, b) => a - b);
969
+ if (failed.length === 0) return { outs, stores, fellBack, fromFallback: 0, report, sharedOutput: [], refused: [], refusedBy: refusedByOf([]) };
970
+
971
+ const chars = failed.reduce((n, i) => n + missed[i].source.length, 0);
972
+ const verdict = budget ? await budget.approve(charsToKeyUnits(chars, fb), fb) : { ok: true };
973
+ if (!verdict.ok) {
974
+ report.skipped = { reason: verdict.reason, items: failed.map(i => `block ${i + 1} of ${texts.length}`) };
975
+ if (primaryError) throw primaryError;
976
+ // The primary's own output stands, exactly as without a fallback —
977
+ // except a block refused as a repeated (memorized) answer, and a block
978
+ // only the fallback was to be asked for (the primary had no answer).
979
+ for (const i of failed) {
980
+ if (sharedIdx.has(i) || primaryFell.has(i) || refusedReason.has(i) || fallbackOnly.has(i)) {
981
+ outs[i] = sharedIdx.has(i) || refusedReason.has(i) || fallbackOnly.has(i)
982
+ ? missed[i].source : restoreBlocks(primary.blocks[i], blocks);
983
+ fellBack.push(i);
984
+ continue;
985
+ }
986
+ outs[i] = restoreBlocks(primary.blocks[i], blocks);
987
+ stores.push({ source: missed[i].source, translation: outs[i], tmKey: primaryKey });
988
+ }
989
+ return {
990
+ outs, stores, fellBack, fromFallback: 0, report, sharedOutput: [...sharedIdx], refused: refusedList(fellBack),
991
+ refusedBy: refusedByOf(fellBack),
992
+ };
993
+ }
994
+
995
+ report.attempted = failed.length;
996
+ let second = null;
997
+ try {
998
+ second = await runBatch(failed.map(i => texts[i]), fb);
999
+ } catch (err) {
1000
+ if (primaryError) {
1001
+ primaryError.message += ` (the fallback, ${fb.method}, failed too: ${err.message})`;
1002
+ throw primaryError;
1003
+ }
1004
+ }
1005
+ if (!second && primaryError) throw primaryError;
1006
+
1007
+ const fbKey = tmMethodKey(fb);
1008
+ const secondFell = new Set(second ? second.fellBack : []);
1009
+ let fromFallback = 0;
1010
+ const fbGood = [];
1011
+ failed.forEach((i, j) => {
1012
+ if (second && !secondFell.has(j)) {
1013
+ const restored = restoreBlocks(second.blocks[j], blocks);
1014
+ const fbFault = blockFault(texts[i], second.blocks[j], restored, missed[i].source, fb);
1015
+ if (!fbFault) fbGood.push({ i, out: restored });
1016
+ else {
1017
+ refusedReason.set(i, refusedReason.has(i) ? `${refusedReason.get(i)}; the fallback's answer: ${fbFault}` : `the fallback's answer: ${fbFault}`);
1018
+ fallbackRefused.add(i);
1019
+ }
1020
+ }
1021
+ });
1022
+ const fbShared = sharedFault(fbGood);
1023
+ for (const i of fbShared) fallbackRefused.add(i);
1024
+ const fbAccepted = new Map(fbGood.filter(({ i }) => !fbShared.has(i)).map(({ i, out }) => [i, out]));
1025
+ remember([...fbAccepted].map(([i, out]) => ({ i, out })));
1026
+ for (const i of failed) {
1027
+ if (fbAccepted.has(i)) {
1028
+ outs[i] = fbAccepted.get(i);
1029
+ stores.push({ source: missed[i].source, translation: outs[i], tmKey: fbKey });
1030
+ report.accepted++;
1031
+ fromFallback++;
1032
+ continue;
1033
+ }
1034
+ // Both methods failed this block: the source text stands.
1035
+ outs[i] = missed[i].source;
1036
+ fellBack.push(i);
1037
+ }
1038
+ return {
1039
+ outs, stores, fellBack, fromFallback, report, sharedOutput: [...sharedIdx].filter(i => !fbAccepted.has(i)),
1040
+ refused: refusedList(fellBack),
1041
+ refusedBy: refusedByOf(fellBack),
1042
+ };
1043
+ }
1044
+
1045
+ /**
1046
+ * Whole-page ('page' segmentation) translation with a fallback. The pair's
1047
+ * method first; when it returns nothing, damages a placeholder or hollows
1048
+ * the body, the fallback translates the whole page once. Returns null for
1049
+ * `body` when both fail — the caller then fails the file exactly as it does
1050
+ * without a fallback (its whole-body checks name the reason).
1051
+ *
1052
+ * Only called for pairs WITH a fallback; the lanes keep their own page path
1053
+ * otherwise.
1054
+ *
1055
+ * @param {object} p
1056
+ * @param {string} p.body - Source body (restored)
1057
+ * @param {Map<string,string>} p.blocks - protectBlocks() map
1058
+ * @param {object} p.pairConfig
1059
+ * @param {(config: object) => Promise<string|null>} p.runPage - One page call for a config
1060
+ * @param {object|null} [p.budget]
1061
+ * @param {boolean} [p.fallbackOnly] - The pair's method refused this page
1062
+ * before (lib/content-refusals.js): it is not asked, only the fallback is
1063
+ * @returns {Promise<{ body: string|null, primaryBody: string|null, tmKey: string|null, report: FallbackReport,
1064
+ * refusedBy: string[] }>} refusedBy: the method keys whose page the whole-body checks refused this run
1065
+ */
1066
+ export async function translatePageWithFallback({ body, blocks, pairConfig, runPage, budget = null, fallbackOnly = false }) {
1067
+ const fb = pairConfig.fallback;
1068
+ const report = newFallbackReport(fb);
1069
+ const fault = (restored) => hasOrphanedPlaceholders(restored) || !!checkContentPreservation(body, restored);
1070
+ const refusedBy = [];
1071
+
1072
+ let primaryBody = null;
1073
+ if (!fallbackOnly) {
1074
+ const first = await runPage(pairConfig);
1075
+ primaryBody = first ? restoreBlocks(first, blocks) : null;
1076
+ if (primaryBody !== null && !fault(primaryBody)) {
1077
+ report.primaryAccepted = 1;
1078
+ return { body: primaryBody, primaryBody, tmKey: tmMethodKey(pairConfig), report, refusedBy };
1079
+ }
1080
+ if (primaryBody !== null) refusedBy.push(tmMethodKey(pairConfig));
1081
+ report.primaryFailedWholesale = first === null;
1082
+ }
1083
+ // Why the page goes to the fallback (warnFallbackMajority).
1084
+ notePrimaryReason(report, fallbackOnly
1085
+ ? refusalCategory(null, { heldBefore: true })
1086
+ : refusalCategory(primaryBody === null ? null : 'content lost or changed', { noAnswer: primaryBody === null }));
1087
+
1088
+ const verdict = budget ? await budget.approve(charsToKeyUnits(body.length, fb), fb) : { ok: true };
1089
+ if (!verdict.ok) {
1090
+ report.skipped = { reason: verdict.reason, items: ['the page body'] };
1091
+ return { body: null, primaryBody, tmKey: null, report, refusedBy };
1092
+ }
1093
+ report.attempted = 1;
1094
+ const second = await runPage(fb);
1095
+ const fallbackBody = second ? restoreBlocks(second, blocks) : null;
1096
+ if (fallbackBody !== null && !fault(fallbackBody)) {
1097
+ report.accepted = 1;
1098
+ return { body: fallbackBody, primaryBody, tmKey: tmMethodKey(fb), report, refusedBy };
1099
+ }
1100
+ if (fallbackBody !== null) refusedBy.push(tmMethodKey(fb));
1101
+ return { body: null, primaryBody, tmKey: null, report, refusedBy };
1102
+ }