champollion 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/LICENSE +133 -0
  2. package/README.md +387 -0
  3. package/bin/cli.js +278 -0
  4. package/index.js +135 -0
  5. package/lib/api-key.js +127 -0
  6. package/lib/autofix.js +432 -0
  7. package/lib/bridge/method_bridge.py +430 -0
  8. package/lib/card-source-resolution.mjs +284 -0
  9. package/lib/cards/cache.js +169 -0
  10. package/lib/cards/env.js +82 -0
  11. package/lib/cards/fetch-card-child.js +38 -0
  12. package/lib/cards/reader.js +435 -0
  13. package/lib/cards/refresh.js +111 -0
  14. package/lib/cards/remote.js +387 -0
  15. package/lib/cldf-export.mjs +540 -0
  16. package/lib/cldf-terms.mjs +62 -0
  17. package/lib/command-help.js +790 -0
  18. package/lib/commands/audit.js +49 -0
  19. package/lib/commands/card.js +454 -0
  20. package/lib/commands/doctor.js +559 -0
  21. package/lib/commands/fonts.js +489 -0
  22. package/lib/commands/help.js +91 -0
  23. package/lib/commands/init.js +1259 -0
  24. package/lib/commands/integrity.js +148 -0
  25. package/lib/commands/leaderboard.js +478 -0
  26. package/lib/commands/lint.js +30 -0
  27. package/lib/commands/models.js +177 -0
  28. package/lib/commands/plugin.js +103 -0
  29. package/lib/commands/provenance.js +45 -0
  30. package/lib/commands/recommend.js +75 -0
  31. package/lib/commands/register-corpus.js +678 -0
  32. package/lib/commands/repair-script.js +42 -0
  33. package/lib/commands/seal-corpus.js +355 -0
  34. package/lib/commands/seo.js +72 -0
  35. package/lib/commands/serve.js +147 -0
  36. package/lib/commands/status.js +265 -0
  37. package/lib/commands/submit.js +332 -0
  38. package/lib/commands/sync.js +89 -0
  39. package/lib/commands/tm.js +573 -0
  40. package/lib/commands/verify.js +39 -0
  41. package/lib/commands/watch.js +20 -0
  42. package/lib/commands/wrap.js +138 -0
  43. package/lib/commands/xliff.js +327 -0
  44. package/lib/commercial-eligibility.js +235 -0
  45. package/lib/concurrent.js +87 -0
  46. package/lib/config.js +523 -0
  47. package/lib/contamination-lane.js +76 -0
  48. package/lib/content-sync.js +731 -0
  49. package/lib/content.js +733 -0
  50. package/lib/corpus-registration.mjs +608 -0
  51. package/lib/cost-report.js +346 -0
  52. package/lib/diff.js +155 -0
  53. package/lib/docusaurus-sync.js +1256 -0
  54. package/lib/flatten.js +55 -0
  55. package/lib/format.js +954 -0
  56. package/lib/hash.js +159 -0
  57. package/lib/icu.js +473 -0
  58. package/lib/integrity.js +689 -0
  59. package/lib/license-gate.mjs +478 -0
  60. package/lib/license-identify.mjs +229 -0
  61. package/lib/lint.js +629 -0
  62. package/lib/method-manifest.js +60 -0
  63. package/lib/methods/anthropic.js +140 -0
  64. package/lib/methods/apertium.js +163 -0
  65. package/lib/methods/api.js +316 -0
  66. package/lib/methods/base.js +184 -0
  67. package/lib/methods/content-separator.js +45 -0
  68. package/lib/methods/deepl.js +426 -0
  69. package/lib/methods/direct-llm.js +586 -0
  70. package/lib/methods/external.js +332 -0
  71. package/lib/methods/fetch-with-retry.js +124 -0
  72. package/lib/methods/gemini.js +147 -0
  73. package/lib/methods/google-translate.js +402 -0
  74. package/lib/methods/http-utils.js +122 -0
  75. package/lib/methods/libretranslate.js +314 -0
  76. package/lib/methods/llm-coached.js +670 -0
  77. package/lib/methods/llm.js +592 -0
  78. package/lib/methods/local.js +76 -0
  79. package/lib/methods/microsoft-translator.js +331 -0
  80. package/lib/methods/openai.js +131 -0
  81. package/lib/methods/openrouter-client.js +327 -0
  82. package/lib/methods/openrouter-pricing.js +156 -0
  83. package/lib/methods/provider-env.js +115 -0
  84. package/lib/methods/provider-pricing.js +310 -0
  85. package/lib/methods/tilde.js +150 -0
  86. package/lib/methods/translated.js +229 -0
  87. package/lib/methods/translation-error.js +80 -0
  88. package/lib/models.js +258 -0
  89. package/lib/no-translate.js +233 -0
  90. package/lib/output.js +238 -0
  91. package/lib/pairs.js +547 -0
  92. package/lib/plugins.js +447 -0
  93. package/lib/provenance.js +323 -0
  94. package/lib/recommend.js +648 -0
  95. package/lib/registers.js +1185 -0
  96. package/lib/repair-script.js +266 -0
  97. package/lib/scripts.js +994 -0
  98. package/lib/seal.mjs +464 -0
  99. package/lib/sealed-qualifier.mjs +211 -0
  100. package/lib/security.js +59 -0
  101. package/lib/segment.js +369 -0
  102. package/lib/seo.js +275 -0
  103. package/lib/serve.js +854 -0
  104. package/lib/string-classify.js +85 -0
  105. package/lib/submit.mjs +344 -0
  106. package/lib/sync.js +969 -0
  107. package/lib/tags/bcp47.js +202 -0
  108. package/lib/tags/resolve.js +314 -0
  109. package/lib/terminology.js +111 -0
  110. package/lib/tm-seed.js +294 -0
  111. package/lib/tm.js +515 -0
  112. package/lib/translate-pair.js +197 -0
  113. package/lib/translate.js +203 -0
  114. package/lib/types.js +230 -0
  115. package/lib/validate.js +510 -0
  116. package/lib/verify.js +451 -0
  117. package/lib/watch.js +145 -0
  118. package/lib/xliff.js +184 -0
  119. package/package.json +93 -0
  120. package/shared/ATTRIBUTION.md +145 -0
  121. package/shared/CORPORA-CARDS.md +288 -0
  122. package/shared/DATA-SOVEREIGNTY.md +500 -0
  123. package/shared/LANGUAGE-CARD-FIELDS.md +532 -0
  124. package/shared/card-lint-baseline.json +3189 -0
  125. package/shared/cards-fallback.json +1 -0
  126. package/shared/catalogue/card-config.json +6091 -0
  127. package/shared/catalogue/external-results.json +3888 -0
  128. package/shared/catalogue/gender-guidance.json +1038 -0
  129. package/shared/catalogue/method-coverage.json +1751 -0
  130. package/shared/catalogue/metric-coverage.json +170 -0
  131. package/shared/catalogue/metric-reliability.json +1 -0
  132. package/shared/catalogue/register-presets.json +3180 -0
  133. package/shared/catalogue/vitality-scales.json +55 -0
  134. package/shared/cldr-index.json +1115 -0
  135. package/shared/code-bridge.json +253 -0
  136. package/shared/corpora-cards-v1-reference.md +281 -0
  137. package/shared/curated-dictionary-flags.json +35 -0
  138. package/shared/curated-endonyms.json +35 -0
  139. package/shared/curated-fsts.json +51 -0
  140. package/shared/curated-orthography-conventions.json +26 -0
  141. package/shared/curated-sil-resources.json +374 -0
  142. package/shared/curated-tools.json +41 -0
  143. package/shared/docent/corpus.json +11333 -0
  144. package/shared/docent/faq.en.json +564 -0
  145. package/shared/docent/register-blocks.json +60 -0
  146. package/shared/docent/system-prompt.md +144 -0
  147. package/shared/domain-taxonomy.json +35 -0
  148. package/shared/explainers/glossary.json +2975 -0
  149. package/shared/explainers/tc-features.json +20112 -0
  150. package/shared/explainers/term-watchlist.json +147 -0
  151. package/shared/human-services.json +59 -0
  152. package/shared/license-corrections.json +261 -0
  153. package/shared/license-evidence.json +13452 -0
  154. package/shared/licenses.json +6781 -0
  155. package/shared/method-registry.json +236 -0
  156. package/shared/metric-registry.json +620 -0
  157. package/shared/model-aliases.json +7 -0
  158. package/shared/schemas/champollion-plugin.schema.json +206 -0
  159. package/shared/schemas/corpora-card.schema.json +957 -0
  160. package/shared/schemas/domain-taxonomy.schema.json +64 -0
  161. package/shared/schemas/external-results.schema.json +314 -0
  162. package/shared/schemas/human-services.schema.json +90 -0
  163. package/shared/schemas/language-card.schema.json +1308 -0
  164. package/shared/schemas/licenses.schema.json +155 -0
  165. package/shared/schemas/method-card.schema.json +412 -0
  166. package/shared/schemas/method-registry.schema.json +85 -0
  167. package/shared/schemas/metric-registry.schema.json +96 -0
  168. package/shared/schemas/metric-reliability.schema.json +178 -0
  169. package/shared/schemas/model-aliases.schema.json +27 -0
  170. package/shared/schemas/source-snapshot.schema.json +96 -0
@@ -0,0 +1,346 @@
1
+ /**
2
+ * cost-report.js — Pre-sync cost estimation display
3
+ *
4
+ * Extracted from sync.js to reduce god-module complexity.
5
+ * This module is purely informational — it reads locale files,
6
+ * estimates translation costs via the pairs API, and prints a
7
+ * formatted table using the output controller. No side effects
8
+ * on sync state.
9
+ *
10
+ * Called once at the start of runSync for EVERY engine — it used to
11
+ * early-return unless OPENROUTER_API_KEY was set, which silently hid
12
+ * the preview from google-translate/deepl/gemini/… users who never
13
+ * touch OpenRouter. Each method implements estimateCost() (or honestly
14
+ * returns null = "unknown", never $0), so the key gate was wrong.
15
+ *
16
+ * The structured estimate is RETURNED to the caller so that:
17
+ * - sync.js can enforce the --max-cost cap BEFORE any API call
18
+ * (unknown ≠ free: an unknowable estimate under a cap aborts), and
19
+ * - the estimate reaches the --json summary (output.raw table lines
20
+ * are invisible in --json mode by design).
21
+ *
22
+ * The estimate is TM-AWARE: keys the Translation Memory already covers are
23
+ * pure cache hits in the real pipeline (translate-pair.js partitions before
24
+ * any API call), so they are reported as tmHits and priced at $0 — pricing
25
+ * them as fresh calls made --max-cost abort nearly-free re-runs.
26
+ *
27
+ * Also home to abortForMaxCost() and the shared printCostTable() renderer,
28
+ * used by both the standard path (sync.js) and the Docusaurus path
29
+ * (docusaurus-sync.js) — sync.js imports docusaurus-sync.js, so shared
30
+ * cap machinery must live below both.
31
+ *
32
+ * Failures stay non-blocking when no cap is set — they log a warning,
33
+ * return null, and the sync continues.
34
+ */
35
+
36
+ import fs from 'node:fs';
37
+ import path from 'node:path';
38
+ import { flattenKeys } from './flatten.js';
39
+ import { diffLocale } from './diff.js';
40
+ import { readLocaleFile } from './format.js';
41
+ import { EST_CHARS_PER_KEY } from './config.js';
42
+ import { countPendingContentTranslations } from './content-sync.js';
43
+ import { loadTM, lookupTM, partitionByTM, tmMethodKey } from './tm.js';
44
+ import { compileNoTranslate } from './no-translate.js';
45
+ import { output } from './output.js';
46
+
47
+ /**
48
+ * Parse and validate a --max-cost flag value.
49
+ *
50
+ * Accepts any finite number >= 0 (a cap of 0 means "only proceed if the
51
+ * estimate is $0"). Rejects NaN, negatives, and empty values loudly —
52
+ * a silently-ignored malformed cap would defeat the entire fail-safe.
53
+ *
54
+ * @param {string|undefined|null} raw - Raw flag value (undefined = flag not set)
55
+ * @returns {number|null} Parsed cap in USD, or null when the flag is not set
56
+ */
57
+ export function parseMaxCost(raw) {
58
+ if (raw === undefined || raw === null || raw === false) return null;
59
+ const val = Number.parseFloat(String(raw));
60
+ if (!Number.isFinite(val) || val < 0 || String(raw).trim() === '') {
61
+ throw new Error(`--max-cost must be a non-negative number in USD (got "${raw}").`);
62
+ }
63
+ return val;
64
+ }
65
+
66
+ /**
67
+ * @typedef {object} CostEstimateSummary
68
+ * @property {string} currency - Always 'USD'
69
+ * @property {Array<{pair: string, method: string, keys: number, tmHits: number, estimatedCost: number|null, source: string}>} pairs
70
+ * Per-pair key-value estimates. `keys` counts only the keys that will
71
+ * actually reach the API (TM misses) — the count the estimate prices.
72
+ * `tmHits` counts keys the Translation Memory already covers ($0).
73
+ * null estimatedCost = unknown, never $0.
74
+ * @property {number} keyCost - Sum of KNOWN per-pair key-value estimates
75
+ * @property {{ files: number, pendingTranslations: number, estimatedCost: number|null, rough: true }|null} content
76
+ * Rough content-file estimate when contentDir is configured, else null
77
+ * @property {number} totalEstimatedCost - keyCost + known content cost
78
+ * @property {boolean} hasUnknownCosts - True when ANY pair or the content
79
+ * estimate is unknown. Consumers enforcing --max-cost must treat this as
80
+ * over-cap (unknown ≠ free).
81
+ */
82
+
83
+ /**
84
+ * Build the --max-cost abort: print the estimate-vs-cap verdict, emit a
85
+ * structured summary (--json consumers get {error} instead of regexing
86
+ * log lines), and return the result object commands/sync.js maps to exit
87
+ * code 2. Nothing has been spent when this runs — it fires BEFORE any
88
+ * translation API call.
89
+ *
90
+ * Lives here (not sync.js) so the Docusaurus path can enforce the same
91
+ * cap without importing sync.js (which imports docusaurus-sync.js).
92
+ *
93
+ * @param {number} maxCost - The user's cap in USD
94
+ * @param {number|null} estimatedCost - Known estimate, or null when unknowable
95
+ * @param {string} reason - Human-readable explanation of the abort
96
+ * @returns {object} runSync result with maxCostAborted set
97
+ */
98
+ export function abortForMaxCost(maxCost, estimatedCost, reason) {
99
+ const estimateStr = estimatedCost !== null ? `~$${estimatedCost.toFixed(4)}` : 'unknown';
100
+ const message = `${reason} Estimated cost: ${estimateStr}, --max-cost cap: $${maxCost.toFixed(4)}. ` +
101
+ 'Aborting before any API call.';
102
+ output.error(message);
103
+ output.summary({
104
+ command: 'sync',
105
+ aborted: 'max-cost',
106
+ error: message,
107
+ estimatedCost,
108
+ maxCost,
109
+ });
110
+ return {
111
+ maxCostAborted: true,
112
+ estimatedCost,
113
+ maxCost,
114
+ totalProcessed: 0,
115
+ totalFailed: 0,
116
+ failedPairs: [],
117
+ verifyErrors: 0,
118
+ verifyWarnings: 0,
119
+ };
120
+ }
121
+
122
+ /**
123
+ * Print the formatted estimate table shared by the standard and Docusaurus
124
+ * sync paths. Pure display — takes an already-computed CostEstimateSummary
125
+ * shape and writes it through the output controller.
126
+ *
127
+ * @param {CostEstimateSummary['pairs']} costEstimates - Per-pair rows
128
+ * @param {CostEstimateSummary['content']} content - Content-file line, or null
129
+ * @param {number} totalEstimatedCost - Sum of known estimates
130
+ * @param {boolean} hasUnknownCosts - Whether any estimate is unknown
131
+ */
132
+ export function printCostTable(costEstimates, content, totalEstimatedCost, hasUnknownCosts) {
133
+ const hasContentLine = content !== null && content.pendingTranslations > 0;
134
+ if (costEstimates.length === 0 && !hasContentLine) return;
135
+
136
+ // raw, not info: the table body below is raw, so in --json mode an info
137
+ // header emitted as its own NDJSON event with no data behind it — a
138
+ // dangling "Estimated translation cost:" line agents had to ignore. The
139
+ // structured estimate rides the end-of-run summary (`costEstimate`); the
140
+ // human table stays header-and-body together in default mode.
141
+ output.raw('Estimated translation cost:');
142
+ output.raw('');
143
+
144
+ if (costEstimates.length > 0) {
145
+ // Column headers. "Keys" is what will reach the API (and what the
146
+ // estimate prices); "TM hits" is served from cache at $0.
147
+ const maxPair = Math.max(4, ...costEstimates.map(e => e.pair.length));
148
+ const maxMethod = Math.max(6, ...costEstimates.map(e => e.method.length));
149
+
150
+ output.raw(` ${'Pair'.padEnd(maxPair)} ${'Method'.padEnd(maxMethod)} ${'Keys'.padStart(6)} ${'TM hits'.padStart(7)} ${'Est. Cost'.padStart(10)}`);
151
+ output.raw(` ${'─'.repeat(maxPair)} ${'─'.repeat(maxMethod)} ${'─'.repeat(6)} ${'─'.repeat(7)} ${'─'.repeat(10)}`);
152
+
153
+ for (const e of costEstimates) {
154
+ const costStr = e.estimatedCost !== null ? `~$${e.estimatedCost.toFixed(4)}` : 'unknown';
155
+ output.raw(` ${e.pair.padEnd(maxPair)} ${e.method.padEnd(maxMethod)} ${String(e.keys).padStart(6)} ${String(e.tmHits ?? 0).padStart(7)} ${costStr.padStart(10)}`);
156
+ }
157
+ }
158
+
159
+ if (hasContentLine) {
160
+ const contentCostStr = content.estimatedCost !== null
161
+ ? `~$${content.estimatedCost.toFixed(4)}`
162
+ : 'unknown';
163
+ output.raw(
164
+ ` Content files: ${content.pendingTranslations} pending translation(s) ` +
165
+ `across ${content.files} source file(s) ${contentCostStr} (rough estimate)`
166
+ );
167
+ }
168
+
169
+ const totalStr = hasUnknownCosts
170
+ ? `~$${totalEstimatedCost.toFixed(4)}+ (some methods have unknown pricing)`
171
+ : `~$${totalEstimatedCost.toFixed(4)}`;
172
+ output.raw(`\n Total: ${totalStr}`);
173
+ output.raw(' Note: Estimates are approximate. Actual cost depends on string length and model pricing.');
174
+ output.raw('');
175
+ }
176
+
177
+ /**
178
+ * Estimate and display translation costs for all pairs that have keys to translate.
179
+ *
180
+ * @param {Array<[string, object]>} pairEntries - Sorted pair graph entries [pairKey, pairConfig]
181
+ * @param {object} sourceFlat - Flattened source locale key-value map
182
+ * @param {object} config - Resolved project config (needs localesDir, fallbackPrefix, forceKeys, contentDir, inputLocale)
183
+ * @param {string} format - Detected format ('json', 'toml', 'yaml')
184
+ * @param {string} ext - File extension for the format (e.g. '.json')
185
+ * @param {string[]} changedKeys - Keys whose source content changed since last sync
186
+ * @param {object} [options]
187
+ * @param {string} [options.cwd] - Project root (content lock file lookup)
188
+ * @param {object} [options.tm] - Loaded Translation Memory (from tm.js loadTM).
189
+ * Pass the SAME object the run will use so the estimate partitions exactly
190
+ * like translate-pair.js does. When omitted, the TM is loaded from cwd.
191
+ * Callers running --no-tm must pass their empty TM object so everything
192
+ * prices as a miss.
193
+ * @param {object} [options.noTranslate] - Compiled no-translate matcher from
194
+ * lib/no-translate.js. Pass the SAME instance the run will use so exempt
195
+ * keys are excluded from the bill by the same decision that excludes them
196
+ * from translation. Derived from config when omitted.
197
+ * @returns {Promise<CostEstimateSummary|null>} Structured estimate, or null
198
+ * when estimation itself failed (callers with --max-cost must fail safe)
199
+ */
200
+ export async function printCostEstimate(pairEntries, sourceFlat, config, format, ext, changedKeys, options = {}) {
201
+ const { cwd = process.cwd() } = options;
202
+
203
+ try {
204
+ // No-translate keys are never sent to a backend, so they must never be
205
+ // priced. Sharing the caller's compiled matcher (rather than re-deriving
206
+ // one here) is what makes "not billed" a property of the same decision
207
+ // that makes it "not translated" — the two cannot drift apart.
208
+ const noTranslate = options.noTranslate || compileNoTranslate(config);
209
+ const { estimateCost } = await import('./pairs.js');
210
+ // TM partition mirror: diff.toProcess includes keys whose translations
211
+ // the TM already holds (recurring "untranslated" brand-name keys, a
212
+ // reverted source string, …). The real pipeline serves those from cache
213
+ // at $0 — pricing them as fresh API calls made --max-cost abort runs
214
+ // that were nearly free. Pure hash lookups; no network.
215
+ const tm = options.tm || loadTM(cwd);
216
+ const costEstimates = [];
217
+ let totalEstimatedCost = 0;
218
+ let hasUnknownCosts = false;
219
+
220
+ for (const [pairKey, pairConfig] of pairEntries) {
221
+ const code = pairConfig.target;
222
+ const filename = `${code}${ext}`;
223
+ const filePath = path.join(config.localesDir, filename);
224
+
225
+ let targetFlat = {};
226
+ if (fs.existsSync(filePath)) {
227
+ const data = readLocaleFile(filePath, format);
228
+ targetFlat = format === 'json' ? flattenKeys(data) : { ...data };
229
+ }
230
+
231
+ // Same confirmed-echo suppression the sync's diff applies, so the
232
+ // estimate prices exactly the keys the run will actually queue.
233
+ const tmKey = tmMethodKey(pairConfig);
234
+ const diff = diffLocale(
235
+ sourceFlat, targetFlat, config.fallbackPrefix, config.forceKeys, changedKeys,
236
+ (key, sourceValue) => lookupTM(tm, sourceValue, code, tmKey) === sourceValue,
237
+ noTranslate.active ? noTranslate.matches : null
238
+ );
239
+ const stringKeys = diff.toProcess.filter(k => typeof sourceFlat[k] === 'string');
240
+ if (stringKeys.length === 0) continue;
241
+
242
+ // Same partition translate-pair.js performs: full method key
243
+ // (method|model|register|coaching), keyed per target locale.
244
+ const { misses } = partitionByTM(tm, sourceFlat, stringKeys, code, tmKey);
245
+ const tmHits = stringKeys.length - misses.length;
246
+
247
+ if (misses.length > 0) {
248
+ // eslint-disable-next-line no-await-in-loop — sequential is fine for cost queries (cached)
249
+ const estimate = await estimateCost(misses.length, pairConfig);
250
+
251
+ if (estimate.estimatedCost !== null) {
252
+ totalEstimatedCost += estimate.estimatedCost;
253
+ } else {
254
+ hasUnknownCosts = true;
255
+ }
256
+
257
+ costEstimates.push({
258
+ pair: pairKey,
259
+ method: pairConfig.method || 'llm',
260
+ keys: misses.length,
261
+ tmHits,
262
+ estimatedCost: estimate.estimatedCost,
263
+ source: estimate.source,
264
+ });
265
+ } else {
266
+ // Fully TM-covered: zero API calls, so the cost is a KNOWN $0 even
267
+ // when the method itself has unknown pricing — this must not trip
268
+ // hasUnknownCosts. (Quality-gate feedback retries can still spend,
269
+ // but they always could and are outside every estimate here.)
270
+ costEstimates.push({
271
+ pair: pairKey,
272
+ method: pairConfig.method || 'llm',
273
+ keys: 0,
274
+ tmHits,
275
+ estimatedCost: 0,
276
+ source: 'translation-memory',
277
+ });
278
+ }
279
+ }
280
+
281
+ // ── Content files (Hugo Markdown) — ROUGH estimate ─────────────
282
+ // Counts only the (file × pair) translations the sync would actually
283
+ // run (hash-manifest skip logic), then prices each pair's pending
284
+ // source characters as EST_CHARS_PER_KEY-char key-equivalents through
285
+ // the pair's own estimateCost(). Exact for char-priced providers
286
+ // (google/deepl/…: key-equivalents × 25 chars = the real characters);
287
+ // deliberately CONSERVATIVE for token-priced LLM models, where each
288
+ // 25-char slice is priced as if it paid full per-key prompt overhead.
289
+ // NOT TM-partitioned (unlike the key-value table above): content-sync
290
+ // caches whole bodies/front-matter fields in the TM, but mirroring that
291
+ // here would mean parsing every pending file — kept rough instead.
292
+ // Rough by design — fail-safe overestimates rather than under.
293
+ let content = null;
294
+ if (config.contentDir) {
295
+ const pending = countPendingContentTranslations(
296
+ config.contentDir, config.inputLocale || 'en', pairEntries, cwd
297
+ );
298
+ let contentCost = 0;
299
+ let contentUnknown = false;
300
+ if (pending.pendingTranslations > 0) {
301
+ for (const [, pairConfig] of pairEntries) {
302
+ const perTarget = pending.byTarget[pairConfig.target];
303
+ if (!perTarget || perTarget.pendingTranslations === 0) continue;
304
+ const keyEquivalents = Math.ceil(perTarget.pendingSourceChars / EST_CHARS_PER_KEY);
305
+ // eslint-disable-next-line no-await-in-loop — sequential is fine for cost queries (cached)
306
+ const estimate = await estimateCost(keyEquivalents, pairConfig);
307
+ if (estimate.estimatedCost !== null) {
308
+ contentCost += estimate.estimatedCost;
309
+ } else {
310
+ contentUnknown = true;
311
+ }
312
+ }
313
+ }
314
+ content = {
315
+ files: pending.sourceFileCount,
316
+ pendingTranslations: pending.pendingTranslations,
317
+ estimatedCost: contentUnknown ? null : contentCost,
318
+ rough: true,
319
+ };
320
+ if (contentUnknown) {
321
+ hasUnknownCosts = true;
322
+ } else {
323
+ totalEstimatedCost += contentCost;
324
+ }
325
+ }
326
+
327
+ // ── Display ────────────────────────────────────────────────────
328
+ printCostTable(costEstimates, content, totalEstimatedCost, hasUnknownCosts);
329
+
330
+ return {
331
+ currency: 'USD',
332
+ pairs: costEstimates,
333
+ keyCost: content && content.estimatedCost !== null
334
+ ? totalEstimatedCost - content.estimatedCost
335
+ : totalEstimatedCost,
336
+ content,
337
+ totalEstimatedCost,
338
+ hasUnknownCosts,
339
+ };
340
+ } catch (costError) {
341
+ // Cost estimation is non-blocking when no cap is set — log and continue.
342
+ // Callers enforcing --max-cost must treat the null return as unknown.
343
+ output.warn(`Cost estimation failed: ${costError.message}`);
344
+ return null;
345
+ }
346
+ }
package/lib/diff.js ADDED
@@ -0,0 +1,155 @@
1
+ /**
2
+ * Key diff engine — compares source locale against target locales.
3
+ *
4
+ * Detects seven categories of keys:
5
+ * 1. Missing: exist in source but not in target
6
+ * 2. Stale: exist in target but removed from source
7
+ * 3. Fallback: exist in target but prefixed with [EN] (need real translation)
8
+ * 4. Untranslated: target value is identical to source value (likely pre-populated
9
+ * by tools like `docusaurus write-translations` with English defaults)
10
+ * 5. Changed: source content hash differs from last sync (auto-detected)
11
+ * 6. Forced: explicitly requested for re-translation via --force-keys
12
+ * 7. No-translate: declared exempt (config `noTranslate` / auto-detected URL) and
13
+ * currently NOT byte-identical to the source — needs a verbatim copy
14
+ *
15
+ * WHY: The diff is the decision layer that determines what work needs
16
+ * to be done. By separating it from the translation and write layers,
17
+ * we can dry-run, audit, or sync with identical detection logic.
18
+ *
19
+ * WHY "untranslated": Docusaurus's `write-translations` pre-populates
20
+ * ALL locale directories with English defaults. Without this check,
21
+ * the diff sees "key exists, value present, no [EN] prefix" and skips
22
+ * it — leaving entire locales (e.g. Thai, Filipino) completely untranslated.
23
+ */
24
+
25
+ import { flattenKeys } from './flatten.js';
26
+
27
+ /**
28
+ * Diff a target locale against the source.
29
+ *
30
+ * @param {object} sourceFlat - Flattened source locale
31
+ * @param {object} targetFlat - Flattened target locale
32
+ * @param {string} fallbackPrefix - The prefix marking untranslated values (default: "[EN] ")
33
+ * @param {string[]} forceKeys - Dot-notation keys to force re-translate (default: [])
34
+ * @param {string[]} changedKeys - Keys detected as changed via content hashing (default: [])
35
+ * @param {((key: string, sourceValue: string) => boolean)|null} isConfirmedEcho -
36
+ * Optional callback consulted ONLY for keys that would be queued for the
37
+ * equals-source ("untranslated") reason. Return true when the echo is
38
+ * CONFIRMED — i.e. the pipeline itself previously produced this exact
39
+ * source-equal value (gate-approved, TM-recorded) — and the key is NOT
40
+ * requeued for that reason. Missing/changed/forced reasons still queue.
41
+ * Invoked lazily so callers pay a TM lookup only for actual echo candidates.
42
+ * @param {((key: string, sourceValue: string) => boolean)|null} isNoTranslate -
43
+ * Optional predicate identifying keys whose correct value in EVERY locale is
44
+ * the source value, verbatim (config `noTranslate` patterns, auto-detected
45
+ * URLs — see lib/no-translate.js). Matching string keys are removed from every
46
+ * translate reason and routed to the `noTranslate` bucket instead, so they are
47
+ * never sent to a backend, never quality-gated, and never billed.
48
+ * @returns {import('./types.js').DiffResult} Diff result with missing, needsTranslation, untranslated, changed, forced, noTranslate, extra, toProcess
49
+ */
50
+ function diffLocale(sourceFlat, targetFlat, fallbackPrefix = '[EN] ', forceKeys = [], changedKeys = [], isConfirmedEcho = null, isNoTranslate = null) {
51
+ const sourceKeys = new Set(Object.keys(sourceFlat));
52
+ const targetKeys = new Set(Object.keys(targetFlat));
53
+
54
+ // Keys declared exempt from translation. Restricted to string-valued source
55
+ // keys: non-strings (numbers, booleans, arrays) already pass through verbatim
56
+ // on every path, so marking them would be a no-op that only muddies counts.
57
+ const exempt = isNoTranslate
58
+ ? new Set([...sourceKeys].filter(k =>
59
+ typeof sourceFlat[k] === 'string' && isNoTranslate(k, sourceFlat[k])))
60
+ : new Set();
61
+
62
+ // Keys in source but not in target
63
+ const missing = [...sourceKeys].filter(k => !targetKeys.has(k) && !exempt.has(k));
64
+
65
+ // Keys in target that are still [EN]-prefixed fallbacks
66
+ const needsTranslation = [...targetKeys].filter(k =>
67
+ !exempt.has(k) && typeof targetFlat[k] === 'string' && targetFlat[k].startsWith(fallbackPrefix)
68
+ );
69
+
70
+ // Keys where target value is identical to source value.
71
+ // This catches locales pre-populated with English defaults by tools
72
+ // like `docusaurus write-translations`. Without this, Thai/Filipino/etc.
73
+ // would appear "fully synced" with all-English content.
74
+ //
75
+ // Only flag string values that aren't already caught by needsTranslation,
76
+ // and skip very short values (1-2 chars) that are likely intentionally
77
+ // identical across locales (e.g. punctuation, symbols, numbers).
78
+ const needsTranslationSet = new Set(needsTranslation);
79
+ const untranslated = [...targetKeys].filter(k => {
80
+ if (needsTranslationSet.has(k)) return false; // already flagged
81
+ if (exempt.has(k)) return false; // source-equal IS the correct state here
82
+ if (!sourceKeys.has(k)) return false; // extra key, not our concern
83
+ const sv = sourceFlat[k];
84
+ const tv = targetFlat[k];
85
+ // Only compare string values
86
+ if (typeof sv !== 'string' || typeof tv !== 'string') return false;
87
+ // Skip very short values — punctuation, numbers, symbols are
88
+ // often intentionally identical across locales.
89
+ if (sv.length <= 2) return false;
90
+ if (sv !== tv) return false;
91
+ // Confirmed echo: the pipeline previously produced this exact
92
+ // source-equal value (it passed the gate and sits in the TM), so
93
+ // requeuing it every sync would just re-serve the same TM hit and
94
+ // rewrite the same file forever. Brand-new echoes (unconfirmed)
95
+ // still queue once — after the API returns the same text and the TM
96
+ // stores it, subsequent syncs skip.
97
+ if (isConfirmedEcho && isConfirmedEcho(k, sv)) return false;
98
+ return true;
99
+ });
100
+
101
+ // Keys whose English source content changed since last sync (auto-detected).
102
+ // Only include keys that exist in the source (defensive filter).
103
+ const changed = changedKeys.filter(k => sourceKeys.has(k) && !exempt.has(k));
104
+
105
+ // Keys explicitly forced for re-translation (only if they exist in source).
106
+ // Silently ignore any forced keys that don't exist in the source.
107
+ //
108
+ // --force-keys does NOT override no-translate: forcing re-translation of a
109
+ // key whose only correct output is the source value would just re-run the
110
+ // failure this bucket exists to prevent. The key still gets re-copied below
111
+ // if the target has drifted.
112
+ const forced = forceKeys.filter(k => sourceKeys.has(k) && !exempt.has(k));
113
+
114
+ // Keys in target but not in source (stale/orphaned)
115
+ const extra = [...targetKeys].filter(k => !sourceKeys.has(k));
116
+
117
+ // Exempt keys whose target is not byte-identical to the source. That covers
118
+ // three states with one rule: never written, [EN]-prefixed from an older
119
+ // run, or CORRUPTED — the gate-dodging edits ("…/view/1954#fr", a prepended
120
+ // U+200E) that shipped to production. All three are repaired by the same
121
+ // verbatim copy, so the drift heals itself on the next sync and the result
122
+ // is idempotent: once equal, the key stops appearing here.
123
+ const noTranslate = [...exempt].filter(k => targetFlat[k] !== sourceFlat[k]);
124
+
125
+ // Combined set of keys that need work (deduplicated). Exempt keys are
126
+ // deliberately absent: nothing here is sent to a backend or billed.
127
+ const toProcess = [...new Set([...missing, ...needsTranslation, ...untranslated, ...changed, ...forced])];
128
+
129
+ return { missing, needsTranslation, untranslated, changed, forced, noTranslate, extra, toProcess };
130
+ }
131
+
132
+ /**
133
+ * Generate a human-readable label for the diff result.
134
+ *
135
+ * @param {import('./types.js').DiffResult} diff - Diff result from diffLocale
136
+ * @returns {string} Human-readable summary (e.g., '3 missing + 1 [EN] fallback(s)')
137
+ */
138
+ function diffLabel(diff) {
139
+ const { missing, needsTranslation, untranslated, changed, forced, noTranslate } = diff;
140
+ const parts = [];
141
+ if (missing.length > 0) parts.push(`${missing.length} missing`);
142
+ if (needsTranslation.length > 0) parts.push(`${needsTranslation.length} [EN] fallback(s)`);
143
+ if (untranslated && untranslated.length > 0) parts.push(`${untranslated.length} untranslated`);
144
+ if (changed && changed.length > 0) parts.push(`${changed.length} changed`);
145
+ // Forced keys were invisible here, so a `--force-keys a,b` run over a
146
+ // locale with requeued echoes printed a total ("Translating 77 key(s)")
147
+ // that nothing itemized — a user could not tell their 2 forced keys from
148
+ // the 75 unstamped echoes riding along.
149
+ if (forced && forced.length > 0) parts.push(`${forced.length} forced`);
150
+ if (noTranslate && noTranslate.length > 0) parts.push(`${noTranslate.length} to copy verbatim`);
151
+ if (parts.length > 0) return parts.join(' + ');
152
+ return 'fully synced';
153
+ }
154
+
155
+ export { diffLocale, diffLabel };