champollion 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +133 -0
- package/README.md +387 -0
- package/bin/cli.js +278 -0
- package/index.js +135 -0
- package/lib/api-key.js +127 -0
- package/lib/autofix.js +432 -0
- package/lib/bridge/method_bridge.py +430 -0
- package/lib/card-source-resolution.mjs +284 -0
- package/lib/cards/cache.js +169 -0
- package/lib/cards/env.js +82 -0
- package/lib/cards/fetch-card-child.js +38 -0
- package/lib/cards/reader.js +435 -0
- package/lib/cards/refresh.js +111 -0
- package/lib/cards/remote.js +387 -0
- package/lib/cldf-export.mjs +540 -0
- package/lib/cldf-terms.mjs +62 -0
- package/lib/command-help.js +790 -0
- package/lib/commands/audit.js +49 -0
- package/lib/commands/card.js +454 -0
- package/lib/commands/doctor.js +559 -0
- package/lib/commands/fonts.js +489 -0
- package/lib/commands/help.js +91 -0
- package/lib/commands/init.js +1259 -0
- package/lib/commands/integrity.js +148 -0
- package/lib/commands/leaderboard.js +478 -0
- package/lib/commands/lint.js +30 -0
- package/lib/commands/models.js +177 -0
- package/lib/commands/plugin.js +103 -0
- package/lib/commands/provenance.js +45 -0
- package/lib/commands/recommend.js +75 -0
- package/lib/commands/register-corpus.js +678 -0
- package/lib/commands/repair-script.js +42 -0
- package/lib/commands/seal-corpus.js +355 -0
- package/lib/commands/seo.js +72 -0
- package/lib/commands/serve.js +147 -0
- package/lib/commands/status.js +265 -0
- package/lib/commands/submit.js +332 -0
- package/lib/commands/sync.js +89 -0
- package/lib/commands/tm.js +573 -0
- package/lib/commands/verify.js +39 -0
- package/lib/commands/watch.js +20 -0
- package/lib/commands/wrap.js +138 -0
- package/lib/commands/xliff.js +327 -0
- package/lib/commercial-eligibility.js +235 -0
- package/lib/concurrent.js +87 -0
- package/lib/config.js +523 -0
- package/lib/contamination-lane.js +76 -0
- package/lib/content-sync.js +731 -0
- package/lib/content.js +733 -0
- package/lib/corpus-registration.mjs +608 -0
- package/lib/cost-report.js +346 -0
- package/lib/diff.js +155 -0
- package/lib/docusaurus-sync.js +1256 -0
- package/lib/flatten.js +55 -0
- package/lib/format.js +954 -0
- package/lib/hash.js +159 -0
- package/lib/icu.js +473 -0
- package/lib/integrity.js +689 -0
- package/lib/license-gate.mjs +478 -0
- package/lib/license-identify.mjs +229 -0
- package/lib/lint.js +629 -0
- package/lib/method-manifest.js +60 -0
- package/lib/methods/anthropic.js +140 -0
- package/lib/methods/apertium.js +163 -0
- package/lib/methods/api.js +316 -0
- package/lib/methods/base.js +184 -0
- package/lib/methods/content-separator.js +45 -0
- package/lib/methods/deepl.js +426 -0
- package/lib/methods/direct-llm.js +586 -0
- package/lib/methods/external.js +332 -0
- package/lib/methods/fetch-with-retry.js +124 -0
- package/lib/methods/gemini.js +147 -0
- package/lib/methods/google-translate.js +402 -0
- package/lib/methods/http-utils.js +122 -0
- package/lib/methods/libretranslate.js +314 -0
- package/lib/methods/llm-coached.js +670 -0
- package/lib/methods/llm.js +592 -0
- package/lib/methods/local.js +76 -0
- package/lib/methods/microsoft-translator.js +331 -0
- package/lib/methods/openai.js +131 -0
- package/lib/methods/openrouter-client.js +327 -0
- package/lib/methods/openrouter-pricing.js +156 -0
- package/lib/methods/provider-env.js +115 -0
- package/lib/methods/provider-pricing.js +310 -0
- package/lib/methods/tilde.js +150 -0
- package/lib/methods/translated.js +229 -0
- package/lib/methods/translation-error.js +80 -0
- package/lib/models.js +258 -0
- package/lib/no-translate.js +233 -0
- package/lib/output.js +238 -0
- package/lib/pairs.js +547 -0
- package/lib/plugins.js +447 -0
- package/lib/provenance.js +323 -0
- package/lib/recommend.js +648 -0
- package/lib/registers.js +1185 -0
- package/lib/repair-script.js +266 -0
- package/lib/scripts.js +994 -0
- package/lib/seal.mjs +464 -0
- package/lib/sealed-qualifier.mjs +211 -0
- package/lib/security.js +59 -0
- package/lib/segment.js +369 -0
- package/lib/seo.js +275 -0
- package/lib/serve.js +854 -0
- package/lib/string-classify.js +85 -0
- package/lib/submit.mjs +344 -0
- package/lib/sync.js +969 -0
- package/lib/tags/bcp47.js +202 -0
- package/lib/tags/resolve.js +314 -0
- package/lib/terminology.js +111 -0
- package/lib/tm-seed.js +294 -0
- package/lib/tm.js +515 -0
- package/lib/translate-pair.js +197 -0
- package/lib/translate.js +203 -0
- package/lib/types.js +230 -0
- package/lib/validate.js +510 -0
- package/lib/verify.js +451 -0
- package/lib/watch.js +145 -0
- package/lib/xliff.js +184 -0
- package/package.json +93 -0
- package/shared/ATTRIBUTION.md +145 -0
- package/shared/CORPORA-CARDS.md +288 -0
- package/shared/DATA-SOVEREIGNTY.md +500 -0
- package/shared/LANGUAGE-CARD-FIELDS.md +532 -0
- package/shared/card-lint-baseline.json +3189 -0
- package/shared/cards-fallback.json +1 -0
- package/shared/catalogue/card-config.json +6091 -0
- package/shared/catalogue/external-results.json +3888 -0
- package/shared/catalogue/gender-guidance.json +1038 -0
- package/shared/catalogue/method-coverage.json +1751 -0
- package/shared/catalogue/metric-coverage.json +170 -0
- package/shared/catalogue/metric-reliability.json +1 -0
- package/shared/catalogue/register-presets.json +3180 -0
- package/shared/catalogue/vitality-scales.json +55 -0
- package/shared/cldr-index.json +1115 -0
- package/shared/code-bridge.json +253 -0
- package/shared/corpora-cards-v1-reference.md +281 -0
- package/shared/curated-dictionary-flags.json +35 -0
- package/shared/curated-endonyms.json +35 -0
- package/shared/curated-fsts.json +51 -0
- package/shared/curated-orthography-conventions.json +26 -0
- package/shared/curated-sil-resources.json +374 -0
- package/shared/curated-tools.json +41 -0
- package/shared/docent/corpus.json +11333 -0
- package/shared/docent/faq.en.json +564 -0
- package/shared/docent/register-blocks.json +60 -0
- package/shared/docent/system-prompt.md +144 -0
- package/shared/domain-taxonomy.json +35 -0
- package/shared/explainers/glossary.json +2975 -0
- package/shared/explainers/tc-features.json +20112 -0
- package/shared/explainers/term-watchlist.json +147 -0
- package/shared/human-services.json +59 -0
- package/shared/license-corrections.json +261 -0
- package/shared/license-evidence.json +13452 -0
- package/shared/licenses.json +6781 -0
- package/shared/method-registry.json +236 -0
- package/shared/metric-registry.json +620 -0
- package/shared/model-aliases.json +7 -0
- package/shared/schemas/champollion-plugin.schema.json +206 -0
- package/shared/schemas/corpora-card.schema.json +957 -0
- package/shared/schemas/domain-taxonomy.schema.json +64 -0
- package/shared/schemas/external-results.schema.json +314 -0
- package/shared/schemas/human-services.schema.json +90 -0
- package/shared/schemas/language-card.schema.json +1308 -0
- package/shared/schemas/licenses.schema.json +155 -0
- package/shared/schemas/method-card.schema.json +412 -0
- package/shared/schemas/method-registry.schema.json +85 -0
- package/shared/schemas/metric-registry.schema.json +96 -0
- package/shared/schemas/metric-reliability.schema.json +178 -0
- package/shared/schemas/model-aliases.schema.json +27 -0
- package/shared/schemas/source-snapshot.schema.json +96 -0
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* cost-report.js — Pre-sync cost estimation display
|
|
3
|
+
*
|
|
4
|
+
* Extracted from sync.js to reduce god-module complexity.
|
|
5
|
+
* This module is purely informational — it reads locale files,
|
|
6
|
+
* estimates translation costs via the pairs API, and prints a
|
|
7
|
+
* formatted table using the output controller. No side effects
|
|
8
|
+
* on sync state.
|
|
9
|
+
*
|
|
10
|
+
* Called once at the start of runSync for EVERY engine — it used to
|
|
11
|
+
* early-return unless OPENROUTER_API_KEY was set, which silently hid
|
|
12
|
+
* the preview from google-translate/deepl/gemini/… users who never
|
|
13
|
+
* touch OpenRouter. Each method implements estimateCost() (or honestly
|
|
14
|
+
* returns null = "unknown", never $0), so the key gate was wrong.
|
|
15
|
+
*
|
|
16
|
+
* The structured estimate is RETURNED to the caller so that:
|
|
17
|
+
* - sync.js can enforce the --max-cost cap BEFORE any API call
|
|
18
|
+
* (unknown ≠ free: an unknowable estimate under a cap aborts), and
|
|
19
|
+
* - the estimate reaches the --json summary (output.raw table lines
|
|
20
|
+
* are invisible in --json mode by design).
|
|
21
|
+
*
|
|
22
|
+
* The estimate is TM-AWARE: keys the Translation Memory already covers are
|
|
23
|
+
* pure cache hits in the real pipeline (translate-pair.js partitions before
|
|
24
|
+
* any API call), so they are reported as tmHits and priced at $0 — pricing
|
|
25
|
+
* them as fresh calls made --max-cost abort nearly-free re-runs.
|
|
26
|
+
*
|
|
27
|
+
* Also home to abortForMaxCost() and the shared printCostTable() renderer,
|
|
28
|
+
* used by both the standard path (sync.js) and the Docusaurus path
|
|
29
|
+
* (docusaurus-sync.js) — sync.js imports docusaurus-sync.js, so shared
|
|
30
|
+
* cap machinery must live below both.
|
|
31
|
+
*
|
|
32
|
+
* Failures stay non-blocking when no cap is set — they log a warning,
|
|
33
|
+
* return null, and the sync continues.
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
import fs from 'node:fs';
|
|
37
|
+
import path from 'node:path';
|
|
38
|
+
import { flattenKeys } from './flatten.js';
|
|
39
|
+
import { diffLocale } from './diff.js';
|
|
40
|
+
import { readLocaleFile } from './format.js';
|
|
41
|
+
import { EST_CHARS_PER_KEY } from './config.js';
|
|
42
|
+
import { countPendingContentTranslations } from './content-sync.js';
|
|
43
|
+
import { loadTM, lookupTM, partitionByTM, tmMethodKey } from './tm.js';
|
|
44
|
+
import { compileNoTranslate } from './no-translate.js';
|
|
45
|
+
import { output } from './output.js';
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Parse and validate a --max-cost flag value.
|
|
49
|
+
*
|
|
50
|
+
* Accepts any finite number >= 0 (a cap of 0 means "only proceed if the
|
|
51
|
+
* estimate is $0"). Rejects NaN, negatives, and empty values loudly —
|
|
52
|
+
* a silently-ignored malformed cap would defeat the entire fail-safe.
|
|
53
|
+
*
|
|
54
|
+
* @param {string|undefined|null} raw - Raw flag value (undefined = flag not set)
|
|
55
|
+
* @returns {number|null} Parsed cap in USD, or null when the flag is not set
|
|
56
|
+
*/
|
|
57
|
+
export function parseMaxCost(raw) {
|
|
58
|
+
if (raw === undefined || raw === null || raw === false) return null;
|
|
59
|
+
const val = Number.parseFloat(String(raw));
|
|
60
|
+
if (!Number.isFinite(val) || val < 0 || String(raw).trim() === '') {
|
|
61
|
+
throw new Error(`--max-cost must be a non-negative number in USD (got "${raw}").`);
|
|
62
|
+
}
|
|
63
|
+
return val;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* @typedef {object} CostEstimateSummary
|
|
68
|
+
* @property {string} currency - Always 'USD'
|
|
69
|
+
* @property {Array<{pair: string, method: string, keys: number, tmHits: number, estimatedCost: number|null, source: string}>} pairs
|
|
70
|
+
* Per-pair key-value estimates. `keys` counts only the keys that will
|
|
71
|
+
* actually reach the API (TM misses) — the count the estimate prices.
|
|
72
|
+
* `tmHits` counts keys the Translation Memory already covers ($0).
|
|
73
|
+
* null estimatedCost = unknown, never $0.
|
|
74
|
+
* @property {number} keyCost - Sum of KNOWN per-pair key-value estimates
|
|
75
|
+
* @property {{ files: number, pendingTranslations: number, estimatedCost: number|null, rough: true }|null} content
|
|
76
|
+
* Rough content-file estimate when contentDir is configured, else null
|
|
77
|
+
* @property {number} totalEstimatedCost - keyCost + known content cost
|
|
78
|
+
* @property {boolean} hasUnknownCosts - True when ANY pair or the content
|
|
79
|
+
* estimate is unknown. Consumers enforcing --max-cost must treat this as
|
|
80
|
+
* over-cap (unknown ≠ free).
|
|
81
|
+
*/
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Build the --max-cost abort: print the estimate-vs-cap verdict, emit a
|
|
85
|
+
* structured summary (--json consumers get {error} instead of regexing
|
|
86
|
+
* log lines), and return the result object commands/sync.js maps to exit
|
|
87
|
+
* code 2. Nothing has been spent when this runs — it fires BEFORE any
|
|
88
|
+
* translation API call.
|
|
89
|
+
*
|
|
90
|
+
* Lives here (not sync.js) so the Docusaurus path can enforce the same
|
|
91
|
+
* cap without importing sync.js (which imports docusaurus-sync.js).
|
|
92
|
+
*
|
|
93
|
+
* @param {number} maxCost - The user's cap in USD
|
|
94
|
+
* @param {number|null} estimatedCost - Known estimate, or null when unknowable
|
|
95
|
+
* @param {string} reason - Human-readable explanation of the abort
|
|
96
|
+
* @returns {object} runSync result with maxCostAborted set
|
|
97
|
+
*/
|
|
98
|
+
export function abortForMaxCost(maxCost, estimatedCost, reason) {
|
|
99
|
+
const estimateStr = estimatedCost !== null ? `~$${estimatedCost.toFixed(4)}` : 'unknown';
|
|
100
|
+
const message = `${reason} Estimated cost: ${estimateStr}, --max-cost cap: $${maxCost.toFixed(4)}. ` +
|
|
101
|
+
'Aborting before any API call.';
|
|
102
|
+
output.error(message);
|
|
103
|
+
output.summary({
|
|
104
|
+
command: 'sync',
|
|
105
|
+
aborted: 'max-cost',
|
|
106
|
+
error: message,
|
|
107
|
+
estimatedCost,
|
|
108
|
+
maxCost,
|
|
109
|
+
});
|
|
110
|
+
return {
|
|
111
|
+
maxCostAborted: true,
|
|
112
|
+
estimatedCost,
|
|
113
|
+
maxCost,
|
|
114
|
+
totalProcessed: 0,
|
|
115
|
+
totalFailed: 0,
|
|
116
|
+
failedPairs: [],
|
|
117
|
+
verifyErrors: 0,
|
|
118
|
+
verifyWarnings: 0,
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Print the formatted estimate table shared by the standard and Docusaurus
|
|
124
|
+
* sync paths. Pure display — takes an already-computed CostEstimateSummary
|
|
125
|
+
* shape and writes it through the output controller.
|
|
126
|
+
*
|
|
127
|
+
* @param {CostEstimateSummary['pairs']} costEstimates - Per-pair rows
|
|
128
|
+
* @param {CostEstimateSummary['content']} content - Content-file line, or null
|
|
129
|
+
* @param {number} totalEstimatedCost - Sum of known estimates
|
|
130
|
+
* @param {boolean} hasUnknownCosts - Whether any estimate is unknown
|
|
131
|
+
*/
|
|
132
|
+
export function printCostTable(costEstimates, content, totalEstimatedCost, hasUnknownCosts) {
|
|
133
|
+
const hasContentLine = content !== null && content.pendingTranslations > 0;
|
|
134
|
+
if (costEstimates.length === 0 && !hasContentLine) return;
|
|
135
|
+
|
|
136
|
+
// raw, not info: the table body below is raw, so in --json mode an info
|
|
137
|
+
// header emitted as its own NDJSON event with no data behind it — a
|
|
138
|
+
// dangling "Estimated translation cost:" line agents had to ignore. The
|
|
139
|
+
// structured estimate rides the end-of-run summary (`costEstimate`); the
|
|
140
|
+
// human table stays header-and-body together in default mode.
|
|
141
|
+
output.raw('Estimated translation cost:');
|
|
142
|
+
output.raw('');
|
|
143
|
+
|
|
144
|
+
if (costEstimates.length > 0) {
|
|
145
|
+
// Column headers. "Keys" is what will reach the API (and what the
|
|
146
|
+
// estimate prices); "TM hits" is served from cache at $0.
|
|
147
|
+
const maxPair = Math.max(4, ...costEstimates.map(e => e.pair.length));
|
|
148
|
+
const maxMethod = Math.max(6, ...costEstimates.map(e => e.method.length));
|
|
149
|
+
|
|
150
|
+
output.raw(` ${'Pair'.padEnd(maxPair)} ${'Method'.padEnd(maxMethod)} ${'Keys'.padStart(6)} ${'TM hits'.padStart(7)} ${'Est. Cost'.padStart(10)}`);
|
|
151
|
+
output.raw(` ${'─'.repeat(maxPair)} ${'─'.repeat(maxMethod)} ${'─'.repeat(6)} ${'─'.repeat(7)} ${'─'.repeat(10)}`);
|
|
152
|
+
|
|
153
|
+
for (const e of costEstimates) {
|
|
154
|
+
const costStr = e.estimatedCost !== null ? `~$${e.estimatedCost.toFixed(4)}` : 'unknown';
|
|
155
|
+
output.raw(` ${e.pair.padEnd(maxPair)} ${e.method.padEnd(maxMethod)} ${String(e.keys).padStart(6)} ${String(e.tmHits ?? 0).padStart(7)} ${costStr.padStart(10)}`);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
if (hasContentLine) {
|
|
160
|
+
const contentCostStr = content.estimatedCost !== null
|
|
161
|
+
? `~$${content.estimatedCost.toFixed(4)}`
|
|
162
|
+
: 'unknown';
|
|
163
|
+
output.raw(
|
|
164
|
+
` Content files: ${content.pendingTranslations} pending translation(s) ` +
|
|
165
|
+
`across ${content.files} source file(s) ${contentCostStr} (rough estimate)`
|
|
166
|
+
);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
const totalStr = hasUnknownCosts
|
|
170
|
+
? `~$${totalEstimatedCost.toFixed(4)}+ (some methods have unknown pricing)`
|
|
171
|
+
: `~$${totalEstimatedCost.toFixed(4)}`;
|
|
172
|
+
output.raw(`\n Total: ${totalStr}`);
|
|
173
|
+
output.raw(' Note: Estimates are approximate. Actual cost depends on string length and model pricing.');
|
|
174
|
+
output.raw('');
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Estimate and display translation costs for all pairs that have keys to translate.
|
|
179
|
+
*
|
|
180
|
+
* @param {Array<[string, object]>} pairEntries - Sorted pair graph entries [pairKey, pairConfig]
|
|
181
|
+
* @param {object} sourceFlat - Flattened source locale key-value map
|
|
182
|
+
* @param {object} config - Resolved project config (needs localesDir, fallbackPrefix, forceKeys, contentDir, inputLocale)
|
|
183
|
+
* @param {string} format - Detected format ('json', 'toml', 'yaml')
|
|
184
|
+
* @param {string} ext - File extension for the format (e.g. '.json')
|
|
185
|
+
* @param {string[]} changedKeys - Keys whose source content changed since last sync
|
|
186
|
+
* @param {object} [options]
|
|
187
|
+
* @param {string} [options.cwd] - Project root (content lock file lookup)
|
|
188
|
+
* @param {object} [options.tm] - Loaded Translation Memory (from tm.js loadTM).
|
|
189
|
+
* Pass the SAME object the run will use so the estimate partitions exactly
|
|
190
|
+
* like translate-pair.js does. When omitted, the TM is loaded from cwd.
|
|
191
|
+
* Callers running --no-tm must pass their empty TM object so everything
|
|
192
|
+
* prices as a miss.
|
|
193
|
+
* @param {object} [options.noTranslate] - Compiled no-translate matcher from
|
|
194
|
+
* lib/no-translate.js. Pass the SAME instance the run will use so exempt
|
|
195
|
+
* keys are excluded from the bill by the same decision that excludes them
|
|
196
|
+
* from translation. Derived from config when omitted.
|
|
197
|
+
* @returns {Promise<CostEstimateSummary|null>} Structured estimate, or null
|
|
198
|
+
* when estimation itself failed (callers with --max-cost must fail safe)
|
|
199
|
+
*/
|
|
200
|
+
export async function printCostEstimate(pairEntries, sourceFlat, config, format, ext, changedKeys, options = {}) {
|
|
201
|
+
const { cwd = process.cwd() } = options;
|
|
202
|
+
|
|
203
|
+
try {
|
|
204
|
+
// No-translate keys are never sent to a backend, so they must never be
|
|
205
|
+
// priced. Sharing the caller's compiled matcher (rather than re-deriving
|
|
206
|
+
// one here) is what makes "not billed" a property of the same decision
|
|
207
|
+
// that makes it "not translated" — the two cannot drift apart.
|
|
208
|
+
const noTranslate = options.noTranslate || compileNoTranslate(config);
|
|
209
|
+
const { estimateCost } = await import('./pairs.js');
|
|
210
|
+
// TM partition mirror: diff.toProcess includes keys whose translations
|
|
211
|
+
// the TM already holds (recurring "untranslated" brand-name keys, a
|
|
212
|
+
// reverted source string, …). The real pipeline serves those from cache
|
|
213
|
+
// at $0 — pricing them as fresh API calls made --max-cost abort runs
|
|
214
|
+
// that were nearly free. Pure hash lookups; no network.
|
|
215
|
+
const tm = options.tm || loadTM(cwd);
|
|
216
|
+
const costEstimates = [];
|
|
217
|
+
let totalEstimatedCost = 0;
|
|
218
|
+
let hasUnknownCosts = false;
|
|
219
|
+
|
|
220
|
+
for (const [pairKey, pairConfig] of pairEntries) {
|
|
221
|
+
const code = pairConfig.target;
|
|
222
|
+
const filename = `${code}${ext}`;
|
|
223
|
+
const filePath = path.join(config.localesDir, filename);
|
|
224
|
+
|
|
225
|
+
let targetFlat = {};
|
|
226
|
+
if (fs.existsSync(filePath)) {
|
|
227
|
+
const data = readLocaleFile(filePath, format);
|
|
228
|
+
targetFlat = format === 'json' ? flattenKeys(data) : { ...data };
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
// Same confirmed-echo suppression the sync's diff applies, so the
|
|
232
|
+
// estimate prices exactly the keys the run will actually queue.
|
|
233
|
+
const tmKey = tmMethodKey(pairConfig);
|
|
234
|
+
const diff = diffLocale(
|
|
235
|
+
sourceFlat, targetFlat, config.fallbackPrefix, config.forceKeys, changedKeys,
|
|
236
|
+
(key, sourceValue) => lookupTM(tm, sourceValue, code, tmKey) === sourceValue,
|
|
237
|
+
noTranslate.active ? noTranslate.matches : null
|
|
238
|
+
);
|
|
239
|
+
const stringKeys = diff.toProcess.filter(k => typeof sourceFlat[k] === 'string');
|
|
240
|
+
if (stringKeys.length === 0) continue;
|
|
241
|
+
|
|
242
|
+
// Same partition translate-pair.js performs: full method key
|
|
243
|
+
// (method|model|register|coaching), keyed per target locale.
|
|
244
|
+
const { misses } = partitionByTM(tm, sourceFlat, stringKeys, code, tmKey);
|
|
245
|
+
const tmHits = stringKeys.length - misses.length;
|
|
246
|
+
|
|
247
|
+
if (misses.length > 0) {
|
|
248
|
+
// eslint-disable-next-line no-await-in-loop — sequential is fine for cost queries (cached)
|
|
249
|
+
const estimate = await estimateCost(misses.length, pairConfig);
|
|
250
|
+
|
|
251
|
+
if (estimate.estimatedCost !== null) {
|
|
252
|
+
totalEstimatedCost += estimate.estimatedCost;
|
|
253
|
+
} else {
|
|
254
|
+
hasUnknownCosts = true;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
costEstimates.push({
|
|
258
|
+
pair: pairKey,
|
|
259
|
+
method: pairConfig.method || 'llm',
|
|
260
|
+
keys: misses.length,
|
|
261
|
+
tmHits,
|
|
262
|
+
estimatedCost: estimate.estimatedCost,
|
|
263
|
+
source: estimate.source,
|
|
264
|
+
});
|
|
265
|
+
} else {
|
|
266
|
+
// Fully TM-covered: zero API calls, so the cost is a KNOWN $0 even
|
|
267
|
+
// when the method itself has unknown pricing — this must not trip
|
|
268
|
+
// hasUnknownCosts. (Quality-gate feedback retries can still spend,
|
|
269
|
+
// but they always could and are outside every estimate here.)
|
|
270
|
+
costEstimates.push({
|
|
271
|
+
pair: pairKey,
|
|
272
|
+
method: pairConfig.method || 'llm',
|
|
273
|
+
keys: 0,
|
|
274
|
+
tmHits,
|
|
275
|
+
estimatedCost: 0,
|
|
276
|
+
source: 'translation-memory',
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// ── Content files (Hugo Markdown) — ROUGH estimate ─────────────
|
|
282
|
+
// Counts only the (file × pair) translations the sync would actually
|
|
283
|
+
// run (hash-manifest skip logic), then prices each pair's pending
|
|
284
|
+
// source characters as EST_CHARS_PER_KEY-char key-equivalents through
|
|
285
|
+
// the pair's own estimateCost(). Exact for char-priced providers
|
|
286
|
+
// (google/deepl/…: key-equivalents × 25 chars = the real characters);
|
|
287
|
+
// deliberately CONSERVATIVE for token-priced LLM models, where each
|
|
288
|
+
// 25-char slice is priced as if it paid full per-key prompt overhead.
|
|
289
|
+
// NOT TM-partitioned (unlike the key-value table above): content-sync
|
|
290
|
+
// caches whole bodies/front-matter fields in the TM, but mirroring that
|
|
291
|
+
// here would mean parsing every pending file — kept rough instead.
|
|
292
|
+
// Rough by design — fail-safe overestimates rather than under.
|
|
293
|
+
let content = null;
|
|
294
|
+
if (config.contentDir) {
|
|
295
|
+
const pending = countPendingContentTranslations(
|
|
296
|
+
config.contentDir, config.inputLocale || 'en', pairEntries, cwd
|
|
297
|
+
);
|
|
298
|
+
let contentCost = 0;
|
|
299
|
+
let contentUnknown = false;
|
|
300
|
+
if (pending.pendingTranslations > 0) {
|
|
301
|
+
for (const [, pairConfig] of pairEntries) {
|
|
302
|
+
const perTarget = pending.byTarget[pairConfig.target];
|
|
303
|
+
if (!perTarget || perTarget.pendingTranslations === 0) continue;
|
|
304
|
+
const keyEquivalents = Math.ceil(perTarget.pendingSourceChars / EST_CHARS_PER_KEY);
|
|
305
|
+
// eslint-disable-next-line no-await-in-loop — sequential is fine for cost queries (cached)
|
|
306
|
+
const estimate = await estimateCost(keyEquivalents, pairConfig);
|
|
307
|
+
if (estimate.estimatedCost !== null) {
|
|
308
|
+
contentCost += estimate.estimatedCost;
|
|
309
|
+
} else {
|
|
310
|
+
contentUnknown = true;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
content = {
|
|
315
|
+
files: pending.sourceFileCount,
|
|
316
|
+
pendingTranslations: pending.pendingTranslations,
|
|
317
|
+
estimatedCost: contentUnknown ? null : contentCost,
|
|
318
|
+
rough: true,
|
|
319
|
+
};
|
|
320
|
+
if (contentUnknown) {
|
|
321
|
+
hasUnknownCosts = true;
|
|
322
|
+
} else {
|
|
323
|
+
totalEstimatedCost += contentCost;
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
// ── Display ────────────────────────────────────────────────────
|
|
328
|
+
printCostTable(costEstimates, content, totalEstimatedCost, hasUnknownCosts);
|
|
329
|
+
|
|
330
|
+
return {
|
|
331
|
+
currency: 'USD',
|
|
332
|
+
pairs: costEstimates,
|
|
333
|
+
keyCost: content && content.estimatedCost !== null
|
|
334
|
+
? totalEstimatedCost - content.estimatedCost
|
|
335
|
+
: totalEstimatedCost,
|
|
336
|
+
content,
|
|
337
|
+
totalEstimatedCost,
|
|
338
|
+
hasUnknownCosts,
|
|
339
|
+
};
|
|
340
|
+
} catch (costError) {
|
|
341
|
+
// Cost estimation is non-blocking when no cap is set — log and continue.
|
|
342
|
+
// Callers enforcing --max-cost must treat the null return as unknown.
|
|
343
|
+
output.warn(`Cost estimation failed: ${costError.message}`);
|
|
344
|
+
return null;
|
|
345
|
+
}
|
|
346
|
+
}
|
package/lib/diff.js
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Key diff engine — compares source locale against target locales.
|
|
3
|
+
*
|
|
4
|
+
* Detects seven categories of keys:
|
|
5
|
+
* 1. Missing: exist in source but not in target
|
|
6
|
+
* 2. Stale: exist in target but removed from source
|
|
7
|
+
* 3. Fallback: exist in target but prefixed with [EN] (need real translation)
|
|
8
|
+
* 4. Untranslated: target value is identical to source value (likely pre-populated
|
|
9
|
+
* by tools like `docusaurus write-translations` with English defaults)
|
|
10
|
+
* 5. Changed: source content hash differs from last sync (auto-detected)
|
|
11
|
+
* 6. Forced: explicitly requested for re-translation via --force-keys
|
|
12
|
+
* 7. No-translate: declared exempt (config `noTranslate` / auto-detected URL) and
|
|
13
|
+
* currently NOT byte-identical to the source — needs a verbatim copy
|
|
14
|
+
*
|
|
15
|
+
* WHY: The diff is the decision layer that determines what work needs
|
|
16
|
+
* to be done. By separating it from the translation and write layers,
|
|
17
|
+
* we can dry-run, audit, or sync with identical detection logic.
|
|
18
|
+
*
|
|
19
|
+
* WHY "untranslated": Docusaurus's `write-translations` pre-populates
|
|
20
|
+
* ALL locale directories with English defaults. Without this check,
|
|
21
|
+
* the diff sees "key exists, value present, no [EN] prefix" and skips
|
|
22
|
+
* it — leaving entire locales (e.g. Thai, Filipino) completely untranslated.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { flattenKeys } from './flatten.js';
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Diff a target locale against the source.
|
|
29
|
+
*
|
|
30
|
+
* @param {object} sourceFlat - Flattened source locale
|
|
31
|
+
* @param {object} targetFlat - Flattened target locale
|
|
32
|
+
* @param {string} fallbackPrefix - The prefix marking untranslated values (default: "[EN] ")
|
|
33
|
+
* @param {string[]} forceKeys - Dot-notation keys to force re-translate (default: [])
|
|
34
|
+
* @param {string[]} changedKeys - Keys detected as changed via content hashing (default: [])
|
|
35
|
+
* @param {((key: string, sourceValue: string) => boolean)|null} isConfirmedEcho -
|
|
36
|
+
* Optional callback consulted ONLY for keys that would be queued for the
|
|
37
|
+
* equals-source ("untranslated") reason. Return true when the echo is
|
|
38
|
+
* CONFIRMED — i.e. the pipeline itself previously produced this exact
|
|
39
|
+
* source-equal value (gate-approved, TM-recorded) — and the key is NOT
|
|
40
|
+
* requeued for that reason. Missing/changed/forced reasons still queue.
|
|
41
|
+
* Invoked lazily so callers pay a TM lookup only for actual echo candidates.
|
|
42
|
+
* @param {((key: string, sourceValue: string) => boolean)|null} isNoTranslate -
|
|
43
|
+
* Optional predicate identifying keys whose correct value in EVERY locale is
|
|
44
|
+
* the source value, verbatim (config `noTranslate` patterns, auto-detected
|
|
45
|
+
* URLs — see lib/no-translate.js). Matching string keys are removed from every
|
|
46
|
+
* translate reason and routed to the `noTranslate` bucket instead, so they are
|
|
47
|
+
* never sent to a backend, never quality-gated, and never billed.
|
|
48
|
+
* @returns {import('./types.js').DiffResult} Diff result with missing, needsTranslation, untranslated, changed, forced, noTranslate, extra, toProcess
|
|
49
|
+
*/
|
|
50
|
+
function diffLocale(sourceFlat, targetFlat, fallbackPrefix = '[EN] ', forceKeys = [], changedKeys = [], isConfirmedEcho = null, isNoTranslate = null) {
|
|
51
|
+
const sourceKeys = new Set(Object.keys(sourceFlat));
|
|
52
|
+
const targetKeys = new Set(Object.keys(targetFlat));
|
|
53
|
+
|
|
54
|
+
// Keys declared exempt from translation. Restricted to string-valued source
|
|
55
|
+
// keys: non-strings (numbers, booleans, arrays) already pass through verbatim
|
|
56
|
+
// on every path, so marking them would be a no-op that only muddies counts.
|
|
57
|
+
const exempt = isNoTranslate
|
|
58
|
+
? new Set([...sourceKeys].filter(k =>
|
|
59
|
+
typeof sourceFlat[k] === 'string' && isNoTranslate(k, sourceFlat[k])))
|
|
60
|
+
: new Set();
|
|
61
|
+
|
|
62
|
+
// Keys in source but not in target
|
|
63
|
+
const missing = [...sourceKeys].filter(k => !targetKeys.has(k) && !exempt.has(k));
|
|
64
|
+
|
|
65
|
+
// Keys in target that are still [EN]-prefixed fallbacks
|
|
66
|
+
const needsTranslation = [...targetKeys].filter(k =>
|
|
67
|
+
!exempt.has(k) && typeof targetFlat[k] === 'string' && targetFlat[k].startsWith(fallbackPrefix)
|
|
68
|
+
);
|
|
69
|
+
|
|
70
|
+
// Keys where target value is identical to source value.
|
|
71
|
+
// This catches locales pre-populated with English defaults by tools
|
|
72
|
+
// like `docusaurus write-translations`. Without this, Thai/Filipino/etc.
|
|
73
|
+
// would appear "fully synced" with all-English content.
|
|
74
|
+
//
|
|
75
|
+
// Only flag string values that aren't already caught by needsTranslation,
|
|
76
|
+
// and skip very short values (1-2 chars) that are likely intentionally
|
|
77
|
+
// identical across locales (e.g. punctuation, symbols, numbers).
|
|
78
|
+
const needsTranslationSet = new Set(needsTranslation);
|
|
79
|
+
const untranslated = [...targetKeys].filter(k => {
|
|
80
|
+
if (needsTranslationSet.has(k)) return false; // already flagged
|
|
81
|
+
if (exempt.has(k)) return false; // source-equal IS the correct state here
|
|
82
|
+
if (!sourceKeys.has(k)) return false; // extra key, not our concern
|
|
83
|
+
const sv = sourceFlat[k];
|
|
84
|
+
const tv = targetFlat[k];
|
|
85
|
+
// Only compare string values
|
|
86
|
+
if (typeof sv !== 'string' || typeof tv !== 'string') return false;
|
|
87
|
+
// Skip very short values — punctuation, numbers, symbols are
|
|
88
|
+
// often intentionally identical across locales.
|
|
89
|
+
if (sv.length <= 2) return false;
|
|
90
|
+
if (sv !== tv) return false;
|
|
91
|
+
// Confirmed echo: the pipeline previously produced this exact
|
|
92
|
+
// source-equal value (it passed the gate and sits in the TM), so
|
|
93
|
+
// requeuing it every sync would just re-serve the same TM hit and
|
|
94
|
+
// rewrite the same file forever. Brand-new echoes (unconfirmed)
|
|
95
|
+
// still queue once — after the API returns the same text and the TM
|
|
96
|
+
// stores it, subsequent syncs skip.
|
|
97
|
+
if (isConfirmedEcho && isConfirmedEcho(k, sv)) return false;
|
|
98
|
+
return true;
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
// Keys whose English source content changed since last sync (auto-detected).
|
|
102
|
+
// Only include keys that exist in the source (defensive filter).
|
|
103
|
+
const changed = changedKeys.filter(k => sourceKeys.has(k) && !exempt.has(k));
|
|
104
|
+
|
|
105
|
+
// Keys explicitly forced for re-translation (only if they exist in source).
|
|
106
|
+
// Silently ignore any forced keys that don't exist in the source.
|
|
107
|
+
//
|
|
108
|
+
// --force-keys does NOT override no-translate: forcing re-translation of a
|
|
109
|
+
// key whose only correct output is the source value would just re-run the
|
|
110
|
+
// failure this bucket exists to prevent. The key still gets re-copied below
|
|
111
|
+
// if the target has drifted.
|
|
112
|
+
const forced = forceKeys.filter(k => sourceKeys.has(k) && !exempt.has(k));
|
|
113
|
+
|
|
114
|
+
// Keys in target but not in source (stale/orphaned)
|
|
115
|
+
const extra = [...targetKeys].filter(k => !sourceKeys.has(k));
|
|
116
|
+
|
|
117
|
+
// Exempt keys whose target is not byte-identical to the source. That covers
|
|
118
|
+
// three states with one rule: never written, [EN]-prefixed from an older
|
|
119
|
+
// run, or CORRUPTED — the gate-dodging edits ("…/view/1954#fr", a prepended
|
|
120
|
+
// U+200E) that shipped to production. All three are repaired by the same
|
|
121
|
+
// verbatim copy, so the drift heals itself on the next sync and the result
|
|
122
|
+
// is idempotent: once equal, the key stops appearing here.
|
|
123
|
+
const noTranslate = [...exempt].filter(k => targetFlat[k] !== sourceFlat[k]);
|
|
124
|
+
|
|
125
|
+
// Combined set of keys that need work (deduplicated). Exempt keys are
|
|
126
|
+
// deliberately absent: nothing here is sent to a backend or billed.
|
|
127
|
+
const toProcess = [...new Set([...missing, ...needsTranslation, ...untranslated, ...changed, ...forced])];
|
|
128
|
+
|
|
129
|
+
return { missing, needsTranslation, untranslated, changed, forced, noTranslate, extra, toProcess };
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Generate a human-readable label for the diff result.
|
|
134
|
+
*
|
|
135
|
+
* @param {import('./types.js').DiffResult} diff - Diff result from diffLocale
|
|
136
|
+
* @returns {string} Human-readable summary (e.g., '3 missing + 1 [EN] fallback(s)')
|
|
137
|
+
*/
|
|
138
|
+
function diffLabel(diff) {
|
|
139
|
+
const { missing, needsTranslation, untranslated, changed, forced, noTranslate } = diff;
|
|
140
|
+
const parts = [];
|
|
141
|
+
if (missing.length > 0) parts.push(`${missing.length} missing`);
|
|
142
|
+
if (needsTranslation.length > 0) parts.push(`${needsTranslation.length} [EN] fallback(s)`);
|
|
143
|
+
if (untranslated && untranslated.length > 0) parts.push(`${untranslated.length} untranslated`);
|
|
144
|
+
if (changed && changed.length > 0) parts.push(`${changed.length} changed`);
|
|
145
|
+
// Forced keys were invisible here, so a `--force-keys a,b` run over a
|
|
146
|
+
// locale with requeued echoes printed a total ("Translating 77 key(s)")
|
|
147
|
+
// that nothing itemized — a user could not tell their 2 forced keys from
|
|
148
|
+
// the 75 unstamped echoes riding along.
|
|
149
|
+
if (forced && forced.length > 0) parts.push(`${forced.length} forced`);
|
|
150
|
+
if (noTranslate && noTranslate.length > 0) parts.push(`${noTranslate.length} to copy verbatim`);
|
|
151
|
+
if (parts.length > 0) return parts.join(' + ');
|
|
152
|
+
return 'fully synced';
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export { diffLocale, diffLabel };
|