champollion 0.3.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -26
- package/bin/cli.js +53 -5
- package/index.js +63 -2
- package/lib/api-key.js +17 -4
- package/lib/autofix.js +83 -36
- package/lib/bridge/method_bridge.py +15 -3
- package/lib/cards/reader.js +34 -0
- package/lib/cards/remote.js +15 -0
- package/lib/cards/search-names.js +178 -0
- package/lib/command-help.js +286 -85
- package/lib/commands/audit.js +10 -3
- package/lib/commands/card.js +583 -226
- package/lib/commands/doctor.js +54 -18
- package/lib/commands/help.js +37 -32
- package/lib/commands/init.js +1689 -87
- package/lib/commands/integrity.js +127 -40
- package/lib/commands/leaderboard.js +187 -67
- package/lib/commands/models.js +9 -2
- package/lib/commands/provenance.js +7 -2
- package/lib/commands/recommend.js +43 -14
- package/lib/commands/register-corpus.js +632 -125
- package/lib/commands/seal-corpus.js +1 -1
- package/lib/commands/status.js +564 -27
- package/lib/commands/submit.js +17 -12
- package/lib/commands/sync.js +31 -7
- package/lib/commands/tm.js +15 -9
- package/lib/commands/verify.js +27 -3
- package/lib/commands/wrap.js +63 -5
- package/lib/commands/xliff.js +135 -64
- package/lib/commercial-eligibility.js +1 -1
- package/lib/config.js +196 -14
- package/lib/content-estimate.js +96 -0
- package/lib/content-refusals.js +270 -0
- package/lib/content-review.js +372 -0
- package/lib/content-sync.js +1127 -344
- package/lib/content.js +94 -7
- package/lib/corpus-registration.mjs +194 -35
- package/lib/cost-label.js +29 -0
- package/lib/cost-report.js +726 -78
- package/lib/diff.js +38 -4
- package/lib/docusaurus-sync.js +965 -253
- package/lib/edit-distance.js +31 -0
- package/lib/fallback.js +964 -0
- package/lib/file-scope.js +106 -0
- package/lib/flatten.js +80 -3
- package/lib/flutter-locales.js +124 -0
- package/lib/format.js +266 -12
- package/lib/hash.js +146 -21
- package/lib/icu-structure.js +929 -0
- package/lib/integrity.js +223 -75
- package/lib/language-pair.js +157 -0
- package/lib/lint.js +78 -16
- package/lib/local-only-marks.js +106 -0
- package/lib/locale-layout.js +1103 -0
- package/lib/locale-state.js +571 -0
- package/lib/methods/anthropic.js +5 -0
- package/lib/methods/apertium.js +6 -3
- package/lib/methods/api.js +138 -25
- package/lib/methods/base.js +17 -0
- package/lib/methods/coaching-data.js +153 -0
- package/lib/methods/content-separator.js +43 -0
- package/lib/methods/deepl.js +1 -1
- package/lib/methods/direct-llm.js +252 -103
- package/lib/methods/external.js +146 -63
- package/lib/methods/gemini.js +1 -0
- package/lib/methods/google-translate.js +1 -0
- package/lib/methods/http-utils.js +41 -0
- package/lib/methods/libretranslate.js +7 -2
- package/lib/methods/llm-coached.js +68 -128
- package/lib/methods/llm.js +80 -31
- package/lib/methods/local.js +93 -10
- package/lib/methods/microsoft-translator.js +1 -2
- package/lib/methods/openai.js +4 -2
- package/lib/methods/openrouter-client.js +20 -19
- package/lib/methods/openrouter-pricing.js +150 -13
- package/lib/methods/prompt-methods.js +20 -0
- package/lib/methods/provider-pricing.js +42 -1
- package/lib/methods/request-capture.js +104 -0
- package/lib/methods/tilde.js +1 -1
- package/lib/methods/translated.js +1 -2
- package/lib/missing-key.js +93 -0
- package/lib/models.js +11 -0
- package/lib/name-rules.js +32 -0
- package/lib/named-keys.js +172 -0
- package/lib/no-translate.js +4 -3
- package/lib/output.js +160 -19
- package/lib/pairs.js +586 -30
- package/lib/placeholders.js +394 -0
- package/lib/plugins.js +8 -0
- package/lib/plural-gap-redo.js +109 -0
- package/lib/plurals.js +323 -0
- package/lib/po.js +1187 -0
- package/lib/public-catalogue.js +74 -0
- package/lib/recommend.js +527 -32
- package/lib/redo.js +95 -0
- package/lib/refusal-category.js +44 -0
- package/lib/registers.js +255 -11
- package/lib/repair-script.js +20 -13
- package/lib/scripts.js +6 -1
- package/lib/seal.mjs +4 -3
- package/lib/sealed-qualifier.mjs +1 -1
- package/lib/segment.js +2 -1
- package/lib/seo.js +19 -9
- package/lib/serve.js +43 -6
- package/lib/shared-output-seed.js +164 -0
- package/lib/source-contexts.js +39 -0
- package/lib/submit.mjs +57 -5
- package/lib/sync.js +2923 -474
- package/lib/terminology.js +13 -4
- package/lib/tm-evict.js +179 -0
- package/lib/tm-seed.js +5 -2
- package/lib/tm.js +818 -36
- package/lib/translate-pair.js +639 -34
- package/lib/translate.js +78 -5
- package/lib/types.js +22 -3
- package/lib/validate.js +880 -17
- package/lib/verify.js +1296 -104
- package/lib/watch.js +32 -13
- package/lib/xliff.js +44 -3
- package/package.json +1 -1
- package/shared/CORPORA-CARDS.md +2 -0
- package/shared/cards-fallback.json +1 -1
- package/shared/curated-orthography-conventions.json +26 -8
- package/shared/gettext-plural-forms.json +45 -0
- package/shared/method-registry.json +2 -0
- package/shared/metric-registry.json +96 -18
- package/shared/schemas/champollion-plugin.schema.json +4 -0
- package/shared/schemas/corpora-card.schema.json +8 -2
- package/shared/schemas/method-index-record.schema.json +67 -0
- package/shared/schemas/method-registry.schema.json +4 -0
- package/shared/schemas/metric-registry.schema.json +55 -1
- package/shared/docent/corpus.json +0 -11739
package/lib/commands/submit.js
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* dataset · resource · method · human-service · external-result · card-correction
|
|
14
14
|
*
|
|
15
15
|
* Interactive wizard when stdin is a TTY; fully scriptable via flags otherwise
|
|
16
|
-
* (--type + --values/--field + --attest, or --yes). See `champollion submit --help`.
|
|
16
|
+
* (--type + --values/--field + --attest, or --yes). See `champollion network submit --help`.
|
|
17
17
|
*
|
|
18
18
|
* Exit codes: 0 = URL produced (or cancelled); 1 = invalid/incomplete request.
|
|
19
19
|
*/
|
|
@@ -29,6 +29,7 @@ import {
|
|
|
29
29
|
resolveType,
|
|
30
30
|
repoBaseFromRepository,
|
|
31
31
|
validateSubmission,
|
|
32
|
+
normalizeSubmissionValues,
|
|
32
33
|
buildIssueUrl,
|
|
33
34
|
buildSubmissionRecord,
|
|
34
35
|
} from '../submit.mjs';
|
|
@@ -181,7 +182,7 @@ async function run(args, cwd) {
|
|
|
181
182
|
return 0;
|
|
182
183
|
}
|
|
183
184
|
console.log('');
|
|
184
|
-
console.log(' Submission types (champollion submit --type <key>):');
|
|
185
|
+
console.log(' Submission types (champollion network submit --type <key>):');
|
|
185
186
|
console.log('');
|
|
186
187
|
SUBMISSION_TYPES.forEach((t, i) => {
|
|
187
188
|
console.log(` ${i + 1}. ${t.key} — ${t.label}`);
|
|
@@ -209,7 +210,7 @@ async function run(args, cwd) {
|
|
|
209
210
|
} else {
|
|
210
211
|
type = resolveType(args.type);
|
|
211
212
|
if (!type) {
|
|
212
|
-
output.error('Choose a submission type with --type (run `champollion submit --list`).');
|
|
213
|
+
output.error('Choose a submission type with --type (run `champollion network submit --list`).');
|
|
213
214
|
return 1;
|
|
214
215
|
}
|
|
215
216
|
values = gatherFromFlags(type, args);
|
|
@@ -220,12 +221,16 @@ async function run(args, cwd) {
|
|
|
220
221
|
if (!v.ok) {
|
|
221
222
|
console.error('[ERR] Cannot build this submission:');
|
|
222
223
|
for (const e of v.errors) console.error(` • ${e}`);
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
224
|
+
if (v.errors.some((e) => e.startsWith('you must confirm'))) {
|
|
225
|
+
console.error('');
|
|
226
|
+
console.error(' The compliance attestation is required. Re-run and confirm it');
|
|
227
|
+
console.error(' (interactively, or pass --attest with the other fields).');
|
|
228
|
+
}
|
|
226
229
|
return 1;
|
|
227
230
|
}
|
|
228
231
|
|
|
232
|
+
// Language pairs go out one way: source>target, one per line (eng-crk in, eng>crk out).
|
|
233
|
+
values = normalizeSubmissionValues(type, values);
|
|
229
234
|
const issueUrl = buildIssueUrl({ type, values, repoBase });
|
|
230
235
|
const generatedAt = new Date().toISOString();
|
|
231
236
|
const record = buildSubmissionRecord({ type, values, repoBase, generatedAt });
|
|
@@ -279,10 +284,10 @@ async function run(args, cwd) {
|
|
|
279
284
|
|
|
280
285
|
function showHelp() {
|
|
281
286
|
console.log(`
|
|
282
|
-
champollion submit — Propose an entry for the Champollion index (review-gated)
|
|
287
|
+
champollion network submit — Propose an entry for the Champollion index (review-gated)
|
|
283
288
|
|
|
284
289
|
USAGE
|
|
285
|
-
champollion submit [options]
|
|
290
|
+
champollion network submit [options]
|
|
286
291
|
|
|
287
292
|
DESCRIPTION
|
|
288
293
|
Gather the fields for a submission and print a PRE-FILLED GitHub issue URL
|
|
@@ -316,15 +321,15 @@ function showHelp() {
|
|
|
316
321
|
"${ATTESTATION_TEXT}"
|
|
317
322
|
|
|
318
323
|
EXAMPLES
|
|
319
|
-
champollion submit # interactive wizard
|
|
320
|
-
champollion submit --list # see the types
|
|
324
|
+
champollion network submit # interactive wizard
|
|
325
|
+
champollion network submit --list # see the types
|
|
321
326
|
# A dataset submission, fully from flags:
|
|
322
|
-
champollion submit --yes --type dataset --attest \\
|
|
327
|
+
champollion network submit --yes --type dataset --attest \\
|
|
323
328
|
--field dataset-name="GlobalVoices eng-amh" \\
|
|
324
329
|
--field pairs=eng-amh --field license=CC-BY-4.0 \\
|
|
325
330
|
--field source-url=https://globalvoices.org
|
|
326
331
|
# An external result, with a saved local copy:
|
|
327
|
-
champollion submit --yes --type external-result --attest --out ./submission.json \\
|
|
332
|
+
champollion network submit --yes --type external-result --attest --out ./submission.json \\
|
|
328
333
|
--values '{"system-name":"NLLB-200","pairs":"eng-crk","dataset":"FLORES-200","metric":"chrF++","score":"28.4","citation":"https://arxiv.org/abs/2207.04672"}'
|
|
329
334
|
`);
|
|
330
335
|
}
|
package/lib/commands/sync.js
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import { runSync } from '../sync.js';
|
|
10
|
+
import { shutdownMethods } from '../translate.js';
|
|
10
11
|
import { output } from '../output.js';
|
|
11
12
|
|
|
12
13
|
/**
|
|
@@ -20,8 +21,9 @@ import { output } from '../output.js';
|
|
|
20
21
|
* totalProcessed > 0`) returned 0 here, so a `sync` wired into a
|
|
21
22
|
* pre-deploy hook went green while silently shipping English
|
|
22
23
|
* fallbacks. Any failure with zero progress must be loud.
|
|
23
|
-
* 2 — partial: some keys translated but others failed
|
|
24
|
-
*
|
|
24
|
+
* 2 — partial: some keys translated but others failed (or were held
|
|
25
|
+
* back: refused before by the same method, so not re-sent), OR
|
|
26
|
+
* verification found errors. The run did real work but isn't clean.
|
|
25
27
|
* ALSO used for a --max-cost abort: the pre-run estimate exceeded the
|
|
26
28
|
* cap (or was unknowable — unknown ≠ free), so the sync deliberately
|
|
27
29
|
* stopped BEFORE any API call. Not catastrophic (nothing broke,
|
|
@@ -29,7 +31,8 @@ import { output } from '../output.js';
|
|
|
29
31
|
*
|
|
30
32
|
* Pure and total (never throws) so it can be unit-tested directly.
|
|
31
33
|
*
|
|
32
|
-
* @param {{ totalProcessed?: number, totalFailed?: number,
|
|
34
|
+
* @param {{ totalProcessed?: number, totalFailed?: number, totalHeld?: number, contentTranslated?: number, contentFailed?: number,
|
|
35
|
+
* contentHeldBack?: number, contentRefused?: number, verifyErrors?: number, maxCostAborted?: boolean } | null | undefined} result
|
|
33
36
|
* @returns {number} exit code
|
|
34
37
|
*/
|
|
35
38
|
function computeExitCode(result) {
|
|
@@ -38,16 +41,34 @@ function computeExitCode(result) {
|
|
|
38
41
|
// --max-cost abort: deliberate pre-run stop, exit 2 by documented contract.
|
|
39
42
|
if (result.maxCostAborted) return 2;
|
|
40
43
|
|
|
41
|
-
|
|
42
|
-
|
|
44
|
+
// A key named for a redo that matches nothing (a typo, a msgid that only
|
|
45
|
+
// exists with a context): the repair asked for did not happen — exit 1,
|
|
46
|
+
// never the 0 of a run that "did" nothing.
|
|
47
|
+
if (Array.isArray(result.unmatchedKeys) && result.unmatchedKeys.length > 0) return 1;
|
|
48
|
+
|
|
49
|
+
// Content files count alongside keys: a run where every content file
|
|
50
|
+
// failed is a failure; one where some failed is partial.
|
|
51
|
+
const processed = (result.totalProcessed || 0) + (result.contentTranslated || 0);
|
|
52
|
+
const failed = (result.totalFailed || 0) + (result.contentFailed || 0);
|
|
43
53
|
const verifyErrors = result.verifyErrors || 0;
|
|
54
|
+
// Keys held back (refused before by this method — lib/locale-state.js):
|
|
55
|
+
// not translated, so not a clean pass, but nothing failed anew. The same
|
|
56
|
+
// for Markdown blocks and front-matter fields (lib/content-refusals.js):
|
|
57
|
+
// held back, or refused by the quality gate this run and left as the
|
|
58
|
+
// '[EN] ' last resort.
|
|
59
|
+
// A plural message left without a form the language uses for ordinary
|
|
60
|
+
// counts (filled with the "other" form, marked in a gettext catalog) is
|
|
61
|
+
// incomplete in the same way.
|
|
62
|
+
const held = (result.totalHeld || 0) + (result.contentHeldBack || 0) + (result.contentRefused || 0)
|
|
63
|
+
+ (result.totalPluralGaps || 0);
|
|
44
64
|
|
|
45
65
|
// Catastrophic: failures with no successful work. A fully-failed sync
|
|
46
66
|
// (bad key, every locale errored) must never exit 0.
|
|
47
67
|
if (failed > 0 && processed === 0) return 1;
|
|
48
68
|
|
|
49
|
-
// Partial: real work done but something failed
|
|
50
|
-
|
|
69
|
+
// Partial: real work done but something failed or was held back, or
|
|
70
|
+
// files didn't verify.
|
|
71
|
+
if (failed > 0 || held > 0 || verifyErrors > 0) return 2;
|
|
51
72
|
|
|
52
73
|
return 0;
|
|
53
74
|
}
|
|
@@ -81,6 +102,9 @@ async function run(args, cwd) {
|
|
|
81
102
|
return 1;
|
|
82
103
|
}
|
|
83
104
|
throw err;
|
|
105
|
+
} finally {
|
|
106
|
+
// Stop the Python bridges external-method pairs started for this run.
|
|
107
|
+
await shutdownMethods();
|
|
84
108
|
}
|
|
85
109
|
|
|
86
110
|
return computeExitCode(result);
|
package/lib/commands/tm.js
CHANGED
|
@@ -27,11 +27,11 @@
|
|
|
27
27
|
import fs from 'node:fs';
|
|
28
28
|
import path from 'node:path';
|
|
29
29
|
import readline from 'node:readline';
|
|
30
|
-
import { loadTM, saveTM, tmSize, pruneTM, TM_DIR, TM_FILENAME } from '../tm.js';
|
|
30
|
+
import { loadTM, saveTM, tmSize, pruneTM, describeMethodKey, TM_DIR, TM_FILENAME } from '../tm.js';
|
|
31
31
|
import { resolveConfig } from '../config.js';
|
|
32
32
|
import { resolveRuntime } from '../sync.js';
|
|
33
33
|
import { seedTMFromExisting } from '../tm-seed.js';
|
|
34
|
-
import { output } from '../output.js';
|
|
34
|
+
import { output, localTimestamp } from '../output.js';
|
|
35
35
|
|
|
36
36
|
/**
|
|
37
37
|
* @param {import('../types.js').CLIArgs} args - Parsed CLI arguments
|
|
@@ -220,6 +220,8 @@ function runStats(args, cwd) {
|
|
|
220
220
|
|
|
221
221
|
// Timestamps from metadata
|
|
222
222
|
const created = tm._meta?.created || 'unknown';
|
|
223
|
+
// JSON keeps the stored (UTC) date for compatibility, plus the full ISO
|
|
224
|
+
// timestamps; the human report prints local time with the zone named.
|
|
223
225
|
const createdDate = created !== 'unknown' ? created.split('T')[0] : 'unknown';
|
|
224
226
|
|
|
225
227
|
// Find newest entry timestamp
|
|
@@ -277,6 +279,8 @@ function runStats(args, cwd) {
|
|
|
277
279
|
sizeBytes: stat.size,
|
|
278
280
|
created: createdDate,
|
|
279
281
|
lastEntry: newestDate,
|
|
282
|
+
createdAt: created !== 'unknown' ? created : null,
|
|
283
|
+
lastEntryAt: newest || null,
|
|
280
284
|
byLocale,
|
|
281
285
|
unknownLocale: unknownCount,
|
|
282
286
|
}, null, 2));
|
|
@@ -286,18 +290,20 @@ function runStats(args, cwd) {
|
|
|
286
290
|
output.raw('\n Translation Memory — .champollion/tm.json\n');
|
|
287
291
|
output.raw(` Entries: ${entryCount.toLocaleString()}`);
|
|
288
292
|
output.raw(` File size: ${sizeStr}`);
|
|
289
|
-
output.raw(` Created: ${
|
|
290
|
-
output.raw(` Last entry: ${
|
|
293
|
+
output.raw(` Created: ${created !== 'unknown' ? localTimestamp(created) : 'unknown'}`);
|
|
294
|
+
output.raw(` Last entry: ${newest ? localTimestamp(newest) : 'unknown'}`);
|
|
291
295
|
|
|
292
296
|
if (sortedLocales.length > 0 || unknownCount > 0) {
|
|
293
297
|
output.raw('\n By locale:');
|
|
294
298
|
|
|
299
|
+
// One line per locale, then what made its entries — method, model and
|
|
300
|
+
// register in words (the raw cache key "local|stub-1|formal-vous|" was
|
|
301
|
+
// printed here before; lib/tm.js describeMethodKey).
|
|
295
302
|
for (const [locale, stats] of sortedLocales) {
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
.
|
|
299
|
-
|
|
300
|
-
output.raw(` ${locale.padEnd(6)} ${String(stats.total).padStart(5)} entries${methodStr}`);
|
|
303
|
+
output.raw(` ${locale.padEnd(6)} ${String(stats.total).padStart(5)} entries`);
|
|
304
|
+
for (const [m, count] of Object.entries(stats.methods).sort((a, b) => b[1] - a[1])) {
|
|
305
|
+
output.raw(` ${''.padEnd(6)} ${String(count).padStart(5)} ${describeMethodKey(m)}`);
|
|
306
|
+
}
|
|
301
307
|
}
|
|
302
308
|
|
|
303
309
|
if (unknownCount > 0) {
|
package/lib/commands/verify.js
CHANGED
|
@@ -6,14 +6,21 @@
|
|
|
6
6
|
* success and keys being wrong in fact.
|
|
7
7
|
*
|
|
8
8
|
* Exit code 1 if errors found (CI-gate compatible).
|
|
9
|
-
* --
|
|
9
|
+
* --strict exits 1 on warnings too (a source echo, a plural form the
|
|
10
|
+
* translation did not supply, two locales with identical text, a
|
|
11
|
+
* translation made from an older source text) — for a CI that must not pass
|
|
12
|
+
* them. --warn-only exits 0 regardless.
|
|
10
13
|
*
|
|
11
14
|
* This runs the same checks that sync's post-sync verification uses,
|
|
12
15
|
* but can be invoked independently for CI pipelines or manual auditing.
|
|
16
|
+
*
|
|
17
|
+
* --pair en:fr (comma-separated) verifies only those pairs' locales — the
|
|
18
|
+
* same scoping `sync --pair` gives its post-sync verification. An unknown
|
|
19
|
+
* pair fails loud, exactly as it does for sync.
|
|
13
20
|
*/
|
|
14
21
|
|
|
15
22
|
import { resolveConfig } from '../config.js';
|
|
16
|
-
import { verifyLocales } from '../verify.js';
|
|
23
|
+
import { verifyLocales, localesForPairFlag } from '../verify.js';
|
|
17
24
|
import { output } from '../output.js';
|
|
18
25
|
|
|
19
26
|
/**
|
|
@@ -27,12 +34,29 @@ async function run(args, cwd) {
|
|
|
27
34
|
if (args.json) output.setMode('json');
|
|
28
35
|
else if (args.quiet) output.setMode('quiet');
|
|
29
36
|
|
|
37
|
+
if (args.strict && args['warn-only']) {
|
|
38
|
+
output.error('--strict (fail on warnings) and --warn-only (never fail) contradict each other — pick one.');
|
|
39
|
+
return 1;
|
|
40
|
+
}
|
|
30
41
|
const config = resolveConfig(args, cwd);
|
|
31
|
-
|
|
42
|
+
let locales = null;
|
|
43
|
+
if (args.pair) {
|
|
44
|
+
try {
|
|
45
|
+
locales = localesForPairFlag(config, args.pair, { cwd });
|
|
46
|
+
} catch (err) {
|
|
47
|
+
output.error(err.message);
|
|
48
|
+
return 1;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
// --strict: the summary line itself says a warning fails the check.
|
|
52
|
+
const { errors, warnings } = await verifyLocales(config, cwd, { locales, strict: !!args.strict && !args['warn-only'] });
|
|
32
53
|
|
|
33
54
|
if (errors > 0 && !args['warn-only']) {
|
|
34
55
|
return 1;
|
|
35
56
|
}
|
|
57
|
+
if (args.strict && warnings > 0) {
|
|
58
|
+
return 1;
|
|
59
|
+
}
|
|
36
60
|
return 0;
|
|
37
61
|
}
|
|
38
62
|
|
package/lib/commands/wrap.js
CHANGED
|
@@ -13,11 +13,12 @@
|
|
|
13
13
|
|
|
14
14
|
import fs from 'node:fs';
|
|
15
15
|
import path from 'node:path';
|
|
16
|
-
import { resolveConfig } from '../config.js';
|
|
16
|
+
import { resolveConfig, autoDetectLanguages } from '../config.js';
|
|
17
|
+
import { discoverLocaleLayout } from '../locale-layout.js';
|
|
17
18
|
import { detectFramework, walkDir } from '../lint.js';
|
|
18
19
|
import {
|
|
19
20
|
checkGitClean, createBackup, restoreFromBackup,
|
|
20
|
-
processFile, generateDiff, addKeysToLocales,
|
|
21
|
+
processFile, generateDiff, addKeysToLocales, wrapUnsupportedReason,
|
|
21
22
|
} from '../autofix.js';
|
|
22
23
|
import { output } from '../output.js';
|
|
23
24
|
|
|
@@ -54,6 +55,15 @@ async function run(args, cwd) {
|
|
|
54
55
|
const framework = detectFramework(cwd);
|
|
55
56
|
const minLength = parseInt(args['min-length'] || 2, 10);
|
|
56
57
|
|
|
58
|
+
// Decide WHERE extracted keys go before any source file is rewritten — a
|
|
59
|
+
// wrap that rewrote components to t('…') and then could not store the
|
|
60
|
+
// keys would leave the app showing raw key names.
|
|
61
|
+
const destination = resolveWrapDestination(config, cwd);
|
|
62
|
+
if (destination.error) {
|
|
63
|
+
output.error(destination.error);
|
|
64
|
+
return 1;
|
|
65
|
+
}
|
|
66
|
+
|
|
57
67
|
output.raw(`\n champollion wrap${isDry ? ' (dry run)' : ''}`);
|
|
58
68
|
output.raw(` Framework: ${framework.name}`);
|
|
59
69
|
output.raw('');
|
|
@@ -117,9 +127,11 @@ async function run(args, cwd) {
|
|
|
117
127
|
|
|
118
128
|
// Add extracted keys to locale files (only if not dry-run)
|
|
119
129
|
if (!isDry && allFixes.length > 0) {
|
|
120
|
-
const
|
|
121
|
-
|
|
122
|
-
|
|
130
|
+
const written = addKeysToLocales(allFixes, destination.source, destination.targets);
|
|
131
|
+
output.info(`Added ${allFixes.length} key(s) to ${destination.source.rel}`
|
|
132
|
+
+ (written.targets > 0 ? ` and ${written.targets} target file(s) (as [EN] placeholders for the next sync)` : ''));
|
|
133
|
+
} else if (isDry && allFixes.length > 0) {
|
|
134
|
+
output.info(`Would add ${allFixes.length} key(s) to ${destination.source.rel}`);
|
|
123
135
|
}
|
|
124
136
|
|
|
125
137
|
output.raw('');
|
|
@@ -135,4 +147,50 @@ async function run(args, cwd) {
|
|
|
135
147
|
return 0;
|
|
136
148
|
}
|
|
137
149
|
|
|
150
|
+
/**
|
|
151
|
+
* The locale files `wrap` writes extracted keys into: the source file (one
|
|
152
|
+
* per flat project; the `defaultNamespace` file — or the only file — of a
|
|
153
|
+
* folder-per-locale project) and the matching file of every configured
|
|
154
|
+
* target locale.
|
|
155
|
+
*
|
|
156
|
+
* @param {object} config - Resolved config
|
|
157
|
+
* @param {string} cwd
|
|
158
|
+
* @returns {{ source?: object, targets?: object[], error?: string }}
|
|
159
|
+
*/
|
|
160
|
+
function resolveWrapDestination(config, cwd) {
|
|
161
|
+
if (config.format === 'docusaurus') {
|
|
162
|
+
return { error: 'wrap writes t() keys into key-value locale files; Docusaurus translates through <Translate> and `docusaurus write-translations` — wrap does not apply.' };
|
|
163
|
+
}
|
|
164
|
+
const layout = discoverLocaleLayout(config, { cwd });
|
|
165
|
+
let ns = '';
|
|
166
|
+
if (layout.namespaced) {
|
|
167
|
+
const namespaces = layout.sourceFiles.map(f => f.ns);
|
|
168
|
+
if (config.defaultNamespace) {
|
|
169
|
+
if (!namespaces.includes(config.defaultNamespace)) {
|
|
170
|
+
return { error: `"defaultNamespace" is "${config.defaultNamespace}", but the source locale has no such file (namespaces: ${namespaces.join(', ') || 'none'}).` };
|
|
171
|
+
}
|
|
172
|
+
ns = config.defaultNamespace;
|
|
173
|
+
} else if (namespaces.length === 1) {
|
|
174
|
+
ns = namespaces[0];
|
|
175
|
+
} else {
|
|
176
|
+
return { error: `This project has ${namespaces.length} namespace files (${namespaces.join(', ') || 'none'}). `
|
|
177
|
+
+ 'Set "defaultNamespace" in champollion.config.json to choose the file wrap adds keys to.' };
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
const source = layout.sourceFiles.find(f => f.ns === ns) || layout.sourceFiles[0];
|
|
181
|
+
// Flutter ARB and gettext catalogs cannot hold t('dotted.key') keys —
|
|
182
|
+
// refuse before any component is rewritten.
|
|
183
|
+
const refusal = source ? wrapUnsupportedReason(source) : null;
|
|
184
|
+
if (refusal) return { error: refusal };
|
|
185
|
+
if (!source || !fs.existsSync(source.path)) {
|
|
186
|
+
return { error: `Source locale file not found: ${source ? source.path : layout.display} — wrap needs it to store the extracted keys.` };
|
|
187
|
+
}
|
|
188
|
+
// Configured targets, or — like sync — the ones on disk when none are.
|
|
189
|
+
const languages = Object.keys(config.resolvedLanguages || {}).length > 0
|
|
190
|
+
? config.resolvedLanguages
|
|
191
|
+
: autoDetectLanguages(config);
|
|
192
|
+
const targetCodes = Object.keys(languages).filter(c => c !== config.inputLocale);
|
|
193
|
+
return { source, targets: targetCodes.map(code => layout.fileFor(code, ns)) };
|
|
194
|
+
}
|
|
195
|
+
|
|
138
196
|
export { run };
|
package/lib/commands/xliff.js
CHANGED
|
@@ -20,8 +20,11 @@ import path from 'node:path';
|
|
|
20
20
|
import { resolveConfig } from '../config.js';
|
|
21
21
|
import { exportXLIFF, importXLIFF } from '../xliff.js';
|
|
22
22
|
import { compileNoTranslate } from '../no-translate.js';
|
|
23
|
-
import {
|
|
24
|
-
import {
|
|
23
|
+
import { setNestedValue, assignInOrder } from '../flatten.js';
|
|
24
|
+
import { writeLocaleFile, detectYAMLStyle } from '../format.js';
|
|
25
|
+
import {
|
|
26
|
+
discoverLocaleLayout, loadSourceUnits, expectedForTarget, readLocaleFlat, lockKey, splitLockKey,
|
|
27
|
+
} from '../locale-layout.js';
|
|
25
28
|
import { output } from '../output.js';
|
|
26
29
|
|
|
27
30
|
/** Default output directory for exported XLIFF files */
|
|
@@ -75,26 +78,34 @@ function runExport(args, cwd) {
|
|
|
75
78
|
}
|
|
76
79
|
|
|
77
80
|
const config = resolveConfig(args, cwd);
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
const
|
|
85
|
-
|
|
86
|
-
|
|
81
|
+
// The project's locale files from the ONE layout module. A folder-per-
|
|
82
|
+
// locale project exports ALL of a locale's namespace files into one XLIFF
|
|
83
|
+
// document; unit ids are "<ns>::<key>" (the lock-manifest convention), so
|
|
84
|
+
// import can route every unit back to its file. Single-file layouts keep
|
|
85
|
+
// bare keys — their XLIFF is exactly what it was.
|
|
86
|
+
const layout = discoverLocaleLayout(config, { cwd });
|
|
87
|
+
const sourceMissing = layout.namespaced
|
|
88
|
+
? layout.sourceFiles.length === 0
|
|
89
|
+
: !fs.existsSync(layout.sourceFiles[0].path);
|
|
90
|
+
if (sourceMissing) {
|
|
91
|
+
const where = layout.namespaced ? `${layout.display} (no files for ${config.inputLocale})` : layout.sourceFiles[0].path;
|
|
92
|
+
output.error(`Source locale file not found: ${where}`);
|
|
87
93
|
return 1;
|
|
88
94
|
}
|
|
89
|
-
const
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
//
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
95
|
+
const units = loadSourceUnits(layout);
|
|
96
|
+
|
|
97
|
+
// Source = what THIS locale must contain (i18next plurals expand to the
|
|
98
|
+
// target's own CLDR categories); target = what its files hold today
|
|
99
|
+
// (files that do not exist yet export as untranslated).
|
|
100
|
+
const sourceFlat = {};
|
|
101
|
+
const targetFlat = {};
|
|
102
|
+
for (const unit of units) {
|
|
103
|
+
const expected = expectedForTarget(unit, config.inputLocale, locale).flat;
|
|
104
|
+
for (const [k, v] of Object.entries(expected)) sourceFlat[lockKey(layout, unit.ns, k)] = v;
|
|
105
|
+
const file = layout.fileFor(locale, unit.ns);
|
|
106
|
+
if (fs.existsSync(file.path)) {
|
|
107
|
+
for (const [k, v] of Object.entries(readLocaleFlat(file))) targetFlat[lockKey(layout, unit.ns, k)] = v;
|
|
108
|
+
}
|
|
98
109
|
}
|
|
99
110
|
|
|
100
111
|
// Generate XLIFF
|
|
@@ -103,8 +114,12 @@ function runExport(args, cwd) {
|
|
|
103
114
|
targetLocale: locale,
|
|
104
115
|
sourceFlat,
|
|
105
116
|
targetFlat,
|
|
106
|
-
original:
|
|
107
|
-
|
|
117
|
+
original: layout.namespaced
|
|
118
|
+
? (layout.kind === 'dir' ? `${config.inputLocale}/` : layout.display)
|
|
119
|
+
: units[0].file.rel,
|
|
120
|
+
// No-translate patterns match a key WITHIN its file — strip the
|
|
121
|
+
// namespace before asking.
|
|
122
|
+
noTranslate: namespacedMatcher(compileNoTranslate(config), layout),
|
|
108
123
|
});
|
|
109
124
|
|
|
110
125
|
// Determine output path
|
|
@@ -198,42 +213,69 @@ function runImport(args, cwd) {
|
|
|
198
213
|
}
|
|
199
214
|
|
|
200
215
|
const config = resolveConfig(args, cwd);
|
|
201
|
-
const
|
|
202
|
-
? config.format
|
|
203
|
-
: detectFormatFromDir(config.localesDir);
|
|
204
|
-
const ext = getExtension(format);
|
|
205
|
-
const targetPath = path.join(config.localesDir, `${locale}${ext}`);
|
|
216
|
+
const layout = discoverLocaleLayout(config, { cwd });
|
|
206
217
|
|
|
207
218
|
const dryRun = args.dry || false;
|
|
208
219
|
|
|
209
|
-
//
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
220
|
+
// Route every unit to its file. Namespaced layouts need "<ns>::<key>" ids
|
|
221
|
+
// (what `xliff export` writes); an id without one — or naming a namespace
|
|
222
|
+
// the source does not have — fails the whole import before anything is
|
|
223
|
+
// written, rather than guessing a file.
|
|
224
|
+
const byNs = new Map();
|
|
225
|
+
const unroutable = [];
|
|
226
|
+
const knownNs = new Set(layout.sourceFiles.map(f => f.ns));
|
|
227
|
+
for (const [id, value] of Object.entries(translations)) {
|
|
228
|
+
const parts = splitLockKey(layout, id);
|
|
229
|
+
if (!parts || (layout.namespaced && knownNs.size > 0 && !knownNs.has(parts.ns))) {
|
|
230
|
+
unroutable.push(id);
|
|
231
|
+
continue;
|
|
232
|
+
}
|
|
233
|
+
if (!byNs.has(parts.ns)) byNs.set(parts.ns, {});
|
|
234
|
+
byNs.get(parts.ns)[parts.key] = value;
|
|
235
|
+
}
|
|
236
|
+
if (unroutable.length > 0) {
|
|
237
|
+
// A gettext context key prints its U+0004 as "␄", as it was exported.
|
|
238
|
+
const sample = unroutable.slice(0, 5).map(id => id.replace(/\u0004/g, '\u2404')).join(', ');
|
|
239
|
+
output.error(
|
|
240
|
+
`${unroutable.length} XLIFF unit id(s) do not name one of this project's locale files: ${sample}`
|
|
241
|
+
+ `${unroutable.length > 5 ? ', …' : ''}. In a folder-per-locale project ids are "<namespace>::<key>" `
|
|
242
|
+
+ `(namespaces: ${[...knownNs].join(', ') || 'none found'}) — export with \`champollion xliff export\`.`);
|
|
243
|
+
return 1;
|
|
215
244
|
}
|
|
216
245
|
|
|
217
|
-
// Merge: XLIFF translations overwrite existing values
|
|
246
|
+
// Merge: XLIFF translations overwrite existing values, file by file.
|
|
218
247
|
let updated = 0;
|
|
219
248
|
let added = 0;
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
249
|
+
const writes = [];
|
|
250
|
+
for (const [ns, entries] of byNs) {
|
|
251
|
+
const file = layout.fileFor(locale, ns);
|
|
252
|
+
const existingFlat = fs.existsSync(file.path) ? readLocaleFlat(file) : {};
|
|
253
|
+
for (const [key, value] of Object.entries(entries)) {
|
|
254
|
+
if (key in existingFlat) {
|
|
255
|
+
if (existingFlat[key] !== value) {
|
|
256
|
+
updated++;
|
|
257
|
+
}
|
|
258
|
+
} else {
|
|
259
|
+
added++;
|
|
224
260
|
}
|
|
225
|
-
|
|
226
|
-
|
|
261
|
+
// A plural form new to the file goes beside its siblings in CLDR order.
|
|
262
|
+
assignInOrder(existingFlat, key, value);
|
|
227
263
|
}
|
|
228
|
-
existingFlat
|
|
264
|
+
writes.push({ ns, file, existingFlat });
|
|
229
265
|
}
|
|
230
266
|
|
|
231
267
|
const skipped = Object.keys(translations).length - updated - added;
|
|
268
|
+
// One file: report its path as before. Several: the layout's pattern.
|
|
269
|
+
const targetPath = writes.length === 1 ? writes[0].file.path : layout.filesFor(locale)[0]?.path || layout.baseDir;
|
|
270
|
+
const targetLabel = writes.length === 1
|
|
271
|
+
? path.relative(cwd, writes[0].file.path)
|
|
272
|
+
: `${writes.length} files (${writes.map(w => w.file.rel).join(', ')})`;
|
|
232
273
|
|
|
233
274
|
if (dryRun) {
|
|
234
275
|
if (args.json) {
|
|
235
276
|
console.log(JSON.stringify({
|
|
236
277
|
command: 'xliff', action: 'import', dryRun: true, locale, path: targetPath,
|
|
278
|
+
...(writes.length > 1 && { files: writes.map(w => w.file.path) }),
|
|
237
279
|
imported: Object.keys(translations).length, updated, added, unchanged: skipped,
|
|
238
280
|
}, null, 2));
|
|
239
281
|
return 0;
|
|
@@ -242,38 +284,48 @@ function runImport(args, cwd) {
|
|
|
242
284
|
output.raw(` Updated: ${updated} (changed from existing)`);
|
|
243
285
|
output.raw(` Added: ${added} (new keys)`);
|
|
244
286
|
output.raw(` Unchanged: ${skipped}`);
|
|
245
|
-
output.raw(` Target: ${
|
|
287
|
+
output.raw(` Target: ${targetLabel}\n`);
|
|
246
288
|
return 0;
|
|
247
289
|
}
|
|
248
290
|
|
|
249
291
|
// Write back
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
292
|
+
for (const { ns, file, existingFlat } of writes) {
|
|
293
|
+
fs.mkdirSync(path.dirname(file.path), { recursive: true });
|
|
294
|
+
if (file.format === 'json') {
|
|
295
|
+
// Re-nest the flat map into JSON structure using setNestedValue
|
|
296
|
+
const nested = {};
|
|
297
|
+
for (const [key, value] of Object.entries(existingFlat)) {
|
|
298
|
+
setNestedValue(nested, key, value);
|
|
299
|
+
}
|
|
300
|
+
fs.writeFileSync(file.path, JSON.stringify(nested, null, 2) + '\n', 'utf-8');
|
|
301
|
+
} else {
|
|
302
|
+
// TOML/YAML — write the merged flat map back through the exact same
|
|
303
|
+
// writer sync uses (lib/format.js writeLocaleFile), so an imported file
|
|
304
|
+
// keeps the serialization style of a synced project. YAML needs the
|
|
305
|
+
// style probe sync performs on the SOURCE locale file (Hugo plural
|
|
306
|
+
// sub-keys vs standard nesting); fall back to the target file when the
|
|
307
|
+
// source is missing.
|
|
308
|
+
let yamlStyle = null;
|
|
309
|
+
if (file.format === 'yaml') {
|
|
310
|
+
const sourcePath = layout.sourceFiles.find(f => f.ns === ns)?.path;
|
|
311
|
+
const stylePath = sourcePath && fs.existsSync(sourcePath) ? sourcePath
|
|
312
|
+
: (fs.existsSync(file.path) ? file.path : null);
|
|
313
|
+
yamlStyle = stylePath ? detectYAMLStyle(fs.readFileSync(stylePath, 'utf-8')) : null;
|
|
314
|
+
}
|
|
315
|
+
// Document formats (.po, .arb) are rebuilt from the SOURCE file's
|
|
316
|
+
// structure — plural entries, metadata, @@locale — not from the flat
|
|
317
|
+
// map alone (a brand-new target has no structure of its own).
|
|
318
|
+
writeLocaleFile(file.path, existingFlat, file.format, existingFlat, yamlStyle, {
|
|
319
|
+
sourcePath: file.sourcePath || null,
|
|
320
|
+
locale: file.code || null,
|
|
321
|
+
});
|
|
270
322
|
}
|
|
271
|
-
writeLocaleFile(targetPath, existingFlat, format, existingFlat, yamlStyle);
|
|
272
323
|
}
|
|
273
324
|
|
|
274
325
|
if (args.json) {
|
|
275
326
|
console.log(JSON.stringify({
|
|
276
327
|
command: 'xliff', action: 'import', dryRun: false, locale, path: targetPath,
|
|
328
|
+
...(writes.length > 1 && { files: writes.map(w => w.file.path) }),
|
|
277
329
|
imported: Object.keys(translations).length, updated, added, unchanged: skipped,
|
|
278
330
|
}, null, 2));
|
|
279
331
|
return 0;
|
|
@@ -283,11 +335,30 @@ function runImport(args, cwd) {
|
|
|
283
335
|
output.raw(` Updated: ${updated} (changed from existing)`);
|
|
284
336
|
output.raw(` Added: ${added} (new keys)`);
|
|
285
337
|
output.raw(` Unchanged: ${skipped}`);
|
|
286
|
-
output.raw(` Written to: ${
|
|
338
|
+
output.raw(` Written to: ${targetLabel}\n`);
|
|
287
339
|
|
|
288
340
|
return 0;
|
|
289
341
|
}
|
|
290
342
|
|
|
343
|
+
/**
|
|
344
|
+
* Wrap a no-translate matcher so it receives the key WITHIN its file:
|
|
345
|
+
* patterns are written against a file's own keys ("**.url"), never against
|
|
346
|
+
* the "<ns>::" prefix the XLIFF ids carry.
|
|
347
|
+
*
|
|
348
|
+
* @param {import('../no-translate.js').NoTranslateMatcher} matcher
|
|
349
|
+
* @param {{ namespaced: boolean }} layout
|
|
350
|
+
*/
|
|
351
|
+
function namespacedMatcher(matcher, layout) {
|
|
352
|
+
if (!layout.namespaced) return matcher;
|
|
353
|
+
return {
|
|
354
|
+
...matcher,
|
|
355
|
+
matches: (id, value) => {
|
|
356
|
+
const parts = splitLockKey(layout, id);
|
|
357
|
+
return matcher.matches(parts ? parts.key : id, value);
|
|
358
|
+
},
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
|
|
291
362
|
// -----------------------------------------------------------------
|
|
292
363
|
// Usage
|
|
293
364
|
// -----------------------------------------------------------------
|