champollion 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +133 -0
- package/README.md +387 -0
- package/bin/cli.js +278 -0
- package/index.js +135 -0
- package/lib/api-key.js +127 -0
- package/lib/autofix.js +432 -0
- package/lib/bridge/method_bridge.py +430 -0
- package/lib/card-source-resolution.mjs +284 -0
- package/lib/cards/cache.js +169 -0
- package/lib/cards/env.js +82 -0
- package/lib/cards/fetch-card-child.js +38 -0
- package/lib/cards/reader.js +435 -0
- package/lib/cards/refresh.js +111 -0
- package/lib/cards/remote.js +387 -0
- package/lib/cldf-export.mjs +540 -0
- package/lib/cldf-terms.mjs +62 -0
- package/lib/command-help.js +790 -0
- package/lib/commands/audit.js +49 -0
- package/lib/commands/card.js +454 -0
- package/lib/commands/doctor.js +559 -0
- package/lib/commands/fonts.js +489 -0
- package/lib/commands/help.js +91 -0
- package/lib/commands/init.js +1259 -0
- package/lib/commands/integrity.js +148 -0
- package/lib/commands/leaderboard.js +478 -0
- package/lib/commands/lint.js +30 -0
- package/lib/commands/models.js +177 -0
- package/lib/commands/plugin.js +103 -0
- package/lib/commands/provenance.js +45 -0
- package/lib/commands/recommend.js +75 -0
- package/lib/commands/register-corpus.js +678 -0
- package/lib/commands/repair-script.js +42 -0
- package/lib/commands/seal-corpus.js +355 -0
- package/lib/commands/seo.js +72 -0
- package/lib/commands/serve.js +147 -0
- package/lib/commands/status.js +265 -0
- package/lib/commands/submit.js +332 -0
- package/lib/commands/sync.js +89 -0
- package/lib/commands/tm.js +573 -0
- package/lib/commands/verify.js +39 -0
- package/lib/commands/watch.js +20 -0
- package/lib/commands/wrap.js +138 -0
- package/lib/commands/xliff.js +327 -0
- package/lib/commercial-eligibility.js +235 -0
- package/lib/concurrent.js +87 -0
- package/lib/config.js +523 -0
- package/lib/contamination-lane.js +76 -0
- package/lib/content-sync.js +731 -0
- package/lib/content.js +733 -0
- package/lib/corpus-registration.mjs +608 -0
- package/lib/cost-report.js +346 -0
- package/lib/diff.js +155 -0
- package/lib/docusaurus-sync.js +1256 -0
- package/lib/flatten.js +55 -0
- package/lib/format.js +954 -0
- package/lib/hash.js +159 -0
- package/lib/icu.js +473 -0
- package/lib/integrity.js +689 -0
- package/lib/license-gate.mjs +478 -0
- package/lib/license-identify.mjs +229 -0
- package/lib/lint.js +629 -0
- package/lib/method-manifest.js +60 -0
- package/lib/methods/anthropic.js +140 -0
- package/lib/methods/apertium.js +163 -0
- package/lib/methods/api.js +316 -0
- package/lib/methods/base.js +184 -0
- package/lib/methods/content-separator.js +45 -0
- package/lib/methods/deepl.js +426 -0
- package/lib/methods/direct-llm.js +586 -0
- package/lib/methods/external.js +332 -0
- package/lib/methods/fetch-with-retry.js +124 -0
- package/lib/methods/gemini.js +147 -0
- package/lib/methods/google-translate.js +402 -0
- package/lib/methods/http-utils.js +122 -0
- package/lib/methods/libretranslate.js +314 -0
- package/lib/methods/llm-coached.js +670 -0
- package/lib/methods/llm.js +592 -0
- package/lib/methods/local.js +76 -0
- package/lib/methods/microsoft-translator.js +331 -0
- package/lib/methods/openai.js +131 -0
- package/lib/methods/openrouter-client.js +327 -0
- package/lib/methods/openrouter-pricing.js +156 -0
- package/lib/methods/provider-env.js +115 -0
- package/lib/methods/provider-pricing.js +310 -0
- package/lib/methods/tilde.js +150 -0
- package/lib/methods/translated.js +229 -0
- package/lib/methods/translation-error.js +80 -0
- package/lib/models.js +258 -0
- package/lib/no-translate.js +233 -0
- package/lib/output.js +238 -0
- package/lib/pairs.js +547 -0
- package/lib/plugins.js +447 -0
- package/lib/provenance.js +323 -0
- package/lib/recommend.js +648 -0
- package/lib/registers.js +1185 -0
- package/lib/repair-script.js +266 -0
- package/lib/scripts.js +994 -0
- package/lib/seal.mjs +464 -0
- package/lib/sealed-qualifier.mjs +211 -0
- package/lib/security.js +59 -0
- package/lib/segment.js +369 -0
- package/lib/seo.js +275 -0
- package/lib/serve.js +854 -0
- package/lib/string-classify.js +85 -0
- package/lib/submit.mjs +344 -0
- package/lib/sync.js +969 -0
- package/lib/tags/bcp47.js +202 -0
- package/lib/tags/resolve.js +314 -0
- package/lib/terminology.js +111 -0
- package/lib/tm-seed.js +294 -0
- package/lib/tm.js +515 -0
- package/lib/translate-pair.js +197 -0
- package/lib/translate.js +203 -0
- package/lib/types.js +230 -0
- package/lib/validate.js +510 -0
- package/lib/verify.js +451 -0
- package/lib/watch.js +145 -0
- package/lib/xliff.js +184 -0
- package/package.json +93 -0
- package/shared/ATTRIBUTION.md +145 -0
- package/shared/CORPORA-CARDS.md +288 -0
- package/shared/DATA-SOVEREIGNTY.md +500 -0
- package/shared/LANGUAGE-CARD-FIELDS.md +532 -0
- package/shared/card-lint-baseline.json +3189 -0
- package/shared/cards-fallback.json +1 -0
- package/shared/catalogue/card-config.json +6091 -0
- package/shared/catalogue/external-results.json +3888 -0
- package/shared/catalogue/gender-guidance.json +1038 -0
- package/shared/catalogue/method-coverage.json +1751 -0
- package/shared/catalogue/metric-coverage.json +170 -0
- package/shared/catalogue/metric-reliability.json +1 -0
- package/shared/catalogue/register-presets.json +3180 -0
- package/shared/catalogue/vitality-scales.json +55 -0
- package/shared/cldr-index.json +1115 -0
- package/shared/code-bridge.json +253 -0
- package/shared/corpora-cards-v1-reference.md +281 -0
- package/shared/curated-dictionary-flags.json +35 -0
- package/shared/curated-endonyms.json +35 -0
- package/shared/curated-fsts.json +51 -0
- package/shared/curated-orthography-conventions.json +26 -0
- package/shared/curated-sil-resources.json +374 -0
- package/shared/curated-tools.json +41 -0
- package/shared/docent/corpus.json +11333 -0
- package/shared/docent/faq.en.json +564 -0
- package/shared/docent/register-blocks.json +60 -0
- package/shared/docent/system-prompt.md +144 -0
- package/shared/domain-taxonomy.json +35 -0
- package/shared/explainers/glossary.json +2975 -0
- package/shared/explainers/tc-features.json +20112 -0
- package/shared/explainers/term-watchlist.json +147 -0
- package/shared/human-services.json +59 -0
- package/shared/license-corrections.json +261 -0
- package/shared/license-evidence.json +13452 -0
- package/shared/licenses.json +6781 -0
- package/shared/method-registry.json +236 -0
- package/shared/metric-registry.json +620 -0
- package/shared/model-aliases.json +7 -0
- package/shared/schemas/champollion-plugin.schema.json +206 -0
- package/shared/schemas/corpora-card.schema.json +957 -0
- package/shared/schemas/domain-taxonomy.schema.json +64 -0
- package/shared/schemas/external-results.schema.json +314 -0
- package/shared/schemas/human-services.schema.json +90 -0
- package/shared/schemas/language-card.schema.json +1308 -0
- package/shared/schemas/licenses.schema.json +155 -0
- package/shared/schemas/method-card.schema.json +412 -0
- package/shared/schemas/method-registry.schema.json +85 -0
- package/shared/schemas/metric-registry.schema.json +96 -0
- package/shared/schemas/metric-reliability.schema.json +178 -0
- package/shared/schemas/model-aliases.schema.json +27 -0
- package/shared/schemas/source-snapshot.schema.json +96 -0
package/lib/verify.js
ADDED
|
@@ -0,0 +1,451 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* verify.js — Post-sync verification module.
|
|
3
|
+
*
|
|
4
|
+
* WHY: The sync pipeline can report "synced 30 keys" but some keys
|
|
5
|
+
* might be wrong in fact — empty values, [EN] fallback markers from
|
|
6
|
+
* prior runs, ASCII-only values for non-Latin locales, missing keys,
|
|
7
|
+
* or broken ICU placeholders. This module re-reads the written locale
|
|
8
|
+
* files from disk and confirms translations are actually present and
|
|
9
|
+
* correct.
|
|
10
|
+
*
|
|
11
|
+
* DESIGN: Runs automatically at the end of every sync (unless --no-verify).
|
|
12
|
+
* Also exposed as a standalone `verify` command for CI gates.
|
|
13
|
+
* Reuses existing validation modules — no new check logic, just orchestration.
|
|
14
|
+
*
|
|
15
|
+
* PHILOSOPHY: This is a trust-but-verify gate. Sync does the work,
|
|
16
|
+
* verify confirms the work is correct. Every issue is logged LOUD.
|
|
17
|
+
*
|
|
18
|
+
* OUTPUT: The decorative section headers and per-check [OK] lines go through
|
|
19
|
+
* output.raw(), which is suppressed in --json mode. The real findings go
|
|
20
|
+
* through output.ok/warn/error, which json-encode themselves — so
|
|
21
|
+
* `champollion sync --json | jq` stays parseable while a human still gets
|
|
22
|
+
* the readable report.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import fs from 'node:fs';
|
|
26
|
+
import path from 'node:path';
|
|
27
|
+
import { readLocaleFile, detectFormatFromDir, getExtension, extractDocusaurusMessages } from './format.js';
|
|
28
|
+
import { flattenKeys } from './flatten.js';
|
|
29
|
+
import { auditLocalePair } from './integrity.js';
|
|
30
|
+
import { NON_LATIN_LOCALES, isAsciiOnly } from './validate.js';
|
|
31
|
+
import { compileNoTranslate } from './no-translate.js';
|
|
32
|
+
import { resolvePairs } from './pairs.js';
|
|
33
|
+
import { loadTM, lookupTM, tmMethodKey } from './tm.js';
|
|
34
|
+
import { parsePairKey } from './pairs.js';
|
|
35
|
+
import { output } from './output.js';
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Target locales the CONFIG promises, independent of what's on disk.
|
|
39
|
+
*
|
|
40
|
+
* Directory-derived discovery can't see a locale whose file was never
|
|
41
|
+
* created — which made `verify` (and sync's post-verify) pass green on a
|
|
42
|
+
* project with zero translation done. This reads the same config surfaces
|
|
43
|
+
* the sync pair graph reads: `languages` (via resolvedLanguages) and the
|
|
44
|
+
* targets of any `pairs` overrides. Auto-detect projects (no languages
|
|
45
|
+
* configured) return an empty set — nothing is promised, nothing to enforce.
|
|
46
|
+
*
|
|
47
|
+
* @param {object} config - Resolved config from resolveConfig()
|
|
48
|
+
* @returns {Set<string>} Configured target locale codes (input locale excluded)
|
|
49
|
+
*/
|
|
50
|
+
function configuredTargetLocales(config) {
|
|
51
|
+
const targets = new Set(Object.keys(config.resolvedLanguages || {}));
|
|
52
|
+
if (config.pairs && typeof config.pairs === 'object') {
|
|
53
|
+
for (const key of Object.keys(config.pairs)) {
|
|
54
|
+
const { target } = parsePairKey(key);
|
|
55
|
+
if (target) targets.add(target);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
targets.delete(config.inputLocale);
|
|
59
|
+
return targets;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Run the per-locale correctness checks against a source/target flat map pair.
|
|
64
|
+
*
|
|
65
|
+
* Pure: collects findings into arrays and returns them; does NOT print. Both
|
|
66
|
+
* the flat (JSON/TOML/YAML) path and the Docusaurus path call this so the two
|
|
67
|
+
* can't drift in what they check.
|
|
68
|
+
*
|
|
69
|
+
* @param {object} sourceFlat - Flattened source locale map
|
|
70
|
+
* @param {object} targetFlat - Flattened target locale map
|
|
71
|
+
* @param {string} locale - Target locale code (for script detection)
|
|
72
|
+
* @param {object} config - Resolved config (fallbackPrefix)
|
|
73
|
+
* @param {import('./no-translate.js').NoTranslateMatcher} [noTranslate] -
|
|
74
|
+
* Compiled no-translate matcher. Exempt keys are excluded from the
|
|
75
|
+
* source-echo warning (identical IS correct for them) and checked for
|
|
76
|
+
* drift instead, which is an error.
|
|
77
|
+
* @returns {{ errors: string[], warnings: string[], sourceKeyCount: number, targetKeyCount: number }}
|
|
78
|
+
*/
|
|
79
|
+
function auditTranslations(sourceFlat, targetFlat, locale, config, noTranslate = null, isConfirmedEcho = null) {
|
|
80
|
+
const errors = [];
|
|
81
|
+
const warnings = [];
|
|
82
|
+
const sourceKeyCount = Object.keys(sourceFlat).length;
|
|
83
|
+
const targetKeyCount = Object.keys(targetFlat).length;
|
|
84
|
+
|
|
85
|
+
// 1. Key parity — are all source keys present in target?
|
|
86
|
+
const missingKeys = Object.keys(sourceFlat).filter(k => !(k in targetFlat));
|
|
87
|
+
if (missingKeys.length > 0) {
|
|
88
|
+
const preview = missingKeys.slice(0, 5).join(', ');
|
|
89
|
+
const suffix = missingKeys.length > 5 ? '...' : '';
|
|
90
|
+
errors.push(`${missingKeys.length} missing key(s): ${preview}${suffix}`);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// 2. [EN] fallback marker scan — any legacy [EN]-prefixed values?
|
|
94
|
+
const fallbackPrefix = config.fallbackPrefix || '[EN] ';
|
|
95
|
+
const fallbackKeys = Object.keys(targetFlat).filter(k =>
|
|
96
|
+
typeof targetFlat[k] === 'string' && targetFlat[k].startsWith(fallbackPrefix)
|
|
97
|
+
);
|
|
98
|
+
if (fallbackKeys.length > 0) {
|
|
99
|
+
const preview = fallbackKeys.slice(0, 3).join(', ');
|
|
100
|
+
const suffix = fallbackKeys.length > 3 ? '...' : '';
|
|
101
|
+
errors.push(`${fallbackKeys.length} [EN] fallback marker(s): ${preview}${suffix}`);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// 3. Empty value scan
|
|
105
|
+
const emptyKeys = Object.keys(targetFlat).filter(k =>
|
|
106
|
+
typeof targetFlat[k] === 'string' && targetFlat[k].trim() === ''
|
|
107
|
+
);
|
|
108
|
+
if (emptyKeys.length > 0) {
|
|
109
|
+
const preview = emptyKeys.slice(0, 3).join(', ');
|
|
110
|
+
const suffix = emptyKeys.length > 3 ? '...' : '';
|
|
111
|
+
errors.push(`${emptyKeys.length} empty translation(s): ${preview}${suffix}`);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// 4. Script compliance — non-Latin locales should have non-ASCII translations
|
|
115
|
+
const isNonLatin = NON_LATIN_LOCALES.has(locale) || NON_LATIN_LOCALES.has(locale.split('-')[0]);
|
|
116
|
+
if (isNonLatin) {
|
|
117
|
+
const asciiOnlyKeys = Object.keys(targetFlat).filter(k => {
|
|
118
|
+
const val = targetFlat[k];
|
|
119
|
+
// Only check string values that are long enough to be real translations
|
|
120
|
+
// (short values like "API", "OK", "ID" are often legitimately ASCII)
|
|
121
|
+
if (typeof val !== 'string' || val.length < 6) return false;
|
|
122
|
+
// A no-translate value is ASCII on purpose. `https://…` copied into an
|
|
123
|
+
// Arabic locale is CORRECT, and flagging it as wrong-script would make
|
|
124
|
+
// a clean sync fail its own post-sync verification.
|
|
125
|
+
if (noTranslate && noTranslate.matches(k, sourceFlat[k])) return false;
|
|
126
|
+
return isAsciiOnly(val);
|
|
127
|
+
});
|
|
128
|
+
if (asciiOnlyKeys.length > 0) {
|
|
129
|
+
const preview = asciiOnlyKeys.slice(0, 3).join(', ');
|
|
130
|
+
const suffix = asciiOnlyKeys.length > 3 ? '...' : '';
|
|
131
|
+
errors.push(`${asciiOnlyKeys.length} wrong script (ASCII-only): ${preview}${suffix}`);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// 5. Placeholder preservation — run the integrity audit for placeholder + encoding checks
|
|
136
|
+
const audit = auditLocalePair(sourceFlat, targetFlat, locale, { noTranslate, isConfirmedEcho });
|
|
137
|
+
|
|
138
|
+
if (audit.placeholderIssues.length > 0) {
|
|
139
|
+
const preview = audit.placeholderIssues.slice(0, 3).map(i => i.key).join(', ');
|
|
140
|
+
const suffix = audit.placeholderIssues.length > 3 ? '...' : '';
|
|
141
|
+
errors.push(`${audit.placeholderIssues.length} placeholder mismatch(es): ${preview}${suffix}`);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// 6. Encoding issues (warning)
|
|
145
|
+
if (audit.encodingIssues.length > 0) {
|
|
146
|
+
const preview = audit.encodingIssues.slice(0, 3).map(i => i.key).join(', ');
|
|
147
|
+
const suffix = audit.encodingIssues.length > 3 ? '...' : '';
|
|
148
|
+
warnings.push(`${audit.encodingIssues.length} encoding issue(s): ${preview}${suffix}`);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// 7. Source echo — untranslated copies (warning, not error — some are legitimate)
|
|
152
|
+
if (audit.copies.length > 0) {
|
|
153
|
+
const preview = audit.copies.slice(0, 5).join(', ');
|
|
154
|
+
const suffix = audit.copies.length > 5 ? '...' : '';
|
|
155
|
+
warnings.push(`${audit.copies.length} source echo(es): ${preview}${suffix}`);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// 8. Hollowed values — the source with its letters deleted, written by a
|
|
159
|
+
// pipeline older than the content-preservation gate. The gate can't reach
|
|
160
|
+
// values already on disk (their manifest hashes read as settled), so this
|
|
161
|
+
// is where old damage surfaces. Error: the value is unreadable in fact.
|
|
162
|
+
if (audit.hollowedValues.length > 0) {
|
|
163
|
+
const preview = audit.hollowedValues.slice(0, 3).map(h => h.key).join(', ');
|
|
164
|
+
const suffix = audit.hollowedValues.length > 3 ? '...' : '';
|
|
165
|
+
errors.push(
|
|
166
|
+
`${audit.hollowedValues.length} hollowed value(s) (source with letters deleted): ${preview}${suffix}`
|
|
167
|
+
+ ' — re-translate with `champollion sync --force-keys <key>` or `--pair <pair> --force`',
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// 9. No-translate drift — a declared-verbatim key that is NOT verbatim.
|
|
172
|
+
// An error, not a warning: unlike a source echo there is no legitimate
|
|
173
|
+
// reading of it. The project declared exactly one correct value and the
|
|
174
|
+
// file holds a different one. `champollion sync` repairs it.
|
|
175
|
+
if (audit.noTranslateDrift.length > 0) {
|
|
176
|
+
const preview = audit.noTranslateDrift.slice(0, 3).map(d => d.key).join(', ');
|
|
177
|
+
const suffix = audit.noTranslateDrift.length > 3 ? '...' : '';
|
|
178
|
+
errors.push(
|
|
179
|
+
`${audit.noTranslateDrift.length} no-translate key(s) differ from the source: ${preview}${suffix}`
|
|
180
|
+
+ ' — run `champollion sync` to restore them verbatim',
|
|
181
|
+
);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
return { errors, warnings, sourceKeyCount, targetKeyCount };
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* Print the final verification summary line and return the counts.
|
|
189
|
+
*
|
|
190
|
+
* @param {number} totalErrors
|
|
191
|
+
* @param {number} totalWarnings
|
|
192
|
+
* @returns {{ errors: number, warnings: number }}
|
|
193
|
+
*/
|
|
194
|
+
function printSummary(totalErrors, totalWarnings) {
|
|
195
|
+
if (totalErrors === 0 && totalWarnings === 0) {
|
|
196
|
+
output.ok('Verification passed — all locales look good.');
|
|
197
|
+
} else if (totalErrors === 0) {
|
|
198
|
+
output.ok(`Verification passed with ${totalWarnings} warning(s).`);
|
|
199
|
+
} else {
|
|
200
|
+
output.error(`Verification: ${totalErrors} error(s), ${totalWarnings} warning(s).`);
|
|
201
|
+
}
|
|
202
|
+
return { errors: totalErrors, warnings: totalWarnings };
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Verify all target locale files against the source.
|
|
207
|
+
*
|
|
208
|
+
* Re-reads files from disk (not memory) to confirm what was actually
|
|
209
|
+
* written. Returns a summary of errors and warnings for the caller.
|
|
210
|
+
*
|
|
211
|
+
* @param {object} config - Resolved config from resolveConfig()
|
|
212
|
+
* @param {string} cwd - Working directory
|
|
213
|
+
* @param {object} [options]
|
|
214
|
+
* @param {import('./no-translate.js').NoTranslateMatcher} [options.noTranslate] -
|
|
215
|
+
* Compiled matcher. Pass the SAME instance the sync used so verification
|
|
216
|
+
* judges the files by the rules that wrote them. Derived from config when
|
|
217
|
+
* omitted (the standalone `verify` command).
|
|
218
|
+
* @returns {Promise<{ errors: number, warnings: number }>}
|
|
219
|
+
*/
|
|
220
|
+
async function verifyLocales(config, cwd, options = {}) {
|
|
221
|
+
const noTranslate = options.noTranslate || compileNoTranslate(config);
|
|
222
|
+
|
|
223
|
+
// TM-confirmed echoes are settled facts, not findings — the same
|
|
224
|
+
// suppression the sync diff and `integrity` apply, so the three tools
|
|
225
|
+
// cannot disagree about a healthy file. Read-only TM load.
|
|
226
|
+
const tm = loadTM(cwd);
|
|
227
|
+
const tmKeys = new Map();
|
|
228
|
+
try {
|
|
229
|
+
for (const [, pc] of resolvePairs(config)) tmKeys.set(pc.target, tmMethodKey(pc));
|
|
230
|
+
} catch { /* invalid pair config — sync reports it; verify still runs */ }
|
|
231
|
+
const echoPredicateFor = (locale) => {
|
|
232
|
+
const tmKey = tmKeys.get(locale);
|
|
233
|
+
return tmKey
|
|
234
|
+
? (key, sourceValue) => lookupTM(tm, sourceValue, locale, tmKey) === sourceValue
|
|
235
|
+
: null;
|
|
236
|
+
};
|
|
237
|
+
|
|
238
|
+
// Docusaurus uses a directory-per-locale layout (i18n/<locale>/code.json,
|
|
239
|
+
// …/<plugin>/*.json) — NOT a flat i18n/<locale>.json file. The flat path
|
|
240
|
+
// below would look for i18n/en.json, never find it, and exit 0 — a
|
|
241
|
+
// false-green gate. Route Docusaurus projects to their own verifier.
|
|
242
|
+
if (config.format === 'docusaurus') {
|
|
243
|
+
return verifyDocusaurusLocales(config, cwd, noTranslate, echoPredicateFor);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
const format = config.format !== 'auto'
|
|
247
|
+
? config.format
|
|
248
|
+
: detectFormatFromDir(config.localesDir);
|
|
249
|
+
const ext = getExtension(format);
|
|
250
|
+
const sourcePath = path.join(config.localesDir, `${config.inputLocale}${ext}`);
|
|
251
|
+
|
|
252
|
+
if (!fs.existsSync(sourcePath)) {
|
|
253
|
+
// No source file — can't verify. This shouldn't happen after a sync,
|
|
254
|
+
// but don't crash the user's workflow over it.
|
|
255
|
+
output.warn('[VERIFY] Source locale file not found — skipping verification.');
|
|
256
|
+
return { errors: 0, warnings: 0 };
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
const sourceRaw = readLocaleFile(sourcePath, format);
|
|
260
|
+
const sourceFlat = format === 'json' ? flattenKeys(sourceRaw) : sourceRaw;
|
|
261
|
+
const sourceKeyCount = Object.keys(sourceFlat).length;
|
|
262
|
+
|
|
263
|
+
// Detect target locales from directory listing
|
|
264
|
+
const files = fs.readdirSync(config.localesDir);
|
|
265
|
+
const targetLocales = files
|
|
266
|
+
.filter(f => f.endsWith(ext) && !f.startsWith(config.inputLocale))
|
|
267
|
+
.map(f => f.replace(ext, ''));
|
|
268
|
+
|
|
269
|
+
// Configured locales whose file doesn't exist at all. These are invisible
|
|
270
|
+
// to the directory listing above, so without this check a CI `verify` gate
|
|
271
|
+
// passed with zero translation done. Only enforced when the source has
|
|
272
|
+
// keys to translate — an empty source promises nothing.
|
|
273
|
+
const missingTargets = sourceKeyCount === 0 ? [] :
|
|
274
|
+
[...configuredTargetLocales(config)]
|
|
275
|
+
.filter(l => !fs.existsSync(path.join(config.localesDir, `${l}${ext}`)))
|
|
276
|
+
.sort();
|
|
277
|
+
|
|
278
|
+
if (targetLocales.length === 0 && missingTargets.length === 0) {
|
|
279
|
+
return { errors: 0, warnings: 0 };
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
output.raw('\n ── Post-Sync Verification ───────────────────────────────\n');
|
|
283
|
+
|
|
284
|
+
let totalErrors = 0;
|
|
285
|
+
let totalWarnings = 0;
|
|
286
|
+
|
|
287
|
+
for (const locale of missingTargets) {
|
|
288
|
+
output.error(`[VERIFY] ${locale}: locale file missing (${locale}${ext}) — configured target has no translations. Run \`champollion sync\` to create it.`);
|
|
289
|
+
totalErrors++;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
for (const locale of targetLocales) {
|
|
293
|
+
const targetPath = path.join(config.localesDir, `${locale}${ext}`);
|
|
294
|
+
if (!fs.existsSync(targetPath)) continue;
|
|
295
|
+
|
|
296
|
+
const targetRaw = readLocaleFile(targetPath, format);
|
|
297
|
+
const targetFlat = format === 'json' ? flattenKeys(targetRaw) : targetRaw;
|
|
298
|
+
|
|
299
|
+
output.raw(` ── ${locale} ──────────────────────────────────────`);
|
|
300
|
+
|
|
301
|
+
const { errors, warnings, targetKeyCount } = auditTranslations(sourceFlat, targetFlat, locale, config, noTranslate, echoPredicateFor(locale));
|
|
302
|
+
|
|
303
|
+
if (!errors.some(e => e.includes('missing key'))) {
|
|
304
|
+
output.raw(` [OK] ${targetKeyCount}/${sourceKeyCount} keys present`);
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
for (const err of errors) {
|
|
308
|
+
output.error(`[VERIFY] ${locale}: ${err}`);
|
|
309
|
+
totalErrors++;
|
|
310
|
+
}
|
|
311
|
+
for (const warn of warnings) {
|
|
312
|
+
output.warn(`[VERIFY] ${locale}: ${warn}`);
|
|
313
|
+
totalWarnings++;
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
if (errors.length === 0 && warnings.length === 0) {
|
|
317
|
+
output.raw(' [OK] All checks passed');
|
|
318
|
+
}
|
|
319
|
+
output.raw('');
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
return printSummary(totalErrors, totalWarnings);
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/**
|
|
326
|
+
* Recursively collect all .json files under a directory.
|
|
327
|
+
*
|
|
328
|
+
* @param {string} dir - Directory to walk
|
|
329
|
+
* @returns {string[]} Absolute paths to .json files, sorted
|
|
330
|
+
*/
|
|
331
|
+
function walkJSONFiles(dir) {
|
|
332
|
+
const files = [];
|
|
333
|
+
function walk(d) {
|
|
334
|
+
if (!fs.existsSync(d)) return;
|
|
335
|
+
for (const entry of fs.readdirSync(d, { withFileTypes: true })) {
|
|
336
|
+
const full = path.join(d, entry.name);
|
|
337
|
+
if (entry.isDirectory()) walk(full);
|
|
338
|
+
else if (entry.isFile() && entry.name.endsWith('.json')) files.push(full);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
walk(dir);
|
|
342
|
+
return files.sort();
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
/**
|
|
346
|
+
* Verify a Docusaurus i18n tree (directory-per-locale, {message} JSON files).
|
|
347
|
+
*
|
|
348
|
+
* Compares the UI-string JSON files under i18n/<locale>/ against the source
|
|
349
|
+
* locale's files (i18n/<inputLocale>/), running the SAME checks as the flat
|
|
350
|
+
* path. Markdown content (docs/blog mirrored under each locale) has no
|
|
351
|
+
* key-parity model and is not key-checked here — this gate is for the
|
|
352
|
+
* {message,description} UI strings that Phase 1 of the Docusaurus sync writes.
|
|
353
|
+
*
|
|
354
|
+
* @param {object} config - Resolved config (format === 'docusaurus')
|
|
355
|
+
* @param {string} cwd - Working directory
|
|
356
|
+
* @param {import('./no-translate.js').NoTranslateMatcher} [noTranslate] - Compiled matcher
|
|
357
|
+
* @returns {Promise<{ errors: number, warnings: number }>}
|
|
358
|
+
*/
|
|
359
|
+
async function verifyDocusaurusLocales(config, cwd, noTranslate = null, echoPredicateFor = () => null) {
|
|
360
|
+
const sourceLocaleDir = path.join(config.localesDir, config.inputLocale);
|
|
361
|
+
|
|
362
|
+
if (!fs.existsSync(sourceLocaleDir)) {
|
|
363
|
+
output.warn(`[VERIFY] Docusaurus source locale dir not found (${sourceLocaleDir}) — skipping verification.`);
|
|
364
|
+
return { errors: 0, warnings: 0 };
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
const sourceFiles = walkJSONFiles(sourceLocaleDir);
|
|
368
|
+
if (sourceFiles.length === 0) {
|
|
369
|
+
output.warn('[VERIFY] No source JSON strings found — skipping verification.');
|
|
370
|
+
return { errors: 0, warnings: 0 };
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
// Target locales = subdirectories of i18n/ other than the source locale.
|
|
374
|
+
const targetLocales = fs.readdirSync(config.localesDir, { withFileTypes: true })
|
|
375
|
+
.filter(e => e.isDirectory() && e.name !== config.inputLocale && !e.name.startsWith('.'))
|
|
376
|
+
.map(e => e.name)
|
|
377
|
+
.sort();
|
|
378
|
+
|
|
379
|
+
// Same false-green hole as the flat path: a configured locale with no
|
|
380
|
+
// i18n/<locale>/ directory is invisible to the listing above. Fail loud.
|
|
381
|
+
const missingTargets = [...configuredTargetLocales(config)]
|
|
382
|
+
.filter(l => !fs.existsSync(path.join(config.localesDir, l)))
|
|
383
|
+
.sort();
|
|
384
|
+
|
|
385
|
+
if (targetLocales.length === 0 && missingTargets.length === 0) {
|
|
386
|
+
return { errors: 0, warnings: 0 };
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
output.raw('\n ── Post-Sync Verification (Docusaurus) ──────────────────\n');
|
|
390
|
+
|
|
391
|
+
let totalErrors = 0;
|
|
392
|
+
let totalWarnings = 0;
|
|
393
|
+
|
|
394
|
+
for (const locale of missingTargets) {
|
|
395
|
+
output.error(`[VERIFY] ${locale}: locale directory missing (${path.join(path.basename(config.localesDir), locale)}/) — configured target has no translations. Run \`champollion sync\` to create it.`);
|
|
396
|
+
totalErrors++;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
for (const locale of targetLocales) {
|
|
400
|
+
output.raw(` ── ${locale} ──────────────────────────────────────`);
|
|
401
|
+
|
|
402
|
+
const localeErrors = [];
|
|
403
|
+
const localeWarnings = [];
|
|
404
|
+
|
|
405
|
+
for (const sourceFile of sourceFiles) {
|
|
406
|
+
const relPath = path.relative(sourceLocaleDir, sourceFile);
|
|
407
|
+
const targetFile = path.join(config.localesDir, locale, relPath);
|
|
408
|
+
|
|
409
|
+
let sourceFlat;
|
|
410
|
+
try {
|
|
411
|
+
sourceFlat = extractDocusaurusMessages(JSON.parse(fs.readFileSync(sourceFile, 'utf-8')));
|
|
412
|
+
} catch (err) {
|
|
413
|
+
// A malformed SOURCE file is a setup problem, not a translation gap.
|
|
414
|
+
localeWarnings.push(`${relPath}: unreadable source JSON (${err.message})`);
|
|
415
|
+
continue;
|
|
416
|
+
}
|
|
417
|
+
if (Object.keys(sourceFlat).length === 0) continue; // nothing to verify in this file
|
|
418
|
+
|
|
419
|
+
let targetFlat = {};
|
|
420
|
+
if (fs.existsSync(targetFile)) {
|
|
421
|
+
try {
|
|
422
|
+
targetFlat = extractDocusaurusMessages(JSON.parse(fs.readFileSync(targetFile, 'utf-8')));
|
|
423
|
+
} catch (err) {
|
|
424
|
+
localeErrors.push(`${relPath}: unreadable target JSON (${err.message})`);
|
|
425
|
+
continue;
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
const { errors, warnings } = auditTranslations(sourceFlat, targetFlat, locale, config, noTranslate, echoPredicateFor(locale));
|
|
430
|
+
for (const e of errors) localeErrors.push(`${relPath}: ${e}`);
|
|
431
|
+
for (const w of warnings) localeWarnings.push(`${relPath}: ${w}`);
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
for (const err of localeErrors) {
|
|
435
|
+
output.error(`[VERIFY] ${locale}: ${err}`);
|
|
436
|
+
totalErrors++;
|
|
437
|
+
}
|
|
438
|
+
for (const warn of localeWarnings) {
|
|
439
|
+
output.warn(`[VERIFY] ${locale}: ${warn}`);
|
|
440
|
+
totalWarnings++;
|
|
441
|
+
}
|
|
442
|
+
if (localeErrors.length === 0 && localeWarnings.length === 0) {
|
|
443
|
+
output.raw(' [OK] All checks passed');
|
|
444
|
+
}
|
|
445
|
+
output.raw('');
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
return printSummary(totalErrors, totalWarnings);
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
export { verifyLocales, auditTranslations };
|
package/lib/watch.js
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Watch mode — monitors the source locale file and re-syncs on changes.
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS EXISTS: Extracted from sync.js to reduce the god-module
|
|
5
|
+
* and give watch its own lifecycle management.
|
|
6
|
+
*
|
|
7
|
+
* Uses fs.watchFile (stat polling) with a configurable interval, followed
|
|
8
|
+
* by a 500ms debounce to prevent duplicate syncs when editors write in
|
|
9
|
+
* multiple steps (write + rename).
|
|
10
|
+
*
|
|
11
|
+
* WHY fs.watchFile INSTEAD OF fs.watch:
|
|
12
|
+
* fs.watch relies on kernel-level events (FSEvents on macOS, inotify on
|
|
13
|
+
* Linux). On macOS, FSEvents is unreliable in /tmp and other special
|
|
14
|
+
* directories, and can miss events when writeFileSync replaces the inode
|
|
15
|
+
* (atomic write = write-to-temp + rename). fs.watchFile uses stat polling,
|
|
16
|
+
* which is slower but works reliably on all platforms and filesystems.
|
|
17
|
+
* The 500ms polling interval keeps CPU overhead negligible for a single file.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import fs from 'node:fs';
|
|
21
|
+
import path from 'node:path';
|
|
22
|
+
import { resolveConfig } from './config.js';
|
|
23
|
+
import { detectFormatFromDir, getExtension } from './format.js';
|
|
24
|
+
import { runSync } from './sync.js';
|
|
25
|
+
import { output } from './output.js';
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Start watch mode — sync once, then re-sync on source file changes.
|
|
29
|
+
*
|
|
30
|
+
* LIFECYCLE:
|
|
31
|
+
* 1. Resolve config and locate the source file
|
|
32
|
+
* 2. Run the initial sync (awaited — completes before watching starts)
|
|
33
|
+
* 3. Set up fs.watchFile on the source file
|
|
34
|
+
* 4. Print [WATCH] Ready to signal the watcher is active
|
|
35
|
+
* 5. Block indefinitely (the returned Promise never resolves until SIGINT)
|
|
36
|
+
*
|
|
37
|
+
* WHY THE PROMISE NEVER RESOLVES:
|
|
38
|
+
* The CLI dispatcher calls process.exit() after the command's run() resolves.
|
|
39
|
+
* Watch mode is a long-running process — it must block the run() promise
|
|
40
|
+
* to prevent the CLI from exiting. SIGINT cleanup handles shutdown.
|
|
41
|
+
*
|
|
42
|
+
* RE-ENTRANCY:
|
|
43
|
+
* If a change arrives while a sync is already in progress, the change is
|
|
44
|
+
* queued via `pendingSync`. When the current sync finishes, it checks
|
|
45
|
+
* `pendingSync` and re-syncs if needed. This prevents both re-entrant
|
|
46
|
+
* syncs AND silently dropped changes.
|
|
47
|
+
*
|
|
48
|
+
* @param {object} options
|
|
49
|
+
* @param {string} [options.cwd] - Working directory
|
|
50
|
+
* @param {object} [options.cliArgs] - CLI arguments
|
|
51
|
+
* @returns {Promise<never>} Never resolves — blocks until SIGINT
|
|
52
|
+
*/
|
|
53
|
+
async function startWatch(options = {}) {
|
|
54
|
+
const { cwd = process.cwd(), cliArgs = {} } = options;
|
|
55
|
+
const config = resolveConfig(cliArgs, cwd);
|
|
56
|
+
const format = config.format !== 'auto'
|
|
57
|
+
? config.format
|
|
58
|
+
: detectFormatFromDir(config.localesDir);
|
|
59
|
+
const ext = getExtension(format);
|
|
60
|
+
const inputLocale = config.inputLocale;
|
|
61
|
+
const sourceFile = `${inputLocale}${ext}`;
|
|
62
|
+
const sourcePath = path.join(config.localesDir, sourceFile);
|
|
63
|
+
|
|
64
|
+
output.info(`Watching ${sourceFile} for changes...`);
|
|
65
|
+
|
|
66
|
+
// Await the initial sync so it completes before we start watching.
|
|
67
|
+
// WHY: runSync is async and writes to target files in the locales directory.
|
|
68
|
+
// Starting the watcher before the initial sync completes could cause the
|
|
69
|
+
// sync's writes to trigger re-entrancy or confuse event delivery.
|
|
70
|
+
await runSync({ cwd, dryRun: !!cliArgs.dry, cliArgs });
|
|
71
|
+
|
|
72
|
+
// --- Watcher state ---
|
|
73
|
+
let debounceTimer = null;
|
|
74
|
+
let syncing = false;
|
|
75
|
+
// Track whether a new change arrived while we were syncing.
|
|
76
|
+
// WHY: Without this, a save during an active sync would be silently dropped.
|
|
77
|
+
// The user would need to save a third time to trigger the re-sync.
|
|
78
|
+
let pendingSync = false;
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Execute a sync cycle. If another change arrives mid-sync, it sets
|
|
82
|
+
* `pendingSync = true`, and we loop back to re-sync after the current
|
|
83
|
+
* one finishes. This guarantees no change is ever silently dropped.
|
|
84
|
+
*/
|
|
85
|
+
async function doSync() {
|
|
86
|
+
syncing = true;
|
|
87
|
+
try {
|
|
88
|
+
output.info(`${sourceFile} changed — syncing locales...`);
|
|
89
|
+
await runSync({ cwd, dryRun: !!cliArgs.dry, cliArgs });
|
|
90
|
+
} catch (err) {
|
|
91
|
+
// Log but don't crash — watch mode should survive transient errors
|
|
92
|
+
// (e.g., malformed JSON during a half-written save).
|
|
93
|
+
output.error(`Sync failed: ${err.message}`);
|
|
94
|
+
}
|
|
95
|
+
syncing = false;
|
|
96
|
+
|
|
97
|
+
// If a change arrived during the sync, run again immediately.
|
|
98
|
+
if (pendingSync) {
|
|
99
|
+
pendingSync = false;
|
|
100
|
+
await doSync();
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// Watch the source file using stat polling (fs.watchFile).
|
|
105
|
+
// On change, debounce for 500ms then sync.
|
|
106
|
+
fs.watchFile(sourcePath, { interval: 500 }, (curr, prev) => {
|
|
107
|
+
// Only react to actual content changes (mtime changed)
|
|
108
|
+
if (curr.mtimeMs === prev.mtimeMs) return;
|
|
109
|
+
|
|
110
|
+
if (debounceTimer) clearTimeout(debounceTimer);
|
|
111
|
+
debounceTimer = setTimeout(() => {
|
|
112
|
+
if (syncing) {
|
|
113
|
+
// A sync is already in progress — mark that we need another one.
|
|
114
|
+
pendingSync = true;
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
// Fire-and-forget but with error handling inside doSync().
|
|
118
|
+
// We intentionally don't await here because we're inside a
|
|
119
|
+
// setTimeout callback (non-async context). Errors are caught
|
|
120
|
+
// inside doSync() and logged, never thrown.
|
|
121
|
+
doSync();
|
|
122
|
+
}, 500);
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
// Signal that the watcher is active and ready for file changes.
|
|
126
|
+
// Tests use this line to know when it's safe to modify the source file.
|
|
127
|
+
// Tests grep for this exact string to detect watcher readiness — use output.raw
|
|
128
|
+
// to preserve the format while still respecting --quiet mode.
|
|
129
|
+
output.raw(`\n[WATCH] Ready — monitoring ${sourceFile} for changes.`);
|
|
130
|
+
|
|
131
|
+
// Return a promise that never resolves — keeps the CLI process alive.
|
|
132
|
+
// SIGINT cleanup handles the actual shutdown via process.exit().
|
|
133
|
+
return new Promise((resolve) => {
|
|
134
|
+
// Use 'once' instead of 'on' to prevent duplicate handler stacking
|
|
135
|
+
// if startWatch were ever called more than once in the same process.
|
|
136
|
+
process.once('SIGINT', () => {
|
|
137
|
+
if (debounceTimer) clearTimeout(debounceTimer);
|
|
138
|
+
fs.unwatchFile(sourcePath);
|
|
139
|
+
resolve(); // Allow the promise to resolve so cleanup can happen
|
|
140
|
+
process.exit(0);
|
|
141
|
+
});
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export { startWatch };
|