champollion 0.3.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -26
- package/bin/cli.js +53 -5
- package/index.js +63 -2
- package/lib/api-key.js +17 -4
- package/lib/autofix.js +83 -36
- package/lib/bridge/method_bridge.py +15 -3
- package/lib/cards/reader.js +34 -0
- package/lib/cards/remote.js +15 -0
- package/lib/cards/search-names.js +178 -0
- package/lib/command-help.js +286 -85
- package/lib/commands/audit.js +10 -3
- package/lib/commands/card.js +583 -226
- package/lib/commands/doctor.js +54 -18
- package/lib/commands/help.js +37 -32
- package/lib/commands/init.js +1689 -87
- package/lib/commands/integrity.js +127 -40
- package/lib/commands/leaderboard.js +187 -67
- package/lib/commands/models.js +9 -2
- package/lib/commands/provenance.js +7 -2
- package/lib/commands/recommend.js +43 -14
- package/lib/commands/register-corpus.js +632 -125
- package/lib/commands/seal-corpus.js +1 -1
- package/lib/commands/status.js +564 -27
- package/lib/commands/submit.js +17 -12
- package/lib/commands/sync.js +31 -7
- package/lib/commands/tm.js +15 -9
- package/lib/commands/verify.js +27 -3
- package/lib/commands/wrap.js +63 -5
- package/lib/commands/xliff.js +135 -64
- package/lib/commercial-eligibility.js +1 -1
- package/lib/config.js +196 -14
- package/lib/content-estimate.js +96 -0
- package/lib/content-refusals.js +270 -0
- package/lib/content-review.js +372 -0
- package/lib/content-sync.js +1127 -344
- package/lib/content.js +94 -7
- package/lib/corpus-registration.mjs +194 -35
- package/lib/cost-label.js +29 -0
- package/lib/cost-report.js +726 -78
- package/lib/diff.js +38 -4
- package/lib/docusaurus-sync.js +965 -253
- package/lib/edit-distance.js +31 -0
- package/lib/fallback.js +964 -0
- package/lib/file-scope.js +106 -0
- package/lib/flatten.js +80 -3
- package/lib/flutter-locales.js +124 -0
- package/lib/format.js +266 -12
- package/lib/hash.js +146 -21
- package/lib/icu-structure.js +929 -0
- package/lib/integrity.js +223 -75
- package/lib/language-pair.js +157 -0
- package/lib/lint.js +78 -16
- package/lib/local-only-marks.js +106 -0
- package/lib/locale-layout.js +1103 -0
- package/lib/locale-state.js +571 -0
- package/lib/methods/anthropic.js +5 -0
- package/lib/methods/apertium.js +6 -3
- package/lib/methods/api.js +138 -25
- package/lib/methods/base.js +17 -0
- package/lib/methods/coaching-data.js +153 -0
- package/lib/methods/content-separator.js +43 -0
- package/lib/methods/deepl.js +1 -1
- package/lib/methods/direct-llm.js +252 -103
- package/lib/methods/external.js +146 -63
- package/lib/methods/gemini.js +1 -0
- package/lib/methods/google-translate.js +1 -0
- package/lib/methods/http-utils.js +41 -0
- package/lib/methods/libretranslate.js +7 -2
- package/lib/methods/llm-coached.js +68 -128
- package/lib/methods/llm.js +80 -31
- package/lib/methods/local.js +93 -10
- package/lib/methods/microsoft-translator.js +1 -2
- package/lib/methods/openai.js +4 -2
- package/lib/methods/openrouter-client.js +20 -19
- package/lib/methods/openrouter-pricing.js +150 -13
- package/lib/methods/prompt-methods.js +20 -0
- package/lib/methods/provider-pricing.js +42 -1
- package/lib/methods/request-capture.js +104 -0
- package/lib/methods/tilde.js +1 -1
- package/lib/methods/translated.js +1 -2
- package/lib/missing-key.js +93 -0
- package/lib/models.js +11 -0
- package/lib/name-rules.js +32 -0
- package/lib/named-keys.js +172 -0
- package/lib/no-translate.js +4 -3
- package/lib/output.js +160 -19
- package/lib/pairs.js +586 -30
- package/lib/placeholders.js +394 -0
- package/lib/plugins.js +8 -0
- package/lib/plural-gap-redo.js +109 -0
- package/lib/plurals.js +323 -0
- package/lib/po.js +1187 -0
- package/lib/public-catalogue.js +74 -0
- package/lib/recommend.js +527 -32
- package/lib/redo.js +95 -0
- package/lib/refusal-category.js +44 -0
- package/lib/registers.js +255 -11
- package/lib/repair-script.js +20 -13
- package/lib/scripts.js +6 -1
- package/lib/seal.mjs +4 -3
- package/lib/sealed-qualifier.mjs +1 -1
- package/lib/segment.js +2 -1
- package/lib/seo.js +19 -9
- package/lib/serve.js +43 -6
- package/lib/shared-output-seed.js +164 -0
- package/lib/source-contexts.js +39 -0
- package/lib/submit.mjs +57 -5
- package/lib/sync.js +2923 -474
- package/lib/terminology.js +13 -4
- package/lib/tm-evict.js +179 -0
- package/lib/tm-seed.js +5 -2
- package/lib/tm.js +818 -36
- package/lib/translate-pair.js +639 -34
- package/lib/translate.js +78 -5
- package/lib/types.js +22 -3
- package/lib/validate.js +880 -17
- package/lib/verify.js +1296 -104
- package/lib/watch.js +32 -13
- package/lib/xliff.js +44 -3
- package/package.json +1 -1
- package/shared/CORPORA-CARDS.md +2 -0
- package/shared/cards-fallback.json +1 -1
- package/shared/curated-orthography-conventions.json +26 -8
- package/shared/gettext-plural-forms.json +45 -0
- package/shared/method-registry.json +2 -0
- package/shared/metric-registry.json +96 -18
- package/shared/schemas/champollion-plugin.schema.json +4 -0
- package/shared/schemas/corpora-card.schema.json +8 -2
- package/shared/schemas/method-index-record.schema.json +67 -0
- package/shared/schemas/method-registry.schema.json +4 -0
- package/shared/schemas/metric-registry.schema.json +55 -1
- package/shared/docent/corpus.json +0 -11739
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What to tell someone whose method cannot run for want of a key — ONE helper
|
|
3
|
+
* for every place that says it (the sync preflight, real and dry; a method
|
|
4
|
+
* that returned nothing; the content and Docusaurus lanes; `models`).
|
|
5
|
+
*
|
|
6
|
+
* On a CI runner the fix is a repository secret passed to the step, not
|
|
7
|
+
* `export …` or `.env.local`: the Round 8 fix reached one of these places and
|
|
8
|
+
* a real CI run still printed the laptop advice from the others (Round 9,
|
|
9
|
+
* i18next persona). Off CI, the method's own setup help stands as before.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Is this a CI runner? `CI` or `GITHUB_ACTIONS` set to anything but an empty
|
|
14
|
+
* string, "false" or "0" (a developer's shell may export CI=false).
|
|
15
|
+
*
|
|
16
|
+
* @param {object} [env]
|
|
17
|
+
* @returns {boolean}
|
|
18
|
+
*/
|
|
19
|
+
export function onCIRunner(env = process.env) {
|
|
20
|
+
const on = (v) => typeof v === 'string' && v.trim() !== '' && !/^(false|0)$/i.test(v.trim());
|
|
21
|
+
return on(env.CI) || on(env.GITHUB_ACTIONS);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* The secret names a readiness reason names, in its first parenthesis:
|
|
26
|
+
* "No OpenRouter API key (OPENROUTER_API_KEY)." → OPENROUTER_API_KEY
|
|
27
|
+
* "No Google Translate API key (GOOGLE_TRANSLATE_API_KEY (or GOOGLE_API_KEY))." → the first (canonical) one
|
|
28
|
+
* "No Lara credentials (LARA_ACCESS_KEY_ID + LARA_ACCESS_KEY_SECRET)." → both (a key pair)
|
|
29
|
+
*
|
|
30
|
+
* @param {Iterable<string>} reasons
|
|
31
|
+
* @returns {string[]} distinct names, in order
|
|
32
|
+
*/
|
|
33
|
+
export function secretNamesIn(reasons) {
|
|
34
|
+
const names = [];
|
|
35
|
+
for (const reason of reasons || []) {
|
|
36
|
+
const m = /\(([A-Z][A-Z0-9]*_[A-Z0-9_]+(?:\s*\+\s*[A-Z][A-Z0-9]*_[A-Z0-9_]+)*)/.exec(String(reason));
|
|
37
|
+
if (!m) continue;
|
|
38
|
+
for (const n of m[1].split('+').map(x => x.trim())) if (!names.includes(n)) names.push(n);
|
|
39
|
+
}
|
|
40
|
+
return names;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* "In CI: add a repository secret named X and pass it to the sync step (env: X: ${{ secrets.X }})."
|
|
45
|
+
*
|
|
46
|
+
* @param {string[]} names
|
|
47
|
+
* @param {string} [step] - what the secret is passed to
|
|
48
|
+
* @returns {string|null} null when no name is known
|
|
49
|
+
*/
|
|
50
|
+
export function ciSecretLine(names, step = 'the sync step') {
|
|
51
|
+
if (!names || names.length === 0) return null;
|
|
52
|
+
const plural = names.length > 1;
|
|
53
|
+
return `In CI: add ${plural ? 'repository secrets' : 'a repository secret'} named ${names.join(' and ')} and pass `
|
|
54
|
+
+ `${plural ? 'them' : 'it'} to ${step} (env: ${names.map(n => `${n}: \${{ secrets.${n} }}`).join(', ')}).`;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* The advice lines for a missing key: on a CI runner, the repository-secret
|
|
59
|
+
* line INSTEAD of the shell advice; elsewhere, the method's own setup help
|
|
60
|
+
* (export …, .env.local). A reason that names no secret (a model server that
|
|
61
|
+
* does not answer) keeps the setup help everywhere — its reason already says
|
|
62
|
+
* what to do on a runner (lib/methods/local.js).
|
|
63
|
+
*
|
|
64
|
+
* @param {object} p
|
|
65
|
+
* @param {Iterable<string>} p.reasons - readiness reasons
|
|
66
|
+
* @param {string[]} [p.setupHelp] - the method's getSetupHelp() lines
|
|
67
|
+
* @param {string} [p.indent]
|
|
68
|
+
* @param {object} [p.env]
|
|
69
|
+
* @param {string} [p.step] - what the secret is passed to (default: the sync step)
|
|
70
|
+
* @returns {string[]}
|
|
71
|
+
*/
|
|
72
|
+
export function missingKeyAdvice({ reasons, setupHelp = [], indent = ' ', env = process.env, step = 'the sync step' }) {
|
|
73
|
+
const line = ciSecretLine(secretNamesIn(reasons), step);
|
|
74
|
+
if (onCIRunner(env) && line) return [`${indent}${line}`];
|
|
75
|
+
return [...setupHelp];
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* The setup help for a method that returned nothing: when its key is missing
|
|
80
|
+
* (its readiness check says so), the missing-key advice above; when the key
|
|
81
|
+
* is there, the method's own HTTP-failure help.
|
|
82
|
+
*
|
|
83
|
+
* @param {object} method - a method instance (lib/methods/base.js)
|
|
84
|
+
* @param {{ apiKey?: string|null, cwd?: string }} [context]
|
|
85
|
+
* @returns {Promise<string[]>}
|
|
86
|
+
*/
|
|
87
|
+
export async function methodSetupAdvice(method, context = {}) {
|
|
88
|
+
const help = method.getSetupHelp();
|
|
89
|
+
let readiness = { ready: true };
|
|
90
|
+
try { readiness = await method.checkReadiness({ apiKey: context.apiKey ?? null, cwd: context.cwd }); } catch { /* the help stands */ }
|
|
91
|
+
if (readiness && readiness.ready === false) return missingKeyAdvice({ reasons: [readiness.reason], setupHelp: help });
|
|
92
|
+
return help;
|
|
93
|
+
}
|
package/lib/models.js
CHANGED
|
@@ -219,6 +219,16 @@ function getProviderLabel(provider) {
|
|
|
219
219
|
return PROVIDERS[provider]?.label || provider;
|
|
220
220
|
}
|
|
221
221
|
|
|
222
|
+
/**
|
|
223
|
+
* The environment variable a provider's key is read from (null: unknown).
|
|
224
|
+
*
|
|
225
|
+
* @param {string} provider - Provider name
|
|
226
|
+
* @returns {string|null}
|
|
227
|
+
*/
|
|
228
|
+
function getProviderEnvVar(provider) {
|
|
229
|
+
return PROVIDERS[provider]?.envVar || null;
|
|
230
|
+
}
|
|
231
|
+
|
|
222
232
|
/**
|
|
223
233
|
* Check if a provider has model listing support.
|
|
224
234
|
*
|
|
@@ -251,6 +261,7 @@ export {
|
|
|
251
261
|
resolveModel,
|
|
252
262
|
resolveProviderApiKey,
|
|
253
263
|
getProviderLabel,
|
|
264
|
+
getProviderEnvVar,
|
|
254
265
|
isListableProvider,
|
|
255
266
|
getListableProviders,
|
|
256
267
|
clearModelCache,
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* name-rules.js — the ONE wording every translation prompt uses for names.
|
|
3
|
+
*
|
|
4
|
+
* The old rule ("proper nouns, product names, and technical terms … role
|
|
5
|
+
* descriptions … should stay in English") let models keep short descriptive
|
|
6
|
+
* labels in English: on the 2026-08-28 dogfood run "Translation CLI" and
|
|
7
|
+
* "Evaluation harness" came back verbatim in Arabic and Thai and failed
|
|
8
|
+
* verification. Names stay; descriptions get translated; a project can
|
|
9
|
+
* declare its real names (config.protectedTerms) so the model never has to
|
|
10
|
+
* guess.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Prompt rule lines about names, each starting with "- ".
|
|
15
|
+
*
|
|
16
|
+
* @param {string[]} [protectedTerms] - config.protectedTerms
|
|
17
|
+
* @returns {string} Newline-joined rule lines (no trailing newline)
|
|
18
|
+
*/
|
|
19
|
+
export function nameRules(protectedTerms = []) {
|
|
20
|
+
const terms = (protectedTerms || []).filter(t => typeof t === 'string' && t.trim());
|
|
21
|
+
const lines = [
|
|
22
|
+
'- Proper names (people, companies, products, places) stay exactly as written.',
|
|
23
|
+
];
|
|
24
|
+
if (terms.length > 0) {
|
|
25
|
+
lines.push(`- Keep these exactly as written wherever they appear: ${terms.map(t => JSON.stringify(t)).join(', ')}.`);
|
|
26
|
+
}
|
|
27
|
+
lines.push(
|
|
28
|
+
'- Descriptive labels and phrases are NOT names, even when short or capitalized (e.g. "Translation CLI", "Evaluation harness", "Getting Started") — translate them.',
|
|
29
|
+
'- Widely used technical acronyms (API, URL, JSON, CLI) may stay as written inside a translated phrase.',
|
|
30
|
+
);
|
|
31
|
+
return lines.join('\n');
|
|
32
|
+
}
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* named-keys.js — keys named for a redo (`--redo keys:` / `--force-keys`)
|
|
3
|
+
* must exist. ONE rule for every sync path (key-value files, Docusaurus UI
|
|
4
|
+
* strings), so the two cannot drift apart.
|
|
5
|
+
*
|
|
6
|
+
* A name that matched nothing used to do nothing in silence ("fully synced
|
|
7
|
+
* … 0 keys", exit 0 — in CI exactly a success), so a typo or a gettext msgid
|
|
8
|
+
* that only exists WITH a context looked like a finished repair (Round 7,
|
|
9
|
+
* Django persona). Now:
|
|
10
|
+
* - no name matches: the run stops before anything is translated or sent
|
|
11
|
+
* (exit 1), naming the closest keys that exist;
|
|
12
|
+
* - some names match: those are redone, then the run fails (exit 1)
|
|
13
|
+
* naming the rest — said again at the very end, where CI looks.
|
|
14
|
+
*
|
|
15
|
+
* Each lane says what its key space is (`known`) — the key-value lane adds
|
|
16
|
+
* the plural forms a target gets and the namespace prefix; the Docusaurus
|
|
17
|
+
* lane collects every source JSON file's message ids.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { fromTypedContext } from './locale-layout.js';
|
|
21
|
+
import { splitKeyList } from './redo.js';
|
|
22
|
+
import { output } from './output.js';
|
|
23
|
+
import { estimateCost } from './pairs.js';
|
|
24
|
+
import { costLabel } from './cost-label.js';
|
|
25
|
+
import { editDistance } from './edit-distance.js';
|
|
26
|
+
|
|
27
|
+
/** A key as a person reads and types it: the gettext context separator as ␄. */
|
|
28
|
+
export const shownKey = (k) => String(k).replace(/\u0004/g, '\u2404');
|
|
29
|
+
|
|
30
|
+
/** "`button␄Cancel` (or type button\x04Cancel)" — both spellings, for a key with a context. */
|
|
31
|
+
function bothSpellings(k) {
|
|
32
|
+
const shown = shownKey(k);
|
|
33
|
+
return /\u0004/.test(k) ? `\`${shown}\` (or type \`${String(k).replace(/\u0004/g, '\\x04')}\`)` : `\`${shown}\``;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Which of the names match a key in `known`, and — for each that does not —
|
|
38
|
+
* the closest keys that do: every context variant of a gettext msgid
|
|
39
|
+
* (`button␄Cancel`, `status␄Cancel` for "Cancel"), the same key in another
|
|
40
|
+
* case, near spellings.
|
|
41
|
+
*
|
|
42
|
+
* @param {string[]} names - As typed (␄ or \x04 for a gettext context)
|
|
43
|
+
* @param {Set<string>} known - Every key a name may match, in the lane's key space
|
|
44
|
+
* @param {object} [opts]
|
|
45
|
+
* @param {(k: string) => string} [opts.bare] - A key without its namespace
|
|
46
|
+
* prefix (key-value layouts with several files); identity otherwise
|
|
47
|
+
* @param {boolean} [opts.bareMatches] - A name with no namespace names that
|
|
48
|
+
* key in every file that has it
|
|
49
|
+
* @returns {{ matched: string[], missed: Array<{ name: string, closest: string[] }> }}
|
|
50
|
+
*/
|
|
51
|
+
export function checkNamedKeys(names, known, { bare = (k) => k, bareMatches = false } = {}) {
|
|
52
|
+
// A name with no namespace of its own (bare(name) === name) names that key
|
|
53
|
+
// in every file that has it.
|
|
54
|
+
const matches = (name) => known.has(name)
|
|
55
|
+
|| (bareMatches && bare(name) === name && [...known].some(k => bare(k) === name));
|
|
56
|
+
const matched = [];
|
|
57
|
+
const missed = [];
|
|
58
|
+
for (const raw of names) {
|
|
59
|
+
const name = fromTypedContext(raw);
|
|
60
|
+
if (matches(name)) { matched.push(raw); continue; }
|
|
61
|
+
const want = bare(name);
|
|
62
|
+
const msgid = want.includes('\u0004') ? want.slice(want.indexOf('\u0004') + 1) : want;
|
|
63
|
+
const lower = want.toLowerCase();
|
|
64
|
+
const max = Math.max(2, Math.floor(want.length / 5));
|
|
65
|
+
const scored = [];
|
|
66
|
+
for (const k of known) {
|
|
67
|
+
const kb = bare(k);
|
|
68
|
+
const kMsgid = kb.includes('\u0004') ? kb.slice(kb.indexOf('\u0004') + 1) : kb;
|
|
69
|
+
let score = null;
|
|
70
|
+
if (kMsgid === msgid) score = 0; // the same msgid, with (another) context
|
|
71
|
+
else if (kb.toLowerCase() === lower || kMsgid.toLowerCase() === msgid.toLowerCase()) score = 1;
|
|
72
|
+
else {
|
|
73
|
+
const d = editDistance(kb.toLowerCase(), lower, max);
|
|
74
|
+
if (d <= max) score = 1 + d;
|
|
75
|
+
}
|
|
76
|
+
if (score !== null) scored.push([score, k]);
|
|
77
|
+
}
|
|
78
|
+
scored.sort((a, b) => a[0] - b[0] || a[1].localeCompare(b[1]));
|
|
79
|
+
missed.push({ name: raw, closest: scored.slice(0, 8).map(([, k]) => k) });
|
|
80
|
+
}
|
|
81
|
+
return { matched, missed };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** The error lines for named keys that matched nothing. */
|
|
85
|
+
export function describeUnmatchedKeys(missed, { flag }) {
|
|
86
|
+
const lines = missed.map(({ name, closest }) => `${flag} names "${shownKey(fromTypedContext(name))}", which matches no key in the source files`
|
|
87
|
+
+ (closest.length > 0 ? ` — closest: ${closest.map(bothSpellings).join(', ')}` : ' (and nothing close to it)') + '.');
|
|
88
|
+
if (missed.some(m => m.closest.some(k => k.includes('\u0004')))) {
|
|
89
|
+
lines.push('A gettext entry with a context is named msgctxt␄msgid; type the ␄ as \\x04 if you cannot (both spellings work).');
|
|
90
|
+
}
|
|
91
|
+
return lines;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** How the run's named keys were given, for its messages. */
|
|
95
|
+
export function namedKeyFlag(cliArgs) {
|
|
96
|
+
return cliArgs.redo !== undefined ? '--redo keys:' : '--force-keys';
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* The rule, applied before any work. With no name matching it throws (the
|
|
101
|
+
* run exits 1, nothing translated or sent). With some matching, it prints
|
|
102
|
+
* the misses, narrows `cliArgs['force-keys']` and `config.forceKeys` to the
|
|
103
|
+
* names that exist, and returns the misses — the caller reports them again
|
|
104
|
+
* at the end (reportUnmatchedKeys) and the run exits 1.
|
|
105
|
+
*
|
|
106
|
+
* @param {object} p
|
|
107
|
+
* @param {object} p.cliArgs - Parsed CLI args (mutated: force-keys narrowed)
|
|
108
|
+
* @param {object} p.config - Resolved config (mutated: forceKeys narrowed)
|
|
109
|
+
* @param {Set<string>|(() => Set<string>)} p.known - The lane's key space (lazy: built only when keys are named)
|
|
110
|
+
* @param {(k: string) => string} [p.bare]
|
|
111
|
+
* @param {boolean} [p.bareMatches]
|
|
112
|
+
* @returns {Array<{ name: string, closest: string[] }>} The names that matched nothing
|
|
113
|
+
*/
|
|
114
|
+
export function applyNamedKeyRule({ cliArgs, config, known, bare, bareMatches = false }) {
|
|
115
|
+
const named = splitKeyList(cliArgs['force-keys'] || '');
|
|
116
|
+
if (named.length === 0) return [];
|
|
117
|
+
const keys = typeof known === 'function' ? known() : known;
|
|
118
|
+
const { matched, missed } = checkNamedKeys(named, keys, { bare, bareMatches });
|
|
119
|
+
if (missed.length === 0) return [];
|
|
120
|
+
const lines = describeUnmatchedKeys(missed, { flag: namedKeyFlag(cliArgs) });
|
|
121
|
+
if (matched.length === 0) {
|
|
122
|
+
throw new Error(`${lines.join('\n')}\nNothing was translated or sent.`);
|
|
123
|
+
}
|
|
124
|
+
for (const l of lines) output.error(l);
|
|
125
|
+
output.error(`Redoing the ${matched.length} named key(s) that exist; the run then fails (exit 1) for the ${missed.length} that do not.`);
|
|
126
|
+
cliArgs['force-keys'] = matched.map(k => k.replace(/,/g, '\\,')).join(',');
|
|
127
|
+
// --redo all / --force re-queues every key anyway; the narrowed names only
|
|
128
|
+
// decide what a named redo replaces.
|
|
129
|
+
config.forceKeys = cliArgs.force ? config.forceKeys : splitKeyList(cliArgs['force-keys']);
|
|
130
|
+
return missed;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** Said again at the end of the run, where a reader (or CI) looks for the verdict. */
|
|
134
|
+
export function reportUnmatchedKeys(missed, cliArgs) {
|
|
135
|
+
if (!missed || missed.length === 0) return;
|
|
136
|
+
for (const l of describeUnmatchedKeys(missed, { flag: namedKeyFlag(cliArgs) })) output.error(l);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** The --json summary's `unmatchedKeys`: each name with the closest keys that exist. */
|
|
140
|
+
export function unmatchedKeysSummary(missed) {
|
|
141
|
+
return (missed || []).map(m => ({ name: m.name, closest: m.closest.map(shownKey) }));
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Keys NAMED for a redo that the cache answered: the model was not asked, and
|
|
146
|
+
* a redo without --fresh said nothing about it — the persona's `--redo
|
|
147
|
+
* keys:<k>` wrote back the text it already had (Round 7, Django). Said, with
|
|
148
|
+
* the --fresh command and what it costs — by every key lane (key-value files,
|
|
149
|
+
* Docusaurus UI strings), in the same words.
|
|
150
|
+
*
|
|
151
|
+
* @param {object} p
|
|
152
|
+
* @param {string} p.filename - The file as the run names it
|
|
153
|
+
* @param {string[]} p.keys - The named keys the cache served
|
|
154
|
+
* @param {string[]} [p.shown] - The same keys as a person names them (default: keys)
|
|
155
|
+
* @param {object} [p.served] - key → the text the cache served
|
|
156
|
+
* @param {object} [p.onDisk] - key → the text the file held before the run
|
|
157
|
+
* @param {object} p.pairConfig
|
|
158
|
+
* @param {string|null} [p.cwd]
|
|
159
|
+
* @param {boolean} [p.dryRun]
|
|
160
|
+
* @param {string} p.command - The `--redo keys: … --fresh` command that asks the model again
|
|
161
|
+
*/
|
|
162
|
+
export async function reportNamedFromCache({ filename, keys, shown = keys, served = {}, onDisk = {}, pairConfig, cwd = null, dryRun = false, command }) {
|
|
163
|
+
if (keys.length === 0) return;
|
|
164
|
+
const same = keys.filter(k => typeof served[k] === 'string' && served[k] === onDisk[k]);
|
|
165
|
+
let price = 'cost unknown';
|
|
166
|
+
try { price = costLabel(await estimateCost(keys.length, pairConfig, { cwd })); } catch { /* unknown */ }
|
|
167
|
+
const sample = `${shown.slice(0, 3).join(', ')}${shown.length > 3 ? `, +${shown.length - 3} more` : ''}`;
|
|
168
|
+
output.info(`${filename} — ${keys.length} key(s) named for a redo ${dryRun ? 'would be' : 'were'} served from the cache, `
|
|
169
|
+
+ `not asked again (${sample})${same.length === keys.length ? ' — the file already holds that text' : ''}: `
|
|
170
|
+
+ 'a redo re-checks and re-writes what the cache holds, at no cost. To ask the model again: '
|
|
171
|
+
+ `\`${command}\` (sends ${keys.length} key(s) — ${price}).`);
|
|
172
|
+
}
|
package/lib/no-translate.js
CHANGED
|
@@ -73,10 +73,11 @@ function isBareUrl(value) {
|
|
|
73
73
|
* - a pattern with no wildcard is an exact dot-path
|
|
74
74
|
*
|
|
75
75
|
* @param {string} pattern - e.g. '**.url', 'pages.software.*.repo'
|
|
76
|
+
* @param {string} [sep='.'] - Segment separator ('/' for file paths — see file-scope.js)
|
|
76
77
|
* @returns {(keySegments: string[]) => boolean} Segment-array matcher
|
|
77
78
|
*/
|
|
78
|
-
function compilePattern(pattern) {
|
|
79
|
-
const patternSegments = pattern.split(
|
|
79
|
+
function compilePattern(pattern, sep = '.') {
|
|
80
|
+
const patternSegments = pattern.split(sep);
|
|
80
81
|
|
|
81
82
|
// Per-segment regexes, built once. `**` is handled structurally below and
|
|
82
83
|
// never reaches this map.
|
|
@@ -88,7 +89,7 @@ function compilePattern(pattern) {
|
|
|
88
89
|
const source = '^' + seg
|
|
89
90
|
.split('*')
|
|
90
91
|
.map(part => part.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
|
|
91
|
-
.join(
|
|
92
|
+
.join(`[^${sep.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}]*`) + '$';
|
|
92
93
|
const re = new RegExp(source);
|
|
93
94
|
return (s) => re.test(s);
|
|
94
95
|
});
|
package/lib/output.js
CHANGED
|
@@ -31,6 +31,44 @@
|
|
|
31
31
|
*/
|
|
32
32
|
let mode = 'default';
|
|
33
33
|
|
|
34
|
+
/**
|
|
35
|
+
* Keys as people can copy them. A gettext key with a context is
|
|
36
|
+
* `msgctxt + U+0004 + msgid` — gettext's own encoding — and U+0004 is an
|
|
37
|
+
* invisible control character on a terminal: a key copied from a report
|
|
38
|
+
* could not be pasted back into `--redo keys:`. Every message printed here
|
|
39
|
+
* shows it as "␄" (U+2404), which `--redo keys:` / `--force-keys` accept
|
|
40
|
+
* back (lib/locale-layout.js keysForNamespace). Structured `data` fields in
|
|
41
|
+
* --json keep the exact key.
|
|
42
|
+
*
|
|
43
|
+
* @param {*} msg
|
|
44
|
+
* @returns {*} msg with U+0004 shown as ␄ (non-strings unchanged)
|
|
45
|
+
*/
|
|
46
|
+
function showKey(msg) {
|
|
47
|
+
return typeof msg === 'string' ? msg.replace(/\u0004/g, '\u2404') : msg;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* LINE-ATOMIC OUTPUT. Locales are translated in parallel, and a progress
|
|
52
|
+
* bar used to be a partial line (`\r…` with no newline) that any other
|
|
53
|
+
* locale's message was appended to mid-line ("Translating 3 key(s) to
|
|
54
|
+
* French (local)...[INFO] ru/django.po — 3 missing"). Now:
|
|
55
|
+
* - every message is one complete line;
|
|
56
|
+
* - a progress bar names its file, and is redrawn in place only on a
|
|
57
|
+
* terminal and only while nothing else has printed since — any other
|
|
58
|
+
* output first ends the open bar line, so nothing is lost or overwritten;
|
|
59
|
+
* - off a terminal (CI logs, pipes) every bar update is its own line.
|
|
60
|
+
*
|
|
61
|
+
* `openBar` is the item whose bar line is currently unterminated.
|
|
62
|
+
*/
|
|
63
|
+
let openBar = null;
|
|
64
|
+
|
|
65
|
+
function endOpenBar() {
|
|
66
|
+
if (openBar !== null) {
|
|
67
|
+
process.stdout.write('\n');
|
|
68
|
+
openBar = null;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
34
72
|
/**
|
|
35
73
|
* Set the output mode. Call once during CLI bootstrap.
|
|
36
74
|
*
|
|
@@ -64,10 +102,11 @@ function getMode() {
|
|
|
64
102
|
function info(msg, data) {
|
|
65
103
|
if (mode === 'quiet') return;
|
|
66
104
|
if (mode === 'json') {
|
|
67
|
-
console.log(JSON.stringify({ level: 'info', message: msg, ...data }));
|
|
105
|
+
console.log(JSON.stringify({ level: 'info', message: showKey(msg), ...data }));
|
|
68
106
|
return;
|
|
69
107
|
}
|
|
70
|
-
|
|
108
|
+
endOpenBar();
|
|
109
|
+
console.log(`[INFO] ${showKey(msg)}`);
|
|
71
110
|
}
|
|
72
111
|
|
|
73
112
|
/**
|
|
@@ -80,10 +119,11 @@ function info(msg, data) {
|
|
|
80
119
|
function ok(msg, data) {
|
|
81
120
|
if (mode === 'quiet') return;
|
|
82
121
|
if (mode === 'json') {
|
|
83
|
-
console.log(JSON.stringify({ level: 'ok', message: msg, ...data }));
|
|
122
|
+
console.log(JSON.stringify({ level: 'ok', message: showKey(msg), ...data }));
|
|
84
123
|
return;
|
|
85
124
|
}
|
|
86
|
-
|
|
125
|
+
endOpenBar();
|
|
126
|
+
console.log(`[OK] ${showKey(msg)}`);
|
|
87
127
|
}
|
|
88
128
|
|
|
89
129
|
/**
|
|
@@ -95,10 +135,11 @@ function ok(msg, data) {
|
|
|
95
135
|
*/
|
|
96
136
|
function warn(msg, data) {
|
|
97
137
|
if (mode === 'json') {
|
|
98
|
-
console.error(JSON.stringify({ level: 'warn', message: msg, ...data }));
|
|
138
|
+
console.error(JSON.stringify({ level: 'warn', message: showKey(msg), ...data }));
|
|
99
139
|
return;
|
|
100
140
|
}
|
|
101
|
-
|
|
141
|
+
endOpenBar();
|
|
142
|
+
console.error(`[WARN] ${showKey(msg)}`);
|
|
102
143
|
}
|
|
103
144
|
|
|
104
145
|
/**
|
|
@@ -110,21 +151,48 @@ function warn(msg, data) {
|
|
|
110
151
|
*/
|
|
111
152
|
function error(msg, data) {
|
|
112
153
|
if (mode === 'json') {
|
|
113
|
-
console.error(JSON.stringify({ level: 'error', message: msg, ...data }));
|
|
154
|
+
console.error(JSON.stringify({ level: 'error', message: showKey(msg), ...data }));
|
|
114
155
|
return;
|
|
115
156
|
}
|
|
116
|
-
|
|
157
|
+
endOpenBar();
|
|
158
|
+
console.error(`[ERR] ${showKey(msg)}`);
|
|
117
159
|
}
|
|
118
160
|
|
|
119
161
|
/**
|
|
120
|
-
* Progress message —
|
|
162
|
+
* Progress message — status for long-running operations, one complete line
|
|
163
|
+
* (a trailing newline is added; see LINE-ATOMIC OUTPUT above). Callers name
|
|
164
|
+
* what the line is about: with locales running in parallel, a bare
|
|
165
|
+
* " [OK]" could belong to any of them.
|
|
121
166
|
* Suppressed in quiet and json modes.
|
|
122
167
|
*
|
|
123
168
|
* @param {string} msg - Progress description
|
|
124
169
|
*/
|
|
125
170
|
function progress(msg) {
|
|
126
171
|
if (mode === 'quiet' || mode === 'json') return;
|
|
127
|
-
|
|
172
|
+
const line = showKey(String(msg)).replace(/\n+$/, '');
|
|
173
|
+
if (line.trim() === '') return;
|
|
174
|
+
endOpenBar();
|
|
175
|
+
process.stdout.write(`${line}\n`);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Finish an item's progress: " [OK]" after its own bar when that bar is the
|
|
180
|
+
* line still open on a terminal, else a complete line naming the item.
|
|
181
|
+
* Suppressed in quiet and json modes.
|
|
182
|
+
*
|
|
183
|
+
* @param {string} item - What finished (the file name the bar showed)
|
|
184
|
+
* @param {string} status - e.g. "[OK]", "[ERR] all translations failed quality gate"
|
|
185
|
+
*/
|
|
186
|
+
function progressDone(item, status) {
|
|
187
|
+
if (mode === 'quiet' || mode === 'json') return;
|
|
188
|
+
const label = showKey(String(item));
|
|
189
|
+
if (openBar === label) {
|
|
190
|
+
process.stdout.write(` ${status}\n`);
|
|
191
|
+
openBar = null;
|
|
192
|
+
return;
|
|
193
|
+
}
|
|
194
|
+
endOpenBar();
|
|
195
|
+
process.stdout.write(` ${label} ${status}\n`);
|
|
128
196
|
}
|
|
129
197
|
|
|
130
198
|
/**
|
|
@@ -135,7 +203,24 @@ function progress(msg) {
|
|
|
135
203
|
*/
|
|
136
204
|
function debug(msg, data) {
|
|
137
205
|
if (mode !== 'verbose') return;
|
|
138
|
-
|
|
206
|
+
endOpenBar();
|
|
207
|
+
console.log(`[DEBUG] ${showKey(msg)}`);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* Event — one machine-readable progress record (json mode only).
|
|
212
|
+
* Emits `{level:'event', event, ...fields}` on stdout; silent otherwise —
|
|
213
|
+
* the human output already says the same thing in prose.
|
|
214
|
+
*
|
|
215
|
+
* Event types used by sync: 'cost' (the pre-run estimate, before the
|
|
216
|
+
* --max-cost gate) and 'file' (one per content file × locale processed).
|
|
217
|
+
*
|
|
218
|
+
* @param {string} event - Event type
|
|
219
|
+
* @param {object} [fields] - Structured data
|
|
220
|
+
*/
|
|
221
|
+
function event(eventType, fields) {
|
|
222
|
+
if (mode !== 'json') return;
|
|
223
|
+
console.log(JSON.stringify({ level: 'event', event: eventType, ...fields }));
|
|
139
224
|
}
|
|
140
225
|
|
|
141
226
|
/**
|
|
@@ -153,6 +238,7 @@ function summary(data) {
|
|
|
153
238
|
return;
|
|
154
239
|
}
|
|
155
240
|
if (data.title) {
|
|
241
|
+
endOpenBar();
|
|
156
242
|
console.log(`\n${data.title}`);
|
|
157
243
|
}
|
|
158
244
|
}
|
|
@@ -165,14 +251,18 @@ function summary(data) {
|
|
|
165
251
|
*/
|
|
166
252
|
function banner(version) {
|
|
167
253
|
if (mode === 'quiet' || mode === 'json') return;
|
|
254
|
+
endOpenBar();
|
|
168
255
|
console.log(`champollion v${version}\n`);
|
|
169
256
|
}
|
|
170
257
|
|
|
171
258
|
/**
|
|
172
|
-
* Progress bar —
|
|
259
|
+
* Progress bar — key-count progress for one item (a locale file).
|
|
173
260
|
*
|
|
174
|
-
* Renders a
|
|
175
|
-
*
|
|
261
|
+
* Renders a line like: ` fr.json ████████░░░░░░░░ 1,440/2,847 keys`
|
|
262
|
+
* On a terminal the line is redrawn in place (\r) while it is still the
|
|
263
|
+
* last thing printed and belongs to the same item; otherwise (another
|
|
264
|
+
* locale's bar, any other message, or output that is not a terminal) each
|
|
265
|
+
* update is its own complete line — see LINE-ATOMIC OUTPUT.
|
|
176
266
|
* Call with `done = true` to finalize the line with a newline.
|
|
177
267
|
*
|
|
178
268
|
* Suppressed in quiet and json modes.
|
|
@@ -180,12 +270,14 @@ function banner(version) {
|
|
|
180
270
|
* @param {number} completed - Number of items completed
|
|
181
271
|
* @param {number} total - Total number of items
|
|
182
272
|
* @param {object} [opts] - Options
|
|
273
|
+
* @param {string} [opts.item=''] - What the bar is for (e.g. the locale file)
|
|
183
274
|
* @param {string} [opts.label='keys'] - Unit label
|
|
184
275
|
* @param {boolean} [opts.done=false] - Whether this is the final update (appends newline)
|
|
185
276
|
*/
|
|
186
277
|
function progressBar(completed, total, opts = {}) {
|
|
187
278
|
if (mode === 'quiet' || mode === 'json') return;
|
|
188
279
|
const { label = 'keys', done = false } = opts;
|
|
280
|
+
const item = showKey(String(opts.item || ''));
|
|
189
281
|
|
|
190
282
|
// Calculate bar width based on terminal width, with sane defaults
|
|
191
283
|
const termWidth = process.stdout.columns || 80;
|
|
@@ -195,12 +287,34 @@ function progressBar(completed, total, opts = {}) {
|
|
|
195
287
|
const filled = Math.round(ratio * barWidth);
|
|
196
288
|
const bar = '█'.repeat(filled) + '░'.repeat(barWidth - filled);
|
|
197
289
|
const nums = `${completed.toLocaleString()}/${total.toLocaleString()} ${label}`;
|
|
290
|
+
const line = ` ${item ? `${item} ` : ''}${bar} ${nums}`;
|
|
198
291
|
|
|
199
|
-
if (
|
|
200
|
-
|
|
292
|
+
if (!process.stdout.isTTY) {
|
|
293
|
+
endOpenBar();
|
|
294
|
+
process.stdout.write(`${line}\n`);
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
if (openBar === item) {
|
|
298
|
+
process.stdout.write(`\r${line}`);
|
|
201
299
|
} else {
|
|
202
|
-
|
|
300
|
+
endOpenBar();
|
|
301
|
+
process.stdout.write(line);
|
|
203
302
|
}
|
|
303
|
+
openBar = item;
|
|
304
|
+
if (done) endOpenBar();
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* A warning block: pre-formatted lines on stderr (the quality-gate and
|
|
309
|
+
* glossary reports), each a complete line with keys shown copyably.
|
|
310
|
+
* Emitted in every human mode, like warn(); callers emit structured
|
|
311
|
+
* records themselves in json mode.
|
|
312
|
+
*
|
|
313
|
+
* @param {string[]} lines
|
|
314
|
+
*/
|
|
315
|
+
function block(lines) {
|
|
316
|
+
endOpenBar();
|
|
317
|
+
for (const line of lines) console.error(showKey(line));
|
|
204
318
|
}
|
|
205
319
|
|
|
206
320
|
/**
|
|
@@ -217,7 +331,8 @@ function progressBar(completed, total, opts = {}) {
|
|
|
217
331
|
*/
|
|
218
332
|
function raw(msg) {
|
|
219
333
|
if (mode === 'quiet' || mode === 'json') return;
|
|
220
|
-
|
|
334
|
+
endOpenBar();
|
|
335
|
+
console.log(showKey(msg));
|
|
221
336
|
}
|
|
222
337
|
|
|
223
338
|
const output = {
|
|
@@ -228,11 +343,37 @@ const output = {
|
|
|
228
343
|
warn,
|
|
229
344
|
error,
|
|
230
345
|
progress,
|
|
346
|
+
progressDone,
|
|
231
347
|
debug,
|
|
232
348
|
summary,
|
|
349
|
+
event,
|
|
233
350
|
banner,
|
|
234
351
|
progressBar,
|
|
235
352
|
raw,
|
|
353
|
+
block,
|
|
236
354
|
};
|
|
237
355
|
|
|
238
|
-
|
|
356
|
+
/**
|
|
357
|
+
* A stored ISO timestamp as this machine's local date and time, with the
|
|
358
|
+
* zone named: "2026-10-03 18:20 MDT". The caches store UTC (`…Z`); printing
|
|
359
|
+
* the UTC date bare read as a day ahead of local time in the evening
|
|
360
|
+
* (Round 5, Next.js persona: `tm stats` "Created"). Zones without an
|
|
361
|
+
* abbreviation print as an offset ("GMT+5:30"). An unparsable value is
|
|
362
|
+
* returned unchanged; a missing one as "unknown".
|
|
363
|
+
*
|
|
364
|
+
* @param {string|null|undefined} iso
|
|
365
|
+
* @param {{ timeZone?: string }} [opts] - For tests; defaults to the machine's zone
|
|
366
|
+
* @returns {string}
|
|
367
|
+
*/
|
|
368
|
+
function localTimestamp(iso, opts = {}) {
|
|
369
|
+
if (!iso) return 'unknown';
|
|
370
|
+
const d = new Date(iso);
|
|
371
|
+
if (Number.isNaN(d.getTime())) return String(iso);
|
|
372
|
+
const parts = Object.fromEntries(new Intl.DateTimeFormat('en-US', {
|
|
373
|
+
year: 'numeric', month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit',
|
|
374
|
+
hourCycle: 'h23', timeZoneName: 'short', ...(opts.timeZone && { timeZone: opts.timeZone }),
|
|
375
|
+
}).formatToParts(d).map(p => [p.type, p.value]));
|
|
376
|
+
return `${parts.year}-${parts.month}-${parts.day} ${parts.hour}:${parts.minute} ${parts.timeZoneName}`;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
export { output, showKey, localTimestamp };
|