champollion 0.3.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -26
- package/bin/cli.js +53 -5
- package/index.js +63 -2
- package/lib/api-key.js +17 -4
- package/lib/autofix.js +83 -36
- package/lib/bridge/method_bridge.py +15 -3
- package/lib/cards/reader.js +34 -0
- package/lib/cards/remote.js +15 -0
- package/lib/cards/search-names.js +178 -0
- package/lib/command-help.js +286 -85
- package/lib/commands/audit.js +10 -3
- package/lib/commands/card.js +583 -226
- package/lib/commands/doctor.js +54 -18
- package/lib/commands/help.js +37 -32
- package/lib/commands/init.js +1689 -87
- package/lib/commands/integrity.js +127 -40
- package/lib/commands/leaderboard.js +187 -67
- package/lib/commands/models.js +9 -2
- package/lib/commands/provenance.js +7 -2
- package/lib/commands/recommend.js +43 -14
- package/lib/commands/register-corpus.js +632 -125
- package/lib/commands/seal-corpus.js +1 -1
- package/lib/commands/status.js +564 -27
- package/lib/commands/submit.js +17 -12
- package/lib/commands/sync.js +31 -7
- package/lib/commands/tm.js +15 -9
- package/lib/commands/verify.js +27 -3
- package/lib/commands/wrap.js +63 -5
- package/lib/commands/xliff.js +135 -64
- package/lib/commercial-eligibility.js +1 -1
- package/lib/config.js +196 -14
- package/lib/content-estimate.js +96 -0
- package/lib/content-refusals.js +270 -0
- package/lib/content-review.js +372 -0
- package/lib/content-sync.js +1127 -344
- package/lib/content.js +94 -7
- package/lib/corpus-registration.mjs +194 -35
- package/lib/cost-label.js +29 -0
- package/lib/cost-report.js +726 -78
- package/lib/diff.js +38 -4
- package/lib/docusaurus-sync.js +965 -253
- package/lib/edit-distance.js +31 -0
- package/lib/fallback.js +964 -0
- package/lib/file-scope.js +106 -0
- package/lib/flatten.js +80 -3
- package/lib/flutter-locales.js +124 -0
- package/lib/format.js +266 -12
- package/lib/hash.js +146 -21
- package/lib/icu-structure.js +929 -0
- package/lib/integrity.js +223 -75
- package/lib/language-pair.js +157 -0
- package/lib/lint.js +78 -16
- package/lib/local-only-marks.js +106 -0
- package/lib/locale-layout.js +1103 -0
- package/lib/locale-state.js +571 -0
- package/lib/methods/anthropic.js +5 -0
- package/lib/methods/apertium.js +6 -3
- package/lib/methods/api.js +138 -25
- package/lib/methods/base.js +17 -0
- package/lib/methods/coaching-data.js +153 -0
- package/lib/methods/content-separator.js +43 -0
- package/lib/methods/deepl.js +1 -1
- package/lib/methods/direct-llm.js +252 -103
- package/lib/methods/external.js +146 -63
- package/lib/methods/gemini.js +1 -0
- package/lib/methods/google-translate.js +1 -0
- package/lib/methods/http-utils.js +41 -0
- package/lib/methods/libretranslate.js +7 -2
- package/lib/methods/llm-coached.js +68 -128
- package/lib/methods/llm.js +80 -31
- package/lib/methods/local.js +93 -10
- package/lib/methods/microsoft-translator.js +1 -2
- package/lib/methods/openai.js +4 -2
- package/lib/methods/openrouter-client.js +20 -19
- package/lib/methods/openrouter-pricing.js +150 -13
- package/lib/methods/prompt-methods.js +20 -0
- package/lib/methods/provider-pricing.js +42 -1
- package/lib/methods/request-capture.js +104 -0
- package/lib/methods/tilde.js +1 -1
- package/lib/methods/translated.js +1 -2
- package/lib/missing-key.js +93 -0
- package/lib/models.js +11 -0
- package/lib/name-rules.js +32 -0
- package/lib/named-keys.js +172 -0
- package/lib/no-translate.js +4 -3
- package/lib/output.js +160 -19
- package/lib/pairs.js +586 -30
- package/lib/placeholders.js +394 -0
- package/lib/plugins.js +8 -0
- package/lib/plural-gap-redo.js +109 -0
- package/lib/plurals.js +323 -0
- package/lib/po.js +1187 -0
- package/lib/public-catalogue.js +74 -0
- package/lib/recommend.js +527 -32
- package/lib/redo.js +95 -0
- package/lib/refusal-category.js +44 -0
- package/lib/registers.js +255 -11
- package/lib/repair-script.js +20 -13
- package/lib/scripts.js +6 -1
- package/lib/seal.mjs +4 -3
- package/lib/sealed-qualifier.mjs +1 -1
- package/lib/segment.js +2 -1
- package/lib/seo.js +19 -9
- package/lib/serve.js +43 -6
- package/lib/shared-output-seed.js +164 -0
- package/lib/source-contexts.js +39 -0
- package/lib/submit.mjs +57 -5
- package/lib/sync.js +2923 -474
- package/lib/terminology.js +13 -4
- package/lib/tm-evict.js +179 -0
- package/lib/tm-seed.js +5 -2
- package/lib/tm.js +818 -36
- package/lib/translate-pair.js +639 -34
- package/lib/translate.js +78 -5
- package/lib/types.js +22 -3
- package/lib/validate.js +880 -17
- package/lib/verify.js +1296 -104
- package/lib/watch.js +32 -13
- package/lib/xliff.js +44 -3
- package/package.json +1 -1
- package/shared/CORPORA-CARDS.md +2 -0
- package/shared/cards-fallback.json +1 -1
- package/shared/curated-orthography-conventions.json +26 -8
- package/shared/gettext-plural-forms.json +45 -0
- package/shared/method-registry.json +2 -0
- package/shared/metric-registry.json +96 -18
- package/shared/schemas/champollion-plugin.schema.json +4 -0
- package/shared/schemas/corpora-card.schema.json +8 -2
- package/shared/schemas/method-index-record.schema.json +67 -0
- package/shared/schemas/method-registry.schema.json +4 -0
- package/shared/schemas/metric-registry.schema.json +55 -1
- package/shared/docent/corpus.json +0 -11739
|
@@ -0,0 +1,571 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* locale-state.js — what sync knows about each target locale's values, kept
|
|
3
|
+
* in .champollion.lock under "locales" (lib/hash.js has the file format).
|
|
4
|
+
*
|
|
5
|
+
* THREE GAPS THIS CLOSES (Round 4 synthetic developers, 2026-10-03):
|
|
6
|
+
*
|
|
7
|
+
* 1. A redo that could not finish was forgotten. `sync --redo all
|
|
8
|
+
* --fresh-on-model-change` refused 3 keys and said they "will be retried
|
|
9
|
+
* on the next sync"; the next sync did 0 keys (the keys exist on disk
|
|
10
|
+
* and their source is unchanged), so the model switch never completed
|
|
11
|
+
* and `status` said nothing. Now those keys are recorded as PENDING and
|
|
12
|
+
* the next plain sync asks for them again — from the model, not the
|
|
13
|
+
* cache (that was the point of the redo).
|
|
14
|
+
*
|
|
15
|
+
* 2. A hand-fixed translation was overwritten by `--redo all` (the very
|
|
16
|
+
* command the model-change notice recommends). The Markdown lane keeps
|
|
17
|
+
* a reviewer's edits (lib/content-review.js); key-value files now do the
|
|
18
|
+
* same: sync records a hash of each value it WRITES, and a value that no
|
|
19
|
+
* longer matches was changed by a person. Bulk redos keep it and say so;
|
|
20
|
+
* a redo that names the key replaces it; a source change replaces it too
|
|
21
|
+
* (the edit was for the old text) but prints the edited wording and
|
|
22
|
+
* appends it to .champollion-replaced-edits.jsonl, so it is never lost.
|
|
23
|
+
*
|
|
24
|
+
* 3. Refused keys were re-billed on every sync. The quality gate refused a
|
|
25
|
+
* key; nothing remembered it; the next sync sent it to the same paid
|
|
26
|
+
* model again, and again. Now a refusal is remembered per (key, source
|
|
27
|
+
* text, method key) and a plain sync holds the key back.
|
|
28
|
+
*
|
|
29
|
+
* And the record answers a question nothing could before: is this
|
|
30
|
+
* translation out of date? A written-record whose source hash differs from
|
|
31
|
+
* the current source text is a STALE translation (status, audit and verify
|
|
32
|
+
* report it).
|
|
33
|
+
*
|
|
34
|
+
* PRECEDENCE — when are queued keys actually sent to the model?
|
|
35
|
+
* 1. Named by `--redo keys:` / `--force-keys`, or queued by `--redo all`,
|
|
36
|
+
* or anything under `--fresh`: always sent (an explicit redo).
|
|
37
|
+
* 2. PENDING (left by a redo that could not finish): the next plain sync
|
|
38
|
+
* asks for it once more, from the model, even when the cache holds an
|
|
39
|
+
* older model's text. That retry is the one the redo promised.
|
|
40
|
+
* 3. REFUSED (the gate refused this method's translation of the key's
|
|
41
|
+
* current source text): held back — not sent again until a redo names
|
|
42
|
+
* it, the source text changes, or the method/model changes (a fallback
|
|
43
|
+
* method that has not refused it still gets it). A pending key whose
|
|
44
|
+
* retry is refused again joins this group: it stays pending (status
|
|
45
|
+
* shows it) but is not re-sent on every sync.
|
|
46
|
+
* The cache is always consulted first (free); holding back only stops a
|
|
47
|
+
* paid call.
|
|
48
|
+
*
|
|
49
|
+
* HAND EDITS — what a sync does with a value that is not what it wrote:
|
|
50
|
+
* - plain sync, source unchanged: never touched (as before);
|
|
51
|
+
* - `--redo all` / `--force`, a pending retry: KEPT, and the run says how
|
|
52
|
+
* many were kept and how to replace one (`--redo keys:<k>`);
|
|
53
|
+
* - `--redo keys:<k>` naming it: replaced (asked for by name) — the edited
|
|
54
|
+
* wording is printed and recorded first;
|
|
55
|
+
* - its SOURCE changed: replaced (the edit was for the old text) — printed
|
|
56
|
+
* and recorded in .champollion-replaced-edits.jsonl.
|
|
57
|
+
* A value with no written-record (written before this version, by hand, or
|
|
58
|
+
* by another tool) counts as Champollion's only when the cache holds that
|
|
59
|
+
* exact text for the key; otherwise it is treated as a person's (kept).
|
|
60
|
+
*/
|
|
61
|
+
|
|
62
|
+
import crypto from 'node:crypto';
|
|
63
|
+
import fs from 'node:fs';
|
|
64
|
+
import path from 'node:path';
|
|
65
|
+
import { hashValue } from './hash.js';
|
|
66
|
+
import { tmTranslationsOf, tmTranslationSet, tmMethodKey } from './tm.js';
|
|
67
|
+
import { tmProofTextsFor } from './tm-evict.js';
|
|
68
|
+
|
|
69
|
+
/** Where replaced hand edits are recorded: the project root, tracked in git. */
|
|
70
|
+
export const REPLACED_EDITS_FILENAME = '.champollion-replaced-edits.jsonl';
|
|
71
|
+
|
|
72
|
+
/** 12 hex of the SHA-256 the source manifest stores for a source value. */
|
|
73
|
+
export function shortSourceHash(sourceValue) {
|
|
74
|
+
return hashValue(sourceValue).slice(0, 12);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** 12 hex of a value as it is in the file. */
|
|
78
|
+
export function valueHash(value) {
|
|
79
|
+
const input = typeof value === 'string' ? value : JSON.stringify(value);
|
|
80
|
+
return crypto.createHash('sha256').update(input, 'utf-8').digest('hex').slice(0, 12);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** "<source hash>:<value hash>" — one written-record. */
|
|
84
|
+
export function encodeWritten(sourceValue, value) {
|
|
85
|
+
return `${shortSourceHash(sourceValue)}:${valueHash(value)}`;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** @returns {{ source: string, value: string }|null} */
|
|
89
|
+
export function decodeWritten(enc) {
|
|
90
|
+
if (typeof enc !== 'string') return null;
|
|
91
|
+
const m = /^([0-9a-f]{12}):([0-9a-f]{12})$/.exec(enc);
|
|
92
|
+
return m ? { source: m[1], value: m[2] } : null;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* "<plural form>:<value hash>" — one forms-record: a borrowed i18next plural
|
|
97
|
+
* form (French `count_many`, translated from the English `_other` text) that
|
|
98
|
+
* the model was ASKED FOR as that form, and the fingerprint of the value its
|
|
99
|
+
* answer left in the file. Committed in the lock, so whether "the model wrote
|
|
100
|
+
* `_many` like `_other`" or "`_many` was filled from `_other`" is decided from
|
|
101
|
+
* committed files alone — never from a machine's own cache (Round 7, i18next
|
|
102
|
+
* persona: one commit passed `verify --strict` on one machine and failed on
|
|
103
|
+
* another).
|
|
104
|
+
*/
|
|
105
|
+
export function encodeForm(form, value) {
|
|
106
|
+
return `${form}:${valueHash(value)}`;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** @returns {{ form: string, value: string }|null} */
|
|
110
|
+
export function decodeForm(enc) {
|
|
111
|
+
if (typeof enc !== 'string') return null;
|
|
112
|
+
const m = /^([a-z-]+):([0-9a-f]{12})$/.exec(enc);
|
|
113
|
+
return m ? { form: m[1], value: m[2] } : null;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* The per-locale part of the lock, mutable for one run.
|
|
118
|
+
*
|
|
119
|
+
* @example
|
|
120
|
+
* const state = new LockState(readLock(cwd).locales);
|
|
121
|
+
* state.of('fr').written['nav.home'] = encodeWritten(src, value);
|
|
122
|
+
* writeManifest(cwd, manifest, state.toJSON());
|
|
123
|
+
*/
|
|
124
|
+
export class LockState {
|
|
125
|
+
/** @param {object} [locales] - The lock's "locales" object */
|
|
126
|
+
constructor(locales = {}) {
|
|
127
|
+
this.locales = {};
|
|
128
|
+
for (const [code, entry] of Object.entries(locales || {})) {
|
|
129
|
+
if (!entry || typeof entry !== 'object') continue;
|
|
130
|
+
// `by` is stored grouped ({ "<method key>": [keys…] }, compact and
|
|
131
|
+
// diff-friendly) and held here per key.
|
|
132
|
+
const by = {};
|
|
133
|
+
for (const [methodKey, keys] of Object.entries(entry.by || {})) {
|
|
134
|
+
if (Array.isArray(keys)) for (const k of keys) if (typeof k === 'string') by[k] = methodKey;
|
|
135
|
+
}
|
|
136
|
+
this.locales[code] = {
|
|
137
|
+
written: { ...(entry.written || {}) },
|
|
138
|
+
pending: { ...(entry.pending || {}) },
|
|
139
|
+
refused: { ...(entry.refused || {}) },
|
|
140
|
+
forms: { ...(entry.forms || {}) },
|
|
141
|
+
// Plural messages answered without a form the language uses, per
|
|
142
|
+
// key: { source, methods } (lib/plural-gap-redo.js).
|
|
143
|
+
gaps: { ...(entry.gaps || {}) },
|
|
144
|
+
by,
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** The state of one locale (created empty when absent). */
|
|
150
|
+
of(code) {
|
|
151
|
+
if (!this.locales[code]) this.locales[code] = { written: {}, pending: {}, refused: {}, forms: {}, gaps: {}, by: {} };
|
|
152
|
+
if (!this.locales[code].forms) this.locales[code].forms = {};
|
|
153
|
+
if (!this.locales[code].gaps) this.locales[code].gaps = {};
|
|
154
|
+
return this.locales[code];
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** Read-only view (never creates an entry). */
|
|
158
|
+
peek(code) {
|
|
159
|
+
return this.locales[code] || { written: {}, pending: {}, refused: {}, forms: {}, gaps: {}, by: {} };
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Drop what no longer applies: locales the config no longer has, and keys
|
|
164
|
+
* a locale's files no longer expect.
|
|
165
|
+
*
|
|
166
|
+
* @param {Map<string, Set<string>|null>} expectedByLocale - locale → its lock
|
|
167
|
+
* keys; null = a configured locale this run did not process (kept as is).
|
|
168
|
+
* A locale not in the map is no longer configured and is dropped.
|
|
169
|
+
*/
|
|
170
|
+
prune(expectedByLocale) {
|
|
171
|
+
for (const code of Object.keys(this.locales)) {
|
|
172
|
+
if (!expectedByLocale.has(code)) { delete this.locales[code]; continue; }
|
|
173
|
+
const keep = expectedByLocale.get(code);
|
|
174
|
+
if (keep === null) continue;
|
|
175
|
+
for (const part of ['written', 'pending', 'refused', 'forms', 'gaps', 'by']) {
|
|
176
|
+
for (const k of Object.keys(this.locales[code][part] || {})) {
|
|
177
|
+
if (!keep.has(k)) delete this.locales[code][part][k];
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/** The lock's "locales" object: `by` grouped by method key, keys sorted. */
|
|
184
|
+
toJSON() {
|
|
185
|
+
const out = {};
|
|
186
|
+
for (const [code, entry] of Object.entries(this.locales)) {
|
|
187
|
+
const grouped = {};
|
|
188
|
+
for (const [k, methodKey] of Object.entries(entry.by || {})) {
|
|
189
|
+
if (typeof methodKey !== 'string') continue;
|
|
190
|
+
(grouped[methodKey] ||= []).push(k);
|
|
191
|
+
}
|
|
192
|
+
for (const keys of Object.values(grouped)) keys.sort();
|
|
193
|
+
out[code] = {
|
|
194
|
+
written: entry.written, pending: entry.pending, refused: entry.refused, forms: entry.forms || {},
|
|
195
|
+
// Only when there is one: a lock without plural gaps stays as it was.
|
|
196
|
+
...(entry.gaps && Object.keys(entry.gaps).length > 0 && { gaps: entry.gaps }),
|
|
197
|
+
by: grouped,
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
return out;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* Who wrote each value on disk, for one locale.
|
|
206
|
+
*
|
|
207
|
+
* @param {object} args
|
|
208
|
+
* @param {object|null} args.tm - Loaded TM (the bootstrap proof); null = none
|
|
209
|
+
* @param {string} args.locale
|
|
210
|
+
* @param {object} args.written - LockState.of(locale).written
|
|
211
|
+
* @returns {(lockKey: string, key: string, value: *, sourceValue: string, sourceChanged?: boolean) =>
|
|
212
|
+
* 'machine'|'edited'|'unknown'}
|
|
213
|
+
* machine — what sync wrote (matches its record), or text the cache holds
|
|
214
|
+
* for the key (pipeline output — e.g. an older translation
|
|
215
|
+
* restored from git);
|
|
216
|
+
* edited — differs from what sync wrote and is no pipeline output: a
|
|
217
|
+
* person changed it;
|
|
218
|
+
* unknown — no record and no cache proof (treated as a person's).
|
|
219
|
+
*/
|
|
220
|
+
export function createEditClassifier({ tm, locale, written, expansion = null }) {
|
|
221
|
+
let everyTranslation = null;
|
|
222
|
+
return (lockKey, key, value, sourceValue, sourceChanged = false) => {
|
|
223
|
+
if (typeof value !== 'string') return 'machine';
|
|
224
|
+
const record = decodeWritten(written[lockKey]);
|
|
225
|
+
if (record && record.value === valueHash(value)) return 'machine';
|
|
226
|
+
// Not what sync last wrote — but text the pipeline produced (an older
|
|
227
|
+
// machine translation restored from git, damage an older version wrote)
|
|
228
|
+
// is still machine text, never a person's.
|
|
229
|
+
if (tm && typeof sourceValue === 'string') {
|
|
230
|
+
// A borrowed plural form is proven by its own entry or — written by an
|
|
231
|
+
// earlier version — by the entry it shared (lib/tm-evict.js).
|
|
232
|
+
for (const text of tmProofTextsFor(key, sourceValue, expansion)) {
|
|
233
|
+
if (tmTranslationsOf(tm, text, locale).has(value)) return 'machine';
|
|
234
|
+
}
|
|
235
|
+
// The source changed since: the old text is known only by its hash, so
|
|
236
|
+
// the proof left is that the cache holds this exact translation text.
|
|
237
|
+
if (sourceChanged) {
|
|
238
|
+
if (!everyTranslation) everyTranslation = tmTranslationSet(tm, locale);
|
|
239
|
+
if (everyTranslation.has(value)) return 'machine';
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
return record ? 'edited' : 'unknown';
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* Does a refusal record hold this key back from `methodKey`?
|
|
248
|
+
*
|
|
249
|
+
* @param {object|undefined} record - LockState.of(code).refused[lockKey]
|
|
250
|
+
* @param {string} sourceValue - Current source text of the key
|
|
251
|
+
* @param {string} methodKey - tmMethodKey of the method about to be asked
|
|
252
|
+
*/
|
|
253
|
+
export function refusedBy(record, sourceValue, methodKey) {
|
|
254
|
+
return !!record && typeof record === 'object' && record.source === shortSourceHash(sourceValue)
|
|
255
|
+
&& Array.isArray(record.methods) && record.methods.includes(methodKey);
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* What a refusal record lets a run do with one unit (a key, a Markdown block,
|
|
260
|
+
* a front-matter field, a whole page) of this source text: 'send' (ask the
|
|
261
|
+
* pair's method), 'fallback-only' (the pair's method refused it; its fallback
|
|
262
|
+
* has not), 'held' (not sent at all). THE rule, shared by every lane: the
|
|
263
|
+
* key-value lane (planQueue), the Docusaurus UI strings, and the content
|
|
264
|
+
* lanes (lib/content-refusals.js contentHolds).
|
|
265
|
+
*
|
|
266
|
+
* @param {object|undefined} record - { source, methods } (a refusal record)
|
|
267
|
+
* @param {string} sourceValue - The unit's current source text
|
|
268
|
+
* @param {object} pairConfig - The pair about to be asked (its `fallback` optional)
|
|
269
|
+
* @returns {'send'|'fallback-only'|'held'}
|
|
270
|
+
*/
|
|
271
|
+
export function holdState(record, sourceValue, pairConfig) {
|
|
272
|
+
if (typeof sourceValue !== 'string' || !refusedBy(record, sourceValue, tmMethodKey(pairConfig))) return 'send';
|
|
273
|
+
if (pairConfig.fallback && !refusedBy(record, sourceValue, tmMethodKey(pairConfig.fallback))) return 'fallback-only';
|
|
274
|
+
return 'held';
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* Remember that the gate refused `methods`' answers for one key's current
|
|
279
|
+
* source text (merged with what an earlier run recorded for the same text).
|
|
280
|
+
*
|
|
281
|
+
* @param {object} localeState - LockState.of(code)
|
|
282
|
+
* @param {string} lockKey
|
|
283
|
+
* @param {string} sourceValue
|
|
284
|
+
* @param {string[]} methods - Method keys whose answer was refused this run
|
|
285
|
+
* @param {{ redo?: boolean }} [opts] - redo: refused under an explicit redo
|
|
286
|
+
* that leaves the key pending (the next plain sync gets one more try)
|
|
287
|
+
*/
|
|
288
|
+
export function recordRefusal(localeState, lockKey, sourceValue, methods, { redo = false } = {}) {
|
|
289
|
+
if (!methods || methods.length === 0 || typeof sourceValue !== 'string') return;
|
|
290
|
+
const prior = localeState.refused[lockKey];
|
|
291
|
+
const src = shortSourceHash(sourceValue);
|
|
292
|
+
const known = prior && prior.source === src && Array.isArray(prior.methods) ? prior.methods : [];
|
|
293
|
+
localeState.refused[lockKey] = {
|
|
294
|
+
source: src,
|
|
295
|
+
methods: [...new Set([...known, ...methods])],
|
|
296
|
+
on: new Date().toISOString().slice(0, 10),
|
|
297
|
+
...(redo && { redo: true }),
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/** "llm (model x/y)" — the method and model a message names. */
|
|
302
|
+
export function describeModel(pairConfig) {
|
|
303
|
+
const model = pairConfig.method === 'api'
|
|
304
|
+
? (pairConfig.endpoint || pairConfig.methodPlugin || null)
|
|
305
|
+
: pairConfig.model;
|
|
306
|
+
return `${pairConfig.method}${model ? ` (${pairConfig.method === 'api' ? 'endpoint' : 'model'} ${model})` : ''}`;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
const sampleOf = (keys, n = 3) => `${keys.slice(0, n).join(', ')}${keys.length > n ? `, +${keys.length - n} more` : ''}`;
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* The warning for one file's keys held back — every key lane says it the
|
|
313
|
+
* same way (the key-value files, Docusaurus UI strings).
|
|
314
|
+
*
|
|
315
|
+
* @param {object} p
|
|
316
|
+
* @param {string} p.filename - The file as the run names it
|
|
317
|
+
* @param {string[]} p.keys - Held keys, as a person names them
|
|
318
|
+
* @param {object} p.pairConfig
|
|
319
|
+
* @param {string} p.command - The `--redo keys:` command that asks again
|
|
320
|
+
* @returns {string}
|
|
321
|
+
*/
|
|
322
|
+
export function describeHeldKeys({ filename, keys, pairConfig, command }) {
|
|
323
|
+
return `${filename} — ${keys.length} key(s) held back (${sampleOf(keys)}): the quality gate refused `
|
|
324
|
+
+ `${describeModel(pairConfig)}'s translation of their current text on an earlier sync${pairConfig.fallback ? `, and the fallback's (${pairConfig.fallback.method})` : ''}, `
|
|
325
|
+
+ 'so they are not sent again (nothing billed; the cache is still read). '
|
|
326
|
+
+ `Ask again: \`${command}\`; or fill them another way — ${pairConfig.fallback ? 'another' : 'a'} "fallback" method on the pair, `
|
|
327
|
+
+ '"noTranslate" for text that stays as written, or write them in the file by hand.';
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/** The note for keys only the pair's fallback is asked for. */
|
|
331
|
+
export function describeFallbackOnlyKeys({ filename, keys, pairConfig }) {
|
|
332
|
+
return `${filename} — ${keys.length} key(s) go to the fallback (${pairConfig.fallback.method}) only: `
|
|
333
|
+
+ `${pairConfig.method} had its translation refused before (${sampleOf(keys)}).`;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/**
|
|
337
|
+
* What the next sync does with a key this run could not translate, said with
|
|
338
|
+
* the key: 'retry' (no usable answer — asked again), 'pending-retry' (an
|
|
339
|
+
* explicit redo could not finish — asked once more), 'held' (refused by the
|
|
340
|
+
* gate — held back).
|
|
341
|
+
*/
|
|
342
|
+
export function keyFateNote(fate, pairConfig) {
|
|
343
|
+
return {
|
|
344
|
+
retry: 'not translated (no usable answer) — the next sync asks again',
|
|
345
|
+
'pending-retry': 'not translated — recorded as pending in .champollion.lock; the next sync asks the model once more',
|
|
346
|
+
held: `refused by the quality gate — held back from now on (the next sync will not re-send it to ${pairConfig.method}; see the summary)`,
|
|
347
|
+
}[fate];
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
/**
|
|
351
|
+
* The end-of-run lines for keys held back from the next sync, with the
|
|
352
|
+
* command per pair that asks again.
|
|
353
|
+
*
|
|
354
|
+
* @param {Array<{ pair: string, key: string }>} heldNext
|
|
355
|
+
* @param {(keys: string[], pair: string) => string} commandFor - The `--redo keys:` command
|
|
356
|
+
* @returns {string[]}
|
|
357
|
+
*/
|
|
358
|
+
export function describeHeldNext(heldNext, commandFor) {
|
|
359
|
+
if (heldNext.length === 0) return [];
|
|
360
|
+
const lines = [` ${heldNext.length} key(s) are held back: the quality gate refused the method's translation of their current text, `
|
|
361
|
+
+ 'so the next sync will not send them to the same method again (it would bill the same answer). To ask again, name them:'];
|
|
362
|
+
const byPair = new Map();
|
|
363
|
+
for (const { pair, key } of heldNext) {
|
|
364
|
+
if (!byPair.has(pair)) byPair.set(pair, []);
|
|
365
|
+
byPair.get(pair).push(key);
|
|
366
|
+
}
|
|
367
|
+
for (const [pair, keys] of byPair) lines.push(` \`${commandFor(keys.slice(0, 8), pair)}\`${keys.length > 8 ? ` (+${keys.length - 8} more)` : ''}`);
|
|
368
|
+
lines.push(' Or fill them another way: a "fallback" method on the pair (it is asked for keys the pair\'s own method refused), '
|
|
369
|
+
+ '"noTranslate" for text that stays as written, or write them in the file by hand.');
|
|
370
|
+
return lines;
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/**
|
|
374
|
+
* Decide what one target file's queued keys become. Shared by the sync and
|
|
375
|
+
* the pre-run cost estimate, so the estimate prices exactly what the run
|
|
376
|
+
* sends. Pure.
|
|
377
|
+
*
|
|
378
|
+
* @param {object} p
|
|
379
|
+
* @param {import('./types.js').DiffResult} p.diff - diffLocale result, computed
|
|
380
|
+
* with the locale's pending keys among the forced ones
|
|
381
|
+
* @param {object} p.sourceFlat - Expected source (target key space)
|
|
382
|
+
* @param {object} p.targetFlat - Values on disk
|
|
383
|
+
* @param {(key: string) => string} p.lockKeyOf - Target key → lock key
|
|
384
|
+
* @param {object} p.localeState - LockState.of(code) (or .peek)
|
|
385
|
+
* @param {Set<string>} p.named - Target keys named by --redo keys: / --force-keys
|
|
386
|
+
* @param {boolean} p.bulk - --redo all / --force
|
|
387
|
+
* @param {Set<string>} p.pending - Target keys pending from an unfinished redo
|
|
388
|
+
* @param {boolean} p.fresh - --fresh / --no-tm (nothing is held back)
|
|
389
|
+
* @param {object} p.pairConfig
|
|
390
|
+
* @param {ReturnType<typeof createEditClassifier>} p.classify
|
|
391
|
+
* @param {string} [p.fallbackPrefix]
|
|
392
|
+
* @param {Set<string>|null} [p.redoGaps] - Target keys `--redo gaps` queued (lib/plural-gap-redo.js)
|
|
393
|
+
* @returns {{ toProcess: string[], kept: string[], keptUnrecorded: string[], replacing: Array<{ key: string, value: string,
|
|
394
|
+
* why: 'source-changed'|'named', unrecorded: boolean }>, held: string[], heldFromPrimary: string[],
|
|
395
|
+
* pendingRetry: string[], forcedByRedo: Set<string> }}
|
|
396
|
+
* toProcess — what goes into the pipeline (the cache is consulted for all of it);
|
|
397
|
+
* kept — hand-edited values a bulk redo / pending retry leaves alone
|
|
398
|
+
* (keptUnrecorded: the ones with no written-record and no cache proof);
|
|
399
|
+
* replacing — hand-edited values this run replaces (printed and recorded);
|
|
400
|
+
* held — keys the pipeline must not send to the pair's method or its fallback;
|
|
401
|
+
* heldFromPrimary — keys only the fallback may be asked for;
|
|
402
|
+
* pendingRetry — pending keys retried now (sent to the model);
|
|
403
|
+
* pendingAll — every pending key (the cache is bypassed for all of them);
|
|
404
|
+
* forcedByRedo — keys this run queued by an explicit redo (pending if they fail)
|
|
405
|
+
*/
|
|
406
|
+
export function planQueue({
|
|
407
|
+
diff, sourceFlat, targetFlat, lockKeyOf, localeState, named, bulk, pending, fresh, pairConfig, classify,
|
|
408
|
+
fallbackPrefix = '[EN] ', redoGaps = null,
|
|
409
|
+
}) {
|
|
410
|
+
const changed = new Set(diff.changed);
|
|
411
|
+
const missing = new Set(diff.missing);
|
|
412
|
+
const fallback = new Set(diff.needsTranslation);
|
|
413
|
+
const echo = new Set(diff.untranslated);
|
|
414
|
+
const forced = new Set(diff.forced);
|
|
415
|
+
|
|
416
|
+
const out = {
|
|
417
|
+
toProcess: [], kept: [], keptUnrecorded: [], replacing: [], held: [], heldFromPrimary: [], pendingRetry: [],
|
|
418
|
+
pendingAll: [], forcedByRedo: new Set(),
|
|
419
|
+
};
|
|
420
|
+
|
|
421
|
+
for (const key of diff.toProcess) {
|
|
422
|
+
const src = sourceFlat[key];
|
|
423
|
+
const onDisk = targetFlat[key];
|
|
424
|
+
const isNamed = named.has(key) && forced.has(key);
|
|
425
|
+
// `--redo gaps` queues the plural messages left incomplete as a bulk
|
|
426
|
+
// redo does: sent (never held back), a person's edit kept.
|
|
427
|
+
const isBulk = (bulk || !!redoGaps?.has(key)) && forced.has(key);
|
|
428
|
+
const isPending = !isNamed && !isBulk && pending.has(key);
|
|
429
|
+
if (isNamed || isBulk) out.forcedByRedo.add(key);
|
|
430
|
+
|
|
431
|
+
// ── A value a person wrote ────────────────────────────────────────
|
|
432
|
+
const present = typeof onDisk === 'string' && onDisk.trim() !== '' && !missing.has(key)
|
|
433
|
+
&& !fallback.has(key) && !onDisk.startsWith(fallbackPrefix);
|
|
434
|
+
if (present && typeof src === 'string') {
|
|
435
|
+
const who = classify(lockKeyOf(key), key, onDisk, src, changed.has(key));
|
|
436
|
+
if (who !== 'machine') {
|
|
437
|
+
const onlyEcho = echo.has(key) && !changed.has(key) && !forced.has(key) && !pending.has(key);
|
|
438
|
+
if (changed.has(key)) {
|
|
439
|
+
out.replacing.push({ key, value: onDisk, why: 'source-changed', unrecorded: who === 'unknown' });
|
|
440
|
+
} else if (isNamed) {
|
|
441
|
+
out.replacing.push({ key, value: onDisk, why: 'named', unrecorded: who === 'unknown' });
|
|
442
|
+
} else if (onlyEcho && who === 'unknown') {
|
|
443
|
+
// An unstamped copy of the source with no record: the pre-populated
|
|
444
|
+
// English this reason exists for (docusaurus write-translations,
|
|
445
|
+
// a copied file). Translated, as before.
|
|
446
|
+
} else {
|
|
447
|
+
// A bulk redo, a pending retry — or a person who set the value
|
|
448
|
+
// equal to the source on purpose: theirs to keep.
|
|
449
|
+
out.kept.push(key);
|
|
450
|
+
if (who === 'unknown') out.keptUnrecorded.push(key);
|
|
451
|
+
continue;
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
// ── Refused before: hold back from the method that refused it ────
|
|
457
|
+
let heldAll = false;
|
|
458
|
+
if (!fresh && !isNamed && !isBulk && typeof src === 'string') {
|
|
459
|
+
const record = localeState.refused[lockKeyOf(key)];
|
|
460
|
+
const hold = holdState(record, src, pairConfig);
|
|
461
|
+
const promisedRetry = isPending && record?.redo === true;
|
|
462
|
+
if (hold !== 'send' && !promisedRetry) {
|
|
463
|
+
if (hold === 'fallback-only') out.heldFromPrimary.push(key);
|
|
464
|
+
else { out.held.push(key); heldAll = true; }
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
if (isPending && !heldAll) out.pendingRetry.push(key);
|
|
469
|
+
// Every pending key bypasses the cache, held back or not: the redo's
|
|
470
|
+
// point was the current model's text, and model carry-over would serve
|
|
471
|
+
// the old model's (silently "completing" the redo with the text it
|
|
472
|
+
// set out to replace).
|
|
473
|
+
if (isPending) out.pendingAll.push(key);
|
|
474
|
+
out.toProcess.push(key);
|
|
475
|
+
}
|
|
476
|
+
return out;
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
/**
|
|
480
|
+
* Append replaced hand edits to the project's record (tracked in git, next
|
|
481
|
+
* to the lock — the .champollion/ cache folder is per machine and ignored).
|
|
482
|
+
*
|
|
483
|
+
* @param {string} cwd
|
|
484
|
+
* @param {Array<object>} entries - { locale, file, key, editedValue, why, newSource }
|
|
485
|
+
* @returns {string|null} The file's name, or null when nothing was written
|
|
486
|
+
*/
|
|
487
|
+
export function recordReplacedEdits(cwd, entries) {
|
|
488
|
+
if (!entries || entries.length === 0) return null;
|
|
489
|
+
const at = new Date().toISOString();
|
|
490
|
+
const lines = entries.map(e => JSON.stringify({ at, ...e })).join('\n') + '\n';
|
|
491
|
+
fs.appendFileSync(path.join(cwd, REPLACED_EDITS_FILENAME), lines, 'utf-8');
|
|
492
|
+
return REPLACED_EDITS_FILENAME;
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
/**
|
|
496
|
+
* How many replaced hand edits the project's record holds (status).
|
|
497
|
+
*
|
|
498
|
+
* @param {string} cwd
|
|
499
|
+
* @returns {number}
|
|
500
|
+
*/
|
|
501
|
+
export function countReplacedEdits(cwd) {
|
|
502
|
+
try {
|
|
503
|
+
return fs.readFileSync(path.join(cwd, REPLACED_EDITS_FILENAME), 'utf-8').split('\n').filter(l => l.trim()).length;
|
|
504
|
+
} catch {
|
|
505
|
+
return 0;
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
/**
|
|
510
|
+
* Per-locale health from the lock and the files: what `status`, `audit` and
|
|
511
|
+
* `verify` report.
|
|
512
|
+
*
|
|
513
|
+
* @param {object} p
|
|
514
|
+
* @param {object} p.layout - lib/locale-layout.js layout
|
|
515
|
+
* @param {Array} p.units - loadSourceUnits(layout)
|
|
516
|
+
* @param {string} p.inputLocale
|
|
517
|
+
* @param {string} p.code - Target locale
|
|
518
|
+
* @param {object} p.localeState - LockState.peek(code)
|
|
519
|
+
* @param {object} p.manifest - The lock's source map
|
|
520
|
+
* @param {object|null} p.tm - Loaded TM (proof for values with no record)
|
|
521
|
+
* @param {object|null} p.pairConfig - The locale's pair (held-back check); null skips it
|
|
522
|
+
* @param {object} p.helpers - { expectedForTarget, readLocaleFlat, lockKey, originKey, fallbackPrefix }
|
|
523
|
+
* @returns {{ stale: string[], pending: Array<{ key: string, reason: string, held: boolean }>, held: string[] }}
|
|
524
|
+
*/
|
|
525
|
+
export function localeHealth({ layout, units, inputLocale, code, localeState, manifest, tm, pairConfig, helpers }) {
|
|
526
|
+
const { expectedForTarget, readLocaleFlat, lockKey, originKey, fallbackPrefix = '[EN] ' } = helpers;
|
|
527
|
+
const stale = [];
|
|
528
|
+
const held = [];
|
|
529
|
+
const pendingOut = [];
|
|
530
|
+
const isHeld = (record, src) => !!pairConfig && holdState(record, src, pairConfig) === 'held';
|
|
531
|
+
|
|
532
|
+
for (const unit of units) {
|
|
533
|
+
let file;
|
|
534
|
+
try { file = layout.fileFor(code, unit.ns); } catch { continue; }
|
|
535
|
+
const { flat: expected, expansion } = expectedForTarget(unit, inputLocale, code);
|
|
536
|
+
let target = {};
|
|
537
|
+
if (file && fs.existsSync(file.path)) {
|
|
538
|
+
try { target = readLocaleFlat(file) || {}; } catch { target = {}; }
|
|
539
|
+
}
|
|
540
|
+
for (const [key, src] of Object.entries(expected)) {
|
|
541
|
+
if (typeof src !== 'string') continue;
|
|
542
|
+
const lk = lockKey(layout, unit.ns, key);
|
|
543
|
+
const value = target[key];
|
|
544
|
+
const present = typeof value === 'string' && value.trim() !== '' && !value.startsWith(fallbackPrefix);
|
|
545
|
+
if (present) {
|
|
546
|
+
const record = decodeWritten(localeState.written[lk]);
|
|
547
|
+
if (record) {
|
|
548
|
+
if (record.source !== shortSourceHash(src)) stale.push(lk);
|
|
549
|
+
} else {
|
|
550
|
+
// No record (older lock): the source manifest — the hash of the
|
|
551
|
+
// source the last successful sync translated — is the evidence,
|
|
552
|
+
// unless the cache proves the value translates the current text.
|
|
553
|
+
const srcKey = lockKey(layout, unit.ns, originKey(key, expansion));
|
|
554
|
+
const sourceNow = unit.flat[originKey(key, expansion)];
|
|
555
|
+
const old = manifest[srcKey];
|
|
556
|
+
if (old && typeof sourceNow === 'string' && old !== hashValue(sourceNow)
|
|
557
|
+
&& !(tm && tmProofTextsFor(key, src, expansion).some(t => tmTranslationsOf(tm, t, code).has(value)))) {
|
|
558
|
+
stale.push(lk);
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
const refusal = localeState.refused[lk];
|
|
563
|
+
const keyHeld = isHeld(refusal, src);
|
|
564
|
+
if (keyHeld && !localeState.pending[lk]) held.push(lk);
|
|
565
|
+
if (localeState.pending[lk]) {
|
|
566
|
+
pendingOut.push({ key: lk, reason: String(localeState.pending[lk]), held: keyHeld && refusal.redo !== true });
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
return { stale, pending: pendingOut, held };
|
|
571
|
+
}
|
package/lib/methods/anthropic.js
CHANGED
|
@@ -35,6 +35,11 @@ class AnthropicMethod extends DirectLLMMethod {
|
|
|
35
35
|
_getApiKeyOptionsKey() { return 'anthropicApiKey'; }
|
|
36
36
|
_getDefaultModel() { return DEFAULT_MODEL; }
|
|
37
37
|
_getProviderLabel() { return 'Anthropic'; }
|
|
38
|
+
_getModelVendor() { return 'anthropic'; }
|
|
39
|
+
|
|
40
|
+
// OpenRouter's "anthropic/claude-haiku-4.5" is the API's "claude-haiku-4-5"
|
|
41
|
+
// (the dotted id is a 404 — verified live by the harness, 2026-09-27).
|
|
42
|
+
_nativeModelName(name) { return name.replace(/(?<=\d)\.(?=\d)/g, '-'); }
|
|
38
43
|
|
|
39
44
|
// ── API request/response shape ───────────────────────────────────
|
|
40
45
|
|
package/lib/methods/apertium.js
CHANGED
|
@@ -31,11 +31,11 @@ const APERTIUM_TIMEOUT_MS = 15000;
|
|
|
31
31
|
class ApertiumMethod extends TranslationMethod {
|
|
32
32
|
constructor(options = {}) {
|
|
33
33
|
super('apertium', options);
|
|
34
|
+
this.translatesRawText = true; // see base.js
|
|
34
35
|
}
|
|
35
36
|
|
|
36
37
|
_resolveEndpoint(options = {}) {
|
|
37
38
|
let ep = options.apertiumApiUrl
|
|
38
|
-
|| getEnvOrFileVar('APERTIUM_API_URL')
|
|
39
39
|
|| getEnvOrFileVar('APERTIUM_API_URL', options.cwd)
|
|
40
40
|
|| APERTIUM_DEFAULT_BASE;
|
|
41
41
|
ep = ep.replace(/\/+$/, '');
|
|
@@ -45,7 +45,6 @@ class ApertiumMethod extends TranslationMethod {
|
|
|
45
45
|
|
|
46
46
|
_resolveApiKey(options = {}) {
|
|
47
47
|
return options.apertiumApiKey
|
|
48
|
-
|| getEnvOrFileVar('APERTIUM_API_KEY')
|
|
49
48
|
|| getEnvOrFileVar('APERTIUM_API_KEY', options.cwd);
|
|
50
49
|
}
|
|
51
50
|
|
|
@@ -110,12 +109,16 @@ class ApertiumMethod extends TranslationMethod {
|
|
|
110
109
|
});
|
|
111
110
|
clearTimeout(timeoutId);
|
|
112
111
|
if (!res.ok) {
|
|
113
|
-
|
|
112
|
+
// A server that is down or failing: sync decides after its plan
|
|
113
|
+
// whether this run needs it (lib/sync.js deferred probes).
|
|
114
|
+
return { ready: false, unreachable: true, endpoint: base, reason: `Apertium API at ${base} responded ${res.status}.` };
|
|
114
115
|
}
|
|
115
116
|
return { ready: true };
|
|
116
117
|
} catch (err) {
|
|
117
118
|
return {
|
|
118
119
|
ready: false,
|
|
120
|
+
unreachable: true,
|
|
121
|
+
endpoint: base,
|
|
119
122
|
reason:
|
|
120
123
|
`Cannot reach Apertium API at ${base}: ` +
|
|
121
124
|
`${err.name === 'AbortError' ? 'timeout' : err.message}. ` +
|