champollion 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/lib/autofix.js +7 -9
- package/lib/commands/init.js +20 -14
- package/lib/commands/wrap.js +2 -2
- package/lib/config.js +35 -4
- package/lib/content-estimate.js +5 -5
- package/lib/content-refusals.js +27 -13
- package/lib/content-sync.js +59 -51
- package/lib/cost-report.js +5 -5
- package/lib/docusaurus-sync.js +42 -41
- package/lib/fallback.js +157 -19
- package/lib/locale-state.js +4 -0
- package/lib/methods/direct-llm.js +13 -8
- package/lib/methods/gemini.js +1 -1
- package/lib/methods/llm-coached.js +3 -1
- package/lib/methods/llm.js +7 -2
- package/lib/methods/openai.js +3 -1
- package/lib/methods/provider-pricing.js +4 -0
- package/lib/models.js +91 -55
- package/lib/pairs.js +8 -8
- package/lib/segment.js +22 -7
- package/lib/validate.js +118 -6
- package/lib/verify.js +38 -2
- package/package.json +1 -1
- package/shared/licenses.json +99 -3
- package/shared/retired-model-aliases.json +9 -0
- package/shared/schemas/human-services.schema.json +1 -1
- package/shared/schemas/retired-model-aliases.schema.json +32 -0
- package/shared/model-aliases.json +0 -7
- package/shared/schemas/model-aliases.schema.json +0 -27
package/lib/methods/openai.js
CHANGED
|
@@ -19,7 +19,9 @@ import { estimateLlmCost } from './provider-pricing.js';
|
|
|
19
19
|
|
|
20
20
|
// The default model, named ONCE — it was previously written twice (here and
|
|
21
21
|
// as an inline fallback in estimateCost()) and the two could drift.
|
|
22
|
-
|
|
22
|
+
// A dated snapshot: plain 'gpt-4o' was an alias OpenAI repoints (founder
|
|
23
|
+
// ruling 2026-10-05: exact model ids only, nothing floating).
|
|
24
|
+
const DEFAULT_MODEL = 'gpt-5.4-mini-2026-03-17';
|
|
23
25
|
|
|
24
26
|
class OpenAIMethod extends DirectLLMMethod {
|
|
25
27
|
constructor(options = {}) {
|
|
@@ -126,6 +126,9 @@ export const LLM_RATES = {
|
|
|
126
126
|
// twice the real input rate. Corrected here.
|
|
127
127
|
'gpt-4o': { provider: 'openai', input: 2.50, output: 10.00, verified: '2026-08-01' },
|
|
128
128
|
'gpt-4o-mini': { provider: 'openai', input: 0.15, output: 0.60, verified: '2026-08-01' },
|
|
129
|
+
// The openai default (a dated snapshot, which OpenRouter lists only
|
|
130
|
+
// undated as openai/gpt-5.4-mini — so this row is what prices it).
|
|
131
|
+
'gpt-5.4-mini-2026-03-17': { provider: 'openai', input: 0.75, output: 4.50, verified: '2026-10-06' },
|
|
129
132
|
|
|
130
133
|
// ── Google Gemini ──────────────────────────────────────────────────
|
|
131
134
|
// LAST VERIFIED: 2026-08-01 against the live OpenRouter draw. The values
|
|
@@ -135,6 +138,7 @@ export const LLM_RATES = {
|
|
|
135
138
|
// should have failed. Corrected here.
|
|
136
139
|
'gemini-2.5-flash': { provider: 'gemini', input: 0.30, output: 2.50, verified: '2026-08-01' },
|
|
137
140
|
'gemini-2.5-pro': { provider: 'gemini', input: 1.25, output: 10.00, verified: '2026-08-01' },
|
|
141
|
+
'gemini-3.8-flash': { provider: 'gemini', input: 0.75, output: 3.75, verified: '2026-10-06' },
|
|
138
142
|
};
|
|
139
143
|
|
|
140
144
|
/**
|
package/lib/models.js
CHANGED
|
@@ -24,77 +24,109 @@ import fs from 'node:fs';
|
|
|
24
24
|
import path from 'node:path';
|
|
25
25
|
import { fileURLToPath } from 'node:url';
|
|
26
26
|
import { getEnvOrFileVar } from './api-key.js';
|
|
27
|
+
// A cycle (config.js imports this module) that is safe: the default is read
|
|
28
|
+
// only inside requireExactModelId, at call time, never while modules load.
|
|
29
|
+
import { DEFAULT_OPENROUTER_MODEL } from './config.js';
|
|
27
30
|
|
|
28
31
|
// Per-process cache: provider name → model ID array (or null if fetch failed)
|
|
29
32
|
const _modelCache = new Map();
|
|
30
33
|
|
|
31
|
-
// Lazy-loaded
|
|
32
|
-
let
|
|
34
|
+
// Lazy-loaded retired-name table (loaded once from shared/retired-model-aliases.json)
|
|
35
|
+
let _retiredCache = null;
|
|
33
36
|
|
|
34
37
|
/**
|
|
35
|
-
*
|
|
38
|
+
* Where to look up the models a name could mean — said in every refusal.
|
|
39
|
+
* `champollion models` lists a direct provider's own names; OpenRouter's
|
|
40
|
+
* catalogue is its public model list.
|
|
41
|
+
*/
|
|
42
|
+
const MODEL_LIST_HINT = 'List models: https://openrouter.ai/models (OpenRouter slugs), '
|
|
43
|
+
+ 'or champollion models --method <gemini|openai|anthropic> (a direct provider\'s own names).';
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Load the RETIRED short model names from shared/retired-model-aliases.json.
|
|
36
47
|
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
* so
|
|
48
|
+
* Founder ruling 2026-10-05: "slugs should be specific, NOT ALIASES — for all
|
|
49
|
+
* models, all slugs, no aliasing." These names no longer resolve to anything.
|
|
50
|
+
* The table exists only so a refusal can say which exact slug the old name
|
|
51
|
+
* used to stand for — it is never used to map a name to a model.
|
|
40
52
|
*
|
|
41
|
-
*
|
|
53
|
+
* Prefers the package-bundled copy (cli/shared/, shipped via sync:shared),
|
|
54
|
+
* then the monorepo-root SSOT for in-repo dev. A missing or unreadable table
|
|
55
|
+
* leaves the refusal of floating ids intact and only loses the "used to stand
|
|
56
|
+
* for" hint for a retired name — which then fails as an unknown model at the
|
|
57
|
+
* provider, still never resolved.
|
|
58
|
+
*
|
|
59
|
+
* @returns {Object<string, string>} Retired short name → the slug it stood for
|
|
42
60
|
*/
|
|
43
|
-
function
|
|
44
|
-
if (
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
]
|
|
56
|
-
|
|
57
|
-
for (const p of aliasPaths) {
|
|
58
|
-
try { raw = fs.readFileSync(p, 'utf-8'); break; } catch { /* try next */ }
|
|
59
|
-
}
|
|
60
|
-
const parsed = raw ? JSON.parse(raw) : {};
|
|
61
|
-
// Strip metadata keys (e.g., _comment) — only keep actual aliases
|
|
62
|
-
_aliasCache = {};
|
|
63
|
-
for (const [key, value] of Object.entries(parsed)) {
|
|
64
|
-
if (!key.startsWith('_') && typeof value === 'string') {
|
|
65
|
-
_aliasCache[key] = value;
|
|
66
|
-
}
|
|
61
|
+
function _loadRetiredAliases() {
|
|
62
|
+
if (_retiredCache) return _retiredCache;
|
|
63
|
+
_retiredCache = Object.create(null);
|
|
64
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
65
|
+
const candidates = [
|
|
66
|
+
path.resolve(__dirname, '..', 'shared', 'retired-model-aliases.json'),
|
|
67
|
+
path.resolve(__dirname, '..', '..', 'shared', 'retired-model-aliases.json'),
|
|
68
|
+
];
|
|
69
|
+
for (const p of candidates) {
|
|
70
|
+
let raw;
|
|
71
|
+
try { raw = fs.readFileSync(p, 'utf-8'); } catch { continue; }
|
|
72
|
+
const retired = JSON.parse(raw).retired || {};
|
|
73
|
+
for (const [key, value] of Object.entries(retired)) {
|
|
74
|
+
if (typeof value === 'string') _retiredCache[key] = value;
|
|
67
75
|
}
|
|
68
|
-
|
|
69
|
-
_aliasCache = {};
|
|
76
|
+
break;
|
|
70
77
|
}
|
|
78
|
+
return _retiredCache;
|
|
79
|
+
}
|
|
71
80
|
|
|
72
|
-
|
|
81
|
+
/**
|
|
82
|
+
* True for a floating id — one that names whatever model a provider points it
|
|
83
|
+
* at today, so a run could not say which model translated: OpenRouter's
|
|
84
|
+
* "~vendor/…" router ids and any "…-latest" / "…:latest" name.
|
|
85
|
+
*
|
|
86
|
+
* @param {string} id
|
|
87
|
+
* @returns {boolean}
|
|
88
|
+
*/
|
|
89
|
+
function isFloatingModelId(id) {
|
|
90
|
+
return typeof id === 'string' && (id.trim().startsWith('~') || /[-:]latest$/i.test(id.trim()));
|
|
73
91
|
}
|
|
74
92
|
|
|
75
93
|
/**
|
|
76
|
-
*
|
|
94
|
+
* Check that a configured model is an EXACT model slug, and return it unchanged.
|
|
77
95
|
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
96
|
+
* Founder ruling 2026-10-05: every model is named by its exact provider slug
|
|
97
|
+
* ("google/gemini-3.5-flash", "anthropic/claude-sonnet-4.6", or a direct
|
|
98
|
+
* provider's own exact name, "gpt-5.5"). Nothing resolves a short name:
|
|
99
|
+
* - a retired alias ("gemini-flash", "gpt", …) is REFUSED, naming the exact
|
|
100
|
+
* slug it used to stand for;
|
|
101
|
+
* - a floating id ("~google/gemini-flash-latest", "…-latest") is REFUSED.
|
|
102
|
+
* Anything else passes through as written — the provider judges whether the
|
|
103
|
+
* model exists (direct providers also check it against their model list).
|
|
82
104
|
*
|
|
83
|
-
* @param {string}
|
|
84
|
-
* @
|
|
105
|
+
* @param {string|null|undefined} id - The model as written
|
|
106
|
+
* @param {{ from?: string|null }} [where] - Where it was set ("from --model")
|
|
107
|
+
* @returns {string|null|undefined} The same id
|
|
108
|
+
* @throws {Error} code CHAMPOLLION_MODEL_ID
|
|
85
109
|
*/
|
|
86
|
-
function
|
|
87
|
-
if (!
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
if (
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
110
|
+
function requireExactModelId(id, { from = null } = {}) {
|
|
111
|
+
if (!id || typeof id !== 'string') return id;
|
|
112
|
+
const at = from ? ` (${from})` : '';
|
|
113
|
+
const retired = _loadRetiredAliases();
|
|
114
|
+
if (Object.prototype.hasOwnProperty.call(retired, id)) {
|
|
115
|
+
const err = new Error(
|
|
116
|
+
`"${id}"${at} is not a model id — Champollion takes exact model slugs only, no aliases. `
|
|
117
|
+
+ `Did you mean ${retired[id]} (what "${id}" used to stand for)? ${MODEL_LIST_HINT}`);
|
|
118
|
+
err.code = 'CHAMPOLLION_MODEL_ID';
|
|
119
|
+
throw err;
|
|
120
|
+
}
|
|
121
|
+
if (isFloatingModelId(id)) {
|
|
122
|
+
const err = new Error(
|
|
123
|
+
`"${id}"${at} is a floating model id — it names whatever model the provider points it at today, `
|
|
124
|
+
+ 'so a run could not say which model translated. Champollion takes exact model slugs only: '
|
|
125
|
+
+ `name the model itself (e.g. "${DEFAULT_OPENROUTER_MODEL}", the CLI default). ${MODEL_LIST_HINT}`);
|
|
126
|
+
err.code = 'CHAMPOLLION_MODEL_ID';
|
|
127
|
+
throw err;
|
|
128
|
+
}
|
|
129
|
+
return id;
|
|
98
130
|
}
|
|
99
131
|
|
|
100
132
|
/**
|
|
@@ -183,7 +215,10 @@ async function fetchAvailableModels(provider, apiKey) {
|
|
|
183
215
|
if (cached !== undefined) return cached;
|
|
184
216
|
|
|
185
217
|
try {
|
|
186
|
-
|
|
218
|
+
// Floating ids ("gemini-flash-latest", "chatgpt-4o-latest") are listed by
|
|
219
|
+
// the providers but refused everywhere (requireExactModelId) — never offer
|
|
220
|
+
// one, so neither `champollion models` nor init's picker suggests it.
|
|
221
|
+
const models = (await config.fetch(apiKey))?.filter(id => !isFloatingModelId(id)) ?? null;
|
|
187
222
|
if (models && models.length > 0) {
|
|
188
223
|
_modelCache.set(provider, models);
|
|
189
224
|
return models;
|
|
@@ -258,7 +293,8 @@ function clearModelCache() {
|
|
|
258
293
|
|
|
259
294
|
export {
|
|
260
295
|
fetchAvailableModels,
|
|
261
|
-
|
|
296
|
+
requireExactModelId,
|
|
297
|
+
isFloatingModelId,
|
|
262
298
|
resolveProviderApiKey,
|
|
263
299
|
getProviderLabel,
|
|
264
300
|
getProviderEnvVar,
|
package/lib/pairs.js
CHANGED
|
@@ -22,7 +22,7 @@ import { resolveTargetScript, validateScriptFallback, converterKeyForLocale } fr
|
|
|
22
22
|
import { getMethod, METHOD_REGISTRY } from './translate.js';
|
|
23
23
|
import { COACHED_PROVIDERS, normalizeProvider } from './methods/llm-coached.js';
|
|
24
24
|
import { DEFAULT_OPENROUTER_MODEL, DEFAULT_BATCH_SIZE, DEFAULT_MAX_RETRIES } from './config.js';
|
|
25
|
-
import {
|
|
25
|
+
import { requireExactModelId } from './models.js';
|
|
26
26
|
import { output } from './output.js';
|
|
27
27
|
import { tmMethodKey } from './tm.js';
|
|
28
28
|
import { PLAIN_LLM_METHODS } from './methods/prompt-methods.js';
|
|
@@ -169,10 +169,10 @@ const MODEL_ID_TRANSPORTS = new Set(['openai', 'anthropic', 'gemini', 'local']);
|
|
|
169
169
|
/**
|
|
170
170
|
* The model id a pair's transport is sent, as written in the config or on
|
|
171
171
|
* --model:
|
|
172
|
-
* - OpenRouter (llm, llm-coached by default):
|
|
173
|
-
*
|
|
174
|
-
*
|
|
175
|
-
*
|
|
172
|
+
* - OpenRouter (llm, llm-coached by default): the exact slug as written.
|
|
173
|
+
* A retired alias ("gemini-flash") or a floating id ("~google/…-latest")
|
|
174
|
+
* THROWS, naming the exact slug to write (founder ruling 2026-10-05:
|
|
175
|
+
* exact slugs only, no aliasing — lib/models.js requireExactModelId).
|
|
176
176
|
* - A direct provider (openai, anthropic, gemini — as a method or as the
|
|
177
177
|
* provider of llm-coached): its own name ("openai/gpt-5.5" → "gpt-5.5");
|
|
178
178
|
* an id it has no name for ("google/…" on openai) THROWS, naming a model
|
|
@@ -185,12 +185,12 @@ const MODEL_ID_TRANSPORTS = new Set(['openai', 'anthropic', 'gemini', 'local']);
|
|
|
185
185
|
* @param {string|null} provider - Resolved transport (null for non-routed methods)
|
|
186
186
|
* @param {string|null} [from] - Where the id was set, for a refusal
|
|
187
187
|
* @returns {string|null}
|
|
188
|
-
* @throws {Error} code CHAMPOLLION_MODEL_ROUTE (the caller prefixes the pair)
|
|
188
|
+
* @throws {Error} code CHAMPOLLION_MODEL_ROUTE or CHAMPOLLION_MODEL_ID (the caller prefixes the pair)
|
|
189
189
|
*/
|
|
190
190
|
function modelIdForTransport(model, method, provider = null, from = null) {
|
|
191
191
|
if (!model) return model;
|
|
192
192
|
const transport = PROVIDER_ROUTED_METHODS.has(method) ? (provider || 'openrouter') : method;
|
|
193
|
-
if (transport === 'openrouter') return
|
|
193
|
+
if (transport === 'openrouter') return requireExactModelId(model, { from });
|
|
194
194
|
if (MODEL_ID_TRANSPORTS.has(transport)) return new METHOD_REGISTRY[transport]().resolveModelId(model, { from });
|
|
195
195
|
return model;
|
|
196
196
|
}
|
|
@@ -203,7 +203,7 @@ function resolveModelNamed(where, from, explicitModel, method, globalDefault, pr
|
|
|
203
203
|
try {
|
|
204
204
|
return resolveModelForPair(explicitModel, method, globalDefault, provider, providerModel, from);
|
|
205
205
|
} catch (err) {
|
|
206
|
-
if (err.code === 'CHAMPOLLION_MODEL_ROUTE') err.message = `${where}: ${err.message}`;
|
|
206
|
+
if (err.code === 'CHAMPOLLION_MODEL_ROUTE' || err.code === 'CHAMPOLLION_MODEL_ID') err.message = `${where}: ${err.message}`;
|
|
207
207
|
throw err;
|
|
208
208
|
}
|
|
209
209
|
}
|
package/lib/segment.js
CHANGED
|
@@ -73,7 +73,10 @@ const SEGMENT_MARKER_SUFFIX = '⟧';
|
|
|
73
73
|
function hasTranslatableText(text) {
|
|
74
74
|
const stripped = text
|
|
75
75
|
.replace(PLACEHOLDER_TOKEN_REGEX, '')
|
|
76
|
-
.replace(IMPORT_EXPORT_LINE_REGEX, '')
|
|
76
|
+
.replace(IMPORT_EXPORT_LINE_REGEX, '')
|
|
77
|
+
// An HTML comment is a note to editors, never shown: not sent, not billed
|
|
78
|
+
// (dogfood 2026-10-05 — comment-only blocks were translated and gated).
|
|
79
|
+
.replace(/<!--[\s\S]*?-->/g, '');
|
|
77
80
|
return /[\p{L}\p{N}]/u.test(stripped);
|
|
78
81
|
}
|
|
79
82
|
|
|
@@ -186,10 +189,21 @@ function buildBlockBatchPrompt(blockTexts, langConfig, options = {}) {
|
|
|
186
189
|
.map((text, i) => `${SEGMENT_MARKER_PREFIX}${i}${SEGMENT_MARKER_SUFFIX}\n${text}`)
|
|
187
190
|
.join('\n\n');
|
|
188
191
|
|
|
192
|
+
// A second ask for segments the quality check refused (lib/fallback.js
|
|
193
|
+
// retryRefusedBlocks): the model is told why, and that text which is
|
|
194
|
+
// correct as written may come back unchanged.
|
|
195
|
+
const notes = langConfig.retryNotes instanceof Map
|
|
196
|
+
? blockTexts.map((text, i) => (langConfig.retryNotes.has(text) ? `- Segment ${i}: ${langConfig.retryNotes.get(text)}` : null)).filter(Boolean)
|
|
197
|
+
: [];
|
|
198
|
+
const retryBlock = notes.length > 0
|
|
199
|
+
? `\nYour previous translation of these segments was refused by an automatic quality check, for the reason given:\n${notes.join('\n')}\n`
|
|
200
|
+
+ `Translate each one again. If a segment is correct exactly as written in ${sourceLanguageName} — a name, title, citation, identifier, code or data — return it unchanged.\n`
|
|
201
|
+
: '';
|
|
202
|
+
|
|
189
203
|
return `You are translating Markdown content from ${sourceLanguageName} to ${langConfig.name}. The document was split into ${blockTexts.length} numbered segment(s); segments not shown are already translated.
|
|
190
204
|
${contextBlock}${titleBlock}
|
|
191
205
|
Register/tone: ${langConfig.register}
|
|
192
|
-
|
|
206
|
+
${retryBlock}
|
|
193
207
|
Rules:
|
|
194
208
|
- Translate ALL human-readable text in every segment.
|
|
195
209
|
- Preserve ALL Markdown formatting: headers (#), bold (**), italic (*), links, images, lists, blockquotes, tables, admonitions (:::), etc.${placeholderRule}
|
|
@@ -285,8 +299,8 @@ function parseBlockBatchResponse(response, expectedCount, opts = {}) {
|
|
|
285
299
|
* 1. Send the full batch. Missing marker(s)? →
|
|
286
300
|
* 2. ONE retry with only the missing segments (a fresh, smaller batch —
|
|
287
301
|
* different neighbors, usually enough). Still missing? →
|
|
288
|
-
* 3.
|
|
289
|
-
*
|
|
302
|
+
* 3. The source block stands for just those segments, unmarked — the
|
|
303
|
+
* caller reports it, records it and asks again on the next sync.
|
|
290
304
|
*
|
|
291
305
|
* The CALLER's contract for fallen-back segments: never store them in the
|
|
292
306
|
* TM (an error cached is an error forever) and never advance the file's
|
|
@@ -297,13 +311,12 @@ function parseBlockBatchResponse(response, expectedCount, opts = {}) {
|
|
|
297
311
|
* @param {string[]} p.texts - Protected block texts to translate
|
|
298
312
|
* @param {(texts: string[]) => string} p.buildPrompt - Batch prompt builder
|
|
299
313
|
* @param {(prompt: string) => Promise<string|null>} p.callModel - One API call
|
|
300
|
-
* @param {string} p.fallbackPrefix - e.g. '[EN] ' — prepended to source text
|
|
301
314
|
* @returns {Promise<{blocks: string[], fellBack: number[]}>} index-aligned
|
|
302
315
|
* translations; `fellBack` lists indexes that carry the fallback
|
|
303
316
|
* @throws {Error} If the FIRST call returns nothing at all, or on
|
|
304
317
|
* duplicate/unknown markers (untrustworthy mapping)
|
|
305
318
|
*/
|
|
306
|
-
async function translateBlockBatchResilient({ texts, buildPrompt, callModel
|
|
319
|
+
async function translateBlockBatchResilient({ texts, buildPrompt, callModel }) {
|
|
307
320
|
const first = await callModel(buildPrompt(texts));
|
|
308
321
|
if (!first) {
|
|
309
322
|
throw new Error('block-batch translation returned no results');
|
|
@@ -331,8 +344,10 @@ async function translateBlockBatchResilient({ texts, buildPrompt, callModel, fal
|
|
|
331
344
|
}
|
|
332
345
|
}
|
|
333
346
|
|
|
347
|
+
// Still missing after the retry: the source block stands, unmarked; the
|
|
348
|
+
// caller records it, does not cache it and does not advance the lock.
|
|
334
349
|
for (const i of stillMissing) {
|
|
335
|
-
blocks[i] =
|
|
350
|
+
blocks[i] = texts[i];
|
|
336
351
|
}
|
|
337
352
|
return { blocks, fellBack: stillMissing };
|
|
338
353
|
}
|
package/lib/validate.js
CHANGED
|
@@ -443,6 +443,106 @@ function isProtectedTermValue(value, protectedTerms = []) {
|
|
|
443
443
|
return rest.replace(/\{[^}]*\}/g, '').replace(/[\d\s\p{P}\p{S}]/gu, '').length === 0;
|
|
444
444
|
}
|
|
445
445
|
|
|
446
|
+
/**
|
|
447
|
+
* The gate's version. A refusal is remembered with the version that made it
|
|
448
|
+
* (lib/content-refusals.js, lib/locale-state.js), and a hold made by an
|
|
449
|
+
* earlier gate lifts by itself: what an over-strict gate refused is asked
|
|
450
|
+
* again once the gate is fixed, instead of staying in the source language
|
|
451
|
+
* until someone names it for a redo. Bump it whenever a check is loosened.
|
|
452
|
+
* 2 — 2026-10-05: names with citations/anchors, reference entries, tables,
|
|
453
|
+
* short titles and source-shown fullwidth letters stopped being refused.
|
|
454
|
+
*/
|
|
455
|
+
const GATE_VERSION = 2;
|
|
456
|
+
|
|
457
|
+
/**
|
|
458
|
+
* Refusals that may be the model keeping text that is correct as written —
|
|
459
|
+
* a name, title, citation, identifier or code: the source handed back, or
|
|
460
|
+
* Latin script kept in a non-Latin target. Told why and asked again, a model
|
|
461
|
+
* that gives the same answer twice is taken at its word (lib/fallback.js
|
|
462
|
+
* retryRefusedBlocks), as the key-value lane takes a name it is asked about
|
|
463
|
+
* twice (LATIN_NAME_OR_LABEL). Every other refusal (repetition, length,
|
|
464
|
+
* hollowing, damaged markup, fullwidth disguise) is never accepted this way.
|
|
465
|
+
*
|
|
466
|
+
* @param {string|null} reason
|
|
467
|
+
* @returns {boolean}
|
|
468
|
+
*/
|
|
469
|
+
function isKeepAsWrittenFault(reason) {
|
|
470
|
+
if (typeof reason !== 'string') return false;
|
|
471
|
+
return reason.startsWith('source echo')
|
|
472
|
+
|| reason === LATIN_NAME_OR_LABEL
|
|
473
|
+
|| reason.startsWith('wrong script (ASCII-only');
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
/**
|
|
477
|
+
* How much longer than its source a value may be before the length ratio
|
|
478
|
+
* counts. The ratio is meaningless on a tiny source: "FAQ" → "Preguntas
|
|
479
|
+
* frecuentes" is 6.7x and correct (dogfood 2026-10-05, a docs page title),
|
|
480
|
+
* while "Feast" → a whole sentence still grows by far more than this.
|
|
481
|
+
*/
|
|
482
|
+
const SHORT_SOURCE_LENGTH_SLACK = 20;
|
|
483
|
+
|
|
484
|
+
/**
|
|
485
|
+
* The part of a string that would need translating, for the "short name kept
|
|
486
|
+
* as written" rule: inline code, quoted strings, parentheticals and Markdown
|
|
487
|
+
* block markers removed. "### METEOR (Banerjee & Lavie, 2005)" is the name
|
|
488
|
+
* METEOR, and "## Hugo (TOML / YAML / Markdown)" the name Hugo — both were
|
|
489
|
+
* refused as source echo on the whole string's length (dogfood 2026-10-05).
|
|
490
|
+
*
|
|
491
|
+
* @param {string} text
|
|
492
|
+
* @returns {string}
|
|
493
|
+
*/
|
|
494
|
+
function nameCore(text) {
|
|
495
|
+
return String(text)
|
|
496
|
+
.replace(/^[ \t]*(?:#{1,6}[ \t]+|[-*+][ \t]+|\d{1,9}[.)][ \t]+)/, '')
|
|
497
|
+
.replace(/\{#[^}\s]+\}\s*$/, ' ')
|
|
498
|
+
.replace(/`[^`]*`/g, ' ')
|
|
499
|
+
.replace(/"[^"]*"|“[^”]*”|「[^」]*」/g, ' ')
|
|
500
|
+
.replace(/\([^()]*\)|([^()]*)/g, ' ')
|
|
501
|
+
.replace(/\s+/g, ' ')
|
|
502
|
+
.trim();
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
/**
|
|
506
|
+
* A reference-list entry — numbered, with a year or a link, and a quoted or
|
|
507
|
+
* italic title: "4. Snover, M., … (2006). \"A Study of …\" *Proceedings …*".
|
|
508
|
+
* Cited works are kept as published, so the same text back is correct, not
|
|
509
|
+
* an echo (dogfood 2026-10-05: twelve references of the scoring spec were
|
|
510
|
+
* refused and published with the fallback prefix).
|
|
511
|
+
*
|
|
512
|
+
* @param {string} text
|
|
513
|
+
* @returns {boolean}
|
|
514
|
+
*/
|
|
515
|
+
function isBibliographicEntry(text) {
|
|
516
|
+
const t = String(text).trim();
|
|
517
|
+
// A reference list in one block: every entry must be one.
|
|
518
|
+
if (t.includes('\n')) {
|
|
519
|
+
const lines = t.split('\n').map(l => l.trim()).filter(Boolean);
|
|
520
|
+
return lines.length > 1 && lines.every(l => !l.includes('\n') && isBibliographicEntry(l));
|
|
521
|
+
}
|
|
522
|
+
return /^(?:\[?[A-Z]?\d{1,3}\]?[.)]?|[-*+])\s+\S/.test(t)
|
|
523
|
+
&& (/\(\d{4}[a-z]?\)/.test(t) || /\]\(https?:\/\//.test(t))
|
|
524
|
+
&& (/"[^"]{8,}"|“[^”]{8,}”/.test(t) || /\*[^*]{8,}\*/.test(t));
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
/**
|
|
528
|
+
* A Markdown table measured as its cells' text: delimiter rows dropped and
|
|
529
|
+
* pipes turned to spaces. The delimiter row ("|---|---|") is repetition by
|
|
530
|
+
* construction, so a table whose model output padded it differently was
|
|
531
|
+
* refused as a repetition hallucination (dogfood 2026-10-05, four tables).
|
|
532
|
+
* Anything that is not a table comes back unchanged.
|
|
533
|
+
*
|
|
534
|
+
* @param {string} text
|
|
535
|
+
* @returns {string}
|
|
536
|
+
*/
|
|
537
|
+
function tableProse(text) {
|
|
538
|
+
const lines = String(text).split('\n');
|
|
539
|
+
if (!lines.some((l) => /^\s*\|/.test(l))) return text;
|
|
540
|
+
return lines
|
|
541
|
+
.filter((l) => !/^\s*\|?(?:\s*:?-{3,}:?\s*\|)+\s*:?-{0,}:?\s*$/.test(l))
|
|
542
|
+
.map((l) => (/^\s*\|/.test(l) ? l.replace(/\|/g, ' ').replace(/[ \t]+/g, ' ').trim() : l))
|
|
543
|
+
.join('\n');
|
|
544
|
+
}
|
|
545
|
+
|
|
446
546
|
/**
|
|
447
547
|
* Why the quality gate refuses one Markdown block or front-matter field, or
|
|
448
548
|
* null when it passes — the key-value gate's own checks (empty, source echo,
|
|
@@ -462,6 +562,9 @@ function isProtectedTermValue(value, protectedTerms = []) {
|
|
|
462
562
|
*/
|
|
463
563
|
function contentGateFault(source, value, pairConfig = {}) {
|
|
464
564
|
if (typeof source !== 'string' || typeof value !== 'string') return null;
|
|
565
|
+
if (value === source && isBibliographicEntry(source)) return null;
|
|
566
|
+
source = tableProse(source);
|
|
567
|
+
value = tableProse(value);
|
|
465
568
|
const { failures } = validateTranslations({ block: value }, { block: source }, pairConfig || {},
|
|
466
569
|
{ prose: true, acceptLatinNames: true, acceptPluralGaps: true });
|
|
467
570
|
return failures.length > 0 ? failures[0].reason : null;
|
|
@@ -547,8 +650,10 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
|
|
|
547
650
|
// it is taken at its word as a name (acceptLatinNames) and cached, so
|
|
548
651
|
// it is never re-billed. One bounded retry — never a retry loop.
|
|
549
652
|
if (translated === source) {
|
|
550
|
-
const
|
|
551
|
-
const
|
|
653
|
+
const core = nameCore(source);
|
|
654
|
+
const asciiRatio = core.replace(/[^\x20-\x7E]/g, '').length / Math.max(core.length, 1);
|
|
655
|
+
// Nothing left once code, quotes and parentheticals go: nothing to translate.
|
|
656
|
+
const isShortAscii = core.length === 0 || (core.length <= 30 && asciiRatio > 0.8);
|
|
552
657
|
if (!isShortAscii) {
|
|
553
658
|
failures.push({ key, reason: 'source echo (identical to English)', value: translated });
|
|
554
659
|
continue;
|
|
@@ -682,7 +787,7 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
|
|
|
682
787
|
// Check 4: Length ratio — catches padding and truncation
|
|
683
788
|
if (source.length > 0) {
|
|
684
789
|
const ratio = translated.length / source.length;
|
|
685
|
-
if (ratio > thresholds.maxLengthRatio) {
|
|
790
|
+
if (ratio > thresholds.maxLengthRatio && translated.length - source.length > SHORT_SOURCE_LENGTH_SLACK) {
|
|
686
791
|
failures.push({
|
|
687
792
|
key,
|
|
688
793
|
reason: `length inflation (${ratio.toFixed(1)}x source, max ${thresholds.maxLengthRatio}x)`,
|
|
@@ -719,7 +824,7 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
|
|
|
719
824
|
// Fullwidth Latin letters ("Book an appointment") outside CJK typography:
|
|
720
825
|
// English in disguise, whatever the target's script. Never a name — a
|
|
721
826
|
// name kept as written is in plain letters.
|
|
722
|
-
if (hasForeignFullwidthLatin(translated, targetLocale)) {
|
|
827
|
+
if (hasForeignFullwidthLatin(translated, targetLocale) && !hasForeignFullwidthLatin(source, targetLocale)) {
|
|
723
828
|
failures.push({
|
|
724
829
|
key,
|
|
725
830
|
reason: `wrong script (fullwidth Latin letters in a ${targetLocale} value — English in disguise, not a translation)`,
|
|
@@ -733,8 +838,9 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
|
|
|
733
838
|
.replace(/\{[^}]*\}/g, '') // ICU placeholders
|
|
734
839
|
.replace(/[\d\s\p{P}\p{S}]/gu, '') // digits, whitespace, punctuation, symbols
|
|
735
840
|
.trim();
|
|
736
|
-
const
|
|
737
|
-
const
|
|
841
|
+
const core = nameCore(source);
|
|
842
|
+
const asciiRatio = core.replace(/[^\x20-\x7E]/g, '').length / Math.max(core.length, 1);
|
|
843
|
+
const isShortAscii = core.length === 0 || (core.length <= 30 && asciiRatio > 0.8);
|
|
738
844
|
// Letters classified by Unicode script: accented Latin is Latin too
|
|
739
845
|
// (an ASCII test passed "Thánk yóú" in a Russian catalog).
|
|
740
846
|
if (translatableText.length > 0 && isLatinOnly(translated, targetLocale)) {
|
|
@@ -1358,6 +1464,12 @@ export {
|
|
|
1358
1464
|
LATIN_NAME_OR_LABEL,
|
|
1359
1465
|
foldForEchoCompare,
|
|
1360
1466
|
letterWordCount,
|
|
1467
|
+
nameCore,
|
|
1468
|
+
GATE_VERSION,
|
|
1469
|
+
isKeepAsWrittenFault,
|
|
1470
|
+
isBibliographicEntry,
|
|
1471
|
+
tableProse,
|
|
1472
|
+
SHORT_SOURCE_LENGTH_SLACK,
|
|
1361
1473
|
isDisguisedEcho,
|
|
1362
1474
|
pluralBranchEcho,
|
|
1363
1475
|
MIN_FOLDED_ECHO_WORDS,
|
package/lib/verify.js
CHANGED
|
@@ -57,6 +57,7 @@ import { pluralGaps, pluralCategoryUse, describeCategories, parseMessage, hasBra
|
|
|
57
57
|
import { pluralCategoriesFor, pluralExtraKeys } from './plurals.js';
|
|
58
58
|
import { poPluralFindings, poPluralSlots } from './po.js';
|
|
59
59
|
import { output } from './output.js';
|
|
60
|
+
import { isPendingLock } from './content-refusals.js';
|
|
60
61
|
|
|
61
62
|
/**
|
|
62
63
|
* The target locales a `--pair` value names (e.g. "en:fr,en:de"), resolved
|
|
@@ -846,9 +847,19 @@ async function verifyLocales(config, cwd, options = {}) {
|
|
|
846
847
|
// (a heading turned into a sentence): sync refuses them since Round 7;
|
|
847
848
|
// what is on disk from before is named here, with the one repair.
|
|
848
849
|
for (const w of contentGateWarnings(locale, contentItems, pairConfigs.get(locale) || { target: locale },
|
|
849
|
-
pairOf.get(locale) || `${config.inputLocale}:${locale}`, config.fallbackPrefix || '[EN] ')) {
|
|
850
|
+
pairOf.get(locale) || `${config.inputLocale}:${locale}`, config.fallbackPrefix || '[EN] ', tm)) {
|
|
850
851
|
localeWarnings.push(w);
|
|
851
852
|
}
|
|
853
|
+
// Pages written with parts left in the source language — what the gate
|
|
854
|
+
// refused twice (asked again with the reason). No marker is written into
|
|
855
|
+
// the page, so the content lock says it: `pending:<hash>`.
|
|
856
|
+
const pendingPages = pendingContentPages(cwd, locale);
|
|
857
|
+
if (pendingPages.length > 0) {
|
|
858
|
+
const pair = pairOf.get(locale) || `${config.inputLocale}:${locale}`;
|
|
859
|
+
localeWarnings.push(`${pendingPages.length} page(s) with parts left in the source language, refused by the quality gate `
|
|
860
|
+
+ `(${pendingPages.slice(0, 3).join(', ')}${pendingPages.length > 3 ? ', …' : ''}) — sync names each part; `
|
|
861
|
+
+ `ask again: \`${contentRedoCommand(pendingPages[0], { pair })}\`${pendingPages.length > 1 ? ' (one per page)' : ''}`);
|
|
862
|
+
}
|
|
852
863
|
|
|
853
864
|
output.raw(` ── ${locale} ──────────────────────────────────────`);
|
|
854
865
|
|
|
@@ -1340,6 +1351,26 @@ function contentItemsFor(config, cwd, locale, { only = null } = {}) {
|
|
|
1340
1351
|
return items;
|
|
1341
1352
|
}
|
|
1342
1353
|
|
|
1354
|
+
/**
|
|
1355
|
+
* Content pages of one locale whose lock entry is `pending:<hash>` (written
|
|
1356
|
+
* with parts left in the source language — lib/content-refusals.js), as sync
|
|
1357
|
+
* names them. Read straight from the content lock; never fails.
|
|
1358
|
+
*
|
|
1359
|
+
* @param {string} cwd
|
|
1360
|
+
* @param {string} locale
|
|
1361
|
+
* @returns {string[]}
|
|
1362
|
+
*/
|
|
1363
|
+
function pendingContentPages(cwd, locale) {
|
|
1364
|
+
let lock = {};
|
|
1365
|
+
try { lock = JSON.parse(fs.readFileSync(path.join(cwd, '.champollion-content.lock'), 'utf-8')); } catch { return []; }
|
|
1366
|
+
const pages = [];
|
|
1367
|
+
for (const [key, value] of Object.entries(lock)) {
|
|
1368
|
+
if (!isPendingLock(value) || !key.endsWith(`:${locale}`)) continue;
|
|
1369
|
+
pages.push(key.slice(0, -(locale.length + 1)).replace(/^docusaurus:/, ''));
|
|
1370
|
+
}
|
|
1371
|
+
return pages.sort();
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1343
1374
|
/**
|
|
1344
1375
|
* The command that re-translates one content file — the ONE repair sync and
|
|
1345
1376
|
* verify both print for a content finding. `--redo files:` matches the path
|
|
@@ -1365,10 +1396,15 @@ function contentRedoCommand(file, { pair = null } = {}) {
|
|
|
1365
1396
|
*
|
|
1366
1397
|
* @returns {string[]} warnings
|
|
1367
1398
|
*/
|
|
1368
|
-
function contentGateWarnings(locale, items, pairConfig, pair, fallbackPrefix) {
|
|
1399
|
+
function contentGateWarnings(locale, items, pairConfig, pair, fallbackPrefix, tm = null) {
|
|
1369
1400
|
const byFile = new Map();
|
|
1401
|
+
const tmKeys = tm ? tmKeysForPair(pairConfig) : [];
|
|
1370
1402
|
for (const it of items || []) {
|
|
1371
1403
|
if (typeof it.value !== 'string' || it.value.startsWith(fallbackPrefix)) continue;
|
|
1404
|
+
// What sync accepted and cached for this source — e.g. a name or citation
|
|
1405
|
+
// the model kept as written when asked again with the reason — is not
|
|
1406
|
+
// judged again here (it would name a deliberate answer as a fault).
|
|
1407
|
+
if (tm && tmHoldsValue(tm, it.source, locale, tmKeys, it.value)) continue;
|
|
1372
1408
|
const reason = contentGateFault(it.source, it.value, pairConfig);
|
|
1373
1409
|
if (!reason) continue;
|
|
1374
1410
|
const rest = it.key.slice('content:'.length);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "champollion",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.1",
|
|
4
4
|
"description": "Research-grade translation engine for i18n projects. Pluggable methods, per-pair quality tiers, and deterministic script converters. Supports JSON (next-intl, i18next), TOML, and YAML (Hugo).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|