champollion 0.3.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -37
- package/bin/cli.js +53 -5
- package/index.js +63 -2
- package/lib/api-key.js +17 -4
- package/lib/autofix.js +83 -36
- package/lib/bridge/method_bridge.py +15 -3
- package/lib/cards/reader.js +51 -3
- package/lib/cards/remote.js +15 -0
- package/lib/cards/search-names.js +178 -0
- package/lib/command-help.js +289 -88
- package/lib/commands/audit.js +10 -3
- package/lib/commands/card.js +583 -226
- package/lib/commands/doctor.js +54 -18
- package/lib/commands/help.js +37 -32
- package/lib/commands/init.js +1689 -87
- package/lib/commands/integrity.js +127 -40
- package/lib/commands/leaderboard.js +187 -67
- package/lib/commands/models.js +9 -2
- package/lib/commands/provenance.js +7 -2
- package/lib/commands/recommend.js +43 -14
- package/lib/commands/register-corpus.js +649 -130
- package/lib/commands/seal-corpus.js +1 -1
- package/lib/commands/status.js +564 -27
- package/lib/commands/submit.js +17 -12
- package/lib/commands/sync.js +31 -7
- package/lib/commands/tm.js +16 -10
- package/lib/commands/verify.js +27 -3
- package/lib/commands/wrap.js +63 -5
- package/lib/commands/xliff.js +135 -64
- package/lib/commercial-eligibility.js +1 -1
- package/lib/config.js +196 -14
- package/lib/content-estimate.js +96 -0
- package/lib/content-refusals.js +270 -0
- package/lib/content-review.js +372 -0
- package/lib/content-sync.js +1127 -344
- package/lib/content.js +94 -7
- package/lib/corpus-registration.mjs +197 -38
- package/lib/cost-label.js +29 -0
- package/lib/cost-report.js +726 -78
- package/lib/diff.js +38 -4
- package/lib/docusaurus-sync.js +965 -253
- package/lib/edit-distance.js +31 -0
- package/lib/fallback.js +964 -0
- package/lib/file-scope.js +106 -0
- package/lib/flatten.js +80 -3
- package/lib/flutter-locales.js +124 -0
- package/lib/format.js +266 -12
- package/lib/hash.js +146 -21
- package/lib/icu-structure.js +929 -0
- package/lib/integrity.js +223 -75
- package/lib/language-pair.js +157 -0
- package/lib/lint.js +78 -16
- package/lib/local-only-marks.js +106 -0
- package/lib/locale-layout.js +1103 -0
- package/lib/locale-state.js +571 -0
- package/lib/methods/anthropic.js +5 -0
- package/lib/methods/apertium.js +6 -3
- package/lib/methods/api.js +138 -25
- package/lib/methods/base.js +17 -0
- package/lib/methods/coaching-data.js +153 -0
- package/lib/methods/content-separator.js +43 -0
- package/lib/methods/deepl.js +1 -1
- package/lib/methods/direct-llm.js +252 -103
- package/lib/methods/external.js +146 -63
- package/lib/methods/gemini.js +1 -0
- package/lib/methods/google-translate.js +1 -0
- package/lib/methods/http-utils.js +41 -0
- package/lib/methods/libretranslate.js +7 -2
- package/lib/methods/llm-coached.js +68 -128
- package/lib/methods/llm.js +80 -31
- package/lib/methods/local.js +93 -10
- package/lib/methods/microsoft-translator.js +1 -2
- package/lib/methods/openai.js +4 -2
- package/lib/methods/openrouter-client.js +20 -19
- package/lib/methods/openrouter-pricing.js +150 -13
- package/lib/methods/prompt-methods.js +20 -0
- package/lib/methods/provider-pricing.js +42 -1
- package/lib/methods/request-capture.js +104 -0
- package/lib/methods/tilde.js +1 -1
- package/lib/methods/translated.js +1 -2
- package/lib/missing-key.js +93 -0
- package/lib/models.js +11 -0
- package/lib/name-rules.js +32 -0
- package/lib/named-keys.js +172 -0
- package/lib/no-translate.js +4 -3
- package/lib/output.js +160 -19
- package/lib/pairs.js +586 -30
- package/lib/placeholders.js +394 -0
- package/lib/plugins.js +8 -0
- package/lib/plural-gap-redo.js +109 -0
- package/lib/plurals.js +323 -0
- package/lib/po.js +1187 -0
- package/lib/public-catalogue.js +74 -0
- package/lib/recommend.js +527 -32
- package/lib/redo.js +95 -0
- package/lib/refusal-category.js +44 -0
- package/lib/registers.js +255 -11
- package/lib/repair-script.js +20 -13
- package/lib/scripts.js +193 -106
- package/lib/seal.mjs +6 -5
- package/lib/sealed-qualifier.mjs +2 -2
- package/lib/segment.js +2 -1
- package/lib/seo.js +19 -9
- package/lib/serve.js +43 -6
- package/lib/shared-output-seed.js +164 -0
- package/lib/source-contexts.js +39 -0
- package/lib/submit.mjs +57 -5
- package/lib/sync.js +2923 -474
- package/lib/terminology.js +13 -4
- package/lib/tm-evict.js +179 -0
- package/lib/tm-seed.js +5 -2
- package/lib/tm.js +818 -36
- package/lib/translate-pair.js +639 -34
- package/lib/translate.js +78 -5
- package/lib/types.js +22 -3
- package/lib/validate.js +880 -17
- package/lib/verify.js +1296 -104
- package/lib/watch.js +32 -13
- package/lib/xliff.js +44 -3
- package/package.json +3 -2
- package/shared/CORPORA-CARDS.md +2 -0
- package/shared/DATA-SOVEREIGNTY.md +19 -20
- package/shared/LANGUAGE-CARD-FIELDS.md +1 -1
- package/shared/cards-fallback.json +1 -1
- package/shared/catalogue/card-config.json +1 -1
- package/shared/curated-orthography-conventions.json +26 -8
- package/shared/docent/faq.en.json +14 -16
- package/shared/docent/system-prompt.md +17 -19
- package/shared/explainers/tc-features.json +15 -15
- package/shared/gettext-plural-forms.json +45 -0
- package/shared/human-services.json +1 -1
- package/shared/method-registry.json +2 -0
- package/shared/metric-registry.json +96 -18
- package/shared/schemas/champollion-plugin.schema.json +4 -0
- package/shared/schemas/corpora-card.schema.json +20 -10
- package/shared/schemas/human-services.schema.json +2 -2
- package/shared/schemas/language-card.schema.json +1 -1
- package/shared/schemas/method-card.schema.json +1 -1
- package/shared/schemas/method-index-record.schema.json +67 -0
- package/shared/schemas/method-registry.schema.json +4 -0
- package/shared/schemas/metric-registry.schema.json +55 -1
- package/shared/docent/corpus.json +0 -11333
|
@@ -22,6 +22,7 @@ const MICROSOFT_REQUEST_TIMEOUT_MS = 15000;
|
|
|
22
22
|
class MicrosoftTranslatorMethod extends TranslationMethod {
|
|
23
23
|
constructor(options = {}) {
|
|
24
24
|
super('microsoft-translator', options);
|
|
25
|
+
this.translatesRawText = true; // see base.js
|
|
25
26
|
}
|
|
26
27
|
|
|
27
28
|
// ── API resolution helpers ──────────────────────────────────────
|
|
@@ -47,7 +48,6 @@ class MicrosoftTranslatorMethod extends TranslationMethod {
|
|
|
47
48
|
*/
|
|
48
49
|
_resolveRegion(options) {
|
|
49
50
|
return options.microsoftRegion
|
|
50
|
-
|| getEnvOrFileVar('MICROSOFT_TRANSLATOR_REGION')
|
|
51
51
|
|| getEnvOrFileVar('MICROSOFT_TRANSLATOR_REGION', options.cwd);
|
|
52
52
|
}
|
|
53
53
|
|
|
@@ -66,7 +66,6 @@ class MicrosoftTranslatorMethod extends TranslationMethod {
|
|
|
66
66
|
*/
|
|
67
67
|
_resolveEndpoint(options = {}) {
|
|
68
68
|
let ep = options.microsoftEndpoint
|
|
69
|
-
|| getEnvOrFileVar('MICROSOFT_TRANSLATOR_ENDPOINT')
|
|
70
69
|
|| getEnvOrFileVar('MICROSOFT_TRANSLATOR_ENDPOINT', options.cwd);
|
|
71
70
|
if (!ep) return MICROSOFT_API_BASE;
|
|
72
71
|
ep = ep.replace(/\/+$/, '');
|
package/lib/methods/openai.js
CHANGED
|
@@ -33,11 +33,12 @@ class OpenAIMethod extends DirectLLMMethod {
|
|
|
33
33
|
_getApiKeyOptionsKey() { return 'openaiApiKey'; }
|
|
34
34
|
_getDefaultModel() { return DEFAULT_MODEL; }
|
|
35
35
|
_getProviderLabel() { return 'OpenAI'; }
|
|
36
|
+
_getModelVendor() { return 'openai'; }
|
|
36
37
|
_getDefaultApiBase() { return 'https://api.openai.com/v1'; }
|
|
37
38
|
|
|
38
39
|
// ── API request/response shape ───────────────────────────────────
|
|
39
40
|
|
|
40
|
-
_buildApiRequest({ prompt, systemMessage, apiKey, model, temperature, isJsonMode }) {
|
|
41
|
+
_buildApiRequest({ prompt, systemMessage, apiKey, model, temperature, isJsonMode, cwd = null }) {
|
|
41
42
|
const messages = systemMessage
|
|
42
43
|
? [{ role: 'system', content: systemMessage }, { role: 'user', content: prompt }]
|
|
43
44
|
: [{ role: 'user', content: prompt }];
|
|
@@ -47,7 +48,8 @@ class OpenAIMethod extends DirectLLMMethod {
|
|
|
47
48
|
body.response_format = { type: 'json_object' };
|
|
48
49
|
}
|
|
49
50
|
|
|
50
|
-
|
|
51
|
+
// The project's .env (cwd) may point it at a gateway or a local server.
|
|
52
|
+
const base = this._resolveApiBase(cwd ? { cwd } : {}) || 'https://api.openai.com/v1';
|
|
51
53
|
return {
|
|
52
54
|
url: `${base}/chat/completions`,
|
|
53
55
|
headers: {
|
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
import { DEFAULT_TEMPERATURE } from '../config.js';
|
|
22
22
|
import { output } from '../output.js';
|
|
23
23
|
import { recordTranslationError } from './translation-error.js';
|
|
24
|
+
import { captureRequest } from './request-capture.js';
|
|
24
25
|
|
|
25
26
|
const OPENROUTER_URL = 'https://openrouter.ai/api/v1/chat/completions';
|
|
26
27
|
|
|
@@ -55,32 +56,32 @@ async function callOpenRouter({
|
|
|
55
56
|
xTitle = 'champollion',
|
|
56
57
|
systemMessage = null,
|
|
57
58
|
}) {
|
|
59
|
+
// Build messages array — when systemMessage is provided, split preamble
|
|
60
|
+
// from payload to enable provider-level prompt caching. The system message
|
|
61
|
+
// (register + rules) is identical across batches for a given locale, so
|
|
62
|
+
// providers like Anthropic and Google cache it automatically.
|
|
63
|
+
const messages = systemMessage
|
|
64
|
+
? [{ role: 'system', content: systemMessage }, { role: 'user', content: prompt }]
|
|
65
|
+
: [{ role: 'user', content: prompt }];
|
|
66
|
+
const headers = {
|
|
67
|
+
'Authorization': `Bearer ${apiKey}`,
|
|
68
|
+
'Content-Type': 'application/json',
|
|
69
|
+
'HTTP-Referer': 'https://github.com/gamedaysuits/Champollion',
|
|
70
|
+
'X-Title': xTitle,
|
|
71
|
+
};
|
|
72
|
+
const body = { model, messages, temperature };
|
|
73
|
+
// `sync --dry --show-prompt`: hand over the exact request, send nothing.
|
|
74
|
+
if (captureRequest({ url: OPENROUTER_URL, headers, body })) return null;
|
|
75
|
+
|
|
58
76
|
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
|
59
77
|
try {
|
|
60
78
|
const controller = new AbortController();
|
|
61
79
|
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
|
62
80
|
|
|
63
|
-
// Build messages array — when systemMessage is provided, split preamble
|
|
64
|
-
// from payload to enable provider-level prompt caching. The system message
|
|
65
|
-
// (register + rules) is identical across batches for a given locale, so
|
|
66
|
-
// providers like Anthropic and Google cache it automatically.
|
|
67
|
-
const messages = systemMessage
|
|
68
|
-
? [{ role: 'system', content: systemMessage }, { role: 'user', content: prompt }]
|
|
69
|
-
: [{ role: 'user', content: prompt }];
|
|
70
|
-
|
|
71
81
|
const response = await fetch(OPENROUTER_URL, {
|
|
72
82
|
method: 'POST',
|
|
73
|
-
headers
|
|
74
|
-
|
|
75
|
-
'Content-Type': 'application/json',
|
|
76
|
-
'HTTP-Referer': 'https://github.com/gamedaysuits/Champollion',
|
|
77
|
-
'X-Title': xTitle,
|
|
78
|
-
},
|
|
79
|
-
body: JSON.stringify({
|
|
80
|
-
model,
|
|
81
|
-
messages,
|
|
82
|
-
temperature,
|
|
83
|
-
}),
|
|
83
|
+
headers,
|
|
84
|
+
body: JSON.stringify(body),
|
|
84
85
|
signal: controller.signal,
|
|
85
86
|
});
|
|
86
87
|
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
*/
|
|
21
21
|
|
|
22
22
|
import { EST_INPUT_TOKENS_PER_KEY, EST_OUTPUT_TOKENS_PER_KEY } from '../config.js';
|
|
23
|
+
import { editDistance } from '../edit-distance.js';
|
|
23
24
|
|
|
24
25
|
const OPENROUTER_MODELS_URL = 'https://openrouter.ai/api/v1/models';
|
|
25
26
|
|
|
@@ -30,6 +31,13 @@ const COACHED_INPUT_MULTIPLIER = 2.5;
|
|
|
30
31
|
// In-memory cache: fetched once per process
|
|
31
32
|
let _pricingCache = null;
|
|
32
33
|
let _pricingFetchPromise = null;
|
|
34
|
+
// What the last fetch saw, beside the prices: whether the list came at all
|
|
35
|
+
// (and why not), and every model id it listed — priced or not — so an
|
|
36
|
+
// unpriced estimate can say WHICH it is: a name the list does not have (a
|
|
37
|
+
// likely typo), a listed model with no per-token price, or no list at all.
|
|
38
|
+
// A mistyped slug used to read exactly like an unpriced model: "unknown (no
|
|
39
|
+
// method in this run has published pricing)" (Round 11, Next.js persona).
|
|
40
|
+
let _catalog = { fetched: false, why: 'not fetched yet', ids: new Set() };
|
|
33
41
|
|
|
34
42
|
/**
|
|
35
43
|
* Fetch pricing for all OpenRouter models.
|
|
@@ -56,6 +64,13 @@ async function fetchModelPricing() {
|
|
|
56
64
|
}
|
|
57
65
|
|
|
58
66
|
async function _doFetch() {
|
|
67
|
+
// The live pricing draw is off (an air-gapped node — the same switch the
|
|
68
|
+
// direct providers' pricing honours, lib/methods/provider-pricing.js
|
|
69
|
+
// pricingOffline): no list, and the estimate says so.
|
|
70
|
+
if (process.env.CHAMPOLLION_PRICING_OFFLINE === '1') {
|
|
71
|
+
_catalog = { fetched: false, why: 'the live price draw is off: CHAMPOLLION_PRICING_OFFLINE=1', ids: new Set() };
|
|
72
|
+
return new Map();
|
|
73
|
+
}
|
|
59
74
|
try {
|
|
60
75
|
const controller = new AbortController();
|
|
61
76
|
const timeoutId = setTimeout(() => controller.abort(), 5000);
|
|
@@ -67,29 +82,93 @@ async function _doFetch() {
|
|
|
67
82
|
|
|
68
83
|
clearTimeout(timeoutId);
|
|
69
84
|
|
|
70
|
-
if (!response.ok)
|
|
85
|
+
if (!response.ok) {
|
|
86
|
+
_catalog = { fetched: false, why: `OpenRouter answered HTTP ${response.status}`, ids: new Set() };
|
|
87
|
+
return new Map();
|
|
88
|
+
}
|
|
71
89
|
|
|
72
90
|
const json = await response.json();
|
|
73
91
|
const models = json.data || [];
|
|
74
92
|
const pricing = new Map();
|
|
93
|
+
const ids = new Set();
|
|
75
94
|
|
|
76
95
|
for (const model of models) {
|
|
77
|
-
if (model.id
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
96
|
+
if (!model.id) continue;
|
|
97
|
+
ids.add(model.id);
|
|
98
|
+
if (!model.pricing) continue;
|
|
99
|
+
const input = parseFloat(model.pricing.prompt);
|
|
100
|
+
const output = parseFloat(model.pricing.completion);
|
|
101
|
+
// A router whose price depends on the model it picks lists "-1": no
|
|
102
|
+
// per-token price — never a negative estimate (listed, unpriced).
|
|
103
|
+
if (!Number.isFinite(input) || !Number.isFinite(output) || input < 0 || output < 0) continue;
|
|
104
|
+
pricing.set(model.id, { input, output });
|
|
83
105
|
}
|
|
84
106
|
|
|
107
|
+
// When the list was read: an estimate says which day's prices it used.
|
|
108
|
+
_catalog = { fetched: true, why: null, ids, fetchedAt: new Date().toISOString() };
|
|
85
109
|
return pricing;
|
|
86
|
-
} catch {
|
|
110
|
+
} catch (err) {
|
|
87
111
|
// Offline, timeout, or API issue — return empty map
|
|
88
|
-
// Cost estimation degrades gracefully to "unknown"
|
|
112
|
+
// Cost estimation degrades gracefully to "unknown", and says why.
|
|
113
|
+
_catalog = {
|
|
114
|
+
fetched: false,
|
|
115
|
+
why: err?.name === 'AbortError' ? 'the request timed out' : `the request failed (${err?.message || err})`,
|
|
116
|
+
ids: new Set(),
|
|
117
|
+
};
|
|
89
118
|
return new Map();
|
|
90
119
|
}
|
|
91
120
|
}
|
|
92
121
|
|
|
122
|
+
/**
|
|
123
|
+
* The listed model ids closest to a name the list does not have — a likely
|
|
124
|
+
* typo's intended slug. Cheap: one bounded edit distance per listed id.
|
|
125
|
+
*
|
|
126
|
+
* @param {string} model
|
|
127
|
+
* @param {Iterable<string>} ids
|
|
128
|
+
* @param {number} [n=3]
|
|
129
|
+
* @returns {string[]}
|
|
130
|
+
*/
|
|
131
|
+
function closestModelIds(model, ids, n = 3) {
|
|
132
|
+
const want = String(model).toLowerCase();
|
|
133
|
+
const max = Math.max(2, Math.floor(want.length / 5));
|
|
134
|
+
const scored = [];
|
|
135
|
+
for (const id of ids) {
|
|
136
|
+
const d = editDistance(id.toLowerCase(), want, max);
|
|
137
|
+
if (d <= max) scored.push([d, id]);
|
|
138
|
+
}
|
|
139
|
+
scored.sort((a, b) => a[0] - b[0] || a[1].localeCompare(b[1]));
|
|
140
|
+
return scored.slice(0, n).map(([, id]) => id);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Why a model has no price, in one sentence that names it — and the
|
|
145
|
+
* estimate's `source` for that case.
|
|
146
|
+
*
|
|
147
|
+
* @param {string} model
|
|
148
|
+
* @returns {{ source: string, note: string, closest?: string[] }}
|
|
149
|
+
*/
|
|
150
|
+
function unpricedReason(model) {
|
|
151
|
+
if (!_catalog.fetched) {
|
|
152
|
+
return {
|
|
153
|
+
source: 'openrouter-price-list-unavailable',
|
|
154
|
+
note: `OpenRouter's price list could not be read (${_catalog.why}), so the price of "${model}" is unknown.`,
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
if (_catalog.ids.has(model)) {
|
|
158
|
+
return {
|
|
159
|
+
source: 'openrouter-no-price',
|
|
160
|
+
note: `"${model}" is in OpenRouter's model list but has no published per-token price, so its cost cannot be estimated.`,
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
const closest = closestModelIds(model, _catalog.ids);
|
|
164
|
+
return {
|
|
165
|
+
source: 'openrouter-not-listed',
|
|
166
|
+
note: `"${model}" is not in OpenRouter's model list — likely a typo in the model name`
|
|
167
|
+
+ (closest.length > 0 ? `. Closest listed: ${closest.join(', ')}.` : ' (nothing listed is close to it).'),
|
|
168
|
+
closest,
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
|
|
93
172
|
/**
|
|
94
173
|
* Estimate cost for translating N keys with a specific model via OpenRouter.
|
|
95
174
|
*
|
|
@@ -115,11 +194,16 @@ async function estimateOpenRouterCost(keyCount, model, options = {}) {
|
|
|
115
194
|
|
|
116
195
|
const modelPricing = pricing.get(model);
|
|
117
196
|
if (!modelPricing) {
|
|
197
|
+
// Named, and why: not in the list (a likely typo — with the closest
|
|
198
|
+
// listed slugs), listed with no price, or no list to look in.
|
|
199
|
+
const why = unpricedReason(model);
|
|
118
200
|
return {
|
|
119
201
|
estimatedCost: null,
|
|
120
202
|
currency: 'USD',
|
|
121
|
-
source:
|
|
122
|
-
note:
|
|
203
|
+
source: why.source,
|
|
204
|
+
note: why.note,
|
|
205
|
+
model,
|
|
206
|
+
...(why.closest && { closest: why.closest }),
|
|
123
207
|
};
|
|
124
208
|
}
|
|
125
209
|
|
|
@@ -136,21 +220,74 @@ async function estimateOpenRouterCost(keyCount, model, options = {}) {
|
|
|
136
220
|
const inputCost = totalInputTokens * modelPricing.input;
|
|
137
221
|
const outputCost = totalOutputTokens * modelPricing.output;
|
|
138
222
|
const totalCost = inputCost + outputCost;
|
|
223
|
+
const rate = openRouterRate(model, modelPricing, { input: inputTokensPerKey, output: outputTokensPerKey });
|
|
139
224
|
|
|
140
225
|
return {
|
|
141
226
|
estimatedCost: Math.round(totalCost * 10000) / 10000,
|
|
142
227
|
currency: 'USD',
|
|
143
228
|
source: `openrouter (${model})`,
|
|
144
|
-
note: `Based on ${model} pricing: $${
|
|
229
|
+
note: `Based on ${model} pricing: $${formatPerMillion(rate.inputPerMillion)}/1M input tokens, `
|
|
230
|
+
+ `$${formatPerMillion(rate.outputPerMillion)}/1M output tokens (OpenRouter's price list, read ${rate.fetchedAt}).`,
|
|
231
|
+
rate,
|
|
145
232
|
};
|
|
146
233
|
}
|
|
147
234
|
|
|
235
|
+
/**
|
|
236
|
+
* The rate an estimate used, said in full: what one million tokens cost in
|
|
237
|
+
* and out, where the figure came from (OpenRouter's public price list) and
|
|
238
|
+
* when it was read, and the tokens per key the estimate assumes. An
|
|
239
|
+
* estimate used to give a figure with none of this (Round 14, Next.js
|
|
240
|
+
* persona) — no way to tell a stale or mistaken price from a real one.
|
|
241
|
+
*
|
|
242
|
+
* @param {string} model - The OpenRouter model id priced
|
|
243
|
+
* @param {{ input: number, output: number }} perToken - USD per token
|
|
244
|
+
* @param {{ input: number, output: number }} tokensPerKey - What the estimate assumes
|
|
245
|
+
* @returns {{ model: string, unit: 'token', inputPerMillion: number, outputPerMillion: number,
|
|
246
|
+
* tokensPerKey: { input: number, output: number }, from: string, url: string, fetchedAt: string|null }}
|
|
247
|
+
*/
|
|
248
|
+
function openRouterRate(model, perToken, tokensPerKey) {
|
|
249
|
+
return {
|
|
250
|
+
model,
|
|
251
|
+
unit: 'token',
|
|
252
|
+
inputPerMillion: perMillion(perToken.input),
|
|
253
|
+
outputPerMillion: perMillion(perToken.output),
|
|
254
|
+
tokensPerKey,
|
|
255
|
+
from: 'openrouter-price-list',
|
|
256
|
+
url: OPENROUTER_MODELS_URL,
|
|
257
|
+
fetchedAt: _catalog.fetchedAt || null,
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** USD per token → USD per million tokens, without float noise (0.0000003 → 0.3). */
|
|
262
|
+
function perMillion(perTokenUsd) {
|
|
263
|
+
return Number((perTokenUsd * 1_000_000).toPrecision(12));
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/** $ per 1M, as people write it: 0.3 → "0.30", 2.5 → "2.50", 0.075 → "0.075". */
|
|
267
|
+
function formatPerMillion(usd) {
|
|
268
|
+
return usd >= 0.01 && Number(usd.toFixed(2)) === usd ? usd.toFixed(2) : String(usd);
|
|
269
|
+
}
|
|
270
|
+
|
|
148
271
|
/**
|
|
149
272
|
* Clear the pricing cache. Useful for testing.
|
|
150
273
|
*/
|
|
151
274
|
function clearPricingCache() {
|
|
152
275
|
_pricingCache = null;
|
|
153
276
|
_pricingFetchPromise = null;
|
|
277
|
+
_catalog = { fetched: false, why: 'not fetched yet', ids: new Set() };
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* When OpenRouter's price list was read in this process (ISO 8601), or
|
|
282
|
+
* null when it was not (offline, failed, never asked).
|
|
283
|
+
*
|
|
284
|
+
* @returns {string|null}
|
|
285
|
+
*/
|
|
286
|
+
function priceListFetchedAt() {
|
|
287
|
+
return _catalog.fetched ? (_catalog.fetchedAt || null) : null;
|
|
154
288
|
}
|
|
155
289
|
|
|
156
|
-
export {
|
|
290
|
+
export {
|
|
291
|
+
fetchModelPricing, estimateOpenRouterCost, clearPricingCache, priceListFetchedAt, formatPerMillion,
|
|
292
|
+
OPENROUTER_MODELS_URL,
|
|
293
|
+
};
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* prompt-methods.js — which methods send the `llm` method's prompt, and so
|
|
3
|
+
* carry a pair's free-text coaching (coachingFile / coachingPrompt).
|
|
4
|
+
*
|
|
5
|
+
* Dependency-free on purpose: the translation-memory key (lib/tm.js
|
|
6
|
+
* tmMethodKey) reads it, and tm.js imports nothing but Node built-ins.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* The plain LLM methods: one prompt (lib/methods/llm.js promptSettingsFor +
|
|
11
|
+
* buildSystemMessage + buildUserMessage), different transports. Their system
|
|
12
|
+
* message carries the pair's coaching text as a "Coaching guidance:" block.
|
|
13
|
+
*/
|
|
14
|
+
export const PLAIN_LLM_METHODS = new Set(['llm', 'openai', 'anthropic', 'gemini', 'local']);
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Every method whose prompt carries the pair's free-text coaching: the plain
|
|
18
|
+
* LLM methods, and llm-coached (which adds its structured coaching).
|
|
19
|
+
*/
|
|
20
|
+
export const COACHING_PROMPT_METHODS = new Set([...PLAIN_LLM_METHODS, 'llm-coached']);
|
|
@@ -21,7 +21,7 @@ import {
|
|
|
21
21
|
EST_INPUT_TOKENS_PER_KEY,
|
|
22
22
|
EST_OUTPUT_TOKENS_PER_KEY,
|
|
23
23
|
} from '../config.js';
|
|
24
|
-
import { fetchModelPricing } from './openrouter-pricing.js';
|
|
24
|
+
import { fetchModelPricing, priceListFetchedAt, OPENROUTER_MODELS_URL } from './openrouter-pricing.js';
|
|
25
25
|
|
|
26
26
|
/**
|
|
27
27
|
* Per-provider pricing data.
|
|
@@ -34,21 +34,29 @@ import { fetchModelPricing } from './openrouter-pricing.js';
|
|
|
34
34
|
export const PROVIDER_RATES = {
|
|
35
35
|
'google-translate': {
|
|
36
36
|
costPerMillionChars: 20,
|
|
37
|
+
verified: '2026-06-08',
|
|
38
|
+
url: 'https://cloud.google.com/translate/pricing',
|
|
37
39
|
source: 'google-cloud-pricing',
|
|
38
40
|
note: 'Google Cloud Translation API v2 ($20/1M chars).',
|
|
39
41
|
},
|
|
40
42
|
'deepl': {
|
|
41
43
|
costPerMillionChars: 25,
|
|
44
|
+
verified: '2026-06-08',
|
|
45
|
+
url: 'https://www.deepl.com/pro-api',
|
|
42
46
|
source: 'deepl-api-pro-pricing',
|
|
43
47
|
note: 'DeepL API Pro (~$25/1M chars). Free tier: 500K chars/month cap.',
|
|
44
48
|
},
|
|
45
49
|
'microsoft-translator': {
|
|
46
50
|
costPerMillionChars: 10,
|
|
51
|
+
verified: '2026-06-08',
|
|
52
|
+
url: 'https://azure.microsoft.com/pricing/details/cognitive-services/translator/',
|
|
47
53
|
source: 'microsoft-translator-pricing',
|
|
48
54
|
note: 'Azure Translator S1 ($10/1M chars). 2M chars/month free tier.',
|
|
49
55
|
},
|
|
50
56
|
'libretranslate': {
|
|
51
57
|
costPerMillionChars: 0,
|
|
58
|
+
verified: null,
|
|
59
|
+
url: null,
|
|
52
60
|
source: 'libretranslate-self-hosted',
|
|
53
61
|
note: 'Self-hosted, free. Infrastructure costs not included.',
|
|
54
62
|
},
|
|
@@ -206,6 +214,8 @@ export async function estimateLlmCost(provider, model, keyCount) {
|
|
|
206
214
|
let rate = null;
|
|
207
215
|
let source = null;
|
|
208
216
|
let divergence = '';
|
|
217
|
+
// Where the rate came from, for the estimate's `rate` (said in the table).
|
|
218
|
+
let from = null;
|
|
209
219
|
|
|
210
220
|
// The OpenRouter namespace must match the provider being billed. Without
|
|
211
221
|
// this, estimateLlmCost('anthropic', 'gpt-4o', …) would happily price the
|
|
@@ -229,6 +239,7 @@ export async function estimateLlmCost(provider, model, keyCount) {
|
|
|
229
239
|
// dollars per million tokens.
|
|
230
240
|
rate = { input: live.input * 1_000_000, output: live.output * 1_000_000 };
|
|
231
241
|
source = `openrouter-live (proxy for ${provider} list price)`;
|
|
242
|
+
from = { from: 'openrouter-price-list', url: OPENROUTER_MODELS_URL, fetchedAt: priceListFetchedAt(), proxyFor: provider };
|
|
232
243
|
|
|
233
244
|
if (pinnedUsable) {
|
|
234
245
|
const drift = (a, b) => (b === 0 ? (a === 0 ? 0 : 1) : Math.abs(a - b) / b);
|
|
@@ -254,6 +265,13 @@ export async function estimateLlmCost(provider, model, keyCount) {
|
|
|
254
265
|
if (!rate && pinnedUsable) {
|
|
255
266
|
rate = { input: pinned.input, output: pinned.output };
|
|
256
267
|
source = `pinned-table (${pinned.verified ? `verified ${pinned.verified}` : 'UNVERIFIED'}; live draw unavailable)`;
|
|
268
|
+
// Why the copy, not the live list: the draw is off, the list could not
|
|
269
|
+
// be read, or it was read and has no price for this model.
|
|
270
|
+
let liveDraw = 'unavailable';
|
|
271
|
+
if (pricingOffline()) liveDraw = 'off';
|
|
272
|
+
else if (!orId || !namespaceOk) liveDraw = 'not-listed';
|
|
273
|
+
else if (priceListFetchedAt()) liveDraw = 'no-price';
|
|
274
|
+
from = { from: 'pinned-table', verified: pinned.verified || null, liveDraw };
|
|
257
275
|
}
|
|
258
276
|
|
|
259
277
|
if (!rate) {
|
|
@@ -279,6 +297,16 @@ export async function estimateLlmCost(provider, model, keyCount) {
|
|
|
279
297
|
note: `Based on ${provider} ${model} pricing `
|
|
280
298
|
+ `($${rate.input.toFixed(2)}/1M input, $${rate.output.toFixed(2)}/1M output, `
|
|
281
299
|
+ `${source}).${divergence}`,
|
|
300
|
+
// The rate used, where it came from and when (cost-report.js says it in
|
|
301
|
+
// one line; --json carries this whole object).
|
|
302
|
+
rate: {
|
|
303
|
+
model,
|
|
304
|
+
unit: 'token',
|
|
305
|
+
inputPerMillion: Number(rate.input.toPrecision(12)),
|
|
306
|
+
outputPerMillion: Number(rate.output.toPrecision(12)),
|
|
307
|
+
tokensPerKey: { input: EST_INPUT_TOKENS_PER_KEY, output: EST_OUTPUT_TOKENS_PER_KEY },
|
|
308
|
+
...from,
|
|
309
|
+
},
|
|
282
310
|
};
|
|
283
311
|
}
|
|
284
312
|
|
|
@@ -306,5 +334,18 @@ export function estimateProviderCost(provider, keyCount) {
|
|
|
306
334
|
currency: 'USD',
|
|
307
335
|
source: rate.source,
|
|
308
336
|
note: rate.note,
|
|
337
|
+
// A published per-character rate, as this file records it (no pricing
|
|
338
|
+
// API exists for these providers): its date says how old the figure is.
|
|
339
|
+
...(rate.costPerMillionChars > 0 && {
|
|
340
|
+
rate: {
|
|
341
|
+
model: provider,
|
|
342
|
+
unit: 'char',
|
|
343
|
+
perMillionChars: rate.costPerMillionChars,
|
|
344
|
+
charsPerKey: EST_CHARS_PER_KEY,
|
|
345
|
+
from: 'published-rate',
|
|
346
|
+
url: rate.url,
|
|
347
|
+
verified: rate.verified,
|
|
348
|
+
},
|
|
349
|
+
}),
|
|
309
350
|
};
|
|
310
351
|
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Request capture — see the exact request a method would send, without
|
|
3
|
+
* sending it (`champollion sync --dry --show-prompt [key]`).
|
|
4
|
+
*
|
|
5
|
+
* WHY: nothing showed whether a gettext `msgctxt`, a `#.` comment or any
|
|
6
|
+
* other per-key instruction actually reaches the model (Round 5, Django
|
|
7
|
+
* persona). Rebuilding "what the prompt probably looks like" beside the real
|
|
8
|
+
* code would be a second prompt builder that drifts; instead the methods'
|
|
9
|
+
* own transports hand their finished request here, at the point where they
|
|
10
|
+
* would call fetch(), and return as if the call failed — no retry, nothing
|
|
11
|
+
* sent, nothing billed.
|
|
12
|
+
*
|
|
13
|
+
* Scoped with AsyncLocalStorage: only code running inside captureRequests()
|
|
14
|
+
* is captured, so a capture never swallows an unrelated call. Inside a
|
|
15
|
+
* capture, fetch() itself is refused (defence in depth: a path that is not
|
|
16
|
+
* hooked fails closed instead of reaching the network).
|
|
17
|
+
*
|
|
18
|
+
* Secrets are redacted before anything is recorded: authorization-type
|
|
19
|
+
* headers and key-bearing URL parameters print as <redacted>.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
23
|
+
|
|
24
|
+
const store = new AsyncLocalStorage();
|
|
25
|
+
|
|
26
|
+
/** The key a preview passes when none is configured (it never leaves the process). */
|
|
27
|
+
export const PREVIEW_KEY = 'preview-no-key';
|
|
28
|
+
|
|
29
|
+
const SECRET_HEADERS = new Set([
|
|
30
|
+
'authorization', 'x-api-key', 'api-key', 'x-goog-api-key', 'ocp-apim-subscription-key',
|
|
31
|
+
'x-champollion-key', 'proxy-authorization', 'cookie',
|
|
32
|
+
]);
|
|
33
|
+
const SECRET_PARAMS = new Set(['key', 'api_key', 'apikey', 'token', 'access_token', 'auth_key']);
|
|
34
|
+
|
|
35
|
+
function redactHeaders(headers) {
|
|
36
|
+
const out = {};
|
|
37
|
+
for (const [k, v] of Object.entries(headers || {})) {
|
|
38
|
+
out[k] = SECRET_HEADERS.has(k.toLowerCase()) ? '<redacted>' : v;
|
|
39
|
+
}
|
|
40
|
+
return out;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function redactUrl(url) {
|
|
44
|
+
try {
|
|
45
|
+
const u = new URL(String(url));
|
|
46
|
+
for (const name of [...u.searchParams.keys()]) {
|
|
47
|
+
if (SECRET_PARAMS.has(name.toLowerCase())) u.searchParams.set(name, '<redacted>');
|
|
48
|
+
}
|
|
49
|
+
return u.toString().replace(/%3Credacted%3E/g, '<redacted>');
|
|
50
|
+
} catch {
|
|
51
|
+
return String(url);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** True inside captureRequests(): the caller must record, not send. */
|
|
56
|
+
export function isCapturing() {
|
|
57
|
+
return store.getStore() !== undefined;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Record the request a transport is about to send. Returns true when it was
|
|
62
|
+
* captured (the transport must then return without calling fetch), false
|
|
63
|
+
* when no capture is active (send as normal).
|
|
64
|
+
*
|
|
65
|
+
* @param {{ url: string, method?: string, headers?: object, body?: unknown }} request
|
|
66
|
+
* @returns {boolean}
|
|
67
|
+
*/
|
|
68
|
+
export function captureRequest({ url, method = 'POST', headers = {}, body }) {
|
|
69
|
+
const sink = store.getStore();
|
|
70
|
+
if (!sink) return false;
|
|
71
|
+
let parsed = body;
|
|
72
|
+
if (typeof body === 'string') {
|
|
73
|
+
try { parsed = JSON.parse(body); } catch { parsed = body; }
|
|
74
|
+
}
|
|
75
|
+
sink.push({ url: redactUrl(url), method, headers: redactHeaders(headers), body: parsed });
|
|
76
|
+
return true;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
let fetchGuarded = false;
|
|
80
|
+
/** Inside a capture fetch() is refused; outside, it is the real fetch. Installed once. */
|
|
81
|
+
function guardFetch() {
|
|
82
|
+
if (fetchGuarded || typeof globalThis.fetch !== 'function') return;
|
|
83
|
+
const realFetch = globalThis.fetch;
|
|
84
|
+
globalThis.fetch = function guardedFetch(...args) {
|
|
85
|
+
if (store.getStore()) {
|
|
86
|
+
return Promise.reject(new Error('request preview: nothing is sent while showing a request'));
|
|
87
|
+
}
|
|
88
|
+
return realFetch.apply(this, args);
|
|
89
|
+
};
|
|
90
|
+
fetchGuarded = true;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Run fn with capture on; return the requests its transports would have sent.
|
|
95
|
+
*
|
|
96
|
+
* @param {() => Promise<unknown>} fn
|
|
97
|
+
* @returns {Promise<Array<{ url: string, method: string, headers: object, body: unknown }>>}
|
|
98
|
+
*/
|
|
99
|
+
export async function captureRequests(fn) {
|
|
100
|
+
guardFetch();
|
|
101
|
+
const sink = [];
|
|
102
|
+
await store.run(sink, fn);
|
|
103
|
+
return sink;
|
|
104
|
+
}
|
package/lib/methods/tilde.js
CHANGED
|
@@ -27,11 +27,11 @@ const TILDE_MAX_BATCH = 25;
|
|
|
27
27
|
class TildeMethod extends TranslationMethod {
|
|
28
28
|
constructor(options = {}) {
|
|
29
29
|
super('tilde', options);
|
|
30
|
+
this.translatesRawText = true; // see base.js
|
|
30
31
|
}
|
|
31
32
|
|
|
32
33
|
_resolveApiKey(options = {}) {
|
|
33
34
|
return options.tildeApiKey
|
|
34
|
-
|| getEnvOrFileVar('TILDE_API_KEY')
|
|
35
35
|
|| getEnvOrFileVar('TILDE_API_KEY', options.cwd);
|
|
36
36
|
}
|
|
37
37
|
|
|
@@ -28,6 +28,7 @@ const TRANSLATED_MAX_BATCH = 50;
|
|
|
28
28
|
class TranslatedMethod extends TranslationMethod {
|
|
29
29
|
constructor(options = {}) {
|
|
30
30
|
super('translated', options);
|
|
31
|
+
this.translatesRawText = true; // see base.js
|
|
31
32
|
this._client = options.laraClient || null; // injectable for tests
|
|
32
33
|
}
|
|
33
34
|
|
|
@@ -40,10 +41,8 @@ class TranslatedMethod extends TranslationMethod {
|
|
|
40
41
|
*/
|
|
41
42
|
_resolveCredentials(options = {}) {
|
|
42
43
|
const id = options.laraAccessKeyId
|
|
43
|
-
|| getEnvOrFileVar('LARA_ACCESS_KEY_ID')
|
|
44
44
|
|| getEnvOrFileVar('LARA_ACCESS_KEY_ID', options.cwd);
|
|
45
45
|
const secret = options.laraAccessKeySecret
|
|
46
|
-
|| getEnvOrFileVar('LARA_ACCESS_KEY_SECRET')
|
|
47
46
|
|| getEnvOrFileVar('LARA_ACCESS_KEY_SECRET', options.cwd);
|
|
48
47
|
return id && secret ? { id, secret } : null;
|
|
49
48
|
}
|