ext-localize 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +538 -2
- package/assets/fill-core.js +338 -0
- package/dist/batch.js +25 -0
- package/dist/check.js +319 -0
- package/dist/cli.js +620 -0
- package/dist/config.js +110 -0
- package/dist/glossary.js +70 -0
- package/dist/listing.js +311 -0
- package/dist/locales.js +86 -0
- package/dist/messages.js +123 -0
- package/dist/placeholders.js +171 -0
- package/dist/providers/anthropic.js +60 -0
- package/dist/providers/deepl.js +269 -0
- package/dist/providers/index.js +40 -0
- package/dist/providers/ollama.js +48 -0
- package/dist/providers/openai.js +50 -0
- package/dist/providers/prompt.js +100 -0
- package/dist/providers/types.js +18 -0
- package/dist/retry.js +66 -0
- package/dist/stale.js +151 -0
- package/dist/state.js +201 -0
- package/dist/stores/chrome-auto.js +205 -0
- package/dist/stores/chrome.js +147 -0
- package/dist/stores/edge.js +43 -0
- package/dist/stores/firefox.js +235 -0
- package/dist/translate.js +308 -0
- package/package.json +54 -4
package/dist/messages.js
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, statSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { dirname, join } from "node:path";
|
|
3
|
+
/** Read and strictly parse a messages.json. Chrome drops a locale whose file does not parse. */
|
|
4
|
+
export function readMessagesFile(path) {
|
|
5
|
+
let text;
|
|
6
|
+
try {
|
|
7
|
+
text = readFileSync(path, "utf8");
|
|
8
|
+
}
|
|
9
|
+
catch (e) {
|
|
10
|
+
return { ok: false, error: `cannot read file: ${e.message}` };
|
|
11
|
+
}
|
|
12
|
+
const hadBom = text.charCodeAt(0) === 0xfeff;
|
|
13
|
+
if (hadBom)
|
|
14
|
+
text = text.slice(1);
|
|
15
|
+
try {
|
|
16
|
+
const data = JSON.parse(text);
|
|
17
|
+
if (data === null || typeof data !== "object" || Array.isArray(data)) {
|
|
18
|
+
return { ok: false, error: "top level must be a JSON object" };
|
|
19
|
+
}
|
|
20
|
+
return { ok: true, data: data, hadBom };
|
|
21
|
+
}
|
|
22
|
+
catch (e) {
|
|
23
|
+
return { ok: false, error: `invalid JSON: ${e.message}` };
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
/** Write JSON atomically (temp file + rename), 2-space indent, trailing newline. */
|
|
27
|
+
export function writeJsonAtomic(path, data) {
|
|
28
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
29
|
+
const tmp = `${path}.tmp-${process.pid}`;
|
|
30
|
+
writeFileSync(tmp, JSON.stringify(data, null, 2) + "\n", "utf8");
|
|
31
|
+
renameSync(tmp, path);
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Remove // and /* *\/ comments outside of strings. Chrome accepts comments in
|
|
35
|
+
* manifest.json, so a strict JSON.parse would fail on some real manifests.
|
|
36
|
+
*/
|
|
37
|
+
export function stripJsonComments(text) {
|
|
38
|
+
let out = "";
|
|
39
|
+
let inString = false;
|
|
40
|
+
for (let i = 0; i < text.length; i++) {
|
|
41
|
+
const c = text[i];
|
|
42
|
+
const next = text[i + 1];
|
|
43
|
+
if (inString) {
|
|
44
|
+
out += c;
|
|
45
|
+
if (c === "\\") {
|
|
46
|
+
out += next ?? "";
|
|
47
|
+
i++;
|
|
48
|
+
}
|
|
49
|
+
else if (c === '"')
|
|
50
|
+
inString = false;
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
if (c === '"') {
|
|
54
|
+
inString = true;
|
|
55
|
+
out += c;
|
|
56
|
+
}
|
|
57
|
+
else if (c === "/" && next === "/") {
|
|
58
|
+
while (i < text.length && text[i] !== "\n")
|
|
59
|
+
i++;
|
|
60
|
+
out += "\n";
|
|
61
|
+
}
|
|
62
|
+
else if (c === "/" && next === "*") {
|
|
63
|
+
i += 2;
|
|
64
|
+
while (i < text.length && !(text[i] === "*" && text[i + 1] === "/"))
|
|
65
|
+
i++;
|
|
66
|
+
i++;
|
|
67
|
+
}
|
|
68
|
+
else
|
|
69
|
+
out += c;
|
|
70
|
+
}
|
|
71
|
+
return out;
|
|
72
|
+
}
|
|
73
|
+
export class ProjectError extends Error {
|
|
74
|
+
}
|
|
75
|
+
export function messagesPath(localesDir, locale) {
|
|
76
|
+
return join(localesDir, locale, "messages.json");
|
|
77
|
+
}
|
|
78
|
+
/** Locale folder names that exist under _locales/ (any name, valid or not). */
|
|
79
|
+
export function listLocaleDirs(localesDir) {
|
|
80
|
+
if (!existsSync(localesDir))
|
|
81
|
+
return [];
|
|
82
|
+
return readdirSync(localesDir)
|
|
83
|
+
.filter((name) => !name.startsWith(".") && statSync(join(localesDir, name)).isDirectory())
|
|
84
|
+
.sort();
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Read manifest.json and the default locale's messages.json.
|
|
88
|
+
* Throws ProjectError with a short, fixable message on any problem.
|
|
89
|
+
*/
|
|
90
|
+
export function loadProject(root) {
|
|
91
|
+
const manifestPath = join(root, "manifest.json");
|
|
92
|
+
if (!existsSync(manifestPath))
|
|
93
|
+
throw new ProjectError(`No manifest.json in ${root}. Use --dir to point at the extension folder.`);
|
|
94
|
+
let manifest;
|
|
95
|
+
try {
|
|
96
|
+
let raw = readFileSync(manifestPath, "utf8");
|
|
97
|
+
if (raw.charCodeAt(0) === 0xfeff)
|
|
98
|
+
raw = raw.slice(1);
|
|
99
|
+
manifest = JSON.parse(stripJsonComments(raw));
|
|
100
|
+
}
|
|
101
|
+
catch (e) {
|
|
102
|
+
throw new ProjectError(`manifest.json does not parse: ${e.message}`);
|
|
103
|
+
}
|
|
104
|
+
const localesDir = join(root, "_locales");
|
|
105
|
+
const defaultLocale = manifest["default_locale"];
|
|
106
|
+
if (typeof defaultLocale !== "string" || !defaultLocale) {
|
|
107
|
+
throw new ProjectError(existsSync(localesDir)
|
|
108
|
+
? `manifest.json has no "default_locale", but _locales/ exists. Chrome will refuse to load the extension.`
|
|
109
|
+
: `manifest.json has no "default_locale". Add it (for example "en") and create _locales/en/messages.json.`);
|
|
110
|
+
}
|
|
111
|
+
const sourcePath = messagesPath(localesDir, defaultLocale);
|
|
112
|
+
const read = readMessagesFile(sourcePath);
|
|
113
|
+
if (!read.ok)
|
|
114
|
+
throw new ProjectError(`Source file _locales/${defaultLocale}/messages.json: ${read.error}`);
|
|
115
|
+
const source = {};
|
|
116
|
+
for (const [key, value] of Object.entries(read.data)) {
|
|
117
|
+
if (!value || typeof value !== "object" || typeof value.message !== "string") {
|
|
118
|
+
throw new ProjectError(`Source key "${key}" has no "message" string.`);
|
|
119
|
+
}
|
|
120
|
+
source[key] = value;
|
|
121
|
+
}
|
|
122
|
+
return { root, manifestPath, manifest, defaultLocale, localesDir, sourcePath, source };
|
|
123
|
+
}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Chrome message format tokens and how we protect them during translation.
|
|
3
|
+
*
|
|
4
|
+
* Chrome rules (https://developer.chrome.com/docs/extensions/mv3/i18n-messages):
|
|
5
|
+
* $$ -> a literal "$"
|
|
6
|
+
* $name$ -> a named placeholder, defined in the "placeholders" block.
|
|
7
|
+
* Names are case-insensitive and use A-Z a-z 0-9 _ @
|
|
8
|
+
* $1 .. $9 -> positional substitutions passed to chrome.i18n.getMessage()
|
|
9
|
+
*
|
|
10
|
+
* We scan left to right and try "$$" first, so "$$1" is a literal "$" then "1".
|
|
11
|
+
*/
|
|
12
|
+
const TOKEN_RE = /\$\$|\$([A-Za-z0-9_@]+)\$|\$([1-9])|\$/g;
|
|
13
|
+
/** Split a message into plain text and format tokens. */
|
|
14
|
+
export function tokenize(message) {
|
|
15
|
+
const out = [];
|
|
16
|
+
let last = 0;
|
|
17
|
+
TOKEN_RE.lastIndex = 0;
|
|
18
|
+
for (let m = TOKEN_RE.exec(message); m; m = TOKEN_RE.exec(message)) {
|
|
19
|
+
if (m.index > last)
|
|
20
|
+
out.push({ kind: "text", raw: message.slice(last, m.index) });
|
|
21
|
+
const raw = m[0];
|
|
22
|
+
if (raw === "$$")
|
|
23
|
+
out.push({ kind: "dollar", raw });
|
|
24
|
+
else if (m[1] !== undefined)
|
|
25
|
+
out.push({ kind: "named", raw, name: m[1].toLowerCase() });
|
|
26
|
+
else if (m[2] !== undefined)
|
|
27
|
+
out.push({ kind: "positional", raw, index: Number(m[2]) });
|
|
28
|
+
else
|
|
29
|
+
out.push({ kind: "lone", raw });
|
|
30
|
+
last = m.index + raw.length;
|
|
31
|
+
}
|
|
32
|
+
if (last < message.length)
|
|
33
|
+
out.push({ kind: "text", raw: message.slice(last) });
|
|
34
|
+
return out;
|
|
35
|
+
}
|
|
36
|
+
export function summarizeFormat(message) {
|
|
37
|
+
const named = new Set();
|
|
38
|
+
const positional = new Set();
|
|
39
|
+
let dollars = 0;
|
|
40
|
+
let lone = 0;
|
|
41
|
+
for (const s of tokenize(message)) {
|
|
42
|
+
if (s.kind === "named")
|
|
43
|
+
named.add(s.name);
|
|
44
|
+
else if (s.kind === "positional")
|
|
45
|
+
positional.add(s.index);
|
|
46
|
+
else if (s.kind === "dollar")
|
|
47
|
+
dollars++;
|
|
48
|
+
else if (s.kind === "lone")
|
|
49
|
+
lone++;
|
|
50
|
+
}
|
|
51
|
+
return {
|
|
52
|
+
named: [...named].sort(),
|
|
53
|
+
positional: [...positional].sort((a, b) => a - b),
|
|
54
|
+
dollars,
|
|
55
|
+
lone,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Compare the format tokens of a source and a translated message.
|
|
60
|
+
* Named and positional sets must match (error). A different "$$" count or a new
|
|
61
|
+
* stray "$" is a warning: a human may write "4,99 EUR" on purpose, but from a
|
|
62
|
+
* translator it usually means a broken token (restoreAndVerify treats all as errors).
|
|
63
|
+
*/
|
|
64
|
+
export function compareFormat(source, target) {
|
|
65
|
+
const a = summarizeFormat(source);
|
|
66
|
+
const b = summarizeFormat(target);
|
|
67
|
+
const problems = [];
|
|
68
|
+
const error = (message) => problems.push({ severity: "error", message });
|
|
69
|
+
const warning = (message) => problems.push({ severity: "warning", message });
|
|
70
|
+
const missingNamed = a.named.filter((n) => !b.named.includes(n));
|
|
71
|
+
const extraNamed = b.named.filter((n) => !a.named.includes(n));
|
|
72
|
+
if (missingNamed.length)
|
|
73
|
+
error(`missing placeholder(s) ${missingNamed.map((n) => `$${n}$`).join(", ")}`);
|
|
74
|
+
if (extraNamed.length)
|
|
75
|
+
error(`unknown placeholder(s) ${extraNamed.map((n) => `$${n}$`).join(", ")}`);
|
|
76
|
+
const missingPos = a.positional.filter((n) => !b.positional.includes(n));
|
|
77
|
+
const extraPos = b.positional.filter((n) => !a.positional.includes(n));
|
|
78
|
+
if (missingPos.length)
|
|
79
|
+
error(`missing substitution(s) ${missingPos.map((n) => `$${n}`).join(", ")}`);
|
|
80
|
+
if (extraPos.length)
|
|
81
|
+
error(`extra substitution(s) ${extraPos.map((n) => `$${n}`).join(", ")}`);
|
|
82
|
+
if (a.dollars !== b.dollars)
|
|
83
|
+
warning(`"$$" count differs (source ${a.dollars}, translation ${b.dollars})`);
|
|
84
|
+
if (b.lone > a.lone)
|
|
85
|
+
warning(`stray "$" in translation (use "$$" for a literal dollar sign)`);
|
|
86
|
+
return problems;
|
|
87
|
+
}
|
|
88
|
+
/** True when the placeholder sets match (warnings allowed). */
|
|
89
|
+
export function placeholdersMatch(source, target) {
|
|
90
|
+
return compareFormat(source, target).every((p) => p.severity !== "error");
|
|
91
|
+
}
|
|
92
|
+
function escapeRegExp(s) {
|
|
93
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Replace every Chrome format token (and every glossary "keep" term) with a
|
|
97
|
+
* marker. Translators are told to copy markers unchanged.
|
|
98
|
+
*/
|
|
99
|
+
export function protect(message, keepTerms = [], opts = {}) {
|
|
100
|
+
const originals = [];
|
|
101
|
+
const terms = [...new Set(keepTerms.filter((t) => t.trim().length > 0))].sort((a, b) => b.length - a.length);
|
|
102
|
+
const termRe = terms.length
|
|
103
|
+
? new RegExp(`(?<![\\p{L}\\p{N}_])(?:${terms.map(escapeRegExp).join("|")})(?![\\p{L}\\p{N}_])`, "gu")
|
|
104
|
+
: null;
|
|
105
|
+
let text = "";
|
|
106
|
+
// Store listing text is not a Chrome message: "$4.99" is just text there.
|
|
107
|
+
const segments = opts.formatTokens === false ? [{ kind: "text", raw: message }] : tokenize(message);
|
|
108
|
+
for (const seg of segments) {
|
|
109
|
+
if (seg.kind !== "text") {
|
|
110
|
+
originals.push(seg.raw);
|
|
111
|
+
text += `<x id="${originals.length - 1}"/>`;
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
if (!termRe) {
|
|
115
|
+
text += seg.raw;
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
text += seg.raw.replace(termRe, (term) => {
|
|
119
|
+
originals.push(term);
|
|
120
|
+
return `<x id="${originals.length - 1}">${term}</x>`;
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
return { text, originals };
|
|
124
|
+
}
|
|
125
|
+
const MARKER_RE = /<x\s+id\s*=\s*["']?(\d+)["']?\s*\/>|<x\s+id\s*=\s*["']?(\d+)["']?\s*>[\s\S]*?<\/x\s*>/g;
|
|
126
|
+
/**
|
|
127
|
+
* Put the original tokens back. Fails when a marker is missing, repeated,
|
|
128
|
+
* unknown, or left half-broken.
|
|
129
|
+
*/
|
|
130
|
+
export function restore(translated, originals) {
|
|
131
|
+
const seen = new Map();
|
|
132
|
+
const unknownIds = [];
|
|
133
|
+
const text = translated.replace(MARKER_RE, (_m, a, b) => {
|
|
134
|
+
const id = Number(a ?? b);
|
|
135
|
+
seen.set(id, (seen.get(id) ?? 0) + 1);
|
|
136
|
+
if (id >= originals.length) {
|
|
137
|
+
unknownIds.push(id);
|
|
138
|
+
return "";
|
|
139
|
+
}
|
|
140
|
+
return originals[id];
|
|
141
|
+
});
|
|
142
|
+
if (unknownIds.length)
|
|
143
|
+
return { ok: false, error: `unknown marker id ${unknownIds.join(", ")}` };
|
|
144
|
+
for (let i = 0; i < originals.length; i++) {
|
|
145
|
+
const n = seen.get(i) ?? 0;
|
|
146
|
+
if (n === 0)
|
|
147
|
+
return { ok: false, error: `marker ${i} (${originals[i]}) is missing` };
|
|
148
|
+
if (n > 1)
|
|
149
|
+
return { ok: false, error: `marker ${i} (${originals[i]}) appears ${n} times` };
|
|
150
|
+
}
|
|
151
|
+
if (/<\/?x[\s>/]/.test(text))
|
|
152
|
+
return { ok: false, error: "broken marker left in text" };
|
|
153
|
+
return { ok: true, text };
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Restore markers, then re-check the Chrome format tokens against the source.
|
|
157
|
+
* This is the gate every translated string passes before it is written.
|
|
158
|
+
*/
|
|
159
|
+
export function restoreAndVerify(source, translated, originals, opts = {}) {
|
|
160
|
+
const r = restore(translated, originals);
|
|
161
|
+
if (!r.ok)
|
|
162
|
+
return r;
|
|
163
|
+
if (r.text.trim() === "" && source.trim() !== "")
|
|
164
|
+
return { ok: false, error: "empty translation" };
|
|
165
|
+
if (opts.checkFormat === false)
|
|
166
|
+
return r;
|
|
167
|
+
const problems = compareFormat(source, r.text);
|
|
168
|
+
if (problems.length)
|
|
169
|
+
return { ok: false, error: problems.map((p) => p.message).join("; ") };
|
|
170
|
+
return r;
|
|
171
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { RESPONSE_SCHEMA, buildUserPrompt, systemPromptFor, parseTranslations } from "./prompt.js";
|
|
2
|
+
import { LLM_LIMITS, OutputTruncatedError, ProviderError } from "./types.js";
|
|
3
|
+
/** Current default Claude model (Claude Opus 5.5). Override with --model. */
|
|
4
|
+
export const ANTHROPIC_DEFAULT_MODEL = "claude-opus-5-5";
|
|
5
|
+
/**
|
|
6
|
+
* Claude via the official SDK. Credentials come from the environment
|
|
7
|
+
* (ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN or an `ant auth login` profile);
|
|
8
|
+
* nothing is written to disk.
|
|
9
|
+
*
|
|
10
|
+
* - Structured outputs (output_config.format) give parseable JSON.
|
|
11
|
+
* - The SDK retries 429 / 5xx / connection errors itself (maxRetries).
|
|
12
|
+
* - Server-side refusal fallback is on ("default"); on models without
|
|
13
|
+
* fallback support the request simply runs on the requested model.
|
|
14
|
+
*/
|
|
15
|
+
export class AnthropicProvider {
|
|
16
|
+
name = "anthropic";
|
|
17
|
+
label;
|
|
18
|
+
limits = LLM_LIMITS;
|
|
19
|
+
model;
|
|
20
|
+
effort;
|
|
21
|
+
client;
|
|
22
|
+
constructor(opts = {}) {
|
|
23
|
+
this.model = opts.model ?? ANTHROPIC_DEFAULT_MODEL;
|
|
24
|
+
this.effort = opts.effort ?? "medium";
|
|
25
|
+
this.client = opts.client;
|
|
26
|
+
this.label = `anthropic/${this.model}`;
|
|
27
|
+
}
|
|
28
|
+
async getClient() {
|
|
29
|
+
if (!this.client) {
|
|
30
|
+
const { default: AnthropicSdk } = await import("@anthropic-ai/sdk");
|
|
31
|
+
this.client = new AnthropicSdk({ maxRetries: 6 });
|
|
32
|
+
}
|
|
33
|
+
return this.client;
|
|
34
|
+
}
|
|
35
|
+
async translateBatch(req) {
|
|
36
|
+
const client = await this.getClient();
|
|
37
|
+
const response = await client.beta.messages.create({
|
|
38
|
+
model: this.model,
|
|
39
|
+
max_tokens: 16000,
|
|
40
|
+
betas: ["server-side-fallback-2026-07-01"],
|
|
41
|
+
fallbacks: "default",
|
|
42
|
+
output_config: {
|
|
43
|
+
effort: this.effort,
|
|
44
|
+
format: { type: "json_schema", schema: RESPONSE_SCHEMA },
|
|
45
|
+
},
|
|
46
|
+
system: systemPromptFor(req.kind),
|
|
47
|
+
messages: [{ role: "user", content: buildUserPrompt(req) }],
|
|
48
|
+
});
|
|
49
|
+
if (response.stop_reason === "refusal") {
|
|
50
|
+
const category = response.stop_details?.category ?? "unknown";
|
|
51
|
+
throw new ProviderError(`Claude declined this batch (category: ${category})`);
|
|
52
|
+
}
|
|
53
|
+
if (response.stop_reason === "max_tokens")
|
|
54
|
+
throw new OutputTruncatedError();
|
|
55
|
+
const text = response.content
|
|
56
|
+
.map((b) => (b.type === "text" ? b.text : ""))
|
|
57
|
+
.join("");
|
|
58
|
+
return parseTranslations(text, req);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
import { fetchJson, withRetry } from "../retry.js";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { ProviderError } from "./types.js";
|
|
4
|
+
/** Chrome locale -> DeepL target code, where they differ from plain upper case. */
|
|
5
|
+
const TARGET_MAP = {
|
|
6
|
+
en: "EN-US",
|
|
7
|
+
en_US: "EN-US",
|
|
8
|
+
en_GB: "EN-GB",
|
|
9
|
+
en_AU: "EN-GB",
|
|
10
|
+
es_419: "ES-419",
|
|
11
|
+
pt_BR: "PT-BR",
|
|
12
|
+
pt_PT: "PT-PT",
|
|
13
|
+
zh_CN: "ZH-HANS",
|
|
14
|
+
zh_TW: "ZH-HANT",
|
|
15
|
+
no: "NB",
|
|
16
|
+
fil: "TL",
|
|
17
|
+
};
|
|
18
|
+
export function deeplTargetCode(locale) {
|
|
19
|
+
return TARGET_MAP[locale] ?? locale.split("_")[0].toUpperCase();
|
|
20
|
+
}
|
|
21
|
+
/** DeepL source codes are language-only (EN, PT, ZH). */
|
|
22
|
+
export function deeplSourceCode(locale) {
|
|
23
|
+
return locale === "fil" ? "TL" : locale === "no" ? "NB" : locale.split("_")[0].toUpperCase();
|
|
24
|
+
}
|
|
25
|
+
const MARKER_SPLIT_RE = /(<x id="\d+"\/>|<x id="\d+">[\s\S]*?<\/x>)/;
|
|
26
|
+
function escapeXml(s) {
|
|
27
|
+
return s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
|
28
|
+
}
|
|
29
|
+
function unescapeXml(s) {
|
|
30
|
+
return s
|
|
31
|
+
.replace(/</g, "<")
|
|
32
|
+
.replace(/>/g, ">")
|
|
33
|
+
.replace(/"/g, '"')
|
|
34
|
+
.replace(/'/g, "'")
|
|
35
|
+
.replace(/&/g, "&");
|
|
36
|
+
}
|
|
37
|
+
/** Escape the text between markers so DeepL's XML mode sees well-formed input. */
|
|
38
|
+
export function toDeeplXml(text) {
|
|
39
|
+
return text
|
|
40
|
+
.split(MARKER_SPLIT_RE)
|
|
41
|
+
.map((part, i) => i % 2 === 1 ? part.replace(/>([\s\S]*?)<\/x>$/, (_m, inner) => `>${escapeXml(inner)}</x>`) : escapeXml(part))
|
|
42
|
+
.join("");
|
|
43
|
+
}
|
|
44
|
+
export function fromDeeplXml(text) {
|
|
45
|
+
return text
|
|
46
|
+
.split(/(<x\s[^>]*\/>|<x\s[^>]*>[\s\S]*?<\/x>)/)
|
|
47
|
+
.map((part, i) => (i % 2 === 1 ? part : unescapeXml(part)))
|
|
48
|
+
.join("");
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Turn glossary terms into DeepL TSV entries. DeepL rejects empty phrases,
|
|
52
|
+
* tabs or line breaks inside a phrase, leading/trailing spaces and duplicate
|
|
53
|
+
* source terms, so those are trimmed or dropped (with a note).
|
|
54
|
+
*/
|
|
55
|
+
export function glossaryEntries(terms) {
|
|
56
|
+
const seen = new Set();
|
|
57
|
+
const lines = [];
|
|
58
|
+
const dropped = [];
|
|
59
|
+
for (const [rawSource, rawTarget] of terms) {
|
|
60
|
+
const source = rawSource.trim();
|
|
61
|
+
const target = rawTarget.trim();
|
|
62
|
+
if (!source || !target || /[\t\r\n\u2028\u2029]/.test(source + target)) {
|
|
63
|
+
dropped.push(rawSource);
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
if (seen.has(source)) {
|
|
67
|
+
dropped.push(rawSource);
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
seen.add(source);
|
|
71
|
+
lines.push(`${source}\t${target}`);
|
|
72
|
+
}
|
|
73
|
+
// Sorted, so the same terms always give the same glossary name.
|
|
74
|
+
lines.sort();
|
|
75
|
+
return { tsv: lines.join("\n"), count: lines.length, dropped };
|
|
76
|
+
}
|
|
77
|
+
/** Name of the DeepL glossary for one language pair and one set of entries. */
|
|
78
|
+
export function glossaryName(source, target, tsv) {
|
|
79
|
+
const hash = createHash("sha256").update(tsv, "utf8").digest("hex").slice(0, 16);
|
|
80
|
+
return `${GLOSSARY_PREFIX}${source}->${target} ${hash}`;
|
|
81
|
+
}
|
|
82
|
+
export const GLOSSARY_PREFIX = "ext-localize ";
|
|
83
|
+
/** DeepL accepts at most 50 texts per /v2/translate request. */
|
|
84
|
+
export const DEEPL_MAX_TEXTS = 50;
|
|
85
|
+
/** Split text into lines; each line keeps its leading and trailing spaces apart. */
|
|
86
|
+
export function splitLines(text) {
|
|
87
|
+
return text.split(/\r?\n/).map((line) => {
|
|
88
|
+
const m = /^(\s*)([\s\S]*?)(\s*)$/.exec(line);
|
|
89
|
+
return { lead: m[1] ?? "", core: m[2] ?? "", trail: m[3] ?? "" };
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* DeepL /v2/translate. Markers are sent as XML tags with tag_handling=xml and
|
|
94
|
+
* ignore_tags=["x"], so placeholders and "keep" terms are never touched.
|
|
95
|
+
* Descriptions go into the per-request "context" field (not billed).
|
|
96
|
+
*
|
|
97
|
+
* Formality: "more"/"less" are sent only to languages that support them
|
|
98
|
+
* (others get the default, with a note); "prefer_more"/"prefer_less" are
|
|
99
|
+
* always sent, DeepL falls back by itself.
|
|
100
|
+
*
|
|
101
|
+
* Glossary terms: one DeepL glossary (v3 API) per target locale, named by a
|
|
102
|
+
* hash of its entries. An existing glossary with the same name is reused, so
|
|
103
|
+
* a new one is created only when the terms change. Old ones are not deleted.
|
|
104
|
+
*/
|
|
105
|
+
export class DeepLProvider {
|
|
106
|
+
name = "deepl";
|
|
107
|
+
label = "deepl";
|
|
108
|
+
limits = { maxItems: 50, maxChars: 30000 };
|
|
109
|
+
apiKey;
|
|
110
|
+
baseUrl;
|
|
111
|
+
targets;
|
|
112
|
+
glossaryLangs;
|
|
113
|
+
glossaries;
|
|
114
|
+
setup = new Map();
|
|
115
|
+
constructor(opts = {}) {
|
|
116
|
+
const key = opts.apiKey ?? process.env.DEEPL_API_KEY;
|
|
117
|
+
if (!key)
|
|
118
|
+
throw new ProviderError("DEEPL_API_KEY is not set");
|
|
119
|
+
this.apiKey = key;
|
|
120
|
+
this.baseUrl = (opts.baseUrl ??
|
|
121
|
+
process.env.DEEPL_API_URL ??
|
|
122
|
+
(key.endsWith(":fx") ? "https://api-free.deepl.com" : "https://api.deepl.com")).replace(/\/+$/, "");
|
|
123
|
+
}
|
|
124
|
+
headers(json = true) {
|
|
125
|
+
const h = { authorization: `DeepL-Auth-Key ${this.apiKey}` };
|
|
126
|
+
if (json)
|
|
127
|
+
h["content-type"] = "application/json";
|
|
128
|
+
return h;
|
|
129
|
+
}
|
|
130
|
+
get(path) {
|
|
131
|
+
return withRetry(() => fetchJson(`${this.baseUrl}${path}`, { method: "GET", headers: this.headers(false) }));
|
|
132
|
+
}
|
|
133
|
+
/** Target languages of this account, with formality support (free call, cached). */
|
|
134
|
+
targetLanguages() {
|
|
135
|
+
this.targets ??= this.get("/v2/languages?type=target").then((list) => new Map(list.map((l) => [
|
|
136
|
+
l.language.toUpperCase(),
|
|
137
|
+
{ formality: l.supports_formality === true },
|
|
138
|
+
])));
|
|
139
|
+
return this.targets;
|
|
140
|
+
}
|
|
141
|
+
async supportsLocale(locale) {
|
|
142
|
+
return (await this.targetLanguages()).has(deeplTargetCode(locale));
|
|
143
|
+
}
|
|
144
|
+
/** Languages usable in glossaries: upper-case code -> code as DeepL spells it. */
|
|
145
|
+
glossaryLanguages() {
|
|
146
|
+
this.glossaryLangs ??= this.get("/v3/languages?resource=glossary").then((list) => new Map(list.map((l) => [l.lang.toUpperCase(), l.lang])));
|
|
147
|
+
return this.glossaryLangs;
|
|
148
|
+
}
|
|
149
|
+
/** Our glossaries on this account: name -> id (cached for the run). */
|
|
150
|
+
glossaryIndex() {
|
|
151
|
+
this.glossaries ??= this.get("/v3/glossaries").then((data) => {
|
|
152
|
+
const list = data.glossaries ?? [];
|
|
153
|
+
return new Map(list.filter((g) => g.name.startsWith(GLOSSARY_PREFIX)).map((g) => [g.name, g.glossary_id]));
|
|
154
|
+
});
|
|
155
|
+
return this.glossaries;
|
|
156
|
+
}
|
|
157
|
+
async ensureGlossary(source, target, tsv) {
|
|
158
|
+
const name = glossaryName(source, target, tsv);
|
|
159
|
+
const index = await this.glossaryIndex();
|
|
160
|
+
const existing = index.get(name);
|
|
161
|
+
if (existing)
|
|
162
|
+
return { id: existing, created: false, name };
|
|
163
|
+
const body = { name, dictionaries: [{ source_lang: source, target_lang: target, entries: tsv, entries_format: "tsv" }] };
|
|
164
|
+
// Not wrapped in withRetry: a retried POST could create the glossary twice.
|
|
165
|
+
const created = (await fetchJson(`${this.baseUrl}/v3/glossaries`, {
|
|
166
|
+
method: "POST",
|
|
167
|
+
headers: this.headers(),
|
|
168
|
+
body: JSON.stringify(body),
|
|
169
|
+
}));
|
|
170
|
+
if (!created.glossary_id)
|
|
171
|
+
throw new ProviderError("DeepL did not return a glossary_id");
|
|
172
|
+
index.set(name, created.glossary_id);
|
|
173
|
+
return { id: created.glossary_id, created: true, name };
|
|
174
|
+
}
|
|
175
|
+
/** Decide formality and glossary for one locale. Returns notes for the log. */
|
|
176
|
+
async prepareLocale(req) {
|
|
177
|
+
const notes = [];
|
|
178
|
+
const setup = {};
|
|
179
|
+
const target = deeplTargetCode(req.targetLocale);
|
|
180
|
+
if (req.formality && req.formality !== "default") {
|
|
181
|
+
const lang = (await this.targetLanguages()).get(target);
|
|
182
|
+
if (req.formality.startsWith("prefer_") || lang?.formality)
|
|
183
|
+
setup.formality = req.formality;
|
|
184
|
+
else
|
|
185
|
+
notes.push(`formality "${req.formality}" is not supported for ${target}; using the default`);
|
|
186
|
+
}
|
|
187
|
+
if (req.terms.length) {
|
|
188
|
+
const { tsv, count, dropped } = glossaryEntries(req.terms);
|
|
189
|
+
if (dropped.length)
|
|
190
|
+
notes.push(`glossary terms not usable on DeepL (empty, tab, line break or duplicate): ${dropped.join(", ")}`);
|
|
191
|
+
const langs = await this.glossaryLanguages();
|
|
192
|
+
const src = langs.get(deeplSourceCode(req.sourceLocale));
|
|
193
|
+
const tgt = langs.get(target);
|
|
194
|
+
if (count === 0) {
|
|
195
|
+
// nothing left to send
|
|
196
|
+
}
|
|
197
|
+
else if (!src || !tgt) {
|
|
198
|
+
notes.push(`DeepL glossaries do not support ${deeplSourceCode(req.sourceLocale)} -> ${target}; glossary terms not applied`);
|
|
199
|
+
}
|
|
200
|
+
else {
|
|
201
|
+
const g = await this.ensureGlossary(src, tgt, tsv);
|
|
202
|
+
setup.glossaryId = g.id;
|
|
203
|
+
notes.push(`${g.created ? "created" : "using"} DeepL glossary "${g.name}" (${count} term${count === 1 ? "" : "s"})`);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
this.setup.set(req.targetLocale, setup);
|
|
207
|
+
return notes;
|
|
208
|
+
}
|
|
209
|
+
async translateBatch(req) {
|
|
210
|
+
let context;
|
|
211
|
+
if (req.kind === "listing") {
|
|
212
|
+
context = "Store listing of a browser extension: the name, a short summary, and the long description.";
|
|
213
|
+
}
|
|
214
|
+
else {
|
|
215
|
+
// DeepL has one context per request and never sees our keys, so each note
|
|
216
|
+
// quotes the source text it belongs to. This is loose background only.
|
|
217
|
+
const notes = req.items
|
|
218
|
+
.filter((i) => i.description)
|
|
219
|
+
.map((i) => `"${i.text}": ${i.description}`)
|
|
220
|
+
.join("\n");
|
|
221
|
+
context = notes ? `Strings from a Chrome browser extension user interface. Notes per string:\n${notes}` : "";
|
|
222
|
+
}
|
|
223
|
+
const params = {
|
|
224
|
+
source_lang: deeplSourceCode(req.sourceLocale),
|
|
225
|
+
target_lang: deeplTargetCode(req.targetLocale),
|
|
226
|
+
tag_handling: "xml",
|
|
227
|
+
ignore_tags: ["x"],
|
|
228
|
+
preserve_formatting: true,
|
|
229
|
+
};
|
|
230
|
+
if (context)
|
|
231
|
+
params.context = context;
|
|
232
|
+
const setup = this.setup.get(req.targetLocale);
|
|
233
|
+
if (setup) {
|
|
234
|
+
if (setup.formality)
|
|
235
|
+
params.formality = setup.formality;
|
|
236
|
+
if (setup.glossaryId)
|
|
237
|
+
params.glossary_id = setup.glossaryId;
|
|
238
|
+
}
|
|
239
|
+
else if (req.formality?.startsWith("prefer_")) {
|
|
240
|
+
params.formality = req.formality;
|
|
241
|
+
}
|
|
242
|
+
// Listing text: each line is sent as its own text. In XML mode DeepL does not
|
|
243
|
+
// treat a line break as the end of a sentence, so list items could be merged.
|
|
244
|
+
// Blank lines and the spaces around a line are kept here and never sent.
|
|
245
|
+
// UI messages are sent whole, as before.
|
|
246
|
+
const split = req.items.map((item) => (req.kind === "listing" ? splitLines(item.text) : [{ lead: "", core: item.text, trail: "" }]));
|
|
247
|
+
const texts = split.flatMap((lines) => lines.filter((l) => l.core !== "" || lines.length === 1).map((l) => toDeeplXml(l.core)));
|
|
248
|
+
const out = [];
|
|
249
|
+
for (let i = 0; i < texts.length; i += DEEPL_MAX_TEXTS) {
|
|
250
|
+
const chunk = texts.slice(i, i + DEEPL_MAX_TEXTS);
|
|
251
|
+
const body = { ...params, text: chunk };
|
|
252
|
+
const data = (await withRetry(() => fetchJson(`${this.baseUrl}/v2/translate`, { method: "POST", headers: this.headers(), body: JSON.stringify(body) })));
|
|
253
|
+
const list = data.translations ?? [];
|
|
254
|
+
if (list.length !== chunk.length) {
|
|
255
|
+
throw new ProviderError(`DeepL returned ${list.length} texts for ${chunk.length} inputs`);
|
|
256
|
+
}
|
|
257
|
+
out.push(...list.map((t) => fromDeeplXml(t.text)));
|
|
258
|
+
}
|
|
259
|
+
const result = {};
|
|
260
|
+
let n = 0;
|
|
261
|
+
req.items.forEach((item, i) => {
|
|
262
|
+
const lines = split[i] ?? [];
|
|
263
|
+
result[item.key] = lines
|
|
264
|
+
.map((l) => (l.core === "" && lines.length > 1 ? l.lead + l.trail : l.lead + (out[n++] ?? "") + l.trail))
|
|
265
|
+
.join("\n");
|
|
266
|
+
});
|
|
267
|
+
return result;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { AnthropicProvider } from "./anthropic.js";
|
|
2
|
+
import { DeepLProvider } from "./deepl.js";
|
|
3
|
+
import { OllamaProvider } from "./ollama.js";
|
|
4
|
+
import { OpenAIProvider } from "./openai.js";
|
|
5
|
+
import { LLM_LIMITS, ProviderError } from "./types.js";
|
|
6
|
+
export const PROVIDER_NAMES = ["anthropic", "openai", "deepl", "ollama"];
|
|
7
|
+
const EFFORTS = ["low", "medium", "high", "xhigh", "max"];
|
|
8
|
+
export function createProvider(cfg) {
|
|
9
|
+
switch (cfg.name) {
|
|
10
|
+
case "anthropic": {
|
|
11
|
+
if (cfg.effort && !EFFORTS.includes(cfg.effort))
|
|
12
|
+
throw new ProviderError(`--effort must be one of ${EFFORTS.join(", ")}`);
|
|
13
|
+
return new AnthropicProvider({ model: cfg.model, effort: cfg.effort });
|
|
14
|
+
}
|
|
15
|
+
case "openai": {
|
|
16
|
+
const model = cfg.model ?? process.env.OPENAI_MODEL;
|
|
17
|
+
if (!model)
|
|
18
|
+
throw new ProviderError("openai needs a model: pass --model (or set OPENAI_MODEL)");
|
|
19
|
+
return new OpenAIProvider({ model });
|
|
20
|
+
}
|
|
21
|
+
case "ollama": {
|
|
22
|
+
const model = cfg.model ?? process.env.OLLAMA_MODEL;
|
|
23
|
+
if (!model)
|
|
24
|
+
throw new ProviderError('ollama needs a model: pass --model (see "ollama list")');
|
|
25
|
+
return new OllamaProvider({ model });
|
|
26
|
+
}
|
|
27
|
+
case "deepl":
|
|
28
|
+
return new DeepLProvider();
|
|
29
|
+
default:
|
|
30
|
+
throw new ProviderError(`unknown provider "${cfg.name}". Use one of: ${PROVIDER_NAMES.join(", ")}`);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
/** Batch limits without building a provider (no key needed). Used by --dry-run. */
|
|
34
|
+
export function limitsFor(name) {
|
|
35
|
+
if (name === "deepl")
|
|
36
|
+
return { maxItems: 50, maxChars: 30000 };
|
|
37
|
+
if (name === "ollama")
|
|
38
|
+
return { maxItems: 15, maxChars: 3000 };
|
|
39
|
+
return LLM_LIMITS;
|
|
40
|
+
}
|