ext-localize 0.0.0-stage → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,309 @@
1
+ import { existsSync } from "node:fs";
2
+ import { itemSize, makeBatches } from "./batch.js";
3
+ import { EMPTY_GLOSSARY, termsFor } from "./glossary.js";
4
+ import { isChromeLocale } from "./locales.js";
5
+ import { listLocaleDirs, messagesPath, readMessagesFile, writeJsonAtomic } from "./messages.js";
6
+ import { protect, restoreAndVerify } from "./placeholders.js";
7
+ import { OutputTruncatedError, ProviderError } from "./providers/types.js";
8
+ import { applyPlan, hashSource, hashTarget, planLocale, saveState, statePath } from "./state.js";
9
+ import { relPath } from "./paths.js";
10
+ function findPlaceholder(msg, name) {
11
+ const ph = msg.placeholders ?? {};
12
+ const key = Object.keys(ph).find((k) => k.toLowerCase() === name.toLowerCase());
13
+ return key ? ph[key] : undefined;
14
+ }
15
+ /** Build the protected item for one source message. */
16
+ export function prepareItem(key, msg, glossary, opts = {}) {
17
+ const p = protect(msg.message, glossary.keep, { formatTokens: opts.formatTokens });
18
+ const hints = [];
19
+ p.originals.forEach((orig, id) => {
20
+ const named = /^\$([A-Za-z0-9_@]+)\$$/.exec(orig);
21
+ if (named) {
22
+ const def = findPlaceholder(msg, named[1]);
23
+ const ex = def?.example ? ` (example: "${def.example}")` : "";
24
+ hints.push(`<x id="${id}"/> = placeholder ${orig}${ex}`);
25
+ }
26
+ else if (/^\$[1-9]$/.test(orig))
27
+ hints.push(`<x id="${id}"/> = runtime value ${orig}`);
28
+ else if (orig === "$$")
29
+ hints.push(`<x id="${id}"/> = a literal "$" sign`);
30
+ });
31
+ const item = { key, text: p.text };
32
+ const description = [msg.description, opts.maxLength ? `Maximum length: ${opts.maxLength} characters.` : undefined]
33
+ .filter(Boolean)
34
+ .join(" ");
35
+ if (description)
36
+ item.description = description;
37
+ if (hints.length)
38
+ item.markerHints = hints;
39
+ return { item, originals: p.originals, source: msg.message };
40
+ }
41
+ /** Run one batch; if the output was cut off, split the batch and try again. */
42
+ async function runBatch(provider, base, items, counter) {
43
+ counter.requests++;
44
+ try {
45
+ return await provider.translateBatch({ ...base, items });
46
+ }
47
+ catch (e) {
48
+ if (e instanceof OutputTruncatedError && items.length > 1) {
49
+ const mid = Math.ceil(items.length / 2);
50
+ const a = await runBatch(provider, base, items.slice(0, mid), counter);
51
+ const b = await runBatch(provider, base, items.slice(mid), counter);
52
+ return { ...a, ...b };
53
+ }
54
+ throw e;
55
+ }
56
+ }
57
+ async function pool(items, limit, fn) {
58
+ let next = 0;
59
+ const workers = Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, async () => {
60
+ while (next < items.length) {
61
+ const item = items[next++];
62
+ await fn(item);
63
+ }
64
+ });
65
+ await Promise.all(workers);
66
+ }
67
+ /** Stop the whole run after this many failed requests in a row (bad key, network down, wrong model). */
68
+ export const MAX_FAILURES_IN_ROW = 3;
69
+ /**
70
+ * How to react to an error thrown by a batch:
71
+ * - fatal: stop the run (401/403 auth, 404 wrong model or URL, 456 DeepL quota)
72
+ * - content: the answer was unusable (bad JSON, refusal); re-send keys one by one
73
+ * - transport: network or HTTP error after the provider's own retries; fail the batch once
74
+ */
75
+ export function classifyError(e) {
76
+ const status = e?.status;
77
+ if (typeof status === "number" && [401, 403, 404, 456].includes(status))
78
+ return "fatal";
79
+ // A single item too long for one answer: that item fails, the run goes on.
80
+ if (e instanceof ProviderError || e instanceof OutputTruncatedError)
81
+ return "content";
82
+ return "transport";
83
+ }
84
+ export function defaultTargets(project) {
85
+ return listLocaleDirs(project.localesDir).filter((l) => l !== project.defaultLocale && isChromeLocale(l));
86
+ }
87
+ /** The _locales/<locale>/messages.json job. */
88
+ export function messagesJob(project) {
89
+ return {
90
+ kind: "messages",
91
+ root: project.root,
92
+ sourceLocale: project.defaultLocale,
93
+ source: project.source,
94
+ section: "messages",
95
+ formatTokens: true,
96
+ read(locale) {
97
+ const path = messagesPath(project.localesDir, locale);
98
+ if (!existsSync(path))
99
+ return { ok: true, data: null };
100
+ const r = readMessagesFile(path);
101
+ return r.ok ? { ok: true, data: r.data } : { ok: false, error: r.error };
102
+ },
103
+ write(locale, file) {
104
+ writeJsonAtomic(messagesPath(project.localesDir, locale), file);
105
+ },
106
+ fileFor: (locale) => `_locales/${locale}/messages.json`,
107
+ };
108
+ }
109
+ /**
110
+ * Translate (or, with dryRun, plan) every target locale of _locales.
111
+ * Writes each locale file and the state file after that locale is done,
112
+ * so an interrupted run keeps the finished locales.
113
+ */
114
+ export async function translateProject(project, state, opts) {
115
+ const targets = opts.targets ?? defaultTargets(project);
116
+ return runJob(messagesJob(project), state, { ...opts, targets });
117
+ }
118
+ /** Run one translation job over its target locales. */
119
+ export async function runJob(job, state, opts) {
120
+ const log = opts.log ?? (() => { });
121
+ const glossary = opts.glossary ?? EMPTY_GLOSSARY;
122
+ const provider = opts.provider;
123
+ const baseLimits = provider?.limits ?? opts.limits ?? { maxItems: 40, maxChars: 8000 };
124
+ const limits = opts.batchSize ? { ...baseLimits, maxItems: opts.batchSize } : baseLimits;
125
+ const targets = opts.targets.filter((l) => l !== job.sourceLocale);
126
+ const source = job.source;
127
+ const checkFormat = job.formatTokens;
128
+ const prepared = new Map();
129
+ for (const [key, msg] of Object.entries(source)) {
130
+ prepared.set(key, prepareItem(key, msg, glossary, { formatTokens: job.formatTokens, maxLength: job.maxLength?.[key] }));
131
+ }
132
+ const results = [];
133
+ const ctx = { fatal: null, failuresInRow: 0 };
134
+ let totalChars = 0;
135
+ let totalPromptChars = 0;
136
+ let stateWritten = false;
137
+ await pool(targets, opts.dryRun ? targets.length : (opts.concurrency ?? 4), async (locale) => {
138
+ const emptyPlan = { locale, translate: [], remove: [], keep: [], adopt: [], manual: [] };
139
+ const r = job.read(locale);
140
+ if (!r.ok) {
141
+ results.push({ locale, plan: emptyPlan, requests: 0, translated: [], failed: [], written: [], skipped: `${r.error}; fix or delete the file first` });
142
+ return;
143
+ }
144
+ const target = r.data;
145
+ const sectionState = { messages: state.locales[locale]?.[job.section] ?? {} };
146
+ const plan = planLocale(source, target, sectionState, locale, { force: opts.force, checkFormat });
147
+ // Fixed keys are copied, never translated.
148
+ const fixed = job.fixed?.(locale) ?? {};
149
+ const isFree = (key) => !(key in fixed);
150
+ plan.translate = plan.translate.filter((t) => isFree(t.key));
151
+ plan.keep = plan.keep.filter(isFree);
152
+ plan.adopt = plan.adopt.filter(isFree);
153
+ plan.manual = plan.manual.filter(isFree);
154
+ const items = plan.translate.map((t) => prepared.get(t.key).item);
155
+ const batches = makeBatches(items, limits);
156
+ const result = { locale, plan, requests: batches.length, translated: [], failed: [], written: [] };
157
+ results.push(result);
158
+ for (const it of items) {
159
+ totalChars += it.text.length;
160
+ totalPromptChars += itemSize(it);
161
+ }
162
+ if (opts.dryRun)
163
+ return;
164
+ if (!provider)
165
+ throw new Error("no provider");
166
+ if (ctx.fatal) {
167
+ result.skipped = `not started: ${ctx.fatal.message}`;
168
+ result.requests = 0;
169
+ return;
170
+ }
171
+ try {
172
+ if (provider.supportsLocale && !(await provider.supportsLocale(locale))) {
173
+ result.skipped = `${provider.name} does not support this locale`;
174
+ result.requests = 0;
175
+ log(`${locale}: skipped (${result.skipped})`);
176
+ return;
177
+ }
178
+ }
179
+ catch (e) {
180
+ // The language list is shared by all locales, so this stops the run.
181
+ ctx.fatal ??= new Error(`cannot get the language list: ${e.message}`);
182
+ result.skipped = `not started: ${ctx.fatal.message}`;
183
+ result.requests = 0;
184
+ return;
185
+ }
186
+ const base = {
187
+ sourceLocale: job.sourceLocale,
188
+ targetLocale: locale,
189
+ terms: termsFor(glossary, locale),
190
+ formality: opts.formality,
191
+ kind: job.kind,
192
+ };
193
+ if (provider.prepareLocale && items.length) {
194
+ try {
195
+ for (const note of await provider.prepareLocale(base))
196
+ log(`${locale}: ${note}`);
197
+ }
198
+ catch (e) {
199
+ if (classifyError(e) === "fatal") {
200
+ ctx.fatal ??= e;
201
+ result.skipped = `not started: ${e.message}`;
202
+ result.requests = 0;
203
+ return;
204
+ }
205
+ // Not fatal: translate without the extras (glossary, formality).
206
+ log(`${locale}: setup failed, translating without glossary/formality: ${e.message}`);
207
+ }
208
+ }
209
+ const counter = { requests: 0 };
210
+ const good = {};
211
+ const errors = new Map();
212
+ // Pass 1 sends the batches. Pass 2 re-sends, one by one, only the keys that
213
+ // came back missing or broken (or whose batch had an unusable answer).
214
+ // A batch that fails on the network or HTTP level is not re-sent key by key.
215
+ let retryOne = [];
216
+ for (let pass = 0; pass < 2; pass++) {
217
+ const passBatches = pass === 0 ? batches : retryOne.map((i) => [i]);
218
+ const nextRetry = [];
219
+ for (const batch of passBatches) {
220
+ if (ctx.fatal) {
221
+ for (const i of batch)
222
+ errors.set(i.key, `not sent: ${ctx.fatal.message}`);
223
+ continue;
224
+ }
225
+ let out;
226
+ try {
227
+ out = await runBatch(provider, base, batch, counter);
228
+ ctx.failuresInRow = 0;
229
+ }
230
+ catch (e) {
231
+ const msg = e.message;
232
+ for (const i of batch)
233
+ errors.set(i.key, msg);
234
+ // An item that does not fit in one answer. Re-send the batch key by key, so the
235
+ // other keys are not lost; the long item alone will fail again and is reported.
236
+ if (e instanceof OutputTruncatedError) {
237
+ if (pass === 0 && batch.length > 1)
238
+ nextRetry.push(...batch);
239
+ continue;
240
+ }
241
+ const kind = classifyError(e);
242
+ if (kind === "content" && pass === 0)
243
+ nextRetry.push(...batch);
244
+ if (kind === "fatal")
245
+ ctx.fatal = e;
246
+ else if (kind === "transport" && ++ctx.failuresInRow >= MAX_FAILURES_IN_ROW) {
247
+ ctx.fatal = new Error(`${MAX_FAILURES_IN_ROW} requests failed in a row, stopping. Last error: ${msg}`);
248
+ }
249
+ continue;
250
+ }
251
+ for (const i of batch) {
252
+ const raw = out[i.key];
253
+ const p = prepared.get(i.key);
254
+ let v = raw === undefined ? { ok: false, error: "no translation returned" } : restoreAndVerify(p.source, raw, p.originals, { checkFormat });
255
+ const max = job.maxLength?.[i.key];
256
+ if (v.ok && max !== undefined && [...v.text].length > max) {
257
+ v = { ok: false, error: `too long: ${[...v.text].length} characters, limit ${max}` };
258
+ }
259
+ if (v.ok) {
260
+ good[i.key] = v.text;
261
+ errors.delete(i.key);
262
+ }
263
+ else {
264
+ errors.set(i.key, v.error);
265
+ if (pass === 0)
266
+ nextRetry.push(i);
267
+ }
268
+ }
269
+ }
270
+ retryOne = nextRetry;
271
+ if (!retryOne.length)
272
+ break;
273
+ }
274
+ result.failed = items.filter((i) => good[i.key] === undefined).map((i) => ({ key: i.key, error: errors.get(i.key) ?? "failed" }));
275
+ result.requests = counter.requests;
276
+ result.translated = Object.keys(good);
277
+ const { file, state: sectionNew } = applyPlan(source, target, sectionState, plan, good, provider.label);
278
+ for (const [key, value] of Object.entries(fixed)) {
279
+ const src = source[key];
280
+ if (!src)
281
+ continue;
282
+ file[key] = { message: value };
283
+ sectionNew.messages[key] = { sourceHash: hashSource(src), targetHash: hashTarget(value), by: "_locales" };
284
+ }
285
+ const changed = !target || JSON.stringify(file) !== JSON.stringify(target);
286
+ if (changed && Object.keys(file).length > 0) {
287
+ job.write(locale, file);
288
+ result.written.push(job.fileFor(locale));
289
+ }
290
+ // Replace only this job's section; the other section of the locale stays as it is.
291
+ state.locales[locale] = { messages: {}, ...state.locales[locale], [job.section]: sectionNew.messages };
292
+ if (saveState(job.root, state))
293
+ stateWritten = true;
294
+ log(`${locale}: ${result.translated.length} translated, ${plan.remove.length} removed, ${plan.keep.length} kept` +
295
+ (result.failed.length ? `, ${result.failed.length} FAILED` : "") +
296
+ ` (${result.requests} request${result.requests === 1 ? "" : "s"})`);
297
+ });
298
+ results.sort((a, b) => (a.locale < b.locale ? -1 : 1));
299
+ return {
300
+ stopped: ctx.fatal?.message,
301
+ locales: results,
302
+ totalRequests: results.reduce((n, r) => n + r.requests, 0),
303
+ totalChars,
304
+ totalPromptChars,
305
+ totalStrings: results.reduce((n, r) => n + r.plan.translate.length, 0),
306
+ failedCount: results.reduce((n, r) => n + r.failed.length, 0),
307
+ filesWritten: [...results.flatMap((r) => r.written), ...(stateWritten ? [relPath(job.root, statePath(job.root))] : [])].sort(),
308
+ };
309
+ }
package/dist/usage.js ADDED
@@ -0,0 +1,3 @@
1
+ /** An error the user can fix by changing the command line. The CLI exits with 2. */
2
+ export class UsageError extends Error {
3
+ }
package/dist/zip.js ADDED
@@ -0,0 +1,88 @@
1
+ import { existsSync, readFileSync } from "node:fs";
2
+ import { ConfigError } from "./config.js";
3
+ import { listListingLocales } from "./listing.js";
4
+ import { isChromeLocale } from "./locales.js";
5
+ /**
6
+ * Read the file names of a .zip (the package you upload to the store) from its
7
+ * central directory. No extraction, no dependency. ZIP64 archives are not supported.
8
+ */
9
+ export function zipEntries(path) {
10
+ if (!existsSync(path))
11
+ throw new ConfigError(`zip file not found: ${path}`);
12
+ const buf = readFileSync(path);
13
+ // End of central directory record: signature 0x06054b50, at most 65,535 + 22 bytes from the end.
14
+ let eocd = -1;
15
+ for (let i = buf.length - 22; i >= Math.max(0, buf.length - 65557); i--) {
16
+ if (buf.readUInt32LE(i) === 0x06054b50) {
17
+ eocd = i;
18
+ break;
19
+ }
20
+ }
21
+ if (eocd < 0)
22
+ throw new ConfigError(`${path} is not a zip file`);
23
+ const count = buf.readUInt16LE(eocd + 10);
24
+ const offset = buf.readUInt32LE(eocd + 16);
25
+ if (offset === 0xffffffff || count === 0xffff)
26
+ throw new ConfigError(`${path}: ZIP64 archives are not supported`);
27
+ const names = [];
28
+ let p = offset;
29
+ for (let n = 0; n < count; n++) {
30
+ if (p + 46 > buf.length || buf.readUInt32LE(p) !== 0x02014b50)
31
+ throw new ConfigError(`${path}: damaged zip central directory`);
32
+ const nameLen = buf.readUInt16LE(p + 28);
33
+ const extraLen = buf.readUInt16LE(p + 30);
34
+ const commentLen = buf.readUInt16LE(p + 32);
35
+ names.push(buf.subarray(p + 46, p + 46 + nameLen).toString("utf8"));
36
+ p += 46 + nameLen + extraLen + commentLen;
37
+ }
38
+ return names;
39
+ }
40
+ /** Locale folders with a messages.json inside the package (at the root, or under one top folder). */
41
+ export function zipLocales(path) {
42
+ const out = new Set();
43
+ for (const name of zipEntries(path)) {
44
+ const m = /^(?:[^/]+\/)?_locales\/([^/]+)\/messages\.json$/.exec(name);
45
+ if (m)
46
+ out.add(m[1]);
47
+ }
48
+ return [...out].sort();
49
+ }
50
+ /**
51
+ * Compare listing files with the package's _locales. The dashboard offers a
52
+ * language only when the uploaded package has that locale folder.
53
+ */
54
+ export function zipIssues(zipPath, listingDirAbs, listingRel, defaultLocale) {
55
+ const inZip = new Set(zipLocales(zipPath));
56
+ const listing = listListingLocales(listingDirAbs);
57
+ const issues = [];
58
+ for (const locale of listing) {
59
+ if (!inZip.has(locale)) {
60
+ issues.push({
61
+ level: "warning",
62
+ locale,
63
+ key: "listing.description",
64
+ message: `listing: the package ${zipPath} has no _locales/${locale}, so the dashboard will not offer this language`,
65
+ file: `${listingRel}/${locale}.json`,
66
+ });
67
+ }
68
+ }
69
+ for (const locale of inZip) {
70
+ // nb and other names the store does not list cannot have a listing file: skip them.
71
+ if (locale !== defaultLocale && isChromeLocale(locale) && !listing.includes(locale)) {
72
+ issues.push({
73
+ level: "warning",
74
+ locale,
75
+ key: "listing.description",
76
+ message: `listing: the package has _locales/${locale} but there is no ${listingRel}/${locale}.json, so its store description stays empty`,
77
+ file: `${listingRel}/${locale}.json`,
78
+ });
79
+ }
80
+ }
81
+ return issues;
82
+ }
83
+ /** Recompute the counts after issues were added. */
84
+ export function recount(report) {
85
+ report.errors = report.issues.filter((i) => i.level === "error").length;
86
+ report.warnings = report.issues.filter((i) => i.level === "warning").length;
87
+ report.stale = report.issues.filter((i) => i.kind === "stale").length;
88
+ }
package/package.json CHANGED
@@ -1,6 +1,57 @@
1
1
  {
2
2
  "name": "ext-localize",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
3
+ "version": "0.2.1",
4
+ "description": "Local-first CLI to translate, validate and deliver Chrome (MV3) extension _locales and store listings (Chrome, Firefox, Edge).",
5
+ "type": "module",
6
+ "bin": {
7
+ "ext-localize": "dist/cli.js"
8
+ },
9
+ "files": [
10
+ "dist",
11
+ "assets",
12
+ "README.md",
13
+ "LICENSE",
14
+ "CHANGELOG.md"
15
+ ],
16
+ "scripts": {
17
+ "build": "tsc -p tsconfig.json",
18
+ "typecheck": "tsc -p tsconfig.json --noEmit",
19
+ "test": "node --test \"test/**/*.test.ts\"",
20
+ "dev": "node src/cli.ts",
21
+ "prepublishOnly": "npm run typecheck && npm test && npm run build",
22
+ "prepack": "npm run build"
23
+ },
24
+ "keywords": [
25
+ "chrome-extension",
26
+ "i18n",
27
+ "l10n",
28
+ "localization",
29
+ "messages.json",
30
+ "translate",
31
+ "_locales",
32
+ "manifest-v3",
33
+ "firefox-addons",
34
+ "store-listing",
35
+ "chrome-web-store"
36
+ ],
37
+ "license": "MIT",
38
+ "engines": {
39
+ "node": ">=20"
40
+ },
41
+ "dependencies": {
42
+ "@anthropic-ai/sdk": "^1.8.0"
43
+ },
44
+ "peerDependencies": {
45
+ "playwright-core": ">=1.40.0"
46
+ },
47
+ "peerDependenciesMeta": {
48
+ "playwright-core": {
49
+ "optional": true
50
+ }
51
+ },
52
+ "devDependencies": {
53
+ "@types/node": "^24.0.0",
54
+ "playwright-core": "^1.65.1",
55
+ "typescript": "^7.0.2"
56
+ }
57
+ }