localewarden 0.1.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +161 -12
- package/dist/budget.d.ts +25 -0
- package/dist/budget.js +74 -0
- package/dist/checks.d.ts +34 -6
- package/dist/checks.js +111 -21
- package/dist/cli.js +66 -10
- package/dist/config.d.ts +37 -1
- package/dist/config.js +118 -3
- package/dist/engine/context.d.ts +81 -0
- package/dist/engine/context.js +75 -0
- package/dist/engine/copies.d.ts +9 -0
- package/dist/engine/copies.js +31 -0
- package/dist/engine/planner.d.ts +66 -0
- package/dist/engine/planner.js +167 -0
- package/dist/engine/repair.d.ts +16 -0
- package/dist/engine/repair.js +61 -0
- package/dist/engine/sources.d.ts +13 -0
- package/dist/engine/sources.js +40 -0
- package/dist/engine/translator.d.ts +29 -0
- package/dist/engine/translator.js +156 -0
- package/dist/engine/writer.d.ts +13 -0
- package/dist/engine/writer.js +63 -0
- package/dist/files.d.ts +30 -0
- package/dist/files.js +125 -12
- package/dist/index.d.ts +6 -3
- package/dist/index.js +6 -3
- package/dist/llm.d.ts +5 -8
- package/dist/llm.js +7 -15
- package/dist/lock.d.ts +13 -0
- package/dist/lock.js +91 -0
- package/dist/output.d.ts +21 -0
- package/dist/output.js +76 -0
- package/dist/placeholders.js +2 -1
- package/dist/plugins.d.ts +72 -0
- package/dist/plugins.js +71 -0
- package/dist/project.d.ts +33 -5
- package/dist/project.js +135 -42
- package/dist/prompt.d.ts +7 -1
- package/dist/prompt.js +10 -3
- package/dist/review.d.ts +1 -1
- package/dist/review.js +49 -23
- package/dist/scope.d.ts +15 -0
- package/dist/scope.js +26 -0
- package/dist/state.d.ts +11 -4
- package/dist/state.js +30 -11
- package/dist/translate.d.ts +9 -49
- package/dist/translate.js +119 -383
- package/dist/ui/data.d.ts +50 -0
- package/dist/ui/data.js +178 -0
- package/dist/ui/page.d.ts +5 -0
- package/dist/ui/page.js +277 -0
- package/dist/ui/server.d.ts +24 -0
- package/dist/ui/server.js +194 -0
- package/dist/util.d.ts +6 -2
- package/dist/util.js +17 -5
- package/package.json +12 -1
package/dist/checks.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { placeholderRegExp, placeholderSignature, placeholdersMatch } from './placeholders.js';
|
|
2
2
|
import { GENDERED_FORMS, NO_AMPERSAND_LANGUAGES, SENTENCE_CASE_LANGUAGES, formalityRule, registerFor, withoutQuotedSpeech, } from './style.js';
|
|
3
|
-
import {
|
|
3
|
+
import { PluginHost } from './plugins.js';
|
|
4
|
+
import { Scope } from './scope.js';
|
|
5
|
+
import { baseLanguage, escapeRegExp, isTraditionalChinese } from './util.js';
|
|
4
6
|
export const CHECKS = [
|
|
5
7
|
'placeholder',
|
|
6
8
|
'unsafe',
|
|
@@ -8,6 +10,7 @@ export const CHECKS = [
|
|
|
8
10
|
'markup',
|
|
9
11
|
'years',
|
|
10
12
|
'formality',
|
|
13
|
+
'length',
|
|
11
14
|
'titlecase',
|
|
12
15
|
'ampersand',
|
|
13
16
|
'glossary',
|
|
@@ -21,17 +24,39 @@ export const FIXABLE_CHECKS = new Set([
|
|
|
21
24
|
'markup',
|
|
22
25
|
'years',
|
|
23
26
|
'formality',
|
|
27
|
+
'length',
|
|
24
28
|
'titlecase',
|
|
25
29
|
'ampersand',
|
|
26
30
|
'glossary',
|
|
27
31
|
'partial',
|
|
28
32
|
]);
|
|
33
|
+
/** Errors break the software (and fail `localewarden check`); everything else is a warning. */
|
|
34
|
+
export const isError = (issue) => issue.severity === 'error' || ERROR_CHECKS.has(issue.check);
|
|
35
|
+
/** Whether `--fix-flagged` may ask the model to fix the issue. */
|
|
36
|
+
export const isFixable = (issue) => issue.fixable ?? FIXABLE_CHECKS.has(issue.check);
|
|
29
37
|
export class Checker {
|
|
30
38
|
config;
|
|
31
39
|
placeholderRe;
|
|
32
|
-
|
|
40
|
+
scope;
|
|
41
|
+
/** Words of termNotes and doNotTranslate: terms a translation may keep in the source language. */
|
|
42
|
+
keptWords;
|
|
43
|
+
plugins;
|
|
44
|
+
constructor(config, plugins = new PluginHost()) {
|
|
33
45
|
this.config = config;
|
|
46
|
+
this.plugins = plugins;
|
|
34
47
|
this.placeholderRe = placeholderRegExp(config.placeholders);
|
|
48
|
+
this.scope = new Scope(config);
|
|
49
|
+
this.keptWords = new Set([...Object.keys(config.termNotes), ...config.doNotTranslate].flatMap(term => term.toLowerCase().split(/[^\p{L}]+/u)).filter(Boolean));
|
|
50
|
+
}
|
|
51
|
+
/** The text without doNotTranslate names, which stay the same in every language. */
|
|
52
|
+
withoutNames(text) {
|
|
53
|
+
return this.config.doNotTranslate.reduce((value, name) => value.split(name).join(' '), text);
|
|
54
|
+
}
|
|
55
|
+
/** "62 characters, limit 60", or null. */
|
|
56
|
+
tooLong(key, text) {
|
|
57
|
+
const max = this.scope.maxLength(key);
|
|
58
|
+
const length = [...text].length;
|
|
59
|
+
return max !== undefined && length > max ? `${length} characters, limit ${max}` : null;
|
|
35
60
|
}
|
|
36
61
|
get englishSource() {
|
|
37
62
|
return baseLanguage(this.config.sourceLanguage) === 'en';
|
|
@@ -43,7 +68,7 @@ export class Checker {
|
|
|
43
68
|
return `source has [${placeholderSignature(source, this.placeholderRe)}], got [${placeholderSignature(text, this.placeholderRe)}]`;
|
|
44
69
|
}
|
|
45
70
|
/** All issues of one translated string. */
|
|
46
|
-
checkString(lang, key, source, text) {
|
|
71
|
+
checkString(lang, key, source, text, file = '') {
|
|
47
72
|
const issues = [];
|
|
48
73
|
const add = (check, note) => issues.push({ check, note });
|
|
49
74
|
const base = baseLanguage(lang);
|
|
@@ -88,16 +113,22 @@ export class Checker {
|
|
|
88
113
|
this.config.sentenceCase &&
|
|
89
114
|
SENTENCE_CASE_LANGUAGES.has(base) &&
|
|
90
115
|
isEnglishTitleCase(source)) {
|
|
91
|
-
const capitals = midCapitals(text, source, this.config.doNotTranslate);
|
|
116
|
+
const capitals = midCapitals(text, source, this.config.doNotTranslate, lang);
|
|
92
117
|
if (capitals.length >= (source.trim().split(/\s+/).length <= 3 ? 1 : 2)) {
|
|
93
118
|
add('titlecase', `capitalised: ${capitals.join(' ')}`);
|
|
94
119
|
}
|
|
95
120
|
}
|
|
121
|
+
const long = this.tooLong(key, text);
|
|
122
|
+
if (long)
|
|
123
|
+
add('length', long);
|
|
96
124
|
const ampersands = (value) => value.split(' & ').length - 1;
|
|
97
125
|
if (NO_AMPERSAND_LANGUAGES.has(base) && ampersands(text) > ampersands(source))
|
|
98
126
|
add('ampersand');
|
|
99
|
-
if (isUnchangedProse(source, text, this.placeholderRe))
|
|
127
|
+
if (isUnchangedProse(this.withoutNames(source), this.withoutNames(text), this.placeholderRe))
|
|
100
128
|
add('untranslated');
|
|
129
|
+
const short = muchShorter(lang, source, text);
|
|
130
|
+
if (short)
|
|
131
|
+
add('partial', short);
|
|
101
132
|
if (this.englishSource && text !== source) {
|
|
102
133
|
const copied = sourceRun(source, text);
|
|
103
134
|
if (copied)
|
|
@@ -105,13 +136,16 @@ export class Checker {
|
|
|
105
136
|
const lead = !copied ? sourceBoldLeadIn(source, text, this.config.doNotTranslate) : null;
|
|
106
137
|
if (lead)
|
|
107
138
|
add('partial', `bold lead-in still in the source language: "${lead}"`);
|
|
108
|
-
const mixed = !copied && !lead ? englishInNativeScript(lang, source, text, this.placeholderRe) : null;
|
|
139
|
+
const mixed = !copied && !lead ? englishInNativeScript(lang, source, text, this.placeholderRe, this.keptWords) : null;
|
|
109
140
|
if (mixed)
|
|
110
141
|
add('partial', mixed);
|
|
111
142
|
if (droppedHedge(lang, source, text)) {
|
|
112
143
|
add('partial', 'hedge dropped: the source says "tend to", the translation states it as certain');
|
|
113
144
|
}
|
|
114
145
|
}
|
|
146
|
+
for (const issue of this.plugins.checks({ lang, key, file, source, text })) {
|
|
147
|
+
issues.push({ severity: 'warning', fixable: false, ...issue });
|
|
148
|
+
}
|
|
115
149
|
return issues;
|
|
116
150
|
}
|
|
117
151
|
/**
|
|
@@ -121,13 +155,17 @@ export class Checker {
|
|
|
121
155
|
* soft: likely loss (tag count, missing year, source words left in). Retried once, then
|
|
122
156
|
* accepted; `localewarden check` keeps reporting it.
|
|
123
157
|
*/
|
|
124
|
-
defect(lang, key, source, text) {
|
|
158
|
+
defect(lang, key, source, text, file = '') {
|
|
125
159
|
if (!text.trim())
|
|
126
160
|
return { hard: 'empty translation', soft: null };
|
|
161
|
+
// A plugin error blocks writing, like a broken placeholder.
|
|
162
|
+
const pluginError = this.plugins.checks({ lang, key, file, source, text }).find(issue => issue.severity === 'error');
|
|
163
|
+
if (pluginError)
|
|
164
|
+
return { hard: `${pluginError.check}${pluginError.note ? `: ${pluginError.note}` : ''}`, soft: null };
|
|
127
165
|
const placeholders = this.placeholdersMatch(key, source, text) ? null : `placeholder mismatch: ${this.placeholderNote(source, text)}`;
|
|
128
166
|
const foreign = foreignScript(this.config.sourceLanguage, source) ? null : foreignScript(lang, text);
|
|
129
167
|
const links = hrefSignature(source) !== hrefSignature(text) ? `links changed: [${hrefSignature(source)}] -> [${hrefSignature(text)}]` : null;
|
|
130
|
-
const echoed = isUnchangedProse(source, text, this.placeholderRe) ? 'returned the source text unchanged' : null;
|
|
168
|
+
const echoed = isUnchangedProse(this.withoutNames(source), this.withoutNames(text), this.placeholderRe) ? 'returned the source text unchanged' : null;
|
|
131
169
|
const broken = brokenMarkup(source) ? null : brokenMarkup(text);
|
|
132
170
|
const boldMarkers = (value) => (value.match(/\*\*/g) ?? []).length % 2;
|
|
133
171
|
const brokenBold = boldMarkers(text) === 1 && boldMarkers(source) === 0 ? 'unbalanced ** markers' : null;
|
|
@@ -135,7 +173,7 @@ export class Checker {
|
|
|
135
173
|
const hard = placeholders ?? unsafeAdditions(source, text) ?? foreign ?? links ?? echoed ?? broken ?? brokenBold ?? droppedBullets(source, text) ?? leaked;
|
|
136
174
|
const tags = tagCount(source) !== tagCount(text) ? `${tagCount(source)} tags in the source, got ${tagCount(text)}` : null;
|
|
137
175
|
const copied = this.englishSource && text !== source ? sourceRun(source, text) : null;
|
|
138
|
-
return { hard, soft: hard ?? tags ?? yearDifference(source, text) ?? (copied ? `source text left in: "${copied}"` : null) };
|
|
176
|
+
return { hard, soft: hard ?? this.tooLong(key, text) ?? muchShorter(lang, source, text) ?? tags ?? yearDifference(source, text) ?? (copied ? `source text left in: "${copied}"` : null) };
|
|
139
177
|
}
|
|
140
178
|
}
|
|
141
179
|
// ---------------------------------------------------------------------------------------
|
|
@@ -188,8 +226,25 @@ const LATIN_LANGUAGES = new Set([
|
|
|
188
226
|
]);
|
|
189
227
|
// Latin glued to a lookalike alphabet inside one word ("Вarda": Cyrillic В + Latin arda).
|
|
190
228
|
const HOMOGLYPH_WORD = /(?=\p{L}*\p{Script=Latin})(?=\p{L}*[\p{Script=Cyrillic}\p{Script=Greek}])\p{L}+/u;
|
|
229
|
+
// Common characters that exist in only one of the two Chinese scripts (pairs at the same index).
|
|
230
|
+
const SIMPLIFIED_ONLY = '们这说时会来对个为发过还让现实动门问间题体关点应开东头书长见认学页电话语读写买卖钱网设计习惯觉机帮爱车钟儿气无边进选择检样经验数据项结种类业务环节';
|
|
231
|
+
const TRADITIONAL_ONLY = '們這說時會來對個為發過還讓現實動門問間題體關點應開東頭書長見認學頁電話語讀寫買賣錢網設計習慣覺機幫愛車鐘兒氣無邊進選擇檢樣經驗數據項結種類業務環節';
|
|
232
|
+
/** Simplified characters in Traditional Chinese text, or the reverse (2+ distinct ones). */
|
|
233
|
+
export function wrongChineseScript(lang, text) {
|
|
234
|
+
if (baseLanguage(lang) !== 'zh')
|
|
235
|
+
return null;
|
|
236
|
+
const traditional = isTraditionalChinese(lang);
|
|
237
|
+
const wrong = traditional ? SIMPLIFIED_ONLY : TRADITIONAL_ONLY;
|
|
238
|
+
const found = [...new Set([...text].filter(ch => wrong.includes(ch)))];
|
|
239
|
+
return found.length >= 2
|
|
240
|
+
? `${traditional ? 'Simplified' : 'Traditional'} Chinese characters in ${lang}: ${found.slice(0, 6).join('')}`
|
|
241
|
+
: null;
|
|
242
|
+
}
|
|
191
243
|
/** Why `text` contains letters that cannot belong to `lang`, or null. */
|
|
192
244
|
export function foreignScript(lang, text) {
|
|
245
|
+
const chinese = wrongChineseScript(lang, text);
|
|
246
|
+
if (chinese)
|
|
247
|
+
return chinese;
|
|
193
248
|
const base = baseLanguage(lang);
|
|
194
249
|
const native = NATIVE_SCRIPTS[base] ?? (LATIN_LANGUAGES.has(base) ? [] : null);
|
|
195
250
|
if (native === null)
|
|
@@ -218,13 +273,18 @@ const unescapeEntities = (text) => text
|
|
|
218
273
|
.replace(/"|�*34;|�*22;/gi, '"')
|
|
219
274
|
.replace(/�*39;|�*27;|'/gi, "'")
|
|
220
275
|
.replace(/:|�*58;|�*3a;/gi, ':');
|
|
221
|
-
|
|
222
|
-
const
|
|
276
|
+
// HTML elements, so text in angle brackets ("<minutes>", "<your name>") is not taken for markup.
|
|
277
|
+
const HTML_ELEMENTS = new Set(('a abbr address area article aside audio b base bdi bdo blockquote body br button canvas caption cite code col colgroup data datalist dd del details dfn dialog div dl dt em embed fieldset figcaption figure footer form frame frameset h1 h2 h3 h4 h5 h6 head header hr html i iframe img input ins kbd label legend li link main map mark math meta meter nav noscript object ol optgroup option output p param picture pre progress q rp rt ruby s samp script section select slot small source span strong style sub summary sup svg table tbody td template textarea tfoot th thead time title tr track u ul var video wbr animate foreignobject use image set').split(' '));
|
|
278
|
+
/** HTML element names, in lower case. Numbered <0> tags (react-i18next) are placeholders. */
|
|
279
|
+
const tagNames = (html) => new Set([...html.matchAll(/<\/?([a-z][\w-]*)/gi)].map(m => m[1].toLowerCase()).filter(tag => HTML_ELEMENTS.has(tag)));
|
|
223
280
|
const TEXT_ATTRIBUTES = new Set(['title', 'alt', 'aria-label', 'aria-description', 'placeholder']);
|
|
224
281
|
/** Every attribute as "name=value" (value without quotes), in lower case. */
|
|
225
282
|
const attributes = (html) => {
|
|
226
283
|
const found = new Set();
|
|
227
|
-
for (const [, inner] of html.matchAll(/<[a-z][\w-]
|
|
284
|
+
for (const [, tag, inner] of html.matchAll(/<([a-z][\w-]*)\s([^>]*)>?/gi)) {
|
|
285
|
+
// "<dakika 20)" (sw: under 20 minutes) is text, not a tag.
|
|
286
|
+
if (!HTML_ELEMENTS.has(tag.toLowerCase()))
|
|
287
|
+
continue;
|
|
228
288
|
for (const [, name, v1, v2, v3] of inner.matchAll(/([^\s"'=<>\/]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+)))?/g)) {
|
|
229
289
|
const attr = name.toLowerCase();
|
|
230
290
|
// Text attributes are translated along with the visible text; only their presence counts.
|
|
@@ -234,7 +294,10 @@ const attributes = (html) => {
|
|
|
234
294
|
}
|
|
235
295
|
return found;
|
|
236
296
|
};
|
|
237
|
-
|
|
297
|
+
// Script URLs in a link target or Markdown link. Plain "data:" in prose is a word (pt/it "date:").
|
|
298
|
+
const DANGEROUS_URL = /(?:(?:href|src|action|formaction|xlink:href|poster|background)\s*=\s*["']?|\]\()\s*(?:javascript|vbscript|data)\s*:/gi;
|
|
299
|
+
// Formatting a translator may add for emphasis or a title (no attributes): a markup warning, not a risk.
|
|
300
|
+
const HARMLESS_TAGS = new Set(['br', 'i', 'b', 'em', 'strong', 'u', 's', 'sub', 'sup', 'small', 'mark', 'q', 'cite']);
|
|
238
301
|
/**
|
|
239
302
|
* Markup the translation adds that the source does not have: a new tag type, a new or changed
|
|
240
303
|
* attribute, an event handler or a script URL. Translations are often rendered as raw HTML
|
|
@@ -244,7 +307,7 @@ const DANGEROUS_URL = /(?:javascript|vbscript|data)\s*:/gi;
|
|
|
244
307
|
export function unsafeAdditions(source, text) {
|
|
245
308
|
const [src, out] = [unescapeEntities(source), unescapeEntities(text)];
|
|
246
309
|
const srcTags = tagNames(src);
|
|
247
|
-
const newTags = [...tagNames(out)].filter(tag => !srcTags.has(tag) && tag
|
|
310
|
+
const newTags = [...tagNames(out)].filter(tag => !srcTags.has(tag) && !HARMLESS_TAGS.has(tag));
|
|
248
311
|
if (newTags.length > 0)
|
|
249
312
|
return `HTML tag not in the source: <${newTags.join('>, <')}>`;
|
|
250
313
|
const srcAttrs = attributes(src);
|
|
@@ -312,7 +375,8 @@ const asciiDigits = (text) => text.replace(/\p{Nd}/gu, ch => {
|
|
|
312
375
|
export function yearDifference(source, text) {
|
|
313
376
|
// Decades ("the 2020s") are written in words or with suffixes in many languages.
|
|
314
377
|
const decades = new Set(asciiDigits(source).match(/(?<!\d)(?:19|20)\d0(?=['’]?s\b)/g) ?? []);
|
|
315
|
-
|
|
378
|
+
// "1,900", "1.900" and "1 900" are numbers, not years: drop thousands separators first.
|
|
379
|
+
const years = (value) => (asciiDigits(value).replace(/(\d)[,.\u00a0\u202f ](?=\d{3}(?!\d))/g, '$1').match(/(?<!\d)(?:19|20)\d\d(?!\d)/g) ?? []).filter(y => !decades.has(y)).sort().join(',');
|
|
316
380
|
const [a, b] = [years(source), years(text)];
|
|
317
381
|
return a === b ? null : `source years [${a}], got [${b}]`;
|
|
318
382
|
}
|
|
@@ -359,20 +423,25 @@ export function keptNameMissing(name, source, text) {
|
|
|
359
423
|
// ---------------------------------------------------------------------------------------
|
|
360
424
|
// Title case
|
|
361
425
|
const COMMON_NAMES = new Set(['iOS', 'Android', 'Apple', 'Google', 'iPhone', 'iPad', 'Mac', 'Windows', 'Linux', 'AI', 'API', 'URL', 'PDF', 'FAQ', 'OK']);
|
|
362
|
-
|
|
426
|
+
// Polish capitalises "you" pronouns as a sign of respect ("W Twoim planie"): correct, not Title Case.
|
|
427
|
+
const POLISH_RESPECT = /^(?:Ty|Twój|Twoja|Twoje|Twojego|Twojej|Twoim|Twoją|Twoich|Twoimi|Ciebie|Cię|Tobie|Tobą|Wy|Wasz|Wasza|Wasze|Wam|Was|Wami)$/u;
|
|
428
|
+
function midCapitals(text, source, names, lang = '') {
|
|
363
429
|
const allowed = new Set([...COMMON_NAMES, ...names.flatMap(name => name.split(/\s+/))]);
|
|
364
430
|
const isName = (word) => allowed.has(word) ||
|
|
365
431
|
new RegExp(`(?<!\\p{L})${escapeRegExp(word)}(?!\\p{L})`, 'u').test(source) ||
|
|
366
432
|
[...allowed].some(name => name.length >= 4 && word.startsWith(name));
|
|
367
433
|
return text
|
|
368
|
-
|
|
434
|
+
// Commas and brackets start segments too: list items ("SMART: Specific, Measurable") and
|
|
435
|
+
// bracketed words ("Customize (Optional)") are capitalised in many languages.
|
|
436
|
+
.split(/[.!?:—–\n•|,;()]+/)
|
|
369
437
|
.flatMap(segment => {
|
|
370
438
|
const words = segment.trim().split(/\s+/);
|
|
371
439
|
const first = words.findIndex(word => /\p{L}/u.test(word));
|
|
372
440
|
return first === -1 ? [] : words.slice(first + 1);
|
|
373
441
|
})
|
|
374
442
|
.map(word => word.replace(/^[^\p{L}]+|[^\p{L}]+$/gu, ''))
|
|
375
|
-
.filter(word => word.length >= 3 && /^\p{Lu}\p{Ll}/u.test(word) && !isName(word))
|
|
443
|
+
.filter(word => word.length >= 3 && /^\p{Lu}\p{Ll}/u.test(word) && !isName(word))
|
|
444
|
+
.filter(word => !(baseLanguage(lang) === 'pl' && POLISH_RESPECT.test(word)));
|
|
376
445
|
}
|
|
377
446
|
function isEnglishTitleCase(source) {
|
|
378
447
|
const rest = source
|
|
@@ -430,26 +499,34 @@ function sourceBoldLeadIn(source, text, names) {
|
|
|
430
499
|
for (const lead of bold(source)) {
|
|
431
500
|
if (lead.length <= 15 || !/\p{Ll}{3}/u.test(lead) || names.some(name => lead.includes(name)))
|
|
432
501
|
continue;
|
|
502
|
+
// Only capitalised words ("Apple App Store:", "Google Play"): a name, kept on purpose.
|
|
503
|
+
if (lead.split(/\s+/).every(word => !/\p{L}/u.test(word) || /^[^\p{L}]*\p{Lu}/u.test(word)))
|
|
504
|
+
continue;
|
|
433
505
|
if (translated.has(lead))
|
|
434
506
|
return lead;
|
|
435
507
|
}
|
|
436
508
|
return null;
|
|
437
509
|
}
|
|
510
|
+
const ICU_HEADER = /\{\s*[\w.-]+\s*,\s*(?:plural|select|selectordinal)\s*,/g;
|
|
511
|
+
const ICU_HEADER_TEST = /\{\s*[\w.-]+\s*,\s*(?:plural|select|selectordinal)\s*,/;
|
|
438
512
|
// Loanwords commonly written in Latin script inside non-Latin text.
|
|
439
513
|
const LATIN_LOANWORDS = new Set(['email', 'online', 'offline', 'emoji', 'smartphone', 'podcast', 'podcasts', 'wifi', 'blog', 'login', 'like', 'likes']);
|
|
440
514
|
/** Source-language words left inside a non-Latin-script translation ("Settings → Privacy"). */
|
|
441
|
-
function englishInNativeScript(lang, source, text, placeholderRe) {
|
|
515
|
+
function englishInNativeScript(lang, source, text, placeholderRe, allowed) {
|
|
442
516
|
const base = baseLanguage(lang);
|
|
443
517
|
// Greek writes many anglicisms in Latin script; not checked.
|
|
444
518
|
if (!NATIVE_SCRIPTS[base] || base === 'el' || base === 'sr' || text === source)
|
|
445
519
|
return null;
|
|
446
520
|
const strip = (value) => withoutUrls(value.replace(/[\w.+-]+@[\w.-]+/g, ' ').replace(/<[^>]+>/g, ' '))
|
|
521
|
+
// ICU syntax ("{count, plural, one {…} other {…}}") is code, not English text.
|
|
522
|
+
.replace(ICU_HEADER, ' ')
|
|
523
|
+
.replace(ICU_HEADER_TEST.test(value) ? /(?:^|[\s}])(?:zero|one|two|few|many|other|=\d+|[\w-]+)\s*(?=\{)/g : /$^/g, ' ')
|
|
447
524
|
.replace(placeholderRe, ' ')
|
|
448
525
|
.replace(/[((][^))]*[))]/g, ' ')
|
|
449
526
|
.replace(/["“„«「『‘'][^"”“»」』’']*["”“»」』’']/g, ' ');
|
|
450
527
|
const sourceWords = new Set(strip(source).match(/\b[a-z]{4,}\b/g) ?? []);
|
|
451
528
|
const left = [
|
|
452
|
-
...new Set((strip(text).match(/(?<![\p{L}-])[a-z]{4,}(?![\p{L}])/gu) ?? []).filter(word => sourceWords.has(word) && !LATIN_LOANWORDS.has(word))),
|
|
529
|
+
...new Set((strip(text).match(/(?<![\p{L}-])[a-z]{4,}(?![\p{L}])/gu) ?? []).filter(word => sourceWords.has(word) && !LATIN_LOANWORDS.has(word) && !allowed.has(word))),
|
|
453
530
|
];
|
|
454
531
|
const menu = /\b[A-Z][a-z]+ → [A-Z][a-z]+/.exec(text);
|
|
455
532
|
if (menu && source.includes(menu[0]))
|
|
@@ -478,9 +555,22 @@ const HEDGE_MARKERS = {
|
|
|
478
555
|
cs: /tendenc|obvykle|často|zpravidla|většinou|bývá|sklon|snadno|častěji|mív/iu,
|
|
479
556
|
sk: /tendenc|obvykle|často|zvyčajne|väčšinou|býva|sklon|ľahko|častejšie|zvyk|skôr/iu,
|
|
480
557
|
el: /τείν|συχνά|συνήθως|τάση|συνήθ|εύκολα|συχνότερα/iu,
|
|
481
|
-
fi: /taipu|usein|yleensä|tapaa|tavallisesti|tuppaa|tyypillisesti|taipumus|helposti|useimmiten|herkästi/iu,
|
|
558
|
+
fi: /taipu|tapana|usein|yleensä|tapaa|tavallisesti|tuppaa|tyypillisesti|taipumus|helposti|useimmiten|herkästi/iu,
|
|
482
559
|
ca: /tendeix|tendència|sol|sovint|generalment|acostum|normalment|fàcilment|freqüent/iu,
|
|
483
560
|
};
|
|
561
|
+
// Chinese, Japanese and Korean need far fewer characters than English; Thai has no spaces.
|
|
562
|
+
const COMPACT_SCRIPTS = new Set(['zh', 'ja', 'ko']);
|
|
563
|
+
/**
|
|
564
|
+
* A translation with a fraction of the source's length lost content: cut off by the model, or
|
|
565
|
+
* the source grew after it was translated. Markup and placeholders are not counted.
|
|
566
|
+
*/
|
|
567
|
+
export function muchShorter(lang, source, text) {
|
|
568
|
+
const visible = (value) => [...value.replace(/<[^>]+>|\{\{?[^{}]*\}?\}/g, '').replace(/\s+/g, ' ').trim()].length;
|
|
569
|
+
const [a, b] = [visible(source), visible(text)];
|
|
570
|
+
// Real translations from English rarely drop below 60% (Chinese/Japanese/Korean: 25%).
|
|
571
|
+
const min = COMPACT_SCRIPTS.has(baseLanguage(lang)) ? 0.2 : 0.5;
|
|
572
|
+
return a >= 200 && b < a * min ? `translation much shorter than the source (${b} of ${a} characters): content missing?` : null;
|
|
573
|
+
}
|
|
484
574
|
function droppedHedge(lang, source, text) {
|
|
485
575
|
const markers = HEDGE_MARKERS[baseLanguage(lang)];
|
|
486
576
|
return Boolean(markers) && /\btends? to\b/i.test(source) && !markers.test(text);
|
package/dist/cli.js
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import fs from 'node:fs';
|
|
3
3
|
import path from 'node:path';
|
|
4
|
-
import { ConfigError, CONFIG_FILE, loadConfig } from './config.js';
|
|
4
|
+
import { ConfigError, CONFIG_FILE, loadConfig, selectGroups } from './config.js';
|
|
5
|
+
import { loadPlugins } from './plugins.js';
|
|
6
|
+
import { startUi } from './ui/server.js';
|
|
5
7
|
import { CHECKS } from './checks.js';
|
|
6
8
|
import { findSourceFiles } from './files.js';
|
|
7
9
|
import { isReasoningModel } from './llm.js';
|
|
8
|
-
import { checkProject, summaryTable } from './project.js';
|
|
10
|
+
import { checkProject, fixPlaceholders, summaryTable } from './project.js';
|
|
9
11
|
import { listReview, updateReview } from './review.js';
|
|
10
12
|
import { run } from './translate.js';
|
|
11
13
|
const HELP = `localewarden - incremental AI translation for JSON locale files
|
|
@@ -14,6 +16,7 @@ Usage:
|
|
|
14
16
|
localewarden [translate] [options] translate new and changed strings
|
|
15
17
|
localewarden check [options] quality check, no API calls (use in CI)
|
|
16
18
|
localewarden review [options] list and resolve hand-edited translations
|
|
19
|
+
localewarden ui [--port 4848] local web interface (127.0.0.1 only)
|
|
17
20
|
localewarden init create ${CONFIG_FILE}
|
|
18
21
|
|
|
19
22
|
Translate options:
|
|
@@ -21,16 +24,21 @@ Translate options:
|
|
|
21
24
|
--lang de,fr only these languages
|
|
22
25
|
--fix-flagged let the model fix strings the quality check flags
|
|
23
26
|
--retranslate-all translate every string again (hand edits stay protected)
|
|
27
|
+
--retranslate-files <patterns> translate every string of these files again, e.g. "locales/{lang}/onboarding.json"
|
|
28
|
+
--refresh-before <YYYY-MM-DD> translate again what was written before that day (or adopted)
|
|
29
|
+
--group app,web only these groups (config "groups")
|
|
24
30
|
--overwrite-manual also replace hand-edited translations
|
|
25
31
|
--max-tokens <n> token budget for this run (default: maxTokensPerRun)
|
|
26
32
|
--verbose more output
|
|
27
33
|
|
|
28
34
|
Check options:
|
|
29
35
|
--lang de,fr only these languages
|
|
36
|
+
--group app,web only these groups
|
|
30
37
|
--verbose, -v list findings, not only counts
|
|
31
38
|
--limit <n> findings shown per check with --verbose (default 20)
|
|
32
|
-
--strict exit 1 on warnings too (default: only
|
|
39
|
+
--strict exit 1 on warnings too (default: only errors)
|
|
33
40
|
--json print findings as JSON
|
|
41
|
+
--fix repair placeholders with exactly one possible fix ({heures} -> {hours})
|
|
34
42
|
|
|
35
43
|
Review options:
|
|
36
44
|
--all include approved entries
|
|
@@ -42,7 +50,7 @@ Global:
|
|
|
42
50
|
--config <path> config file (default ./${CONFIG_FILE})
|
|
43
51
|
--help, --version
|
|
44
52
|
`;
|
|
45
|
-
const VALUE_FLAGS = new Set(['--lang', '--max-tokens', '--config', '--limit', '--approve', '--release']);
|
|
53
|
+
const VALUE_FLAGS = new Set(['--port', '--lang', '--group', '--max-tokens', '--config', '--limit', '--approve', '--release', '--retranslate-files', '--refresh-before']);
|
|
46
54
|
function parseArgs(argv) {
|
|
47
55
|
const args = { command: 'translate', flags: new Set(), values: new Map() };
|
|
48
56
|
let commandSet = false;
|
|
@@ -75,6 +83,7 @@ function parseArgs(argv) {
|
|
|
75
83
|
}
|
|
76
84
|
return args;
|
|
77
85
|
}
|
|
86
|
+
const listArg = (args, name) => args.values.get(name)?.flatMap(value => value.split(',')).map(v => v.trim()).filter(Boolean);
|
|
78
87
|
const languagesArg = (args) => args.values.get('--lang')?.flatMap(value => value.split(',')).map(l => l.trim()).filter(Boolean);
|
|
79
88
|
function version() {
|
|
80
89
|
const pkg = JSON.parse(fs.readFileSync(new URL('../package.json', import.meta.url), 'utf8'));
|
|
@@ -93,6 +102,8 @@ const LAYOUTS = [
|
|
|
93
102
|
'src/assets/i18n/{lang}.json',
|
|
94
103
|
'i18n/{lang}.json',
|
|
95
104
|
'lang/{lang}.json',
|
|
105
|
+
'lib/l10n/app_{lang}.arb',
|
|
106
|
+
'lib/l10n/intl_{lang}.arb',
|
|
96
107
|
];
|
|
97
108
|
function init() {
|
|
98
109
|
const file = path.resolve(CONFIG_FILE);
|
|
@@ -128,8 +139,14 @@ async function translateCommand(args) {
|
|
|
128
139
|
const maxTokens = args.values.get('--max-tokens')?.[0];
|
|
129
140
|
if (maxTokens !== undefined && !(Number(maxTokens) > 0))
|
|
130
141
|
throw new ConfigError('--max-tokens must be a positive number');
|
|
142
|
+
const refreshBefore = args.values.get('--refresh-before')?.[0];
|
|
143
|
+
if (refreshBefore !== undefined && !/^\d{4}-\d{2}-\d{2}$/.test(refreshBefore))
|
|
144
|
+
throw new ConfigError('--refresh-before must be a date like 2026-10-01');
|
|
131
145
|
const summary = await run(config, {
|
|
132
146
|
languages: languagesArg(args),
|
|
147
|
+
groups: listArg(args, '--group'),
|
|
148
|
+
retranslateFiles: listArg(args, '--retranslate-files'),
|
|
149
|
+
refreshBefore,
|
|
133
150
|
dryRun: args.flags.has('--dry-run'),
|
|
134
151
|
retranslateAll: args.flags.has('--retranslate-all'),
|
|
135
152
|
overwriteManual: args.flags.has('--overwrite-manual'),
|
|
@@ -147,6 +164,7 @@ async function translateCommand(args) {
|
|
|
147
164
|
}
|
|
148
165
|
// Rough: prompt overhead per request plus input and output text (~4 characters per token).
|
|
149
166
|
const requests = Math.ceil(strings / config.batchSize) + Math.ceil(strings * 0.05);
|
|
167
|
+
// Groups may use different models; the estimate uses the first group's.
|
|
150
168
|
const low = Math.round(requests * 900 + (chars / 4) * 2.2);
|
|
151
169
|
const high = Math.round(low * (isReasoningModel(config.model) ? 3 : 1.5));
|
|
152
170
|
console.log(`Dry run: ${strings} string(s), ${chars.toLocaleString('en')} characters to translate.`);
|
|
@@ -168,8 +186,10 @@ async function translateCommand(args) {
|
|
|
168
186
|
if (rows.length === 0)
|
|
169
187
|
console.log('Everything is up to date.');
|
|
170
188
|
console.log(`\n${summary.filesWritten.length} file(s) written, ${summary.requests} request(s), ${summary.tokens.toLocaleString('en')} tokens.`);
|
|
189
|
+
if (summary.filesCopied.length > 0)
|
|
190
|
+
console.log(`${summary.filesCopied.length} file(s) copied to their regional locales.`);
|
|
171
191
|
if (summary.stoppedByBudget)
|
|
172
|
-
console.log(
|
|
192
|
+
console.log(`Stopped: ${summary.stopReason ?? 'token budget reached'}. Run again to continue.`);
|
|
173
193
|
if (summary.pendingReview > 0)
|
|
174
194
|
console.log(`${summary.pendingReview} hand-edited translation(s) to review: npx localewarden review`);
|
|
175
195
|
const failed = Object.values(summary.languages).some(s => s.failed > 0);
|
|
@@ -177,10 +197,22 @@ async function translateCommand(args) {
|
|
|
177
197
|
console.log('Failed strings were not written and will be retried on the next run.');
|
|
178
198
|
return 0;
|
|
179
199
|
}
|
|
180
|
-
function checkCommand(args) {
|
|
200
|
+
async function checkCommand(args) {
|
|
181
201
|
const config = loadConfig(args.values.get('--config')?.[0]);
|
|
182
|
-
const
|
|
183
|
-
const
|
|
202
|
+
const plugins = await loadPlugins(config);
|
|
203
|
+
const groupNames = listArg(args, '--group');
|
|
204
|
+
const groups = selectGroups(config, groupNames, languagesArg(args));
|
|
205
|
+
const languages = languagesArg(args) ?? [...new Set(groups.flatMap(g => g.targetLanguages))];
|
|
206
|
+
const options = { languages, groups: groupNames, plugins };
|
|
207
|
+
let findings = checkProject(config, options);
|
|
208
|
+
if (args.flags.has('--fix')) {
|
|
209
|
+
const fixed = fixPlaceholders(config, findings);
|
|
210
|
+
for (const f of fixed)
|
|
211
|
+
console.log(`fixed [${f.lang}] ${f.file} ${f.key}: ${f.text}`);
|
|
212
|
+
console.log(`${fixed.length} placeholder(s) repaired.\n`);
|
|
213
|
+
if (fixed.length > 0)
|
|
214
|
+
findings = checkProject(config, options);
|
|
215
|
+
}
|
|
184
216
|
if (args.flags.has('--json')) {
|
|
185
217
|
console.log(JSON.stringify(findings, null, 2));
|
|
186
218
|
}
|
|
@@ -200,7 +232,17 @@ function checkCommand(args) {
|
|
|
200
232
|
console.log(` ... ${items.length - limit} more (--limit <n>)`);
|
|
201
233
|
}
|
|
202
234
|
}
|
|
203
|
-
|
|
235
|
+
// Errors fail CI, so show them even without --verbose.
|
|
236
|
+
const errorItems = findings.filter(f => f.severity === 'error');
|
|
237
|
+
if (!args.flags.has('--verbose') && errorItems.length > 0) {
|
|
238
|
+
console.log('\nErrors:');
|
|
239
|
+
for (const f of errorItems.slice(0, 20)) {
|
|
240
|
+
console.log(` [${f.lang}] ${f.file} ${f.key} - ${f.check}${f.note ? `: ${f.note}` : ''}`);
|
|
241
|
+
}
|
|
242
|
+
if (errorItems.length > 20)
|
|
243
|
+
console.log(` ... ${errorItems.length - 20} more (--verbose)`);
|
|
244
|
+
}
|
|
245
|
+
const errors = errorItems.length;
|
|
204
246
|
console.log(`\n${errors} error(s), ${findings.length - errors} warning(s).${findings.length && !args.flags.has('--verbose') ? ' Details: --verbose' : ''}`);
|
|
205
247
|
}
|
|
206
248
|
const errors = findings.some(f => f.severity === 'error');
|
|
@@ -231,6 +273,9 @@ function reviewCommand(args) {
|
|
|
231
273
|
return 0;
|
|
232
274
|
}
|
|
233
275
|
async function main() {
|
|
276
|
+
const major = Number(process.versions.node.split('.')[0]);
|
|
277
|
+
if (major < 20)
|
|
278
|
+
throw new ConfigError(`localewarden needs Node.js 20 or newer; this is ${process.versions.node}.`);
|
|
234
279
|
const args = parseArgs(process.argv.slice(2));
|
|
235
280
|
if (args.flags.has('--help')) {
|
|
236
281
|
console.log(HELP);
|
|
@@ -244,12 +289,23 @@ async function main() {
|
|
|
244
289
|
case 'translate':
|
|
245
290
|
return translateCommand(args);
|
|
246
291
|
case 'check':
|
|
247
|
-
return checkCommand(args);
|
|
292
|
+
return await checkCommand(args);
|
|
248
293
|
case 'review':
|
|
249
294
|
return reviewCommand(args);
|
|
250
295
|
case 'init':
|
|
251
296
|
init();
|
|
252
297
|
return 0;
|
|
298
|
+
case 'ui': {
|
|
299
|
+
const config = loadConfig(args.values.get('--config')?.[0]);
|
|
300
|
+
const port = args.values.get('--port')?.[0];
|
|
301
|
+
if (port !== undefined && !/^\d+$/.test(port))
|
|
302
|
+
throw new ConfigError('--port must be a number');
|
|
303
|
+
const ui = await startUi(config, { port: port === undefined ? undefined : Number(port) });
|
|
304
|
+
console.log(`localewarden interface: ${ui.url}`);
|
|
305
|
+
console.log('Only reachable from this computer. Press Ctrl+C to stop.');
|
|
306
|
+
await new Promise(() => { }); // runs until Ctrl+C
|
|
307
|
+
return 0;
|
|
308
|
+
}
|
|
253
309
|
default:
|
|
254
310
|
throw new ConfigError(`Unknown command "${args.command}". See --help.`);
|
|
255
311
|
}
|
package/dist/config.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { PluginSpec } from './plugins.js';
|
|
1
2
|
export type Register = 'formal' | 'informal';
|
|
2
3
|
export interface Config {
|
|
3
4
|
/** Language of the source files. Default "en". Several checks assume English. */
|
|
@@ -24,6 +25,16 @@ export interface Config {
|
|
|
24
25
|
termNotes: Record<string, string>;
|
|
25
26
|
/** Extra instructions per language; "*" applies to every language. */
|
|
26
27
|
instructions: Record<string, string>;
|
|
28
|
+
/**
|
|
29
|
+
* Keys whose values are not text (ids, types, image paths). Copied from the source, never
|
|
30
|
+
* translated. "*" matches within one key segment, "**" across segments; a pattern without
|
|
31
|
+
* a dot matches the last segment anywhere ("id" matches "steps.2.id").
|
|
32
|
+
*/
|
|
33
|
+
ignoreKeys: string[];
|
|
34
|
+
/** Source files to skip, as path patterns relative to the config ("locales/{lang}/nav.json"). */
|
|
35
|
+
exclude: string[];
|
|
36
|
+
/** Maximum characters per key pattern: { "**.meta.title": 60 }. Told to the model and checked. */
|
|
37
|
+
maxLength: Record<string, number>;
|
|
27
38
|
/** Regular expressions (as strings) that match placeholders. Replaces the built-in list. */
|
|
28
39
|
placeholders?: string[];
|
|
29
40
|
model: string;
|
|
@@ -42,14 +53,39 @@ export interface Config {
|
|
|
42
53
|
batchSize: number;
|
|
43
54
|
/** Where state, review list and repair report live, relative to the config file. */
|
|
44
55
|
stateDir: string;
|
|
56
|
+
/**
|
|
57
|
+
* Target locales that are a copy of another one instead of a translation, e.g. store
|
|
58
|
+
* listings: { "en-GB": "en-US", "fr-CA": "fr-FR" }. The value may be the source language.
|
|
59
|
+
*/
|
|
60
|
+
copies: Record<string, string>;
|
|
61
|
+
/** Strings longer than this many characters are translated paragraph by paragraph. */
|
|
62
|
+
chunkChars: number;
|
|
63
|
+
/** Plugin modules (relative to the config file), see plugins.ts. */
|
|
64
|
+
plugins: PluginSpec[];
|
|
65
|
+
/** Token limit per UTC day across runs; usage is kept in <stateDir>/usage.json. */
|
|
66
|
+
dailyTokenBudget?: number;
|
|
67
|
+
/** Group name, when this config is one of `groups`. */
|
|
68
|
+
name?: string;
|
|
69
|
+
/**
|
|
70
|
+
* Parts of the project with their own files and settings, translated in this order
|
|
71
|
+
* (e.g. app UI first, long-form content last). Each group inherits the top-level settings.
|
|
72
|
+
*/
|
|
73
|
+
groups?: Config[];
|
|
45
74
|
/** Directory of the config file; all relative paths resolve against it. */
|
|
46
75
|
root: string;
|
|
47
76
|
}
|
|
48
77
|
export declare const CONFIG_FILE = "localewarden.config.json";
|
|
49
|
-
export declare const DEFAULTS: Omit<Config, 'targetLanguages' | 'files' | 'root'>;
|
|
78
|
+
export declare const DEFAULTS: Omit<Config, 'targetLanguages' | 'files' | 'root' | 'groups' | 'name'>;
|
|
50
79
|
export declare class ConfigError extends Error {
|
|
51
80
|
}
|
|
52
81
|
/** Validates a parsed config object and fills in defaults. */
|
|
53
82
|
export declare function resolveConfig(raw: unknown, root: string): Config;
|
|
83
|
+
/** The groups of a config, or the config itself as its only group. */
|
|
84
|
+
export declare const groupsOf: (config: Config) => Config[];
|
|
85
|
+
/**
|
|
86
|
+
* The groups to work on, in config order, limited by name; unknown group or language names
|
|
87
|
+
* are an error (a typo must not turn into "nothing to do" or a green check).
|
|
88
|
+
*/
|
|
89
|
+
export declare function selectGroups(config: Config, names?: string[], languages?: string[]): Config[];
|
|
54
90
|
/** Loads the config file (default: localewarden.config.json in the working directory). */
|
|
55
91
|
export declare function loadConfig(configPath?: string): Config;
|