localewarden 0.1.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +161 -12
- package/dist/budget.d.ts +25 -0
- package/dist/budget.js +74 -0
- package/dist/checks.d.ts +34 -6
- package/dist/checks.js +111 -21
- package/dist/cli.js +66 -10
- package/dist/config.d.ts +37 -1
- package/dist/config.js +118 -3
- package/dist/engine/context.d.ts +81 -0
- package/dist/engine/context.js +75 -0
- package/dist/engine/copies.d.ts +9 -0
- package/dist/engine/copies.js +31 -0
- package/dist/engine/planner.d.ts +66 -0
- package/dist/engine/planner.js +167 -0
- package/dist/engine/repair.d.ts +16 -0
- package/dist/engine/repair.js +61 -0
- package/dist/engine/sources.d.ts +13 -0
- package/dist/engine/sources.js +40 -0
- package/dist/engine/translator.d.ts +29 -0
- package/dist/engine/translator.js +156 -0
- package/dist/engine/writer.d.ts +13 -0
- package/dist/engine/writer.js +63 -0
- package/dist/files.d.ts +30 -0
- package/dist/files.js +125 -12
- package/dist/index.d.ts +6 -3
- package/dist/index.js +6 -3
- package/dist/llm.d.ts +5 -8
- package/dist/llm.js +7 -15
- package/dist/lock.d.ts +13 -0
- package/dist/lock.js +91 -0
- package/dist/output.d.ts +21 -0
- package/dist/output.js +76 -0
- package/dist/placeholders.js +2 -1
- package/dist/plugins.d.ts +72 -0
- package/dist/plugins.js +71 -0
- package/dist/project.d.ts +33 -5
- package/dist/project.js +135 -42
- package/dist/prompt.d.ts +7 -1
- package/dist/prompt.js +10 -3
- package/dist/review.d.ts +1 -1
- package/dist/review.js +49 -23
- package/dist/scope.d.ts +15 -0
- package/dist/scope.js +26 -0
- package/dist/state.d.ts +11 -4
- package/dist/state.js +30 -11
- package/dist/translate.d.ts +9 -49
- package/dist/translate.js +119 -383
- package/dist/ui/data.d.ts +50 -0
- package/dist/ui/data.js +178 -0
- package/dist/ui/page.d.ts +5 -0
- package/dist/ui/page.js +277 -0
- package/dist/ui/server.d.ts +24 -0
- package/dist/ui/server.js +194 -0
- package/dist/util.d.ts +6 -2
- package/dist/util.js +17 -5
- package/package.json +12 -1
package/dist/project.js
CHANGED
|
@@ -1,66 +1,159 @@
|
|
|
1
1
|
import path from 'node:path';
|
|
2
|
-
import { CHECKS, Checker,
|
|
3
|
-
import {
|
|
2
|
+
import { CHECKS, Checker, isError } from './checks.js';
|
|
3
|
+
import { groupsOf, selectGroups } from './config.js';
|
|
4
|
+
import { listFiles } from './engine/sources.js';
|
|
5
|
+
import { findSourceFiles, flatten, isArbMetadata, parseDoc, readText, writeText } from './files.js';
|
|
6
|
+
import { PluginHost } from './plugins.js';
|
|
4
7
|
import { reviewId, State } from './state.js';
|
|
5
8
|
import { hash } from './util.js';
|
|
9
|
+
/**
|
|
10
|
+
* Short sibling strings (same parent key) that differ in the source but got the same
|
|
11
|
+
* translation: answer options, tabs or menu items the user can no longer tell apart.
|
|
12
|
+
*/
|
|
13
|
+
export function collapsedSiblings(source, target) {
|
|
14
|
+
const groups = new Map();
|
|
15
|
+
for (const key of target.keys()) {
|
|
16
|
+
// Only nested keys: in a flat file every string would be a "sibling" of every other.
|
|
17
|
+
if (!key.includes('.'))
|
|
18
|
+
continue;
|
|
19
|
+
const parent = key.slice(0, key.lastIndexOf('.'));
|
|
20
|
+
// Plural forms of one string are meant to look alike.
|
|
21
|
+
if (/_(zero|one|two|few|many|other)$/.test(key))
|
|
22
|
+
continue;
|
|
23
|
+
groups.set(parent, [...(groups.get(parent) ?? []), key]);
|
|
24
|
+
}
|
|
25
|
+
const out = [];
|
|
26
|
+
for (const keys of groups.values()) {
|
|
27
|
+
const seen = new Map();
|
|
28
|
+
for (const key of keys) {
|
|
29
|
+
const src = source.get(key)?.trim();
|
|
30
|
+
const text = target.get(key)?.trim().toLocaleLowerCase();
|
|
31
|
+
if (!src || !text || src.length > 40)
|
|
32
|
+
continue;
|
|
33
|
+
const earlier = seen.get(text);
|
|
34
|
+
if (earlier !== undefined && source.get(earlier)?.trim().toLocaleLowerCase() !== src.toLocaleLowerCase()) {
|
|
35
|
+
out.push({ key, note: `same translation as "${earlier}", although the source differs ("${source.get(earlier)}" / "${src}")` });
|
|
36
|
+
}
|
|
37
|
+
else if (earlier === undefined) {
|
|
38
|
+
seen.set(text, key);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return out;
|
|
43
|
+
}
|
|
6
44
|
function readJson(file) {
|
|
7
45
|
const text = readText(file);
|
|
8
|
-
return text === null ? null :
|
|
46
|
+
return text === null ? null : parseDoc(file, text);
|
|
9
47
|
}
|
|
10
48
|
/**
|
|
11
|
-
* Runs the quality checks over every translated string. Approved
|
|
12
|
-
* except for placeholder
|
|
49
|
+
* Runs the quality checks over every translated string of every group. Approved strings are
|
|
50
|
+
* skipped except for errors (placeholder, unsafe, script, plugin errors), which break the
|
|
51
|
+
* software either way.
|
|
13
52
|
*/
|
|
14
|
-
export function checkProject(config,
|
|
15
|
-
const
|
|
53
|
+
export function checkProject(config, optionsOrLanguages = {}) {
|
|
54
|
+
const options = Array.isArray(optionsOrLanguages) ? { languages: optionsOrLanguages } : optionsOrLanguages;
|
|
55
|
+
const host = new PluginHost(options.plugins ?? []);
|
|
16
56
|
const state = new State(path.join(config.root, config.stateDir));
|
|
17
57
|
const findings = [];
|
|
18
|
-
for (const
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
const rel = file.pathFor(lang);
|
|
25
|
-
let doc;
|
|
26
|
-
try {
|
|
27
|
-
doc = readJson(path.join(config.root, rel));
|
|
28
|
-
}
|
|
29
|
-
catch (error) {
|
|
30
|
-
findings.push({ lang, check: 'markup', severity: 'error', file: rel, key: '(file)', text: '', note: `invalid JSON: ${error.message}` });
|
|
31
|
-
continue;
|
|
32
|
-
}
|
|
33
|
-
if (doc === null)
|
|
58
|
+
for (const group of selectGroups(config, options.groups, options.languages)) {
|
|
59
|
+
const checker = new Checker(group, host);
|
|
60
|
+
const languages = group.targetLanguages.filter(lang => !options.languages?.length || options.languages.includes(lang));
|
|
61
|
+
for (const file of listFiles(group, checker.scope, host)) {
|
|
62
|
+
const sourceDoc = readJson(path.join(group.root, file.pathFor(group.sourceLanguage)));
|
|
63
|
+
if (sourceDoc === null)
|
|
34
64
|
continue;
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
const
|
|
38
|
-
|
|
65
|
+
const source = flatten(sourceDoc);
|
|
66
|
+
for (const lang of languages) {
|
|
67
|
+
const rel = file.pathFor(lang);
|
|
68
|
+
let doc;
|
|
69
|
+
try {
|
|
70
|
+
doc = readJson(path.join(group.root, rel));
|
|
71
|
+
}
|
|
72
|
+
catch (error) {
|
|
73
|
+
findings.push({ lang, check: 'markup', severity: 'error', file: rel, key: '(file)', text: '', note: `invalid JSON: ${error.message}`, group: group.name });
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
if (doc === null)
|
|
39
77
|
continue;
|
|
40
|
-
const
|
|
41
|
-
const
|
|
42
|
-
|
|
43
|
-
if (
|
|
78
|
+
const translated = flatten(doc);
|
|
79
|
+
for (const { key, note } of collapsedSiblings(source, translated)) {
|
|
80
|
+
const review = state.review[reviewId(lang, file.id, key)];
|
|
81
|
+
if (review?.status === 'approved')
|
|
44
82
|
continue;
|
|
45
|
-
findings.push({
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
83
|
+
findings.push({ lang, check: 'partial', severity: 'warning', file: rel, key, text: translated.get(key) ?? '', note, group: group.name });
|
|
84
|
+
}
|
|
85
|
+
for (const [key, text] of translated) {
|
|
86
|
+
// Plural forms only the target language has are checked against the source "_other".
|
|
87
|
+
const sourceText = source.get(key) ?? (/_(zero|one|two|few|many)$/.test(key) ? source.get(key.replace(/_(zero|one|two|few|many)$/, '_other')) : undefined);
|
|
88
|
+
if (sourceText === undefined || sourceText.trim() === '' || isArbMetadata(rel, key) || checker.scope.isLiteral(key, sourceText))
|
|
89
|
+
continue;
|
|
90
|
+
const review = state.review[reviewId(lang, file.id, key)];
|
|
91
|
+
const approved = review?.status === 'approved' && review.valueHash === hash(text);
|
|
92
|
+
for (const issue of checker.checkString(lang, key, sourceText, text, file.id)) {
|
|
93
|
+
if (approved && !isError(issue))
|
|
94
|
+
continue;
|
|
95
|
+
findings.push({
|
|
96
|
+
lang,
|
|
97
|
+
check: issue.check,
|
|
98
|
+
severity: isError(issue) ? 'error' : 'warning',
|
|
99
|
+
file: rel,
|
|
100
|
+
key,
|
|
101
|
+
text,
|
|
102
|
+
note: issue.note,
|
|
103
|
+
group: group.name,
|
|
104
|
+
});
|
|
105
|
+
}
|
|
54
106
|
}
|
|
55
107
|
}
|
|
56
108
|
}
|
|
57
109
|
}
|
|
58
110
|
return findings;
|
|
59
111
|
}
|
|
112
|
+
/**
|
|
113
|
+
* Repairs placeholder findings that have exactly one possible fix: the source has one
|
|
114
|
+
* placeholder and the translation one brace token with another name ("{heures}" for
|
|
115
|
+
* "{hours}"). The file text is edited in place, so its formatting stays as it is.
|
|
116
|
+
* Returns the repaired findings.
|
|
117
|
+
*/
|
|
118
|
+
export function fixPlaceholders(config, findings) {
|
|
119
|
+
const fixed = [];
|
|
120
|
+
const sources = new Map();
|
|
121
|
+
for (const group of groupsOf(config)) {
|
|
122
|
+
const checker = new Checker(group);
|
|
123
|
+
for (const file of findSourceFiles(group.root, group.files, group.sourceLanguage)) {
|
|
124
|
+
const doc = readJson(path.join(group.root, file.pathFor(group.sourceLanguage)));
|
|
125
|
+
if (doc)
|
|
126
|
+
for (const lang of group.targetLanguages)
|
|
127
|
+
sources.set(file.pathFor(lang), { values: flatten(doc), checker });
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
for (const finding of findings.filter(f => f.check === 'placeholder')) {
|
|
131
|
+
const entry = sources.get(finding.file);
|
|
132
|
+
const source = entry?.values.get(finding.key);
|
|
133
|
+
if (!entry || source === undefined)
|
|
134
|
+
continue;
|
|
135
|
+
const expected = source.match(entry.checker.placeholderRe) ?? [];
|
|
136
|
+
const tokens = finding.text.match(/\{\{?[^{}]+\}?\}/g) ?? [];
|
|
137
|
+
const [want, got] = [expected[0], tokens[0]];
|
|
138
|
+
if (new Set(expected).size !== 1 || tokens.length !== 1 || !want || !got || got === want)
|
|
139
|
+
continue;
|
|
140
|
+
const repaired = finding.text.replace(got, want);
|
|
141
|
+
const file = path.join(config.root, finding.file);
|
|
142
|
+
const text = readText(file);
|
|
143
|
+
const before = JSON.stringify(finding.text).slice(1, -1);
|
|
144
|
+
if (text === null || text.split(before).length !== 2)
|
|
145
|
+
continue; // not found exactly once
|
|
146
|
+
writeText(file, text.replace(before, JSON.stringify(repaired).slice(1, -1)));
|
|
147
|
+
fixed.push({ ...finding, text: repaired });
|
|
148
|
+
}
|
|
149
|
+
return fixed;
|
|
150
|
+
}
|
|
60
151
|
/** Table of finding counts per language and check. */
|
|
61
152
|
export function summaryTable(findings, languages) {
|
|
62
|
-
|
|
63
|
-
const
|
|
153
|
+
// Built-in checks always; plugin checks when they found something.
|
|
154
|
+
const checks = [...CHECKS, ...new Set(findings.map(f => f.check).filter(c => !CHECKS.includes(c)))];
|
|
155
|
+
const header = ['lang', ...checks];
|
|
156
|
+
const rows = languages.map(lang => [lang, ...checks.map(check => String(findings.filter(f => f.lang === lang && f.check === check).length))]);
|
|
64
157
|
const widths = header.map((h, i) => Math.max(h.length, ...rows.map(row => row[i].length)));
|
|
65
158
|
const line = (cells) => cells.map((cell, i) => cell.padEnd(widths[i])).join(' ').trimEnd();
|
|
66
159
|
return [line(header), ...rows.map(line)].join('\n');
|
package/dist/prompt.d.ts
CHANGED
|
@@ -4,9 +4,15 @@ export interface PromptItem {
|
|
|
4
4
|
source: string;
|
|
5
5
|
/** Existing translation of an older version of the source (revision mode). */
|
|
6
6
|
previous?: string;
|
|
7
|
+
/** Maximum characters of the translation (store listings, SEO titles, buttons). */
|
|
8
|
+
maxLength?: number;
|
|
9
|
+
/** Set when --fix-flagged revises a translation that lost content. */
|
|
10
|
+
completing?: boolean;
|
|
11
|
+
/** File id ("locales/{lang}/common.json"), for plugins. */
|
|
12
|
+
file?: string;
|
|
7
13
|
}
|
|
8
14
|
/** System prompt for translating a JSON array of strings. */
|
|
9
|
-
export declare function batchPrompt(config: Config, lang: string, items: PromptItem[]): string;
|
|
15
|
+
export declare function batchPrompt(config: Config, lang: string, items: PromptItem[], notes?: string): string;
|
|
10
16
|
/** System prompt for translating one string as plain text. */
|
|
11
17
|
export declare function singlePrompt(config: Config, lang: string, item: PromptItem, extra?: string): string;
|
|
12
18
|
/** Turns a translation request into a minimal correction of an existing translation. */
|
package/dist/prompt.js
CHANGED
|
@@ -36,7 +36,7 @@ function rules(config, lang, texts) {
|
|
|
36
36
|
}
|
|
37
37
|
const REVISION_RULE = 'start from the existing translation: keep its wording, terms and sentence structure wherever it still says what the source now says, and change only the parts where the source differs. Remove anything the existing translation says that the source no longer says, and add what is new. Do not reword text that is still correct.';
|
|
38
38
|
/** System prompt for translating a JSON array of strings. */
|
|
39
|
-
export function batchPrompt(config, lang, items) {
|
|
39
|
+
export function batchPrompt(config, lang, items, notes = '') {
|
|
40
40
|
const keys = ` CONTEXT: each element is a UI string. Its key (by 1-based position) hints at the screen and role; use it to pick the right meaning, never translate or output it: ${items
|
|
41
41
|
.map((item, i) => `${i + 1}=${item.key}`)
|
|
42
42
|
.join(', ')}.`;
|
|
@@ -46,17 +46,24 @@ export function batchPrompt(config, lang, items) {
|
|
|
46
46
|
const revision = revised.length > 0
|
|
47
47
|
? ` REVISION: the source of some elements was edited after they had been translated. Their existing translation (of the older source) by 1-based position: ${JSON.stringify(Object.fromEntries(revised))}. For these elements, ${REVISION_RULE}`
|
|
48
48
|
: '';
|
|
49
|
+
const limited = items.map((item, i) => (item.maxLength ? `element ${i + 1} at most ${item.maxLength}` : null)).filter(Boolean);
|
|
50
|
+
const lengths = limited.length > 0
|
|
51
|
+
? ` LENGTH LIMIT: these elements are cut off past a character limit (counting spaces): ${limited.join(', ')} characters. Stay under it even if that means a shorter, freer phrasing; never pad.`
|
|
52
|
+
: '';
|
|
49
53
|
const plural = items.some(item => /_(zero|one|two|few|many|other)$/.test(item.key))
|
|
50
54
|
? ` PLURALS: keys ending in _zero, _one, _two, _few, _many or _other are plural forms (Unicode CLDR categories) of ${languageName(lang)}. Write the form of that category, even when the source text given is the English plural (_zero is used when the count is 0, _two when it is 2; keep the {{count}} placeholder).`
|
|
51
55
|
: '';
|
|
52
|
-
return `${intro(config, lang)} Translate each string in the JSON array. Return ONLY a valid JSON array of strings with the same number of elements in the same order. No explanations, no code fences.${rules(config, lang, items.map(i => i.source))}${keys}${plural}${revision}`;
|
|
56
|
+
return `${intro(config, lang)} Translate each string in the JSON array. Return ONLY a valid JSON array of strings with the same number of elements in the same order. No explanations, no code fences.${rules(config, lang, items.map(i => i.source))}${keys}${lengths}${plural}${revision}${notes}`;
|
|
53
57
|
}
|
|
54
58
|
/** System prompt for translating one string as plain text. */
|
|
55
59
|
export function singlePrompt(config, lang, item, extra = '') {
|
|
56
60
|
const revision = item.previous
|
|
57
61
|
? ` REVISION: the source was edited after it had been translated. Existing translation (of the older source): ${JSON.stringify(item.previous)}. ${REVISION_RULE[0].toUpperCase()}${REVISION_RULE.slice(1)}`
|
|
58
62
|
: '';
|
|
59
|
-
|
|
63
|
+
const length = item.maxLength
|
|
64
|
+
? ` LENGTH LIMIT: at most ${item.maxLength} characters including spaces; the text is cut off past that. Prefer a shorter, freer phrasing over a literal one.`
|
|
65
|
+
: '';
|
|
66
|
+
return `${intro(config, lang)}${length} The user message is one UI string (key: ${item.key}). Output only the translation, with no explanations, quotes or commentary.${rules(config, lang, [item.source])}${revision}${extra}`;
|
|
60
67
|
}
|
|
61
68
|
/** Turns a translation request into a minimal correction of an existing translation. */
|
|
62
69
|
export function repairInstruction(existing, problems) {
|
package/dist/review.d.ts
CHANGED
package/dist/review.js
CHANGED
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
import path from 'node:path';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
2
|
+
import { groupsOf } from './config.js';
|
|
3
|
+
import { findSourceFiles, flatten, parseDoc, readText } from './files.js';
|
|
4
|
+
import { parseReviewId, reviewId, State } from './state.js';
|
|
5
|
+
import { withLock } from './lock.js';
|
|
6
|
+
import { hash, today } from './util.js';
|
|
5
7
|
function currentValue(config, lang, fileId, key) {
|
|
6
|
-
const
|
|
8
|
+
const rel = fileId.split('{lang}').join(lang);
|
|
9
|
+
const text = readText(path.join(config.root, rel));
|
|
7
10
|
if (text === null)
|
|
8
11
|
return undefined;
|
|
9
12
|
try {
|
|
10
|
-
return flatten(
|
|
13
|
+
return flatten(parseDoc(rel, text)).get(key);
|
|
11
14
|
}
|
|
12
15
|
catch {
|
|
13
16
|
return undefined;
|
|
@@ -40,25 +43,48 @@ function matches(selector, lang, key) {
|
|
|
40
43
|
* Returns the number of entries changed.
|
|
41
44
|
*/
|
|
42
45
|
export function updateReview(config, action, selectors) {
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
const value = currentValue(config, lang, file, key);
|
|
51
|
-
if (value === undefined)
|
|
46
|
+
// Under the run lock: a run in progress would otherwise overwrite these changes.
|
|
47
|
+
return withLock(path.join(config.root, config.stateDir), () => {
|
|
48
|
+
const state = new State(path.join(config.root, config.stateDir));
|
|
49
|
+
let changed = 0;
|
|
50
|
+
for (const [id, entry] of Object.entries(state.review)) {
|
|
51
|
+
const { lang, file, key } = parseReviewId(id);
|
|
52
|
+
if (!selectors.some(selector => matches(selector, lang, key)))
|
|
52
53
|
continue;
|
|
53
|
-
|
|
54
|
+
if (action === 'approve') {
|
|
55
|
+
const value = currentValue(config, lang, file, key);
|
|
56
|
+
if (value === undefined)
|
|
57
|
+
continue;
|
|
58
|
+
state.review[id] = { ...entry, status: 'approved', valueHash: hash(value) };
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
delete state.review[id];
|
|
62
|
+
state.invalidate(lang, file, key);
|
|
63
|
+
}
|
|
64
|
+
changed++;
|
|
54
65
|
}
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
66
|
+
// Approving a string that is not on the list marks it as checked by a person: it is then
|
|
67
|
+
// protected like a hand edit and the quality check no longer reports warnings for it.
|
|
68
|
+
if (action === 'approve') {
|
|
69
|
+
const exact = selectors.filter(sel => sel !== 'all' && !sel.endsWith(':*') && sel.includes(':'));
|
|
70
|
+
for (const selector of exact) {
|
|
71
|
+
const colon = selector.indexOf(':');
|
|
72
|
+
const [lang, key] = [selector.slice(0, colon), selector.slice(colon + 1)];
|
|
73
|
+
const files = groupsOf(config).flatMap(group => findSourceFiles(group.root, group.files, group.sourceLanguage));
|
|
74
|
+
for (const file of files) {
|
|
75
|
+
const id = reviewId(lang, file.id, key);
|
|
76
|
+
if (state.review[id])
|
|
77
|
+
continue;
|
|
78
|
+
const value = currentValue(config, lang, file.id, key);
|
|
79
|
+
if (value === undefined)
|
|
80
|
+
continue;
|
|
81
|
+
state.review[id] = { status: 'approved', reason: 'approved-by-hand', file: file.pathFor(lang), since: today(), valueHash: hash(value) };
|
|
82
|
+
changed++;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
58
85
|
}
|
|
59
|
-
changed
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
return changed;
|
|
86
|
+
if (changed > 0)
|
|
87
|
+
state.save();
|
|
88
|
+
return changed;
|
|
89
|
+
});
|
|
64
90
|
}
|
package/dist/scope.d.ts
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { Config } from './config.js';
|
|
2
|
+
import { type LocaleFile } from './files.js';
|
|
3
|
+
/** Which files and strings localewarden translates, and their length limits. */
|
|
4
|
+
export declare class Scope {
|
|
5
|
+
private readonly config;
|
|
6
|
+
private readonly excluded;
|
|
7
|
+
private readonly ignored;
|
|
8
|
+
private readonly limits;
|
|
9
|
+
constructor(config: Config);
|
|
10
|
+
/** Excluded by id ("locales/{lang}/nav.json") or by the source path ("locales/en/nav.json"). */
|
|
11
|
+
isExcluded(file: LocaleFile): boolean;
|
|
12
|
+
/** Not text: an ignored key or a URL, email, file path or number. Copied, never translated. */
|
|
13
|
+
isLiteral(key: string, value: string): boolean;
|
|
14
|
+
maxLength(key: string): number | undefined;
|
|
15
|
+
}
|
package/dist/scope.js
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { isLiteralValue, keyPattern, pathPattern } from './files.js';
|
|
2
|
+
/** Which files and strings localewarden translates, and their length limits. */
|
|
3
|
+
export class Scope {
|
|
4
|
+
config;
|
|
5
|
+
excluded;
|
|
6
|
+
ignored;
|
|
7
|
+
limits;
|
|
8
|
+
constructor(config) {
|
|
9
|
+
this.config = config;
|
|
10
|
+
this.excluded = config.exclude.map(pathPattern);
|
|
11
|
+
this.ignored = config.ignoreKeys.map(keyPattern);
|
|
12
|
+
this.limits = Object.entries(config.maxLength).map(([pattern, max]) => [keyPattern(pattern), max]);
|
|
13
|
+
}
|
|
14
|
+
/** Excluded by id ("locales/{lang}/nav.json") or by the source path ("locales/en/nav.json"). */
|
|
15
|
+
isExcluded(file) {
|
|
16
|
+
const paths = [file.id, file.pathFor(this.config.sourceLanguage)];
|
|
17
|
+
return this.excluded.some(re => paths.some(p => re.test(p)));
|
|
18
|
+
}
|
|
19
|
+
/** Not text: an ignored key or a URL, email, file path or number. Copied, never translated. */
|
|
20
|
+
isLiteral(key, value) {
|
|
21
|
+
return this.ignored.some(re => re.test(key)) || isLiteralValue(value);
|
|
22
|
+
}
|
|
23
|
+
maxLength(key) {
|
|
24
|
+
return this.limits.find(([re]) => re.test(key))?.[1];
|
|
25
|
+
}
|
|
26
|
+
}
|
package/dist/state.d.ts
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* repair-failures.json targeted repairs (--fix-flagged) that were rejected, so the same
|
|
10
10
|
* value is not sent to the model again.
|
|
11
11
|
*/
|
|
12
|
-
export type ReviewReason = 'manual-edit' | 'source-changed' | 'edited-after-approval';
|
|
12
|
+
export type ReviewReason = 'manual-edit' | 'source-changed' | 'edited-after-approval' | 'approved-by-hand';
|
|
13
13
|
export interface ReviewEntry {
|
|
14
14
|
status: 'pending' | 'approved';
|
|
15
15
|
reason: ReviewReason;
|
|
@@ -39,13 +39,20 @@ export declare class State {
|
|
|
39
39
|
review: Record<string, ReviewEntry>;
|
|
40
40
|
repairFailures: Record<string, RepairFailure>;
|
|
41
41
|
constructor(dir: string);
|
|
42
|
-
/**
|
|
42
|
+
/**
|
|
43
|
+
* Hashes recorded for a string, or undefined if localewarden has not seen it yet. `date` is
|
|
44
|
+
* the UTC day (YYYY-MM-DD) localewarden wrote it; empty for adopted existing translations.
|
|
45
|
+
*/
|
|
43
46
|
get(lang: string, fileId: string, key: string): {
|
|
44
47
|
source: string;
|
|
45
48
|
value: string;
|
|
49
|
+
date: string;
|
|
46
50
|
} | undefined;
|
|
47
|
-
/**
|
|
48
|
-
|
|
51
|
+
/**
|
|
52
|
+
* Records that `value` is the current translation of `source`, written today; pass
|
|
53
|
+
* `written: false` for a translation that was adopted or protected, not written by us.
|
|
54
|
+
*/
|
|
55
|
+
set(lang: string, fileId: string, key: string, source: string, value: string, written?: boolean): void;
|
|
49
56
|
/** Marks a recorded string as needing re-translation (source hash cleared). */
|
|
50
57
|
invalidate(lang: string, fileId: string, key: string): void;
|
|
51
58
|
delete(lang: string, fileId: string, key: string): void;
|
package/dist/state.js
CHANGED
|
@@ -1,28 +1,37 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
|
+
import { writeText } from './files.js';
|
|
3
4
|
import { hash, sortObject, today } from './util.js';
|
|
4
5
|
/** "<file id>#<key>" */
|
|
5
6
|
export const stringId = (fileId, key) => `${fileId}#${key}`;
|
|
6
7
|
/** "<lang>|<file id>#<key>" */
|
|
7
8
|
export const reviewId = (lang, fileId, key) => `${lang}|${stringId(fileId, key)}`;
|
|
8
9
|
export function parseReviewId(id) {
|
|
10
|
+
// File ids are paths and contain no "#"; keys may ("faq.#1"), so split at the first one.
|
|
9
11
|
const bar = id.indexOf('|');
|
|
10
|
-
const hashMark = id.
|
|
12
|
+
const hashMark = id.indexOf('#', bar + 1);
|
|
11
13
|
return { lang: id.slice(0, bar), file: id.slice(bar + 1, hashMark), key: id.slice(hashMark + 1) };
|
|
12
14
|
}
|
|
13
15
|
function readJson(file, fallback) {
|
|
16
|
+
let text;
|
|
14
17
|
try {
|
|
15
|
-
|
|
18
|
+
text = fs.readFileSync(file, 'utf8');
|
|
16
19
|
}
|
|
17
20
|
catch (error) {
|
|
18
21
|
if (error.code === 'ENOENT')
|
|
19
22
|
return fallback;
|
|
20
|
-
throw
|
|
23
|
+
throw error;
|
|
24
|
+
}
|
|
25
|
+
try {
|
|
26
|
+
return JSON.parse(text.replace(/^\uFEFF/, ''));
|
|
27
|
+
}
|
|
28
|
+
catch (error) {
|
|
29
|
+
throw new Error(`${file} is not valid JSON (${error.message}). It is localewarden's own bookkeeping: ` +
|
|
30
|
+
'restore it from git, or delete it (translations are kept and adopted again; hand edits made since are then not detected).');
|
|
21
31
|
}
|
|
22
32
|
}
|
|
23
33
|
function writeJson(file, value) {
|
|
24
|
-
|
|
25
|
-
fs.writeFileSync(file, JSON.stringify(value, null, 2) + '\n', 'utf8');
|
|
34
|
+
writeText(file, JSON.stringify(value, null, 2) + '\n');
|
|
26
35
|
}
|
|
27
36
|
export class State {
|
|
28
37
|
dir;
|
|
@@ -36,17 +45,27 @@ export class State {
|
|
|
36
45
|
this.review = readJson(path.join(dir, 'review.json'), {});
|
|
37
46
|
this.repairFailures = readJson(path.join(dir, 'repair-failures.json'), {});
|
|
38
47
|
}
|
|
39
|
-
/**
|
|
48
|
+
/**
|
|
49
|
+
* Hashes recorded for a string, or undefined if localewarden has not seen it yet. `date` is
|
|
50
|
+
* the UTC day (YYYY-MM-DD) localewarden wrote it; empty for adopted existing translations.
|
|
51
|
+
*/
|
|
40
52
|
get(lang, fileId, key) {
|
|
41
53
|
const raw = this.entries[lang]?.[stringId(fileId, key)];
|
|
42
54
|
if (!raw)
|
|
43
55
|
return undefined;
|
|
44
|
-
const [source, value] = raw.split(':');
|
|
45
|
-
return { source, value };
|
|
56
|
+
const [source, value, date = ''] = raw.split(':');
|
|
57
|
+
return { source, value, date: date ? `${date.slice(0, 4)}-${date.slice(4, 6)}-${date.slice(6, 8)}` : '' };
|
|
46
58
|
}
|
|
47
|
-
/**
|
|
48
|
-
|
|
49
|
-
|
|
59
|
+
/**
|
|
60
|
+
* Records that `value` is the current translation of `source`, written today; pass
|
|
61
|
+
* `written: false` for a translation that was adopted or protected, not written by us.
|
|
62
|
+
*/
|
|
63
|
+
set(lang, fileId, key, source, value, written = true) {
|
|
64
|
+
const id = stringId(fileId, key);
|
|
65
|
+
const previous = this.get(lang, fileId, key);
|
|
66
|
+
const keepDate = !written && previous && previous.value === hash(value) ? previous.date.replace(/-/g, '') : '';
|
|
67
|
+
const date = written ? today().replace(/-/g, '') : keepDate;
|
|
68
|
+
(this.entries[lang] ??= {})[id] = `${hash(source)}:${hash(value)}${date ? `:${date}` : ''}`;
|
|
50
69
|
}
|
|
51
70
|
/** Marks a recorded string as needing re-translation (source hash cleared). */
|
|
52
71
|
invalidate(lang, fileId, key) {
|
package/dist/translate.d.ts
CHANGED
|
@@ -1,50 +1,10 @@
|
|
|
1
|
-
import type
|
|
2
|
-
import { type
|
|
3
|
-
export
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
export interface RunOptions {
|
|
11
|
-
/** Subset of the configured target languages. */
|
|
12
|
-
languages?: string[];
|
|
13
|
-
/** Show what would be translated; no API calls, no writes. */
|
|
14
|
-
dryRun?: boolean;
|
|
15
|
-
/** Re-translate every string, not only new and changed ones. Hand edits stay protected. */
|
|
16
|
-
retranslateAll?: boolean;
|
|
17
|
-
/** Also replace hand-edited translations. */
|
|
18
|
-
overwriteManual?: boolean;
|
|
19
|
-
/** Ask the model to fix strings the quality check flags, changing as little as possible. */
|
|
20
|
-
fixFlagged?: boolean;
|
|
21
|
-
/** Overrides maxTokensPerRun from the config. */
|
|
22
|
-
maxTokens?: number;
|
|
23
|
-
logger?: Logger;
|
|
24
|
-
/** Model to use instead of the configured OpenAI-compatible endpoint (tests, other SDKs). */
|
|
25
|
-
model?: Model;
|
|
26
|
-
}
|
|
27
|
-
export interface LanguageSummary {
|
|
28
|
-
translated: number;
|
|
29
|
-
revised: number;
|
|
30
|
-
repaired: number;
|
|
31
|
-
failed: number;
|
|
32
|
-
protected: number;
|
|
33
|
-
removed: number;
|
|
34
|
-
planned: number;
|
|
35
|
-
plannedChars: number;
|
|
36
|
-
}
|
|
37
|
-
export interface RunSummary {
|
|
38
|
-
languages: Record<string, LanguageSummary>;
|
|
39
|
-
filesWritten: string[];
|
|
40
|
-
tokens: number;
|
|
41
|
-
requests: number;
|
|
42
|
-
stoppedByBudget: boolean;
|
|
43
|
-
pendingReview: number;
|
|
44
|
-
dryRun: boolean;
|
|
45
|
-
}
|
|
46
|
-
/** Words not shared by both texts, counted on the longer side (word-level LCS). */
|
|
47
|
-
export declare function changedWords(before: string, after: string): number;
|
|
48
|
-
/** A targeted repair may change a few words of a short string or a quarter of a long one. */
|
|
49
|
-
export declare function tooManyChanges(before: string, after: string): string | null;
|
|
1
|
+
import { type Config } from './config.js';
|
|
2
|
+
import { type RunOptions, type RunSummary } from './engine/context.js';
|
|
3
|
+
export { consoleLogger, type Logger, type LanguageSummary, type RunOptions, type RunSummary } from './engine/context.js';
|
|
4
|
+
export { changedWords, tooManyChanges } from './engine/repair.js';
|
|
5
|
+
/**
|
|
6
|
+
* Translates new and changed strings, group by group (in config order), language by language.
|
|
7
|
+
* Within a language, all files of a group share requests, so many small files do not cost a
|
|
8
|
+
* request each. See engine/planner.ts for what is translated, revised, adopted or protected.
|
|
9
|
+
*/
|
|
50
10
|
export declare function run(config: Config, options?: RunOptions): Promise<RunSummary>;
|