localewarden 0.1.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +161 -12
- package/dist/budget.d.ts +25 -0
- package/dist/budget.js +74 -0
- package/dist/checks.d.ts +34 -6
- package/dist/checks.js +111 -21
- package/dist/cli.js +66 -10
- package/dist/config.d.ts +37 -1
- package/dist/config.js +118 -3
- package/dist/engine/context.d.ts +81 -0
- package/dist/engine/context.js +75 -0
- package/dist/engine/copies.d.ts +9 -0
- package/dist/engine/copies.js +31 -0
- package/dist/engine/planner.d.ts +66 -0
- package/dist/engine/planner.js +167 -0
- package/dist/engine/repair.d.ts +16 -0
- package/dist/engine/repair.js +61 -0
- package/dist/engine/sources.d.ts +13 -0
- package/dist/engine/sources.js +40 -0
- package/dist/engine/translator.d.ts +29 -0
- package/dist/engine/translator.js +156 -0
- package/dist/engine/writer.d.ts +13 -0
- package/dist/engine/writer.js +63 -0
- package/dist/files.d.ts +30 -0
- package/dist/files.js +125 -12
- package/dist/index.d.ts +6 -3
- package/dist/index.js +6 -3
- package/dist/llm.d.ts +5 -8
- package/dist/llm.js +7 -15
- package/dist/lock.d.ts +13 -0
- package/dist/lock.js +91 -0
- package/dist/output.d.ts +21 -0
- package/dist/output.js +76 -0
- package/dist/placeholders.js +2 -1
- package/dist/plugins.d.ts +72 -0
- package/dist/plugins.js +71 -0
- package/dist/project.d.ts +33 -5
- package/dist/project.js +135 -42
- package/dist/prompt.d.ts +7 -1
- package/dist/prompt.js +10 -3
- package/dist/review.d.ts +1 -1
- package/dist/review.js +49 -23
- package/dist/scope.d.ts +15 -0
- package/dist/scope.js +26 -0
- package/dist/state.d.ts +11 -4
- package/dist/state.js +30 -11
- package/dist/translate.d.ts +9 -49
- package/dist/translate.js +119 -383
- package/dist/ui/data.d.ts +50 -0
- package/dist/ui/data.js +178 -0
- package/dist/ui/page.d.ts +5 -0
- package/dist/ui/page.js +277 -0
- package/dist/ui/server.d.ts +24 -0
- package/dist/ui/server.js +194 -0
- package/dist/util.d.ts +6 -2
- package/dist/util.js +17 -5
- package/package.json +12 -1
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { Config } from '../config.js';
|
|
2
|
+
import { type LocaleFile } from '../files.js';
|
|
3
|
+
import type { PluginHost } from '../plugins.js';
|
|
4
|
+
import type { Scope } from '../scope.js';
|
|
5
|
+
import type { Logger } from './context.js';
|
|
6
|
+
import type { SourceFile } from './planner.js';
|
|
7
|
+
/**
|
|
8
|
+
* The source files of a group, without excluded ones, in the order plugins choose (they may
|
|
9
|
+
* also leave files out, e.g. blog posts scheduled for a later date).
|
|
10
|
+
*/
|
|
11
|
+
export declare function listFiles(config: Config, scope: Scope, plugins: PluginHost): LocaleFile[];
|
|
12
|
+
/** Reads and parses the source files; unreadable ones are reported and skipped. */
|
|
13
|
+
export declare function loadSources(config: Config, files: LocaleFile[], log: Logger): SourceFile[];
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { findSourceFiles, parseDoc, readText, stringLeaves } from '../files.js';
|
|
3
|
+
/**
|
|
4
|
+
* The source files of a group, without excluded ones, in the order plugins choose (they may
|
|
5
|
+
* also leave files out, e.g. blog posts scheduled for a later date).
|
|
6
|
+
*/
|
|
7
|
+
export function listFiles(config, scope, plugins) {
|
|
8
|
+
const files = findSourceFiles(config.root, config.files, config.sourceLanguage).filter(file => !scope.isExcluded(file));
|
|
9
|
+
const byId = new Map(files.map(file => [file.id, file]));
|
|
10
|
+
return plugins
|
|
11
|
+
.order(files.map(file => ({ id: file.id, sourcePath: file.pathFor(config.sourceLanguage) })), config.name)
|
|
12
|
+
.map(info => byId.get(info.id))
|
|
13
|
+
.filter((file) => file !== undefined);
|
|
14
|
+
}
|
|
15
|
+
/** Reads and parses the source files; unreadable ones are reported and skipped. */
|
|
16
|
+
export function loadSources(config, files, log) {
|
|
17
|
+
const sources = [];
|
|
18
|
+
for (const file of files) {
|
|
19
|
+
const sourceRel = file.pathFor(config.sourceLanguage);
|
|
20
|
+
const text = readText(path.join(config.root, sourceRel));
|
|
21
|
+
if (text === null)
|
|
22
|
+
continue;
|
|
23
|
+
try {
|
|
24
|
+
const doc = parseDoc(sourceRel, text);
|
|
25
|
+
const leaves = stringLeaves(doc);
|
|
26
|
+
// {"a.b": …} next to {"a": {"b": …}} flatten to the same key; only one would be kept.
|
|
27
|
+
const seen = new Set();
|
|
28
|
+
for (const leaf of leaves) {
|
|
29
|
+
if (seen.has(leaf.key))
|
|
30
|
+
log.warn(`${sourceRel}: key "${leaf.key}" exists twice (as a dotted and a nested key); rename one of them.`);
|
|
31
|
+
seen.add(leaf.key);
|
|
32
|
+
}
|
|
33
|
+
sources.push({ file, doc, text, leaves });
|
|
34
|
+
}
|
|
35
|
+
catch (error) {
|
|
36
|
+
log.error(`${sourceRel} is not valid JSON (${error.message}); skipped.`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return sources;
|
|
40
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { Checker } from '../checks.js';
|
|
2
|
+
import type { Config } from '../config.js';
|
|
3
|
+
import type { Client } from '../llm.js';
|
|
4
|
+
import { type PromptItem } from '../prompt.js';
|
|
5
|
+
import type { RunContext } from './context.js';
|
|
6
|
+
/**
|
|
7
|
+
* Turns prompt items into checked translations: batches strings into JSON-array requests,
|
|
8
|
+
* falls back to one request per string, splits very long texts at paragraph breaks, cleans
|
|
9
|
+
* and post-processes answers, and retries a defective answer once on its own.
|
|
10
|
+
*/
|
|
11
|
+
export declare class Translator {
|
|
12
|
+
private readonly ctx;
|
|
13
|
+
private readonly config;
|
|
14
|
+
private readonly checker;
|
|
15
|
+
private readonly client;
|
|
16
|
+
constructor(ctx: RunContext, config: Config, checker: Checker, client: Client);
|
|
17
|
+
private timeoutFor;
|
|
18
|
+
/** Model answer cleaned and run through plugin post-processing. */
|
|
19
|
+
private finish;
|
|
20
|
+
private notes;
|
|
21
|
+
/** One string as plain text; very long new strings go paragraph by paragraph. */
|
|
22
|
+
translateOne(lang: string, item: PromptItem, extra?: string): Promise<string | null>;
|
|
23
|
+
/** Splits items into batches by count and size; list markup and long texts go alone. */
|
|
24
|
+
private batches;
|
|
25
|
+
/** Translates items; returns the ones that passed the output checks. */
|
|
26
|
+
translateItems(lang: string, items: PromptItem[]): Promise<Map<PromptItem, string>>;
|
|
27
|
+
/** Output checks for one answer, with one retry on its own; null when it is not written. */
|
|
28
|
+
private accept;
|
|
29
|
+
}
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import { isUnchangedProse } from '../checks.js';
|
|
2
|
+
import { cleanOutput, parseArray, splitLongText } from '../output.js';
|
|
3
|
+
import { batchPrompt, singlePrompt } from '../prompt.js';
|
|
4
|
+
import { baseLanguage } from '../util.js';
|
|
5
|
+
// Scripts whose output costs many tokens per character; smaller batches avoid timeouts.
|
|
6
|
+
const SLOW_LANGUAGES = new Set(['ar', 'fa', 'ur', 'he', 'hi', 'mr', 'ne', 'bn', 'th', 'ta', 'te', 'kn', 'ml', 'gu', 'pa', 'si', 'my', 'km', 'lo', 'am', 'ka', 'hy']);
|
|
7
|
+
const MAX_BATCH_CHARS = 6000;
|
|
8
|
+
const rank = (d) => (d.hard ? 2 : d.soft ? 1 : 0);
|
|
9
|
+
/**
|
|
10
|
+
* Turns prompt items into checked translations: batches strings into JSON-array requests,
|
|
11
|
+
* falls back to one request per string, splits very long texts at paragraph breaks, cleans
|
|
12
|
+
* and post-processes answers, and retries a defective answer once on its own.
|
|
13
|
+
*/
|
|
14
|
+
export class Translator {
|
|
15
|
+
ctx;
|
|
16
|
+
config;
|
|
17
|
+
checker;
|
|
18
|
+
client;
|
|
19
|
+
constructor(ctx, config, checker, client) {
|
|
20
|
+
this.ctx = ctx;
|
|
21
|
+
this.config = config;
|
|
22
|
+
this.checker = checker;
|
|
23
|
+
this.client = client;
|
|
24
|
+
}
|
|
25
|
+
timeoutFor(lang) {
|
|
26
|
+
return SLOW_LANGUAGES.has(baseLanguage(lang)) ? 240_000 : 120_000;
|
|
27
|
+
}
|
|
28
|
+
/** Model answer cleaned and run through plugin post-processing. */
|
|
29
|
+
finish(lang, item, raw) {
|
|
30
|
+
const text = cleanOutput(item.source, raw);
|
|
31
|
+
return this.ctx.plugins.postProcess({ lang, key: item.key, file: item.file ?? '', source: item.source, text });
|
|
32
|
+
}
|
|
33
|
+
notes(lang, items) {
|
|
34
|
+
return this.ctx.plugins.promptNotes(lang, items.map(i => ({ key: i.key, source: i.source })));
|
|
35
|
+
}
|
|
36
|
+
/** One string as plain text; very long new strings go paragraph by paragraph. */
|
|
37
|
+
async translateOne(lang, item, extra = '') {
|
|
38
|
+
if (this.ctx.stopped)
|
|
39
|
+
return null;
|
|
40
|
+
const notes = this.notes(lang, [item]);
|
|
41
|
+
try {
|
|
42
|
+
if (!extra && !item.previous && item.source.length > this.config.chunkChars) {
|
|
43
|
+
const { parts, separators } = splitLongText(item.source, this.config.chunkChars);
|
|
44
|
+
let out = '';
|
|
45
|
+
for (const [i, part] of parts.entries()) {
|
|
46
|
+
const piece = await this.client.complete(singlePrompt(this.config, lang, { ...item, source: part }, notes), part, this.timeoutFor(lang));
|
|
47
|
+
out += cleanOutput(part, piece) + (separators[i] ?? '');
|
|
48
|
+
}
|
|
49
|
+
return this.finish(lang, item, out);
|
|
50
|
+
}
|
|
51
|
+
const raw = await this.client.complete(singlePrompt(this.config, lang, item, extra + notes), item.source, this.timeoutFor(lang));
|
|
52
|
+
return this.finish(lang, item, raw);
|
|
53
|
+
}
|
|
54
|
+
catch (error) {
|
|
55
|
+
this.ctx.handleError(error, lang);
|
|
56
|
+
return null;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/** Splits items into batches by count and size; list markup and long texts go alone. */
|
|
60
|
+
batches(lang, items) {
|
|
61
|
+
const size = SLOW_LANGUAGES.has(baseLanguage(lang)) ? Math.min(5, this.config.batchSize) : this.config.batchSize;
|
|
62
|
+
const batches = [];
|
|
63
|
+
let current = [];
|
|
64
|
+
let chars = 0;
|
|
65
|
+
const flush = () => {
|
|
66
|
+
if (current.length > 0)
|
|
67
|
+
batches.push(current);
|
|
68
|
+
current = [];
|
|
69
|
+
chars = 0;
|
|
70
|
+
};
|
|
71
|
+
for (const item of items) {
|
|
72
|
+
// Models split list markup into several array elements; send it alone.
|
|
73
|
+
const alone = /<li\b/i.test(item.source) || item.source.length > MAX_BATCH_CHARS / 2;
|
|
74
|
+
if (alone || current.length >= size || chars + item.source.length > MAX_BATCH_CHARS)
|
|
75
|
+
flush();
|
|
76
|
+
current.push(item);
|
|
77
|
+
chars += item.source.length;
|
|
78
|
+
if (alone)
|
|
79
|
+
flush();
|
|
80
|
+
}
|
|
81
|
+
flush();
|
|
82
|
+
return batches;
|
|
83
|
+
}
|
|
84
|
+
/** Translates items; returns the ones that passed the output checks. */
|
|
85
|
+
async translateItems(lang, items) {
|
|
86
|
+
const results = new Map();
|
|
87
|
+
for (const batch of this.batches(lang, items)) {
|
|
88
|
+
if (this.ctx.stopped)
|
|
89
|
+
break;
|
|
90
|
+
let candidates = null;
|
|
91
|
+
if (batch.length > 1) {
|
|
92
|
+
try {
|
|
93
|
+
const raw = await this.client.complete(batchPrompt(this.config, lang, batch, this.notes(lang, batch)), JSON.stringify(batch.map(i => i.source)), this.timeoutFor(lang));
|
|
94
|
+
const parsed = parseArray(raw, batch.length);
|
|
95
|
+
candidates = parsed ? parsed.map((text, i) => (text === null ? null : this.finish(lang, batch[i], text))) : null;
|
|
96
|
+
// An element the model left empty (null) gets its own request.
|
|
97
|
+
if (candidates)
|
|
98
|
+
for (const [i, text] of candidates.entries())
|
|
99
|
+
if (text === null)
|
|
100
|
+
candidates[i] = await this.translateOne(lang, batch[i]);
|
|
101
|
+
if (!candidates)
|
|
102
|
+
this.ctx.log.warn(`[${lang}] batch answer was not a matching JSON array; translating one by one`);
|
|
103
|
+
}
|
|
104
|
+
catch (error) {
|
|
105
|
+
this.ctx.handleError(error, lang);
|
|
106
|
+
if (this.ctx.stopped)
|
|
107
|
+
break;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
if (!candidates) {
|
|
111
|
+
candidates = [];
|
|
112
|
+
for (const item of batch)
|
|
113
|
+
candidates.push(await this.translateOne(lang, item));
|
|
114
|
+
}
|
|
115
|
+
for (const [i, item] of batch.entries()) {
|
|
116
|
+
const text = await this.accept(lang, item, candidates[i], batch.length > 1);
|
|
117
|
+
if (text !== null)
|
|
118
|
+
results.set(item, text);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
return results;
|
|
122
|
+
}
|
|
123
|
+
/** Output checks for one answer, with one retry on its own; null when it is not written. */
|
|
124
|
+
async accept(lang, item, candidate, fromBatch) {
|
|
125
|
+
if (candidate === null)
|
|
126
|
+
return null;
|
|
127
|
+
let text = candidate;
|
|
128
|
+
let defect = this.checker.defect(lang, item.key, item.source, text, item.file);
|
|
129
|
+
const echoed = isUnchangedProse(item.source, text, this.checker.placeholderRe);
|
|
130
|
+
if (defect.soft && (fromBatch || echoed)) {
|
|
131
|
+
const retry = await this.translateOne(lang, item);
|
|
132
|
+
if (retry !== null) {
|
|
133
|
+
const retryDefect = this.checker.defect(lang, item.key, item.source, retry, item.file);
|
|
134
|
+
if (rank(retryDefect) < rank(defect)) {
|
|
135
|
+
text = retry;
|
|
136
|
+
defect = retryDefect;
|
|
137
|
+
}
|
|
138
|
+
else if (echoed && isUnchangedProse(item.source, retry, this.checker.placeholderRe)) {
|
|
139
|
+
// Asked twice, the model keeps the source text: names and product lists are often
|
|
140
|
+
// the same in every language. Accept it instead of retrying on every run.
|
|
141
|
+
defect = { hard: null, soft: 'identical to the source (accepted after a retry)' };
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
if (defect.hard) {
|
|
146
|
+
this.ctx.log.warn(`[${lang}] ${item.key}: ${defect.hard}; not written`);
|
|
147
|
+
return null;
|
|
148
|
+
}
|
|
149
|
+
const long = this.checker.tooLong(item.key, text);
|
|
150
|
+
if (long)
|
|
151
|
+
this.ctx.log.warn(`[${lang}] ${item.key}: ${long}; written, shorten it by hand or run --fix-flagged`);
|
|
152
|
+
else if (defect.soft)
|
|
153
|
+
this.ctx.log.debug?.(`[${lang}] ${item.key}: ${defect.soft} (kept)`);
|
|
154
|
+
return text;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { type JsonValue } from '../files.js';
|
|
2
|
+
import type { PromptItem } from '../prompt.js';
|
|
3
|
+
import type { RunContext } from './context.js';
|
|
4
|
+
import type { Plan } from './planner.js';
|
|
5
|
+
/** JSON with object keys sorted, to compare content regardless of key order and formatting. */
|
|
6
|
+
export declare function canonical(value: JsonValue): string;
|
|
7
|
+
/** Puts the translations that passed into the plan's values and records them in the state. */
|
|
8
|
+
export declare function applyTranslations(ctx: RunContext, plan: Plan, translated: Map<PromptItem, string>): void;
|
|
9
|
+
/**
|
|
10
|
+
* Builds the target document in the source's shape and writes it, unless its content is the
|
|
11
|
+
* same as on disk (formatting and key order of the existing file are then left alone).
|
|
12
|
+
*/
|
|
13
|
+
export declare function writePlan(ctx: RunContext, plan: Plan): void;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { muchShorter } from '../checks.js';
|
|
2
|
+
import { buildTarget, detectFormat, serializeDoc, setAt, writeText } from '../files.js';
|
|
3
|
+
import { reviewId } from '../state.js';
|
|
4
|
+
import { hash, today } from '../util.js';
|
|
5
|
+
/** JSON with object keys sorted, to compare content regardless of key order and formatting. */
|
|
6
|
+
export function canonical(value) {
|
|
7
|
+
return JSON.stringify(value, (_, v) => v && typeof v === 'object' && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => (a < b ? -1 : 1))) : v);
|
|
8
|
+
}
|
|
9
|
+
/** Puts the translations that passed into the plan's values and records them in the state. */
|
|
10
|
+
export function applyTranslations(ctx, plan, translated) {
|
|
11
|
+
const { state, log } = ctx;
|
|
12
|
+
const { lang, file, values, counts } = plan;
|
|
13
|
+
for (const item of plan.work) {
|
|
14
|
+
const text = translated.get(item);
|
|
15
|
+
if (text === undefined) {
|
|
16
|
+
counts.failed++;
|
|
17
|
+
continue;
|
|
18
|
+
}
|
|
19
|
+
if (item.completing && item.previous && muchShorter(lang, item.source, text)) {
|
|
20
|
+
// Still missing content: keep the old text and do not spend tokens on it again.
|
|
21
|
+
state.repairFailures[reviewId(lang, file.id, item.key)] = {
|
|
22
|
+
valueHash: hash(item.previous),
|
|
23
|
+
date: today(),
|
|
24
|
+
problems: ['content missing'],
|
|
25
|
+
reason: 'revision still much shorter than the source',
|
|
26
|
+
attempted: text,
|
|
27
|
+
};
|
|
28
|
+
log.warn(`[${lang}] ${item.key}: could not complete the missing content; listed in repair-failures.json`);
|
|
29
|
+
counts.failed++;
|
|
30
|
+
continue;
|
|
31
|
+
}
|
|
32
|
+
values.set(item.key, text);
|
|
33
|
+
plan.pending.push({ key: item.key, source: item.source, value: text });
|
|
34
|
+
if (item.previous)
|
|
35
|
+
counts.revised++;
|
|
36
|
+
else
|
|
37
|
+
counts.translated++;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Builds the target document in the source's shape and writes it, unless its content is the
|
|
42
|
+
* same as on disk (formatting and key order of the existing file are then left alone).
|
|
43
|
+
*/
|
|
44
|
+
export function writePlan(ctx, plan) {
|
|
45
|
+
const { targetRel, targetFile, targetText, targetDoc, values, sourceDoc, sourceText, pluralExtras } = plan;
|
|
46
|
+
plan.counts.removed += plan.removed.length;
|
|
47
|
+
const built = buildTarget(sourceDoc, values) ?? {};
|
|
48
|
+
for (const leaf of pluralExtras) {
|
|
49
|
+
const value = values.get(leaf.key);
|
|
50
|
+
if (value !== undefined)
|
|
51
|
+
setAt(built, leaf.path, value);
|
|
52
|
+
}
|
|
53
|
+
const unchanged = targetDoc !== null && canonical(built) === canonical(targetDoc);
|
|
54
|
+
const text = serializeDoc(targetRel, built, detectFormat(targetText ?? sourceText));
|
|
55
|
+
if (!unchanged && text !== null && text !== targetText && !(targetText === null && Object.keys(built).length === 0)) {
|
|
56
|
+
writeText(targetFile, text);
|
|
57
|
+
ctx.summary.filesWritten.push(targetRel);
|
|
58
|
+
}
|
|
59
|
+
// The file now holds these translations: record them (see Plan.pending).
|
|
60
|
+
for (const { key, source, value } of plan.pending)
|
|
61
|
+
ctx.state.set(plan.lang, plan.file.id, key, source, value);
|
|
62
|
+
plan.pending = [];
|
|
63
|
+
}
|
package/dist/files.d.ts
CHANGED
|
@@ -10,6 +10,18 @@ export interface LocaleFile {
|
|
|
10
10
|
* Supported: {lang} (once or more), * within one path segment, and **\/ for any depth.
|
|
11
11
|
*/
|
|
12
12
|
export declare function findSourceFiles(root: string, pattern: string, sourceLanguage: string): LocaleFile[];
|
|
13
|
+
/**
|
|
14
|
+
* Key pattern -> RegExp. "*" matches within one segment, "**" any number of segments. A
|
|
15
|
+
* pattern without a dot matches the last segment anywhere ("id" matches "steps.2.id").
|
|
16
|
+
*/
|
|
17
|
+
export declare function keyPattern(pattern: string): RegExp;
|
|
18
|
+
/** Path pattern -> RegExp ("*" within a folder, "**" across folders, {lang} as written). */
|
|
19
|
+
export declare function pathPattern(pattern: string): RegExp;
|
|
20
|
+
/**
|
|
21
|
+
* Values that are not text and stay as they are in every language: URLs, email addresses,
|
|
22
|
+
* file paths and plain numbers or codes without spaces.
|
|
23
|
+
*/
|
|
24
|
+
export declare function isLiteralValue(value: string): boolean;
|
|
13
25
|
export type JsonValue = string | number | boolean | null | JsonValue[] | {
|
|
14
26
|
[key: string]: JsonValue;
|
|
15
27
|
};
|
|
@@ -44,9 +56,27 @@ export declare function setAt(doc: JsonValue, segments: PathSegment[], value: st
|
|
|
44
56
|
export interface JsonFormat {
|
|
45
57
|
indent: string;
|
|
46
58
|
finalNewline: boolean;
|
|
59
|
+
/** Line ending of the existing file ("\r\n" on Windows checkouts). */
|
|
60
|
+
eol: '\n' | '\r\n';
|
|
61
|
+
/** The existing file starts with a byte order mark. */
|
|
62
|
+
bom: boolean;
|
|
47
63
|
}
|
|
48
64
|
/** Indentation and final newline of an existing JSON text (2 spaces by default). */
|
|
65
|
+
/** Indentation, final newline, line ending and BOM of an existing text (2 spaces, LF by default). */
|
|
49
66
|
export declare function detectFormat(text: string | null): JsonFormat;
|
|
50
67
|
export declare function serialize(value: JsonValue, format: JsonFormat): string;
|
|
68
|
+
/**
|
|
69
|
+
* Parses a locale file. A .txt file (fastlane metadata: description.txt, keywords.txt) is one
|
|
70
|
+
* string, keyed by its file name, so maxLength patterns like "keywords" apply to it.
|
|
71
|
+
*/
|
|
72
|
+
export declare function parseDoc(rel: string, text: string): JsonValue;
|
|
73
|
+
/** Flutter ARB metadata ("@@locale", "@title": { description, placeholders }): not text. */
|
|
74
|
+
export declare const isArbMetadata: (rel: string, key: string) => boolean;
|
|
75
|
+
/** Text of a locale file, or null when a .txt file has no value to write. */
|
|
76
|
+
export declare function serializeDoc(rel: string, doc: JsonValue, format: JsonFormat): string | null;
|
|
51
77
|
export declare function readText(file: string): string | null;
|
|
78
|
+
/**
|
|
79
|
+
* Writes via a temporary file and a rename, so an interrupted run never leaves a half-written
|
|
80
|
+
* locale file behind.
|
|
81
|
+
*/
|
|
52
82
|
export declare function writeText(file: string, text: string): void;
|
package/dist/files.js
CHANGED
|
@@ -16,21 +16,45 @@ function tokenize(pattern) {
|
|
|
16
16
|
});
|
|
17
17
|
}
|
|
18
18
|
const SKIP_DIRS = new Set(['node_modules', '.git', '.localewarden']);
|
|
19
|
-
|
|
19
|
+
/** Lists files below `dir`, following symbolic links once (a link loop is not followed again). */
|
|
20
|
+
function walk(root, dir, out, seen = new Set()) {
|
|
21
|
+
const full = path.join(root, dir);
|
|
22
|
+
let real;
|
|
23
|
+
try {
|
|
24
|
+
real = fs.realpathSync(full);
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
if (seen.has(real))
|
|
30
|
+
return;
|
|
31
|
+
seen.add(real);
|
|
20
32
|
let entries;
|
|
21
33
|
try {
|
|
22
|
-
entries = fs.readdirSync(
|
|
34
|
+
entries = fs.readdirSync(full, { withFileTypes: true });
|
|
23
35
|
}
|
|
24
36
|
catch {
|
|
25
37
|
return;
|
|
26
38
|
}
|
|
27
39
|
for (const entry of entries) {
|
|
28
40
|
const rel = dir ? `${dir}/${entry.name}` : entry.name;
|
|
29
|
-
|
|
41
|
+
let isDir = entry.isDirectory();
|
|
42
|
+
let isFile = entry.isFile();
|
|
43
|
+
if (entry.isSymbolicLink()) {
|
|
44
|
+
try {
|
|
45
|
+
const stat = fs.statSync(path.join(root, rel));
|
|
46
|
+
isDir = stat.isDirectory();
|
|
47
|
+
isFile = stat.isFile();
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
continue; // broken link
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
if (isDir) {
|
|
30
54
|
if (!SKIP_DIRS.has(entry.name))
|
|
31
|
-
walk(root, rel, out);
|
|
55
|
+
walk(root, rel, out, seen);
|
|
32
56
|
}
|
|
33
|
-
else if (
|
|
57
|
+
else if (isFile) {
|
|
34
58
|
out.push(rel);
|
|
35
59
|
}
|
|
36
60
|
}
|
|
@@ -87,7 +111,7 @@ export function findSourceFiles(root, pattern, sourceLanguage) {
|
|
|
87
111
|
};
|
|
88
112
|
const files = [];
|
|
89
113
|
for (const rel of all.sort()) {
|
|
90
|
-
if (
|
|
114
|
+
if (!/\.(json|arb|txt)$/.test(rel))
|
|
91
115
|
continue;
|
|
92
116
|
const match = matcher.exec(rel);
|
|
93
117
|
if (!match || match.groups?.lang !== sourceLanguage)
|
|
@@ -99,6 +123,38 @@ export function findSourceFiles(root, pattern, sourceLanguage) {
|
|
|
99
123
|
}
|
|
100
124
|
return files;
|
|
101
125
|
}
|
|
126
|
+
/**
|
|
127
|
+
* Key pattern -> RegExp. "*" matches within one segment, "**" any number of segments. A
|
|
128
|
+
* pattern without a dot matches the last segment anywhere ("id" matches "steps.2.id").
|
|
129
|
+
*/
|
|
130
|
+
export function keyPattern(pattern) {
|
|
131
|
+
const body = pattern
|
|
132
|
+
.split(/(\*\*|\*)/)
|
|
133
|
+
.map(part => (part === '**' ? '.*' : part === '*' ? '[^.]*' : escapeRegExp(part)))
|
|
134
|
+
.join('');
|
|
135
|
+
return new RegExp(pattern.includes('.') ? `^${body}$` : `(?:^|\\.)${body}$`);
|
|
136
|
+
}
|
|
137
|
+
/** Path pattern -> RegExp ("*" within a folder, "**" across folders, {lang} as written). */
|
|
138
|
+
export function pathPattern(pattern) {
|
|
139
|
+
const body = pattern
|
|
140
|
+
.replace(/\\/g, '/')
|
|
141
|
+
.replace(/^\.\//, '')
|
|
142
|
+
.split(/(\*\*\/?|\*)/)
|
|
143
|
+
.map(part => (part.startsWith('**') ? '(?:.*/)?' : part === '*' ? '[^/]*' : escapeRegExp(part)))
|
|
144
|
+
.join('');
|
|
145
|
+
return new RegExp(`^${body}$`);
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Values that are not text and stay as they are in every language: URLs, email addresses,
|
|
149
|
+
* file paths and plain numbers or codes without spaces.
|
|
150
|
+
*/
|
|
151
|
+
export function isLiteralValue(value) {
|
|
152
|
+
const v = value.trim();
|
|
153
|
+
return (/^(?:https?:\/\/|mailto:|tel:)\S+$/i.test(v) ||
|
|
154
|
+
/^[\w.+-]+@[\w-]+(?:\.[\w-]+)+$/.test(v) ||
|
|
155
|
+
/^(?:\.{0,2}\/)?[\w@.-]+(?:\/[\w@.-]+)*\.(?:png|jpe?g|gif|svg|webp|avif|ico|mp3|mp4|webm|wav|pdf|json|css|js|html?)$/i.test(v) ||
|
|
156
|
+
/^[\d\s.,:%+\-–/×x#]+$/.test(v));
|
|
157
|
+
}
|
|
102
158
|
export const keyOf = (segments) => segments.join('.');
|
|
103
159
|
/** All string values of a parsed JSON document, in document order. */
|
|
104
160
|
export function stringLeaves(value, prefix = [], out = []) {
|
|
@@ -142,7 +198,8 @@ export function buildTarget(source, values, prefix = []) {
|
|
|
142
198
|
if (built && typeof built === 'object' && !Array.isArray(built) && Object.keys(built).length === 0) {
|
|
143
199
|
continue;
|
|
144
200
|
}
|
|
145
|
-
|
|
201
|
+
// defineProperty: a key named "__proto__" stays a key instead of setting the prototype.
|
|
202
|
+
Object.defineProperty(result, key, { value: built, enumerable: true, writable: true, configurable: true });
|
|
146
203
|
}
|
|
147
204
|
return result;
|
|
148
205
|
}
|
|
@@ -202,12 +259,44 @@ export function setAt(doc, segments, value) {
|
|
|
202
259
|
Object.assign(node, Object.fromEntries(entries));
|
|
203
260
|
}
|
|
204
261
|
/** Indentation and final newline of an existing JSON text (2 spaces by default). */
|
|
262
|
+
/** Indentation, final newline, line ending and BOM of an existing text (2 spaces, LF by default). */
|
|
205
263
|
export function detectFormat(text) {
|
|
206
|
-
const indent = text?.match(
|
|
207
|
-
return {
|
|
264
|
+
const indent = text?.match(/^\uFEFF?[{[][ \t]*\r?\n([ \t]+)\S/)?.[1] ?? ' ';
|
|
265
|
+
return {
|
|
266
|
+
indent,
|
|
267
|
+
finalNewline: text === null ? true : text.endsWith('\n'),
|
|
268
|
+
eol: text?.includes('\r\n') ? '\r\n' : '\n',
|
|
269
|
+
bom: text?.startsWith('\uFEFF') ?? false,
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
/** Line endings, final newline and BOM applied to text written with "\n". */
|
|
273
|
+
function applyFormat(text, format) {
|
|
274
|
+
const body = format.eol === '\r\n' ? text.replace(/\r?\n/g, '\r\n') : text;
|
|
275
|
+
return (format.bom ? '\uFEFF' : '') + body + (format.finalNewline ? format.eol : '');
|
|
208
276
|
}
|
|
209
277
|
export function serialize(value, format) {
|
|
210
|
-
|
|
278
|
+
// JSON.stringify escapes newlines inside strings, so every raw "\n" is structural.
|
|
279
|
+
return applyFormat(JSON.stringify(value, null, format.indent), format);
|
|
280
|
+
}
|
|
281
|
+
const txtKey = (rel) => path.posix.basename(rel.replace(/\\/g, '/'), '.txt');
|
|
282
|
+
/**
|
|
283
|
+
* Parses a locale file. A .txt file (fastlane metadata: description.txt, keywords.txt) is one
|
|
284
|
+
* string, keyed by its file name, so maxLength patterns like "keywords" apply to it.
|
|
285
|
+
*/
|
|
286
|
+
export function parseDoc(rel, text) {
|
|
287
|
+
const body = text.replace(/^\uFEFF/, '');
|
|
288
|
+
if (rel.endsWith('.txt'))
|
|
289
|
+
return { [txtKey(rel)]: body.replace(/\r?\n$/, '').replace(/\r\n/g, '\n') };
|
|
290
|
+
return JSON.parse(body);
|
|
291
|
+
}
|
|
292
|
+
/** Flutter ARB metadata ("@@locale", "@title": { description, placeholders }): not text. */
|
|
293
|
+
export const isArbMetadata = (rel, key) => rel.endsWith('.arb') && key.startsWith('@');
|
|
294
|
+
/** Text of a locale file, or null when a .txt file has no value to write. */
|
|
295
|
+
export function serializeDoc(rel, doc, format) {
|
|
296
|
+
if (!rel.endsWith('.txt'))
|
|
297
|
+
return serialize(doc, format);
|
|
298
|
+
const value = doc[txtKey(rel)];
|
|
299
|
+
return typeof value === 'string' ? applyFormat(value, format) : null;
|
|
211
300
|
}
|
|
212
301
|
export function readText(file) {
|
|
213
302
|
try {
|
|
@@ -219,7 +308,31 @@ export function readText(file) {
|
|
|
219
308
|
throw error;
|
|
220
309
|
}
|
|
221
310
|
}
|
|
311
|
+
/**
|
|
312
|
+
* Writes via a temporary file and a rename, so an interrupted run never leaves a half-written
|
|
313
|
+
* locale file behind.
|
|
314
|
+
*/
|
|
222
315
|
export function writeText(file, text) {
|
|
223
|
-
|
|
224
|
-
|
|
316
|
+
// A symbolic link is written through (the link stays a link), with the file's own mode.
|
|
317
|
+
let target = file;
|
|
318
|
+
let mode;
|
|
319
|
+
try {
|
|
320
|
+
target = fs.realpathSync(file);
|
|
321
|
+
mode = fs.statSync(target).mode & 0o7777;
|
|
322
|
+
}
|
|
323
|
+
catch {
|
|
324
|
+
// does not exist yet
|
|
325
|
+
}
|
|
326
|
+
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
327
|
+
const tmp = `${target}.${process.pid}.localewarden-tmp`;
|
|
328
|
+
try {
|
|
329
|
+
fs.writeFileSync(tmp, text, { encoding: 'utf8', ...(mode !== undefined ? { mode } : {}) });
|
|
330
|
+
if (mode !== undefined)
|
|
331
|
+
fs.chmodSync(tmp, mode);
|
|
332
|
+
fs.renameSync(tmp, target);
|
|
333
|
+
}
|
|
334
|
+
catch (error) {
|
|
335
|
+
fs.rmSync(tmp, { force: true });
|
|
336
|
+
throw error;
|
|
337
|
+
}
|
|
225
338
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
|
-
export { loadConfig, resolveConfig, ConfigError, DEFAULTS, type Config, type Register } from './config.js';
|
|
1
|
+
export { loadConfig, resolveConfig, groupsOf, ConfigError, DEFAULTS, type Config, type Register } from './config.js';
|
|
2
2
|
export { run, consoleLogger, type RunOptions, type RunSummary, type LanguageSummary, type Logger } from './translate.js';
|
|
3
|
-
export { checkProject, summaryTable, type Finding } from './project.js';
|
|
4
|
-
export { Checker, CHECKS, type CheckName, type Issue } from './checks.js';
|
|
3
|
+
export { checkProject, fixPlaceholders, summaryTable, type CheckOptions, type Finding } from './project.js';
|
|
4
|
+
export { Checker, CHECKS, isError, isFixable, type CheckName, type Issue } from './checks.js';
|
|
5
5
|
export { listReview, updateReview, type ReviewItem } from './review.js';
|
|
6
6
|
export { OpenAICompatibleModel, type Model, type Completion } from './llm.js';
|
|
7
|
+
export { Budget, BudgetExceededError } from './budget.js';
|
|
8
|
+
export { loadPlugins, type Plugin, type PluginIssue, type PluginSpec, type StringContext, type FileInfo } from './plugins.js';
|
|
9
|
+
export { startUi, type UiOptions, type UiServer } from './ui/server.js';
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
|
-
export { loadConfig, resolveConfig, ConfigError, DEFAULTS } from './config.js';
|
|
1
|
+
export { loadConfig, resolveConfig, groupsOf, ConfigError, DEFAULTS } from './config.js';
|
|
2
2
|
export { run, consoleLogger } from './translate.js';
|
|
3
|
-
export { checkProject, summaryTable } from './project.js';
|
|
4
|
-
export { Checker, CHECKS } from './checks.js';
|
|
3
|
+
export { checkProject, fixPlaceholders, summaryTable } from './project.js';
|
|
4
|
+
export { Checker, CHECKS, isError, isFixable } from './checks.js';
|
|
5
5
|
export { listReview, updateReview } from './review.js';
|
|
6
6
|
export { OpenAICompatibleModel } from './llm.js';
|
|
7
|
+
export { Budget, BudgetExceededError } from './budget.js';
|
|
8
|
+
export { loadPlugins } from './plugins.js';
|
|
9
|
+
export { startUi } from './ui/server.js';
|
package/dist/llm.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import type { Config } from './config.js';
|
|
2
|
+
import type { Budget } from './budget.js';
|
|
3
|
+
export { BudgetExceededError } from './budget.js';
|
|
2
4
|
export interface Completion {
|
|
3
5
|
text: string;
|
|
4
6
|
tokens: number;
|
|
@@ -27,26 +29,21 @@ export declare class OpenAICompatibleModel implements Model {
|
|
|
27
29
|
timeoutMs: number;
|
|
28
30
|
}): Promise<Completion>;
|
|
29
31
|
}
|
|
30
|
-
export declare class BudgetExceededError extends Error {
|
|
31
|
-
}
|
|
32
32
|
/**
|
|
33
|
-
* Wraps a Model with retries, spacing between requests and the
|
|
33
|
+
* Wraps a Model with retries, spacing between requests and the shared token budget.
|
|
34
34
|
* The budget is checked before each request; the request that crosses it still completes.
|
|
35
35
|
*/
|
|
36
36
|
export declare class Client {
|
|
37
37
|
private readonly model;
|
|
38
|
-
|
|
38
|
+
readonly budget: Budget;
|
|
39
39
|
private readonly options;
|
|
40
|
-
tokens: number;
|
|
41
|
-
requests: number;
|
|
42
40
|
private lastStart;
|
|
43
41
|
private queue;
|
|
44
|
-
constructor(model: Model,
|
|
42
|
+
constructor(model: Model, budget: Budget, options?: {
|
|
45
43
|
retries?: number;
|
|
46
44
|
minSpacingMs?: number;
|
|
47
45
|
onRetry?: (message: string) => void;
|
|
48
46
|
});
|
|
49
|
-
get budgetExceeded(): boolean;
|
|
50
47
|
/** Enforces a minimum gap between request starts across all parallel workers. */
|
|
51
48
|
private spacing;
|
|
52
49
|
complete(system: string, user: string, timeoutMs: number): Promise<string>;
|