localewarden 0.1.1 → 0.3.0

package/dist/translate.js

@@ -1,401 +1,137 @@
 import path from 'node:path';
-import { Checker, FIXABLE_CHECKS, isUnchangedProse } from './checks.js';
-import { buildTarget, detectFormat, findSourceFiles, missingPluralLeaves, setAt, flatten, readText, serialize, stringLeaves, writeText, } from './files.js';
-import { BudgetExceededError, Client, FatalModelError, OpenAICompatibleModel } from './llm.js';
-import { batchPrompt, repairInstruction, singlePrompt } from './prompt.js';
-import { reviewId, State } from './state.js';
-import { baseLanguage, hash, inParallel, today } from './util.js';
-export const consoleLogger = {
-    info: message => console.log(message),
-    warn: message => console.warn(`warning: ${message}`),
-    error: message => console.error(`error: ${message}`),
-};
-// Scripts whose output costs many tokens per character; smaller batches avoid timeouts.
-const SLOW_LANGUAGES = new Set(['ar', 'fa', 'ur', 'he', 'hi', 'mr', 'ne', 'bn', 'th', 'ta', 'te', 'kn', 'ml', 'gu', 'pa', 'si', 'my', 'km', 'lo', 'am', 'ka', 'hy']);
-const MAX_BATCH_CHARS = 6000;
-/** Words not shared by both texts, counted on the longer side (word-level LCS). */
-export function changedWords(before, after) {
-    const split = (text) => text.replace(/<[^>]+>/g, ' ').split(/\s+/).filter(Boolean).map(w => w.toLocaleLowerCase());
-    const a = split(before);
-    const b = split(after);
-    let prev = new Array(b.length + 1).fill(0);
-    for (let i = 1; i <= a.length; i++) {
-        const row = new Array(b.length + 1).fill(0);
-        for (let j = 1; j <= b.length; j++)
-            row[j] = a[i - 1] === b[j - 1] ? prev[j - 1] + 1 : Math.max(prev[j], row[j - 1]);
-        prev = row;
-    }
-    return Math.max(a.length, b.length) - prev[b.length];
-}
-/** A targeted repair may change a few words of a short string or a quarter of a long one. */
-export function tooManyChanges(before, after) {
-    const changed = changedWords(before, after);
-    const allowed = Math.max(6, Math.ceil(before.split(/\s+/).filter(Boolean).length * 0.25));
-    return changed > allowed ? `changed ${changed} words (allowed ${allowed})` : null;
-}
-function parseArray(raw, expected) {
-    const json = raw.replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '');
-    try {
-        const parsed = JSON.parse(json);
-        if (!Array.isArray(parsed) || parsed.length !== expected)
-            return null;
-        return parsed.map(value => String(value));
-    }
-    catch {
-        return null;
-    }
-}
-const emptySummary = () => ({
-    translated: 0,
-    revised: 0,
-    repaired: 0,
-    failed: 0,
-    protected: 0,
-    removed: 0,
-    planned: 0,
-    plannedChars: 0,
-});
+import { Budget } from './budget.js';
+import { Checker } from './checks.js';
+import { selectGroups } from './config.js';
+import { applyCopies } from './engine/copies.js';
+import { consoleLogger, RunContext } from './engine/context.js';
+import { planFile } from './engine/planner.js';
+import { runRepairs } from './engine/repair.js';
+import { listFiles, loadSources } from './engine/sources.js';
+import { Translator } from './engine/translator.js';
+import { applyTranslations, writePlan } from './engine/writer.js';
+import { Client, OpenAICompatibleModel } from './llm.js';
+import { acquireLock } from './lock.js';
+import { loadPlugins, PluginHost } from './plugins.js';
+import { Scope } from './scope.js';
+import { State } from './state.js';
+import { inParallel } from './util.js';
+export { consoleLogger } from './engine/context.js';
+export { changedWords, tooManyChanges } from './engine/repair.js';
+const noModel = { complete: async () => ({ text: '', tokens: 0 }) };
+/**
+ * Translates new and changed strings, group by group (in config order), language by language.
+ * Within a language, all files of a group share requests, so many small files do not cost a
+ * request each. See engine/planner.ts for what is translated, revised, adopted or protected.
+ */
 export async function run(config, options = {}) {
     const log = options.logger ?? consoleLogger;
-    const languages = options.languages?.length ? options.languages : config.targetLanguages;
-    const unknown = languages.filter(lang => !config.targetLanguages.includes(lang));
-    if (unknown.length > 0)
-        throw new Error(`Not in targetLanguages: ${unknown.join(', ')}`);
-    if (config.context?.startsWith('Describe your product')) {
-        log.warn('"context" still has the example text from init; it is ignored. Describe your product in localewarden.config.json for better translations.');
-        config = { ...config, context: undefined };
+    const groups = selectGroups(config, options.groups, options.languages);
+    const dryRun = Boolean(options.dryRun);
+    if (!options.model && !dryRun) {
+        const needsKey = groups.find(g => !process.env[g.apiKeyEnv] && g.baseUrl.includes('api.openai.com'));
+        if (needsKey)
+            throw new Error(`Set the ${needsKey.apiKeyEnv} environment variable to your OpenAI API key (or run with --dry-run).`);
     }
-    const files = findSourceFiles(config.root, config.files, config.sourceLanguage);
-    if (files.length === 0) {
-        throw new Error(`No ${config.sourceLanguage} files match "${config.files}" under ${config.root}.`);
+    const stateDir = path.join(config.root, config.stateDir);
+    const plugins = new PluginHost(options.plugins ?? (await loadPlugins(config)));
+    const budget = new Budget(options.maxTokens ?? config.maxTokensPerRun, config.dailyTokenBudget, path.join(stateDir, 'usage.json'));
+    const release = dryRun ? () => { } : acquireLock(stateDir);
+    let state;
+    try {
+        state = new State(stateDir);
     }
-    const dryRun = Boolean(options.dryRun);
-    let model = options.model;
-    if (!model && !dryRun) {
-        if (!process.env[config.apiKeyEnv] && config.baseUrl.includes('api.openai.com')) {
-            throw new Error(`Set the ${config.apiKeyEnv} environment variable to your OpenAI API key (or run with --dry-run).`);
-        }
-        model = new OpenAICompatibleModel(config);
+    catch (error) {
+        release();
+        throw error;
     }
-    const client = new Client(model ?? { complete: async () => ({ text: '', tokens: 0 }) }, options.maxTokens ?? config.maxTokensPerRun, {
-        onRetry: message => log.warn(message),
-    });
-    const state = new State(path.join(config.root, config.stateDir));
-    const checker = new Checker(config);
-    const summary = {
-        languages: Object.fromEntries(languages.map(lang => [lang, emptySummary()])),
-        filesWritten: [],
-        tokens: 0,
-        requests: 0,
-        stoppedByBudget: false,
-        pendingReview: 0,
-        dryRun,
-    };
-    let fatal = null;
-    const stopped = () => fatal !== null || summary.stoppedByBudget;
-    const handleError = (error, lang) => {
-        if (error instanceof BudgetExceededError) {
-            if (!summary.stoppedByBudget)
-                log.warn(`${error.message}; stopping. Run again to continue.`);
-            summary.stoppedByBudget = true;
-        }
-        else if (error instanceof FatalModelError) {
-            fatal ??= error;
-        }
-        else {
-            log.warn(`[${lang}] ${error.message}`);
-        }
+    const ctx = new RunContext(options, log, state, budget, plugins);
+    // Ctrl+C or a CI timeout: keep what was translated so far, then stop.
+    const onSignal = (signal) => {
+        log.warn(`${signal}: saving progress and stopping.`);
+        if (!dryRun)
+            state.save();
+        release();
+        process.exit(signal === 'SIGINT' ? 130 : 143);
     };
-    const timeoutFor = (lang) => (SLOW_LANGUAGES.has(baseLanguage(lang)) ? 240_000 : 120_000);
-    async function translateOne(lang, item, extra = '') {
-        if (stopped())
-            return null;
-        try {
-            return (await client.complete(singlePrompt(config, lang, item, extra), item.source, timeoutFor(lang))).trim();
-        }
-        catch (error) {
-            handleError(error, lang);
-            return null;
-        }
+    process.once('SIGINT', onSignal);
+    process.once('SIGTERM', onSignal);
+    if (!dryRun && budget.exceeded) {
+        log.warn(`${budget.exceeded}; nothing to do until tomorrow (UTC).`);
+        ctx.summary.stoppedByBudget = true;
+        ctx.summary.stopReason = budget.exceeded;
     }
-    /** Translates items in batches; returns key -> translation for the ones that passed. */
-    async function translateItems(lang, items) {
-        const results = new Map();
-        const size = SLOW_LANGUAGES.has(baseLanguage(lang)) ? Math.min(5, config.batchSize) : config.batchSize;
-        const batches = [];
-        let current = [];
-        let chars = 0;
-        for (const item of items) {
-            // List markup gets split into several array elements by models; send it alone.
-            const alone = /<li\b/i.test(item.source) || item.source.length > MAX_BATCH_CHARS / 2;
-            if (current.length > 0 && (alone || current.length >= size || chars + item.source.length > MAX_BATCH_CHARS)) {
-                batches.push(current);
-                current = [];
-                chars = 0;
-            }
-            current.push(item);
-            chars += item.source.length;
-            if (alone) {
-                batches.push(current);
-                current = [];
-                chars = 0;
-            }
-        }
-        if (current.length > 0)
-            batches.push(current);
-        for (const batch of batches) {
-            if (stopped())
+    try {
+        for (const raw of groups) {
+            if (ctx.stopped)
                 break;
-            let candidates = null;
-            if (batch.length > 1) {
-                try {
-                    const raw = await client.complete(batchPrompt(config, lang, batch), JSON.stringify(batch.map(i => i.source)), timeoutFor(lang));
-                    candidates = parseArray(raw, batch.length);
-                    if (!candidates)
-                        log.warn(`[${lang}] batch answer was not a matching JSON array; translating one by one`);
-                }
-                catch (error) {
-                    handleError(error, lang);
-                    if (stopped())
-                        break;
-                }
-            }
-            if (!candidates) {
-                candidates = [];
-                for (const item of batch)
-                    candidates.push(await translateOne(lang, item));
-            }
-            for (const [i, item] of batch.entries()) {
-                let text = candidates[i];
-                if (text === null)
-                    continue;
-                let defect = checker.defect(lang, item.key, item.source, text);
-                const echoed = isUnchangedProse(item.source, text, checker.placeholderRe);
-                if (defect.soft && (batch.length > 1 || echoed)) {
-                    // One retry on its own, then keep whichever version is cleaner.
-                    const retry = await translateOne(lang, item);
-                    if (retry !== null) {
-                        const retryDefect = checker.defect(lang, item.key, item.source, retry);
-                        const rank = (d) => (d.hard ? 2 : d.soft ? 1 : 0);
-                        if (rank(retryDefect) < rank(defect)) {
-                            text = retry;
-                            defect = retryDefect;
-                        }
-                        else if (echoed && isUnchangedProse(item.source, retry, checker.placeholderRe)) {
-                            // Asked twice, the model keeps the source text: names and product lists are often
-                            // the same in every language. Accept it instead of retrying on every run.
-                            defect = { hard: null, soft: 'identical to the source (accepted after a retry)' };
-                        }
-                    }
-                }
-                if (defect.hard) {
-                    log.warn(`[${lang}] ${item.key}: ${defect.hard}; not written`);
-                    continue;
-                }
-                if (defect.soft)
-                    log.debug?.(`[${lang}] ${item.key}: ${defect.soft} (kept)`);
-                results.set(item.key, text);
+            let group = raw;
+            if (group.context?.startsWith('Describe your product')) {
+                log.warn('"context" still has the example text from init; it is ignored. Describe your product in localewarden.config.json for better translations.');
+                group = { ...group, context: undefined };
             }
+            await runGroup(ctx, group, options.model);
         }
-        return results;
-    }
-    async function processFile(file, sourceDoc, sourceText) {
-        const sourceLeaves = stringLeaves(sourceDoc);
-        await inParallel(languages, config.concurrency, async (lang) => {
-            if (stopped())
-                return;
-            const counts = summary.languages[lang];
-            const pluralExtras = missingPluralLeaves(sourceLeaves, lang);
-            const leaves = [...sourceLeaves, ...pluralExtras];
-            const sourceKeys = new Set(leaves.map(leaf => leaf.key));
-            const targetRel = file.pathFor(lang);
-            const targetFile = path.join(config.root, targetRel);
-            if (!path.resolve(targetFile).startsWith(path.resolve(config.root) + path.sep)) {
-                log.error(`${targetRel} is outside the project; skipped.`);
-                return;
-            }
-            const targetText = readText(targetFile);
-            let targetDoc = null;
-            if (targetText !== null) {
-                try {
-                    targetDoc = JSON.parse(targetText);
-                }
-                catch (error) {
-                    log.error(`${targetRel} is not valid JSON (${error.message}); skipped. Fix it by hand.`);
-                    return;
-                }
-            }
-            const current = targetDoc === null ? new Map() : flatten(targetDoc);
-            const values = new Map(current);
-            const fresh = [];
-            const revise = [];
-            const repairs = [];
-            for (const { key, value: source } of leaves) {
-                const cur = current.get(key);
-                const entry = state.get(lang, file.id, key);
-                const sourceHash = hash(source);
-                if (source.trim() === '') {
-                    values.set(key, source);
-                    continue;
-                }
-                const hasValue = cur !== undefined && cur.trim() !== '';
-                if (hasValue) {
-                    const id = reviewId(lang, file.id, key);
-                    const review = state.review[id];
-                    const curHash = hash(cur);
-                    if (options.overwriteManual && (review || (entry && entry.value !== curHash))) {
-                        delete state.review[id];
-                        fresh.push({ key, source });
-                        continue;
-                    }
-                    if (review) {
-                        // Counted only when something new needs a person's attention, not on every run.
-                        if (review.status === 'approved' && review.valueHash !== curHash) {
-                            state.addReview(lang, file.id, key, 'edited-after-approval', cur);
-                            counts.protected++;
-                        }
-                        else if (entry && entry.source !== sourceHash) {
-                            state.addReview(lang, file.id, key, 'source-changed', cur);
-                            log.warn(`[${lang}] ${key}: source changed, but the translation was edited by hand; kept and listed for review`);
-                            counts.protected++;
-                        }
-                        state.set(lang, file.id, key, source, cur);
-                        continue;
-                    }
-                    if (entry && entry.value !== curHash) {
-                        const sourceChanged = entry.source !== sourceHash;
-                        state.addReview(lang, file.id, key, sourceChanged ? 'source-changed' : 'manual-edit', cur);
-                        state.set(lang, file.id, key, source, cur);
-                        log.info(`[${lang}] ${key}: edited by hand; protected and listed for review`);
-                        counts.protected++;
-                        continue;
-                    }
-                }
-                if (!hasValue || options.retranslateAll) {
-                    fresh.push({ key, source });
-                }
-                else if (!entry) {
-                    // First time localewarden sees this string: adopt an existing translation, unless
-                    // it is just the source text copied over.
-                    if (isUnchangedProse(source, cur, checker.placeholderRe))
-                        fresh.push({ key, source });
-                    else
-                        state.set(lang, file.id, key, source, cur);
-                }
-                else if (entry.source !== sourceHash) {
-                    revise.push({ key, source, previous: cur });
-                }
-                else if (options.fixFlagged) {
-                    const issues = checker.checkString(lang, key, source, cur).filter(issue => FIXABLE_CHECKS.has(issue.check));
-                    const failed = state.repairFailures[reviewId(lang, file.id, key)];
-                    if (issues.length > 0 && failed?.valueHash !== hash(cur))
-                        repairs.push({ key, source, issues });
-                }
-            }
-            // Strings removed from the source.
-            const removed = [...current.keys()].filter(key => !sourceKeys.has(key));
-            for (const key of new Set([...removed, ...state.keys(lang, file.id).filter(key => !sourceKeys.has(key))])) {
-                state.delete(lang, file.id, key);
-                delete state.review[reviewId(lang, file.id, key)];
-            }
-            const work = [...fresh, ...revise];
-            if (dryRun) {
-                counts.planned += work.length + repairs.length;
-                counts.plannedChars += [...work, ...repairs].reduce((sum, item) => sum + item.source.length, 0);
-                counts.removed += removed.length;
-                if (work.length + repairs.length + removed.length > 0) {
-                    log.info(`[${lang}] ${targetRel}: ${fresh.length} new, ${revise.length} changed${repairs.length ? `, ${repairs.length} to repair` : ''}${removed.length ? `, ${removed.length} to remove` : ''}`);
-                }
-                return;
-            }
-            if (work.length > 0) {
-                log.info(`[${lang}] ${targetRel}: translating ${fresh.length} new, ${revise.length} changed`);
-                const translated = await translateItems(lang, work);
-                for (const item of work) {
-                    const text = translated.get(item.key);
-                    if (text === undefined) {
-                        counts.failed++;
-                        continue;
-                    }
-                    values.set(item.key, text);
-                    state.set(lang, file.id, item.key, item.source, text);
-                    if (item.previous)
-                        counts.revised++;
-                    else
-                        counts.translated++;
-                }
-            }
-            for (const { key, source, issues } of repairs) {
-                if (stopped())
-                    break;
-                const before = values.get(key) ?? '';
-                const problems = issues.map(issue => (issue.note ? `${issue.check}: ${issue.note}` : issue.check));
-                const after = await translateOne(lang, { key, source }, repairInstruction(before, problems));
-                const id = reviewId(lang, file.id, key);
-                let reason = after === null ? 'no answer' : null;
-                if (after !== null) {
-                    const checks = new Set(issues.map(issue => issue.check));
-                    const remaining = checker.checkString(lang, key, source, after).filter(issue => checks.has(issue.check));
-                    reason =
-                        checker.defect(lang, key, source, after).hard ??
-                            (remaining.length > 0 ? `still flagged: ${remaining.map(r => r.note ?? r.check).join('; ')}` : null) ??
-                            tooManyChanges(before, after);
-                }
-                if (reason || after === null) {
-                    if (after !== null || !stopped()) {
-                        state.repairFailures[id] = { valueHash: hash(before), date: today(), problems, reason: reason ?? 'no answer', ...(after ? { attempted: after } : {}) };
-                        log.warn(`[${lang}] ${key}: repair rejected (${reason})`);
-                    }
-                    continue;
-                }
-                delete state.repairFailures[id];
-                values.set(key, after);
-                state.set(lang, file.id, key, source, after);
-                counts.repaired++;
-            }
-            counts.removed += removed.length;
-            const built = buildTarget(sourceDoc, values) ?? {};
-            for (const leaf of pluralExtras) {
-                const value = values.get(leaf.key);
-                if (value !== undefined)
-                    setAt(built, leaf.path, value);
-            }
-            const text = serialize(built, detectFormat(targetText ?? sourceText));
-            if (text !== targetText && !(targetText === null && Object.keys(built).length === 0)) {
-                writeText(targetFile, text);
-                summary.filesWritten.push(targetRel);
-            }
-        });
     }
-    try {
-        for (const file of files) {
-            if (stopped())
-                break;
-            const sourceRel = file.pathFor(config.sourceLanguage);
-            const sourceText = readText(path.join(config.root, sourceRel));
-            if (sourceText === null)
-                continue;
-            let sourceDoc;
-            try {
-                sourceDoc = JSON.parse(sourceText);
-            }
-            catch (error) {
-                log.error(`${sourceRel} is not valid JSON (${error.message}); skipped.`);
-                continue;
-            }
-            await processFile(file, sourceDoc, sourceText);
+    finally {
+        process.off('SIGINT', onSignal);
+        process.off('SIGTERM', onSignal);
+        try {
             if (!dryRun)
                 state.save();
         }
+        finally {
+            release();
+        }
     }
-    finally {
-        if (!dryRun)
-            state.save();
+    ctx.summary.tokens = budget.tokens;
+    ctx.summary.requests = budget.requests;
+    ctx.summary.pendingReview = Object.values(state.review).filter(entry => entry.status === 'pending').length;
+    ctx.rethrow();
+    return ctx.summary;
+}
+async function runGroup(ctx, group, model) {
+    const { log, options } = ctx;
+    const scope = new Scope(group);
+    const checker = new Checker(group, ctx.plugins);
+    const files = listFiles(group, scope, ctx.plugins);
+    if (files.length === 0) {
+        const label = group.name ? `Group "${group.name}": no` : 'No';
+        if (group.groups || group.name)
+            log.warn(`${label} ${group.sourceLanguage} files match "${group.files}".`);
+        else
+            throw new Error(`No ${group.sourceLanguage} files match "${group.files}" under ${group.root}.`);
+        return;
     }
-    summary.tokens = client.tokens;
-    summary.requests = client.requests;
-    summary.pendingReview = Object.values(state.review).filter(entry => entry.status === 'pending').length;
-    if (fatal)
-        throw fatal;
-    return summary;
+    if (group.name)
+        log.info(`== ${group.name}`);
+    const client = new Client(model ?? (ctx.dryRun ? noModel : new OpenAICompatibleModel(group)), ctx.budget, {
+        onRetry: message => log.warn(message),
+    });
+    const translator = new Translator(ctx, group, checker, client);
+    const sources = loadSources(group, files, log);
+    const languages = group.targetLanguages.filter(lang => !options.languages?.length || options.languages.includes(lang));
+    await inParallel(languages, group.concurrency, async (lang) => {
+        if (ctx.stopped)
+            return;
+        const plans = sources.map(source => planFile(ctx, { config: group, scope, checker }, source, lang)).filter((p) => p !== null);
+        const counts = ctx.counts(lang);
+        if (ctx.dryRun) {
+            for (const plan of plans) {
+                counts.planned += plan.work.length + plan.repairs.length;
+                counts.plannedChars += [...plan.work, ...plan.repairs].reduce((sum, item) => sum + item.source.length, 0);
+                counts.removed += plan.removed.length;
+            }
+            return;
+        }
+        const work = plans.flatMap(plan => plan.work);
+        const translated = work.length > 0 ? await translator.translateItems(lang, work) : new Map();
+        for (const plan of plans) {
+            applyTranslations(ctx, plan, translated);
+            await runRepairs(ctx, plan, checker, scope, translator);
+            writePlan(ctx, plan);
+        }
+        ctx.state.save();
+    });
+    if (!ctx.stopped)
+        applyCopies(ctx, group, files);
 }

package/dist/ui/data.d.ts

@@ -0,0 +1,50 @@
+import { type Config } from '../config.js';
+import { PluginHost } from '../plugins.js';
+/** What the interface shows about one string. */
+export interface StringRow {
+    group?: string;
+    file: string;
+    key: string;
+    source: string;
+    text: string | null;
+    status: 'missing' | 'translated' | 'pending-review' | 'approved' | 'not-text';
+}
+export interface GroupStatus {
+    name?: string;
+    files: number;
+    sourceLanguage: string;
+    languages: {
+        lang: string;
+        strings: number;
+        translated: number;
+        pendingReview: number;
+    }[];
+}
+/** Per group and language: strings, translated strings and hand edits waiting for review. */
+export declare function status(config: Config, plugins: PluginHost): {
+    groups: GroupStatus[];
+    usage: unknown;
+    limits: unknown;
+};
+/** Strings of a group in one language, filtered by a search text (key, source or translation). */
+export declare function strings(config: Config, plugins: PluginHost, query: {
+    group?: string;
+    lang: string;
+    q?: string;
+    only?: 'missing' | 'pending-review';
+    limit?: number;
+}): {
+    rows: StringRow[];
+    total: number;
+};
+/**
+ * Saves a translation edited in the interface. It counts as checked by a person: it is
+ * approved on the review list, so runs leave it alone and the check skips its warnings.
+ */
+export declare function editString(config: Config, plugins: PluginHost, edit: {
+    group?: string;
+    lang: string;
+    file: string;
+    key: string;
+    value: string;
+}): StringRow;