localewarden 0.1.0 → 0.2.0

package/dist/translate.js

@@ -1,8 +1,9 @@
 import path from 'node:path';
-import { Checker, FIXABLE_CHECKS, isUnchangedProse } from './checks.js';
-import { buildTarget, detectFormat, findSourceFiles, flatten, readText, serialize, stringLeaves, writeText, } from './files.js';
+import { Checker, FIXABLE_CHECKS, isUnchangedProse, muchShorter } from './checks.js';
+import { buildTarget, detectFormat, findSourceFiles, isArbMetadata, missingPluralLeaves, setAt, flatten, parseDoc, readText, serializeDoc, stringLeaves, writeText, } from './files.js';
 import { BudgetExceededError, Client, FatalModelError, OpenAICompatibleModel } from './llm.js';
 import { batchPrompt, repairInstruction, singlePrompt } from './prompt.js';
+import { Scope } from './scope.js';
 import { reviewId, State } from './state.js';
 import { baseLanguage, hash, inParallel, today } from './util.js';
 export const consoleLogger = {
@@ -13,6 +14,10 @@ export const consoleLogger = {
 // Scripts whose output costs many tokens per character; smaller batches avoid timeouts.
 const SLOW_LANGUAGES = new Set(['ar', 'fa', 'ur', 'he', 'hi', 'mr', 'ne', 'bn', 'th', 'ta', 'te', 'kn', 'ml', 'gu', 'pa', 'si', 'my', 'km', 'lo', 'am', 'ka', 'hy']);
 const MAX_BATCH_CHARS = 6000;
+/** JSON with object keys sorted, to compare content regardless of key order and formatting. */
+function canonical(value) {
+    return JSON.stringify(value, (_, v) => v && typeof v === 'object' && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => (a < b ? -1 : 1))) : v);
+}
 /** Words not shared by both texts, counted on the longer side (word-level LCS). */
 export function changedWords(before, after) {
     const split = (text) => text.replace(/<[^>]+>/g, ' ').split(/\s+/).filter(Boolean).map(w => w.toLocaleLowerCase());
@@ -61,7 +66,12 @@ export async function run(config, options = {}) {
     const unknown = languages.filter(lang => !config.targetLanguages.includes(lang));
     if (unknown.length > 0)
         throw new Error(`Not in targetLanguages: ${unknown.join(', ')}`);
-    const files = findSourceFiles(config.root, config.files, config.sourceLanguage);
+    if (config.context?.startsWith('Describe your product')) {
+        log.warn('"context" still has the example text from init; it is ignored. Describe your product in localewarden.config.json for better translations.');
+        config = { ...config, context: undefined };
+    }
+    const scope = new Scope(config);
+    const files = findSourceFiles(config.root, config.files, config.sourceLanguage).filter(file => !scope.isExcluded(file));
     if (files.length === 0) {
         throw new Error(`No ${config.sourceLanguage} files match "${config.files}" under ${config.root}.`);
     }
@@ -166,7 +176,8 @@ export async function run(config, options = {}) {
                 if (text === null)
                     continue;
                 let defect = checker.defect(lang, item.key, item.source, text);
-                if (defect.soft && batch.length > 1) {
+                const echoed = isUnchangedProse(item.source, text, checker.placeholderRe);
+                if (defect.soft && (batch.length > 1 || echoed)) {
                     // One retry on its own, then keep whichever version is cleaner.
                     const retry = await translateOne(lang, item);
                     if (retry !== null) {
@@ -176,194 +187,253 @@ export async function run(config, options = {}) {
                             text = retry;
                             defect = retryDefect;
                         }
+                        else if (echoed && isUnchangedProse(item.source, retry, checker.placeholderRe)) {
+                            // Asked twice, the model keeps the source text: names and product lists are often
+                            // the same in every language. Accept it instead of retrying on every run.
+                            defect = { hard: null, soft: 'identical to the source (accepted after a retry)' };
+                        }
                     }
                 }
                 if (defect.hard) {
                     log.warn(`[${lang}] ${item.key}: ${defect.hard}; not written`);
                     continue;
                 }
-                if (defect.soft)
+                const long = checker.tooLong(item.key, text);
+                if (long)
+                    log.warn(`[${lang}] ${item.key}: ${long}; written, shorten it by hand or run --fix-flagged`);
+                else if (defect.soft)
                     log.debug?.(`[${lang}] ${item.key}: ${defect.soft} (kept)`);
-                results.set(item.key, text);
+                results.set(item, text);
             }
         }
         return results;
     }
-    async function processFile(file, sourceDoc, sourceText) {
-        const leaves = stringLeaves(sourceDoc);
+    /**
+     * Decides what to do with one file in one language: what to translate, revise or repair,
+     * what is protected and what was removed. Returns null when the file is skipped.
+     */
+    function planFile({ file, doc: sourceDoc, text: sourceText, leaves: sourceLeaves }, lang) {
+        const counts = summary.languages[lang];
+        const pluralExtras = missingPluralLeaves(sourceLeaves, lang);
+        const leaves = [...sourceLeaves, ...pluralExtras];
         const sourceKeys = new Set(leaves.map(leaf => leaf.key));
-        await inParallel(languages, config.concurrency, async (lang) => {
-            if (stopped())
-                return;
-            const counts = summary.languages[lang];
-            const targetRel = file.pathFor(lang);
-            const targetFile = path.join(config.root, targetRel);
-            const targetText = readText(targetFile);
-            let targetDoc = null;
-            if (targetText !== null) {
-                try {
-                    targetDoc = JSON.parse(targetText);
-                }
-                catch (error) {
-                    log.error(`${targetRel} is not valid JSON (${error.message}); skipped. Fix it by hand.`);
-                    return;
-                }
+        const targetRel = file.pathFor(lang);
+        const targetFile = path.join(config.root, targetRel);
+        if (!path.resolve(targetFile).startsWith(path.resolve(config.root) + path.sep)) {
+            log.error(`${targetRel} is outside the project; skipped.`);
+            return null;
+        }
+        const targetText = readText(targetFile);
+        let targetDoc = null;
+        if (targetText !== null) {
+            try {
+                targetDoc = parseDoc(targetRel, targetText);
+            }
+            catch (error) {
+                log.error(`${targetRel} is not valid JSON (${error.message}); skipped. Fix it by hand.`);
+                return null;
+            }
+        }
+        const current = targetDoc === null ? new Map() : flatten(targetDoc);
+        const values = new Map(current);
+        const fresh = [];
+        const revise = [];
+        const repairs = [];
+        for (const { key, value: source } of leaves) {
+            const cur = current.get(key);
+            const entry = state.get(lang, file.id, key);
+            const sourceHash = hash(source);
+            if (source.trim() === '') {
+                values.set(key, source);
+                continue;
             }
-            const current = targetDoc === null ? new Map() : flatten(targetDoc);
-            const values = new Map(current);
-            const fresh = [];
-            const revise = [];
-            const repairs = [];
-            for (const { key, value: source } of leaves) {
-                const cur = current.get(key);
-                const entry = state.get(lang, file.id, key);
-                const sourceHash = hash(source);
-                if (source.trim() === '') {
+            if (isArbMetadata(targetRel, key)) {
+                // Flutter ARB: "@@locale" names the target language; other metadata is copied.
+                values.set(key, key === '@@locale' ? lang : cur ?? source);
+                continue;
+            }
+            if (scope.isLiteral(key, source)) {
+                // Not text: keep a localized value someone set (e.g. a /de/ URL), else copy the source.
+                if (cur === undefined || cur.trim() === '')
                     values.set(key, source);
+                continue;
+            }
+            const hasValue = cur !== undefined && cur.trim() !== '';
+            if (hasValue) {
+                const id = reviewId(lang, file.id, key);
+                const review = state.review[id];
+                const curHash = hash(cur);
+                if (options.overwriteManual && (review || (entry && entry.value !== curHash))) {
+                    delete state.review[id];
+                    fresh.push({ key, source, maxLength: scope.maxLength(key) });
                     continue;
                 }
-                const hasValue = cur !== undefined && cur.trim() !== '';
-                if (hasValue) {
-                    const id = reviewId(lang, file.id, key);
-                    const review = state.review[id];
-                    const curHash = hash(cur);
-                    if (options.overwriteManual && (review || (entry && entry.value !== curHash))) {
-                        delete state.review[id];
-                        fresh.push({ key, source });
-                        continue;
-                    }
-                    if (review) {
-                        if (review.status === 'approved' && review.valueHash !== curHash) {
-                            state.addReview(lang, file.id, key, 'edited-after-approval', cur);
-                        }
-                        else if (entry && entry.source !== sourceHash) {
-                            state.addReview(lang, file.id, key, 'source-changed', cur);
-                            log.warn(`[${lang}] ${key}: source changed, but the translation was edited by hand; kept and listed for review`);
-                        }
-                        state.set(lang, file.id, key, source, cur);
+                if (review) {
+                    // Counted only when something new needs a person's attention, not on every run.
+                    if (review.status === 'approved' && review.valueHash !== curHash) {
+                        state.addReview(lang, file.id, key, 'edited-after-approval', cur);
                         counts.protected++;
-                        continue;
                     }
-                    if (entry && entry.value !== curHash) {
-                        const sourceChanged = entry.source !== sourceHash;
-                        state.addReview(lang, file.id, key, sourceChanged ? 'source-changed' : 'manual-edit', cur);
-                        state.set(lang, file.id, key, source, cur);
-                        log.info(`[${lang}] ${key}: edited by hand; protected and listed for review`);
+                    else if (entry && entry.source !== sourceHash) {
+                        state.addReview(lang, file.id, key, 'source-changed', cur);
+                        log.warn(`[${lang}] ${key}: source changed, but the translation was edited by hand; kept and listed for review`);
                         counts.protected++;
-                        continue;
                     }
+                    state.set(lang, file.id, key, source, cur);
+                    continue;
                 }
-                if (!hasValue || options.retranslateAll) {
-                    fresh.push({ key, source });
-                }
-                else if (!entry) {
-                    // First time localewarden sees this string: adopt an existing translation, unless
-                    // it is just the source text copied over.
-                    if (isUnchangedProse(source, cur, checker.placeholderRe))
-                        fresh.push({ key, source });
-                    else
-                        state.set(lang, file.id, key, source, cur);
-                }
-                else if (entry.source !== sourceHash) {
-                    revise.push({ key, source, previous: cur });
-                }
-                else if (options.fixFlagged) {
-                    const issues = checker.checkString(lang, key, source, cur).filter(issue => FIXABLE_CHECKS.has(issue.check));
-                    const failed = state.repairFailures[reviewId(lang, file.id, key)];
-                    if (issues.length > 0 && failed?.valueHash !== hash(cur))
-                        repairs.push({ key, source, issues });
+                if (entry && entry.value !== curHash) {
+                    const sourceChanged = entry.source !== sourceHash;
+                    state.addReview(lang, file.id, key, sourceChanged ? 'source-changed' : 'manual-edit', cur);
+                    state.set(lang, file.id, key, source, cur);
+                    log.info(`[${lang}] ${key}: edited by hand; protected and listed for review`);
+                    counts.protected++;
+                    continue;
                 }
             }
-            // Strings removed from the source.
-            const removed = [...current.keys()].filter(key => !sourceKeys.has(key));
-            for (const key of new Set([...removed, ...state.keys(lang, file.id).filter(key => !sourceKeys.has(key))])) {
-                state.delete(lang, file.id, key);
-                delete state.review[reviewId(lang, file.id, key)];
+            if (!hasValue || options.retranslateAll) {
+                fresh.push({ key, source, maxLength: scope.maxLength(key) });
             }
-            const work = [...fresh, ...revise];
-            if (dryRun) {
-                counts.planned += work.length + repairs.length;
-                counts.plannedChars += [...work, ...repairs].reduce((sum, item) => sum + item.source.length, 0);
-                counts.removed += removed.length;
-                if (work.length + repairs.length + removed.length > 0) {
-                    log.info(`[${lang}] ${targetRel}: ${fresh.length} new, ${revise.length} changed${repairs.length ? `, ${repairs.length} to repair` : ''}${removed.length ? `, ${removed.length} to remove` : ''}`);
-                }
-                return;
+            else if (!entry) {
+                // First time localewarden sees this string: adopt an existing translation, unless
+                // it is just the source text copied over.
+                if (isUnchangedProse(source, cur, checker.placeholderRe))
+                    fresh.push({ key, source, maxLength: scope.maxLength(key) });
+                else
+                    state.set(lang, file.id, key, source, cur);
             }
-            if (work.length > 0) {
-                log.info(`[${lang}] ${targetRel}: translating ${fresh.length} new, ${revise.length} changed`);
-                const translated = await translateItems(lang, work);
-                for (const item of work) {
-                    const text = translated.get(item.key);
-                    if (text === undefined) {
-                        counts.failed++;
-                        continue;
-                    }
-                    values.set(item.key, text);
-                    state.set(lang, file.id, item.key, item.source, text);
-                    if (item.previous)
-                        counts.revised++;
-                    else
-                        counts.translated++;
-                }
+            else if (entry.source !== sourceHash) {
+                revise.push({ key, source, previous: cur, maxLength: scope.maxLength(key) });
             }
-            for (const { key, source, issues } of repairs) {
-                if (stopped())
-                    break;
-                const before = values.get(key) ?? '';
-                const problems = issues.map(issue => (issue.note ? `${issue.check}: ${issue.note}` : issue.check));
-                const after = await translateOne(lang, { key, source }, repairInstruction(before, problems));
-                const id = reviewId(lang, file.id, key);
-                let reason = after === null ? 'no answer' : null;
-                if (after !== null) {
-                    const checks = new Set(issues.map(issue => issue.check));
-                    const remaining = checker.checkString(lang, key, source, after).filter(issue => checks.has(issue.check));
-                    reason =
-                        checker.defect(lang, key, source, after).hard ??
-                            (remaining.length > 0 ? `still flagged: ${remaining.map(r => r.note ?? r.check).join('; ')}` : null) ??
-                            tooManyChanges(before, after);
+            else if (options.fixFlagged) {
+                const issues = checker.checkString(lang, key, source, cur).filter(issue => FIXABLE_CHECKS.has(issue.check));
+                const failed = state.repairFailures[reviewId(lang, file.id, key)];
+                // Missing content needs more than a minimal correction: revise from the existing text.
+                if (failed?.valueHash === hash(cur))
+                    continue; // this value could not be fixed before
+                if (muchShorter(lang, source, cur))
+                    revise.push({ key, source, previous: cur, maxLength: scope.maxLength(key), completing: true });
+                else if (issues.length > 0)
+                    repairs.push({ key, source, issues });
+            }
+        }
+        // Strings removed from the source.
+        const removed = [...current.keys()].filter(key => !sourceKeys.has(key));
+        for (const key of new Set([...removed, ...state.keys(lang, file.id).filter(key => !sourceKeys.has(key))])) {
+            state.delete(lang, file.id, key);
+            delete state.review[reviewId(lang, file.id, key)];
+        }
+        const work = [...fresh, ...revise];
+        if (work.length + repairs.length + removed.length > 0) {
+            log.info(`[${lang}] ${targetRel}: ${fresh.length} new, ${revise.length} changed${repairs.length ? `, ${repairs.length} to repair` : ''}${removed.length ? `, ${removed.length} to remove` : ''}`);
+        }
+        return { file, lang, counts, sourceDoc, sourceText, targetRel, targetFile, targetText, targetDoc, values, work, repairs, removed, pluralExtras };
+    }
+    /** Applies translations, runs repairs and writes the target file. */
+    async function finishPlan(plan, translated) {
+        const { file, lang, counts, sourceDoc, sourceText, targetRel, targetFile, targetText, targetDoc, values, work, repairs, removed, pluralExtras } = plan;
+        if (work.length > 0) {
+            for (const item of work) {
+                const text = translated.get(item);
+                if (text === undefined) {
+                    counts.failed++;
+                    continue;
                 }
-                if (reason || after === null) {
-                    if (after !== null || !stopped()) {
-                        state.repairFailures[id] = { valueHash: hash(before), date: today(), problems, reason: reason ?? 'no answer', ...(after ? { attempted: after } : {}) };
-                        log.warn(`[${lang}] ${key}: repair rejected (${reason})`);
-                    }
+                if (item.completing && item.previous && muchShorter(lang, item.source, text)) {
+                    // Still missing content: keep the old text and do not spend tokens on it again.
+                    state.repairFailures[reviewId(lang, file.id, item.key)] = { valueHash: hash(item.previous), date: today(), problems: ['content missing'], reason: 'revision still much shorter than the source', attempted: text };
+                    log.warn(`[${lang}] ${item.key}: could not complete the missing content; listed in repair-failures.json`);
+                    counts.failed++;
                     continue;
                 }
-                delete state.repairFailures[id];
-                values.set(key, after);
-                state.set(lang, file.id, key, source, after);
-                counts.repaired++;
-            }
-            counts.removed += removed.length;
-            const built = buildTarget(sourceDoc, values) ?? {};
-            const text = serialize(built, detectFormat(targetText ?? sourceText));
-            if (text !== targetText && !(targetText === null && Object.keys(built).length === 0)) {
-                writeText(targetFile, text);
-                summary.filesWritten.push(targetRel);
+                values.set(item.key, text);
+                state.set(lang, file.id, item.key, item.source, text);
+                if (item.previous)
+                    counts.revised++;
+                else
+                    counts.translated++;
             }
-        });
-    }
-    try {
-        for (const file of files) {
+        }
+        for (const { key, source, issues } of repairs) {
             if (stopped())
                 break;
-            const sourceRel = file.pathFor(config.sourceLanguage);
-            const sourceText = readText(path.join(config.root, sourceRel));
-            if (sourceText === null)
-                continue;
-            let sourceDoc;
-            try {
-                sourceDoc = JSON.parse(sourceText);
+            const before = values.get(key) ?? '';
+            const problems = issues.map(issue => (issue.note ? `${issue.check}: ${issue.note}` : issue.check));
+            const after = await translateOne(lang, { key, source, maxLength: scope.maxLength(key) }, repairInstruction(before, problems));
+            const id = reviewId(lang, file.id, key);
+            let reason = after === null ? 'no answer' : null;
+            if (after !== null) {
+                const checks = new Set(issues.map(issue => issue.check));
+                const remaining = checker.checkString(lang, key, source, after).filter(issue => checks.has(issue.check));
+                reason =
+                    checker.defect(lang, key, source, after).hard ??
+                        (remaining.length > 0 ? `still flagged: ${remaining.map(r => r.note ?? r.check).join('; ')}` : null) ??
+                        tooManyChanges(before, after);
             }
-            catch (error) {
-                log.error(`${sourceRel} is not valid JSON (${error.message}); skipped.`);
+            if (reason || after === null) {
+                if (after !== null || !stopped()) {
+                    state.repairFailures[id] = { valueHash: hash(before), date: today(), problems, reason: reason ?? 'no answer', ...(after ? { attempted: after } : {}) };
+                    log.warn(`[${lang}] ${key}: repair rejected (${reason})`);
+                }
                 continue;
             }
-            await processFile(file, sourceDoc, sourceText);
-            if (!dryRun)
-                state.save();
+            delete state.repairFailures[id];
+            values.set(key, after);
+            state.set(lang, file.id, key, source, after);
+            counts.repaired++;
+        }
+        counts.removed += removed.length;
+        const built = buildTarget(sourceDoc, values) ?? {};
+        for (const leaf of pluralExtras) {
+            const value = values.get(leaf.key);
+            if (value !== undefined)
+                setAt(built, leaf.path, value);
         }
+        // Same content as on disk (only formatting or key order differs): leave the file alone, so
+        // projects formatted with Prettier do not get a diff on every run.
+        const unchanged = targetDoc !== null && canonical(built) === canonical(targetDoc);
+        const text = serializeDoc(targetRel, built, detectFormat(targetText ?? sourceText));
+        if (!unchanged && text !== null && text !== targetText && !(targetText === null && Object.keys(built).length === 0)) {
+            writeText(targetFile, text);
+            summary.filesWritten.push(targetRel);
+        }
+    }
+    const sources = [];
+    for (const file of files) {
+        const sourceRel = file.pathFor(config.sourceLanguage);
+        const text = readText(path.join(config.root, sourceRel));
+        if (text === null)
+            continue;
+        try {
+            const doc = parseDoc(sourceRel, text);
+            sources.push({ file, doc, text, leaves: stringLeaves(doc) });
+        }
+        catch (error) {
+            log.error(`${sourceRel} is not valid JSON (${error.message}); skipped.`);
+        }
+    }
+    try {
+        // Per language: plan every file, translate all their strings in shared batches (many
+        // small files, e.g. store listings, do not cost a request each), then write each file.
+        await inParallel(languages, config.concurrency, async (lang) => {
+            if (stopped())
+                return;
+            const plans = sources.map(source => planFile(source, lang)).filter((plan) => plan !== null);
+            const counts = summary.languages[lang];
+            if (dryRun) {
+                for (const plan of plans) {
+                    counts.planned += plan.work.length + plan.repairs.length;
+                    counts.plannedChars += [...plan.work, ...plan.repairs].reduce((sum, item) => sum + item.source.length, 0);
+                    counts.removed += plan.removed.length;
+                }
+                return;
+            }
+            const work = plans.flatMap(plan => plan.work);
+            const translated = work.length > 0 ? await translateItems(lang, work) : new Map();
+            for (const plan of plans)
+                await finishPlan(plan, translated);
+            state.save();
+        });
     }
     finally {
         if (!dryRun)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "localewarden",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Incremental AI translation for JSON locale files: translates only what changed, never overwrites hand edits, and checks every result before writing it.",
   "type": "module",
   "bin": {
@@ -39,6 +39,17 @@
     "json",
     "locales",
     "i18next",
+    "react-intl",
+    "vue-i18n",
+    "next-intl",
+    "fastlane",
+    "flutter",
+    "arb",
+    "app-store",
+    "ai-translation",
+    "gpt",
+    "chatgpt",
+    "translation-quality",
     "cli"
   ],
   "repository": {