@babelize/engine 0.0.0-stage → 0.1.2

package/dist/modules/translator/language-codes/language-mapper.js

@@ -0,0 +1,132 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.toFlores = toFlores;
+exports.fromFlores = fromFlores;
+exports.isSupported = isSupported;
+exports.isNllbSupported = isNllbSupported;
+exports.storageLangCode = storageLangCode;
+exports.localeFromStorageCode = localeFromStorageCode;
+exports.getSupportedLanguages = getSupportedLanguages;
+const errors_1 = require("./errors");
+const flores_map_1 = require("./flores-map");
+/**
+ * Script variants that have their own FLORES-200 code (NLLB can translate
+ * them). Keys are `language-script` in lower case.
+ */
+const SCRIPT_FLORES = {
+    'zh-hans': 'zho_Hans',
+    'zh-hant': 'zho_Hant',
+    'pa-arab': 'pnb_Arab',
+    'az-arab': 'azb_Arab',
+};
+/** Chinese regions imply a script. */
+const REGION_SCRIPT = {
+    'zh-tw': 'hant',
+    'zh-hk': 'hant',
+    'zh-mo': 'hant',
+    'zh-cn': 'hans',
+    'zh-sg': 'hans',
+};
+/** language[-Script][-REGION], e.g. pt-BR, sr-Latn, zh-Hant-TW, es-419. */
+const LOCALE_TAG = /^([a-z]{2,3})(?:-([a-z]{4}))?(?:-([a-z]{2}|\d{3}))?$/;
+function normalize(code) {
+    return code.trim().toLowerCase().replace(/_/g, '-');
+}
+function own(map, key) {
+    // Own keys only: `in` would also accept prototype keys like "constructor".
+    return Object.prototype.hasOwnProperty.call(map, key) ? map[key] : undefined;
+}
+function titleCase(script) {
+    return script.charAt(0).toUpperCase() + script.slice(1);
+}
+function resolve(isoCode) {
+    const n = normalize(isoCode);
+    const plain = own(flores_map_1.FLORES_MAP, n);
+    if (plain)
+        return { flores: plain, nllb: true, plain: true };
+    const m = LOCALE_TAG.exec(n);
+    if (!m)
+        return undefined;
+    const [, lang, explicitScript, region] = m;
+    const base = own(flores_map_1.FLORES_MAP, lang);
+    if (!base)
+        return undefined;
+    const script = explicitScript ?? (region ? own(REGION_SCRIPT, `${lang}-${region}`) : undefined);
+    if (!script)
+        return { flores: base, nllb: true, plain: false, region };
+    const variant = own(SCRIPT_FLORES, `${lang}-${script}`);
+    if (variant)
+        return { flores: variant, nllb: true, plain: false, region };
+    const [baseLang, baseScript] = base.split('_');
+    if (script === baseScript.toLowerCase())
+        return { flores: base, nllb: true, plain: false, region };
+    // A script FLORES-200 doesn't have for this language (sr-Latn, uz-Cyrl,
+    // az-Cyrl): LLM providers translate it from the ISO locale; NLLB can't.
+    return { flores: `${baseLang}_${titleCase(script)}`, nllb: false, plain: false, region };
+}
+/**
+ * FLORES-200 code for an ISO locale.
+ * - Region variants map to their language (pt-BR → por_Latn); Chinese regions
+ *   pick the script (zh-TW → zho_Hant, zh-CN → zho_Hans).
+ * - Script subtags are honoured: sr-Cyrl → srp_Cyrl, pa-Arab → pnb_Arab. A
+ *   script FLORES-200 lacks gets a FLORES-style code (sr-Latn → srp_Latn) that
+ *   only LLM providers can use — see isNllbSupported().
+ */
+function toFlores(isoCode) {
+    const r = resolve(isoCode);
+    if (!r) {
+        throw new errors_1.BabelizeError('UNSUPPORTED_LANGUAGE', `Unsupported language code: "${isoCode}". Use getSupportedLanguages() to see available codes.`);
+    }
+    return r.flores;
+}
+function fromFlores(floresCode) {
+    const entry = Object.entries(flores_map_1.FLORES_MAP).find(([, code]) => code === floresCode);
+    if (entry)
+        return entry[0];
+    const variant = Object.entries(SCRIPT_FLORES).find(([, code]) => code === floresCode);
+    if (variant) {
+        const [lang, script] = variant[0].split('-');
+        return `${lang}-${titleCase(script)}`;
+    }
+    const [langPart, script] = floresCode.split('_');
+    const base = Object.entries(flores_map_1.FLORES_MAP).find(([, code]) => code.split('_')[0] === langPart);
+    if (base && script)
+        return `${base[0]}-${script}`;
+    throw new errors_1.BabelizeError('UNSUPPORTED_LANGUAGE', `Flores code '${floresCode}' is not supported.`);
+}
+/** True when any translator can handle the locale (LLM providers use the ISO tag). */
+function isSupported(isoCode) {
+    return resolve(isoCode) !== undefined;
+}
+/** True when the NLLB translator service can translate into the locale. */
+function isNllbSupported(isoCode) {
+    return resolve(isoCode)?.nllb === true;
+}
+/**
+ * Key for storing translations of `isoCode` (translation cache, translation
+ * memory): the FLORES code including the script, plus the region when there
+ * is one, so variants never share translations — pt-BR → por_Latn-br,
+ * sr-Latn → srp_Latn, sr-Cyrl-RS → srp_Cyrl-rs. A plain language code keeps
+ * its FLORES code (pt → por_Latn), as before.
+ */
+function storageLangCode(isoCode) {
+    const r = resolve(isoCode);
+    if (!r)
+        return toFlores(isoCode); // throws the usual error
+    if (r.plain || !r.region)
+        return r.flores;
+    return `${r.flores}-${r.region}`;
+}
+/** BCP-47 tag for a storage code (por_Latn-br → pt-BR, srp_Latn → sr-Latn). */
+function localeFromStorageCode(code) {
+    const [flores, region] = code.split('-');
+    if (!region)
+        return fromFlores(flores);
+    // Chinese regions imply the script: zho_Hant-tw → zh-TW, zho_Hans-cn → zh-CN.
+    if (flores === 'zho_Hant' || flores === 'zho_Hans')
+        return `zh-${region.toUpperCase()}`;
+    return `${fromFlores(flores)}-${/^\d+$/.test(region) ? region : region.toUpperCase()}`;
+}
+function getSupportedLanguages() {
+    return Object.keys(flores_map_1.FLORES_MAP);
+}

package/dist/modules/translator/language-codes/rtl-languages.d.ts

@@ -0,0 +1,2 @@
+export declare const RTL_LANGUAGES: string[];
+export declare function isRTL(locale: string): boolean;

package/dist/modules/translator/language-codes/rtl-languages.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RTL_LANGUAGES = void 0;
+exports.isRTL = isRTL;
+exports.RTL_LANGUAGES = ['ar', 'he', 'fa', 'ur', 'ps'];
+function isRTL(locale) {
+    const base = locale.split('-')[0];
+    return exports.RTL_LANGUAGES.includes(base);
+}

package/dist/modules/translator/memory/memory-matcher.d.ts

@@ -0,0 +1,8 @@
+import type { MemoryEntry, MemoryLookupResult } from './types';
+/**
+ * Match from memory with exact and fuzzy matching.
+ * Exact matches are a map lookup; fuzzy matches (≥90% similar) are only
+ * computed when there is no exact match, and only against candidates whose
+ * length could possibly reach the threshold.
+ */
+export declare function matchFromMemory(entries: MemoryEntry[], sourceText: string, targetLocale: string): MemoryLookupResult;

package/dist/modules/translator/memory/memory-matcher.js

@@ -0,0 +1,218 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.matchFromMemory = matchFromMemory;
+const FUZZY_THRESHOLD = 0.9;
+const MAX_FUZZY_MATCHES = 3;
+const Q = 3;
+/**
+ * Fuzzy results are suggestions only (never auto-applied) and only the top
+ * MAX_FUZZY_MATCHES are kept, so Levenshtein is run on at most this many
+ * candidates — those sharing the most q-grams with the query. Without a cap,
+ * a memory full of near-identical templated strings makes every entry a
+ * candidate.
+ */
+const MAX_FUZZY_CANDIDATES = 50;
+/** Largest edit distance that still scores ≥ FUZZY_THRESHOLD (float-safe). */
+function maxDistanceFor(maxLength) {
+    return Math.floor(maxLength - FUZZY_THRESHOLD * maxLength + 1e-9);
+}
+/** Multiset of the string's q-grams: gram → occurrences. */
+function qgrams(text) {
+    const grams = new Map();
+    for (let i = 0; i + Q <= text.length; i++) {
+        const g = text.slice(i, i + Q);
+        grams.set(g, (grams.get(g) ?? 0) + 1);
+    }
+    return grams;
+}
+/**
+ * Levenshtein distance, or `maxDistance + 1` as soon as it is known to exceed
+ * `maxDistance`. Only cells within `maxDistance` of the diagonal can lead to
+ * an answer under the bound, so each row computes that band only, and the
+ * loop exits once a whole band row is over the bound.
+ */
+function boundedLevenshtein(s1, s2, maxDistance) {
+    const m = s1.length;
+    const n = s2.length;
+    const over = maxDistance + 1;
+    if (Math.abs(m - n) > maxDistance)
+        return over;
+    let prev = new Array(n + 1).fill(over);
+    let curr = new Array(n + 1).fill(over);
+    for (let j = 0; j <= Math.min(n, maxDistance); j++)
+        prev[j] = j;
+    for (let i = 1; i <= m; i++) {
+        const from = Math.max(1, i - maxDistance);
+        const to = Math.min(n, i + maxDistance);
+        curr.fill(over);
+        curr[0] = i <= maxDistance ? i : over;
+        let rowMin = curr[0];
+        for (let j = from; j <= to; j++) {
+            const cost = s1[i - 1] === s2[j - 1] ? 0 : 1;
+            curr[j] = Math.min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost, over);
+            if (curr[j] < rowMin)
+                rowMin = curr[j];
+        }
+        if (rowMin > maxDistance)
+            return over;
+        [prev, curr] = [curr, prev];
+    }
+    return Math.min(prev[n], over);
+}
+/**
+ * Indexes are built once per (memory array, target locale) and reused for
+ * every key in the run. The caller fetches one array per locale and matches
+ * every uncached key against it, so rebuilding (or re-hashing every entry)
+ * per key was O(keys × entries).
+ */
+const indexCache = new WeakMap();
+function getIndex(entries, targetLocale) {
+    let byLocale = indexCache.get(entries);
+    if (!byLocale) {
+        byLocale = new Map();
+        indexCache.set(entries, byLocale);
+    }
+    let index = byLocale.get(targetLocale);
+    if (!index) {
+        const forLocale = entries.filter((e) => e.targetLocale === targetLocale);
+        const exact = new Map();
+        for (const e of forLocale) {
+            // Prefer a human-approved entry if the same text appears twice.
+            const existing = exact.get(e.sourceText);
+            if (!existing || (!existing.approvedAt && e.approvedAt))
+                exact.set(e.sourceText, e);
+        }
+        const lists = new Map();
+        forLocale.forEach((e, idx) => {
+            for (const [g, n] of qgrams(e.sourceText)) {
+                let list = lists.get(g);
+                if (!list)
+                    lists.set(g, (list = []));
+                list.push(idx, n);
+            }
+        });
+        const postings = new Map();
+        for (const [g, list] of lists) {
+            const idx = new Int32Array(list.length / 2);
+            const count = new Uint16Array(list.length / 2);
+            for (let p = 0; p < idx.length; p++) {
+                idx[p] = list[2 * p];
+                count[p] = Math.min(list[2 * p + 1], 0xffff);
+            }
+            postings.set(g, { idx, count });
+        }
+        index = { exact, entries: forLocale, postings, shared: new Int32Array(forLocale.length) };
+        byLocale.set(targetLocale, index);
+    }
+    return index;
+}
+/**
+ * Match from memory with exact and fuzzy matching.
+ * Exact matches are a map lookup; fuzzy matches (≥90% similar) are only
+ * computed when there is no exact match, and only against candidates whose
+ * length could possibly reach the threshold.
+ */
+function matchFromMemory(entries, sourceText, targetLocale) {
+    if (!entries.length)
+        return { hit: false };
+    const index = getIndex(entries, targetLocale);
+    const exactMatch = index.exact.get(sourceText);
+    if (exactMatch) {
+        return {
+            hit: true,
+            translatedText: exactMatch.translatedText,
+            confidence: 1,
+            fromMemory: true
+        };
+    }
+    const fuzzyMatches = [];
+    const consider = (i) => {
+        const e = index.entries[i];
+        const maxLength = Math.max(sourceText.length, e.sourceText.length);
+        if (maxLength === 0)
+            return;
+        const maxDistance = maxDistanceFor(maxLength);
+        if (Math.abs(sourceText.length - e.sourceText.length) > maxDistance)
+            return;
+        const distance = boundedLevenshtein(sourceText, e.sourceText, maxDistance);
+        if (distance > maxDistance)
+            return;
+        fuzzyMatches.push({
+            translatedText: e.translatedText,
+            sourceText: e.sourceText,
+            similarity: (maxLength - distance) / maxLength,
+        });
+    };
+    if (sourceText.length <= Q + 1) {
+        // Too short for the q-gram filter to prune safely; short strings are cheap.
+        for (let i = 0; i < index.entries.length; i++)
+            consider(i);
+    }
+    else {
+        // q-gram lemma: strings within edit distance d share at least
+        // max(len) - Q + 1 - Q·d q-grams. Count shared grams via the inverted
+        // index and only run Levenshtein on entries that clear that bar.
+        const { shared } = index;
+        const touched = [];
+        for (const [g, qn] of qgrams(sourceText)) {
+            const posting = index.postings.get(g);
+            if (!posting)
+                continue;
+            const { idx: ids, count: counts } = posting;
+            for (let p = 0; p < ids.length; p++) {
+                const idx = ids[p];
+                if (shared[idx] === 0)
+                    touched.push(idx);
+                shared[idx] += counts[p] < qn ? counts[p] : qn;
+            }
+        }
+        const candidates = [];
+        const candidateCounts = [];
+        let maxCount = 0;
+        for (const idx of touched) {
+            const count = shared[idx];
+            shared[idx] = 0;
+            const maxLength = Math.max(sourceText.length, index.entries[idx].sourceText.length);
+            if (count >= maxLength - Q + 1 - Q * maxDistanceFor(maxLength)) {
+                candidates.push(idx);
+                candidateCounts.push(count);
+                if (count > maxCount)
+                    maxCount = count;
+            }
+        }
+        if (candidates.length <= MAX_FUZZY_CANDIDATES) {
+            for (const idx of candidates)
+                consider(idx);
+        }
+        else {
+            // Top-K by shared count in O(n): counts are small integers, so bucket them.
+            const buckets = new Int32Array(maxCount + 1);
+            for (const c of candidateCounts)
+                buckets[c]++;
+            let cutoff = maxCount;
+            for (let taken = 0; cutoff > 0 && taken + buckets[cutoff] < MAX_FUZZY_CANDIDATES; cutoff--)
+                taken += buckets[cutoff];
+            let budget = MAX_FUZZY_CANDIDATES;
+            for (let i = 0; i < candidates.length && budget > 0; i++) {
+                if (candidateCounts[i] > cutoff) {
+                    consider(candidates[i]);
+                    budget--;
+                }
+            }
+            for (let i = 0; i < candidates.length && budget > 0; i++) {
+                if (candidateCounts[i] === cutoff) {
+                    consider(candidates[i]);
+                    budget--;
+                }
+            }
+        }
+    }
+    if (fuzzyMatches.length > 0) {
+        fuzzyMatches.sort((a, b) => b.similarity - a.similarity);
+        return {
+            hit: false, // Don't auto-apply fuzzy matches
+            fuzzyMatches: fuzzyMatches.slice(0, MAX_FUZZY_MATCHES),
+        };
+    }
+    return { hit: false };
+}

package/dist/modules/translator/memory/types.d.ts

@@ -0,0 +1,22 @@
+export interface MemoryEntry {
+    sourceText: string;
+    sourceLocale: string;
+    targetLocale: string;
+    translatedText: string;
+    projectId?: string;
+    projectName?: string;
+    approvedAt?: string;
+    isoKey?: string;
+}
+export interface FuzzyMatch {
+    translatedText: string;
+    similarity: number;
+    sourceText: string;
+}
+export interface MemoryLookupResult {
+    hit: boolean;
+    translatedText?: string;
+    confidence?: number;
+    fromMemory?: boolean;
+    fuzzyMatches?: FuzzyMatch[];
+}

package/dist/modules/translator/memory/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/modules/translator/translate.d.ts

@@ -0,0 +1,74 @@
+import type { LocaleFile, WorkspaceInfo } from '../../runtime/types';
+import type { ByokProvider } from '@babelize/types';
+import { type GlossaryRules } from '../glossary';
+import type { EngineProviders, ProjectRulesStore, TranslationFailure, TranslationCache } from '../../providers/types';
+/** QA note on keys the backup (NLLB) translator produced. */
+export declare const FALLBACK_QA_WARNING = "translated with the backup translator; will be retried with the AI translator next run";
+export interface TranslateOptions {
+    /**
+     * The project's runtime formats ICU (next-intl, react-i18next + i18next-icu):
+     * a translation may turn a count placeholder into an ICU plural. An LLM
+     * decides per message; for a plain-text translator (NLLB) the plural is
+     * built branch by branch for languages with more than one plural category.
+     * Otherwise a plain source must stay plain.
+     */
+    targetPlurals?: boolean;
+    /**
+     * Per key: literal text in an ICU plural's branches that stands for the
+     * plural's value besides `#` (a gettext plural turned into ICU keeps its
+     * `%(count)d`). Lets a plain-text translator inflect each branch.
+     */
+    pluralValuePlaceholders?: Record<string, string>;
+    /**
+     * The project's glossary and brand voice, already loaded (the orchestrator
+     * loads them once to partition the lockfile). Loaded here when absent.
+     */
+    rules?: LoadedTranslationRules;
+    /**
+     * Running count of fresh primary-translator translations, bumped as each
+     * batch lands (when it is also written to the cache and TM). Lets a run
+     * stopped mid-translation bill what it already produced.
+     */
+    progress?: {
+        freshTranslations: number;
+    };
+}
+export interface LoadedTranslationRules {
+    glossary: GlossaryRules;
+    brandVoice: string | null;
+    /** False if the glossary or brand voice failed to load. */
+    complete: boolean;
+}
+/**
+ * Load a project's glossary and brand voice. Never throws: a failure means
+ * translating without that rule, as before. The two lookups are independent,
+ * so a glossary failure doesn't also drop the brand voice. `complete` is false
+ * if either failed — callers that copy text into shared TM must then assume
+ * project-specific wording may be baked in.
+ */
+export declare function loadProjectTranslationRules(projectId: string, rules?: ProjectRulesStore): Promise<LoadedTranslationRules>;
+export declare function translateLocales(workspace: WorkspaceInfo, sourceLocale: LocaleFile, targetLocales: string[], forceTranslation?: boolean, dryRun?: boolean, projectId?: string, onProgress?: (message: string) => void, keysToTranslateByLocale?: Record<string, string[]>, cacheProvider?: TranslationCache, byokConfig?: {
+    provider: ByokProvider;
+    apiKey: string;
+    model?: string;
+}, contextMap?: Record<string, string>, extractionContextMap?: Record<string, string>, providers?: EngineProviders, options?: TranslateOptions): Promise<{
+    translation: {
+        locales: LocaleFile[];
+        errors: {
+            locale: string;
+            error: string;
+        }[];
+        failures: TranslationFailure[];
+    };
+    stats: {
+        tmHits: number;
+    };
+    metadata: Record<string, Record<string, any>>;
+}>;
+/**
+ * Backfill TM from translations that were reused without calling the
+ * translator (lockfile hits). Existing TM rows are left untouched.
+ * Locales are ISO codes here; identity translations (source === target) are
+ * skipped since they mean a previous run had no working translator.
+ */
+export declare function backfillTranslationMemory(projectId: string, sourceLocale: string, flatSource: Record<string, string>, translationsByLocale: Record<string, Record<string, string>>, excludeKeys?: Set<string>, providers?: EngineProviders): Promise<Record<string, string[]>>;