@babelize/engine 0.0.0-stage → 0.1.0

package/dist/modules/translator/client/retry.js

@@ -0,0 +1,94 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.withRetry503 = exports.ProviderHttpError = void 0;
+exports.parseRetryAfter = parseRetryAfter;
+exports.isRetryable = isRetryable;
+exports.withRetry = withRetry;
+function abortError() {
+    const err = new Error('Translation request aborted');
+    err.name = 'AbortError';
+    return err;
+}
+function sleep(ms, signal) {
+    return new Promise((resolve, reject) => {
+        if (signal?.aborted)
+            return reject(abortError());
+        const onAbort = () => {
+            clearTimeout(timer);
+            reject(abortError());
+        };
+        const timer = setTimeout(() => {
+            signal?.removeEventListener('abort', onAbort);
+            resolve();
+        }, ms);
+        signal?.addEventListener('abort', onAbort, { once: true });
+    });
+}
+/** An HTTP error from a translation API, carrying what retry decisions need. */
+class ProviderHttpError extends Error {
+    constructor(message, status, 
+    /** Server-requested wait from a Retry-After header, in ms. */
+    retryAfterMs) {
+        super(message);
+        this.status = status;
+        this.retryAfterMs = retryAfterMs;
+        this.name = 'ProviderHttpError';
+    }
+}
+exports.ProviderHttpError = ProviderHttpError;
+/** Parse a Retry-After header (seconds or HTTP date) into milliseconds. */
+function parseRetryAfter(header) {
+    if (!header)
+        return undefined;
+    const seconds = Number(header);
+    if (Number.isFinite(seconds) && seconds >= 0)
+        return seconds * 1000;
+    const date = Date.parse(header);
+    if (!Number.isNaN(date))
+        return Math.max(0, date - Date.now());
+    return undefined;
+}
+/**
+ * Retry only what can succeed on a second try: timeouts, network errors,
+ * 408/409/425/429 and 5xx. Auth, bad request and unknown model (other 4xx)
+ * fail immediately — retrying a bad key just delays the error.
+ */
+function isRetryable(err) {
+    if (err instanceof ProviderHttpError) {
+        return err.status >= 500 || [408, 409, 425, 429].includes(err.status);
+    }
+    return true;
+}
+async function withRetry(fn, opts) {
+    const maxDelay = opts.maxDelayMs ?? 60000;
+    let attempt = 0;
+    const outer = opts.signal;
+    while (true) {
+        if (outer?.aborted)
+            throw abortError();
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), opts.timeoutMs);
+        const onAbort = () => controller.abort();
+        outer?.addEventListener('abort', onAbort, { once: true });
+        try {
+            return await fn(controller.signal);
+        }
+        catch (err) {
+            if (outer?.aborted)
+                throw abortError();
+            attempt += 1;
+            if (attempt > opts.maxRetries || !isRetryable(err))
+                throw err;
+            const backoff = opts.baseDelayMs * Math.pow(2, attempt - 1);
+            const requested = err instanceof ProviderHttpError ? err.retryAfterMs : undefined;
+            const delay = Math.min(maxDelay, Math.max(backoff, requested ?? 0));
+            await sleep(delay, outer);
+        }
+        finally {
+            clearTimeout(timer);
+            outer?.removeEventListener('abort', onAbort);
+        }
+    }
+}
+/** @deprecated Use withRetry — kept for existing callers. */
+exports.withRetry503 = withRetry;

package/dist/modules/translator/client/test-utils/mock.client.d.ts

@@ -0,0 +1,31 @@
+import type { TranslatorClient } from '../translator.interface';
+import type { HFTranslateRequest, HFTranslateResponse } from '../request.types';
+/**
+ * Mock translator client for unit tests.
+ *
+ * - `health()` / `warmup()` → always succeed (configurable)
+ * - `translate()` → deterministic: prefixes each value with `[targetLang]`
+ * - `calls` array records every invocation for assertion
+ *
+ * Usage:
+ *   const mock = new MockTranslatorClient();
+ *   // … run code under test …
+ *   expect(mock.calls.length).toBe(2);
+ */
+export declare class MockTranslatorClient implements TranslatorClient {
+    /** Every translate() call is pushed here for inspection. */
+    readonly calls: HFTranslateRequest[];
+    /** Configure health() return value. */
+    healthResult: boolean;
+    /** Configure warmup() return value. */
+    warmupResult: boolean;
+    /** If set, translate() will throw this error. */
+    translateError: Error | null;
+    /** Custom translate handler — overrides default deterministic behaviour. */
+    translateHandler: ((req: HFTranslateRequest) => HFTranslateResponse) | null;
+    health(): Promise<boolean>;
+    warmup(): Promise<boolean>;
+    translate(req: HFTranslateRequest): Promise<HFTranslateResponse>;
+    /** Reset call history and configuration. */
+    reset(): void;
+}

package/dist/modules/translator/client/test-utils/mock.client.js

@@ -0,0 +1,59 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MockTranslatorClient = void 0;
+/**
+ * Mock translator client for unit tests.
+ *
+ * - `health()` / `warmup()` → always succeed (configurable)
+ * - `translate()` → deterministic: prefixes each value with `[targetLang]`
+ * - `calls` array records every invocation for assertion
+ *
+ * Usage:
+ *   const mock = new MockTranslatorClient();
+ *   // … run code under test …
+ *   expect(mock.calls.length).toBe(2);
+ */
+class MockTranslatorClient {
+    constructor() {
+        /** Every translate() call is pushed here for inspection. */
+        this.calls = [];
+        /** Configure health() return value. */
+        this.healthResult = true;
+        /** Configure warmup() return value. */
+        this.warmupResult = true;
+        /** If set, translate() will throw this error. */
+        this.translateError = null;
+        /** Custom translate handler — overrides default deterministic behaviour. */
+        this.translateHandler = null;
+    }
+    async health() {
+        return this.healthResult;
+    }
+    async warmup() {
+        return this.warmupResult;
+    }
+    async translate(req) {
+        this.calls.push(req);
+        if (this.translateError)
+            throw this.translateError;
+        if (this.translateHandler)
+            return this.translateHandler(req);
+        // Deterministic: prefix each value with the target language code
+        return {
+            translations: req.entries.map((e) => ({
+                key: e.key,
+                value: `[${req.targetLang}] ${e.value}`,
+            })),
+            model: 'mock-nllb-200',
+        };
+    }
+    /** Reset call history and configuration. */
+    reset() {
+        this.calls.length = 0;
+        this.healthResult = true;
+        this.warmupResult = true;
+        this.translateError = null;
+        this.translateHandler = null;
+    }
+}
+exports.MockTranslatorClient = MockTranslatorClient;

package/dist/modules/translator/client/translator.interface.d.ts

@@ -0,0 +1 @@
+export type { TranslatorClient } from '../../../types';

package/dist/modules/translator/client/translator.interface.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/modules/translator/language-codes/errors.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Babelize Error Types
+ *
+ * Typed error class for structured error handling across the engine.
+ */
+export type BabelizeErrorCode = 'UNSUPPORTED_LANGUAGE' | 'TRANSLATION_FAILED' | 'INVALID_INPUT';
+export declare class BabelizeError extends Error {
+    readonly code: BabelizeErrorCode;
+    constructor(code: BabelizeErrorCode, message: string);
+}

package/dist/modules/translator/language-codes/errors.js

@@ -0,0 +1,18 @@
+"use strict";
+/**
+ * Babelize Error Types
+ *
+ * Typed error class for structured error handling across the engine.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BabelizeError = void 0;
+class BabelizeError extends Error {
+    constructor(code, message) {
+        super(message);
+        this.name = 'BabelizeError';
+        this.code = code;
+        // Maintains proper prototype chain for instanceof checks
+        Object.setPrototypeOf(this, BabelizeError.prototype);
+    }
+}
+exports.BabelizeError = BabelizeError;

package/dist/modules/translator/language-codes/flores-map.d.ts

@@ -0,0 +1 @@
+export declare const FLORES_MAP: Record<string, string>;

package/dist/modules/translator/language-codes/flores-map.js

@@ -0,0 +1,107 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FLORES_MAP = void 0;
+exports.FLORES_MAP = {
+    af: "afr_Latn",
+    sq: "als_Latn",
+    am: "amh_Ethi",
+    ar: "arb_Arab",
+    hy: "hye_Armn",
+    az: "azj_Latn",
+    eu: "eus_Latn",
+    be: "bel_Cyrl",
+    bn: "ben_Beng",
+    bs: "bos_Latn",
+    bg: "bul_Cyrl",
+    my: "mya_Mymr",
+    ca: "cat_Latn",
+    ceb: "ceb_Latn",
+    hr: "hrv_Latn",
+    cs: "ces_Latn",
+    da: "dan_Latn",
+    nl: "nld_Latn",
+    en: "eng_Latn",
+    eo: "epo_Latn",
+    et: "est_Latn",
+    tl: "tgl_Latn",
+    fi: "fin_Latn",
+    fr: "fra_Latn",
+    gl: "glg_Latn",
+    ka: "kat_Geor",
+    de: "deu_Latn",
+    el: "ell_Grek",
+    gu: "guj_Gujr",
+    ht: "hat_Latn",
+    ha: "hau_Latn",
+    haw: "haw_Latn",
+    he: "heb_Hebr",
+    hi: "hin_Deva",
+    hmn: "hmn_Latn",
+    hu: "hun_Latn",
+    is: "isl_Latn",
+    ig: "ibo_Latn",
+    id: "ind_Latn",
+    ga: "gle_Latn",
+    it: "ita_Latn",
+    ja: "jpn_Jpan",
+    jw: "jav_Latn",
+    kn: "kan_Knda",
+    kk: "kaz_Cyrl",
+    km: "khm_Khmr",
+    ko: "kor_Hang",
+    ku: "kmr_Latn",
+    ky: "kir_Cyrl",
+    lo: "lao_Laoo",
+    la: "lat_Latn",
+    lv: "lvs_Latn",
+    lt: "lit_Latn",
+    lb: "ltz_Latn",
+    mk: "mkd_Cyrl",
+    mg: "plt_Latn",
+    ms: "zsm_Latn",
+    ml: "mal_Mlym",
+    mt: "mlt_Latn",
+    mi: "mri_Latn",
+    mr: "mar_Deva",
+    mn: "khk_Cyrl",
+    ne: "npi_Deva",
+    no: "nob_Latn",
+    nb: "nob_Latn",
+    ny: "nya_Latn",
+    ps: "pbt_Arab",
+    fa: "pes_Arab",
+    pl: "pol_Latn",
+    pt: "por_Latn",
+    pa: "pan_Guru",
+    ro: "ron_Latn",
+    ru: "rus_Cyrl",
+    sm: "smo_Latn",
+    gd: "gla_Latn",
+    sr: "srp_Cyrl",
+    st: "sot_Latn",
+    sn: "sna_Latn",
+    sd: "snd_Arab",
+    si: "sin_Sinh",
+    sk: "slk_Latn",
+    sl: "slv_Latn",
+    so: "som_Latn",
+    es: "spa_Latn",
+    su: "sun_Latn",
+    sw: "swh_Latn",
+    sv: "swe_Latn",
+    tg: "tgk_Cyrl",
+    ta: "tam_Taml",
+    te: "tel_Telu",
+    th: "tha_Thai",
+    tr: "tur_Latn",
+    uk: "ukr_Cyrl",
+    ur: "urd_Arab",
+    uz: "uzn_Latn",
+    vi: "vie_Latn",
+    cy: "cym_Latn",
+    xh: "xho_Latn",
+    yi: "ydd_Hebr",
+    yo: "yor_Latn",
+    zu: "zul_Latn",
+    zh: "zho_Hans",
+};

package/dist/modules/translator/language-codes/index.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Language Codes Module — barrel export
+ *
+ * Single import point for all language code utilities:
+ *   import { toFlores, isSupported, RTL_LANGUAGES } from '../modules/translator/language-codes';
+ */
+export { BabelizeError } from './errors';
+export type { BabelizeErrorCode } from './errors';
+export { FLORES_MAP } from './flores-map';
+export { toFlores, fromFlores, isSupported, isNllbSupported, getSupportedLanguages, storageLangCode, localeFromStorageCode } from './language-mapper';
+export { RTL_LANGUAGES, isRTL } from './rtl-languages';

package/dist/modules/translator/language-codes/index.js

@@ -0,0 +1,24 @@
+"use strict";
+/**
+ * Language Codes Module — barrel export
+ *
+ * Single import point for all language code utilities:
+ *   import { toFlores, isSupported, RTL_LANGUAGES } from '../modules/translator/language-codes';
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isRTL = exports.RTL_LANGUAGES = exports.localeFromStorageCode = exports.storageLangCode = exports.getSupportedLanguages = exports.isNllbSupported = exports.isSupported = exports.fromFlores = exports.toFlores = exports.FLORES_MAP = exports.BabelizeError = void 0;
+var errors_1 = require("./errors");
+Object.defineProperty(exports, "BabelizeError", { enumerable: true, get: function () { return errors_1.BabelizeError; } });
+var flores_map_1 = require("./flores-map");
+Object.defineProperty(exports, "FLORES_MAP", { enumerable: true, get: function () { return flores_map_1.FLORES_MAP; } });
+var language_mapper_1 = require("./language-mapper");
+Object.defineProperty(exports, "toFlores", { enumerable: true, get: function () { return language_mapper_1.toFlores; } });
+Object.defineProperty(exports, "fromFlores", { enumerable: true, get: function () { return language_mapper_1.fromFlores; } });
+Object.defineProperty(exports, "isSupported", { enumerable: true, get: function () { return language_mapper_1.isSupported; } });
+Object.defineProperty(exports, "isNllbSupported", { enumerable: true, get: function () { return language_mapper_1.isNllbSupported; } });
+Object.defineProperty(exports, "getSupportedLanguages", { enumerable: true, get: function () { return language_mapper_1.getSupportedLanguages; } });
+Object.defineProperty(exports, "storageLangCode", { enumerable: true, get: function () { return language_mapper_1.storageLangCode; } });
+Object.defineProperty(exports, "localeFromStorageCode", { enumerable: true, get: function () { return language_mapper_1.localeFromStorageCode; } });
+var rtl_languages_1 = require("./rtl-languages");
+Object.defineProperty(exports, "RTL_LANGUAGES", { enumerable: true, get: function () { return rtl_languages_1.RTL_LANGUAGES; } });
+Object.defineProperty(exports, "isRTL", { enumerable: true, get: function () { return rtl_languages_1.isRTL; } });

package/dist/modules/translator/language-codes/language-mapper.d.ts

@@ -0,0 +1,25 @@
+/**
+ * FLORES-200 code for an ISO locale.
+ * - Region variants map to their language (pt-BR → por_Latn); Chinese regions
+ *   pick the script (zh-TW → zho_Hant, zh-CN → zho_Hans).
+ * - Script subtags are honoured: sr-Cyrl → srp_Cyrl, pa-Arab → pnb_Arab. A
+ *   script FLORES-200 lacks gets a FLORES-style code (sr-Latn → srp_Latn) that
+ *   only LLM providers can use — see isNllbSupported().
+ */
+export declare function toFlores(isoCode: string): string;
+export declare function fromFlores(floresCode: string): string;
+/** True when any translator can handle the locale (LLM providers use the ISO tag). */
+export declare function isSupported(isoCode: string): boolean;
+/** True when the NLLB translator service can translate into the locale. */
+export declare function isNllbSupported(isoCode: string): boolean;
+/**
+ * Key for storing translations of `isoCode` (translation cache, translation
+ * memory): the FLORES code including the script, plus the region when there
+ * is one, so variants never share translations — pt-BR → por_Latn-br,
+ * sr-Latn → srp_Latn, sr-Cyrl-RS → srp_Cyrl-rs. A plain language code keeps
+ * its FLORES code (pt → por_Latn), as before.
+ */
+export declare function storageLangCode(isoCode: string): string;
+/** BCP-47 tag for a storage code (por_Latn-br → pt-BR, srp_Latn → sr-Latn). */
+export declare function localeFromStorageCode(code: string): string;
+export declare function getSupportedLanguages(): string[];

package/dist/modules/translator/language-codes/language-mapper.js

@@ -0,0 +1,132 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.toFlores = toFlores;
+exports.fromFlores = fromFlores;
+exports.isSupported = isSupported;
+exports.isNllbSupported = isNllbSupported;
+exports.storageLangCode = storageLangCode;
+exports.localeFromStorageCode = localeFromStorageCode;
+exports.getSupportedLanguages = getSupportedLanguages;
+const errors_1 = require("./errors");
+const flores_map_1 = require("./flores-map");
+/**
+ * Script variants that have their own FLORES-200 code (NLLB can translate
+ * them). Keys are `language-script` in lower case.
+ */
+const SCRIPT_FLORES = {
+    'zh-hans': 'zho_Hans',
+    'zh-hant': 'zho_Hant',
+    'pa-arab': 'pnb_Arab',
+    'az-arab': 'azb_Arab',
+};
+/** Chinese regions imply a script. */
+const REGION_SCRIPT = {
+    'zh-tw': 'hant',
+    'zh-hk': 'hant',
+    'zh-mo': 'hant',
+    'zh-cn': 'hans',
+    'zh-sg': 'hans',
+};
+/** language[-Script][-REGION], e.g. pt-BR, sr-Latn, zh-Hant-TW, es-419. */
+const LOCALE_TAG = /^([a-z]{2,3})(?:-([a-z]{4}))?(?:-([a-z]{2}|\d{3}))?$/;
+function normalize(code) {
+    return code.trim().toLowerCase().replace(/_/g, '-');
+}
+function own(map, key) {
+    // Own keys only: `in` would also accept prototype keys like "constructor".
+    return Object.prototype.hasOwnProperty.call(map, key) ? map[key] : undefined;
+}
+function titleCase(script) {
+    return script.charAt(0).toUpperCase() + script.slice(1);
+}
+function resolve(isoCode) {
+    const n = normalize(isoCode);
+    const plain = own(flores_map_1.FLORES_MAP, n);
+    if (plain)
+        return { flores: plain, nllb: true, plain: true };
+    const m = LOCALE_TAG.exec(n);
+    if (!m)
+        return undefined;
+    const [, lang, explicitScript, region] = m;
+    const base = own(flores_map_1.FLORES_MAP, lang);
+    if (!base)
+        return undefined;
+    const script = explicitScript ?? (region ? own(REGION_SCRIPT, `${lang}-${region}`) : undefined);
+    if (!script)
+        return { flores: base, nllb: true, plain: false, region };
+    const variant = own(SCRIPT_FLORES, `${lang}-${script}`);
+    if (variant)
+        return { flores: variant, nllb: true, plain: false, region };
+    const [baseLang, baseScript] = base.split('_');
+    if (script === baseScript.toLowerCase())
+        return { flores: base, nllb: true, plain: false, region };
+    // A script FLORES-200 doesn't have for this language (sr-Latn, uz-Cyrl,
+    // az-Cyrl): LLM providers translate it from the ISO locale; NLLB can't.
+    return { flores: `${baseLang}_${titleCase(script)}`, nllb: false, plain: false, region };
+}
+/**
+ * FLORES-200 code for an ISO locale.
+ * - Region variants map to their language (pt-BR → por_Latn); Chinese regions
+ *   pick the script (zh-TW → zho_Hant, zh-CN → zho_Hans).
+ * - Script subtags are honoured: sr-Cyrl → srp_Cyrl, pa-Arab → pnb_Arab. A
+ *   script FLORES-200 lacks gets a FLORES-style code (sr-Latn → srp_Latn) that
+ *   only LLM providers can use — see isNllbSupported().
+ */
+function toFlores(isoCode) {
+    const r = resolve(isoCode);
+    if (!r) {
+        throw new errors_1.BabelizeError('UNSUPPORTED_LANGUAGE', `Unsupported language code: "${isoCode}". Use getSupportedLanguages() to see available codes.`);
+    }
+    return r.flores;
+}
+function fromFlores(floresCode) {
+    const entry = Object.entries(flores_map_1.FLORES_MAP).find(([, code]) => code === floresCode);
+    if (entry)
+        return entry[0];
+    const variant = Object.entries(SCRIPT_FLORES).find(([, code]) => code === floresCode);
+    if (variant) {
+        const [lang, script] = variant[0].split('-');
+        return `${lang}-${titleCase(script)}`;
+    }
+    const [langPart, script] = floresCode.split('_');
+    const base = Object.entries(flores_map_1.FLORES_MAP).find(([, code]) => code.split('_')[0] === langPart);
+    if (base && script)
+        return `${base[0]}-${script}`;
+    throw new errors_1.BabelizeError('UNSUPPORTED_LANGUAGE', `Flores code '${floresCode}' is not supported.`);
+}
+/** True when any translator can handle the locale (LLM providers use the ISO tag). */
+function isSupported(isoCode) {
+    return resolve(isoCode) !== undefined;
+}
+/** True when the NLLB translator service can translate into the locale. */
+function isNllbSupported(isoCode) {
+    return resolve(isoCode)?.nllb === true;
+}
+/**
+ * Key for storing translations of `isoCode` (translation cache, translation
+ * memory): the FLORES code including the script, plus the region when there
+ * is one, so variants never share translations — pt-BR → por_Latn-br,
+ * sr-Latn → srp_Latn, sr-Cyrl-RS → srp_Cyrl-rs. A plain language code keeps
+ * its FLORES code (pt → por_Latn), as before.
+ */
+function storageLangCode(isoCode) {
+    const r = resolve(isoCode);
+    if (!r)
+        return toFlores(isoCode); // throws the usual error
+    if (r.plain || !r.region)
+        return r.flores;
+    return `${r.flores}-${r.region}`;
+}
+/** BCP-47 tag for a storage code (por_Latn-br → pt-BR, srp_Latn → sr-Latn). */
+function localeFromStorageCode(code) {
+    const [flores, region] = code.split('-');
+    if (!region)
+        return fromFlores(flores);
+    // Chinese regions imply the script: zho_Hant-tw → zh-TW, zho_Hans-cn → zh-CN.
+    if (flores === 'zho_Hant' || flores === 'zho_Hans')
+        return `zh-${region.toUpperCase()}`;
+    return `${fromFlores(flores)}-${/^\d+$/.test(region) ? region : region.toUpperCase()}`;
+}
+function getSupportedLanguages() {
+    return Object.keys(flores_map_1.FLORES_MAP);
+}

package/dist/modules/translator/language-codes/rtl-languages.d.ts

@@ -0,0 +1,2 @@
+export declare const RTL_LANGUAGES: string[];
+export declare function isRTL(locale: string): boolean;

package/dist/modules/translator/language-codes/rtl-languages.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RTL_LANGUAGES = void 0;
+exports.isRTL = isRTL;
+exports.RTL_LANGUAGES = ['ar', 'he', 'fa', 'ur', 'ps'];
+function isRTL(locale) {
+    const base = locale.split('-')[0];
+    return exports.RTL_LANGUAGES.includes(base);
+}

package/dist/modules/translator/memory/memory-matcher.d.ts

@@ -0,0 +1,8 @@
+import type { MemoryEntry, MemoryLookupResult } from './types';
+/**
+ * Match from memory with exact and fuzzy matching.
+ * Exact matches are a map lookup; fuzzy matches (≥90% similar) are only
+ * computed when there is no exact match, and only against candidates whose
+ * length could possibly reach the threshold.
+ */
+export declare function matchFromMemory(entries: MemoryEntry[], sourceText: string, targetLocale: string): MemoryLookupResult;