@babelize/engine 0.0.0-stage → 0.1.0

package/dist/modules/translator/memory/memory-matcher.js

@@ -0,0 +1,218 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.matchFromMemory = matchFromMemory;
+const FUZZY_THRESHOLD = 0.9;
+const MAX_FUZZY_MATCHES = 3;
+const Q = 3;
+/**
+ * Fuzzy results are suggestions only (never auto-applied) and only the top
+ * MAX_FUZZY_MATCHES are kept, so Levenshtein is run on at most this many
+ * candidates — those sharing the most q-grams with the query. Without a cap,
+ * a memory full of near-identical templated strings makes every entry a
+ * candidate.
+ */
+const MAX_FUZZY_CANDIDATES = 50;
+/** Largest edit distance that still scores ≥ FUZZY_THRESHOLD (float-safe). */
+function maxDistanceFor(maxLength) {
+    return Math.floor(maxLength - FUZZY_THRESHOLD * maxLength + 1e-9);
+}
+/** Multiset of the string's q-grams: gram → occurrences. */
+function qgrams(text) {
+    const grams = new Map();
+    for (let i = 0; i + Q <= text.length; i++) {
+        const g = text.slice(i, i + Q);
+        grams.set(g, (grams.get(g) ?? 0) + 1);
+    }
+    return grams;
+}
+/**
+ * Levenshtein distance, or `maxDistance + 1` as soon as it is known to exceed
+ * `maxDistance`. Only cells within `maxDistance` of the diagonal can lead to
+ * an answer under the bound, so each row computes that band only, and the
+ * loop exits once a whole band row is over the bound.
+ */
+function boundedLevenshtein(s1, s2, maxDistance) {
+    const m = s1.length;
+    const n = s2.length;
+    const over = maxDistance + 1;
+    if (Math.abs(m - n) > maxDistance)
+        return over;
+    let prev = new Array(n + 1).fill(over);
+    let curr = new Array(n + 1).fill(over);
+    for (let j = 0; j <= Math.min(n, maxDistance); j++)
+        prev[j] = j;
+    for (let i = 1; i <= m; i++) {
+        const from = Math.max(1, i - maxDistance);
+        const to = Math.min(n, i + maxDistance);
+        curr.fill(over);
+        curr[0] = i <= maxDistance ? i : over;
+        let rowMin = curr[0];
+        for (let j = from; j <= to; j++) {
+            const cost = s1[i - 1] === s2[j - 1] ? 0 : 1;
+            curr[j] = Math.min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost, over);
+            if (curr[j] < rowMin)
+                rowMin = curr[j];
+        }
+        if (rowMin > maxDistance)
+            return over;
+        [prev, curr] = [curr, prev];
+    }
+    return Math.min(prev[n], over);
+}
+/**
+ * Indexes are built once per (memory array, target locale) and reused for
+ * every key in the run. The caller fetches one array per locale and matches
+ * every uncached key against it, so rebuilding (or re-hashing every entry)
+ * per key was O(keys × entries).
+ */
+const indexCache = new WeakMap();
+function getIndex(entries, targetLocale) {
+    let byLocale = indexCache.get(entries);
+    if (!byLocale) {
+        byLocale = new Map();
+        indexCache.set(entries, byLocale);
+    }
+    let index = byLocale.get(targetLocale);
+    if (!index) {
+        const forLocale = entries.filter((e) => e.targetLocale === targetLocale);
+        const exact = new Map();
+        for (const e of forLocale) {
+            // Prefer a human-approved entry if the same text appears twice.
+            const existing = exact.get(e.sourceText);
+            if (!existing || (!existing.approvedAt && e.approvedAt))
+                exact.set(e.sourceText, e);
+        }
+        const lists = new Map();
+        forLocale.forEach((e, idx) => {
+            for (const [g, n] of qgrams(e.sourceText)) {
+                let list = lists.get(g);
+                if (!list)
+                    lists.set(g, (list = []));
+                list.push(idx, n);
+            }
+        });
+        const postings = new Map();
+        for (const [g, list] of lists) {
+            const idx = new Int32Array(list.length / 2);
+            const count = new Uint16Array(list.length / 2);
+            for (let p = 0; p < idx.length; p++) {
+                idx[p] = list[2 * p];
+                count[p] = Math.min(list[2 * p + 1], 0xffff);
+            }
+            postings.set(g, { idx, count });
+        }
+        index = { exact, entries: forLocale, postings, shared: new Int32Array(forLocale.length) };
+        byLocale.set(targetLocale, index);
+    }
+    return index;
+}
+/**
+ * Match from memory with exact and fuzzy matching.
+ * Exact matches are a map lookup; fuzzy matches (≥90% similar) are only
+ * computed when there is no exact match, and only against candidates whose
+ * length could possibly reach the threshold.
+ */
+function matchFromMemory(entries, sourceText, targetLocale) {
+    if (!entries.length)
+        return { hit: false };
+    const index = getIndex(entries, targetLocale);
+    const exactMatch = index.exact.get(sourceText);
+    if (exactMatch) {
+        return {
+            hit: true,
+            translatedText: exactMatch.translatedText,
+            confidence: 1,
+            fromMemory: true
+        };
+    }
+    const fuzzyMatches = [];
+    const consider = (i) => {
+        const e = index.entries[i];
+        const maxLength = Math.max(sourceText.length, e.sourceText.length);
+        if (maxLength === 0)
+            return;
+        const maxDistance = maxDistanceFor(maxLength);
+        if (Math.abs(sourceText.length - e.sourceText.length) > maxDistance)
+            return;
+        const distance = boundedLevenshtein(sourceText, e.sourceText, maxDistance);
+        if (distance > maxDistance)
+            return;
+        fuzzyMatches.push({
+            translatedText: e.translatedText,
+            sourceText: e.sourceText,
+            similarity: (maxLength - distance) / maxLength,
+        });
+    };
+    if (sourceText.length <= Q + 1) {
+        // Too short for the q-gram filter to prune safely; short strings are cheap.
+        for (let i = 0; i < index.entries.length; i++)
+            consider(i);
+    }
+    else {
+        // q-gram lemma: strings within edit distance d share at least
+        // max(len) - Q + 1 - Q·d q-grams. Count shared grams via the inverted
+        // index and only run Levenshtein on entries that clear that bar.
+        const { shared } = index;
+        const touched = [];
+        for (const [g, qn] of qgrams(sourceText)) {
+            const posting = index.postings.get(g);
+            if (!posting)
+                continue;
+            const { idx: ids, count: counts } = posting;
+            for (let p = 0; p < ids.length; p++) {
+                const idx = ids[p];
+                if (shared[idx] === 0)
+                    touched.push(idx);
+                shared[idx] += counts[p] < qn ? counts[p] : qn;
+            }
+        }
+        const candidates = [];
+        const candidateCounts = [];
+        let maxCount = 0;
+        for (const idx of touched) {
+            const count = shared[idx];
+            shared[idx] = 0;
+            const maxLength = Math.max(sourceText.length, index.entries[idx].sourceText.length);
+            if (count >= maxLength - Q + 1 - Q * maxDistanceFor(maxLength)) {
+                candidates.push(idx);
+                candidateCounts.push(count);
+                if (count > maxCount)
+                    maxCount = count;
+            }
+        }
+        if (candidates.length <= MAX_FUZZY_CANDIDATES) {
+            for (const idx of candidates)
+                consider(idx);
+        }
+        else {
+            // Top-K by shared count in O(n): counts are small integers, so bucket them.
+            const buckets = new Int32Array(maxCount + 1);
+            for (const c of candidateCounts)
+                buckets[c]++;
+            let cutoff = maxCount;
+            for (let taken = 0; cutoff > 0 && taken + buckets[cutoff] < MAX_FUZZY_CANDIDATES; cutoff--)
+                taken += buckets[cutoff];
+            let budget = MAX_FUZZY_CANDIDATES;
+            for (let i = 0; i < candidates.length && budget > 0; i++) {
+                if (candidateCounts[i] > cutoff) {
+                    consider(candidates[i]);
+                    budget--;
+                }
+            }
+            for (let i = 0; i < candidates.length && budget > 0; i++) {
+                if (candidateCounts[i] === cutoff) {
+                    consider(candidates[i]);
+                    budget--;
+                }
+            }
+        }
+    }
+    if (fuzzyMatches.length > 0) {
+        fuzzyMatches.sort((a, b) => b.similarity - a.similarity);
+        return {
+            hit: false, // Don't auto-apply fuzzy matches
+            fuzzyMatches: fuzzyMatches.slice(0, MAX_FUZZY_MATCHES),
+        };
+    }
+    return { hit: false };
+}

package/dist/modules/translator/memory/types.d.ts

@@ -0,0 +1,22 @@
+export interface MemoryEntry {
+    sourceText: string;
+    sourceLocale: string;
+    targetLocale: string;
+    translatedText: string;
+    projectId?: string;
+    projectName?: string;
+    approvedAt?: string;
+    isoKey?: string;
+}
+export interface FuzzyMatch {
+    translatedText: string;
+    similarity: number;
+    sourceText: string;
+}
+export interface MemoryLookupResult {
+    hit: boolean;
+    translatedText?: string;
+    confidence?: number;
+    fromMemory?: boolean;
+    fuzzyMatches?: FuzzyMatch[];
+}

package/dist/modules/translator/memory/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/modules/translator/translate.d.ts

@@ -0,0 +1,38 @@
+import type { LocaleFile, WorkspaceInfo } from '../../runtime/types';
+import type { ByokProvider } from '@babelize/types';
+import type { EngineProviders, TranslationFailure, TranslationCache } from '../../providers/types';
+/** QA note on keys the backup (NLLB) translator produced. */
+export declare const FALLBACK_QA_WARNING = "translated with the backup translator; will be retried with the AI translator next run";
+export interface TranslateOptions {
+    /**
+     * The project's runtime formats ICU (next-intl, react-i18next + i18next-icu)
+     * and the translator is an LLM: a translation may turn a count placeholder
+     * into an ICU plural. Otherwise a plain source must stay plain.
+     */
+    targetPlurals?: boolean;
+}
+export declare function translateLocales(workspace: WorkspaceInfo, sourceLocale: LocaleFile, targetLocales: string[], forceTranslation?: boolean, dryRun?: boolean, projectId?: string, onProgress?: (message: string) => void, keysToTranslateByLocale?: Record<string, string[]>, cacheProvider?: TranslationCache, byokConfig?: {
+    provider: ByokProvider;
+    apiKey: string;
+    model?: string;
+}, contextMap?: Record<string, string>, extractionContextMap?: Record<string, string>, providers?: EngineProviders, options?: TranslateOptions): Promise<{
+    translation: {
+        locales: LocaleFile[];
+        errors: {
+            locale: string;
+            error: string;
+        }[];
+        failures: TranslationFailure[];
+    };
+    stats: {
+        tmHits: number;
+    };
+    metadata: Record<string, Record<string, any>>;
+}>;
+/**
+ * Backfill TM from translations that were reused without calling the
+ * translator (lockfile hits). Existing TM rows are left untouched.
+ * Locales are ISO codes here; identity translations (source === target) are
+ * skipped since they mean a previous run had no working translator.
+ */
+export declare function backfillTranslationMemory(projectId: string, sourceLocale: string, flatSource: Record<string, string>, translationsByLocale: Record<string, Record<string, string>>, excludeKeys?: Set<string>, providers?: EngineProviders): Promise<Record<string, string[]>>;