@babelize/engine 0.0.0-stage → 0.1.0

package/dist/modules/lockfile/lockfile-manager.d.ts

@@ -0,0 +1,90 @@
+import type { LockfileEntry } from './schema';
+import { type LockfileData, type LockfilePartition, type LockfileStats, type PartitionedKey } from './schema';
+/**
+ * For each target locale, list of translation keys that must go through the translator
+ * (new keys, source-changed keys, or keys missing that locale in the lockfile).
+ */
+export declare function buildKeysToTranslateByTargetLocale(partition: LockfilePartition, targetLocales: string[]): Record<string, string[]>;
+/**
+ * Resolve the lockfile path for a project:
+ * `<dataDir>/.babelize/{projectId}/babelize-lock.json`. `dataDir` defaults to
+ * $BABELIZE_DATA_DIR, else `process.cwd()` (the cloud backend). Local runs
+ * pass it explicitly so concurrent runs in one process never share it.
+ */
+export declare function getLockfilePath(projectId: string, dataDir?: string): string;
+/**
+ * Load and validate the lockfile from disk.
+ * Returns an empty lockfile if the file doesn't exist or is malformed.
+ */
+export declare function loadLockfile(projectId: string, dataDir?: string): Promise<LockfileData>;
+/**
+ * Persist lockfile to disk atomically (write→tmp→rename).
+ * Uses advisory file locking via atomic mkdir to prevent concurrent writes
+ * from overwriting each other (two simultaneous runs for the same project).
+ */
+export declare function saveLockfile(projectId: string, data: LockfileData, dataDir?: string): Promise<void>;
+/**
+ * Delete the lockfile for a project (used by "Force full re-translate").
+ */
+export declare function clearLockfile(projectId: string, dataDir?: string): Promise<void>;
+/**
+ * Purge a specific locale's translations from the lockfile.
+ * Used when a language is removed from a project so re-adding it
+ * triggers fresh translation instead of resurrecting stale cache.
+ */
+export declare function removeLocaleFromLockfile(projectId: string, localeCode: string, dataDir?: string): Promise<void>;
+/**
+ * Partition generated keys into unchanged / new / changed / missingLocales
+ * by comparing source hashes against the lockfile.
+ *
+ * @param lockfile  - Current lockfile data
+ * @param keys      - Flat map of translation key → source string value
+ * @param targetLocales - Locales the user is targeting this run
+ */
+/** Options for reading the lockfile during a run. */
+export interface LockfileReadOptions {
+    /** This run is pseudo-localization: pseudo entries count as done. Otherwise they count as missing. */
+    pseudo?: boolean;
+}
+/**
+ * True when the stored value for `locale` is pseudo-localization and this run
+ * is a real one: it must be translated, never reused. Recognised by the
+ * `pseudo` metadata flag, or by the pseudo markers for lockfiles written
+ * before the flag existed.
+ */
+export declare function isStalePseudoEntry(entry: LockfileEntry, locale: string, options?: LockfileReadOptions): boolean;
+export declare function partitionKeys(lockfile: LockfileData, keys: Record<string, string>, targetLocales: string[], options?: LockfileReadOptions): LockfilePartition;
+/**
+ * Merge newly translated values and metadata into the lockfile.
+ *
+ * @param lockfile     - Current lockfile (will be mutated)
+ * @param keys         - Flat map of key → source value (for hashing)
+ * @param translations - Map of locale → { key → translated string }
+ * @param projectId    - Project identifier
+ * @param metadata     - Map of locale → { key → metadata object }
+ */
+/**
+ * Remove the stored translation of every key that failed this run.
+ * updateLockfile records the new source fingerprint for every key and keeps
+ * the previous translation where a locale produced none — for a key whose
+ * source text changed and then failed, that previous value translates the
+ * *old* text. Dropping it makes the next run treat the locale as missing and
+ * retry it, instead of silently reusing a stale translation.
+ */
+export declare function dropFailedTranslations(lockfile: LockfileData, failures: Array<{
+    key: string;
+    locale: string;
+}>): LockfileData;
+/** True when `next` differs from `previous` in anything but timestamps worth saving. */
+export declare function lockfileChanged(previous: LockfileData, next: LockfileData): boolean;
+export declare function updateLockfile(lockfile: LockfileData, keys: Record<string, string>, translations: Record<string, Record<string, string>>, projectId: string, metadata?: Record<string, Record<string, any>>): LockfileData;
+/**
+ * Extract cached translations for unchanged keys from the lockfile.
+ *
+ * @returns Map of locale → { key → translated string }
+ */
+export declare function collectCachedTranslations(lockfile: LockfileData, unchangedKeys: PartitionedKey[], targetLocales: string[], options?: LockfileReadOptions): Record<string, Record<string, string>>;
+/**
+ * Compute stats for the UI: X translated, Y reused, Z changed.
+ */
+export declare function computeLockfileStats(partition: LockfilePartition): LockfileStats;

package/dist/modules/lockfile/lockfile-manager.js

@@ -0,0 +1,482 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildKeysToTranslateByTargetLocale = buildKeysToTranslateByTargetLocale;
+exports.getLockfilePath = getLockfilePath;
+exports.loadLockfile = loadLockfile;
+exports.saveLockfile = saveLockfile;
+exports.clearLockfile = clearLockfile;
+exports.removeLocaleFromLockfile = removeLocaleFromLockfile;
+exports.isStalePseudoEntry = isStalePseudoEntry;
+exports.partitionKeys = partitionKeys;
+exports.dropFailedTranslations = dropFailedTranslations;
+exports.lockfileChanged = lockfileChanged;
+exports.updateLockfile = updateLockfile;
+exports.collectCachedTranslations = collectCachedTranslations;
+exports.computeLockfileStats = computeLockfileStats;
+const path = __importStar(require("path"));
+const pseudo_markers_1 = require("../../utils/pseudo-markers");
+const fs = __importStar(require("fs-extra"));
+const logger_1 = require("../../utils/logger");
+const hasher_1 = require("./hasher");
+const schema_1 = require("./schema");
+const LOCK_RETRIES = 20;
+const LOCK_RETRY_DELAY_MS = 50;
+async function acquireLock(lockDir) {
+    let lastErr = null;
+    for (let i = 0; i < LOCK_RETRIES; i++) {
+        try {
+            await fs.mkdir(lockDir, { recursive: false });
+            return;
+        }
+        catch (err) {
+            lastErr = err;
+            if (err.code !== 'EEXIST')
+                throw err;
+            await new Promise((r) => setTimeout(r, LOCK_RETRY_DELAY_MS));
+        }
+    }
+    throw new Error(`Could not acquire lockfile lock after ${LOCK_RETRIES} retries: ${lastErr?.message}`);
+}
+async function releaseLock(lockDir) {
+    try {
+        await fs.rmdir(lockDir);
+    }
+    catch {
+        // Best-effort cleanup
+    }
+}
+/**
+ * For each target locale, list of translation keys that must go through the translator
+ * (new keys, source-changed keys, or keys missing that locale in the lockfile).
+ */
+function buildKeysToTranslateByTargetLocale(partition, targetLocales) {
+    const byLocale = {};
+    for (const loc of targetLocales) {
+        byLocale[loc] = new Set();
+    }
+    for (const { key } of partition.newKeys) {
+        for (const loc of targetLocales) {
+            byLocale[loc].add(key);
+        }
+    }
+    for (const { key } of partition.changed) {
+        for (const loc of targetLocales) {
+            byLocale[loc].add(key);
+        }
+    }
+    for (const { key, missingLocales } of partition.missingLocales) {
+        for (const loc of missingLocales) {
+            if (byLocale[loc])
+                byLocale[loc].add(key);
+        }
+    }
+    const result = {};
+    for (const loc of targetLocales) {
+        result[loc] = Array.from(byLocale[loc]);
+    }
+    return result;
+}
+// ─────────────────────────────────────────────────────────────
+// Path helpers
+// ─────────────────────────────────────────────────────────────
+/**
+ * Resolve the lockfile path for a project:
+ * `<dataDir>/.babelize/{projectId}/babelize-lock.json`. `dataDir` defaults to
+ * $BABELIZE_DATA_DIR, else `process.cwd()` (the cloud backend). Local runs
+ * pass it explicitly so concurrent runs in one process never share it.
+ */
+function getLockfilePath(projectId, dataDir) {
+    const baseDir = dataDir || process.env.BABELIZE_DATA_DIR || process.cwd();
+    return path.join(baseDir, '.babelize', projectId, 'babelize-lock.json');
+}
+// ─────────────────────────────────────────────────────────────
+// Load
+// ─────────────────────────────────────────────────────────────
+/**
+ * Load and validate the lockfile from disk.
+ * Returns an empty lockfile if the file doesn't exist or is malformed.
+ */
+async function loadLockfile(projectId, dataDir) {
+    const lockfilePath = getLockfilePath(projectId, dataDir);
+    try {
+        if (!(await fs.pathExists(lockfilePath))) {
+            logger_1.engineLogger.info('No existing lockfile found, starting fresh', { projectId });
+            return (0, schema_1.createEmptyLockfile)(projectId);
+        }
+        const raw = await fs.readJson(lockfilePath);
+        const parsed = schema_1.lockfileDataSchema.safeParse(raw);
+        if (!parsed.success) {
+            logger_1.engineLogger.warn('Lockfile validation failed, starting fresh', {
+                projectId,
+                errors: parsed.error.issues,
+            });
+            return (0, schema_1.createEmptyLockfile)(projectId);
+        }
+        logger_1.engineLogger.info('Loaded lockfile', {
+            projectId,
+            keyCount: Object.keys(parsed.data.keys).length,
+        });
+        return parsed.data;
+    }
+    catch (err) {
+        logger_1.engineLogger.warn('Failed to read lockfile, starting fresh', {
+            projectId,
+            error: err instanceof Error ? err.message : String(err),
+        });
+        return (0, schema_1.createEmptyLockfile)(projectId);
+    }
+}
+// ─────────────────────────────────────────────────────────────
+// Save
+// ─────────────────────────────────────────────────────────────
+/**
+ * Persist lockfile to disk atomically (write→tmp→rename).
+ * Uses advisory file locking via atomic mkdir to prevent concurrent writes
+ * from overwriting each other (two simultaneous runs for the same project).
+ */
+async function saveLockfile(projectId, data, dataDir) {
+    const lockfilePath = getLockfilePath(projectId, dataDir);
+    const lockDir = lockfilePath + '.lock';
+    const tmpPath = lockfilePath + '.tmp';
+    await fs.ensureDir(path.dirname(lockfilePath));
+    await acquireLock(lockDir);
+    try {
+        // Re-read the current lockfile and merge any keys written by a concurrent run
+        // between when we loaded data and now.
+        const merged = data;
+        try {
+            if (await fs.pathExists(lockfilePath)) {
+                const current = await fs.readJson(lockfilePath);
+                for (const [key, entry] of Object.entries(current.keys || {})) {
+                    if (!merged.keys[key]) {
+                        merged.keys[key] = entry;
+                    }
+                }
+            }
+        }
+        catch (readErr) {
+            logger_1.engineLogger.warn('Failed to re-read lockfile for merge, proceeding with current data', {
+                projectId,
+                error: readErr instanceof Error ? readErr.message : String(readErr),
+            });
+        }
+        merged.updatedAt = new Date().toISOString();
+        await fs.writeJson(tmpPath, merged, { spaces: 2 });
+        await fs.rename(tmpPath, lockfilePath);
+        logger_1.engineLogger.info('Saved lockfile', {
+            projectId,
+            keyCount: Object.keys(merged.keys).length,
+        });
+    }
+    catch (err) {
+        logger_1.engineLogger.error('Failed to save lockfile', {
+            projectId,
+            error: err instanceof Error ? err.message : String(err),
+        });
+        await fs.remove(tmpPath).catch((cleanupErr) => {
+            logger_1.engineLogger.warn('Failed to clean up temp lockfile', { tmpPath, error: cleanupErr });
+        });
+        throw err;
+    }
+    finally {
+        await releaseLock(lockDir);
+    }
+}
+// ─────────────────────────────────────────────────────────────
+// Clear (force re-translate)
+// ─────────────────────────────────────────────────────────────
+/**
+ * Delete the lockfile for a project (used by "Force full re-translate").
+ */
+async function clearLockfile(projectId, dataDir) {
+    const lockfilePath = getLockfilePath(projectId, dataDir);
+    try {
+        if (await fs.pathExists(lockfilePath)) {
+            await fs.remove(lockfilePath);
+            logger_1.engineLogger.info('Cleared lockfile', { projectId });
+        }
+    }
+    catch (err) {
+        logger_1.engineLogger.warn('Failed to clear lockfile', {
+            projectId,
+            error: err instanceof Error ? err.message : String(err),
+        });
+    }
+}
+/**
+ * Purge a specific locale's translations from the lockfile.
+ * Used when a language is removed from a project so re-adding it
+ * triggers fresh translation instead of resurrecting stale cache.
+ */
+async function removeLocaleFromLockfile(projectId, localeCode, dataDir) {
+    const lockfilePath = getLockfilePath(projectId, dataDir);
+    try {
+        if (!(await fs.pathExists(lockfilePath))) {
+            return;
+        }
+        const lockfile = await loadLockfile(projectId, dataDir);
+        let removedCount = 0;
+        for (const [key, entry] of Object.entries(lockfile.keys)) {
+            if (localeCode in entry.languages) {
+                delete entry.languages[localeCode];
+                removedCount++;
+            }
+            if (entry.metadata && localeCode in entry.metadata) {
+                delete entry.metadata[localeCode];
+            }
+            if (Object.keys(entry.languages).length === 0) {
+                delete lockfile.keys[key];
+            }
+        }
+        if (removedCount > 0) {
+            lockfile.updatedAt = new Date().toISOString();
+            // Write directly (bypass saveLockfile's merge logic which would
+            // re-add deleted keys from the on-disk file)
+            const lockDir = lockfilePath + '.lock';
+            const tmpPath = lockfilePath + '.tmp';
+            await fs.ensureDir(path.dirname(lockfilePath));
+            await acquireLock(lockDir);
+            try {
+                await fs.writeJson(tmpPath, lockfile, { spaces: 2 });
+                await fs.rename(tmpPath, lockfilePath);
+            }
+            finally {
+                await releaseLock(lockDir);
+            }
+            logger_1.engineLogger.info('Purged locale from lockfile', {
+                projectId,
+                localeCode,
+                keysAffected: removedCount,
+            });
+        }
+    }
+    catch (err) {
+        logger_1.engineLogger.warn('Failed to purge locale from lockfile', {
+            projectId,
+            localeCode,
+            error: err instanceof Error ? err.message : String(err),
+        });
+    }
+}
+/**
+ * True when the stored value for `locale` is pseudo-localization and this run
+ * is a real one: it must be translated, never reused. Recognised by the
+ * `pseudo` metadata flag, or by the pseudo markers for lockfiles written
+ * before the flag existed.
+ */
+function isStalePseudoEntry(entry, locale, options = {}) {
+    if (options.pseudo)
+        return false;
+    return entry.metadata?.[locale]?.pseudo === true || (0, pseudo_markers_1.isPseudoLocalized)(entry.languages[locale] ?? '');
+}
+function partitionKeys(lockfile, keys, targetLocales, options = {}) {
+    const result = {
+        unchanged: [],
+        newKeys: [],
+        changed: [],
+        missingLocales: [],
+    };
+    for (const [key, value] of Object.entries(keys)) {
+        const currentHash = (0, hasher_1.hashSourceString)(value);
+        const entry = lockfile.keys[key];
+        if (!entry) {
+            // Key not seen before
+            result.newKeys.push({ key, value });
+            continue;
+        }
+        if (entry.sourceHash !== currentHash) {
+            // Source string changed
+            result.changed.push({ key, value });
+            continue;
+        }
+        // Hash matches — check if all target locales are present
+        const missing = targetLocales.filter((locale) => !(locale in entry.languages) || isStalePseudoEntry(entry, locale, options));
+        if (missing.length === 0) {
+            // Fully translated and unchanged
+            result.unchanged.push({ key, value });
+        }
+        else {
+            // Some locales missing
+            result.missingLocales.push({ key, value, missingLocales: missing });
+        }
+    }
+    return result;
+}
+// ─────────────────────────────────────────────────────────────
+// Update lockfile with new translations
+// ─────────────────────────────────────────────────────────────
+/**
+ * Merge newly translated values and metadata into the lockfile.
+ *
+ * @param lockfile     - Current lockfile (will be mutated)
+ * @param keys         - Flat map of key → source value (for hashing)
+ * @param translations - Map of locale → { key → translated string }
+ * @param projectId    - Project identifier
+ * @param metadata     - Map of locale → { key → metadata object }
+ */
+/**
+ * Remove the stored translation of every key that failed this run.
+ * updateLockfile records the new source fingerprint for every key and keeps
+ * the previous translation where a locale produced none — for a key whose
+ * source text changed and then failed, that previous value translates the
+ * *old* text. Dropping it makes the next run treat the locale as missing and
+ * retry it, instead of silently reusing a stale translation.
+ */
+function dropFailedTranslations(lockfile, failures) {
+    if (failures.length === 0)
+        return lockfile;
+    const cloned = JSON.parse(JSON.stringify(lockfile));
+    for (const { key, locale } of failures) {
+        const entry = cloned.keys[key];
+        if (entry?.languages)
+            delete entry.languages[locale];
+        if (entry?.metadata)
+            delete entry.metadata[locale];
+    }
+    return cloned;
+}
+/** The metadata fields the lockfile schema keeps (anything else is dropped on load anyway). */
+function pickLockfileMetadata(meta) {
+    const out = {};
+    if (meta.pseudo === true)
+        out.pseudo = true;
+    if (typeof meta.fromMemory === 'boolean')
+        out.fromMemory = meta.fromMemory;
+    if (typeof meta.sourceHash === 'string')
+        out.sourceHash = meta.sourceHash;
+    if (typeof meta.inTm === 'boolean')
+        out.inTm = meta.inTm;
+    return out;
+}
+function sameJson(a, b) {
+    return JSON.stringify(a ?? {}) === JSON.stringify(b ?? {});
+}
+/** True when `next` differs from `previous` in anything but timestamps worth saving. */
+function lockfileChanged(previous, next) {
+    return previous.projectId !== next.projectId || JSON.stringify(previous.keys) !== JSON.stringify(next.keys);
+}
+function updateLockfile(lockfile, keys, translations, projectId, metadata) {
+    // Deep-clone to prevent in-place mutation of the caller's reference.
+    // Prevents races when multiple concurrent runs share the same lockfile object.
+    const cloned = JSON.parse(JSON.stringify(lockfile));
+    const now = new Date().toISOString();
+    cloned.projectId = projectId;
+    for (const [key, value] of Object.entries(keys)) {
+        const sourceHash = (0, hasher_1.hashSourceString)(value);
+        const existing = cloned.keys[key];
+        // An entry is only touched (and re-stamped) when something in it
+        // actually changes, so a no-change rerun leaves the committed
+        // lockfile byte-identical.
+        const entry = existing ?? { sourceHash, translatedAt: now, languages: {}, metadata: {} };
+        let changed = !existing || existing.sourceHash !== sourceHash;
+        entry.sourceHash = sourceHash;
+        // Merge per-locale translations and metadata
+        for (const [locale, translatedKeys] of Object.entries(translations)) {
+            if (translatedKeys[key]) {
+                // Skip caching identity translations (same as source text) for non-trivial strings.
+                // Identity translations mean the translator fell back to source text
+                // (e.g., because the service was unavailable). Caching them would cause
+                // subsequent runs to skip re-translation and produce fake locale files.
+                const translated = translatedKeys[key];
+                if (translated === value && value.length > 4) {
+                    logger_1.engineLogger.warn(`Skipping cache for "${key}" → "${locale}": identity translation (same as source). Translator may be unavailable.`);
+                    continue;
+                }
+                if (entry.languages[locale] !== translated) {
+                    entry.languages[locale] = translated;
+                    changed = true;
+                }
+                // Metadata from this run replaces the previous run's rather than
+                // merging into it, so a stale `fromMemory: true` doesn't survive
+                // the key being re-translated. Keys reused from the lockfile
+                // carry no new metadata and keep what they had.
+                if (metadata?.[locale]?.[key]) {
+                    const next = pickLockfileMetadata(metadata[locale][key]);
+                    if (!sameJson(entry.metadata?.[locale], next)) {
+                        if (!entry.metadata)
+                            entry.metadata = {};
+                        entry.metadata[locale] = next;
+                        changed = true;
+                    }
+                }
+            }
+        }
+        if (changed) {
+            entry.translatedAt = now;
+            if (!entry.metadata)
+                entry.metadata = {};
+            cloned.keys[key] = entry;
+        }
+    }
+    return cloned;
+}
+// ─────────────────────────────────────────────────────────────
+// Collect cached translations from lockfile
+// ─────────────────────────────────────────────────────────────
+/**
+ * Extract cached translations for unchanged keys from the lockfile.
+ *
+ * @returns Map of locale → { key → translated string }
+ */
+function collectCachedTranslations(lockfile, unchangedKeys, targetLocales, options = {}) {
+    const cached = {};
+    for (const locale of targetLocales) {
+        cached[locale] = {};
+    }
+    for (const { key } of unchangedKeys) {
+        const entry = lockfile.keys[key];
+        if (!entry)
+            continue;
+        for (const locale of targetLocales) {
+            if (entry.languages[locale] && !isStalePseudoEntry(entry, locale, options)) {
+                cached[locale][key] = entry.languages[locale];
+            }
+        }
+    }
+    return cached;
+}
+/**
+ * Compute stats for the UI: X translated, Y reused, Z changed.
+ */
+function computeLockfileStats(partition) {
+    return {
+        reused: partition.unchanged.length,
+        translated: partition.newKeys.length +
+            partition.changed.length +
+            partition.missingLocales.length,
+        changed: partition.changed.length,
+    };
+}