@babelize/engine 0.0.0-stage → 0.1.0

package/dist/modules/sync-detector/sync-detector.js

@@ -0,0 +1,374 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.detectSyncIssues = detectSyncIssues;
+const path = __importStar(require("path"));
+const fs = __importStar(require("fs-extra"));
+const logger_1 = require("../../utils/logger");
+const extract_1 = require("../extractor/extract");
+const lockfile_manager_1 = require("../lockfile/lockfile-manager");
+const types_1 = require("../../runtime/types");
+const extraction_options_1 = require("../../utils/extraction-options");
+/**
+ * Extraction options the project's engine runs use (sentence grouping, ICU
+ * plurals, data-literal rules), so a sync check compares like with like and
+ * doesn't report every sentence as new.
+ */
+async function runExtractionOptions(workspacePath, options) {
+    let packageJson;
+    try {
+        packageJson = (await fs.readJson(path.join(workspacePath, 'package.json'))) ?? undefined;
+    }
+    catch {
+        packageJson = undefined;
+    }
+    const deps = { ...packageJson?.dependencies, ...packageJson?.devDependencies };
+    const framework = options.framework ?? (deps.next ? 'nextjs' : deps.react ? 'react' : deps.vue ? 'vue' : 'unknown');
+    const workspace = {
+        path: workspacePath, framework, sourceDirs: ['.'], hasLocalesDir: false, existingLocales: [], publicDir: 'public',
+        localeFormat: 'json', packageJson,
+    };
+    const icu = await (0, extraction_options_1.resolveIcuOptions)(workspace, !!options.llmTranslator);
+    return { framework, sentences: icu.sentences, plurals: icu.plurals };
+}
+// ─────────────────────────────────────────────────────────────
+// File filtering
+// ─────────────────────────────────────────────────────────────
+/**
+ * Patterns for files that should never trigger sync detection.
+ * Covers config, test, build, and documentation files.
+ */
+const SYNC_IGNORE_PATTERNS = [
+    ...types_1.EXCLUDED_PATTERNS.map(p => typeof p === 'string' ? new RegExp(p.replace(/\*/g, '.*')) : p),
+    // Additional patterns specific to sync detection
+    /\.test\.(ts|tsx|js|jsx)$/,
+    /\.spec\.(ts|tsx|js|jsx)$/,
+    /\.stories\.(ts|tsx|js|jsx)$/,
+    /__tests__\//,
+    /__mocks__\//,
+    /\.config\.(ts|js|mjs|cjs)$/,
+    /jest\.config/,
+    /playwright\.config/,
+    /next\.config/,
+    /tailwind\.config/,
+    /postcss\.config/,
+    /tsconfig/,
+    /\.eslintrc/,
+    /\.prettierrc/,
+    /\.env/,
+    /package\.json$/,
+    /package-lock\.json$/,
+    /bun\.lock$/,
+    /yarn\.lock$/,
+    /README\.md$/i,
+    /CHANGELOG\.md$/i,
+    /LICENSE/i,
+    /\.d\.ts$/,
+    /\.map$/,
+    /dist\//,
+    /build\//,
+    /\.next\//,
+    /node_modules\//,
+    // Locale files themselves should not trigger detection
+    /locales?\//,
+    /i18n\//,
+    /translations?\//,
+    /lang\//,
+    /messages\//,
+];
+/**
+ * Check if a file path matches supported source extensions
+ * and is not excluded by ignore patterns.
+ */
+function isSourceFile(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    if (!types_1.SUPPORTED_EXTENSIONS.includes(ext))
+        return false;
+    for (const pattern of SYNC_IGNORE_PATTERNS) {
+        if (pattern.test(filePath))
+            return false;
+    }
+    return true;
+}
+// ─────────────────────────────────────────────────────────────
+// Core detection
+// ─────────────────────────────────────────────────────────────
+/**
+ * Detect sync issues by comparing extracted strings from changed files
+ * against the lockfile and locale files.
+ *
+ * @param projectId      The project identifier (for lockfile lookup)
+ * @param workspacePath  Absolute path to the project workspace
+ * @param changedFiles   List of file paths changed in the push (relative to repo root)
+ * @param commitSha      The commit SHA that triggered this check
+ * @param targetLocales  Target locales configured for this project
+ */
+async function detectSyncIssues(projectId, workspacePath, changedFiles, commitSha, targetLocales, options = {}) {
+    const now = new Date().toISOString();
+    const filesChecked = [];
+    const filesSkipped = [];
+    // 1. Filter changed files to only source files
+    const sourceFiles = [];
+    for (const file of changedFiles) {
+        if (isSourceFile(file)) {
+            sourceFiles.push(file);
+        }
+        else {
+            filesSkipped.push(file);
+        }
+    }
+    if (sourceFiles.length === 0) {
+        logger_1.engineLogger.info('No source files in push, skipping sync check', {
+            projectId,
+            commitSha,
+            totalChanged: changedFiles.length,
+        });
+        return {
+            newStrings: [],
+            changedStrings: [],
+            missingLocaleKeys: [],
+            totalIssues: 0,
+            commitSha,
+            checkedAt: now,
+            filesChecked: [],
+            filesSkipped,
+        };
+    }
+    // 2. Extract strings from changed source files, the way runs do
+    const extractedStrings = {};
+    const extractOptions = await runExtractionOptions(workspacePath, options);
+    for (const relativePath of sourceFiles) {
+        const absolutePath = path.join(workspacePath, relativePath);
+        try {
+            if (!(await fs.pathExists(absolutePath))) {
+                filesSkipped.push(relativePath);
+                continue;
+            }
+            const content = await fs.readFile(absolutePath, 'utf-8');
+            const { strings } = await (0, extract_1.extractStringsFromFile)(content, relativePath, false, extractOptions);
+            for (const s of strings) {
+                // Use the text as both key and value since we're doing pre-keygen detection
+                const key = s.id || `${relativePath}:${s.line}:${s.text}`;
+                extractedStrings[key] = s.text;
+            }
+            filesChecked.push(relativePath);
+        }
+        catch (err) {
+            logger_1.engineLogger.warn('Failed to extract strings from file during sync check', {
+                projectId,
+                file: relativePath,
+                error: err instanceof Error ? err.message : String(err),
+            });
+            filesSkipped.push(relativePath);
+        }
+    }
+    if (Object.keys(extractedStrings).length === 0) {
+        logger_1.engineLogger.info('No translatable strings found in changed files', {
+            projectId,
+            commitSha,
+            filesChecked: filesChecked.length,
+        });
+        return {
+            newStrings: [],
+            changedStrings: [],
+            missingLocaleKeys: [],
+            totalIssues: 0,
+            commitSha,
+            checkedAt: now,
+            filesChecked,
+            filesSkipped,
+        };
+    }
+    // 3. Load lockfile and partition keys
+    const lockfile = await (0, lockfile_manager_1.loadLockfile)(projectId);
+    const partition = (0, lockfile_manager_1.partitionKeys)(lockfile, extractedStrings, targetLocales);
+    // 4. Build detection results
+    const newStrings = partition.newKeys.map(k => ({
+        key: k.key,
+        value: k.value,
+        file: k.key.split(':')[0] || 'unknown',
+        line: parseInt(k.key.split(':')[1] || '0', 10),
+    }));
+    const changedStrings = partition.changed.map(k => ({
+        key: k.key,
+        value: k.value,
+        file: k.key.split(':')[0] || 'unknown',
+        line: parseInt(k.key.split(':')[1] || '0', 10),
+    }));
+    // 5. Detect missing locale keys
+    const missingLocaleKeys = partition.missingLocales.map(k => ({
+        key: k.key,
+        presentIn: targetLocales.filter(loc => !k.missingLocales.includes(loc)),
+        missingFrom: k.missingLocales,
+    }));
+    // 6. Also check locale files on disk for cross-locale missing keys
+    const diskMissing = await detectMissingLocaleKeysFromDisk(workspacePath, targetLocales);
+    for (const m of diskMissing) {
+        // Only add if not already detected via lockfile
+        if (!missingLocaleKeys.some(existing => existing.key === m.key)) {
+            missingLocaleKeys.push(m);
+        }
+    }
+    const totalIssues = newStrings.length + changedStrings.length + missingLocaleKeys.length;
+    logger_1.engineLogger.info('Sync detection complete', {
+        projectId,
+        commitSha,
+        newStrings: newStrings.length,
+        changedStrings: changedStrings.length,
+        missingLocaleKeys: missingLocaleKeys.length,
+        totalIssues,
+        filesChecked: filesChecked.length,
+    });
+    return {
+        newStrings,
+        changedStrings,
+        missingLocaleKeys,
+        totalIssues,
+        commitSha,
+        checkedAt: now,
+        filesChecked,
+        filesSkipped,
+    };
+}
+// ─────────────────────────────────────────────────────────────
+// Locale file disk check
+// ─────────────────────────────────────────────────────────────
+/**
+ * Scan locale JSON files on disk to detect keys present in the
+ * base locale (en) but missing from target locales.
+ */
+async function detectMissingLocaleKeysFromDisk(workspacePath, targetLocales) {
+    const missing = [];
+    // Common locale directory patterns
+    const localeDirCandidates = [
+        'locales',
+        'locale',
+        'i18n',
+        'translations',
+        'lang',
+        'messages',
+        'public/locales',
+        'src/locales',
+        'src/i18n',
+    ];
+    let localeDir = null;
+    for (const candidate of localeDirCandidates) {
+        const dirPath = path.join(workspacePath, candidate);
+        if (await fs.pathExists(dirPath)) {
+            localeDir = dirPath;
+            break;
+        }
+    }
+    if (!localeDir)
+        return missing;
+    try {
+        // Load the base locale file (en.json)
+        const baseLocalePath = path.join(localeDir, 'en.json');
+        if (!(await fs.pathExists(baseLocalePath)))
+            return missing;
+        const baseLocale = await fs.readJson(baseLocalePath);
+        const baseKeys = flattenKeys(baseLocale);
+        // Check each target locale
+        for (const locale of targetLocales) {
+            if (locale === 'en')
+                continue;
+            const localeFilePath = path.join(localeDir, `${locale}.json`);
+            if (!(await fs.pathExists(localeFilePath))) {
+                // Entire locale file missing — all keys are missing
+                for (const key of Object.keys(baseKeys)) {
+                    const existing = missing.find(m => m.key === key);
+                    if (existing) {
+                        if (!existing.missingFrom.includes(locale)) {
+                            existing.missingFrom.push(locale);
+                        }
+                    }
+                    else {
+                        missing.push({
+                            key,
+                            presentIn: ['en'],
+                            missingFrom: [locale],
+                        });
+                    }
+                }
+                continue;
+            }
+            const localeData = await fs.readJson(localeFilePath);
+            const localeKeys = flattenKeys(localeData);
+            for (const key of Object.keys(baseKeys)) {
+                if (!(key in localeKeys)) {
+                    const existing = missing.find(m => m.key === key);
+                    if (existing) {
+                        if (!existing.missingFrom.includes(locale)) {
+                            existing.missingFrom.push(locale);
+                        }
+                        if (!existing.presentIn.includes('en')) {
+                            existing.presentIn.push('en');
+                        }
+                    }
+                    else {
+                        missing.push({
+                            key,
+                            presentIn: ['en'],
+                            missingFrom: [locale],
+                        });
+                    }
+                }
+            }
+        }
+    }
+    catch (err) {
+        logger_1.engineLogger.warn('Failed to check locale files on disk', {
+            error: err instanceof Error ? err.message : String(err),
+        });
+    }
+    return missing;
+}
+/**
+ * Flatten a nested JSON object into a flat key map.
+ * e.g., { "a": { "b": "hello" } } → { "a.b": "hello" }
+ */
+function flattenKeys(obj, prefix = '') {
+    const result = {};
+    for (const [key, value] of Object.entries(obj)) {
+        const fullKey = prefix ? `${prefix}.${key}` : key;
+        if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+            Object.assign(result, flattenKeys(value, fullKey));
+        }
+        else {
+            result[fullKey] = String(value);
+        }
+    }
+    return result;
+}

package/dist/modules/translator/batch-optimizer.d.ts

@@ -0,0 +1,12 @@
+import type { TranslationUnit } from '../../types';
+export interface Batch {
+    units: TranslationUnit[];
+}
+/**
+ * Groups translation units into batches that respect a rough token limit and
+ * ensures we do NOT make parallel HTTP calls to the HF Space.
+ */
+export declare function createBatches(units: TranslationUnit[], options?: {
+    maxBatchSize?: number;
+    maxCharsPerBatch?: number;
+}): Batch[];

package/dist/modules/translator/batch-optimizer.js

@@ -0,0 +1,30 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createBatches = createBatches;
+/**
+ * Groups translation units into batches that respect a rough token limit and
+ * ensures we do NOT make parallel HTTP calls to the HF Space.
+ */
+function createBatches(units, options = {}) {
+    const maxBatchSize = options.maxBatchSize ?? 50;
+    const maxCharsPerBatch = options.maxCharsPerBatch ?? 2000;
+    const batches = [];
+    let current = [];
+    let currentChars = 0;
+    for (const unit of units) {
+        const textLen = unit.sourceText.length;
+        const wouldExceedSize = current.length >= maxBatchSize;
+        const wouldExceedChars = currentChars + textLen > maxCharsPerBatch;
+        if (current.length > 0 && (wouldExceedSize || wouldExceedChars)) {
+            batches.push({ units: current });
+            current = [];
+            currentChars = 0;
+        }
+        current.push(unit);
+        currentChars += textLen;
+    }
+    if (current.length > 0) {
+        batches.push({ units: current });
+    }
+    return batches;
+}

package/dist/modules/translator/client/byok.provider.d.ts

@@ -0,0 +1,64 @@
+/**
+ * BYOK (Bring Your Own Key) Translation Provider
+ *
+ * Direct integration with Anthropic, OpenAI, Gemini, Groq and OpenRouter for
+ * users who provide their own API keys. Zero Babelize AI credits consumed.
+ *
+ * Contract: returns a translation only for entries the model actually
+ * translated. An entry it skipped or garbled is simply absent from the
+ * response, and the engine reports it as a failed key — never as the source
+ * text.
+ */
+import type { HFTranslateRequest, HFTranslateResponse } from './request.types';
+import type { TranslatorClient } from './translator.interface';
+import type { ByokProvider } from '@babelize/types';
+interface BYOKProviderConfig {
+    provider: ByokProvider;
+    apiKey: string;
+    /** Optional model override */
+    model?: string;
+}
+interface PromptEntry {
+    id: string;
+    text: string;
+    context?: string;
+    element?: string;
+    note?: string;
+}
+export declare class BYOKTranslatorClient implements TranslatorClient {
+    private readonly config;
+    constructor(config: BYOKProviderConfig);
+    health(): Promise<boolean>;
+    warmup(): Promise<boolean>;
+    translate(req: HFTranslateRequest, signal?: AbortSignal): Promise<HFTranslateResponse>;
+    private getModel;
+    private getBaseUrl;
+    private getProviderDisplayName;
+    /** Anthropic: force a tool call so the output is schema-shaped JSON. */
+    private callAnthropic;
+    private callOpenAICompatible;
+    private httpError;
+    private formatApiError;
+    private redactApiKey;
+    /**
+     * Estimate token count and cost for a translation request
+     */
+    static estimateCost(entries: Array<{
+        value: string;
+    }>, provider: ByokProvider): {
+        estimatedTokens: number;
+        estimatedCost: number | null;
+        currency: 'USD';
+    };
+}
+export declare function buildSystemPrompt(req: HFTranslateRequest, entries: PromptEntry[]): string;
+/**
+ * Pull `{translations:[{id, value}]}` out of a tool input object or a text
+ * reply (tolerating code fences and surrounding prose). Returns only entries
+ * whose id was requested and whose value is a string; nothing is invented.
+ */
+export declare function parseTranslations(content: unknown, requestedIds: Set<string>): Array<{
+    key: string;
+    value: string;
+}>;
+export {};