i18nizer 0.5.0 → 0.5.1

package/dist/commands/extract.js

@@ -8,6 +8,8 @@ import { extractTexts } from "../core/ast/extract-text.js";
 import { insertUseTranslations } from "../core/ast/insert-user-translations.js";
 import { parseFile } from "../core/ast/parse-file.js";
 import { replaceTempKeysWithT } from "../core/ast/replace-text-with-text.js";
+import { TranslationCache } from "../core/cache/translation-cache.js";
+import { Deduplicator } from "../core/deduplication/deduplicator.js";
 import { parseAiJson } from "../core/i18n/parse-ai-json.js";
 import { saveSourceFile } from "../core/i18n/sace-source-file.js";
 import { writeLocaleFiles } from "../core/i18n/write-files.js";
@@ -30,6 +32,11 @@ export default class Extract extends Command {
             char: "p",
             description: "AI provider (gemini | huggingface), optional",
         }),
+        "use-ai-keys": Flags.boolean({
+            allowNo: true,
+            default: true,
+            description: "Use AI to generate human-readable keys (default: true)",
+        }),
     };
     async run() {
         const { args, flags } = await this.parse(Extract);
@@ -51,17 +58,58 @@ export default class Extract extends Command {
             this.log(chalk.yellow("⚠️  No translatable texts found."));
             return;
         }
-        this.log(`🔍 Found ${chalk.green(texts.length)} translatable texts`);
         const componentName = path
             .basename(args.file)
             .replace(/\.(tsx|jsx)$/, "");
         const locales = flags.locales.split(",");
+        // Count unique strings
+        const uniqueTexts = new Set(texts.map((t) => t.text));
+        this.log(`🔍 Extracting strings... (${chalk.green(texts.length)} found, ${chalk.green(uniqueTexts.size)} unique)`);
+        // Initialize cache and deduplicator
+        const cwd = process.cwd();
+        const projectDir = path.join(cwd, ".i18nizer");
+        const cache = new TranslationCache(projectDir);
+        const deduplicator = new Deduplicator(cache, flags["use-ai-keys"], provider);
+        // Step 1: Generate keys (batch, cache-first)
+        let keySpinner;
+        if (flags["use-ai-keys"]) {
+            keySpinner = ora("🧠 Generating keys (batch)...").start();
+        }
+        else {
+            this.log("🔑 Generating keys (deterministic)...");
+        }
+        const textList = texts.map((t) => t.text);
+        const deduplicationResults = await deduplicator.deduplicateBatch(textList, componentName, false // Standalone extraction: generate fresh keys without reusing from cache
+        );
+        const stats = deduplicator.getStats();
+        if (keySpinner) {
+            keySpinner.succeed(`🧠 Generated ${chalk.green(uniqueTexts.size)} keys`);
+        }
+        else {
+            this.log(`✅ Generated ${chalk.green(uniqueTexts.size)} keys (deterministic)`);
+        }
+        this.log(`💾 Cache hits: ${chalk.green(stats.cacheHits)}`);
+        this.log(`🤖 AI requests used: ${chalk.green(stats.aiRequestsUsed)}`);
+        // Map results back to texts
+        const mappedTexts = texts.map((t) => {
+            const result = deduplicationResults.get(t.text);
+            return {
+                isCached: result.isCached,
+                key: result.key,
+                node: t.node,
+                placeholders: t.placeholders,
+                tempKey: t.tempKey,
+                text: t.text,
+            };
+        });
+        // Step 2: Generate translations (batch, separate from key generation)
         const spinner = ora(`💬 Generating translations with ${provider}...`).start();
         try {
+            // Build prompt with the keys we already generated
             const prompt = buildPrompt({
                 componentName,
                 locales,
-                texts: texts.map((t) => ({ tempKey: t.tempKey, text: t.text })),
+                texts: mappedTexts.map((t) => ({ tempKey: t.tempKey, text: t.text })),
             });
             const raw = await generateTranslations(prompt, provider);
             if (!raw)
@@ -70,32 +118,37 @@ export default class Extract extends Command {
             const namespace = componentName;
             const jsonNamespace = json[namespace] || {};
             const i18nJson = {};
-            for (const [, translations] of Object.entries(jsonNamespace)) {
-                const { key } = translations;
-                i18nJson[key] = {};
+            // Use our generated keys, not the AI's keys
+            for (const mapped of mappedTexts) {
+                const translations = jsonNamespace[mapped.tempKey];
+                if (!translations) {
+                    throw new Error(`No translations found for tempKey: ${mapped.tempKey}`);
+                }
+                i18nJson[mapped.key] = {};
                 for (const locale of locales) {
-                    i18nJson[key][locale] = translations[locale];
+                    i18nJson[mapped.key][locale] = translations[locale];
                 }
+                // Update cache
+                cache.set({
+                    componentName,
+                    key: mapped.key,
+                    locales: i18nJson[mapped.key],
+                    text: mapped.text,
+                });
             }
             writeLocaleFiles(componentName, { [componentName]: i18nJson }, locales);
             spinner.succeed(`✅ Translations generated with ${provider}`);
-            // const aiGeneratedKeys = Object.keys(json[namespace] || {});
-            const mapped = texts.map(t => {
-                const translations = jsonNamespace[t.tempKey];
-                if (!translations) {
-                    throw new Error(`No translations found for tempKey: ${t.tempKey}`);
-                }
-                return {
-                    key: translations.key,
-                    node: t.node,
-                    placeholders: t.placeholders,
-                    tempKey: t.tempKey
-                };
-            });
-            this.log(`🔗 Mapped ${chalk.green(mapped.length)} texts to keys`);
+            this.log(`🔗 Mapped ${chalk.green(mappedTexts.length)} texts to keys`);
             insertUseTranslations(sourceFile, componentName);
-            replaceTempKeysWithT(mapped);
+            replaceTempKeysWithT(mappedTexts.map((m) => ({
+                key: m.key,
+                node: m.node,
+                placeholders: m.placeholders,
+                tempKey: m.tempKey,
+            })));
             saveSourceFile(sourceFile);
+            // Save cache
+            cache.save();
             this.log(chalk.green("✨ Component rewritten with t() calls"));
             this.log(chalk.green(`🌍 JSON files generated using ${provider}`));
         }

package/dist/commands/translate.js

@@ -123,10 +123,13 @@ export default class Translate extends Command {
                     continue;
                 }
                 totalExtracted += texts.length;
-                // Deduplicate and assign keys (now async)
+                // Deduplicate and assign keys using batch processing
+                const textList = texts.map((t) => t.text);
                 // eslint-disable-next-line no-await-in-loop
-                const mappedTexts = await Promise.all(texts.map(async (t) => {
-                    const result = await deduplicator.deduplicate(t.text, componentName, config.behavior.detectDuplicates);
+                const deduplicationResults = await deduplicator.deduplicateBatch(textList, componentName, config.behavior.detectDuplicates);
+                // Map results back to texts
+                const mappedTexts = texts.map((t) => {
+                    const result = deduplicationResults.get(t.text);
                     if (result.isReused)
                         totalReused++;
                     if (result.isCached)
@@ -139,7 +142,7 @@ export default class Translate extends Command {
                         tempKey: t.tempKey,
                         text: t.text,
                     };
-                }));
+                });
                 // Build translations JSON
                 const i18nJson = {};
                 for (const mapped of mappedTexts) {
@@ -241,14 +244,18 @@ export default class Translate extends Command {
         if (!flags["dry-run"]) {
             cache.save();
         }
+        // Get statistics from deduplicator
+        const stats = deduplicator.getStats();
         // Summary
         this.log("");
         this.log(chalk.green("🎉 Translation complete!"));
         this.log(chalk.cyan("📊 Summary:"));
         this.log(`  Files processed: ${chalk.bold(filesToProcess.length)}`);
         this.log(`  Strings extracted: ${chalk.bold(totalExtracted)}`);
+        this.log(`  Unique strings: ${chalk.bold(stats.uniqueStrings)}`);
         this.log(`  Keys reused: ${chalk.bold(totalReused)}`);
         this.log(`  Cached translations: ${chalk.bold(totalCached)}`);
+        this.log(`  AI requests (keys): ${chalk.bold(stats.aiRequestsUsed)}`);
         this.log("");
     }
 }

package/dist/core/ai/generate-english-key.js

@@ -1,7 +1,77 @@
 import { generateTranslations } from "./client.js";
+/**
+ * Generate English camelCase keys for multiple texts in a single AI request
+ * This is the preferred method for efficiency
+ *
+ * @param texts - Array of texts to generate keys for
+ * @param provider - AI provider to use
+ * @returns Map of text to generated key, or empty map on failure
+ */
+export async function generateEnglishKeysBatch(texts, provider = "huggingface") {
+    if (texts.length === 0) {
+        return new Map();
+    }
+    const prompt = `
+You are an i18n key generator. Generate concise, meaningful camelCase keys in English for the following texts.
+
+RULES:
+- Output ONLY a JSON object mapping each text to its camelCase key
+- Keys should be 2-4 words maximum
+- Keys must be in English
+- Keys should describe the content/purpose
+- Do NOT include explanations, comments, or any other text
+- Output format: { "original text": "camelCaseKey" }
+
+TEXTS:
+${texts.map((t, i) => `${i + 1}. ${JSON.stringify(t)}`).join("\n")}
+
+EXAMPLES:
+{ "Bienvenido de nuevo": "welcomeBack", "Por favor inicia sesión": "pleaseSignIn" }
+
+OUTPUT (JSON only):`.trim();
+    try {
+        const response = await generateTranslations(prompt, provider);
+        if (!response)
+            return new Map();
+        // Extract JSON from response (handle cases where AI adds markdown or extra text)
+        let jsonText = response.trim();
+        // Remove markdown code blocks if present
+        if (jsonText.startsWith("```")) {
+            jsonText = jsonText.replace(/^```(?:json)?\s*\n/, "").replace(/\n```\s*$/, "");
+        }
+        const parsed = JSON.parse(jsonText);
+        const result = new Map();
+        // Validate and add each key
+        for (const [text, key] of Object.entries(parsed)) {
+            // Validate it's a valid camelCase identifier
+            if (typeof key === "string" && /^[a-z][a-zA-Z0-9]*$/.test(key)) {
+                result.set(text, key);
+            }
+        }
+        return result;
+    }
+    catch (error) {
+        console.error("❌ Error generating English keys with AI:", error);
+        return new Map();
+    }
+}
 /**
  * Generate an English camelCase key from text using AI
  * This ensures keys are always in English regardless of source language
+ *
+ * @deprecated Use generateEnglishKeysBatch for better efficiency. This method
+ * will be removed in a future version. To migrate, collect all texts and call
+ * generateEnglishKeysBatch once:
+ *
+ * @example
+ * // Before:
+ * const key1 = await generateEnglishKey(text1);
+ * const key2 = await generateEnglishKey(text2);
+ *
+ * // After:
+ * const keyMap = await generateEnglishKeysBatch([text1, text2]);
+ * const key1 = keyMap.get(text1);
+ * const key2 = keyMap.get(text2);
  */
 export async function generateEnglishKey(text, provider = "huggingface") {
     const prompt = `
@@ -14,7 +84,7 @@ RULES:
 - Key should describe the content/purpose
 - Do NOT include quotes, explanations, or any other text
 
-TEXT: "${text.replaceAll('"', '\\"')}"
+TEXT: ${JSON.stringify(text)}
 
 EXAMPLES:
 TEXT: "Bienvenido de nuevo" -> welcomeBack

package/dist/core/deduplication/deduplicator.js

@@ -1,4 +1,4 @@
-import { generateEnglishKey } from "../ai/generate-english-key.js";
+import { generateEnglishKeysBatch } from "../ai/generate-english-key.js";
 import { generateKey, generateUniqueKey } from "../i18n/generate-key.js";
 /**
  * Deduplicate strings and assign deterministic keys
@@ -8,45 +8,96 @@ export class Deduplicator {
     provider;
     useAiForKeys;
     usedKeys = new Set();
+    seenTexts = new Set(); // Track all texts seen across all batches
+    stats = {
+        aiRequestsUsed: 0,
+        cacheHits: 0,
+        totalStrings: 0,
+        uniqueStrings: 0,
+    };
     constructor(cache, useAiForKeys = true, provider = "huggingface") {
         this.cache = cache;
         this.provider = provider;
         this.useAiForKeys = useAiForKeys;
     }
     /**
-     * Process a text and return a deterministic key
-     * Reuses existing keys if the text was seen before
+     * Process multiple texts in batch and return deterministic keys
+     * This is the preferred method for efficiency with AI
      */
-    async deduplicate(text, componentName, detectDuplicates) {
-        // Check cache first
-        const cached = this.cache.get(text);
-        if (cached && detectDuplicates) {
-            // Reuse existing key from cache
-            this.usedKeys.add(cached.key);
-            return {
-                isCached: true,
-                isReused: true,
-                key: cached.key,
-            };
+    async deduplicateBatch(texts, componentName, detectDuplicates) {
+        this.stats.totalStrings += texts.length;
+        const results = new Map();
+        const uncachedTexts = [];
+        // First pass: check cache and collect uncached texts
+        for (const text of texts) {
+            const cached = this.cache.get(text);
+            if (cached && detectDuplicates) {
+                // Reuse existing key from cache
+                this.usedKeys.add(cached.key);
+                results.set(text, {
+                    isCached: true,
+                    isReused: true,
+                    key: cached.key,
+                });
+                this.stats.cacheHits++;
+            }
+            else {
+                uncachedTexts.push(text);
+            }
         }
-        // Generate a new deterministic key
-        let baseKey;
-        if (this.useAiForKeys && !cached) {
-            // Try to generate English key using AI
-            const aiKey = await generateEnglishKey(text, this.provider);
-            baseKey = aiKey ?? generateKey(text);
+        // Track unique strings across all batches
+        for (const text of texts) {
+            if (!this.seenTexts.has(text)) {
+                this.seenTexts.add(text);
+                this.stats.uniqueStrings++;
+            }
         }
-        else {
-            // Use deterministic key generation (fallback or when AI is disabled)
-            baseKey = generateKey(text);
+        // Second pass: generate keys for uncached texts
+        if (uncachedTexts.length > 0) {
+            let aiKeyMap = new Map();
+            if (this.useAiForKeys) {
+                // Batch AI request for all uncached texts
+                aiKeyMap = await generateEnglishKeysBatch(uncachedTexts, this.provider);
+                if (aiKeyMap.size > 0) {
+                    this.stats.aiRequestsUsed++;
+                }
+            }
+            // Generate keys for remaining uncached texts
+            for (const text of uncachedTexts) {
+                const aiKey = aiKeyMap.get(text);
+                const baseKey = aiKey ?? generateKey(text);
+                const uniqueKey = generateUniqueKey(baseKey, this.usedKeys);
+                this.usedKeys.add(uniqueKey);
+                results.set(text, {
+                    isCached: false,
+                    isReused: false,
+                    key: uniqueKey,
+                });
+            }
         }
-        const uniqueKey = generateUniqueKey(baseKey, this.usedKeys);
-        this.usedKeys.add(uniqueKey);
-        return {
-            isCached: false,
-            isReused: false,
-            key: uniqueKey,
-        };
+        return results;
+    }
+    /**
+     * Process a single text and return a deterministic key
+     * Reuses existing keys if the text was seen before
+     *
+     * @deprecated Use deduplicateBatch for better efficiency. This method will be
+     * removed in v1.0. To migrate, collect texts and call deduplicateBatch once:
+     *
+     * @example
+     * // Before:
+     * const result1 = await deduplicator.deduplicate(text1, comp, true);
+     * const result2 = await deduplicator.deduplicate(text2, comp, true);
+     *
+     * // After:
+     * const results = await deduplicator.deduplicateBatch([text1, text2], comp, true);
+     * const result1 = results.get(text1);
+     * const result2 = results.get(text2);
+     */
+    async deduplicate(text, componentName, detectDuplicates) {
+        // Use batch method with single item for consistency
+        const results = await this.deduplicateBatch([text], componentName, detectDuplicates);
+        return results.get(text);
     }
     /**
      * Get all used keys
@@ -54,4 +105,22 @@ export class Deduplicator {
     getUsedKeys() {
         return new Set(this.usedKeys);
     }
+    /**
+     * Get deduplication statistics
+     */
+    getStats() {
+        return { ...this.stats };
+    }
+    /**
+     * Reset statistics
+     */
+    resetStats() {
+        this.stats = {
+            aiRequestsUsed: 0,
+            cacheHits: 0,
+            totalStrings: 0,
+            uniqueStrings: 0,
+        };
+        this.seenTexts.clear();
+    }
 }

package/oclif.manifest.json

@@ -27,6 +27,12 @@
           "hasDynamicHelp": false,
           "multiple": false,
           "type": "option"
+        },
+        "use-ai-keys": {
+          "description": "Use AI to generate human-readable keys (default: true)",
+          "name": "use-ai-keys",
+          "allowNo": true,
+          "type": "boolean"
         }
       },
       "hasDynamicHelp": false,
@@ -301,5 +307,5 @@
       ]
     }
   },
-  "version": "0.5.0"
+  "version": "0.5.1"
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "i18nizer",
   "description": "CLI to extract texts from JSX/TSX and generate i18n JSON with AI translations",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "author": "Yoannis Sanchez Soto",
   "bin": "./bin/run.js",
   "bugs": "https://github.com/yossTheDev/i18nizer/issues",