gptrans 1.8.6 → 1.8.8

package/demo/REFERENCES_USAGE.md

@@ -0,0 +1,101 @@
+# GPTrans - Referencias Múltiples y Idioma Base Alternativo
+
+## Nuevas Funcionalidades
+
+El método `preload()` ahora soporta dos parámetros opcionales que mejoran la precisión de las traducciones:
+
+### 1. `references` - Referencias Múltiples
+
+Permite incluir traducciones existentes en otros idiomas como contexto adicional para el modelo de IA.
+
+```javascript
+await gptrans.preload({ 
+  references: ['en', 'pt']  // Usar inglés y portugués como referencia
+});
+```
+
+**Caso de uso**: Cuando tienes traducciones en varios idiomas y quieres que la nueva traducción sea consistente con las existentes.
+
+### 2. `baseLanguage` - Idioma Base Alternativo
+
+Permite usar un idioma diferente al original como base para la traducción.
+
+```javascript
+await gptrans.preload({ 
+  baseLanguage: 'en'  // Traducir desde inglés en vez del idioma original
+});
+```
+
+**Caso de uso**: Cuando el texto original tiene características específicas del idioma (como he/she en inglés) que pueden omitirse en la traducción final.
+
+## Ejemplos de Uso
+
+### Ejemplo 1: Traducción Básica (sin cambios)
+
+```javascript
+const gptrans = new GPTrans({ from: 'en', target: 'es' });
+await gptrans.preload();  // Comportamiento original
+```
+
+### Ejemplo 2: Con Referencias
+
+```javascript
+// Original en español, traducir a francés usando inglés como referencia
+const gptrans = new GPTrans({ from: 'es', target: 'fr' });
+await gptrans.preload({ 
+  references: ['en']  // El modelo verá la traducción en inglés como contexto
+});
+```
+
+### Ejemplo 3: Con Idioma Base Alternativo
+
+```javascript
+// Original en español, pero traducir DE inglés A portugués
+const gptrans = new GPTrans({ from: 'es', target: 'pt' });
+await gptrans.preload({ 
+  baseLanguage: 'en'  // Usa la traducción en inglés como base
+});
+```
+
+### Ejemplo 4: Combinado
+
+```javascript
+// Original en español, traducir de inglés a alemán, mostrando español y portugués como referencia
+const gptrans = new GPTrans({ from: 'es', target: 'de' });
+await gptrans.preload({ 
+  baseLanguage: 'en',     // Traduce desde inglés
+  references: ['es', 'pt'] // Muestra español y portugués como contexto adicional
+});
+```
+
+## Caso de Uso Real: Evitar Problemas de Género
+
+### Problema
+En inglés: "The student is very good" (neutral)
+En español: "El estudiante es muy bueno" / "La estudiante es muy buena" (con género)
+En portugués: Se puede omitir el género en algunos contextos
+
+### Solución
+```javascript
+// Si el original está en español pero queremos la traducción al portugués
+// basada en la versión inglesa (que es neutral):
+
+const ptTranslator = new GPTrans({ from: 'es', target: 'pt' });
+await ptTranslator.preload({ 
+  baseLanguage: 'en',  // Usa la versión inglesa como base
+  references: ['es']    // Muestra el español original como referencia
+});
+```
+
+## Compatibilidad
+
+✅ **Totalmente retrocompatible**: Si no especificas ningún parámetro, `preload()` funciona exactamente igual que antes.
+
+```javascript
+// Esto sigue funcionando sin cambios:
+await gptrans.preload();
+```
+
+## Archivo de Prueba
+
+Ejecuta `demo/case_references.js` para ver ejemplos completos de todas las funcionalidades nuevas.

package/demo/case_references.js

@@ -0,0 +1,139 @@
+import GPTrans from '../index.js';
+import dotenv from 'dotenv';
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+
+// Load .env from demo folder
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+dotenv.config({ path: join(__dirname, '.env') });
+
+console.log('🚀 Testing GPTrans with Reference Translations\n');
+console.log('='.repeat(70));
+
+async function testReferences() {
+    console.log('\n📋 Test 1: Basic usage without references (baseline)\n');
+    
+    // First, create translations in English and Portuguese
+    const enTranslator = new GPTrans({ 
+        from: 'es', 
+        target: 'en',
+        model: 'sonnet45',
+        name: 'ref_test',
+        debug: true
+    });
+    
+    const ptTranslator = new GPTrans({ 
+        from: 'es', 
+        target: 'pt',
+        model: 'sonnet45',
+        name: 'ref_test',
+        debug: true
+    });
+    
+    // Sample Spanish texts with gendered language
+    const spanishTexts = [
+        'El estudiante es muy bueno',
+        'La estudiante es muy buena',
+        'Tienes que ir al doctor',
+        'Está muy cansada'
+    ];
+    
+    console.log('📝 Creating English translations from Spanish...');
+    spanishTexts.forEach(text => {
+        console.log(`   EN: ${enTranslator.t(text)}`);
+    });
+    
+    await new Promise(resolve => setTimeout(resolve, 5000));
+    
+    console.log('\n📝 Creating Portuguese translations from Spanish...');
+    spanishTexts.forEach(text => {
+        console.log(`   PT: ${ptTranslator.t(text)}`);
+    });
+    
+    await new Promise(resolve => setTimeout(resolve, 5000));
+    
+    console.log('\n' + '='.repeat(70));
+    console.log('\n📋 Test 2: Translation with English as reference\n');
+    
+    // Now translate to French using English as reference
+    const frTranslator = new GPTrans({ 
+        from: 'es', 
+        target: 'fr',
+        model: 'sonnet45',
+        name: 'ref_test',
+        debug: true
+    });
+    
+    console.log('🔄 Preloading French translations with English as reference...');
+    await frTranslator.preload({ 
+        references: ['en']
+    });
+    
+    console.log('\n📝 French translations (with EN reference):');
+    spanishTexts.forEach(text => {
+        console.log(`   ES: ${text}`);
+        console.log(`   FR: ${frTranslator.t(text)}\n`);
+    });
+    
+    console.log('='.repeat(70));
+    console.log('\n📋 Test 3: Translation using alternate base language\n');
+    
+    // Translate from English to Italian (using English as base instead of Spanish)
+    const itTranslator = new GPTrans({ 
+        from: 'es', 
+        target: 'it',
+        model: 'sonnet45',
+        name: 'ref_test',
+        debug: true
+    });
+    
+    console.log('🔄 Preloading Italian translations with English as base language...');
+    await itTranslator.preload({ 
+        baseLanguage: 'en',
+        references: ['es', 'pt']  // Show original Spanish and Portuguese as reference
+    });
+    
+    console.log('\n📝 Italian translations (from EN, with ES+PT references):');
+    spanishTexts.forEach(text => {
+        console.log(`   ES (original): ${text}`);
+        console.log(`   IT (from EN):  ${itTranslator.t(text)}\n`);
+    });
+    
+    console.log('='.repeat(70));
+    console.log('\n📋 Test 4: Multiple references\n');
+    
+    // Translate to German with multiple references
+    const deTranslator = new GPTrans({ 
+        from: 'es', 
+        target: 'de',
+        model: 'sonnet45',
+        name: 'ref_test',
+        debug: true
+    });
+    
+    console.log('🔄 Preloading German translations with multiple references...');
+    await deTranslator.preload({ 
+        references: ['en', 'pt', 'fr']
+    });
+    
+    console.log('\n📝 German translations (with EN+PT+FR references):');
+    spanishTexts.forEach(text => {
+        console.log(`   ES: ${text}`);
+        console.log(`   DE: ${deTranslator.t(text)}\n`);
+    });
+    
+    console.log('='.repeat(70));
+    console.log('\n✅ All tests completed!\n');
+    console.log('💡 Key features demonstrated:');
+    console.log('   1. Baseline translations without references');
+    console.log('   2. Using one language as reference for better context');
+    console.log('   3. Using alternate base language (translate from intermediate language)');
+    console.log('   4. Using multiple references for maximum accuracy\n');
+}
+
+// Run tests
+testReferences().catch(error => {
+    console.error('\n❌ Error during tests:', error.message);
+    console.error(error.stack);
+});

package/index.js

@@ -11,14 +11,14 @@ class GPTrans {
     static #mmixInstances = new Map();
     static #translationLocks = new Map();
 
-    static mmix(models = 'sonnet45') {
+    static mmix(models = 'sonnet45', { debug = false } = {}) {
         const key = Array.isArray(models) ? models.join(',') : models;
 
         if (!this.#mmixInstances.has(key)) {
             const mmix = new ModelMix({
                 config: {
                     max_history: 1,
-                    debug: false,
+                    debug,
                     bottleneck: {
                         minTime: 15000,
                         maxConcurrent: 1
@@ -49,16 +49,16 @@ class GPTrans {
         if (!this.#translationLocks.has(modelKey)) {
             this.#translationLocks.set(modelKey, Promise.resolve());
         }
-        
+
         const previousLock = this.#translationLocks.get(modelKey);
         let releaseLock;
-        
+
         const currentLock = new Promise(resolve => {
             releaseLock = resolve;
         });
-        
+
         this.#translationLocks.set(modelKey, previousLock.then(() => currentLock));
-        
+
         await previousLock;
         return releaseLock;
     }
@@ -67,7 +67,7 @@ class GPTrans {
         return isLanguageAvailable(langCode);
     }
 
-    constructor({ from = 'en-US', target = 'es', model = 'sonnet45', batchThreshold = 1500, debounceTimeout = 500, promptFile = null, name = '', context = '', freeze = false } = {}) {
+    constructor({ from = 'en-US', target = 'es', model = 'sonnet45', batchThreshold = 1500, debounceTimeout = 500, promptFile = null, name = '', context = '', freeze = false, debug = false } = {}) {
 
         target = this.normalizeBCP47(target);
         from = this.normalizeBCP47(from);
@@ -79,9 +79,11 @@ class GPTrans {
         }
 
         const path = new URL('../../db', import.meta.url).pathname;
-        name = name ? '_' + name : '';
-        this.dbTarget = new DeepBase({ name: 'gptrans' + name + '_' + target, path });
-        this.dbFrom = new DeepBase({ name: 'gptrans' + name + '_from_' + from, path });
+        const namePrefix = name ? '_' + name : '';
+        this.dbPath = path;
+        this.instanceName = name;
+        this.dbTarget = new DeepBase({ name: 'gptrans' + namePrefix + '_' + target, path });
+        this.dbFrom = new DeepBase({ name: 'gptrans' + namePrefix + '_from_' + from, path });
 
         try {
             this.replaceTarget = isoAssoc(target, 'TARGET_');
@@ -95,6 +97,8 @@ class GPTrans {
         this.pendingTranslations = new Map(); // [key, text]
         this.pendingCharCount = 0; // Add character count tracker
         this.debounceTimer = null;
+        this.isProcessingBatch = false; // Track if a batch is currently being processed
+        this.modelMixOptions = { debug };
         this.modelKey = model;
         this.promptFile = promptFile ?? new URL('./prompt/translate.md', import.meta.url).pathname;
         this.context = context;
@@ -196,13 +200,24 @@ class GPTrans {
 
         this.modelConfig.options.max_tokens = this.pendingCharCount + 1000;
         const minTime = Math.floor((60000 / (8000 / this.pendingCharCount)) * 1.4);
-        GPTrans.mmix(this.modelKey).limiter.updateSettings({ minTime });
+        GPTrans.mmix(this.modelKey, this.modelMixOptions).limiter.updateSettings({ minTime });
 
         this.pendingCharCount = 0;
 
+        // Load references for each text in the batch if preloadReferences is set
+        const batchReferences = {};
+        if (this.preloadReferences && this.preloadReferences.length > 0) {
+            batch.forEach(([key]) => {
+                const refs = this._loadReferenceTranslations(key, this.preloadReferences);
+                if (Object.keys(refs).length > 0) {
+                    batchReferences[key] = refs;
+                }
+            });
+        }
+
         const textsToTranslate = batch.map(([_, text]) => text).join(`\n${this.divider}\n`);
         try {
-            const translations = await this._translate(textsToTranslate);
+            const translations = await this._translate(textsToTranslate, batch, batchReferences, this.preloadBaseLanguage);
             const translatedTexts = translations.split(`\n${this.divider}\n`);
 
             const contextHash = this._hash(context);
@@ -223,20 +238,57 @@ class GPTrans {
         }
     }
 
-    async _translate(text) {
+    async _translate(text, batch = [], batchReferences = {}, baseLanguage = null) {
         // Acquire lock to ensure atomic model configuration and translation
         const releaseLock = await GPTrans.#acquireTranslationLock(this.modelKey);
-        
+
         try {
-            const model = GPTrans.mmix(this.modelKey);
+            const model = GPTrans.mmix(this.modelKey, this.modelMixOptions);
 
             model.setSystem("You are an expert translator specialized in literary translation between FROM_LANG and TARGET_DENONYM TARGET_LANG.");
 
             model.addTextFromFile(this.promptFile);
 
-            model.replace({ INPUT: text, CONTEXT: this.context });
+            // Format references if available
+            let referencesText = '';
+            if (Object.keys(batchReferences).length > 0 && batch.length > 0) {
+                const textsArray = text.split(`\n${this.divider}\n`);
+
+                referencesText = textsArray.map((txt, index) => {
+                    const key = batch[index] ? batch[index][0] : null;
+                    if (key && batchReferences[key]) {
+                        const refs = batchReferences[key];
+                        const refLines = Object.entries(refs).map(([lang, translation]) => {
+                            try {
+                                const langInfo = isoAssoc(lang);
+                                return `${langInfo.DENONYM} ${langInfo.LANG} (${lang}): ${translation}`;
+                            } catch (e) {
+                                return `${lang}: ${translation}`;
+                            }
+                        }).join(`\n${this.divider}\n`);
+                        return refLines;
+                    }
+                    return '';
+                }).filter(r => r).join(`\n\n`);
+            }
+
+            // Determine which FROM_ values to use
+            let fromReplace = this.replaceFrom;
+            if (baseLanguage) {
+                try {
+                    fromReplace = isoAssoc(baseLanguage, 'FROM_');
+                } catch (e) {
+                    console.warn(`Invalid baseLanguage: ${baseLanguage}, using default`);
+                }
+            }
+
+            model.replace({
+                INPUT: text,
+                CONTEXT: this.context,
+                REFERENCES: referencesText || 'None'
+            });
             model.replace(this.replaceTarget);
-            model.replace(this.replaceFrom);
+            model.replace(fromReplace);
 
             const response = await model.message();
 
@@ -269,29 +321,98 @@ class GPTrans {
         return stringHash(input).toString(36);
     }
 
-    async preload() {
+    _loadReferenceTranslations(key, referenceLangs = []) {
+        const references = {};
+        const contextHash = this._hash(this.context);
+
+        for (const lang of referenceLangs) {
+            const namePrefix = this.instanceName ? '_' + this.instanceName : '';
+            const dbRef = new DeepBase({
+                name: `gptrans${namePrefix}_${lang}`,
+                path: this.dbPath
+            });
+
+            const translation = dbRef.get(contextHash, key);
+            if (translation) {
+                references[lang] = translation;
+            }
+        }
+
+        return references;
+    }
+
+    async preload({ references = [], baseLanguage = null } = {}) {
 
         if (!this.context && this.replaceFrom.FROM_ISO === this.replaceTarget.TARGET_ISO) {
             return this;
         }
 
+        // Store preload options for use in translation
+        this.preloadReferences = references;
+        this.preloadBaseLanguage = baseLanguage;
+
+        // Track which keys need translation
+        const keysNeedingTranslation = [];
+        
         for (const [context, pairs] of this.dbFrom.entries()) {
+            // Skip the _context metadata
+            if (context === '_context') continue;
+            
             this.setContext(context);
+            const contextHash = this._hash(context);
+            
             for (const [key, text] of Object.entries(pairs)) {
+                // Check if translation already exists
+                if (!this.dbTarget.get(contextHash, key)) {
+                    keysNeedingTranslation.push({ context, contextHash, key });
+                }
                 this.get(key, text);
             }
         }
 
+        // If nothing needs translation, return immediately
+        if (keysNeedingTranslation.length === 0) {
+            this.preloadReferences = [];
+            this.preloadBaseLanguage = null;
+            return this;
+        }
+
         // Wait for any pending translations to complete
-        await new Promise(resolve => {
+        const maxWaitTime = 120000; // 120 seconds timeout
+        const startTime = Date.now();
+        
+        await new Promise((resolve, reject) => {
             const checkInterval = setInterval(() => {
-                if (this.dbFrom.keys().length === this.dbTarget.keys().length) {
+                // Check if there are still pending translations or batch being processed
+                const hasPending = this.pendingTranslations.size > 0 || this.isProcessingBatch;
+                
+                // Check if all needed translations are now complete
+                let allTranslated = true;
+                for (const { contextHash, key } of keysNeedingTranslation) {
+                    if (!this.dbTarget.get(contextHash, key)) {
+                        allTranslated = false;
+                        break;
+                    }
+                }
+                
+                if (allTranslated && !hasPending) {
                     clearInterval(checkInterval);
                     resolve();
                 }
+                
+                // Timeout check
+                if (Date.now() - startTime > maxWaitTime) {
+                    clearInterval(checkInterval);
+                    console.warn(`Preload timeout: ${keysNeedingTranslation.length} translations pending`);
+                    resolve(); // Resolve instead of reject to allow partial completion
+                }
             }, 100);
         });
 
+        // Clear preload options after completion
+        this.preloadReferences = [];
+        this.preloadBaseLanguage = null;
+
         return this;
     }
 
@@ -304,7 +425,7 @@ class GPTrans {
         // Iterate through dbTarget and remove keys that don't exist in dbFrom
         for (const [contextHash, pairs] of this.dbTarget.entries()) {
             for (const key of Object.keys(pairs)) {
-                
+
                 const context = this.dbFrom.get('_context', contextHash);
                 if (!this.dbFrom.get(context, key)) {
                     console.log(contextHash, key);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gptrans",
   "type": "module",
-  "version": "1.8.6",
+  "version": "1.8.8",
   "description": "🚆 GPTrans - The smarter AI-powered way to translate.",
   "keywords": [
     "translate",
@@ -37,7 +37,7 @@
     "dotenv": "^16.4.7",
     "form-data": "^4.0.4",
     "genmix": "^1.0.4",
-    "modelmix": "^3.9.0",
+    "modelmix": "^4.2.8",
     "string-hash": "^1.1.3"
   }
 }

package/prompt/translate.md

@@ -6,6 +6,12 @@ Translation from FROM_ISO to TARGET_ISO (TARGET_DENONYM TARGET_LANG) with cultur
 INPUT
 ```
 
+## Reference Translations (for context)
+These are existing translations in other languages that may help you provide a more accurate translation. Use them as reference but do not simply copy them:
+```
+REFERENCES
+```
+
 # Return Format
    - Provide the final translation within a code block using ```.
    - Do not include alternative translations, only provide the best translation.