gptrans 1.4.4 → 1.4.8

package/README.md

@@ -9,10 +9,10 @@ Whether you're building a multilingual website, a mobile app, or a localization
 ## ✨ Features
 
 - **AI-Powered Translations:** Harness advanced models like OpenAI's GPT and Anthropic's Sonnet for high-quality translations
-- **Smart Batching & Debouncing:** Automatically groups translation requests to optimize API usage
-- **Caching with DeepBase:** Quickly retrieves cached translations to boost performance
+- **Smart Batching & Debouncing:** Translations are processed in batches, not only for efficiency but also to provide better context. By sending multiple related texts together, the AI model can better understand the overall context and produce more accurate and consistent translations across related terms and phrases.
+- **Caching with JSON:** Quickly retrieves cached translations to boost performance
 - **Parameter Substitution:** Dynamically replace placeholders in your translations
-- **Flexible Configuration:** Customize source and target locales, model keys, and batching settings to fit your needs
+- **Smart Context Handling:** Add contextual information to improve translation accuracy. Perfect for gender-aware translations, domain-specific content, or any scenario where additional context helps produce better results. The context is automatically cleared after each translation to prevent unintended effects.
 
 ## 📦 Installation
 
@@ -65,20 +65,31 @@ When creating a new instance of GPTrans, you can customize:
 
 | Option | Description | Default |
 |--------|-------------|---------|
-| `from` | Source language locale | `es-AR` |
-| `target` | Target language locale | `en-US` |
+| `from` | Source language locale (BCP 47) | `es-AR` |
+| `target` | Target language locale (BCP 47) | `en-US` |
 | `model` | Translation model key | `claude-3-7-sonnet` |
 | `batchThreshold` | Maximum number of characters to accumulate before triggering batch processing | `1000` |
 | `debounceTimeout` | Time in milliseconds to wait before processing translations | `500` |
 
+### BCP 47 Language Tags
+
+GPTrans uses BCP 47 language tags for language specification. BCP 47 is the standard for language tags that combines language, script, and region codes. Here are some common examples:
+
+- `en-US` - English (United States)
+- `es-AR` - Spanish (Argentina)
+- `pt-BR` - Portuguese (Brazil)
+
+For simplified or universal language codes, you can omit the region specification:
+- `es` - Spanish (Universal)
+
 ## 🔍 How It Works
 
 1. **First-Time Translation Behavior:** On the first request, Gptrans will return the original text while processing the translation in the background. This ensures your application remains responsive without waiting for API calls.
-2. **Translation Caching:** Once processed, translations are stored in `db/gptrans_<iso>.json`. Subsequent requests for the same text will be served instantly from the cache.
-3. **Smart Batch Processing:** Translations are processed in batches, providing better context for more accurate results.
+2. **Translation Caching:** Once processed, translations are stored in `db/gptrans_<tag>.json`. Subsequent requests for the same text will be served instantly from the cache.
+3. **Smart Batch Processing:** Automatically groups translation requests to optimize API usage and provide better context.
 4. **Dynamic Model Integration:** Easily plug in multiple AI translation providers with the ModelMix library.
 5. **Customizable Prompts:** Load and modify translation prompts (see the `prompt/translate.md` file) to fine-tune the translation output.
-6. **Manual Corrections:** A JSON file stores key-translation pairs, allowing you to override specific translations and make manual corrections when needed. Simply edit the `db/gptrans_<iso>.json` file:
+6. **Manual Corrections:** A JSON file stores key-translation pairs, allowing you to override specific translations and make manual corrections when needed. Simply edit the `db/gptrans_<tag>.json` file:
 
 ```json
 {

package/db/gptrans_es.json

@@ -0,0 +1,5 @@
+{
+    "1sfvxng": {
+        "eres_muy_bueno_26czme": "Eres muy buena"
+    }
+}

package/db/gptrans_from_es.json

@@ -1,3 +1,9 @@
 {
-    "cargan_1998owo": "Cargando..."
+    "": {
+        "eres_muy_bueno_26czme": "Eres muy bueno",
+        "tienes_fuego_1i2o3ok": "Tienes fuego?"
+    },
+    "El mensaje es para una mujer": {
+        "eres_muy_bueno_26czme": "Eres muy bueno"
+    }
 }

package/demo/case_4.js

@@ -0,0 +1,17 @@
+import GPTrans from '../index.js';
+
+
+// Case 2: Translate from Spanish Spain to Spanish Argentina
+const model = new GPTrans({
+    from: 'es',
+    target: 'es',
+});
+
+await model.preload();
+
+
+console.log(model.setContext().t('Eres muy bueno'));
+console.log(model.setContext('El mensaje es para una mujer').t('Eres muy bueno'));
+
+
+console.log(model.setContext().t('Tienes fuego?'));

package/index.js

@@ -35,8 +35,8 @@ class GPTrans {
         this.dbFrom = new DeepBase({ name: 'gptrans_from_' + from });
 
         try {
-            this.replace_target = isoAssoc(target, 'TARGET_');
-            this.replace_from = isoAssoc(from, 'FROM_');
+            this.replaceTarget = isoAssoc(target, 'TARGET_');
+            this.replaceFrom = isoAssoc(from, 'FROM_');
         } catch (e) {
             throw new Error(`Invalid target: ${target}`);
         }
@@ -57,7 +57,7 @@ class GPTrans {
                     maxConcurrent: 2,
                 }
             },
-            options: { 
+            options: {
                 max_tokens: batchThreshold,
                 temperature: 0
             }
@@ -68,7 +68,7 @@ class GPTrans {
     setContext(context = '') {
         if (this.context !== context && this.pendingTranslations.size > 0) {
             clearTimeout(this.debounceTimer);
-            this._processBatch();
+            this._processBatch(this.context);
         }
         this.context = context;
         return this;
@@ -85,15 +85,22 @@ class GPTrans {
     }
 
     get(key, text) {
-        const translation = this.dbTarget.get(key);
+        const contextHash = this._hash(this.context);
+        const translation = this.dbTarget.get(contextHash, key);
+
         if (!translation) {
-            this.pendingTranslations.set(key, text);
-            this.pendingCharCount += text.length; // Update character count
+            if (!this.dbFrom.get(this.context, key)) {
+                this.dbFrom.set(this.context, key, text);
+            }
 
-            if (!this.dbFrom.get(key)) {
-                this.dbFrom.set(key, text);
+            // Skip translation if context is empty and languages are the same
+            if (!this.context && this.replaceFrom.FROM_ISO === this.replaceTarget.TARGET_ISO) {
+                return text;
             }
 
+            this.pendingTranslations.set(key, text);
+            this.pendingCharCount += text.length; // Update character count
+
             // Clear existing timer
             if (this.debounceTimer) {
                 clearTimeout(this.debounceTimer);
@@ -102,20 +109,20 @@ class GPTrans {
             // Set new timer
             this.debounceTimer = setTimeout(() => {
                 if (this.pendingTranslations.size > 0) {
-                    this._processBatch();
+                    this._processBatch(this.context);
                 }
             }, this.debounceTimeout);
 
             // Process if we hit the character count threshold
             if (this.pendingCharCount >= this.batchThreshold) {
                 clearTimeout(this.debounceTimer);
-                this._processBatch();
+                this._processBatch(this.context);
             }
         }
         return translation;
     }
 
-    async _processBatch() {
+    async _processBatch(context) {
         this.processing = true;
 
         const batch = Array.from(this.pendingTranslations.entries());
@@ -130,8 +137,9 @@ class GPTrans {
             const translations = await this._translate(textsToTranslate);
             const translatedTexts = translations.split('\n---\n');
 
+            const contextHash = this._hash(context);
             batch.forEach(([key], index) => {
-                this.dbTarget.set(key, translatedTexts[index].trim());
+                this.dbTarget.set(contextHash, key, translatedTexts[index].trim());
             });
 
         } catch (e) {
@@ -142,6 +150,7 @@ class GPTrans {
     }
 
     async _translate(text) {
+
         const model = GPTrans.mmix.create(this.modelKey, this.modelConfig);
 
         model.setSystem("You are an expert translator specialized in literary translation between FROM_LANG and TARGET_DENONYM TARGET_LANG.");
@@ -149,8 +158,8 @@ class GPTrans {
         model.addTextFromFile(this.promptFile);
 
         model.replace({ INPUT: text, CONTEXT: this.context });
-        model.replace(this.replace_target);
-        model.replace(this.replace_from);
+        model.replace(this.replaceTarget);
+        model.replace(this.replaceFrom);
 
         const response = await model.message();
 
@@ -171,11 +180,15 @@ class GPTrans {
 
         let key = words.map((x) => x.slice(0, maxlen)).join("_");
         key += key ? '_' : '';
-        key += stringHash(text + this.context).toString(36);
+        key += this._hash(text);
         return key;
     }
 
-    async preload({ target = this.replace_target.TARGET_ISO, model = this.modelKey, from = this.replace_from.FROM_ISO, batchThreshold = this.batchThreshold, debounceTimeout = this.debounceTimeout } = {}) {
+    _hash(input) {
+        return stringHash(input).toString(36);
+    }
+
+    async preload({ target = this.replaceTarget.TARGET_ISO, model = this.modelKey, from = this.replaceFrom.FROM_ISO, batchThreshold = this.batchThreshold, debounceTimeout = this.debounceTimeout } = {}) {
 
         // Create new GPTrans instance for the target language
         const translator = new GPTrans({
@@ -187,8 +200,11 @@ class GPTrans {
         });
 
         // Process all entries in batches
-        for (const [key, text] of this.dbFrom.entries()) {
-            translator.get(key, text);
+        for (const [context, pairs] of this.dbFrom.entries()) {
+            translator.setContext(context);
+            for (const [key, text] of Object.entries(pairs)) {
+                translator.get(key, text);
+            }
         }
 
         // Wait for any pending translations to complete

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gptrans",
   "type": "module",
-  "version": "1.4.4",
+  "version": "1.4.8",
   "description": "🚆 GPTrans - The smarter AI-powered way to translate.",
   "keywords": [
     "translate",
@@ -32,7 +32,7 @@
   },
   "homepage": "https://github.com/clasen/GPTrans#readme",
   "dependencies": {
-    "deepbase": "^1.4.6",
+    "deepbase": "^1.4.8",
     "dotenv": "^16.4.7",
     "modelmix": "^2.9.0",
     "string-hash": "^1.1.3"

package/db/gptrans_ar.json

@@ -1,3 +0,0 @@
-{
-    "cargan_1998owo": "جارٍ التحميل..."
-}

package/db/gptrans_es-AR.json

@@ -1,13 +0,0 @@
-{
-    "eres_muy_bueno_26czme": "Sos muy bueno",
-    "eres_muy_bueno_k3ml5b": "Sos muy buena",
-    "hello_name_1987p1n": "¡Hola, {name}!",
-    "topup_uzdh5y": "Recargar",
-    "transf_176pc1a": "Transferir",
-    "deposi_wg2ec5": "Depositar",
-    "balanc_1rv8if7": "Saldo",
-    "transa_1wtqm5d": "Transacción",
-    "accoun_x1y0v8": "Cuenta",
-    "card_yis1ox": "Tarjeta",
-    "tienes_fuego_1i2o3ok": "¿Tenés fuego?"
-}

package/db/gptrans_es-ES.json

@@ -1,3 +0,0 @@
-{
-    "tenes_fuego_1fs98im": "¿Tienes fuego?"
-}

package/db/gptrans_from_en-US.json

@@ -1,11 +0,0 @@
-{
-    "hello_name_1987p1n": "Hello, {name}!",
-    "topup_uzdh5y": "Top-up",
-    "transf_176pc1a": "Transfer",
-    "deposi_wg2ec5": "Deposit",
-    "balanc_1rv8if7": "Balance",
-    "transa_1wtqm5d": "Transaction",
-    "accoun_x1y0v8": "Account",
-    "card_yis1ox": "Card",
-    "loadin_21q3nx": "Loading..."
-}

package/db/gptrans_from_es-AR.json

@@ -1,3 +0,0 @@
-{
-    "tenes_fuego_1fs98im": "¿Tenés fuego?"
-}

package/db/gptrans_from_es-ES.json

@@ -1,5 +0,0 @@
-{
-    "eres_muy_bueno_26czme": "Eres muy bueno",
-    "eres_muy_bueno_k3ml5b": "Eres muy bueno",
-    "tienes_fuego_1i2o3ok": "Tienes fuego?"
-}

package/db/gptrans_it.json

@@ -1,11 +0,0 @@
-{
-    "hello_name_1987p1n": "Ciao, {name}!",
-    "topup_uzdh5y": "Ricarica",
-    "transf_176pc1a": "Trasferimento",
-    "deposi_wg2ec5": "Deposito",
-    "balanc_1rv8if7": "Saldo",
-    "transa_1wtqm5d": "Transazione",
-    "accoun_x1y0v8": "Account",
-    "card_yis1ox": "Carta",
-    "loadin_21q3nx": "Caricamento in corso..."
-}