gptrans 2.0.6 → 2.0.9

package/index.js

@@ -131,17 +131,45 @@ class GPTrans {
     t(text, params = {}) {
         const key = this._textToKey(text);
         const translation = this.get(key, text) || text;
+        return this._applyParams(translation, params);
+    }
 
-        return Object.entries(params).reduce(
-            (text, [key, value]) => text.replace(`{${key}}`, value),
-            translation
-        );
+    async tAsync(text, params = {}) {
+        const key = this._textToKey(text);
+        const { contextHash, translation, needsTranslation } = this._resolveTranslationState(key, text);
+
+        if (!needsTranslation) {
+            return this._applyParams(translation || text, params);
+        }
+
+        // If this key was enqueued via t(), avoid duplicate work in a later batch.
+        this._dequeuePendingTranslation(key);
+
+        const translatedText = await this._translate(text, [[key, text]], {}, this.preloadBaseLanguage);
+        const immediateTranslation = translatedText.trim();
+        this.dbTarget.set(contextHash, key, immediateTranslation);
+
+        return this._applyParams(immediateTranslation, params);
     }
 
     get(key, text) {
+        const { translation, needsTranslation } = this._resolveTranslationState(key, text);
+        if (needsTranslation) {
+            this._enqueuePendingTranslation(key, text);
+        }
+        return translation;
+    }
 
+    _applyParams(text, params = {}) {
+        return Object.entries(params).reduce(
+            (translated, [paramKey, value]) => translated.replace(`{${paramKey}}`, value),
+            text
+        );
+    }
+
+    _resolveTranslationState(key, text) {
         if (!text || !text.trim()) {
-            return text;
+            return { contextHash: this._hash(this.context), translation: text, needsTranslation: false };
         }
 
         const contextHash = this._hash(this.context);
@@ -152,41 +180,56 @@ class GPTrans {
             this.dbFrom.set('_context', contextHash, this.context);
         }
 
-        if (!translation) {
+        if (translation) {
+            return { contextHash, translation, needsTranslation: false };
+        }
 
-            // Skip translation if context is empty and languages are the same
-            if (!this.context && this.replaceFrom.FROM_ISO === this.replaceTarget.TARGET_ISO) {
-                return text;
-            }
+        if (!this.context && this.replaceFrom.FROM_ISO === this.replaceTarget.TARGET_ISO) {
+            return { contextHash, translation: text, needsTranslation: false };
+        }
 
-            if (this.freeze) {
-                console.log(`Freeze mode: [${key}] ${text}`);
-                return text;
-            }
+        if (this.freeze) {
+            console.log(`Freeze mode: [${key}] ${text}`);
+            return { contextHash, translation: text, needsTranslation: false };
+        }
 
-            this.pendingTranslations.set(key, text);
-            this.pendingCharCount += text.length; // Update character count
+        return { contextHash, translation: null, needsTranslation: true };
+    }
 
-            // Clear existing timer
-            if (this.debounceTimer) {
-                clearTimeout(this.debounceTimer);
-            }
+    _enqueuePendingTranslation(key, text) {
+        const existingText = this.pendingTranslations.get(key);
+        if (existingText) {
+            this.pendingCharCount -= existingText.length;
+        }
 
-            // Set new timer - capture context at scheduling time
-            const capturedContext = this.context;
-            this.debounceTimer = setTimeout(() => {
-                if (this.pendingTranslations.size > 0) {
-                    this._processBatch(capturedContext);
-                }
-            }, this.debounceTimeout);
+        this.pendingTranslations.set(key, text);
+        this.pendingCharCount += text.length;
 
-            // Process if we hit the character count threshold
-            if (this.pendingCharCount >= this.batchThreshold) {
-                clearTimeout(this.debounceTimer);
-                this._processBatch(this.context);
+        if (this.debounceTimer) {
+            clearTimeout(this.debounceTimer);
+        }
+
+        const capturedContext = this.context;
+        this.debounceTimer = setTimeout(() => {
+            if (this.pendingTranslations.size > 0) {
+                this._processBatch(capturedContext);
             }
+        }, this.debounceTimeout);
+
+        if (this.pendingCharCount >= this.batchThreshold) {
+            clearTimeout(this.debounceTimer);
+            this._processBatch(this.context);
         }
-        return translation;
+    }
+
+    _dequeuePendingTranslation(key) {
+        const queuedText = this.pendingTranslations.get(key);
+        if (!queuedText) {
+            return;
+        }
+
+        this.pendingTranslations.delete(key);
+        this.pendingCharCount = Math.max(0, this.pendingCharCount - queuedText.length);
     }
 
     async _processBatch(context) {
@@ -220,15 +263,9 @@ class GPTrans {
             // Try different split strategies to be more robust
             let translatedTexts = translations.split(`\n${this.divider}\n`);
 
-            // If split doesn't match batch size, try alternative separators
+            // If split doesn't match batch size, try without newlines around divider
             if (translatedTexts.length !== batch.length) {
-                // Try without newlines around divider
                 translatedTexts = translations.split(this.divider);
-
-                // If still doesn't match, try with just newline
-                if (translatedTexts.length !== batch.length) {
-                    translatedTexts = translations.split(/\n{2,}/); // Split by multiple newlines
-                }
             }
 
             const contextHash = this._hash(context);
@@ -251,14 +288,23 @@ class GPTrans {
                 return;
             }
 
+            // Detect suspicious duplicates
+            const trimmed = translatedTexts.map(t => t.trim());
+            const uniqueCount = new Set(trimmed).size;
+            if (batch.length > 2 && uniqueCount === 1) {
+                console.error(`❌ All ${batch.length} translations are identical ("${trimmed[0].slice(0, 60)}..."), discarding batch`);
+                console.error(`   Batch keys: ${batch.map(([key]) => key).join(', ')}`);
+                return;
+            }
+
             batch.forEach(([key], index) => {
-                if (!translatedTexts[index] || !translatedTexts[index].trim()) {
+                if (!trimmed[index]) {
                     console.error(`❌ No translation found for ${key} at index ${index}`);
                     console.error(`   Original text: ${batch[index][1]}`);
                     return;
                 }
 
-                this.dbTarget.set(contextHash, key, translatedTexts[index].trim());
+                this.dbTarget.set(contextHash, key, trimmed[index]);
             });
 
         } catch (e) {
@@ -551,10 +597,6 @@ class GPTrans {
 
             if (refinedTexts.length !== entries.length) {
                 refinedTexts = refined.split(this.divider);
-
-                if (refinedTexts.length !== entries.length) {
-                    refinedTexts = refined.split(/\n{2,}/);
-                }
             }
 
             if (refinedTexts.length !== entries.length) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gptrans",
   "type": "module",
-  "version": "2.0.6",
+  "version": "2.0.9",
   "description": "🚆 GPTrans - The smarter AI-powered way to translate.",
   "keywords": [
     "translate",
@@ -26,7 +26,7 @@
   },
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "node --test test/**/*.test.js"
   },
   "author": "Martin Clasen",
   "license": "MIT",

package/prompt/translate.md

@@ -10,6 +10,7 @@ Translation from {FROM_ISO} to {TARGET_ISO} ({TARGET_DENONYM} {TARGET_LANG}) wit
 {REFERENCES}
 
 # Return Format
+   - The input may contain multiple texts separated by `------`. Translate each one independently and return them in the same order, separated by `------`. The number of segments in your output must exactly match the number of segments in the input.
    - Provide the final translation within a code block using ```.
    - Do not include alternative translations, only provide the best translation.
 

package/test/gptrans.batch.test.js

@@ -0,0 +1,237 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import GPTrans from '../index.js';
+
+function createMemoryDb() {
+    const store = new Map();
+    return {
+        get(context, key) {
+            return store.get(context)?.get(key);
+        },
+        set(context, key, value) {
+            if (!store.has(context)) {
+                store.set(context, new Map());
+            }
+            store.get(context).set(key, value);
+        },
+        entries() {
+            return Array.from(store.entries()).map(([context, pairs]) => [
+                context,
+                Object.fromEntries(pairs.entries())
+            ]);
+        },
+        async del(context, key) {
+            const pairs = store.get(context);
+            if (!pairs) return;
+            pairs.delete(key);
+            if (pairs.size === 0) store.delete(context);
+        }
+    };
+}
+
+function createTestInstance() {
+    const gp = new GPTrans({
+        from: 'es',
+        target: 'en-US',
+        debounceTimeout: 10_000,
+        batchThreshold: 50_000,
+        name: `unit_${Date.now()}_${Math.random().toString(36).slice(2)}`
+    });
+
+    gp.dbFrom = createMemoryDb();
+    gp.dbTarget = createMemoryDb();
+
+    return gp;
+}
+
+const stubMmix = () => ({ limiter: { updateSettings() {} } });
+
+// --- Split logic ---
+
+test('_processBatch splits correctly with \\n------\\n separators', async (t) => {
+    const gp = createTestInstance();
+    const originalMmix = GPTrans.mmix;
+    GPTrans.mmix = stubMmix;
+
+    gp._translate = async () => 'Hello\n------\nGoodbye\n------\nThank you';
+
+    gp.pendingTranslations.set('k1', 'Hola');
+    gp.pendingTranslations.set('k2', 'Adiós');
+    gp.pendingTranslations.set('k3', 'Gracias');
+    gp.pendingCharCount = 15;
+
+    await gp._processBatch('');
+
+    const h = gp._hash('');
+    assert.equal(gp.dbTarget.get(h, 'k1'), 'Hello');
+    assert.equal(gp.dbTarget.get(h, 'k2'), 'Goodbye');
+    assert.equal(gp.dbTarget.get(h, 'k3'), 'Thank you');
+
+    GPTrans.mmix = originalMmix;
+});
+
+test('_processBatch falls back to split by divider without newlines', async (t) => {
+    const gp = createTestInstance();
+    const originalMmix = GPTrans.mmix;
+    GPTrans.mmix = stubMmix;
+
+    // Model returned divider without surrounding newlines
+    gp._translate = async () => 'Hello------Goodbye------Thank you';
+
+    gp.pendingTranslations.set('k1', 'Hola');
+    gp.pendingTranslations.set('k2', 'Adiós');
+    gp.pendingTranslations.set('k3', 'Gracias');
+    gp.pendingCharCount = 15;
+
+    await gp._processBatch('');
+
+    const h = gp._hash('');
+    assert.equal(gp.dbTarget.get(h, 'k1'), 'Hello');
+    assert.equal(gp.dbTarget.get(h, 'k2'), 'Goodbye');
+    assert.equal(gp.dbTarget.get(h, 'k3'), 'Thank you');
+
+    GPTrans.mmix = originalMmix;
+});
+
+// --- Double-newline no longer used as fallback ---
+
+test('_processBatch does NOT split by double newlines', async (t) => {
+    const gp = createTestInstance();
+    const originalMmix = GPTrans.mmix;
+    GPTrans.mmix = stubMmix;
+
+    // Response has no divider but has double newlines — old code would split by \n\n
+    // and accidentally "match" 3 chunks. New code should treat this as a mismatch.
+    gp._translate = async () => 'Hello world\n\nThis is a paragraph\n\nAnother paragraph';
+
+    gp.pendingTranslations.set('k1', 'Hola');
+    gp.pendingTranslations.set('k2', 'Adiós');
+    gp.pendingTranslations.set('k3', 'Gracias');
+    gp.pendingCharCount = 15;
+
+    const errors = [];
+    const origError = console.error;
+    console.error = (...args) => errors.push(args.join(' '));
+
+    await gp._processBatch('');
+
+    console.error = origError;
+
+    // Should hit mismatch path, not silently save wrong translations
+    assert.ok(errors.some(e => e.includes('Translation count mismatch')));
+
+    GPTrans.mmix = originalMmix;
+});
+
+// --- Duplicate detection ---
+
+test('_processBatch discards batch when all translations are identical (3+)', async (t) => {
+    const gp = createTestInstance();
+    const originalMmix = GPTrans.mmix;
+    GPTrans.mmix = stubMmix;
+
+    gp._translate = async () =>
+        'Everyone understands.\n------\nEveryone understands.\n------\nEveryone understands.\n------\nEveryone understands.';
+
+    gp.pendingTranslations.set('k1', 'La fusión será completa');
+    gp.pendingTranslations.set('k2', 'Y entonces entenderás');
+    gp.pendingTranslations.set('k3', 'Qué libertad tiene el que');
+    gp.pendingTranslations.set('k4', 'Todos lo entienden');
+    gp.pendingCharCount = 80;
+
+    const errors = [];
+    const origError = console.error;
+    console.error = (...args) => errors.push(args.join(' '));
+
+    await gp._processBatch('');
+
+    console.error = origError;
+
+    const h = gp._hash('');
+    // Nothing should be saved
+    assert.equal(gp.dbTarget.get(h, 'k1'), undefined);
+    assert.equal(gp.dbTarget.get(h, 'k2'), undefined);
+    assert.equal(gp.dbTarget.get(h, 'k3'), undefined);
+    assert.equal(gp.dbTarget.get(h, 'k4'), undefined);
+
+    assert.ok(errors.some(e => e.includes('translations are identical')));
+
+    GPTrans.mmix = originalMmix;
+});
+
+test('_processBatch allows batch of 2 with identical translations (edge case)', async (t) => {
+    const gp = createTestInstance();
+    const originalMmix = GPTrans.mmix;
+    GPTrans.mmix = stubMmix;
+
+    // Two items could legitimately have the same translation (e.g. synonyms)
+    gp._translate = async () => 'Yes\n------\nYes';
+
+    gp.pendingTranslations.set('k1', 'Sí');
+    gp.pendingTranslations.set('k2', 'Claro que sí');
+    gp.pendingCharCount = 15;
+
+    await gp._processBatch('');
+
+    const h = gp._hash('');
+    assert.equal(gp.dbTarget.get(h, 'k1'), 'Yes');
+    assert.equal(gp.dbTarget.get(h, 'k2'), 'Yes');
+
+    GPTrans.mmix = originalMmix;
+});
+
+test('_processBatch saves partial duplicates normally', async (t) => {
+    const gp = createTestInstance();
+    const originalMmix = GPTrans.mmix;
+    GPTrans.mmix = stubMmix;
+
+    // Some translations are the same but not all — this is valid
+    gp._translate = async () => 'Hello\n------\nHello\n------\nGoodbye';
+
+    gp.pendingTranslations.set('k1', 'Hola');
+    gp.pendingTranslations.set('k2', 'Buenos días');
+    gp.pendingTranslations.set('k3', 'Adiós');
+    gp.pendingCharCount = 20;
+
+    await gp._processBatch('');
+
+    const h = gp._hash('');
+    assert.equal(gp.dbTarget.get(h, 'k1'), 'Hello');
+    assert.equal(gp.dbTarget.get(h, 'k2'), 'Hello');
+    assert.equal(gp.dbTarget.get(h, 'k3'), 'Goodbye');
+
+    GPTrans.mmix = originalMmix;
+});
+
+// --- Count mismatch saves partial ---
+
+test('_processBatch saves partial results on count mismatch', async (t) => {
+    const gp = createTestInstance();
+    const originalMmix = GPTrans.mmix;
+    GPTrans.mmix = stubMmix;
+
+    // Model returned only 2 translations for a batch of 3
+    gp._translate = async () => 'Hello\n------\nGoodbye';
+
+    gp.pendingTranslations.set('k1', 'Hola');
+    gp.pendingTranslations.set('k2', 'Adiós');
+    gp.pendingTranslations.set('k3', 'Gracias');
+    gp.pendingCharCount = 15;
+
+    const errors = [];
+    const origError = console.error;
+    console.error = (...args) => errors.push(args.join(' '));
+
+    await gp._processBatch('');
+
+    console.error = origError;
+
+    const h = gp._hash('');
+    assert.equal(gp.dbTarget.get(h, 'k1'), 'Hello');
+    assert.equal(gp.dbTarget.get(h, 'k2'), 'Goodbye');
+    assert.equal(gp.dbTarget.get(h, 'k3'), undefined); // Not saved
+
+    assert.ok(errors.some(e => e.includes('Translation count mismatch')));
+
+    GPTrans.mmix = originalMmix;
+});

package/test/gptrans.tAsync.test.js

@@ -0,0 +1,143 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import GPTrans from '../index.js';
+
+function createMemoryDb() {
+    const store = new Map();
+    return {
+        get(context, key) {
+            return store.get(context)?.get(key);
+        },
+        set(context, key, value) {
+            if (!store.has(context)) {
+                store.set(context, new Map());
+            }
+            store.get(context).set(key, value);
+        },
+        entries() {
+            return Array.from(store.entries()).map(([context, pairs]) => [
+                context,
+                Object.fromEntries(pairs.entries())
+            ]);
+        },
+        async del(context, key) {
+            const pairs = store.get(context);
+            if (!pairs) {
+                return;
+            }
+            pairs.delete(key);
+            if (pairs.size === 0) {
+                store.delete(context);
+            }
+        }
+    };
+}
+
+function createTestInstance() {
+    const gp = new GPTrans({
+        from: 'en-US',
+        target: 'es',
+        debounceTimeout: 10_000,
+        batchThreshold: 5000,
+        name: `unit_${Date.now()}_${Math.random().toString(36).slice(2)}`
+    });
+
+    gp.dbFrom = createMemoryDb();
+    gp.dbTarget = createMemoryDb();
+
+    return gp;
+}
+
+test('tAsync translates immediately and caches result', async () => {
+    const gp = createTestInstance();
+    let translateCalls = 0;
+
+    gp._translate = async (text) => {
+        translateCalls += 1;
+        assert.equal(text, 'Hello {name}');
+        return 'Hola {name}';
+    };
+
+    const first = await gp.tAsync('Hello {name}', { name: 'Martin' });
+    const second = await gp.tAsync('Hello {name}', { name: 'Martin' });
+
+    assert.equal(first, 'Hola Martin');
+    assert.equal(second, 'Hola Martin');
+    assert.equal(translateCalls, 1);
+
+    if (gp.debounceTimer) {
+        clearTimeout(gp.debounceTimer);
+    }
+});
+
+test('tAsync removes queued batch item to avoid duplicate work', async () => {
+    const gp = createTestInstance();
+    let translateCalls = 0;
+
+    gp._translate = async () => {
+        translateCalls += 1;
+        return 'Comprar';
+    };
+
+    const key = gp._textToKey('Buy');
+
+    // Enqueue via sync API (batch/background path).
+    const syncValue = gp.t('Buy');
+    assert.equal(syncValue, 'Buy');
+    assert.equal(gp.pendingTranslations.get(key), 'Buy');
+    assert.equal(gp.pendingCharCount, 'Buy'.length);
+
+    // Force immediate translation for same key.
+    const asyncValue = await gp.tAsync('Buy');
+    assert.equal(asyncValue, 'Comprar');
+
+    assert.equal(translateCalls, 1);
+    assert.equal(gp.pendingTranslations.has(key), false);
+    assert.equal(gp.pendingCharCount, 0);
+
+    if (gp.debounceTimer) {
+        clearTimeout(gp.debounceTimer);
+    }
+});
+
+test('preload translates missing keys from dbFrom into dbTarget', async () => {
+    const gp = createTestInstance();
+    gp.debounceTimeout = 1;
+    let translateCalls = 0;
+
+    const context = 'checkout';
+    const sourceText = 'Buy now';
+    const key = gp._textToKey(sourceText);
+    const contextHash = gp._hash(context);
+
+    gp.dbFrom.set(context, key, sourceText);
+    gp.dbFrom.set('_context', contextHash, context);
+
+    gp._translate = async (text) => {
+        translateCalls += 1;
+        assert.equal(text, sourceText);
+        return 'Comprar ahora';
+    };
+
+    const originalMmix = GPTrans.mmix;
+    GPTrans.mmix = () => ({
+        limiter: {
+            updateSettings() {
+            }
+        }
+    });
+
+    try {
+        await gp.preload();
+
+        assert.equal(translateCalls, 1);
+        assert.equal(gp.dbTarget.get(contextHash, key), 'Comprar ahora');
+        assert.deepEqual(gp.preloadReferences, []);
+        assert.equal(gp.preloadBaseLanguage, null);
+    } finally {
+        GPTrans.mmix = originalMmix;
+        if (gp.debounceTimer) {
+            clearTimeout(gp.debounceTimer);
+        }
+    }
+});