gptrans 2.1.10 → 2.2.4

package/index.js

@@ -1,20 +1,26 @@
 import DeepBase from 'deepbase';
+import { JsonDriver } from 'deepbase-json';
 import stringHash from 'string-hash';
 import { ModelMix } from 'modelmix';
 import { isoAssoc, isLanguageAvailable } from './isoAssoc.js';
 import { GeminiGenerator } from 'genmix';
 import fs from 'fs';
-import path from 'path';
+import pathModule from 'path';
+
+const TRANSLATE_PROMPT_FILE = pathModule.join(import.meta.dirname, 'prompt', 'translate.md');
+const REFINE_PROMPT_FILE = pathModule.join(import.meta.dirname, 'prompt', 'refine.md');
+const DEFAULT_MODEL_CHAIN = Object.freeze(['sonnet5@50', 'gpt56luna@100']);
 
 class GPTrans {
     static #mmixInstances = new Map();
     static #translationLocks = new Map();
 
-    static mmix(models = 'sonnet45', { debug = 0 } = {}) {
-        const key = Array.isArray(models) ? models.join(',') : models;
+    static mmix(models = DEFAULT_MODEL_CHAIN, { debug = 0 } = {}) {
+        const modelChain = Array.isArray(models) ? models : [models];
+        const key = modelChain.join(',');
 
         if (!this.#mmixInstances.has(key)) {
-            let instance = ModelMix.new({
+            const instance = ModelMix.new({
                 config: {
                     debug,
                     bottleneck: {
@@ -22,18 +28,7 @@ class GPTrans {
                         maxConcurrent: 1
                     }
                 }
-            });
-            const modelArray = Array.isArray(models) ? models : [models];
-
-            for (const model of modelArray) {
-                if (typeof instance[model] !== 'function') {
-                    throw new Error(
-                        `Model "${model}" is not available. Please check the model name. ` +
-                        `Available models include: gpt51, gpt52, sonnet46, sonnet45, opus46, haiku45, etc.`
-                    );
-                }
-                instance = instance[model]();
-            }
+            }).chain(...modelChain);
 
             this.#mmixInstances.set(key, instance);
         }
@@ -62,23 +57,21 @@ class GPTrans {
         return isLanguageAvailable(langCode);
     }
 
-    constructor({ from = 'en-US', target = 'es', model = 'sonnet46', batchThreshold = 1500, debounceTimeout = 500, promptFile = null, name = '', context = '', instruction = '', freeze = false, debug = false } = {}) {
+    constructor({ from = 'en-US', target = 'es', model = DEFAULT_MODEL_CHAIN, batchThreshold = 1500, debounceTimeout = 500, promptFile = null, name = '', context = '', instruction = '', freeze = false, debug = false, path: dbPath } = {}) {
+
+        if (typeof dbPath !== 'string' || !pathModule.isAbsolute(dbPath)) {
+            throw new TypeError('GPTrans requires an absolute "path" option.');
+        }
 
         target = this.normalizeBCP47(target);
         from = this.normalizeBCP47(from);
 
-        try {
-            process.loadEnvFile();
-        } catch {
-            /* optional .env missing or unreadable */
-        }
-
-        const path = new URL('../../db', import.meta.url).pathname;
         const namePrefix = name ? '_' + name : '';
-        this.dbPath = path;
+        this.dbPath = dbPath;
         this.instanceName = name;
-        this.dbTarget = new DeepBase({ name: 'gptrans' + namePrefix + '_' + target, path });
-        this.dbFrom = new DeepBase({ name: 'gptrans' + namePrefix + '_from_' + from, path });
+        this.dbTarget = this._createDatabase('gptrans' + namePrefix + '_' + target);
+        this.dbFrom = this._createDatabase('gptrans' + namePrefix + '_from_' + from);
+        this.pendingSourceWrites = Promise.resolve();
 
         try {
             this.replaceTarget = isoAssoc(target, 'TARGET_');
@@ -95,7 +88,7 @@ class GPTrans {
         this.isProcessingBatch = false; // Track if a batch is currently being processed
         this.modelMixOptions = { debug };
         this.modelKey = model;
-        this.promptFile = promptFile ?? new URL('./prompt/translate.md', import.meta.url).pathname;
+        this.promptFile = promptFile ? pathModule.resolve(promptFile) : TRANSLATE_PROMPT_FILE;
         this.context = context;
         this.instruction = instruction;
         this.freeze = freeze;
@@ -116,6 +109,17 @@ class GPTrans {
         return iso.toLowerCase();
     }
 
+    _createDatabase(name) {
+        return new DeepBase(new JsonDriver({ name, path: this.dbPath }));
+    }
+
+    _saveSourceText(context, contextHash, key, text) {
+        this.pendingSourceWrites = this.pendingSourceWrites.then(async () => {
+            await this.dbFrom.set(context, key, text);
+            await this.dbFrom.set('_context', contextHash, context);
+        });
+    }
+
     setContext(context = '') {
         if (this.context !== context && this.pendingTranslations.size > 0) {
             clearTimeout(this.debounceTimer);
@@ -143,9 +147,10 @@ class GPTrans {
         // If this key was enqueued via t(), avoid duplicate work in a later batch.
         this._dequeuePendingTranslation(key);
 
+        await this.pendingSourceWrites;
         const translatedText = await this._translate(text, [[key, text]], {}, this.preloadBaseLanguage);
         const immediateTranslation = translatedText.trim();
-        this.dbTarget.set(contextHash, key, immediateTranslation);
+        await this.dbTarget.set(contextHash, key, immediateTranslation);
 
         return this._applyParams(immediateTranslation, params);
     }
@@ -171,11 +176,10 @@ class GPTrans {
         }
 
         const contextHash = this._hash(this.context);
-        const translation = this.dbTarget.get(contextHash, key);
+        const translation = this.dbTarget.getSync(contextHash, key);
 
-        if (!this.freeze && !this.dbFrom.get(this.context, key)) {
-            this.dbFrom.set(this.context, key, text);
-            this.dbFrom.set('_context', contextHash, this.context);
+        if (!this.freeze && !this.dbFrom.getSync(this.context, key)) {
+            this._saveSourceText(this.context, contextHash, key, text);
         }
 
         if (translation) {
@@ -233,16 +237,17 @@ class GPTrans {
     async _processBatch(context) {
 
         const batch = Array.from(this.pendingTranslations.entries());
+        const batchCharCount = this.pendingCharCount;
 
         // Clear pending translations and character count before awaiting translation
         this.pendingTranslations.clear();
+        this.pendingCharCount = 0;
+        await this.pendingSourceWrites;
 
-        this.modelConfig.options.max_tokens = this.pendingCharCount + 1000;
-        const minTime = Math.floor((60000 / (8000 / this.pendingCharCount)) * 1.4);
+        this.modelConfig.options.max_tokens = batchCharCount + 1000;
+        const minTime = Math.floor((60000 / (8000 / batchCharCount)) * 1.4);
         GPTrans.mmix(this.modelKey, this.modelMixOptions).limiter.updateSettings({ minTime });
 
-        this.pendingCharCount = 0;
-
         // Load references for each text in the batch if preloadReferences is set
         const batchReferences = {};
         if (this.preloadReferences && this.preloadReferences.length > 0) {
@@ -280,7 +285,7 @@ class GPTrans {
                 const minLength = Math.min(translatedTexts.length, batch.length);
                 for (let i = 0; i < minLength; i++) {
                     if (translatedTexts[i] && translatedTexts[i].trim()) {
-                        this.dbTarget.set(contextHash, batch[i][0], translatedTexts[i].trim());
+                        await this.dbTarget.set(contextHash, batch[i][0], translatedTexts[i].trim());
                     }
                 }
                 return;
@@ -295,15 +300,15 @@ class GPTrans {
                 return;
             }
 
-            batch.forEach(([key], index) => {
+            for (const [index, [key]] of batch.entries()) {
                 if (!trimmed[index]) {
                     console.error(`❌ No translation found for ${key} at index ${index}`);
                     console.error(`   Original text: ${batch[index][1]}`);
-                    return;
+                    continue;
                 }
 
-                this.dbTarget.set(contextHash, key, trimmed[index]);
-            });
+                await this.dbTarget.set(contextHash, key, trimmed[index]);
+            }
 
         } catch (e) {
             console.error('❌ Error in _processBatch:', e.message);
@@ -318,7 +323,7 @@ class GPTrans {
         try {
             const model = GPTrans.mmix(this.modelKey, this.modelMixOptions);
 
-            model.setSystem("You are an expert translator specialized in literary translation between {FROM_LANG} and {TARGET_DENONYM} {TARGET_LANG}.");
+            model.setSystem("You are an expert translator specialized in literary translation between <%- FROM_LANG %> and <%- TARGET_DENONYM %> <%- TARGET_LANG %>.");
 
             // Build references section (includes header when references exist, empty otherwise)
             let referencesText = '';
@@ -362,21 +367,21 @@ class GPTrans {
                 }
             }
 
-            // Use ModelMix templates: addTextFromFile + replace + block
+            // Use ModelMix templates: addTextFromFile + assign + block
             model.addTextFromFile(this.promptFile);
-            model.replace({
-                '{INPUT}': text,
-                '{CONTEXT}': this.context,
-                '{INSTRUCTION}': this.instruction,
-                '{REFERENCES}': referencesText,
-                '{TARGET_ISO}': this.replaceTarget.TARGET_ISO,
-                '{TARGET_LANG}': this.replaceTarget.TARGET_LANG,
-                '{TARGET_COUNTRY}': this.replaceTarget.TARGET_COUNTRY,
-                '{TARGET_DENONYM}': this.replaceTarget.TARGET_DENONYM,
-                '{FROM_ISO}': fromReplace.FROM_ISO,
-                '{FROM_LANG}': fromReplace.FROM_LANG,
-                '{FROM_COUNTRY}': fromReplace.FROM_COUNTRY,
-                '{FROM_DENONYM}': fromReplace.FROM_DENONYM,
+            model.assign({
+                INPUT: text,
+                CONTEXT: this.context,
+                INSTRUCTION: this.instruction,
+                REFERENCES: referencesText,
+                TARGET_ISO: this.replaceTarget.TARGET_ISO,
+                TARGET_LANG: this.replaceTarget.TARGET_LANG,
+                TARGET_COUNTRY: this.replaceTarget.TARGET_COUNTRY,
+                TARGET_DENONYM: this.replaceTarget.TARGET_DENONYM,
+                FROM_ISO: fromReplace.FROM_ISO,
+                FROM_LANG: fromReplace.FROM_LANG,
+                FROM_COUNTRY: fromReplace.FROM_COUNTRY,
+                FROM_DENONYM: fromReplace.FROM_DENONYM,
             });
 
             return await model.block({ addSystemExtra: false });
@@ -411,12 +416,9 @@ class GPTrans {
 
         for (const lang of referenceLangs) {
             const namePrefix = this.instanceName ? '_' + this.instanceName : '';
-            const dbRef = new DeepBase({
-                name: `gptrans${namePrefix}_${lang}`,
-                path: this.dbPath
-            });
+            const dbRef = this._createDatabase(`gptrans${namePrefix}_${lang}`);
 
-            const translation = dbRef.get(contextHash, key);
+            const translation = dbRef.getSync(contextHash, key);
             if (translation) {
                 references[lang] = translation;
             }
@@ -427,6 +429,8 @@ class GPTrans {
 
     async preload({ references = [], baseLanguage = null } = {}) {
 
+        await this.pendingSourceWrites;
+
         if (!this.context && this.replaceFrom.FROM_ISO === this.replaceTarget.TARGET_ISO) {
             return this;
         }
@@ -454,7 +458,7 @@ class GPTrans {
         // Track which keys need translation
         const keysNeedingTranslation = [];
 
-        for (const [context, pairs] of this.dbFrom.entries()) {
+        for (const [context, pairs] of Object.entries(this.dbFrom.getSync() ?? {})) {
             // Skip the _context metadata
             if (context === '_context') continue;
 
@@ -463,7 +467,7 @@ class GPTrans {
 
             for (const [key, text] of Object.entries(pairs)) {
                 // Check if translation already exists
-                if (!this.dbTarget.get(contextHash, key)) {
+                if (!this.dbTarget.getSync(contextHash, key)) {
                     keysNeedingTranslation.push({ context, contextHash, key });
                     // Only call get() if translation doesn't exist
                     this.get(key, text);
@@ -488,7 +492,7 @@ class GPTrans {
                 // Check if all needed translations are now complete
                 let allTranslated = true;
                 for (const { contextHash, key } of keysNeedingTranslation) {
-                    if (!this.dbTarget.get(contextHash, key)) {
+                    if (!this.dbTarget.getSync(contextHash, key)) {
                         allTranslated = false;
                         break;
                     }
@@ -514,12 +518,14 @@ class GPTrans {
     }
 
     async purge() {
+        await this.pendingSourceWrites;
+
         // Iterate through dbTarget and remove keys that don't exist in dbFrom
-        for (const [contextHash, pairs] of this.dbTarget.entries()) {
+        for (const [contextHash, pairs] of Object.entries(this.dbTarget.getSync() ?? {})) {
             for (const key of Object.keys(pairs)) {
 
-                const context = this.dbFrom.get('_context', contextHash);
-                if (!this.dbFrom.get(context, key)) {
+                const context = this.dbFrom.getSync('_context', contextHash);
+                if (!this.dbFrom.getSync(context, key)) {
                     console.log(contextHash, key);
                     await this.dbTarget.del(contextHash, key);
                 }
@@ -539,14 +545,14 @@ class GPTrans {
         }
 
         const finalInstruction = instructions.length > 1 ? `- ${merged}` : merged;
-        const refinePromptFile = promptFile ?? new URL('./prompt/refine.md', import.meta.url).pathname;
+        const refinePromptFile = promptFile ? pathModule.resolve(promptFile) : REFINE_PROMPT_FILE;
 
         // Collect all existing translations grouped by contextHash
         const allBatches = [];
         let currentBatch = [];
         let currentCharCount = 0;
 
-        for (const [contextHash, pairs] of this.dbTarget.entries()) {
+        for (const [contextHash, pairs] of Object.entries(this.dbTarget.getSync() ?? {})) {
             for (const [key, translation] of Object.entries(pairs)) {
                 const entryCharCount = translation.length;
 
@@ -607,20 +613,20 @@ class GPTrans {
                 const minLength = Math.min(refinedTexts.length, entries.length);
                 for (let i = 0; i < minLength; i++) {
                     if (refinedTexts[i] && refinedTexts[i].trim()) {
-                        this.dbTarget.set(entries[i].contextHash, entries[i].key, refinedTexts[i].trim());
+                        await this.dbTarget.set(entries[i].contextHash, entries[i].key, refinedTexts[i].trim());
                     }
                 }
                 return;
             }
 
-            entries.forEach((entry, index) => {
+            for (const [index, entry] of entries.entries()) {
                 const refinedText = refinedTexts[index]?.trim();
                 if (!refinedText) {
                     console.error(`❌ No refined text for ${entry.key} at index ${index}`);
-                    return;
+                    continue;
                 }
-                this.dbTarget.set(entry.contextHash, entry.key, refinedText);
-            });
+                await this.dbTarget.set(entry.contextHash, entry.key, refinedText);
+            }
 
         } catch (e) {
             console.error('❌ Error in _processRefineBatch:', e.message);
@@ -634,22 +640,22 @@ class GPTrans {
         try {
             const model = GPTrans.mmix(this.modelKey, this.modelMixOptions);
 
-            model.setSystem("You are an expert translator and editor specialized in refining {TARGET_DENONYM} {TARGET_LANG} translations.");
+            model.setSystem("You are an expert translator and editor specialized in refining <%- TARGET_DENONYM %> <%- TARGET_LANG %> translations.");
 
-            // Use ModelMix templates: addTextFromFile + replace + block
+            // Use ModelMix templates: addTextFromFile + assign + block
             model.addTextFromFile(refinePromptFile);
-            model.replace({
-                '{INPUT}': text,
-                '{INSTRUCTION}': instruction,
-                '{CONTEXT}': this.context,
-                '{TARGET_ISO}': this.replaceTarget.TARGET_ISO,
-                '{TARGET_LANG}': this.replaceTarget.TARGET_LANG,
-                '{TARGET_COUNTRY}': this.replaceTarget.TARGET_COUNTRY,
-                '{TARGET_DENONYM}': this.replaceTarget.TARGET_DENONYM,
-                '{FROM_ISO}': this.replaceFrom.FROM_ISO,
-                '{FROM_LANG}': this.replaceFrom.FROM_LANG,
-                '{FROM_COUNTRY}': this.replaceFrom.FROM_COUNTRY,
-                '{FROM_DENONYM}': this.replaceFrom.FROM_DENONYM,
+            model.assign({
+                INPUT: text,
+                INSTRUCTION: instruction,
+                CONTEXT: this.context,
+                TARGET_ISO: this.replaceTarget.TARGET_ISO,
+                TARGET_LANG: this.replaceTarget.TARGET_LANG,
+                TARGET_COUNTRY: this.replaceTarget.TARGET_COUNTRY,
+                TARGET_DENONYM: this.replaceTarget.TARGET_DENONYM,
+                FROM_ISO: this.replaceFrom.FROM_ISO,
+                FROM_LANG: this.replaceFrom.FROM_LANG,
+                FROM_COUNTRY: this.replaceFrom.FROM_COUNTRY,
+                FROM_DENONYM: this.replaceFrom.FROM_DENONYM,
             });
 
             return await model.block({ addSystemExtra: false });
@@ -668,13 +674,13 @@ class GPTrans {
         } = options;
 
         // Parse image filename and extension
-        const parsedPath = path.parse(imagePath);
+        const parsedPath = pathModule.parse(imagePath);
         const filename = parsedPath.base;
         const targetLang = this.replaceTarget.TARGET_ISO || 'en';
 
         // Check if image is already in a language folder
-        const dirName = path.basename(path.dirname(imagePath));
-        const parentDir = path.dirname(path.dirname(imagePath));
+        const dirName = pathModule.basename(pathModule.dirname(imagePath));
+        const parentDir = pathModule.dirname(pathModule.dirname(imagePath));
 
         // If the image is in a language folder (e.g., en/image.jpg)
         // create the target at the same level (e.g., es/image.jpg)
@@ -683,12 +689,12 @@ class GPTrans {
 
         if (this._isLanguageFolder(dirName)) {
             // Image is in a language folder: create sibling folder
-            targetDir = path.join(parentDir, targetLang);
-            targetPath = path.join(targetDir, filename);
+            targetDir = pathModule.join(parentDir, targetLang);
+            targetPath = pathModule.join(targetDir, filename);
         } else {
             // Image is not in a language folder: create subfolder
-            targetDir = path.join(path.dirname(imagePath), targetLang);
-            targetPath = path.join(targetDir, filename);
+            targetDir = pathModule.join(pathModule.dirname(imagePath), targetLang);
+            targetPath = pathModule.join(targetDir, filename);
         }
 
         // Check if translated image already exists
@@ -736,7 +742,7 @@ class GPTrans {
             }
 
             // Save translated image - preserve original file format
-            const filename = path.basename(targetPath, path.extname(targetPath));
+            const filename = pathModule.basename(targetPath, pathModule.extname(targetPath));
             const formatOptions = generator.getReferenceMetadata();
 
             // Apply default quality settings for JPEG images
@@ -755,4 +761,4 @@ class GPTrans {
     }
 }
 
-export default GPTrans;
+export default GPTrans;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gptrans",
   "type": "module",
-  "version": "2.1.10",
+  "version": "2.2.4",
   "description": "🚆 GPTrans - The smarter AI-powered way to translate.",
   "keywords": [
     "translate",
@@ -32,12 +32,13 @@
   },
   "homepage": "https://github.com/clasen/GPTrans#readme",
   "engines": {
-    "node": ">=20.6.0"
+    "node": ">=20.12.0"
   },
   "dependencies": {
-    "deepbase": "^1.5.6",
-    "genmix": "^1.2.4",
-    "modelmix": "^4.6.5",
+    "deepbase": "^3.9.0",
+    "deepbase-json": "3.9.0",
+    "genmix": "^1.2.5",
+    "modelmix": "^5.1.6",
     "string-hash": "^1.1.3"
   },
   "scripts": {

package/pnpm-workspace.yaml

@@ -1,5 +1,10 @@
 allowBuilds:
   sharp: true
+minimumReleaseAgeExclude:
+  - genmix@1.2.5
+  - deepbase-json@3.9.0
+  - deepbase@3.9.0
+  - modelmix@5.1.6
 overrides:
   sharp: "^0.35.0"
   "@hono/node-server": "^2.0.5"

package/prompt/refine.md

@@ -1,12 +1,12 @@
 # Goal
-Refine existing translations in {TARGET_ISO} ({TARGET_DENONYM} {TARGET_LANG}) based on the following instruction.
+Refine existing translations in <%- TARGET_ISO %> (<%- TARGET_DENONYM %> <%- TARGET_LANG %>) based on the following instruction.
 
 ## Refinement Instruction
-{INSTRUCTION}
+<%- INSTRUCTION %>
 
 ## Current Translations to Evaluate
 ```
-{INPUT}
+<%- INPUT %>
 ```
 
 # Return Format
@@ -23,4 +23,4 @@ Refine existing translations in {TARGET_ISO} ({TARGET_DENONYM} {TARGET_LANG}) ba
    - **Consistency:** Maintain consistent terminology across all translations in the batch.
 
 # Context
-{CONTEXT}
+<%- CONTEXT %>

package/prompt/translate.md

@@ -1,13 +1,13 @@
 # Goal
-Translation from {FROM_ISO} to {TARGET_ISO} ({TARGET_DENONYM} {TARGET_LANG}) with cultural adaptations.
-{INSTRUCTION}
+Translation from <%- FROM_ISO %> to <%- TARGET_ISO %> (<%- TARGET_DENONYM %> <%- TARGET_LANG %>) with cultural adaptations.
+<%- INSTRUCTION %>
 
 ## Text to translate
 ```
-{INPUT}
+<%- INPUT %>
 ```
 
-{REFERENCES}
+<%- REFERENCES %>
 
 # Return Format
    - The input may contain multiple texts separated by `------`. Translate each one independently and return them in the same order, separated by `------`. The number of segments in your output must exactly match the number of segments in the input.
@@ -15,13 +15,13 @@ Translation from {FROM_ISO} to {TARGET_ISO} ({TARGET_DENONYM} {TARGET_LANG}) wit
    - Do not include alternative translations, only provide the best translation.
 
 # Warnings
-   - **Context:** I will provide you with a text in {FROM_DENONYM} {FROM_LANG}. The goal is to translate it to {TARGET_ISO} ({TARGET_DENONYM} {TARGET_LANG}) while maintaining the essence, style, intention, and tone of the original.
+   - **Context:** I will provide you with a text in <%- FROM_DENONYM %> <%- FROM_LANG %>. The goal is to translate it to <%- TARGET_ISO %> (<%- TARGET_DENONYM %> <%- TARGET_LANG %>) while maintaining the essence, style, intention, and tone of the original.
    - **Proper names:** Do not translate proper names (people, places, brands, etc.) unless they have an officially recognized translation in the target language.
-   - **Cultural references:** Adapt or explain references that are not familiar in {TARGET_DENONYM} culture, whenever necessary.
+   - **Cultural references:** Adapt or explain references that are not familiar in <%- TARGET_DENONYM %> culture, whenever necessary.
    - **Wordplay and humor:** When it's impossible to directly translate wordplay, find a resource that recreates the playful effect.
    - **Idioms:** Do not introduce new idioms or expressions that are not present in the original text.
    - **Variables:** Do not translate content between curly braces. These are system variables and must remain exactly the same.
 
 
 # Context
-{CONTEXT}
+<%- CONTEXT %>