gpt-po 1.1.2 → 1.2.0

package/.github/FUNDING.yml

@@ -0,0 +1 @@
+buy_me_a_coffee: ryanhex

package/CHANGELOG.md

@@ -0,0 +1,17 @@
+# Changelog
+
+## [1.2.0] - 2024-11-26
+- Submit multiple entries per chat to boost speed and reduce tokens
+- Command `systemprompt` has been removed
+- Bug fixes
+
+## [1.1.2] - 2024-11-12
+- Update command options for OpenAI model
+- Dependency updates, target es2022, module(Nodejs 18+)
+
+## [1.0.11] - 2023-10-26
+- Add `gpt-po remove` command
+- improve prompt for translation
+
+## [1.0.9] - 2023-09-08
+- Can add dictionaries for each languages

package/README.md

@@ -7,6 +7,8 @@ Translation tool for gettext (po) files that supports custom system prompts and
 
 Read in other languages: English | [简体中文](./README_zh-CN.md)
 
+<a href="https://buymeacoffee.com/ryanhex" target="_blank"><img src="https://cdn.buymeacoffee.com/buttons/default-red.png" alt="Buy Me A Coffee" height="41" width="174"></a>
+
 ## Installation
 
 ```
@@ -25,8 +27,6 @@ Set `OPENAI_API_KEY` before using this tool.
 - `gpt-po --dir .` Translate all po files in current directory to a designated target language.
 - `gpt-po userdict` Modify or view user dictionaries
 - `gpt-po userdict --explore` Explore user dictionaries, if you want add new dictionaries or modify existing dictionaries. dictionaries can be named as `dictionary-<lang>.json`, for example, `dictionary-zh.json` is the dictionary for Simplified Chinese.
-- `gpt-po systemprompt` Modify or view system prompts, if you are not sure how to use it, you can leave it alone
-- `gpt-po systemprompt --reset` Reset system prompts
 
 ```
 Usage: gpt-po [options] [command]
@@ -40,7 +40,6 @@ Options:
 Commands:
   translate [options]     translate po file (default command)
   sync [options]          update po from pot file
-  systemprompt [options]  open/edit system prompt
   userdict [options]      open/edit user dictionary
   remove [options]        remove po entries by options
   help [command]          display help for command
@@ -81,3 +80,14 @@ Options:
   -rc, --reference-contains <text>  remove entries whose reference contains text, text can be a regular expression like /text/ig
   -h, --help                        display help for command
 ```
+
+```
+Usage: gpt-po sync [options]
+
+update po from pot file
+
+Options:
+  --po <file>   po file path
+  --pot <file>  pot file path
+  -h, --help    display help for command
+```

package/README_zh-CN.md

@@ -7,6 +7,8 @@ gettext(po)文件翻译工具，支持自定义系统提示词和用户字典，
 
 使用其他语言阅读：[English](./README.md) | 简体中文
 
+<a href="https://buymeacoffee.com/ryanhex" target="_blank"><img src="https://cdn.buymeacoffee.com/buttons/default-red.png" alt="请我喝杯咖啡" height="41" width="174"></a>
+
 ## 安装
 
 ```
@@ -27,8 +29,6 @@ npm install gpt-po
 - `gpt-po --dir .` 将当前目录下的所有 po 文件翻译成目标语言。
 - `gpt-po userdict` 修改或查看用户词典。
 - `gpt-po userdict --explore` 浏览用户词典，如果您想添加新词典或修改现有词典，词典可以命名为 `dictionary-<lang>.json`，例如 `dictionary-zh.json` 是简体中文词典。
-- `gpt-po systemprompt` 修改或查看系统提示，如果您不确定如何使用它，可以忽略。
-- `gpt-po systemprompt --reset` 重置系统提示。
 
 ```
 用法: gpt-po [options] [command]
@@ -42,7 +42,6 @@ npm install gpt-po
 命令:
   translate [options]     翻译 po 文件（默认命令）
   sync [options]          根据 pot 文件更新 po 文件
-  systemprompt [options]  打开/编辑系统提示
   userdict [options]      打开/编辑用户词典
   remove [options]        通过选项删除 po 条目
   help [command]          显示命令帮助
@@ -83,3 +82,14 @@ npm install gpt-po
   -rc, --reference-contains <text>  删除引用包含文本的条目，文本可以是正则表达式，例如 /text/ig
   -h, --help                        显示命令帮助
 ```
+
+```
+用法: gpt-po sync [options]
+
+从 pot 文件中更新条目到 po 文件
+
+Options:
+  --po <file>   po 文件路径
+  --pot <file>  pot 文件路径
+  -h, --help    显示命令帮助
+```

package/lib/package.json

@@ -1,6 +1,6 @@
 {
     "name": "gpt-po",
-    "version": "1.1.2",
+    "version": "1.2.0",
     "description": "command tool for translate po files by gpt",
     "main": "lib/src/index.js",
     "type": "module",

package/lib/src/index.d.ts

@@ -1,2 +1,2 @@
-#!/usr/bin/env node
+#!/usr/bin/env node --no-warnings=ExperimentalWarning
 export {};

package/lib/src/index.js

@@ -1,4 +1,4 @@
-#!/usr/bin/env node
+#!/usr/bin/env node --no-warnings=ExperimentalWarning
 import { Command, Option } from "commander";
 import path from "path";
 import { fileURLToPath } from "url";
@@ -16,7 +16,7 @@ program
     .description("translate po file (default command)")
     .addOption(new Option("-k, --key <key>", "openai api key").env("OPENAI_API_KEY"))
     .addOption(new Option("--host <host>", "openai api host").env("OPENAI_API_HOST"))
-    .addOption(new Option("--model <model>", "openai model").env("OPENAI_MODEL").default("gpt-4o"))
+    .addOption(new Option("--model <model>", "openai model").env("OPENAI_MODEL").default("gpt-4o-mini"))
     .addOption(new Option("--po <file>", "po file path").conflicts("dir"))
     .addOption(new Option("--dir <dir>", "po file directory").conflicts("po"))
     .option("-src, --source <lang>", "source language (ISO 639-1)", "en")
@@ -57,39 +57,24 @@ program
     .action(async ({ po, pot }) => {
     await sync(po, pot);
 });
-// program command `systemprompt` with help text `open/edit system prompt`
-program
-    .command("systemprompt")
-    .description("open/edit system prompt")
-    .option("--reset", "reset system prompt to default")
-    .action((args) => {
-    const { reset } = args;
-    // open `systemprompt.txt` file by system text default editor
-    const copyFile = __dirname + "/systemprompt.txt";
-    // user home path
-    const promptFile = findConfig("systemprompt.txt");
-    copyFileIfNotExists(promptFile, copyFile, reset);
-    if (reset) {
-        console.log("systemprompt.txt reset to default");
-    }
-    openFileByDefault(promptFile);
-});
 // program command `userdict` with help text `open/edit user dictionary`
 program
     .command("userdict")
     .description("open/edit user dictionary")
     .option("--explore", "open user dictionary directory")
+    .option("-l, --lang <lang>", "target language (ISO 639-1)")
     .action((args) => {
-    const { explore } = args;
+    const { explore, lang } = args;
     // open `dictionary.json` file by system text default editor
     const copyFile = __dirname + "/dictionary.json";
-    // user home path
-    const dictFile = findConfig("dictionary.json");
+    // find from user home path
+    const dictFile = findConfig(`dictionary${lang ? "-" + lang : ""}.json`);
     if (explore) {
         // open user dictionary directory
         return openFileExplorer(dictFile);
     }
-    copyFileIfNotExists(dictFile, copyFile);
+    if (!lang)
+        copyFileIfNotExists(dictFile, copyFile);
     openFileByDefault(dictFile);
 });
 // program command `remove` with help text `remove po entries by options`
@@ -113,7 +98,7 @@ program
         translated,
         translatedNotFuzzy,
         fuzzyTranslated,
-        referenceContains,
+        referenceContains
     };
     const output = args.output || po;
     const translations = await parsePo(po);

package/lib/src/systemprompt.txt

@@ -1,31 +1 @@
-You are a language translation expert. You will translate text from one language to another, following these guidelines:
-
-1. **Language Codes**:
-   - If provided with an ISO 639 two-letter language code (e.g., `de`), use the language’s main dialect (e.g., `de` is equivalent to `de_DE`).
-   - If provided with both an ISO 639 language code and an ISO 3166 country code (e.g., `es_MX`), translate into that specific regional dialect.
-
-2. **Placeholder Handling**:
-   - Maintain the positions of placeholders (e.g., %s, %d, {example}) in the translated text. Do not translate placeholders.
-
-3. **Formatting**:
-   - Preserve the formatting of untranslatable portions.
-   - Retain any whitespace at the beginning or end of the message.
-   - Add or omit a period (.) at the end of your translation to match the incoming message.
-
-4. **XML Tags**:
-   - Input messages will be wrapped in `<translate>` XML tags.
-   - Respond with the translated message wrapped in `<translated>` XML tags.
-
-5. **Quality**:
-   - Translate in a colloquial, professional, and elegant manner without sounding like a machine translation.
-
-6. **Error Handling**:
-   - If you cannot reliably translate a message, respond without `<translated>` XML tags and provide a short reason why.
-
-**Examples**:
-- Input: `<translate>This is a message. </translate>`
-  - Output: `<translated>Este es un mensaje. </translated>`
-- Input: `<translate> Hello %s</translate>`
-  - Output: `<translated> Hola %s</translated>`
-
-Do not answer questions or explain concepts. Only provide translations within `<translated>` XML tags unless you need to respond with a short error reason.
+You are a language translation expert. You will carefully follow the translation guidelines to translate the incoming XML messages from one language to another.

package/lib/src/translate.d.ts

@@ -1,6 +1,6 @@
 import { GetTextTranslation } from "gettext-parser";
 import { OpenAI } from "openai";
 export declare function init(force?: boolean): OpenAI;
-export declare function translate(text: string, src: string, lang: string, model: string, comments: GetTextTranslation["comments"] | undefined, contextFile: string): import("openai/core.mjs").APIPromise<OpenAI.Chat.Completions.ChatCompletion>;
+export declare function translate(src: string, lang: string, model: string, translations: GetTextTranslation[], contextFile: string): Promise<void>;
 export declare function translatePo(model: string, po: string, source: string, lang: string, verbose: boolean, output: string, contextFile: string): Promise<void>;
 export declare function translatePoDir(model: string | undefined, dir: string, source: string, lang: string, verbose: boolean, contextFile: string): Promise<void>;

package/lib/src/translate.js

@@ -8,47 +8,59 @@ const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 let _openai;
 let _systemprompt;
+let _userprompt;
 let _userdict;
 export function init(force) {
     if (!_openai || force) {
         let configuration = {
-            apiKey: process.env.OPENAI_API_KEY,
+            apiKey: process.env.OPENAI_API_KEY
         };
         _openai = new OpenAI(configuration);
         if (process.env.OPENAI_API_HOST) {
             _openai.baseURL = process.env.OPENAI_API_HOST.replace(/\/+$/, "") + "/v1";
         }
     }
-    // load systemprompt.txt from homedir
+    // load systemprompt.txt from project
     if (!_systemprompt || force) {
-        const systemprompt = findConfig("systemprompt.txt");
-        copyFileIfNotExists(systemprompt, path.join(__dirname, "systemprompt.txt"));
-        _systemprompt = fs.readFileSync(systemprompt, "utf-8");
+        _systemprompt = fs.readFileSync(path.join(__dirname, "systemprompt.txt"), "utf-8");
+    }
+    // load userprompt.txt from project
+    if (!_userprompt || force) {
+        _userprompt = fs.readFileSync(path.join(__dirname, "userprompt.txt"), "utf-8");
     }
     // load dictionary.json from homedir
     if (!_userdict || force) {
         const userdict = findConfig("dictionary.json");
         copyFileIfNotExists(userdict, path.join(__dirname, "dictionary.json"));
-        _userdict = { "default": JSON.parse(fs.readFileSync(userdict, "utf-8")) };
+        _userdict = { default: JSON.parse(fs.readFileSync(userdict, "utf-8")) };
     }
     return _openai;
 }
-export function translate(text, src, lang, model, comments, contextFile) {
-    const lang_code = lang.toLowerCase().trim().replace(/[\W_]+/g, "-");
-    const dicts = Object.entries(_userdict[lang_code] || _userdict["default"])
-        .filter(([k, _]) => text.toLowerCase().includes(k.toLowerCase()))
-        .map(([k, v]) => [
-        { role: "user", content: k },
-        { role: "assistant", content: v },
-    ])
-        .flat();
-    var notes = "";
-    if (comments != undefined && comments.extracted != undefined)
-        notes = comments.extracted;
-    var context = "";
-    if (contextFile !== undefined)
-        context = "\n\n" + fs.readFileSync(contextFile, "utf-8");
-    return _openai.chat.completions.create({
+export async function translate(src, lang, model, translations, contextFile) {
+    const lang_code = lang
+        .toLowerCase()
+        .trim()
+        .replace(/[\W_]+/g, "-");
+    const dicts = Object.entries(_userdict[lang_code] || _userdict["default"]).reduce((acc, [k, v], idx) => {
+        if (translations.some((tr) => tr.msgid.toLowerCase().includes(k.toLowerCase()))) {
+            acc.user.push(`<translate index="${idx + 1}">${k}</translate>`);
+            acc.assistant.push(`<translated index="${idx + 1}">${v}</translated>`);
+        }
+        return acc;
+    }, { user: [], assistant: [] });
+    const notes = translations
+        .reduce((acc, tr) => {
+        if (tr.comments?.extracted) {
+            acc.push(tr.comments?.extracted);
+        }
+        return acc;
+    }, [])
+        .join("\n");
+    const context = contextFile ? "\n\nContext: " + fs.readFileSync(contextFile, "utf-8") : "";
+    const translationsContent = translations
+        .map((tr, idx) => `<translate index="${idx + dicts.user.length + 1}">${tr.msgid}</translate>`)
+        .join("\n");
+    const res = await _openai.chat.completions.create({
         model: model,
         temperature: 0.1,
         messages: [
@@ -58,21 +70,41 @@ export function translate(text, src, lang, model, comments, contextFile) {
             },
             {
                 role: "user",
-                content: `Wait for my incoming message in "${src}" and translate it into "${lang}", carefully following your system prompt. ` + notes
+                content: `${_userprompt}\n\nWait for my incoming message in "${src}" and translate it into "${lang}"(a language code and an optional region code). ` +
+                    notes
             },
             {
                 role: "assistant",
-                content: `Understood, I will translate your incoming "${src}" message into "${lang}", carefully following my system prompt. Please go ahead and send your message for translation.`
+                content: `Understood, I will translate your incoming "${src}" message into "${lang}", carefully following guidelines. Please go ahead and send your message for translation.`
             },
-            // add userdict here
-            ...dicts,
+            // add userdict
+            ...(dicts.user.length > 0
+                ? [
+                    { role: "user", content: dicts.user.join("\n") },
+                    { role: "assistant", content: dicts.assistant.join("\n") }
+                ]
+                : []),
+            // add user translations
             {
                 role: "user",
-                content: "<translate>" + text + "</translate>"
-            },
-        ],
+                content: translationsContent
+            }
+        ]
     }, {
         timeout: 20000,
+        stream: false
+    });
+    const content = res.choices[0].message.content ?? "";
+    translations.forEach((trans, idx) => {
+        const tag = `<translated index="${idx + dicts.user.length + 1}">`;
+        const s = content.indexOf(tag);
+        if (s > -1) {
+            const e = content.indexOf("</translated>", s);
+            trans.msgstr[0] = content.slice(s + tag.length, e);
+        }
+        else {
+            console.error("Error: Unable to find translation for string [" + trans.msgid + "]");
+        }
     });
 }
 export async function translatePo(model, po, source, lang, verbose, output, contextFile) {
@@ -84,7 +116,10 @@ export async function translatePo(model, po, source, lang, verbose, output, cont
         return;
     }
     // try to load dictionary by lang-code if it not loaded
-    const lang_code = lang.toLowerCase().trim().replace(/[\W_]+/g, "-");
+    const lang_code = lang
+        .toLowerCase()
+        .trim()
+        .replace(/[\W_]+/g, "-");
     if (!_userdict[lang_code]) {
         const lang_dic_file = findConfig(`dictionary-${lang_code}.json`);
         if (fs.existsSync(lang_dic_file)) {
@@ -117,9 +152,9 @@ export async function translatePo(model, po, source, lang, verbose, output, cont
         return;
     }
     potrans.headers["Last-Translator"] = `gpt-po v${pkg.version}`;
+    const translations = [];
     let err429 = false;
-    let modified = false;
-    for (let i = 0; i < list.length; i++) {
+    for (let i = 0, c = 0; i < list.length; i++) {
         if (i == 0)
             printProgress(i, list.length);
         if (err429) {
@@ -127,40 +162,43 @@ export async function translatePo(model, po, source, lang, verbose, output, cont
             await new Promise((resolve) => setTimeout(resolve, 20000));
         }
         const trans = list[i];
-        try {
-            const res = await translate(trans.msgid, source, lang, model, trans.comments, contextFile);
-            var translated = res.choices[0].message?.content || trans.msgstr[0];
-            if (!translated.startsWith('<translated>') && !translated.endsWith('</translated>')) {
-                // We got an error response
-                console.log("Error: Unable to translate string [" + trans.msgid + "]. Bot says [" + translated + "]");
-                continue;
-            }
-            // We got a valid translation response
-            trans.msgstr[0] = translated.replace('<translated>', '').replace('</translated>', '');
-            modified = true;
-            if (verbose) {
-                console.log(trans.msgid);
-                console.log(trans.msgstr[0]);
-            }
-            printProgress(i + 1, list.length);
-            await compilePo(potrans, output || po);
+        if (c < 2000) {
+            translations.push(trans);
+            c += trans.msgid.length;
         }
-        catch (error) {
-            if (error.response) {
-                if (error.response.status == 429) {
-                    // caused by rate limit exceeded, should sleep for 20 seconds.
-                    err429 = true;
-                    --i;
-                }
-                else {
-                    console.error(error.response.status);
-                    console.log(error.response.data);
+        if (c >= 2000 || i == list.length - 1) {
+            try {
+                await translate(source, lang, model, translations, contextFile);
+                if (verbose) {
+                    translations.forEach((trans) => {
+                        console.log(trans.msgid);
+                        console.log(trans.msgstr[0]);
+                    });
                 }
+                translations.length = 0;
+                c = 0;
+                // update progress
+                printProgress(i + 1, list.length);
+                // save po file after each 2000 characters
+                await compilePo(potrans, output || po);
             }
-            else {
-                console.error(error.message);
-                if (error.code == "ECONNABORTED") {
-                    console.log('you may need to set "HTTPS_PROXY" to reach openai api.');
+            catch (error) {
+                if (error.response) {
+                    if (error.response.status == 429) {
+                        // caused by rate limit exceeded, should sleep for 20 seconds.
+                        err429 = true;
+                        --i;
+                    }
+                    else {
+                        console.error(error.response.status);
+                        console.log(error.response.data);
+                    }
+                }
+                else {
+                    console.error(error.message);
+                    if (error.code == "ECONNABORTED") {
+                        console.log('you may need to set "HTTPS_PROXY" to reach openai api.');
+                    }
                 }
             }
         }

package/lib/src/userprompt.txt

@@ -0,0 +1,27 @@
+Translation guidelines are as follows:
+
+1. **Placeholder Handling**:
+   - Maintain the positions of placeholders (e.g., %s, %d, {example}) in the translated text. Do not translate placeholders.
+
+2. **Formatting**:
+   - Preserve the formatting of untranslatable portions.
+   - Retain any whitespace at the beginning or end of the message.
+   - Add or omit a period (.) at the end of your translation to match the incoming message.
+
+3. **XML Tags and Indexing**:
+   - Input messages will be wrapped in `<translate>` XML tags with an index attribute, e.g., `<translate index="1">`.
+   - Respond with the translated message wrapped in `<translated>` XML tags, including the same index attribute, e.g., `<translated index="1">`.
+
+4. **Multiple Translations**:
+   - You may receive multiple translation requests in a single input, each with a unique index.
+   - Ensuring the complete number of requests are translated, even if they are repeated, while maintaining the original order when possible.
+
+**Examples**:
+- Input:
+  `<translate index="1">This is a message. </translate>`
+  `<translate index="2"> Hello %s</translate>`
+- Output:
+  `<translated index="1">这是一条信息</translated>`
+  `<translated index="2"> 你好 %s</translated>`
+
+Do not answer questions or explain concepts. Only provide translations within `<translated>` XML tags unless you need to respond with a short error reason.

package/lib/src/utils.d.ts

@@ -8,7 +8,12 @@ import { GetTextTranslations } from "gettext-parser";
 export declare function copyFileIfNotExists(file: string, copyFile: string, force?: boolean): void;
 export declare function openFileByDefault(filePath: string): void;
 export declare function parsePo(poFile: string, defaultCharset?: string): Promise<GetTextTranslations>;
-export declare function compilePo(data: GetTextTranslations, poFile: string): Promise<void>;
+export type compileOptions = {
+    foldLength?: number;
+    sort?: boolean | ((a: never, b: never) => number);
+    escapeCharacters?: boolean;
+};
+export declare function compilePo(data: GetTextTranslations, poFile: string, options?: compileOptions): Promise<void>;
 export declare function printProgress(progress: number, total: number, extra?: string): void;
 export declare function gitRootDir(dir?: string): string | null;
 /**

package/lib/src/utils.js

@@ -36,13 +36,13 @@ export function parsePo(poFile, defaultCharset) {
         fs.readFile(poFile, (err, buffer) => {
             if (err)
                 reject(err);
-            var result = po.parse(buffer, defaultCharset ?? "utf-8");
+            const result = po.parse(buffer, defaultCharset ?? "utf-8");
             resolve(result);
         });
     });
 }
-export function compilePo(data, poFile) {
-    const buffer = po.compile(data, { foldLength: 120 });
+export function compilePo(data, poFile, options = { foldLength: 120, sort: false, escapeCharacters: true }) {
+    const buffer = po.compile(data, options);
     return new Promise((resolve, reject) => {
         fs.writeFile(poFile, buffer, (err) => {
             if (err)
@@ -95,7 +95,7 @@ export function findConfig(fileName) {
         path.join(currentDir, ".gpt-po", fileName),
         path.join(currentDir, fileName),
         path.join(gitDir, ".gpt-po", fileName),
-        path.join(homeDir, ".gpt-po", fileName)
+        path.join(homeDir, ".gpt-po", fileName),
     ];
     // check if file exists and return the first one
     for (const filePath of filePaths) {
@@ -111,10 +111,10 @@ export function findConfig(fileName) {
  * @param location folder or file path
  */
 export function openFileExplorer(location) {
-    if (platform() === 'win32') {
+    if (platform() === "win32") {
         exec(`explorer.exe "${path.dirname(location)}"`);
     }
-    else if (platform() === 'darwin') {
+    else if (platform() === "darwin") {
         exec(`open "${path.dirname(location)}"`);
     }
     else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gpt-po",
-  "version": "1.1.2",
+  "version": "1.2.0",
   "description": "command tool for translate po files by gpt",
   "main": "lib/src/index.js",
   "type": "module",