cc-translate 0.4.0

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "cc-translate",
+  "version": "0.4.0",
+  "description": "A language translation tool based on OpenAI",
+  "author": "maybe",
+  "packageManager": "yarn@1.22.22",
+  "types": "./index.d.ts",
+  "main": "./index.mjs",
+  "module": "./index.mjs",
+  "publishConfig": {
+    "access": "public"
+  },
+  "keywords": [
+    "translate",
+    "openai"
+  ],
+  "license": "ISC",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/cctip/cwallet-translate-script.git"
+  },
+  "homepage": "https://github.com/cctip/cwallet-translate-script",
+  "bugs": "https://github.com/xxx/cwallet-translate-script/issues",
+  "type": "module",
+  "scripts": {
+    "publish": "npx tsc && npm publish",
+    "compile": "tsc"
+  },
+  "exports": {
+    ".": {
+      "import": "./index.js",
+      "require": "./index.js"
+    }
+  },
+  "dependencies": {
+    "ansi-colors": "^4.1.3",
+    "cli-progress": "^3.12.0",
+    "openai": "^4.68.1",
+    "ts-node": "^10.9.2"
+  },
+  "devDependencies": {
+    "@types/cli-progress": "^3.11.6",
+    "@types/node": "^22.7.9",
+    "typescript": "^5.6.3"
+  }
+}

package/src/index.ts

@@ -0,0 +1,361 @@
+import path from "path";
+import fs from "fs";
+import colors from "ansi-colors";
+import { OpenAI } from "openai";
+import cliProgress from "cli-progress";
+import {
+  ICwalletTranslateParams,
+  IJson,
+  IOpenaiConfig,
+  IOutputLanguageFile,
+  ISingleTranslate,
+  ITranslateChat,
+  ITranslateChatResponse,
+  SupportLanguageType,
+} from "./types";
+import {
+  chunkArray,
+  getRandomNumber,
+  notExistsToCreateFile,
+  readFileOfDirSync,
+  readJsonFileSync,
+} from "./lib/utils.js";
+import {
+  getCacheFileSync,
+  registerLanguageCacheFile,
+  translateJSONDiffToJson,
+} from "./lib/cache/index.js";
+import { logErrorToFile } from "./lib/log/index.js";
+import { SUPPORT_LANGUAGE_MAP } from "./lib/support.js";
+
+const __dirname = process.cwd();
+
+const DEFAULT_OPENAI_CONFIG: IOpenaiConfig = {
+  model: "gpt-4o",
+};
+
+export class CwalletTranslate {
+  /** open ai api key  */
+  private OPENAI_KEY: string;
+  /** */
+  CACHE_ROOT_PATH: string;
+  ENTRY_ROOT_PATH: string;
+  /** default en */
+  SOURCE_LANGUAGE: SupportLanguageType;
+  OUTPUT_ROOT_PATH: string | undefined;
+  languages: SupportLanguageType[];
+  client: OpenAI | null = null;
+  /** default model gpt-4o */
+  openaiConfig: IOpenaiConfig;
+  fineTune: string[];
+
+  constructor(params: ICwalletTranslateParams) {
+    this.OPENAI_KEY = params.key;
+    this.CACHE_ROOT_PATH = params.cacheFileRootPath;
+    this.ENTRY_ROOT_PATH = params.fileRootPath;
+    this.openaiConfig = params.openaiConfig ?? DEFAULT_OPENAI_CONFIG;
+    this.SOURCE_LANGUAGE = params.sourceLanguage ?? "en";
+    this.OUTPUT_ROOT_PATH = params.outputRootPath;
+    this.fineTune = params.fineTune;
+    this.languages = params.languages ?? [];
+    this.createOpenAIClient();
+  }
+
+  get supportLanguages() {
+    return Object.entries(SUPPORT_LANGUAGE_MAP)
+      .map(([key, val]) => val)
+      .filter(
+        ({ code }) =>
+          this.languages.includes(code) || code === this.SOURCE_LANGUAGE
+      );
+  }
+
+  get outputPath() {
+    return this.OUTPUT_ROOT_PATH ?? this.ENTRY_ROOT_PATH;
+  }
+
+  searchLanguage(code: SupportLanguageType) {
+    return this.supportLanguages.find((item) => item.code === code);
+  }
+
+  createOpenAIClient = () => {
+    /** 初始化openAi */
+    const client = new OpenAI({
+      apiKey: this.OPENAI_KEY,
+    });
+
+    this.client = client;
+  };
+  /**
+   * 翻译入口文件的所有支持的语言文件夹和其中的文件
+   */
+  translate = async () => {
+    console.log("🚀 开始翻译");
+    console.log(`🚀 使用的模型: ${this.openaiConfig.model} 🚀`);
+    console.log(`🚀 微调: ${this.fineTune} 🚀`);
+
+    const translateFolderPath = path.join(
+      this.ENTRY_ROOT_PATH,
+      this.SOURCE_LANGUAGE
+    );
+    // 翻译源语言问价夹下的所有json文件
+    const translateFolders = await readFileOfDirSync(translateFolderPath);
+    console.log("🚀 ~ 需要翻译语言的文件:", translateFolders);
+    // 创建进度条
+    const multiBar = new cliProgress.MultiBar(
+      {
+        clearOnComplete: false,
+        hideCursor: true,
+        format:
+          colors.cyan("{bar}") +
+          "| {percentage}% || {filename} {value}/{total} ",
+      },
+      cliProgress.Presets.legacy
+    );
+
+    let promises = [];
+    const arr: (() => Promise<void>)[] = [];
+
+    for (const item of this.supportLanguages) {
+      // 源语言不翻译
+      if (item.code === this.SOURCE_LANGUAGE) continue;
+      for (const fileName of translateFolders) {
+        const translateJson = await this.getTranslateContent(
+          item.code,
+          fileName
+        );
+        if (!translateJson) {
+          console.log(`${item.code}:${fileName} 没有需要翻译的内容`);
+          continue;
+        }
+        arr.push(() =>
+          this.singleTranslate({
+            language: item.code,
+            fileName,
+            multiBar,
+            translateJson,
+          })
+        );
+      }
+    }
+
+    promises = chunkArray(arr, 8);
+
+    for (const chunk of promises) {
+      await Promise.all(chunk.map((fn) => fn()));
+    }
+
+    multiBar.stop();
+    console.log("🚀 翻译完毕");
+  };
+  /**
+   * 翻译单个文件
+   * @param params
+   * @returns
+   */
+  singleTranslate = async (params: ISingleTranslate) => {
+    const {
+      /** 待翻译的语言 */
+      language,
+      /** 待翻译的文件名 */
+      fileName,
+      translateJson,
+      multiBar,
+      callback,
+    } = params;
+
+    try {
+      // 等待翻译的数组
+      const jsonMap: IJson = {};
+
+      // 生成chat循环代码
+      const promiseList = Object.entries(translateJson).map(
+        ([key, value], index) =>
+          () =>
+            this.translateChat({
+              key,
+              value,
+              language,
+              index,
+              fileName,
+            })
+      );
+
+      const progressBar = multiBar.create(promiseList.length, 0);
+
+      for (const fn of promiseList) {
+        const result = await fn();
+        jsonMap[result.key] = result.value;
+        progressBar.update(result.index + 1, {
+          filename: `${language}:${fileName}`,
+        });
+      }
+      this.outputLanguageFile({
+        jsonMap,
+        folderName: language,
+        fileName,
+      });
+      callback && callback();
+    } catch (error) {
+      logErrorToFile({
+        error: error as Error,
+        language,
+        fileName,
+        key: "",
+      });
+      return;
+    }
+  };
+
+  /**
+   * 使用open ai 进行翻译
+   * @param {string} key
+   * @param {string} value
+   * @param {OpenAI} client
+   * @param {string} language
+   * @returns
+   */
+  translateChat = (params: ITranslateChat): Promise<ITranslateChatResponse> => {
+    return new Promise((resolve) => {
+      const { key, value, language, index, fileName } = params;
+      try {
+        if (!this.client) throw new Error("Connection failed");
+        const targetLanguage = this.searchLanguage(language);
+        const originLanguage = this.searchLanguage(this.SOURCE_LANGUAGE);
+
+        if (!targetLanguage) {
+          throw new Error(`不支持的语言：${language}`);
+        }
+
+        if (!originLanguage) {
+          throw new Error(`不支持的语言：${this.SOURCE_LANGUAGE}`);
+        }
+
+        setTimeout(async () => {
+          const chatCompletion = await this.client!.chat.completions.create({
+            model: this.openaiConfig?.model,
+            messages: [
+              ...this.fineTune.map(
+                (val) =>
+                  ({
+                    role: "system",
+                    content: val,
+                  } as OpenAI.Chat.Completions.ChatCompletionMessageParam)
+              ),
+              {
+                role: "system",
+                content: `请将${originLanguage!.name}翻译成${
+                  targetLanguage!.name
+                }`,
+              },
+              {
+                role: "system",
+                content: `翻译完成直接输出后对应意思的内容不要携带任何无关内容`,
+              },
+              {
+                role: "user",
+                content: value,
+              },
+            ],
+          });
+          resolve({
+            key,
+            value: chatCompletion?.choices[0]?.message.content ?? value,
+            index,
+          });
+        }, getRandomNumber(200, 300));
+      } catch (error) {
+        logErrorToFile({ error: error as Error, key, fileName, language });
+        resolve({
+          key,
+          value,
+          index,
+          error: error as Error,
+        });
+      }
+    });
+  };
+  /**
+   * 对比缓存文件 获取需要翻译的内容
+   * @param language
+   * @param fileName
+   * @returns
+   */
+  getTranslateContent = async (
+    language: SupportLanguageType,
+    fileName: string
+  ): Promise<IJson | undefined> => {
+    const translateFilePath = path.join(
+      this.ENTRY_ROOT_PATH,
+      this.SOURCE_LANGUAGE,
+      fileName
+    );
+    /** 缓存文件路径 */
+    const cacheFilePath = path.join(this.CACHE_ROOT_PATH, language, fileName);
+    if (!fs.existsSync(translateFilePath)) {
+      console.dir(`File not found: ${translateFilePath}`);
+      return;
+    }
+    const translateFileObject = await readJsonFileSync(translateFilePath);
+    const cacheObject = await getCacheFileSync(cacheFilePath);
+    const diffObject = translateJSONDiffToJson(
+      cacheObject,
+      translateFileObject
+    );
+    if (Object.values(diffObject).length === 0) {
+      if (Object.keys(translateFileObject).length === 0) {
+        this.outputLanguageFile({
+          jsonMap: {},
+          folderName: language,
+          fileName,
+        });
+      }
+      return;
+    }
+    return diffObject;
+  };
+
+  /**
+   * 输出语言文件
+   * @param {Object} jsonMap
+   */
+  outputLanguageFile = async (params: IOutputLanguageFile) => {
+    const { folderName, fileName, jsonMap } = params;
+    const outputFilePath = path.join(this.outputPath, folderName, fileName);
+    //创建输出文件夹
+    notExistsToCreateFile(this.outputPath);
+    //创建输出的语言文件夹
+    notExistsToCreateFile(`${this.outputPath}/${folderName}`);
+    let oldJsonData: string = "";
+    // 检查是否存在文件
+    if (!fs.existsSync(outputFilePath)) {
+      oldJsonData = await fs.readFileSync(
+        path.join(this.ENTRY_ROOT_PATH, this.SOURCE_LANGUAGE, fileName),
+        "utf8"
+      );
+    } else {
+      oldJsonData = await fs.readFileSync(outputFilePath, "utf8");
+    }
+    const oldJsonMap: IJson = JSON.parse(oldJsonData);
+    const newJsonMap: IJson = Object.assign(oldJsonMap, jsonMap);
+
+    await fs.writeFileSync(
+      path.resolve(outputFilePath),
+      JSON.stringify(newJsonMap, null, 2),
+      "utf8"
+    );
+
+    // 注册缓存
+    registerLanguageCacheFile({
+      sourceFilePath: path.join(
+        this.ENTRY_ROOT_PATH,
+        this.SOURCE_LANGUAGE,
+        fileName
+      ),
+      jsonMap: newJsonMap,
+      fileName,
+      language: folderName,
+      folderName: this.CACHE_ROOT_PATH,
+    });
+  };
+}

package/src/lib/cache/index.ts

@@ -0,0 +1,70 @@
+import path from "path";
+import fs from "fs";
+import { notExistsToCreateFile, readJsonFileSync } from "../utils.js";
+import { IJson, IRegisterLanguageCacheFile } from "../../types";
+
+/**
+ * 和翻译缓存json文件对比 返回存在更改的json文件
+ * @param {object} cacheObject 已经缓存的对象
+ * @param {object} translateObject 需要翻译的对象
+ * @returns {object} 存在修改的对象
+ */
+export const translateJSONDiffToJson = (
+  cacheObject: IJson,
+  translateObject: IJson
+) => {
+  if (Object.values(cacheObject).length === 0) return translateObject;
+  // json文件内容diff
+  const pendingTranslateMap: IJson = {};
+  Object.entries(translateObject).forEach(([key, value]) => {
+    //不存在该key 是新增的key
+    if (!cacheObject[key]) {
+      pendingTranslateMap[key] = value;
+    }
+    // 存在缓存key但是内容不一样 需要重新翻译
+    else if (translateObject[key] !== cacheObject[key]) {
+      pendingTranslateMap[key] = value;
+    }
+  });
+
+  return pendingTranslateMap;
+};
+
+/**
+ * 获取缓存文件
+ * @param {string} filePath 缓存文件路径
+ * @returns {Promise<{key:value}>}
+ */
+export const getCacheFileSync = async (filePath: string): Promise<IJson> => {
+  if (fs.existsSync(filePath)) {
+    return await readJsonFileSync(filePath);
+  } else {
+    return {};
+  }
+};
+
+/**
+ * 注册语言缓存文件
+ * @param {string} language
+ */
+export const registerLanguageCacheFile = async (
+  params: IRegisterLanguageCacheFile
+) => {
+  const { jsonMap, sourceFilePath, fileName, language, folderName } = params;
+  const cacheFilePath = path.join(folderName, language, fileName);
+  const sourceObject = await readJsonFileSync(sourceFilePath);
+  const cacheObject = await readJsonFileSync(cacheFilePath);
+
+  Object.entries(jsonMap).forEach(([key, value]) => {
+    cacheObject[key] = sourceObject[key];
+  });
+
+  if (Object.values(jsonMap).length === 0) return;
+  notExistsToCreateFile(folderName);
+  notExistsToCreateFile(`${folderName}/${language}`);
+  await fs.writeFileSync(
+    cacheFilePath,
+    JSON.stringify(cacheObject, null, 2),
+    "utf8"
+  );
+};

package/src/lib/log/index.ts

@@ -0,0 +1,19 @@
+import { ITranslateLogError } from "../../types";
+import fs from "fs";
+
+export function logErrorToFile(params: ITranslateLogError) {
+  const { error, key, language, fileName } = params;
+  const logMessage = `${new Date().toISOString()} - Error: ${error.message}
+  \nLanguage: ${language}
+  \nFileName: ${fileName}
+  \nkey:${key}
+  \nStack: ${error.stack}\n\n`;
+
+  fs.appendFile("error.log", logMessage, (err) => {
+    if (err) {
+      console.error("无法写入日志文件:", err);
+    } else {
+      console.log("错误日志已写入 error.log 文件");
+    }
+  });
+}

package/src/lib/support.ts

@@ -0,0 +1,104 @@
+import { SupportLanguageMap } from "../types";
+
+export const SUPPORT_LANGUAGE_MAP: SupportLanguageMap = {
+  en: {
+    code: "en",
+    name: "英语",
+  },
+  ["zh-CN"]: {
+    code: "zh-CN",
+    name: "简体中文",
+  },
+  ["zh-TW"]: {
+    code: "zh-TW",
+    name: "繁体中文",
+  },
+  ja: {
+    code: "ja",
+    name: "日语",
+  },
+  ar: {
+    code: "ar",
+    name: "阿拉伯语",
+  },
+  bn: {
+    code: "bn",
+    name: "孟加拉语",
+  },
+  de: {
+    code: "de",
+    name: "德语",
+  },
+  ["es-ES"]: {
+    code: "es-ES",
+    name: "西班牙语",
+  },
+  fr: {
+    code: "fr",
+    name: "法语",
+  },
+  hi: {
+    code: "hi",
+    name: "印地语",
+  },
+  id: {
+    code: "id",
+    name: "印度尼西亚语",
+  },
+  it: {
+    code: "it",
+    name: "意大利语",
+  },
+  ko: {
+    code: "ko",
+    name: "韩语",
+  },
+  ms: {
+    code: "ms",
+    name: "马来语",
+  },
+  my: {
+    code: "my",
+    name: "缅甸语",
+  },
+  ["ne-NP"]: {
+    code: "ne-NP",
+    name: "尼泊尔语",
+  },
+  nl: {
+    code: "nl",
+    name: "荷兰语",
+  },
+  pl: {
+    code: "pl",
+    name: "波兰语",
+  },
+  ["pt-PT"]: {
+    code: "pt-PT",
+    name: "葡萄牙语",
+  },
+  ru: {
+    code: "ru",
+    name: "俄罗斯语",
+  },
+  tl: {
+    code: "tl",
+    name: "菲律宾语",
+  },
+  tr: {
+    code: "tr",
+    name: "土耳其语",
+  },
+  vi: {
+    code: "vi",
+    name: "越南语",
+  },
+  uk: {
+    code: "uk",
+    name: "乌克兰语",
+  },
+  ["ur-PK"]: {
+    code: "ur-PK",
+    name: "乌尔都语",
+  },
+};

package/src/lib/utils.ts

@@ -0,0 +1,74 @@
+import path from "path";
+import fs from "fs";
+import { ICreateJsonFileParams } from "../types";
+
+/**
+ * 不存在的文件夹则创建
+ * @param {string} path
+ */
+export const notExistsToCreateFile = (path: string) => {
+  if (fs.existsSync(path)) return;
+  fs.mkdirSync(path);
+};
+
+/**
+ * 获取json文件 并返回 [[key,value],[key,value]...]]
+ * @param {*} path
+ * @returns
+ */
+export const readJsonFileSync = async (path: string) => {
+  try {
+    if (!fs.existsSync(path)) return {};
+    const jsonStr = await fs.readFileSync(path, "utf8");
+    // 将JSON字符串解析为对象
+    return JSON.parse(jsonStr);
+  } catch (error) {
+    console.error("解析JSON时出错:", error);
+    return {};
+  }
+};
+
+/**
+ * 创建json文件
+ * @param {string} fileName 文件名
+ * @param {string} folderName  文件夹名
+ * @param {string} language 语言环境
+ * @param {object} jsonData 文件数据
+ */
+export const createJsonFile = (params: ICreateJsonFileParams) => {
+  const { fileName, folderName, jsonData, language } = params;
+  notExistsToCreateFile(folderName);
+  notExistsToCreateFile(path.resolve(`${folderName}/${language}`));
+  fs.writeFileSync(
+    path.resolve(`${folderName}/${language}/${fileName}`),
+    JSON.stringify(jsonData, null, 2),
+    "utf8"
+  );
+};
+
+export const getRandomNumber = (min: number, max: number) => {
+  return Math.floor(Math.random() * (max - min + 1)) + min;
+};
+
+export const isDirectoryPath = (path: string) => {
+  if (!fs.existsSync(path)) return false;
+  return fs.statSync(path).isDirectory();
+};
+
+export const readFileOfDirSync = (dirPath: string) => {
+  if (!isDirectoryPath(dirPath)) return [];
+  const files = fs.readdirSync(dirPath);
+  // 筛选出所有文件夹
+  return files.filter((file) => path.extname(file) === ".json");
+};
+
+export function chunkArray<T extends object>(array: T[], chunkSize: number) {
+  const result = [];
+  let index = 0;
+
+  while (index < array.length) {
+    result.push(array.slice(index, index + chunkSize));
+    index += chunkSize;
+  }
+  return result;
+}