@agentscope-ai/i18n 0.1.2

package/lib/utils/translation-utils.js

@@ -0,0 +1,572 @@
+const { callDashScope } = require('../services/dashscope');
+const { callDashScopeMTBatch } = require('../services/dashscopeMT');
+
+// 生成带长度限制的key
+const generateKeyWithLengthLimit = (translatedText, originalKey) => {
+  const MAX_KEY_LENGTH = 256;
+
+  // 将翻译结果转换为适合的key格式
+  let cleanKey = translatedText
+    .toLowerCase()
+    .replace(/[^a-z0-9\s]/g, '') // 移除特殊字符
+    .trim()
+    .split(/\s+/)
+    .filter((word) => word.length > 0)
+    .map((word, index) => (index === 0 ? word : word.charAt(0).toUpperCase() + word.slice(1)))
+    .join('');
+
+  // 替换原key的下标部分，保留前缀和文件路径
+  const keyParts = originalKey.split('.');
+  let newKey;
+
+  if (keyParts.length > 0) {
+    // 替换最后一部分（下标）为翻译后的key
+    keyParts[keyParts.length - 1] = cleanKey;
+    newKey = keyParts.join('.');
+  } else {
+    // 如果key格式不符合预期，直接使用翻译结果
+    newKey = cleanKey;
+  }
+
+  // 检查长度是否超过限制
+  if (newKey.length <= MAX_KEY_LENGTH) {
+    return { newKey, isShortened: false };
+  }
+
+  // 长度超过限制，开始缩短处理
+  console.log(`    [警告] key长度超过${MAX_KEY_LENGTH}字符 (${newKey.length}字符)，开始缩短处理`);
+
+  // 方法1：cleanKey转化时只取翻译文本的后七个单词
+  const words = translatedText
+    .toLowerCase()
+    .replace(/[^a-z0-9\s]/g, '')
+    .trim()
+    .split(/\s+/)
+    .filter((word) => word.length > 0);
+
+  const lastSevenWords = words.slice(-7); // 取后七个单词
+  const shortenedCleanKey = lastSevenWords
+    .map((word, index) => (index === 0 ? word : word.charAt(0).toUpperCase() + word.slice(1)))
+    .join('');
+
+  // 重新构建key
+  const originalKeyParts = originalKey.split('.');
+  if (originalKeyParts.length > 0) {
+    originalKeyParts[originalKeyParts.length - 1] = shortenedCleanKey;
+    newKey = originalKeyParts.join('.');
+  } else {
+    newKey = shortenedCleanKey;
+  }
+
+  // 如果还是超过长度，使用方法2：缩短originalKey路径
+  if (newKey.length > MAX_KEY_LENGTH) {
+    console.log(`    [警告] 方法1后仍超过长度限制，使用方法2缩短路径`);
+
+    const parts = originalKey.split('.');
+    if (parts.length > 7) {
+      // 只保留第一个索引和最后六个索引
+      const shortenedParts = [parts[0], ...parts.slice(-6)];
+      shortenedParts[shortenedParts.length - 1] = shortenedCleanKey;
+      newKey = shortenedParts.join('.');
+    } else {
+      // 如果原key部分数量不足7个，保持原有逻辑
+      parts[parts.length - 1] = shortenedCleanKey;
+      newKey = parts.join('.');
+    }
+  }
+
+  // 如果仍然超过长度，进行最终截断
+  if (newKey.length > MAX_KEY_LENGTH) {
+    console.log(`    [警告] 缩短后仍超过长度限制，进行截断处理`);
+    newKey = newKey.substring(0, MAX_KEY_LENGTH);
+
+    // 确保截断后不以'.'结尾
+    if (newKey.endsWith('.')) {
+      newKey = newKey.substring(0, newKey.length - 1);
+    }
+  }
+
+  return { newKey, isShortened: true };
+};
+const fs = require('fs-extra');
+const path = require('path');
+
+// 日志工具函数
+const logTranslationError = (error, context) => {
+  const logDir = path.join(process.cwd(), 'logs');
+  const logFile = path.join(logDir, 'i18n-translation-errors.log');
+
+  // 确保日志目录存在
+  fs.ensureDirSync(logDir);
+
+  const timestamp = new Date().toISOString();
+  const logEntry = {
+    timestamp,
+    error: {
+      message: error.message,
+      stack: error.stack,
+      ...(error.response && {
+        response: {
+          status: error.response.status,
+          statusText: error.response.statusText,
+          data: error.response.data,
+          headers: error.response.headers,
+        },
+      }),
+    },
+    context,
+  };
+
+  // 追加到日志文件
+  const logLine = JSON.stringify(logEntry, null, 2) + '\n' + '-'.repeat(80) + '\n';
+  fs.appendFileSync(logFile, logLine);
+
+  console.error(`    [日志] 翻译错误已记录到: ${logFile}`);
+};
+
+// 统一的批处理翻译函数，支持重试机制
+const batchTranslateWithRetry = async (content, tokenName, appId, apiKey, retryCount = 3) => {
+  for (let i = 0; i < retryCount; i++) {
+    try {
+      const translatedText = await callDashScope(
+        JSON.stringify(content, null, 2),
+        tokenName,
+        appId,
+        apiKey,
+      );
+      const translatedJson = extractJsonContent(translatedText);
+      if (translatedJson) {
+        return translatedJson;
+      }
+      throw new Error('翻译结果解析失败');
+    } catch (error) {
+      // 记录翻译失败信息
+      const context = {
+        attempt: i + 1,
+        totalAttempts: retryCount,
+        tokenName,
+        appId,
+        contentSize: JSON.stringify(content).length,
+        timestamp: new Date().toISOString(),
+      };
+
+      logTranslationError(error, context);
+
+      if (i < retryCount - 1) {
+        console.log(`    [重试] 第${i + 1}次翻译失败，10秒后重试...`);
+        console.error(`    [错误] ${error.message}`);
+        await new Promise((resolve) => setTimeout(resolve, 10000));
+      } else {
+        throw error;
+      }
+    }
+  }
+};
+
+// 分批处理key翻译，每批100个
+const batchKeyTranslate = async (i18n, dashScope) => {
+  const entries = Object.entries(i18n);
+  const batchSize = dashScope.batchSize || 100;
+  const batches = [];
+
+  // 将内容分成每批100个
+  for (let i = 0; i < entries.length; i += batchSize) {
+    batches.push(entries.slice(i, i + batchSize));
+  }
+
+  console.log(`    [进度] 共${batches.length}批需要翻译`);
+  const keyMap = {};
+  const zhCN = {};
+
+  // 逐批处理
+  for (let i = 0; i < batches.length; i++) {
+    const batch = batches[i];
+    const batchContent = {
+      type: 'i18n',
+      content: Object.fromEntries(batch),
+    };
+
+    console.log(`    [进度] 正在翻译第${i + 1}/${batches.length}批...`);
+    const translatedJson = await batchTranslateWithRetry(
+      batchContent,
+      'i18n',
+      dashScope.keyTranslateAppId,
+      dashScope.apiKey,
+    );
+
+    if (translatedJson.keyMap && translatedJson['zh-cn']) {
+      Object.assign(keyMap, translatedJson.keyMap);
+      Object.assign(zhCN, translatedJson['zh-cn']);
+    }
+    // 每批之间暂停1秒，避免请求过快
+    if (i < batches.length - 1) {
+      await new Promise((resolve) => setTimeout(resolve, 1000));
+    }
+  }
+
+  return { keyMap, zhCN };
+};
+
+// 分批处理value翻译，每批100个，支持多语言
+const batchTranslate = async (i18n, dashScope) => {
+  const entries = Object.entries(i18n);
+  const batchSize = dashScope.batchSize || 100;
+  const batches = [];
+
+  // 将内容分成每批100个
+  for (let i = 0; i < entries.length; i += batchSize) {
+    batches.push(entries.slice(i, i + batchSize));
+  }
+
+  console.log(`    [进度] 共${batches.length}批需要翻译`);
+
+  // 支持多语言翻译，为每种语言创建翻译结果对象
+  const translatedResults = {};
+
+  // 检查valueTranslateAppId配置
+  if (typeof dashScope.valueTranslateAppId === 'string') {
+    // 兼容旧版本配置，只翻译成英文
+    console.log(`    [进度] 检测到单语言配置，翻译为英文...`);
+    const translatedI18n = {};
+
+    for (let i = 0; i < batches.length; i++) {
+      const batch = batches[i];
+      const batchContent = Object.fromEntries(batch);
+
+      console.log(`    [进度] 正在翻译第${i + 1}/${batches.length}批...`);
+      const translatedJson = await batchTranslateWithRetry(
+        batchContent,
+        'i18n',
+        dashScope.valueTranslateAppId,
+        dashScope.apiKey,
+      );
+
+      if (translatedJson) {
+        Object.assign(translatedI18n, translatedJson);
+      }
+
+      // 每批之间暂停1秒，避免请求过快
+      if (i < batches.length - 1) {
+        await new Promise((resolve) => setTimeout(resolve, 1000));
+      }
+    }
+
+    translatedResults['en-us'] = translatedI18n;
+  } else if (typeof dashScope.valueTranslateAppId === 'object') {
+    // 新版本配置，支持多语言翻译
+    const languages = Object.keys(dashScope.valueTranslateAppId);
+    console.log(`    [进度] 检测到多语言配置，将翻译为: ${languages.join(', ')}`);
+
+    // 为每种语言进行翻译
+    for (const language of languages) {
+      const appId = dashScope.valueTranslateAppId[language];
+      console.log(`    [进度] 开始翻译为 ${language}...`);
+
+      const translatedI18n = {};
+
+      for (let i = 0; i < batches.length; i++) {
+        const batch = batches[i];
+        const batchContent = Object.fromEntries(batch);
+
+        console.log(`    [进度] ${language}: 正在翻译第${i + 1}/${batches.length}批...`);
+        const translatedJson = await batchTranslateWithRetry(
+          batchContent,
+          'i18n',
+          appId,
+          dashScope.apiKey,
+        );
+
+        if (translatedJson) {
+          Object.assign(translatedI18n, translatedJson);
+        }
+
+        // 每批之间暂停1秒，避免请求过快
+        if (i < batches.length - 1) {
+          await new Promise((resolve) => setTimeout(resolve, 1000));
+        }
+      }
+
+      translatedResults[language] = translatedI18n;
+    }
+  } else {
+    throw new Error('valueTranslateAppId 配置无效，必须是字符串或对象');
+  }
+
+  return translatedResults;
+};
+
+const extractJsonContent = (text, retryCount = 3) => {
+  if (typeof text !== 'string') {
+    const error = new Error('传入的文本不是字符串类型');
+    const context = {
+      type: 'type_error',
+      actualType: typeof text,
+      content: text,
+      timestamp: new Date().toISOString(),
+    };
+    logTranslationError(error, context);
+    console.error('    # 错误：传入的文本不是字符串类型');
+    console.error('    # 实际类型:', typeof text);
+    console.error('    # 内容:', text);
+    return null;
+  }
+
+  // 清理文本中可能导致解析失败的字符
+  const cleanText = text
+    .replace(/[\u0000-\u001F\u007F-\u009F]/g, '') // 移除控制字符
+    .replace(/\n\s*\n/g, '\n') // 移除多余的空行
+    .trim(); // 移除首尾空白
+
+  try {
+    // 首先尝试直接解析为JSON
+    return JSON.parse(cleanText);
+  } catch (e) {
+    try {
+      // 如果直接解析失败，尝试提取```json```中的内容
+      const regex = /```json\n([\s\S]*?)```/;
+      const match = cleanText.match(regex);
+      if (match && match[1]) {
+        return JSON.parse(match[1].trim());
+      }
+    } catch (innerError) {
+      // 记录JSON解析失败
+      const error = new Error(`JSON解析失败: ${innerError.message}`);
+      const context = {
+        type: 'json_parse_error',
+        originalText: text,
+        cleanText: cleanText,
+        innerError: innerError.message,
+        timestamp: new Date().toISOString(),
+      };
+      logTranslationError(error, context);
+
+      console.error('    # JSON解析错误，准备重试翻译');
+      console.error('    # 原始文本:', text);
+      console.error('    # 错误信息:', innerError.message);
+    }
+    return null;
+  }
+};
+
+// 单条翻译key（使用MT方式）
+const singleKeyTranslate = async (i18n, dashScope) => {
+  console.log(`    [进度] 使用MT方式翻译key，共${Object.keys(i18n).length}条`);
+
+  const keyMap = {};
+  const zhCN = {};
+  const entries = Object.entries(i18n);
+  const concurrency = dashScope.mtConcurrency || 1; // 默认并发数为1
+  const delay = dashScope.mtDelay || 100; // 默认延迟100ms
+  const mtBatchSize = dashScope.mtBatchSize || 5; // 默认每次调用翻译5条
+
+  console.log(
+    `    [配置] 并发数: ${concurrency}, 请求间隔: ${delay}ms, 每次调用翻译条数: ${mtBatchSize}`,
+  );
+
+  // 批量翻译key
+  const translateKeyBatch = async (batch, startIndex) => {
+    try {
+      const prompts = batch.map(([, chineseValue]) => chineseValue);
+      console.log(
+        `    [进度] 翻译key批次 ${startIndex + 1}-${startIndex + batch.length}/${entries.length}`,
+      );
+
+      // 调用批量MT翻译
+      const translatedTexts = await callDashScopeMTBatch(
+        prompts,
+        'en-us',
+        dashScope.apiKey,
+        3,
+        'key',
+      );
+
+      const results = [];
+      for (let i = 0; i < batch.length; i++) {
+        const [originalKey, chineseValue] = batch[i];
+        const translatedText = translatedTexts[i];
+
+        if (translatedText && translatedText.trim()) {
+          // 生成新key，并处理长度限制
+          const { newKey, isShortened } = generateKeyWithLengthLimit(translatedText, originalKey);
+
+          if (isShortened) {
+            console.log(`    [长度限制] ${originalKey} -> ${newKey} (已缩短)`);
+          } else {
+            console.log(`    [完成] ${originalKey} -> ${newKey}`);
+          }
+          results.push({ originalKey, newKey, chineseValue });
+        } else {
+          console.warn(`    [警告] key翻译失败，保持原key: ${originalKey}`);
+          results.push({ originalKey, newKey: originalKey, chineseValue });
+        }
+      }
+
+      return results;
+    } catch (error) {
+      console.error(`    [错误] 翻译key批次失败:`, error.message);
+      // 翻译失败时保持原key
+      return batch.map(([originalKey, chineseValue]) => ({
+        originalKey,
+        newKey: originalKey,
+        chineseValue,
+      }));
+    }
+  };
+
+  // 分组处理：先按mtBatchSize分组，再按并发数并行处理
+  for (let i = 0; i < entries.length; i += mtBatchSize * concurrency) {
+    const superBatch = entries.slice(i, i + mtBatchSize * concurrency);
+    const batches = [];
+
+    // 将superBatch分成多个mtBatchSize大小的批次
+    for (let j = 0; j < superBatch.length; j += mtBatchSize) {
+      batches.push(superBatch.slice(j, j + mtBatchSize));
+    }
+
+    // 并发处理这些批次
+    const promises = batches.map((batch, batchIndex) =>
+      translateKeyBatch(batch, i + batchIndex * mtBatchSize),
+    );
+
+    const batchResults = await Promise.all(promises);
+
+    // 收集结果
+    batchResults.forEach((results) => {
+      results.forEach(({ originalKey, newKey, chineseValue }) => {
+        keyMap[originalKey] = newKey;
+        zhCN[newKey] = chineseValue;
+      });
+    });
+
+    // 避免请求过快，超级批次间暂停
+    if (i + mtBatchSize * concurrency < entries.length) {
+      await new Promise((resolve) => setTimeout(resolve, delay));
+    }
+  }
+
+  return { keyMap, zhCN };
+};
+
+// 单条翻译value（使用MT方式）
+const singleTranslate = async (i18n, dashScope) => {
+  console.log(`    [进度] 使用MT方式翻译value，共${Object.keys(i18n).length}条`);
+
+  // 支持多语言翻译，为每种语言创建翻译结果对象
+  const translatedResults = {};
+  const concurrency = dashScope.mtConcurrency || 1; // 默认并发数为1
+  const delay = dashScope.mtDelay || 100; // 默认延迟100ms
+  const mtBatchSize = dashScope.mtBatchSize || 5; // 默认每次调用翻译5条
+
+  console.log(
+    `    [配置] 并发数: ${concurrency}, 请求间隔: ${delay}ms, 每次调用翻译条数: ${mtBatchSize}`,
+  );
+
+  // 检查valueTranslateAppId配置，如果没有配置就使用默认的英文翻译
+  let languages = ['en-us']; // 默认翻译为英文
+
+  if (dashScope.valueTranslateAppId) {
+    if (typeof dashScope.valueTranslateAppId === 'string') {
+      // 兼容旧版本配置，只翻译成英文
+      console.log(`    [进度] 检测到单语言配置，使用MT翻译为英文...`);
+      languages = ['en-us'];
+    } else if (typeof dashScope.valueTranslateAppId === 'object') {
+      // 新版本配置，支持多语言翻译
+      languages = Object.keys(dashScope.valueTranslateAppId);
+      console.log(`    [进度] 检测到多语言配置，使用MT翻译为: ${languages.join(', ')}`);
+    }
+  } else {
+    console.log(`    [进度] 未配置valueTranslateAppId，默认使用MT翻译为英文...`);
+  }
+
+  // 批量翻译value函数
+  const translateValueBatch = async (batch, language, startIndex, total) => {
+    try {
+      const prompts = batch.map(([, chineseValue]) => chineseValue);
+      console.log(
+        `    [进度] ${language}: 翻译批次 ${startIndex + 1}-${startIndex + batch.length}/${total}`,
+      );
+
+      // 调用批量MT翻译
+      const translatedValues = await callDashScopeMTBatch(
+        prompts,
+        language,
+        dashScope.apiKey,
+        3,
+        'value',
+      );
+
+      const results = [];
+      for (let i = 0; i < batch.length; i++) {
+        const [key, chineseValue] = batch[i];
+        const translatedValue = translatedValues[i];
+
+        if (translatedValue && translatedValue.trim()) {
+          console.log(`    [完成] ${chineseValue} -> ${translatedValue.trim()}`);
+          results.push({ key, value: translatedValue.trim() });
+        } else {
+          console.warn(`    [警告] value翻译失败，保持原文: ${chineseValue}`);
+          results.push({ key, value: chineseValue });
+        }
+      }
+
+      return results;
+    } catch (error) {
+      console.error(`    [错误] 翻译value批次失败:`, error.message);
+      // 翻译失败时保持原文
+      return batch.map(([key, chineseValue]) => ({
+        key,
+        value: chineseValue,
+      }));
+    }
+  };
+
+  // 为每种语言进行翻译
+  for (const language of languages) {
+    console.log(`    [进度] 开始MT翻译为 ${language}...`);
+    const translatedI18n = {};
+    const entries = Object.entries(i18n);
+
+    // 分组处理：先按mtBatchSize分组，再按并发数并行处理
+    for (let i = 0; i < entries.length; i += mtBatchSize * concurrency) {
+      const superBatch = entries.slice(i, i + mtBatchSize * concurrency);
+      const batches = [];
+
+      // 将superBatch分成多个mtBatchSize大小的批次
+      for (let j = 0; j < superBatch.length; j += mtBatchSize) {
+        batches.push(superBatch.slice(j, j + mtBatchSize));
+      }
+
+      // 并发处理这些批次
+      const promises = batches.map((batch, batchIndex) =>
+        translateValueBatch(batch, language, i + batchIndex * mtBatchSize, entries.length),
+      );
+
+      const batchResults = await Promise.all(promises);
+
+      // 收集结果
+      batchResults.forEach((results) => {
+        results.forEach(({ key, value }) => {
+          translatedI18n[key] = value;
+        });
+      });
+
+      // 避免请求过快，超级批次间暂停
+      if (i + mtBatchSize * concurrency < entries.length) {
+        await new Promise((resolve) => setTimeout(resolve, delay));
+      }
+    }
+
+    translatedResults[language] = translatedI18n;
+  }
+
+  return translatedResults;
+};
+
+module.exports = {
+  batchTranslateWithRetry,
+  batchKeyTranslate,
+  batchTranslate,
+  extractJsonContent,
+  singleKeyTranslate,
+  singleTranslate,
+};

package/package.json

@@ -0,0 +1,66 @@
+{
+  "name": "@agentscope-ai/i18n",
+  "version": "0.1.2",
+  "description": "A tool for translating Chinese content in frontend code repositories",
+  "keywords": [
+    "i18n",
+    "translation",
+    "chinese",
+    "frontend"
+  ],
+  "bin": {
+    "i18n": "./lib/cli.js"
+  },
+  "files": [
+    "lib/*",
+    "README.md"
+  ],
+  "scripts": {
+    "lint": "eslint .",
+    "format": "prettier --write \"lib/**/*.{ts,tsx}\""
+  },
+  "lint-staged": {
+    "**/*.{js,jsx,html,css,json}": [
+      "npx prettier --write"
+    ]
+  },
+  "author": [],
+  "license": "ISC",
+  "dependencies": {
+    "@babel/generator": "^7.23.6",
+    "@babel/parser": "^7.23.6",
+    "@babel/traverse": "^7.23.6",
+    "@babel/types": "^7.23.6",
+    "axios": "^1.6.2",
+    "commander": "^11.1.0",
+    "dotenv": "^16.3.1",
+    "find": "^0.3.0",
+    "fs-extra": "^11.1.1",
+    "inquirer": "^12.9.3",
+    "lodash-es": "^4.17.21",
+    "node-fetch": "^3.3.2",
+    "xlsx": "^0.18.5"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.23.6",
+    "@babel/preset-env": "^7.23.6",
+    "@babel/preset-react": "^7.23.6",
+    "@babel/preset-typescript": "^7.23.6",
+    "@types/jest": "^29.5.10",
+    "@types/node": "^20.10.4",
+    "@types/react": "^19.1.6",
+    "babel-plugin-module-resolver": "^5.0.0",
+    "eslint": "^8.55.0",
+    "jest": "^29.7.0",
+    "prettier": "^3.1.0",
+    "react": "^19.1.0",
+    "typescript": "^5.3.3"
+  },
+  "engines": {
+    "node": ">=14.0.0"
+  },
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org/",
+    "access": "public"
+  }
+}