@agentscope-ai/i18n 1.0.1 → 1.0.2

package/lib/core/checker.js

@@ -0,0 +1,544 @@
+const fs = require('fs-extra');
+const find = require('find');
+const { parse } = require('@babel/parser');
+const traverse = require('@babel/traverse').default;
+const generate = require('@babel/generator').default;
+const {
+  extractI18nKey,
+  chnRegExp,
+  onlyContainsChinesePunctuation,
+  onlyContainsEmoji,
+  isI18nCall,
+  isInConsoleCall,
+} = require('../utils/ast-utils');
+const { askQuestion } = require('../utils/cli-utils');
+const {
+  shouldIgnoreLineByComment,
+  shouldIgnoreBlockByComment,
+  shouldIgnoreByComment,
+  shouldIgnoreFile,
+} = require('../utils/file-utils');
+const { generateMedusaExcel } = require('../utils/excel-utils');
+
+const parseAST = (code) => {
+  return parse(code, {
+    sourceType: 'module',
+    plugins: ['jsx', 'typescript'],
+  });
+};
+
+const scanFilesAndCollectKeys = (files, callExpr) => {
+  const usedKeys = new Set();
+  const usedKeysData = {};
+
+  files.forEach((file) => {
+    try {
+      const code = fs.readFileSync(file, 'utf8');
+      const ast = parseAST(code);
+
+      traverse(ast, {
+        CallExpression(path) {
+          const result = extractI18nKey(path.node, callExpr);
+          if (result && result.id) {
+            usedKeys.add(result.id);
+            if (result.dm && typeof result.dm === 'string') {
+              usedKeysData[result.id] = result.dm;
+            }
+          }
+        },
+      });
+    } catch (error) {
+      console.error(`    # 处理文件失败: ${file}`, error);
+      if (error.loc) {
+        console.error(`    # 错误位置: 第${error.loc.line}行, 第${error.loc.column}列`);
+      }
+    }
+  });
+
+  return { usedKeys, usedKeysData };
+};
+
+const scanUntranslatedTexts = (files, doNotTranslateFiles, callExpr) => {
+  const untranslatedTexts = [];
+
+  const filteredFiles = files.filter((file) => {
+    if (shouldIgnoreFile(file, doNotTranslateFiles || [])) return false;
+    if (shouldIgnoreByComment(file)) return false;
+    return true;
+  });
+
+  filteredFiles.forEach((file) => {
+    try {
+      const code = fs.readFileSync(file, 'utf8');
+      const ast = parseAST(code);
+
+      const shouldIgnoreNode = (nodePath) => {
+        if (!nodePath.node.loc) return false;
+        const lineNumber = nodePath.node.loc.start.line;
+        return (
+          shouldIgnoreLineByComment(file, lineNumber) ||
+          shouldIgnoreBlockByComment(file, lineNumber)
+        );
+      };
+
+      traverse(ast, {
+        StringLiteral(astPath) {
+          const { value } = astPath.node;
+          if (
+            chnRegExp.test(value) &&
+            !isI18nCall(astPath.parent, callExpr) &&
+            !astPath.findParent((p) => isI18nCall(p.node, callExpr)) &&
+            !isInConsoleCall(astPath) &&
+            !shouldIgnoreNode(astPath) &&
+            !onlyContainsChinesePunctuation(value) &&
+            !onlyContainsEmoji(value)
+          ) {
+            untranslatedTexts.push({
+              file,
+              line: astPath.node.loc?.start.line,
+              column: astPath.node.loc?.start.column,
+              type: 'StringLiteral',
+              text: value,
+            });
+          }
+        },
+        TemplateLiteral(astPath) {
+          if (
+            !isI18nCall(astPath.parent, callExpr) &&
+            !astPath.findParent((p) => isI18nCall(p.node, callExpr)) &&
+            !isInConsoleCall(astPath) &&
+            !shouldIgnoreNode(astPath)
+          ) {
+            const staticParts = astPath.node.quasis.map((q) => q.value.raw).join('');
+            if (
+              chnRegExp.test(staticParts) &&
+              !onlyContainsChinesePunctuation(staticParts) &&
+              !onlyContainsEmoji(staticParts)
+            ) {
+              const fullText = astPath.node.quasis
+                .map((q, i) => {
+                  const expr = astPath.node.expressions[i];
+                  const exprStr = expr ? `\${${generate(expr, { compact: true }).code}}` : '';
+                  return q.value.raw + exprStr;
+                })
+                .join('');
+              untranslatedTexts.push({
+                file,
+                line: astPath.node.loc?.start.line,
+                column: astPath.node.loc?.start.column,
+                type: 'TemplateLiteral',
+                text: fullText,
+              });
+            }
+          }
+        },
+        JSXText(astPath) {
+          const { value } = astPath.node;
+          if (
+            chnRegExp.test(value) &&
+            !isI18nCall(astPath.parent, callExpr) &&
+            !astPath.findParent((p) => isI18nCall(p.node, callExpr)) &&
+            !isInConsoleCall(astPath) &&
+            !shouldIgnoreNode(astPath) &&
+            !onlyContainsChinesePunctuation(value) &&
+            !onlyContainsEmoji(value)
+          ) {
+            const trimmedValue = value.replace(/^\s+|\s+$/g, '');
+            if (trimmedValue) {
+              untranslatedTexts.push({
+                file,
+                line: astPath.node.loc?.start.line,
+                column: astPath.node.loc?.start.column,
+                type: 'JSXText',
+                text: trimmedValue,
+              });
+            }
+          }
+        },
+      });
+    } catch (error) {
+      console.error(`    # 扫描未翻译文案失败: ${file}`, error.message);
+    }
+  });
+
+  return untranslatedTexts;
+};
+
+const readTranslationFiles = (localesFilePath, dashScope, checkLanguages) => {
+  let zhCN = {};
+  try {
+    zhCN = fs.readJsonSync(`${localesFilePath}/zh-cn.json`);
+  } catch (error) {
+    console.log('    未找到zh-cn.json文件');
+  }
+
+  let languages = [];
+  if (Array.isArray(checkLanguages) && checkLanguages.length > 0) {
+    languages = checkLanguages.filter((lang) => lang !== 'zh-cn');
+  } else if (dashScope && dashScope.valueTranslateAppId) {
+    if (typeof dashScope.valueTranslateAppId === 'string') {
+      languages = ['en-us'];
+    } else if (typeof dashScope.valueTranslateAppId === 'object') {
+      languages = Object.keys(dashScope.valueTranslateAppId);
+    }
+  }
+
+  const languageFiles = {};
+  for (const language of languages) {
+    try {
+      languageFiles[language] = fs.readJsonSync(`${localesFilePath}/${language}.json`);
+    } catch (error) {
+      console.log(`    未找到${language}.json文件`);
+    }
+  }
+
+  return { zhCN, languageFiles };
+};
+
+const analyzeMissingAndUnusedKeys = (usedKeys, zhCN, languageFiles) => {
+  const zhCNKeys = new Set(Object.keys(zhCN));
+  const languageKeys = {};
+  for (const [language, content] of Object.entries(languageFiles)) {
+    languageKeys[language] = new Set(Object.keys(content));
+  }
+
+  const missingKeys = {
+    zhCN: Array.from(usedKeys).filter((key) => !zhCNKeys.has(key)),
+  };
+  for (const [language, keys] of Object.entries(languageKeys)) {
+    missingKeys[language] = Array.from(usedKeys).filter((key) => !keys.has(key));
+  }
+
+  const unusedKeys = {
+    zhCN: Array.from(zhCNKeys).filter((key) => !usedKeys.has(key)),
+  };
+  for (const [language, keys] of Object.entries(languageKeys)) {
+    unusedKeys[language] = Array.from(keys).filter((key) => !usedKeys.has(key));
+  }
+
+  return { missingKeys, unusedKeys };
+};
+
+const generateCheckOutput = (missingKeys, unusedKeys, untranslatedTexts) => {
+  const outputLines = [];
+  outputLines.push('=== 检查结果 ===');
+  outputLines.push('');
+  outputLines.push('1. 缺失的翻译key:');
+
+  if (missingKeys.zhCN.length > 0) {
+    outputLines.push('');
+    outputLines.push('zh-cn.json 缺失的key:');
+    missingKeys.zhCN.forEach((key) => outputLines.push(`    ${key}`));
+  } else {
+    outputLines.push('');
+    outputLines.push('zh-cn.json 没有缺失的key');
+  }
+
+  for (const [language, missing] of Object.entries(missingKeys)) {
+    if (language === 'zhCN') continue;
+    if (missing.length > 0) {
+      outputLines.push('');
+      outputLines.push(`${language}.json 缺失的key:`);
+      missing.forEach((key) => outputLines.push(`    ${key}`));
+    } else {
+      outputLines.push('');
+      outputLines.push(`${language}.json 没有缺失的key`);
+    }
+  }
+
+  outputLines.push('');
+  outputLines.push('2. 未使用的翻译key:');
+
+  if (unusedKeys.zhCN.length > 0) {
+    outputLines.push('');
+    outputLines.push('zh-cn.json 中未使用的key:');
+    unusedKeys.zhCN.forEach((key) => outputLines.push(`    ${key}`));
+  } else {
+    outputLines.push('');
+    outputLines.push('zh-cn.json 没有未使用的key');
+  }
+
+  for (const [language, unused] of Object.entries(unusedKeys)) {
+    if (language === 'zhCN') continue;
+    if (unused.length > 0) {
+      outputLines.push('');
+      outputLines.push(`${language}.json 中未使用的key:`);
+      unused.forEach((key) => outputLines.push(`    ${key}`));
+    } else {
+      outputLines.push('');
+      outputLines.push(`${language}.json 没有未使用的key`);
+    }
+  }
+
+  outputLines.push('');
+  outputLines.push('3. 未翻译的中文文案:');
+
+  if (untranslatedTexts.length > 0) {
+    outputLines.push('');
+    outputLines.push(`共发现 ${untranslatedTexts.length} 条未翻译的中文文案:`);
+    untranslatedTexts.forEach((item) => {
+      outputLines.push(`    ${item.file}:${item.line}:${item.column}  "${item.text}"`);
+    });
+  } else {
+    outputLines.push('');
+    outputLines.push('没有发现未翻译的中文文案');
+  }
+
+  outputLines.push('');
+  outputLines.push('=== 检查完成 ===');
+
+  return outputLines;
+};
+
+const handleOutputAndJsonGeneration = (
+  outputLines,
+  missingKeys,
+  unusedKeys,
+  untranslatedTexts,
+  autoDeleteUnused,
+) => {
+  if (outputLines.length > 100 && !autoDeleteUnused) {
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+    const jsonFileName = `check-result-${timestamp}.json`;
+
+    const jsonResult = {
+      timestamp: new Date().toISOString(),
+      summary: {
+        totalMissingKeys: Object.values(missingKeys).reduce((sum, keys) => sum + keys.length, 0),
+        totalUnusedKeys: Object.values(unusedKeys).reduce((sum, keys) => sum + keys.length, 0),
+        totalUntranslatedTexts: untranslatedTexts.length,
+        totalOutputLines: outputLines.length,
+      },
+      missingKeys,
+      unusedKeys,
+      untranslatedTexts,
+    };
+
+    try {
+      fs.writeJsonSync(jsonFileName, jsonResult, { spaces: 2 });
+
+      console.log('\n=== 检查结果 ===');
+      console.log(`\n⚠️  输出内容较多(${outputLines.length}行)，详细结果已保存到: ${jsonFileName}`);
+      console.log('\n📊 概要信息:');
+      console.log(`   • 缺失的翻译key总数: ${jsonResult.summary.totalMissingKeys}`);
+      console.log(`   • 未使用的翻译key总数: ${jsonResult.summary.totalUnusedKeys}`);
+      console.log(`   • 未翻译的中文文案总数: ${jsonResult.summary.totalUntranslatedTexts}`);
+
+      console.log('\n📋 各语言统计:');
+      console.log('   缺失的key:');
+      for (const [language, missing] of Object.entries(missingKeys)) {
+        const displayLang = language === 'zhCN' ? 'zh-cn' : language;
+        console.log(`     ${displayLang}.json: ${missing.length}个`);
+      }
+
+      console.log('   未使用的key:');
+      for (const [language, unused] of Object.entries(unusedKeys)) {
+        const displayLang = language === 'zhCN' ? 'zh-cn' : language;
+        console.log(`     ${displayLang}.json: ${unused.length}个`);
+      }
+
+      if (untranslatedTexts.length > 0) {
+        console.log(`   未翻译的中文文案: ${untranslatedTexts.length}条`);
+      }
+
+      console.log('\n=== 检查完成 ===\n');
+    } catch (error) {
+      console.error(`\n❌ 生成JSON文件失败: ${error.message}`);
+      if (!autoDeleteUnused) {
+        outputLines.forEach((line) => console.log(line));
+      }
+    }
+  } else {
+    if (!autoDeleteUnused) {
+      console.log('');
+      outputLines.forEach((line) => console.log(line));
+      console.log('');
+    }
+  }
+};
+
+const handleUnusedKeysDeletion = async (
+  unusedKeys,
+  zhCN,
+  languageFiles,
+  localesFilePath,
+  autoDeleteUnused,
+) => {
+  const hasUnusedKeys = Object.values(unusedKeys).some((keys) => keys.length > 0);
+  if (!hasUnusedKeys) return;
+
+  let shouldDelete = false;
+
+  if (autoDeleteUnused) {
+    shouldDelete = true;
+    console.log('    # 自动删除未使用的key...');
+  } else {
+    const answer = await askQuestion('是否删除未使用的key？(y/n): ');
+    shouldDelete = answer === 'y' || answer === 'yes';
+  }
+
+  if (shouldDelete) {
+    if (unusedKeys.zhCN.length > 0) {
+      unusedKeys.zhCN.forEach((key) => {
+        delete zhCN[key];
+      });
+      fs.writeJsonSync(`${localesFilePath}/zh-cn.json`, zhCN, { spaces: 2 });
+      console.log(`    # 已从zh-cn.json中删除${unusedKeys.zhCN.length}个未使用的key`);
+    }
+
+    for (const [language, unused] of Object.entries(unusedKeys)) {
+      if (language === 'zhCN') continue;
+      if (unused.length > 0) {
+        unused.forEach((key) => {
+          delete languageFiles[language][key];
+        });
+        fs.writeJsonSync(`${localesFilePath}/${language}.json`, languageFiles[language], {
+          spaces: 2,
+        });
+        console.log(`    # 已从${language}.json中删除${unused.length}个未使用的key`);
+      }
+    }
+  } else {
+    console.log('    # 已取消删除操作');
+  }
+};
+
+const handleExcelGeneration = async (missingKeys, usedKeysData, zhCN, params, autoDeleteUnused) => {
+  const hasAnyMissingKeys = Object.values(missingKeys).some((keys) => keys.length > 0);
+
+  if (!hasAnyMissingKeys || !params.medusa || autoDeleteUnused) {
+    if (hasAnyMissingKeys && !params.medusa && !autoDeleteUnused) {
+      console.log('\n=== 提示 ===');
+      console.log('    [提示] 检测到丢失的翻译key，但未配置medusa，无法生成Excel文件');
+      console.log('    [提示] 请在配置文件中添加medusa配置以启用Excel生成功能');
+    }
+    return;
+  }
+
+  console.log('\n=== Excel生成选项 ===');
+  const answer = await askQuestion('检测到丢失的翻译key，是否生成Excel文件用于翻译？(y/n): ');
+
+  if (answer !== 'y' && answer !== 'yes') {
+    console.log('    [取消] 已取消Excel文件生成');
+    return;
+  }
+
+  try {
+    console.log('\n各语言缺失的翻译key数量：');
+    const languageOptions = [];
+
+    for (const [language, keys] of Object.entries(missingKeys)) {
+      if (keys.length > 0) {
+        const languageName = language === 'zhCN' ? 'zh-cn' : language;
+        console.log(`  ${languageName}: ${keys.length}个缺失key`);
+        languageOptions.push({ code: languageName, keys: keys, originalCode: language });
+      }
+    }
+
+    if (languageOptions.length === 0) {
+      console.log('  没有缺失的key');
+      return;
+    }
+
+    let selectedLanguage;
+
+    try {
+      const inquirer = await import('inquirer');
+
+      const choices = languageOptions.map((option) => ({
+        name: `${option.code} (${option.keys.length}个缺失key)`,
+        value: option,
+      }));
+
+      const result = await inquirer.default.prompt([
+        {
+          type: 'list',
+          name: 'selectedLanguage',
+          message: '请选择要生成Excel的语言（使用上下键选择，回车确认）：',
+          choices: choices,
+          pageSize: 10,
+        },
+      ]);
+
+      selectedLanguage = result.selectedLanguage;
+    } catch (error) {
+      if (error.isTtyError) {
+        console.log('\n当前环境不支持交互式选择，使用数字输入模式：');
+        languageOptions.forEach((option, index) => {
+          console.log(`  ${index + 1}. ${option.code} (${option.keys.length}个缺失key)`);
+        });
+
+        const languageAnswer = await askQuestion(
+          `请输入数字选择语言 (1-${languageOptions.length}): `,
+        );
+        const selectedIndex = parseInt(languageAnswer) - 1;
+
+        if (selectedIndex < 0 || selectedIndex >= languageOptions.length) {
+          console.log('    [取消] 选择无效，已取消Excel文件生成');
+          return;
+        }
+
+        selectedLanguage = languageOptions[selectedIndex];
+      } else {
+        throw error;
+      }
+    }
+
+    const missingKeysArray = selectedLanguage.keys;
+
+    console.log(
+      `\n已选择语言: ${selectedLanguage.code}，将为${missingKeysArray.length}个缺失key生成Excel`,
+    );
+
+    if (missingKeysArray.length > 0) {
+      const customData = {};
+      console.log(`    [信息] 准备为${missingKeysArray.length}个缺失的key生成Excel`);
+
+      for (const key of missingKeysArray) {
+        customData[key] = {};
+
+        for (const [languageFile] of Object.entries(params.medusa.keyMap)) {
+          if (languageFile === 'zh-cn') {
+            if (selectedLanguage.originalCode === 'zhCN') {
+              const dmValue = usedKeysData[key] || '';
+              customData[key][languageFile] = dmValue;
+            } else {
+              const existingZhValue = zhCN[key] || usedKeysData[key] || '';
+              customData[key][languageFile] = existingZhValue;
+            }
+          } else {
+            customData[key][languageFile] = '';
+          }
+        }
+      }
+
+      const excelPath = await generateMedusaExcel({
+        localesFilePath: params.localesFilePath,
+        medusa: params.medusa,
+        newKeys: missingKeysArray,
+        customData,
+      });
+
+      console.log(`    [完成] Excel文件已生成: ${excelPath}`);
+      console.log('    [提示] 请在Excel中完成翻译后，使用相应的导入功能更新翻译文件');
+    } else {
+      console.log('    [信息] 没有需要生成Excel的丢失key');
+    }
+  } catch (error) {
+    console.error(`    [错误] 生成Excel文件失败: ${error.message}`);
+    if (error.stack) {
+      console.error(error.stack);
+    }
+  }
+};
+
+module.exports = {
+  scanFilesAndCollectKeys,
+  scanUntranslatedTexts,
+  readTranslationFiles,
+  analyzeMissingAndUnusedKeys,
+  generateCheckOutput,
+  handleOutputAndJsonGeneration,
+  handleUnusedKeysDeletion,
+  handleExcelGeneration,
+};