i18ntk 3.2.0 → 4.0.0

package/main/i18ntk-complete.js

@@ -161,9 +161,14 @@ class I18nCompletionTool {
     return lowered.startsWith('backup-') || lowered === 'backup' || lowered === 'reports' || lowered === 'i18ntk-reports';
   }
 
-  // Get all JSON files from a language directory
-  getLanguageFiles(language) {
-    const languageDir = path.join(this.sourceDir, language);
+  // Get all JSON files from a language directory
+  getLanguageFiles(language) {
+    const monolithFile = path.join(this.sourceDir, `${language}.json`);
+    if (SecurityUtils.safeExistsSync(monolithFile, this.config.projectRoot)) {
+      return [`${language}.json`];
+    }
+
+    const languageDir = path.join(this.sourceDir, language);
     
     if (!SecurityUtils.safeExistsSync(languageDir, this.config.projectRoot)) {
       return [];
@@ -173,8 +178,20 @@ class I18nCompletionTool {
       .filter(file => {
         return file.endsWith('.json') && 
                !this.config.excludeFiles.includes(file);
-      });
-  }
+      });
+  }
+
+  getLanguageFilePath(language, fileName) {
+    if (fileName === `${language}.json`) {
+      return path.join(this.sourceDir, fileName);
+    }
+
+    return path.join(this.sourceDir, language, fileName);
+  }
+
+  usesMonolithFile(language) {
+    return SecurityUtils.safeExistsSync(path.join(this.sourceDir, `${language}.json`), this.config.projectRoot);
+  }
 
   // Parse key path and determine which file it belongs to
   parseKeyPath(keyPath) {
@@ -238,25 +255,30 @@ class I18nCompletionTool {
   }
 
   // Add missing keys to a language
-  addMissingKeysToLanguage(language, missingKeys, dryRun = false) {
-    const languageDir = path.join(this.sourceDir, language);
-    const changes = [];
+  addMissingKeysToLanguage(language, missingKeys, dryRun = false) {
+    const languageDir = path.join(this.sourceDir, language);
+    const changes = [];
+    const usesMonolith = this.usesMonolithFile(language);
     
     // Group keys by file
     const keysByFile = {};
     
-    missingKeys.forEach(keyPath => {
-      const { file, key } = this.parseKeyPath(keyPath);
-      if (!keysByFile[file]) {
-        keysByFile[file] = [];
-      }
+    missingKeys.forEach(keyPath => {
+      const { file, key } = usesMonolith
+        ? { file: `${language}.json`, key: keyPath }
+        : this.parseKeyPath(keyPath);
+      if (!keysByFile[file]) {
+        keysByFile[file] = [];
+      }
       keysByFile[file].push({ keyPath, key });
     });
     
-    // Process each file
-    for (const [fileName, keys] of Object.entries(keysByFile)) {
-      const filePath = path.join(languageDir, fileName);
-      let fileContent = {};
+    // Process each file
+    for (const [fileName, keys] of Object.entries(keysByFile)) {
+      const filePath = usesMonolith
+        ? path.join(this.sourceDir, fileName)
+        : path.join(languageDir, fileName);
+      let fileContent = {};
       
       // Load existing file or create new
       if (SecurityUtils.safeExistsSync(filePath, this.config.projectRoot)) {
@@ -266,12 +288,12 @@ class I18nCompletionTool {
           console.warn(t("completeTranslations.warning_could_not_parse_filepa", { filePath })); ;
           fileContent = {};
         }
-      } else {
-        // Create directory if it doesn't exist
-        if (!SecurityUtils.safeExistsSync(languageDir, this.config.projectRoot)) {
-          if (!dryRun) {
-            SecurityUtils.safeMkdirSync(languageDir, this.config.projectRoot, { recursive: true });
-          }
+      } else {
+        // Create directory if it doesn't exist
+        if (!usesMonolith && !SecurityUtils.safeExistsSync(languageDir, this.config.projectRoot)) {
+          if (!dryRun) {
+            SecurityUtils.safeMkdirSync(languageDir, this.config.projectRoot, { recursive: true });
+          }
         }
       }
       
@@ -303,19 +325,66 @@ class I18nCompletionTool {
     return changes;
   }
 
-  // Generate appropriate translation value based on key and language
-  generateTranslationValue(keyPath, language) {
-    // Generate value from key path for source language
-    const baseValue = this.generateValueFromKey(keyPath);
-    
-    // For source language, use the generated value
-    if (language === this.config.sourceLanguage) {
-      return baseValue;
-    }
-    
-    // For other languages, use the not translated marker
-    return this.config.notTranslatedMarker || 'NOT_TRANSLATED';
-  }
+  // Generate appropriate translation value based on key and language
+  generateTranslationValue(keyPath, language) {
+    const sourceValue = this.getSourceValueForKeyPath(keyPath);
+    const baseValue = typeof sourceValue === 'string' && sourceValue.trim() !== ''
+      ? sourceValue
+      : this.generateValueFromKey(keyPath);
+    
+    // For source language, use the generated value
+    if (language === this.config.sourceLanguage) {
+      return baseValue;
+    }
+    
+    return `[${language.toUpperCase()}] ${baseValue}`;
+  }
+
+  getNestedValue(obj, keyPath) {
+    const keys = String(keyPath || '').split('.');
+    let current = obj;
+
+    for (const key of keys) {
+      if (!current || typeof current !== 'object' || !(key in current)) {
+        return undefined;
+      }
+      current = current[key];
+    }
+
+    return current;
+  }
+
+  getSourceValueForKeyPath(keyPath) {
+    if (!this.sourceLanguageDir && !this.usesMonolithFile(this.config.sourceLanguage)) {
+      return undefined;
+    }
+
+    const sourceFiles = this.getLanguageFiles(this.config.sourceLanguage);
+    const keyPathStr = String(keyPath || '');
+    const parsed = this.parseKeyPath(keyPathStr);
+
+    for (const fileName of sourceFiles) {
+      const sourceFilePath = this.getLanguageFilePath(this.config.sourceLanguage, fileName);
+      try {
+        const sourceContent = SecurityUtils.safeParseJSON(SecurityUtils.safeReadFileSync(sourceFilePath, this.config.projectRoot, 'utf8'));
+        if (!sourceContent || typeof sourceContent !== 'object') continue;
+
+        const candidates = [keyPathStr];
+        if (fileName === parsed.file) {
+          candidates.push(parsed.key);
+        }
+
+        for (const candidate of candidates) {
+          const value = this.getNestedValue(sourceContent, candidate);
+          if (value !== undefined) return value;
+        }
+      } catch (error) {
+        console.warn(t("complete.couldNotParseSource", { file: sourceFilePath }));
+      }
+    }
+
+    return undefined;
+  }
 
   // Generate a readable value from a key path
   generateValueFromKey(keyPath) {
@@ -351,18 +420,18 @@ class I18nCompletionTool {
   }
 
   // Get missing keys by comparing source language with target languages
-  getMissingKeysFromComparison() {
-    const sourceFiles = this.getLanguageFiles(this.config.sourceLanguage);
-    const missingKeys = [];
-    
-    if (!SecurityUtils.safeExistsSync(this.sourceLanguageDir, this.config.projectRoot)) {
-      console.log(t("complete.sourceLanguageNotFound", { sourceLanguage: this.config.sourceLanguage }));
-      return [];
-    }
-    
-    // Process each file in source language
-    for (const fileName of sourceFiles) {
-      const sourceFilePath = path.join(this.sourceLanguageDir, fileName);
+  getMissingKeysFromComparison() {
+    const sourceFiles = this.getLanguageFiles(this.config.sourceLanguage);
+    const missingKeys = [];
+    
+    if (sourceFiles.length === 0) {
+      console.log(t("complete.sourceLanguageNotFound", { sourceLanguage: this.config.sourceLanguage }));
+      return [];
+    }
+    
+    // Process each file in source language
+    for (const fileName of sourceFiles) {
+      const sourceFilePath = this.getLanguageFilePath(this.config.sourceLanguage, fileName);
       
       try {
         const sourceContent = SecurityUtils.safeParseJSON(SecurityUtils.safeReadFileSync(sourceFilePath, this.config.projectRoot, 'utf8'));
@@ -373,7 +442,9 @@ class I18nCompletionTool {
         for (const language of languages) {
           if (language === this.config.sourceLanguage) continue;
           
-          const targetFilePath = path.join(this.sourceDir, language, fileName);
+          const targetFilePath = fileName === `${this.config.sourceLanguage}.json` || this.usesMonolithFile(language)
+            ? path.join(this.sourceDir, `${language}.json`)
+            : path.join(this.sourceDir, language, fileName);
           let targetKeys = [];
           
           if (SecurityUtils.safeExistsSync(targetFilePath, this.config.projectRoot)) {

package/main/i18ntk-scanner.js

@@ -109,49 +109,64 @@ class I18nTextScanner {
     const args = process.argv.slice(2);
     const parsed = {};
 
-    args.forEach(arg => {
-      if (arg.startsWith('--')) {
-        const [key, ...valueParts] = arg.substring(2).split('=');
-        const value = valueParts.join('=');
-
-        switch (key) {
-          case 'source-dir':
-            parsed.sourceDir = value || '';
-            break;
-          case 'framework':
-            parsed.framework = value || '';
-            break;
-          case 'patterns':
-            parsed.patterns = value ? value.split(',').map(p => p.trim()).filter(Boolean) : [];
-            break;
-          case 'exclude':
-            parsed.exclude = value ? value.split(',').map(e => e.trim()).filter(Boolean) : [];
-            break;
-          case 'output-dir':
-            parsed.outputDir = value || '';
-            break;
-          case 'min-length':
-            parsed.minLength = parseInt(value) || 3;
-            break;
-          case 'max-length':
-            parsed.maxLength = parseInt(value) || 100;
-            break;
-          case 'output-report':
-            parsed.outputReport = true;
+    for (let i = 0; i < args.length; i++) {
+      const arg = args[i];
+      if (arg.startsWith('--')) {
+        const [key, ...valueParts] = arg.substring(2).split('=');
+        let value = valueParts.join('=');
+        if (!value && args[i + 1] && !args[i + 1].startsWith('--')) {
+          value = args[i + 1];
+        }
+
+        switch (key) {
+          case 'source-dir':
+            parsed.sourceDir = value || '';
+            if (value === args[i + 1]) i++;
+            break;
+          case 'framework':
+            parsed.framework = value || '';
+            if (value === args[i + 1]) i++;
+            break;
+          case 'patterns':
+            parsed.patterns = value ? value.split(',').map(p => p.trim()).filter(Boolean) : [];
+            if (value === args[i + 1]) i++;
+            break;
+          case 'exclude':
+            parsed.exclude = value ? value.split(',').map(e => e.trim()).filter(Boolean) : [];
+            if (value === args[i + 1]) i++;
+            break;
+          case 'output-dir':
+            parsed.outputDir = value || '';
+            if (value === args[i + 1]) i++;
+            break;
+          case 'min-length':
+            parsed.minLength = parseInt(value) || 3;
+            if (value === args[i + 1]) i++;
+            break;
+          case 'max-length':
+            parsed.maxLength = parseInt(value) || 100;
+            if (value === args[i + 1]) i++;
+            break;
+          case 'output-report':
+            parsed.outputReport = true;
             break;
           case 'include-tests':
-            parsed.includeTests = true;
-            break;
-          case 'help':
-          case 'h':
-            parsed.help = true;
-            break;
-        }
-      }
-    });
-
-    return parsed;
-  }
+            parsed.includeTests = true;
+            break;
+          case 'source-language':
+            parsed.sourceLanguage = value || '';
+            if (value === args[i + 1]) i++;
+            break;
+          case 'help':
+          case 'h':
+            parsed.help = true;
+            break;
+        }
+      }
+    }
+
+    return parsed;
+  }
 
   detectFramework(projectRoot) {
     const packagePath = path.join(projectRoot, 'package.json');
@@ -293,43 +308,148 @@ class I18nTextScanner {
   }
 
   isEnglishText(text) {
-    // Enhanced text detection for Unicode and multilingual support
     const trimmed = text.trim();
     if (trimmed.length < 3) return false;
 
-    // Skip if it's just numbers or special characters
     if (/^\d+$/.test(trimmed)) return false;
     if (/^[!@#$%^&*()_+\-=\[\]{};':"\\|,.<>?]+$/.test(trimmed)) return false;
 
-    // Allow Unicode characters including CJK, Cyrillic, etc.
     const validChars = trimmed.match(/[\p{L}\p{N}\s\-,.!?':"()\[\]{}]/gu) || [];
     const validRatio = validChars.length / trimmed.length;
 
-    // Must have at least 50% valid characters and some alphabetic characters
     const hasAlpha = /[a-zA-Z\u00C0-\u024F\u1E00-\u1EFF\u0400-\u04FF\u4E00-\u9FFF\uAC00-\uD7AF]/u.test(trimmed);
 
     return validRatio >= 0.5 && hasAlpha;
   }
 
+  getLanguageProfile(langCode) {
+    const profiles = {
+      en: {
+        name: 'English',
+        charRegex: /[a-zA-Z\u00C0-\u024F]/u,
+        stopwords: ['the', 'and', 'for', 'are', 'but', 'not', 'you', 'all', 'can', 'had', 'her', 'was', 'one', 'our', 'out', 'has', 'have', 'from', 'they', 'that', 'with', 'this', 'will', 'your', 'which', 'their', 'them', 'than', 'then', 'been', 'being', 'would', 'should', 'could', 'about', 'after'],
+        minLength: 3,
+        maxLength: 150
+      },
+      de: {
+        name: 'German',
+        charRegex: /[a-zA-Z\u00C0-\u00FF\u0100-\u017F\u00DF\u1E00-\u1EFF]/u,
+        stopwords: ['der', 'die', 'das', 'und', 'ist', 'von', 'mit', 'sich', 'des', 'auf', 'dem', 'nicht', 'ein', 'eine', 'auch', 'als', 'aus', 'bei', 'nach', 'wie', 'oder', 'war', 'hat', 'ich', 'sie', 'einem', 'um', 'am', 'im', 'es'],
+        minLength: 3,
+        maxLength: 180
+      },
+      fr: {
+        name: 'French',
+        charRegex: /[a-zA-Z\u00C0-\u00FF\u0152\u0153]/u,
+        stopwords: ['le', 'la', 'les', 'des', 'est', 'pas', 'que', 'une', 'dans', 'sur', 'plus', 'par', 'pour', 'avec', 'aux', 'ces', 'ses', 'mes', 'tes', 'notre', 'votre', 'leur', 'dont', 'sont', 'comme', 'mais', 'alors', 'peut', 'tout', 'tous', 'fait'],
+        minLength: 3,
+        maxLength: 170
+      },
+      es: {
+        name: 'Spanish',
+        charRegex: /[a-zA-Z\u00C0-\u00FF\u00F1\u00D1]/u,
+        stopwords: ['que', 'los', 'las', 'del', 'como', 'por', 'para', 'con', 'una', 'sus', 'muy', 'más', 'pero', 'este', 'esta', 'hay', 'son', 'eran', 'fue', 'han', 'será', 'está', 'todo', 'otro', 'otra'],
+        minLength: 3,
+        maxLength: 150
+      },
+      ja: {
+        name: 'Japanese',
+        charRegex: /[\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\uFF66-\uFF9F]/u,
+        stopwords: ['の', 'に', 'は', 'を', 'た', 'が', 'で', 'て', 'と', 'し', 'れ', 'さ', 'る', 'す', 'ん', 'な', 'い', 'か', 'ま', 'も', 'こ', 'り', 'ち', 'き', 'ょ', 'う'],
+        minLength: 2,
+        maxLength: 80
+      },
+      zh: {
+        name: 'Chinese',
+        charRegex: /[\u4E00-\u9FFF\u3400-\u4DBF\uF900-\uFAFF]/u,
+        stopwords: ['的', '是', '在', '不', '了', '有', '和', '人', '这', '中', '大', '为', '上', '个', '国', '我', '以', '要', '他', '时', '来', '用', '们', '生', '到', '作', '地'],
+        minLength: 1,
+        maxLength: 50
+      },
+      ru: {
+        name: 'Russian',
+        charRegex: /[\u0400-\u04FF\u0500-\u052F]/u,
+        stopwords: ['и', 'в', 'не', 'на', 'что', 'как', 'по', 'к', 'от', 'это', 'за', 'то', 'для', 'все', 'его', 'она', 'так', 'же', 'но', 'был', 'быть', 'еще', 'уже', 'кто', 'мой', 'ее', 'их', 'из'],
+        minLength: 2,
+        maxLength: 200
+      },
+      ko: {
+        name: 'Korean',
+        charRegex: /[\uAC00-\uD7AF\u1100-\u11FF\u3130-\u318F]/u,
+        stopwords: ['이', '그', '저', '것', '수', '등', '들', '및', '년', '월', '일', '에서', '에게', '으로', '보다', '에게서', '의', '에', '는', '은', '가', '를', '과', '와', '도', '만', '까지', '부터'],
+        minLength: 1,
+        maxLength: 70
+      },
+      ar: {
+        name: 'Arabic',
+        charRegex: /[\u0600-\u06FF\u0750-\u077F\u08A0-\u08FF]/u,
+        stopwords: ['في', 'من', 'على', 'عن', 'مع', 'هو', 'هي', 'كان', 'هذا', 'ذلك', 'بين', 'بعد', 'قبل', 'عند', 'حتى', 'الى', 'او', 'لا', 'ما', 'لم', 'لن', 'كل', 'بعض', 'أي'],
+        minLength: 2,
+        maxLength: 150
+      },
+      hi: {
+        name: 'Hindi',
+        charRegex: /[\u0900-\u097F]/u,
+        stopwords: ['का', 'की', 'के', 'है', 'हैं', 'था', 'थे', 'होगा', 'होगी', 'में', 'से', 'पर', 'को', 'तक', 'और', 'या', 'लेकिन', 'जब', 'तब', 'कि', 'यह', 'वह', 'एक', 'दो'],
+        minLength: 2,
+        maxLength: 160
+      },
+      vanilla: {
+        name: 'Generic Latin',
+        charRegex: /[a-zA-Z\u00C0-\u024F]/u,
+        stopwords: [],
+        minLength: 3,
+        maxLength: 150
+      }
+    };
+    return profiles[langCode] || profiles.en;
+  }
+
+  isTextInLanguage(text, langCode) {
+    const profile = this.getLanguageProfile(langCode);
+    const trimmed = text.trim();
+
+    if (trimmed.length < profile.minLength) return false;
+    if (trimmed.length > profile.maxLength) return false;
+
+    if (/^\d+$/.test(trimmed)) return false;
+    if (/^[!@#$%^&*()_+\-=\[\]{};':"\\|,.<>?]+$/.test(trimmed)) return false;
+
+    const hasScriptChar = profile.charRegex.test(trimmed);
+    if (!hasScriptChar) return false;
+
+    if (profile.stopwords.length > 0) {
+      const words = trimmed.toLowerCase().split(/\s+/);
+      for (const word of words) {
+        if (profile.stopwords.includes(word)) return true;
+      }
+      const validChars = trimmed.match(/[\p{L}\p{N}\s\-,.!?':"()\[\]{}]/gu) || [];
+      const validRatio = validChars.length / trimmed.length;
+      return validRatio >= 0.5;
+    }
+
+    return true;
+  }
+
   scanFile(filePath, patterns, minLength, maxLength) {
     try {
       const content = SecurityUtils.safeReadFileSync(filePath, path.dirname(filePath), 'utf8');
       const lines = content.split('\n');
       const results = [];
+      const sourceLang = this.config.sourceLanguage || 'en';
 
       patterns.forEach(pattern => {
         let match;
         while ((match = pattern.exec(content)) !== null) {
           const text = match[1] || match[0];
 
-          // Skip translation function calls
           const beforeMatch = content.substring(Math.max(0, match.index - 20), match.index);
           if (beforeMatch.includes('t(') || beforeMatch.includes('i18next.t(') ||
               beforeMatch.includes('$t(') || beforeMatch.includes('translate(')) {
             continue;
           }
 
-          if (text && this.isEnglishText(text) &&
+          if (text && this.isTextInLanguage(text, sourceLang) &&
               text.length >= minLength && text.length <= maxLength) {
 
             const lineNumber = content.substring(0, match.index).split('\n').length;
@@ -355,7 +475,23 @@ class I18nTextScanner {
   }
 
   generateSuggestion(text) {
-    const key = text.toLowerCase()
+    const sourceLang = this.config.sourceLanguage || 'en';
+    const transliterations = {
+      ja: { 'あ': 'a', 'い': 'i', 'う': 'u', 'え': 'e', 'お': 'o', 'か': 'ka', 'き': 'ki', 'く': 'ku', 'け': 'ke', 'こ': 'ko', 'さ': 'sa', 'し': 'shi', 'す': 'su', 'せ': 'se', 'そ': 'so', 'た': 'ta', 'ち': 'chi', 'つ': 'tsu', 'て': 'te', 'と': 'to', 'な': 'na', 'に': 'ni', 'ぬ': 'nu', 'ね': 'ne', 'の': 'no', 'は': 'ha', 'ひ': 'hi', 'ふ': 'fu', 'へ': 'he', 'ほ': 'ho', 'ま': 'ma', 'み': 'mi', 'む': 'mu', 'め': 'me', 'も': 'mo', 'や': 'ya', 'ゆ': 'yu', 'よ': 'yo', 'ら': 'ra', 'り': 'ri', 'る': 'ru', 'れ': 're', 'ろ': 'ro', 'わ': 'wa', 'を': 'wo', 'ん': 'n' },
+      ru: { 'а': 'a', 'б': 'b', 'в': 'v', 'г': 'g', 'д': 'd', 'е': 'e', 'ё': 'yo', 'ж': 'zh', 'з': 'z', 'и': 'i', 'й': 'y', 'к': 'k', 'л': 'l', 'м': 'm', 'н': 'n', 'о': 'o', 'п': 'p', 'р': 'r', 'с': 's', 'т': 't', 'у': 'u', 'ф': 'f', 'х': 'kh', 'ц': 'ts', 'ч': 'ch', 'ш': 'sh', 'щ': 'sch', 'ъ': '', 'ы': 'y', 'ь': '', 'э': 'e', 'ю': 'yu', 'я': 'ya' },
+      zh: { '的': 'de', '一': 'yi', '是': 'shi', '在': 'zai', '不': 'bu', '了': 'le', '有': 'you', '和': 'he', '人': 'ren', '这': 'zhe', '中': 'zhong', '大': 'da', '为': 'wei', '上': 'shang', '个': 'ge', '国': 'guo', '我': 'wo', '以': 'yi_t', '要': 'yao', '他': 'ta', '时': 'shi_t', '来': 'lai', '用': 'yong', '们': 'men', '生': 'sheng', '到': 'dao', '作': 'zuo', '地': 'di' }
+    };
+
+    let transliterated = text;
+    const table = transliterations[sourceLang];
+    if (table) {
+      transliterated = '';
+      for (const ch of text) {
+        transliterated += table[ch] || ch;
+      }
+    }
+
+    const key = transliterated.toLowerCase()
       .replace(/[^a-z0-9\s]/g, '')
       .replace(/\s+/g, '_')
       .substring(0, 50);
@@ -559,6 +695,9 @@ class I18nTextScanner {
 
     this.sourceDir = this.config.sourceDir || './src';
 
+    // Source language for multi-language detection
+    this.sourceLanguage = args['source-language'] || this.config.sourceLanguage || 'en';
+
     // Resolve framework with precedence: CLI arg > config.framework.preference|string > auto-detect > fallback
     const cliFramework = args.framework;
     const cfgFramework = this.config.framework;
@@ -637,4 +776,4 @@ if (require.main === module) {
   })();
 }
 
-module.exports = I18nTextScanner;
+module.exports = I18nTextScanner;