i18ntk 2.6.0 → 3.1.0

package/utils/translate/report.js

@@ -0,0 +1,117 @@
+const path = require('path');
+const SecurityUtils = require('../security');
+
+function generateReport(skippedKeys, translatedCount, totalCount, options = {}) {
+  const {
+    sourceFile,
+    targetLang,
+    dryRun = false,
+    timestamp = new Date().toISOString(),
+    placeholderProtected = 0,
+    protectedSkipped = 0,
+  } = options;
+  const placeholderSkipped = skippedKeys.filter(key => key.skipReason !== 'protected');
+  const protectedKeys = skippedKeys.filter(key => key.skipReason === 'protected');
+
+  const lines = [];
+  lines.push('='.repeat(72));
+  lines.push('  I18NTK POST-TRANSLATION REPORT');
+  lines.push('='.repeat(72));
+  lines.push(`  Generated:       ${timestamp}`);
+  lines.push(`  Source file:     ${sourceFile || 'N/A'}`);
+  lines.push(`  Target language: ${targetLang || 'N/A'}`);
+  if (dryRun) {
+    lines.push(`  Mode:            DRY-RUN (no API calls made)`);
+  }
+  lines.push(`  Total keys:      ${totalCount}`);
+  lines.push(`  Translated:      ${translatedCount}`);
+  lines.push(`  Placeholder-safe: ${String(placeholderProtected).padStart(6)}`);
+  lines.push(`  Protected:       ${String(protectedSkipped).padStart(6)}`);
+  lines.push(`  Skipped:         ${skippedKeys.length}`);
+  lines.push('='.repeat(72));
+
+  if (skippedKeys.length === 0) {
+    lines.push('');
+    lines.push('  All strings were processed. No keys were skipped.');
+    lines.push('');
+  } else {
+    if (placeholderSkipped.length > 0) {
+      lines.push('');
+      lines.push('  WARNING: The following keys were SKIPPED because they contain');
+      lines.push('  dynamic placeholder tokens that should be manually translated');
+      lines.push('  to avoid runtime substitution breakage.');
+      lines.push('');
+      lines.push('  These entries were copied verbatim into the output file.');
+      lines.push('  You MUST manually translate them before using the file.');
+    }
+    if (protectedKeys.length > 0) {
+      lines.push('');
+      lines.push('  The following keys were copied unchanged because they matched');
+      lines.push('  Auto Translate protection rules for keys or exact values.');
+    }
+    lines.push('');
+    lines.push(`  ${'-'.repeat(64)}`);
+    lines.push(`  Key Path                                            Original Value`);
+    lines.push(`  ${'-'.repeat(64)}`);
+
+    for (const skip of skippedKeys) {
+      const keyDisplay = skip.keyPath.length > 50
+        ? skip.keyPath.substring(0, 47) + '...'
+        : skip.keyPath.padEnd(50);
+      const valDisplay = skip.value.length > 80
+        ? skip.value.substring(0, 77) + '...'
+        : skip.value;
+      lines.push(`  ${keyDisplay} ${valDisplay}`);
+    }
+
+    lines.push(`  ${'-'.repeat(64)}`);
+    if (placeholderSkipped.length > 0) {
+      lines.push('');
+      lines.push('  REMINDER:');
+      lines.push('  1. Open the target JSON file');
+      lines.push('  2. Search for the placeholder-skipped keys listed above');
+      lines.push('  3. Manually translate each value, preserving all placeholders');
+      lines.push('     exactly as they appear in the original');
+      lines.push('  4. Verify placeholder integrity before runtime use');
+      lines.push('');
+    }
+  }
+
+  lines.push('');
+  if (skippedKeys.length === 0) {
+    lines.push('  The generated file can be used immediately after review.');
+    lines.push('  Placeholder tokens were preserved automatically where found.');
+  } else if (placeholderSkipped.length > 0) {
+    lines.push('  The generated file can be used immediately for all');
+    lines.push('  translated text. Only the skipped keys need');
+    lines.push('  manual attention.');
+  } else {
+    lines.push('  Protected keys and values were intentionally copied unchanged.');
+    lines.push('  Review the output file before using it in production.');
+  }
+  lines.push('='.repeat(72));
+
+  return lines.join('\n');
+}
+
+function writeReport(reportText, filePath) {
+  if (!filePath) return;
+  try {
+    const resolvedPath = path.resolve(process.cwd(), filePath);
+    SecurityUtils.safeWriteFileSync(resolvedPath, reportText + '\n', path.dirname(resolvedPath), 'utf-8');
+  } catch (e) {
+    console.error('Failed to write report file:', e.message);
+  }
+}
+
+function formatSummaryLine(skippedCount, translatedCount, totalCount, placeholderProtected = 0, protectedSkipped = 0) {
+  const protectedPart = placeholderProtected > 0 ? `, ${placeholderProtected} placeholder-safe` : '';
+  const glossaryPart = protectedSkipped > 0 ? `, ${protectedSkipped} protected` : '';
+  return `[translate] ${translatedCount} translated${protectedPart}${glossaryPart}, ${skippedCount} skipped (of ${totalCount} total keys)`;
+}
+
+module.exports = {
+  generateReport,
+  writeReport,
+  formatSummaryLine,
+};

package/utils/translate/traverse.js

@@ -0,0 +1,148 @@
+function deepTraverse(obj, visitor, keyPath = '') {
+  if (obj === null || obj === undefined) {
+    visitor({ type: 'null', keyPath, value: null });
+    return;
+  }
+  if (typeof obj === 'string') {
+    visitor({ type: 'string', keyPath, value: obj });
+    return;
+  }
+  if (typeof obj === 'number' || typeof obj === 'boolean') {
+    visitor({ type: 'leaf', keyPath, value: obj });
+    return;
+  }
+  if (Array.isArray(obj)) {
+    visitor({ type: 'array-start', keyPath, value: obj });
+    for (let i = 0; i < obj.length; i++) {
+      const childPath = keyPath ? `${keyPath}[${i}]` : `[${i}]`;
+      deepTraverse(obj[i], visitor, childPath);
+    }
+    visitor({ type: 'array-end', keyPath, value: obj });
+    return;
+  }
+  if (typeof obj === 'object') {
+    visitor({ type: 'object-start', keyPath, value: obj });
+    for (const key of Object.keys(obj)) {
+      const childPath = keyPath ? `${keyPath}.${key}` : key;
+      deepTraverse(obj[key], visitor, childPath);
+    }
+    visitor({ type: 'object-end', keyPath, value: obj });
+    return;
+  }
+  visitor({ type: 'unknown', keyPath, value: String(obj) });
+}
+
+function collectLeaves(obj, prefix = '') {
+  const entries = [];
+  for (const [key, value] of Object.entries(obj)) {
+    const fullKey = prefix ? `${prefix}.${key}` : key;
+    if (typeof value === 'string') {
+      entries.push({ keyPath: fullKey, value });
+    } else if (Array.isArray(value)) {
+      for (let i = 0; i < value.length; i++) {
+        const itemPath = `${fullKey}[${i}]`;
+        if (typeof value[i] === 'string') {
+          entries.push({ keyPath: itemPath, value: value[i] });
+        } else if (typeof value[i] === 'object' && value[i] !== null && !Array.isArray(value[i])) {
+          entries.push(...collectLeaves(value[i], itemPath));
+        }
+      }
+    } else if (typeof value === 'object' && value !== null) {
+      entries.push(...collectLeaves(value, fullKey));
+    }
+  }
+  return entries;
+}
+
+function setLeaf(obj, keyPath, value) {
+  const parts = [];
+  let current = '';
+  for (let i = 0; i < keyPath.length; i++) {
+    const ch = keyPath[i];
+    if (ch === '.' && keyPath[i - 1] !== '\\') {
+      if (current) parts.push(current);
+      current = '';
+    } else if (ch === '[') {
+      if (current) parts.push(current);
+      current = '';
+      i++;
+      while (i < keyPath.length && keyPath[i] !== ']') {
+        current += keyPath[i];
+        i++;
+      }
+      parts.push(`[${current}]`);
+      current = '';
+    } else {
+      current += ch;
+    }
+  }
+  if (current) parts.push(current);
+
+  let target = obj;
+  for (let i = 0; i < parts.length - 1; i++) {
+    const part = parts[i];
+    if (part.startsWith('[') && part.endsWith(']')) {
+      const idx = parseInt(part.slice(1, -1), 10);
+      if (!(idx in target)) target[idx] = {};
+      target = target[idx];
+    } else {
+      if (!(part in target)) target[part] = {};
+      target = target[part];
+    }
+  }
+
+  const last = parts[parts.length - 1];
+  if (last.startsWith('[') && last.endsWith(']')) {
+    target[parseInt(last.slice(1, -1), 10)] = value;
+  } else {
+    target[last] = value;
+  }
+}
+
+function getLeaf(obj, keyPath) {
+  const parts = [];
+  let current = '';
+  for (let i = 0; i < keyPath.length; i++) {
+    const ch = keyPath[i];
+    if (ch === '.' && keyPath[i - 1] !== '\\') {
+      if (current) parts.push(current);
+      current = '';
+    } else if (ch === '[') {
+      if (current) parts.push(current);
+      current = '';
+      i++;
+      while (i < keyPath.length && keyPath[i] !== ']') {
+        current += keyPath[i];
+        i++;
+      }
+      parts.push(`[${current}]`);
+      current = '';
+    } else {
+      current += ch;
+    }
+  }
+  if (current) parts.push(current);
+
+  let target = obj;
+  for (const part of parts) {
+    if (target === null || target === undefined) return undefined;
+    if (part.startsWith('[') && part.endsWith(']')) {
+      target = target[parseInt(part.slice(1, -1), 10)];
+    } else {
+      target = target[part];
+    }
+  }
+  return target;
+}
+
+function deepClone(obj) {
+  return JSON.parse(JSON.stringify(obj));
+}
+
+module.exports = {
+  deepTraverse,
+  collectLeaves,
+  setLeaf,
+  getLeaf,
+  deepClone,
+};

package/utils/validation-risk.js

@@ -0,0 +1,175 @@
+const DEFAULT_ENGLISH_THRESHOLD_PERCENT = 10;
+
+const URL_PATTERN = /https?:\/\/[^\s"'<>]+/i;
+const EMAIL_PATTERN = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/;
+const PRIVATE_KEY_PATTERN = /-----BEGIN [A-Z ]*PRIVATE KEY-----/;
+const BEARER_TOKEN_PATTERN = /\bBearer\s+[A-Za-z0-9._~+/=-]{16,}/i;
+const CREDENTIAL_ASSIGNMENT_PATTERN = /\b(api[_-]?key|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|password|private[_-]?key|client[_-]?secret)\b\s*[:=]\s*["']?[A-Za-z0-9._~+/=-]{16,}/i;
+const CREDENTIAL_KEY_PATTERN = /\b(api[_-]?key|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|password|private[_-]?key|client[_-]?secret)\b/i;
+const OPAQUE_SECRET_PATTERN = /\b(AKIA[0-9A-Z]{16}|ASIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{35}|sk_live_[0-9A-Za-z]{16,}|xox[baprs]-[0-9A-Za-z-]{16,}|gh[pousr]_[0-9A-Za-z_]{20,}|[A-Za-z0-9._~+/=-]{32,})\b/;
+const JWT_PATTERN = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/;
+
+const ENGLISH_WORDS = new Set([
+  'a', 'about', 'above', 'after', 'again', 'all', 'also', 'an', 'and', 'any', 'are', 'as', 'at', 'available',
+  'back', 'based', 'be', 'because', 'before', 'below', 'best', 'book', 'build', 'by',
+  'can', 'cash', 'challenge', 'change', 'check', 'choose', 'click', 'close', 'coming', 'complete', 'confirm',
+  'continue', 'copy', 'create', 'current',
+  'data', 'delete', 'depth', 'details', 'done', 'down',
+  'earn', 'edit', 'email', 'empty', 'enter', 'error', 'exchange',
+  'failed', 'file', 'filter', 'for', 'from',
+  'governance',
+  'has', 'have', 'help', 'hide', 'history', 'home',
+  'if', 'in', 'into', 'is', 'it', 'its',
+  'key', 'keys',
+  'language', 'last', 'latest', 'learn', 'live', 'loading', 'log', 'login', 'logout',
+  'market', 'markets', 'message', 'missing', 'more',
+  'new', 'next', 'no', 'not',
+  'of', 'off', 'on', 'open', 'or', 'order', 'other', 'out', 'over', 'overview',
+  'page', 'participate', 'players', 'please', 'platform', 'predict', 'prediction', 'previous', 'public',
+  'real', 'record', 'remove', 'required', 'reset', 'result', 'results', 'rewards',
+  'save', 'search', 'select', 'settings', 'show', 'soon', 'start', 'status', 'structured', 'submit', 'success',
+  'the', 'their', 'this', 'through', 'to', 'track', 'try',
+  'up', 'update', 'use', 'used', 'using',
+  'value', 'view',
+  'warning', 'when', 'with', 'without'
+]);
+
+const DEFAULT_ALLOWED_ENGLISH_TERMS = new Set([
+  'api',
+]);
+
+function normalizeLanguage(language) {
+  return String(language || '').toLowerCase().split(/[-_]/)[0];
+}
+
+function toAllowedTermSet(terms) {
+  const allowed = new Set(DEFAULT_ALLOWED_ENGLISH_TERMS);
+  if (Array.isArray(terms)) {
+    terms.forEach(term => {
+      if (typeof term === 'string' && term.trim()) {
+        allowed.add(term.trim().toLowerCase());
+      }
+    });
+  }
+  return allowed;
+}
+
+function stripNonLanguageTokens(value) {
+  return String(value || '')
+    .replace(URL_PATTERN, ' ')
+    .replace(EMAIL_PATTERN, ' ')
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/\{\{[^}]+\}\}|\{[^}]+\}|%[sdifjoO]/g, ' ');
+}
+
+function isIgnoredEnglishToken(word, allowedTerms) {
+  const normalized = word.toLowerCase().replace(/^['-]+|['-]+$/g, '');
+  if (!normalized || normalized.length <= 2) return true;
+  if (allowedTerms.has(normalized)) return true;
+  if (/^[A-Z0-9_-]{2,}$/.test(word)) return true;
+  if (/\d/.test(word)) return true;
+  return false;
+}
+
+function analyzeEnglishContent(value, options = {}) {
+  const allowedTerms = toAllowedTermSet(options.allowedEnglishTerms);
+  const words = stripNonLanguageTokens(value).match(/[A-Za-z][A-Za-z'-]*/g) || [];
+  const countedWords = [];
+  const englishWords = [];
+
+  words.forEach(word => {
+    if (isIgnoredEnglishToken(word, allowedTerms)) return;
+
+    const normalized = word.toLowerCase().replace(/^['-]+|['-]+$/g, '');
+    countedWords.push(normalized);
+    if (ENGLISH_WORDS.has(normalized)) {
+      englishWords.push(normalized);
+    }
+  });
+
+  const totalWordCount = countedWords.length;
+  const englishWordCount = englishWords.length;
+  const englishPercentage = totalWordCount === 0
+    ? 0
+    : Number(((englishWordCount / totalWordCount) * 100).toFixed(2));
+
+  return {
+    englishPercentage,
+    englishWordCount,
+    totalWordCount,
+    englishWords: [...new Set(englishWords)].slice(0, 8)
+  };
+}
+
+function hasSecretLikeValue(value, keyPath = '') {
+  const valueStr = String(value || '');
+  if (
+    PRIVATE_KEY_PATTERN.test(valueStr) ||
+    BEARER_TOKEN_PATTERN.test(valueStr) ||
+    CREDENTIAL_ASSIGNMENT_PATTERN.test(valueStr) ||
+    JWT_PATTERN.test(valueStr)
+  ) {
+    return true;
+  }
+
+  return CREDENTIAL_KEY_PATTERN.test(keyPath) && OPAQUE_SECRET_PATTERN.test(valueStr);
+}
+
+function detectTranslationContentRisks(value, options = {}) {
+  const valueStr = String(value || '');
+  const issues = [];
+
+  if (URL_PATTERN.test(valueStr)) {
+    issues.push({
+      type: 'url',
+      reason: 'Contains a URL; verify it is intentional and localized where needed.'
+    });
+  }
+
+  if (EMAIL_PATTERN.test(valueStr)) {
+    issues.push({
+      type: 'email',
+      reason: 'Contains an email address; verify it is intentional public contact content.'
+    });
+  }
+
+  if (hasSecretLikeValue(valueStr, options.keyPath)) {
+    issues.push({
+      type: 'secret',
+      reason: 'Looks like a credential or secret value, not ordinary translated content.'
+    });
+  }
+
+  const sourceLanguage = normalizeLanguage(options.sourceLanguage || 'en');
+  const targetLanguage = normalizeLanguage(options.targetLanguage);
+  if (targetLanguage && targetLanguage !== sourceLanguage) {
+    const threshold = Number.isFinite(Number(options.englishThresholdPercent))
+      ? Number(options.englishThresholdPercent)
+      : DEFAULT_ENGLISH_THRESHOLD_PERCENT;
+    const english = analyzeEnglishContent(valueStr, options);
+
+    if (
+      english.englishPercentage > threshold &&
+      english.englishWordCount >= 3
+    ) {
+      issues.push({
+        type: 'english_content',
+        reason: `Possible untranslated English content (${english.englishPercentage}% English words, threshold ${threshold}%).`,
+        englishPercentage: english.englishPercentage,
+        englishThresholdPercent: threshold,
+        englishWordCount: english.englishWordCount,
+        totalWordCount: english.totalWordCount,
+        englishWords: english.englishWords
+      });
+    }
+  }
+
+  return issues;
+}
+
+module.exports = {
+  DEFAULT_ENGLISH_THRESHOLD_PERCENT,
+  analyzeEnglishContent,
+  detectTranslationContentRisks,
+  hasSecretLikeValue
+};