i18ntk 3.0.0 → 3.1.0

package/utils/translate/protection.js

@@ -0,0 +1,243 @@
+const path = require('path');
+const SecurityUtils = require('../security');
+
+const DEFAULT_PROTECTION_FILE = 'i18ntk-auto-translate.json';
+const TOKEN_PREFIX = '__I18NTK_KEEP_';
+
+function defaultProtectionConfig() {
+  return {
+    version: 1,
+    description: 'Auto Translate protection rules. Terms are masked before translation and restored after translation. Keys and values are copied from the source without translation.',
+    terms: [],
+    keys: [],
+    values: [],
+    patterns: []
+  };
+}
+
+function resolveProtectionFile(filePath, cwd = process.cwd()) {
+  const requested = filePath || DEFAULT_PROTECTION_FILE;
+  const resolved = path.isAbsolute(requested)
+    ? path.resolve(requested)
+    : path.resolve(cwd, requested);
+  const root = path.resolve(cwd);
+
+  if (!resolved.startsWith(root + path.sep) && resolved !== root) {
+    throw new Error(`Protection file must be inside the project: ${requested}`);
+  }
+
+  return resolved;
+}
+
+function normalizeList(value) {
+  if (!Array.isArray(value)) return [];
+  return value
+    .filter(item => typeof item === 'string')
+    .map(item => item.trim())
+    .filter(Boolean);
+}
+
+function compilePatterns(patterns) {
+  const compiled = [];
+  for (const pattern of patterns) {
+    try {
+      compiled.push(new RegExp(pattern, 'g'));
+    } catch (error) {
+      console.warn(`Invalid Auto Translate protection pattern ignored: ${pattern} (${error.message})`);
+    }
+  }
+  return compiled;
+}
+
+function normalizeProtectionConfig(config = {}, filePath = null) {
+  const terms = normalizeList(config.terms);
+  const keys = normalizeList(config.keys);
+  const values = normalizeList(config.values);
+  const patterns = normalizeList(config.patterns);
+
+  return {
+    enabled: true,
+    filePath,
+    terms,
+    keys,
+    values,
+    patterns,
+    compiledPatterns: compilePatterns(patterns)
+  };
+}
+
+function createProtectionFile(filePath, options = {}) {
+  const resolved = resolveProtectionFile(filePath, options.cwd);
+  const dir = path.dirname(resolved);
+  if (!SecurityUtils.safeExistsSync(dir, path.dirname(dir))) {
+    SecurityUtils.safeMkdirSync(dir, path.dirname(dir), { recursive: true });
+  }
+
+  if (!SecurityUtils.safeExistsSync(resolved, dir)) {
+    SecurityUtils.safeWriteFileSync(
+      resolved,
+      JSON.stringify(defaultProtectionConfig(), null, 2) + '\n',
+      dir,
+      'utf8'
+    );
+  }
+
+  return resolved;
+}
+
+function readProtectionFile(filePath, options = {}) {
+  const resolved = resolveProtectionFile(filePath, options.cwd);
+  const raw = SecurityUtils.safeReadFileSync(resolved, path.dirname(resolved), 'utf8');
+  if (!raw) return defaultProtectionConfig();
+  return JSON.parse(raw.replace(/^\uFEFF/, ''));
+}
+
+function saveProtectionFile(filePath, config, options = {}) {
+  const resolved = resolveProtectionFile(filePath, options.cwd);
+  const dir = path.dirname(resolved);
+  if (!SecurityUtils.safeExistsSync(dir, path.dirname(dir))) {
+    SecurityUtils.safeMkdirSync(dir, path.dirname(dir), { recursive: true });
+  }
+  const nextConfig = {
+    ...defaultProtectionConfig(),
+    ...(config || {}),
+    terms: normalizeList(config?.terms),
+    keys: normalizeList(config?.keys),
+    values: normalizeList(config?.values),
+    patterns: normalizeList(config?.patterns)
+  };
+  SecurityUtils.safeWriteFileSync(resolved, JSON.stringify(nextConfig, null, 2) + '\n', dir, 'utf8');
+  return resolved;
+}
+
+function loadProtectionConfig(filePath, options = {}) {
+  if (options.enabled === false) {
+    return normalizeProtectionConfig({}, null);
+  }
+
+  const resolved = resolveProtectionFile(filePath, options.cwd);
+  const dir = path.dirname(resolved);
+
+  if (!SecurityUtils.safeExistsSync(resolved, dir)) {
+    if (options.create) {
+      createProtectionFile(resolved, options);
+    } else {
+      return normalizeProtectionConfig({}, resolved);
+    }
+  }
+
+  const raw = SecurityUtils.safeReadFileSync(resolved, dir, 'utf8');
+  if (!raw) {
+    return normalizeProtectionConfig({}, resolved);
+  }
+
+  let parsed;
+  try {
+    parsed = JSON.parse(raw.replace(/^\uFEFF/, ''));
+  } catch (error) {
+    throw new Error(`Invalid Auto Translate protection JSON at ${resolved}: ${error.message}`);
+  }
+
+  return normalizeProtectionConfig(parsed, resolved);
+}
+
+function escapeRegExp(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+function keyMatchesRule(keyPath, rule) {
+  if (rule === keyPath) return true;
+  if (!rule.includes('*')) return false;
+  const regex = new RegExp(`^${rule.split('*').map(escapeRegExp).join('.*')}$`);
+  return regex.test(keyPath);
+}
+
+function shouldPreserveWholeValue(keyPath, value, protection) {
+  if (!protection || protection.enabled === false) return false;
+  if (protection.keys.some(rule => keyMatchesRule(keyPath, rule))) return true;
+  const valueText = String(value);
+  return protection.values.includes(valueText) || protection.terms.includes(valueText);
+}
+
+function addReplacement(replacements, original) {
+  if (!original) return;
+  if (replacements.some(item => item.original === original)) return;
+  replacements.push({ original });
+}
+
+function collectReplacements(value, protection) {
+  const text = String(value);
+  const replacements = [];
+
+  for (const term of protection.terms || []) {
+    if (text.includes(term)) {
+      addReplacement(replacements, term);
+    }
+  }
+
+  for (const pattern of protection.compiledPatterns || []) {
+    pattern.lastIndex = 0;
+    let match;
+    while ((match = pattern.exec(text)) !== null) {
+      addReplacement(replacements, match[0]);
+      if (match[0] === '') pattern.lastIndex++;
+    }
+  }
+
+  return replacements.sort((a, b) => b.original.length - a.original.length);
+}
+
+function protectText(value, protection) {
+  if (!protection || protection.enabled === false || typeof value !== 'string') {
+    return { value, map: new Map(), count: 0 };
+  }
+
+  const replacements = collectReplacements(value, protection);
+  if (replacements.length === 0) {
+    return { value, map: new Map(), count: 0 };
+  }
+
+  let protectedValue = value;
+  const map = new Map();
+  replacements.forEach((replacement, index) => {
+    const token = `${TOKEN_PREFIX}${index}__`;
+    map.set(token, replacement.original);
+    protectedValue = protectedValue.split(replacement.original).join(token);
+  });
+
+  return { value: protectedValue, map, count: map.size };
+}
+
+function restoreText(value, map) {
+  if (!(map instanceof Map) || map.size === 0 || typeof value !== 'string') return value;
+  let restored = value;
+  for (const [token, original] of map.entries()) {
+    restored = restored.split(token).join(original);
+  }
+  return restored;
+}
+
+function hasProtectionRules(protection) {
+  return Boolean(
+    protection &&
+    (
+      protection.terms.length ||
+      protection.keys.length ||
+      protection.values.length ||
+      protection.patterns.length
+    )
+  );
+}
+
+module.exports = {
+  DEFAULT_PROTECTION_FILE,
+  createProtectionFile,
+  defaultProtectionConfig,
+  hasProtectionRules,
+  loadProtectionConfig,
+  protectText,
+  readProtectionFile,
+  restoreText,
+  saveProtectionFile,
+  shouldPreserveWholeValue
+};

package/utils/translate/report.js

@@ -1,4 +1,5 @@
-const fs = require('fs');
+const path = require('path');
+const SecurityUtils = require('../security');
 
 function generateReport(skippedKeys, translatedCount, totalCount, options = {}) {
   const {
@@ -6,7 +7,11 @@ function generateReport(skippedKeys, translatedCount, totalCount, options = {})
     targetLang,
     dryRun = false,
     timestamp = new Date().toISOString(),
+    placeholderProtected = 0,
+    protectedSkipped = 0,
   } = options;
+  const placeholderSkipped = skippedKeys.filter(key => key.skipReason !== 'protected');
+  const protectedKeys = skippedKeys.filter(key => key.skipReason === 'protected');
 
   const lines = [];
   lines.push('='.repeat(72));
@@ -20,6 +25,8 @@ function generateReport(skippedKeys, translatedCount, totalCount, options = {})
   }
   lines.push(`  Total keys:      ${totalCount}`);
   lines.push(`  Translated:      ${translatedCount}`);
+  lines.push(`  Placeholder-safe: ${String(placeholderProtected).padStart(6)}`);
+  lines.push(`  Protected:       ${String(protectedSkipped).padStart(6)}`);
   lines.push(`  Skipped:         ${skippedKeys.length}`);
   lines.push('='.repeat(72));
 
@@ -28,13 +35,20 @@ function generateReport(skippedKeys, translatedCount, totalCount, options = {})
     lines.push('  All strings were processed. No keys were skipped.');
     lines.push('');
   } else {
-    lines.push('');
-    lines.push('  WARNING: The following keys were SKIPPED because they contain');
-    lines.push('  dynamic placeholder tokens that should be manually translated');
-    lines.push('  to avoid runtime substitution breakage.');
-    lines.push('');
-    lines.push('  These entries were copied verbatim into the output file.');
-    lines.push('  You MUST manually translate them before using the file.');
+    if (placeholderSkipped.length > 0) {
+      lines.push('');
+      lines.push('  WARNING: The following keys were SKIPPED because they contain');
+      lines.push('  dynamic placeholder tokens that should be manually translated');
+      lines.push('  to avoid runtime substitution breakage.');
+      lines.push('');
+      lines.push('  These entries were copied verbatim into the output file.');
+      lines.push('  You MUST manually translate them before using the file.');
+    }
+    if (protectedKeys.length > 0) {
+      lines.push('');
+      lines.push('  The following keys were copied unchanged because they matched');
+      lines.push('  Auto Translate protection rules for keys or exact values.');
+    }
     lines.push('');
     lines.push(`  ${'-'.repeat(64)}`);
     lines.push(`  Key Path                                            Original Value`);
@@ -51,20 +65,30 @@ function generateReport(skippedKeys, translatedCount, totalCount, options = {})
     }
 
     lines.push(`  ${'-'.repeat(64)}`);
-    lines.push('');
-    lines.push('  REMINDER:');
-    lines.push('  1. Open the target JSON file');
-    lines.push('  2. Search for the keys listed above');
-    lines.push('  3. Manually translate each value, preserving all placeholders');
-    lines.push('     exactly as they appear in the original');
-    lines.push('  4. Verify placeholder integrity before runtime use');
-    lines.push('');
+    if (placeholderSkipped.length > 0) {
+      lines.push('');
+      lines.push('  REMINDER:');
+      lines.push('  1. Open the target JSON file');
+      lines.push('  2. Search for the placeholder-skipped keys listed above');
+      lines.push('  3. Manually translate each value, preserving all placeholders');
+      lines.push('     exactly as they appear in the original');
+      lines.push('  4. Verify placeholder integrity before runtime use');
+      lines.push('');
+    }
   }
 
   lines.push('');
-  lines.push('  The generated file can be used immediately for all');
-  lines.push('  non-placeholder text. Only the skipped keys need');
-  lines.push('  manual attention.');
+  if (skippedKeys.length === 0) {
+    lines.push('  The generated file can be used immediately after review.');
+    lines.push('  Placeholder tokens were preserved automatically where found.');
+  } else if (placeholderSkipped.length > 0) {
+    lines.push('  The generated file can be used immediately for all');
+    lines.push('  translated text. Only the skipped keys need');
+    lines.push('  manual attention.');
+  } else {
+    lines.push('  Protected keys and values were intentionally copied unchanged.');
+    lines.push('  Review the output file before using it in production.');
+  }
   lines.push('='.repeat(72));
 
   return lines.join('\n');
@@ -73,14 +97,17 @@ function generateReport(skippedKeys, translatedCount, totalCount, options = {})
 function writeReport(reportText, filePath) {
   if (!filePath) return;
   try {
-    fs.writeFileSync(filePath, reportText + '\n', 'utf-8');
+    const resolvedPath = path.resolve(process.cwd(), filePath);
+    SecurityUtils.safeWriteFileSync(resolvedPath, reportText + '\n', path.dirname(resolvedPath), 'utf-8');
   } catch (e) {
     console.error('Failed to write report file:', e.message);
   }
 }
 
-function formatSummaryLine(skippedCount, translatedCount, totalCount) {
-  return `[translate] ${translatedCount} translated, ${skippedCount} skipped (of ${totalCount} total keys)`;
+function formatSummaryLine(skippedCount, translatedCount, totalCount, placeholderProtected = 0, protectedSkipped = 0) {
+  const protectedPart = placeholderProtected > 0 ? `, ${placeholderProtected} placeholder-safe` : '';
+  const glossaryPart = protectedSkipped > 0 ? `, ${protectedSkipped} protected` : '';
+  return `[translate] ${translatedCount} translated${protectedPart}${glossaryPart}, ${skippedCount} skipped (of ${totalCount} total keys)`;
 }
 
 module.exports = {

package/utils/validation-risk.js

@@ -0,0 +1,175 @@
+const DEFAULT_ENGLISH_THRESHOLD_PERCENT = 10;
+
+const URL_PATTERN = /https?:\/\/[^\s"'<>]+/i;
+const EMAIL_PATTERN = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/;
+const PRIVATE_KEY_PATTERN = /-----BEGIN [A-Z ]*PRIVATE KEY-----/;
+const BEARER_TOKEN_PATTERN = /\bBearer\s+[A-Za-z0-9._~+/=-]{16,}/i;
+const CREDENTIAL_ASSIGNMENT_PATTERN = /\b(api[_-]?key|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|password|private[_-]?key|client[_-]?secret)\b\s*[:=]\s*["']?[A-Za-z0-9._~+/=-]{16,}/i;
+const CREDENTIAL_KEY_PATTERN = /\b(api[_-]?key|access[_-]?token|auth[_-]?token|refresh[_-]?token|secret|password|private[_-]?key|client[_-]?secret)\b/i;
+const OPAQUE_SECRET_PATTERN = /\b(AKIA[0-9A-Z]{16}|ASIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{35}|sk_live_[0-9A-Za-z]{16,}|xox[baprs]-[0-9A-Za-z-]{16,}|gh[pousr]_[0-9A-Za-z_]{20,}|[A-Za-z0-9._~+/=-]{32,})\b/;
+const JWT_PATTERN = /\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/;
+
+const ENGLISH_WORDS = new Set([
+  'a', 'about', 'above', 'after', 'again', 'all', 'also', 'an', 'and', 'any', 'are', 'as', 'at', 'available',
+  'back', 'based', 'be', 'because', 'before', 'below', 'best', 'book', 'build', 'by',
+  'can', 'cash', 'challenge', 'change', 'check', 'choose', 'click', 'close', 'coming', 'complete', 'confirm',
+  'continue', 'copy', 'create', 'current',
+  'data', 'delete', 'depth', 'details', 'done', 'down',
+  'earn', 'edit', 'email', 'empty', 'enter', 'error', 'exchange',
+  'failed', 'file', 'filter', 'for', 'from',
+  'governance',
+  'has', 'have', 'help', 'hide', 'history', 'home',
+  'if', 'in', 'into', 'is', 'it', 'its',
+  'key', 'keys',
+  'language', 'last', 'latest', 'learn', 'live', 'loading', 'log', 'login', 'logout',
+  'market', 'markets', 'message', 'missing', 'more',
+  'new', 'next', 'no', 'not',
+  'of', 'off', 'on', 'open', 'or', 'order', 'other', 'out', 'over', 'overview',
+  'page', 'participate', 'players', 'please', 'platform', 'predict', 'prediction', 'previous', 'public',
+  'real', 'record', 'remove', 'required', 'reset', 'result', 'results', 'rewards',
+  'save', 'search', 'select', 'settings', 'show', 'soon', 'start', 'status', 'structured', 'submit', 'success',
+  'the', 'their', 'this', 'through', 'to', 'track', 'try',
+  'up', 'update', 'use', 'used', 'using',
+  'value', 'view',
+  'warning', 'when', 'with', 'without'
+]);
+
+const DEFAULT_ALLOWED_ENGLISH_TERMS = new Set([
+  'api',
+]);
+
+function normalizeLanguage(language) {
+  return String(language || '').toLowerCase().split(/[-_]/)[0];
+}
+
+function toAllowedTermSet(terms) {
+  const allowed = new Set(DEFAULT_ALLOWED_ENGLISH_TERMS);
+  if (Array.isArray(terms)) {
+    terms.forEach(term => {
+      if (typeof term === 'string' && term.trim()) {
+        allowed.add(term.trim().toLowerCase());
+      }
+    });
+  }
+  return allowed;
+}
+
+function stripNonLanguageTokens(value) {
+  return String(value || '')
+    .replace(URL_PATTERN, ' ')
+    .replace(EMAIL_PATTERN, ' ')
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/\{\{[^}]+\}\}|\{[^}]+\}|%[sdifjoO]/g, ' ');
+}
+
+function isIgnoredEnglishToken(word, allowedTerms) {
+  const normalized = word.toLowerCase().replace(/^['-]+|['-]+$/g, '');
+  if (!normalized || normalized.length <= 2) return true;
+  if (allowedTerms.has(normalized)) return true;
+  if (/^[A-Z0-9_-]{2,}$/.test(word)) return true;
+  if (/\d/.test(word)) return true;
+  return false;
+}
+
+function analyzeEnglishContent(value, options = {}) {
+  const allowedTerms = toAllowedTermSet(options.allowedEnglishTerms);
+  const words = stripNonLanguageTokens(value).match(/[A-Za-z][A-Za-z'-]*/g) || [];
+  const countedWords = [];
+  const englishWords = [];
+
+  words.forEach(word => {
+    if (isIgnoredEnglishToken(word, allowedTerms)) return;
+
+    const normalized = word.toLowerCase().replace(/^['-]+|['-]+$/g, '');
+    countedWords.push(normalized);
+    if (ENGLISH_WORDS.has(normalized)) {
+      englishWords.push(normalized);
+    }
+  });
+
+  const totalWordCount = countedWords.length;
+  const englishWordCount = englishWords.length;
+  const englishPercentage = totalWordCount === 0
+    ? 0
+    : Number(((englishWordCount / totalWordCount) * 100).toFixed(2));
+
+  return {
+    englishPercentage,
+    englishWordCount,
+    totalWordCount,
+    englishWords: [...new Set(englishWords)].slice(0, 8)
+  };
+}
+
+function hasSecretLikeValue(value, keyPath = '') {
+  const valueStr = String(value || '');
+  if (
+    PRIVATE_KEY_PATTERN.test(valueStr) ||
+    BEARER_TOKEN_PATTERN.test(valueStr) ||
+    CREDENTIAL_ASSIGNMENT_PATTERN.test(valueStr) ||
+    JWT_PATTERN.test(valueStr)
+  ) {
+    return true;
+  }
+
+  return CREDENTIAL_KEY_PATTERN.test(keyPath) && OPAQUE_SECRET_PATTERN.test(valueStr);
+}
+
+function detectTranslationContentRisks(value, options = {}) {
+  const valueStr = String(value || '');
+  const issues = [];
+
+  if (URL_PATTERN.test(valueStr)) {
+    issues.push({
+      type: 'url',
+      reason: 'Contains a URL; verify it is intentional and localized where needed.'
+    });
+  }
+
+  if (EMAIL_PATTERN.test(valueStr)) {
+    issues.push({
+      type: 'email',
+      reason: 'Contains an email address; verify it is intentional public contact content.'
+    });
+  }
+
+  if (hasSecretLikeValue(valueStr, options.keyPath)) {
+    issues.push({
+      type: 'secret',
+      reason: 'Looks like a credential or secret value, not ordinary translated content.'
+    });
+  }
+
+  const sourceLanguage = normalizeLanguage(options.sourceLanguage || 'en');
+  const targetLanguage = normalizeLanguage(options.targetLanguage);
+  if (targetLanguage && targetLanguage !== sourceLanguage) {
+    const threshold = Number.isFinite(Number(options.englishThresholdPercent))
+      ? Number(options.englishThresholdPercent)
+      : DEFAULT_ENGLISH_THRESHOLD_PERCENT;
+    const english = analyzeEnglishContent(valueStr, options);
+
+    if (
+      english.englishPercentage > threshold &&
+      english.englishWordCount >= 3
+    ) {
+      issues.push({
+        type: 'english_content',
+        reason: `Possible untranslated English content (${english.englishPercentage}% English words, threshold ${threshold}%).`,
+        englishPercentage: english.englishPercentage,
+        englishThresholdPercent: threshold,
+        englishWordCount: english.englishWordCount,
+        totalWordCount: english.totalWordCount,
+        englishWords: english.englishWords
+      });
+    }
+  }
+
+  return issues;
+}
+
+module.exports = {
+  DEFAULT_ENGLISH_THRESHOLD_PERCENT,
+  analyzeEnglishContent,
+  detectTranslationContentRisks,
+  hasSecretLikeValue
+};