i18ntk 3.0.0 → 3.1.1

package/utils/config-manager.js

@@ -121,7 +121,7 @@ const DEFAULT_CONFIG = {
     "complete": null,
     "manage": null
   },
-  "processing": {
+  "processing": {
     "batchSize": 2000,
     "concurrency": 32,
     "maxFileSize": 524288,
@@ -150,9 +150,25 @@ const DEFAULT_CONFIG = {
     "streaming": true,
     "compression": "brotli",
     "parallelProcessing": true,
-    "minimalLogging": true
-  },
-  "reports": {
+    "minimalLogging": true
+  },
+  "autoTranslate": {
+    "placeholderMode": "preserve",
+    "concurrency": 6,
+    "batchSize": 100,
+    "progressInterval": 25,
+    "retryCount": 3,
+    "retryDelay": 1000,
+    "timeout": 15000,
+    "dryRunFirst": true,
+    "reportStdout": true,
+    "bom": false,
+    "protectionEnabled": true,
+    "protectionFile": "./i18ntk-auto-translate.json",
+    "promptProtectionSetup": true,
+    "promptProtectionUpdate": true
+  },
+  "reports": {
     "format": "json",
     "includeStats": true,
     "includeMissingKeys": true,

package/utils/security.js

@@ -687,12 +687,13 @@ static _logging = false;
       'projectRoot', 'sourceDir', 'i18nDir', 'outputDir', 'backupDir', 'tempDir', 'cacheDir', 'configDir',
       // Language settings
       'sourceLanguage', 'uiLanguage', 'language', 'defaultLanguages', 'supportedLanguages',
-      // Translation markers and content
-      'notTranslatedMarker', 'notTranslatedMarkers', 'translatedMarker', 'translatedMarkers',
+      // Translation markers and content
+      'notTranslatedMarker', 'notTranslatedMarkers', 'translatedMarker', 'translatedMarkers',
+      'allowedEnglishTerms', 'englishContentThresholdPercent',
       // File handling
       'supportedExtensions', 'excludeFiles', 'excludeDirs', 'includeFiles', 'includeDirs',
       // Operational settings
-      'strictMode', 'debug', 'displayPaths', 'version', 'scriptDirectories',
+      'strictMode', 'debug', 'displayPaths', 'version', 'scriptDirectories', 'autoTranslate',
       // Framework and processing
       'framework', 'processing', 'performance', 'advanced',
       // UI and theme settings

package/utils/translate/cli.js

@@ -6,18 +6,19 @@ const PLACEHOLDER_WARNING = [
   '  WARNING: DYNAMIC PLACEHOLDER TOKENS DETECTED',
   '============================================================',
   '',
-  '  Google Translate will attempt to translate the ENTIRE',
-  '  string value, including any placeholder tokens like:',
+  '  Auto Translate can preserve placeholders while translating',
+  '  only the text around tokens like:',
   '',
   '    {name}    {{count}}    %d    %s    :param    ${var}',
   '',
-  '  This WILL corrupt or alter your placeholders, which',
-  '  will break runtime substitution in your application.',
+  '  Sending placeholders to a translation provider can corrupt',
+  '  runtime substitution in your application.',
   '',
-  '  You have two choices for strings containing placeholders:',
+  '  You have three choices for strings containing placeholders:',
   '',
-  '    SKIP  - Copy verbatim (safe); manually translate later',
-  '    SEND  - Translate anyway (risky); may corrupt placeholders',
+  '    PRESERVE - Translate text segments and reinsert placeholders',
+  '    SKIP     - Copy verbatim; manually translate later',
+  '    SEND     - Translate anyway with masking',
   '',
   '============================================================',
 ].join('\n');
@@ -28,18 +29,20 @@ async function confirmGlobalChoice() {
   console.log('  What should we do with ALL strings that contain');
   console.log('  dynamic placeholder tokens?');
   console.log('');
-  console.log('  [s] SKIP all  - Copy verbatim, translate nothing with placeholders');
-  console.log('  [t] SEND all  - Translate everything, accept corruption risk');
-  console.log('  [i] ASK each  - Decide individually for each key');
+  console.log('  [p] PRESERVE all - Translate text around placeholders (recommended)');
+  console.log('  [s] SKIP all     - Copy verbatim, translate nothing with placeholders');
+  console.log('  [t] SEND all     - Translate everything with placeholder masking');
+  console.log('  [i] ASK each     - Decide individually for each key');
   console.log('');
 
   while (true) {
-    const answer = await ask('  Choice [s/t/i]: ');
+    const answer = await ask('  Choice [p/s/t/i]: ');
     const lower = answer.toLowerCase().trim();
+    if (lower === '' || lower === 'p' || lower === 'preserve') return { strategy: 'preserve', interactive: false };
     if (lower === 's' || lower === 'skip') return { strategy: 'skip', interactive: false };
     if (lower === 't' || lower === 'send') return { strategy: 'send', interactive: false };
-    if (lower === 'i' || lower === 'ask' || lower === 'interactive') return { strategy: 'skip', interactive: true };
-    console.log('  Please enter s, t, or i.');
+    if (lower === 'i' || lower === 'ask' || lower === 'interactive') return { strategy: 'preserve', interactive: true };
+    console.log('  Please enter p, s, t, or i.');
   }
 }
 
@@ -52,13 +55,14 @@ async function confirmPerKey(keyPath, value, placeholders) {
   console.log('');
 
   while (true) {
-    const answer = await ask('  [s]kip / [t]ranslate anyway / s[k]ip all remaining / [a]ll remaining? ');
+    const answer = await ask('  [p]reserve / [s]kip / [t]ranslate masked / s[k]ip all / [a]ll preserve? ');
     const lower = answer.toLowerCase().trim();
+    if (lower === '' || lower === 'p' || lower === 'preserve') return 'preserve';
     if (lower === 's' || lower === 'skip') return 'skip';
     if (lower === 't' || lower === 'translate') return 'send';
     if (lower === 'k' || lower === 'skipall') return 'skip-all';
-    if (lower === 'a' || lower === 'all') return 'send-all';
-    console.log('  Please enter s, t, k, or a.');
+    if (lower === 'a' || lower === 'all') return 'preserve-all';
+    console.log('  Please enter p, s, t, k, or a.');
   }
 }
 

package/utils/translate/placeholder.js

@@ -40,6 +40,7 @@ function detectPlaceholders(value, customPatterns) {
   const patterns = compilePatterns(customPatterns);
   const found = new Set();
   for (const pattern of patterns) {
+    pattern.lastIndex = 0;
     const matches = value.match(pattern);
     if (matches) {
       for (const m of matches) found.add(m);
@@ -52,11 +53,68 @@ function hasPlaceholders(value, customPatterns) {
   if (!value || typeof value !== 'string') return false;
   const patterns = compilePatterns(customPatterns);
   for (const pattern of patterns) {
+    pattern.lastIndex = 0;
     if (pattern.test(value)) return true;
   }
   return false;
 }
 
+function splitByPlaceholders(value, customPatterns) {
+  if (!value || typeof value !== 'string') {
+    return [{ type: 'text', value: value || '' }];
+  }
+
+  const patterns = compilePatterns(customPatterns);
+  const matches = [];
+
+  for (const pattern of patterns) {
+    pattern.lastIndex = 0;
+    let match;
+    while ((match = pattern.exec(value)) !== null) {
+      if (!match[0]) {
+        pattern.lastIndex++;
+        continue;
+      }
+      matches.push({
+        start: match.index,
+        end: match.index + match[0].length,
+        value: match[0],
+      });
+    }
+  }
+
+  if (matches.length === 0) {
+    return [{ type: 'text', value }];
+  }
+
+  matches.sort((a, b) => {
+    if (a.start !== b.start) return a.start - b.start;
+    return (b.end - b.start) - (a.end - a.start);
+  });
+
+  const accepted = [];
+  for (const match of matches) {
+    const overlaps = accepted.some((item) => match.start < item.end && match.end > item.start);
+    if (!overlaps) accepted.push(match);
+  }
+  accepted.sort((a, b) => a.start - b.start);
+
+  const segments = [];
+  let cursor = 0;
+  for (const match of accepted) {
+    if (match.start > cursor) {
+      segments.push({ type: 'text', value: value.slice(cursor, match.start) });
+    }
+    segments.push({ type: 'placeholder', value: match.value });
+    cursor = match.end;
+  }
+  if (cursor < value.length) {
+    segments.push({ type: 'text', value: value.slice(cursor) });
+  }
+
+  return segments;
+}
+
 function maskPlaceholders(value, customPatterns) {
   if (!value || typeof value !== 'string') return { masked: value, map: new Map() };
   const patterns = compilePatterns(customPatterns);
@@ -64,6 +122,7 @@ function maskPlaceholders(value, customPatterns) {
   let idx = 0;
   let masked = value;
   for (const pattern of patterns) {
+    pattern.lastIndex = 0;
     masked = masked.replace(pattern, (match) => {
       const ph = `\uE000${idx}\uE001`;
       map.set(ph, match);
@@ -88,6 +147,7 @@ module.exports = {
   compilePatterns,
   detectPlaceholders,
   hasPlaceholders,
+  splitByPlaceholders,
   maskPlaceholders,
   unmaskPlaceholders,
 };

package/utils/translate/protection.js

@@ -0,0 +1,243 @@
+const path = require('path');
+const SecurityUtils = require('../security');
+
+const DEFAULT_PROTECTION_FILE = 'i18ntk-auto-translate.json';
+const TOKEN_PREFIX = '__I18NTK_KEEP_';
+
+function defaultProtectionConfig() {
+  return {
+    version: 1,
+    description: 'Auto Translate protection rules. Terms are masked before translation and restored after translation. Keys and values are copied from the source without translation.',
+    terms: [],
+    keys: [],
+    values: [],
+    patterns: []
+  };
+}
+
+function resolveProtectionFile(filePath, cwd = process.cwd()) {
+  const requested = filePath || DEFAULT_PROTECTION_FILE;
+  const resolved = path.isAbsolute(requested)
+    ? path.resolve(requested)
+    : path.resolve(cwd, requested);
+  const root = path.resolve(cwd);
+
+  if (!resolved.startsWith(root + path.sep) && resolved !== root) {
+    throw new Error(`Protection file must be inside the project: ${requested}`);
+  }
+
+  return resolved;
+}
+
+function normalizeList(value) {
+  if (!Array.isArray(value)) return [];
+  return value
+    .filter(item => typeof item === 'string')
+    .map(item => item.trim())
+    .filter(Boolean);
+}
+
+function compilePatterns(patterns) {
+  const compiled = [];
+  for (const pattern of patterns) {
+    try {
+      compiled.push(new RegExp(pattern, 'g'));
+    } catch (error) {
+      console.warn(`Invalid Auto Translate protection pattern ignored: ${pattern} (${error.message})`);
+    }
+  }
+  return compiled;
+}
+
+function normalizeProtectionConfig(config = {}, filePath = null) {
+  const terms = normalizeList(config.terms);
+  const keys = normalizeList(config.keys);
+  const values = normalizeList(config.values);
+  const patterns = normalizeList(config.patterns);
+
+  return {
+    enabled: true,
+    filePath,
+    terms,
+    keys,
+    values,
+    patterns,
+    compiledPatterns: compilePatterns(patterns)
+  };
+}
+
+function createProtectionFile(filePath, options = {}) {
+  const resolved = resolveProtectionFile(filePath, options.cwd);
+  const dir = path.dirname(resolved);
+  if (!SecurityUtils.safeExistsSync(dir, path.dirname(dir))) {
+    SecurityUtils.safeMkdirSync(dir, path.dirname(dir), { recursive: true });
+  }
+
+  if (!SecurityUtils.safeExistsSync(resolved, dir)) {
+    SecurityUtils.safeWriteFileSync(
+      resolved,
+      JSON.stringify(defaultProtectionConfig(), null, 2) + '\n',
+      dir,
+      'utf8'
+    );
+  }
+
+  return resolved;
+}
+
+function readProtectionFile(filePath, options = {}) {
+  const resolved = resolveProtectionFile(filePath, options.cwd);
+  const raw = SecurityUtils.safeReadFileSync(resolved, path.dirname(resolved), 'utf8');
+  if (!raw) return defaultProtectionConfig();
+  return JSON.parse(raw.replace(/^\uFEFF/, ''));
+}
+
+function saveProtectionFile(filePath, config, options = {}) {
+  const resolved = resolveProtectionFile(filePath, options.cwd);
+  const dir = path.dirname(resolved);
+  if (!SecurityUtils.safeExistsSync(dir, path.dirname(dir))) {
+    SecurityUtils.safeMkdirSync(dir, path.dirname(dir), { recursive: true });
+  }
+  const nextConfig = {
+    ...defaultProtectionConfig(),
+    ...(config || {}),
+    terms: normalizeList(config?.terms),
+    keys: normalizeList(config?.keys),
+    values: normalizeList(config?.values),
+    patterns: normalizeList(config?.patterns)
+  };
+  SecurityUtils.safeWriteFileSync(resolved, JSON.stringify(nextConfig, null, 2) + '\n', dir, 'utf8');
+  return resolved;
+}
+
+function loadProtectionConfig(filePath, options = {}) {
+  if (options.enabled === false) {
+    return normalizeProtectionConfig({}, null);
+  }
+
+  const resolved = resolveProtectionFile(filePath, options.cwd);
+  const dir = path.dirname(resolved);
+
+  if (!SecurityUtils.safeExistsSync(resolved, dir)) {
+    if (options.create) {
+      createProtectionFile(resolved, options);
+    } else {
+      return normalizeProtectionConfig({}, resolved);
+    }
+  }
+
+  const raw = SecurityUtils.safeReadFileSync(resolved, dir, 'utf8');
+  if (!raw) {
+    return normalizeProtectionConfig({}, resolved);
+  }
+
+  let parsed;
+  try {
+    parsed = JSON.parse(raw.replace(/^\uFEFF/, ''));
+  } catch (error) {
+    throw new Error(`Invalid Auto Translate protection JSON at ${resolved}: ${error.message}`);
+  }
+
+  return normalizeProtectionConfig(parsed, resolved);
+}
+
+function escapeRegExp(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+function keyMatchesRule(keyPath, rule) {
+  if (rule === keyPath) return true;
+  if (!rule.includes('*')) return false;
+  const regex = new RegExp(`^${rule.split('*').map(escapeRegExp).join('.*')}$`);
+  return regex.test(keyPath);
+}
+
+function shouldPreserveWholeValue(keyPath, value, protection) {
+  if (!protection || protection.enabled === false) return false;
+  if (protection.keys.some(rule => keyMatchesRule(keyPath, rule))) return true;
+  const valueText = String(value);
+  return protection.values.includes(valueText) || protection.terms.includes(valueText);
+}
+
+function addReplacement(replacements, original) {
+  if (!original) return;
+  if (replacements.some(item => item.original === original)) return;
+  replacements.push({ original });
+}
+
+function collectReplacements(value, protection) {
+  const text = String(value);
+  const replacements = [];
+
+  for (const term of protection.terms || []) {
+    if (text.includes(term)) {
+      addReplacement(replacements, term);
+    }
+  }
+
+  for (const pattern of protection.compiledPatterns || []) {
+    pattern.lastIndex = 0;
+    let match;
+    while ((match = pattern.exec(text)) !== null) {
+      addReplacement(replacements, match[0]);
+      if (match[0] === '') pattern.lastIndex++;
+    }
+  }
+
+  return replacements.sort((a, b) => b.original.length - a.original.length);
+}
+
+function protectText(value, protection) {
+  if (!protection || protection.enabled === false || typeof value !== 'string') {
+    return { value, map: new Map(), count: 0 };
+  }
+
+  const replacements = collectReplacements(value, protection);
+  if (replacements.length === 0) {
+    return { value, map: new Map(), count: 0 };
+  }
+
+  let protectedValue = value;
+  const map = new Map();
+  replacements.forEach((replacement, index) => {
+    const token = `${TOKEN_PREFIX}${index}__`;
+    map.set(token, replacement.original);
+    protectedValue = protectedValue.split(replacement.original).join(token);
+  });
+
+  return { value: protectedValue, map, count: map.size };
+}
+
+function restoreText(value, map) {
+  if (!(map instanceof Map) || map.size === 0 || typeof value !== 'string') return value;
+  let restored = value;
+  for (const [token, original] of map.entries()) {
+    restored = restored.split(token).join(original);
+  }
+  return restored;
+}
+
+function hasProtectionRules(protection) {
+  return Boolean(
+    protection &&
+    (
+      protection.terms.length ||
+      protection.keys.length ||
+      protection.values.length ||
+      protection.patterns.length
+    )
+  );
+}
+
+module.exports = {
+  DEFAULT_PROTECTION_FILE,
+  createProtectionFile,
+  defaultProtectionConfig,
+  hasProtectionRules,
+  loadProtectionConfig,
+  protectText,
+  readProtectionFile,
+  restoreText,
+  saveProtectionFile,
+  shouldPreserveWholeValue
+};

package/utils/translate/report.js

@@ -1,4 +1,5 @@
-const fs = require('fs');
+const path = require('path');
+const SecurityUtils = require('../security');
 
 function generateReport(skippedKeys, translatedCount, totalCount, options = {}) {
   const {
@@ -6,7 +7,11 @@ function generateReport(skippedKeys, translatedCount, totalCount, options = {})
     targetLang,
     dryRun = false,
     timestamp = new Date().toISOString(),
+    placeholderProtected = 0,
+    protectedSkipped = 0,
   } = options;
+  const placeholderSkipped = skippedKeys.filter(key => key.skipReason !== 'protected');
+  const protectedKeys = skippedKeys.filter(key => key.skipReason === 'protected');
 
   const lines = [];
   lines.push('='.repeat(72));
@@ -20,6 +25,8 @@ function generateReport(skippedKeys, translatedCount, totalCount, options = {})
   }
   lines.push(`  Total keys:      ${totalCount}`);
   lines.push(`  Translated:      ${translatedCount}`);
+  lines.push(`  Placeholder-safe: ${String(placeholderProtected).padStart(6)}`);
+  lines.push(`  Protected:       ${String(protectedSkipped).padStart(6)}`);
   lines.push(`  Skipped:         ${skippedKeys.length}`);
   lines.push('='.repeat(72));
 
@@ -28,13 +35,20 @@ function generateReport(skippedKeys, translatedCount, totalCount, options = {})
     lines.push('  All strings were processed. No keys were skipped.');
     lines.push('');
   } else {
-    lines.push('');
-    lines.push('  WARNING: The following keys were SKIPPED because they contain');
-    lines.push('  dynamic placeholder tokens that should be manually translated');
-    lines.push('  to avoid runtime substitution breakage.');
-    lines.push('');
-    lines.push('  These entries were copied verbatim into the output file.');
-    lines.push('  You MUST manually translate them before using the file.');
+    if (placeholderSkipped.length > 0) {
+      lines.push('');
+      lines.push('  WARNING: The following keys were SKIPPED because they contain');
+      lines.push('  dynamic placeholder tokens that should be manually translated');
+      lines.push('  to avoid runtime substitution breakage.');
+      lines.push('');
+      lines.push('  These entries were copied verbatim into the output file.');
+      lines.push('  You MUST manually translate them before using the file.');
+    }
+    if (protectedKeys.length > 0) {
+      lines.push('');
+      lines.push('  The following keys were copied unchanged because they matched');
+      lines.push('  Auto Translate protection rules for keys or exact values.');
+    }
     lines.push('');
     lines.push(`  ${'-'.repeat(64)}`);
     lines.push(`  Key Path                                            Original Value`);
@@ -51,20 +65,30 @@ function generateReport(skippedKeys, translatedCount, totalCount, options = {})
     }
 
     lines.push(`  ${'-'.repeat(64)}`);
-    lines.push('');
-    lines.push('  REMINDER:');
-    lines.push('  1. Open the target JSON file');
-    lines.push('  2. Search for the keys listed above');
-    lines.push('  3. Manually translate each value, preserving all placeholders');
-    lines.push('     exactly as they appear in the original');
-    lines.push('  4. Verify placeholder integrity before runtime use');
-    lines.push('');
+    if (placeholderSkipped.length > 0) {
+      lines.push('');
+      lines.push('  REMINDER:');
+      lines.push('  1. Open the target JSON file');
+      lines.push('  2. Search for the placeholder-skipped keys listed above');
+      lines.push('  3. Manually translate each value, preserving all placeholders');
+      lines.push('     exactly as they appear in the original');
+      lines.push('  4. Verify placeholder integrity before runtime use');
+      lines.push('');
+    }
   }
 
   lines.push('');
-  lines.push('  The generated file can be used immediately for all');
-  lines.push('  non-placeholder text. Only the skipped keys need');
-  lines.push('  manual attention.');
+  if (skippedKeys.length === 0) {
+    lines.push('  The generated file can be used immediately after review.');
+    lines.push('  Placeholder tokens were preserved automatically where found.');
+  } else if (placeholderSkipped.length > 0) {
+    lines.push('  The generated file can be used immediately for all');
+    lines.push('  translated text. Only the skipped keys need');
+    lines.push('  manual attention.');
+  } else {
+    lines.push('  Protected keys and values were intentionally copied unchanged.');
+    lines.push('  Review the output file before using it in production.');
+  }
   lines.push('='.repeat(72));
 
   return lines.join('\n');
@@ -73,14 +97,17 @@ function generateReport(skippedKeys, translatedCount, totalCount, options = {})
 function writeReport(reportText, filePath) {
   if (!filePath) return;
   try {
-    fs.writeFileSync(filePath, reportText + '\n', 'utf-8');
+    const resolvedPath = path.resolve(process.cwd(), filePath);
+    SecurityUtils.safeWriteFileSync(resolvedPath, reportText + '\n', path.dirname(resolvedPath), 'utf-8');
   } catch (e) {
     console.error('Failed to write report file:', e.message);
   }
 }
 
-function formatSummaryLine(skippedCount, translatedCount, totalCount) {
-  return `[translate] ${translatedCount} translated, ${skippedCount} skipped (of ${totalCount} total keys)`;
+function formatSummaryLine(skippedCount, translatedCount, totalCount, placeholderProtected = 0, protectedSkipped = 0) {
+  const protectedPart = placeholderProtected > 0 ? `, ${placeholderProtected} placeholder-safe` : '';
+  const glossaryPart = protectedSkipped > 0 ? `, ${protectedSkipped} protected` : '';
+  return `[translate] ${translatedCount} translated${protectedPart}${glossaryPart}, ${skippedCount} skipped (of ${totalCount} total keys)`;
 }
 
 module.exports = {