i18ntk 2.6.0 → 3.1.0

package/utils/translate/api.js

@@ -0,0 +1,168 @@
+const https = require('https');
+const http = require('http');
+const { URL } = require('url');
+
+const DEFAULT_CONCURRENCY = 3;
+const DEFAULT_RETRY_COUNT = 3;
+const DEFAULT_RETRY_DELAY = 1000;
+const MAX_BACKOFF_DELAY = 30000;
+
+function httpGet(urlString, timeout = 15000) {
+  return new Promise((resolve) => {
+    const url = new URL(urlString);
+    const client = url.protocol === 'https:' ? https : http;
+    const req = client.get(urlString, { timeout }, (res) => {
+      let data = '';
+      res.on('data', (chunk) => { data += chunk; });
+      res.on('end', () => {
+        try {
+          const parsed = JSON.parse(data);
+          resolve({ ok: true, data: parsed, status: res.statusCode });
+        } catch (e) {
+          resolve({ ok: false, error: 'ParseError', raw: data.substring(0, 500), status: res.statusCode });
+        }
+      });
+    });
+    req.on('error', (e) => {
+      resolve({ ok: false, error: e.code || 'NetworkError', message: e.message });
+    });
+    req.on('timeout', () => {
+      req.destroy();
+      resolve({ ok: false, error: 'TimeoutError', message: 'Request timed out' });
+    });
+  });
+}
+
+function extractTranslation(result) {
+  if (result && Array.isArray(result) && result[0]) {
+    return result[0]
+      .filter((part) => part && typeof part[0] === 'string')
+      .map((part) => part[0])
+      .join('')
+      .trim() || null;
+  }
+  return null;
+}
+
+function detectRateLimitError(result) {
+  if (!result.ok && result.status === 429) return true;
+  if (result.ok && result.status === 429) return true;
+  if (!result.ok && result.error === 'TimeoutError') return false;
+  if (result.data && (result.data.error || result.data.error_description)) return true;
+  return false;
+}
+
+async function translateText(text, targetLang, options = {}) {
+  const {
+    sourceLang = 'en',
+    retryCount = DEFAULT_RETRY_COUNT,
+    retryDelay = DEFAULT_RETRY_DELAY,
+    customFn,
+    timeout = 15000,
+  } = options;
+
+  if (!text || text.trim().length === 0) return { ok: true, translated: text };
+
+  if (typeof customFn === 'function') {
+    try {
+      const translated = await customFn(text, { sourceLang, targetLang });
+      return { ok: true, translated: translated || text };
+    } catch (e) {
+      return { ok: false, error: 'CustomFnError', message: e.message };
+    }
+  }
+
+  const params = new URLSearchParams({
+    client: 'gtx',
+    sl: sourceLang,
+    tl: targetLang,
+    dt: 't',
+    q: text,
+  });
+  const url = `https://translate.googleapis.com/translate_a/single?${params.toString()}`;
+
+  let lastError = null;
+  for (let attempt = 0; attempt < retryCount; attempt++) {
+    if (attempt > 0) {
+      const delay = Math.min(retryDelay * Math.pow(2, attempt - 1), MAX_BACKOFF_DELAY);
+      await new Promise((resolve) => setTimeout(resolve, delay));
+    }
+
+    const result = await httpGet(url, timeout);
+
+    if (result.ok) {
+      const translated = extractTranslation(result.data);
+      if (translated !== null && translated !== text) {
+        return { ok: true, translated };
+      }
+      if (translated === text) {
+        return { ok: true, translated: text };
+      }
+      if (result.status === 429) {
+        lastError = { error: 'RateLimited', message: 'Google Translate rate limit hit' };
+        continue;
+      }
+    }
+
+    if (detectRateLimitError(result)) {
+      lastError = { error: 'RateLimited', message: 'Rate limit detected' };
+      continue;
+    }
+
+    lastError = { error: result.error || 'UnknownError', message: result.message || 'Request failed' };
+  }
+
+  return { ok: false, translated: null, ...lastError };
+}
+
+async function translateBatch(batch, targetLang, options = {}) {
+  const {
+    concurrency = DEFAULT_CONCURRENCY,
+    onProgress,
+    onError,
+  } = options;
+
+  const results = new Array(batch.length).fill(null);
+  let idx = 0;
+  let completed = 0;
+
+  async function worker() {
+    while (idx < batch.length) {
+      const i = idx++;
+      const item = batch[i];
+      const value = typeof item === 'string' ? item : item.value;
+      const result = await translateText(value, targetLang, options);
+
+      if (result.ok) {
+        results[i] = result.translated;
+      } else {
+        results[i] = value;
+        if (typeof onError === 'function') {
+          onError({ index: i, item, error: result.error, message: result.message });
+        }
+      }
+
+      completed++;
+      if (typeof onProgress === 'function') {
+        onProgress({ completed, total: batch.length, index: i, ok: result.ok });
+      }
+    }
+  }
+
+  const workerCount = Math.min(concurrency, batch.length);
+  const workers = Array.from({ length: workerCount }, () => worker());
+  await Promise.all(workers);
+
+  return results;
+}
+
+module.exports = {
+  httpGet,
+  extractTranslation,
+  detectRateLimitError,
+  translateText,
+  translateBatch,
+  DEFAULT_CONCURRENCY,
+  DEFAULT_RETRY_COUNT,
+  DEFAULT_RETRY_DELAY,
+};

package/utils/translate/cli.js

@@ -0,0 +1,95 @@
+const { ask } = require('../cli');
+
+const PLACEHOLDER_WARNING = [
+  '',
+  '============================================================',
+  '  WARNING: DYNAMIC PLACEHOLDER TOKENS DETECTED',
+  '============================================================',
+  '',
+  '  Auto Translate can preserve placeholders while translating',
+  '  only the text around tokens like:',
+  '',
+  '    {name}    {{count}}    %d    %s    :param    ${var}',
+  '',
+  '  Sending placeholders to a translation provider can corrupt',
+  '  runtime substitution in your application.',
+  '',
+  '  You have three choices for strings containing placeholders:',
+  '',
+  '    PRESERVE - Translate text segments and reinsert placeholders',
+  '    SKIP     - Copy verbatim; manually translate later',
+  '    SEND     - Translate anyway with masking',
+  '',
+  '============================================================',
+].join('\n');
+
+async function confirmGlobalChoice() {
+  console.log(PLACEHOLDER_WARNING);
+  console.log('');
+  console.log('  What should we do with ALL strings that contain');
+  console.log('  dynamic placeholder tokens?');
+  console.log('');
+  console.log('  [p] PRESERVE all - Translate text around placeholders (recommended)');
+  console.log('  [s] SKIP all     - Copy verbatim, translate nothing with placeholders');
+  console.log('  [t] SEND all     - Translate everything with placeholder masking');
+  console.log('  [i] ASK each     - Decide individually for each key');
+  console.log('');
+
+  while (true) {
+    const answer = await ask('  Choice [p/s/t/i]: ');
+    const lower = answer.toLowerCase().trim();
+    if (lower === '' || lower === 'p' || lower === 'preserve') return { strategy: 'preserve', interactive: false };
+    if (lower === 's' || lower === 'skip') return { strategy: 'skip', interactive: false };
+    if (lower === 't' || lower === 'send') return { strategy: 'send', interactive: false };
+    if (lower === 'i' || lower === 'ask' || lower === 'interactive') return { strategy: 'preserve', interactive: true };
+    console.log('  Please enter p, s, t, or i.');
+  }
+}
+
+async function confirmPerKey(keyPath, value, placeholders) {
+  const displayVal = value.length > 60 ? value.substring(0, 57) + '...' : value;
+  console.log('');
+  console.log(`  Key:   ${keyPath}`);
+  console.log(`  Value: "${displayVal}"`);
+  console.log(`  Placeholders: ${placeholders.join(', ')}`);
+  console.log('');
+
+  while (true) {
+    const answer = await ask('  [p]reserve / [s]kip / [t]ranslate masked / s[k]ip all / [a]ll preserve? ');
+    const lower = answer.toLowerCase().trim();
+    if (lower === '' || lower === 'p' || lower === 'preserve') return 'preserve';
+    if (lower === 's' || lower === 'skip') return 'skip';
+    if (lower === 't' || lower === 'translate') return 'send';
+    if (lower === 'k' || lower === 'skipall') return 'skip-all';
+    if (lower === 'a' || lower === 'all') return 'preserve-all';
+    console.log('  Please enter p, s, t, k, or a.');
+  }
+}
+
+async function previewSkipped(skippedLeaves) {
+  console.log('');
+  console.log('============================================================');
+  console.log('  DRY-RUN: Keys that would be skipped (placeholders found):');
+  console.log('============================================================');
+  console.log('');
+  if (skippedLeaves.length === 0) {
+    console.log('  No keys would be skipped.');
+  } else {
+    for (const leaf of skippedLeaves) {
+      const displayVal = leaf.value.length > 60 ? leaf.value.substring(0, 57) + '...' : leaf.value;
+      console.log(`  ${leaf.keyPath}`);
+      console.log(`    "${displayVal}"`);
+      console.log(`    Placeholders: ${leaf.placeholders.join(', ')}`);
+      console.log('');
+    }
+    console.log(`  Total keys that would be skipped: ${skippedLeaves.length}`);
+  }
+  console.log('');
+}
+
+module.exports = {
+  PLACEHOLDER_WARNING,
+  confirmGlobalChoice,
+  confirmPerKey,
+  previewSkipped,
+};

package/utils/translate/placeholder.js

@@ -0,0 +1,153 @@
+const DEFAULT_PLACEHOLDER_PATTERNS = [
+  /\{\{[^}]+\}\}/g,         // {{variable}} - double curly (Handlebars, Mustache)
+  /\{[a-zA-Z_]\w*\}/g,       // {name} - single curly (i18next, Python format named)
+  /\{\d+\}/g,                // {0} - indexed curly
+  /%\d?\$?[sd]/g,            // %d, %s, %1$s, %2$d (printf-style)
+  /:[a-zA-Z_]\w*/g,          // :param - colon-style (Rails, Swift)
+  /%\{[a-zA-Z_]\w*\}/g,      // %{name} - Ruby/Perl named
+  /%\([a-zA-Z_]\w*\)[sd]/g,  // %(name)s - Python named format (with type)
+  /\$\{[a-zA-Z_]\w*\}/g,     // ${variable} - JS template literal style
+  /<[a-zA-Z_]\w*>/g,         // <name> - XML/HTML-style
+  /@[a-zA-Z_]\w*/g,          // @param - Java/Spring-style
+  /&[a-zA-Z_]\w*;?/g,        // &amp; HTML entity style (careful, broad match)
+];
+
+function compilePatterns(customPatterns) {
+  const patterns = [...DEFAULT_PLACEHOLDER_PATTERNS];
+  if (customPatterns) {
+    const customs = Array.isArray(customPatterns) ? customPatterns : [customPatterns];
+    for (const pat of customs) {
+      if (typeof pat === 'string') {
+        try {
+          patterns.push(new RegExp(pat, 'g'));
+        } catch (e) {
+          console.warn('Invalid custom regex pattern ignored:', pat, e.message);
+        }
+      } else if (pat instanceof RegExp) {
+        if (!pat.global) {
+          patterns.push(new RegExp(pat.source, 'g'));
+        } else {
+          patterns.push(pat);
+        }
+      }
+    }
+  }
+  return patterns;
+}
+
+function detectPlaceholders(value, customPatterns) {
+  if (!value || typeof value !== 'string') return [];
+  const patterns = compilePatterns(customPatterns);
+  const found = new Set();
+  for (const pattern of patterns) {
+    pattern.lastIndex = 0;
+    const matches = value.match(pattern);
+    if (matches) {
+      for (const m of matches) found.add(m);
+    }
+  }
+  return Array.from(found);
+}
+
+function hasPlaceholders(value, customPatterns) {
+  if (!value || typeof value !== 'string') return false;
+  const patterns = compilePatterns(customPatterns);
+  for (const pattern of patterns) {
+    pattern.lastIndex = 0;
+    if (pattern.test(value)) return true;
+  }
+  return false;
+}
+
+function splitByPlaceholders(value, customPatterns) {
+  if (!value || typeof value !== 'string') {
+    return [{ type: 'text', value: value || '' }];
+  }
+
+  const patterns = compilePatterns(customPatterns);
+  const matches = [];
+
+  for (const pattern of patterns) {
+    pattern.lastIndex = 0;
+    let match;
+    while ((match = pattern.exec(value)) !== null) {
+      if (!match[0]) {
+        pattern.lastIndex++;
+        continue;
+      }
+      matches.push({
+        start: match.index,
+        end: match.index + match[0].length,
+        value: match[0],
+      });
+    }
+  }
+
+  if (matches.length === 0) {
+    return [{ type: 'text', value }];
+  }
+
+  matches.sort((a, b) => {
+    if (a.start !== b.start) return a.start - b.start;
+    return (b.end - b.start) - (a.end - a.start);
+  });
+
+  const accepted = [];
+  for (const match of matches) {
+    const overlaps = accepted.some((item) => match.start < item.end && match.end > item.start);
+    if (!overlaps) accepted.push(match);
+  }
+  accepted.sort((a, b) => a.start - b.start);
+
+  const segments = [];
+  let cursor = 0;
+  for (const match of accepted) {
+    if (match.start > cursor) {
+      segments.push({ type: 'text', value: value.slice(cursor, match.start) });
+    }
+    segments.push({ type: 'placeholder', value: match.value });
+    cursor = match.end;
+  }
+  if (cursor < value.length) {
+    segments.push({ type: 'text', value: value.slice(cursor) });
+  }
+
+  return segments;
+}
+
+function maskPlaceholders(value, customPatterns) {
+  if (!value || typeof value !== 'string') return { masked: value, map: new Map() };
+  const patterns = compilePatterns(customPatterns);
+  const map = new Map();
+  let idx = 0;
+  let masked = value;
+  for (const pattern of patterns) {
+    pattern.lastIndex = 0;
+    masked = masked.replace(pattern, (match) => {
+      const ph = `\uE000${idx}\uE001`;
+      map.set(ph, match);
+      idx++;
+      return ph;
+    });
+  }
+  return { masked, map };
+}
+
+function unmaskPlaceholders(value, map) {
+  if (!map || map.size === 0) return value;
+  let result = value;
+  map.forEach((original, placeholder) => {
+    result = result.split(placeholder).join(original);
+  });
+  return result;
+}
+
+module.exports = {
+  DEFAULT_PLACEHOLDER_PATTERNS,
+  compilePatterns,
+  detectPlaceholders,
+  hasPlaceholders,
+  splitByPlaceholders,
+  maskPlaceholders,
+  unmaskPlaceholders,
+};

package/utils/translate/protection.js

@@ -0,0 +1,243 @@
+const path = require('path');
+const SecurityUtils = require('../security');
+
+const DEFAULT_PROTECTION_FILE = 'i18ntk-auto-translate.json';
+const TOKEN_PREFIX = '__I18NTK_KEEP_';
+
+function defaultProtectionConfig() {
+  return {
+    version: 1,
+    description: 'Auto Translate protection rules. Terms are masked before translation and restored after translation. Keys and values are copied from the source without translation.',
+    terms: [],
+    keys: [],
+    values: [],
+    patterns: []
+  };
+}
+
+function resolveProtectionFile(filePath, cwd = process.cwd()) {
+  const requested = filePath || DEFAULT_PROTECTION_FILE;
+  const resolved = path.isAbsolute(requested)
+    ? path.resolve(requested)
+    : path.resolve(cwd, requested);
+  const root = path.resolve(cwd);
+
+  if (!resolved.startsWith(root + path.sep) && resolved !== root) {
+    throw new Error(`Protection file must be inside the project: ${requested}`);
+  }
+
+  return resolved;
+}
+
+function normalizeList(value) {
+  if (!Array.isArray(value)) return [];
+  return value
+    .filter(item => typeof item === 'string')
+    .map(item => item.trim())
+    .filter(Boolean);
+}
+
+function compilePatterns(patterns) {
+  const compiled = [];
+  for (const pattern of patterns) {
+    try {
+      compiled.push(new RegExp(pattern, 'g'));
+    } catch (error) {
+      console.warn(`Invalid Auto Translate protection pattern ignored: ${pattern} (${error.message})`);
+    }
+  }
+  return compiled;
+}
+
+function normalizeProtectionConfig(config = {}, filePath = null) {
+  const terms = normalizeList(config.terms);
+  const keys = normalizeList(config.keys);
+  const values = normalizeList(config.values);
+  const patterns = normalizeList(config.patterns);
+
+  return {
+    enabled: true,
+    filePath,
+    terms,
+    keys,
+    values,
+    patterns,
+    compiledPatterns: compilePatterns(patterns)
+  };
+}
+
+function createProtectionFile(filePath, options = {}) {
+  const resolved = resolveProtectionFile(filePath, options.cwd);
+  const dir = path.dirname(resolved);
+  if (!SecurityUtils.safeExistsSync(dir, path.dirname(dir))) {
+    SecurityUtils.safeMkdirSync(dir, path.dirname(dir), { recursive: true });
+  }
+
+  if (!SecurityUtils.safeExistsSync(resolved, dir)) {
+    SecurityUtils.safeWriteFileSync(
+      resolved,
+      JSON.stringify(defaultProtectionConfig(), null, 2) + '\n',
+      dir,
+      'utf8'
+    );
+  }
+
+  return resolved;
+}
+
+function readProtectionFile(filePath, options = {}) {
+  const resolved = resolveProtectionFile(filePath, options.cwd);
+  const raw = SecurityUtils.safeReadFileSync(resolved, path.dirname(resolved), 'utf8');
+  if (!raw) return defaultProtectionConfig();
+  return JSON.parse(raw.replace(/^\uFEFF/, ''));
+}
+
+function saveProtectionFile(filePath, config, options = {}) {
+  const resolved = resolveProtectionFile(filePath, options.cwd);
+  const dir = path.dirname(resolved);
+  if (!SecurityUtils.safeExistsSync(dir, path.dirname(dir))) {
+    SecurityUtils.safeMkdirSync(dir, path.dirname(dir), { recursive: true });
+  }
+  const nextConfig = {
+    ...defaultProtectionConfig(),
+    ...(config || {}),
+    terms: normalizeList(config?.terms),
+    keys: normalizeList(config?.keys),
+    values: normalizeList(config?.values),
+    patterns: normalizeList(config?.patterns)
+  };
+  SecurityUtils.safeWriteFileSync(resolved, JSON.stringify(nextConfig, null, 2) + '\n', dir, 'utf8');
+  return resolved;
+}
+
+function loadProtectionConfig(filePath, options = {}) {
+  if (options.enabled === false) {
+    return normalizeProtectionConfig({}, null);
+  }
+
+  const resolved = resolveProtectionFile(filePath, options.cwd);
+  const dir = path.dirname(resolved);
+
+  if (!SecurityUtils.safeExistsSync(resolved, dir)) {
+    if (options.create) {
+      createProtectionFile(resolved, options);
+    } else {
+      return normalizeProtectionConfig({}, resolved);
+    }
+  }
+
+  const raw = SecurityUtils.safeReadFileSync(resolved, dir, 'utf8');
+  if (!raw) {
+    return normalizeProtectionConfig({}, resolved);
+  }
+
+  let parsed;
+  try {
+    parsed = JSON.parse(raw.replace(/^\uFEFF/, ''));
+  } catch (error) {
+    throw new Error(`Invalid Auto Translate protection JSON at ${resolved}: ${error.message}`);
+  }
+
+  return normalizeProtectionConfig(parsed, resolved);
+}
+
+function escapeRegExp(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+function keyMatchesRule(keyPath, rule) {
+  if (rule === keyPath) return true;
+  if (!rule.includes('*')) return false;
+  const regex = new RegExp(`^${rule.split('*').map(escapeRegExp).join('.*')}$`);
+  return regex.test(keyPath);
+}
+
+function shouldPreserveWholeValue(keyPath, value, protection) {
+  if (!protection || protection.enabled === false) return false;
+  if (protection.keys.some(rule => keyMatchesRule(keyPath, rule))) return true;
+  const valueText = String(value);
+  return protection.values.includes(valueText) || protection.terms.includes(valueText);
+}
+
+function addReplacement(replacements, original) {
+  if (!original) return;
+  if (replacements.some(item => item.original === original)) return;
+  replacements.push({ original });
+}
+
+function collectReplacements(value, protection) {
+  const text = String(value);
+  const replacements = [];
+
+  for (const term of protection.terms || []) {
+    if (text.includes(term)) {
+      addReplacement(replacements, term);
+    }
+  }
+
+  for (const pattern of protection.compiledPatterns || []) {
+    pattern.lastIndex = 0;
+    let match;
+    while ((match = pattern.exec(text)) !== null) {
+      addReplacement(replacements, match[0]);
+      if (match[0] === '') pattern.lastIndex++;
+    }
+  }
+
+  return replacements.sort((a, b) => b.original.length - a.original.length);
+}
+
+function protectText(value, protection) {
+  if (!protection || protection.enabled === false || typeof value !== 'string') {
+    return { value, map: new Map(), count: 0 };
+  }
+
+  const replacements = collectReplacements(value, protection);
+  if (replacements.length === 0) {
+    return { value, map: new Map(), count: 0 };
+  }
+
+  let protectedValue = value;
+  const map = new Map();
+  replacements.forEach((replacement, index) => {
+    const token = `${TOKEN_PREFIX}${index}__`;
+    map.set(token, replacement.original);
+    protectedValue = protectedValue.split(replacement.original).join(token);
+  });
+
+  return { value: protectedValue, map, count: map.size };
+}
+
+function restoreText(value, map) {
+  if (!(map instanceof Map) || map.size === 0 || typeof value !== 'string') return value;
+  let restored = value;
+  for (const [token, original] of map.entries()) {
+    restored = restored.split(token).join(original);
+  }
+  return restored;
+}
+
+function hasProtectionRules(protection) {
+  return Boolean(
+    protection &&
+    (
+      protection.terms.length ||
+      protection.keys.length ||
+      protection.values.length ||
+      protection.patterns.length
+    )
+  );
+}
+
+module.exports = {
+  DEFAULT_PROTECTION_FILE,
+  createProtectionFile,
+  defaultProtectionConfig,
+  hasProtectionRules,
+  loadProtectionConfig,
+  protectText,
+  readProtectionFile,
+  restoreText,
+  saveProtectionFile,
+  shouldPreserveWholeValue
+};