i18ntk 2.6.0 → 3.0.0

package/utils/translate/api.js

@@ -0,0 +1,168 @@
+const https = require('https');
+const http = require('http');
+const { URL } = require('url');
+
+const DEFAULT_CONCURRENCY = 3;
+const DEFAULT_RETRY_COUNT = 3;
+const DEFAULT_RETRY_DELAY = 1000;
+const MAX_BACKOFF_DELAY = 30000;
+
+function httpGet(urlString, timeout = 15000) {
+  return new Promise((resolve) => {
+    const url = new URL(urlString);
+    const client = url.protocol === 'https:' ? https : http;
+    const req = client.get(urlString, { timeout }, (res) => {
+      let data = '';
+      res.on('data', (chunk) => { data += chunk; });
+      res.on('end', () => {
+        try {
+          const parsed = JSON.parse(data);
+          resolve({ ok: true, data: parsed, status: res.statusCode });
+        } catch (e) {
+          resolve({ ok: false, error: 'ParseError', raw: data.substring(0, 500), status: res.statusCode });
+        }
+      });
+    });
+    req.on('error', (e) => {
+      resolve({ ok: false, error: e.code || 'NetworkError', message: e.message });
+    });
+    req.on('timeout', () => {
+      req.destroy();
+      resolve({ ok: false, error: 'TimeoutError', message: 'Request timed out' });
+    });
+  });
+}
+
+function extractTranslation(result) {
+  if (result && Array.isArray(result) && result[0]) {
+    return result[0]
+      .filter((part) => part && typeof part[0] === 'string')
+      .map((part) => part[0])
+      .join('')
+      .trim() || null;
+  }
+  return null;
+}
+
+function detectRateLimitError(result) {
+  if (!result.ok && result.status === 429) return true;
+  if (result.ok && result.status === 429) return true;
+  if (!result.ok && result.error === 'TimeoutError') return false;
+  if (result.data && (result.data.error || result.data.error_description)) return true;
+  return false;
+}
+
+async function translateText(text, targetLang, options = {}) {
+  const {
+    sourceLang = 'en',
+    retryCount = DEFAULT_RETRY_COUNT,
+    retryDelay = DEFAULT_RETRY_DELAY,
+    customFn,
+    timeout = 15000,
+  } = options;
+
+  if (!text || text.trim().length === 0) return { ok: true, translated: text };
+
+  if (typeof customFn === 'function') {
+    try {
+      const translated = await customFn(text, { sourceLang, targetLang });
+      return { ok: true, translated: translated || text };
+    } catch (e) {
+      return { ok: false, error: 'CustomFnError', message: e.message };
+    }
+  }
+
+  const params = new URLSearchParams({
+    client: 'gtx',
+    sl: sourceLang,
+    tl: targetLang,
+    dt: 't',
+    q: text,
+  });
+  const url = `https://translate.googleapis.com/translate_a/single?${params.toString()}`;
+
+  let lastError = null;
+  for (let attempt = 0; attempt < retryCount; attempt++) {
+    if (attempt > 0) {
+      const delay = Math.min(retryDelay * Math.pow(2, attempt - 1), MAX_BACKOFF_DELAY);
+      await new Promise((resolve) => setTimeout(resolve, delay));
+    }
+
+    const result = await httpGet(url, timeout);
+
+    if (result.ok) {
+      const translated = extractTranslation(result.data);
+      if (translated !== null && translated !== text) {
+        return { ok: true, translated };
+      }
+      if (translated === text) {
+        return { ok: true, translated: text };
+      }
+      if (result.status === 429) {
+        lastError = { error: 'RateLimited', message: 'Google Translate rate limit hit' };
+        continue;
+      }
+    }
+
+    if (detectRateLimitError(result)) {
+      lastError = { error: 'RateLimited', message: 'Rate limit detected' };
+      continue;
+    }
+
+    lastError = { error: result.error || 'UnknownError', message: result.message || 'Request failed' };
+  }
+
+  return { ok: false, translated: null, ...lastError };
+}
+
+async function translateBatch(batch, targetLang, options = {}) {
+  const {
+    concurrency = DEFAULT_CONCURRENCY,
+    onProgress,
+    onError,
+  } = options;
+
+  const results = new Array(batch.length).fill(null);
+  let idx = 0;
+  let completed = 0;
+
+  async function worker() {
+    while (idx < batch.length) {
+      const i = idx++;
+      const item = batch[i];
+      const value = typeof item === 'string' ? item : item.value;
+      const result = await translateText(value, targetLang, options);
+
+      if (result.ok) {
+        results[i] = result.translated;
+      } else {
+        results[i] = value;
+        if (typeof onError === 'function') {
+          onError({ index: i, item, error: result.error, message: result.message });
+        }
+      }
+
+      completed++;
+      if (typeof onProgress === 'function') {
+        onProgress({ completed, total: batch.length, index: i, ok: result.ok });
+      }
+    }
+  }
+
+  const workerCount = Math.min(concurrency, batch.length);
+  const workers = Array.from({ length: workerCount }, () => worker());
+  await Promise.all(workers);
+
+  return results;
+}
+
+module.exports = {
+  httpGet,
+  extractTranslation,
+  detectRateLimitError,
+  translateText,
+  translateBatch,
+  DEFAULT_CONCURRENCY,
+  DEFAULT_RETRY_COUNT,
+  DEFAULT_RETRY_DELAY,
+};

package/utils/translate/cli.js

@@ -0,0 +1,91 @@
+const { ask } = require('../cli');
+
+const PLACEHOLDER_WARNING = [
+  '',
+  '============================================================',
+  '  WARNING: DYNAMIC PLACEHOLDER TOKENS DETECTED',
+  '============================================================',
+  '',
+  '  Google Translate will attempt to translate the ENTIRE',
+  '  string value, including any placeholder tokens like:',
+  '',
+  '    {name}    {{count}}    %d    %s    :param    ${var}',
+  '',
+  '  This WILL corrupt or alter your placeholders, which',
+  '  will break runtime substitution in your application.',
+  '',
+  '  You have two choices for strings containing placeholders:',
+  '',
+  '    SKIP  - Copy verbatim (safe); manually translate later',
+  '    SEND  - Translate anyway (risky); may corrupt placeholders',
+  '',
+  '============================================================',
+].join('\n');
+
+async function confirmGlobalChoice() {
+  console.log(PLACEHOLDER_WARNING);
+  console.log('');
+  console.log('  What should we do with ALL strings that contain');
+  console.log('  dynamic placeholder tokens?');
+  console.log('');
+  console.log('  [s] SKIP all  - Copy verbatim, translate nothing with placeholders');
+  console.log('  [t] SEND all  - Translate everything, accept corruption risk');
+  console.log('  [i] ASK each  - Decide individually for each key');
+  console.log('');
+
+  while (true) {
+    const answer = await ask('  Choice [s/t/i]: ');
+    const lower = answer.toLowerCase().trim();
+    if (lower === 's' || lower === 'skip') return { strategy: 'skip', interactive: false };
+    if (lower === 't' || lower === 'send') return { strategy: 'send', interactive: false };
+    if (lower === 'i' || lower === 'ask' || lower === 'interactive') return { strategy: 'skip', interactive: true };
+    console.log('  Please enter s, t, or i.');
+  }
+}
+
+async function confirmPerKey(keyPath, value, placeholders) {
+  const displayVal = value.length > 60 ? value.substring(0, 57) + '...' : value;
+  console.log('');
+  console.log(`  Key:   ${keyPath}`);
+  console.log(`  Value: "${displayVal}"`);
+  console.log(`  Placeholders: ${placeholders.join(', ')}`);
+  console.log('');
+
+  while (true) {
+    const answer = await ask('  [s]kip / [t]ranslate anyway / s[k]ip all remaining / [a]ll remaining? ');
+    const lower = answer.toLowerCase().trim();
+    if (lower === 's' || lower === 'skip') return 'skip';
+    if (lower === 't' || lower === 'translate') return 'send';
+    if (lower === 'k' || lower === 'skipall') return 'skip-all';
+    if (lower === 'a' || lower === 'all') return 'send-all';
+    console.log('  Please enter s, t, k, or a.');
+  }
+}
+
+async function previewSkipped(skippedLeaves) {
+  console.log('');
+  console.log('============================================================');
+  console.log('  DRY-RUN: Keys that would be skipped (placeholders found):');
+  console.log('============================================================');
+  console.log('');
+  if (skippedLeaves.length === 0) {
+    console.log('  No keys would be skipped.');
+  } else {
+    for (const leaf of skippedLeaves) {
+      const displayVal = leaf.value.length > 60 ? leaf.value.substring(0, 57) + '...' : leaf.value;
+      console.log(`  ${leaf.keyPath}`);
+      console.log(`    "${displayVal}"`);
+      console.log(`    Placeholders: ${leaf.placeholders.join(', ')}`);
+      console.log('');
+    }
+    console.log(`  Total keys that would be skipped: ${skippedLeaves.length}`);
+  }
+  console.log('');
+}
+
+module.exports = {
+  PLACEHOLDER_WARNING,
+  confirmGlobalChoice,
+  confirmPerKey,
+  previewSkipped,
+};

package/utils/translate/placeholder.js

@@ -0,0 +1,93 @@
+const DEFAULT_PLACEHOLDER_PATTERNS = [
+  /\{\{[^}]+\}\}/g,         // {{variable}} - double curly (Handlebars, Mustache)
+  /\{[a-zA-Z_]\w*\}/g,       // {name} - single curly (i18next, Python format named)
+  /\{\d+\}/g,                // {0} - indexed curly
+  /%\d?\$?[sd]/g,            // %d, %s, %1$s, %2$d (printf-style)
+  /:[a-zA-Z_]\w*/g,          // :param - colon-style (Rails, Swift)
+  /%\{[a-zA-Z_]\w*\}/g,      // %{name} - Ruby/Perl named
+  /%\([a-zA-Z_]\w*\)[sd]/g,  // %(name)s - Python named format (with type)
+  /\$\{[a-zA-Z_]\w*\}/g,     // ${variable} - JS template literal style
+  /<[a-zA-Z_]\w*>/g,         // <name> - XML/HTML-style
+  /@[a-zA-Z_]\w*/g,          // @param - Java/Spring-style
+  /&[a-zA-Z_]\w*;?/g,        // &amp; HTML entity style (careful, broad match)
+];
+
+function compilePatterns(customPatterns) {
+  const patterns = [...DEFAULT_PLACEHOLDER_PATTERNS];
+  if (customPatterns) {
+    const customs = Array.isArray(customPatterns) ? customPatterns : [customPatterns];
+    for (const pat of customs) {
+      if (typeof pat === 'string') {
+        try {
+          patterns.push(new RegExp(pat, 'g'));
+        } catch (e) {
+          console.warn('Invalid custom regex pattern ignored:', pat, e.message);
+        }
+      } else if (pat instanceof RegExp) {
+        if (!pat.global) {
+          patterns.push(new RegExp(pat.source, 'g'));
+        } else {
+          patterns.push(pat);
+        }
+      }
+    }
+  }
+  return patterns;
+}
+
+function detectPlaceholders(value, customPatterns) {
+  if (!value || typeof value !== 'string') return [];
+  const patterns = compilePatterns(customPatterns);
+  const found = new Set();
+  for (const pattern of patterns) {
+    const matches = value.match(pattern);
+    if (matches) {
+      for (const m of matches) found.add(m);
+    }
+  }
+  return Array.from(found);
+}
+
+function hasPlaceholders(value, customPatterns) {
+  if (!value || typeof value !== 'string') return false;
+  const patterns = compilePatterns(customPatterns);
+  for (const pattern of patterns) {
+    if (pattern.test(value)) return true;
+  }
+  return false;
+}
+
+function maskPlaceholders(value, customPatterns) {
+  if (!value || typeof value !== 'string') return { masked: value, map: new Map() };
+  const patterns = compilePatterns(customPatterns);
+  const map = new Map();
+  let idx = 0;
+  let masked = value;
+  for (const pattern of patterns) {
+    masked = masked.replace(pattern, (match) => {
+      const ph = `\uE000${idx}\uE001`;
+      map.set(ph, match);
+      idx++;
+      return ph;
+    });
+  }
+  return { masked, map };
+}
+
+function unmaskPlaceholders(value, map) {
+  if (!map || map.size === 0) return value;
+  let result = value;
+  map.forEach((original, placeholder) => {
+    result = result.split(placeholder).join(original);
+  });
+  return result;
+}
+
+module.exports = {
+  DEFAULT_PLACEHOLDER_PATTERNS,
+  compilePatterns,
+  detectPlaceholders,
+  hasPlaceholders,
+  maskPlaceholders,
+  unmaskPlaceholders,
+};

package/utils/translate/report.js

@@ -0,0 +1,90 @@
+const fs = require('fs');
+
+function generateReport(skippedKeys, translatedCount, totalCount, options = {}) {
+  const {
+    sourceFile,
+    targetLang,
+    dryRun = false,
+    timestamp = new Date().toISOString(),
+  } = options;
+
+  const lines = [];
+  lines.push('='.repeat(72));
+  lines.push('  I18NTK POST-TRANSLATION REPORT');
+  lines.push('='.repeat(72));
+  lines.push(`  Generated:       ${timestamp}`);
+  lines.push(`  Source file:     ${sourceFile || 'N/A'}`);
+  lines.push(`  Target language: ${targetLang || 'N/A'}`);
+  if (dryRun) {
+    lines.push(`  Mode:            DRY-RUN (no API calls made)`);
+  }
+  lines.push(`  Total keys:      ${totalCount}`);
+  lines.push(`  Translated:      ${translatedCount}`);
+  lines.push(`  Skipped:         ${skippedKeys.length}`);
+  lines.push('='.repeat(72));
+
+  if (skippedKeys.length === 0) {
+    lines.push('');
+    lines.push('  All strings were processed. No keys were skipped.');
+    lines.push('');
+  } else {
+    lines.push('');
+    lines.push('  WARNING: The following keys were SKIPPED because they contain');
+    lines.push('  dynamic placeholder tokens that should be manually translated');
+    lines.push('  to avoid runtime substitution breakage.');
+    lines.push('');
+    lines.push('  These entries were copied verbatim into the output file.');
+    lines.push('  You MUST manually translate them before using the file.');
+    lines.push('');
+    lines.push(`  ${'-'.repeat(64)}`);
+    lines.push(`  Key Path                                            Original Value`);
+    lines.push(`  ${'-'.repeat(64)}`);
+
+    for (const skip of skippedKeys) {
+      const keyDisplay = skip.keyPath.length > 50
+        ? skip.keyPath.substring(0, 47) + '...'
+        : skip.keyPath.padEnd(50);
+      const valDisplay = skip.value.length > 80
+        ? skip.value.substring(0, 77) + '...'
+        : skip.value;
+      lines.push(`  ${keyDisplay} ${valDisplay}`);
+    }
+
+    lines.push(`  ${'-'.repeat(64)}`);
+    lines.push('');
+    lines.push('  REMINDER:');
+    lines.push('  1. Open the target JSON file');
+    lines.push('  2. Search for the keys listed above');
+    lines.push('  3. Manually translate each value, preserving all placeholders');
+    lines.push('     exactly as they appear in the original');
+    lines.push('  4. Verify placeholder integrity before runtime use');
+    lines.push('');
+  }
+
+  lines.push('');
+  lines.push('  The generated file can be used immediately for all');
+  lines.push('  non-placeholder text. Only the skipped keys need');
+  lines.push('  manual attention.');
+  lines.push('='.repeat(72));
+
+  return lines.join('\n');
+}
+
+function writeReport(reportText, filePath) {
+  if (!filePath) return;
+  try {
+    fs.writeFileSync(filePath, reportText + '\n', 'utf-8');
+  } catch (e) {
+    console.error('Failed to write report file:', e.message);
+  }
+}
+
+function formatSummaryLine(skippedCount, translatedCount, totalCount) {
+  return `[translate] ${translatedCount} translated, ${skippedCount} skipped (of ${totalCount} total keys)`;
+}
+
+module.exports = {
+  generateReport,
+  writeReport,
+  formatSummaryLine,
+};

package/utils/translate/traverse.js

@@ -0,0 +1,148 @@
+function deepTraverse(obj, visitor, keyPath = '') {
+  if (obj === null || obj === undefined) {
+    visitor({ type: 'null', keyPath, value: null });
+    return;
+  }
+  if (typeof obj === 'string') {
+    visitor({ type: 'string', keyPath, value: obj });
+    return;
+  }
+  if (typeof obj === 'number' || typeof obj === 'boolean') {
+    visitor({ type: 'leaf', keyPath, value: obj });
+    return;
+  }
+  if (Array.isArray(obj)) {
+    visitor({ type: 'array-start', keyPath, value: obj });
+    for (let i = 0; i < obj.length; i++) {
+      const childPath = keyPath ? `${keyPath}[${i}]` : `[${i}]`;
+      deepTraverse(obj[i], visitor, childPath);
+    }
+    visitor({ type: 'array-end', keyPath, value: obj });
+    return;
+  }
+  if (typeof obj === 'object') {
+    visitor({ type: 'object-start', keyPath, value: obj });
+    for (const key of Object.keys(obj)) {
+      const childPath = keyPath ? `${keyPath}.${key}` : key;
+      deepTraverse(obj[key], visitor, childPath);
+    }
+    visitor({ type: 'object-end', keyPath, value: obj });
+    return;
+  }
+  visitor({ type: 'unknown', keyPath, value: String(obj) });
+}
+
+function collectLeaves(obj, prefix = '') {
+  const entries = [];
+  for (const [key, value] of Object.entries(obj)) {
+    const fullKey = prefix ? `${prefix}.${key}` : key;
+    if (typeof value === 'string') {
+      entries.push({ keyPath: fullKey, value });
+    } else if (Array.isArray(value)) {
+      for (let i = 0; i < value.length; i++) {
+        const itemPath = `${fullKey}[${i}]`;
+        if (typeof value[i] === 'string') {
+          entries.push({ keyPath: itemPath, value: value[i] });
+        } else if (typeof value[i] === 'object' && value[i] !== null && !Array.isArray(value[i])) {
+          entries.push(...collectLeaves(value[i], itemPath));
+        }
+      }
+    } else if (typeof value === 'object' && value !== null) {
+      entries.push(...collectLeaves(value, fullKey));
+    }
+  }
+  return entries;
+}
+
+function setLeaf(obj, keyPath, value) {
+  const parts = [];
+  let current = '';
+  for (let i = 0; i < keyPath.length; i++) {
+    const ch = keyPath[i];
+    if (ch === '.' && keyPath[i - 1] !== '\\') {
+      if (current) parts.push(current);
+      current = '';
+    } else if (ch === '[') {
+      if (current) parts.push(current);
+      current = '';
+      i++;
+      while (i < keyPath.length && keyPath[i] !== ']') {
+        current += keyPath[i];
+        i++;
+      }
+      parts.push(`[${current}]`);
+      current = '';
+    } else {
+      current += ch;
+    }
+  }
+  if (current) parts.push(current);
+
+  let target = obj;
+  for (let i = 0; i < parts.length - 1; i++) {
+    const part = parts[i];
+    if (part.startsWith('[') && part.endsWith(']')) {
+      const idx = parseInt(part.slice(1, -1), 10);
+      if (!(idx in target)) target[idx] = {};
+      target = target[idx];
+    } else {
+      if (!(part in target)) target[part] = {};
+      target = target[part];
+    }
+  }
+
+  const last = parts[parts.length - 1];
+  if (last.startsWith('[') && last.endsWith(']')) {
+    target[parseInt(last.slice(1, -1), 10)] = value;
+  } else {
+    target[last] = value;
+  }
+}
+
+function getLeaf(obj, keyPath) {
+  const parts = [];
+  let current = '';
+  for (let i = 0; i < keyPath.length; i++) {
+    const ch = keyPath[i];
+    if (ch === '.' && keyPath[i - 1] !== '\\') {
+      if (current) parts.push(current);
+      current = '';
+    } else if (ch === '[') {
+      if (current) parts.push(current);
+      current = '';
+      i++;
+      while (i < keyPath.length && keyPath[i] !== ']') {
+        current += keyPath[i];
+        i++;
+      }
+      parts.push(`[${current}]`);
+      current = '';
+    } else {
+      current += ch;
+    }
+  }
+  if (current) parts.push(current);
+
+  let target = obj;
+  for (const part of parts) {
+    if (target === null || target === undefined) return undefined;
+    if (part.startsWith('[') && part.endsWith(']')) {
+      target = target[parseInt(part.slice(1, -1), 10)];
+    } else {
+      target = target[part];
+    }
+  }
+  return target;
+}
+
+function deepClone(obj) {
+  return JSON.parse(JSON.stringify(obj));
+}
+
+module.exports = {
+  deepTraverse,
+  collectLeaves,
+  setLeaf,
+  getLeaf,
+  deepClone,
+};