bctranslate 1.0.0-beta.1

package/src/parsers/vue.js

@@ -0,0 +1,282 @@
+import * as compiler from '@vue/compiler-dom';
+import { parse as babelParse } from '@babel/parser';
+import _traverse from '@babel/traverse';
+const traverse = _traverse.default;
+import MagicString from 'magic-string';
+import { contextKey, isTranslatable } from '../utils.js';
+
+const ATTR_BLACKLIST = new Set([
+  'id', 'class', 'style', 'src', 'href', 'ref', 'key', 'is',
+  'v-model', 'v-bind', 'v-on', 'v-if', 'v-else', 'v-else-if',
+  'v-for', 'v-show', 'v-html', 'v-text', 'v-slot', 'v-pre',
+  'v-cloak', 'v-once', 'v-memo', 'name', 'type', 'value',
+  'action', 'method', 'target', 'rel', 'media', 'lang',
+  'charset', 'content', 'http-equiv', 'for', 'tabindex',
+  'role', 'xmlns', 'viewBox', 'fill', 'stroke', 'd', 'cx', 'cy',
+  'r', 'rx', 'ry', 'x', 'y', 'width', 'height', 'transform',
+  'xmlns:xlink', 'xlink:href', 'data-testid', 'data-cy',
+]);
+
+const ATTR_WHITELIST = new Set([
+  'title', 'placeholder', 'label', 'alt', 'aria-label',
+  'aria-placeholder', 'aria-description', 'aria-roledescription',
+]);
+
+/**
+ * Detect which t-function call to emit in templates.
+ *
+ * Rules:
+ *  - If <script setup> and file already destructures `t` from a composable
+ *    → use  t('key')          (matches user's existing pattern)
+ *  - If <script setup> but no `t` yet
+ *    → use  t('key')   AND inject `const { t } = useI18n()` into script
+ *  - Options API (no setup)
+ *    → use  $t('key')         (global plugin property)
+ */
+
+/**
+ * Parse a .vue file and extract translatable strings.
+ */
+export function parseVue(source, filePath) {
+  const extracted = [];
+  const s = new MagicString(source);
+
+  const scriptSetupMatch = source.match(/(<script\b[^>]*\bsetup\b[^>]*>)([\s\S]*?)<\/script>/i);
+  const hasScriptSetup = !!scriptSetupMatch;
+
+  // Prefer what the file already uses in templates: `$t()` vs `t()`.
+  const templateBlockMatch = source.match(/<template\b[^>]*>([\s\S]*?)<\/template>/);
+  const templateBlock = templateBlockMatch ? templateBlockMatch[1] : '';
+  const templatePrefersDollarT = /\$t\s*\(/.test(templateBlock);
+  const templatePrefersT = !templatePrefersDollarT && /\bt\s*\(/.test(templateBlock);
+
+  const tpl = (key) => (templatePrefersT ? `t('${key}')` : `$t('${key}')`);
+  const scr = (key) => (hasScriptSetup ? `t('${key}')` : `this.$t('${key}')`);
+
+  // ── Template ────────────────────────────────────────────────────────────────
+  const templateMatch = source.match(/<template\b[^>]*>([\s\S]*?)<\/template>/);
+  if (templateMatch) {
+    const templateContent = templateMatch[1];
+    const templateOffset =
+      source.indexOf(templateMatch[0]) + templateMatch[0].indexOf(templateContent);
+
+    try {
+      const ast = compiler.parse(templateContent, { comments: true, getTextMode: () => 0 });
+      walkTemplate(ast.children, s, templateOffset, extracted, filePath, tpl);
+    } catch {
+      extractTemplateRegex(source, s, extracted, filePath, tpl);
+    }
+  }
+
+  // ── Script (prefer <script setup>, else first non-setup <script>) ────────────
+  if (scriptSetupMatch) {
+    const scriptContent = scriptSetupMatch[2];
+    const scriptOffset =
+      source.indexOf(scriptSetupMatch[0]) + scriptSetupMatch[0].indexOf(scriptContent);
+    extractScriptStrings(scriptContent, s, scriptOffset, extracted, filePath, scr);
+  } else {
+    const scriptMatch = source.match(/<script\b(?![^>]*\bsetup\b)[^>]*>([\s\S]*?)<\/script>/i);
+    if (scriptMatch) {
+      const scriptContent = scriptMatch[1];
+      const scriptOffset =
+        source.indexOf(scriptMatch[0]) + scriptMatch[0].indexOf(scriptContent);
+      extractScriptStrings(scriptContent, s, scriptOffset, extracted, filePath, scr);
+    }
+  }
+
+  // ── Inject `const { t } = useI18n()` if not yet declared ────────
+  if (extracted.length > 0 && hasScriptSetup) {
+    const hasT = /const\s*\{[^}]*\bt\b[^}]*\}\s*=/.test(source);
+
+    if (!hasT) {
+      const insertAt = source.indexOf(scriptSetupMatch[0]) + scriptSetupMatch[1].length;
+      const needsImport = !source.includes('useI18n');
+      const importLine = needsImport ? `import { useI18n } from 'vue-i18n';\n` : '';
+      s.appendRight(insertAt, `\n${importLine}const { t } = useI18n();\n`);
+    }
+  }
+
+  return {
+    source: extracted.length > 0 ? s.toString() : source,
+    extracted,
+    modified: extracted.length > 0,
+  };
+}
+
+// ── Template walker ───────────────────────────────────────────────────────────
+
+function walkTemplate(nodes, s, baseOffset, extracted, filePath, tpl) {
+  for (const node of nodes) {
+    // Text node (type 2)
+    if (node.type === 2) {
+      const text = node.content.trim();
+      if (isTranslatable(text)) {
+        const start = baseOffset + node.loc.start.offset;
+        const end   = baseOffset + node.loc.end.offset;
+
+        if (!isAlreadyWrapped(s.original, start, end)) {
+          const key = contextKey(text, filePath);
+          const orig = node.content;
+          const lws  = orig.match(/^(\s*)/)[1];
+          const tws  = orig.match(/(\s*)$/)[1];
+          s.overwrite(start, end, `${lws}{{ ${tpl(key)} }}${tws}`);
+          extracted.push({ key, text, context: 'template-text' });
+        }
+      }
+    }
+
+    // Element (type 1) — check translatable attributes, then recurse
+    if (node.type === 1) {
+      for (const prop of node.props ?? []) {
+        if (prop.type === 6 && prop.value) {
+          const attrName = prop.name.toLowerCase();
+          if (ATTR_WHITELIST.has(attrName) && isTranslatable(prop.value.content)) {
+            const text = prop.value.content;
+            if (!ATTR_BLACKLIST.has(attrName)) {
+              const key      = contextKey(text, filePath);
+              const attrStart = baseOffset + prop.loc.start.offset;
+              const attrEnd   = baseOffset + prop.loc.end.offset;
+              s.overwrite(attrStart, attrEnd, `:${attrName}="${tpl(key)}"`);
+              extracted.push({ key, text, context: `template-attr-${attrName}` });
+            }
+          }
+        }
+      }
+      if (node.children) walkTemplate(node.children, s, baseOffset, extracted, filePath, tpl);
+    }
+
+    // ForNode (type 11)
+    if (node.type === 11 && node.children) {
+      walkTemplate(node.children, s, baseOffset, extracted, filePath, tpl);
+    }
+
+    // IfNode (type 9) — walk branches
+    if (node.type === 9 && node.branches) {
+      for (const branch of node.branches) {
+        if (branch.children) walkTemplate(branch.children, s, baseOffset, extracted, filePath, tpl);
+      }
+    }
+  }
+}
+
+// ── Script string extractor ───────────────────────────────────────────────────
+
+function extractScriptStrings(scriptContent, s, baseOffset, extracted, filePath, scr) {
+  try {
+    const ast = babelParse(scriptContent, {
+      sourceType: 'module',
+      plugins: ['typescript', 'jsx'], // Enable TS and JSX support
+    });
+
+    const replacements = [];
+
+    traverse(ast, {
+      enter(path) {
+        let text;
+
+        if (path.isStringLiteral()) {
+          text = path.node.value;
+        } else if (path.isTemplateLiteral()) {
+          if (path.node.expressions.length > 0 || path.node.quasis.length !== 1) {
+            return;
+          }
+          text = path.node.quasis[0].value.cooked;
+        } else {
+          return;
+        }
+        
+        if (!isTranslatable(text)) return;
+
+        // --- Parent checks to avoid replacing the wrong strings ---
+        if (path.parent.type === 'CallExpression' && path.parent.callee.name === 't') return;
+        if (['ImportDeclaration', 'ExportNamedDeclaration', 'ExportAllDeclaration'].includes(path.parent.type)) return;
+        if (path.parent.type === 'ObjectProperty' && path.parent.key === path.node) return;
+        if (path.parent.type === 'Property' && ['name'].includes(path.parent.key.name)) return;
+
+        // --- Passed all checks, schedule replacement ---
+        const key = contextKey(text, filePath);
+        const start = baseOffset + path.node.start;
+        const end = baseOffset + path.node.end;
+        replacements.push({ start, end, key });
+        extracted.push({ key, text, context: 'script' });
+      }
+    });
+
+    // Apply replacements in reverse order to avoid offset issues
+    for (let i = replacements.length - 1; i >= 0; i--) {
+      const { start, end, key } = replacements[i];
+      s.overwrite(start, end, scr(key));
+    }
+
+  } catch (e) {
+    console.error('Babel parsing failed:', e);
+    // If babel parsing fails, fallback to regex.
+    extractScriptStringsRegex(scriptContent, s, baseOffset, extracted, filePath, scr);
+  }
+}
+
+
+function extractScriptStringsRegex(scriptContent, s, baseOffset, extracted, filePath, scr) {
+  // This is the old regex-based implementation, kept as a fallback.
+  const patterns = [
+    /\b(alert|confirm|toast|notify|message\.(?:success|error|warning|info))\s*\(\s*(['"`])((?:(?!\2).)+)\2\s*\)/g,
+    /\b(title|label|placeholder|message|text|description|tooltip|hint|caption|header|subtitle|errorMessage|successMessage|emptyText|noData|loadingText|buttonText|confirmText|cancelText|successText|failText|warningText|helperText|hintText)\s*:\s*(['"`])((?:(?!\2).)+)\2/g,
+    /\bref\s*\(\s*(['"`])((?:(?!\1)[^\\]|\\.)+)\1\s*\)/g,
+    /\bcomputed\s*\(\s*\(\s*\)\s*=>\s*(['"`])((?:(?!\1)[^\\]|\\.)+)\1\s*\)/g,
+  ];
+
+  for (const pattern of patterns) {
+    let match;
+    while ((match = pattern.exec(scriptContent)) !== null) {
+      const text = match[3] || match[2];
+      if (!isTranslatable(text) || text.length <= 1) continue;
+      
+      const quoteChar = match[2] || match[1];
+      const innerStr = quoteChar + text + quoteChar;
+      const relPos = match.index + match[0].lastIndexOf(innerStr);
+
+      if (isAlreadyWrappedScript(scriptContent, relPos)) continue;
+
+      const key = contextKey(text, filePath);
+      const textStart = baseOffset + relPos;
+      const textEnd = textStart + innerStr.length;
+
+      s.overwrite(textStart, textEnd, scr(key));
+      extracted.push({ key, text, context: 'script-regex' });
+    }
+  }
+}
+
+function isAlreadyWrappedScript(scriptContent, pos) {
+  const before = scriptContent.slice(Math.max(0, pos - 30), pos);
+  return /\$?t\s*\(\s*$/.test(before);
+}
+
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+function isAlreadyWrapped(source, start, end) {
+  // Look back 25 chars for an open t( call — node is inside an interpolation
+  const before = source.slice(Math.max(0, start - 25), start);
+  return /\$?t\s*\(\s*['"]/.test(before);
+}
+
+function extractTemplateRegex(source, s, extracted, filePath, tpl) {
+  const pattern = />([^<]+)</g;
+  let match;
+  while ((match = pattern.exec(source)) !== null) {
+    const text = match[1].trim();
+    if (isTranslatable(text)) {
+      const textStart = match.index + 1;
+      const textEnd   = textStart + match[1].length;
+      if (!isAlreadyWrapped(source, textStart, textEnd)) {
+        const key  = contextKey(text, filePath);
+        const orig = match[1];
+        const lws  = orig.match(/^(\s*)/)[1];
+        const tws  = orig.match(/(\s*)$/)[1];
+        s.overwrite(textStart, textEnd, `${lws}{{ ${tpl(key)} }}${tws}`);
+        extracted.push({ key, text, context: 'template-text' });
+      }
+    }
+  }
+}

package/src/utils.js

@@ -0,0 +1,227 @@
+import { createHash } from 'crypto';
+import { basename, extname } from 'path';
+
+// Words too generic to use as the sole semantic component of a key
+const STOP_WORDS = new Set([
+  'a', 'an', 'the', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+  'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'shall',
+  'should', 'may', 'might', 'must', 'can', 'could',
+  'to', 'for', 'and', 'or', 'but', 'of', 'in', 'on', 'at', 'by',
+  'as', 'if', 'its', 'it', 'this', 'that', 'these', 'those',
+  'my', 'your', 'our', 'their', 'with', 'from', 'up', 'about',
+  'no', 'not', 'so',
+]);
+
+/**
+ * Generate a readable, slug-based i18n key from a string.
+ * "Submit" → "submit"
+ * "Please enter your email" → "please_enter_your_email"
+ * Falls back to a hash prefix for non-Latin or symbol-only strings.
+ */
+export function textKey(text) {
+  const trimmed = text.trim();
+
+  const slug = trimmed
+    .toLowerCase()
+    .replace(/[^\w\s]/g, ' ')   // punctuation → space
+    .replace(/\s+/g, '_')        // spaces → underscores
+    .replace(/^[^a-z]+/, '')     // strip non-alpha prefix
+    .replace(/[^a-z0-9_]/g, '') // remove remaining non-ASCII
+    .replace(/_+/g, '_')         // collapse multiple underscores
+    .replace(/^_|_$/g, '')       // trim underscores
+    .slice(0, 40)
+    .replace(/_+$/, '');
+
+  if (slug && slug.length >= 2 && /[a-z]/.test(slug)) {
+    return slug;
+  }
+
+  // Fallback: hash (for Chinese, Arabic, emoji, symbols, etc.)
+  const hash = createHash('sha256').update(trimmed).digest('hex').slice(0, 8);
+  return `key_${hash}`;
+}
+
+/**
+ * Generate a context-aware i18n key using the component name as namespace
+ * and key content words as the slug.
+ *
+ * "Notes"           in HomeView.vue  → home.notes
+ * "Quick Note"      in HomeView.vue  → home.quickNote
+ * "View livestock"  in HomeView.vue  → home.viewLivestock
+ * "Submit"          in LoginForm.vue → loginForm.submit
+ * "你好"             in App.vue       → app.key_3d2a1f
+ *
+ * @param {string} text     The source string to key
+ * @param {string} filePath Absolute or relative path of the source file
+ */
+export function contextKey(text, filePath) {
+  const trimmed = text.trim();
+
+  // ── Namespace: derive from filename ──────────────────────────────────────
+  const fileName = basename(filePath, extname(filePath));
+  // Strip common Vue/React suffixes: HomeView → Home, UserCard → User, etc.
+  const stripped = fileName.replace(
+    /(?:View|Component|Page|Screen|Modal|Dialog|Card|Panel|Widget|Layout|Container)$/,
+    ''
+  ) || fileName;
+  // camelCase namespace: "UserProfile" → "userProfile", "home" → "home"
+  const ns = stripped[0].toLowerCase() + stripped.slice(1);
+
+  // ── Slug: 1-3 meaningful words in camelCase ───────────────────────────────
+  const words = trimmed
+    .replace(/[^\w\s]/g, ' ')
+    .split(/\s+/)
+    .map((w) => w.toLowerCase())
+    .filter((w) => w.length >= 2 && /[a-z]/.test(w) && !STOP_WORDS.has(w));
+
+  if (!words.length) {
+    // Non-Latin scripts, emoji, or all stop words — fall back to hash suffix
+    const hash = createHash('sha256').update(trimmed).digest('hex').slice(0, 6);
+    return `${ns}.key${hash}`;
+  }
+
+  const slug =
+    words[0] + words.slice(1, 3).map((w) => w[0].toUpperCase() + w.slice(1)).join('');
+
+  return `${ns}.${slug}`;
+}
+
+/**
+ * @deprecated Use contextKey() for new code. textKey() kept for non-file contexts.
+ */
+export const hashKey = textKey;
+
+/**
+ * Determine if a string is "translatable" — contains actual human-readable
+ * text rather than whitespace, numbers, symbols, or code identifiers.
+ */
+export function isTranslatable(text) {
+  if (!text || typeof text !== 'string') return false;
+
+  const trimmed = text.trim();
+  if (trimmed.length === 0) return false;
+
+  // Skip pure whitespace / newlines
+  if (/^\s*$/.test(trimmed)) return false;
+
+  // Skip pure numbers
+  if (/^[\d.,]+$/.test(trimmed)) return false;
+
+  // Skip single characters that are punctuation/symbols
+  if (
+    trimmed.length === 1 &&
+    /[^a-zA-Z\u00C0-\u024F\u0400-\u04FF\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff]/.test(trimmed)
+  )
+    return false;
+
+  // Skip code identifiers (camelCase, snake_case with no spaces)
+  if (/^[a-zA-Z_$][a-zA-Z0-9_$.]*$/.test(trimmed) && !trimmed.includes(' ') && trimmed.length > 1) {
+    // Allow capitalised real words: "Submit", "Cancel", "Home"
+    if (/[aeiouAEIOU]/.test(trimmed) && trimmed.length > 2 && /^[A-Z][a-z]+$/.test(trimmed)) {
+      return true;
+    }
+    // Allow ALL-CAPS short labels: "OK", "FAQ"
+    if (/^[A-Z]{2,12}$/.test(trimmed)) return true;
+    return false;
+  }
+
+  // Skip URLs, paths, emails
+  if (/^(https?:\/\/|\/|\.\/|\.\.\/)/.test(trimmed)) return false;
+  if (/^[\w.+-]+@[\w.-]+\.\w+$/.test(trimmed)) return false;
+
+  // Skip pure template expressions {{ something }}
+  if (/^\{\{[^}]+\}\}$/.test(trimmed)) return false;
+
+  // Must contain at least one letter from any script
+  if (
+    !/[a-zA-Z\u00C0-\u024F\u0400-\u04FF\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uAC00-\uD7AF]/.test(
+      trimmed
+    )
+  )
+    return false;
+
+  return true;
+}
+
+/**
+ * Shield interpolation variables before sending to Argos Translate.
+ * Replaces {{ name }}, {name}, ${name}, %{name} with XML-like tokens <xi/>
+ * that NMT models are trained to preserve verbatim.
+ *
+ * Returns { shielded, tokens } — call unshieldInterpolations() to restore.
+ */
+export function shieldInterpolations(text) {
+  const tokens = [];
+  let shielded = text;
+
+  // Vue {{ expr }} — must come first to avoid matching inner {
+  shielded = shielded.replace(/\{\{[^}]*\}\}/g, (m) => {
+    const i = tokens.length;
+    tokens.push(m);
+    return `<x${i}/>`;
+  });
+
+  // Template literal ${expr}
+  shielded = shielded.replace(/\$\{[^}]*\}/g, (m) => {
+    const i = tokens.length;
+    tokens.push(m);
+    return `<x${i}/>`;
+  });
+
+  // i18next / vue-i18n {varName} or {0}
+  shielded = shielded.replace(/\{[^{}\s][^{}]*\}/g, (m) => {
+    const i = tokens.length;
+    tokens.push(m);
+    return `<x${i}/>`;
+  });
+
+  // Ruby / Rails %{varName}
+  shielded = shielded.replace(/%\{[^}]+\}/g, (m) => {
+    const i = tokens.length;
+    tokens.push(m);
+    return `<x${i}/>`;
+  });
+
+  return { shielded, tokens };
+}
+
+/**
+ * Restore interpolation variables after translation.
+ * Tolerates minor whitespace changes the MT model may introduce.
+ */
+export function unshieldInterpolations(text, tokens) {
+  if (!tokens || tokens.length === 0) return text;
+  return text.replace(/<x(\d+)\s*\/>/gi, (_, idx) => tokens[parseInt(idx, 10)] ?? '');
+}
+
+/**
+ * Check if a string contains interpolation expressions.
+ */
+export function parseInterpolation(text) {
+  const vuePattern = /\{\{\s*([^}]+?)\s*\}\}/g;
+
+  const parts = [];
+  let hasInterpolation = false;
+
+  if (vuePattern.test(text)) {
+    hasInterpolation = true;
+    vuePattern.lastIndex = 0;
+
+    let lastIndex = 0;
+    let match;
+    while ((match = vuePattern.exec(text)) !== null) {
+      if (match.index > lastIndex) {
+        const staticPart = text.slice(lastIndex, match.index);
+        if (staticPart.trim()) parts.push({ type: 'static', value: staticPart });
+      }
+      parts.push({ type: 'expr', value: match[1].trim() });
+      lastIndex = match.index + match[0].length;
+    }
+    if (lastIndex < text.length) {
+      const remainder = text.slice(lastIndex);
+      if (remainder.trim()) parts.push({ type: 'static', value: remainder });
+    }
+  }
+
+  return { hasInterpolation, parts };
+}