@djangocfg/llm 2.1.164

package/src/translator/index.ts

@@ -0,0 +1,55 @@
+/**
+ * JSON Translation module
+ *
+ * Features:
+ * - Smart text-level caching (memory + persistent)
+ * - Technical content detection (URLs, emails, paths)
+ * - Parallel multi-language translation
+ * - Partial JSON - only sends uncached texts to LLM
+ * - Language auto-detection
+ */
+
+// Main translator
+export { JsonTranslator, createTranslator } from './json-translator';
+
+// Types
+export type {
+  LanguageCode,
+  TranslateOptions,
+  TranslateResult,
+  BatchTranslateItem,
+  CacheStats,
+} from './types';
+
+export { LANGUAGE_NAMES } from './types';
+
+// Cache
+export { TranslationCache, createCache } from './cache';
+
+// Validation
+export {
+  validateTranslation,
+  validateJsonKeys,
+  validatePlaceholders,
+  type ValidationResult,
+} from './validator';
+
+// Text utilities
+export {
+  isTechnicalContent,
+  extractPlaceholders,
+  extractTranslatableValues,
+  detectScript,
+  containsCJK,
+  containsCyrillic,
+  containsArabic,
+  validatePlaceholders as validateTextPlaceholders,
+} from './text-utils';
+
+// Prompts (for advanced usage)
+export {
+  buildJsonTranslationPrompt,
+  buildTextTranslationPrompt,
+  buildRetryPrompt,
+  getLanguageName,
+} from './prompts';

package/src/translator/json-translator.ts

@@ -0,0 +1,408 @@
+/**
+ * JSON Translator using LLM
+ *
+ * Smart text-level caching for efficiency:
+ * 1. Extract all translatable strings from JSON
+ * 2. Check cache for each string
+ * 3. Send only uncached strings to LLM
+ * 4. Cache new translations
+ * 5. Apply all translations to original structure
+ */
+
+import type { LLMClient } from '../types';
+import { extractJson } from '../utils/json';
+import type { TranslateOptions, TranslateResult, LanguageCode } from './types';
+import { buildJsonTranslationPrompt } from './prompts';
+import { isTechnicalContent, detectScript } from './text-utils';
+import { TranslationCache, createCache } from './cache';
+
+/** Default model for translation - gpt-4o-mini is reliable and cost-effective */
+const DEFAULT_TRANSLATION_MODEL = 'openai/gpt-4o-mini';
+
+/**
+ * JSON Translator class with smart caching
+ */
+export class JsonTranslator {
+  private cache: TranslationCache;
+  private defaultModel: string;
+
+  constructor(
+    private client: LLMClient,
+    cache?: TranslationCache,
+    defaultModel?: string
+  ) {
+    this.cache = cache ?? createCache();
+    this.defaultModel = defaultModel ?? DEFAULT_TRANSLATION_MODEL;
+  }
+
+  /**
+   * Detect language from text
+   */
+  detectLanguage(text: string): LanguageCode {
+    const script = detectScript(text);
+    switch (script) {
+      case 'cjk':
+        // Could be zh, ja, ko - default to zh
+        return 'zh';
+      case 'cyrillic':
+        return 'ru';
+      case 'arabic':
+        return 'ar';
+      default:
+        return 'en';
+    }
+  }
+
+  /**
+   * Check if text needs translation
+   */
+  needsTranslation(
+    text: string,
+    sourceLang: string,
+    targetLang: string
+  ): boolean {
+    if (!text || !text.trim()) return false;
+    if (isTechnicalContent(text)) return false;
+    if (sourceLang === targetLang) return false;
+    return true;
+  }
+
+  /**
+   * Translate single text
+   */
+  async translateText(
+    text: string,
+    targetLanguage: LanguageCode,
+    options?: TranslateOptions
+  ): Promise<string> {
+    if (!text || !text.trim()) return text;
+
+    const sourceLang = options?.sourceLanguage ?? 'auto';
+    const actualSource = sourceLang === 'auto' ? this.detectLanguage(text) : sourceLang;
+
+    if (!this.needsTranslation(text, actualSource, targetLanguage)) {
+      return text;
+    }
+
+    // Check cache
+    const cached = this.cache.get(text, actualSource, targetLanguage);
+    if (cached) return cached;
+
+    // Call LLM
+    const response = await this.client.chat(
+      `Translate this text to ${targetLanguage}. Return ONLY the translation:\n\n${text}`,
+      {
+        ...options,
+        model: options?.model ?? this.defaultModel,
+        temperature: 0,
+      }
+    );
+
+    const translated = response.content.trim();
+    this.cache.set(text, actualSource, targetLanguage, translated);
+
+    return translated;
+  }
+
+  /**
+   * Translate JSON object with smart text-level caching
+   *
+   * @example
+   * ```ts
+   * const translator = new JsonTranslator(llm)
+   * const result = await translator.translate(
+   *   { title: 'Hello', items: ['World', 'Earth'] },
+   *   'ru'
+   * )
+   * // { data: { title: 'Привет', items: ['Мир', 'Земля'] }, valid: true, ... }
+   * ```
+   */
+  async translate<T extends Record<string, unknown>>(
+    data: T,
+    targetLanguage: LanguageCode,
+    options?: TranslateOptions
+  ): Promise<TranslateResult<T>> {
+    const sourceLang = options?.sourceLanguage ?? 'auto';
+
+    // Extract all translatable texts
+    const translatableTexts = this.extractTranslatableTexts(data, sourceLang, targetLanguage);
+
+    if (translatableTexts.size === 0) {
+      return {
+        data,
+        valid: true,
+        errors: [],
+        retries: 0,
+        sourceLanguage: sourceLang,
+        targetLanguage,
+      };
+    }
+
+    // Detect source language from first text if auto
+    let actualSource = sourceLang;
+    if (sourceLang === 'auto') {
+      const firstText = [...translatableTexts][0];
+      actualSource = this.detectLanguage(firstText);
+    }
+
+    // Check cache for each text
+    const { cached, uncached } = this.cache.getMany(
+      [...translatableTexts],
+      actualSource,
+      targetLanguage
+    );
+
+    // If everything cached, apply and return
+    if (uncached.length === 0) {
+      return {
+        data: this.applyTranslations(data, cached),
+        valid: true,
+        errors: [],
+        retries: 0,
+        sourceLanguage: actualSource,
+        targetLanguage,
+      };
+    }
+
+    // Create partial JSON with only uncached texts
+    const partialJson = this.createPartialJson(data, new Set(uncached));
+    const jsonStr = JSON.stringify(partialJson, null, 2);
+
+    try {
+      // Build prompt and call LLM
+      const prompt = buildJsonTranslationPrompt(jsonStr, actualSource, targetLanguage);
+
+      const response = await this.client.chat(prompt, {
+        ...options,
+        model: options?.model ?? this.defaultModel,
+        temperature: 0,
+      });
+
+      // Parse response
+      const translatedPartial = extractJson<Record<string, unknown>>(response.content);
+
+      // Extract translations by comparison
+      const newTranslations = this.extractTranslationsByComparison(
+        partialJson,
+        translatedPartial,
+        uncached
+      );
+
+      // Cache new translations
+      this.cache.setMany(newTranslations, actualSource, targetLanguage);
+
+      // Combine all translations
+      const allTranslations = new Map([...cached, ...newTranslations]);
+
+      return {
+        data: this.applyTranslations(data, allTranslations),
+        valid: true,
+        errors: [],
+        retries: 0,
+        sourceLanguage: actualSource,
+        targetLanguage,
+      };
+    } catch (error) {
+      // On error, return what we have from cache
+      const errorMsg = error instanceof Error ? error.message : String(error);
+      console.error('Translation failed:', errorMsg);
+
+      return {
+        data: cached.size > 0 ? this.applyTranslations(data, cached) : data,
+        valid: false,
+        errors: [errorMsg],
+        retries: 0,
+        sourceLanguage: actualSource,
+        targetLanguage,
+      };
+    }
+  }
+
+  /**
+   * Translate to multiple languages in parallel
+   */
+  async translateToMany<T extends Record<string, unknown>>(
+    data: T,
+    targetLanguages: LanguageCode[],
+    options?: TranslateOptions
+  ): Promise<Map<LanguageCode, TranslateResult<T>>> {
+    const results = new Map<LanguageCode, TranslateResult<T>>();
+
+    // Run translations in parallel
+    const promises = targetLanguages.map(async (lang) => {
+      const result = await this.translate(data, lang, options);
+      return { lang, result };
+    });
+
+    const settled = await Promise.all(promises);
+    for (const { lang, result } of settled) {
+      results.set(lang, result);
+    }
+
+    return results;
+  }
+
+  /**
+   * Extract all translatable texts from object
+   */
+  private extractTranslatableTexts(
+    obj: unknown,
+    sourceLang: string,
+    targetLang: string
+  ): Set<string> {
+    const texts = new Set<string>();
+
+    const extract = (item: unknown): void => {
+      if (typeof item === 'string') {
+        if (this.needsTranslation(item, sourceLang, targetLang)) {
+          texts.add(item);
+        }
+      } else if (Array.isArray(item)) {
+        for (const i of item) extract(i);
+      } else if (item !== null && typeof item === 'object') {
+        for (const v of Object.values(item)) extract(v);
+      }
+    };
+
+    extract(obj);
+    return texts;
+  }
+
+  /**
+   * Apply translations to object
+   */
+  private applyTranslations<T>(obj: T, translations: Map<string, string>): T {
+    const apply = (item: unknown): unknown => {
+      if (typeof item === 'string') {
+        return translations.get(item) ?? item;
+      } else if (Array.isArray(item)) {
+        return item.map(apply);
+      } else if (item !== null && typeof item === 'object') {
+        const result: Record<string, unknown> = {};
+        for (const [k, v] of Object.entries(item)) {
+          result[k] = apply(v);
+        }
+        return result;
+      }
+      return item;
+    };
+
+    return apply(obj) as T;
+  }
+
+  /**
+   * Create partial JSON with only texts that need translation
+   */
+  private createPartialJson(data: unknown, textsToInclude: Set<string>): unknown {
+    const filter = (obj: unknown): unknown => {
+      if (typeof obj === 'string') {
+        return textsToInclude.has(obj) ? obj : 'SKIP';
+      } else if (Array.isArray(obj)) {
+        return obj.map(filter).filter((i) => this.hasTranslatable(i, textsToInclude));
+      } else if (obj !== null && typeof obj === 'object') {
+        const result: Record<string, unknown> = {};
+        for (const [k, v] of Object.entries(obj)) {
+          const filtered = filter(v);
+          if (this.hasTranslatable(filtered, textsToInclude)) {
+            result[k] = filtered;
+          }
+        }
+        return result;
+      }
+      return obj;
+    };
+
+    return filter(data);
+  }
+
+  /**
+   * Check if object contains translatable text
+   */
+  private hasTranslatable(obj: unknown, textsSet: Set<string>): boolean {
+    if (typeof obj === 'string') return textsSet.has(obj);
+    if (Array.isArray(obj)) return obj.some((i) => this.hasTranslatable(i, textsSet));
+    if (obj !== null && typeof obj === 'object') {
+      return Object.values(obj).some((v) => this.hasTranslatable(v, textsSet));
+    }
+    return false;
+  }
+
+  /**
+   * Extract translations by comparing original and translated JSON
+   */
+  private extractTranslationsByComparison(
+    original: unknown,
+    translated: unknown,
+    uncachedTexts: string[]
+  ): Map<string, string> {
+    const translations = new Map<string, string>();
+    const uncachedSet = new Set(uncachedTexts);
+
+    const compare = (orig: unknown, trans: unknown): void => {
+      if (typeof orig === 'string' && typeof trans === 'string') {
+        // Save translation if original was in uncached list and translation is valid
+        if (uncachedSet.has(orig) && trans !== 'SKIP' && trans.trim()) {
+          translations.set(orig, trans);
+        }
+      } else if (Array.isArray(orig) && Array.isArray(trans)) {
+        for (let i = 0; i < Math.min(orig.length, trans.length); i++) {
+          compare(orig[i], trans[i]);
+        }
+      } else if (
+        orig !== null &&
+        typeof orig === 'object' &&
+        trans !== null &&
+        typeof trans === 'object'
+      ) {
+        for (const key of Object.keys(orig as Record<string, unknown>)) {
+          if (key in (trans as Record<string, unknown>)) {
+            compare(
+              (orig as Record<string, unknown>)[key],
+              (trans as Record<string, unknown>)[key]
+            );
+          }
+        }
+      }
+    };
+
+    compare(original, translated);
+    return translations;
+  }
+
+  /**
+   * Get translation statistics
+   */
+  getStats() {
+    return this.cache.getStats();
+  }
+
+  /**
+   * Clear translation cache
+   */
+  clearCache(sourceLang?: string, targetLang?: string) {
+    this.cache.clear(sourceLang, targetLang);
+  }
+}
+
+/**
+ * Translator options
+ */
+export interface TranslatorConfig {
+  cache?: TranslationCache;
+  /** Model for translation (default: openai/gpt-4o-mini) */
+  model?: string;
+}
+
+/**
+ * Create JSON translator
+ */
+export function createTranslator(
+  client: LLMClient,
+  configOrCache?: TranslationCache | TranslatorConfig
+): JsonTranslator {
+  // Support both old and new API
+  if (configOrCache instanceof TranslationCache) {
+    return new JsonTranslator(client, configOrCache);
+  }
+  return new JsonTranslator(client, configOrCache?.cache, configOrCache?.model);
+}

package/src/translator/prompts.ts

@@ -0,0 +1,90 @@
+/**
+ * Translation prompts
+ */
+
+import { LANGUAGE_NAMES } from './types';
+
+/**
+ * Get language name from code
+ */
+export function getLanguageName(code: string): string {
+  return LANGUAGE_NAMES[code] ?? code;
+}
+
+/**
+ * Build JSON translation prompt
+ */
+export function buildJsonTranslationPrompt(
+  json: string,
+  sourceLanguage: string,
+  targetLanguage: string
+): string {
+  const sourceName = getLanguageName(sourceLanguage);
+  const targetName = getLanguageName(targetLanguage);
+
+  return `Translate all string VALUES in this JSON from ${sourceName} to ${targetName}.
+
+Rules:
+- Translate ALL string values to ${targetName}
+- Keep JSON keys unchanged (English)
+- Skip: URLs, emails, numbers, "SKIP"
+- Keep placeholders: {name}, {{var}}, %s
+
+${json}
+
+Return ONLY the JSON with all values translated to ${targetName}:`;
+}
+
+/**
+ * Build text translation prompt
+ */
+export function buildTextTranslationPrompt(
+  text: string,
+  sourceLanguage: string,
+  targetLanguage: string
+): string {
+  const sourceName = getLanguageName(sourceLanguage);
+  const targetName = getLanguageName(targetLanguage);
+
+  return `Translate from ${sourceName} to ${targetName}.
+
+RULES:
+1. Translate ONLY the text provided
+2. Preserve formatting, numbers, URLs
+3. Keep placeholders like {name}, {{var}} unchanged
+4. Return ONLY the translation, no explanations
+
+Text: ${text}
+
+Translation:`;
+}
+
+/**
+ * Build retry prompt with errors
+ */
+export function buildRetryPrompt(
+  originalJson: string,
+  wrongJson: string,
+  errors: string[],
+  targetLanguage: string
+): string {
+  const targetName = getLanguageName(targetLanguage);
+
+  return `Your previous translation had errors. Fix them.
+
+ERRORS FOUND:
+${errors.map((e) => `- ${e}`).join('\n')}
+
+ORIGINAL JSON:
+${originalJson}
+
+YOUR WRONG TRANSLATION:
+${wrongJson}
+
+FIX THE ERRORS. Remember:
+- JSON keys must stay in English
+- Only translate string values
+- Preserve all placeholders
+
+Return ONLY the corrected JSON translated to ${targetName}:`;
+}

package/src/translator/text-utils.ts

@@ -0,0 +1,148 @@
+/**
+ * Text utilities for translation
+ */
+
+/**
+ * Check if text is technical content that shouldn't be translated
+ */
+export function isTechnicalContent(text: string): boolean {
+  const trimmed = text.trim();
+
+  // Empty or whitespace only
+  if (!trimmed) return true;
+
+  // URLs
+  if (/^(https?:\/\/|\/\/|www\.)/i.test(trimmed)) return true;
+
+  // Email addresses
+  if (/^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(trimmed)) return true;
+
+  // File paths (starts with / or contains common extensions)
+  if (/^\/[a-zA-Z]/.test(trimmed)) return true;
+  if (/\.(js|ts|tsx|jsx|json|css|scss|html|md|py|go|rs)$/i.test(trimmed)) return true;
+
+  // Numbers only (including decimals and percentages)
+  if (/^[\d.,]+%?$/.test(trimmed)) return true;
+
+  // Technical identifiers (SCREAMING_SNAKE_CASE)
+  if (/^[A-Z][A-Z0-9_]*$/.test(trimmed)) return true;
+
+  // Placeholders like {name}, {{var}}, %s, $1
+  if (/^(\{[^}]+\}|\{\{[^}]+\}\}|%[sd]|\$\d+)$/.test(trimmed)) return true;
+
+  // Single special characters
+  if (/^[!@#$%^&*()_+\-=\[\]{};':"\\|,.<>\/?]+$/.test(trimmed)) return true;
+
+  return false;
+}
+
+/**
+ * Check if text contains CJK characters (Chinese, Japanese, Korean)
+ */
+export function containsCJK(text: string): boolean {
+  // CJK Unified Ideographs, Hiragana, Katakana, Hangul
+  return /[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]/.test(text);
+}
+
+/**
+ * Check if text contains Cyrillic characters
+ */
+export function containsCyrillic(text: string): boolean {
+  return /[\u0400-\u04ff]/.test(text);
+}
+
+/**
+ * Check if text contains Arabic characters
+ */
+export function containsArabic(text: string): boolean {
+  return /[\u0600-\u06ff]/.test(text);
+}
+
+/**
+ * Detect script/language from text
+ */
+export function detectScript(
+  text: string
+): 'cjk' | 'cyrillic' | 'arabic' | 'latin' | 'unknown' {
+  if (containsCJK(text)) return 'cjk';
+  if (containsCyrillic(text)) return 'cyrillic';
+  if (containsArabic(text)) return 'arabic';
+  if (/[a-zA-Z]/.test(text)) return 'latin';
+  return 'unknown';
+}
+
+/**
+ * Extract placeholders from text
+ */
+export function extractPlaceholders(text: string): string[] {
+  const patterns = [
+    /\{[^}]+\}/g, // {name}
+    /\{\{[^}]+\}\}/g, // {{name}}
+    /%[sd]/g, // %s, %d
+    /\$\d+/g, // $1, $2
+  ];
+
+  const placeholders: string[] = [];
+  for (const pattern of patterns) {
+    const matches = text.match(pattern);
+    if (matches) {
+      placeholders.push(...matches);
+    }
+  }
+
+  return [...new Set(placeholders)];
+}
+
+/**
+ * Check if all placeholders are preserved in translation
+ */
+export function validatePlaceholders(
+  original: string,
+  translated: string
+): { valid: boolean; missing: string[]; extra: string[] } {
+  const originalPlaceholders = extractPlaceholders(original);
+  const translatedPlaceholders = extractPlaceholders(translated);
+
+  const missing = originalPlaceholders.filter(
+    (p) => !translatedPlaceholders.includes(p)
+  );
+  const extra = translatedPlaceholders.filter(
+    (p) => !originalPlaceholders.includes(p)
+  );
+
+  return {
+    valid: missing.length === 0 && extra.length === 0,
+    missing,
+    extra,
+  };
+}
+
+/**
+ * Get values that need translation from JSON
+ */
+export function extractTranslatableValues(
+  obj: unknown,
+  path: string = ''
+): Array<{ path: string; value: string }> {
+  const values: Array<{ path: string; value: string }> = [];
+
+  if (typeof obj === 'string') {
+    if (!isTechnicalContent(obj)) {
+      values.push({ path, value: obj });
+    }
+  } else if (Array.isArray(obj)) {
+    obj.forEach((item, index) => {
+      values.push(
+        ...extractTranslatableValues(item, path ? `${path}[${index}]` : `[${index}]`)
+      );
+    });
+  } else if (obj !== null && typeof obj === 'object') {
+    for (const [key, value] of Object.entries(obj)) {
+      values.push(
+        ...extractTranslatableValues(value, path ? `${path}.${key}` : key)
+      );
+    }
+  }
+
+  return values;
+}