@yellowpanther/shared 1.2.1 → 1.2.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yellowpanther/shared",
-  "version": "1.2.1",
+  "version": "1.2.2",
   "publishConfig": {
     "access": "public"
   },
@@ -31,6 +31,7 @@
     "./queue/quizPublishQueue": "./src/queue/quizPublishQueue.js",
     "./queue/predictPublishQueue": "./src/queue/predictPublishQueue.js",
     "./queue/queueRegistry": "./src/queue/queueRegistry.js",
+    "./translationHelper": "./src/lang/translationHelper.js",
     "./*": "./src/*"
   },
   "scripts": {

package/src/index.js

@@ -2,6 +2,9 @@ module.exports = {
   // ✅ Redis
   redisClient: require('./redis/redisClient'),
 
+  // ✅ Language tranlation Helper
+  translationHelper: require('./lang/translationHelper'),
+
   // ✅ Config
   config: require('./config'),
 

package/src/lang/translationHelper.js

@@ -0,0 +1,223 @@
+const axios = require('axios');
+const { JSDOM } = require('jsdom'); // Required for HTML parsing
+
+// ♻️ Shared Redis connection
+const redis = require('../redis/redisClient');
+
+const BASE_URLS = [
+  'https://lingva.ml',
+  'https://translate.mentality.rip',
+  'https://translate.plausibility.cloud',
+  'https://lingva.translate.garudalinux.org',
+];
+
+const MAX_CHUNK_SIZE = 300;
+const RETRY_LIMIT = 3;
+const DELAY_BETWEEN_REQUESTS = 300;
+const REDIS_TTL = 60 * 60 * 24 * 7; // 7 days
+const MAX_FAILURES = 3; // Maximum consecutive mirror failures before Redis key is invalidated
+
+const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+function splitTextIntoChunks(text, maxChunkSize = MAX_CHUNK_SIZE) {
+  const sentences = text.match(/[^.!?]+[.!?]*[\s]*/g) || [text];
+  const chunks = [];
+  let current = '';
+
+  for (const sentence of sentences) {
+    if ((current + sentence).length > maxChunkSize) {
+      if (current.trim()) chunks.push(current.trim());
+      current = sentence;
+    } else {
+      current += sentence;
+    }
+  }
+
+  if (current.trim()) chunks.push(current.trim());
+  return chunks;
+}
+
+function isLikelyURL(text) {
+  return /(https?:\/\/[^\s]+)/.test(text);
+}
+
+async function getFromCache(cacheKey) {
+  return await redis.get(cacheKey);
+}
+
+async function setInCache(cacheKey, value, ttl = REDIS_TTL) {
+  await redis.set(cacheKey, value, 'EX', ttl);
+}
+
+async function deleteCacheKey(cacheKey) {
+  await redis.del(cacheKey);
+}
+
+/**
+ * 🔁 Translates a chunk with retries and mirror fallback.
+ */
+async function translateChunk(chunk, from, to, debug = false, fallbackToOriginal = true) {
+  const encoded = encodeURIComponent(chunk);
+  const langFrom = from?.toLowerCase?.() || 'auto';
+  const langTo = to?.toLowerCase?.() || 'en';
+
+  let totalFailures = 0;
+
+  for (let baseUrl of BASE_URLS) {
+    const url = `${baseUrl}/api/v1/${langFrom}/${langTo}/${encoded}`;
+
+    for (let attempt = 1; attempt <= RETRY_LIMIT; attempt++) {
+      try {
+        if (debug) console.log(`🌐 Translating "${chunk}" via ${baseUrl} (attempt ${attempt})`);
+        const response = await axios.get(url);
+        const translated = response?.data?.translation?.trim();
+        if (translated) return translated;
+        break;
+      } catch (error) {
+        const status = error?.response?.status;
+        const message = error?.message || 'Unknown error';
+
+        if (status === 429) {
+          if (debug) console.warn(`⚠️ 429 Too Many Requests – waiting ${attempt * 500}ms`);
+          await delay(attempt * 500);
+        } else {
+          totalFailures++;
+          if (debug) {
+            console.error(`❌ Mirror ${baseUrl} failed:`, { chunk, status, message });
+          }
+          break;
+        }
+      }
+    }
+  }
+
+  if (debug) console.warn(`🚫 Final fallback for chunk "${chunk}"`);
+  return fallbackToOriginal ? chunk : '';
+}
+
+/**
+ * 🧠 Translates plain text (cached in Redis, safe retries)
+ */
+async function translateWithLingva(text, to = 'fr', from = 'en', options = {}) {
+  const {
+    debug = false,
+    fallbackToOriginal = true,
+    cacheEnabled = true,
+    html = false,
+  } = options;
+
+  try {
+    if (!text || typeof text !== 'string' || !text.trim()) {
+      if (debug) console.warn('⚠️ Skipping invalid input');
+      return text;
+    }
+
+    if (isLikelyURL(text)) return text;
+
+    const trimmedText = text.trim();
+    const langFrom = from?.toLowerCase?.() || 'auto';
+    const langTo = to?.toLowerCase?.() || 'en';
+    const cacheKey = `lingva:${langFrom}:${langTo}:${trimmedText}`;
+
+    if (cacheEnabled) {
+      const cached = await getFromCache(cacheKey);
+      if (cached) {
+        if (debug) console.log('✅ Redis cache hit');
+        return cached;
+      }
+    }
+
+    let translated;
+    if (html) {
+      // calling infor with html content
+      /*
+        const { translateWithLingva } = require('./translateHelperRedis');
+        const htmlInput = `
+          <div>
+            <h1>Hello World!</h1>
+            <p>This is <strong>translated</strong> text with <a href="https://example.com">link</a>.</p>
+          </div>
+        `; 
+        const result = await translateWithLingva(htmlInput, 'fr', 'en', {
+          debug: true,
+          html: true,
+        });
+      */
+      translated = await translateHtmlContent(trimmedText, from, to, debug, fallbackToOriginal);
+    } else {
+      const chunks = splitTextIntoChunks(trimmedText);
+      const translations = [];
+
+      for (const chunk of chunks) {
+        const result = await translateChunk(chunk, from, to, debug, fallbackToOriginal);
+        translations.push(result);
+        await delay(DELAY_BETWEEN_REQUESTS);
+      }
+
+      translated = translations.join(' ').replace(/\s+/g, ' ').trim();
+    }
+
+    // Cache only if result is non-empty and not equal to input
+    if (translated && cacheEnabled && translated !== trimmedText) {
+      await setInCache(cacheKey, translated);
+    }
+
+    // Auto-invalidate on mirror failures
+    if (!translated || translated === trimmedText) {
+      const mirrorFailures = await redis.incr(`lingva:fail:${cacheKey}`);
+      await redis.expire(`lingva:fail:${cacheKey}`, 3600); // 1h expiration
+
+      if (mirrorFailures >= MAX_FAILURES) {
+        await deleteCacheKey(cacheKey);
+        if (debug) console.warn(`🔥 Invalidated cache due to repeated failures`);
+      }
+    }
+
+    return translated || (fallbackToOriginal ? trimmedText : '');
+  } catch (err) {
+    if (debug) console.error('💥 Unexpected error:', err.message);
+    return fallbackToOriginal ? text : '';
+  }
+}
+
+/**
+ * 🧠 Translate inner text of HTML while preserving tags
+ */
+async function translateHtmlContent(htmlText, from, to, debug = false, fallbackToOriginal = true) {
+  const dom = new JSDOM(`<body>${htmlText}</body>`);
+  const walker = dom.window.document.createTreeWalker(dom.window.document.body, NodeFilter.SHOW_TEXT);
+
+  while (walker.nextNode()) {
+    const node = walker.currentNode;
+    const raw = node.nodeValue.trim();
+    if (raw.length > 0) {
+      const translated = await translateWithLingva(raw, to, from, {
+        debug,
+        fallbackToOriginal,
+        cacheEnabled: true,
+        html: false, // prevent recursion
+      });
+      node.nodeValue = translated;
+      await delay(DELAY_BETWEEN_REQUESTS);
+    }
+  }
+
+  return dom.window.document.body.innerHTML;
+}
+
+/**
+ * 📚 Batch translation of plain text strings
+ */
+async function batchTranslateWithLingva(texts = [], to = 'fr', from = 'en', options = {}) {
+  const results = [];
+  for (const text of texts) {
+    const result = await translateWithLingva(text, to, from, options);
+    results.push(result);
+  }
+  return results;
+}
+
+module.exports = {
+  translateWithLingva,
+  batchTranslateWithLingva,
+};