@yellowpanther/shared 1.2.0 → 1.2.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yellowpanther/shared",
-  "version": "1.2.0",
+  "version": "1.2.2",
   "publishConfig": {
     "access": "public"
   },
@@ -25,11 +25,13 @@
     "./queue/articlePublishQueue": "./src/queue/articlePublishQueue.js",
     "./queue/videoPublishQueue": "./src/queue/videoPublishQueue.js",
     "./queue/imagePublishQueue": "./src/queue/imagePublishQueue.js",
+    "./queue/imageAlbumPublishQueue": "./src/queue/imageAlbumPublishQueue.js",
     "./queue/pagePublishQueue": "./src/queue/pagePublishQueue.js",
     "./queue/productPublishQueue": "./src/queue/productPublishQueue.js",
     "./queue/quizPublishQueue": "./src/queue/quizPublishQueue.js",
     "./queue/predictPublishQueue": "./src/queue/predictPublishQueue.js",
     "./queue/queueRegistry": "./src/queue/queueRegistry.js",
+    "./translationHelper": "./src/lang/translationHelper.js",
     "./*": "./src/*"
   },
   "scripts": {

package/src/index.js

@@ -2,6 +2,9 @@ module.exports = {
   // ✅ Redis
   redisClient: require('./redis/redisClient'),
 
+  // ✅ Language tranlation Helper
+  translationHelper: require('./lang/translationHelper'),
+
   // ✅ Config
   config: require('./config'),
 
@@ -18,5 +21,5 @@ module.exports = {
   addProductPublishJob: require('./queue/productPublishQueue').addProductPublishJob,
   addQuizPublishJob: require('./queue/quizPublishQueue').addQuizPublishJob,
   addPredictionPublishJob: require('./queue/predictionPublishQueue').addPredictionPublishJob,
-
+  addImageAlbumPublishJob: require('./queue/imageAlbumPublishQueue').addImageAlbumPublishJob,
 };

package/src/lang/translationHelper.js

@@ -0,0 +1,223 @@
+const axios = require('axios');
+const { JSDOM } = require('jsdom'); // Required for HTML parsing
+
+// ♻️ Shared Redis connection
+const redis = require('../redis/redisClient');
+
+const BASE_URLS = [
+  'https://lingva.ml',
+  'https://translate.mentality.rip',
+  'https://translate.plausibility.cloud',
+  'https://lingva.translate.garudalinux.org',
+];
+
+const MAX_CHUNK_SIZE = 300;
+const RETRY_LIMIT = 3;
+const DELAY_BETWEEN_REQUESTS = 300;
+const REDIS_TTL = 60 * 60 * 24 * 7; // 7 days
+const MAX_FAILURES = 3; // Maximum consecutive mirror failures before Redis key is invalidated
+
+const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+function splitTextIntoChunks(text, maxChunkSize = MAX_CHUNK_SIZE) {
+  const sentences = text.match(/[^.!?]+[.!?]*[\s]*/g) || [text];
+  const chunks = [];
+  let current = '';
+
+  for (const sentence of sentences) {
+    if ((current + sentence).length > maxChunkSize) {
+      if (current.trim()) chunks.push(current.trim());
+      current = sentence;
+    } else {
+      current += sentence;
+    }
+  }
+
+  if (current.trim()) chunks.push(current.trim());
+  return chunks;
+}
+
+function isLikelyURL(text) {
+  return /(https?:\/\/[^\s]+)/.test(text);
+}
+
+async function getFromCache(cacheKey) {
+  return await redis.get(cacheKey);
+}
+
+async function setInCache(cacheKey, value, ttl = REDIS_TTL) {
+  await redis.set(cacheKey, value, 'EX', ttl);
+}
+
+async function deleteCacheKey(cacheKey) {
+  await redis.del(cacheKey);
+}
+
+/**
+ * 🔁 Translates a chunk with retries and mirror fallback.
+ */
+async function translateChunk(chunk, from, to, debug = false, fallbackToOriginal = true) {
+  const encoded = encodeURIComponent(chunk);
+  const langFrom = from?.toLowerCase?.() || 'auto';
+  const langTo = to?.toLowerCase?.() || 'en';
+
+  let totalFailures = 0;
+
+  for (let baseUrl of BASE_URLS) {
+    const url = `${baseUrl}/api/v1/${langFrom}/${langTo}/${encoded}`;
+
+    for (let attempt = 1; attempt <= RETRY_LIMIT; attempt++) {
+      try {
+        if (debug) console.log(`🌐 Translating "${chunk}" via ${baseUrl} (attempt ${attempt})`);
+        const response = await axios.get(url);
+        const translated = response?.data?.translation?.trim();
+        if (translated) return translated;
+        break;
+      } catch (error) {
+        const status = error?.response?.status;
+        const message = error?.message || 'Unknown error';
+
+        if (status === 429) {
+          if (debug) console.warn(`⚠️ 429 Too Many Requests – waiting ${attempt * 500}ms`);
+          await delay(attempt * 500);
+        } else {
+          totalFailures++;
+          if (debug) {
+            console.error(`❌ Mirror ${baseUrl} failed:`, { chunk, status, message });
+          }
+          break;
+        }
+      }
+    }
+  }
+
+  if (debug) console.warn(`🚫 Final fallback for chunk "${chunk}"`);
+  return fallbackToOriginal ? chunk : '';
+}
+
+/**
+ * 🧠 Translates plain text (cached in Redis, safe retries)
+ */
+async function translateWithLingva(text, to = 'fr', from = 'en', options = {}) {
+  const {
+    debug = false,
+    fallbackToOriginal = true,
+    cacheEnabled = true,
+    html = false,
+  } = options;
+
+  try {
+    if (!text || typeof text !== 'string' || !text.trim()) {
+      if (debug) console.warn('⚠️ Skipping invalid input');
+      return text;
+    }
+
+    if (isLikelyURL(text)) return text;
+
+    const trimmedText = text.trim();
+    const langFrom = from?.toLowerCase?.() || 'auto';
+    const langTo = to?.toLowerCase?.() || 'en';
+    const cacheKey = `lingva:${langFrom}:${langTo}:${trimmedText}`;
+
+    if (cacheEnabled) {
+      const cached = await getFromCache(cacheKey);
+      if (cached) {
+        if (debug) console.log('✅ Redis cache hit');
+        return cached;
+      }
+    }
+
+    let translated;
+    if (html) {
+      // calling infor with html content
+      /*
+        const { translateWithLingva } = require('./translateHelperRedis');
+        const htmlInput = `
+          <div>
+            <h1>Hello World!</h1>
+            <p>This is <strong>translated</strong> text with <a href="https://example.com">link</a>.</p>
+          </div>
+        `; 
+        const result = await translateWithLingva(htmlInput, 'fr', 'en', {
+          debug: true,
+          html: true,
+        });
+      */
+      translated = await translateHtmlContent(trimmedText, from, to, debug, fallbackToOriginal);
+    } else {
+      const chunks = splitTextIntoChunks(trimmedText);
+      const translations = [];
+
+      for (const chunk of chunks) {
+        const result = await translateChunk(chunk, from, to, debug, fallbackToOriginal);
+        translations.push(result);
+        await delay(DELAY_BETWEEN_REQUESTS);
+      }
+
+      translated = translations.join(' ').replace(/\s+/g, ' ').trim();
+    }
+
+    // Cache only if result is non-empty and not equal to input
+    if (translated && cacheEnabled && translated !== trimmedText) {
+      await setInCache(cacheKey, translated);
+    }
+
+    // Auto-invalidate on mirror failures
+    if (!translated || translated === trimmedText) {
+      const mirrorFailures = await redis.incr(`lingva:fail:${cacheKey}`);
+      await redis.expire(`lingva:fail:${cacheKey}`, 3600); // 1h expiration
+
+      if (mirrorFailures >= MAX_FAILURES) {
+        await deleteCacheKey(cacheKey);
+        if (debug) console.warn(`🔥 Invalidated cache due to repeated failures`);
+      }
+    }
+
+    return translated || (fallbackToOriginal ? trimmedText : '');
+  } catch (err) {
+    if (debug) console.error('💥 Unexpected error:', err.message);
+    return fallbackToOriginal ? text : '';
+  }
+}
+
+/**
+ * 🧠 Translate inner text of HTML while preserving tags
+ */
+async function translateHtmlContent(htmlText, from, to, debug = false, fallbackToOriginal = true) {
+  const dom = new JSDOM(`<body>${htmlText}</body>`);
+  const walker = dom.window.document.createTreeWalker(dom.window.document.body, NodeFilter.SHOW_TEXT);
+
+  while (walker.nextNode()) {
+    const node = walker.currentNode;
+    const raw = node.nodeValue.trim();
+    if (raw.length > 0) {
+      const translated = await translateWithLingva(raw, to, from, {
+        debug,
+        fallbackToOriginal,
+        cacheEnabled: true,
+        html: false, // prevent recursion
+      });
+      node.nodeValue = translated;
+      await delay(DELAY_BETWEEN_REQUESTS);
+    }
+  }
+
+  return dom.window.document.body.innerHTML;
+}
+
+/**
+ * 📚 Batch translation of plain text strings
+ */
+async function batchTranslateWithLingva(texts = [], to = 'fr', from = 'en', options = {}) {
+  const results = [];
+  for (const text of texts) {
+    const result = await translateWithLingva(text, to, from, options);
+    results.push(result);
+  }
+  return results;
+}
+
+module.exports = {
+  translateWithLingva,
+  batchTranslateWithLingva,
+};

package/src/queue/imageAlbumPublishQueue.js

@@ -0,0 +1,164 @@
+// src/queue/imageAlbumPublishQueue.js
+// One-off *publish* scheduler for image album items (BullMQ).
+// Strong upsert: purge any prior job for the same album before adding a new delayed job.
+
+const { Queue } = require('bullmq');
+const redisClient = require('../redis/redisClient');
+// const logger = require('../logger'); // uncomment if you have a shared logger
+
+const DEBUG    = String(process.env.DEBUG_LOGGER || '').trim() === '1';
+const DRIFT_MS = Math.max(0, Number(process.env.SCHEDULE_DRIFT_MS || 250));
+
+const imageAlbumPublishQueue = new Queue('imageAlbumPublishQueue', {
+  connection: redisClient,
+  defaultJobOptions: {
+    attempts: 5,
+    backoff: { type: 'exponential', delay: 2000 },
+    removeOnComplete: 500,
+    removeOnFail: 500,
+  },
+});
+
+// ── ID helpers ────────────────────────────────────────────────────────────────
+
+function stableJobId(album_id) {
+  return `imageAlbum:publish:${album_id}`;
+}
+function versionedJobId(album_id, runAtIso) {
+  return `imageAlbum:publish:${album_id}:${runAtIso}`;
+}
+
+// ── Time helpers ──────────────────────────────────────────────────────────────
+
+/** Parse anything into a UTC Date. Strings without timezone are treated as UTC. */
+function normalizeToUtcDate(input) {
+  if (input instanceof Date) return new Date(input.getTime());
+  if (typeof input === 'number') return new Date(input); // epoch ms
+
+  if (typeof input === 'string') {
+    // Has timezone (Z or ±hh:mm)
+    if (/[zZ]|[+\-]\d{2}:\d{2}$/.test(input)) {
+      const d = new Date(input);
+      if (Number.isNaN(d.getTime())) throw new Error(`Invalid ISO datetime: ${input}`);
+      return d;
+    }
+    // No timezone => treat as UTC
+    const s = input.trim().replace(' ', 'T');
+    const d = new Date(`${s}Z`);
+    if (Number.isNaN(d.getTime())) throw new Error(`Invalid datetime (no tz): ${input}`);
+    return d;
+  }
+
+  throw new Error(`Unsupported runAtUtc type: ${typeof input}`);
+}
+
+// ── Purge helpers ────────────────────────────────────────────────────────────
+
+async function findPendingAlbumJobs(album_id) {
+  const states = ['delayed', 'waiting', 'waiting-children', 'active'];
+  const jobs = await imageAlbumPublishQueue.getJobs(states);
+  return jobs.filter(
+    (j) => j?.name === 'imageAlbum:publish' && j?.data?.album_id === String(album_id)
+  );
+}
+
+async function purgeExistingForAlbum(album_id) {
+  let removed = 0;
+
+  // Remove any pending/waiting/active jobs for this album
+  const pendings = await findPendingAlbumJobs(album_id);
+  for (const j of pendings) {
+    try {
+      await j.remove();
+      removed++;
+    } catch (e) {
+      if (DEBUG) console.warn(`[imageAlbumPublishQueue] failed to remove pending job ${j.id}: ${e.message}`);
+    }
+  }
+
+  // Also remove a leftover stable job if present
+  const last = await imageAlbumPublishQueue.getJob(stableJobId(album_id));
+  if (last) {
+    try {
+      await last.remove();
+      removed++;
+    } catch (e) {
+      if (DEBUG) console.warn(`[imageAlbumPublishQueue] failed to remove stable job ${last.id}: ${e.message}`);
+    }
+  }
+
+  if (DEBUG) {
+    console.info('[imageAlbumPublishQueue] purgeExistingForAlbum', {
+      album_id: String(album_id),
+      removed,
+    });
+  }
+  return removed;
+}
+
+// ── Public API ───────────────────────────────────────────────────────────────
+
+/**
+ * Upsert a one-off publish job for an image album.
+ * - Purges any previous jobs for this album
+ * - Uses a versioned jobId by default (or stable if useStableId=true)
+ *
+ * @param {Object} params
+ * @param {string|number} params.album_id
+ * @param {string|number|Date} params.runAtUtc - ISO string, epoch ms, or Date (UTC)
+ * @param {object} [params.extra]
+ * @param {boolean} [params.useStableId=false] - if true, uses stable jobId
+ */
+async function addImageAlbumPublishJob({ album_id, runAtUtc, extra = {}, useStableId = false }) {
+  if (!album_id || !runAtUtc) {
+    throw new Error('addImageAlbumPublishJob: album_id and runAtUtc are required');
+  }
+
+  const runAt = normalizeToUtcDate(runAtUtc);
+  const runAtIso = runAt.toISOString();
+
+  // Drift guard to avoid immediate fire due to ms skew
+  const now = Date.now();
+  let delayMs = runAt.getTime() - now;
+  if (delayMs < DRIFT_MS) delayMs = Math.max(0, DRIFT_MS);
+
+  // Purge any prior jobs for this album (across states)
+  await purgeExistingForAlbum(album_id);
+
+  // Choose jobId strategy
+  const jobId = useStableId ? stableJobId(album_id) : versionedJobId(album_id, runAtIso);
+
+  const job = await imageAlbumPublishQueue.add(
+    'imageAlbum:publish',
+    { album_id: String(album_id), runAtUtc: runAtIso, extra },
+    { jobId, delay: delayMs }
+  );
+
+  if (DEBUG) {
+    console.info('[imageAlbumPublishQueue] upsert', {
+      album_id: String(album_id),
+      jobId,
+      runAtIso,
+      delayMs,
+      nowIso: new Date(now).toISOString(),
+    });
+  }
+
+  return job;
+}
+
+/** Cancel any scheduled publish for a given image album (if present). */
+async function cancelImageAlbumPublishJob(album_id) {
+  const removed = await purgeExistingForAlbum(album_id);
+  return removed > 0;
+}
+
+module.exports = {
+  imageAlbumPublishQueue,
+  addImageAlbumPublishJob,
+  cancelImageAlbumPublishJob,
+  // helpers for tooling/tests
+  stableJobId,
+  versionedJobId,
+  normalizeToUtcDate,
+};