@kaikybrofc/omnizap-system 2.1.8

package/app/modules/stickerPackModule/stickerWorkerPipelineRuntime.js

@@ -0,0 +1,233 @@
+import logger from '../../utils/logger/loggerModule.js';
+import { setQueueDepth } from '../../observability/metrics.js';
+import { runStickerClassificationCycle } from './stickerClassificationBackgroundRuntime.js';
+import { runStickerAutoPackByTagsCycle } from './stickerAutoPackByTagsRuntime.js';
+import {
+  claimWorkerTask,
+  completeWorkerTask,
+  countWorkerTasksByStatus,
+  enqueueWorkerTask,
+  failWorkerTask,
+  hasPendingWorkerTask,
+} from './stickerWorkerTaskQueueRepository.js';
+
+const parseEnvBool = (value, fallback) => {
+  if (value === undefined || value === null || value === '') return fallback;
+  const normalized = String(value).trim().toLowerCase();
+  if (['1', 'true', 'yes', 'y', 'on'].includes(normalized)) return true;
+  if (['0', 'false', 'no', 'n', 'off'].includes(normalized)) return false;
+  return fallback;
+};
+
+const PIPELINE_ENABLED = parseEnvBool(process.env.STICKER_WORKER_PIPELINE_ENABLED, false);
+const STARTUP_DELAY_MS = Math.max(1_000, Number(process.env.STICKER_WORKER_PIPELINE_STARTUP_DELAY_MS) || 12_000);
+const SCHEDULER_INTERVAL_MS = Math.max(2_000, Number(process.env.STICKER_WORKER_PIPELINE_SCHEDULER_INTERVAL_MS) || 15_000);
+const POLLER_INTERVAL_MS = Math.max(1_000, Number(process.env.STICKER_WORKER_PIPELINE_POLLER_INTERVAL_MS) || 4_000);
+const WORKER_RETRY_DELAY_SECONDS = Math.max(5, Math.min(3600, Number(process.env.STICKER_WORKER_PIPELINE_RETRY_DELAY_SECONDS) || 45));
+
+const TASK_CADENCE_MS = {
+  classification_cycle: Math.max(10_000, Number(process.env.STICKER_WORKER_CLASSIFICATION_CADENCE_MS) || 120_000),
+  curation_cycle: Math.max(15_000, Number(process.env.STICKER_WORKER_CURATION_CADENCE_MS) || 180_000),
+  rebuild_cycle: Math.max(15_000, Number(process.env.STICKER_WORKER_REBUILD_CADENCE_MS) || 240_000),
+};
+
+const TASK_PRIORITY = {
+  classification_cycle: 70,
+  curation_cycle: 55,
+  rebuild_cycle: 50,
+};
+
+let startupHandle = null;
+let schedulerHandle = null;
+let pollerHandle = null;
+let runningPoll = false;
+let taskQueueAvailable = true;
+const nextScheduleByTask = new Map();
+
+const taskHandlers = {
+  classification_cycle: async () => runStickerClassificationCycle({ processPending: true, processReprocess: true }),
+  curation_cycle: async () => runStickerAutoPackByTagsCycle({ enableAdditions: true, enableRebuild: false }),
+  rebuild_cycle: async () => runStickerAutoPackByTagsCycle({ enableAdditions: false, enableRebuild: true }),
+};
+
+const refreshQueueDepthMetrics = async () => {
+  if (!taskQueueAvailable) return;
+  const [pending, processing, failed] = await Promise.all([
+    countWorkerTasksByStatus('pending'),
+    countWorkerTasksByStatus('processing'),
+    countWorkerTasksByStatus('failed'),
+  ]);
+  setQueueDepth('sticker_worker_tasks_pending', pending);
+  setQueueDepth('sticker_worker_tasks_processing', processing);
+  setQueueDepth('sticker_worker_tasks_failed', failed);
+};
+
+const scheduleTaskIfNeeded = async (taskType) => {
+  if (!taskQueueAvailable) return;
+  const cadence = TASK_CADENCE_MS[taskType];
+  if (!cadence) return;
+
+  const now = Date.now();
+  const nextDueAt = nextScheduleByTask.get(taskType) || 0;
+  if (now < nextDueAt) return;
+
+  const hasPending = await hasPendingWorkerTask(taskType);
+  if (hasPending) {
+    nextScheduleByTask.set(taskType, now + Math.floor(cadence / 2));
+    return;
+  }
+
+  await enqueueWorkerTask({
+    taskType,
+    payload: { scheduled_by: 'sticker_worker_pipeline' },
+    priority: TASK_PRIORITY[taskType] || 50,
+  });
+  nextScheduleByTask.set(taskType, now + cadence);
+};
+
+const schedulerTick = async () => {
+  if (!PIPELINE_ENABLED) return;
+
+  try {
+    await Promise.all([
+      scheduleTaskIfNeeded('classification_cycle'),
+      scheduleTaskIfNeeded('curation_cycle'),
+      scheduleTaskIfNeeded('rebuild_cycle'),
+    ]);
+    await refreshQueueDepthMetrics();
+  } catch (error) {
+    if (error?.code === 'ER_NO_SUCH_TABLE') {
+      taskQueueAvailable = false;
+      logger.warn('Fila do pipeline de workers indisponivel (migração pendente).', {
+        action: 'sticker_worker_pipeline_queue_unavailable',
+      });
+      return;
+    }
+    throw error;
+  }
+};
+
+const processSingleTaskType = async (taskType) => {
+  if (!taskQueueAvailable) return false;
+  const task = await claimWorkerTask({ taskType });
+  if (!task) return false;
+
+  try {
+    const handler = taskHandlers[taskType];
+    if (typeof handler !== 'function') {
+      throw new Error(`handler_not_found:${taskType}`);
+    }
+
+    await handler(task.payload || {});
+    await completeWorkerTask(task.id);
+
+    logger.debug('Task de worker concluída.', {
+      action: 'sticker_worker_task_completed',
+      task_type: taskType,
+      task_id: task.id,
+      attempts: task.attempts,
+    });
+    return true;
+  } catch (error) {
+    await failWorkerTask(task.id, {
+      error: error?.message || 'worker_task_failed',
+      retryDelaySeconds: WORKER_RETRY_DELAY_SECONDS,
+    });
+
+    logger.warn('Task de worker falhou.', {
+      action: 'sticker_worker_task_failed',
+      task_type: taskType,
+      task_id: task.id,
+      attempts: task.attempts,
+      error: error?.message,
+    });
+    return true;
+  }
+};
+
+const pollerTick = async () => {
+  if (runningPoll || !PIPELINE_ENABLED) return;
+  runningPoll = true;
+
+  try {
+    let progressed = false;
+
+    progressed = (await processSingleTaskType('classification_cycle')) || progressed;
+    progressed = (await processSingleTaskType('curation_cycle')) || progressed;
+    progressed = (await processSingleTaskType('rebuild_cycle')) || progressed;
+
+    if (progressed) {
+      await refreshQueueDepthMetrics();
+    }
+  } catch (error) {
+    if (error?.code === 'ER_NO_SUCH_TABLE') {
+      taskQueueAvailable = false;
+      logger.warn('Fila do pipeline de workers indisponivel durante o poll.', {
+        action: 'sticker_worker_pipeline_queue_unavailable_poll',
+      });
+    } else {
+      logger.error('Falha no poller do pipeline de workers.', {
+        action: 'sticker_worker_pipeline_poll_failed',
+        error: error?.message,
+      });
+    }
+  } finally {
+    runningPoll = false;
+  }
+};
+
+export const startStickerWorkerPipeline = () => {
+  if (!PIPELINE_ENABLED) {
+    logger.info('Pipeline de workers de sticker desabilitado.', {
+      action: 'sticker_worker_pipeline_disabled',
+    });
+    return;
+  }
+
+  if (startupHandle || schedulerHandle || pollerHandle) return;
+
+  logger.info('Iniciando pipeline de workers de sticker.', {
+    action: 'sticker_worker_pipeline_start',
+    startup_delay_ms: STARTUP_DELAY_MS,
+    scheduler_interval_ms: SCHEDULER_INTERVAL_MS,
+    poller_interval_ms: POLLER_INTERVAL_MS,
+    cadence_ms: TASK_CADENCE_MS,
+  });
+
+  startupHandle = setTimeout(() => {
+    startupHandle = null;
+
+    void schedulerTick();
+    void pollerTick();
+
+    schedulerHandle = setInterval(() => {
+      void schedulerTick();
+    }, SCHEDULER_INTERVAL_MS);
+
+    pollerHandle = setInterval(() => {
+      void pollerTick();
+    }, POLLER_INTERVAL_MS);
+
+    if (typeof schedulerHandle.unref === 'function') schedulerHandle.unref();
+    if (typeof pollerHandle.unref === 'function') pollerHandle.unref();
+  }, STARTUP_DELAY_MS);
+
+  if (typeof startupHandle.unref === 'function') startupHandle.unref();
+};
+
+export const stopStickerWorkerPipeline = () => {
+  if (startupHandle) {
+    clearTimeout(startupHandle);
+    startupHandle = null;
+  }
+  if (schedulerHandle) {
+    clearInterval(schedulerHandle);
+    schedulerHandle = null;
+  }
+  if (pollerHandle) {
+    clearInterval(pollerHandle);
+    pollerHandle = null;
+  }
+};
+
+export const isStickerWorkerPipelineEnabled = () => PIPELINE_ENABLED;

package/app/modules/stickerPackModule/stickerWorkerTaskQueueRepository.js

@@ -0,0 +1,205 @@
+import { randomUUID } from 'node:crypto';
+
+import { executeQuery, TABLES } from '../../../database/index.js';
+
+const normalizeTaskType = (value) => {
+  const normalized = String(value || '').trim().toLowerCase();
+  if (['classification_cycle', 'curation_cycle', 'rebuild_cycle'].includes(normalized)) return normalized;
+  return null;
+};
+
+const normalizeStatus = (value) => {
+  const normalized = String(value || '').trim().toLowerCase();
+  if (['pending', 'processing', 'completed', 'failed'].includes(normalized)) return normalized;
+  return null;
+};
+
+const clampInt = (value, fallback, min, max) => {
+  const numeric = Number(value);
+  if (!Number.isFinite(numeric)) return fallback;
+  return Math.max(min, Math.min(max, Math.floor(numeric)));
+};
+
+const parseJson = (value, fallback = null) => {
+  if (value === null || value === undefined) return fallback;
+  if (typeof value === 'object') return value;
+  if (Buffer.isBuffer(value)) {
+    try {
+      return JSON.parse(value.toString('utf8'));
+    } catch {
+      return fallback;
+    }
+  }
+
+  if (typeof value === 'string') {
+    try {
+      return JSON.parse(value);
+    } catch {
+      return fallback;
+    }
+  }
+
+  return fallback;
+};
+
+const normalizeRow = (row) => {
+  if (!row) return null;
+  return {
+    id: Number(row.id),
+    task_type: row.task_type,
+    payload: parseJson(row.payload, {}),
+    priority: Number(row.priority || 0),
+    scheduled_at: row.scheduled_at || null,
+    status: row.status,
+    attempts: Number(row.attempts || 0),
+    max_attempts: Number(row.max_attempts || 0),
+    worker_token: row.worker_token || null,
+    last_error: row.last_error || null,
+    locked_at: row.locked_at || null,
+    processed_at: row.processed_at || null,
+    created_at: row.created_at || null,
+    updated_at: row.updated_at || null,
+  };
+};
+
+export async function enqueueWorkerTask(
+  { taskType, payload = {}, priority = 50, scheduledAt = null, maxAttempts = 5 },
+  connection = null,
+) {
+  const normalizedTaskType = normalizeTaskType(taskType);
+  if (!normalizedTaskType) return false;
+
+  const safePriority = clampInt(priority, 50, 1, 100);
+  const safeMaxAttempts = clampInt(maxAttempts, 5, 1, 20);
+  const safeScheduledAt = scheduledAt ? new Date(scheduledAt) : null;
+  const scheduledValue = safeScheduledAt && Number.isFinite(safeScheduledAt.valueOf()) ? safeScheduledAt : null;
+
+  await executeQuery(
+    `INSERT INTO ${TABLES.STICKER_WORKER_TASK_QUEUE}
+      (task_type, payload, priority, scheduled_at, status, attempts, max_attempts)
+     VALUES (?, ?, ?, COALESCE(?, UTC_TIMESTAMP()), 'pending', 0, ?)`,
+    [normalizedTaskType, JSON.stringify(payload || {}), safePriority, scheduledValue, safeMaxAttempts],
+    connection,
+  );
+  return true;
+}
+
+export async function hasPendingWorkerTask(taskType, connection = null) {
+  const normalizedTaskType = normalizeTaskType(taskType);
+  if (!normalizedTaskType) return false;
+
+  const rows = await executeQuery(
+    `SELECT id
+     FROM ${TABLES.STICKER_WORKER_TASK_QUEUE}
+     WHERE task_type = ?
+       AND status IN ('pending', 'processing')
+     LIMIT 1`,
+    [normalizedTaskType],
+    connection,
+  );
+  return rows.length > 0;
+}
+
+export async function claimWorkerTask({ taskType, allowRetryFailed = true } = {}, connection = null) {
+  const normalizedTaskType = normalizeTaskType(taskType);
+  if (!normalizedTaskType) return null;
+
+  const workerToken = randomUUID();
+  const statusClause = allowRetryFailed
+    ? "(status = 'pending' OR (status = 'failed' AND attempts < max_attempts))"
+    : "status = 'pending'";
+
+  await executeQuery(
+    `UPDATE ${TABLES.STICKER_WORKER_TASK_QUEUE}
+     SET status = 'processing',
+         worker_token = ?,
+         locked_at = UTC_TIMESTAMP(),
+         attempts = attempts + 1,
+         updated_at = UTC_TIMESTAMP()
+     WHERE id = (
+       SELECT id FROM (
+         SELECT id
+         FROM ${TABLES.STICKER_WORKER_TASK_QUEUE}
+         WHERE task_type = ?
+           AND ${statusClause}
+           AND scheduled_at <= UTC_TIMESTAMP()
+         ORDER BY priority DESC, scheduled_at ASC, id ASC
+         LIMIT 1
+       ) pick
+     )`,
+    [workerToken, normalizedTaskType],
+    connection,
+  );
+
+  const rows = await executeQuery(
+    `SELECT *
+     FROM ${TABLES.STICKER_WORKER_TASK_QUEUE}
+     WHERE worker_token = ?
+       AND status = 'processing'
+     ORDER BY id DESC
+     LIMIT 1`,
+    [workerToken],
+    connection,
+  );
+
+  return normalizeRow(rows?.[0] || null);
+}
+
+export async function completeWorkerTask(taskId, connection = null) {
+  if (!taskId) return false;
+  await executeQuery(
+    `UPDATE ${TABLES.STICKER_WORKER_TASK_QUEUE}
+     SET status = 'completed',
+         processed_at = UTC_TIMESTAMP(),
+         worker_token = NULL,
+         locked_at = NULL,
+         last_error = NULL,
+         updated_at = UTC_TIMESTAMP()
+     WHERE id = ?`,
+    [taskId],
+    connection,
+  );
+  return true;
+}
+
+export async function failWorkerTask(
+  taskId,
+  {
+    error = null,
+    retryDelaySeconds = 0,
+  } = {},
+  connection = null,
+) {
+  if (!taskId) return false;
+  const safeDelay = clampInt(retryDelaySeconds, 0, 0, 86400 * 7);
+  const message = String(error || '').trim().slice(0, 255) || null;
+
+  await executeQuery(
+    `UPDATE ${TABLES.STICKER_WORKER_TASK_QUEUE}
+     SET status = IF(attempts >= max_attempts, 'failed', 'pending'),
+         worker_token = NULL,
+         locked_at = NULL,
+         last_error = ?,
+         scheduled_at = IF(attempts >= max_attempts, scheduled_at, UTC_TIMESTAMP() + INTERVAL ${safeDelay} SECOND),
+         updated_at = UTC_TIMESTAMP(),
+         processed_at = IF(attempts >= max_attempts, UTC_TIMESTAMP(), processed_at)
+     WHERE id = ?`,
+    [message, taskId],
+    connection,
+  );
+  return true;
+}
+
+export async function countWorkerTasksByStatus(status = 'pending', connection = null) {
+  const normalizedStatus = normalizeStatus(status);
+  if (!normalizedStatus) return 0;
+
+  const rows = await executeQuery(
+    `SELECT COUNT(*) AS total
+     FROM ${TABLES.STICKER_WORKER_TASK_QUEUE}
+     WHERE status = ?`,
+    [normalizedStatus],
+    connection,
+  );
+  return Number(rows?.[0]?.total || 0);
+}