azify-logger 1.0.26 → 1.0.28

package/scripts/redis-worker.js

@@ -0,0 +1,467 @@
+#!/usr/bin/env node
+
+const os = require('os')
+const http = require('http')
+const https = require('https')
+const Redis = require('ioredis')
+const axios = require('axios')
+const { DEFAULT_STREAM_KEY, DEFAULT_MAXLEN } = require('../queue/redisQueue')
+
+const STREAM_KEY = process.env.AZIFY_LOGGER_REDIS_STREAM || process.env.AZIFY_LOGGER_REDIS_QUEUE_KEY || DEFAULT_STREAM_KEY
+const DEAD_LETTER_STREAM_KEY = process.env.AZIFY_LOGGER_REDIS_DLQ || `${STREAM_KEY}:dlq`
+const REDIS_URL = process.env.AZIFY_LOGGER_REDIS_URL || 'redis://localhost:6381'
+const WORKER_GROUP = process.env.AZIFY_LOGGER_REDIS_GROUP || 'azify-logger-workers'
+const CONSUMER_NAME = process.env.AZIFY_LOGGER_REDIS_CONSUMER || `${os.hostname()}-${process.pid}`
+const MAX_BATCH = Number(process.env.AZIFY_LOGGER_REDIS_BATCH || 100)
+const BLOCK_MS = Number(process.env.AZIFY_LOGGER_REDIS_BLOCK || 5000)
+const MAX_DELIVERY_ATTEMPTS = Number(process.env.AZIFY_LOGGER_MAX_DELIVERY_ATTEMPTS || 10)
+const PENDING_IDLE_TIMEOUT = Number(process.env.AZIFY_LOGGER_PENDING_IDLE_TIMEOUT || 60000)
+
+const TRANSPORT_TIMEOUT = Number(process.env.AZIFY_LOGGER_HTTP_TIMEOUT || 250)
+const MAX_SOCKETS = Number(process.env.AZIFY_LOGGER_MAX_SOCKETS || 20)
+const WORKER_CONCURRENCY = Math.max(1, Number(process.env.AZIFY_LOGGER_WORKER_CONCURRENCY || 25))
+const NO_GROUP_RETRY_DELAY = Number(process.env.AZIFY_LOGGER_WORKER_NOGROUP_DELAY || 250)
+
+const httpAgent = new http.Agent({ keepAlive: true, maxSockets: MAX_SOCKETS })
+const httpsAgent = new https.Agent({ keepAlive: true, maxSockets: MAX_SOCKETS })
+
+const redisOptions = {
+  enableAutoPipelining: true,
+  maxRetriesPerRequest: null,
+  retryStrategy(times) {
+    const delay = Math.min(1000 * Math.pow(1.5, times), 10000)
+    return delay
+  },
+  reconnectOnError(err) {
+    const msg = err && err.message ? err.message : ''
+    if (msg.includes('READONLY') || msg.includes('ECONNRESET') || msg.includes('ECONNREFUSED')) {
+      return true
+    }
+    return false
+  }
+}
+
+let stopRequested = false
+let deliveries = 0
+let lastRedisErrorLog = 0
+let consecutiveNoGroupErrors = 0
+let redisErrorCount = 0
+const REDIS_ERROR_LOG_INTERVAL = 300000 // 5 minutos entre logs (evitar logs repetidos)
+
+// Usar flag global compartilhada com redisQueue.js para garantir apenas 1 log por processo
+if (typeof global.__azifyLoggerRedisErrorLogged === 'undefined') {
+  global.__azifyLoggerRedisErrorLogged = false
+  global.__azifyLoggerRedisErrorLastLog = 0
+}
+
+const redis = new Redis(REDIS_URL, redisOptions)
+redis.on('error', (err) => {
+  // Log apenas uma vez por processo inteiro (compartilhado com producer)
+  // Se já foi logado pelo producer, não logar novamente
+  const now = Date.now()
+  if (!global.__azifyLoggerRedisErrorLogged && now - global.__azifyLoggerRedisErrorLastLog > REDIS_ERROR_LOG_INTERVAL) {
+    if (err && (err.code === 'ECONNREFUSED' || err.message?.includes('ECONNREFUSED') || err.message?.includes('Redis'))) {
+      global.__azifyLoggerRedisErrorLogged = true
+      global.__azifyLoggerRedisErrorLastLog = now
+      redisErrorCount++
+      // Mensagem clara: aplicação continua funcionando, apenas logging está desabilitado
+      process.stderr.write('[azify-logger] ⚠️  Redis indisponível. O sistema de logging está desabilitado. A aplicação continua funcionando normalmente.\n')
+      lastRedisErrorLog = now
+    }
+  }
+  // Após primeira mensagem, não logar mais - silenciar completamente
+})
+redis.on('connect', () => {
+  // Resetar contador quando conectar com sucesso (sem logar para não poluir)
+  if (redisErrorCount > 0 || global.__azifyLoggerRedisErrorLogged) {
+    redisErrorCount = 0
+    lastRedisErrorLog = 0
+    consecutiveNoGroupErrors = 0
+    global.__azifyLoggerRedisErrorLogged = false
+    global.__azifyLoggerRedisErrorLastLog = 0
+  }
+})
+
+process.on('uncaughtException', (err) => {
+  console.error('[azify-logger][worker] uncaughtException:', err)
+  process.exit(1)
+})
+
+process.on('unhandledRejection', (reason) => {
+  console.error('[azify-logger][worker] unhandledRejection:', reason)
+  process.exit(1)
+})
+
+async function ensureGroup() {
+  try {
+    await redis.xgroup('CREATE', STREAM_KEY, WORKER_GROUP, '0', 'MKSTREAM')
+  } catch (err) {
+    if (!String(err && err.message).includes('BUSYGROUP')) {
+      throw err
+    }
+  }
+}
+
+function isNoGroupError(err) {
+  return typeof (err && err.message) === 'string' && err.message.includes('NOGROUP')
+}
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms))
+}
+
+// Headers sensíveis que devem ser mascarados
+const SENSITIVE_HEADER_KEYS = new Set([
+  'authorization',
+  'cookie',
+  'set-cookie',
+  'x-api-key',
+  'x-auth-token',
+  'x-access-token',
+  'proxy-authorization',
+  'x-signature',
+  'x-timestamp'
+])
+
+// Campos sensíveis no body que devem ser mascarados
+const SENSITIVE_BODY_FIELDS = new Set([
+  'password',
+  'token',
+  'secret',
+  'apiKey',
+  'api_key',
+  'accessToken',
+  'access_token',
+  'refreshToken',
+  'refresh_token',
+  'clientSecret',
+  'client_secret',
+  'creditCard',
+  'credit_card',
+  'cvv',
+  'cvc'
+])
+
+// Função para sanitizar headers
+function sanitizeHeaders(headers) {
+  if (!headers || typeof headers !== 'object') {
+    return {}
+  }
+  const sanitized = {}
+  for (const key in headers) {
+    if (!headers.hasOwnProperty(key)) continue
+    const lower = String(key).toLowerCase()
+    if (SENSITIVE_HEADER_KEYS.has(lower)) {
+      sanitized[key] = '***'
+    } else {
+      const value = headers[key]
+      sanitized[key] = Array.isArray(value) ? value.map(String) : (value != null ? String(value) : value)
+    }
+  }
+  return sanitized
+}
+
+// Função para sanitizar body (sem truncamento - manter tamanho completo)
+function sanitizeBody(body) {
+  if (!body || typeof body !== 'object') {
+    // Se não for objeto, retornar como está (não truncar strings)
+    return body
+  }
+  
+  try {
+    // Sanitizar campos sensíveis recursivamente (sem limitação de tamanho)
+    const sanitized = Array.isArray(body) ? [] : {}
+    
+    for (const key in body) {
+      if (!body.hasOwnProperty(key)) continue
+      const lower = String(key).toLowerCase()
+      
+      if (SENSITIVE_BODY_FIELDS.has(lower) || lower.includes('password') || lower.includes('secret')) {
+        // Mascarar campos sensíveis
+        sanitized[key] = '***'
+      } else if (typeof body[key] === 'object' && body[key] !== null) {
+        // Recursivamente sanitizar objetos aninhados (sem limitação de profundidade)
+        sanitized[key] = sanitizeBody(body[key])
+      } else {
+        // Copiar valores não sensíveis (mantendo tamanho completo)
+        sanitized[key] = body[key]
+      }
+    }
+    
+    return sanitized
+  } catch (err) {
+    // Se houver erro na sanitização, retornar body original (não truncar)
+    return body
+  }
+}
+
+// Função para sanitizar payload completo
+function sanitizePayload(payload) {
+  if (!payload || typeof payload !== 'object') {
+    return payload
+  }
+  
+  const sanitized = { ...payload }
+  
+  // Sanitizar meta se existir
+  if (sanitized.meta && typeof sanitized.meta === 'object') {
+    // Sanitizar headers da request
+    if (sanitized.meta.request && sanitized.meta.request.headers) {
+      sanitized.meta.request.headers = sanitizeHeaders(sanitized.meta.request.headers)
+    }
+    
+    // Sanitizar headers da response
+    if (sanitized.meta.response && sanitized.meta.response.headers) {
+      sanitized.meta.response.headers = sanitizeHeaders(sanitized.meta.response.headers)
+    }
+    
+    // Sanitizar body da request
+    if (sanitized.meta.request && sanitized.meta.request.body) {
+      sanitized.meta.request.body = sanitizeBody(sanitized.meta.request.body)
+    }
+    
+    // Sanitizar body da response
+    if (sanitized.meta.response && sanitized.meta.response.body) {
+      sanitized.meta.response.body = sanitizeBody(sanitized.meta.response.body)
+    }
+    
+    // Sanitizar headers de HTTP client (interceptors)
+    if (sanitized.meta.headers) {
+      sanitized.meta.headers = sanitizeHeaders(sanitized.meta.headers)
+    }
+    if (sanitized.meta.responseHeaders) {
+      sanitized.meta.responseHeaders = sanitizeHeaders(sanitized.meta.responseHeaders)
+    }
+    if (sanitized.meta.responseBody) {
+      sanitized.meta.responseBody = sanitizeBody(sanitized.meta.responseBody)
+    }
+  }
+  
+  return sanitized
+}
+
+async function deliver(entry) {
+  if (!entry || typeof entry !== 'object') {
+    return
+  }
+  const target = entry.loggerUrl
+  if (!target) {
+    return
+  }
+
+  // Sanitizar payload antes de enviar
+  const sanitizedPayload = entry.payload ? sanitizePayload(entry.payload) : entry.payload
+
+  await axios.post(target, sanitizedPayload, {
+    headers: entry.headers || {},
+    timeout: TRANSPORT_TIMEOUT,
+    httpAgent,
+    httpsAgent,
+    validateStatus: () => true
+  })
+  
+  deliveries += 1
+  // Log removido para reduzir ruído nos logs
+}
+
+async function requeue(entry, attempts) {
+  const next = {
+    ...entry,
+    attempts: attempts + 1,
+    lastAttemptAt: Date.now()
+  }
+  await redis.xadd(STREAM_KEY, 'MAXLEN', '~', DEFAULT_MAXLEN, '*', 'entry', JSON.stringify(next))
+}
+
+async function deadLetter(entry, reason) {
+  const payload = {
+    ...entry,
+    deadLetterReason: reason,
+    deadLetterAt: Date.now()
+  }
+  await redis.xadd(DEAD_LETTER_STREAM_KEY, '*', 'entry', JSON.stringify(payload))
+}
+
+async function acknowledge(id) {
+  await redis.xack(STREAM_KEY, WORKER_GROUP, id)
+}
+
+async function claimPending() {
+  try {
+    const pending = await redis.xpending(STREAM_KEY, WORKER_GROUP, '-', '+', MAX_BATCH)
+    const stale = pending.filter((item) => item[1] >= PENDING_IDLE_TIMEOUT)
+    for (const item of stale) {
+      const id = item[0]
+      const claim = await redis.xclaim(STREAM_KEY, WORKER_GROUP, CONSUMER_NAME, 0, id, 'JUSTID')
+      if (claim && claim.length) {
+        const entries = await redis.xrange(STREAM_KEY, id, id)
+        if (entries && entries.length) {
+          await processBatch(entries)
+        }
+      }
+    }
+  } catch (err) {
+    if (isNoGroupError(err)) {
+      consecutiveNoGroupErrors += 1
+      if (consecutiveNoGroupErrors === 1) {
+        console.warn('[azify-logger][worker] grupo não encontrado ao recuperar pendentes, recriando…')
+      }
+      await ensureGroup()
+      if (NO_GROUP_RETRY_DELAY > 0) {
+        await sleep(NO_GROUP_RETRY_DELAY)
+      }
+      return
+    }
+    console.error('[azify-logger][worker] erro ao reivindicar pendentes:', err && err.message ? err.message : err)
+  }
+}
+
+async function processEntry(raw) {
+  const id = raw[0]
+  const fields = raw[1]
+  const serialized = fields[1]
+  let entry
+  try {
+    entry = JSON.parse(serialized)
+  } catch (_) {
+    await acknowledge(id)
+    return
+  }
+
+  const attempts = Number(entry.attempts || 0)
+  try {
+    await deliver(entry)
+    await acknowledge(id)
+    consecutiveNoGroupErrors = 0
+  } catch (error) {
+    // Silenciar logs de erro - não poluir logs da aplicação
+    // Apenas enviar para DLQ após max tentativas ou requeue silenciosamente
+    if (attempts + 1 >= MAX_DELIVERY_ATTEMPTS) {
+      await acknowledge(id)
+      await deadLetter(entry, error && error.message ? error.message : 'delivery-error')
+    } else {
+      await acknowledge(id)
+      await requeue(entry, attempts)
+    }
+  }
+}
+
+async function processBatch(entries) {
+  if (!entries || !entries.length) {
+    return
+  }
+
+  const executing = []
+  for (const entry of entries) {
+    executing.push(processEntry(entry))
+    if (executing.length >= WORKER_CONCURRENCY) {
+      await Promise.allSettled(executing.splice(0, executing.length))
+    }
+  }
+
+  if (executing.length) {
+    await Promise.allSettled(executing)
+  }
+}
+
+async function consumeLoop() {
+  let groupEnsured = false
+  
+  while (!stopRequested) {
+    // Garantir que o grupo existe antes de tentar ler
+    if (!groupEnsured) {
+      try {
+        await ensureGroup()
+        groupEnsured = true
+        consecutiveNoGroupErrors = 0
+      } catch (err) {
+        const errMsg = err && err.message ? err.message : String(err)
+        if (!errMsg.includes('BUSYGROUP')) {
+          const now = Date.now()
+          if (now - lastRedisErrorLog > 5000) {
+            console.error('[azify-logger][worker] erro ao garantir grupo:', errMsg)
+            lastRedisErrorLog = now
+          }
+          await sleep(1000)
+        } else {
+          groupEnsured = true
+        }
+        continue
+      }
+    }
+    
+    let messages = null
+    try {
+      // Usar xreadgroup diretamente do ioredis
+      // Sintaxe: xreadgroup('GROUP', group, consumer, 'COUNT', count, 'BLOCK', block, 'STREAMS', key, id)
+      messages = await redis.xreadgroup(
+        'GROUP', WORKER_GROUP, CONSUMER_NAME,
+        'COUNT', MAX_BATCH,
+        'BLOCK', BLOCK_MS,
+        'STREAMS', STREAM_KEY, '>'
+      )
+      
+      // Se retornar mensagens, processar
+      if (messages && Array.isArray(messages) && messages.length > 0) {
+        // XREADGROUP retorna [[streamName, [entries]]]
+        for (const streamData of messages) {
+          if (Array.isArray(streamData) && streamData.length >= 2) {
+            const entries = streamData[1]
+            if (Array.isArray(entries) && entries.length > 0) {
+              await processBatch(entries)
+            }
+          }
+        }
+        consecutiveNoGroupErrors = 0
+        continue
+      }
+      
+      // Se não houver mensagens novas, verificar pendentes
+      await claimPending()
+    } catch (err) {
+      const errMsg = err && err.message ? err.message : String(err)
+      
+      // Se for erro de grupo ou sintaxe, recriar o grupo
+      if (isNoGroupError(err) || errMsg.includes('syntax error') || errMsg.includes('NOGROUP')) {
+        groupEnsured = false
+        consecutiveNoGroupErrors += 1
+        if (consecutiveNoGroupErrors === 1) {
+          console.warn('[azify-logger][worker] grupo não encontrado ou erro de sintaxe, recriando…')
+        }
+        await sleep(NO_GROUP_RETRY_DELAY)
+        continue
+      }
+      
+      // Outros erros - log apenas a cada 5 segundos para evitar spam
+      const now = Date.now()
+      if (now - lastRedisErrorLog > 5000) {
+        console.error('[azify-logger][worker] erro ao ler stream:', errMsg)
+        lastRedisErrorLog = now
+      }
+      await sleep(250)
+    }
+  }
+}
+
+async function shutdown() {
+  stopRequested = true
+  console.log('[azify-logger][worker] encerrando...')
+  await redis.quit().catch(() => {})
+  process.exit(0)
+}
+
+process.on('SIGINT', shutdown)
+process.on('SIGTERM', shutdown)
+
+ensureGroup()
+  .then(() => {
+    console.log('[azify-logger][worker] consumindo stream', STREAM_KEY, 'como', CONSUMER_NAME)
+    if (process.send) {
+      process.send({ type: 'azify-logger:ready', pid: process.pid, stream: STREAM_KEY })
+    }
+    return consumeLoop()
+  })
+  .catch((err) => {
+    console.error('[azify-logger][worker] não foi possível iniciar:', err)
+    process.exit(1)
+  })