azify-logger 1.0.26 → 1.0.29

package/scripts/redis-worker.js

@@ -0,0 +1,439 @@
+#!/usr/bin/env node
+
+const os = require('os')
+const http = require('http')
+const https = require('https')
+const Redis = require('ioredis')
+const axios = require('axios')
+const { DEFAULT_STREAM_KEY, DEFAULT_MAXLEN } = require('../queue/redisQueue')
+
+const STREAM_KEY = process.env.AZIFY_LOGGER_REDIS_STREAM || process.env.AZIFY_LOGGER_REDIS_QUEUE_KEY || DEFAULT_STREAM_KEY
+const DEAD_LETTER_STREAM_KEY = process.env.AZIFY_LOGGER_REDIS_DLQ || `${STREAM_KEY}:dlq`
+const REDIS_URL = process.env.AZIFY_LOGGER_REDIS_URL || 'redis://localhost:6381'
+const WORKER_GROUP = process.env.AZIFY_LOGGER_REDIS_GROUP || 'azify-logger-workers'
+const CONSUMER_NAME = process.env.AZIFY_LOGGER_REDIS_CONSUMER || `${os.hostname()}-${process.pid}`
+const MAX_BATCH = Number(process.env.AZIFY_LOGGER_REDIS_BATCH || 500)
+const BLOCK_MS = Number(process.env.AZIFY_LOGGER_REDIS_BLOCK || 5000)
+const MAX_DELIVERY_ATTEMPTS = Number(process.env.AZIFY_LOGGER_MAX_DELIVERY_ATTEMPTS || 10)
+const PENDING_IDLE_TIMEOUT = Number(process.env.AZIFY_LOGGER_PENDING_IDLE_TIMEOUT || 60000)
+
+const TRANSPORT_TIMEOUT = Number(process.env.AZIFY_LOGGER_HTTP_TIMEOUT || 250)
+const MAX_SOCKETS = Number(process.env.AZIFY_LOGGER_MAX_SOCKETS || 50)
+const WORKER_CONCURRENCY = Math.max(1, Number(process.env.AZIFY_LOGGER_WORKER_CONCURRENCY || 100))
+const NO_GROUP_RETRY_DELAY = Number(process.env.AZIFY_LOGGER_WORKER_NOGROUP_DELAY || 250)
+
+const httpAgent = new http.Agent({ keepAlive: true, maxSockets: MAX_SOCKETS })
+const httpsAgent = new https.Agent({ keepAlive: true, maxSockets: MAX_SOCKETS })
+
+const redisOptions = {
+  enableAutoPipelining: true,
+  maxRetriesPerRequest: null,
+  retryStrategy(times) {
+    const delay = Math.min(1000 * Math.pow(1.5, times), 10000)
+    return delay
+  },
+  reconnectOnError(err) {
+    const msg = err && err.message ? err.message : ''
+    if (msg.includes('READONLY') || msg.includes('ECONNRESET') || msg.includes('ECONNREFUSED')) {
+      return true
+    }
+    return false
+  }
+}
+
+let stopRequested = false
+let deliveries = 0
+let lastRedisErrorLog = 0
+let consecutiveNoGroupErrors = 0
+let redisErrorCount = 0
+const REDIS_ERROR_LOG_INTERVAL = 300000
+if (typeof global.__azifyLoggerRedisErrorLogged === 'undefined') {
+  global.__azifyLoggerRedisErrorLogged = false
+  global.__azifyLoggerRedisErrorLastLog = 0
+}
+
+const redis = new Redis(REDIS_URL, redisOptions)
+redis.on('error', (err) => {
+  const now = Date.now()
+  if (!global.__azifyLoggerRedisErrorLogged && now - global.__azifyLoggerRedisErrorLastLog > REDIS_ERROR_LOG_INTERVAL) {
+    if (err && (err.code === 'ECONNREFUSED' || err.message?.includes('ECONNREFUSED') || err.message?.includes('Redis'))) {
+      global.__azifyLoggerRedisErrorLogged = true
+      global.__azifyLoggerRedisErrorLastLog = now
+      redisErrorCount++
+      process.stderr.write('[azify-logger] ⚠️  Redis indisponível. O sistema de logging está desabilitado. A aplicação continua funcionando normalmente.\n')
+      lastRedisErrorLog = now
+    }
+  }
+})
+redis.on('connect', () => {
+  if (redisErrorCount > 0 || global.__azifyLoggerRedisErrorLogged) {
+    redisErrorCount = 0
+    lastRedisErrorLog = 0
+    consecutiveNoGroupErrors = 0
+    global.__azifyLoggerRedisErrorLogged = false
+    global.__azifyLoggerRedisErrorLastLog = 0
+  }
+})
+
+process.on('uncaughtException', (err) => {
+  console.error('[azify-logger][worker] uncaughtException:', err)
+  process.exit(1)
+})
+
+process.on('unhandledRejection', (reason) => {
+  console.error('[azify-logger][worker] unhandledRejection:', reason)
+  process.exit(1)
+})
+
+async function ensureGroup() {
+  try {
+    await redis.xgroup('CREATE', STREAM_KEY, WORKER_GROUP, '0', 'MKSTREAM')
+  } catch (err) {
+    if (!String(err && err.message).includes('BUSYGROUP')) {
+      throw err
+    }
+  }
+}
+
+function isNoGroupError(err) {
+  return typeof (err && err.message) === 'string' && err.message.includes('NOGROUP')
+}
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms))
+}
+
+const SENSITIVE_HEADER_KEYS = new Set([
+  'authorization',
+  'cookie',
+  'set-cookie',
+  'x-api-key',
+  'x-auth-token',
+  'x-access-token',
+  'proxy-authorization',
+  'x-signature',
+  'x-timestamp'
+])
+
+const SENSITIVE_BODY_FIELDS = new Set([
+  'password',
+  'token',
+  'secret',
+  'apiKey',
+  'api_key',
+  'accessToken',
+  'access_token',
+  'refreshToken',
+  'refresh_token',
+  'clientSecret',
+  'client_secret',
+  'creditCard',
+  'credit_card',
+  'cvv',
+  'cvc'
+])
+
+function sanitizeHeaders(headers) {
+  if (!headers || typeof headers !== 'object') {
+    return {}
+  }
+  const sanitized = {}
+  for (const key in headers) {
+    if (!headers.hasOwnProperty(key)) continue
+    const lower = String(key).toLowerCase()
+    if (SENSITIVE_HEADER_KEYS.has(lower)) {
+      sanitized[key] = '***'
+    } else {
+      const value = headers[key]
+      sanitized[key] = Array.isArray(value) ? value.map(String) : (value != null ? String(value) : value)
+    }
+  }
+  return sanitized
+}
+
+function sanitizeBody(body) {
+  if (!body || typeof body !== 'object') {
+    return body
+  }
+  
+  try {
+    const sanitized = Array.isArray(body) ? [] : {}
+    
+    for (const key in body) {
+      if (!body.hasOwnProperty(key)) continue
+      const lower = String(key).toLowerCase()
+      
+      if (SENSITIVE_BODY_FIELDS.has(lower) || lower.includes('password') || lower.includes('secret')) {
+        sanitized[key] = '***'
+      } else if (typeof body[key] === 'object' && body[key] !== null) {
+        sanitized[key] = sanitizeBody(body[key])
+      } else {
+        sanitized[key] = body[key]
+      }
+    }
+    
+    return sanitized
+  } catch (err) {
+    return body
+  }
+}
+
+function sanitizePayload(payload) {
+  if (!payload || typeof payload !== 'object') {
+    return payload
+  }
+  
+  const sanitized = { ...payload }
+  
+  if (sanitized.meta && typeof sanitized.meta === 'object') {
+    if (sanitized.meta.request && sanitized.meta.request.headers) {
+      sanitized.meta.request.headers = sanitizeHeaders(sanitized.meta.request.headers)
+    }
+    
+    if (sanitized.meta.response && sanitized.meta.response.headers) {
+      sanitized.meta.response.headers = sanitizeHeaders(sanitized.meta.response.headers)
+    }
+    
+    if (sanitized.meta.request && sanitized.meta.request.body) {
+      sanitized.meta.request.body = sanitizeBody(sanitized.meta.request.body)
+    }
+    
+    if (sanitized.meta.response && sanitized.meta.response.body) {
+      sanitized.meta.response.body = sanitizeBody(sanitized.meta.response.body)
+    }
+    if (sanitized.meta.headers) {
+      sanitized.meta.headers = sanitizeHeaders(sanitized.meta.headers)
+    }
+    if (sanitized.meta.responseHeaders) {
+      sanitized.meta.responseHeaders = sanitizeHeaders(sanitized.meta.responseHeaders)
+    }
+    if (sanitized.meta.responseBody) {
+      sanitized.meta.responseBody = sanitizeBody(sanitized.meta.responseBody)
+    }
+  }
+  
+  return sanitized
+}
+
+async function deliver(entry) {
+  if (!entry || typeof entry !== 'object') {
+    return
+  }
+  const target = entry.loggerUrl
+  if (!target) {
+    return
+  }
+
+  let sanitizedPayload = entry.payload
+  if (entry.payload && typeof entry.payload === 'object') {
+    try {
+      sanitizedPayload = sanitizePayload(entry.payload)
+    } catch (err) {
+      sanitizedPayload = entry.payload
+    }
+  }
+
+  await axios.post(target, sanitizedPayload, {
+    headers: entry.headers || {},
+    timeout: TRANSPORT_TIMEOUT,
+    httpAgent,
+    httpsAgent,
+    validateStatus: () => true,
+    maxRedirects: 0
+  })
+  
+  deliveries += 1
+}
+
+async function requeue(entry, attempts) {
+  const next = {
+    ...entry,
+    attempts: attempts + 1,
+    lastAttemptAt: Date.now()
+  }
+  await redis.xadd(STREAM_KEY, 'MAXLEN', '~', DEFAULT_MAXLEN, '*', 'entry', JSON.stringify(next))
+}
+
+async function deadLetter(entry, reason) {
+  const payload = {
+    ...entry,
+    deadLetterReason: reason,
+    deadLetterAt: Date.now()
+  }
+  await redis.xadd(DEAD_LETTER_STREAM_KEY, '*', 'entry', JSON.stringify(payload))
+}
+
+async function acknowledge(id) {
+  await redis.xack(STREAM_KEY, WORKER_GROUP, id)
+}
+
+async function claimPending() {
+  try {
+    const pending = await redis.xpending(STREAM_KEY, WORKER_GROUP, '-', '+', MAX_BATCH)
+    const stale = pending.filter((item) => item[1] >= PENDING_IDLE_TIMEOUT)
+    for (const item of stale) {
+      const id = item[0]
+      const claim = await redis.xclaim(STREAM_KEY, WORKER_GROUP, CONSUMER_NAME, 0, id, 'JUSTID')
+      if (claim && claim.length) {
+        const entries = await redis.xrange(STREAM_KEY, id, id)
+        if (entries && entries.length) {
+          await processBatch(entries)
+        }
+      }
+    }
+  } catch (err) {
+    if (isNoGroupError(err)) {
+      consecutiveNoGroupErrors += 1
+      if (consecutiveNoGroupErrors === 1) {
+        console.warn('[azify-logger][worker] grupo não encontrado ao recuperar pendentes, recriando…')
+      }
+      await ensureGroup()
+      if (NO_GROUP_RETRY_DELAY > 0) {
+        await sleep(NO_GROUP_RETRY_DELAY)
+      }
+      return
+    }
+    console.error('[azify-logger][worker] erro ao reivindicar pendentes:', err && err.message ? err.message : err)
+  }
+}
+
+async function processEntry(raw) {
+  const id = raw[0]
+  const fields = raw[1]
+  const serialized = fields[1]
+  let entry
+  try {
+    entry = JSON.parse(serialized)
+  } catch (_) {
+    await acknowledge(id)
+    return
+  }
+
+  const attempts = Number(entry.attempts || 0)
+  try {
+    await deliver(entry)
+    await acknowledge(id)
+    consecutiveNoGroupErrors = 0
+  } catch (error) {
+    if (attempts + 1 >= MAX_DELIVERY_ATTEMPTS) {
+      await acknowledge(id)
+      await deadLetter(entry, error && error.message ? error.message : 'delivery-error')
+    } else {
+      await acknowledge(id)
+      await requeue(entry, attempts)
+    }
+  }
+}
+
+async function processBatch(entries) {
+  if (!entries || !entries.length) {
+    return
+  }
+
+  const chunkSize = WORKER_CONCURRENCY
+  const chunks = []
+  
+  for (let i = 0; i < entries.length; i += chunkSize) {
+    chunks.push(entries.slice(i, i + chunkSize))
+  }
+  
+  const chunkPromises = chunks.map(chunk => {
+    return Promise.allSettled(chunk.map(entry => processEntry(entry)))
+  })
+  
+  await Promise.all(chunkPromises)
+}
+
+async function consumeLoop() {
+  let groupEnsured = false
+  
+  while (!stopRequested) {
+    if (!groupEnsured) {
+      try {
+        await ensureGroup()
+        groupEnsured = true
+        consecutiveNoGroupErrors = 0
+      } catch (err) {
+        const errMsg = err && err.message ? err.message : String(err)
+        if (!errMsg.includes('BUSYGROUP')) {
+          const now = Date.now()
+          if (now - lastRedisErrorLog > 5000) {
+            console.error('[azify-logger][worker] erro ao garantir grupo:', errMsg)
+            lastRedisErrorLog = now
+          }
+          await sleep(1000)
+        } else {
+          groupEnsured = true
+        }
+        continue
+      }
+    }
+    
+    let messages = null
+    try {
+      messages = await redis.xreadgroup(
+        'GROUP', WORKER_GROUP, CONSUMER_NAME,
+        'COUNT', MAX_BATCH,
+        'BLOCK', BLOCK_MS,
+        'STREAMS', STREAM_KEY, '>'
+      )
+      
+      if (messages && Array.isArray(messages) && messages.length > 0) {
+        for (const streamData of messages) {
+          if (Array.isArray(streamData) && streamData.length >= 2) {
+            const entries = streamData[1]
+            if (Array.isArray(entries) && entries.length > 0) {
+              await processBatch(entries)
+            }
+          }
+        }
+        consecutiveNoGroupErrors = 0
+        continue
+      }
+      
+      await claimPending()
+    } catch (err) {
+      const errMsg = err && err.message ? err.message : String(err)
+      
+      if (isNoGroupError(err) || errMsg.includes('syntax error') || errMsg.includes('NOGROUP')) {
+        groupEnsured = false
+        consecutiveNoGroupErrors += 1
+        if (consecutiveNoGroupErrors === 1) {
+          console.warn('[azify-logger][worker] grupo não encontrado ou erro de sintaxe, recriando…')
+        }
+        await sleep(NO_GROUP_RETRY_DELAY)
+        continue
+      }
+      
+      const now = Date.now()
+      if (now - lastRedisErrorLog > 5000) {
+        console.error('[azify-logger][worker] erro ao ler stream:', errMsg)
+        lastRedisErrorLog = now
+      }
+      await sleep(250)
+    }
+  }
+}
+
+async function shutdown() {
+  stopRequested = true
+  console.log('[azify-logger][worker] encerrando...')
+  await redis.quit().catch(() => {})
+  process.exit(0)
+}
+
+process.on('SIGINT', shutdown)
+process.on('SIGTERM', shutdown)
+
+ensureGroup()
+  .then(() => {
+    console.log('[azify-logger][worker] consumindo stream', STREAM_KEY, 'como', CONSUMER_NAME)
+    if (process.send) {
+      process.send({ type: 'azify-logger:ready', pid: process.pid, stream: STREAM_KEY })
+    }
+    return consumeLoop()
+  })
+  .catch((err) => {
+    console.error('[azify-logger][worker] não foi possível iniciar:', err)
+    process.exit(1)
+  })