azify-logger 1.0.28 → 1.0.30

package/scripts/redis-worker.js

@@ -12,14 +12,14 @@ const DEAD_LETTER_STREAM_KEY = process.env.AZIFY_LOGGER_REDIS_DLQ || `${STREAM_K
 const REDIS_URL = process.env.AZIFY_LOGGER_REDIS_URL || 'redis://localhost:6381'
 const WORKER_GROUP = process.env.AZIFY_LOGGER_REDIS_GROUP || 'azify-logger-workers'
 const CONSUMER_NAME = process.env.AZIFY_LOGGER_REDIS_CONSUMER || `${os.hostname()}-${process.pid}`
-const MAX_BATCH = Number(process.env.AZIFY_LOGGER_REDIS_BATCH || 100)
+const MAX_BATCH = Number(process.env.AZIFY_LOGGER_REDIS_BATCH || 500)
 const BLOCK_MS = Number(process.env.AZIFY_LOGGER_REDIS_BLOCK || 5000)
 const MAX_DELIVERY_ATTEMPTS = Number(process.env.AZIFY_LOGGER_MAX_DELIVERY_ATTEMPTS || 10)
 const PENDING_IDLE_TIMEOUT = Number(process.env.AZIFY_LOGGER_PENDING_IDLE_TIMEOUT || 60000)
 
 const TRANSPORT_TIMEOUT = Number(process.env.AZIFY_LOGGER_HTTP_TIMEOUT || 250)
-const MAX_SOCKETS = Number(process.env.AZIFY_LOGGER_MAX_SOCKETS || 20)
-const WORKER_CONCURRENCY = Math.max(1, Number(process.env.AZIFY_LOGGER_WORKER_CONCURRENCY || 25))
+const MAX_SOCKETS = Number(process.env.AZIFY_LOGGER_MAX_SOCKETS || 50)
+const WORKER_CONCURRENCY = Math.max(1, Number(process.env.AZIFY_LOGGER_WORKER_CONCURRENCY || 100))
 const NO_GROUP_RETRY_DELAY = Number(process.env.AZIFY_LOGGER_WORKER_NOGROUP_DELAY || 250)
 
 const httpAgent = new http.Agent({ keepAlive: true, maxSockets: MAX_SOCKETS })
@@ -46,9 +46,7 @@ let deliveries = 0
 let lastRedisErrorLog = 0
 let consecutiveNoGroupErrors = 0
 let redisErrorCount = 0
-const REDIS_ERROR_LOG_INTERVAL = 300000 // 5 minutos entre logs (evitar logs repetidos)
-
-// Usar flag global compartilhada com redisQueue.js para garantir apenas 1 log por processo
+const REDIS_ERROR_LOG_INTERVAL = 300000
 if (typeof global.__azifyLoggerRedisErrorLogged === 'undefined') {
   global.__azifyLoggerRedisErrorLogged = false
   global.__azifyLoggerRedisErrorLastLog = 0
@@ -56,23 +54,18 @@ if (typeof global.__azifyLoggerRedisErrorLogged === 'undefined') {
 
 const redis = new Redis(REDIS_URL, redisOptions)
 redis.on('error', (err) => {
-  // Log apenas uma vez por processo inteiro (compartilhado com producer)
-  // Se já foi logado pelo producer, não logar novamente
   const now = Date.now()
   if (!global.__azifyLoggerRedisErrorLogged && now - global.__azifyLoggerRedisErrorLastLog > REDIS_ERROR_LOG_INTERVAL) {
     if (err && (err.code === 'ECONNREFUSED' || err.message?.includes('ECONNREFUSED') || err.message?.includes('Redis'))) {
       global.__azifyLoggerRedisErrorLogged = true
       global.__azifyLoggerRedisErrorLastLog = now
       redisErrorCount++
-      // Mensagem clara: aplicação continua funcionando, apenas logging está desabilitado
       process.stderr.write('[azify-logger] ⚠️  Redis indisponível. O sistema de logging está desabilitado. A aplicação continua funcionando normalmente.\n')
       lastRedisErrorLog = now
     }
   }
-  // Após primeira mensagem, não logar mais - silenciar completamente
 })
 redis.on('connect', () => {
-  // Resetar contador quando conectar com sucesso (sem logar para não poluir)
   if (redisErrorCount > 0 || global.__azifyLoggerRedisErrorLogged) {
     redisErrorCount = 0
     lastRedisErrorLog = 0
@@ -110,7 +103,6 @@ function sleep(ms) {
   return new Promise(resolve => setTimeout(resolve, ms))
 }
 
-// Headers sensíveis que devem ser mascarados
 const SENSITIVE_HEADER_KEYS = new Set([
   'authorization',
   'cookie',
@@ -123,7 +115,6 @@ const SENSITIVE_HEADER_KEYS = new Set([
   'x-timestamp'
 ])
 
-// Campos sensíveis no body que devem ser mascarados
 const SENSITIVE_BODY_FIELDS = new Set([
   'password',
   'token',
@@ -142,7 +133,6 @@ const SENSITIVE_BODY_FIELDS = new Set([
   'cvc'
 ])
 
-// Função para sanitizar headers
 function sanitizeHeaders(headers) {
   if (!headers || typeof headers !== 'object') {
     return {}
@@ -161,15 +151,12 @@ function sanitizeHeaders(headers) {
   return sanitized
 }
 
-// Função para sanitizar body (sem truncamento - manter tamanho completo)
 function sanitizeBody(body) {
   if (!body || typeof body !== 'object') {
-    // Se não for objeto, retornar como está (não truncar strings)
     return body
   }
   
   try {
-    // Sanitizar campos sensíveis recursivamente (sem limitação de tamanho)
     const sanitized = Array.isArray(body) ? [] : {}
     
     for (const key in body) {
@@ -177,25 +164,20 @@ function sanitizeBody(body) {
       const lower = String(key).toLowerCase()
       
       if (SENSITIVE_BODY_FIELDS.has(lower) || lower.includes('password') || lower.includes('secret')) {
-        // Mascarar campos sensíveis
         sanitized[key] = '***'
       } else if (typeof body[key] === 'object' && body[key] !== null) {
-        // Recursivamente sanitizar objetos aninhados (sem limitação de profundidade)
         sanitized[key] = sanitizeBody(body[key])
       } else {
-        // Copiar valores não sensíveis (mantendo tamanho completo)
         sanitized[key] = body[key]
       }
     }
     
     return sanitized
   } catch (err) {
-    // Se houver erro na sanitização, retornar body original (não truncar)
     return body
   }
 }
 
-// Função para sanitizar payload completo
 function sanitizePayload(payload) {
   if (!payload || typeof payload !== 'object') {
     return payload
@@ -203,29 +185,22 @@ function sanitizePayload(payload) {
   
   const sanitized = { ...payload }
   
-  // Sanitizar meta se existir
   if (sanitized.meta && typeof sanitized.meta === 'object') {
-    // Sanitizar headers da request
     if (sanitized.meta.request && sanitized.meta.request.headers) {
       sanitized.meta.request.headers = sanitizeHeaders(sanitized.meta.request.headers)
     }
     
-    // Sanitizar headers da response
     if (sanitized.meta.response && sanitized.meta.response.headers) {
       sanitized.meta.response.headers = sanitizeHeaders(sanitized.meta.response.headers)
     }
     
-    // Sanitizar body da request
     if (sanitized.meta.request && sanitized.meta.request.body) {
       sanitized.meta.request.body = sanitizeBody(sanitized.meta.request.body)
     }
     
-    // Sanitizar body da response
     if (sanitized.meta.response && sanitized.meta.response.body) {
       sanitized.meta.response.body = sanitizeBody(sanitized.meta.response.body)
     }
-    
-    // Sanitizar headers de HTTP client (interceptors)
     if (sanitized.meta.headers) {
       sanitized.meta.headers = sanitizeHeaders(sanitized.meta.headers)
     }
@@ -249,19 +224,25 @@ async function deliver(entry) {
     return
   }
 
-  // Sanitizar payload antes de enviar
-  const sanitizedPayload = entry.payload ? sanitizePayload(entry.payload) : entry.payload
+  let sanitizedPayload = entry.payload
+  if (entry.payload && typeof entry.payload === 'object') {
+    try {
+      sanitizedPayload = sanitizePayload(entry.payload)
+    } catch (err) {
+      sanitizedPayload = entry.payload
+    }
+  }
 
   await axios.post(target, sanitizedPayload, {
     headers: entry.headers || {},
     timeout: TRANSPORT_TIMEOUT,
     httpAgent,
     httpsAgent,
-    validateStatus: () => true
+    validateStatus: () => true,
+    maxRedirects: 0
   })
   
   deliveries += 1
-  // Log removido para reduzir ruído nos logs
 }
 
 async function requeue(entry, attempts) {
@@ -334,8 +315,6 @@ async function processEntry(raw) {
     await acknowledge(id)
     consecutiveNoGroupErrors = 0
   } catch (error) {
-    // Silenciar logs de erro - não poluir logs da aplicação
-    // Apenas enviar para DLQ após max tentativas ou requeue silenciosamente
     if (attempts + 1 >= MAX_DELIVERY_ATTEMPTS) {
       await acknowledge(id)
       await deadLetter(entry, error && error.message ? error.message : 'delivery-error')
@@ -351,24 +330,24 @@ async function processBatch(entries) {
     return
   }
 
-  const executing = []
-  for (const entry of entries) {
-    executing.push(processEntry(entry))
-    if (executing.length >= WORKER_CONCURRENCY) {
-      await Promise.allSettled(executing.splice(0, executing.length))
-    }
-  }
-
-  if (executing.length) {
-    await Promise.allSettled(executing)
+  const chunkSize = WORKER_CONCURRENCY
+  const chunks = []
+  
+  for (let i = 0; i < entries.length; i += chunkSize) {
+    chunks.push(entries.slice(i, i + chunkSize))
   }
+  
+  const chunkPromises = chunks.map(chunk => {
+    return Promise.allSettled(chunk.map(entry => processEntry(entry)))
+  })
+  
+  await Promise.all(chunkPromises)
 }
 
 async function consumeLoop() {
   let groupEnsured = false
   
   while (!stopRequested) {
-    // Garantir que o grupo existe antes de tentar ler
     if (!groupEnsured) {
       try {
         await ensureGroup()
@@ -392,8 +371,6 @@ async function consumeLoop() {
     
     let messages = null
     try {
-      // Usar xreadgroup diretamente do ioredis
-      // Sintaxe: xreadgroup('GROUP', group, consumer, 'COUNT', count, 'BLOCK', block, 'STREAMS', key, id)
       messages = await redis.xreadgroup(
         'GROUP', WORKER_GROUP, CONSUMER_NAME,
         'COUNT', MAX_BATCH,
@@ -401,9 +378,7 @@ async function consumeLoop() {
         'STREAMS', STREAM_KEY, '>'
       )
       
-      // Se retornar mensagens, processar
       if (messages && Array.isArray(messages) && messages.length > 0) {
-        // XREADGROUP retorna [[streamName, [entries]]]
         for (const streamData of messages) {
           if (Array.isArray(streamData) && streamData.length >= 2) {
             const entries = streamData[1]
@@ -416,12 +391,10 @@ async function consumeLoop() {
         continue
       }
       
-      // Se não houver mensagens novas, verificar pendentes
       await claimPending()
     } catch (err) {
       const errMsg = err && err.message ? err.message : String(err)
       
-      // Se for erro de grupo ou sintaxe, recriar o grupo
       if (isNoGroupError(err) || errMsg.includes('syntax error') || errMsg.includes('NOGROUP')) {
         groupEnsured = false
         consecutiveNoGroupErrors += 1
@@ -432,7 +405,6 @@ async function consumeLoop() {
         continue
       }
       
-      // Outros erros - log apenas a cada 5 segundos para evitar spam
       const now = Date.now()
       if (now - lastRedisErrorLog > 5000) {
         console.error('[azify-logger][worker] erro ao ler stream:', errMsg)

package/server.js

@@ -120,7 +120,7 @@ async function ensureIndexTemplate() {
           mappings: {
             properties: {
             '@timestamp': { type: 'date' },
-              level: { type: 'keyword' },
+              level: { type: 'text', fields: { keyword: { type: 'keyword' } } },
               message: { type: 'text' },
               service: {
                 properties: {
@@ -399,7 +399,7 @@ async function setupGrafanaForApp(appName) {
         esVersion: '2.11.1',
         version: '2.11.1',
         logMessageField: 'message',
-        logLevelField: 'level',
+        logLevelField: 'level.keyword',
         maxConcurrentShardRequests: 5,
         includeFrozen: false,
         xpack: false,
@@ -481,6 +481,90 @@ async function setupGrafanaForApp(appName) {
       }
     }
 
+    // Datasource Grafana Tempo para tracing completo (opcional - não bloqueia criação da org)
+    try {
+      const tempoDatasourceUid = `tempo-${appName.toLowerCase()}`
+      const tempoUrl = runningInDocker ? 'http://azify-tempo:3200' : 'http://localhost:3200'
+      const tempoDatasourceConfig = {
+        name: `Tempo-${appName}`,
+        type: 'tempo',
+        access: 'proxy',
+        url: tempoUrl,
+        uid: tempoDatasourceUid,
+        isDefault: false,
+        jsonData: {
+          httpMethod: 'GET',
+          tracesToLogs: {
+            datasourceUid: datasourceUid,
+            tags: ['job', 'service', 'pod'],
+            mappedTags: [{ key: 'service.name', value: 'service' }],
+            mapTagNamesEnabled: false,
+            spanStartTimeShift: '1h',
+            spanEndTimeShift: '1h',
+            filterByTraceID: false,
+            filterBySpanID: false
+          },
+          serviceMap: {
+            datasourceUid: datasourceUid
+          },
+          nodeGraph: {
+            enabled: true
+          },
+          search: {
+            hide: false
+          }
+        },
+        editable: true,
+        version: 1
+      }
+
+      try {
+        console.log(`[setupGrafana] Verificando datasource Tempo existente: ${tempoDatasourceUid} na org ${org.id}`)
+        const existingTempo = await axios.get(`${grafanaUrl}/api/datasources/uid/${tempoDatasourceUid}`, {
+          auth,
+          headers: { 'X-Grafana-Org-Id': org.id },
+          timeout: 3000
+        })
+        console.log(`[setupGrafana] Datasource Tempo existente encontrado: ${existingTempo.data?.id || 'N/A'}`)
+        try {
+          await axios.put(
+            `${grafanaUrl}/api/datasources/${existingTempo.data.id}`,
+            tempoDatasourceConfig,
+            {
+              auth,
+              headers: { 'X-Grafana-Org-Id': org.id },
+              timeout: 3000
+            }
+          )
+          console.log(`[setupGrafana] ✅ Datasource Tempo atualizado: ${tempoDatasourceUid}`)
+        } catch (updateTempoErr) {
+          console.error(`[setupGrafana] ⚠️  Erro ao atualizar datasource Tempo: ${updateTempoErr.response?.data?.message || updateTempoErr.message}`)
+        }
+      } catch (tempoError) {
+        console.log(`[setupGrafana] Erro ao verificar datasource Tempo: status=${tempoError.response?.status || 'N/A'}, message=${tempoError.response?.data?.message || tempoError.message}`)
+        if (tempoError.response?.status === 404) {
+          try {
+            const tempoCreate = await axios.post(
+              `${grafanaUrl}/api/datasources`,
+              tempoDatasourceConfig,
+              {
+                auth,
+                headers: { 'X-Grafana-Org-Id': org.id },
+                timeout: 3000
+              }
+            )
+            console.log(`[setupGrafana] ✅ Datasource Tempo criado: ${tempoDatasourceUid} (ID: ${tempoCreate.data?.datasource?.id || 'N/A'})`)
+          } catch (createTempoError) {
+            console.error(`[setupGrafana] ⚠️  Erro ao criar datasource Tempo via API: ${createTempoError.response?.data?.message || createTempoError.message}`)
+            console.log(`[setupGrafana] 💡 Datasource Tempo será criado manualmente ou na próxima tentativa`)
+          }
+        }
+      }
+    } catch (tempoGeneralError) {
+      console.error(`[setupGrafana] ⚠️  Erro geral ao configurar datasource Tempo: ${tempoGeneralError.message}`)
+      console.log(`[setupGrafana] 💡 Continuando sem datasource Tempo - não é crítico para funcionamento básico`)
+    }
+
     console.log(`[setupGrafana] 📈 Criando dashboard para ${appName}...`)
     
     const appNameLower = appName.toLowerCase()
@@ -505,7 +589,7 @@ async function setupGrafanaForApp(appName) {
               datasource: { uid: datasourceUid, type: 'grafana-opensearch-datasource' },
               targets: [{
                 refId: 'A',
-                query: `message:"[REQUEST]" AND NOT message:"[RESPONSE]" AND ${indexFilter}`,
+                query: `message:"[RESPONSE]" AND ${indexFilter}`,
                 bucketAggs: [{
                   id: '2',
                   type: 'date_histogram',
@@ -958,55 +1042,37 @@ async function handleLog(req, res) {
         }
         
         const truncateBody = (bodyValue, forResponse = false) => {
-          if (forResponse && typeof bodyValue === 'string') {
-            if (bodyValue.length > 10000) {
-              return bodyValue.substring(0, 10000) + '... [truncated]'
+          if (forResponse) {
+            if (typeof bodyValue === 'string') {
+              return bodyValue
+            } else if (Buffer.isBuffer(bodyValue)) {
+              return bodyValue.toString('utf8')
+            } else if (typeof bodyValue === 'object' && bodyValue !== null) {
+              try {
+                return JSON.stringify(bodyValue)
+              } catch (_) {
+                return String(bodyValue)
+              }
             }
-            return bodyValue
+            return String(bodyValue)
           }
           
           if (typeof bodyValue === 'string') {
-            if (!forResponse && (bodyValue.trim().startsWith('{') || bodyValue.trim().startsWith('['))) {
+            if (bodyValue.trim().startsWith('{') || bodyValue.trim().startsWith('[')) {
               try {
                 let parsed = JSON.parse(bodyValue)
                 if (typeof parsed === 'object') {
-                  let serialized = JSON.stringify(parsed)
-                  if (serialized.length > 10000) {
-                    return bodyValue.substring(0, 10000) + '... [truncated]'
-                  }
                   return parsed
                 }
               } catch (_) { }
             }
-            if (bodyValue.length > 10000) {
-              return bodyValue.substring(0, 10000) + '... [truncated]'
-            }
             return bodyValue
           } else if (typeof bodyValue === 'object' && bodyValue !== null) {
-            try {
-              let serialized = JSON.stringify(bodyValue)
-              if (serialized.length > 10000 || forResponse) {
-                return serialized.substring(0, 10000) + '... [truncated]'
-              }
-              return bodyValue
-            } catch (e) {
-              try {
-                let str = String(bodyValue)
-                if (str.length > 10000) {
-                  return str.substring(0, 10000) + '... [truncated]'
-                }
-                return str
-              } catch (_) {
-                return '[Unable to serialize body]'
-              }
-            }
+            return bodyValue
           } else if (Buffer.isBuffer(bodyValue)) {
             try {
               let str = bodyValue.toString('utf8')
-              if (str.length > 10000) {
-                return str.substring(0, 10000) + '... [truncated]'
-              }
-              if (!forResponse && (str.trim().startsWith('{') || str.trim().startsWith('['))) {
+              if (str.trim().startsWith('{') || str.trim().startsWith('[')) {
                 try {
                   return JSON.parse(str)
                 } catch (_) {
@@ -1029,14 +1095,33 @@ async function handleLog(req, res) {
           } else {
             value.body = processedBody
           }
-        } else if (key === 'response' && value && typeof value === 'object' && value.body !== undefined) {
-          const processedBody = truncateBody(value.body, true)
-          const bodyString = typeof processedBody === 'string' ? processedBody : JSON.stringify(processedBody)
-          logEntry.responseBody = bodyString
-          const { body, ...responseWithoutBody } = value
-          logEntry[key] = responseWithoutBody
+        } else if (key === 'response' && value && typeof value === 'object') {
+          if (value.statusCode != null) {
+            logEntry.statusCode = value.statusCode
+          }
+          if (value.durationMs != null) {
+            logEntry.responseTime = value.durationMs
+          }
+          if (value.body !== undefined) {
+            const processedBody = truncateBody(value.body, true)
+            let bodyString
+            if (typeof processedBody === 'string') {
+              bodyString = processedBody
+            } else {
+              bodyString = JSON.stringify(processedBody)
+            }
+            logEntry.responseBody = bodyString
+            const { body, ...responseWithoutBody } = value
+            logEntry[key] = responseWithoutBody
+          } else {
+            logEntry[key] = value
+          }
         } else if (key === 'responseBody' || key === 'requestBody') {
-          value = truncateBody(value)
+          if (key === 'responseBody') {
+            value = truncateBody(value, true)
+          } else {
+            value = truncateBody(value, false)
+          }
           logEntry[key] = value
         } else {
           logEntry[key] = value
@@ -1053,10 +1138,6 @@ async function handleLog(req, res) {
     const serviceName = appName.toLowerCase().replace(/[^a-z0-9-]/g, '-')
     const indexName = `logs-${serviceName}`
     
-    if (logEntry.responseBody && typeof logEntry.responseBody === 'string' && logEntry.responseBody.length > 10000) {
-      logEntry.responseBody = logEntry.responseBody.substring(0, 10000) + '... [truncated]'
-    }
-    
     await axios.post(`${osUrl}/${indexName}/_doc`, logEntry, {
       headers: { 'Content-Type': 'application/json' }
     })

package/store.js

@@ -2,8 +2,14 @@ const { AsyncLocalStorage } = require('async_hooks')
 
 const als = new AsyncLocalStorage()
 
-function generateId(bytes = 16) {
-  return require('crypto').randomBytes(bytes).toString('hex')
+function fastGenerateId(length = 16) {
+  const chars = '0123456789abcdef'
+  let result = ''
+  const totalChars = length * 2
+  for (let i = 0; i < totalChars; i++) {
+    result += chars[Math.floor(Math.random() * chars.length)]
+  }
+  return result
 }
 
 function toTraceId(hex32) {
@@ -12,8 +18,8 @@ function toTraceId(hex32) {
 }
 
 function startRequestContext(initial = {}) {
-  const traceHex = initial.traceHex || generateId(16)
-  const spanHex = initial.spanHex || generateId(8)
+  const traceHex = initial.traceHex || fastGenerateId(16)
+  const spanHex = initial.spanHex || fastGenerateId(8)
   const ctx = {
     traceId: toTraceId(traceHex),
     spanId: spanHex,