azify-logger 1.0.55 → 1.0.57

package/server.js

@@ -232,6 +232,16 @@ async function ensureIndexTemplate() {
               hostname: { type: 'keyword' },
               source: { type: 'keyword' },
               responseBody: { type: 'text' },
+              businessStatus: { type: 'keyword' },
+              metaJson: { type: 'text' },
+              dataJson: { type: 'text' },
+              reqJson: { type: 'text' },
+              headersJson: { type: 'text' },
+              paramsJson: { type: 'text' },
+              queryJson: { type: 'text' },
+              requestJson: { type: 'text' },
+              responseJson: { type: 'text' },
+              bodyJson: { type: 'text' },
               error: {
                 properties: {
                   message: { type: 'text' },
@@ -2110,20 +2120,103 @@ function _trimLogEntryForOpenSearch(doc) {
   return out
 }
 
+const _OPENSEARCH_ROOT_KEEP_KEYS = new Set([
+  '@timestamp', 'time', 'timestamp', 'level', 'message', 'traceId', 'spanId', 'parentSpanId',
+  'userId', 'requestId', 'method', 'url', 'baseUrl', 'statusCode', 'responseTime',
+  'cpu', 'cpuUsage', 'memory', 'memoryUsage', 'totalMemory', 'freeMemory', 'usedMemory',
+  'ip', 'userAgent', 'environment', 'hostname', 'source', 'responseBody', 'requestBody',
+  'bodyJson', 'businessStatus', 'appName', 'duration', 'name'
+])
+
+const _OPENSEARCH_ROOT_STRUCTURED_KEYS = new Set(['error', 'service'])
+
+function _coerceLogValueToString(v) {
+  if (v == null) return v
+  if (typeof v === 'string') return v
+  if (Buffer.isBuffer(v)) return v.toString('utf8')
+  if (typeof v === 'object') {
+    try {
+      return JSON.stringify(v)
+    } catch (_) {
+      return String(v)
+    }
+  }
+  return String(v)
+}
+
+/** Campos fora da allowlist viram {campo}Json (text) — evita conflitos de dynamic mapping. */
+function _serializeUnmappedRootFieldsToJson(doc) {
+  if (!doc || typeof doc !== 'object' || Array.isArray(doc) || Buffer.isBuffer(doc)) return doc
+  const o = { ...doc }
+  for (const key of Object.keys(o)) {
+    if (key.endsWith('Json')) continue
+    if (_OPENSEARCH_ROOT_KEEP_KEYS.has(key)) continue
+    if (_OPENSEARCH_ROOT_STRUCTURED_KEYS.has(key)) continue
+    if (o[key] === undefined) continue
+    o[key + 'Json'] = _coerceLogValueToString(o[key])
+    delete o[key]
+  }
+  return o
+}
+
+function _extractMappingConflictRootField(errorMsg) {
+  const m = String(errorMsg || '').match(/(?:object mapping for|failed to parse field) \[([^\]]+)\]/)
+  return m ? m[1].split('.')[0] : null
+}
+
+function _forceRootFieldToJson(doc, rootKey) {
+  if (!rootKey || !doc || typeof doc !== 'object' || Array.isArray(doc)) return doc
+  if (rootKey.endsWith('Json') || _OPENSEARCH_ROOT_KEEP_KEYS.has(rootKey) || _OPENSEARCH_ROOT_STRUCTURED_KEYS.has(rootKey)) {
+    return doc
+  }
+  const o = { ...doc }
+  if (!Object.prototype.hasOwnProperty.call(o, rootKey)) return o
+  o[rootKey + 'Json'] = _coerceLogValueToString(o[rootKey])
+  delete o[rootKey]
+  return o
+}
+
+function _prepareDocForOpenSearch(doc) {
+  return _serializeUnmappedRootFieldsToJson(doc)
+}
+
+function _isOpenSearchMappingError(errorMsg) {
+  const m = String(errorMsg || '')
+  return m.includes('failed to parse field') ||
+    m.includes('object mapping for') ||
+    m.includes('mapper_parsing_exception') ||
+    m.includes('Limit of total fields')
+}
+
 function _normalizeLogEntryForOpenSearch(doc) {
   if (!doc || typeof doc !== 'object') return doc
   function coerceToString(v) {
-    if (v == null) return v
-    if (typeof v === 'string') return v
-    if (Buffer.isBuffer(v)) return v.toString('utf8')
-    if (typeof v === 'object') {
-      try {
-        return JSON.stringify(v)
-      } catch (_) {
-        return String(v)
+    return _coerceLogValueToString(v)
+  }
+  /**
+   * Índices antigos mapearam meta.status como long (primeiro doc numérico).
+   * Status de negócio (ex. KYC_FAILED) vai para businessStatus (keyword).
+   */
+  function normalizeStatusFieldConflicts(obj) {
+    if (!obj || typeof obj !== 'object' || Array.isArray(obj) || Buffer.isBuffer(obj)) return obj
+    const o = { ...obj }
+    if (Object.prototype.hasOwnProperty.call(o, 'status')) {
+      const st = o.status
+      const isHttpCode = typeof st === 'number' && Number.isFinite(st) && st >= 100 && st <= 599
+      if (typeof st === 'number' && Number.isFinite(st)) {
+        if (isHttpCode && o.statusCode == null) o.statusCode = st
+      } else if (st != null) {
+        o.businessStatus = coerceToString(st)
+        delete o.status
+      }
+    }
+    for (const key of Object.keys(o)) {
+      const v = o[key]
+      if (v != null && typeof v === 'object' && !Array.isArray(v) && !Buffer.isBuffer(v)) {
+        o[key] = normalizeStatusFieldConflicts(v)
       }
     }
-    return String(v)
+    return o
   }
   function normalizeNested(obj) {
     if (!obj || typeof obj !== 'object' || Buffer.isBuffer(obj)) return obj
@@ -2137,11 +2230,52 @@ function _normalizeLogEntryForOpenSearch(doc) {
     }
     return o
   }
-  const out = normalizeNested(doc)
-  if (out.meta != null && typeof out.meta === 'object') {
-    out.meta = normalizeNested(out.meta)
+  /**
+   * Conflitos de dynamic mapping no OpenSearch (campos restantes após serializeVolatileFieldsToJson):
+   * - campo text + valor object → serializa como JSON string
+   */
+  function isPlainLeafObject(v) {
+    if (v == null || typeof v !== 'object' || Array.isArray(v) || Buffer.isBuffer(v)) return false
+    return Object.values(v).every(val => {
+      if (val == null) return true
+      if (typeof val !== 'object') return true
+      if (Array.isArray(val)) return val.every(x => x == null || typeof x !== 'object')
+      return false
+    })
   }
-  return out
+  /**
+   * Campos restantes (fora da lista volátil): plain objects → JSON string.
+   */
+  function normalizeDynamicMappingConflicts(obj) {
+    if (!obj || typeof obj !== 'object' || Array.isArray(obj) || Buffer.isBuffer(obj)) return obj
+    const textCompatObjectKeys = new Set(['errorDetails', 'additionalInfo'])
+    const o = { ...obj }
+    for (const key of Object.keys(o)) {
+      const v = o[key]
+      if (v == null) continue
+      const keepObjectSubfields = key === 'error' || key === 'service'
+      if (Array.isArray(v)) {
+        o[key] = v.map(item => {
+          if (item != null && typeof item === 'object' && !Buffer.isBuffer(item) && !Array.isArray(item)) {
+            return isPlainLeafObject(item) ? coerceToString(item) : normalizeDynamicMappingConflicts(item)
+          }
+          return item
+        })
+      } else if (typeof v === 'object' && !Buffer.isBuffer(v)) {
+        if (keepObjectSubfields) {
+          o[key] = normalizeDynamicMappingConflicts(v)
+        } else if (textCompatObjectKeys.has(key) || isPlainLeafObject(v)) {
+          o[key] = coerceToString(v)
+        } else {
+          o[key] = normalizeDynamicMappingConflicts(v)
+        }
+      }
+    }
+    return o
+  }
+  return _prepareDocForOpenSearch(
+    normalizeStatusFieldConflicts(normalizeDynamicMappingConflicts(normalizeNested(doc)))
+  )
 }
 
 const _openSearchWriteQueue = []
@@ -2282,7 +2416,8 @@ function _drainOpenSearchQueue() {
     const job = _openSearchWriteQueue.shift()
     if (!job) break
     _openSearchWriteInFlight++
-    axios.post(`${job.osUrl}/${job.indexName}/_doc`, job.logEntry, {
+    const payload = _prepareDocForOpenSearch(job.logEntry)
+    axios.post(`${job.osUrl}/${job.indexName}/_doc`, payload, {
       headers: { 'Content-Type': 'application/json' },
       timeout: _openSearchWriteTimeout
     })
@@ -2319,6 +2454,23 @@ function _drainOpenSearchQueue() {
         }
       })
       .catch((error) => {
+        const status = error?.response?.status
+        const rawMsg = error?.response?.data?.error?.reason ||
+          error?.response?.data?.error?.type ||
+          error?.response?.data?.message ||
+          error?.message ||
+          'Erro desconhecido'
+        const errorMsg = typeof rawMsg === 'string' ? rawMsg.slice(0, 800) : String(rawMsg).slice(0, 800)
+        if (status === 400 && !job._mappingRetry && _isOpenSearchMappingError(errorMsg)) {
+          job._mappingRetry = true
+          let retried = _prepareDocForOpenSearch(JSON.parse(JSON.stringify(job.logEntry)))
+          const conflictRoot = _extractMappingConflictRootField(errorMsg)
+          if (conflictRoot) retried = _forceRootFieldToJson(retried, conflictRoot)
+          job.logEntry = retried
+          console.warn('[opensearch] Reenviando log com allowlist+*Json após mapping error:', job.indexName, conflictRoot || '')
+          _openSearchWriteQueue.unshift(job)
+          return
+        }
         _openSearchFailuresInRow++
         if (_openSearchFailuresInRow >= _openSearchCircuitThreshold) {
           _openSearchCircuitOpenUntil = Date.now() + _openSearchCircuitPauseMs
@@ -2331,13 +2483,6 @@ function _drainOpenSearchQueue() {
             if (_openSearchCircuitRetryTimer.unref) _openSearchCircuitRetryTimer.unref()
           }
         }
-        const status = error?.response?.status
-        const rawMsg = error?.response?.data?.error?.reason ||
-          error?.response?.data?.error?.type ||
-          error?.response?.data?.message ||
-          error?.message ||
-          'Erro desconhecido'
-        const errorMsg = typeof rawMsg === 'string' ? rawMsg.slice(0, 800) : String(rawMsg).slice(0, 800)
         const logPayload = {
           index: job.indexName,
           serviceName: job.serviceName,
@@ -2630,6 +2775,7 @@ async function startServer() {
 
   app.listen(port, () => {
     console.log('[startup] server listening on port', port, '- /health ready')
+    console.log('[startup] OpenSearch normalizer: volatile-*-json (req/headers/params/meta → *Json)')
     setTimeout(() => writeOtelCollectorConfigIfEnabled().catch(() => {}), 3000)
     setTimeout(() => {
       fetchGrafanaOrgs()

package/utils/sanitizeSensitiveHttpBody.js

@@ -0,0 +1,222 @@
+'use strict'
+
+const SENSITIVE_FIELD_KEYS_BASE = new Set([
+  'password',
+  'passwd',
+  'pwd',
+  'token',
+  'secret',
+  'apikey',
+  'api_key',
+  'accesstoken',
+  'access_token',
+  'refreshtoken',
+  'refresh_token',
+  'clientsecret',
+  'client_secret',
+  'client_id',
+  'secret_id',
+  'credential',
+  'credentials',
+  'authorization',
+  'creditcard',
+  'credit_card',
+  'cardnumber',
+  'card_number',
+  'cvv',
+  'cvc',
+  'privatekey',
+  'private_key',
+  'csrf',
+  'session',
+  'id_token',
+  'bearer'
+])
+
+/** Chaves lowercase adicionadas em runtime pelo env */
+function getExtraSensitiveKeys() {
+  const raw = process.env.AZIFY_LOGGER_SENSITIVE_BODY_KEYS
+  if (!raw || typeof raw !== 'string') return EMPTY_EXTRA
+  if (_cachedExtraRaw === raw && _cachedExtraSet) return _cachedExtraSet
+  const set = new Set()
+  for (const part of raw.split(',')) {
+    const k = String(part).trim().toLowerCase()
+    if (k) set.add(k)
+  }
+  _cachedExtraRaw = raw
+  _cachedExtraSet = set
+  return set
+}
+
+const EMPTY_EXTRA = new Set()
+let _cachedExtraRaw = null
+let _cachedExtraSet = null
+
+function fieldKeyLooksSensitive(lower) {
+  if (!lower) return false
+  if (SENSITIVE_FIELD_KEYS_BASE.has(lower)) return true
+  if (getExtraSensitiveKeys().has(lower)) return true
+  if (lower.includes('password') || lower.includes('passwd')) return true
+  if (lower.includes('secret')) return true
+  if (lower.includes('credential')) return true
+  if (lower === 'authorization' || lower.endsWith('_token') || /^[a-z0-9]+token$/i.test(lower)) {
+    return true
+  }
+  return false
+}
+
+function sanitizeObjectOrArray(body) {
+  if (body == null) return body
+
+  try {
+    if (Array.isArray(body)) {
+      const out = []
+      for (let i = 0; i < body.length; i++) {
+        const v = body[i]
+        if (v !== null && typeof v === 'object' && !(v instanceof Date)) {
+          out.push(sanitizeObjectOrArray(v))
+        } else {
+          out.push(v)
+        }
+      }
+      return out
+    }
+
+    if (typeof body !== 'object') {
+      return body
+    }
+
+    if (typeof Buffer !== 'undefined' && Buffer.isBuffer(body)) {
+      return body
+    }
+
+    const sanitized = {}
+    for (const key in body) {
+      if (!Object.prototype.hasOwnProperty.call(body, key)) continue
+      const lower = String(key).toLowerCase()
+
+      if (fieldKeyLooksSensitive(lower)) {
+        sanitized[key] = '***'
+      } else {
+        const v = body[key]
+        if (v !== null && typeof v === 'object' && !(v instanceof Date)) {
+          sanitized[key] = sanitizeObjectOrArray(v)
+        } else {
+          sanitized[key] = v
+        }
+      }
+    }
+    return sanitized
+  } catch (_) {
+    return body
+  }
+}
+
+function sanitizeSensitiveBodyForHttpLog(value, maxChars) {
+  const mc =
+    typeof maxChars === 'number' && Number.isFinite(maxChars) && maxChars > 0 ? maxChars : 5000
+
+  function clip(s) {
+    if (typeof s !== 'string' || !s.length) return s
+    return s.length > mc ? s.slice(0, mc) : s
+  }
+
+  try {
+    if (value === null || value === undefined) return value
+
+    if (typeof Buffer !== 'undefined' && Buffer.isBuffer(value)) {
+      return sanitizeSensitiveBodyForHttpLog(value.toString('utf8'), maxChars)
+    }
+
+    if (typeof value === 'object' && !(value instanceof Date)) {
+      const sanitized = sanitizeObjectOrArray(value)
+      return clip(JSON.stringify(sanitized))
+    }
+
+    if (typeof value !== 'string') {
+      return clip(String(value))
+    }
+
+    const t = value.trim()
+    const looksJson = (t.startsWith('{') && t.endsWith('}')) || (t.startsWith('[') && t.endsWith(']'))
+    if (looksJson && t.length <= mc + 4096) {
+      try {
+        const parsed = JSON.parse(value)
+        if (parsed !== null && typeof parsed === 'object') {
+          const sanitized = sanitizeObjectOrArray(parsed)
+          return clip(JSON.stringify(sanitized))
+        }
+      } catch (_) {
+        /* manter texto bruto cortado */
+      }
+    }
+
+    return clip(value)
+  } catch (_) {
+    try {
+      return clip(typeof value === 'string' ? value : String(value))
+    } catch (_) {
+      return '[unreadable]'
+    }
+  }
+}
+
+function sanitizeBodyJsonValue(body) {
+  if (body == null || typeof body !== 'object') {
+    if (typeof body === 'string') {
+      try {
+        const trimmed = body.trim()
+        const looksJson = (trimmed.startsWith('{') && trimmed.endsWith('}')) || (trimmed.startsWith('[') && trimmed.endsWith(']'))
+        if (!looksJson) return body
+        const parsed = JSON.parse(body)
+        if (parsed !== null && typeof parsed === 'object') {
+          return sanitizeObjectOrArray(parsed)
+        }
+      } catch (_) {
+        return body
+      }
+    }
+    return body
+  }
+
+  try {
+    return sanitizeObjectOrArray(body)
+  } catch (_) {
+    return body
+  }
+}
+
+function getHttpLogBodyMaxChars() {
+  return Math.min(
+    Math.max(parseInt(String(process.env.AZIFY_LOGGER_HTTP_BODY_MAX_CHARS || '5000'), 10) || 5000, 100),
+    100000
+  )
+}
+
+function sanitizeOutboundHttpMetaBodies(meta, maxChars) {
+  const mc =
+    typeof maxChars === 'number' && Number.isFinite(maxChars) && maxChars > 0 ? maxChars : getHttpLogBodyMaxChars()
+
+  if (!meta || typeof meta !== 'object') return meta
+
+  try {
+    const out = { ...meta }
+    if (Object.prototype.hasOwnProperty.call(out, 'requestBody') && out.requestBody !== undefined) {
+      out.requestBody = sanitizeSensitiveBodyForHttpLog(out.requestBody, mc)
+    }
+    if (Object.prototype.hasOwnProperty.call(out, 'responseBody') && out.responseBody !== undefined) {
+      out.responseBody = sanitizeSensitiveBodyForHttpLog(out.responseBody, mc)
+    }
+    return out
+  } catch (_) {
+    return meta
+  }
+}
+
+module.exports = {
+  sanitizeSensitiveBodyForHttpLog,
+  sanitizeBodyJsonValue,
+  sanitizeObjectOrArray,
+  getHttpLogBodyMaxChars,
+  sanitizeOutboundHttpMetaBodies
+}

package/utils/undiciLogBodies.js

@@ -0,0 +1,83 @@
+'use strict'
+
+function maxBodyChars() {
+  return Math.min(
+    Math.max(parseInt(String(process.env.AZIFY_LOGGER_HTTP_BODY_MAX_CHARS || '5000'), 10) || 5000, 100),
+    100000
+  )
+}
+
+function clip(s) {
+  const m = maxBodyChars()
+  if (typeof s !== 'string' || !s.length) return null
+  return s.length > m ? s.slice(0, m) : s
+}
+
+/** undici: request(url, opts) ou request(opts) — retorna o objeto de opções com body/method. */
+function resolveUndiciRequestOpts(url, options) {
+  if (typeof url === 'object' && url !== null && !(url instanceof URL)) {
+    return url
+  }
+  return options && typeof options === 'object' ? options : {}
+}
+
+function serializeUndiciRequestBody(opts) {
+  if (!opts || opts.body == null || opts.body === undefined) return null
+  try {
+    if (typeof opts.body === 'string') return clip(opts.body)
+    if (Buffer.isBuffer(opts.body)) return clip(opts.body.toString('utf8'))
+    if (opts.body && typeof opts.body.pipe === 'function') return '[Stream]'
+    return clip(String(opts.body))
+  } catch (_) {
+    return null
+  }
+}
+
+function responseReadTimeoutMs() {
+  return Math.min(
+    Math.max(parseInt(String(process.env.AZIFY_LOGGER_HTTP_RESPONSE_READ_MS || '4000'), 10) || 4000, 200),
+    30000
+  )
+}
+
+/**
+ * Lê o body da resposta undici para log e substitui `data.body` por um objeto com .text()/.json()
+ * compatível com o que o undici expõe, para o chamador ainda conseguir fazer parse.
+ */
+async function restoreUndiciResponseBodyAfterReadForLog(data) {
+  if (!data || !data.body || typeof data.body.text !== 'function') {
+    return { logStr: null, data }
+  }
+  const timeoutMs = responseReadTimeoutMs()
+  try {
+    const text = await Promise.race([
+      data.body.text(),
+      new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), timeoutMs))
+    ])
+    if (text == null || typeof text !== 'string') {
+      return { logStr: null, data }
+    }
+    const logStr = clip(text)
+    data.body = {
+      async text() {
+        return text
+      },
+      async json() {
+        return JSON.parse(text)
+      },
+      async arrayBuffer() {
+        return Buffer.from(text, 'utf8').buffer
+      }
+    }
+    return { logStr, data }
+  } catch (_) {
+    return { logStr: null, data }
+  }
+}
+
+module.exports = {
+  resolveUndiciRequestOpts,
+  serializeUndiciRequestBody,
+  restoreUndiciResponseBodyAfterReadForLog,
+  maxBodyChars
+}