azify-logger 1.0.26 → 1.0.29

package/middleware-restify.js

@@ -1,348 +1,192 @@
-/**
- * Creates a Restify middleware for automatic request/response logging with azify-logger
- * @param {Object} options - Configuration options
- * @param {string} [options.serviceName] - Name of the service (defaults to APP_NAME env var or 'azipay')
- * @param {string} [options.loggerUrl] - URL of the azify-logger service (defaults to AZIFY_LOGGER_URL env var or 'http://localhost:3001')
- * @param {string} [options.environment] - Environment name (defaults to NODE_ENV env var or 'development')
- * @returns {Function} Restify middleware function
- * @example
- * const azifyMiddleware = require('azify-logger/middleware-restify');
- * server.use(azifyMiddleware({ serviceName: 'my-app' }));
- */
-const axios = require('axios')
-const { als, runWithRequestContext, startRequestContext, getRequestContext } = require('./store')
-
-function createRestifyLoggingMiddleware(options = {}) {
-  const config = {
-    serviceName: options.serviceName || process.env.APP_NAME || 'azipay',
-    loggerUrl: (options.loggerUrl || process.env.AZIFY_LOGGER_URL || 'http://localhost:3001/log').replace(/\/log$/, '') + '/log',
-    environment: options.environment || process.env.NODE_ENV || 'development'
-  }
+const { startRequestContext } = require('./store')
+const { createHttpLoggerTransport } = require('./streams/httpQueue')
+const os = require('os')
+
+function fastUUID() {
+  const timestamp = Date.now().toString(36)
+  const randomPart = Math.random().toString(36).substring(2, 15)
+  const randomPart2 = Math.random().toString(36).substring(2, 15)
+  return `${timestamp}-${randomPart}-${randomPart2}`.substring(0, 36)
+}
 
-  const pendingLogs = new Set()
+function sanitizeTraceHex(value) {
+  if (!value || typeof value !== 'string') return null
+  const hex = value.replace(/[^0-9a-fA-F]/g, '').toLowerCase()
+  if (!hex) return null
+  return hex.padEnd(32, '0').slice(0, 32)
+}
 
-  let exitHandlerInstalled = false
-  if (!exitHandlerInstalled) {
-    exitHandlerInstalled = true
-    
-    process.on('uncaughtException', (err) => {
-      console.error('[AZIFY] Uncaught exception:', err.message)
-      
-      if (pendingLogs.size > 0) {
-        setTimeout(() => {
-          process.exit(1)
-        }, 500)
-      } else {
-        process.exit(1)
-      }
-    })
+const HEADER_WHITELIST = new Set([
+  'content-type',
+  'content-length',
+  'accept',
+  'accept-encoding',
+  'user-agent',
+  'host',
+  'x-request-id',
+  'x-trace-id',
+  'x-span-id',
+  'x-parent-span-id'
+])
+
+function pickHeaders (source) {
+  if (!source || typeof source !== 'object') {
+    return {}
   }
-
-  function sendLog(level, message, meta = {}) {
-    const logData = {
-      level,
-      message,
-      meta: {
-        ...meta,
-        service: {
-          name: config.serviceName,
-          version: '1.0.0'
-        },
-        environment: config.environment,
-        timestamp: new Date().toISOString(),
-        hostname: require('os').hostname()
-      }
+  const result = {}
+  for (const key in source) {
+    const lower = key.toLowerCase()
+    if (!HEADER_WHITELIST.has(lower)) continue
+    
+    if (!Object.prototype.hasOwnProperty.call(source, key)) continue
+    
+    const value = source[key]
+    if (Array.isArray(value)) {
+      result[key] = value.map(String)
+    } else if (value != null) {
+      result[key] = String(value)
     }
-
-    const logId = Math.random().toString(36).substring(7)
-    pendingLogs.add(logId)
-    
-    axios.post(`${config.loggerUrl}`, logData, {
-      timeout: 2000
-    }).then(() => {
-      pendingLogs.delete(logId)
-    }).catch(() => {
-      pendingLogs.delete(logId)
-    })
   }
+  return result
+}
 
-  function sanitizeHeaders(headers) {
-    const sanitized = { ...headers }
-    const sensitiveHeaders = ['authorization', 'cookie', 'x-api-key', 'x-auth-token', 'x-access-token']
-    
-    for (const key of Object.keys(sanitized)) {
-      if (sensitiveHeaders.includes(key.toLowerCase())) {
-        sanitized[key] = '***'
-      }
-    }
-    
-    return sanitized
+function createRestifyLoggingMiddleware (options = {}) {
+  const config = {
+    serviceName: options.serviceName || process.env.APP_NAME,
+    loggerUrl: options.loggerUrl || process.env.AZIFY_LOGGER_URL || 'http://localhost:3001/log',
+    environment: options.environment || process.env.NODE_ENV
   }
 
-  function sanitizeBody(body) {
-    if (!body || typeof body !== 'object') return body
-    
-    const sanitized = Array.isArray(body) ? [...body] : { ...body }
-    const sensitiveFields = ['password', 'token', 'secret', 'apiKey', 'api_key', 'accessToken', 'access_token', 'refreshToken', 'refresh_token', 'clientSecret', 'client_secret']
+  const transport = createHttpLoggerTransport(config.loggerUrl, {})
+  const hostname = os.hostname()
+  const serviceObj = config.serviceName ? { name: config.serviceName, version: '1.0.0' } : null
+
+  function sendLog (level, message, meta = {}) {
+    if (!transport || typeof transport.enqueue !== 'function') return
     
-    for (const key of Object.keys(sanitized)) {
-      if (sensitiveFields.includes(key) || key.toLowerCase().includes('password') || key.toLowerCase().includes('secret')) {
-        sanitized[key] = '***'
-      }
-    }
+    if (serviceObj) meta.service = serviceObj
+    if (config.environment) meta.environment = config.environment
+    meta.timestamp = Date.now()
+    meta.hostname = hostname
     
-    return sanitized
+    transport.enqueue({
+      level,
+      message,
+      meta
+    }, { 'content-type': 'application/json' })
   }
 
-  return function azifyLoggingMiddleware(req, res, next) {
+  return function azifyLoggingMiddleware (req, res, next) {
     const startTime = Date.now()
-    const requestId = req.requestId || require('uuid').v4()
-    
+    const requestId = req.requestId || fastUUID()
+
     if (res._azifySetup) {
       return next()
     }
     res._azifySetup = true
-    
-    const reqCtx = startRequestContext({ requestId })
-    const requestTraceId = reqCtx.traceId
-    const requestSpanId = reqCtx.spanId
-    const requestParentSpanId = reqCtx.parentSpanId
-    
-    let baseUrl = req.url
-    if (baseUrl.includes('?')) {
-      baseUrl = baseUrl.substring(0, baseUrl.indexOf('?'))
-    }
-    baseUrl = baseUrl.replace(/\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, '/{id}')
-    baseUrl = baseUrl.replace(/\/[0-9]+/g, '/{id}')
 
-    req._azifyRequestData = {
+    const traceHex = sanitizeTraceHex(req.headers['x-trace-id'])
+    const reqCtx = startRequestContext({
       requestId,
-      method: req.method,
-      url: req.url,
-      baseUrl: baseUrl,
-      path: req.url,
-      headers: sanitizeHeaders(req.headers),
-      query: req.query,
-      userAgent: req.headers['user-agent'],
-      ip: req.connection.remoteAddress || req.socket.remoteAddress,
-      traceId: requestTraceId,
-      spanId: requestSpanId,
-      parentSpanId: requestParentSpanId
+      traceHex,
+      parentSpanId: req.headers['x-parent-span-id'] || null
+    })
+
+    let normalizedPath = req.url
+    if (normalizedPath.includes('?')) {
+      normalizedPath = normalizedPath.slice(0, normalizedPath.indexOf('?'))
     }
+    normalizedPath = normalizedPath.replace(/\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, '/{id}')
+    normalizedPath = normalizedPath.replace(/\/[0-9]+/g, '/{id}')
 
-    if (req.method === 'GET') {
-      sendLog('info', `[REQUEST] ${req.method} ${req.url}`, req._azifyRequestData)
-    } else {
-      if (req.body !== undefined) {
-        req._azifyRequestData.requestBody = sanitizeBody(req.body)
-      }
-      sendLog('info', `[REQUEST] ${req.method} ${req.url}`, req._azifyRequestData)
+    const requestSnapshot = {
+      id: requestId,
+      method: req.method,
+      url: req.url,
+      baseUrl: normalizedPath,
+      headers: pickHeaders(req.headers || {}),
+      query: req.query || {},
+      ip: req.connection.remoteAddress || req.socket.remoteAddress
     }
 
-    res._azifyResponseLogged = false
-    let sentBody
-    
-    const originalSend = res.send && res.send.bind(res)
-    if (originalSend) {
-      res.send = function patchedSend() {
-        try {
-          if (arguments.length === 1) {
-            sentBody = arguments[0]
-          } else if (arguments.length >= 2) {
-            sentBody = typeof arguments[0] === 'number' ? arguments[1] : (arguments[1] || arguments[0])
-          }
-        } catch (_) {}
-        
-        if (!res._azifyResponseLogged) {
-          res._azifyResponseLogged = true
-          logResponse()
+    let responseStatus
+    const originalStatus = res.status
+    if (typeof originalStatus === 'function') {
+      res.status = function patchedStatus (code) {
+        if (typeof code === 'number') {
+          responseStatus = code
         }
-        
-        return originalSend.apply(this, arguments)
+        return originalStatus.call(this, code)
       }
     }
 
-
-    const originalStatus = res.status
-    res.status = function(code) {
-      res._actualStatusCode = code
-      return originalStatus.call(this, code)
-    }
-    
     const originalWriteHead = res.writeHead
-    res.writeHead = function(statusCode, statusMessage, headers) {
-      res._actualStatusCode = statusCode
-      if (typeof statusMessage === 'object') {
-        headers = statusMessage
-        statusMessage = undefined
-      }
-      return originalWriteHead.call(this, statusCode, statusMessage, headers)
-    }
-    
-    const originalJsonMethod = res.json
-    res.json = function(code, body) {
-      try {
-        if (arguments.length === 1) {
-          sentBody = arguments[0]
-        } else if (arguments.length >= 2) {
-          sentBody = typeof arguments[0] === 'number' ? arguments[1] : (arguments[1] || arguments[0])
+    if (typeof originalWriteHead === 'function') {
+      res.writeHead = function patchedWriteHead (statusCode, statusMessage, headers) {
+        if (typeof statusCode === 'number') {
+          responseStatus = statusCode
         }
-      } catch (_) {}
-      
-      if (typeof code === 'number') {
-        res._actualStatusCode = code
-      } else {
-        const errorObj = arguments.length === 1 ? arguments[0] : (typeof arguments[0] === 'number' ? arguments[1] : arguments[0])
-        
-        if (errorObj && errorObj.constructor && errorObj.constructor.name === 'ErrCtor') {
-          const errorName = errorObj.toString()
-          if (errorName.includes('InternalServerError') || errorName.includes('InternalError')) {
-            res._actualStatusCode = 500
-            res._actualStatusCode = 500
-          } else if (errorName.includes('BadRequest') || errorName.includes('BadDigest')) {
-            res._actualStatusCode = 400
-          } else if (errorName.includes('NotFound')) {
-            res._actualStatusCode = 404
-          } else if (errorName.includes('Unauthorized')) {
-            res._actualStatusCode = 401
-          } else if (errorName.includes('Forbidden')) {
-            res._actualStatusCode = 403
-          } else {
-            res._actualStatusCode = 500
-          }
-        } else {
-          res._actualStatusCode = res.statusCode || 200
+        if (typeof statusMessage === 'object') {
+          headers = statusMessage
         }
+        return originalWriteHead.call(this, statusCode, statusMessage, headers)
       }
-      return originalJsonMethod.apply(this, arguments)
     }
 
-    const originalEnd = res.end
-    res.end = function(chunk, encoding) {
-      const duration = Date.now() - startTime
-      
-      let responseBody = sentBody
-      try {
-        if (responseBody == null && chunk) {
-          if (Buffer.isBuffer(chunk)) {
-            responseBody = chunk.toString('utf8')
-          } else if (typeof chunk === 'string') {
-            responseBody = chunk
-          } else {
-            responseBody = JSON.stringify(chunk)
-          }
-        }
-      } catch (_) {}
-
-      if (!res._azifyResponseLogged) {
-        logResponse()
-        res._azifyResponseLogged = true
-      }
+    let logSent = false
 
-      originalEnd.call(this, chunk, encoding)
-    }
-    
-    function logResponse() {
-      const duration = Date.now() - startTime
-      
-      let responseBody = sentBody
+    function emitLog (level, message, extraMeta = {}) {
+      if (logSent) return
+      logSent = true
       
-      let serializedResponseBody
-      try {
-        if (typeof responseBody === 'string') {
-          serializedResponseBody = responseBody
-        } else if (Array.isArray(responseBody)) {
-          serializedResponseBody = JSON.stringify(responseBody)
-        } else if (responseBody && typeof responseBody === 'object') {
-          if (responseBody.toJSON && typeof responseBody.toJSON === 'function') {
-            serializedResponseBody = JSON.stringify(responseBody.toJSON())
-          } else if (responseBody.toString && typeof responseBody.toString === 'function' && responseBody.toString() !== '[object Object]') {
-            serializedResponseBody = responseBody.toString()
-          } else {
-            serializedResponseBody = JSON.stringify(responseBody, (key, value) => {
-              if (typeof value === 'function') {
-                return '[Function]'
-              }
-              if (value instanceof Error) {
-                return { name: value.name, message: value.message, stack: value.stack }
-              }
-              return value
-            }, null, 0)
-          }
-        } else {
-          serializedResponseBody = responseBody != null ? String(responseBody) : ''
+      process.nextTick(() => {
+        const duration = Date.now() - startTime
+        const statusCode = responseStatus || res.statusCode || 200
+        const responseHeaders = typeof res.getHeaders === 'function' ? res.getHeaders() : {}
+
+        const meta = {
+          traceId: reqCtx.traceId,
+          spanId: reqCtx.spanId,
+          parentSpanId: reqCtx.parentSpanId || null,
+          requestId,
+          request: requestSnapshot,
+          response: {
+            statusCode,
+            headers: pickHeaders(responseHeaders),
+            durationMs: duration
+          },
+          ...extraMeta
         }
-      } catch (error) {
-        try {
-          serializedResponseBody = JSON.stringify(responseBody, null, 2)
-        } catch (secondError) {
-          serializedResponseBody = '[Complex object - serialization failed]'
-        }
-      }
-      
-      const statusCode = res._actualStatusCode || res._statusCode || res.statusCode || 200
-      
-      const responseMessage = serializedResponseBody && serializedResponseBody.length > 0
-        ? `[RESPONSE] ${serializedResponseBody}`
-        : `[RESPONSE] ${req.method} ${req.url} ${statusCode} ${duration}ms`
-      
-      const responseData = {
-        ...req._azifyRequestData,
-        requestBody: req.body,
-        statusCode: statusCode,
-        responseTime: duration,
-        responseHeaders: res.getHeaders ? res.getHeaders() : {},
-        responseBody: serializedResponseBody
-      }
-      
-      try { res._azifyResponseLogged = true } catch (_) {}
-      sendLog('info', responseMessage, responseData)
+
+        sendLog(level, message, meta)
+      })
     }
 
-    req._azifyContext = reqCtx
-    
-    const originalNext = next
-    const wrappedNext = (err) => {
-      if (err && !res._azifyResponseLogged) {
-        const errorData = {
-          ...req._azifyRequestData,
-          statusCode: res.statusCode || 500,
-          responseTime: Date.now() - startTime,
-          error: {
-            message: err.message || String(err),
-            stack: err.stack,
-            name: err.name,
-            code: err.code
-          }
-        }
-        sendLog('error', `[ERROR] ${req.method} ${req.url}: ${err.message || String(err)}`, errorData)
-        res._azifyResponseLogged = true
-      }
-      return originalNext(err)
+    function finalize (level = 'info', errorMeta) {
+      if (logSent) return
+      const message = `${req.method} ${req.url}`
+      const meta = errorMeta ? { error: errorMeta } : {}
+      emitLog(level, message, meta)
     }
-    
-    res.on('error', (error) => {
-      if (!res._azifyResponseLogged) {
-        const errorData = {
-          ...req._azifyRequestData,
-          statusCode: res.statusCode || 500,
-          responseTime: Date.now() - startTime,
-          error: {
-            message: error.message,
-            stack: error.stack,
-            name: error.name
-          }
-        }
-        sendLog('error', `[ERROR] ${req.method} ${req.url}: ${error.message}`, errorData)
-        res._azifyResponseLogged = true
-      }
+
+    res.on('finish', () => {
+      finalize('info')
     })
-    
-    runWithRequestContext(reqCtx, () => {
-      wrappedNext()
+
+    res.on('close', () => {
+      finalize('warn', { message: 'response closed before finish' })
     })
+
+    res.on('error', (err) => {
+      finalize('error', {
+        message: err && err.message ? err.message : String(err),
+        name: err && err.name ? err.name : 'Error',
+        stack: err && err.stack ? err.stack : undefined
+      })
+    })
+
+    next()
   }
 }
 
 module.exports = createRestifyLoggingMiddleware
-

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "azify-logger",
-  "version": "1.0.26",
+  "version": "1.0.29",
   "description": "Azify Logger Client - Centralized logging for OpenSearch",
   "main": "index.js",
   "types": "index.d.ts",
@@ -9,14 +9,15 @@
     "test": "echo \"No tests specified\" && exit 0",
     "start": "node server.js",
     "dev": "node server.js",
-    "dev:full": "docker-compose up -d && sleep 10 && npm start",
+    "dev:full": "docker compose up -d && sleep 10 && npm start",
     "pm2:start": "pm2 start ecosystem.config.js",
     "pm2:stop": "pm2 stop ecosystem.config.js",
     "pm2:restart": "pm2 restart ecosystem.config.js",
     "pm2:logs": "pm2 logs azify-logger",
     "docker:up": "docker-compose up -d",
     "docker:down": "docker-compose down",
-    "docker:logs": "docker-compose logs -f"
+    "docker:logs": "docker-compose logs -f",
+    "worker": "node scripts/redis-worker.js"
   },
   "keywords": [
     "logging",
@@ -27,45 +28,41 @@
   "author": "Azify",
   "license": "MIT",
   "dependencies": {
+    "@opentelemetry/api": "1.0.4",
+    "@opentelemetry/auto-instrumentations-node": "0.27.0",
+    "@opentelemetry/core": "1.0.1",
+    "@opentelemetry/exporter-trace-otlp-http": "0.27.0",
+    "@opentelemetry/instrumentation-express": "0.27.0",
+    "@opentelemetry/instrumentation-http": "0.27.0",
+    "@opentelemetry/instrumentation-restify": "0.27.0",
+    "@opentelemetry/resources": "1.0.1",
+    "@opentelemetry/sdk-node": "0.27.0",
+    "@opentelemetry/semantic-conventions": "1.0.1",
     "axios": "^1.6.0",
     "cors": "^2.8.5",
     "express": "^4.18.2",
     "express-session": "^1.17.3",
+    "ioredis": "^5.8.2",
+    "js-yaml": "^4.1.0",
     "passport": "^0.6.0",
     "passport-azure-ad": "^4.3.5",
-    "js-yaml": "^4.1.0",
     "require-in-the-middle": "^7.4.0",
     "uuid": "^9.0.1"
   },
-  "peerDependencies": {
-    "@opentelemetry/api": ">=1.1.0 <1.6.0",
-    "@opentelemetry/core": ">=1.1.0 <1.6.0",
-    "@opentelemetry/resources": ">=1.1.0 <1.6.0",
-    "@opentelemetry/semantic-conventions": ">=1.1.0 <1.6.0",
-    "@opentelemetry/sdk-node": ">=0.39.1 <0.42.0",
-    "@opentelemetry/exporter-trace-otlp-http": ">=0.39.1 <0.42.0",
-    "@opentelemetry/auto-instrumentations-node": ">=0.39.1 <0.42.0"
-  },
-  "optionalDependencies": {
-    "@opentelemetry/api": "1.1.0",
-    "@opentelemetry/core": "1.1.0",
-    "@opentelemetry/resources": "1.1.0",
-    "@opentelemetry/semantic-conventions": "1.1.0",
-    "@opentelemetry/sdk-node": "0.39.1",
-    "@opentelemetry/exporter-trace-otlp-http": "0.39.1",
-    "@opentelemetry/auto-instrumentations-node": "0.39.1"
-  },
   "overrides": {
-    "@opentelemetry/api": "1.1.0",
-    "@opentelemetry/core": "1.1.0",
-    "@opentelemetry/resources": "1.1.0",
-    "@opentelemetry/semantic-conventions": "1.1.0",
-    "@opentelemetry/sdk-node": "0.39.1",
-    "@opentelemetry/exporter-trace-otlp-http": "0.39.1",
-    "@opentelemetry/auto-instrumentations-node": "0.39.1"
+    "@opentelemetry/api": "1.0.4",
+    "@opentelemetry/core": "1.0.1",
+    "@opentelemetry/resources": "1.0.1",
+    "@opentelemetry/semantic-conventions": "1.0.1",
+    "@opentelemetry/sdk-node": "0.27.0",
+    "@opentelemetry/exporter-trace-otlp-http": "0.27.0",
+    "@opentelemetry/auto-instrumentations-node": "0.27.0"
   },
   "engines": {
-    "node": ">=12.0.0"
+    "node": ">=12 <=22"
+  },
+  "bin": {
+    "azify-logger-worker": "scripts/redis-worker.js"
   },
   "files": [
     "index.js",
@@ -78,7 +75,11 @@
     "middleware-restify.js",
     "middleware-express.js",
     "middleware-express.d.ts",
+    "middleware-fastify.js",
     "server.js",
+    "sampling.js",
+    "queue/",
+    "scripts/redis-worker.js",
     "streams/",
     "package.json",
     "README.md"