azify-logger 1.0.25 → 1.0.28

package/middleware-restify.js

@@ -1,348 +1,177 @@
-/**
- * Creates a Restify middleware for automatic request/response logging with azify-logger
- * @param {Object} options - Configuration options
- * @param {string} [options.serviceName] - Name of the service (defaults to APP_NAME env var or 'azipay')
- * @param {string} [options.loggerUrl] - URL of the azify-logger service (defaults to AZIFY_LOGGER_URL env var or 'http://localhost:3001')
- * @param {string} [options.environment] - Environment name (defaults to NODE_ENV env var or 'development')
- * @returns {Function} Restify middleware function
- * @example
- * const azifyMiddleware = require('azify-logger/middleware-restify');
- * server.use(azifyMiddleware({ serviceName: 'my-app' }));
- */
-const axios = require('axios')
-const { als, runWithRequestContext, startRequestContext, getRequestContext } = require('./store')
-
-function createRestifyLoggingMiddleware(options = {}) {
-  const config = {
-    serviceName: options.serviceName || process.env.APP_NAME || 'azipay',
-    loggerUrl: (options.loggerUrl || process.env.AZIFY_LOGGER_URL || 'http://localhost:3001/log').replace(/\/log$/, '') + '/log',
-    environment: options.environment || process.env.NODE_ENV || 'development'
-  }
-
-  const pendingLogs = new Set()
-
-  let exitHandlerInstalled = false
-  if (!exitHandlerInstalled) {
-    exitHandlerInstalled = true
-    
-    process.on('uncaughtException', (err) => {
-      console.error('[AZIFY] Uncaught exception:', err.message)
-      
-      if (pendingLogs.size > 0) {
-        setTimeout(() => {
-          process.exit(1)
-        }, 500)
-      } else {
-        process.exit(1)
-      }
-    })
+const { runWithRequestContext, startRequestContext, getRequestContext } = require('./store')
+const { createHttpLoggerTransport } = require('./streams/httpQueue')
+const { randomUUID } = require('crypto')
+const os = require('os')
+
+const HEADER_WHITELIST = new Set([
+  'content-type',
+  'content-length',
+  'accept',
+  'accept-encoding',
+  'user-agent',
+  'host',
+  'x-request-id',
+  'x-trace-id',
+  'x-span-id',
+  'x-parent-span-id'
+])
+
+function pickHeaders (source) {
+  if (!source || typeof source !== 'object') {
+    return {}
   }
-
-  function sendLog(level, message, meta = {}) {
-    const logData = {
-      level,
-      message,
-      meta: {
-        ...meta,
-        service: {
-          name: config.serviceName,
-          version: '1.0.0'
-        },
-        environment: config.environment,
-        timestamp: new Date().toISOString(),
-        hostname: require('os').hostname()
-      }
+  const result = {}
+  for (const [key, value] of Object.entries(source)) {
+    const lower = key.toLowerCase()
+    if (HEADER_WHITELIST.has(lower)) {
+      result[key] = Array.isArray(value) ? value.map(String) : (value != null ? String(value) : value)
     }
-
-    const logId = Math.random().toString(36).substring(7)
-    pendingLogs.add(logId)
-    
-    axios.post(`${config.loggerUrl}`, logData, {
-      timeout: 2000
-    }).then(() => {
-      pendingLogs.delete(logId)
-    }).catch(() => {
-      pendingLogs.delete(logId)
-    })
   }
+  return result
+}
 
-  function sanitizeHeaders(headers) {
-    const sanitized = { ...headers }
-    const sensitiveHeaders = ['authorization', 'cookie', 'x-api-key', 'x-auth-token', 'x-access-token']
-    
-    for (const key of Object.keys(sanitized)) {
-      if (sensitiveHeaders.includes(key.toLowerCase())) {
-        sanitized[key] = '***'
-      }
-    }
-    
-    return sanitized
+function createRestifyLoggingMiddleware (options = {}) {
+  const config = {
+    serviceName: options.serviceName || process.env.APP_NAME,
+    loggerUrl: options.loggerUrl || process.env.AZIFY_LOGGER_URL || 'http://localhost:3001/log',
+    environment: options.environment || process.env.NODE_ENV
   }
 
-  function sanitizeBody(body) {
-    if (!body || typeof body !== 'object') return body
-    
-    const sanitized = Array.isArray(body) ? [...body] : { ...body }
-    const sensitiveFields = ['password', 'token', 'secret', 'apiKey', 'api_key', 'accessToken', 'access_token', 'refreshToken', 'refresh_token', 'clientSecret', 'client_secret']
-    
-    for (const key of Object.keys(sanitized)) {
-      if (sensitiveFields.includes(key) || key.toLowerCase().includes('password') || key.toLowerCase().includes('secret')) {
-        sanitized[key] = '***'
-      }
-    }
-    
-    return sanitized
+  const transport = createHttpLoggerTransport(config.loggerUrl)
+
+  function sendLog (level, message, meta = {}) {
+    try {
+      transport.enqueue({
+        level,
+        message,
+        meta: {
+          ...meta,
+          service: {
+            name: config.serviceName,
+            version: '1.0.0'
+          },
+          environment: config.environment,
+          timestamp: new Date().toISOString(),
+          hostname: os.hostname()
+        }
+      }, { 'content-type': 'application/json' })
+    } catch (err) { }
   }
 
-  return function azifyLoggingMiddleware(req, res, next) {
+  return function azifyLoggingMiddleware (req, res, next) {
     const startTime = Date.now()
-    const requestId = req.requestId || require('uuid').v4()
-    
+    const requestId = req.requestId || randomUUID()
+
     if (res._azifySetup) {
       return next()
     }
     res._azifySetup = true
-    
-    const reqCtx = startRequestContext({ requestId })
-    const requestTraceId = reqCtx.traceId
-    const requestSpanId = reqCtx.spanId
-    const requestParentSpanId = reqCtx.parentSpanId
-    
-    let baseUrl = req.url
-    if (baseUrl.includes('?')) {
-      baseUrl = baseUrl.substring(0, baseUrl.indexOf('?'))
+
+    const currentCtx = getRequestContext()
+    const traceId = currentCtx && currentCtx.traceId ? currentCtx.traceId : (req.headers['x-trace-id'] || randomUUID())
+    const parentSpanId = currentCtx && currentCtx.spanId ? currentCtx.spanId : (req.headers['x-parent-span-id'] || null)
+    const spanId = randomUUID().replace(/-/g, '').slice(0, 16)
+
+    const reqCtx = startRequestContext({ requestId, traceId, spanId, parentSpanId })
+
+    let normalizedPath = req.url
+    if (normalizedPath.includes('?')) {
+      normalizedPath = normalizedPath.slice(0, normalizedPath.indexOf('?'))
     }
-    baseUrl = baseUrl.replace(/\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, '/{id}')
-    baseUrl = baseUrl.replace(/\/[0-9]+/g, '/{id}')
+    normalizedPath = normalizedPath.replace(/\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, '/{id}')
+    normalizedPath = normalizedPath.replace(/\/[0-9]+/g, '/{id}')
 
-    req._azifyRequestData = {
-      requestId,
+    const requestSnapshot = {
+      id: requestId,
       method: req.method,
       url: req.url,
-      baseUrl: baseUrl,
-      path: req.url,
-      headers: sanitizeHeaders(req.headers),
-      query: req.query,
-      userAgent: req.headers['user-agent'],
-      ip: req.connection.remoteAddress || req.socket.remoteAddress,
-      traceId: requestTraceId,
-      spanId: requestSpanId,
-      parentSpanId: requestParentSpanId
-    }
-
-    if (req.method === 'GET') {
-      sendLog('info', `[REQUEST] ${req.method} ${req.url}`, req._azifyRequestData)
-    } else {
-      if (req.body !== undefined) {
-        req._azifyRequestData.requestBody = sanitizeBody(req.body)
-      }
-      sendLog('info', `[REQUEST] ${req.method} ${req.url}`, req._azifyRequestData)
+      baseUrl: normalizedPath,
+      headers: pickHeaders(req.headers || {}),
+      query: req.query || {},
+      ip: req.connection.remoteAddress || req.socket.remoteAddress
     }
 
-    res._azifyResponseLogged = false
-    let sentBody
-    
-    const originalSend = res.send && res.send.bind(res)
-    if (originalSend) {
-      res.send = function patchedSend() {
-        try {
-          if (arguments.length === 1) {
-            sentBody = arguments[0]
-          } else if (arguments.length >= 2) {
-            sentBody = typeof arguments[0] === 'number' ? arguments[1] : (arguments[1] || arguments[0])
-          }
-        } catch (_) {}
-        
-        if (!res._azifyResponseLogged) {
-          res._azifyResponseLogged = true
-          logResponse()
+    let responseStatus
+    const originalStatus = res.status
+    if (typeof originalStatus === 'function') {
+      res.status = function patchedStatus (code) {
+        if (typeof code === 'number') {
+          responseStatus = code
         }
-        
-        return originalSend.apply(this, arguments)
+        return originalStatus.call(this, code)
       }
     }
 
-
-    const originalStatus = res.status
-    res.status = function(code) {
-      res._actualStatusCode = code
-      return originalStatus.call(this, code)
-    }
-    
     const originalWriteHead = res.writeHead
-    res.writeHead = function(statusCode, statusMessage, headers) {
-      res._actualStatusCode = statusCode
-      if (typeof statusMessage === 'object') {
-        headers = statusMessage
-        statusMessage = undefined
-      }
-      return originalWriteHead.call(this, statusCode, statusMessage, headers)
-    }
-    
-    const originalJsonMethod = res.json
-    res.json = function(code, body) {
-      try {
-        if (arguments.length === 1) {
-          sentBody = arguments[0]
-        } else if (arguments.length >= 2) {
-          sentBody = typeof arguments[0] === 'number' ? arguments[1] : (arguments[1] || arguments[0])
+    if (typeof originalWriteHead === 'function') {
+      res.writeHead = function patchedWriteHead (statusCode, statusMessage, headers) {
+        if (typeof statusCode === 'number') {
+          responseStatus = statusCode
         }
-      } catch (_) {}
-      
-      if (typeof code === 'number') {
-        res._actualStatusCode = code
-      } else {
-        const errorObj = arguments.length === 1 ? arguments[0] : (typeof arguments[0] === 'number' ? arguments[1] : arguments[0])
-        
-        if (errorObj && errorObj.constructor && errorObj.constructor.name === 'ErrCtor') {
-          const errorName = errorObj.toString()
-          if (errorName.includes('InternalServerError') || errorName.includes('InternalError')) {
-            res._actualStatusCode = 500
-            res._actualStatusCode = 500
-          } else if (errorName.includes('BadRequest') || errorName.includes('BadDigest')) {
-            res._actualStatusCode = 400
-          } else if (errorName.includes('NotFound')) {
-            res._actualStatusCode = 404
-          } else if (errorName.includes('Unauthorized')) {
-            res._actualStatusCode = 401
-          } else if (errorName.includes('Forbidden')) {
-            res._actualStatusCode = 403
-          } else {
-            res._actualStatusCode = 500
-          }
-        } else {
-          res._actualStatusCode = res.statusCode || 200
+        if (typeof statusMessage === 'object') {
+          headers = statusMessage
         }
+        return originalWriteHead.call(this, statusCode, statusMessage, headers)
       }
-      return originalJsonMethod.apply(this, arguments)
     }
 
-    const originalEnd = res.end
-    res.end = function(chunk, encoding) {
+    function emitLog (level, message, extraMeta = {}) {
       const duration = Date.now() - startTime
-      
-      let responseBody = sentBody
-      try {
-        if (responseBody == null && chunk) {
-          if (Buffer.isBuffer(chunk)) {
-            responseBody = chunk.toString('utf8')
-          } else if (typeof chunk === 'string') {
-            responseBody = chunk
-          } else {
-            responseBody = JSON.stringify(chunk)
-          }
-        }
-      } catch (_) {}
-
-      if (!res._azifyResponseLogged) {
-        logResponse()
-        res._azifyResponseLogged = true
+      const statusCode = responseStatus || res.statusCode || 200
+      const responseHeaders = typeof res.getHeaders === 'function' ? res.getHeaders() : {}
+
+      const meta = {
+        traceId: reqCtx.traceId,
+        spanId: reqCtx.spanId,
+        parentSpanId: reqCtx.parentSpanId,
+        requestId,
+        request: requestSnapshot,
+        response: {
+          statusCode,
+          headers: pickHeaders(responseHeaders),
+          durationMs: duration
+        },
+        ...extraMeta
       }
 
-      originalEnd.call(this, chunk, encoding)
-    }
-    
-    function logResponse() {
-      const duration = Date.now() - startTime
-      
-      let responseBody = sentBody
-      
-      let serializedResponseBody
-      try {
-        if (typeof responseBody === 'string') {
-          serializedResponseBody = responseBody
-        } else if (Array.isArray(responseBody)) {
-          serializedResponseBody = JSON.stringify(responseBody)
-        } else if (responseBody && typeof responseBody === 'object') {
-          if (responseBody.toJSON && typeof responseBody.toJSON === 'function') {
-            serializedResponseBody = JSON.stringify(responseBody.toJSON())
-          } else if (responseBody.toString && typeof responseBody.toString === 'function' && responseBody.toString() !== '[object Object]') {
-            serializedResponseBody = responseBody.toString()
-          } else {
-            serializedResponseBody = JSON.stringify(responseBody, (key, value) => {
-              if (typeof value === 'function') {
-                return '[Function]'
-              }
-              if (value instanceof Error) {
-                return { name: value.name, message: value.message, stack: value.stack }
-              }
-              return value
-            }, null, 0)
-          }
-        } else {
-          serializedResponseBody = responseBody != null ? String(responseBody) : ''
-        }
-      } catch (error) {
-        try {
-          serializedResponseBody = JSON.stringify(responseBody, null, 2)
-        } catch (secondError) {
-          serializedResponseBody = '[Complex object - serialization failed]'
-        }
-      }
-      
-      const statusCode = res._actualStatusCode || res._statusCode || res.statusCode || 200
-      
-      const responseMessage = serializedResponseBody && serializedResponseBody.length > 0
-        ? `[RESPONSE] ${serializedResponseBody}`
-        : `[RESPONSE] ${req.method} ${req.url} ${statusCode} ${duration}ms`
-      
-      const responseData = {
-        ...req._azifyRequestData,
-        requestBody: req.body,
-        statusCode: statusCode,
-        responseTime: duration,
-        responseHeaders: res.getHeaders ? res.getHeaders() : {},
-        responseBody: serializedResponseBody
-      }
-      
-      try { res._azifyResponseLogged = true } catch (_) {}
-      sendLog('info', responseMessage, responseData)
+      sendLog(level, message, meta)
     }
 
-    req._azifyContext = reqCtx
-    
-    const originalNext = next
-    const wrappedNext = (err) => {
-      if (err && !res._azifyResponseLogged) {
-        const errorData = {
-          ...req._azifyRequestData,
-          statusCode: res.statusCode || 500,
-          responseTime: Date.now() - startTime,
-          error: {
-            message: err.message || String(err),
-            stack: err.stack,
-            name: err.name,
-            code: err.code
-          }
-        }
-        sendLog('error', `[ERROR] ${req.method} ${req.url}: ${err.message || String(err)}`, errorData)
-        res._azifyResponseLogged = true
-      }
-      return originalNext(err)
+    function finalize (level = 'info', errorMeta) {
+      const message = `${req.method} ${req.url}`
+      const meta = errorMeta ? { error: errorMeta } : {}
+      emitLog(level, message, meta)
     }
-    
-    res.on('error', (error) => {
-      if (!res._azifyResponseLogged) {
-        const errorData = {
-          ...req._azifyRequestData,
-          statusCode: res.statusCode || 500,
-          responseTime: Date.now() - startTime,
-          error: {
-            message: error.message,
-            stack: error.stack,
-            name: error.name
-          }
-        }
-        sendLog('error', `[ERROR] ${req.method} ${req.url}: ${error.message}`, errorData)
-        res._azifyResponseLogged = true
-      }
+
+    res.on('finish', () => {
+      finalize('info')
+    })
+
+    res.on('close', () => {
+      finalize('warn', { message: 'response closed before finish' })
+    })
+
+    res.on('error', (err) => {
+      finalize('error', {
+        message: err && err.message ? err.message : String(err),
+        name: err && err.name ? err.name : 'Error',
+        stack: err && err.stack ? err.stack : undefined
+      })
     })
-    
+
     runWithRequestContext(reqCtx, () => {
-      wrappedNext()
+      try {
+        next()
+      } catch (err) {
+        finalize('error', {
+          message: err && err.message ? err.message : String(err),
+          name: err && err.name ? err.name : 'Error',
+          stack: err && err.stack ? err.stack : undefined
+        })
+        throw err
+      }
     })
   }
 }
 
 module.exports = createRestifyLoggingMiddleware
-

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "azify-logger",
-  "version": "1.0.25",
+  "version": "1.0.28",
   "description": "Azify Logger Client - Centralized logging for OpenSearch",
   "main": "index.js",
   "types": "index.d.ts",
@@ -16,7 +16,8 @@
     "pm2:logs": "pm2 logs azify-logger",
     "docker:up": "docker-compose up -d",
     "docker:down": "docker-compose down",
-    "docker:logs": "docker-compose logs -f"
+    "docker:logs": "docker-compose logs -f",
+    "worker": "node scripts/redis-worker.js"
   },
   "keywords": [
     "logging",
@@ -28,6 +29,7 @@
   "license": "MIT",
   "dependencies": {
     "axios": "^1.6.0",
+    "ioredis": "^5.4.1",
     "cors": "^2.8.5",
     "express": "^4.18.2",
     "express-session": "^1.17.3",
@@ -36,26 +38,33 @@
     "js-yaml": "^4.1.0",
     "require-in-the-middle": "^7.4.0",
     "uuid": "^9.0.1",
-    "@opentelemetry/api": "1.1.0",
-    "@opentelemetry/core": "1.1.0",
-    "@opentelemetry/resources": "1.1.0",
-    "@opentelemetry/semantic-conventions": "1.1.0",
-    "@opentelemetry/sdk-node": "0.39.1",
-    "@opentelemetry/exporter-trace-otlp-http": "0.39.1",
-    "@opentelemetry/auto-instrumentations-node": "0.39.1"
+    "@opentelemetry/instrumentation-http": "0.27.0",
+    "@opentelemetry/instrumentation-express": "0.27.0",
+    "@opentelemetry/instrumentation-restify": "0.27.0",
+
+    "@opentelemetry/api": "1.0.4",
+    "@opentelemetry/core": "1.0.1",
+    "@opentelemetry/resources": "1.0.1",
+    "@opentelemetry/semantic-conventions": "1.0.1",
+    "@opentelemetry/sdk-node": "0.27.0",
+    "@opentelemetry/exporter-trace-otlp-http": "0.27.0",
+    "@opentelemetry/auto-instrumentations-node": "0.27.0"
   },
   "overrides": {
-    "@opentelemetry/api": "1.1.0",
-    "@opentelemetry/core": "1.1.0",
-    "@opentelemetry/resources": "1.1.0",
-    "@opentelemetry/semantic-conventions": "1.1.0",
-    "@opentelemetry/sdk-node": "0.39.1",
-    "@opentelemetry/exporter-trace-otlp-http": "0.39.1",
-    "@opentelemetry/auto-instrumentations-node": "0.39.1"
+    "@opentelemetry/api": "1.0.4",
+    "@opentelemetry/core": "1.0.1",
+    "@opentelemetry/resources": "1.0.1",
+    "@opentelemetry/semantic-conventions": "1.0.1",
+    "@opentelemetry/sdk-node": "0.27.0",
+    "@opentelemetry/exporter-trace-otlp-http": "0.27.0",
+    "@opentelemetry/auto-instrumentations-node": "0.27.0"
   },
   "optionalDependencies": {},
   "engines": {
-    "node": ">=12.0.0"
+    "node": ">=12 <=22"
+  },
+  "bin": {
+    "azify-logger-worker": "scripts/redis-worker.js"
   },
   "files": [
     "index.js",
@@ -69,6 +78,9 @@
     "middleware-express.js",
     "middleware-express.d.ts",
     "server.js",
+    "sampling.js",
+    "queue/",
+    "scripts/redis-worker.js",
     "streams/",
     "package.json",
     "README.md"

package/queue/fileQueue.js

@@ -0,0 +1,100 @@
+const fs = require('fs')
+const path = require('path')
+
+function createFileSpool(options = {}) {
+  const directory = options.directory || path.join(process.cwd(), '.azify-logger-spool')
+  const flushInterval = options.flushInterval || 5000
+  const batchSize = options.batchSize || 100
+  const pushFn = typeof options.pushFn === 'function' ? options.pushFn : null
+
+  if (!pushFn) {
+    return null
+  }
+
+  fs.mkdirSync(directory, { recursive: true })
+
+  let scheduled = false
+  let flushing = false
+
+  const scheduleFlush = () => {
+    if (scheduled) {
+      return
+    }
+    scheduled = true
+    const timer = setTimeout(async () => {
+      scheduled = false
+      await flush().catch(() => {})
+    }, flushInterval)
+    if (typeof timer.unref === 'function') {
+      timer.unref()
+    }
+  }
+
+  async function append(entry) {
+    const filePath = path.join(directory, `spool-${process.pid}.ndjson`)
+    await fs.promises.appendFile(filePath, JSON.stringify(entry) + '\n')
+    scheduleFlush()
+  }
+
+  async function flush() {
+    if (flushing) {
+      return
+    }
+    flushing = true
+    try {
+      const files = (await fs.promises.readdir(directory)).filter((file) => file.endsWith('.ndjson'))
+      for (const file of files) {
+        const fullPath = path.join(directory, file)
+        const drainingPath = `${fullPath}.draining`
+        try {
+          await fs.promises.rename(fullPath, drainingPath)
+        } catch (err) {
+          if (err.code === 'ENOENT') {
+            continue
+          }
+          throw err
+        }
+
+        const content = await fs.promises.readFile(drainingPath, 'utf8').catch(() => '')
+        if (!content) {
+          await fs.promises.unlink(drainingPath).catch(() => {})
+          continue
+        }
+
+        const lines = content.split('\n').filter(Boolean)
+        while (lines.length) {
+          const slice = lines.splice(0, batchSize)
+          const entries = slice.map((line) => {
+            try {
+              return JSON.parse(line)
+            } catch (_) {
+              return null
+            }
+          }).filter(Boolean)
+          if (entries.length > 0) {
+            await pushFn(entries).catch(async () => {
+              const remaining = entries.map((entry) => JSON.stringify(entry)).join('\n') + '\n'
+              await fs.promises.appendFile(fullPath, remaining)
+              throw new Error('Failed to flush file spool entries back to Redis')
+            })
+          }
+        }
+
+        await fs.promises.unlink(drainingPath).catch(() => {})
+      }
+    } finally {
+      flushing = false
+    }
+  }
+
+  return {
+    append,
+    flush
+  }
+}
+
+module.exports = {
+  createFileSpool
+}
+
+