@nm-logger/logger 1.1.4 → 1.1.5

package/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2025 Virtual Employee
+Copyright (c) 2025
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package/README.md

@@ -1,236 +1,14 @@
 # @nm-logger/logger
 
-Daily JSON logger for Node.js / Express APIs with:
-
-- S3 upload + queue + daily rotation  
-- Correlation IDs (with `X-Correlation-ID` header)  
-- External API logging (Axios)  
-- Sensitive field masking (password, token, otp, etc.)  
-
----
-
-## Installation
-
-After you publish the package to npm:
-
-```bash
-npm install @nm-logger/logger
-```
-
----
-
-## Log Format
-
-Each log line in `daily_logs.json` is a JSON object:
-
-```json
-{
-  "url": "/api/v1/attendance/get",
-  "body": "{\"month\":\"2025-12\"}",
-  "params": "{\"params\":{},\"query\":{}}",
-  "type": "get",
-  "error": "",
-  "date": "2025-12-05 12:24:00",
-  "employee_id": "TAKK122",
-  "correlation_id": "cid-abcd1234-17f5d3c9a"
-}
-```
-
-Fields:
-
-- `url` – `req.originalUrl`
-- `body` – stringified (and masked) `req.body`
-- `params` – stringified (and masked) object `{ params: req.params, query: req.query }`
-- `type` – last segment of the URL (e.g. `/api/v1/attendance/get` → `"get"`)
-- `error` – error message if any
-- `date` – `YYYY-MM-DD HH:mm:ss`
-- `employee_id` – from argument or `req.user.employee_id / emp_code / id`
-- `correlation_id` – unique per request chain (also added as `X-Correlation-ID` header)
-
----
-
-## Basic Usage
-
-### 1. Create the logger
-
-```js
-const Logger = require("@nm-logger/logger");
-
-const logger = new Logger(
-  {
-    accessKeyId: process.env.AWS_KEY,
-    secretAccessKey: process.env.AWS_SECRET,
-    region: "ap-south-1",
-    bucket: "your-log-bucket-name"
-  },
-  {
-    baseDir: "logs",        // optional, default "logs"
-    watchIntervalMs: 60000, // optional, default 60s
-    maskFields: ["aadhaar", "panNumber"] // extra fields to mask
-  }
-);
-```
-
-### 2. Log every request + set correlation ID header
-
-```js
-app.use(logger.requestLoggerMiddleware());
-```
-
-### 3. Log errors via Express error middleware
-
-```js
-// your routes above...
-
-app.use(logger.expressMiddleware()); // or logger.expressErrorMiddleware()
-```
-
-### 4. Manual logging in routes
-
-```js
-app.post("/api/v1/attendance/get", async (req, res) => {
-  try {
-    // ... your logic, external APIs etc ...
-
-    await logger.logRequest(req, req.user?.employee_id);
-    res.json({ success: true });
-  } catch (err) {
-    await logger.logError(err, req, req.user?.employee_id);
-    res.status(500).json({ error: err.message });
-  }
-});
-```
-
----
-
-## External API logging with Axios
-
-```js
-const axios = require("axios");
-
-// Attach once at startup
-logger.attachAxiosLogger(axios);
-
-app.get("/api/v1/some-data", async (req, res) => {
-  try {
-    const response = await axios.get("https://api.example.com/data", {
-      headers: {
-        "X-Correlation-ID": req.correlationId,          // propagated
-        "X-Employee-ID": req.user?.employee_id || ""    // optional
-      },
-      params: {
-        id: 123
-      }
-    });
-
-    res.json(response.data);
-  } catch (err) {
-    await logger.logError(err, req, req.user?.employee_id);
-    res.status(500).json({ error: err.message });
-  }
-});
-```
-
-This will produce external log lines like:
-
-```json
-{
-  "url": "https://api.example.com/data",
-  "body": "{}",
-  "params": "{\"id\":123}",
-  "type": "external_api",
-  "error": "",
-  "date": "2025-12-05 12:24:00",
-  "employee_id": "TAKK122",
-  "correlation_id": "cid-abcd1234-17f5d3c9a"
-}
-```
-
----
-
-## Sensitive Data Masking
-
-Built-in masked keys (case-insensitive, partial match):
-
-- password, pass  
-- token, secret  
-- otp  
-- auth, authorization  
-- apiKey, api_key  
-- session  
-- ssn  
-
-Plus anything you pass in `maskFields` option.
-
-Any object like:
-
-```json
-{
-  "password": "MyPass123",
-  "otp": "111222",
-  "aadhaar": "9999-8888-7777",
-  "email": "user@example.com"
-}
-```
-
-will be logged as:
-
-```json
-{
-  "password": "*****",
-  "otp": "*****",
-  "aadhaar": "*****",
-  "email": "user@example.com"
-}
-```
-
----
-
-## S3 Upload Behavior
-
-- Logs are stored locally under:
-  - `logs/YYYY/MM/DD/daily_logs.json`
-- A watcher runs every `watchIntervalMs` (default 60 seconds)
-  - When the date changes (e.g., from `2025-12-05` to `2025-12-06`),
-  - The logger uploads the **previous day's** log file to S3:
-
-Example S3 key:
-
-```txt
-2025/12/05/daily_logs.json
-```
-
-So final S3 path:
-
-```txt
-s3://<bucket>/<year>/<month>/<day>/daily_logs.json
-```
-
----
-
-## TypeScript Usage
-
-```ts
-import Logger, { S3Config, LoggerOptions } from "@ve/logger";
-
-const s3config: S3Config = {
-  accessKeyId: process.env.AWS_KEY!,
-  secretAccessKey: process.env.AWS_SECRET!,
-  region: "ap-south-1",
-  bucket: "your-log-bucket"
-};
-
-const options: LoggerOptions = {
-  baseDir: "logs",
-  watchIntervalMs: 60000,
-  maskFields: ["aadhaar", "pan"]
-};
-
-const logger = new Logger(s3config, options);
-```
-
----
-
-## License
-
-MIT
+Express JSON logger with:
+
+- Per-request logging (success, error, external API)
+- Separate daily files:
+  - `daily_logs_success.json`
+  - `daily_logs_error.json`
+  - `daily_logs_external.json`
+- Periodic S3 upload (and local file deletion)
+- Correlation IDs (`X-Correlation-ID`)
+- Sensitive field masking
+
+See source comments for usage.

package/index.d.ts

@@ -8,23 +8,9 @@ export interface S3Config {
 }
 
 export interface LoggerOptions {
-  /**
-   * Local base directory where logs are stored.
-   * Default: "logs"
-   */
   baseDir?: string;
-
-  /**
-   * Interval in milliseconds for the daily watcher to check for date change.
-   * Default: 60000 (1 minute)
-   */
-  watchIntervalMs?: number;
-
-  /**
-   * Extra field names to mask in logs (in addition to built-in ones like password, token, otp, etc.).
-   * Matching is case-insensitive and uses "includes".
-   */
   maskFields?: string[];
+  uploadIntervalMs?: number;
 }
 
 export interface LogEntryShape {
@@ -32,10 +18,12 @@ export interface LogEntryShape {
   body: string;
   params: string;
   type: string;
+  method: string;
   error: string;
-  date: string;
   employee_id: string;
   correlation_id: string;
+  date: string;
+  category: string;
 }
 
 export interface ExternalApiLogOptions {
@@ -51,32 +39,10 @@ export interface ExternalApiLogOptions {
 declare class Logger {
   constructor(s3config?: S3Config, options?: LoggerOptions);
 
-  /**
-   * Log an error for a given request.
-   * The log will be appended to the daily JSON file.
-   */
-  logError(
-    err: any,
-    req?: express.Request,
-    employee_id?: string
-  ): Promise<void>;
-
-  /**
-   * Log a normal API request (no error).
-   */
+  logError(err: any, req?: express.Request, employee_id?: string): Promise<void>;
   logRequest(req: express.Request, employee_id?: string): Promise<void>;
-
-  /**
-   * Log an external API call (axios, etc.).
-   * Type will be "external_api".
-   */
   logExternalApi(options: ExternalApiLogOptions): Promise<void>;
 
-  /**
-   * Express error-handling middleware.
-   * Use: app.use(logger.expressMiddleware());
-   * Also ensures X-Correlation-ID header is present.
-   */
   expressMiddleware(): (
     err: any,
     req: express.Request,
@@ -84,9 +50,6 @@ declare class Logger {
     next: express.NextFunction
   ) => void;
 
-  /**
-   * Alias for expressMiddleware (for clarity).
-   */
   expressErrorMiddleware(): (
     err: any,
     req: express.Request,
@@ -94,23 +57,12 @@ declare class Logger {
     next: express.NextFunction
   ) => void;
 
-  /**
-   * Express request-logging middleware (non-error).
-   * Also generates/propagates correlation ID and sets X-Correlation-ID header.
-   * Use near the top of your middleware chain.
-   */
   requestLoggerMiddleware(): (
     req: express.Request,
     res: express.Response,
     next: express.NextFunction
   ) => void;
 
-  /**
-   * Attach axios interceptors to log external API calls.
-   * Usage:
-   *   const axios = require("axios");
-   *   logger.attachAxiosLogger(axios);
-   */
   attachAxiosLogger(axiosInstance: any): void;
 }
 

package/index.js

@@ -1,4 +1,3 @@
 const Logger = require("./src/Logger");
-
 module.exports = Logger;
 module.exports.Logger = Logger;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@nm-logger/logger",
-  "version": "1.1.4",
-  "description": "Daily JSON logger with S3 upload queue, correlation IDs, external API logging, and sensitive field masking for Express-based APIs.",
+  "version": "1.1.5",
+  "description": "Express JSON logger with S3 upload, correlation IDs, and separate success/error/external daily logs.",
   "main": "index.js",
   "types": "index.d.ts",
   "scripts": {
@@ -16,11 +16,9 @@
     "json-logger",
     "daily-logs",
     "correlation-id",
-    "axios",
-    "masking"
+    "axios"
   ],
-  "author": "Virtual Employee (https://virtualemployee.com)",
-  "homepage": "https://virtualemployee.com",
+  "author": "nm-logger",
   "license": "MIT",
   "dependencies": {
     "aws-sdk": "^2.1554.0",
@@ -34,4 +32,4 @@
     "@types/node": "^22.0.0",
     "typescript": "^5.6.0"
   }
-}
+}

package/src/DailyWatcher.js

@@ -1,42 +1,55 @@
-const { getDatePath } = require("./utils");
-const path = require("path");
 const fs = require("fs-extra");
+const path = require("path");
+const { getDatePath } = require("./utils");
+
+const CATEGORY_FILE_MAP = {
+  success: "daily_logs_success.json",
+  error: "daily_logs_error.json",
+  external: "daily_logs_external.json"
+};
 
 class DailyWatcher {
   constructor(baseDir, queue, s3Uploader, options = {}) {
     this.baseDir = baseDir;
     this.queue = queue;
     this.s3Uploader = s3Uploader;
+    this.intervalMs = options.uploadIntervalMs || 60_000; // default 1 minute
 
-    // default upload every 15 minutes
-    this.intervalMs = options.uploadIntervalMs || 15 * 60 * 1000;
+    console.log(
+      "⏱ [@nm-logger/logger] periodic S3 upload every",
+      this.intervalMs,
+      "ms"
+    );
 
-    console.log("🕒 S3 Periodic Upload Enabled. Interval:", this.intervalMs);
-
-    this.startPeriodicUpload();
+    this.start();
   }
 
-  startPeriodicUpload() {
+  start() {
     setInterval(() => {
       const { Y, M, D } = getDatePath();
 
-      const logFile = path.join(
-        this.baseDir,
-        `${Y}/${M}/${D}`,
-        "daily_logs.json"
-      );
-
-      if (!fs.existsSync(logFile)) {
-        console.log("⚠️ No log file found yet:", logFile);
-        return;
-      }
-
-      const s3Key = `${Y}/${M}/${D}/daily_logs.json`;
-
-      this.queue.add(async () => {
-        console.log(`📤 Periodic Upload → ${s3Key}`);
-        await this.s3Uploader.upload(logFile, s3Key);
-        console.log("✅ Uploaded");
+      Object.entries(CATEGORY_FILE_MAP).forEach(([category, fileName]) => {
+        const file = path.join(
+          this.baseDir,
+          `${Y}/${M}/${D}`,
+          fileName
+        );
+        const key = `${Y}/${M}/${D}/${fileName}`;
+
+        if (!fs.existsSync(file)) {
+          return;
+        }
+
+        this.queue.add(async () => {
+          try {
+            console.log(`📤 Uploading [${category}] logs to S3:`, key);
+            await this.s3Uploader.upload(file, key);
+            console.log("✅ Uploaded. Deleting local file:", file);
+            await fs.remove(file);
+          } catch (err) {
+            console.error(`❌ Error uploading [${category}] logs to S3:`, err);
+          }
+        });
       });
     }, this.intervalMs);
   }

package/src/LogWriter.js

@@ -2,27 +2,61 @@ const fs = require("fs-extra");
 const path = require("path");
 const { getDatePath, formatDate } = require("./utils");
 
+const CATEGORY_FILE_MAP = {
+  success: "daily_logs_success.json",
+  error: "daily_logs_error.json",
+  external: "daily_logs_external.json"
+};
+
 class LogWriter {
   constructor(baseDir = "logs") {
     this.baseDir = baseDir;
   }
 
-  /**
-   * Writes a single log entry in the required JSON format
-   * and appends it as one line to daily_logs.json
-   */
-  async writeLog(log) {
-    const { Y, M, D } = getDatePath();
-    const logDir = path.join(this.baseDir, `${Y}/${M}/${D}`);
-    const logFile = path.join(logDir, "daily_logs.json");
+  async writeLog(log, category = "success") {
+    const filePath = this.getFilePath(category);
+
+    await fs.ensureDir(path.dirname(filePath));
+
+    let logsWrapper = { logs: [] };
 
-    await fs.ensureDir(logDir);
+    if (await fs.pathExists(filePath)) {
+      try {
+        const content = await fs.readFile(filePath, "utf8");
+        if (content.trim()) {
+          const parsed = JSON.parse(content);
+          if (Array.isArray(parsed.logs)) {
+            logsWrapper.logs = parsed.logs;
+          }
+        }
+      } catch (e) {
+        logsWrapper = { logs: [] };
+      }
+    }
 
-    // Ensure date field is always set in required format
-    log.date = formatDate(new Date());
+    const enriched = {
+      url: log.url || "",
+      body: log.body || "",
+      params: log.params || "",
+      type: log.type || "",
+      method: log.method || "",
+      error: log.error || "",
+      employee_id: log.employee_id || "",
+      correlation_id: log.correlation_id || "",
+      date: log.date || formatDate(new Date()),
+      category: category || ""
+    };
 
-    await fs.appendFile(logFile, JSON.stringify(log) + "\n");
-    return logFile;
+    logsWrapper.logs.push(enriched);
+
+    await fs.writeFile(filePath, JSON.stringify(logsWrapper, null, 2), "utf8");
+    return filePath;
+  }
+
+  getFilePath(category = "success") {
+    const { Y, M, D } = getDatePath();
+    const fileName = CATEGORY_FILE_MAP[category] || CATEGORY_FILE_MAP.success;
+    return path.join(this.baseDir, `${Y}/${M}/${D}/${fileName}`);
   }
 }
 

package/src/Logger.js

@@ -16,30 +16,19 @@ class Logger {
     this.queue = new Queue();
     this.s3Uploader = new S3Uploader(s3config);
 
-    // Extra fields user wants masked (e.g. ["aadhaar", "pan"])
     this.extraMaskFields = (options.maskFields || []).map((f) =>
       String(f).toLowerCase()
     );
 
-    // Start daily watcher for S3 uploads
     new DailyWatcher(this.baseDir, this.queue, this.s3Uploader, {
-      watchIntervalMs: options.watchIntervalMs
+      uploadIntervalMs: options.uploadIntervalMs
     });
   }
 
-  /**
-   * Mask any sensitive data using default + custom fields.
-   */
   mask(value) {
     return maskSensitive(value, this.extraMaskFields);
   }
 
-  /**
-   * Build the base log entry with required shape.
-   * {
-   *   url, body, params, type, error, date (added by LogWriter), employee_id, correlation_id
-   * }
-   */
   buildBaseLog(req, employee_id = "") {
     const url = req?.originalUrl || "";
     const bodySrc = req?.body || {};
@@ -51,18 +40,20 @@ class Logger {
     const maskedBody = this.mask(bodySrc);
     const maskedParams = this.mask(paramsObj);
 
-    // derive correlation ID from request or headers
     let correlationId =
       req?.correlationId ||
       (req?.headers &&
         (req.headers["x-correlation-id"] || req.headers["X-Correlation-ID"])) ||
       "";
 
+    const method = (req?.method || "").toUpperCase();
+
     return {
       url,
       body: JSON.stringify(maskedBody || {}),
       params: JSON.stringify(maskedParams || {}),
       type: getApiType(url),
+      method,
       error: "",
       employee_id:
         employee_id ||
@@ -74,32 +65,20 @@ class Logger {
     };
   }
 
-  /**
-   * Log an error (incoming API).
-   */
   async logError(err, req, employee_id = "") {
     const base = this.buildBaseLog(req || {}, employee_id);
-
     const logData = {
       ...base,
       error: err && err.message ? err.message : String(err || "")
     };
-
-    await this.logWriter.writeLog(logData);
+    await this.logWriter.writeLog(logData, "error");
   }
 
-  /**
-   * Log a normal request (incoming API).
-   */
   async logRequest(req, employee_id = "") {
     const logData = this.buildBaseLog(req, employee_id);
-    await this.logWriter.writeLog(logData);
+    await this.logWriter.writeLog(logData, "success");
   }
 
-  /**
-   * Log an external API call (axios, etc.).
-   * For external APIs, `type` is always "external_api".
-   */
   async logExternalApi({
     url,
     method,
@@ -117,20 +96,16 @@ class Logger {
       body: JSON.stringify(maskedBody || {}),
       params: JSON.stringify(maskedParams || {}),
       type: "external_api",
+      method: (method || "").toUpperCase(),
       error: error || "",
-      date: undefined, // set by LogWriter
       employee_id: employeeId || "",
-      correlation_id: correlationId || ""
+      correlation_id: correlationId || "",
+      date: undefined
     };
 
-    await this.logWriter.writeLog(logData);
+    await this.logWriter.writeLog(logData, "external");
   }
 
-  /**
-   * Express error-handling middleware.
-   * Use: app.use(logger.expressMiddleware());
-   * Also ensures correlation ID is present and added to response header.
-   */
   expressMiddleware() {
     return (err, req, res, next) => {
       try {
@@ -161,22 +136,10 @@ class Logger {
     };
   }
 
-  /**
-   * Alias for expressMiddleware (for clarity).
-   */
   expressErrorMiddleware() {
     return this.expressMiddleware();
   }
 
-  /**
-   * Express request-logging middleware.
-   * - Generates or reuses correlation ID
-   * - Sets X-Correlation-ID header on response
-   * - Logs each incoming request
-   *
-   * Use near the top of middleware stack:
-   *   app.use(logger.requestLoggerMiddleware());
-   */
   requestLoggerMiddleware() {
     return (req, res, next) => {
       try {
@@ -203,25 +166,10 @@ class Logger {
     };
   }
 
-  /**
-   * Attach axios interceptors to log EXTERNAL API calls.
-   *
-   * Usage:
-   *   const axios = require("axios");
-   *   logger.attachAxiosLogger(axios);
-   *
-   * In your route:
-   *   await axios.get("https://api.example.com", {
-   *     headers: {
-   *       "X-Correlation-ID": req.correlationId,
-   *       "X-Employee-ID": req.user?.employee_id
-   *     }
-   *   });
-   */
   attachAxiosLogger(axiosInstance) {
     if (!axiosInstance || !axiosInstance.interceptors) {
       console.warn(
-        "[@ve/logger] attachAxiosLogger: provided axios instance is invalid"
+        "[@nm-logger/logger] attachAxiosLogger: provided axios instance is invalid"
       );
       return;
     }

package/src/Queue.js

@@ -11,7 +11,6 @@ class Queue {
 
   async run() {
     if (this.processing) return;
-
     this.processing = true;
 
     while (this.jobs.length) {
@@ -19,9 +18,7 @@ class Queue {
       try {
         await job();
       } catch (err) {
-        console.error("Queue job failed → re-added to queue:", err);
-        // Re-add the job for retry (very simple retry mechanism)
-        this.jobs.push(job);
+        console.error("[@nm-logger/logger] Queue job failed:", err);
       }
     }
 

package/src/S3Uploader.js

@@ -1,5 +1,5 @@
-const AWS = require("aws-sdk");
 const fs = require("fs");
+const AWS = require("aws-sdk");
 
 class S3Uploader {
   constructor(config) {
@@ -8,7 +8,6 @@ class S3Uploader {
       secretAccessKey: config.secretAccessKey,
       region: config.region
     });
-
     this.bucket = config.bucket;
   }
 

package/src/utils.js

@@ -1,4 +1,3 @@
-// Build path segments for logs: YYYY/MM/DD
 exports.getDatePath = () => {
   const now = new Date();
   const Y = now.getFullYear();
@@ -7,10 +6,8 @@ exports.getDatePath = () => {
   return { Y, M, D };
 };
 
-// Format date as: 2025-12-05 12:24:00
 exports.formatDate = (d) => {
   const pad = (n) => String(n).padStart(2, "0");
-
   return (
     d.getFullYear() +
     "-" +
@@ -26,15 +23,12 @@ exports.formatDate = (d) => {
   );
 };
 
-// Extract last segment from URL, used as "type"
-// e.g. "/api/v1/attendance/get" → "get"
 exports.getApiType = (url) => {
   if (!url) return "";
   const segments = url.split("/").filter(Boolean);
   return segments[segments.length - 1] || "";
 };
 
-// Default keys to mask (case-insensitive, "includes" match)
 const DEFAULT_MASK_KEYS = [
   "password",
   "pass",
@@ -49,12 +43,6 @@ const DEFAULT_MASK_KEYS = [
   "ssn"
 ];
 
-/**
- * Deep mask of sensitive fields in any object/array.
- * - Keys containing any of the mask keys are replaced with "*****"
- * - Works recursively
- * - If input is string, tries JSON.parse then masks
- */
 exports.maskSensitive = (value, extraFields = []) => {
   const allKeys = [
     ...DEFAULT_MASK_KEYS,
@@ -74,13 +62,7 @@ exports.maskSensitive = (value, extraFields = []) => {
       const out = {};
       for (const [k, v] of Object.entries(val)) {
         if (shouldMaskKey(k)) {
-          if (v === null || v === undefined) {
-            out[k] = v;
-          } else if (typeof v === "string" && v.length) {
-            out[k] = "*****";
-          } else {
-            out[k] = "*****";
-          }
+          out[k] = "*****";
         } else {
           out[k] = maskAny(v);
         }
@@ -90,7 +72,6 @@ exports.maskSensitive = (value, extraFields = []) => {
     return val;
   };
 
-  // If it's a string, try JSON.parse and mask
   if (typeof value === "string") {
     try {
       const parsed = JSON.parse(value);
@@ -103,7 +84,6 @@ exports.maskSensitive = (value, extraFields = []) => {
   return maskAny(value);
 };
 
-// Simple correlation ID generator
 exports.generateCorrelationId = () => {
   const rand = Math.random().toString(16).slice(2, 10);
   const ts = Date.now().toString(16);