@nm-logger/logger 1.1.6 → 1.1.8

package/README.md

@@ -1,14 +1,241 @@
 # @nm-logger/logger
 
-Express JSON logger with:
-
+Daily JSON logger for Node.js / Express APIs with:
 - Per-request logging (success, error, external API)
 - Separate daily files:
   - `daily_logs_success.json`
   - `daily_logs_error.json`
   - `daily_logs_external.json`
-- Periodic S3 upload (and local file deletion)
-- Correlation IDs (`X-Correlation-ID`)
-- Sensitive field masking
+- S3 upload + queue + daily rotation  
+- Correlation IDs (with `X-Correlation-ID` header)  
+- External API logging (Axios)  
+- Sensitive field masking (password, token, otp, etc.)  
+
+---
+
+## Installation
+
+After you publish the package to npm:
+
+```bash
+npm install @nm-logger/logger
+```
+
+---
+
+## Log Format
+
+Each log line in `daily_logs.json` is a JSON object:
+
+```json
+{
+  "url": "/api/v1/attendance/get",
+  "body": "{\"month\":\"2025-12\"}",
+  "params": "{\"params\":{},\"query\":{}}",
+  "type": "get",
+  "error": "",
+  "date": "2025-12-05 12:24:00",
+  "employee_id": "TAKK122",
+  "correlation_id": "cid-abcd1234-17f5d3c9a"
+}
+```
+
+Fields:
+
+- `url` – `req.originalUrl`
+- `body` – stringified (and masked) `req.body`
+- `params` – stringified (and masked) object `{ params: req.params, query: req.query }`
+- `type` – last segment of the URL (e.g. `/api/v1/attendance/get` → `"get"`)
+- `error` – error message if any
+- `date` – `YYYY-MM-DD HH:mm:ss`
+- `employee_id` – from argument or `req.user.employee_id / emp_code / id`
+- `correlation_id` – unique per request chain (also added as `X-Correlation-ID` header)
+
+---
+
+## Basic Usage
+
+### 1. Create the logger
+
+```js
+const Logger = require("@nm-logger/logger");
+
+const logger = new Logger(
+  {
+    accessKeyId: process.env.AWS_KEY,
+    secretAccessKey: process.env.AWS_SECRET,
+    region: "ap-south-1",
+    bucket: "your-log-bucket-name"
+  },
+  {
+    baseDir: "logs",        // optional, default "logs"
+    watchIntervalMs: 60000, // optional, default 60s
+    maskFields: ["aadhaar", "panNumber"] // extra fields to mask
+  }
+);
+```
+
+### 2. Log every request + set correlation ID header
+
+```js
+app.use(logger.requestLoggerMiddleware());
+```
+
+### 3. Log errors via Express error middleware
+
+```js
+// your routes above...
+
+app.use(logger.expressMiddleware()); // or logger.expressErrorMiddleware()
+```
+
+### 4. Manual logging in routes
+
+```js
+app.post("/api/v1/attendance/get", async (req, res) => {
+  try {
+    // ... your logic, external APIs etc ...
+
+    await logger.logRequest(req, req.user?.employee_id);
+    res.json({ success: true });
+  } catch (err) {
+    await logger.logError(err, req, req.user?.employee_id);
+    res.status(500).json({ error: err.message });
+  }
+});
+```
+
+---
+
+## External API logging with Axios
+
+```js
+const axios = require("axios");
+
+// Attach once at startup
+logger.attachAxiosLogger(axios);
+
+app.get("/api/v1/some-data", async (req, res) => {
+  try {
+    const response = await axios.get("https://api.example.com/data", {
+      headers: {
+        "X-Correlation-ID": req.correlationId,          // propagated
+        "X-Employee-ID": req.user?.employee_id || ""    // optional
+      },
+      params: {
+        id: 123
+      }
+    });
+
+    res.json(response.data);
+  } catch (err) {
+    await logger.logError(err, req, req.user?.employee_id);
+    res.status(500).json({ error: err.message });
+  }
+});
+```
+
+This will produce external log lines like:
+
+```json
+{
+  "url": "https://api.example.com/data",
+  "body": "{}",
+  "params": "{\"id\":123}",
+  "type": "external_api",
+  "error": "",
+  "date": "2025-12-05 12:24:00",
+  "employee_id": "TAKK122",
+  "correlation_id": "cid-abcd1234-17f5d3c9a"
+}
+```
+
+---
+
+## Sensitive Data Masking
+
+Built-in masked keys (case-insensitive, partial match):
+
+- password, pass  
+- token, secret  
+- otp  
+- auth, authorization  
+- apiKey, api_key  
+- session  
+- ssn  
+
+Plus anything you pass in `maskFields` option.
+
+Any object like:
+
+```json
+{
+  "password": "MyPass123",
+  "otp": "111222",
+  "aadhaar": "9999-8888-7777",
+  "email": "user@example.com"
+}
+```
+
+will be logged as:
+
+```json
+{
+  "password": "*****",
+  "otp": "*****",
+  "aadhaar": "*****",
+  "email": "user@example.com"
+}
+```
+
+---
+
+## S3 Upload Behavior
+
+- Logs are stored locally under:
+  - `logs/YYYY/MM/DD/daily_logs.json`
+- A watcher runs every `watchIntervalMs` (default 60 seconds)
+  - When the date changes (e.g., from `2025-12-05` to `2025-12-06`),
+  - The logger uploads the **previous day's** log file to S3:
+
+Example S3 key:
+
+```txt
+2025/12/05/daily_logs.json
+```
+
+So final S3 path:
+
+```txt
+s3://<bucket>/<year>/<month>/<day>/daily_logs.json
+```
+
+---
+
+## TypeScript Usage
+
+```ts
+import Logger, { S3Config, LoggerOptions } from "@ve/logger";
+
+const s3config: S3Config = {
+  accessKeyId: process.env.AWS_KEY!,
+  secretAccessKey: process.env.AWS_SECRET!,
+  region: "ap-south-1",
+  bucket: "your-log-bucket"
+};
+
+const options: LoggerOptions = {
+  baseDir: "logs",
+  watchIntervalMs: 60000,
+  maskFields: ["aadhaar", "pan"]
+};
+
+const logger = new Logger(s3config, options);
+```
+
+---
+
+## License
+
+MIT
 
-See source comments for usage.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nm-logger/logger",
-  "version": "1.1.6",
+  "version": "1.1.8",
   "description": "Express JSON logger with S3 upload, correlation IDs, and separate success/error/external daily logs.",
   "main": "index.js",
   "types": "index.d.ts",
@@ -22,7 +22,8 @@
   "license": "MIT",
   "dependencies": {
     "@aws-sdk/client-s3": "^3.600.0",
-    "fs-extra": "^11.1.1"
+    "fs-extra": "^11.1.1",
+     "@nm-logger/logger": "^1.1.8"
   },
   "peerDependencies": {
     "express": ">=4.0.0"

package/src/DailyWatcher.js

@@ -40,16 +40,42 @@ class DailyWatcher {
           return;
         }
 
-        this.queue.add(async () => {
-          try {
-            console.log(`📤 Uploading [${category}] logs to S3:`, key);
-            await this.s3Uploader.upload(file, key);
-            console.log("✅ Uploaded. Deleting local file:", file);
-            await fs.remove(file);
-          } catch (err) {
-            console.error(`❌ Error uploading [${category}] logs to S3:`, err);
-          }
-        });
+       this.queue.add(async () => {
+  try {
+    const now = new Date();
+    const key = `${Y}/${M}/${D}/${fileName}`;
+
+    console.log(`📤 Uploading [${category}] logs to S3: ${key}`);
+
+    // Step 1 — Read local file
+    const localContent = JSON.parse(await fs.readFile(file, "utf8"));
+
+    // Step 2 — Try loading existing file from S3
+    let existing = { logs: [] };
+    try {
+      const s3Content = await this.s3Uploader.getObject(key);
+      if (s3Content) {
+        existing = JSON.parse(s3Content);
+      }
+    } catch (err) {
+      // File may not exist on first upload — ignore
+    }
+
+    // Step 3 — Append logs
+    const merged = {
+      logs: [...existing.logs, ...localContent.logs]
+    };
+
+    // Step 4 — Upload merged logs
+    await this.s3Uploader.putObject(key, JSON.stringify(merged, null, 2));
+
+    console.log("✅ Logs appended to S3 successfully");
+
+  } catch (err) {
+    console.error(`❌ Error uploading logs to S3:`, err);
+  }
+});
+
       });
     }, this.intervalMs);
   }

package/src/S3Uploader.js

@@ -1,8 +1,10 @@
 const fs = require("fs");
 const {
   S3Client,
-  PutObjectCommand
+  PutObjectCommand,
+  GetObjectCommand
 } = require("@aws-sdk/client-s3");
+const { streamToString } = require("./utils");
 
 class S3Uploader {
   constructor(config) {
@@ -17,16 +19,28 @@ class S3Uploader {
     this.bucket = config.bucket;
   }
 
-  async upload(filePath, s3Key) {
-    const fileData = fs.readFileSync(filePath);
-
-    const command = new PutObjectCommand({
-      Bucket: this.bucket,
-      Key: s3Key,
-      Body: fileData
-    });
+  async getObject(key) {
+    try {
+      const result = await this.client.send(
+        new GetObjectCommand({
+          Bucket: this.bucket,
+          Key: key
+        })
+      );
+      return await streamToString(result.Body);
+    } catch (err) {
+      return null; // file not found
+    }
+  }
 
-    await this.client.send(command);
+  async putObject(key, body) {
+    await this.client.send(
+      new PutObjectCommand({
+        Bucket: this.bucket,
+        Key: key,
+        Body: body
+      })
+    );
   }
 }
 

package/src/utils.js

@@ -89,3 +89,12 @@ exports.generateCorrelationId = () => {
   const ts = Date.now().toString(16);
   return `cid-${rand}-${ts}`;
 };
+
+exports.streamToString = async (stream) => {
+  return await new Promise((resolve, reject) => {
+    const chunks = [];
+    stream.on("data", (chunk) => chunks.push(chunk));
+    stream.on("error", reject);
+    stream.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")));
+  });
+};