@nm-logger/logger 1.2.5 → 1.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +285 -16
- package/package.json +12 -1
- package/src/DailyWatcher.js +85 -2
- package/src/LogWriter.js +32 -4
- package/src/Logger.js +132 -19
package/README.md
CHANGED
|
@@ -1,48 +1,317 @@
|
|
|
1
|
-
# @nm-logger/logger
|
|
1
|
+
# @nm-logger/logger
|
|
2
2
|
|
|
3
|
-
Minimal JSON logger for Express
|
|
3
|
+
Minimal JSON logger for Express + Axios with optional S3 sync.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- Logs handled Express requests as **success**
|
|
6
8
|
- Logs Express errors as **error**
|
|
7
9
|
- Logs Axios calls as **external**
|
|
10
|
+
- Optional **database error** logs as `db_error`
|
|
8
11
|
- Local daily files under configurable `baseDir`:
|
|
9
12
|
- `YYYY/MM/DD/daily_logs_success.json`
|
|
10
13
|
- `YYYY/MM/DD/daily_logs_error.json`
|
|
11
14
|
- `YYYY/MM/DD/daily_logs_external.json`
|
|
12
|
-
-
|
|
15
|
+
- `YYYY/MM/DD/daily_logs_db_error.json`
|
|
16
|
+
- S3: one JSON file per day per category (append by read + merge + put)
|
|
13
17
|
- Previous day's local folder is removed when the date changes
|
|
14
|
-
- `employee_id` taken from `req.user.employee_id || req.user.emp_code || req.user.id`
|
|
15
|
-
-
|
|
18
|
+
- `employee_id` taken from `req.user.employee_id || req.user.emp_code || req.user.id` (used internally only, **not** written to JSON)
|
|
19
|
+
- Each log entry includes:
|
|
20
|
+
- `url`, `method`, `type` (internal/external/db_error)
|
|
21
|
+
- `status_code`
|
|
22
|
+
- `user_agent`
|
|
23
|
+
- `project_version` (from Logger options)
|
|
24
|
+
- `error`
|
|
25
|
+
- optional `response` (for external errors)
|
|
26
|
+
- `count` (when consecutive identical entries are compacted)
|
|
27
|
+
- External API logging:
|
|
28
|
+
- Logs **response body only for error-like calls** (4xx/5xx or when a custom `externalErrorDetector` marks it as error)
|
|
29
|
+
- Long responses are truncated with `maxResponseLength`
|
|
30
|
+
|
|
31
|
+
## Install
|
|
16
32
|
|
|
33
|
+
```bash
|
|
34
|
+
npm install @nm-logger/logger
|
|
35
|
+
```
|
|
17
36
|
|
|
18
|
-
Basic usage
|
|
37
|
+
## Basic usage (Express + Axios)
|
|
19
38
|
|
|
20
39
|
```js
|
|
21
|
-
const
|
|
40
|
+
const express = require("express");
|
|
22
41
|
const axios = require("axios");
|
|
42
|
+
const Logger = require("@nm-logger/logger");
|
|
43
|
+
|
|
44
|
+
const app = express();
|
|
45
|
+
|
|
46
|
+
const nm_logger = new Logger(
|
|
47
|
+
{
|
|
48
|
+
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
|
49
|
+
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
|
|
50
|
+
region: "us-east-1",
|
|
51
|
+
bucket: "project",
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
baseDir: "/var/log/nm-logger",
|
|
55
|
+
maskFields: ["aadhaar", "pan"],
|
|
56
|
+
|
|
57
|
+
// Optional metadata for logs
|
|
58
|
+
projectVersion: "Project-1.0.0",
|
|
59
|
+
|
|
60
|
+
// Background S3 uploader (set to false if you prefer cron; see below)
|
|
61
|
+
enableDailyWatcher: true,
|
|
62
|
+
|
|
63
|
+
// Interval for background S3 uploads when enableDailyWatcher = true
|
|
64
|
+
uploadIntervalMs: 60 * 1000, // 1 minute
|
|
65
|
+
|
|
66
|
+
// How many characters of external response to keep (after masking)
|
|
67
|
+
maxResponseLength: 4000,
|
|
68
|
+
|
|
69
|
+
// If true (default), include external response body only for error-like calls
|
|
70
|
+
logExternalResponseOnError: true,
|
|
71
|
+
|
|
72
|
+
// Control logging of successful external calls
|
|
73
|
+
// - true (default): log all external calls
|
|
74
|
+
// - false: log only error-like calls (4xx/5xx or externalErrorDetector = true)
|
|
75
|
+
logExternalSuccess: true,
|
|
76
|
+
|
|
77
|
+
// Optional semantic error detector for 2xx/3xx external responses
|
|
78
|
+
// Example below in "External error-only logging"
|
|
79
|
+
externalErrorDetector: null,
|
|
80
|
+
}
|
|
81
|
+
);
|
|
82
|
+
|
|
83
|
+
// Express middlewares
|
|
84
|
+
app.use(express.json());
|
|
85
|
+
app.use(nm_logger.requestLoggerMiddleware());
|
|
86
|
+
app.use("/api", require("./routes"));
|
|
87
|
+
app.use(nm_logger.expressMiddleware());
|
|
88
|
+
|
|
89
|
+
// Attach Axios interceptor
|
|
90
|
+
nm_logger.attachAxiosLogger(axios);
|
|
23
91
|
|
|
92
|
+
app.listen(3000, () => {
|
|
93
|
+
console.log("Server listening on 3000");
|
|
94
|
+
});
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
### Folder permissions
|
|
98
|
+
|
|
99
|
+
Make sure the base directory exists and is writable by your Node process:
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
mkdir -p /var/log/nm-logger
|
|
103
|
+
chmod -R 777 /var/log/nm-logger
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
## External error-only logging (recommended for high load)
|
|
107
|
+
|
|
108
|
+
If your external API returns HTTP 200 but the **body** contains error info, you can:
|
|
109
|
+
|
|
110
|
+
- Skip logging successful calls completely
|
|
111
|
+
- Still log error-like responses with full (masked + truncated) body
|
|
112
|
+
|
|
113
|
+
Example:
|
|
114
|
+
|
|
115
|
+
```js
|
|
24
116
|
const nm_logger = new Logger(
|
|
25
117
|
{
|
|
26
118
|
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
|
27
119
|
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
|
|
28
120
|
region: "us-east-1",
|
|
29
|
-
bucket: "
|
|
121
|
+
bucket: "project",
|
|
30
122
|
},
|
|
31
123
|
{
|
|
32
|
-
baseDir: "/
|
|
33
|
-
|
|
34
|
-
|
|
124
|
+
baseDir: "/var/log/nm-logger",
|
|
125
|
+
maskFields: ["aadhaar", "pan"],
|
|
126
|
+
|
|
127
|
+
enableDailyWatcher: false, // use cron for S3 (see next section)
|
|
128
|
+
logExternalSuccess: false, // do NOT log success responses
|
|
129
|
+
|
|
130
|
+
externalErrorDetector: ({ response }) => {
|
|
131
|
+
try {
|
|
132
|
+
// Adjust this according to your external API response format
|
|
133
|
+
if (response && response.success === false) return true;
|
|
134
|
+
if (response && typeof response.status === "string" &&
|
|
135
|
+
response.status.toLowerCase() === "error") return true;
|
|
136
|
+
if (response && (response.error || response.error_code)) return true;
|
|
137
|
+
} catch (e) {
|
|
138
|
+
return false;
|
|
139
|
+
}
|
|
140
|
+
return false;
|
|
141
|
+
},
|
|
35
142
|
}
|
|
36
143
|
);
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
With this config:
|
|
147
|
+
|
|
148
|
+
- Only external calls that look like **errors** are logged (4xx/5xx or `externalErrorDetector` = true).
|
|
149
|
+
- For these, the masked + truncated `response` body is stored in JSON.
|
|
150
|
+
|
|
151
|
+
## S3 upload modes
|
|
37
152
|
|
|
38
|
-
|
|
153
|
+
There are two ways to sync daily logs to S3.
|
|
39
154
|
|
|
40
|
-
|
|
41
|
-
|
|
155
|
+
### Mode 1: Background timer (default)
|
|
156
|
+
|
|
157
|
+
If `enableDailyWatcher: true` (or omitted), the logger will:
|
|
158
|
+
|
|
159
|
+
- Start an internal `setInterval` in the same Node process
|
|
160
|
+
- Every `uploadIntervalMs`:
|
|
161
|
+
- Read local daily log JSONs
|
|
162
|
+
- Merge them with S3 files
|
|
163
|
+
- Upload merged logs back to S3
|
|
164
|
+
- Clean up previous day's local folder
|
|
165
|
+
|
|
166
|
+
This is simple but adds some extra work inside your API process.
|
|
167
|
+
|
|
168
|
+
### Mode 2: Cron-based upload (recommended for production)
|
|
169
|
+
|
|
170
|
+
To keep your API process as light as possible, you can:
|
|
171
|
+
|
|
172
|
+
1. **Disable the internal watcher in your API**
|
|
173
|
+
2. **Use a separate cron script** to upload logs to S3
|
|
174
|
+
|
|
175
|
+
#### 1. API server: disable watcher
|
|
176
|
+
|
|
177
|
+
```js
|
|
178
|
+
const nm_logger = new Logger(
|
|
179
|
+
{
|
|
180
|
+
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
|
181
|
+
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
|
|
182
|
+
region: "us-east-1",
|
|
183
|
+
bucket: "project",
|
|
184
|
+
},
|
|
185
|
+
{
|
|
186
|
+
baseDir: "/var/log/nm-logger",
|
|
187
|
+
maskFields: ["aadhaar", "pan"],
|
|
188
|
+
enableDailyWatcher: false, // 🔴 no setInterval in API process
|
|
189
|
+
}
|
|
190
|
+
);
|
|
42
191
|
|
|
43
192
|
app.use(nm_logger.requestLoggerMiddleware());
|
|
44
193
|
app.use("/api", routes);
|
|
45
194
|
app.use(nm_logger.expressMiddleware());
|
|
46
195
|
nm_logger.attachAxiosLogger(axios);
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
This way, the API only writes local JSON files.
|
|
199
|
+
|
|
200
|
+
### Mode 2b: node-cron inside Node (no system crontab)
|
|
201
|
+
|
|
202
|
+
If you are already using [`node-cron`](https://www.npmjs.com/package/node-cron) in your Node project, you can avoid Linux `crontab` entirely and schedule uploads from your app code.
|
|
203
|
+
|
|
204
|
+
1. Disable the internal watcher in your API (same as before):
|
|
205
|
+
|
|
206
|
+
```js
|
|
207
|
+
const nm_logger = new Logger(
|
|
208
|
+
{
|
|
209
|
+
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
|
210
|
+
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
|
|
211
|
+
region: "us-east-1",
|
|
212
|
+
bucket: "project",
|
|
213
|
+
},
|
|
214
|
+
{
|
|
215
|
+
baseDir: "/var/log/nm-logger",
|
|
216
|
+
maskFields: ["aadhaar", "pan"],
|
|
217
|
+
enableDailyWatcher: false, // no setInterval
|
|
218
|
+
}
|
|
219
|
+
);
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
2. Use `node-cron` in your Node app to trigger uploads:
|
|
223
|
+
|
|
224
|
+
```js
|
|
225
|
+
const cron = require("node-cron");
|
|
226
|
+
const Logger = require("@nm-logger/logger");
|
|
227
|
+
|
|
228
|
+
const nm_logger = new Logger(
|
|
229
|
+
{
|
|
230
|
+
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
|
231
|
+
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
|
|
232
|
+
region: "us-east-1",
|
|
233
|
+
bucket: "project",
|
|
234
|
+
},
|
|
235
|
+
{
|
|
236
|
+
baseDir: "/var/log/nm-logger",
|
|
237
|
+
maskFields: ["aadhaar", "pan"],
|
|
238
|
+
enableDailyWatcher: false,
|
|
239
|
+
}
|
|
240
|
+
);
|
|
241
|
+
|
|
242
|
+
// Every 5 minutes
|
|
243
|
+
cron.schedule("*/5 * * * *", async () => {
|
|
244
|
+
try {
|
|
245
|
+
await nm_logger.uploadDailyLogsOnce();
|
|
246
|
+
console.log("[nm-logger] ✅ Daily logs uploaded to S3 (node-cron)");
|
|
247
|
+
} catch (err) {
|
|
248
|
+
console.error("[nm-logger] ❌ Upload failed (node-cron):", err);
|
|
249
|
+
}
|
|
250
|
+
});
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
This way you:
|
|
254
|
+
- Do **not** depend on OS-level `crontab`.
|
|
255
|
+
- Still avoid `setInterval`-based uploads inside the main request flow.
|
|
256
|
+
- Keep all scheduling logic inside Node where you are already comfortable.
|
|
257
|
+
|
|
258
|
+
#### 3. System Crontab entry (optional, Linux)
|
|
259
|
+
|
|
260
|
+
If you prefer OS-level cron instead of `node-cron`, you can still use a separate script as before.
|
|
261
|
+
|
|
262
|
+
Create a separate script (for example in your backend root):
|
|
263
|
+
|
|
264
|
+
```js
|
|
265
|
+
// upload-logs.js
|
|
266
|
+
require("dotenv").config();
|
|
267
|
+
const Logger = require("@nm-logger/logger");
|
|
268
|
+
|
|
269
|
+
async function main() {
|
|
270
|
+
const logger = new Logger(
|
|
271
|
+
{
|
|
272
|
+
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
|
273
|
+
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
|
|
274
|
+
region: "us-east-1",
|
|
275
|
+
bucket: "project",
|
|
276
|
+
},
|
|
277
|
+
{
|
|
278
|
+
baseDir: "/var/log/nm-logger",
|
|
279
|
+
maskFields: ["aadhaar", "pan"],
|
|
280
|
+
enableDailyWatcher: false, // no internal timers here either
|
|
281
|
+
}
|
|
282
|
+
);
|
|
283
|
+
|
|
284
|
+
try {
|
|
285
|
+
await logger.uploadDailyLogsOnce();
|
|
286
|
+
console.log("[nm-logger] ✅ Daily logs uploaded to S3");
|
|
287
|
+
} catch (err) {
|
|
288
|
+
console.error("[nm-logger] ❌ Failed to upload daily logs:", err);
|
|
289
|
+
process.exitCode = 1;
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
main();
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
On your server (Linux):
|
|
297
|
+
|
|
298
|
+
```bash
|
|
299
|
+
crontab -e
|
|
300
|
+
```
|
|
301
|
+
|
|
302
|
+
Add a line like:
|
|
303
|
+
|
|
304
|
+
```cron
|
|
305
|
+
*/5 * * * * /usr/bin/node /path/to/upload-logs.js >> /var/log/nm-logger-cron.log 2>&1
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
- This will **not** be created automatically by the package.
|
|
309
|
+
- You must add it manually using `crontab -e`.
|
|
310
|
+
- Make sure the `node` path (`/usr/bin/node`) and script path (`/path/to/upload-logs.js`) are correct for your server.
|
|
311
|
+
|
|
312
|
+
## Notes
|
|
47
313
|
|
|
48
|
-
|
|
314
|
+
- The logger does **not** create or manage cron jobs automatically; you must configure `node-cron` or `crontab` yourself if you choose cron mode.
|
|
315
|
+
- For lower CPU and disk usage on busy servers:
|
|
316
|
+
- Prefer `logExternalSuccess: false` + `externalErrorDetector` to log only problematic external calls.
|
|
317
|
+
- Use cron-based upload (Mode 2 / Mode 2b) so S3 sync runs in a separate short-lived context.
|
package/package.json
CHANGED
|
@@ -1 +1,12 @@
|
|
|
1
|
-
{
|
|
1
|
+
{
|
|
2
|
+
"name": "@nm-logger/logger",
|
|
3
|
+
"version": "1.2.7",
|
|
4
|
+
"description": "Express JSON logger with daily success/error/external logs, S3 append uploads, and configurable baseDir.",
|
|
5
|
+
"main": "index.js",
|
|
6
|
+
"types": "index.d.ts",
|
|
7
|
+
"license": "MIT",
|
|
8
|
+
"dependencies": {
|
|
9
|
+
"@aws-sdk/client-s3": "^3.600.0",
|
|
10
|
+
"fs-extra": "^11.1.1"
|
|
11
|
+
}
|
|
12
|
+
}
|
package/src/DailyWatcher.js
CHANGED
|
@@ -1,2 +1,85 @@
|
|
|
1
|
-
const fs=require("fs-extra");
|
|
2
|
-
|
|
1
|
+
const fs = require("fs-extra");
|
|
2
|
+
const path = require("path");
|
|
3
|
+
const { getDatePath } = require("./utils");
|
|
4
|
+
|
|
5
|
+
const MAP = {
|
|
6
|
+
success: "daily_logs_success.json",
|
|
7
|
+
error: "daily_logs_error.json",
|
|
8
|
+
external: "daily_logs_external.json",
|
|
9
|
+
db_error: "daily_logs_db_error.json",
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
class DailyWatcher {
|
|
13
|
+
constructor(baseDir, q, s3, opt = {}) {
|
|
14
|
+
this.baseDir = baseDir;
|
|
15
|
+
this.q = q;
|
|
16
|
+
this.s3 = s3;
|
|
17
|
+
this.uploadIntervalMs = opt.uploadIntervalMs || 60_000;
|
|
18
|
+
this.currentDate = getDatePath();
|
|
19
|
+
this.disableTimer = !!opt.disableTimer;
|
|
20
|
+
|
|
21
|
+
if (!this.disableTimer) {
|
|
22
|
+
this.start();
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
async cleanup() {
|
|
27
|
+
const { Y, M, D } = getDatePath();
|
|
28
|
+
const p = this.currentDate;
|
|
29
|
+
|
|
30
|
+
if (p && (p.Y !== Y || p.M !== M || p.D !== D)) {
|
|
31
|
+
const dir = path.join(this.baseDir, `${p.Y}/${p.M}/${p.D}`);
|
|
32
|
+
try {
|
|
33
|
+
if (await fs.pathExists(dir)) {
|
|
34
|
+
await fs.remove(dir);
|
|
35
|
+
}
|
|
36
|
+
} catch (_) {}
|
|
37
|
+
this.currentDate = { Y, M, D };
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
start() {
|
|
42
|
+
setInterval(() => {
|
|
43
|
+
this.tick().catch(() => {});
|
|
44
|
+
}, this.uploadIntervalMs);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
async tick() {
|
|
48
|
+
await this.cleanup();
|
|
49
|
+
|
|
50
|
+
const { Y, M, D } = getDatePath();
|
|
51
|
+
|
|
52
|
+
for (const [cat, fileName] of Object.entries(MAP)) {
|
|
53
|
+
const file = path.join(this.baseDir, `${Y}/${M}/${D}`, fileName);
|
|
54
|
+
const key = `${Y}/${M}/${D}/${fileName}`;
|
|
55
|
+
|
|
56
|
+
if (!(await fs.pathExists(file))) continue;
|
|
57
|
+
|
|
58
|
+
this.q.add(async () => {
|
|
59
|
+
try {
|
|
60
|
+
const localTxt = await fs.readFile(file, "utf8");
|
|
61
|
+
let local = { logs: [] };
|
|
62
|
+
if (localTxt && localTxt.trim()) {
|
|
63
|
+
local = JSON.parse(localTxt);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
let existing = { logs: [] };
|
|
67
|
+
try {
|
|
68
|
+
const s3Txt = await this.s3.getObject(key);
|
|
69
|
+
if (s3Txt && s3Txt.trim()) {
|
|
70
|
+
existing = JSON.parse(s3Txt);
|
|
71
|
+
}
|
|
72
|
+
} catch (_) {}
|
|
73
|
+
|
|
74
|
+
const merged = {
|
|
75
|
+
logs: [...(existing.logs || []), ...(local.logs || [])],
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
await this.s3.putObject(key, JSON.stringify(merged, null, 2));
|
|
79
|
+
} catch (_) {}
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
module.exports = DailyWatcher;
|
package/src/LogWriter.js
CHANGED
|
@@ -6,6 +6,7 @@ const MAP = {
|
|
|
6
6
|
success: "daily_logs_success.json",
|
|
7
7
|
error: "daily_logs_error.json",
|
|
8
8
|
external: "daily_logs_external.json",
|
|
9
|
+
db_error: "daily_logs_db_error.json",
|
|
9
10
|
};
|
|
10
11
|
|
|
11
12
|
class LogWriter {
|
|
@@ -37,17 +38,44 @@ class LogWriter {
|
|
|
37
38
|
} catch (_) {}
|
|
38
39
|
}
|
|
39
40
|
|
|
40
|
-
|
|
41
|
+
const entry = {
|
|
41
42
|
url: log.url || "",
|
|
42
43
|
body: log.body || "",
|
|
43
44
|
params: log.params || "",
|
|
44
|
-
response: log.response || "",
|
|
45
|
+
response: log.response || "",
|
|
45
46
|
type: log.type || "",
|
|
46
47
|
method: log.method || "",
|
|
47
48
|
error: log.error || "",
|
|
48
|
-
|
|
49
|
+
status_code: log.status_code || "",
|
|
50
|
+
user_agent: log.user_agent || "",
|
|
51
|
+
project_version: log.project_version || "",
|
|
49
52
|
date: log.date || formatDate(new Date()),
|
|
50
|
-
}
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
const logs = wrapper.logs;
|
|
56
|
+
|
|
57
|
+
if (logs.length > 0) {
|
|
58
|
+
const last = logs[logs.length - 1];
|
|
59
|
+
|
|
60
|
+
if (
|
|
61
|
+
last.url === entry.url &&
|
|
62
|
+
last.method === entry.method &&
|
|
63
|
+
last.type === entry.type &&
|
|
64
|
+
String(last.status_code || "") === String(entry.status_code || "") &&
|
|
65
|
+
String(last.user_agent || "") === String(entry.user_agent || "") &&
|
|
66
|
+
String(last.project_version || "") === String(entry.project_version || "")
|
|
67
|
+
) {
|
|
68
|
+
// Same URL/method/type/status/user_agent/version → compact into one entry
|
|
69
|
+
last.count = (last.count || 1) + 1;
|
|
70
|
+
last.date = entry.date;
|
|
71
|
+
} else {
|
|
72
|
+
entry.count = entry.count || 1;
|
|
73
|
+
logs.push(entry);
|
|
74
|
+
}
|
|
75
|
+
} else {
|
|
76
|
+
entry.count = entry.count || 1;
|
|
77
|
+
logs.push(entry);
|
|
78
|
+
}
|
|
51
79
|
|
|
52
80
|
await fs.writeFile(filePath, JSON.stringify(wrapper, null, 2), "utf8");
|
|
53
81
|
return filePath;
|
package/src/Logger.js
CHANGED
|
@@ -40,9 +40,48 @@ class Logger {
|
|
|
40
40
|
String(x).toLowerCase()
|
|
41
41
|
);
|
|
42
42
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
43
|
+
// Optional: application/version metadata
|
|
44
|
+
this.projectVersion = opt.projectVersion || opt.version || "";
|
|
45
|
+
|
|
46
|
+
// Limit size of stored external responses to avoid huge log files
|
|
47
|
+
this.maxResponseLength =
|
|
48
|
+
typeof opt.maxResponseLength === "number" ? opt.maxResponseLength : 4000;
|
|
49
|
+
|
|
50
|
+
// Only include external API responses for errors by default
|
|
51
|
+
this.logExternalResponseOnError =
|
|
52
|
+
opt.logExternalResponseOnError !== false; // default: true
|
|
53
|
+
|
|
54
|
+
// Control logging of successful external calls
|
|
55
|
+
this.logExternalSuccess =
|
|
56
|
+
opt.logExternalSuccess !== false; // default: true (for backward compatibility)
|
|
57
|
+
|
|
58
|
+
// Optional semantic error detector for 2xx/3xx responses
|
|
59
|
+
this.externalErrorDetector =
|
|
60
|
+
typeof opt.externalErrorDetector === "function"
|
|
61
|
+
? opt.externalErrorDetector
|
|
62
|
+
: null;
|
|
63
|
+
|
|
64
|
+
// Control background S3 uploader (DailyWatcher)
|
|
65
|
+
this.enableDailyWatcher = opt.enableDailyWatcher !== false;
|
|
66
|
+
|
|
67
|
+
if (this.enableDailyWatcher) {
|
|
68
|
+
this.dailyWatcher = new DailyWatcher(this.baseDir, this.queue, this.s3, {
|
|
69
|
+
uploadIntervalMs: opt.uploadIntervalMs || 60_000,
|
|
70
|
+
disableTimer: false,
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
async uploadDailyLogsOnce() {
|
|
76
|
+
if (!this._manualDailyWatcher) {
|
|
77
|
+
this._manualDailyWatcher = new DailyWatcher(
|
|
78
|
+
this.baseDir,
|
|
79
|
+
this.queue,
|
|
80
|
+
this.s3,
|
|
81
|
+
{ disableTimer: true }
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
await this._manualDailyWatcher.tick();
|
|
46
85
|
}
|
|
47
86
|
|
|
48
87
|
// Mask helper
|
|
@@ -51,7 +90,7 @@ class Logger {
|
|
|
51
90
|
}
|
|
52
91
|
|
|
53
92
|
// Build base log object for success/error logs
|
|
54
|
-
buildBaseLog(req, employee_id = "") {
|
|
93
|
+
buildBaseLog(req, employee_id = "", statusCode) {
|
|
55
94
|
const url = req && req.originalUrl ? req.originalUrl : "";
|
|
56
95
|
const body = (req && req.body) || {};
|
|
57
96
|
const params = {
|
|
@@ -62,6 +101,11 @@ class Logger {
|
|
|
62
101
|
const maskedBody = this.mask(body);
|
|
63
102
|
const maskedParams = this.mask(params);
|
|
64
103
|
const method = (req && req.method ? req.method : "").toUpperCase();
|
|
104
|
+
const userAgent =
|
|
105
|
+
(req &&
|
|
106
|
+
req.headers &&
|
|
107
|
+
(req.headers["user-agent"] || req.headers["User-Agent"])) ||
|
|
108
|
+
"";
|
|
65
109
|
|
|
66
110
|
const emp =
|
|
67
111
|
employee_id ||
|
|
@@ -79,12 +123,16 @@ class Logger {
|
|
|
79
123
|
method,
|
|
80
124
|
error: "",
|
|
81
125
|
employee_id: emp,
|
|
126
|
+
status_code:
|
|
127
|
+
typeof statusCode === "number" ? statusCode : statusCode || "",
|
|
128
|
+
user_agent: userAgent,
|
|
129
|
+
project_version: this.projectVersion || "",
|
|
82
130
|
};
|
|
83
131
|
}
|
|
84
132
|
|
|
85
133
|
// ---- Error logging (Express errors) ----
|
|
86
|
-
async logError(err, req, employee_id = "") {
|
|
87
|
-
const base = this.buildBaseLog(req || {}, employee_id);
|
|
134
|
+
async logError(err, req, employee_id = "", statusCode) {
|
|
135
|
+
const base = this.buildBaseLog(req || {}, employee_id, statusCode);
|
|
88
136
|
|
|
89
137
|
const log = {
|
|
90
138
|
...base,
|
|
@@ -95,8 +143,8 @@ class Logger {
|
|
|
95
143
|
}
|
|
96
144
|
|
|
97
145
|
// ---- Success logging (normal requests) ----
|
|
98
|
-
async logRequest(req, employee_id = "") {
|
|
99
|
-
const log = this.buildBaseLog(req, employee_id);
|
|
146
|
+
async logRequest(req, employee_id = "", statusCode) {
|
|
147
|
+
const log = this.buildBaseLog(req, employee_id, statusCode);
|
|
100
148
|
await this.logWriter.writeLog(log, "success");
|
|
101
149
|
}
|
|
102
150
|
|
|
@@ -107,22 +155,70 @@ class Logger {
|
|
|
107
155
|
data,
|
|
108
156
|
params,
|
|
109
157
|
error,
|
|
110
|
-
|
|
111
|
-
|
|
158
|
+
response,
|
|
159
|
+
statusCode,
|
|
160
|
+
userAgent,
|
|
112
161
|
}) {
|
|
113
162
|
const maskedBody = this.mask(data || {});
|
|
114
163
|
const maskedParams = this.mask(params || {});
|
|
115
|
-
|
|
164
|
+
|
|
165
|
+
const codeStr = statusCode ? String(statusCode) : "";
|
|
166
|
+
const isErrorStatus =
|
|
167
|
+
codeStr.startsWith("4") || codeStr.startsWith("5");
|
|
168
|
+
|
|
169
|
+
let semanticError = false;
|
|
170
|
+
if (this.externalErrorDetector) {
|
|
171
|
+
try {
|
|
172
|
+
semanticError = !!this.externalErrorDetector({
|
|
173
|
+
url,
|
|
174
|
+
method,
|
|
175
|
+
statusCode,
|
|
176
|
+
response,
|
|
177
|
+
error,
|
|
178
|
+
});
|
|
179
|
+
} catch (_) {
|
|
180
|
+
semanticError = false;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
const hasErrorFlag = !!(error || isErrorStatus || semanticError);
|
|
185
|
+
|
|
186
|
+
// If logExternalSuccess is false, skip non-error calls completely
|
|
187
|
+
if (!this.logExternalSuccess && !hasErrorFlag) {
|
|
188
|
+
return;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
let respStr = "";
|
|
192
|
+
const includeResponse =
|
|
193
|
+
this.logExternalResponseOnError && hasErrorFlag;
|
|
194
|
+
|
|
195
|
+
if (includeResponse && typeof response !== "undefined") {
|
|
196
|
+
try {
|
|
197
|
+
const maskedResponse = this.mask(response || {});
|
|
198
|
+
respStr = JSON.stringify(maskedResponse || {});
|
|
199
|
+
} catch (_) {
|
|
200
|
+
respStr = "";
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
if (this.maxResponseLength && respStr.length > this.maxResponseLength) {
|
|
204
|
+
respStr =
|
|
205
|
+
respStr.slice(0, this.maxResponseLength) +
|
|
206
|
+
`... [truncated ${respStr.length - this.maxResponseLength} chars]`;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
116
209
|
|
|
117
210
|
const log = {
|
|
118
211
|
url: url || "",
|
|
119
212
|
body: JSON.stringify(maskedBody || {}),
|
|
120
213
|
params: JSON.stringify(maskedParams || {}),
|
|
121
|
-
response:
|
|
214
|
+
response: respStr,
|
|
122
215
|
type: "external_api",
|
|
123
216
|
method: (method || "").toUpperCase(),
|
|
124
217
|
error: error || "",
|
|
125
|
-
|
|
218
|
+
status_code:
|
|
219
|
+
typeof statusCode === "number" ? statusCode : statusCode || "",
|
|
220
|
+
user_agent: userAgent || "",
|
|
221
|
+
project_version: this.projectVersion || "",
|
|
126
222
|
};
|
|
127
223
|
|
|
128
224
|
await this.logWriter.writeLog(log, "external");
|
|
@@ -132,7 +228,8 @@ class Logger {
|
|
|
132
228
|
expressMiddleware() {
|
|
133
229
|
return (err, req, res, next) => {
|
|
134
230
|
try {
|
|
135
|
-
|
|
231
|
+
const statusCode = res && res.statusCode;
|
|
232
|
+
this.logError(err, req, "", statusCode).catch(() => {});
|
|
136
233
|
} catch (_) {}
|
|
137
234
|
next(err);
|
|
138
235
|
};
|
|
@@ -144,7 +241,12 @@ class Logger {
|
|
|
144
241
|
try {
|
|
145
242
|
if (!req.__nm_logger_logged) {
|
|
146
243
|
req.__nm_logger_logged = true;
|
|
147
|
-
|
|
244
|
+
res.on("finish", () => {
|
|
245
|
+
try {
|
|
246
|
+
const statusCode = res && res.statusCode;
|
|
247
|
+
this.logRequest(req, "", statusCode).catch(() => {});
|
|
248
|
+
} catch (_) {}
|
|
249
|
+
});
|
|
148
250
|
}
|
|
149
251
|
} catch (_) {}
|
|
150
252
|
next();
|
|
@@ -160,14 +262,20 @@ class Logger {
|
|
|
160
262
|
(response) => {
|
|
161
263
|
try {
|
|
162
264
|
const cfg = response.config || {};
|
|
265
|
+
const statusCode = response && response.status;
|
|
266
|
+
const headers = (cfg && cfg.headers) || {};
|
|
267
|
+
const userAgent =
|
|
268
|
+
headers["User-Agent"] || headers["user-agent"] || "";
|
|
269
|
+
|
|
163
270
|
this.logExternalApi({
|
|
164
271
|
url: cfg.url,
|
|
165
272
|
method: cfg.method,
|
|
166
273
|
data: cfg.data,
|
|
167
274
|
params: cfg.params,
|
|
168
|
-
response: response.data,
|
|
275
|
+
response: response.data,
|
|
169
276
|
error: null,
|
|
170
|
-
|
|
277
|
+
statusCode,
|
|
278
|
+
userAgent,
|
|
171
279
|
}).catch(() => {});
|
|
172
280
|
} catch (_) {}
|
|
173
281
|
|
|
@@ -179,15 +287,20 @@ class Logger {
|
|
|
179
287
|
try {
|
|
180
288
|
const cfg = (error && error.config) || {};
|
|
181
289
|
const res = error && error.response;
|
|
290
|
+
const statusCode = res && res.status;
|
|
291
|
+
const headers = (cfg && cfg.headers) || {};
|
|
292
|
+
const userAgent =
|
|
293
|
+
headers["User-Agent"] || headers["user-agent"] || "";
|
|
182
294
|
|
|
183
295
|
this.logExternalApi({
|
|
184
296
|
url: cfg && cfg.url,
|
|
185
297
|
method: cfg && cfg.method,
|
|
186
298
|
data: cfg && cfg.data,
|
|
187
299
|
params: cfg && cfg.params,
|
|
188
|
-
response: res && res.data,
|
|
300
|
+
response: res && res.data,
|
|
189
301
|
error: error && error.message,
|
|
190
|
-
|
|
302
|
+
statusCode,
|
|
303
|
+
userAgent,
|
|
191
304
|
}).catch(() => {});
|
|
192
305
|
} catch (_) {}
|
|
193
306
|
|