badmfck-api-server 4.1.59 → 5.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apiServer/APIService.d.ts +1 -0
- package/dist/apiServer/APIService.js +2 -1
- package/dist/apiServer/MonitorService.d.ts +37 -1
- package/dist/apiServer/MonitorService.js +387 -17
- package/dist/apiServer/documentation/index.html +43 -43
- package/dist/apiServer/helper/Validator.js +4 -0
- package/dist/apiServer/http/Http.d.ts +7 -0
- package/dist/apiServer/http/Http.js +33 -6
- package/dist/apiServer/monitor/Monitor.d.ts +2 -0
- package/dist/apiServer/monitor/Monitor.js +12 -0
- package/dist/apiServer/monitor/index.html +72 -56
- package/package.json +14 -14
|
@@ -51,6 +51,7 @@ export interface APIServiceOptions<TInterceptor = unknown> {
|
|
|
51
51
|
appVersion?: string;
|
|
52
52
|
fileTempDir: string;
|
|
53
53
|
fileLimit: number;
|
|
54
|
+
metricsDir?: string;
|
|
54
55
|
}
|
|
55
56
|
type InferInterceptor<T> = T extends IInterceptor<infer U> ? U : unknown;
|
|
56
57
|
export declare const getDefaultOptions: <I extends IInterceptor<any> | undefined>(overrides?: Omit<Partial<APIServiceOptions<InferInterceptor<NonNullable<I>>>>, "interceptor"> & {
|
|
@@ -79,6 +79,7 @@ function defaultOptions() {
|
|
|
79
79
|
documentation: []
|
|
80
80
|
},
|
|
81
81
|
appVersion: undefined,
|
|
82
|
+
metricsDir: undefined,
|
|
82
83
|
fileTempDir: "/tmp",
|
|
83
84
|
fileLimit: 10_000_000,
|
|
84
85
|
keepAliveTimeout: 25 * 1000,
|
|
@@ -165,7 +166,7 @@ class APIService extends BaseService_1.BaseService {
|
|
|
165
166
|
new Monitor_1.Monitor(this.options),
|
|
166
167
|
];
|
|
167
168
|
}
|
|
168
|
-
this.monitor = new MonitorService_1.MonitorService();
|
|
169
|
+
this.monitor = new MonitorService_1.MonitorService(this.options.metricsDir);
|
|
169
170
|
this.monitor.init();
|
|
170
171
|
new DocumentService_1.DocumentGenerator(this.options.endpoints);
|
|
171
172
|
this.options.endpoints.push(new Liveness_1.Liveness(this.started), new Readiness_1.Readiness(this.started));
|
|
@@ -15,6 +15,11 @@ export declare const S_STAT_REGISTRATE_SERVICE: Signal<string>;
|
|
|
15
15
|
export declare const S_STAT_REGISTRATE_HTTP_REQUEST: Signal<{
|
|
16
16
|
url: string;
|
|
17
17
|
}>;
|
|
18
|
+
export declare const S_STAT_REGISTRATE_HTTP_RESULT: Signal<{
|
|
19
|
+
host: string;
|
|
20
|
+
ok: boolean;
|
|
21
|
+
ms: number;
|
|
22
|
+
}>;
|
|
18
23
|
export declare const S_STAT_REGISTRATE_SQL_REQUEST: Signal<{
|
|
19
24
|
success: boolean;
|
|
20
25
|
query: string;
|
|
@@ -22,6 +27,10 @@ export declare const S_STAT_REGISTRATE_SQL_REQUEST: Signal<{
|
|
|
22
27
|
export declare const REQ_STAT: Req<{
|
|
23
28
|
range: "week" | "day" | "hour";
|
|
24
29
|
}, IStat[]>;
|
|
30
|
+
export declare const REQ_STAT_WEEKS: Req<void, string[]>;
|
|
31
|
+
export declare const REQ_STAT_WEEK: Req<{
|
|
32
|
+
weekKey: string;
|
|
33
|
+
}, IStat[]>;
|
|
25
34
|
type IMonitorSlot = {
|
|
26
35
|
yyyymmddhhii: number;
|
|
27
36
|
cpu: number;
|
|
@@ -35,6 +44,9 @@ type IMonitorSlot = {
|
|
|
35
44
|
successEp: Record<string, number>;
|
|
36
45
|
failEp: Record<string, number>;
|
|
37
46
|
http: Record<string, number>;
|
|
47
|
+
http_ok: Record<string, number>;
|
|
48
|
+
http_fail: Record<string, number>;
|
|
49
|
+
httpP95: Record<string, number>;
|
|
38
50
|
success_sql: Record<string, number>;
|
|
39
51
|
fail_sql: Record<string, number>;
|
|
40
52
|
};
|
|
@@ -43,8 +55,15 @@ export declare class MonitorService extends BaseService {
|
|
|
43
55
|
intervalMs: number;
|
|
44
56
|
lastCpuUsage: NodeJS.CpuUsage | null;
|
|
45
57
|
lastTimestamp: number | null;
|
|
46
|
-
|
|
58
|
+
private metricsDir;
|
|
59
|
+
private static MAX_WEEKS;
|
|
60
|
+
private static SCHEMA_VERSION;
|
|
61
|
+
private _persistReady;
|
|
62
|
+
private _flushedUpTo;
|
|
63
|
+
private _lastSeenWeekKey;
|
|
64
|
+
constructor(metricsDir?: string | null);
|
|
47
65
|
init(): Promise<void>;
|
|
66
|
+
finishService(): Promise<void>;
|
|
48
67
|
getEPStat(req: {
|
|
49
68
|
range: "week" | "day" | "hour";
|
|
50
69
|
}): Promise<IStat[]>;
|
|
@@ -57,7 +76,24 @@ export declare class MonitorService extends BaseService {
|
|
|
57
76
|
static normalizeUrl(url: string): string;
|
|
58
77
|
private bumpCappedKey;
|
|
59
78
|
private onHttpRequestRegistrate;
|
|
79
|
+
private _latByMinute;
|
|
80
|
+
private static LAT_CAP_PER_MIN;
|
|
81
|
+
private onHttpResultRegistrate;
|
|
82
|
+
private finalizeLatency;
|
|
60
83
|
private onSQLRequestRegistrate;
|
|
61
84
|
private getSlot;
|
|
85
|
+
private liveFile;
|
|
86
|
+
private archiveFile;
|
|
87
|
+
private weekKey;
|
|
88
|
+
private atomicWrite;
|
|
89
|
+
private normalizeSlot;
|
|
90
|
+
private readJsonl;
|
|
91
|
+
private persistTick;
|
|
92
|
+
private rotate;
|
|
93
|
+
private downsampleWeek;
|
|
94
|
+
private mergeSlots;
|
|
95
|
+
private hydrate;
|
|
96
|
+
private listPersistedWeeks;
|
|
97
|
+
private readPersistedWeek;
|
|
62
98
|
}
|
|
63
99
|
export {};
|
|
@@ -36,32 +36,68 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
36
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
37
|
};
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
-
exports.MonitorService = exports.REQ_STAT = exports.S_STAT_REGISTRATE_SQL_REQUEST = exports.S_STAT_REGISTRATE_HTTP_REQUEST = exports.S_STAT_REGISTRATE_SERVICE = exports.S_STAT_REGISTRATE_CUSTOM_EVENT = exports.S_STAT_REGISTRATE_REQUEST = void 0;
|
|
39
|
+
exports.MonitorService = exports.REQ_STAT_WEEK = exports.REQ_STAT_WEEKS = exports.REQ_STAT = exports.S_STAT_REGISTRATE_SQL_REQUEST = exports.S_STAT_REGISTRATE_HTTP_RESULT = exports.S_STAT_REGISTRATE_HTTP_REQUEST = exports.S_STAT_REGISTRATE_SERVICE = exports.S_STAT_REGISTRATE_CUSTOM_EVENT = exports.S_STAT_REGISTRATE_REQUEST = void 0;
|
|
40
40
|
const badmfck_signal_1 = __importStar(require("badmfck-signal"));
|
|
41
41
|
const BaseService_1 = require("./BaseService");
|
|
42
42
|
const YYYYMMDDHH_1 = require("./helper/YYYYMMDDHH");
|
|
43
|
+
const LogService_1 = require("./LogService");
|
|
43
44
|
const os_1 = __importDefault(require("os"));
|
|
45
|
+
const promises_1 = __importDefault(require("fs/promises"));
|
|
46
|
+
const path_1 = __importDefault(require("path"));
|
|
44
47
|
exports.S_STAT_REGISTRATE_REQUEST = new badmfck_signal_1.default();
|
|
45
48
|
exports.S_STAT_REGISTRATE_CUSTOM_EVENT = new badmfck_signal_1.default();
|
|
46
49
|
exports.S_STAT_REGISTRATE_SERVICE = new badmfck_signal_1.default();
|
|
47
50
|
exports.S_STAT_REGISTRATE_HTTP_REQUEST = new badmfck_signal_1.default();
|
|
51
|
+
exports.S_STAT_REGISTRATE_HTTP_RESULT = new badmfck_signal_1.default();
|
|
48
52
|
exports.S_STAT_REGISTRATE_SQL_REQUEST = new badmfck_signal_1.default();
|
|
49
53
|
exports.REQ_STAT = new badmfck_signal_1.Req(undefined, "REQ_STAT");
|
|
54
|
+
exports.REQ_STAT_WEEKS = new badmfck_signal_1.Req(undefined, "REQ_STAT_WEEKS");
|
|
55
|
+
exports.REQ_STAT_WEEK = new badmfck_signal_1.Req(undefined, "REQ_STAT_WEEK");
|
|
50
56
|
class MonitorService extends BaseService_1.BaseService {
|
|
51
57
|
slots = new Map();
|
|
52
58
|
intervalMs = 59_000;
|
|
53
59
|
lastCpuUsage = null;
|
|
54
60
|
lastTimestamp = null;
|
|
55
|
-
|
|
61
|
+
metricsDir;
|
|
62
|
+
static MAX_WEEKS = 10;
|
|
63
|
+
static SCHEMA_VERSION = 2;
|
|
64
|
+
_persistReady = false;
|
|
65
|
+
_flushedUpTo = 0;
|
|
66
|
+
_lastSeenWeekKey = null;
|
|
67
|
+
constructor(metricsDir) {
|
|
56
68
|
super("MonitorService");
|
|
69
|
+
this.metricsDir = metricsDir && metricsDir.length > 0 ? metricsDir : null;
|
|
57
70
|
setInterval(() => this.collectUsage(), this.intervalMs);
|
|
58
71
|
}
|
|
59
72
|
async init() {
|
|
60
73
|
super.init();
|
|
61
74
|
exports.REQ_STAT.listener = async (req) => this.getEPStat(req);
|
|
75
|
+
exports.REQ_STAT_WEEKS.listener = async () => this.listPersistedWeeks();
|
|
76
|
+
exports.REQ_STAT_WEEK.listener = async (req) => this.readPersistedWeek(req.weekKey);
|
|
62
77
|
exports.S_STAT_REGISTRATE_HTTP_REQUEST.subscribe((data) => this.onHttpRequestRegistrate(data));
|
|
78
|
+
exports.S_STAT_REGISTRATE_HTTP_RESULT.subscribe((data) => this.onHttpResultRegistrate(data));
|
|
63
79
|
exports.S_STAT_REGISTRATE_SQL_REQUEST.subscribe((data) => this.onSQLRequestRegistrate(data));
|
|
64
80
|
exports.S_STAT_REGISTRATE_REQUEST.subscribe((data) => this.onStatRegistrate(data));
|
|
81
|
+
if (this.metricsDir) {
|
|
82
|
+
try {
|
|
83
|
+
await promises_1.default.mkdir(this.metricsDir, { recursive: true });
|
|
84
|
+
}
|
|
85
|
+
catch (e) {
|
|
86
|
+
(0, LogService_1.logWarn)("metrics: cannot create metricsDir, disk persistence disabled: " + (e?.message ?? e));
|
|
87
|
+
this.metricsDir = null;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
if (this.metricsDir) {
|
|
91
|
+
await this.hydrate();
|
|
92
|
+
await this.rotate();
|
|
93
|
+
this._lastSeenWeekKey = this.weekKey(new Date());
|
|
94
|
+
this._persistReady = true;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
async finishService() {
|
|
98
|
+
await super.finishService();
|
|
99
|
+
if (this.metricsDir && this._persistReady)
|
|
100
|
+
await this.persistTick(true).catch(() => { });
|
|
65
101
|
}
|
|
66
102
|
async getEPStat(req) {
|
|
67
103
|
const allSlots = [];
|
|
@@ -91,33 +127,45 @@ class MonitorService extends BaseService_1.BaseService {
|
|
|
91
127
|
successEp: {},
|
|
92
128
|
failEp: {},
|
|
93
129
|
http: {},
|
|
130
|
+
http_ok: {},
|
|
131
|
+
http_fail: {},
|
|
132
|
+
httpP95: {},
|
|
94
133
|
success_sql: {},
|
|
95
134
|
fail_sql: {}
|
|
96
135
|
};
|
|
136
|
+
const sum = (dst, src) => {
|
|
137
|
+
if (!src)
|
|
138
|
+
return;
|
|
139
|
+
for (const [k, v] of Object.entries(src))
|
|
140
|
+
dst[k] = (dst[k] || 0) + v;
|
|
141
|
+
};
|
|
142
|
+
const maxInto = (dst, src) => {
|
|
143
|
+
if (!src)
|
|
144
|
+
return;
|
|
145
|
+
for (const [k, v] of Object.entries(src))
|
|
146
|
+
dst[k] = Math.max(dst[k] || 0, v);
|
|
147
|
+
};
|
|
97
148
|
chunk.forEach(s => {
|
|
98
149
|
agg.cpu += s.cpu;
|
|
99
150
|
agg.memory.rss += s.memory.rss;
|
|
100
151
|
agg.memory.heapUsed += s.memory.heapUsed;
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
agg.success_sql[query] = (agg.success_sql[query] || 0) + count;
|
|
112
|
-
}
|
|
113
|
-
for (const [query, count] of Object.entries(s.fail_sql)) {
|
|
114
|
-
agg.fail_sql[query] = (agg.fail_sql[query] || 0) + count;
|
|
115
|
-
}
|
|
152
|
+
agg.memory.heapTotal += s.memory.heapTotal;
|
|
153
|
+
agg.memory.external += s.memory.external;
|
|
154
|
+
sum(agg.successEp, s.successEp);
|
|
155
|
+
sum(agg.failEp, s.failEp);
|
|
156
|
+
sum(agg.http, s.http);
|
|
157
|
+
sum(agg.http_ok, s.http_ok);
|
|
158
|
+
sum(agg.http_fail, s.http_fail);
|
|
159
|
+
sum(agg.success_sql, s.success_sql);
|
|
160
|
+
sum(agg.fail_sql, s.fail_sql);
|
|
161
|
+
maxInto(agg.httpP95, s.httpP95);
|
|
116
162
|
});
|
|
117
163
|
const len = chunk.length;
|
|
118
164
|
agg.cpu = Number((agg.cpu / len).toFixed(2));
|
|
119
165
|
agg.memory.rss = Math.round(agg.memory.rss / len);
|
|
120
166
|
agg.memory.heapUsed = Math.round(agg.memory.heapUsed / len);
|
|
167
|
+
agg.memory.heapTotal = Math.round(agg.memory.heapTotal / len);
|
|
168
|
+
agg.memory.external = Math.round(agg.memory.external / len);
|
|
121
169
|
result.push({
|
|
122
170
|
timestamp: this.slotToTimestamp(agg.yyyymmddhhii),
|
|
123
171
|
slot: agg
|
|
@@ -144,6 +192,7 @@ class MonitorService extends BaseService_1.BaseService {
|
|
|
144
192
|
const usage = process.cpuUsage();
|
|
145
193
|
const now = Date.now();
|
|
146
194
|
const yyyymmddhhmm = parseInt((0, YYYYMMDDHH_1.YYYYMMDDHH)(new Date(now), "yyyymmddhhmm"));
|
|
195
|
+
this.finalizeLatency(yyyymmddhhmm);
|
|
147
196
|
const slot = this.getSlot(yyyymmddhhmm);
|
|
148
197
|
if (this.lastCpuUsage && this.lastTimestamp) {
|
|
149
198
|
const diff = usage.user - this.lastCpuUsage.user;
|
|
@@ -160,6 +209,8 @@ class MonitorService extends BaseService_1.BaseService {
|
|
|
160
209
|
heapTotal: Math.round(mem.heapTotal / 1024 / 1024),
|
|
161
210
|
external: Math.round(mem.external / 1024 / 1024),
|
|
162
211
|
};
|
|
212
|
+
if (this.metricsDir && this._persistReady)
|
|
213
|
+
await this.persistTick().catch(() => { });
|
|
163
214
|
}
|
|
164
215
|
static MAX_SLOT_KEYS = 100;
|
|
165
216
|
static normalizeQuery(query) {
|
|
@@ -200,6 +251,42 @@ class MonitorService extends BaseService_1.BaseService {
|
|
|
200
251
|
const slot = this.getSlot(yyyymmddhhmm);
|
|
201
252
|
this.bumpCappedKey(slot.http, MonitorService.normalizeUrl(data.url));
|
|
202
253
|
}
|
|
254
|
+
_latByMinute = new Map();
|
|
255
|
+
static LAT_CAP_PER_MIN = 3000;
|
|
256
|
+
onHttpResultRegistrate(data) {
|
|
257
|
+
const minuteKey = parseInt((0, YYYYMMDDHH_1.YYYYMMDDHH)(new Date(), "yyyymmddhhmm"));
|
|
258
|
+
const slot = this.getSlot(minuteKey);
|
|
259
|
+
this.bumpCappedKey(data.ok ? slot.http_ok : slot.http_fail, data.host);
|
|
260
|
+
let hosts = this._latByMinute.get(minuteKey);
|
|
261
|
+
if (!hosts) {
|
|
262
|
+
hosts = new Map();
|
|
263
|
+
this._latByMinute.set(minuteKey, hosts);
|
|
264
|
+
}
|
|
265
|
+
let arr = hosts.get(data.host);
|
|
266
|
+
if (!arr) {
|
|
267
|
+
arr = [];
|
|
268
|
+
hosts.set(data.host, arr);
|
|
269
|
+
}
|
|
270
|
+
if (arr.length < MonitorService.LAT_CAP_PER_MIN)
|
|
271
|
+
arr.push(data.ms);
|
|
272
|
+
}
|
|
273
|
+
finalizeLatency(currentMinute, includeCurrent = false) {
|
|
274
|
+
for (const [minuteKey, hosts] of this._latByMinute) {
|
|
275
|
+
if (includeCurrent ? minuteKey > currentMinute : minuteKey >= currentMinute)
|
|
276
|
+
continue;
|
|
277
|
+
const slot = this.slots.get(Math.floor(minuteKey / 100))?.get(minuteKey);
|
|
278
|
+
if (slot) {
|
|
279
|
+
for (const [host, samples] of hosts) {
|
|
280
|
+
if (!samples.length)
|
|
281
|
+
continue;
|
|
282
|
+
samples.sort((a, b) => a - b);
|
|
283
|
+
const idx = Math.min(samples.length - 1, Math.max(0, Math.ceil(0.95 * samples.length) - 1));
|
|
284
|
+
slot.httpP95[host] = samples[idx];
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
this._latByMinute.delete(minuteKey);
|
|
288
|
+
}
|
|
289
|
+
}
|
|
203
290
|
onSQLRequestRegistrate(data) {
|
|
204
291
|
const date = new Date();
|
|
205
292
|
const yyyymmddhhmm = parseInt((0, YYYYMMDDHH_1.YYYYMMDDHH)(date, "yyyymmddhhmm"));
|
|
@@ -233,6 +320,9 @@ class MonitorService extends BaseService_1.BaseService {
|
|
|
233
320
|
successEp: {},
|
|
234
321
|
failEp: {},
|
|
235
322
|
http: {},
|
|
323
|
+
http_ok: {},
|
|
324
|
+
http_fail: {},
|
|
325
|
+
httpP95: {},
|
|
236
326
|
success_sql: {},
|
|
237
327
|
fail_sql: {}
|
|
238
328
|
};
|
|
@@ -240,5 +330,285 @@ class MonitorService extends BaseService_1.BaseService {
|
|
|
240
330
|
}
|
|
241
331
|
return slot;
|
|
242
332
|
}
|
|
333
|
+
liveFile(weekKey) {
|
|
334
|
+
return path_1.default.join(this.metricsDir, `metrics_${weekKey}.jsonl`);
|
|
335
|
+
}
|
|
336
|
+
archiveFile(weekKey) {
|
|
337
|
+
return path_1.default.join(this.metricsDir, `metrics_${weekKey}.json`);
|
|
338
|
+
}
|
|
339
|
+
weekKey(date) {
|
|
340
|
+
const d = new Date(Date.UTC(date.getFullYear(), date.getMonth(), date.getDate()));
|
|
341
|
+
const dayNum = (d.getUTCDay() + 6) % 7;
|
|
342
|
+
d.setUTCDate(d.getUTCDate() - dayNum + 3);
|
|
343
|
+
const firstThursday = new Date(Date.UTC(d.getUTCFullYear(), 0, 4));
|
|
344
|
+
const firstDayNum = (firstThursday.getUTCDay() + 6) % 7;
|
|
345
|
+
firstThursday.setUTCDate(firstThursday.getUTCDate() - firstDayNum + 3);
|
|
346
|
+
const week = 1 + Math.round((d.getTime() - firstThursday.getTime()) / (7 * 24 * 3600 * 1000));
|
|
347
|
+
return `${d.getUTCFullYear()}-W${String(week).padStart(2, "0")}`;
|
|
348
|
+
}
|
|
349
|
+
async atomicWrite(file, content) {
|
|
350
|
+
const tmp = file + ".tmp";
|
|
351
|
+
await promises_1.default.writeFile(tmp, content, "utf-8");
|
|
352
|
+
await promises_1.default.rename(tmp, file);
|
|
353
|
+
}
|
|
354
|
+
normalizeSlot(raw) {
|
|
355
|
+
if (!raw || typeof raw !== "object")
|
|
356
|
+
return null;
|
|
357
|
+
const key = Number(raw.yyyymmddhhii);
|
|
358
|
+
if (!Number.isFinite(key))
|
|
359
|
+
return null;
|
|
360
|
+
const num = (v) => (typeof v === "number" && Number.isFinite(v) ? v : 0);
|
|
361
|
+
const rec = (v) => {
|
|
362
|
+
const out = {};
|
|
363
|
+
if (v && typeof v === "object" && !Array.isArray(v))
|
|
364
|
+
for (const [k, val] of Object.entries(v))
|
|
365
|
+
if (typeof val === "number" && Number.isFinite(val))
|
|
366
|
+
out[k] = val;
|
|
367
|
+
return out;
|
|
368
|
+
};
|
|
369
|
+
const m = raw.memory && typeof raw.memory === "object" ? raw.memory : {};
|
|
370
|
+
return {
|
|
371
|
+
yyyymmddhhii: key,
|
|
372
|
+
cpu: num(raw.cpu),
|
|
373
|
+
memory: { rss: num(m.rss), heapUsed: num(m.heapUsed), heapTotal: num(m.heapTotal), external: num(m.external) },
|
|
374
|
+
successEp: rec(raw.successEp), failEp: rec(raw.failEp),
|
|
375
|
+
http: rec(raw.http), http_ok: rec(raw.http_ok), http_fail: rec(raw.http_fail), httpP95: rec(raw.httpP95),
|
|
376
|
+
success_sql: rec(raw.success_sql), fail_sql: rec(raw.fail_sql),
|
|
377
|
+
};
|
|
378
|
+
}
|
|
379
|
+
async readJsonl(file) {
|
|
380
|
+
const txt = await promises_1.default.readFile(file, "utf-8");
|
|
381
|
+
const out = [];
|
|
382
|
+
for (const line of txt.split("\n")) {
|
|
383
|
+
const t = line.trim();
|
|
384
|
+
if (!t)
|
|
385
|
+
continue;
|
|
386
|
+
try {
|
|
387
|
+
const s = this.normalizeSlot(JSON.parse(t));
|
|
388
|
+
if (s)
|
|
389
|
+
out.push(s);
|
|
390
|
+
}
|
|
391
|
+
catch { }
|
|
392
|
+
}
|
|
393
|
+
return out;
|
|
394
|
+
}
|
|
395
|
+
async persistTick(includeCurrentMinute = false) {
|
|
396
|
+
if (!this.metricsDir)
|
|
397
|
+
return;
|
|
398
|
+
const nowKey = parseInt((0, YYYYMMDDHH_1.YYYYMMDDHH)(new Date(), "yyyymmddhhmm"));
|
|
399
|
+
this.finalizeLatency(nowKey, includeCurrentMinute);
|
|
400
|
+
const pending = [];
|
|
401
|
+
for (const dayMap of this.slots.values()) {
|
|
402
|
+
for (const slot of dayMap.values()) {
|
|
403
|
+
const k = slot.yyyymmddhhii;
|
|
404
|
+
if (k <= this._flushedUpTo)
|
|
405
|
+
continue;
|
|
406
|
+
if (!includeCurrentMinute && k >= nowKey)
|
|
407
|
+
continue;
|
|
408
|
+
pending.push(slot);
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
pending.sort((a, b) => a.yyyymmddhhii - b.yyyymmddhhii);
|
|
412
|
+
if (pending.length) {
|
|
413
|
+
try {
|
|
414
|
+
const byWeek = new Map();
|
|
415
|
+
for (const s of pending) {
|
|
416
|
+
const wk = this.weekKey(new Date(this.slotToTimestamp(s.yyyymmddhhii)));
|
|
417
|
+
byWeek.set(wk, (byWeek.get(wk) ?? "") + JSON.stringify(s) + "\n");
|
|
418
|
+
}
|
|
419
|
+
for (const [wk, text] of byWeek)
|
|
420
|
+
await promises_1.default.appendFile(this.liveFile(wk), text, "utf-8");
|
|
421
|
+
this._flushedUpTo = pending[pending.length - 1].yyyymmddhhii;
|
|
422
|
+
}
|
|
423
|
+
catch (e) {
|
|
424
|
+
(0, LogService_1.logWarn)("metrics: persist append failed: " + (e?.message ?? e));
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
const curWeek = this.weekKey(new Date());
|
|
428
|
+
if (this._lastSeenWeekKey && this._lastSeenWeekKey !== curWeek)
|
|
429
|
+
await this.rotate().catch(() => { });
|
|
430
|
+
this._lastSeenWeekKey = curWeek;
|
|
431
|
+
}
|
|
432
|
+
async rotate() {
|
|
433
|
+
if (!this.metricsDir)
|
|
434
|
+
return;
|
|
435
|
+
try {
|
|
436
|
+
const curWeek = this.weekKey(new Date());
|
|
437
|
+
const files = await promises_1.default.readdir(this.metricsDir);
|
|
438
|
+
for (const f of files) {
|
|
439
|
+
const m = f.match(/^metrics_(\d{4}-W\d{2})\.jsonl$/);
|
|
440
|
+
if (m && m[1] !== curWeek)
|
|
441
|
+
await this.downsampleWeek(m[1]);
|
|
442
|
+
}
|
|
443
|
+
const after = await promises_1.default.readdir(this.metricsDir);
|
|
444
|
+
const byWeek = new Map();
|
|
445
|
+
for (const f of after) {
|
|
446
|
+
const m = f.match(/^metrics_(\d{4}-W\d{2})\.(jsonl|json)$/);
|
|
447
|
+
if (!m)
|
|
448
|
+
continue;
|
|
449
|
+
const arr = byWeek.get(m[1]) ?? [];
|
|
450
|
+
arr.push(f);
|
|
451
|
+
byWeek.set(m[1], arr);
|
|
452
|
+
}
|
|
453
|
+
const weeks = Array.from(byWeek.keys()).sort();
|
|
454
|
+
const excess = weeks.slice(0, Math.max(0, weeks.length - MonitorService.MAX_WEEKS));
|
|
455
|
+
for (const wk of excess)
|
|
456
|
+
for (const f of byWeek.get(wk))
|
|
457
|
+
await promises_1.default.unlink(path_1.default.join(this.metricsDir, f)).catch(() => { });
|
|
458
|
+
}
|
|
459
|
+
catch (e) {
|
|
460
|
+
(0, LogService_1.logWarn)("metrics: rotate failed: " + (e?.message ?? e));
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
async downsampleWeek(weekKey) {
|
|
464
|
+
const live = this.liveFile(weekKey);
|
|
465
|
+
let slots;
|
|
466
|
+
try {
|
|
467
|
+
slots = await this.readJsonl(live);
|
|
468
|
+
}
|
|
469
|
+
catch {
|
|
470
|
+
return;
|
|
471
|
+
}
|
|
472
|
+
if (!slots.length) {
|
|
473
|
+
await promises_1.default.unlink(live).catch(() => { });
|
|
474
|
+
return;
|
|
475
|
+
}
|
|
476
|
+
const byHour = new Map();
|
|
477
|
+
for (const s of slots) {
|
|
478
|
+
const hourKey = Math.floor(s.yyyymmddhhii / 100) * 100;
|
|
479
|
+
const arr = byHour.get(hourKey) ?? [];
|
|
480
|
+
arr.push(s);
|
|
481
|
+
byHour.set(hourKey, arr);
|
|
482
|
+
}
|
|
483
|
+
const hourly = Array.from(byHour.entries())
|
|
484
|
+
.sort((a, b) => a[0] - b[0])
|
|
485
|
+
.map(([hourKey, arr]) => this.mergeSlots(hourKey, arr));
|
|
486
|
+
await this.atomicWrite(this.archiveFile(weekKey), JSON.stringify({ version: MonitorService.SCHEMA_VERSION, weekKey, resolution: "hour", slots: hourly }));
|
|
487
|
+
await promises_1.default.unlink(live).catch(() => { });
|
|
488
|
+
}
|
|
489
|
+
mergeSlots(key, arr) {
|
|
490
|
+
const out = {
|
|
491
|
+
yyyymmddhhii: key, cpu: 0,
|
|
492
|
+
memory: { rss: 0, heapUsed: 0, heapTotal: 0, external: 0 },
|
|
493
|
+
successEp: {}, failEp: {}, http: {}, http_ok: {}, http_fail: {}, httpP95: {},
|
|
494
|
+
success_sql: {}, fail_sql: {},
|
|
495
|
+
};
|
|
496
|
+
const add = (dst, src) => {
|
|
497
|
+
if (!src)
|
|
498
|
+
return;
|
|
499
|
+
for (const [k, v] of Object.entries(src))
|
|
500
|
+
dst[k] = (dst[k] || 0) + v;
|
|
501
|
+
};
|
|
502
|
+
const maxInto = (dst, src) => {
|
|
503
|
+
if (!src)
|
|
504
|
+
return;
|
|
505
|
+
for (const [k, v] of Object.entries(src))
|
|
506
|
+
dst[k] = Math.max(dst[k] || 0, v);
|
|
507
|
+
};
|
|
508
|
+
for (const s of arr) {
|
|
509
|
+
out.cpu += s.cpu;
|
|
510
|
+
out.memory.rss += s.memory.rss;
|
|
511
|
+
out.memory.heapUsed += s.memory.heapUsed;
|
|
512
|
+
out.memory.heapTotal += s.memory.heapTotal;
|
|
513
|
+
out.memory.external += s.memory.external;
|
|
514
|
+
add(out.successEp, s.successEp);
|
|
515
|
+
add(out.failEp, s.failEp);
|
|
516
|
+
add(out.http, s.http);
|
|
517
|
+
add(out.http_ok, s.http_ok);
|
|
518
|
+
add(out.http_fail, s.http_fail);
|
|
519
|
+
add(out.success_sql, s.success_sql);
|
|
520
|
+
add(out.fail_sql, s.fail_sql);
|
|
521
|
+
maxInto(out.httpP95, s.httpP95);
|
|
522
|
+
}
|
|
523
|
+
const n = arr.length || 1;
|
|
524
|
+
out.cpu = Number((out.cpu / n).toFixed(2));
|
|
525
|
+
out.memory.rss = Math.round(out.memory.rss / n);
|
|
526
|
+
out.memory.heapUsed = Math.round(out.memory.heapUsed / n);
|
|
527
|
+
out.memory.heapTotal = Math.round(out.memory.heapTotal / n);
|
|
528
|
+
out.memory.external = Math.round(out.memory.external / n);
|
|
529
|
+
return out;
|
|
530
|
+
}
|
|
531
|
+
async hydrate() {
|
|
532
|
+
if (!this.metricsDir)
|
|
533
|
+
return;
|
|
534
|
+
try {
|
|
535
|
+
const cutoff = parseInt((0, YYYYMMDDHH_1.YYYYMMDDHH)(new Date(Date.now() - 7 * 24 * 3600 * 1000), "yyyymmddhhmm"));
|
|
536
|
+
const weeks = new Set([
|
|
537
|
+
this.weekKey(new Date()),
|
|
538
|
+
this.weekKey(new Date(Date.now() - 7 * 24 * 3600 * 1000)),
|
|
539
|
+
]);
|
|
540
|
+
let maxKey = 0;
|
|
541
|
+
for (const wk of weeks) {
|
|
542
|
+
let slots;
|
|
543
|
+
try {
|
|
544
|
+
slots = await this.readJsonl(this.liveFile(wk));
|
|
545
|
+
}
|
|
546
|
+
catch {
|
|
547
|
+
continue;
|
|
548
|
+
}
|
|
549
|
+
for (const s of slots) {
|
|
550
|
+
if (s.yyyymmddhhii < cutoff)
|
|
551
|
+
continue;
|
|
552
|
+
const dayKey = Math.floor(s.yyyymmddhhii / 100);
|
|
553
|
+
let dayMap = this.slots.get(dayKey);
|
|
554
|
+
if (!dayMap) {
|
|
555
|
+
dayMap = new Map();
|
|
556
|
+
this.slots.set(dayKey, dayMap);
|
|
557
|
+
}
|
|
558
|
+
dayMap.set(s.yyyymmddhhii, s);
|
|
559
|
+
if (s.yyyymmddhhii > maxKey)
|
|
560
|
+
maxKey = s.yyyymmddhhii;
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
this._flushedUpTo = maxKey;
|
|
564
|
+
while (this.slots.size > 7) {
|
|
565
|
+
const oldest = Array.from(this.slots.keys()).sort((a, b) => a - b)[0];
|
|
566
|
+
this.slots.delete(oldest);
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
catch (e) {
|
|
570
|
+
(0, LogService_1.logWarn)("metrics: hydrate failed: " + (e?.message ?? e));
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
async listPersistedWeeks() {
|
|
574
|
+
if (!this.metricsDir)
|
|
575
|
+
return [];
|
|
576
|
+
try {
|
|
577
|
+
const files = await promises_1.default.readdir(this.metricsDir);
|
|
578
|
+
const weeks = new Set();
|
|
579
|
+
for (const f of files) {
|
|
580
|
+
const m = f.match(/^metrics_(\d{4}-W\d{2})\.(jsonl|json)$/);
|
|
581
|
+
if (m)
|
|
582
|
+
weeks.add(m[1]);
|
|
583
|
+
}
|
|
584
|
+
return Array.from(weeks).sort();
|
|
585
|
+
}
|
|
586
|
+
catch {
|
|
587
|
+
return [];
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
async readPersistedWeek(weekKey) {
|
|
591
|
+
if (!this.metricsDir || !/^\d{4}-W\d{2}$/.test(weekKey))
|
|
592
|
+
return [];
|
|
593
|
+
let slots = [];
|
|
594
|
+
try {
|
|
595
|
+
try {
|
|
596
|
+
slots = await this.readJsonl(this.liveFile(weekKey));
|
|
597
|
+
}
|
|
598
|
+
catch {
|
|
599
|
+
const txt = await promises_1.default.readFile(this.archiveFile(weekKey), "utf-8");
|
|
600
|
+
const rawSlots = JSON.parse(txt)?.slots;
|
|
601
|
+
slots = Array.isArray(rawSlots)
|
|
602
|
+
? rawSlots.map((r) => this.normalizeSlot(r)).filter((s) => s !== null)
|
|
603
|
+
: [];
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
catch {
|
|
607
|
+
return [];
|
|
608
|
+
}
|
|
609
|
+
return slots
|
|
610
|
+
.sort((a, b) => a.yyyymmddhhii - b.yyyymmddhhii)
|
|
611
|
+
.map(s => ({ timestamp: this.slotToTimestamp(s.yyyymmddhhii), slot: s }));
|
|
612
|
+
}
|
|
243
613
|
}
|
|
244
614
|
exports.MonitorService = MonitorService;
|