@open-mercato/shared 0.6.6-develop.6384.1.f06fc0b42c → 0.6.6-develop.6385.1.9a81faa5f0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/AGENTS.md +1 -0
- package/dist/lib/modules/resource-usage.js +904 -0
- package/dist/lib/modules/resource-usage.js.map +7 -0
- package/dist/lib/number.js +14 -0
- package/dist/lib/number.js.map +7 -0
- package/dist/lib/search/config.js +2 -5
- package/dist/lib/search/config.js.map +2 -2
- package/dist/lib/version.js +1 -1
- package/dist/lib/version.js.map +1 -1
- package/dist/modules/registry.js.map +2 -2
- package/package.json +2 -2
- package/src/lib/__tests__/number.test.ts +33 -0
- package/src/lib/modules/__tests__/resource-usage.test.ts +351 -0
- package/src/lib/modules/resource-usage.ts +1213 -0
- package/src/lib/number.ts +14 -0
- package/src/lib/search/config.ts +2 -5
- package/src/modules/registry.ts +2 -0
|
@@ -0,0 +1,904 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { parseBooleanWithDefault } from "../boolean.js";
|
|
4
|
+
import { parseNumberWithDefault } from "../number.js";
|
|
5
|
+
const GLOBAL_KEY = "__openMercatoModuleResourceUsage__";
|
|
6
|
+
const NS_PER_MS = 1e6;
|
|
7
|
+
const MICROS_PER_MS = 1e3;
|
|
8
|
+
const DEFAULT_RECENT_SAMPLE_LIMIT = 128;
|
|
9
|
+
const DEFAULT_TOP_OPERATIONS_LIMIT = 5;
|
|
10
|
+
const TIME_BUCKET_RETENTION_MS = 24 * 60 * 60 * 1e3;
|
|
11
|
+
const SNAPSHOT_THROTTLE_MS = 5e3;
|
|
12
|
+
const SNAPSHOT_MAX_AGE_MS = 24 * 60 * 60 * 1e3;
|
|
13
|
+
const MINUTE_MS = 60 * 1e3;
|
|
14
|
+
const MODULE_RESOURCE_USAGE_BUCKET_INTERVAL_MS = 5 * MINUTE_MS;
|
|
15
|
+
const MODULE_RESOURCE_USAGE_STARTUP_STAGE_MS = 5 * MINUTE_MS;
|
|
16
|
+
function getState() {
|
|
17
|
+
const existing = globalThis[GLOBAL_KEY];
|
|
18
|
+
if (existing && typeof existing === "object" && existing.entries instanceof Map) {
|
|
19
|
+
const state2 = existing;
|
|
20
|
+
if (!(state2.buckets instanceof Map)) state2.buckets = /* @__PURE__ */ new Map();
|
|
21
|
+
if (!(state2.activeCalls instanceof Set)) state2.activeCalls = /* @__PURE__ */ new Set();
|
|
22
|
+
const currentIntervalMs = moduleResourceUsageBucketIntervalMs();
|
|
23
|
+
if (state2.migratedIntervalMs !== currentIntervalMs) {
|
|
24
|
+
migrateMutableTimeBuckets(state2, currentIntervalMs);
|
|
25
|
+
state2.migratedIntervalMs = currentIntervalMs;
|
|
26
|
+
}
|
|
27
|
+
registerSnapshotShutdownHook(state2);
|
|
28
|
+
return state2;
|
|
29
|
+
}
|
|
30
|
+
const state = {
|
|
31
|
+
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
32
|
+
entries: /* @__PURE__ */ new Map(),
|
|
33
|
+
buckets: /* @__PURE__ */ new Map(),
|
|
34
|
+
activeCalls: /* @__PURE__ */ new Set(),
|
|
35
|
+
lastSnapshotAt: 0,
|
|
36
|
+
shutdownHookRegistered: false,
|
|
37
|
+
migratedIntervalMs: moduleResourceUsageBucketIntervalMs()
|
|
38
|
+
};
|
|
39
|
+
globalThis[GLOBAL_KEY] = state;
|
|
40
|
+
registerSnapshotShutdownHook(state);
|
|
41
|
+
return state;
|
|
42
|
+
}
|
|
43
|
+
function migrateMutableTimeBuckets(state, intervalMs) {
|
|
44
|
+
const migratedBuckets = /* @__PURE__ */ new Map();
|
|
45
|
+
for (const bucket of state.buckets.values()) {
|
|
46
|
+
if (!(bucket.modules instanceof Map)) continue;
|
|
47
|
+
const bucketStartMs = Number.isFinite(bucket.bucketStartMs) ? Math.floor(bucket.bucketStartMs / intervalMs) * intervalMs : null;
|
|
48
|
+
if (bucketStartMs === null) continue;
|
|
49
|
+
const key = String(bucketStartMs);
|
|
50
|
+
const targetBucket = migratedBuckets.get(key) ?? {
|
|
51
|
+
bucketStartMs,
|
|
52
|
+
bucketEndMs: bucketStartMs + intervalMs,
|
|
53
|
+
modules: /* @__PURE__ */ new Map()
|
|
54
|
+
};
|
|
55
|
+
mergeMutableTimeBucket(targetBucket, bucket);
|
|
56
|
+
migratedBuckets.set(key, targetBucket);
|
|
57
|
+
}
|
|
58
|
+
state.buckets = migratedBuckets;
|
|
59
|
+
}
|
|
60
|
+
function trimRecentDurations(values) {
|
|
61
|
+
if (values.length <= DEFAULT_RECENT_SAMPLE_LIMIT) return values;
|
|
62
|
+
return values.slice(values.length - DEFAULT_RECENT_SAMPLE_LIMIT);
|
|
63
|
+
}
|
|
64
|
+
function mergeMutableEntry(target, source) {
|
|
65
|
+
target.calls += source.calls;
|
|
66
|
+
target.errors += source.errors;
|
|
67
|
+
target.concurrentCalls += source.concurrentCalls;
|
|
68
|
+
target.totalDurationMs += source.totalDurationMs;
|
|
69
|
+
target.maxDurationMs = Math.max(target.maxDurationMs, source.maxDurationMs);
|
|
70
|
+
target.totalCpuUserMs += source.totalCpuUserMs;
|
|
71
|
+
target.totalCpuSystemMs += source.totalCpuSystemMs;
|
|
72
|
+
target.maxCpuMs = Math.max(target.maxCpuMs, source.maxCpuMs);
|
|
73
|
+
target.totalHeapDeltaBytes += source.totalHeapDeltaBytes;
|
|
74
|
+
target.positiveHeapDeltaBytes += source.positiveHeapDeltaBytes;
|
|
75
|
+
target.maxHeapDeltaBytes = Math.max(target.maxHeapDeltaBytes, source.maxHeapDeltaBytes);
|
|
76
|
+
target.totalRssDeltaBytes += source.totalRssDeltaBytes;
|
|
77
|
+
target.positiveRssDeltaBytes += source.positiveRssDeltaBytes;
|
|
78
|
+
target.maxRssDeltaBytes = Math.max(target.maxRssDeltaBytes, source.maxRssDeltaBytes);
|
|
79
|
+
target.firstSeenAt = target.firstSeenAt < source.firstSeenAt ? target.firstSeenAt : source.firstSeenAt;
|
|
80
|
+
target.lastSeenAt = target.lastSeenAt > source.lastSeenAt ? target.lastSeenAt : source.lastSeenAt;
|
|
81
|
+
target.recentDurationsMs = trimRecentDurations([
|
|
82
|
+
...target.recentDurationsMs,
|
|
83
|
+
...Array.isArray(source.recentDurationsMs) ? source.recentDurationsMs : []
|
|
84
|
+
]);
|
|
85
|
+
}
|
|
86
|
+
function cloneMutableEntry(entry) {
|
|
87
|
+
return {
|
|
88
|
+
...entry,
|
|
89
|
+
recentDurationsMs: trimRecentDurations(Array.isArray(entry.recentDurationsMs) ? [...entry.recentDurationsMs] : [])
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
function mergeMutableTimeBucket(targetBucket, sourceBucket) {
|
|
93
|
+
for (const sourceModule of sourceBucket.modules.values()) {
|
|
94
|
+
if (!Array.isArray(sourceModule.recentDurationsMs)) sourceModule.recentDurationsMs = [];
|
|
95
|
+
if (!(sourceModule.operations instanceof Map)) sourceModule.operations = /* @__PURE__ */ new Map();
|
|
96
|
+
const targetModule = targetBucket.modules.get(sourceModule.moduleId);
|
|
97
|
+
if (!targetModule) {
|
|
98
|
+
targetBucket.modules.set(sourceModule.moduleId, {
|
|
99
|
+
moduleId: sourceModule.moduleId,
|
|
100
|
+
calls: sourceModule.calls,
|
|
101
|
+
errors: sourceModule.errors,
|
|
102
|
+
totalDurationMs: sourceModule.totalDurationMs,
|
|
103
|
+
totalCpuMs: sourceModule.totalCpuMs,
|
|
104
|
+
positiveHeapDeltaBytes: sourceModule.positiveHeapDeltaBytes,
|
|
105
|
+
positiveRssDeltaBytes: sourceModule.positiveRssDeltaBytes,
|
|
106
|
+
recentDurationsMs: trimRecentDurations([...sourceModule.recentDurationsMs]),
|
|
107
|
+
operations: new Map(Array.from(sourceModule.operations.entries()).map(([key, entry]) => [key, cloneMutableEntry(entry)]))
|
|
108
|
+
});
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
targetModule.calls += sourceModule.calls;
|
|
112
|
+
targetModule.errors += sourceModule.errors;
|
|
113
|
+
targetModule.totalDurationMs += sourceModule.totalDurationMs;
|
|
114
|
+
targetModule.totalCpuMs += sourceModule.totalCpuMs;
|
|
115
|
+
targetModule.positiveHeapDeltaBytes += sourceModule.positiveHeapDeltaBytes;
|
|
116
|
+
targetModule.positiveRssDeltaBytes += sourceModule.positiveRssDeltaBytes;
|
|
117
|
+
targetModule.recentDurationsMs = trimRecentDurations([
|
|
118
|
+
...targetModule.recentDurationsMs,
|
|
119
|
+
...sourceModule.recentDurationsMs
|
|
120
|
+
]);
|
|
121
|
+
for (const [operationKey, sourceOperation] of sourceModule.operations) {
|
|
122
|
+
const targetOperation = targetModule.operations.get(operationKey);
|
|
123
|
+
if (!targetOperation) {
|
|
124
|
+
targetModule.operations.set(operationKey, cloneMutableEntry(sourceOperation));
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
mergeMutableEntry(targetOperation, sourceOperation);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
function resetModuleResourceUsage() {
|
|
132
|
+
;
|
|
133
|
+
globalThis[GLOBAL_KEY] = {
|
|
134
|
+
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
135
|
+
entries: /* @__PURE__ */ new Map(),
|
|
136
|
+
buckets: /* @__PURE__ */ new Map(),
|
|
137
|
+
activeCalls: /* @__PURE__ */ new Set(),
|
|
138
|
+
lastSnapshotAt: 0,
|
|
139
|
+
shutdownHookRegistered: false,
|
|
140
|
+
migratedIntervalMs: moduleResourceUsageBucketIntervalMs()
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
function clearModuleResourceUsageData() {
|
|
144
|
+
resetModuleResourceUsage();
|
|
145
|
+
try {
|
|
146
|
+
fs.rmSync(getSnapshotDir(), { recursive: true, force: true });
|
|
147
|
+
} catch {
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
function isModuleResourceUsageEnabled() {
|
|
151
|
+
return parseBooleanWithDefault(process.env.OM_MODULE_RESOURCE_USAGE, true);
|
|
152
|
+
}
|
|
153
|
+
function isSnapshotEnabled() {
|
|
154
|
+
return parseBooleanWithDefault(process.env.OM_MODULE_RESOURCE_USAGE_SNAPSHOT, process.env.NODE_ENV !== "test");
|
|
155
|
+
}
|
|
156
|
+
function inferModuleIdFromResourceId(resourceId) {
|
|
157
|
+
if (!resourceId) return null;
|
|
158
|
+
const trimmed = resourceId.trim();
|
|
159
|
+
if (!trimmed) return null;
|
|
160
|
+
const colon = trimmed.indexOf(":");
|
|
161
|
+
if (colon > 0) return trimmed.slice(0, colon);
|
|
162
|
+
const dot = trimmed.indexOf(".");
|
|
163
|
+
if (dot > 0) return trimmed.slice(0, dot);
|
|
164
|
+
return null;
|
|
165
|
+
}
|
|
166
|
+
function normalizeModuleId(input) {
|
|
167
|
+
const explicit = input.moduleId?.trim();
|
|
168
|
+
if (explicit) return explicit;
|
|
169
|
+
return inferModuleIdFromResourceId(input.resourceId) ?? inferModuleIdFromResourceId(input.operation);
|
|
170
|
+
}
|
|
171
|
+
function nowNs() {
|
|
172
|
+
return process.hrtime.bigint();
|
|
173
|
+
}
|
|
174
|
+
function durationMs(startNs) {
|
|
175
|
+
const elapsed = nowNs() - startNs;
|
|
176
|
+
if (elapsed <= BigInt(0)) return 0;
|
|
177
|
+
return Number(elapsed) / NS_PER_MS;
|
|
178
|
+
}
|
|
179
|
+
function round(value, precision = 100) {
|
|
180
|
+
if (!Number.isFinite(value)) return 0;
|
|
181
|
+
return Math.round(value * precision) / precision;
|
|
182
|
+
}
|
|
183
|
+
function safeMemoryUsage() {
|
|
184
|
+
try {
|
|
185
|
+
return process.memoryUsage();
|
|
186
|
+
} catch {
|
|
187
|
+
return null;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
function entryKey(input) {
|
|
191
|
+
return `${input.moduleId}\0${input.surface}\0${input.operation}`;
|
|
192
|
+
}
|
|
193
|
+
function getOrCreateEntry(input, timestamp) {
|
|
194
|
+
const state = getState();
|
|
195
|
+
const key = entryKey(input);
|
|
196
|
+
const existing = state.entries.get(key);
|
|
197
|
+
if (existing) return existing;
|
|
198
|
+
const created = {
|
|
199
|
+
moduleId: input.moduleId,
|
|
200
|
+
surface: input.surface,
|
|
201
|
+
operation: input.operation,
|
|
202
|
+
resourceId: input.resourceId ?? null,
|
|
203
|
+
calls: 0,
|
|
204
|
+
errors: 0,
|
|
205
|
+
concurrentCalls: 0,
|
|
206
|
+
totalDurationMs: 0,
|
|
207
|
+
maxDurationMs: 0,
|
|
208
|
+
totalCpuUserMs: 0,
|
|
209
|
+
totalCpuSystemMs: 0,
|
|
210
|
+
maxCpuMs: 0,
|
|
211
|
+
totalHeapDeltaBytes: 0,
|
|
212
|
+
positiveHeapDeltaBytes: 0,
|
|
213
|
+
maxHeapDeltaBytes: 0,
|
|
214
|
+
totalRssDeltaBytes: 0,
|
|
215
|
+
positiveRssDeltaBytes: 0,
|
|
216
|
+
maxRssDeltaBytes: 0,
|
|
217
|
+
firstSeenAt: timestamp,
|
|
218
|
+
lastSeenAt: timestamp,
|
|
219
|
+
recentDurationsMs: []
|
|
220
|
+
};
|
|
221
|
+
state.entries.set(key, created);
|
|
222
|
+
return created;
|
|
223
|
+
}
|
|
224
|
+
function getOrCreateTimeBucket(state, timestamp) {
|
|
225
|
+
const timestampMs = Date.parse(timestamp);
|
|
226
|
+
if (!Number.isFinite(timestampMs)) return null;
|
|
227
|
+
const intervalMs = moduleResourceUsageBucketIntervalMs();
|
|
228
|
+
const bucketStartMs = Math.floor(timestampMs / intervalMs) * intervalMs;
|
|
229
|
+
const key = String(bucketStartMs);
|
|
230
|
+
const existing = state.buckets.get(key);
|
|
231
|
+
if (existing) return existing;
|
|
232
|
+
const created = {
|
|
233
|
+
bucketStartMs,
|
|
234
|
+
bucketEndMs: bucketStartMs + intervalMs,
|
|
235
|
+
modules: /* @__PURE__ */ new Map()
|
|
236
|
+
};
|
|
237
|
+
state.buckets.set(key, created);
|
|
238
|
+
pruneTimeBuckets(state);
|
|
239
|
+
return created;
|
|
240
|
+
}
|
|
241
|
+
function getOrCreateTimeBucketModule(bucket, moduleId) {
|
|
242
|
+
const existing = bucket.modules.get(moduleId);
|
|
243
|
+
if (existing) {
|
|
244
|
+
if (!Array.isArray(existing.recentDurationsMs)) existing.recentDurationsMs = [];
|
|
245
|
+
if (!(existing.operations instanceof Map)) existing.operations = /* @__PURE__ */ new Map();
|
|
246
|
+
return existing;
|
|
247
|
+
}
|
|
248
|
+
const created = {
|
|
249
|
+
moduleId,
|
|
250
|
+
calls: 0,
|
|
251
|
+
errors: 0,
|
|
252
|
+
totalDurationMs: 0,
|
|
253
|
+
totalCpuMs: 0,
|
|
254
|
+
positiveHeapDeltaBytes: 0,
|
|
255
|
+
positiveRssDeltaBytes: 0,
|
|
256
|
+
recentDurationsMs: [],
|
|
257
|
+
operations: /* @__PURE__ */ new Map()
|
|
258
|
+
};
|
|
259
|
+
bucket.modules.set(moduleId, created);
|
|
260
|
+
return created;
|
|
261
|
+
}
|
|
262
|
+
function getOrCreateTimeBucketOperation(bucketModule, input, timestamp) {
|
|
263
|
+
const key = entryKey(input);
|
|
264
|
+
const existing = bucketModule.operations.get(key);
|
|
265
|
+
if (existing) return existing;
|
|
266
|
+
const created = {
|
|
267
|
+
moduleId: input.moduleId,
|
|
268
|
+
surface: input.surface,
|
|
269
|
+
operation: input.operation,
|
|
270
|
+
resourceId: input.resourceId ?? null,
|
|
271
|
+
calls: 0,
|
|
272
|
+
errors: 0,
|
|
273
|
+
concurrentCalls: 0,
|
|
274
|
+
totalDurationMs: 0,
|
|
275
|
+
maxDurationMs: 0,
|
|
276
|
+
totalCpuUserMs: 0,
|
|
277
|
+
totalCpuSystemMs: 0,
|
|
278
|
+
maxCpuMs: 0,
|
|
279
|
+
totalHeapDeltaBytes: 0,
|
|
280
|
+
positiveHeapDeltaBytes: 0,
|
|
281
|
+
maxHeapDeltaBytes: 0,
|
|
282
|
+
totalRssDeltaBytes: 0,
|
|
283
|
+
positiveRssDeltaBytes: 0,
|
|
284
|
+
maxRssDeltaBytes: 0,
|
|
285
|
+
firstSeenAt: timestamp,
|
|
286
|
+
lastSeenAt: timestamp,
|
|
287
|
+
recentDurationsMs: []
|
|
288
|
+
};
|
|
289
|
+
bucketModule.operations.set(key, created);
|
|
290
|
+
return created;
|
|
291
|
+
}
|
|
292
|
+
function pruneTimeBuckets(state) {
|
|
293
|
+
const limit = moduleResourceUsageBucketLimit();
|
|
294
|
+
if (state.buckets.size <= limit) return;
|
|
295
|
+
const keys = Array.from(state.buckets.entries()).sort((a, b) => a[1].bucketStartMs - b[1].bucketStartMs).map(([key]) => key);
|
|
296
|
+
while (state.buckets.size > limit) {
|
|
297
|
+
const key = keys.shift();
|
|
298
|
+
if (!key) break;
|
|
299
|
+
state.buckets.delete(key);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
function recordTimeBucket(input, timestamp, metrics) {
|
|
303
|
+
const state = getState();
|
|
304
|
+
const bucket = getOrCreateTimeBucket(state, timestamp);
|
|
305
|
+
if (!bucket) return;
|
|
306
|
+
const bucketModule = getOrCreateTimeBucketModule(bucket, input.moduleId);
|
|
307
|
+
bucketModule.calls += 1;
|
|
308
|
+
if (metrics.status === "error") bucketModule.errors += 1;
|
|
309
|
+
bucketModule.totalDurationMs += metrics.durationMs;
|
|
310
|
+
bucketModule.totalCpuMs += metrics.cpuMs;
|
|
311
|
+
bucketModule.positiveHeapDeltaBytes += Math.max(metrics.heapDeltaBytes, 0);
|
|
312
|
+
bucketModule.positiveRssDeltaBytes += Math.max(metrics.rssDeltaBytes, 0);
|
|
313
|
+
bucketModule.recentDurationsMs.push(metrics.durationMs);
|
|
314
|
+
if (bucketModule.recentDurationsMs.length > DEFAULT_RECENT_SAMPLE_LIMIT) {
|
|
315
|
+
bucketModule.recentDurationsMs.splice(0, bucketModule.recentDurationsMs.length - DEFAULT_RECENT_SAMPLE_LIMIT);
|
|
316
|
+
}
|
|
317
|
+
const operation = getOrCreateTimeBucketOperation(bucketModule, input, timestamp);
|
|
318
|
+
operation.calls += 1;
|
|
319
|
+
if (metrics.status === "error") operation.errors += 1;
|
|
320
|
+
if (metrics.concurrentTainted) operation.concurrentCalls += 1;
|
|
321
|
+
operation.totalDurationMs += metrics.durationMs;
|
|
322
|
+
operation.maxDurationMs = Math.max(operation.maxDurationMs, metrics.durationMs);
|
|
323
|
+
operation.totalCpuUserMs += metrics.cpuUserMs;
|
|
324
|
+
operation.totalCpuSystemMs += metrics.cpuSystemMs;
|
|
325
|
+
operation.maxCpuMs = Math.max(operation.maxCpuMs, metrics.cpuMs);
|
|
326
|
+
operation.totalHeapDeltaBytes += metrics.heapDeltaBytes;
|
|
327
|
+
operation.positiveHeapDeltaBytes += Math.max(metrics.heapDeltaBytes, 0);
|
|
328
|
+
operation.maxHeapDeltaBytes = Math.max(operation.maxHeapDeltaBytes, metrics.heapDeltaBytes);
|
|
329
|
+
operation.totalRssDeltaBytes += metrics.rssDeltaBytes;
|
|
330
|
+
operation.positiveRssDeltaBytes += Math.max(metrics.rssDeltaBytes, 0);
|
|
331
|
+
operation.maxRssDeltaBytes = Math.max(operation.maxRssDeltaBytes, metrics.rssDeltaBytes);
|
|
332
|
+
operation.lastSeenAt = timestamp;
|
|
333
|
+
operation.recentDurationsMs.push(metrics.durationMs);
|
|
334
|
+
if (operation.recentDurationsMs.length > DEFAULT_RECENT_SAMPLE_LIMIT) {
|
|
335
|
+
operation.recentDurationsMs.splice(0, operation.recentDurationsMs.length - DEFAULT_RECENT_SAMPLE_LIMIT);
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
function recordModuleResourceUsage(input, metrics) {
|
|
339
|
+
const moduleId = normalizeModuleId(input);
|
|
340
|
+
if (!moduleId) return;
|
|
341
|
+
const timestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
342
|
+
const entry = getOrCreateEntry({ ...input, moduleId }, timestamp);
|
|
343
|
+
const cpuUserMs = metrics.cpuUserMicros / MICROS_PER_MS;
|
|
344
|
+
const cpuSystemMs = metrics.cpuSystemMicros / MICROS_PER_MS;
|
|
345
|
+
const cpuMs = cpuUserMs + cpuSystemMs;
|
|
346
|
+
entry.calls += 1;
|
|
347
|
+
if (metrics.status === "error") entry.errors += 1;
|
|
348
|
+
if (metrics.concurrentTainted) entry.concurrentCalls += 1;
|
|
349
|
+
entry.totalDurationMs += metrics.durationMs;
|
|
350
|
+
entry.maxDurationMs = Math.max(entry.maxDurationMs, metrics.durationMs);
|
|
351
|
+
entry.totalCpuUserMs += cpuUserMs;
|
|
352
|
+
entry.totalCpuSystemMs += cpuSystemMs;
|
|
353
|
+
entry.maxCpuMs = Math.max(entry.maxCpuMs, cpuMs);
|
|
354
|
+
entry.totalHeapDeltaBytes += metrics.heapDeltaBytes;
|
|
355
|
+
entry.positiveHeapDeltaBytes += Math.max(metrics.heapDeltaBytes, 0);
|
|
356
|
+
entry.maxHeapDeltaBytes = Math.max(entry.maxHeapDeltaBytes, metrics.heapDeltaBytes);
|
|
357
|
+
entry.totalRssDeltaBytes += metrics.rssDeltaBytes;
|
|
358
|
+
entry.positiveRssDeltaBytes += Math.max(metrics.rssDeltaBytes, 0);
|
|
359
|
+
entry.maxRssDeltaBytes = Math.max(entry.maxRssDeltaBytes, metrics.rssDeltaBytes);
|
|
360
|
+
entry.lastSeenAt = timestamp;
|
|
361
|
+
entry.recentDurationsMs.push(metrics.durationMs);
|
|
362
|
+
if (entry.recentDurationsMs.length > DEFAULT_RECENT_SAMPLE_LIMIT) {
|
|
363
|
+
entry.recentDurationsMs.splice(0, entry.recentDurationsMs.length - DEFAULT_RECENT_SAMPLE_LIMIT);
|
|
364
|
+
}
|
|
365
|
+
recordTimeBucket({ ...input, moduleId }, timestamp, {
|
|
366
|
+
durationMs: metrics.durationMs,
|
|
367
|
+
cpuUserMs,
|
|
368
|
+
cpuSystemMs,
|
|
369
|
+
cpuMs,
|
|
370
|
+
heapDeltaBytes: metrics.heapDeltaBytes,
|
|
371
|
+
rssDeltaBytes: metrics.rssDeltaBytes,
|
|
372
|
+
status: metrics.status,
|
|
373
|
+
concurrentTainted: metrics.concurrentTainted
|
|
374
|
+
});
|
|
375
|
+
flushModuleResourceUsageSnapshot(false);
|
|
376
|
+
}
|
|
377
|
+
async function withModuleResourceUsage(input, fn) {
|
|
378
|
+
if (!isModuleResourceUsageEnabled() || !normalizeModuleId(input)) {
|
|
379
|
+
return Promise.resolve(fn());
|
|
380
|
+
}
|
|
381
|
+
const state = getState();
|
|
382
|
+
const token = { tainted: state.activeCalls.size > 0 };
|
|
383
|
+
if (token.tainted) {
|
|
384
|
+
for (const other of state.activeCalls) other.tainted = true;
|
|
385
|
+
}
|
|
386
|
+
state.activeCalls.add(token);
|
|
387
|
+
const startNs = nowNs();
|
|
388
|
+
const startCpu = process.cpuUsage();
|
|
389
|
+
const startMemory = safeMemoryUsage();
|
|
390
|
+
let result;
|
|
391
|
+
try {
|
|
392
|
+
result = await Promise.resolve(fn());
|
|
393
|
+
} catch (error) {
|
|
394
|
+
state.activeCalls.delete(token);
|
|
395
|
+
try {
|
|
396
|
+
const cpu = process.cpuUsage(startCpu);
|
|
397
|
+
const endMemory = safeMemoryUsage();
|
|
398
|
+
recordModuleResourceUsage(input, {
|
|
399
|
+
durationMs: durationMs(startNs),
|
|
400
|
+
cpuUserMicros: cpu.user,
|
|
401
|
+
cpuSystemMicros: cpu.system,
|
|
402
|
+
heapDeltaBytes: endMemory && startMemory ? endMemory.heapUsed - startMemory.heapUsed : 0,
|
|
403
|
+
rssDeltaBytes: endMemory && startMemory ? endMemory.rss - startMemory.rss : 0,
|
|
404
|
+
status: "error",
|
|
405
|
+
concurrentTainted: token.tainted
|
|
406
|
+
});
|
|
407
|
+
} catch {
|
|
408
|
+
}
|
|
409
|
+
throw error;
|
|
410
|
+
}
|
|
411
|
+
state.activeCalls.delete(token);
|
|
412
|
+
try {
|
|
413
|
+
const cpu = process.cpuUsage(startCpu);
|
|
414
|
+
const endMemory = safeMemoryUsage();
|
|
415
|
+
recordModuleResourceUsage(input, {
|
|
416
|
+
durationMs: durationMs(startNs),
|
|
417
|
+
cpuUserMicros: cpu.user,
|
|
418
|
+
cpuSystemMicros: cpu.system,
|
|
419
|
+
heapDeltaBytes: endMemory && startMemory ? endMemory.heapUsed - startMemory.heapUsed : 0,
|
|
420
|
+
rssDeltaBytes: endMemory && startMemory ? endMemory.rss - startMemory.rss : 0,
|
|
421
|
+
status: "ok",
|
|
422
|
+
concurrentTainted: token.tainted
|
|
423
|
+
});
|
|
424
|
+
} catch {
|
|
425
|
+
}
|
|
426
|
+
return result;
|
|
427
|
+
}
|
|
428
|
+
function percentile(values, p) {
|
|
429
|
+
if (!values.length) return 0;
|
|
430
|
+
const sorted = values.slice().sort((a, b) => a - b);
|
|
431
|
+
const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(p / 100 * sorted.length) - 1));
|
|
432
|
+
return sorted[index];
|
|
433
|
+
}
|
|
434
|
+
function serializeEntry(entry) {
|
|
435
|
+
return {
|
|
436
|
+
moduleId: entry.moduleId,
|
|
437
|
+
surface: entry.surface,
|
|
438
|
+
operation: entry.operation,
|
|
439
|
+
resourceId: entry.resourceId,
|
|
440
|
+
calls: entry.calls,
|
|
441
|
+
errors: entry.errors,
|
|
442
|
+
concurrentCalls: entry.concurrentCalls,
|
|
443
|
+
totalDurationMs: round(entry.totalDurationMs),
|
|
444
|
+
maxDurationMs: round(entry.maxDurationMs),
|
|
445
|
+
p95DurationMs: round(percentile(entry.recentDurationsMs, 95)),
|
|
446
|
+
totalCpuUserMs: round(entry.totalCpuUserMs),
|
|
447
|
+
totalCpuSystemMs: round(entry.totalCpuSystemMs),
|
|
448
|
+
maxCpuMs: round(entry.maxCpuMs),
|
|
449
|
+
totalHeapDeltaBytes: Math.round(entry.totalHeapDeltaBytes),
|
|
450
|
+
positiveHeapDeltaBytes: Math.round(entry.positiveHeapDeltaBytes),
|
|
451
|
+
maxHeapDeltaBytes: Math.round(entry.maxHeapDeltaBytes),
|
|
452
|
+
totalRssDeltaBytes: Math.round(entry.totalRssDeltaBytes),
|
|
453
|
+
positiveRssDeltaBytes: Math.round(entry.positiveRssDeltaBytes),
|
|
454
|
+
maxRssDeltaBytes: Math.round(entry.maxRssDeltaBytes),
|
|
455
|
+
firstSeenAt: entry.firstSeenAt,
|
|
456
|
+
lastSeenAt: entry.lastSeenAt
|
|
457
|
+
};
|
|
458
|
+
}
|
|
459
|
+
function timeBucketTotals(modules) {
|
|
460
|
+
return {
|
|
461
|
+
modules: modules.length,
|
|
462
|
+
calls: modules.reduce((sum, module) => sum + module.calls, 0),
|
|
463
|
+
errors: modules.reduce((sum, module) => sum + module.errors, 0),
|
|
464
|
+
totalDurationMs: round(modules.reduce((sum, module) => sum + module.totalDurationMs, 0)),
|
|
465
|
+
totalCpuMs: round(modules.reduce((sum, module) => sum + module.totalCpuMs, 0)),
|
|
466
|
+
positiveHeapDeltaBytes: modules.reduce((sum, module) => sum + module.positiveHeapDeltaBytes, 0),
|
|
467
|
+
positiveRssDeltaBytes: modules.reduce((sum, module) => sum + module.positiveRssDeltaBytes, 0)
|
|
468
|
+
};
|
|
469
|
+
}
|
|
470
|
+
function surfaceSummaries(entries) {
|
|
471
|
+
const surfaces = /* @__PURE__ */ new Map();
|
|
472
|
+
for (const entry of entries) {
|
|
473
|
+
const list = surfaces.get(entry.surface) ?? [];
|
|
474
|
+
list.push(entry);
|
|
475
|
+
surfaces.set(entry.surface, list);
|
|
476
|
+
}
|
|
477
|
+
return Array.from(surfaces.entries()).map(([surface, surfaceEntries]) => ({
|
|
478
|
+
surface,
|
|
479
|
+
calls: surfaceEntries.reduce((sum, entry) => sum + entry.calls, 0),
|
|
480
|
+
errors: surfaceEntries.reduce((sum, entry) => sum + entry.errors, 0),
|
|
481
|
+
totalDurationMs: round(surfaceEntries.reduce((sum, entry) => sum + entry.totalDurationMs, 0)),
|
|
482
|
+
p95DurationMs: round(Math.max(...surfaceEntries.map((entry) => entry.p95DurationMs), 0)),
|
|
483
|
+
totalCpuMs: round(surfaceEntries.reduce((sum, entry) => sum + entry.totalCpuUserMs + entry.totalCpuSystemMs, 0)),
|
|
484
|
+
positiveHeapDeltaBytes: surfaceEntries.reduce((sum, entry) => sum + entry.positiveHeapDeltaBytes, 0),
|
|
485
|
+
positiveRssDeltaBytes: surfaceEntries.reduce((sum, entry) => sum + entry.positiveRssDeltaBytes, 0)
|
|
486
|
+
})).sort((a, b) => b.totalCpuMs - a.totalCpuMs || b.totalDurationMs - a.totalDurationMs);
|
|
487
|
+
}
|
|
488
|
+
function moduleResourceUsageBucketStage(bucketStartMs, bucketEndMs, startedAtMs) {
|
|
489
|
+
if (!Number.isFinite(bucketStartMs) || !Number.isFinite(bucketEndMs) || !Number.isFinite(startedAtMs)) return "running";
|
|
490
|
+
return bucketEndMs > startedAtMs && bucketStartMs < startedAtMs + MODULE_RESOURCE_USAGE_STARTUP_STAGE_MS ? "startup" : "running";
|
|
491
|
+
}
|
|
492
|
+
function serializeTimeBucket(bucket, nowMs, startedAtMs) {
|
|
493
|
+
const thresholds = getModuleResourceUsageThresholds();
|
|
494
|
+
const modules = Array.from(bucket.modules.values()).map((module) => ({
|
|
495
|
+
module,
|
|
496
|
+
operations: module.operations instanceof Map ? Array.from(module.operations.values()).map(serializeEntry) : []
|
|
497
|
+
})).map(({ module, operations }) => {
|
|
498
|
+
const summaryBase = {
|
|
499
|
+
moduleId: module.moduleId,
|
|
500
|
+
calls: module.calls,
|
|
501
|
+
errors: module.errors,
|
|
502
|
+
totalDurationMs: round(module.totalDurationMs),
|
|
503
|
+
p95DurationMs: round(percentile(module.recentDurationsMs, 95)),
|
|
504
|
+
totalCpuMs: round(module.totalCpuMs),
|
|
505
|
+
positiveHeapDeltaBytes: Math.round(module.positiveHeapDeltaBytes),
|
|
506
|
+
positiveRssDeltaBytes: Math.round(module.positiveRssDeltaBytes),
|
|
507
|
+
surfaces: surfaceSummaries(operations),
|
|
508
|
+
topOperations: operations.slice().sort((a, b) => b.totalCpuUserMs + b.totalCpuSystemMs - (a.totalCpuUserMs + a.totalCpuSystemMs) || b.totalDurationMs - a.totalDurationMs).slice(0, DEFAULT_TOP_OPERATIONS_LIMIT)
|
|
509
|
+
};
|
|
510
|
+
return {
|
|
511
|
+
...summaryBase,
|
|
512
|
+
candidateReasons: buildCandidateReasons(summaryBase, thresholds)
|
|
513
|
+
};
|
|
514
|
+
}).sort(
|
|
515
|
+
(a, b) => b.candidateReasons.length - a.candidateReasons.length || b.totalCpuMs - a.totalCpuMs || b.positiveRssDeltaBytes - a.positiveRssDeltaBytes || b.calls - a.calls
|
|
516
|
+
);
|
|
517
|
+
return {
|
|
518
|
+
bucketStart: new Date(bucket.bucketStartMs).toISOString(),
|
|
519
|
+
bucketEnd: new Date(bucket.bucketEndMs).toISOString(),
|
|
520
|
+
bucketIntervalMs: bucket.bucketEndMs - bucket.bucketStartMs,
|
|
521
|
+
stage: moduleResourceUsageBucketStage(bucket.bucketStartMs, bucket.bucketEndMs, startedAtMs),
|
|
522
|
+
partial: bucket.bucketStartMs < startedAtMs || nowMs < bucket.bucketEndMs,
|
|
523
|
+
totals: timeBucketTotals(modules),
|
|
524
|
+
modules
|
|
525
|
+
};
|
|
526
|
+
}
|
|
527
|
+
function serializeTimeBuckets(state) {
|
|
528
|
+
const nowMs = Date.now();
|
|
529
|
+
const startedAtMs = Date.parse(state.startedAt);
|
|
530
|
+
return Array.from(state.buckets.values()).sort((a, b) => a.bucketStartMs - b.bucketStartMs).map((bucket) => serializeTimeBucket(bucket, nowMs, Number.isFinite(startedAtMs) ? startedAtMs : nowMs));
|
|
531
|
+
}
|
|
532
|
+
function getSnapshotDir() {
|
|
533
|
+
const configured = process.env.OM_MODULE_RESOURCE_USAGE_DIR?.trim();
|
|
534
|
+
return configured ? path.resolve(configured) : path.resolve(process.cwd(), ".mercato/module-resource-usage");
|
|
535
|
+
}
|
|
536
|
+
function getSnapshotPath() {
|
|
537
|
+
return path.join(getSnapshotDir(), `process-${process.pid}.json`);
|
|
538
|
+
}
|
|
539
|
+
function registerSnapshotShutdownHook(state) {
|
|
540
|
+
if (state.shutdownHookRegistered) return;
|
|
541
|
+
const flush = () => {
|
|
542
|
+
try {
|
|
543
|
+
flushModuleResourceUsageSnapshot(true);
|
|
544
|
+
} catch {
|
|
545
|
+
}
|
|
546
|
+
};
|
|
547
|
+
process.once("beforeExit", flush);
|
|
548
|
+
state.shutdownHookRegistered = true;
|
|
549
|
+
}
|
|
550
|
+
let snapshotWriteSequence = 0;
|
|
551
|
+
function writeSnapshotFileAsync(dir, snapshotPath, payload) {
|
|
552
|
+
const tempPath = `${snapshotPath}.${process.pid}.${snapshotWriteSequence++}.tmp`;
|
|
553
|
+
fs.promises.mkdir(dir, { recursive: true }).then(() => fs.promises.writeFile(tempPath, JSON.stringify(payload, null, 2))).then(() => fs.promises.rename(tempPath, snapshotPath)).catch(() => {
|
|
554
|
+
fs.promises.unlink(tempPath).catch(() => {
|
|
555
|
+
});
|
|
556
|
+
});
|
|
557
|
+
}
|
|
558
|
+
function flushModuleResourceUsageSnapshot(force) {
|
|
559
|
+
if (!isSnapshotEnabled()) return;
|
|
560
|
+
const state = getState();
|
|
561
|
+
const now = Date.now();
|
|
562
|
+
if (!force && now - state.lastSnapshotAt < SNAPSHOT_THROTTLE_MS) return;
|
|
563
|
+
state.lastSnapshotAt = now;
|
|
564
|
+
try {
|
|
565
|
+
const payload = {
|
|
566
|
+
pid: process.pid,
|
|
567
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
568
|
+
startedAt: state.startedAt,
|
|
569
|
+
entries: Array.from(state.entries.values()).map(serializeEntry),
|
|
570
|
+
buckets: serializeTimeBuckets(state)
|
|
571
|
+
};
|
|
572
|
+
writeSnapshotFileAsync(getSnapshotDir(), getSnapshotPath(), payload);
|
|
573
|
+
} catch {
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
function isFreshSnapshot(payload) {
|
|
577
|
+
if (typeof payload.generatedAt !== "string") return false;
|
|
578
|
+
const generatedAt = Date.parse(payload.generatedAt);
|
|
579
|
+
return Number.isFinite(generatedAt) && Date.now() - generatedAt <= SNAPSHOT_MAX_AGE_MS;
|
|
580
|
+
}
|
|
581
|
+
function pruneStaleSnapshotFile(filePath) {
|
|
582
|
+
fs.promises.unlink(filePath).catch(() => {
|
|
583
|
+
});
|
|
584
|
+
}
|
|
585
|
+
function readSnapshotPayloads() {
|
|
586
|
+
if (!isSnapshotEnabled()) return [];
|
|
587
|
+
const dir = getSnapshotDir();
|
|
588
|
+
if (!fs.existsSync(dir)) return [];
|
|
589
|
+
const payloads = [];
|
|
590
|
+
for (const file of fs.readdirSync(dir)) {
|
|
591
|
+
if (!file.endsWith(".json")) continue;
|
|
592
|
+
const filePath = path.join(dir, file);
|
|
593
|
+
try {
|
|
594
|
+
const payload = JSON.parse(fs.readFileSync(filePath, "utf8"));
|
|
595
|
+
if (payload.pid === process.pid) continue;
|
|
596
|
+
if (!isFreshSnapshot(payload)) {
|
|
597
|
+
pruneStaleSnapshotFile(filePath);
|
|
598
|
+
continue;
|
|
599
|
+
}
|
|
600
|
+
payloads.push(payload);
|
|
601
|
+
} catch {
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
return payloads;
|
|
605
|
+
}
|
|
606
|
+
function readSnapshotEntries(payloads) {
|
|
607
|
+
const entries = [];
|
|
608
|
+
for (const payload of payloads) {
|
|
609
|
+
if (!Array.isArray(payload.entries)) continue;
|
|
610
|
+
for (const entry of payload.entries) {
|
|
611
|
+
if (!entry || typeof entry !== "object") continue;
|
|
612
|
+
const candidate = entry;
|
|
613
|
+
if (typeof candidate.moduleId !== "string" || typeof candidate.operation !== "string") continue;
|
|
614
|
+
if (!candidate.surface) continue;
|
|
615
|
+
entries.push(candidate);
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
return entries;
|
|
619
|
+
}
|
|
620
|
+
function readSnapshotBuckets(payloads) {
|
|
621
|
+
const buckets = [];
|
|
622
|
+
for (const payload of payloads) {
|
|
623
|
+
try {
|
|
624
|
+
if (!Array.isArray(payload.buckets)) continue;
|
|
625
|
+
for (const bucket of payload.buckets) {
|
|
626
|
+
if (!bucket || typeof bucket !== "object") continue;
|
|
627
|
+
const candidate = bucket;
|
|
628
|
+
if (typeof candidate.bucketStart !== "string" || typeof candidate.bucketEnd !== "string") continue;
|
|
629
|
+
if (!Array.isArray(candidate.modules)) continue;
|
|
630
|
+
const modules = candidate.modules.map(normalizeTimeBucketModule);
|
|
631
|
+
buckets.push({
|
|
632
|
+
bucketStart: candidate.bucketStart,
|
|
633
|
+
bucketEnd: candidate.bucketEnd,
|
|
634
|
+
bucketIntervalMs: normalizeBucketIntervalMs(candidate),
|
|
635
|
+
stage: candidate.stage === "startup" || candidate.stage === "running" ? candidate.stage : moduleResourceUsageBucketStage(
|
|
636
|
+
Date.parse(candidate.bucketStart),
|
|
637
|
+
Date.parse(candidate.bucketEnd),
|
|
638
|
+
typeof payload.startedAt === "string" ? Date.parse(payload.startedAt) : Date.parse(candidate.bucketStart)
|
|
639
|
+
),
|
|
640
|
+
partial: candidate.partial === true,
|
|
641
|
+
totals: candidate.totals ?? timeBucketTotals(modules),
|
|
642
|
+
modules
|
|
643
|
+
});
|
|
644
|
+
}
|
|
645
|
+
} catch {
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
return buckets;
|
|
649
|
+
}
|
|
650
|
+
function normalizeBucketIntervalMs(bucket) {
|
|
651
|
+
if (Number.isFinite(bucket.bucketIntervalMs) && Number(bucket.bucketIntervalMs) > 0) {
|
|
652
|
+
return Number(bucket.bucketIntervalMs);
|
|
653
|
+
}
|
|
654
|
+
const bucketStartMs = typeof bucket.bucketStart === "string" ? Date.parse(bucket.bucketStart) : NaN;
|
|
655
|
+
const bucketEndMs = typeof bucket.bucketEnd === "string" ? Date.parse(bucket.bucketEnd) : NaN;
|
|
656
|
+
const inferred = bucketEndMs - bucketStartMs;
|
|
657
|
+
return Number.isFinite(inferred) && inferred > 0 ? inferred : moduleResourceUsageBucketIntervalMs();
|
|
658
|
+
}
|
|
659
|
+
function normalizeTimeBucketModule(module) {
|
|
660
|
+
return {
|
|
661
|
+
moduleId: typeof module.moduleId === "string" ? module.moduleId : "unknown",
|
|
662
|
+
calls: Number.isFinite(module.calls) ? Number(module.calls) : 0,
|
|
663
|
+
errors: Number.isFinite(module.errors) ? Number(module.errors) : 0,
|
|
664
|
+
totalDurationMs: Number.isFinite(module.totalDurationMs) ? Number(module.totalDurationMs) : 0,
|
|
665
|
+
p95DurationMs: Number.isFinite(module.p95DurationMs) ? Number(module.p95DurationMs) : 0,
|
|
666
|
+
totalCpuMs: Number.isFinite(module.totalCpuMs) ? Number(module.totalCpuMs) : 0,
|
|
667
|
+
positiveHeapDeltaBytes: Number.isFinite(module.positiveHeapDeltaBytes) ? Number(module.positiveHeapDeltaBytes) : 0,
|
|
668
|
+
positiveRssDeltaBytes: Number.isFinite(module.positiveRssDeltaBytes) ? Number(module.positiveRssDeltaBytes) : 0,
|
|
669
|
+
surfaces: Array.isArray(module.surfaces) ? module.surfaces : [],
|
|
670
|
+
topOperations: Array.isArray(module.topOperations) ? module.topOperations : [],
|
|
671
|
+
candidateReasons: Array.isArray(module.candidateReasons) ? module.candidateReasons : []
|
|
672
|
+
};
|
|
673
|
+
}
|
|
674
|
+
function mergeEntries(entries) {
|
|
675
|
+
const merged = /* @__PURE__ */ new Map();
|
|
676
|
+
for (const entry of entries) {
|
|
677
|
+
const key = entryKey({ moduleId: entry.moduleId, surface: entry.surface, operation: entry.operation });
|
|
678
|
+
const existing = merged.get(key);
|
|
679
|
+
if (!existing) {
|
|
680
|
+
merged.set(key, {
|
|
681
|
+
...entry,
|
|
682
|
+
// Snapshot files written by an older process before concurrentCalls existed lack it.
|
|
683
|
+
concurrentCalls: Number.isFinite(entry.concurrentCalls) ? entry.concurrentCalls : 0
|
|
684
|
+
});
|
|
685
|
+
continue;
|
|
686
|
+
}
|
|
687
|
+
existing.calls += entry.calls;
|
|
688
|
+
existing.errors += entry.errors;
|
|
689
|
+
existing.concurrentCalls += Number.isFinite(entry.concurrentCalls) ? entry.concurrentCalls : 0;
|
|
690
|
+
existing.totalDurationMs = round(existing.totalDurationMs + entry.totalDurationMs);
|
|
691
|
+
existing.maxDurationMs = Math.max(existing.maxDurationMs, entry.maxDurationMs);
|
|
692
|
+
existing.p95DurationMs = Math.max(existing.p95DurationMs, entry.p95DurationMs);
|
|
693
|
+
existing.totalCpuUserMs = round(existing.totalCpuUserMs + entry.totalCpuUserMs);
|
|
694
|
+
existing.totalCpuSystemMs = round(existing.totalCpuSystemMs + entry.totalCpuSystemMs);
|
|
695
|
+
existing.maxCpuMs = Math.max(existing.maxCpuMs, entry.maxCpuMs);
|
|
696
|
+
existing.totalHeapDeltaBytes += entry.totalHeapDeltaBytes;
|
|
697
|
+
existing.positiveHeapDeltaBytes += entry.positiveHeapDeltaBytes;
|
|
698
|
+
existing.maxHeapDeltaBytes = Math.max(existing.maxHeapDeltaBytes, entry.maxHeapDeltaBytes);
|
|
699
|
+
existing.totalRssDeltaBytes += entry.totalRssDeltaBytes;
|
|
700
|
+
existing.positiveRssDeltaBytes += entry.positiveRssDeltaBytes;
|
|
701
|
+
existing.maxRssDeltaBytes = Math.max(existing.maxRssDeltaBytes, entry.maxRssDeltaBytes);
|
|
702
|
+
existing.firstSeenAt = existing.firstSeenAt < entry.firstSeenAt ? existing.firstSeenAt : entry.firstSeenAt;
|
|
703
|
+
existing.lastSeenAt = existing.lastSeenAt > entry.lastSeenAt ? existing.lastSeenAt : entry.lastSeenAt;
|
|
704
|
+
}
|
|
705
|
+
return Array.from(merged.values());
|
|
706
|
+
}
|
|
707
|
+
function mergeSurfaceSummaries(surfaces) {
|
|
708
|
+
const merged = /* @__PURE__ */ new Map();
|
|
709
|
+
for (const surface of surfaces) {
|
|
710
|
+
const existing = merged.get(surface.surface);
|
|
711
|
+
if (!existing) {
|
|
712
|
+
merged.set(surface.surface, { ...surface });
|
|
713
|
+
continue;
|
|
714
|
+
}
|
|
715
|
+
existing.calls += surface.calls;
|
|
716
|
+
existing.errors += surface.errors;
|
|
717
|
+
existing.totalDurationMs = round(existing.totalDurationMs + surface.totalDurationMs);
|
|
718
|
+
existing.p95DurationMs = Math.max(existing.p95DurationMs, surface.p95DurationMs);
|
|
719
|
+
existing.totalCpuMs = round(existing.totalCpuMs + surface.totalCpuMs);
|
|
720
|
+
existing.positiveHeapDeltaBytes += surface.positiveHeapDeltaBytes;
|
|
721
|
+
existing.positiveRssDeltaBytes += surface.positiveRssDeltaBytes;
|
|
722
|
+
}
|
|
723
|
+
return Array.from(merged.values()).sort((a, b) => b.totalCpuMs - a.totalCpuMs || b.totalDurationMs - a.totalDurationMs);
|
|
724
|
+
}
|
|
725
|
+
function mergeOperationEntries(entries) {
|
|
726
|
+
return mergeEntries(entries).sort((a, b) => b.totalCpuUserMs + b.totalCpuSystemMs - (a.totalCpuUserMs + a.totalCpuSystemMs) || b.totalDurationMs - a.totalDurationMs).slice(0, DEFAULT_TOP_OPERATIONS_LIMIT);
|
|
727
|
+
}
|
|
728
|
+
function mergeTimeBuckets(buckets) {
|
|
729
|
+
const merged = /* @__PURE__ */ new Map();
|
|
730
|
+
const thresholds = getModuleResourceUsageThresholds();
|
|
731
|
+
for (const bucket of buckets) {
|
|
732
|
+
const bucketStartMs = Date.parse(bucket.bucketStart);
|
|
733
|
+
if (!Number.isFinite(bucketStartMs)) continue;
|
|
734
|
+
const bucketIntervalMs = normalizeBucketIntervalMs(bucket);
|
|
735
|
+
const key = `${bucket.bucketStart}\0${bucketIntervalMs}`;
|
|
736
|
+
const existing = merged.get(key);
|
|
737
|
+
if (!existing) {
|
|
738
|
+
const modules = bucket.modules.map((module) => {
|
|
739
|
+
const normalizedModule = normalizeTimeBucketModule(module);
|
|
740
|
+
return {
|
|
741
|
+
...normalizedModule,
|
|
742
|
+
candidateReasons: buildCandidateReasons(normalizedModule, thresholds)
|
|
743
|
+
};
|
|
744
|
+
});
|
|
745
|
+
merged.set(key, {
|
|
746
|
+
bucketStart: bucket.bucketStart,
|
|
747
|
+
bucketEnd: bucket.bucketEnd,
|
|
748
|
+
bucketIntervalMs,
|
|
749
|
+
stage: bucket.stage,
|
|
750
|
+
partial: bucket.partial,
|
|
751
|
+
totals: timeBucketTotals(modules),
|
|
752
|
+
modules
|
|
753
|
+
});
|
|
754
|
+
continue;
|
|
755
|
+
}
|
|
756
|
+
existing.partial = existing.partial || bucket.partial;
|
|
757
|
+
if (existing.stage !== "startup") existing.stage = bucket.stage;
|
|
758
|
+
const modulesById = new Map(existing.modules.map((module) => [module.moduleId, module]));
|
|
759
|
+
for (const module of bucket.modules) {
|
|
760
|
+
const normalizedModule = normalizeTimeBucketModule(module);
|
|
761
|
+
const current = modulesById.get(normalizedModule.moduleId);
|
|
762
|
+
if (!current) {
|
|
763
|
+
const created = {
|
|
764
|
+
...normalizedModule,
|
|
765
|
+
candidateReasons: buildCandidateReasons(normalizedModule, thresholds)
|
|
766
|
+
};
|
|
767
|
+
existing.modules.push(created);
|
|
768
|
+
modulesById.set(normalizedModule.moduleId, created);
|
|
769
|
+
continue;
|
|
770
|
+
}
|
|
771
|
+
current.calls += normalizedModule.calls;
|
|
772
|
+
current.errors += normalizedModule.errors;
|
|
773
|
+
current.totalDurationMs = round(current.totalDurationMs + normalizedModule.totalDurationMs);
|
|
774
|
+
current.p95DurationMs = Math.max(current.p95DurationMs, normalizedModule.p95DurationMs);
|
|
775
|
+
current.totalCpuMs = round(current.totalCpuMs + normalizedModule.totalCpuMs);
|
|
776
|
+
current.positiveHeapDeltaBytes += normalizedModule.positiveHeapDeltaBytes;
|
|
777
|
+
current.positiveRssDeltaBytes += normalizedModule.positiveRssDeltaBytes;
|
|
778
|
+
current.surfaces = mergeSurfaceSummaries([...current.surfaces, ...normalizedModule.surfaces]);
|
|
779
|
+
current.topOperations = mergeOperationEntries([...current.topOperations, ...normalizedModule.topOperations]);
|
|
780
|
+
current.candidateReasons = buildCandidateReasons(current, thresholds);
|
|
781
|
+
}
|
|
782
|
+
existing.modules.sort(
|
|
783
|
+
(a, b) => b.totalCpuMs - a.totalCpuMs || b.positiveRssDeltaBytes - a.positiveRssDeltaBytes || b.calls - a.calls
|
|
784
|
+
);
|
|
785
|
+
existing.totals = timeBucketTotals(existing.modules);
|
|
786
|
+
}
|
|
787
|
+
return Array.from(merged.values()).sort((a, b) => a.bucketStart.localeCompare(b.bucketStart)).slice(-moduleResourceUsageBucketLimit());
|
|
788
|
+
}
|
|
789
|
+
function readNumberEnv(name, fallback) {
|
|
790
|
+
return parseNumberWithDefault(process.env[name], fallback, { min: 0 });
|
|
791
|
+
}
|
|
792
|
+
function moduleResourceUsageBucketIntervalMs() {
|
|
793
|
+
return MODULE_RESOURCE_USAGE_BUCKET_INTERVAL_MS;
|
|
794
|
+
}
|
|
795
|
+
function moduleResourceUsageBucketLimit() {
|
|
796
|
+
return Math.max(1, Math.ceil(TIME_BUCKET_RETENTION_MS / moduleResourceUsageBucketIntervalMs()));
|
|
797
|
+
}
|
|
798
|
+
function getModuleResourceUsageThresholds() {
|
|
799
|
+
return {
|
|
800
|
+
p95DurationMs: readNumberEnv("OM_MODULE_RESOURCE_HEAVY_P95_MS", 5e3),
|
|
801
|
+
cpuMs: readNumberEnv("OM_MODULE_RESOURCE_HEAVY_CPU_MS", 25e3),
|
|
802
|
+
positiveHeapDeltaBytes: readNumberEnv("OM_MODULE_RESOURCE_HEAVY_HEAP_BYTES", 250 * 1024 * 1024),
|
|
803
|
+
positiveRssDeltaBytes: readNumberEnv("OM_MODULE_RESOURCE_HEAVY_RSS_BYTES", 250 * 1024 * 1024),
|
|
804
|
+
errors: readNumberEnv("OM_MODULE_RESOURCE_HEAVY_ERRORS", 10)
|
|
805
|
+
};
|
|
806
|
+
}
|
|
807
|
+
function buildCandidateReasons(summary, thresholds) {
|
|
808
|
+
const reasons = [];
|
|
809
|
+
if (summary.p95DurationMs >= thresholds.p95DurationMs) reasons.push("p95_duration");
|
|
810
|
+
if (summary.totalCpuMs >= thresholds.cpuMs) reasons.push("cpu");
|
|
811
|
+
if (summary.positiveHeapDeltaBytes >= thresholds.positiveHeapDeltaBytes) reasons.push("heap_allocations");
|
|
812
|
+
if (summary.positiveRssDeltaBytes >= thresholds.positiveRssDeltaBytes) reasons.push("rss_growth");
|
|
813
|
+
if (summary.errors >= thresholds.errors) reasons.push("errors");
|
|
814
|
+
return reasons;
|
|
815
|
+
}
|
|
816
|
+
function getModuleResourceUsageReport() {
|
|
817
|
+
const state = getState();
|
|
818
|
+
flushModuleResourceUsageSnapshot(true);
|
|
819
|
+
const snapshotPayloads = readSnapshotPayloads();
|
|
820
|
+
const entries = mergeEntries([
|
|
821
|
+
...Array.from(state.entries.values()).map(serializeEntry),
|
|
822
|
+
...readSnapshotEntries(snapshotPayloads)
|
|
823
|
+
]);
|
|
824
|
+
const buckets = mergeTimeBuckets([
|
|
825
|
+
...serializeTimeBuckets(state),
|
|
826
|
+
...readSnapshotBuckets(snapshotPayloads)
|
|
827
|
+
]);
|
|
828
|
+
const thresholds = getModuleResourceUsageThresholds();
|
|
829
|
+
const modules = /* @__PURE__ */ new Map();
|
|
830
|
+
for (const entry of entries) {
|
|
831
|
+
const list = modules.get(entry.moduleId) ?? [];
|
|
832
|
+
list.push(entry);
|
|
833
|
+
modules.set(entry.moduleId, list);
|
|
834
|
+
}
|
|
835
|
+
const moduleSummaries = [];
|
|
836
|
+
for (const [moduleId, moduleEntries] of modules) {
|
|
837
|
+
const surfaces = /* @__PURE__ */ new Map();
|
|
838
|
+
for (const entry of moduleEntries) {
|
|
839
|
+
const list = surfaces.get(entry.surface) ?? [];
|
|
840
|
+
list.push(entry);
|
|
841
|
+
surfaces.set(entry.surface, list);
|
|
842
|
+
}
|
|
843
|
+
const totalDurationMs = moduleEntries.reduce((sum, entry) => sum + entry.totalDurationMs, 0);
|
|
844
|
+
const totalCpuMs = moduleEntries.reduce((sum, entry) => sum + entry.totalCpuUserMs + entry.totalCpuSystemMs, 0);
|
|
845
|
+
const summaryBase = {
|
|
846
|
+
moduleId,
|
|
847
|
+
calls: moduleEntries.reduce((sum, entry) => sum + entry.calls, 0),
|
|
848
|
+
errors: moduleEntries.reduce((sum, entry) => sum + entry.errors, 0),
|
|
849
|
+
totalDurationMs: round(totalDurationMs),
|
|
850
|
+
p95DurationMs: round(Math.max(...moduleEntries.map((entry) => entry.p95DurationMs), 0)),
|
|
851
|
+
totalCpuMs: round(totalCpuMs),
|
|
852
|
+
positiveHeapDeltaBytes: moduleEntries.reduce((sum, entry) => sum + entry.positiveHeapDeltaBytes, 0),
|
|
853
|
+
positiveRssDeltaBytes: moduleEntries.reduce((sum, entry) => sum + entry.positiveRssDeltaBytes, 0),
|
|
854
|
+
surfaces: Array.from(surfaces.entries()).map(([surface, surfaceEntries]) => ({
|
|
855
|
+
surface,
|
|
856
|
+
calls: surfaceEntries.reduce((sum, entry) => sum + entry.calls, 0),
|
|
857
|
+
errors: surfaceEntries.reduce((sum, entry) => sum + entry.errors, 0),
|
|
858
|
+
totalDurationMs: round(surfaceEntries.reduce((sum, entry) => sum + entry.totalDurationMs, 0)),
|
|
859
|
+
p95DurationMs: round(Math.max(...surfaceEntries.map((entry) => entry.p95DurationMs), 0)),
|
|
860
|
+
totalCpuMs: round(surfaceEntries.reduce((sum, entry) => sum + entry.totalCpuUserMs + entry.totalCpuSystemMs, 0)),
|
|
861
|
+
positiveHeapDeltaBytes: surfaceEntries.reduce((sum, entry) => sum + entry.positiveHeapDeltaBytes, 0),
|
|
862
|
+
positiveRssDeltaBytes: surfaceEntries.reduce((sum, entry) => sum + entry.positiveRssDeltaBytes, 0)
|
|
863
|
+
})).sort((a, b) => b.totalCpuMs - a.totalCpuMs || b.totalDurationMs - a.totalDurationMs),
|
|
864
|
+
topOperations: moduleEntries.slice().sort((a, b) => b.totalCpuUserMs + b.totalCpuSystemMs - (a.totalCpuUserMs + a.totalCpuSystemMs) || b.totalDurationMs - a.totalDurationMs).slice(0, DEFAULT_TOP_OPERATIONS_LIMIT)
|
|
865
|
+
};
|
|
866
|
+
moduleSummaries.push({
|
|
867
|
+
...summaryBase,
|
|
868
|
+
candidateReasons: buildCandidateReasons(summaryBase, thresholds)
|
|
869
|
+
});
|
|
870
|
+
}
|
|
871
|
+
const sortedModules = moduleSummaries.sort(
|
|
872
|
+
(a, b) => b.candidateReasons.length - a.candidateReasons.length || b.totalCpuMs - a.totalCpuMs || b.totalDurationMs - a.totalDurationMs
|
|
873
|
+
);
|
|
874
|
+
return {
|
|
875
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
876
|
+
startedAt: state.startedAt,
|
|
877
|
+
enabled: isModuleResourceUsageEnabled(),
|
|
878
|
+
bucketIntervalMs: moduleResourceUsageBucketIntervalMs(),
|
|
879
|
+
totals: {
|
|
880
|
+
modules: sortedModules.length,
|
|
881
|
+
operations: entries.length,
|
|
882
|
+
calls: entries.reduce((sum, entry) => sum + entry.calls, 0),
|
|
883
|
+
errors: entries.reduce((sum, entry) => sum + entry.errors, 0),
|
|
884
|
+
totalDurationMs: round(entries.reduce((sum, entry) => sum + entry.totalDurationMs, 0)),
|
|
885
|
+
totalCpuMs: round(entries.reduce((sum, entry) => sum + entry.totalCpuUserMs + entry.totalCpuSystemMs, 0)),
|
|
886
|
+
positiveHeapDeltaBytes: entries.reduce((sum, entry) => sum + entry.positiveHeapDeltaBytes, 0),
|
|
887
|
+
positiveRssDeltaBytes: entries.reduce((sum, entry) => sum + entry.positiveRssDeltaBytes, 0)
|
|
888
|
+
},
|
|
889
|
+
thresholds,
|
|
890
|
+
modules: sortedModules,
|
|
891
|
+
candidates: sortedModules.filter((module) => module.candidateReasons.length > 0),
|
|
892
|
+
buckets
|
|
893
|
+
};
|
|
894
|
+
}
|
|
895
|
+
export {
|
|
896
|
+
clearModuleResourceUsageData,
|
|
897
|
+
getModuleResourceUsageReport,
|
|
898
|
+
getModuleResourceUsageThresholds,
|
|
899
|
+
inferModuleIdFromResourceId,
|
|
900
|
+
isModuleResourceUsageEnabled,
|
|
901
|
+
resetModuleResourceUsage,
|
|
902
|
+
withModuleResourceUsage
|
|
903
|
+
};
|
|
904
|
+
//# sourceMappingURL=resource-usage.js.map
|