@open-mercato/shared 0.6.6-develop.6384.1.f06fc0b42c → 0.6.6-develop.6386.1.391a1afb9e
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/AGENTS.md +1 -0
- package/dist/lib/modules/resource-usage.js +904 -0
- package/dist/lib/modules/resource-usage.js.map +7 -0
- package/dist/lib/number.js +14 -0
- package/dist/lib/number.js.map +7 -0
- package/dist/lib/search/config.js +2 -5
- package/dist/lib/search/config.js.map +2 -2
- package/dist/lib/version.js +1 -1
- package/dist/lib/version.js.map +1 -1
- package/dist/modules/registry.js.map +2 -2
- package/package.json +2 -2
- package/src/lib/__tests__/number.test.ts +33 -0
- package/src/lib/modules/__tests__/resource-usage.test.ts +351 -0
- package/src/lib/modules/resource-usage.ts +1213 -0
- package/src/lib/number.ts +14 -0
- package/src/lib/search/config.ts +2 -5
- package/src/modules/registry.ts +2 -0
|
@@ -0,0 +1,1213 @@
|
|
|
1
|
+
import fs from 'node:fs'
|
|
2
|
+
import path from 'node:path'
|
|
3
|
+
import { parseBooleanWithDefault } from '../boolean'
|
|
4
|
+
import { parseNumberWithDefault } from '../number'
|
|
5
|
+
|
|
6
|
+
const GLOBAL_KEY = '__openMercatoModuleResourceUsage__'
|
|
7
|
+
const NS_PER_MS = 1_000_000
|
|
8
|
+
const MICROS_PER_MS = 1000
|
|
9
|
+
const DEFAULT_RECENT_SAMPLE_LIMIT = 128
|
|
10
|
+
const DEFAULT_TOP_OPERATIONS_LIMIT = 5
|
|
11
|
+
const TIME_BUCKET_RETENTION_MS = 24 * 60 * 60 * 1000
|
|
12
|
+
const SNAPSHOT_THROTTLE_MS = 5_000
|
|
13
|
+
const SNAPSHOT_MAX_AGE_MS = 24 * 60 * 60 * 1000
|
|
14
|
+
const MINUTE_MS = 60 * 1000
|
|
15
|
+
const MODULE_RESOURCE_USAGE_BUCKET_INTERVAL_MS = 5 * MINUTE_MS
|
|
16
|
+
const MODULE_RESOURCE_USAGE_STARTUP_STAGE_MS = 5 * MINUTE_MS
|
|
17
|
+
|
|
18
|
+
export type ModuleResourceSurface = 'api' | 'subscriber' | 'worker' | 'custom'
|
|
19
|
+
|
|
20
|
+
export type ModuleResourceUsageInput = {
|
|
21
|
+
moduleId?: string | null
|
|
22
|
+
surface: ModuleResourceSurface
|
|
23
|
+
operation: string
|
|
24
|
+
resourceId?: string | null
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export type ModuleResourceUsageEntry = {
|
|
28
|
+
moduleId: string
|
|
29
|
+
surface: ModuleResourceSurface
|
|
30
|
+
operation: string
|
|
31
|
+
resourceId: string | null
|
|
32
|
+
calls: number
|
|
33
|
+
errors: number
|
|
34
|
+
// How many of `calls` overlapped in wall-clock time with another tracked call anywhere in
|
|
35
|
+
// the process. process.cpuUsage()/process.memoryUsage() are process-wide, so a call's own
|
|
36
|
+
// CPU/heap/RSS deltas can include work actually done by a concurrent call during that window.
|
|
37
|
+
// A high concurrentCalls share means the CPU/heap/RSS numbers here are less trustworthy as
|
|
38
|
+
// this specific operation's own cost.
|
|
39
|
+
concurrentCalls: number
|
|
40
|
+
totalDurationMs: number
|
|
41
|
+
maxDurationMs: number
|
|
42
|
+
p95DurationMs: number
|
|
43
|
+
totalCpuUserMs: number
|
|
44
|
+
totalCpuSystemMs: number
|
|
45
|
+
maxCpuMs: number
|
|
46
|
+
totalHeapDeltaBytes: number
|
|
47
|
+
positiveHeapDeltaBytes: number
|
|
48
|
+
maxHeapDeltaBytes: number
|
|
49
|
+
totalRssDeltaBytes: number
|
|
50
|
+
positiveRssDeltaBytes: number
|
|
51
|
+
maxRssDeltaBytes: number
|
|
52
|
+
firstSeenAt: string
|
|
53
|
+
lastSeenAt: string
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export type ModuleResourceUsageSurfaceSummary = {
|
|
57
|
+
surface: ModuleResourceSurface
|
|
58
|
+
calls: number
|
|
59
|
+
errors: number
|
|
60
|
+
totalDurationMs: number
|
|
61
|
+
p95DurationMs: number
|
|
62
|
+
totalCpuMs: number
|
|
63
|
+
positiveHeapDeltaBytes: number
|
|
64
|
+
positiveRssDeltaBytes: number
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export type ModuleResourceUsageModuleSummary = {
|
|
68
|
+
moduleId: string
|
|
69
|
+
calls: number
|
|
70
|
+
errors: number
|
|
71
|
+
totalDurationMs: number
|
|
72
|
+
p95DurationMs: number
|
|
73
|
+
totalCpuMs: number
|
|
74
|
+
positiveHeapDeltaBytes: number
|
|
75
|
+
positiveRssDeltaBytes: number
|
|
76
|
+
surfaces: ModuleResourceUsageSurfaceSummary[]
|
|
77
|
+
topOperations: ModuleResourceUsageEntry[]
|
|
78
|
+
candidateReasons: string[]
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export type ModuleResourceUsageTimeBucketModule = {
|
|
82
|
+
moduleId: string
|
|
83
|
+
calls: number
|
|
84
|
+
errors: number
|
|
85
|
+
totalDurationMs: number
|
|
86
|
+
p95DurationMs: number
|
|
87
|
+
totalCpuMs: number
|
|
88
|
+
positiveHeapDeltaBytes: number
|
|
89
|
+
positiveRssDeltaBytes: number
|
|
90
|
+
surfaces: ModuleResourceUsageSurfaceSummary[]
|
|
91
|
+
topOperations: ModuleResourceUsageEntry[]
|
|
92
|
+
candidateReasons: string[]
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export type ModuleResourceUsageTimeBucket = {
|
|
96
|
+
bucketStart: string
|
|
97
|
+
bucketEnd: string
|
|
98
|
+
bucketIntervalMs: number
|
|
99
|
+
stage: 'startup' | 'running'
|
|
100
|
+
partial: boolean
|
|
101
|
+
totals: {
|
|
102
|
+
modules: number
|
|
103
|
+
calls: number
|
|
104
|
+
errors: number
|
|
105
|
+
totalDurationMs: number
|
|
106
|
+
totalCpuMs: number
|
|
107
|
+
positiveHeapDeltaBytes: number
|
|
108
|
+
positiveRssDeltaBytes: number
|
|
109
|
+
}
|
|
110
|
+
modules: ModuleResourceUsageTimeBucketModule[]
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export type ModuleResourceUsageReport = {
|
|
114
|
+
generatedAt: string
|
|
115
|
+
startedAt: string
|
|
116
|
+
enabled: boolean
|
|
117
|
+
bucketIntervalMs: number
|
|
118
|
+
totals: {
|
|
119
|
+
modules: number
|
|
120
|
+
operations: number
|
|
121
|
+
calls: number
|
|
122
|
+
errors: number
|
|
123
|
+
totalDurationMs: number
|
|
124
|
+
totalCpuMs: number
|
|
125
|
+
positiveHeapDeltaBytes: number
|
|
126
|
+
positiveRssDeltaBytes: number
|
|
127
|
+
}
|
|
128
|
+
thresholds: ModuleResourceUsageThresholds
|
|
129
|
+
modules: ModuleResourceUsageModuleSummary[]
|
|
130
|
+
candidates: ModuleResourceUsageModuleSummary[]
|
|
131
|
+
buckets: ModuleResourceUsageTimeBucket[]
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export type ModuleResourceUsageThresholds = {
|
|
135
|
+
p95DurationMs: number
|
|
136
|
+
cpuMs: number
|
|
137
|
+
positiveHeapDeltaBytes: number
|
|
138
|
+
positiveRssDeltaBytes: number
|
|
139
|
+
errors: number
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
type MutableEntry = Omit<ModuleResourceUsageEntry, 'p95DurationMs'> & {
|
|
143
|
+
recentDurationsMs: number[]
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
type MutableTimeBucketModule = Omit<ModuleResourceUsageTimeBucketModule, 'p95DurationMs' | 'surfaces' | 'topOperations' | 'candidateReasons'> & {
|
|
147
|
+
recentDurationsMs: number[]
|
|
148
|
+
operations: Map<string, MutableEntry>
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
type MutableTimeBucket = {
|
|
152
|
+
bucketStartMs: number
|
|
153
|
+
bucketEndMs: number
|
|
154
|
+
modules: Map<string, MutableTimeBucketModule>
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// A live token per currently in-flight withModuleResourceUsage call. Used to detect when two
|
|
158
|
+
// tracked calls overlap in wall-clock time, since process.cpuUsage()/process.memoryUsage() can't
|
|
159
|
+
// distinguish which concurrent call actually did the work (see ModuleResourceUsageEntry.concurrentCalls).
|
|
160
|
+
type ActiveCallToken = { tainted: boolean }
|
|
161
|
+
|
|
162
|
+
type ResourceUsageState = {
|
|
163
|
+
startedAt: string
|
|
164
|
+
entries: Map<string, MutableEntry>
|
|
165
|
+
buckets: Map<string, MutableTimeBucket>
|
|
166
|
+
activeCalls: Set<ActiveCallToken>
|
|
167
|
+
lastSnapshotAt: number
|
|
168
|
+
shutdownHookRegistered: boolean
|
|
169
|
+
// The bucket interval buckets were last re-keyed for. migrateMutableTimeBuckets only needs to
|
|
170
|
+
// run again if this stops matching moduleResourceUsageBucketIntervalMs() (e.g. a dev HMR reload
|
|
171
|
+
// picked up a source change to the interval constant) — it's a hardcoded constant today, so in
|
|
172
|
+
// the common case this lets getState() skip rebuilding the whole buckets map on every call.
|
|
173
|
+
migratedIntervalMs: number
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
function getState(): ResourceUsageState {
|
|
177
|
+
const existing = (globalThis as Record<string, unknown>)[GLOBAL_KEY]
|
|
178
|
+
if (existing && typeof existing === 'object' && (existing as ResourceUsageState).entries instanceof Map) {
|
|
179
|
+
const state = existing as ResourceUsageState
|
|
180
|
+
if (!(state.buckets instanceof Map)) state.buckets = new Map()
|
|
181
|
+
if (!(state.activeCalls instanceof Set)) state.activeCalls = new Set()
|
|
182
|
+
const currentIntervalMs = moduleResourceUsageBucketIntervalMs()
|
|
183
|
+
if (state.migratedIntervalMs !== currentIntervalMs) {
|
|
184
|
+
migrateMutableTimeBuckets(state, currentIntervalMs)
|
|
185
|
+
state.migratedIntervalMs = currentIntervalMs
|
|
186
|
+
}
|
|
187
|
+
registerSnapshotShutdownHook(state)
|
|
188
|
+
return state
|
|
189
|
+
}
|
|
190
|
+
const state: ResourceUsageState = {
|
|
191
|
+
startedAt: new Date().toISOString(),
|
|
192
|
+
entries: new Map(),
|
|
193
|
+
buckets: new Map(),
|
|
194
|
+
activeCalls: new Set(),
|
|
195
|
+
lastSnapshotAt: 0,
|
|
196
|
+
shutdownHookRegistered: false,
|
|
197
|
+
migratedIntervalMs: moduleResourceUsageBucketIntervalMs(),
|
|
198
|
+
}
|
|
199
|
+
;(globalThis as Record<string, unknown>)[GLOBAL_KEY] = state
|
|
200
|
+
registerSnapshotShutdownHook(state)
|
|
201
|
+
return state
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
function migrateMutableTimeBuckets(state: ResourceUsageState, intervalMs: number): void {
|
|
205
|
+
const migratedBuckets = new Map<string, MutableTimeBucket>()
|
|
206
|
+
for (const bucket of state.buckets.values()) {
|
|
207
|
+
if (!(bucket.modules instanceof Map)) continue
|
|
208
|
+
const bucketStartMs = Number.isFinite(bucket.bucketStartMs)
|
|
209
|
+
? Math.floor(bucket.bucketStartMs / intervalMs) * intervalMs
|
|
210
|
+
: null
|
|
211
|
+
if (bucketStartMs === null) continue
|
|
212
|
+
const key = String(bucketStartMs)
|
|
213
|
+
const targetBucket = migratedBuckets.get(key) ?? {
|
|
214
|
+
bucketStartMs,
|
|
215
|
+
bucketEndMs: bucketStartMs + intervalMs,
|
|
216
|
+
modules: new Map(),
|
|
217
|
+
}
|
|
218
|
+
mergeMutableTimeBucket(targetBucket, bucket)
|
|
219
|
+
migratedBuckets.set(key, targetBucket)
|
|
220
|
+
}
|
|
221
|
+
state.buckets = migratedBuckets
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function trimRecentDurations(values: number[]): number[] {
|
|
225
|
+
if (values.length <= DEFAULT_RECENT_SAMPLE_LIMIT) return values
|
|
226
|
+
return values.slice(values.length - DEFAULT_RECENT_SAMPLE_LIMIT)
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
function mergeMutableEntry(target: MutableEntry, source: MutableEntry): void {
|
|
230
|
+
target.calls += source.calls
|
|
231
|
+
target.errors += source.errors
|
|
232
|
+
target.concurrentCalls += source.concurrentCalls
|
|
233
|
+
target.totalDurationMs += source.totalDurationMs
|
|
234
|
+
target.maxDurationMs = Math.max(target.maxDurationMs, source.maxDurationMs)
|
|
235
|
+
target.totalCpuUserMs += source.totalCpuUserMs
|
|
236
|
+
target.totalCpuSystemMs += source.totalCpuSystemMs
|
|
237
|
+
target.maxCpuMs = Math.max(target.maxCpuMs, source.maxCpuMs)
|
|
238
|
+
target.totalHeapDeltaBytes += source.totalHeapDeltaBytes
|
|
239
|
+
target.positiveHeapDeltaBytes += source.positiveHeapDeltaBytes
|
|
240
|
+
target.maxHeapDeltaBytes = Math.max(target.maxHeapDeltaBytes, source.maxHeapDeltaBytes)
|
|
241
|
+
target.totalRssDeltaBytes += source.totalRssDeltaBytes
|
|
242
|
+
target.positiveRssDeltaBytes += source.positiveRssDeltaBytes
|
|
243
|
+
target.maxRssDeltaBytes = Math.max(target.maxRssDeltaBytes, source.maxRssDeltaBytes)
|
|
244
|
+
target.firstSeenAt = target.firstSeenAt < source.firstSeenAt ? target.firstSeenAt : source.firstSeenAt
|
|
245
|
+
target.lastSeenAt = target.lastSeenAt > source.lastSeenAt ? target.lastSeenAt : source.lastSeenAt
|
|
246
|
+
target.recentDurationsMs = trimRecentDurations([
|
|
247
|
+
...target.recentDurationsMs,
|
|
248
|
+
...(Array.isArray(source.recentDurationsMs) ? source.recentDurationsMs : []),
|
|
249
|
+
])
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
function cloneMutableEntry(entry: MutableEntry): MutableEntry {
|
|
253
|
+
return {
|
|
254
|
+
...entry,
|
|
255
|
+
recentDurationsMs: trimRecentDurations(Array.isArray(entry.recentDurationsMs) ? [...entry.recentDurationsMs] : []),
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
function mergeMutableTimeBucket(targetBucket: MutableTimeBucket, sourceBucket: MutableTimeBucket): void {
|
|
260
|
+
for (const sourceModule of sourceBucket.modules.values()) {
|
|
261
|
+
if (!Array.isArray(sourceModule.recentDurationsMs)) sourceModule.recentDurationsMs = []
|
|
262
|
+
if (!(sourceModule.operations instanceof Map)) sourceModule.operations = new Map()
|
|
263
|
+
const targetModule = targetBucket.modules.get(sourceModule.moduleId)
|
|
264
|
+
if (!targetModule) {
|
|
265
|
+
targetBucket.modules.set(sourceModule.moduleId, {
|
|
266
|
+
moduleId: sourceModule.moduleId,
|
|
267
|
+
calls: sourceModule.calls,
|
|
268
|
+
errors: sourceModule.errors,
|
|
269
|
+
totalDurationMs: sourceModule.totalDurationMs,
|
|
270
|
+
totalCpuMs: sourceModule.totalCpuMs,
|
|
271
|
+
positiveHeapDeltaBytes: sourceModule.positiveHeapDeltaBytes,
|
|
272
|
+
positiveRssDeltaBytes: sourceModule.positiveRssDeltaBytes,
|
|
273
|
+
recentDurationsMs: trimRecentDurations([...sourceModule.recentDurationsMs]),
|
|
274
|
+
operations: new Map(Array.from(sourceModule.operations.entries()).map(([key, entry]) => [key, cloneMutableEntry(entry)])),
|
|
275
|
+
})
|
|
276
|
+
continue
|
|
277
|
+
}
|
|
278
|
+
targetModule.calls += sourceModule.calls
|
|
279
|
+
targetModule.errors += sourceModule.errors
|
|
280
|
+
targetModule.totalDurationMs += sourceModule.totalDurationMs
|
|
281
|
+
targetModule.totalCpuMs += sourceModule.totalCpuMs
|
|
282
|
+
targetModule.positiveHeapDeltaBytes += sourceModule.positiveHeapDeltaBytes
|
|
283
|
+
targetModule.positiveRssDeltaBytes += sourceModule.positiveRssDeltaBytes
|
|
284
|
+
targetModule.recentDurationsMs = trimRecentDurations([
|
|
285
|
+
...targetModule.recentDurationsMs,
|
|
286
|
+
...sourceModule.recentDurationsMs,
|
|
287
|
+
])
|
|
288
|
+
for (const [operationKey, sourceOperation] of sourceModule.operations) {
|
|
289
|
+
const targetOperation = targetModule.operations.get(operationKey)
|
|
290
|
+
if (!targetOperation) {
|
|
291
|
+
targetModule.operations.set(operationKey, cloneMutableEntry(sourceOperation))
|
|
292
|
+
continue
|
|
293
|
+
}
|
|
294
|
+
mergeMutableEntry(targetOperation, sourceOperation)
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
export function resetModuleResourceUsage(): void {
|
|
300
|
+
;(globalThis as Record<string, unknown>)[GLOBAL_KEY] = {
|
|
301
|
+
startedAt: new Date().toISOString(),
|
|
302
|
+
entries: new Map(),
|
|
303
|
+
buckets: new Map(),
|
|
304
|
+
activeCalls: new Set(),
|
|
305
|
+
lastSnapshotAt: 0,
|
|
306
|
+
shutdownHookRegistered: false,
|
|
307
|
+
migratedIntervalMs: moduleResourceUsageBucketIntervalMs(),
|
|
308
|
+
} satisfies ResourceUsageState
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
export function clearModuleResourceUsageData(): void {
|
|
312
|
+
resetModuleResourceUsage()
|
|
313
|
+
try {
|
|
314
|
+
fs.rmSync(getSnapshotDir(), { recursive: true, force: true })
|
|
315
|
+
} catch {
|
|
316
|
+
// Telemetry is diagnostic only; clearing in-memory state is still useful if files cannot be removed.
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
export function isModuleResourceUsageEnabled(): boolean {
|
|
321
|
+
return parseBooleanWithDefault(process.env.OM_MODULE_RESOURCE_USAGE, true)
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
function isSnapshotEnabled(): boolean {
|
|
325
|
+
return parseBooleanWithDefault(process.env.OM_MODULE_RESOURCE_USAGE_SNAPSHOT, process.env.NODE_ENV !== 'test')
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
export function inferModuleIdFromResourceId(resourceId: string | null | undefined): string | null {
|
|
329
|
+
if (!resourceId) return null
|
|
330
|
+
const trimmed = resourceId.trim()
|
|
331
|
+
if (!trimmed) return null
|
|
332
|
+
const colon = trimmed.indexOf(':')
|
|
333
|
+
if (colon > 0) return trimmed.slice(0, colon)
|
|
334
|
+
const dot = trimmed.indexOf('.')
|
|
335
|
+
if (dot > 0) return trimmed.slice(0, dot)
|
|
336
|
+
return null
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
function normalizeModuleId(input: ModuleResourceUsageInput): string | null {
|
|
340
|
+
const explicit = input.moduleId?.trim()
|
|
341
|
+
if (explicit) return explicit
|
|
342
|
+
return inferModuleIdFromResourceId(input.resourceId) ?? inferModuleIdFromResourceId(input.operation)
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
function nowNs(): bigint {
|
|
346
|
+
return process.hrtime.bigint()
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
function durationMs(startNs: bigint): number {
|
|
350
|
+
const elapsed = nowNs() - startNs
|
|
351
|
+
if (elapsed <= BigInt(0)) return 0
|
|
352
|
+
return Number(elapsed) / NS_PER_MS
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
function round(value: number, precision = 100): number {
|
|
356
|
+
if (!Number.isFinite(value)) return 0
|
|
357
|
+
return Math.round(value * precision) / precision
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
function safeMemoryUsage(): NodeJS.MemoryUsage | null {
|
|
361
|
+
try {
|
|
362
|
+
return process.memoryUsage()
|
|
363
|
+
} catch {
|
|
364
|
+
return null
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
function entryKey(input: Required<Pick<ModuleResourceUsageInput, 'surface' | 'operation'>> & { moduleId: string }): string {
|
|
369
|
+
return `${input.moduleId}\u0000${input.surface}\u0000${input.operation}`
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
function getOrCreateEntry(input: ModuleResourceUsageInput & { moduleId: string }, timestamp: string): MutableEntry {
|
|
373
|
+
const state = getState()
|
|
374
|
+
const key = entryKey(input)
|
|
375
|
+
const existing = state.entries.get(key)
|
|
376
|
+
if (existing) return existing
|
|
377
|
+
const created: MutableEntry = {
|
|
378
|
+
moduleId: input.moduleId,
|
|
379
|
+
surface: input.surface,
|
|
380
|
+
operation: input.operation,
|
|
381
|
+
resourceId: input.resourceId ?? null,
|
|
382
|
+
calls: 0,
|
|
383
|
+
errors: 0,
|
|
384
|
+
concurrentCalls: 0,
|
|
385
|
+
totalDurationMs: 0,
|
|
386
|
+
maxDurationMs: 0,
|
|
387
|
+
totalCpuUserMs: 0,
|
|
388
|
+
totalCpuSystemMs: 0,
|
|
389
|
+
maxCpuMs: 0,
|
|
390
|
+
totalHeapDeltaBytes: 0,
|
|
391
|
+
positiveHeapDeltaBytes: 0,
|
|
392
|
+
maxHeapDeltaBytes: 0,
|
|
393
|
+
totalRssDeltaBytes: 0,
|
|
394
|
+
positiveRssDeltaBytes: 0,
|
|
395
|
+
maxRssDeltaBytes: 0,
|
|
396
|
+
firstSeenAt: timestamp,
|
|
397
|
+
lastSeenAt: timestamp,
|
|
398
|
+
recentDurationsMs: [],
|
|
399
|
+
}
|
|
400
|
+
state.entries.set(key, created)
|
|
401
|
+
return created
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
function getOrCreateTimeBucket(state: ResourceUsageState, timestamp: string): MutableTimeBucket | null {
|
|
405
|
+
const timestampMs = Date.parse(timestamp)
|
|
406
|
+
if (!Number.isFinite(timestampMs)) return null
|
|
407
|
+
const intervalMs = moduleResourceUsageBucketIntervalMs()
|
|
408
|
+
const bucketStartMs = Math.floor(timestampMs / intervalMs) * intervalMs
|
|
409
|
+
const key = String(bucketStartMs)
|
|
410
|
+
const existing = state.buckets.get(key)
|
|
411
|
+
if (existing) return existing
|
|
412
|
+
const created: MutableTimeBucket = {
|
|
413
|
+
bucketStartMs,
|
|
414
|
+
bucketEndMs: bucketStartMs + intervalMs,
|
|
415
|
+
modules: new Map(),
|
|
416
|
+
}
|
|
417
|
+
state.buckets.set(key, created)
|
|
418
|
+
pruneTimeBuckets(state)
|
|
419
|
+
return created
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
function getOrCreateTimeBucketModule(bucket: MutableTimeBucket, moduleId: string): MutableTimeBucketModule {
|
|
423
|
+
const existing = bucket.modules.get(moduleId)
|
|
424
|
+
if (existing) {
|
|
425
|
+
if (!Array.isArray(existing.recentDurationsMs)) existing.recentDurationsMs = []
|
|
426
|
+
if (!(existing.operations instanceof Map)) existing.operations = new Map()
|
|
427
|
+
return existing
|
|
428
|
+
}
|
|
429
|
+
const created: MutableTimeBucketModule = {
|
|
430
|
+
moduleId,
|
|
431
|
+
calls: 0,
|
|
432
|
+
errors: 0,
|
|
433
|
+
totalDurationMs: 0,
|
|
434
|
+
totalCpuMs: 0,
|
|
435
|
+
positiveHeapDeltaBytes: 0,
|
|
436
|
+
positiveRssDeltaBytes: 0,
|
|
437
|
+
recentDurationsMs: [],
|
|
438
|
+
operations: new Map(),
|
|
439
|
+
}
|
|
440
|
+
bucket.modules.set(moduleId, created)
|
|
441
|
+
return created
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
function getOrCreateTimeBucketOperation(
|
|
445
|
+
bucketModule: MutableTimeBucketModule,
|
|
446
|
+
input: ModuleResourceUsageInput & { moduleId: string },
|
|
447
|
+
timestamp: string,
|
|
448
|
+
): MutableEntry {
|
|
449
|
+
const key = entryKey(input)
|
|
450
|
+
const existing = bucketModule.operations.get(key)
|
|
451
|
+
if (existing) return existing
|
|
452
|
+
const created: MutableEntry = {
|
|
453
|
+
moduleId: input.moduleId,
|
|
454
|
+
surface: input.surface,
|
|
455
|
+
operation: input.operation,
|
|
456
|
+
resourceId: input.resourceId ?? null,
|
|
457
|
+
calls: 0,
|
|
458
|
+
errors: 0,
|
|
459
|
+
concurrentCalls: 0,
|
|
460
|
+
totalDurationMs: 0,
|
|
461
|
+
maxDurationMs: 0,
|
|
462
|
+
totalCpuUserMs: 0,
|
|
463
|
+
totalCpuSystemMs: 0,
|
|
464
|
+
maxCpuMs: 0,
|
|
465
|
+
totalHeapDeltaBytes: 0,
|
|
466
|
+
positiveHeapDeltaBytes: 0,
|
|
467
|
+
maxHeapDeltaBytes: 0,
|
|
468
|
+
totalRssDeltaBytes: 0,
|
|
469
|
+
positiveRssDeltaBytes: 0,
|
|
470
|
+
maxRssDeltaBytes: 0,
|
|
471
|
+
firstSeenAt: timestamp,
|
|
472
|
+
lastSeenAt: timestamp,
|
|
473
|
+
recentDurationsMs: [],
|
|
474
|
+
}
|
|
475
|
+
bucketModule.operations.set(key, created)
|
|
476
|
+
return created
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
function pruneTimeBuckets(state: ResourceUsageState): void {
|
|
480
|
+
const limit = moduleResourceUsageBucketLimit()
|
|
481
|
+
if (state.buckets.size <= limit) return
|
|
482
|
+
const keys = Array.from(state.buckets.entries())
|
|
483
|
+
.sort((a, b) => a[1].bucketStartMs - b[1].bucketStartMs)
|
|
484
|
+
.map(([key]) => key)
|
|
485
|
+
while (state.buckets.size > limit) {
|
|
486
|
+
const key = keys.shift()
|
|
487
|
+
if (!key) break
|
|
488
|
+
state.buckets.delete(key)
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
function recordTimeBucket(input: ModuleResourceUsageInput & { moduleId: string }, timestamp: string, metrics: {
|
|
493
|
+
durationMs: number
|
|
494
|
+
cpuUserMs: number
|
|
495
|
+
cpuSystemMs: number
|
|
496
|
+
cpuMs: number
|
|
497
|
+
heapDeltaBytes: number
|
|
498
|
+
rssDeltaBytes: number
|
|
499
|
+
status: 'ok' | 'error'
|
|
500
|
+
concurrentTainted: boolean
|
|
501
|
+
}): void {
|
|
502
|
+
const state = getState()
|
|
503
|
+
const bucket = getOrCreateTimeBucket(state, timestamp)
|
|
504
|
+
if (!bucket) return
|
|
505
|
+
const bucketModule = getOrCreateTimeBucketModule(bucket, input.moduleId)
|
|
506
|
+
bucketModule.calls += 1
|
|
507
|
+
if (metrics.status === 'error') bucketModule.errors += 1
|
|
508
|
+
bucketModule.totalDurationMs += metrics.durationMs
|
|
509
|
+
bucketModule.totalCpuMs += metrics.cpuMs
|
|
510
|
+
bucketModule.positiveHeapDeltaBytes += Math.max(metrics.heapDeltaBytes, 0)
|
|
511
|
+
bucketModule.positiveRssDeltaBytes += Math.max(metrics.rssDeltaBytes, 0)
|
|
512
|
+
bucketModule.recentDurationsMs.push(metrics.durationMs)
|
|
513
|
+
if (bucketModule.recentDurationsMs.length > DEFAULT_RECENT_SAMPLE_LIMIT) {
|
|
514
|
+
bucketModule.recentDurationsMs.splice(0, bucketModule.recentDurationsMs.length - DEFAULT_RECENT_SAMPLE_LIMIT)
|
|
515
|
+
}
|
|
516
|
+
const operation = getOrCreateTimeBucketOperation(bucketModule, input, timestamp)
|
|
517
|
+
operation.calls += 1
|
|
518
|
+
if (metrics.status === 'error') operation.errors += 1
|
|
519
|
+
if (metrics.concurrentTainted) operation.concurrentCalls += 1
|
|
520
|
+
operation.totalDurationMs += metrics.durationMs
|
|
521
|
+
operation.maxDurationMs = Math.max(operation.maxDurationMs, metrics.durationMs)
|
|
522
|
+
operation.totalCpuUserMs += metrics.cpuUserMs
|
|
523
|
+
operation.totalCpuSystemMs += metrics.cpuSystemMs
|
|
524
|
+
operation.maxCpuMs = Math.max(operation.maxCpuMs, metrics.cpuMs)
|
|
525
|
+
operation.totalHeapDeltaBytes += metrics.heapDeltaBytes
|
|
526
|
+
operation.positiveHeapDeltaBytes += Math.max(metrics.heapDeltaBytes, 0)
|
|
527
|
+
operation.maxHeapDeltaBytes = Math.max(operation.maxHeapDeltaBytes, metrics.heapDeltaBytes)
|
|
528
|
+
operation.totalRssDeltaBytes += metrics.rssDeltaBytes
|
|
529
|
+
operation.positiveRssDeltaBytes += Math.max(metrics.rssDeltaBytes, 0)
|
|
530
|
+
operation.maxRssDeltaBytes = Math.max(operation.maxRssDeltaBytes, metrics.rssDeltaBytes)
|
|
531
|
+
operation.lastSeenAt = timestamp
|
|
532
|
+
operation.recentDurationsMs.push(metrics.durationMs)
|
|
533
|
+
if (operation.recentDurationsMs.length > DEFAULT_RECENT_SAMPLE_LIMIT) {
|
|
534
|
+
operation.recentDurationsMs.splice(0, operation.recentDurationsMs.length - DEFAULT_RECENT_SAMPLE_LIMIT)
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
function recordModuleResourceUsage(input: ModuleResourceUsageInput, metrics: {
|
|
539
|
+
durationMs: number
|
|
540
|
+
cpuUserMicros: number
|
|
541
|
+
cpuSystemMicros: number
|
|
542
|
+
heapDeltaBytes: number
|
|
543
|
+
rssDeltaBytes: number
|
|
544
|
+
status: 'ok' | 'error'
|
|
545
|
+
concurrentTainted: boolean
|
|
546
|
+
}): void {
|
|
547
|
+
const moduleId = normalizeModuleId(input)
|
|
548
|
+
if (!moduleId) return
|
|
549
|
+
const timestamp = new Date().toISOString()
|
|
550
|
+
const entry = getOrCreateEntry({ ...input, moduleId }, timestamp)
|
|
551
|
+
const cpuUserMs = metrics.cpuUserMicros / MICROS_PER_MS
|
|
552
|
+
const cpuSystemMs = metrics.cpuSystemMicros / MICROS_PER_MS
|
|
553
|
+
const cpuMs = cpuUserMs + cpuSystemMs
|
|
554
|
+
entry.calls += 1
|
|
555
|
+
if (metrics.status === 'error') entry.errors += 1
|
|
556
|
+
if (metrics.concurrentTainted) entry.concurrentCalls += 1
|
|
557
|
+
entry.totalDurationMs += metrics.durationMs
|
|
558
|
+
entry.maxDurationMs = Math.max(entry.maxDurationMs, metrics.durationMs)
|
|
559
|
+
entry.totalCpuUserMs += cpuUserMs
|
|
560
|
+
entry.totalCpuSystemMs += cpuSystemMs
|
|
561
|
+
entry.maxCpuMs = Math.max(entry.maxCpuMs, cpuMs)
|
|
562
|
+
entry.totalHeapDeltaBytes += metrics.heapDeltaBytes
|
|
563
|
+
entry.positiveHeapDeltaBytes += Math.max(metrics.heapDeltaBytes, 0)
|
|
564
|
+
entry.maxHeapDeltaBytes = Math.max(entry.maxHeapDeltaBytes, metrics.heapDeltaBytes)
|
|
565
|
+
entry.totalRssDeltaBytes += metrics.rssDeltaBytes
|
|
566
|
+
entry.positiveRssDeltaBytes += Math.max(metrics.rssDeltaBytes, 0)
|
|
567
|
+
entry.maxRssDeltaBytes = Math.max(entry.maxRssDeltaBytes, metrics.rssDeltaBytes)
|
|
568
|
+
entry.lastSeenAt = timestamp
|
|
569
|
+
entry.recentDurationsMs.push(metrics.durationMs)
|
|
570
|
+
if (entry.recentDurationsMs.length > DEFAULT_RECENT_SAMPLE_LIMIT) {
|
|
571
|
+
entry.recentDurationsMs.splice(0, entry.recentDurationsMs.length - DEFAULT_RECENT_SAMPLE_LIMIT)
|
|
572
|
+
}
|
|
573
|
+
recordTimeBucket({ ...input, moduleId }, timestamp, {
|
|
574
|
+
durationMs: metrics.durationMs,
|
|
575
|
+
cpuUserMs,
|
|
576
|
+
cpuSystemMs,
|
|
577
|
+
cpuMs,
|
|
578
|
+
heapDeltaBytes: metrics.heapDeltaBytes,
|
|
579
|
+
rssDeltaBytes: metrics.rssDeltaBytes,
|
|
580
|
+
status: metrics.status,
|
|
581
|
+
concurrentTainted: metrics.concurrentTainted,
|
|
582
|
+
})
|
|
583
|
+
flushModuleResourceUsageSnapshot(false)
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
export async function withModuleResourceUsage<T>(
|
|
587
|
+
input: ModuleResourceUsageInput,
|
|
588
|
+
fn: () => Promise<T> | T,
|
|
589
|
+
): Promise<T> {
|
|
590
|
+
if (!isModuleResourceUsageEnabled() || !normalizeModuleId(input)) {
|
|
591
|
+
return Promise.resolve(fn())
|
|
592
|
+
}
|
|
593
|
+
const state = getState()
|
|
594
|
+
// Mark this call tainted if another tracked call is already in flight, and mark every
|
|
595
|
+
// currently in-flight call tainted too, since process.cpuUsage()/process.memoryUsage() can't
|
|
596
|
+
// separate their concurrent work from ours (see ModuleResourceUsageEntry.concurrentCalls).
|
|
597
|
+
const token: ActiveCallToken = { tainted: state.activeCalls.size > 0 }
|
|
598
|
+
if (token.tainted) {
|
|
599
|
+
for (const other of state.activeCalls) other.tainted = true
|
|
600
|
+
}
|
|
601
|
+
state.activeCalls.add(token)
|
|
602
|
+
const startNs = nowNs()
|
|
603
|
+
const startCpu = process.cpuUsage()
|
|
604
|
+
const startMemory = safeMemoryUsage()
|
|
605
|
+
// Recording must never override the wrapped call's real outcome: `fn()` is awaited in its own
|
|
606
|
+
// try/catch, and the bookkeeping below always runs in a try/catch of its own so a bug in the
|
|
607
|
+
// telemetry path can only be lost silently, never surface as a fake success/failure for `fn()`.
|
|
608
|
+
let result: T
|
|
609
|
+
try {
|
|
610
|
+
result = await Promise.resolve(fn())
|
|
611
|
+
} catch (error) {
|
|
612
|
+
state.activeCalls.delete(token)
|
|
613
|
+
try {
|
|
614
|
+
const cpu = process.cpuUsage(startCpu)
|
|
615
|
+
const endMemory = safeMemoryUsage()
|
|
616
|
+
recordModuleResourceUsage(input, {
|
|
617
|
+
durationMs: durationMs(startNs),
|
|
618
|
+
cpuUserMicros: cpu.user,
|
|
619
|
+
cpuSystemMicros: cpu.system,
|
|
620
|
+
heapDeltaBytes: endMemory && startMemory ? endMemory.heapUsed - startMemory.heapUsed : 0,
|
|
621
|
+
rssDeltaBytes: endMemory && startMemory ? endMemory.rss - startMemory.rss : 0,
|
|
622
|
+
status: 'error',
|
|
623
|
+
concurrentTainted: token.tainted,
|
|
624
|
+
})
|
|
625
|
+
} catch {
|
|
626
|
+
// Telemetry bookkeeping is diagnostic only; never mask the real error from the wrapped call.
|
|
627
|
+
}
|
|
628
|
+
throw error
|
|
629
|
+
}
|
|
630
|
+
state.activeCalls.delete(token)
|
|
631
|
+
try {
|
|
632
|
+
const cpu = process.cpuUsage(startCpu)
|
|
633
|
+
const endMemory = safeMemoryUsage()
|
|
634
|
+
recordModuleResourceUsage(input, {
|
|
635
|
+
durationMs: durationMs(startNs),
|
|
636
|
+
cpuUserMicros: cpu.user,
|
|
637
|
+
cpuSystemMicros: cpu.system,
|
|
638
|
+
heapDeltaBytes: endMemory && startMemory ? endMemory.heapUsed - startMemory.heapUsed : 0,
|
|
639
|
+
rssDeltaBytes: endMemory && startMemory ? endMemory.rss - startMemory.rss : 0,
|
|
640
|
+
status: 'ok',
|
|
641
|
+
concurrentTainted: token.tainted,
|
|
642
|
+
})
|
|
643
|
+
} catch {
|
|
644
|
+
// Telemetry bookkeeping is diagnostic only; never mask the real result from the wrapped call.
|
|
645
|
+
}
|
|
646
|
+
return result
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
function percentile(values: number[], p: number): number {
|
|
650
|
+
if (!values.length) return 0
|
|
651
|
+
const sorted = values.slice().sort((a, b) => a - b)
|
|
652
|
+
const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1))
|
|
653
|
+
return sorted[index]
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
function serializeEntry(entry: MutableEntry): ModuleResourceUsageEntry {
|
|
657
|
+
return {
|
|
658
|
+
moduleId: entry.moduleId,
|
|
659
|
+
surface: entry.surface,
|
|
660
|
+
operation: entry.operation,
|
|
661
|
+
resourceId: entry.resourceId,
|
|
662
|
+
calls: entry.calls,
|
|
663
|
+
errors: entry.errors,
|
|
664
|
+
concurrentCalls: entry.concurrentCalls,
|
|
665
|
+
totalDurationMs: round(entry.totalDurationMs),
|
|
666
|
+
maxDurationMs: round(entry.maxDurationMs),
|
|
667
|
+
p95DurationMs: round(percentile(entry.recentDurationsMs, 95)),
|
|
668
|
+
totalCpuUserMs: round(entry.totalCpuUserMs),
|
|
669
|
+
totalCpuSystemMs: round(entry.totalCpuSystemMs),
|
|
670
|
+
maxCpuMs: round(entry.maxCpuMs),
|
|
671
|
+
totalHeapDeltaBytes: Math.round(entry.totalHeapDeltaBytes),
|
|
672
|
+
positiveHeapDeltaBytes: Math.round(entry.positiveHeapDeltaBytes),
|
|
673
|
+
maxHeapDeltaBytes: Math.round(entry.maxHeapDeltaBytes),
|
|
674
|
+
totalRssDeltaBytes: Math.round(entry.totalRssDeltaBytes),
|
|
675
|
+
positiveRssDeltaBytes: Math.round(entry.positiveRssDeltaBytes),
|
|
676
|
+
maxRssDeltaBytes: Math.round(entry.maxRssDeltaBytes),
|
|
677
|
+
firstSeenAt: entry.firstSeenAt,
|
|
678
|
+
lastSeenAt: entry.lastSeenAt,
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
function timeBucketTotals(modules: ModuleResourceUsageTimeBucketModule[]): ModuleResourceUsageTimeBucket['totals'] {
|
|
683
|
+
return {
|
|
684
|
+
modules: modules.length,
|
|
685
|
+
calls: modules.reduce((sum, module) => sum + module.calls, 0),
|
|
686
|
+
errors: modules.reduce((sum, module) => sum + module.errors, 0),
|
|
687
|
+
totalDurationMs: round(modules.reduce((sum, module) => sum + module.totalDurationMs, 0)),
|
|
688
|
+
totalCpuMs: round(modules.reduce((sum, module) => sum + module.totalCpuMs, 0)),
|
|
689
|
+
positiveHeapDeltaBytes: modules.reduce((sum, module) => sum + module.positiveHeapDeltaBytes, 0),
|
|
690
|
+
positiveRssDeltaBytes: modules.reduce((sum, module) => sum + module.positiveRssDeltaBytes, 0),
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
function surfaceSummaries(entries: ModuleResourceUsageEntry[]): ModuleResourceUsageSurfaceSummary[] {
|
|
695
|
+
const surfaces = new Map<ModuleResourceSurface, ModuleResourceUsageEntry[]>()
|
|
696
|
+
for (const entry of entries) {
|
|
697
|
+
const list = surfaces.get(entry.surface) ?? []
|
|
698
|
+
list.push(entry)
|
|
699
|
+
surfaces.set(entry.surface, list)
|
|
700
|
+
}
|
|
701
|
+
return Array.from(surfaces.entries()).map(([surface, surfaceEntries]) => ({
|
|
702
|
+
surface,
|
|
703
|
+
calls: surfaceEntries.reduce((sum, entry) => sum + entry.calls, 0),
|
|
704
|
+
errors: surfaceEntries.reduce((sum, entry) => sum + entry.errors, 0),
|
|
705
|
+
totalDurationMs: round(surfaceEntries.reduce((sum, entry) => sum + entry.totalDurationMs, 0)),
|
|
706
|
+
p95DurationMs: round(Math.max(...surfaceEntries.map((entry) => entry.p95DurationMs), 0)),
|
|
707
|
+
totalCpuMs: round(surfaceEntries.reduce((sum, entry) => sum + entry.totalCpuUserMs + entry.totalCpuSystemMs, 0)),
|
|
708
|
+
positiveHeapDeltaBytes: surfaceEntries.reduce((sum, entry) => sum + entry.positiveHeapDeltaBytes, 0),
|
|
709
|
+
positiveRssDeltaBytes: surfaceEntries.reduce((sum, entry) => sum + entry.positiveRssDeltaBytes, 0),
|
|
710
|
+
})).sort((a, b) => b.totalCpuMs - a.totalCpuMs || b.totalDurationMs - a.totalDurationMs)
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
function moduleResourceUsageBucketStage(bucketStartMs: number, bucketEndMs: number, startedAtMs: number): ModuleResourceUsageTimeBucket['stage'] {
|
|
714
|
+
if (!Number.isFinite(bucketStartMs) || !Number.isFinite(bucketEndMs) || !Number.isFinite(startedAtMs)) return 'running'
|
|
715
|
+
return bucketEndMs > startedAtMs && bucketStartMs < startedAtMs + MODULE_RESOURCE_USAGE_STARTUP_STAGE_MS
|
|
716
|
+
? 'startup'
|
|
717
|
+
: 'running'
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
function serializeTimeBucket(bucket: MutableTimeBucket, nowMs: number, startedAtMs: number): ModuleResourceUsageTimeBucket {
|
|
721
|
+
const thresholds = getModuleResourceUsageThresholds()
|
|
722
|
+
const modules = Array.from(bucket.modules.values()).map((module) => ({
|
|
723
|
+
module,
|
|
724
|
+
operations: module.operations instanceof Map
|
|
725
|
+
? Array.from(module.operations.values()).map(serializeEntry)
|
|
726
|
+
: [],
|
|
727
|
+
})).map(({ module, operations }) => {
|
|
728
|
+
const summaryBase = {
|
|
729
|
+
moduleId: module.moduleId,
|
|
730
|
+
calls: module.calls,
|
|
731
|
+
errors: module.errors,
|
|
732
|
+
totalDurationMs: round(module.totalDurationMs),
|
|
733
|
+
p95DurationMs: round(percentile(module.recentDurationsMs, 95)),
|
|
734
|
+
totalCpuMs: round(module.totalCpuMs),
|
|
735
|
+
positiveHeapDeltaBytes: Math.round(module.positiveHeapDeltaBytes),
|
|
736
|
+
positiveRssDeltaBytes: Math.round(module.positiveRssDeltaBytes),
|
|
737
|
+
surfaces: surfaceSummaries(operations),
|
|
738
|
+
topOperations: operations
|
|
739
|
+
.slice()
|
|
740
|
+
.sort((a, b) => (b.totalCpuUserMs + b.totalCpuSystemMs) - (a.totalCpuUserMs + a.totalCpuSystemMs) || b.totalDurationMs - a.totalDurationMs)
|
|
741
|
+
.slice(0, DEFAULT_TOP_OPERATIONS_LIMIT),
|
|
742
|
+
}
|
|
743
|
+
return {
|
|
744
|
+
...summaryBase,
|
|
745
|
+
candidateReasons: buildCandidateReasons(summaryBase, thresholds),
|
|
746
|
+
}
|
|
747
|
+
}).sort((a, b) =>
|
|
748
|
+
b.candidateReasons.length - a.candidateReasons.length
|
|
749
|
+
|| b.totalCpuMs - a.totalCpuMs
|
|
750
|
+
|| b.positiveRssDeltaBytes - a.positiveRssDeltaBytes
|
|
751
|
+
|| b.calls - a.calls
|
|
752
|
+
)
|
|
753
|
+
return {
|
|
754
|
+
bucketStart: new Date(bucket.bucketStartMs).toISOString(),
|
|
755
|
+
bucketEnd: new Date(bucket.bucketEndMs).toISOString(),
|
|
756
|
+
bucketIntervalMs: bucket.bucketEndMs - bucket.bucketStartMs,
|
|
757
|
+
stage: moduleResourceUsageBucketStage(bucket.bucketStartMs, bucket.bucketEndMs, startedAtMs),
|
|
758
|
+
partial: bucket.bucketStartMs < startedAtMs || nowMs < bucket.bucketEndMs,
|
|
759
|
+
totals: timeBucketTotals(modules),
|
|
760
|
+
modules,
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
function serializeTimeBuckets(state: ResourceUsageState): ModuleResourceUsageTimeBucket[] {
|
|
765
|
+
const nowMs = Date.now()
|
|
766
|
+
const startedAtMs = Date.parse(state.startedAt)
|
|
767
|
+
return Array.from(state.buckets.values())
|
|
768
|
+
.sort((a, b) => a.bucketStartMs - b.bucketStartMs)
|
|
769
|
+
.map((bucket) => serializeTimeBucket(bucket, nowMs, Number.isFinite(startedAtMs) ? startedAtMs : nowMs))
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
function getSnapshotDir(): string {
|
|
773
|
+
const configured = process.env.OM_MODULE_RESOURCE_USAGE_DIR?.trim()
|
|
774
|
+
return configured ? path.resolve(configured) : path.resolve(process.cwd(), '.mercato/module-resource-usage')
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
function getSnapshotPath(): string {
|
|
778
|
+
return path.join(getSnapshotDir(), `process-${process.pid}.json`)
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
function registerSnapshotShutdownHook(state: ResourceUsageState): void {
|
|
782
|
+
if (state.shutdownHookRegistered) return
|
|
783
|
+
const flush = () => {
|
|
784
|
+
try {
|
|
785
|
+
flushModuleResourceUsageSnapshot(true)
|
|
786
|
+
} catch {
|
|
787
|
+
// Best-effort diagnostic snapshot; never block process shutdown.
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
process.once('beforeExit', flush)
|
|
791
|
+
state.shutdownHookRegistered = true
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
// Fire-and-forget: nothing in this process depends on the write completing (readers always
|
|
795
|
+
// skip their own pid's file — see readSnapshotPayloads), so there's no reason for the write to
|
|
796
|
+
// block the tracked call, event dispatch, or report request that triggered it. This mirrors the
|
|
797
|
+
// existing fire-and-forget queue-close pattern in registerProducerShutdownHook (bus.ts): Node's
|
|
798
|
+
// event loop stays alive for the pending promise, so even the beforeExit-triggered flush below
|
|
799
|
+
// still completes before the process actually exits, as long as nothing calls process.exit()
|
|
800
|
+
// first.
|
|
801
|
+
let snapshotWriteSequence = 0
|
|
802
|
+
|
|
803
|
+
// Writes atomically: serialize to a unique temp file, then rename onto the final path. rename is
|
|
804
|
+
// atomic on POSIX, so a concurrent reader (this process's report, another process's directory scan,
|
|
805
|
+
// or a test) always observes either the previous complete file or the new complete file — never the
|
|
806
|
+
// empty/partial content window that a direct writeFile to snapshotPath would briefly expose.
|
|
807
|
+
function writeSnapshotFileAsync(dir: string, snapshotPath: string, payload: unknown): void {
|
|
808
|
+
const tempPath = `${snapshotPath}.${process.pid}.${snapshotWriteSequence++}.tmp`
|
|
809
|
+
fs.promises.mkdir(dir, { recursive: true })
|
|
810
|
+
.then(() => fs.promises.writeFile(tempPath, JSON.stringify(payload, null, 2)))
|
|
811
|
+
.then(() => fs.promises.rename(tempPath, snapshotPath))
|
|
812
|
+
.catch(() => {
|
|
813
|
+
// Snapshot files are diagnostic only; in-memory reporting still works. Best-effort cleanup of
|
|
814
|
+
// the temp file if the rename never happened; ignore if it is already gone.
|
|
815
|
+
fs.promises.unlink(tempPath).catch(() => {})
|
|
816
|
+
})
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
function flushModuleResourceUsageSnapshot(force: boolean): void {
|
|
820
|
+
if (!isSnapshotEnabled()) return
|
|
821
|
+
const state = getState()
|
|
822
|
+
const now = Date.now()
|
|
823
|
+
if (!force && now - state.lastSnapshotAt < SNAPSHOT_THROTTLE_MS) return
|
|
824
|
+
state.lastSnapshotAt = now
|
|
825
|
+
try {
|
|
826
|
+
const payload = {
|
|
827
|
+
pid: process.pid,
|
|
828
|
+
generatedAt: new Date().toISOString(),
|
|
829
|
+
startedAt: state.startedAt,
|
|
830
|
+
entries: Array.from(state.entries.values()).map(serializeEntry),
|
|
831
|
+
buckets: serializeTimeBuckets(state),
|
|
832
|
+
}
|
|
833
|
+
writeSnapshotFileAsync(getSnapshotDir(), getSnapshotPath(), payload)
|
|
834
|
+
} catch {
|
|
835
|
+
// Snapshot files are diagnostic only; in-memory reporting still works.
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
type SnapshotPayload = {
|
|
840
|
+
pid?: unknown
|
|
841
|
+
generatedAt?: unknown
|
|
842
|
+
startedAt?: unknown
|
|
843
|
+
entries?: unknown
|
|
844
|
+
buckets?: unknown
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
function isFreshSnapshot(payload: SnapshotPayload): boolean {
|
|
848
|
+
if (typeof payload.generatedAt !== 'string') return false
|
|
849
|
+
const generatedAt = Date.parse(payload.generatedAt)
|
|
850
|
+
return Number.isFinite(generatedAt) && Date.now() - generatedAt <= SNAPSHOT_MAX_AGE_MS
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
// Reads every other process's snapshot file exactly once. Both readSnapshotEntries and
|
|
854
|
+
// readSnapshotBuckets used to each force a flush and independently re-scan/re-parse the whole
|
|
855
|
+
// directory; callers now do that single scan once (see getModuleResourceUsageReport) and pass
|
|
856
|
+
// the result to both.
|
|
857
|
+
// Restarts/redeploys/autoscaling leave one snapshot file behind per historical process. Prune
|
|
858
|
+
// stale ones opportunistically here (best-effort, never blocks the read) so the directory doesn't
|
|
859
|
+
// grow without bound in a long-running deployment.
|
|
860
|
+
function pruneStaleSnapshotFile(filePath: string): void {
|
|
861
|
+
fs.promises.unlink(filePath).catch(() => {
|
|
862
|
+
// Snapshot files are diagnostic only; leave it for a later pass if it can't be removed now.
|
|
863
|
+
})
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
function readSnapshotPayloads(): SnapshotPayload[] {
|
|
867
|
+
if (!isSnapshotEnabled()) return []
|
|
868
|
+
const dir = getSnapshotDir()
|
|
869
|
+
if (!fs.existsSync(dir)) return []
|
|
870
|
+
const payloads: SnapshotPayload[] = []
|
|
871
|
+
for (const file of fs.readdirSync(dir)) {
|
|
872
|
+
if (!file.endsWith('.json')) continue
|
|
873
|
+
const filePath = path.join(dir, file)
|
|
874
|
+
try {
|
|
875
|
+
const payload = JSON.parse(fs.readFileSync(filePath, 'utf8')) as SnapshotPayload
|
|
876
|
+
if (payload.pid === process.pid) continue
|
|
877
|
+
if (!isFreshSnapshot(payload)) {
|
|
878
|
+
pruneStaleSnapshotFile(filePath)
|
|
879
|
+
continue
|
|
880
|
+
}
|
|
881
|
+
payloads.push(payload)
|
|
882
|
+
} catch {
|
|
883
|
+
// Ignore malformed or concurrently-written snapshots.
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
return payloads
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
function readSnapshotEntries(payloads: SnapshotPayload[]): ModuleResourceUsageEntry[] {
|
|
890
|
+
const entries: ModuleResourceUsageEntry[] = []
|
|
891
|
+
for (const payload of payloads) {
|
|
892
|
+
if (!Array.isArray(payload.entries)) continue
|
|
893
|
+
for (const entry of payload.entries) {
|
|
894
|
+
if (!entry || typeof entry !== 'object') continue
|
|
895
|
+
const candidate = entry as Partial<ModuleResourceUsageEntry>
|
|
896
|
+
if (typeof candidate.moduleId !== 'string' || typeof candidate.operation !== 'string') continue
|
|
897
|
+
if (!candidate.surface) continue
|
|
898
|
+
entries.push(candidate as ModuleResourceUsageEntry)
|
|
899
|
+
}
|
|
900
|
+
}
|
|
901
|
+
return entries
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
function readSnapshotBuckets(payloads: SnapshotPayload[]): ModuleResourceUsageTimeBucket[] {
|
|
905
|
+
const buckets: ModuleResourceUsageTimeBucket[] = []
|
|
906
|
+
for (const payload of payloads) {
|
|
907
|
+
try {
|
|
908
|
+
if (!Array.isArray(payload.buckets)) continue
|
|
909
|
+
for (const bucket of payload.buckets) {
|
|
910
|
+
if (!bucket || typeof bucket !== 'object') continue
|
|
911
|
+
const candidate = bucket as Partial<ModuleResourceUsageTimeBucket>
|
|
912
|
+
if (typeof candidate.bucketStart !== 'string' || typeof candidate.bucketEnd !== 'string') continue
|
|
913
|
+
if (!Array.isArray(candidate.modules)) continue
|
|
914
|
+
const modules = (candidate.modules as Partial<ModuleResourceUsageTimeBucketModule>[]).map(normalizeTimeBucketModule)
|
|
915
|
+
buckets.push({
|
|
916
|
+
bucketStart: candidate.bucketStart,
|
|
917
|
+
bucketEnd: candidate.bucketEnd,
|
|
918
|
+
bucketIntervalMs: normalizeBucketIntervalMs(candidate),
|
|
919
|
+
stage: candidate.stage === 'startup' || candidate.stage === 'running'
|
|
920
|
+
? candidate.stage
|
|
921
|
+
: moduleResourceUsageBucketStage(
|
|
922
|
+
Date.parse(candidate.bucketStart),
|
|
923
|
+
Date.parse(candidate.bucketEnd),
|
|
924
|
+
typeof payload.startedAt === 'string' ? Date.parse(payload.startedAt) : Date.parse(candidate.bucketStart),
|
|
925
|
+
),
|
|
926
|
+
partial: candidate.partial === true,
|
|
927
|
+
totals: candidate.totals ?? timeBucketTotals(modules),
|
|
928
|
+
modules,
|
|
929
|
+
})
|
|
930
|
+
}
|
|
931
|
+
} catch {
|
|
932
|
+
// Ignore malformed or concurrently-written snapshots.
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
return buckets
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
function normalizeBucketIntervalMs(bucket: Partial<ModuleResourceUsageTimeBucket>): number {
|
|
939
|
+
if (Number.isFinite(bucket.bucketIntervalMs) && Number(bucket.bucketIntervalMs) > 0) {
|
|
940
|
+
return Number(bucket.bucketIntervalMs)
|
|
941
|
+
}
|
|
942
|
+
const bucketStartMs = typeof bucket.bucketStart === 'string' ? Date.parse(bucket.bucketStart) : NaN
|
|
943
|
+
const bucketEndMs = typeof bucket.bucketEnd === 'string' ? Date.parse(bucket.bucketEnd) : NaN
|
|
944
|
+
const inferred = bucketEndMs - bucketStartMs
|
|
945
|
+
return Number.isFinite(inferred) && inferred > 0 ? inferred : moduleResourceUsageBucketIntervalMs()
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
function normalizeTimeBucketModule(module: Partial<ModuleResourceUsageTimeBucketModule>): ModuleResourceUsageTimeBucketModule {
|
|
949
|
+
return {
|
|
950
|
+
moduleId: typeof module.moduleId === 'string' ? module.moduleId : 'unknown',
|
|
951
|
+
calls: Number.isFinite(module.calls) ? Number(module.calls) : 0,
|
|
952
|
+
errors: Number.isFinite(module.errors) ? Number(module.errors) : 0,
|
|
953
|
+
totalDurationMs: Number.isFinite(module.totalDurationMs) ? Number(module.totalDurationMs) : 0,
|
|
954
|
+
p95DurationMs: Number.isFinite(module.p95DurationMs) ? Number(module.p95DurationMs) : 0,
|
|
955
|
+
totalCpuMs: Number.isFinite(module.totalCpuMs) ? Number(module.totalCpuMs) : 0,
|
|
956
|
+
positiveHeapDeltaBytes: Number.isFinite(module.positiveHeapDeltaBytes) ? Number(module.positiveHeapDeltaBytes) : 0,
|
|
957
|
+
positiveRssDeltaBytes: Number.isFinite(module.positiveRssDeltaBytes) ? Number(module.positiveRssDeltaBytes) : 0,
|
|
958
|
+
surfaces: Array.isArray(module.surfaces) ? module.surfaces : [],
|
|
959
|
+
topOperations: Array.isArray(module.topOperations) ? module.topOperations : [],
|
|
960
|
+
candidateReasons: Array.isArray(module.candidateReasons) ? module.candidateReasons : [],
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
function mergeEntries(entries: ModuleResourceUsageEntry[]): ModuleResourceUsageEntry[] {
|
|
965
|
+
const merged = new Map<string, ModuleResourceUsageEntry>()
|
|
966
|
+
for (const entry of entries) {
|
|
967
|
+
const key = entryKey({ moduleId: entry.moduleId, surface: entry.surface, operation: entry.operation })
|
|
968
|
+
const existing = merged.get(key)
|
|
969
|
+
if (!existing) {
|
|
970
|
+
merged.set(key, {
|
|
971
|
+
...entry,
|
|
972
|
+
// Snapshot files written by an older process before concurrentCalls existed lack it.
|
|
973
|
+
concurrentCalls: Number.isFinite(entry.concurrentCalls) ? entry.concurrentCalls : 0,
|
|
974
|
+
})
|
|
975
|
+
continue
|
|
976
|
+
}
|
|
977
|
+
existing.calls += entry.calls
|
|
978
|
+
existing.errors += entry.errors
|
|
979
|
+
existing.concurrentCalls += Number.isFinite(entry.concurrentCalls) ? entry.concurrentCalls : 0
|
|
980
|
+
existing.totalDurationMs = round(existing.totalDurationMs + entry.totalDurationMs)
|
|
981
|
+
existing.maxDurationMs = Math.max(existing.maxDurationMs, entry.maxDurationMs)
|
|
982
|
+
existing.p95DurationMs = Math.max(existing.p95DurationMs, entry.p95DurationMs)
|
|
983
|
+
existing.totalCpuUserMs = round(existing.totalCpuUserMs + entry.totalCpuUserMs)
|
|
984
|
+
existing.totalCpuSystemMs = round(existing.totalCpuSystemMs + entry.totalCpuSystemMs)
|
|
985
|
+
existing.maxCpuMs = Math.max(existing.maxCpuMs, entry.maxCpuMs)
|
|
986
|
+
existing.totalHeapDeltaBytes += entry.totalHeapDeltaBytes
|
|
987
|
+
existing.positiveHeapDeltaBytes += entry.positiveHeapDeltaBytes
|
|
988
|
+
existing.maxHeapDeltaBytes = Math.max(existing.maxHeapDeltaBytes, entry.maxHeapDeltaBytes)
|
|
989
|
+
existing.totalRssDeltaBytes += entry.totalRssDeltaBytes
|
|
990
|
+
existing.positiveRssDeltaBytes += entry.positiveRssDeltaBytes
|
|
991
|
+
existing.maxRssDeltaBytes = Math.max(existing.maxRssDeltaBytes, entry.maxRssDeltaBytes)
|
|
992
|
+
existing.firstSeenAt = existing.firstSeenAt < entry.firstSeenAt ? existing.firstSeenAt : entry.firstSeenAt
|
|
993
|
+
existing.lastSeenAt = existing.lastSeenAt > entry.lastSeenAt ? existing.lastSeenAt : entry.lastSeenAt
|
|
994
|
+
}
|
|
995
|
+
return Array.from(merged.values())
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
function mergeSurfaceSummaries(surfaces: ModuleResourceUsageSurfaceSummary[]): ModuleResourceUsageSurfaceSummary[] {
|
|
999
|
+
const merged = new Map<ModuleResourceSurface, ModuleResourceUsageSurfaceSummary>()
|
|
1000
|
+
for (const surface of surfaces) {
|
|
1001
|
+
const existing = merged.get(surface.surface)
|
|
1002
|
+
if (!existing) {
|
|
1003
|
+
merged.set(surface.surface, { ...surface })
|
|
1004
|
+
continue
|
|
1005
|
+
}
|
|
1006
|
+
existing.calls += surface.calls
|
|
1007
|
+
existing.errors += surface.errors
|
|
1008
|
+
existing.totalDurationMs = round(existing.totalDurationMs + surface.totalDurationMs)
|
|
1009
|
+
existing.p95DurationMs = Math.max(existing.p95DurationMs, surface.p95DurationMs)
|
|
1010
|
+
existing.totalCpuMs = round(existing.totalCpuMs + surface.totalCpuMs)
|
|
1011
|
+
existing.positiveHeapDeltaBytes += surface.positiveHeapDeltaBytes
|
|
1012
|
+
existing.positiveRssDeltaBytes += surface.positiveRssDeltaBytes
|
|
1013
|
+
}
|
|
1014
|
+
return Array.from(merged.values()).sort((a, b) => b.totalCpuMs - a.totalCpuMs || b.totalDurationMs - a.totalDurationMs)
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
function mergeOperationEntries(entries: ModuleResourceUsageEntry[]): ModuleResourceUsageEntry[] {
|
|
1018
|
+
return mergeEntries(entries)
|
|
1019
|
+
.sort((a, b) => (b.totalCpuUserMs + b.totalCpuSystemMs) - (a.totalCpuUserMs + a.totalCpuSystemMs) || b.totalDurationMs - a.totalDurationMs)
|
|
1020
|
+
.slice(0, DEFAULT_TOP_OPERATIONS_LIMIT)
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
function mergeTimeBuckets(buckets: ModuleResourceUsageTimeBucket[]): ModuleResourceUsageTimeBucket[] {
|
|
1024
|
+
const merged = new Map<string, ModuleResourceUsageTimeBucket>()
|
|
1025
|
+
const thresholds = getModuleResourceUsageThresholds()
|
|
1026
|
+
for (const bucket of buckets) {
|
|
1027
|
+
const bucketStartMs = Date.parse(bucket.bucketStart)
|
|
1028
|
+
if (!Number.isFinite(bucketStartMs)) continue
|
|
1029
|
+
const bucketIntervalMs = normalizeBucketIntervalMs(bucket)
|
|
1030
|
+
const key = `${bucket.bucketStart}\u0000${bucketIntervalMs}`
|
|
1031
|
+
const existing = merged.get(key)
|
|
1032
|
+
if (!existing) {
|
|
1033
|
+
const modules = bucket.modules.map((module) => {
|
|
1034
|
+
const normalizedModule = normalizeTimeBucketModule(module)
|
|
1035
|
+
return {
|
|
1036
|
+
...normalizedModule,
|
|
1037
|
+
candidateReasons: buildCandidateReasons(normalizedModule, thresholds),
|
|
1038
|
+
}
|
|
1039
|
+
})
|
|
1040
|
+
merged.set(key, {
|
|
1041
|
+
bucketStart: bucket.bucketStart,
|
|
1042
|
+
bucketEnd: bucket.bucketEnd,
|
|
1043
|
+
bucketIntervalMs,
|
|
1044
|
+
stage: bucket.stage,
|
|
1045
|
+
partial: bucket.partial,
|
|
1046
|
+
totals: timeBucketTotals(modules),
|
|
1047
|
+
modules,
|
|
1048
|
+
})
|
|
1049
|
+
continue
|
|
1050
|
+
}
|
|
1051
|
+
existing.partial = existing.partial || bucket.partial
|
|
1052
|
+
if (existing.stage !== 'startup') existing.stage = bucket.stage
|
|
1053
|
+
const modulesById = new Map(existing.modules.map((module) => [module.moduleId, module]))
|
|
1054
|
+
for (const module of bucket.modules) {
|
|
1055
|
+
const normalizedModule = normalizeTimeBucketModule(module)
|
|
1056
|
+
const current = modulesById.get(normalizedModule.moduleId)
|
|
1057
|
+
if (!current) {
|
|
1058
|
+
const created = {
|
|
1059
|
+
...normalizedModule,
|
|
1060
|
+
candidateReasons: buildCandidateReasons(normalizedModule, thresholds),
|
|
1061
|
+
}
|
|
1062
|
+
existing.modules.push(created)
|
|
1063
|
+
modulesById.set(normalizedModule.moduleId, created)
|
|
1064
|
+
continue
|
|
1065
|
+
}
|
|
1066
|
+
current.calls += normalizedModule.calls
|
|
1067
|
+
current.errors += normalizedModule.errors
|
|
1068
|
+
current.totalDurationMs = round(current.totalDurationMs + normalizedModule.totalDurationMs)
|
|
1069
|
+
current.p95DurationMs = Math.max(current.p95DurationMs, normalizedModule.p95DurationMs)
|
|
1070
|
+
current.totalCpuMs = round(current.totalCpuMs + normalizedModule.totalCpuMs)
|
|
1071
|
+
current.positiveHeapDeltaBytes += normalizedModule.positiveHeapDeltaBytes
|
|
1072
|
+
current.positiveRssDeltaBytes += normalizedModule.positiveRssDeltaBytes
|
|
1073
|
+
current.surfaces = mergeSurfaceSummaries([...current.surfaces, ...normalizedModule.surfaces])
|
|
1074
|
+
current.topOperations = mergeOperationEntries([...current.topOperations, ...normalizedModule.topOperations])
|
|
1075
|
+
current.candidateReasons = buildCandidateReasons(current, thresholds)
|
|
1076
|
+
}
|
|
1077
|
+
existing.modules.sort((a, b) =>
|
|
1078
|
+
b.totalCpuMs - a.totalCpuMs
|
|
1079
|
+
|| b.positiveRssDeltaBytes - a.positiveRssDeltaBytes
|
|
1080
|
+
|| b.calls - a.calls
|
|
1081
|
+
)
|
|
1082
|
+
existing.totals = timeBucketTotals(existing.modules)
|
|
1083
|
+
}
|
|
1084
|
+
return Array.from(merged.values())
|
|
1085
|
+
.sort((a, b) => a.bucketStart.localeCompare(b.bucketStart))
|
|
1086
|
+
.slice(-moduleResourceUsageBucketLimit())
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
function readNumberEnv(name: string, fallback: number): number {
|
|
1090
|
+
return parseNumberWithDefault(process.env[name], fallback, { min: 0 })
|
|
1091
|
+
}
|
|
1092
|
+
|
|
1093
|
+
function moduleResourceUsageBucketIntervalMs(): number {
|
|
1094
|
+
return MODULE_RESOURCE_USAGE_BUCKET_INTERVAL_MS
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
function moduleResourceUsageBucketLimit(): number {
|
|
1098
|
+
return Math.max(1, Math.ceil(TIME_BUCKET_RETENTION_MS / moduleResourceUsageBucketIntervalMs()))
|
|
1099
|
+
}
|
|
1100
|
+
|
|
1101
|
+
export function getModuleResourceUsageThresholds(): ModuleResourceUsageThresholds {
|
|
1102
|
+
return {
|
|
1103
|
+
p95DurationMs: readNumberEnv('OM_MODULE_RESOURCE_HEAVY_P95_MS', 5_000),
|
|
1104
|
+
cpuMs: readNumberEnv('OM_MODULE_RESOURCE_HEAVY_CPU_MS', 25_000),
|
|
1105
|
+
positiveHeapDeltaBytes: readNumberEnv('OM_MODULE_RESOURCE_HEAVY_HEAP_BYTES', 250 * 1024 * 1024),
|
|
1106
|
+
positiveRssDeltaBytes: readNumberEnv('OM_MODULE_RESOURCE_HEAVY_RSS_BYTES', 250 * 1024 * 1024),
|
|
1107
|
+
errors: readNumberEnv('OM_MODULE_RESOURCE_HEAVY_ERRORS', 10),
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
function buildCandidateReasons(
|
|
1112
|
+
summary: Omit<ModuleResourceUsageModuleSummary, 'candidateReasons'>,
|
|
1113
|
+
thresholds: ModuleResourceUsageThresholds,
|
|
1114
|
+
): string[] {
|
|
1115
|
+
const reasons: string[] = []
|
|
1116
|
+
if (summary.p95DurationMs >= thresholds.p95DurationMs) reasons.push('p95_duration')
|
|
1117
|
+
if (summary.totalCpuMs >= thresholds.cpuMs) reasons.push('cpu')
|
|
1118
|
+
if (summary.positiveHeapDeltaBytes >= thresholds.positiveHeapDeltaBytes) reasons.push('heap_allocations')
|
|
1119
|
+
if (summary.positiveRssDeltaBytes >= thresholds.positiveRssDeltaBytes) reasons.push('rss_growth')
|
|
1120
|
+
if (summary.errors >= thresholds.errors) reasons.push('errors')
|
|
1121
|
+
return reasons
|
|
1122
|
+
}
|
|
1123
|
+
|
|
1124
|
+
export function getModuleResourceUsageReport(): ModuleResourceUsageReport {
|
|
1125
|
+
const state = getState()
|
|
1126
|
+
// Force our own snapshot to disk once (fire-and-forget — see writeSnapshotFileAsync) so other
|
|
1127
|
+
// processes viewing the report around the same time see fresh data, then read every other
|
|
1128
|
+
// process's snapshot file exactly once and derive both entries and buckets from that one scan.
|
|
1129
|
+
flushModuleResourceUsageSnapshot(true)
|
|
1130
|
+
const snapshotPayloads = readSnapshotPayloads()
|
|
1131
|
+
const entries = mergeEntries([
|
|
1132
|
+
...Array.from(state.entries.values()).map(serializeEntry),
|
|
1133
|
+
...readSnapshotEntries(snapshotPayloads),
|
|
1134
|
+
])
|
|
1135
|
+
const buckets = mergeTimeBuckets([
|
|
1136
|
+
...serializeTimeBuckets(state),
|
|
1137
|
+
...readSnapshotBuckets(snapshotPayloads),
|
|
1138
|
+
])
|
|
1139
|
+
const thresholds = getModuleResourceUsageThresholds()
|
|
1140
|
+
const modules = new Map<string, ModuleResourceUsageEntry[]>()
|
|
1141
|
+
for (const entry of entries) {
|
|
1142
|
+
const list = modules.get(entry.moduleId) ?? []
|
|
1143
|
+
list.push(entry)
|
|
1144
|
+
modules.set(entry.moduleId, list)
|
|
1145
|
+
}
|
|
1146
|
+
|
|
1147
|
+
const moduleSummaries: ModuleResourceUsageModuleSummary[] = []
|
|
1148
|
+
for (const [moduleId, moduleEntries] of modules) {
|
|
1149
|
+
const surfaces = new Map<ModuleResourceSurface, ModuleResourceUsageEntry[]>()
|
|
1150
|
+
for (const entry of moduleEntries) {
|
|
1151
|
+
const list = surfaces.get(entry.surface) ?? []
|
|
1152
|
+
list.push(entry)
|
|
1153
|
+
surfaces.set(entry.surface, list)
|
|
1154
|
+
}
|
|
1155
|
+
const totalDurationMs = moduleEntries.reduce((sum, entry) => sum + entry.totalDurationMs, 0)
|
|
1156
|
+
const totalCpuMs = moduleEntries.reduce((sum, entry) => sum + entry.totalCpuUserMs + entry.totalCpuSystemMs, 0)
|
|
1157
|
+
const summaryBase = {
|
|
1158
|
+
moduleId,
|
|
1159
|
+
calls: moduleEntries.reduce((sum, entry) => sum + entry.calls, 0),
|
|
1160
|
+
errors: moduleEntries.reduce((sum, entry) => sum + entry.errors, 0),
|
|
1161
|
+
totalDurationMs: round(totalDurationMs),
|
|
1162
|
+
p95DurationMs: round(Math.max(...moduleEntries.map((entry) => entry.p95DurationMs), 0)),
|
|
1163
|
+
totalCpuMs: round(totalCpuMs),
|
|
1164
|
+
positiveHeapDeltaBytes: moduleEntries.reduce((sum, entry) => sum + entry.positiveHeapDeltaBytes, 0),
|
|
1165
|
+
positiveRssDeltaBytes: moduleEntries.reduce((sum, entry) => sum + entry.positiveRssDeltaBytes, 0),
|
|
1166
|
+
surfaces: Array.from(surfaces.entries()).map(([surface, surfaceEntries]) => ({
|
|
1167
|
+
surface,
|
|
1168
|
+
calls: surfaceEntries.reduce((sum, entry) => sum + entry.calls, 0),
|
|
1169
|
+
errors: surfaceEntries.reduce((sum, entry) => sum + entry.errors, 0),
|
|
1170
|
+
totalDurationMs: round(surfaceEntries.reduce((sum, entry) => sum + entry.totalDurationMs, 0)),
|
|
1171
|
+
p95DurationMs: round(Math.max(...surfaceEntries.map((entry) => entry.p95DurationMs), 0)),
|
|
1172
|
+
totalCpuMs: round(surfaceEntries.reduce((sum, entry) => sum + entry.totalCpuUserMs + entry.totalCpuSystemMs, 0)),
|
|
1173
|
+
positiveHeapDeltaBytes: surfaceEntries.reduce((sum, entry) => sum + entry.positiveHeapDeltaBytes, 0),
|
|
1174
|
+
positiveRssDeltaBytes: surfaceEntries.reduce((sum, entry) => sum + entry.positiveRssDeltaBytes, 0),
|
|
1175
|
+
})).sort((a, b) => b.totalCpuMs - a.totalCpuMs || b.totalDurationMs - a.totalDurationMs),
|
|
1176
|
+
topOperations: moduleEntries
|
|
1177
|
+
.slice()
|
|
1178
|
+
.sort((a, b) => (b.totalCpuUserMs + b.totalCpuSystemMs) - (a.totalCpuUserMs + a.totalCpuSystemMs) || b.totalDurationMs - a.totalDurationMs)
|
|
1179
|
+
.slice(0, DEFAULT_TOP_OPERATIONS_LIMIT),
|
|
1180
|
+
}
|
|
1181
|
+
moduleSummaries.push({
|
|
1182
|
+
...summaryBase,
|
|
1183
|
+
candidateReasons: buildCandidateReasons(summaryBase, thresholds),
|
|
1184
|
+
})
|
|
1185
|
+
}
|
|
1186
|
+
|
|
1187
|
+
const sortedModules = moduleSummaries.sort((a, b) =>
|
|
1188
|
+
b.candidateReasons.length - a.candidateReasons.length
|
|
1189
|
+
|| b.totalCpuMs - a.totalCpuMs
|
|
1190
|
+
|| b.totalDurationMs - a.totalDurationMs
|
|
1191
|
+
)
|
|
1192
|
+
|
|
1193
|
+
return {
|
|
1194
|
+
generatedAt: new Date().toISOString(),
|
|
1195
|
+
startedAt: state.startedAt,
|
|
1196
|
+
enabled: isModuleResourceUsageEnabled(),
|
|
1197
|
+
bucketIntervalMs: moduleResourceUsageBucketIntervalMs(),
|
|
1198
|
+
totals: {
|
|
1199
|
+
modules: sortedModules.length,
|
|
1200
|
+
operations: entries.length,
|
|
1201
|
+
calls: entries.reduce((sum, entry) => sum + entry.calls, 0),
|
|
1202
|
+
errors: entries.reduce((sum, entry) => sum + entry.errors, 0),
|
|
1203
|
+
totalDurationMs: round(entries.reduce((sum, entry) => sum + entry.totalDurationMs, 0)),
|
|
1204
|
+
totalCpuMs: round(entries.reduce((sum, entry) => sum + entry.totalCpuUserMs + entry.totalCpuSystemMs, 0)),
|
|
1205
|
+
positiveHeapDeltaBytes: entries.reduce((sum, entry) => sum + entry.positiveHeapDeltaBytes, 0),
|
|
1206
|
+
positiveRssDeltaBytes: entries.reduce((sum, entry) => sum + entry.positiveRssDeltaBytes, 0),
|
|
1207
|
+
},
|
|
1208
|
+
thresholds,
|
|
1209
|
+
modules: sortedModules,
|
|
1210
|
+
candidates: sortedModules.filter((module) => module.candidateReasons.length > 0),
|
|
1211
|
+
buckets,
|
|
1212
|
+
}
|
|
1213
|
+
}
|