@open-mercato/shared 0.6.6-develop.6383.1.27fcc83455 → 0.6.6-develop.6385.1.9a81faa5f0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1213 @@
1
+ import fs from 'node:fs'
2
+ import path from 'node:path'
3
+ import { parseBooleanWithDefault } from '../boolean'
4
+ import { parseNumberWithDefault } from '../number'
5
+
6
+ const GLOBAL_KEY = '__openMercatoModuleResourceUsage__'
7
+ const NS_PER_MS = 1_000_000
8
+ const MICROS_PER_MS = 1000
9
+ const DEFAULT_RECENT_SAMPLE_LIMIT = 128
10
+ const DEFAULT_TOP_OPERATIONS_LIMIT = 5
11
+ const TIME_BUCKET_RETENTION_MS = 24 * 60 * 60 * 1000
12
+ const SNAPSHOT_THROTTLE_MS = 5_000
13
+ const SNAPSHOT_MAX_AGE_MS = 24 * 60 * 60 * 1000
14
+ const MINUTE_MS = 60 * 1000
15
+ const MODULE_RESOURCE_USAGE_BUCKET_INTERVAL_MS = 5 * MINUTE_MS
16
+ const MODULE_RESOURCE_USAGE_STARTUP_STAGE_MS = 5 * MINUTE_MS
17
+
18
+ export type ModuleResourceSurface = 'api' | 'subscriber' | 'worker' | 'custom'
19
+
20
+ export type ModuleResourceUsageInput = {
21
+ moduleId?: string | null
22
+ surface: ModuleResourceSurface
23
+ operation: string
24
+ resourceId?: string | null
25
+ }
26
+
27
+ export type ModuleResourceUsageEntry = {
28
+ moduleId: string
29
+ surface: ModuleResourceSurface
30
+ operation: string
31
+ resourceId: string | null
32
+ calls: number
33
+ errors: number
34
+ // How many of `calls` overlapped in wall-clock time with another tracked call anywhere in
35
+ // the process. process.cpuUsage()/process.memoryUsage() are process-wide, so a call's own
36
+ // CPU/heap/RSS deltas can include work actually done by a concurrent call during that window.
37
+ // A high concurrentCalls share means the CPU/heap/RSS numbers here are less trustworthy as
38
+ // this specific operation's own cost.
39
+ concurrentCalls: number
40
+ totalDurationMs: number
41
+ maxDurationMs: number
42
+ p95DurationMs: number
43
+ totalCpuUserMs: number
44
+ totalCpuSystemMs: number
45
+ maxCpuMs: number
46
+ totalHeapDeltaBytes: number
47
+ positiveHeapDeltaBytes: number
48
+ maxHeapDeltaBytes: number
49
+ totalRssDeltaBytes: number
50
+ positiveRssDeltaBytes: number
51
+ maxRssDeltaBytes: number
52
+ firstSeenAt: string
53
+ lastSeenAt: string
54
+ }
55
+
56
+ export type ModuleResourceUsageSurfaceSummary = {
57
+ surface: ModuleResourceSurface
58
+ calls: number
59
+ errors: number
60
+ totalDurationMs: number
61
+ p95DurationMs: number
62
+ totalCpuMs: number
63
+ positiveHeapDeltaBytes: number
64
+ positiveRssDeltaBytes: number
65
+ }
66
+
67
+ export type ModuleResourceUsageModuleSummary = {
68
+ moduleId: string
69
+ calls: number
70
+ errors: number
71
+ totalDurationMs: number
72
+ p95DurationMs: number
73
+ totalCpuMs: number
74
+ positiveHeapDeltaBytes: number
75
+ positiveRssDeltaBytes: number
76
+ surfaces: ModuleResourceUsageSurfaceSummary[]
77
+ topOperations: ModuleResourceUsageEntry[]
78
+ candidateReasons: string[]
79
+ }
80
+
81
+ export type ModuleResourceUsageTimeBucketModule = {
82
+ moduleId: string
83
+ calls: number
84
+ errors: number
85
+ totalDurationMs: number
86
+ p95DurationMs: number
87
+ totalCpuMs: number
88
+ positiveHeapDeltaBytes: number
89
+ positiveRssDeltaBytes: number
90
+ surfaces: ModuleResourceUsageSurfaceSummary[]
91
+ topOperations: ModuleResourceUsageEntry[]
92
+ candidateReasons: string[]
93
+ }
94
+
95
+ export type ModuleResourceUsageTimeBucket = {
96
+ bucketStart: string
97
+ bucketEnd: string
98
+ bucketIntervalMs: number
99
+ stage: 'startup' | 'running'
100
+ partial: boolean
101
+ totals: {
102
+ modules: number
103
+ calls: number
104
+ errors: number
105
+ totalDurationMs: number
106
+ totalCpuMs: number
107
+ positiveHeapDeltaBytes: number
108
+ positiveRssDeltaBytes: number
109
+ }
110
+ modules: ModuleResourceUsageTimeBucketModule[]
111
+ }
112
+
113
+ export type ModuleResourceUsageReport = {
114
+ generatedAt: string
115
+ startedAt: string
116
+ enabled: boolean
117
+ bucketIntervalMs: number
118
+ totals: {
119
+ modules: number
120
+ operations: number
121
+ calls: number
122
+ errors: number
123
+ totalDurationMs: number
124
+ totalCpuMs: number
125
+ positiveHeapDeltaBytes: number
126
+ positiveRssDeltaBytes: number
127
+ }
128
+ thresholds: ModuleResourceUsageThresholds
129
+ modules: ModuleResourceUsageModuleSummary[]
130
+ candidates: ModuleResourceUsageModuleSummary[]
131
+ buckets: ModuleResourceUsageTimeBucket[]
132
+ }
133
+
134
+ export type ModuleResourceUsageThresholds = {
135
+ p95DurationMs: number
136
+ cpuMs: number
137
+ positiveHeapDeltaBytes: number
138
+ positiveRssDeltaBytes: number
139
+ errors: number
140
+ }
141
+
142
+ type MutableEntry = Omit<ModuleResourceUsageEntry, 'p95DurationMs'> & {
143
+ recentDurationsMs: number[]
144
+ }
145
+
146
+ type MutableTimeBucketModule = Omit<ModuleResourceUsageTimeBucketModule, 'p95DurationMs' | 'surfaces' | 'topOperations' | 'candidateReasons'> & {
147
+ recentDurationsMs: number[]
148
+ operations: Map<string, MutableEntry>
149
+ }
150
+
151
+ type MutableTimeBucket = {
152
+ bucketStartMs: number
153
+ bucketEndMs: number
154
+ modules: Map<string, MutableTimeBucketModule>
155
+ }
156
+
157
+ // A live token per currently in-flight withModuleResourceUsage call. Used to detect when two
158
+ // tracked calls overlap in wall-clock time, since process.cpuUsage()/process.memoryUsage() can't
159
+ // distinguish which concurrent call actually did the work (see ModuleResourceUsageEntry.concurrentCalls).
160
+ type ActiveCallToken = { tainted: boolean }
161
+
162
+ type ResourceUsageState = {
163
+ startedAt: string
164
+ entries: Map<string, MutableEntry>
165
+ buckets: Map<string, MutableTimeBucket>
166
+ activeCalls: Set<ActiveCallToken>
167
+ lastSnapshotAt: number
168
+ shutdownHookRegistered: boolean
169
+ // The bucket interval buckets were last re-keyed for. migrateMutableTimeBuckets only needs to
170
+ // run again if this stops matching moduleResourceUsageBucketIntervalMs() (e.g. a dev HMR reload
171
+ // picked up a source change to the interval constant) — it's a hardcoded constant today, so in
172
+ // the common case this lets getState() skip rebuilding the whole buckets map on every call.
173
+ migratedIntervalMs: number
174
+ }
175
+
176
+ function getState(): ResourceUsageState {
177
+ const existing = (globalThis as Record<string, unknown>)[GLOBAL_KEY]
178
+ if (existing && typeof existing === 'object' && (existing as ResourceUsageState).entries instanceof Map) {
179
+ const state = existing as ResourceUsageState
180
+ if (!(state.buckets instanceof Map)) state.buckets = new Map()
181
+ if (!(state.activeCalls instanceof Set)) state.activeCalls = new Set()
182
+ const currentIntervalMs = moduleResourceUsageBucketIntervalMs()
183
+ if (state.migratedIntervalMs !== currentIntervalMs) {
184
+ migrateMutableTimeBuckets(state, currentIntervalMs)
185
+ state.migratedIntervalMs = currentIntervalMs
186
+ }
187
+ registerSnapshotShutdownHook(state)
188
+ return state
189
+ }
190
+ const state: ResourceUsageState = {
191
+ startedAt: new Date().toISOString(),
192
+ entries: new Map(),
193
+ buckets: new Map(),
194
+ activeCalls: new Set(),
195
+ lastSnapshotAt: 0,
196
+ shutdownHookRegistered: false,
197
+ migratedIntervalMs: moduleResourceUsageBucketIntervalMs(),
198
+ }
199
+ ;(globalThis as Record<string, unknown>)[GLOBAL_KEY] = state
200
+ registerSnapshotShutdownHook(state)
201
+ return state
202
+ }
203
+
204
+ function migrateMutableTimeBuckets(state: ResourceUsageState, intervalMs: number): void {
205
+ const migratedBuckets = new Map<string, MutableTimeBucket>()
206
+ for (const bucket of state.buckets.values()) {
207
+ if (!(bucket.modules instanceof Map)) continue
208
+ const bucketStartMs = Number.isFinite(bucket.bucketStartMs)
209
+ ? Math.floor(bucket.bucketStartMs / intervalMs) * intervalMs
210
+ : null
211
+ if (bucketStartMs === null) continue
212
+ const key = String(bucketStartMs)
213
+ const targetBucket = migratedBuckets.get(key) ?? {
214
+ bucketStartMs,
215
+ bucketEndMs: bucketStartMs + intervalMs,
216
+ modules: new Map(),
217
+ }
218
+ mergeMutableTimeBucket(targetBucket, bucket)
219
+ migratedBuckets.set(key, targetBucket)
220
+ }
221
+ state.buckets = migratedBuckets
222
+ }
223
+
224
+ function trimRecentDurations(values: number[]): number[] {
225
+ if (values.length <= DEFAULT_RECENT_SAMPLE_LIMIT) return values
226
+ return values.slice(values.length - DEFAULT_RECENT_SAMPLE_LIMIT)
227
+ }
228
+
229
+ function mergeMutableEntry(target: MutableEntry, source: MutableEntry): void {
230
+ target.calls += source.calls
231
+ target.errors += source.errors
232
+ target.concurrentCalls += source.concurrentCalls
233
+ target.totalDurationMs += source.totalDurationMs
234
+ target.maxDurationMs = Math.max(target.maxDurationMs, source.maxDurationMs)
235
+ target.totalCpuUserMs += source.totalCpuUserMs
236
+ target.totalCpuSystemMs += source.totalCpuSystemMs
237
+ target.maxCpuMs = Math.max(target.maxCpuMs, source.maxCpuMs)
238
+ target.totalHeapDeltaBytes += source.totalHeapDeltaBytes
239
+ target.positiveHeapDeltaBytes += source.positiveHeapDeltaBytes
240
+ target.maxHeapDeltaBytes = Math.max(target.maxHeapDeltaBytes, source.maxHeapDeltaBytes)
241
+ target.totalRssDeltaBytes += source.totalRssDeltaBytes
242
+ target.positiveRssDeltaBytes += source.positiveRssDeltaBytes
243
+ target.maxRssDeltaBytes = Math.max(target.maxRssDeltaBytes, source.maxRssDeltaBytes)
244
+ target.firstSeenAt = target.firstSeenAt < source.firstSeenAt ? target.firstSeenAt : source.firstSeenAt
245
+ target.lastSeenAt = target.lastSeenAt > source.lastSeenAt ? target.lastSeenAt : source.lastSeenAt
246
+ target.recentDurationsMs = trimRecentDurations([
247
+ ...target.recentDurationsMs,
248
+ ...(Array.isArray(source.recentDurationsMs) ? source.recentDurationsMs : []),
249
+ ])
250
+ }
251
+
252
+ function cloneMutableEntry(entry: MutableEntry): MutableEntry {
253
+ return {
254
+ ...entry,
255
+ recentDurationsMs: trimRecentDurations(Array.isArray(entry.recentDurationsMs) ? [...entry.recentDurationsMs] : []),
256
+ }
257
+ }
258
+
259
+ function mergeMutableTimeBucket(targetBucket: MutableTimeBucket, sourceBucket: MutableTimeBucket): void {
260
+ for (const sourceModule of sourceBucket.modules.values()) {
261
+ if (!Array.isArray(sourceModule.recentDurationsMs)) sourceModule.recentDurationsMs = []
262
+ if (!(sourceModule.operations instanceof Map)) sourceModule.operations = new Map()
263
+ const targetModule = targetBucket.modules.get(sourceModule.moduleId)
264
+ if (!targetModule) {
265
+ targetBucket.modules.set(sourceModule.moduleId, {
266
+ moduleId: sourceModule.moduleId,
267
+ calls: sourceModule.calls,
268
+ errors: sourceModule.errors,
269
+ totalDurationMs: sourceModule.totalDurationMs,
270
+ totalCpuMs: sourceModule.totalCpuMs,
271
+ positiveHeapDeltaBytes: sourceModule.positiveHeapDeltaBytes,
272
+ positiveRssDeltaBytes: sourceModule.positiveRssDeltaBytes,
273
+ recentDurationsMs: trimRecentDurations([...sourceModule.recentDurationsMs]),
274
+ operations: new Map(Array.from(sourceModule.operations.entries()).map(([key, entry]) => [key, cloneMutableEntry(entry)])),
275
+ })
276
+ continue
277
+ }
278
+ targetModule.calls += sourceModule.calls
279
+ targetModule.errors += sourceModule.errors
280
+ targetModule.totalDurationMs += sourceModule.totalDurationMs
281
+ targetModule.totalCpuMs += sourceModule.totalCpuMs
282
+ targetModule.positiveHeapDeltaBytes += sourceModule.positiveHeapDeltaBytes
283
+ targetModule.positiveRssDeltaBytes += sourceModule.positiveRssDeltaBytes
284
+ targetModule.recentDurationsMs = trimRecentDurations([
285
+ ...targetModule.recentDurationsMs,
286
+ ...sourceModule.recentDurationsMs,
287
+ ])
288
+ for (const [operationKey, sourceOperation] of sourceModule.operations) {
289
+ const targetOperation = targetModule.operations.get(operationKey)
290
+ if (!targetOperation) {
291
+ targetModule.operations.set(operationKey, cloneMutableEntry(sourceOperation))
292
+ continue
293
+ }
294
+ mergeMutableEntry(targetOperation, sourceOperation)
295
+ }
296
+ }
297
+ }
298
+
299
+ export function resetModuleResourceUsage(): void {
300
+ ;(globalThis as Record<string, unknown>)[GLOBAL_KEY] = {
301
+ startedAt: new Date().toISOString(),
302
+ entries: new Map(),
303
+ buckets: new Map(),
304
+ activeCalls: new Set(),
305
+ lastSnapshotAt: 0,
306
+ shutdownHookRegistered: false,
307
+ migratedIntervalMs: moduleResourceUsageBucketIntervalMs(),
308
+ } satisfies ResourceUsageState
309
+ }
310
+
311
+ export function clearModuleResourceUsageData(): void {
312
+ resetModuleResourceUsage()
313
+ try {
314
+ fs.rmSync(getSnapshotDir(), { recursive: true, force: true })
315
+ } catch {
316
+ // Telemetry is diagnostic only; clearing in-memory state is still useful if files cannot be removed.
317
+ }
318
+ }
319
+
320
+ export function isModuleResourceUsageEnabled(): boolean {
321
+ return parseBooleanWithDefault(process.env.OM_MODULE_RESOURCE_USAGE, true)
322
+ }
323
+
324
+ function isSnapshotEnabled(): boolean {
325
+ return parseBooleanWithDefault(process.env.OM_MODULE_RESOURCE_USAGE_SNAPSHOT, process.env.NODE_ENV !== 'test')
326
+ }
327
+
328
+ export function inferModuleIdFromResourceId(resourceId: string | null | undefined): string | null {
329
+ if (!resourceId) return null
330
+ const trimmed = resourceId.trim()
331
+ if (!trimmed) return null
332
+ const colon = trimmed.indexOf(':')
333
+ if (colon > 0) return trimmed.slice(0, colon)
334
+ const dot = trimmed.indexOf('.')
335
+ if (dot > 0) return trimmed.slice(0, dot)
336
+ return null
337
+ }
338
+
339
+ function normalizeModuleId(input: ModuleResourceUsageInput): string | null {
340
+ const explicit = input.moduleId?.trim()
341
+ if (explicit) return explicit
342
+ return inferModuleIdFromResourceId(input.resourceId) ?? inferModuleIdFromResourceId(input.operation)
343
+ }
344
+
345
+ function nowNs(): bigint {
346
+ return process.hrtime.bigint()
347
+ }
348
+
349
+ function durationMs(startNs: bigint): number {
350
+ const elapsed = nowNs() - startNs
351
+ if (elapsed <= BigInt(0)) return 0
352
+ return Number(elapsed) / NS_PER_MS
353
+ }
354
+
355
+ function round(value: number, precision = 100): number {
356
+ if (!Number.isFinite(value)) return 0
357
+ return Math.round(value * precision) / precision
358
+ }
359
+
360
+ function safeMemoryUsage(): NodeJS.MemoryUsage | null {
361
+ try {
362
+ return process.memoryUsage()
363
+ } catch {
364
+ return null
365
+ }
366
+ }
367
+
368
+ function entryKey(input: Required<Pick<ModuleResourceUsageInput, 'surface' | 'operation'>> & { moduleId: string }): string {
369
+ return `${input.moduleId}\u0000${input.surface}\u0000${input.operation}`
370
+ }
371
+
372
+ function getOrCreateEntry(input: ModuleResourceUsageInput & { moduleId: string }, timestamp: string): MutableEntry {
373
+ const state = getState()
374
+ const key = entryKey(input)
375
+ const existing = state.entries.get(key)
376
+ if (existing) return existing
377
+ const created: MutableEntry = {
378
+ moduleId: input.moduleId,
379
+ surface: input.surface,
380
+ operation: input.operation,
381
+ resourceId: input.resourceId ?? null,
382
+ calls: 0,
383
+ errors: 0,
384
+ concurrentCalls: 0,
385
+ totalDurationMs: 0,
386
+ maxDurationMs: 0,
387
+ totalCpuUserMs: 0,
388
+ totalCpuSystemMs: 0,
389
+ maxCpuMs: 0,
390
+ totalHeapDeltaBytes: 0,
391
+ positiveHeapDeltaBytes: 0,
392
+ maxHeapDeltaBytes: 0,
393
+ totalRssDeltaBytes: 0,
394
+ positiveRssDeltaBytes: 0,
395
+ maxRssDeltaBytes: 0,
396
+ firstSeenAt: timestamp,
397
+ lastSeenAt: timestamp,
398
+ recentDurationsMs: [],
399
+ }
400
+ state.entries.set(key, created)
401
+ return created
402
+ }
403
+
404
+ function getOrCreateTimeBucket(state: ResourceUsageState, timestamp: string): MutableTimeBucket | null {
405
+ const timestampMs = Date.parse(timestamp)
406
+ if (!Number.isFinite(timestampMs)) return null
407
+ const intervalMs = moduleResourceUsageBucketIntervalMs()
408
+ const bucketStartMs = Math.floor(timestampMs / intervalMs) * intervalMs
409
+ const key = String(bucketStartMs)
410
+ const existing = state.buckets.get(key)
411
+ if (existing) return existing
412
+ const created: MutableTimeBucket = {
413
+ bucketStartMs,
414
+ bucketEndMs: bucketStartMs + intervalMs,
415
+ modules: new Map(),
416
+ }
417
+ state.buckets.set(key, created)
418
+ pruneTimeBuckets(state)
419
+ return created
420
+ }
421
+
422
+ function getOrCreateTimeBucketModule(bucket: MutableTimeBucket, moduleId: string): MutableTimeBucketModule {
423
+ const existing = bucket.modules.get(moduleId)
424
+ if (existing) {
425
+ if (!Array.isArray(existing.recentDurationsMs)) existing.recentDurationsMs = []
426
+ if (!(existing.operations instanceof Map)) existing.operations = new Map()
427
+ return existing
428
+ }
429
+ const created: MutableTimeBucketModule = {
430
+ moduleId,
431
+ calls: 0,
432
+ errors: 0,
433
+ totalDurationMs: 0,
434
+ totalCpuMs: 0,
435
+ positiveHeapDeltaBytes: 0,
436
+ positiveRssDeltaBytes: 0,
437
+ recentDurationsMs: [],
438
+ operations: new Map(),
439
+ }
440
+ bucket.modules.set(moduleId, created)
441
+ return created
442
+ }
443
+
444
+ function getOrCreateTimeBucketOperation(
445
+ bucketModule: MutableTimeBucketModule,
446
+ input: ModuleResourceUsageInput & { moduleId: string },
447
+ timestamp: string,
448
+ ): MutableEntry {
449
+ const key = entryKey(input)
450
+ const existing = bucketModule.operations.get(key)
451
+ if (existing) return existing
452
+ const created: MutableEntry = {
453
+ moduleId: input.moduleId,
454
+ surface: input.surface,
455
+ operation: input.operation,
456
+ resourceId: input.resourceId ?? null,
457
+ calls: 0,
458
+ errors: 0,
459
+ concurrentCalls: 0,
460
+ totalDurationMs: 0,
461
+ maxDurationMs: 0,
462
+ totalCpuUserMs: 0,
463
+ totalCpuSystemMs: 0,
464
+ maxCpuMs: 0,
465
+ totalHeapDeltaBytes: 0,
466
+ positiveHeapDeltaBytes: 0,
467
+ maxHeapDeltaBytes: 0,
468
+ totalRssDeltaBytes: 0,
469
+ positiveRssDeltaBytes: 0,
470
+ maxRssDeltaBytes: 0,
471
+ firstSeenAt: timestamp,
472
+ lastSeenAt: timestamp,
473
+ recentDurationsMs: [],
474
+ }
475
+ bucketModule.operations.set(key, created)
476
+ return created
477
+ }
478
+
479
+ function pruneTimeBuckets(state: ResourceUsageState): void {
480
+ const limit = moduleResourceUsageBucketLimit()
481
+ if (state.buckets.size <= limit) return
482
+ const keys = Array.from(state.buckets.entries())
483
+ .sort((a, b) => a[1].bucketStartMs - b[1].bucketStartMs)
484
+ .map(([key]) => key)
485
+ while (state.buckets.size > limit) {
486
+ const key = keys.shift()
487
+ if (!key) break
488
+ state.buckets.delete(key)
489
+ }
490
+ }
491
+
492
+ function recordTimeBucket(input: ModuleResourceUsageInput & { moduleId: string }, timestamp: string, metrics: {
493
+ durationMs: number
494
+ cpuUserMs: number
495
+ cpuSystemMs: number
496
+ cpuMs: number
497
+ heapDeltaBytes: number
498
+ rssDeltaBytes: number
499
+ status: 'ok' | 'error'
500
+ concurrentTainted: boolean
501
+ }): void {
502
+ const state = getState()
503
+ const bucket = getOrCreateTimeBucket(state, timestamp)
504
+ if (!bucket) return
505
+ const bucketModule = getOrCreateTimeBucketModule(bucket, input.moduleId)
506
+ bucketModule.calls += 1
507
+ if (metrics.status === 'error') bucketModule.errors += 1
508
+ bucketModule.totalDurationMs += metrics.durationMs
509
+ bucketModule.totalCpuMs += metrics.cpuMs
510
+ bucketModule.positiveHeapDeltaBytes += Math.max(metrics.heapDeltaBytes, 0)
511
+ bucketModule.positiveRssDeltaBytes += Math.max(metrics.rssDeltaBytes, 0)
512
+ bucketModule.recentDurationsMs.push(metrics.durationMs)
513
+ if (bucketModule.recentDurationsMs.length > DEFAULT_RECENT_SAMPLE_LIMIT) {
514
+ bucketModule.recentDurationsMs.splice(0, bucketModule.recentDurationsMs.length - DEFAULT_RECENT_SAMPLE_LIMIT)
515
+ }
516
+ const operation = getOrCreateTimeBucketOperation(bucketModule, input, timestamp)
517
+ operation.calls += 1
518
+ if (metrics.status === 'error') operation.errors += 1
519
+ if (metrics.concurrentTainted) operation.concurrentCalls += 1
520
+ operation.totalDurationMs += metrics.durationMs
521
+ operation.maxDurationMs = Math.max(operation.maxDurationMs, metrics.durationMs)
522
+ operation.totalCpuUserMs += metrics.cpuUserMs
523
+ operation.totalCpuSystemMs += metrics.cpuSystemMs
524
+ operation.maxCpuMs = Math.max(operation.maxCpuMs, metrics.cpuMs)
525
+ operation.totalHeapDeltaBytes += metrics.heapDeltaBytes
526
+ operation.positiveHeapDeltaBytes += Math.max(metrics.heapDeltaBytes, 0)
527
+ operation.maxHeapDeltaBytes = Math.max(operation.maxHeapDeltaBytes, metrics.heapDeltaBytes)
528
+ operation.totalRssDeltaBytes += metrics.rssDeltaBytes
529
+ operation.positiveRssDeltaBytes += Math.max(metrics.rssDeltaBytes, 0)
530
+ operation.maxRssDeltaBytes = Math.max(operation.maxRssDeltaBytes, metrics.rssDeltaBytes)
531
+ operation.lastSeenAt = timestamp
532
+ operation.recentDurationsMs.push(metrics.durationMs)
533
+ if (operation.recentDurationsMs.length > DEFAULT_RECENT_SAMPLE_LIMIT) {
534
+ operation.recentDurationsMs.splice(0, operation.recentDurationsMs.length - DEFAULT_RECENT_SAMPLE_LIMIT)
535
+ }
536
+ }
537
+
538
+ function recordModuleResourceUsage(input: ModuleResourceUsageInput, metrics: {
539
+ durationMs: number
540
+ cpuUserMicros: number
541
+ cpuSystemMicros: number
542
+ heapDeltaBytes: number
543
+ rssDeltaBytes: number
544
+ status: 'ok' | 'error'
545
+ concurrentTainted: boolean
546
+ }): void {
547
+ const moduleId = normalizeModuleId(input)
548
+ if (!moduleId) return
549
+ const timestamp = new Date().toISOString()
550
+ const entry = getOrCreateEntry({ ...input, moduleId }, timestamp)
551
+ const cpuUserMs = metrics.cpuUserMicros / MICROS_PER_MS
552
+ const cpuSystemMs = metrics.cpuSystemMicros / MICROS_PER_MS
553
+ const cpuMs = cpuUserMs + cpuSystemMs
554
+ entry.calls += 1
555
+ if (metrics.status === 'error') entry.errors += 1
556
+ if (metrics.concurrentTainted) entry.concurrentCalls += 1
557
+ entry.totalDurationMs += metrics.durationMs
558
+ entry.maxDurationMs = Math.max(entry.maxDurationMs, metrics.durationMs)
559
+ entry.totalCpuUserMs += cpuUserMs
560
+ entry.totalCpuSystemMs += cpuSystemMs
561
+ entry.maxCpuMs = Math.max(entry.maxCpuMs, cpuMs)
562
+ entry.totalHeapDeltaBytes += metrics.heapDeltaBytes
563
+ entry.positiveHeapDeltaBytes += Math.max(metrics.heapDeltaBytes, 0)
564
+ entry.maxHeapDeltaBytes = Math.max(entry.maxHeapDeltaBytes, metrics.heapDeltaBytes)
565
+ entry.totalRssDeltaBytes += metrics.rssDeltaBytes
566
+ entry.positiveRssDeltaBytes += Math.max(metrics.rssDeltaBytes, 0)
567
+ entry.maxRssDeltaBytes = Math.max(entry.maxRssDeltaBytes, metrics.rssDeltaBytes)
568
+ entry.lastSeenAt = timestamp
569
+ entry.recentDurationsMs.push(metrics.durationMs)
570
+ if (entry.recentDurationsMs.length > DEFAULT_RECENT_SAMPLE_LIMIT) {
571
+ entry.recentDurationsMs.splice(0, entry.recentDurationsMs.length - DEFAULT_RECENT_SAMPLE_LIMIT)
572
+ }
573
+ recordTimeBucket({ ...input, moduleId }, timestamp, {
574
+ durationMs: metrics.durationMs,
575
+ cpuUserMs,
576
+ cpuSystemMs,
577
+ cpuMs,
578
+ heapDeltaBytes: metrics.heapDeltaBytes,
579
+ rssDeltaBytes: metrics.rssDeltaBytes,
580
+ status: metrics.status,
581
+ concurrentTainted: metrics.concurrentTainted,
582
+ })
583
+ flushModuleResourceUsageSnapshot(false)
584
+ }
585
+
586
+ export async function withModuleResourceUsage<T>(
587
+ input: ModuleResourceUsageInput,
588
+ fn: () => Promise<T> | T,
589
+ ): Promise<T> {
590
+ if (!isModuleResourceUsageEnabled() || !normalizeModuleId(input)) {
591
+ return Promise.resolve(fn())
592
+ }
593
+ const state = getState()
594
+ // Mark this call tainted if another tracked call is already in flight, and mark every
595
+ // currently in-flight call tainted too, since process.cpuUsage()/process.memoryUsage() can't
596
+ // separate their concurrent work from ours (see ModuleResourceUsageEntry.concurrentCalls).
597
+ const token: ActiveCallToken = { tainted: state.activeCalls.size > 0 }
598
+ if (token.tainted) {
599
+ for (const other of state.activeCalls) other.tainted = true
600
+ }
601
+ state.activeCalls.add(token)
602
+ const startNs = nowNs()
603
+ const startCpu = process.cpuUsage()
604
+ const startMemory = safeMemoryUsage()
605
+ // Recording must never override the wrapped call's real outcome: `fn()` is awaited in its own
606
+ // try/catch, and the bookkeeping below always runs in a try/catch of its own so a bug in the
607
+ // telemetry path can only be lost silently, never surface as a fake success/failure for `fn()`.
608
+ let result: T
609
+ try {
610
+ result = await Promise.resolve(fn())
611
+ } catch (error) {
612
+ state.activeCalls.delete(token)
613
+ try {
614
+ const cpu = process.cpuUsage(startCpu)
615
+ const endMemory = safeMemoryUsage()
616
+ recordModuleResourceUsage(input, {
617
+ durationMs: durationMs(startNs),
618
+ cpuUserMicros: cpu.user,
619
+ cpuSystemMicros: cpu.system,
620
+ heapDeltaBytes: endMemory && startMemory ? endMemory.heapUsed - startMemory.heapUsed : 0,
621
+ rssDeltaBytes: endMemory && startMemory ? endMemory.rss - startMemory.rss : 0,
622
+ status: 'error',
623
+ concurrentTainted: token.tainted,
624
+ })
625
+ } catch {
626
+ // Telemetry bookkeeping is diagnostic only; never mask the real error from the wrapped call.
627
+ }
628
+ throw error
629
+ }
630
+ state.activeCalls.delete(token)
631
+ try {
632
+ const cpu = process.cpuUsage(startCpu)
633
+ const endMemory = safeMemoryUsage()
634
+ recordModuleResourceUsage(input, {
635
+ durationMs: durationMs(startNs),
636
+ cpuUserMicros: cpu.user,
637
+ cpuSystemMicros: cpu.system,
638
+ heapDeltaBytes: endMemory && startMemory ? endMemory.heapUsed - startMemory.heapUsed : 0,
639
+ rssDeltaBytes: endMemory && startMemory ? endMemory.rss - startMemory.rss : 0,
640
+ status: 'ok',
641
+ concurrentTainted: token.tainted,
642
+ })
643
+ } catch {
644
+ // Telemetry bookkeeping is diagnostic only; never mask the real result from the wrapped call.
645
+ }
646
+ return result
647
+ }
648
+
649
+ function percentile(values: number[], p: number): number {
650
+ if (!values.length) return 0
651
+ const sorted = values.slice().sort((a, b) => a - b)
652
+ const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1))
653
+ return sorted[index]
654
+ }
655
+
656
+ function serializeEntry(entry: MutableEntry): ModuleResourceUsageEntry {
657
+ return {
658
+ moduleId: entry.moduleId,
659
+ surface: entry.surface,
660
+ operation: entry.operation,
661
+ resourceId: entry.resourceId,
662
+ calls: entry.calls,
663
+ errors: entry.errors,
664
+ concurrentCalls: entry.concurrentCalls,
665
+ totalDurationMs: round(entry.totalDurationMs),
666
+ maxDurationMs: round(entry.maxDurationMs),
667
+ p95DurationMs: round(percentile(entry.recentDurationsMs, 95)),
668
+ totalCpuUserMs: round(entry.totalCpuUserMs),
669
+ totalCpuSystemMs: round(entry.totalCpuSystemMs),
670
+ maxCpuMs: round(entry.maxCpuMs),
671
+ totalHeapDeltaBytes: Math.round(entry.totalHeapDeltaBytes),
672
+ positiveHeapDeltaBytes: Math.round(entry.positiveHeapDeltaBytes),
673
+ maxHeapDeltaBytes: Math.round(entry.maxHeapDeltaBytes),
674
+ totalRssDeltaBytes: Math.round(entry.totalRssDeltaBytes),
675
+ positiveRssDeltaBytes: Math.round(entry.positiveRssDeltaBytes),
676
+ maxRssDeltaBytes: Math.round(entry.maxRssDeltaBytes),
677
+ firstSeenAt: entry.firstSeenAt,
678
+ lastSeenAt: entry.lastSeenAt,
679
+ }
680
+ }
681
+
682
+ function timeBucketTotals(modules: ModuleResourceUsageTimeBucketModule[]): ModuleResourceUsageTimeBucket['totals'] {
683
+ return {
684
+ modules: modules.length,
685
+ calls: modules.reduce((sum, module) => sum + module.calls, 0),
686
+ errors: modules.reduce((sum, module) => sum + module.errors, 0),
687
+ totalDurationMs: round(modules.reduce((sum, module) => sum + module.totalDurationMs, 0)),
688
+ totalCpuMs: round(modules.reduce((sum, module) => sum + module.totalCpuMs, 0)),
689
+ positiveHeapDeltaBytes: modules.reduce((sum, module) => sum + module.positiveHeapDeltaBytes, 0),
690
+ positiveRssDeltaBytes: modules.reduce((sum, module) => sum + module.positiveRssDeltaBytes, 0),
691
+ }
692
+ }
693
+
694
+ function surfaceSummaries(entries: ModuleResourceUsageEntry[]): ModuleResourceUsageSurfaceSummary[] {
695
+ const surfaces = new Map<ModuleResourceSurface, ModuleResourceUsageEntry[]>()
696
+ for (const entry of entries) {
697
+ const list = surfaces.get(entry.surface) ?? []
698
+ list.push(entry)
699
+ surfaces.set(entry.surface, list)
700
+ }
701
+ return Array.from(surfaces.entries()).map(([surface, surfaceEntries]) => ({
702
+ surface,
703
+ calls: surfaceEntries.reduce((sum, entry) => sum + entry.calls, 0),
704
+ errors: surfaceEntries.reduce((sum, entry) => sum + entry.errors, 0),
705
+ totalDurationMs: round(surfaceEntries.reduce((sum, entry) => sum + entry.totalDurationMs, 0)),
706
+ p95DurationMs: round(Math.max(...surfaceEntries.map((entry) => entry.p95DurationMs), 0)),
707
+ totalCpuMs: round(surfaceEntries.reduce((sum, entry) => sum + entry.totalCpuUserMs + entry.totalCpuSystemMs, 0)),
708
+ positiveHeapDeltaBytes: surfaceEntries.reduce((sum, entry) => sum + entry.positiveHeapDeltaBytes, 0),
709
+ positiveRssDeltaBytes: surfaceEntries.reduce((sum, entry) => sum + entry.positiveRssDeltaBytes, 0),
710
+ })).sort((a, b) => b.totalCpuMs - a.totalCpuMs || b.totalDurationMs - a.totalDurationMs)
711
+ }
712
+
713
+ function moduleResourceUsageBucketStage(bucketStartMs: number, bucketEndMs: number, startedAtMs: number): ModuleResourceUsageTimeBucket['stage'] {
714
+ if (!Number.isFinite(bucketStartMs) || !Number.isFinite(bucketEndMs) || !Number.isFinite(startedAtMs)) return 'running'
715
+ return bucketEndMs > startedAtMs && bucketStartMs < startedAtMs + MODULE_RESOURCE_USAGE_STARTUP_STAGE_MS
716
+ ? 'startup'
717
+ : 'running'
718
+ }
719
+
720
+ function serializeTimeBucket(bucket: MutableTimeBucket, nowMs: number, startedAtMs: number): ModuleResourceUsageTimeBucket {
721
+ const thresholds = getModuleResourceUsageThresholds()
722
+ const modules = Array.from(bucket.modules.values()).map((module) => ({
723
+ module,
724
+ operations: module.operations instanceof Map
725
+ ? Array.from(module.operations.values()).map(serializeEntry)
726
+ : [],
727
+ })).map(({ module, operations }) => {
728
+ const summaryBase = {
729
+ moduleId: module.moduleId,
730
+ calls: module.calls,
731
+ errors: module.errors,
732
+ totalDurationMs: round(module.totalDurationMs),
733
+ p95DurationMs: round(percentile(module.recentDurationsMs, 95)),
734
+ totalCpuMs: round(module.totalCpuMs),
735
+ positiveHeapDeltaBytes: Math.round(module.positiveHeapDeltaBytes),
736
+ positiveRssDeltaBytes: Math.round(module.positiveRssDeltaBytes),
737
+ surfaces: surfaceSummaries(operations),
738
+ topOperations: operations
739
+ .slice()
740
+ .sort((a, b) => (b.totalCpuUserMs + b.totalCpuSystemMs) - (a.totalCpuUserMs + a.totalCpuSystemMs) || b.totalDurationMs - a.totalDurationMs)
741
+ .slice(0, DEFAULT_TOP_OPERATIONS_LIMIT),
742
+ }
743
+ return {
744
+ ...summaryBase,
745
+ candidateReasons: buildCandidateReasons(summaryBase, thresholds),
746
+ }
747
+ }).sort((a, b) =>
748
+ b.candidateReasons.length - a.candidateReasons.length
749
+ || b.totalCpuMs - a.totalCpuMs
750
+ || b.positiveRssDeltaBytes - a.positiveRssDeltaBytes
751
+ || b.calls - a.calls
752
+ )
753
+ return {
754
+ bucketStart: new Date(bucket.bucketStartMs).toISOString(),
755
+ bucketEnd: new Date(bucket.bucketEndMs).toISOString(),
756
+ bucketIntervalMs: bucket.bucketEndMs - bucket.bucketStartMs,
757
+ stage: moduleResourceUsageBucketStage(bucket.bucketStartMs, bucket.bucketEndMs, startedAtMs),
758
+ partial: bucket.bucketStartMs < startedAtMs || nowMs < bucket.bucketEndMs,
759
+ totals: timeBucketTotals(modules),
760
+ modules,
761
+ }
762
+ }
763
+
764
+ function serializeTimeBuckets(state: ResourceUsageState): ModuleResourceUsageTimeBucket[] {
765
+ const nowMs = Date.now()
766
+ const startedAtMs = Date.parse(state.startedAt)
767
+ return Array.from(state.buckets.values())
768
+ .sort((a, b) => a.bucketStartMs - b.bucketStartMs)
769
+ .map((bucket) => serializeTimeBucket(bucket, nowMs, Number.isFinite(startedAtMs) ? startedAtMs : nowMs))
770
+ }
771
+
772
+ function getSnapshotDir(): string {
773
+ const configured = process.env.OM_MODULE_RESOURCE_USAGE_DIR?.trim()
774
+ return configured ? path.resolve(configured) : path.resolve(process.cwd(), '.mercato/module-resource-usage')
775
+ }
776
+
777
+ function getSnapshotPath(): string {
778
+ return path.join(getSnapshotDir(), `process-${process.pid}.json`)
779
+ }
780
+
781
+ function registerSnapshotShutdownHook(state: ResourceUsageState): void {
782
+ if (state.shutdownHookRegistered) return
783
+ const flush = () => {
784
+ try {
785
+ flushModuleResourceUsageSnapshot(true)
786
+ } catch {
787
+ // Best-effort diagnostic snapshot; never block process shutdown.
788
+ }
789
+ }
790
+ process.once('beforeExit', flush)
791
+ state.shutdownHookRegistered = true
792
+ }
793
+
794
+ // Fire-and-forget: nothing in this process depends on the write completing (readers always
795
+ // skip their own pid's file — see readSnapshotPayloads), so there's no reason for the write to
796
+ // block the tracked call, event dispatch, or report request that triggered it. This mirrors the
797
+ // existing fire-and-forget queue-close pattern in registerProducerShutdownHook (bus.ts): Node's
798
+ // event loop stays alive for the pending promise, so even the beforeExit-triggered flush below
799
+ // still completes before the process actually exits, as long as nothing calls process.exit()
800
+ // first.
801
+ let snapshotWriteSequence = 0
802
+
803
+ // Writes atomically: serialize to a unique temp file, then rename onto the final path. rename is
804
+ // atomic on POSIX, so a concurrent reader (this process's report, another process's directory scan,
805
+ // or a test) always observes either the previous complete file or the new complete file — never the
806
+ // empty/partial content window that a direct writeFile to snapshotPath would briefly expose.
807
+ function writeSnapshotFileAsync(dir: string, snapshotPath: string, payload: unknown): void {
808
+ const tempPath = `${snapshotPath}.${process.pid}.${snapshotWriteSequence++}.tmp`
809
+ fs.promises.mkdir(dir, { recursive: true })
810
+ .then(() => fs.promises.writeFile(tempPath, JSON.stringify(payload, null, 2)))
811
+ .then(() => fs.promises.rename(tempPath, snapshotPath))
812
+ .catch(() => {
813
+ // Snapshot files are diagnostic only; in-memory reporting still works. Best-effort cleanup of
814
+ // the temp file if the rename never happened; ignore if it is already gone.
815
+ fs.promises.unlink(tempPath).catch(() => {})
816
+ })
817
+ }
818
+
819
+ function flushModuleResourceUsageSnapshot(force: boolean): void {
820
+ if (!isSnapshotEnabled()) return
821
+ const state = getState()
822
+ const now = Date.now()
823
+ if (!force && now - state.lastSnapshotAt < SNAPSHOT_THROTTLE_MS) return
824
+ state.lastSnapshotAt = now
825
+ try {
826
+ const payload = {
827
+ pid: process.pid,
828
+ generatedAt: new Date().toISOString(),
829
+ startedAt: state.startedAt,
830
+ entries: Array.from(state.entries.values()).map(serializeEntry),
831
+ buckets: serializeTimeBuckets(state),
832
+ }
833
+ writeSnapshotFileAsync(getSnapshotDir(), getSnapshotPath(), payload)
834
+ } catch {
835
+ // Snapshot files are diagnostic only; in-memory reporting still works.
836
+ }
837
+ }
838
+
839
+ type SnapshotPayload = {
840
+ pid?: unknown
841
+ generatedAt?: unknown
842
+ startedAt?: unknown
843
+ entries?: unknown
844
+ buckets?: unknown
845
+ }
846
+
847
+ function isFreshSnapshot(payload: SnapshotPayload): boolean {
848
+ if (typeof payload.generatedAt !== 'string') return false
849
+ const generatedAt = Date.parse(payload.generatedAt)
850
+ return Number.isFinite(generatedAt) && Date.now() - generatedAt <= SNAPSHOT_MAX_AGE_MS
851
+ }
852
+
853
+ // Reads every other process's snapshot file exactly once. Both readSnapshotEntries and
854
+ // readSnapshotBuckets used to each force a flush and independently re-scan/re-parse the whole
855
+ // directory; callers now do that single scan once (see getModuleResourceUsageReport) and pass
856
+ // the result to both.
857
+ // Restarts/redeploys/autoscaling leave one snapshot file behind per historical process. Prune
858
+ // stale ones opportunistically here (best-effort, never blocks the read) so the directory doesn't
859
+ // grow without bound in a long-running deployment.
860
+ function pruneStaleSnapshotFile(filePath: string): void {
861
+ fs.promises.unlink(filePath).catch(() => {
862
+ // Snapshot files are diagnostic only; leave it for a later pass if it can't be removed now.
863
+ })
864
+ }
865
+
866
+ function readSnapshotPayloads(): SnapshotPayload[] {
867
+ if (!isSnapshotEnabled()) return []
868
+ const dir = getSnapshotDir()
869
+ if (!fs.existsSync(dir)) return []
870
+ const payloads: SnapshotPayload[] = []
871
+ for (const file of fs.readdirSync(dir)) {
872
+ if (!file.endsWith('.json')) continue
873
+ const filePath = path.join(dir, file)
874
+ try {
875
+ const payload = JSON.parse(fs.readFileSync(filePath, 'utf8')) as SnapshotPayload
876
+ if (payload.pid === process.pid) continue
877
+ if (!isFreshSnapshot(payload)) {
878
+ pruneStaleSnapshotFile(filePath)
879
+ continue
880
+ }
881
+ payloads.push(payload)
882
+ } catch {
883
+ // Ignore malformed or concurrently-written snapshots.
884
+ }
885
+ }
886
+ return payloads
887
+ }
888
+
889
+ function readSnapshotEntries(payloads: SnapshotPayload[]): ModuleResourceUsageEntry[] {
890
+ const entries: ModuleResourceUsageEntry[] = []
891
+ for (const payload of payloads) {
892
+ if (!Array.isArray(payload.entries)) continue
893
+ for (const entry of payload.entries) {
894
+ if (!entry || typeof entry !== 'object') continue
895
+ const candidate = entry as Partial<ModuleResourceUsageEntry>
896
+ if (typeof candidate.moduleId !== 'string' || typeof candidate.operation !== 'string') continue
897
+ if (!candidate.surface) continue
898
+ entries.push(candidate as ModuleResourceUsageEntry)
899
+ }
900
+ }
901
+ return entries
902
+ }
903
+
904
+ function readSnapshotBuckets(payloads: SnapshotPayload[]): ModuleResourceUsageTimeBucket[] {
905
+ const buckets: ModuleResourceUsageTimeBucket[] = []
906
+ for (const payload of payloads) {
907
+ try {
908
+ if (!Array.isArray(payload.buckets)) continue
909
+ for (const bucket of payload.buckets) {
910
+ if (!bucket || typeof bucket !== 'object') continue
911
+ const candidate = bucket as Partial<ModuleResourceUsageTimeBucket>
912
+ if (typeof candidate.bucketStart !== 'string' || typeof candidate.bucketEnd !== 'string') continue
913
+ if (!Array.isArray(candidate.modules)) continue
914
+ const modules = (candidate.modules as Partial<ModuleResourceUsageTimeBucketModule>[]).map(normalizeTimeBucketModule)
915
+ buckets.push({
916
+ bucketStart: candidate.bucketStart,
917
+ bucketEnd: candidate.bucketEnd,
918
+ bucketIntervalMs: normalizeBucketIntervalMs(candidate),
919
+ stage: candidate.stage === 'startup' || candidate.stage === 'running'
920
+ ? candidate.stage
921
+ : moduleResourceUsageBucketStage(
922
+ Date.parse(candidate.bucketStart),
923
+ Date.parse(candidate.bucketEnd),
924
+ typeof payload.startedAt === 'string' ? Date.parse(payload.startedAt) : Date.parse(candidate.bucketStart),
925
+ ),
926
+ partial: candidate.partial === true,
927
+ totals: candidate.totals ?? timeBucketTotals(modules),
928
+ modules,
929
+ })
930
+ }
931
+ } catch {
932
+ // Ignore malformed or concurrently-written snapshots.
933
+ }
934
+ }
935
+ return buckets
936
+ }
937
+
938
+ function normalizeBucketIntervalMs(bucket: Partial<ModuleResourceUsageTimeBucket>): number {
939
+ if (Number.isFinite(bucket.bucketIntervalMs) && Number(bucket.bucketIntervalMs) > 0) {
940
+ return Number(bucket.bucketIntervalMs)
941
+ }
942
+ const bucketStartMs = typeof bucket.bucketStart === 'string' ? Date.parse(bucket.bucketStart) : NaN
943
+ const bucketEndMs = typeof bucket.bucketEnd === 'string' ? Date.parse(bucket.bucketEnd) : NaN
944
+ const inferred = bucketEndMs - bucketStartMs
945
+ return Number.isFinite(inferred) && inferred > 0 ? inferred : moduleResourceUsageBucketIntervalMs()
946
+ }
947
+
948
+ function normalizeTimeBucketModule(module: Partial<ModuleResourceUsageTimeBucketModule>): ModuleResourceUsageTimeBucketModule {
949
+ return {
950
+ moduleId: typeof module.moduleId === 'string' ? module.moduleId : 'unknown',
951
+ calls: Number.isFinite(module.calls) ? Number(module.calls) : 0,
952
+ errors: Number.isFinite(module.errors) ? Number(module.errors) : 0,
953
+ totalDurationMs: Number.isFinite(module.totalDurationMs) ? Number(module.totalDurationMs) : 0,
954
+ p95DurationMs: Number.isFinite(module.p95DurationMs) ? Number(module.p95DurationMs) : 0,
955
+ totalCpuMs: Number.isFinite(module.totalCpuMs) ? Number(module.totalCpuMs) : 0,
956
+ positiveHeapDeltaBytes: Number.isFinite(module.positiveHeapDeltaBytes) ? Number(module.positiveHeapDeltaBytes) : 0,
957
+ positiveRssDeltaBytes: Number.isFinite(module.positiveRssDeltaBytes) ? Number(module.positiveRssDeltaBytes) : 0,
958
+ surfaces: Array.isArray(module.surfaces) ? module.surfaces : [],
959
+ topOperations: Array.isArray(module.topOperations) ? module.topOperations : [],
960
+ candidateReasons: Array.isArray(module.candidateReasons) ? module.candidateReasons : [],
961
+ }
962
+ }
963
+
964
+ function mergeEntries(entries: ModuleResourceUsageEntry[]): ModuleResourceUsageEntry[] {
965
+ const merged = new Map<string, ModuleResourceUsageEntry>()
966
+ for (const entry of entries) {
967
+ const key = entryKey({ moduleId: entry.moduleId, surface: entry.surface, operation: entry.operation })
968
+ const existing = merged.get(key)
969
+ if (!existing) {
970
+ merged.set(key, {
971
+ ...entry,
972
+ // Snapshot files written by an older process before concurrentCalls existed lack it.
973
+ concurrentCalls: Number.isFinite(entry.concurrentCalls) ? entry.concurrentCalls : 0,
974
+ })
975
+ continue
976
+ }
977
+ existing.calls += entry.calls
978
+ existing.errors += entry.errors
979
+ existing.concurrentCalls += Number.isFinite(entry.concurrentCalls) ? entry.concurrentCalls : 0
980
+ existing.totalDurationMs = round(existing.totalDurationMs + entry.totalDurationMs)
981
+ existing.maxDurationMs = Math.max(existing.maxDurationMs, entry.maxDurationMs)
982
+ existing.p95DurationMs = Math.max(existing.p95DurationMs, entry.p95DurationMs)
983
+ existing.totalCpuUserMs = round(existing.totalCpuUserMs + entry.totalCpuUserMs)
984
+ existing.totalCpuSystemMs = round(existing.totalCpuSystemMs + entry.totalCpuSystemMs)
985
+ existing.maxCpuMs = Math.max(existing.maxCpuMs, entry.maxCpuMs)
986
+ existing.totalHeapDeltaBytes += entry.totalHeapDeltaBytes
987
+ existing.positiveHeapDeltaBytes += entry.positiveHeapDeltaBytes
988
+ existing.maxHeapDeltaBytes = Math.max(existing.maxHeapDeltaBytes, entry.maxHeapDeltaBytes)
989
+ existing.totalRssDeltaBytes += entry.totalRssDeltaBytes
990
+ existing.positiveRssDeltaBytes += entry.positiveRssDeltaBytes
991
+ existing.maxRssDeltaBytes = Math.max(existing.maxRssDeltaBytes, entry.maxRssDeltaBytes)
992
+ existing.firstSeenAt = existing.firstSeenAt < entry.firstSeenAt ? existing.firstSeenAt : entry.firstSeenAt
993
+ existing.lastSeenAt = existing.lastSeenAt > entry.lastSeenAt ? existing.lastSeenAt : entry.lastSeenAt
994
+ }
995
+ return Array.from(merged.values())
996
+ }
997
+
998
+ function mergeSurfaceSummaries(surfaces: ModuleResourceUsageSurfaceSummary[]): ModuleResourceUsageSurfaceSummary[] {
999
+ const merged = new Map<ModuleResourceSurface, ModuleResourceUsageSurfaceSummary>()
1000
+ for (const surface of surfaces) {
1001
+ const existing = merged.get(surface.surface)
1002
+ if (!existing) {
1003
+ merged.set(surface.surface, { ...surface })
1004
+ continue
1005
+ }
1006
+ existing.calls += surface.calls
1007
+ existing.errors += surface.errors
1008
+ existing.totalDurationMs = round(existing.totalDurationMs + surface.totalDurationMs)
1009
+ existing.p95DurationMs = Math.max(existing.p95DurationMs, surface.p95DurationMs)
1010
+ existing.totalCpuMs = round(existing.totalCpuMs + surface.totalCpuMs)
1011
+ existing.positiveHeapDeltaBytes += surface.positiveHeapDeltaBytes
1012
+ existing.positiveRssDeltaBytes += surface.positiveRssDeltaBytes
1013
+ }
1014
+ return Array.from(merged.values()).sort((a, b) => b.totalCpuMs - a.totalCpuMs || b.totalDurationMs - a.totalDurationMs)
1015
+ }
1016
+
1017
+ function mergeOperationEntries(entries: ModuleResourceUsageEntry[]): ModuleResourceUsageEntry[] {
1018
+ return mergeEntries(entries)
1019
+ .sort((a, b) => (b.totalCpuUserMs + b.totalCpuSystemMs) - (a.totalCpuUserMs + a.totalCpuSystemMs) || b.totalDurationMs - a.totalDurationMs)
1020
+ .slice(0, DEFAULT_TOP_OPERATIONS_LIMIT)
1021
+ }
1022
+
1023
+ function mergeTimeBuckets(buckets: ModuleResourceUsageTimeBucket[]): ModuleResourceUsageTimeBucket[] {
1024
+ const merged = new Map<string, ModuleResourceUsageTimeBucket>()
1025
+ const thresholds = getModuleResourceUsageThresholds()
1026
+ for (const bucket of buckets) {
1027
+ const bucketStartMs = Date.parse(bucket.bucketStart)
1028
+ if (!Number.isFinite(bucketStartMs)) continue
1029
+ const bucketIntervalMs = normalizeBucketIntervalMs(bucket)
1030
+ const key = `${bucket.bucketStart}\u0000${bucketIntervalMs}`
1031
+ const existing = merged.get(key)
1032
+ if (!existing) {
1033
+ const modules = bucket.modules.map((module) => {
1034
+ const normalizedModule = normalizeTimeBucketModule(module)
1035
+ return {
1036
+ ...normalizedModule,
1037
+ candidateReasons: buildCandidateReasons(normalizedModule, thresholds),
1038
+ }
1039
+ })
1040
+ merged.set(key, {
1041
+ bucketStart: bucket.bucketStart,
1042
+ bucketEnd: bucket.bucketEnd,
1043
+ bucketIntervalMs,
1044
+ stage: bucket.stage,
1045
+ partial: bucket.partial,
1046
+ totals: timeBucketTotals(modules),
1047
+ modules,
1048
+ })
1049
+ continue
1050
+ }
1051
+ existing.partial = existing.partial || bucket.partial
1052
+ if (existing.stage !== 'startup') existing.stage = bucket.stage
1053
+ const modulesById = new Map(existing.modules.map((module) => [module.moduleId, module]))
1054
+ for (const module of bucket.modules) {
1055
+ const normalizedModule = normalizeTimeBucketModule(module)
1056
+ const current = modulesById.get(normalizedModule.moduleId)
1057
+ if (!current) {
1058
+ const created = {
1059
+ ...normalizedModule,
1060
+ candidateReasons: buildCandidateReasons(normalizedModule, thresholds),
1061
+ }
1062
+ existing.modules.push(created)
1063
+ modulesById.set(normalizedModule.moduleId, created)
1064
+ continue
1065
+ }
1066
+ current.calls += normalizedModule.calls
1067
+ current.errors += normalizedModule.errors
1068
+ current.totalDurationMs = round(current.totalDurationMs + normalizedModule.totalDurationMs)
1069
+ current.p95DurationMs = Math.max(current.p95DurationMs, normalizedModule.p95DurationMs)
1070
+ current.totalCpuMs = round(current.totalCpuMs + normalizedModule.totalCpuMs)
1071
+ current.positiveHeapDeltaBytes += normalizedModule.positiveHeapDeltaBytes
1072
+ current.positiveRssDeltaBytes += normalizedModule.positiveRssDeltaBytes
1073
+ current.surfaces = mergeSurfaceSummaries([...current.surfaces, ...normalizedModule.surfaces])
1074
+ current.topOperations = mergeOperationEntries([...current.topOperations, ...normalizedModule.topOperations])
1075
+ current.candidateReasons = buildCandidateReasons(current, thresholds)
1076
+ }
1077
+ existing.modules.sort((a, b) =>
1078
+ b.totalCpuMs - a.totalCpuMs
1079
+ || b.positiveRssDeltaBytes - a.positiveRssDeltaBytes
1080
+ || b.calls - a.calls
1081
+ )
1082
+ existing.totals = timeBucketTotals(existing.modules)
1083
+ }
1084
+ return Array.from(merged.values())
1085
+ .sort((a, b) => a.bucketStart.localeCompare(b.bucketStart))
1086
+ .slice(-moduleResourceUsageBucketLimit())
1087
+ }
1088
+
1089
+ function readNumberEnv(name: string, fallback: number): number {
1090
+ return parseNumberWithDefault(process.env[name], fallback, { min: 0 })
1091
+ }
1092
+
1093
+ function moduleResourceUsageBucketIntervalMs(): number {
1094
+ return MODULE_RESOURCE_USAGE_BUCKET_INTERVAL_MS
1095
+ }
1096
+
1097
+ function moduleResourceUsageBucketLimit(): number {
1098
+ return Math.max(1, Math.ceil(TIME_BUCKET_RETENTION_MS / moduleResourceUsageBucketIntervalMs()))
1099
+ }
1100
+
1101
+ export function getModuleResourceUsageThresholds(): ModuleResourceUsageThresholds {
1102
+ return {
1103
+ p95DurationMs: readNumberEnv('OM_MODULE_RESOURCE_HEAVY_P95_MS', 5_000),
1104
+ cpuMs: readNumberEnv('OM_MODULE_RESOURCE_HEAVY_CPU_MS', 25_000),
1105
+ positiveHeapDeltaBytes: readNumberEnv('OM_MODULE_RESOURCE_HEAVY_HEAP_BYTES', 250 * 1024 * 1024),
1106
+ positiveRssDeltaBytes: readNumberEnv('OM_MODULE_RESOURCE_HEAVY_RSS_BYTES', 250 * 1024 * 1024),
1107
+ errors: readNumberEnv('OM_MODULE_RESOURCE_HEAVY_ERRORS', 10),
1108
+ }
1109
+ }
1110
+
1111
+ function buildCandidateReasons(
1112
+ summary: Omit<ModuleResourceUsageModuleSummary, 'candidateReasons'>,
1113
+ thresholds: ModuleResourceUsageThresholds,
1114
+ ): string[] {
1115
+ const reasons: string[] = []
1116
+ if (summary.p95DurationMs >= thresholds.p95DurationMs) reasons.push('p95_duration')
1117
+ if (summary.totalCpuMs >= thresholds.cpuMs) reasons.push('cpu')
1118
+ if (summary.positiveHeapDeltaBytes >= thresholds.positiveHeapDeltaBytes) reasons.push('heap_allocations')
1119
+ if (summary.positiveRssDeltaBytes >= thresholds.positiveRssDeltaBytes) reasons.push('rss_growth')
1120
+ if (summary.errors >= thresholds.errors) reasons.push('errors')
1121
+ return reasons
1122
+ }
1123
+
1124
+ export function getModuleResourceUsageReport(): ModuleResourceUsageReport {
1125
+ const state = getState()
1126
+ // Force our own snapshot to disk once (fire-and-forget — see writeSnapshotFileAsync) so other
1127
+ // processes viewing the report around the same time see fresh data, then read every other
1128
+ // process's snapshot file exactly once and derive both entries and buckets from that one scan.
1129
+ flushModuleResourceUsageSnapshot(true)
1130
+ const snapshotPayloads = readSnapshotPayloads()
1131
+ const entries = mergeEntries([
1132
+ ...Array.from(state.entries.values()).map(serializeEntry),
1133
+ ...readSnapshotEntries(snapshotPayloads),
1134
+ ])
1135
+ const buckets = mergeTimeBuckets([
1136
+ ...serializeTimeBuckets(state),
1137
+ ...readSnapshotBuckets(snapshotPayloads),
1138
+ ])
1139
+ const thresholds = getModuleResourceUsageThresholds()
1140
+ const modules = new Map<string, ModuleResourceUsageEntry[]>()
1141
+ for (const entry of entries) {
1142
+ const list = modules.get(entry.moduleId) ?? []
1143
+ list.push(entry)
1144
+ modules.set(entry.moduleId, list)
1145
+ }
1146
+
1147
+ const moduleSummaries: ModuleResourceUsageModuleSummary[] = []
1148
+ for (const [moduleId, moduleEntries] of modules) {
1149
+ const surfaces = new Map<ModuleResourceSurface, ModuleResourceUsageEntry[]>()
1150
+ for (const entry of moduleEntries) {
1151
+ const list = surfaces.get(entry.surface) ?? []
1152
+ list.push(entry)
1153
+ surfaces.set(entry.surface, list)
1154
+ }
1155
+ const totalDurationMs = moduleEntries.reduce((sum, entry) => sum + entry.totalDurationMs, 0)
1156
+ const totalCpuMs = moduleEntries.reduce((sum, entry) => sum + entry.totalCpuUserMs + entry.totalCpuSystemMs, 0)
1157
+ const summaryBase = {
1158
+ moduleId,
1159
+ calls: moduleEntries.reduce((sum, entry) => sum + entry.calls, 0),
1160
+ errors: moduleEntries.reduce((sum, entry) => sum + entry.errors, 0),
1161
+ totalDurationMs: round(totalDurationMs),
1162
+ p95DurationMs: round(Math.max(...moduleEntries.map((entry) => entry.p95DurationMs), 0)),
1163
+ totalCpuMs: round(totalCpuMs),
1164
+ positiveHeapDeltaBytes: moduleEntries.reduce((sum, entry) => sum + entry.positiveHeapDeltaBytes, 0),
1165
+ positiveRssDeltaBytes: moduleEntries.reduce((sum, entry) => sum + entry.positiveRssDeltaBytes, 0),
1166
+ surfaces: Array.from(surfaces.entries()).map(([surface, surfaceEntries]) => ({
1167
+ surface,
1168
+ calls: surfaceEntries.reduce((sum, entry) => sum + entry.calls, 0),
1169
+ errors: surfaceEntries.reduce((sum, entry) => sum + entry.errors, 0),
1170
+ totalDurationMs: round(surfaceEntries.reduce((sum, entry) => sum + entry.totalDurationMs, 0)),
1171
+ p95DurationMs: round(Math.max(...surfaceEntries.map((entry) => entry.p95DurationMs), 0)),
1172
+ totalCpuMs: round(surfaceEntries.reduce((sum, entry) => sum + entry.totalCpuUserMs + entry.totalCpuSystemMs, 0)),
1173
+ positiveHeapDeltaBytes: surfaceEntries.reduce((sum, entry) => sum + entry.positiveHeapDeltaBytes, 0),
1174
+ positiveRssDeltaBytes: surfaceEntries.reduce((sum, entry) => sum + entry.positiveRssDeltaBytes, 0),
1175
+ })).sort((a, b) => b.totalCpuMs - a.totalCpuMs || b.totalDurationMs - a.totalDurationMs),
1176
+ topOperations: moduleEntries
1177
+ .slice()
1178
+ .sort((a, b) => (b.totalCpuUserMs + b.totalCpuSystemMs) - (a.totalCpuUserMs + a.totalCpuSystemMs) || b.totalDurationMs - a.totalDurationMs)
1179
+ .slice(0, DEFAULT_TOP_OPERATIONS_LIMIT),
1180
+ }
1181
+ moduleSummaries.push({
1182
+ ...summaryBase,
1183
+ candidateReasons: buildCandidateReasons(summaryBase, thresholds),
1184
+ })
1185
+ }
1186
+
1187
+ const sortedModules = moduleSummaries.sort((a, b) =>
1188
+ b.candidateReasons.length - a.candidateReasons.length
1189
+ || b.totalCpuMs - a.totalCpuMs
1190
+ || b.totalDurationMs - a.totalDurationMs
1191
+ )
1192
+
1193
+ return {
1194
+ generatedAt: new Date().toISOString(),
1195
+ startedAt: state.startedAt,
1196
+ enabled: isModuleResourceUsageEnabled(),
1197
+ bucketIntervalMs: moduleResourceUsageBucketIntervalMs(),
1198
+ totals: {
1199
+ modules: sortedModules.length,
1200
+ operations: entries.length,
1201
+ calls: entries.reduce((sum, entry) => sum + entry.calls, 0),
1202
+ errors: entries.reduce((sum, entry) => sum + entry.errors, 0),
1203
+ totalDurationMs: round(entries.reduce((sum, entry) => sum + entry.totalDurationMs, 0)),
1204
+ totalCpuMs: round(entries.reduce((sum, entry) => sum + entry.totalCpuUserMs + entry.totalCpuSystemMs, 0)),
1205
+ positiveHeapDeltaBytes: entries.reduce((sum, entry) => sum + entry.positiveHeapDeltaBytes, 0),
1206
+ positiveRssDeltaBytes: entries.reduce((sum, entry) => sum + entry.positiveRssDeltaBytes, 0),
1207
+ },
1208
+ thresholds,
1209
+ modules: sortedModules,
1210
+ candidates: sortedModules.filter((module) => module.candidateReasons.length > 0),
1211
+ buckets,
1212
+ }
1213
+ }