@prometheus-ai/utils 0.5.3 → 0.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +36 -0
- package/dist/types/abortable.d.ts +5 -0
- package/dist/types/dirs.d.ts +4 -0
- package/dist/types/env.d.ts +4 -0
- package/dist/types/fetch-retry.d.ts +8 -0
- package/dist/types/index.d.ts +3 -0
- package/dist/types/logger.d.ts +22 -5
- package/dist/types/loop-phase.d.ts +10 -0
- package/dist/types/module-timer.d.ts +1 -0
- package/dist/types/path-tree.d.ts +76 -0
- package/dist/types/procmgr.d.ts +4 -0
- package/dist/types/runtime-install.d.ts +68 -0
- package/dist/types/tab-spacing.d.ts +16 -0
- package/dist/types/timing-buffer.d.ts +22 -0
- package/package.json +4 -2
- package/src/abortable.ts +86 -1
- package/src/dirs.ts +10 -0
- package/src/env.ts +17 -0
- package/src/fetch-retry.ts +8 -0
- package/src/index.ts +3 -0
- package/src/logger.ts +320 -73
- package/src/loop-phase.ts +49 -0
- package/src/module-timer.ts +148 -0
- package/src/path-tree.ts +147 -0
- package/src/procmgr.ts +1 -1
- package/src/runtime-install.ts +372 -0
- package/src/tab-spacing.ts +51 -0
- package/src/temp.ts +57 -4
- package/src/timing-buffer.ts +47 -0
package/src/logger.ts
CHANGED
|
@@ -15,6 +15,7 @@ import { isPromise } from "node:util/types";
|
|
|
15
15
|
import winston from "winston";
|
|
16
16
|
import DailyRotateFile from "winston-daily-rotate-file";
|
|
17
17
|
import { getLogsDir } from "./dirs";
|
|
18
|
+
import { drainModuleLoadEvents } from "./timing-buffer";
|
|
18
19
|
|
|
19
20
|
/** Ensure a logs directory exists; return the resolved path. */
|
|
20
21
|
function ensureDir(dir: string): string {
|
|
@@ -46,25 +47,30 @@ function jsonReplacer(_key: string, value: unknown): unknown {
|
|
|
46
47
|
return value;
|
|
47
48
|
}
|
|
48
49
|
|
|
49
|
-
/** Custom format that includes pid and flattens metadata */
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
50
|
+
/** Custom format that includes pid and flattens metadata; built on first use. */
|
|
51
|
+
let logFormat: winston.Logform.Format | undefined;
|
|
52
|
+
|
|
53
|
+
function getLogFormat(): winston.Logform.Format {
|
|
54
|
+
logFormat ??= winston.format.combine(
|
|
55
|
+
winston.format.timestamp({ format: "YYYY-MM-DDTHH:mm:ss.SSSZ" }),
|
|
56
|
+
winston.format.printf(({ timestamp, level, message, ...meta }) => {
|
|
57
|
+
const entry: Record<string, unknown> = {
|
|
58
|
+
timestamp,
|
|
59
|
+
level,
|
|
60
|
+
pid: process.pid,
|
|
61
|
+
message,
|
|
62
|
+
};
|
|
63
|
+
// Flatten metadata into entry
|
|
64
|
+
for (const [key, value] of Object.entries(meta)) {
|
|
65
|
+
if (key !== "level" && key !== "timestamp" && key !== "message") {
|
|
66
|
+
entry[key] = value;
|
|
67
|
+
}
|
|
63
68
|
}
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
69
|
+
return JSON.stringify(entry, jsonReplacer);
|
|
70
|
+
}),
|
|
71
|
+
);
|
|
72
|
+
return logFormat;
|
|
73
|
+
}
|
|
68
74
|
|
|
69
75
|
/** Build a rotating file transport, materializing the target directory lazily. */
|
|
70
76
|
function makeFileTransport(dir?: string): winston.transport {
|
|
@@ -79,17 +85,35 @@ function makeFileTransport(dir?: string): winston.transport {
|
|
|
79
85
|
}
|
|
80
86
|
|
|
81
87
|
function makeConsoleTransport(): winston.transport {
|
|
82
|
-
return new winston.transports.Console({ format:
|
|
88
|
+
return new winston.transports.Console({ format: getLogFormat() });
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Desired transport configuration, applied when the winston logger is built.
|
|
93
|
+
* Default: file ON (TUI-safe), console OFF.
|
|
94
|
+
*/
|
|
95
|
+
let transportOpts: { console?: boolean; file?: boolean | string } = { file: true };
|
|
96
|
+
|
|
97
|
+
/** The winston logger instance, created lazily on first log emission. */
|
|
98
|
+
let winstonLogger: winston.Logger | undefined;
|
|
99
|
+
|
|
100
|
+
function buildTransports(opts: { console?: boolean; file?: boolean | string }): winston.transport[] {
|
|
101
|
+
const transports: winston.transport[] = [];
|
|
102
|
+
if (opts.file) transports.push(makeFileTransport(typeof opts.file === "string" ? opts.file : undefined));
|
|
103
|
+
if (opts.console) transports.push(makeConsoleTransport());
|
|
104
|
+
return transports;
|
|
83
105
|
}
|
|
84
106
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
});
|
|
107
|
+
function getWinstonLogger(): winston.Logger {
|
|
108
|
+
winstonLogger ??= winston.createLogger({
|
|
109
|
+
level: "debug",
|
|
110
|
+
format: getLogFormat(),
|
|
111
|
+
transports: buildTransports(transportOpts),
|
|
112
|
+
// Don't exit on error - logging failures shouldn't crash the app
|
|
113
|
+
exitOnError: false,
|
|
114
|
+
});
|
|
115
|
+
return winstonLogger;
|
|
116
|
+
}
|
|
93
117
|
|
|
94
118
|
/**
|
|
95
119
|
* Replace the active log transports. Pass `console: true, file: false` for
|
|
@@ -97,11 +121,10 @@ const winstonLogger = winston.createLogger({
|
|
|
97
121
|
* logs piped into a process supervisor instead of the rotating file.
|
|
98
122
|
*/
|
|
99
123
|
export function setTransports(opts: { console?: boolean; file?: boolean | string }): void {
|
|
124
|
+
transportOpts = opts;
|
|
125
|
+
if (!winstonLogger) return; // applied lazily when the logger is first built
|
|
100
126
|
winstonLogger.clear();
|
|
101
|
-
|
|
102
|
-
winstonLogger.add(makeFileTransport(typeof opts.file === "string" ? opts.file : undefined));
|
|
103
|
-
}
|
|
104
|
-
if (opts.console) winstonLogger.add(makeConsoleTransport());
|
|
127
|
+
for (const transport of buildTransports(opts)) winstonLogger.add(transport);
|
|
105
128
|
}
|
|
106
129
|
|
|
107
130
|
/**
|
|
@@ -111,7 +134,7 @@ export function setTransports(opts: { console?: boolean; file?: boolean | string
|
|
|
111
134
|
*/
|
|
112
135
|
export function error(message: string, context?: Record<string, unknown>): void {
|
|
113
136
|
try {
|
|
114
|
-
|
|
137
|
+
getWinstonLogger().error(message, context);
|
|
115
138
|
} catch {
|
|
116
139
|
// Silently ignore logging failures
|
|
117
140
|
}
|
|
@@ -124,7 +147,7 @@ export function error(message: string, context?: Record<string, unknown>): void
|
|
|
124
147
|
*/
|
|
125
148
|
export function warn(message: string, context?: Record<string, unknown>): void {
|
|
126
149
|
try {
|
|
127
|
-
|
|
150
|
+
getWinstonLogger().warn(message, context);
|
|
128
151
|
} catch {
|
|
129
152
|
// Silently ignore logging failures
|
|
130
153
|
}
|
|
@@ -137,7 +160,7 @@ export function warn(message: string, context?: Record<string, unknown>): void {
|
|
|
137
160
|
*/
|
|
138
161
|
export function info(message: string, context?: Record<string, unknown>): void {
|
|
139
162
|
try {
|
|
140
|
-
|
|
163
|
+
getWinstonLogger().info(message, context);
|
|
141
164
|
} catch {
|
|
142
165
|
// Silently ignore logging failures
|
|
143
166
|
}
|
|
@@ -150,12 +173,29 @@ export function info(message: string, context?: Record<string, unknown>): void {
|
|
|
150
173
|
*/
|
|
151
174
|
export function debug(message: string, context?: Record<string, unknown>): void {
|
|
152
175
|
try {
|
|
153
|
-
|
|
176
|
+
getWinstonLogger().debug(message, context);
|
|
154
177
|
} catch {
|
|
155
178
|
// Silently ignore logging failures
|
|
156
179
|
}
|
|
157
180
|
}
|
|
158
181
|
|
|
182
|
+
/**
|
|
183
|
+
* Streaming startup markers, enabled by `PROMETHEUS_DEBUG_STARTUP`. Unlike the
|
|
184
|
+
* PROMETHEUS_TIMING tree (printed only after startup completes), these write one
|
|
185
|
+
* synchronous stderr line as each phase begins/ends, so a hard hang still
|
|
186
|
+
* shows the last phase that started. `fs.writeSync(2)` is used deliberately:
|
|
187
|
+
* it cannot be reordered or buffered past a synchronous block of the event
|
|
188
|
+
* loop (dlopen, sync fs on a dead mount, spawnSync).
|
|
189
|
+
*/
|
|
190
|
+
export function startupMarker(text: string): void {
|
|
191
|
+
if (!process.env.PROMETHEUS_DEBUG_STARTUP) return;
|
|
192
|
+
try {
|
|
193
|
+
fs.writeSync(2, `[startup] ${text}\n`);
|
|
194
|
+
} catch {
|
|
195
|
+
// stderr unavailable; markers are best-effort
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
159
199
|
const LOGGED_TIMING_THRESHOLD_MS = 0.5;
|
|
160
200
|
|
|
161
201
|
interface Span {
|
|
@@ -166,12 +206,36 @@ interface Span {
|
|
|
166
206
|
children: Span[];
|
|
167
207
|
/** Marker / point event without a duration. */
|
|
168
208
|
point?: boolean;
|
|
209
|
+
/** Absolute module path for module-load spans. */
|
|
210
|
+
modulePath?: string;
|
|
211
|
+
/** Own top-level module body / TLA duration for module-load spans. */
|
|
212
|
+
moduleBodyMs?: number;
|
|
213
|
+
/** Resolved static imports for module-load spans. */
|
|
214
|
+
moduleImports?: string[];
|
|
169
215
|
}
|
|
170
|
-
|
|
171
216
|
const spanStorage = new AsyncLocalStorage<Span>();
|
|
172
217
|
let gRootSpan: Span | undefined;
|
|
173
218
|
let gRecordTimings = false;
|
|
174
219
|
|
|
220
|
+
export function timingModeIncludes(option: "full" | "x"): boolean {
|
|
221
|
+
const value = process.env.PROMETHEUS_TIMING;
|
|
222
|
+
if (!value) return false;
|
|
223
|
+
if (value === option) return true;
|
|
224
|
+
let start = 0;
|
|
225
|
+
for (let i = 0; i <= value.length; i++) {
|
|
226
|
+
const code = i === value.length ? 44 : value.charCodeAt(i);
|
|
227
|
+
const separator = code === 44 || code === 58 || code === 59 || code === 43 || code <= 32;
|
|
228
|
+
if (!separator) continue;
|
|
229
|
+
if (i > start && value.slice(start, i) === option) return true;
|
|
230
|
+
start = i + 1;
|
|
231
|
+
}
|
|
232
|
+
return false;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
export function shouldExitAfterTimings(): boolean {
|
|
236
|
+
return timingModeIncludes("x") || timingModeIncludes("full");
|
|
237
|
+
}
|
|
238
|
+
|
|
175
239
|
/**
|
|
176
240
|
* Print collected timings as an indented tree.
|
|
177
241
|
* Each span shows wall duration; parents with children also show "(self)" for unattributed time.
|
|
@@ -184,9 +248,22 @@ export function printTimings(): void {
|
|
|
184
248
|
}
|
|
185
249
|
|
|
186
250
|
gRootSpan.end = performance.now();
|
|
251
|
+
// Splice any preload-captured module-load events into the tree as root
|
|
252
|
+
// children and back-extend the root window over them, so the static-import
|
|
253
|
+
// phase that ran before the first explicit marker becomes visible (the
|
|
254
|
+
// `(modules)` summary below) instead of being lumped into the opaque
|
|
255
|
+
// `(before instrumentation)` figure.
|
|
256
|
+
spliceModuleLoadBuffer();
|
|
187
257
|
const lines: string[] = [];
|
|
188
258
|
lines.push("");
|
|
189
259
|
lines.push("--- Startup timings (hierarchical) ---");
|
|
260
|
+
// performance.now() shares the process-start origin, so the root span's start
|
|
261
|
+
// is the wall time before the first marker — runtime init plus any module
|
|
262
|
+
// loads not captured below. With the module-load preload active this shrinks
|
|
263
|
+
// to ~runtime init because the load phase is back-folded into the window.
|
|
264
|
+
if (gRootSpan.start > LOGGED_TIMING_THRESHOLD_MS) {
|
|
265
|
+
lines.push(`(before instrumentation): ${fmtMs(gRootSpan.start)} [runtime init + module load]`);
|
|
266
|
+
}
|
|
190
267
|
const work: Span[] = [];
|
|
191
268
|
const loads: Span[] = [];
|
|
192
269
|
for (const child of gRootSpan.children) {
|
|
@@ -199,8 +276,14 @@ export function printTimings(): void {
|
|
|
199
276
|
if (loads.length > 0) {
|
|
200
277
|
printModuleLoadSummary(loads, 0, lines);
|
|
201
278
|
}
|
|
279
|
+
// Surface the root's own unattributed time so the gap between the visible
|
|
280
|
+
// top-level spans and Total isn't silently swallowed.
|
|
281
|
+
const rootSelf = selfTimeOf(gRootSpan);
|
|
282
|
+
if (gRootSpan.children.length > 0 && rootSelf > LOGGED_TIMING_THRESHOLD_MS) {
|
|
283
|
+
lines.push(`(unattributed self): ${fmtMs(rootSelf)}`);
|
|
284
|
+
}
|
|
202
285
|
const totalMs = (gRootSpan.end - gRootSpan.start).toFixed(1);
|
|
203
|
-
lines.push(`Total: ${totalMs}ms`);
|
|
286
|
+
lines.push(`Total: ${totalMs}ms (since first marker)`);
|
|
204
287
|
lines.push("--------------------------------------");
|
|
205
288
|
lines.push("");
|
|
206
289
|
console.error(lines.join("\n"));
|
|
@@ -209,8 +292,8 @@ export function printTimings(): void {
|
|
|
209
292
|
|
|
210
293
|
/**
|
|
211
294
|
* Begin recording startup timings under a new root span.
|
|
212
|
-
* Idempotent: a second call while already recording is a no-op so
|
|
213
|
-
*
|
|
295
|
+
* Idempotent: a second call while already recording is a no-op, so an explicit
|
|
296
|
+
* starter (main.ts) and any future early starter can coexist.
|
|
214
297
|
*/
|
|
215
298
|
export function startTiming(): void {
|
|
216
299
|
if (gRecordTimings) return;
|
|
@@ -225,10 +308,16 @@ export function startTiming(): void {
|
|
|
225
308
|
|
|
226
309
|
/**
|
|
227
310
|
* Record an externally-measured span as a leaf child of the active span (or root
|
|
228
|
-
* when no span is active). Used by
|
|
229
|
-
*
|
|
311
|
+
* when no span is active). Used by {@link spliceModuleLoadBuffer} to fold
|
|
312
|
+
* preload-captured module windows into the tree.
|
|
230
313
|
*/
|
|
231
|
-
export function recordModuleLoadSpan(
|
|
314
|
+
export function recordModuleLoadSpan(
|
|
315
|
+
path: string,
|
|
316
|
+
start: number,
|
|
317
|
+
durationMs: number,
|
|
318
|
+
bodyMs?: number,
|
|
319
|
+
imports: string[] = [],
|
|
320
|
+
): void {
|
|
232
321
|
if (!gRecordTimings || !gRootSpan) return;
|
|
233
322
|
const parent = spanStorage.getStore() ?? gRootSpan;
|
|
234
323
|
const span: Span = {
|
|
@@ -237,10 +326,32 @@ export function recordModuleLoadSpan(path: string, start: number, durationMs: nu
|
|
|
237
326
|
end: start + durationMs,
|
|
238
327
|
parent,
|
|
239
328
|
children: [],
|
|
329
|
+
modulePath: path,
|
|
330
|
+
moduleBodyMs: bodyMs,
|
|
331
|
+
moduleImports: imports,
|
|
240
332
|
};
|
|
241
333
|
parent.children.push(span);
|
|
242
334
|
}
|
|
243
335
|
|
|
336
|
+
/**
|
|
337
|
+
* Drain the preload's module-load buffer (see module-timer.ts) into the tree as
|
|
338
|
+
* `load:` children of the root, then back-extend the root window to the earliest
|
|
339
|
+
* captured read so the pre-marker load phase is counted in Total rather than
|
|
340
|
+
* hidden as `(before instrumentation)`. No-op when nothing was captured (e.g. no
|
|
341
|
+
* `--preload`, or a compiled binary where module reads are not interceptable).
|
|
342
|
+
*/
|
|
343
|
+
function spliceModuleLoadBuffer(): void {
|
|
344
|
+
if (!gRootSpan) return;
|
|
345
|
+
const events = drainModuleLoadEvents();
|
|
346
|
+
if (events.length === 0) return;
|
|
347
|
+
let earliest = gRootSpan.start;
|
|
348
|
+
for (const event of events) {
|
|
349
|
+
recordModuleLoadSpan(event.path, event.start, event.durationMs, event.bodyMs, event.imports);
|
|
350
|
+
if (event.start < earliest) earliest = event.start;
|
|
351
|
+
}
|
|
352
|
+
gRootSpan.start = earliest;
|
|
353
|
+
}
|
|
354
|
+
|
|
244
355
|
function shortenLoadPath(p: string): string {
|
|
245
356
|
const cwd = process.cwd();
|
|
246
357
|
if (p.startsWith(`${cwd}/`)) return p.slice(cwd.length + 1);
|
|
@@ -257,6 +368,29 @@ export function endTiming(): void {
|
|
|
257
368
|
gRecordTimings = false;
|
|
258
369
|
}
|
|
259
370
|
|
|
371
|
+
/**
|
|
372
|
+
* Ops of the currently-open span chain (root → deepest), following the most
|
|
373
|
+
* recently started unfinished child at each level. Lets a startup watchdog
|
|
374
|
+
* name the phase a stalled startup is stuck in.
|
|
375
|
+
*/
|
|
376
|
+
export function openSpanPath(): string[] {
|
|
377
|
+
const ops: string[] = [];
|
|
378
|
+
let node = gRootSpan;
|
|
379
|
+
while (node) {
|
|
380
|
+
let next: Span | undefined;
|
|
381
|
+
for (let i = node.children.length - 1; i >= 0; i--) {
|
|
382
|
+
if (node.children[i].end === undefined) {
|
|
383
|
+
next = node.children[i];
|
|
384
|
+
break;
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
if (!next) break;
|
|
388
|
+
ops.push(next.op);
|
|
389
|
+
node = next;
|
|
390
|
+
}
|
|
391
|
+
return ops;
|
|
392
|
+
}
|
|
393
|
+
|
|
260
394
|
function durationOf(span: Span): number {
|
|
261
395
|
if (span.point || span.end === undefined) return 0;
|
|
262
396
|
return span.end - span.start;
|
|
@@ -296,6 +430,16 @@ function fmtMs(ms: number): string {
|
|
|
296
430
|
|
|
297
431
|
const MODULE_LOAD_PREFIX = "load:";
|
|
298
432
|
const MODULE_LOAD_VERBOSE_TOP = 10;
|
|
433
|
+
const MODULE_TREE_MAX_DEPTH = 5;
|
|
434
|
+
const MODULE_TREE_ROOT_TOP = 5;
|
|
435
|
+
const MODULE_TREE_CHILD_TOP = 8;
|
|
436
|
+
|
|
437
|
+
interface ModuleTimingNode {
|
|
438
|
+
span: Span;
|
|
439
|
+
children: ModuleTimingNode[];
|
|
440
|
+
parents: number;
|
|
441
|
+
body: number;
|
|
442
|
+
}
|
|
299
443
|
|
|
300
444
|
function isModuleLoadSpan(span: Span): boolean {
|
|
301
445
|
return span.op.startsWith(MODULE_LOAD_PREFIX);
|
|
@@ -330,35 +474,120 @@ function printSpan(span: Span, depth: number, lines: string[]): void {
|
|
|
330
474
|
}
|
|
331
475
|
}
|
|
332
476
|
|
|
333
|
-
/**
|
|
477
|
+
/** Render module-load spans as a dependency-aware DAG/tree. */
|
|
334
478
|
function printModuleLoadSummary(loads: Span[], depth: number, lines: string[]): void {
|
|
335
479
|
const childIndent = " ".repeat(depth);
|
|
336
480
|
const grandIndent = " ".repeat(depth + 1);
|
|
337
481
|
let unionStart = Number.POSITIVE_INFINITY;
|
|
338
482
|
let unionEnd = 0;
|
|
339
|
-
let totalSelf = 0;
|
|
340
483
|
for (const span of loads) {
|
|
341
484
|
if (span.end === undefined) continue;
|
|
342
485
|
if (span.start < unionStart) unionStart = span.start;
|
|
343
486
|
if (span.end > unionEnd) unionEnd = span.end;
|
|
344
|
-
totalSelf += span.end - span.start;
|
|
345
487
|
}
|
|
346
488
|
const wall = unionEnd > unionStart ? unionEnd - unionStart : 0;
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
489
|
+
const nodes = buildModuleTimingGraph(loads);
|
|
490
|
+
lines.push(`${childIndent}(modules): ${loads.length} loaded, wall ${fmtMs(wall)}`);
|
|
491
|
+
if (nodes.length === 0) return;
|
|
492
|
+
|
|
493
|
+
const showAll = timingModeIncludes("full");
|
|
494
|
+
const byBody = [...nodes].sort(compareModuleNodes);
|
|
495
|
+
const topBody = showAll ? byBody : byBody.slice(0, MODULE_LOAD_VERBOSE_TOP);
|
|
496
|
+
lines.push(`${grandIndent}top body/TLA:`);
|
|
497
|
+
for (const node of topBody) {
|
|
498
|
+
if (!showAll && node.body < LOGGED_TIMING_THRESHOLD_MS) break;
|
|
499
|
+
lines.push(`${grandIndent} ${node.span.op}: body ${fmtMs(node.body)} (total ${fmtMs(durationOf(node.span))})`);
|
|
500
|
+
}
|
|
501
|
+
if (!showAll && byBody.length > MODULE_LOAD_VERBOSE_TOP) {
|
|
502
|
+
lines.push(
|
|
503
|
+
`${grandIndent} … ${byBody.length - MODULE_LOAD_VERBOSE_TOP} more (PROMETHEUS_TIMING=full to show all)`,
|
|
504
|
+
);
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
const roots = nodes.filter(node => node.parents === 0);
|
|
508
|
+
const treeRoots = (roots.length > 0 ? roots : nodes).sort((a, b) => durationOf(b.span) - durationOf(a.span));
|
|
509
|
+
const visibleRoots = showAll ? treeRoots : treeRoots.slice(0, MODULE_TREE_ROOT_TOP);
|
|
510
|
+
lines.push(`${grandIndent}tree:`);
|
|
511
|
+
const rendered = new Set<string>();
|
|
512
|
+
for (const node of visibleRoots) {
|
|
513
|
+
renderModuleTimingNode(node, depth + 2, lines, rendered, new Set<string>(), showAll);
|
|
514
|
+
}
|
|
515
|
+
if (!showAll && treeRoots.length > MODULE_TREE_ROOT_TOP) {
|
|
516
|
+
lines.push(
|
|
517
|
+
`${grandIndent} … ${treeRoots.length - MODULE_TREE_ROOT_TOP} more roots (PROMETHEUS_TIMING=full to show all)`,
|
|
518
|
+
);
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
function buildModuleTimingGraph(loads: Span[]): ModuleTimingNode[] {
|
|
523
|
+
const nodes = new Map<string, ModuleTimingNode>();
|
|
524
|
+
for (const span of loads) {
|
|
525
|
+
if (!span.modulePath || span.end === undefined) continue;
|
|
526
|
+
nodes.set(span.modulePath, { span, children: [], parents: 0, body: span.moduleBodyMs ?? 0 });
|
|
527
|
+
}
|
|
528
|
+
for (const node of nodes.values()) {
|
|
529
|
+
for (const childPath of node.span.moduleImports ?? []) {
|
|
530
|
+
const child = nodes.get(childPath);
|
|
531
|
+
if (!child || child === node) continue;
|
|
532
|
+
node.children.push(child);
|
|
533
|
+
child.parents++;
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
for (const node of nodes.values()) {
|
|
537
|
+
node.children.sort(compareModuleNodes);
|
|
538
|
+
}
|
|
539
|
+
return [...nodes.values()];
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
function compareModuleNodes(a: ModuleTimingNode, b: ModuleTimingNode): number {
|
|
543
|
+
const bodyDiff = b.body - a.body;
|
|
544
|
+
if (Math.abs(bodyDiff) > 0.001) return bodyDiff;
|
|
545
|
+
return durationOf(b.span) - durationOf(a.span);
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
function renderModuleTimingNode(
|
|
549
|
+
node: ModuleTimingNode,
|
|
550
|
+
depth: number,
|
|
551
|
+
lines: string[],
|
|
552
|
+
rendered: Set<string>,
|
|
553
|
+
ancestors: Set<string>,
|
|
554
|
+
showAll: boolean,
|
|
555
|
+
): void {
|
|
556
|
+
const path = node.span.modulePath;
|
|
557
|
+
if (!path) return;
|
|
558
|
+
const indent = " ".repeat(depth);
|
|
559
|
+
const total = durationOf(node.span);
|
|
560
|
+
if (!showAll && total < LOGGED_TIMING_THRESHOLD_MS && node.children.length === 0) return;
|
|
561
|
+
const wait = Math.max(0, total - node.body);
|
|
562
|
+
const shared = node.parents > 1 ? " [shared]" : "";
|
|
563
|
+
const timing =
|
|
564
|
+
node.body > LOGGED_TIMING_THRESHOLD_MS || node.children.length > 0
|
|
565
|
+
? ` (body ${fmtMs(node.body)}, wait ${fmtMs(wait)})`
|
|
566
|
+
: "";
|
|
567
|
+
const alreadyRendered = rendered.has(path);
|
|
568
|
+
const cycle = ancestors.has(path);
|
|
569
|
+
const suffix = cycle ? " [cycle]" : alreadyRendered ? " [already shown]" : "";
|
|
570
|
+
lines.push(`${indent}${node.span.op}: ${fmtMs(total)}${timing}${shared}${suffix}`);
|
|
571
|
+
if (cycle || alreadyRendered) return;
|
|
572
|
+
rendered.add(path);
|
|
573
|
+
ancestors.add(path);
|
|
574
|
+
if (!showAll && ancestors.size >= MODULE_TREE_MAX_DEPTH) {
|
|
575
|
+
if (node.children.length > 0) {
|
|
576
|
+
lines.push(`${indent} … ${node.children.length} imports deeper (PROMETHEUS_TIMING=full to show all)`);
|
|
577
|
+
}
|
|
578
|
+
ancestors.delete(path);
|
|
579
|
+
return;
|
|
580
|
+
}
|
|
581
|
+
const visibleChildren = showAll ? node.children : node.children.slice(0, MODULE_TREE_CHILD_TOP);
|
|
582
|
+
for (const child of visibleChildren) {
|
|
583
|
+
renderModuleTimingNode(child, depth + 1, lines, rendered, ancestors, showAll);
|
|
584
|
+
}
|
|
585
|
+
if (!showAll && node.children.length > MODULE_TREE_CHILD_TOP) {
|
|
358
586
|
lines.push(
|
|
359
|
-
`${
|
|
587
|
+
`${indent} … ${node.children.length - MODULE_TREE_CHILD_TOP} more imports (PROMETHEUS_TIMING=full to show all)`,
|
|
360
588
|
);
|
|
361
589
|
}
|
|
590
|
+
ancestors.delete(path);
|
|
362
591
|
}
|
|
363
592
|
|
|
364
593
|
/** A span is parallel if it overlaps a sibling that started before it. */
|
|
@@ -385,33 +614,51 @@ function isParallel(span: Span): boolean {
|
|
|
385
614
|
export function time(op: string): void;
|
|
386
615
|
export function time<T, A extends unknown[]>(op: string, fn: (...args: A) => T, ...args: A): T;
|
|
387
616
|
export function time<T, A extends unknown[]>(op: string, fn?: (...args: A) => T, ...args: A): T | undefined {
|
|
388
|
-
|
|
389
|
-
if (fn === undefined) return undefined as T;
|
|
390
|
-
return fn(...args);
|
|
391
|
-
}
|
|
392
|
-
|
|
393
|
-
const parent = spanStorage.getStore() ?? gRootSpan;
|
|
394
|
-
const span: Span = { op, start: performance.now(), parent, children: [] };
|
|
395
|
-
parent.children.push(span);
|
|
617
|
+
const recording = gRecordTimings && gRootSpan !== undefined;
|
|
396
618
|
|
|
397
619
|
if (fn === undefined) {
|
|
398
|
-
|
|
399
|
-
|
|
620
|
+
startupMarker(op);
|
|
621
|
+
if (!recording) return undefined as T;
|
|
622
|
+
const parent = spanStorage.getStore() ?? gRootSpan!;
|
|
623
|
+
const now = performance.now();
|
|
624
|
+
parent.children.push({ op, start: now, end: now, parent, children: [], point: true });
|
|
400
625
|
return undefined as T;
|
|
401
626
|
}
|
|
402
627
|
|
|
403
|
-
|
|
404
|
-
|
|
628
|
+
if (!recording && !process.env.PROMETHEUS_DEBUG_STARTUP) {
|
|
629
|
+
return fn(...args);
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
startupMarker(`${op}:start`);
|
|
633
|
+
let span: Span | undefined;
|
|
634
|
+
if (recording) {
|
|
635
|
+
const parent = spanStorage.getStore() ?? gRootSpan!;
|
|
636
|
+
span = { op, start: performance.now(), parent, children: [] };
|
|
637
|
+
parent.children.push(span);
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
const finish = (ok: boolean): void => {
|
|
641
|
+
if (span) span.end = performance.now();
|
|
642
|
+
startupMarker(ok ? `${op}:done` : `${op}:fail`);
|
|
405
643
|
};
|
|
406
644
|
try {
|
|
407
|
-
const result = spanStorage.run(span, () => fn(...args));
|
|
645
|
+
const result = span ? spanStorage.run(span, () => fn(...args)) : fn(...args);
|
|
408
646
|
if (isPromise(result)) {
|
|
409
|
-
return result.
|
|
647
|
+
return result.then(
|
|
648
|
+
value => {
|
|
649
|
+
finish(true);
|
|
650
|
+
return value;
|
|
651
|
+
},
|
|
652
|
+
error => {
|
|
653
|
+
finish(false);
|
|
654
|
+
throw error;
|
|
655
|
+
},
|
|
656
|
+
) as T;
|
|
410
657
|
}
|
|
411
|
-
finish();
|
|
658
|
+
finish(true);
|
|
412
659
|
return result;
|
|
413
660
|
} catch (error) {
|
|
414
|
-
finish();
|
|
661
|
+
finish(false);
|
|
415
662
|
throw error;
|
|
416
663
|
}
|
|
417
664
|
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Live event-loop phase breadcrumb. Hot synchronous paths push a short label
|
|
3
|
+
* before running and pop it after (via `try`/`finally`); the loop watchdog
|
|
4
|
+
* reads {@link takeRecentLoopPhase} when it detects a block, so a stall is
|
|
5
|
+
* logged with the work that caused it instead of an opaque "unknown".
|
|
6
|
+
*
|
|
7
|
+
* This is deliberately a process-global stack and not part of the logger span
|
|
8
|
+
* machinery: `main.ts` ends timing spans before the interactive TUI starts, so
|
|
9
|
+
* `logger.openSpanPath()` is empty in a live session.
|
|
10
|
+
*
|
|
11
|
+
* Correctness constraint: each `pushLoopPhase` must be balanced by a
|
|
12
|
+
* `popLoopPhase` within the SAME synchronous execution (always via `try`/
|
|
13
|
+
* `finally`). The stack is global and shared, so a label held across an
|
|
14
|
+
* `await`/async boundary — or interleaved between concurrent tasks — would
|
|
15
|
+
* misattribute or leak phases. Instrument only synchronous spans; for async
|
|
16
|
+
* work, push/pop around each synchronous chunk, not across the await.
|
|
17
|
+
*/
|
|
18
|
+
const stack: string[] = [];
|
|
19
|
+
// The most recent label pushed, retained after it is popped. A hot path pushes
|
|
20
|
+
// and pops a phase entirely within one synchronous macrotask, so by the time
|
|
21
|
+
// the watchdog's delayed tick runs the stack is already empty; this slot keeps
|
|
22
|
+
// the culprit available for that one tick. Consumed (cleared) on read so it
|
|
23
|
+
// only attributes the just-elapsed interval.
|
|
24
|
+
let recentPhase: string | undefined;
|
|
25
|
+
|
|
26
|
+
export function pushLoopPhase(label: string): void {
|
|
27
|
+
stack.push(label);
|
|
28
|
+
recentPhase = label;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function popLoopPhase(): void {
|
|
32
|
+
stack.pop();
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function currentLoopPhase(): string | undefined {
|
|
36
|
+
return stack[stack.length - 1];
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Phase to blame for a just-detected loop block: the live top phase if one is
|
|
41
|
+
* still held, else the most recent phase pushed since the last call. Clears the
|
|
42
|
+
* recent slot so a block in a later, phase-less interval is not misattributed
|
|
43
|
+
* to a phase that already finished.
|
|
44
|
+
*/
|
|
45
|
+
export function takeRecentLoopPhase(): string | undefined {
|
|
46
|
+
const phase = stack[stack.length - 1] ?? recentPhase;
|
|
47
|
+
recentPhase = undefined;
|
|
48
|
+
return phase;
|
|
49
|
+
}
|