@origintrail-official/dkg-core 10.0.11 → 10.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backpressure-observability.d.ts +175 -0
- package/dist/backpressure-observability.d.ts.map +1 -0
- package/dist/backpressure-observability.js +495 -0
- package/dist/backpressure-observability.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/ka-ual-identity.d.ts +33 -0
- package/dist/ka-ual-identity.d.ts.map +1 -0
- package/dist/ka-ual-identity.js +89 -0
- package/dist/ka-ual-identity.js.map +1 -0
- package/dist/telemetry-api.d.ts +59 -0
- package/dist/telemetry-api.d.ts.map +1 -1
- package/dist/telemetry-api.js +93 -0
- package/dist/telemetry-api.js.map +1 -1
- package/dist/vm-update-convergence.d.ts +277 -0
- package/dist/vm-update-convergence.d.ts.map +1 -0
- package/dist/vm-update-convergence.js +715 -0
- package/dist/vm-update-convergence.js.map +1 -0
- package/package.json +2 -2
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
export type BackpressureState = 'healthy' | 'degraded' | 'saturated' | 'stalled';
|
|
2
|
+
export type SchedulerPressureOutcome = 'completed' | 'failed' | 'cancelled' | 'released';
|
|
3
|
+
export interface SchedulerPressureCapacity {
|
|
4
|
+
queueLimit?: number | null;
|
|
5
|
+
inflightLimit?: number | null;
|
|
6
|
+
lanes?: Record<string, {
|
|
7
|
+
queueLimit?: number | null;
|
|
8
|
+
inflightLimit?: number | null;
|
|
9
|
+
}>;
|
|
10
|
+
}
|
|
11
|
+
export interface SchedulerPressureThresholds {
|
|
12
|
+
/** Queue age that turns otherwise-low utilization into degraded pressure. */
|
|
13
|
+
degradedQueueAgeMs?: number;
|
|
14
|
+
/** Active-work age that indicates an admitted operation may be stuck. */
|
|
15
|
+
stalledActiveAgeMs?: number;
|
|
16
|
+
/** Fraction of a bounded queue that marks a lane as degraded. */
|
|
17
|
+
degradedQueueUtilization?: number;
|
|
18
|
+
/** Keep a recent admission rejection visible as saturation for this long. */
|
|
19
|
+
rejectionStateWindowMs?: number;
|
|
20
|
+
}
|
|
21
|
+
export interface SchedulerPressureWork {
|
|
22
|
+
lane: string;
|
|
23
|
+
operation: string;
|
|
24
|
+
}
|
|
25
|
+
export interface SchedulerPressureTicket {
|
|
26
|
+
readonly id: number;
|
|
27
|
+
}
|
|
28
|
+
export interface BackpressureOperationSummary {
|
|
29
|
+
operation: string;
|
|
30
|
+
count: number;
|
|
31
|
+
oldestAgeMs: number;
|
|
32
|
+
}
|
|
33
|
+
export interface BackpressureLaneSnapshot {
|
|
34
|
+
lane: string;
|
|
35
|
+
state: BackpressureState;
|
|
36
|
+
queued: number;
|
|
37
|
+
queueLimit: number | null;
|
|
38
|
+
inflight: number;
|
|
39
|
+
inflightLimit: number | null;
|
|
40
|
+
oldestQueuedAgeMs: number;
|
|
41
|
+
oldestActiveAgeMs: number;
|
|
42
|
+
queuedOperations: BackpressureOperationSummary[];
|
|
43
|
+
activeOperations: BackpressureOperationSummary[];
|
|
44
|
+
events: Record<string, number>;
|
|
45
|
+
rejectedTotal: number;
|
|
46
|
+
rejectedByReason: Record<string, number>;
|
|
47
|
+
/** Age of the latest rejection; safe for monotonic or epoch-based clocks. */
|
|
48
|
+
lastRejectedAgeMs: number | null;
|
|
49
|
+
}
|
|
50
|
+
export interface BackpressureSnapshot {
|
|
51
|
+
scheduler: string;
|
|
52
|
+
state: BackpressureState;
|
|
53
|
+
totals: {
|
|
54
|
+
queued: number;
|
|
55
|
+
queueLimit: number | null;
|
|
56
|
+
inflight: number;
|
|
57
|
+
inflightLimit: number | null;
|
|
58
|
+
oldestQueuedAgeMs: number;
|
|
59
|
+
oldestActiveAgeMs: number;
|
|
60
|
+
rejectedTotal: number;
|
|
61
|
+
};
|
|
62
|
+
lanes: BackpressureLaneSnapshot[];
|
|
63
|
+
}
|
|
64
|
+
export interface BackpressureSource {
|
|
65
|
+
readonly backpressureId: string;
|
|
66
|
+
getBackpressureSnapshot(): BackpressureSnapshot;
|
|
67
|
+
}
|
|
68
|
+
export interface BackpressureRegistrySnapshot {
|
|
69
|
+
capturedAt: string;
|
|
70
|
+
state: BackpressureState;
|
|
71
|
+
schedulers: BackpressureSnapshot[];
|
|
72
|
+
failures: Array<{
|
|
73
|
+
scheduler: string;
|
|
74
|
+
error: string;
|
|
75
|
+
}>;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Keep metric/log dimensions bounded and strip payload-like punctuation.
|
|
79
|
+
* Callers should still pass static operation names rather than graph, peer, or
|
|
80
|
+
* job identifiers.
|
|
81
|
+
*/
|
|
82
|
+
export declare function normalizeBackpressureLabel(value: string, fallback?: string): string;
|
|
83
|
+
export interface SchedulerPressureTrackerOptions {
|
|
84
|
+
scheduler: string;
|
|
85
|
+
capacity?: SchedulerPressureCapacity;
|
|
86
|
+
thresholds?: SchedulerPressureThresholds;
|
|
87
|
+
now?: () => number;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Scheduling-policy-neutral lifecycle tracker.
|
|
91
|
+
*
|
|
92
|
+
* Queue implementations call the five transition methods at their existing
|
|
93
|
+
* admission boundaries. The tracker owns timings, state classification,
|
|
94
|
+
* bounded metrics, and diagnostic snapshots, but never decides which work may
|
|
95
|
+
* run. All observability calls are fail-open so instrumentation cannot change
|
|
96
|
+
* scheduler behaviour.
|
|
97
|
+
*/
|
|
98
|
+
export declare class SchedulerPressureTracker {
|
|
99
|
+
readonly scheduler: string;
|
|
100
|
+
private readonly queued;
|
|
101
|
+
private readonly active;
|
|
102
|
+
private readonly lanes;
|
|
103
|
+
private readonly now;
|
|
104
|
+
private readonly thresholds;
|
|
105
|
+
private capacity;
|
|
106
|
+
private nextTicketId;
|
|
107
|
+
constructor(options: SchedulerPressureTrackerOptions);
|
|
108
|
+
updateCapacity(capacity: SchedulerPressureCapacity): void;
|
|
109
|
+
enqueue(work: SchedulerPressureWork): SchedulerPressureTicket;
|
|
110
|
+
start(ticket: SchedulerPressureTicket): void;
|
|
111
|
+
reject(work: SchedulerPressureWork, reason: string): void;
|
|
112
|
+
rejectQueued(ticket: SchedulerPressureTicket, reason: string): void;
|
|
113
|
+
cancelQueued(ticket: SchedulerPressureTicket, reason?: string): void;
|
|
114
|
+
finish(ticket: SchedulerPressureTicket, outcome: SchedulerPressureOutcome): void;
|
|
115
|
+
snapshot(): BackpressureSnapshot;
|
|
116
|
+
private runtimeFor;
|
|
117
|
+
private recordEvent;
|
|
118
|
+
private recordRejection;
|
|
119
|
+
private laneSnapshot;
|
|
120
|
+
private sumLaneLimits;
|
|
121
|
+
private safeMetric;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Base class for in-memory schedulers. Subclasses retain complete ownership of
|
|
125
|
+
* queue ordering, admission, coalescing, and release semantics and call these
|
|
126
|
+
* protected lifecycle methods at their existing boundaries.
|
|
127
|
+
*/
|
|
128
|
+
export declare abstract class ObservableScheduler implements BackpressureSource {
|
|
129
|
+
protected readonly pressure: SchedulerPressureTracker;
|
|
130
|
+
protected constructor(options: SchedulerPressureTrackerOptions);
|
|
131
|
+
get backpressureId(): string;
|
|
132
|
+
getBackpressureSnapshot(): BackpressureSnapshot;
|
|
133
|
+
protected updatePressureCapacity(capacity: SchedulerPressureCapacity): void;
|
|
134
|
+
protected pressureEnqueue(work: SchedulerPressureWork): SchedulerPressureTicket;
|
|
135
|
+
protected pressureStart(ticket: SchedulerPressureTicket): void;
|
|
136
|
+
protected pressureReject(work: SchedulerPressureWork, reason: string): void;
|
|
137
|
+
protected pressureRejectQueued(ticket: SchedulerPressureTicket, reason: string): void;
|
|
138
|
+
protected pressureCancelQueued(ticket: SchedulerPressureTicket, reason?: string): void;
|
|
139
|
+
protected pressureFinish(ticket: SchedulerPressureTicket, outcome: SchedulerPressureOutcome): void;
|
|
140
|
+
}
|
|
141
|
+
export declare class BackpressureRegistry {
|
|
142
|
+
private readonly sources;
|
|
143
|
+
register(source: BackpressureSource): () => void;
|
|
144
|
+
capture(): BackpressureRegistrySnapshot;
|
|
145
|
+
}
|
|
146
|
+
export declare const backpressureRegistry: BackpressureRegistry;
|
|
147
|
+
export declare function recordBackpressureSnapshotMetrics(snapshot: BackpressureSnapshot): void;
|
|
148
|
+
export interface BackpressureMonitorOptions {
|
|
149
|
+
registry?: BackpressureRegistry;
|
|
150
|
+
intervalMs?: number;
|
|
151
|
+
summaryIntervalMs?: number;
|
|
152
|
+
now?: () => number;
|
|
153
|
+
emit: (level: 'info' | 'warn', message: string, snapshot: BackpressureSnapshot, lane: BackpressureLaneSnapshot | null) => void;
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* One process-wide sampler provides transition/recovery logging and periodic
|
|
157
|
+
* summaries. It logs state changes rather than individual queue operations.
|
|
158
|
+
*/
|
|
159
|
+
export declare class BackpressureMonitor {
|
|
160
|
+
private readonly registry;
|
|
161
|
+
private readonly intervalMs;
|
|
162
|
+
private readonly summaryIntervalMs;
|
|
163
|
+
private readonly now;
|
|
164
|
+
private readonly emit;
|
|
165
|
+
private readonly logged;
|
|
166
|
+
private timer;
|
|
167
|
+
constructor(options: BackpressureMonitorOptions);
|
|
168
|
+
start(): void;
|
|
169
|
+
stop(): void;
|
|
170
|
+
sample(): void;
|
|
171
|
+
private observeSample;
|
|
172
|
+
private message;
|
|
173
|
+
private safeEmit;
|
|
174
|
+
}
|
|
175
|
+
//# sourceMappingURL=backpressure-observability.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"backpressure-observability.d.ts","sourceRoot":"","sources":["../src/backpressure-observability.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,iBAAiB,GAAG,SAAS,GAAG,UAAU,GAAG,WAAW,GAAG,SAAS,CAAC;AACjF,MAAM,MAAM,wBAAwB,GAAG,WAAW,GAAG,QAAQ,GAAG,WAAW,GAAG,UAAU,CAAC;AAEzF,MAAM,WAAW,yBAAyB;IACxC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE;QACrB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;KAC/B,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,2BAA2B;IAC1C,6EAA6E;IAC7E,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,yEAAyE;IACzE,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,iEAAiE;IACjE,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,6EAA6E;IAC7E,sBAAsB,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,4BAA4B;IAC3C,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,wBAAwB;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,iBAAiB,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,4BAA4B,EAAE,CAAC;IACjD,gBAAgB,EAAE,4BAA4B,EAAE,CAAC;IACjD,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACzC,6EAA6E;IAC7E,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC;AAED,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,iBAAiB,CAAC;IACzB,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1B,QAAQ,EAAE,MAAM,CAAC;QACjB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;QAC7B,iBAAiB,EAAE,MAAM,CAAC;QAC1B,iBAAiB,EAAE,MAAM,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,KAAK,EAAE,wBAAwB,EAAE,CAAC;CACnC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,uBAAuB,IAAI,oBAAoB,CAAC;CACjD;AAED,MAAM,WAAW,4BAA4B;IAC3C,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,iBAAiB,CAAC;IACzB,UAAU,EAAE,oBAAoB,EAAE,CAAC;IACnC,QAAQ,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACvD;AAqCD;;;;GAIG;AACH,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,SAAY,GAAG,MAAM,CAItF;AAgCD,MAAM,WAAW,+BAA+B;IAC9C,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,yBAAyB,CAAC;IACrC,UAAU,CAAC,EAAE,2BAA2B,CAAC;IACzC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED;;;;;;;;GAQG;AACH,qBAAa,wBAAwB;IACnC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAE3B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAyC;IAChE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAyC;IAChE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAkC;IACxD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAwC;IACnE,OAAO,CAAC,QAAQ,CAA4B;IAC5C,OAAO,CAAC,YAAY,CAAK;gBAEb,OAAO,EAAE,+BAA+B;IAgBpD,cAAc,CAAC,QAAQ,EAAE,yBAAyB,GAAG,IAAI;IAIzD,OAAO,CAAC,IAAI,EAAE,qBAAqB,GAAG,uBAAuB;IAY7D,KAAK,CAAC,MAAM,EAAE,uBAAuB,GAAG,IAAI;IAa5C,MAAM,CAAC,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAKzD,YAAY,CAAC,MAAM,EAAE,uBAAuB,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAOnE,YAAY,CAAC,MAAM,EAAE,uBAAuB,EAAE,MAAM,SAAc,GAAG,IAAI;IAOzE,MAAM,CAAC,MAAM,EAAE,uBAAuB,EAAE,OAAO,EAAE,wBAAwB,GAAG,IAAI;IAgBhF,QAAQ,IAAI,oBAAoB;IAqDhC,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,WAAW;IAWnB,OAAO,CAAC,eAAe;IAWvB,OAAO,CAAC,YAAY;IA4DpB,OAAO,CAAC,aAAa;IAQrB,OAAO,CAAC,UAAU;CAOnB;AAED;;;;GAIG;AACH,8BAAsB,mBAAoB,YAAW,kBAAkB;IACrE,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,wBAAwB,CAAC;IAEtD,SAAS,aAAa,OAAO,EAAE,+BAA+B;IAI9D,IAAI,cAAc,IAAI,MAAM,CAE3B;IAED,uBAAuB,IAAI,oBAAoB;IAI/C,SAAS,CAAC,sBAAsB,CAAC,QAAQ,EAAE,yBAAyB,GAAG,IAAI;IAI3E,SAAS,CAAC,eAAe,CAAC,IAAI,EAAE,qBAAqB,GAAG,uBAAuB;IAI/E,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,uBAAuB,GAAG,IAAI;IAI9D,SAAS,CAAC,cAAc,CAAC,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAI3E,SAAS,CAAC,oBAAoB,CAAC,MAAM,EAAE,uBAAuB,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAIrF,SAAS,CAAC,oBAAoB,CAAC,MAAM,EAAE,uBAAuB,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI;IAItF,SAAS,CAAC,cAAc,CACtB,MAAM,EAAE,uBAAuB,EAC/B,OAAO,EAAE,wBAAwB,GAChC,IAAI;CAGR;AAED,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyC;IAEjE,QAAQ,CAAC,MAAM,EAAE,kBAAkB,GAAG,MAAM,IAAI;IAYhD,OAAO,IAAI,4BAA4B;CAuBxC;AAED,eAAO,MAAM,oBAAoB,sBAA6B,CAAC;AAE/D,wBAAgB,iCAAiC,CAAC,QAAQ,EAAE,oBAAoB,GAAG,IAAI,CAkCtF;AAED,MAAM,WAAW,0BAA0B;IACzC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;IAChC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,IAAI,EAAE,CACJ,KAAK,EAAE,MAAM,GAAG,MAAM,EACtB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,oBAAoB,EAC9B,IAAI,EAAE,wBAAwB,GAAG,IAAI,KAClC,IAAI,CAAC;CACX;AAOD;;;GAGG;AACH,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAuB;IAChD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAqC;IAC1D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0C;IACjE,OAAO,CAAC,KAAK,CAA+C;gBAEhD,OAAO,EAAE,0BAA0B;IAQ/C,KAAK,IAAI,IAAI;IAOb,IAAI,IAAI,IAAI;IAMZ,MAAM,IAAI,IAAI;IAgCd,OAAO,CAAC,aAAa;IAkCrB,OAAO,CAAC,OAAO;IAyCf,OAAO,CAAC,QAAQ;CAYjB"}
|
|
@@ -0,0 +1,495 @@
|
|
|
1
|
+
import { getMetrics } from './telemetry-api.js';
|
|
2
|
+
const DEFAULT_DEGRADED_QUEUE_AGE_MS = 5_000;
|
|
3
|
+
const DEFAULT_STALLED_ACTIVE_AGE_MS = 30_000;
|
|
4
|
+
const DEFAULT_DEGRADED_QUEUE_UTILIZATION = 0.75;
|
|
5
|
+
const DEFAULT_REJECTION_STATE_WINDOW_MS = 60_000;
|
|
6
|
+
const MAX_OPERATION_SUMMARIES = 8;
|
|
7
|
+
const STATE_RANK = {
|
|
8
|
+
healthy: 0,
|
|
9
|
+
degraded: 1,
|
|
10
|
+
saturated: 2,
|
|
11
|
+
stalled: 3,
|
|
12
|
+
};
|
|
13
|
+
function maxState(a, b) {
|
|
14
|
+
return STATE_RANK[a] >= STATE_RANK[b] ? a : b;
|
|
15
|
+
}
|
|
16
|
+
function normalizeLimit(value) {
|
|
17
|
+
return Number.isFinite(value) && value >= 0 ? value : null;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Keep metric/log dimensions bounded and strip payload-like punctuation.
|
|
21
|
+
* Callers should still pass static operation names rather than graph, peer, or
|
|
22
|
+
* job identifiers.
|
|
23
|
+
*/
|
|
24
|
+
export function normalizeBackpressureLabel(value, fallback = 'unknown') {
|
|
25
|
+
const trimmed = value.trim();
|
|
26
|
+
if (!trimmed)
|
|
27
|
+
return fallback;
|
|
28
|
+
return trimmed.replace(/[^\w:./-]/g, '_').slice(0, 80) || fallback;
|
|
29
|
+
}
|
|
30
|
+
function mapToRecord(map) {
|
|
31
|
+
return Object.fromEntries([...map.entries()].sort(([a], [b]) => a.localeCompare(b)));
|
|
32
|
+
}
|
|
33
|
+
function operationSummaries(records, timestamp, now) {
|
|
34
|
+
const byOperation = new Map();
|
|
35
|
+
for (const record of records) {
|
|
36
|
+
const existing = byOperation.get(record.operation);
|
|
37
|
+
const at = timestamp(record);
|
|
38
|
+
if (existing) {
|
|
39
|
+
existing.count += 1;
|
|
40
|
+
existing.oldestAt = Math.min(existing.oldestAt, at);
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
byOperation.set(record.operation, { count: 1, oldestAt: at });
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return [...byOperation.entries()]
|
|
47
|
+
.map(([operation, value]) => ({
|
|
48
|
+
operation,
|
|
49
|
+
count: value.count,
|
|
50
|
+
oldestAgeMs: Math.max(0, Math.floor(now - value.oldestAt)),
|
|
51
|
+
}))
|
|
52
|
+
.sort((a, b) => b.oldestAgeMs - a.oldestAgeMs || b.count - a.count || a.operation.localeCompare(b.operation))
|
|
53
|
+
.slice(0, MAX_OPERATION_SUMMARIES);
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Scheduling-policy-neutral lifecycle tracker.
|
|
57
|
+
*
|
|
58
|
+
* Queue implementations call the five transition methods at their existing
|
|
59
|
+
* admission boundaries. The tracker owns timings, state classification,
|
|
60
|
+
* bounded metrics, and diagnostic snapshots, but never decides which work may
|
|
61
|
+
* run. All observability calls are fail-open so instrumentation cannot change
|
|
62
|
+
* scheduler behaviour.
|
|
63
|
+
*/
|
|
64
|
+
export class SchedulerPressureTracker {
|
|
65
|
+
scheduler;
|
|
66
|
+
queued = new Map();
|
|
67
|
+
active = new Map();
|
|
68
|
+
lanes = new Map();
|
|
69
|
+
now;
|
|
70
|
+
thresholds;
|
|
71
|
+
capacity;
|
|
72
|
+
nextTicketId = 1;
|
|
73
|
+
constructor(options) {
|
|
74
|
+
this.scheduler = normalizeBackpressureLabel(options.scheduler, 'scheduler');
|
|
75
|
+
this.capacity = options.capacity ?? {};
|
|
76
|
+
this.now = options.now ?? Date.now;
|
|
77
|
+
this.thresholds = {
|
|
78
|
+
degradedQueueAgeMs: options.thresholds?.degradedQueueAgeMs ?? DEFAULT_DEGRADED_QUEUE_AGE_MS,
|
|
79
|
+
stalledActiveAgeMs: options.thresholds?.stalledActiveAgeMs ?? DEFAULT_STALLED_ACTIVE_AGE_MS,
|
|
80
|
+
degradedQueueUtilization: options.thresholds?.degradedQueueUtilization ?? DEFAULT_DEGRADED_QUEUE_UTILIZATION,
|
|
81
|
+
rejectionStateWindowMs: options.thresholds?.rejectionStateWindowMs ?? DEFAULT_REJECTION_STATE_WINDOW_MS,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
updateCapacity(capacity) {
|
|
85
|
+
this.capacity = capacity;
|
|
86
|
+
}
|
|
87
|
+
enqueue(work) {
|
|
88
|
+
const record = {
|
|
89
|
+
id: this.nextTicketId++,
|
|
90
|
+
lane: normalizeBackpressureLabel(work.lane, 'default'),
|
|
91
|
+
operation: normalizeBackpressureLabel(work.operation),
|
|
92
|
+
queuedAt: this.now(),
|
|
93
|
+
};
|
|
94
|
+
this.queued.set(record.id, record);
|
|
95
|
+
this.recordEvent(record.lane, 'enqueued');
|
|
96
|
+
return { id: record.id };
|
|
97
|
+
}
|
|
98
|
+
start(ticket) {
|
|
99
|
+
const record = this.queued.get(ticket.id);
|
|
100
|
+
if (!record)
|
|
101
|
+
return;
|
|
102
|
+
this.queued.delete(ticket.id);
|
|
103
|
+
record.startedAt = this.now();
|
|
104
|
+
this.active.set(ticket.id, record);
|
|
105
|
+
this.recordEvent(record.lane, 'started');
|
|
106
|
+
this.safeMetric(() => getMetrics().backpressureQueueWaitMs.record(Math.max(0, record.startedAt - record.queuedAt), { scheduler: this.scheduler, lane: record.lane }));
|
|
107
|
+
}
|
|
108
|
+
reject(work, reason) {
|
|
109
|
+
const lane = normalizeBackpressureLabel(work.lane, 'default');
|
|
110
|
+
this.recordRejection(lane, reason);
|
|
111
|
+
}
|
|
112
|
+
rejectQueued(ticket, reason) {
|
|
113
|
+
const record = this.queued.get(ticket.id);
|
|
114
|
+
if (!record)
|
|
115
|
+
return;
|
|
116
|
+
this.queued.delete(ticket.id);
|
|
117
|
+
this.recordRejection(record.lane, reason);
|
|
118
|
+
}
|
|
119
|
+
cancelQueued(ticket, reason = 'cancelled') {
|
|
120
|
+
const record = this.queued.get(ticket.id);
|
|
121
|
+
if (!record)
|
|
122
|
+
return;
|
|
123
|
+
this.queued.delete(ticket.id);
|
|
124
|
+
this.recordEvent(record.lane, 'cancelled', reason);
|
|
125
|
+
}
|
|
126
|
+
finish(ticket, outcome) {
|
|
127
|
+
const record = this.active.get(ticket.id);
|
|
128
|
+
if (!record)
|
|
129
|
+
return;
|
|
130
|
+
this.active.delete(ticket.id);
|
|
131
|
+
const finishedAt = this.now();
|
|
132
|
+
this.recordEvent(record.lane, outcome);
|
|
133
|
+
this.safeMetric(() => getMetrics().backpressureActiveDurationMs.record(Math.max(0, finishedAt - (record.startedAt ?? finishedAt)), {
|
|
134
|
+
scheduler: this.scheduler,
|
|
135
|
+
lane: record.lane,
|
|
136
|
+
outcome,
|
|
137
|
+
}));
|
|
138
|
+
}
|
|
139
|
+
snapshot() {
|
|
140
|
+
const now = this.now();
|
|
141
|
+
const laneNames = new Set([
|
|
142
|
+
...this.lanes.keys(),
|
|
143
|
+
...Object.keys(this.capacity.lanes ?? {}),
|
|
144
|
+
...[...this.queued.values()].map((entry) => entry.lane),
|
|
145
|
+
...[...this.active.values()].map((entry) => entry.lane),
|
|
146
|
+
]);
|
|
147
|
+
const snapshots = [...laneNames]
|
|
148
|
+
.sort()
|
|
149
|
+
.map((lane) => this.laneSnapshot(lane, now));
|
|
150
|
+
const totals = {
|
|
151
|
+
queued: snapshots.reduce((sum, lane) => sum + lane.queued, 0),
|
|
152
|
+
queueLimit: normalizeLimit(this.capacity.queueLimit)
|
|
153
|
+
?? this.sumLaneLimits('queueLimit', snapshots),
|
|
154
|
+
inflight: snapshots.reduce((sum, lane) => sum + lane.inflight, 0),
|
|
155
|
+
inflightLimit: normalizeLimit(this.capacity.inflightLimit)
|
|
156
|
+
?? this.sumLaneLimits('inflightLimit', snapshots),
|
|
157
|
+
oldestQueuedAgeMs: Math.max(0, ...snapshots.map((lane) => lane.oldestQueuedAgeMs)),
|
|
158
|
+
oldestActiveAgeMs: Math.max(0, ...snapshots.map((lane) => lane.oldestActiveAgeMs)),
|
|
159
|
+
rejectedTotal: snapshots.reduce((sum, lane) => sum + lane.rejectedTotal, 0),
|
|
160
|
+
};
|
|
161
|
+
let state = snapshots.reduce((current, lane) => maxState(current, lane.state), 'healthy');
|
|
162
|
+
if (totals.queueLimit !== null
|
|
163
|
+
&& totals.queueLimit > 0
|
|
164
|
+
&& totals.queued >= totals.queueLimit) {
|
|
165
|
+
state = maxState(state, 'saturated');
|
|
166
|
+
}
|
|
167
|
+
else if (totals.queueLimit !== null
|
|
168
|
+
&& totals.queueLimit > 0
|
|
169
|
+
&& totals.queued / totals.queueLimit >= this.thresholds.degradedQueueUtilization) {
|
|
170
|
+
state = maxState(state, 'degraded');
|
|
171
|
+
}
|
|
172
|
+
if (totals.oldestQueuedAgeMs >= this.thresholds.degradedQueueAgeMs) {
|
|
173
|
+
state = maxState(state, 'degraded');
|
|
174
|
+
}
|
|
175
|
+
if (totals.oldestActiveAgeMs >= this.thresholds.stalledActiveAgeMs) {
|
|
176
|
+
state = maxState(state, 'stalled');
|
|
177
|
+
}
|
|
178
|
+
return {
|
|
179
|
+
scheduler: this.scheduler,
|
|
180
|
+
state,
|
|
181
|
+
totals,
|
|
182
|
+
lanes: snapshots,
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
runtimeFor(lane) {
|
|
186
|
+
let runtime = this.lanes.get(lane);
|
|
187
|
+
if (!runtime) {
|
|
188
|
+
runtime = {
|
|
189
|
+
events: new Map(),
|
|
190
|
+
rejectedByReason: new Map(),
|
|
191
|
+
lastRejectedAt: null,
|
|
192
|
+
};
|
|
193
|
+
this.lanes.set(lane, runtime);
|
|
194
|
+
}
|
|
195
|
+
return runtime;
|
|
196
|
+
}
|
|
197
|
+
recordEvent(lane, event, reason) {
|
|
198
|
+
const runtime = this.runtimeFor(lane);
|
|
199
|
+
runtime.events.set(event, (runtime.events.get(event) ?? 0) + 1);
|
|
200
|
+
this.safeMetric(() => getMetrics().backpressureEventsTotal.add(1, {
|
|
201
|
+
scheduler: this.scheduler,
|
|
202
|
+
lane,
|
|
203
|
+
event,
|
|
204
|
+
...(reason ? { reason: normalizeBackpressureLabel(reason) } : {}),
|
|
205
|
+
}));
|
|
206
|
+
}
|
|
207
|
+
recordRejection(lane, reason) {
|
|
208
|
+
const normalizedReason = normalizeBackpressureLabel(reason, 'rejected');
|
|
209
|
+
const runtime = this.runtimeFor(lane);
|
|
210
|
+
runtime.rejectedByReason.set(normalizedReason, (runtime.rejectedByReason.get(normalizedReason) ?? 0) + 1);
|
|
211
|
+
runtime.lastRejectedAt = this.now();
|
|
212
|
+
this.recordEvent(lane, 'rejected', normalizedReason);
|
|
213
|
+
}
|
|
214
|
+
laneSnapshot(lane, now) {
|
|
215
|
+
const runtime = this.runtimeFor(lane);
|
|
216
|
+
const queued = [...this.queued.values()].filter((entry) => entry.lane === lane);
|
|
217
|
+
const active = [...this.active.values()].filter((entry) => entry.lane === lane);
|
|
218
|
+
const queueLimit = normalizeLimit(this.capacity.lanes?.[lane]?.queueLimit);
|
|
219
|
+
const inflightLimit = normalizeLimit(this.capacity.lanes?.[lane]?.inflightLimit);
|
|
220
|
+
const oldestQueuedAgeMs = queued.length === 0
|
|
221
|
+
? 0
|
|
222
|
+
: Math.max(...queued.map((entry) => Math.max(0, Math.floor(now - entry.queuedAt))));
|
|
223
|
+
const oldestActiveAgeMs = active.length === 0
|
|
224
|
+
? 0
|
|
225
|
+
: Math.max(...active.map((entry) => Math.max(0, Math.floor(now - (entry.startedAt ?? now)))));
|
|
226
|
+
let state = 'healthy';
|
|
227
|
+
if (oldestActiveAgeMs >= this.thresholds.stalledActiveAgeMs) {
|
|
228
|
+
state = 'stalled';
|
|
229
|
+
}
|
|
230
|
+
else if ((queueLimit !== null && queueLimit > 0 && queued.length >= queueLimit)
|
|
231
|
+
|| (runtime.lastRejectedAt !== null
|
|
232
|
+
&& now - runtime.lastRejectedAt <= this.thresholds.rejectionStateWindowMs)) {
|
|
233
|
+
state = 'saturated';
|
|
234
|
+
}
|
|
235
|
+
else if (oldestQueuedAgeMs >= this.thresholds.degradedQueueAgeMs
|
|
236
|
+
|| (queueLimit !== null
|
|
237
|
+
&& queueLimit > 0
|
|
238
|
+
&& queued.length / queueLimit >= this.thresholds.degradedQueueUtilization)) {
|
|
239
|
+
state = 'degraded';
|
|
240
|
+
}
|
|
241
|
+
return {
|
|
242
|
+
lane,
|
|
243
|
+
state,
|
|
244
|
+
queued: queued.length,
|
|
245
|
+
queueLimit,
|
|
246
|
+
inflight: active.length,
|
|
247
|
+
inflightLimit,
|
|
248
|
+
oldestQueuedAgeMs,
|
|
249
|
+
oldestActiveAgeMs,
|
|
250
|
+
queuedOperations: operationSummaries(queued, (entry) => entry.queuedAt, now),
|
|
251
|
+
activeOperations: operationSummaries(active, (entry) => entry.startedAt ?? now, now),
|
|
252
|
+
events: mapToRecord(runtime.events),
|
|
253
|
+
rejectedTotal: [...runtime.rejectedByReason.values()].reduce((sum, value) => sum + value, 0),
|
|
254
|
+
rejectedByReason: mapToRecord(runtime.rejectedByReason),
|
|
255
|
+
lastRejectedAgeMs: runtime.lastRejectedAt === null
|
|
256
|
+
? null
|
|
257
|
+
: Math.max(0, Math.floor(now - runtime.lastRejectedAt)),
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
sumLaneLimits(field, lanes) {
|
|
261
|
+
if (lanes.length === 0 || lanes.some((lane) => lane[field] === null))
|
|
262
|
+
return null;
|
|
263
|
+
return lanes.reduce((sum, lane) => sum + (lane[field] ?? 0), 0);
|
|
264
|
+
}
|
|
265
|
+
safeMetric(record) {
|
|
266
|
+
try {
|
|
267
|
+
record();
|
|
268
|
+
}
|
|
269
|
+
catch {
|
|
270
|
+
// Metrics are strictly fail-open.
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
/**
|
|
275
|
+
* Base class for in-memory schedulers. Subclasses retain complete ownership of
|
|
276
|
+
* queue ordering, admission, coalescing, and release semantics and call these
|
|
277
|
+
* protected lifecycle methods at their existing boundaries.
|
|
278
|
+
*/
|
|
279
|
+
export class ObservableScheduler {
|
|
280
|
+
pressure;
|
|
281
|
+
constructor(options) {
|
|
282
|
+
this.pressure = new SchedulerPressureTracker(options);
|
|
283
|
+
}
|
|
284
|
+
get backpressureId() {
|
|
285
|
+
return this.pressure.scheduler;
|
|
286
|
+
}
|
|
287
|
+
getBackpressureSnapshot() {
|
|
288
|
+
return this.pressure.snapshot();
|
|
289
|
+
}
|
|
290
|
+
updatePressureCapacity(capacity) {
|
|
291
|
+
this.pressure.updateCapacity(capacity);
|
|
292
|
+
}
|
|
293
|
+
pressureEnqueue(work) {
|
|
294
|
+
return this.pressure.enqueue(work);
|
|
295
|
+
}
|
|
296
|
+
pressureStart(ticket) {
|
|
297
|
+
this.pressure.start(ticket);
|
|
298
|
+
}
|
|
299
|
+
pressureReject(work, reason) {
|
|
300
|
+
this.pressure.reject(work, reason);
|
|
301
|
+
}
|
|
302
|
+
pressureRejectQueued(ticket, reason) {
|
|
303
|
+
this.pressure.rejectQueued(ticket, reason);
|
|
304
|
+
}
|
|
305
|
+
pressureCancelQueued(ticket, reason) {
|
|
306
|
+
this.pressure.cancelQueued(ticket, reason);
|
|
307
|
+
}
|
|
308
|
+
pressureFinish(ticket, outcome) {
|
|
309
|
+
this.pressure.finish(ticket, outcome);
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
export class BackpressureRegistry {
|
|
313
|
+
sources = new Map();
|
|
314
|
+
register(source) {
|
|
315
|
+
const id = normalizeBackpressureLabel(source.backpressureId, 'scheduler');
|
|
316
|
+
const existing = this.sources.get(id);
|
|
317
|
+
if (existing && existing !== source) {
|
|
318
|
+
throw new Error(`Backpressure source "${id}" is already registered`);
|
|
319
|
+
}
|
|
320
|
+
this.sources.set(id, source);
|
|
321
|
+
return () => {
|
|
322
|
+
if (this.sources.get(id) === source)
|
|
323
|
+
this.sources.delete(id);
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
capture() {
|
|
327
|
+
const schedulers = [];
|
|
328
|
+
const failures = [];
|
|
329
|
+
for (const [id, source] of [...this.sources.entries()].sort(([a], [b]) => a.localeCompare(b))) {
|
|
330
|
+
try {
|
|
331
|
+
schedulers.push(source.getBackpressureSnapshot());
|
|
332
|
+
}
|
|
333
|
+
catch (error) {
|
|
334
|
+
failures.push({
|
|
335
|
+
scheduler: id,
|
|
336
|
+
error: error instanceof Error ? error.message : String(error),
|
|
337
|
+
});
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
return {
|
|
341
|
+
capturedAt: new Date().toISOString(),
|
|
342
|
+
state: schedulers.reduce((current, scheduler) => maxState(current, scheduler.state), 'healthy'),
|
|
343
|
+
schedulers,
|
|
344
|
+
failures,
|
|
345
|
+
};
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
export const backpressureRegistry = new BackpressureRegistry();
|
|
349
|
+
export function recordBackpressureSnapshotMetrics(snapshot) {
|
|
350
|
+
const metrics = getMetrics();
|
|
351
|
+
const record = (lane, values) => {
|
|
352
|
+
const attributes = { scheduler: snapshot.scheduler, lane };
|
|
353
|
+
metrics.backpressureQueueDepth.record(values.queued, attributes);
|
|
354
|
+
metrics.backpressureInflight.record(values.inflight, attributes);
|
|
355
|
+
metrics.backpressureOldestQueuedAgeMs.record(values.oldestQueuedAgeMs, attributes);
|
|
356
|
+
metrics.backpressureOldestActiveAgeMs.record(values.oldestActiveAgeMs, attributes);
|
|
357
|
+
if (values.queueLimit !== null) {
|
|
358
|
+
metrics.backpressureQueueLimit.record(values.queueLimit, attributes);
|
|
359
|
+
}
|
|
360
|
+
if (values.inflightLimit !== null) {
|
|
361
|
+
metrics.backpressureInflightLimit.record(values.inflightLimit, attributes);
|
|
362
|
+
}
|
|
363
|
+
};
|
|
364
|
+
record('all', {
|
|
365
|
+
queued: snapshot.totals.queued,
|
|
366
|
+
queueLimit: snapshot.totals.queueLimit,
|
|
367
|
+
inflight: snapshot.totals.inflight,
|
|
368
|
+
inflightLimit: snapshot.totals.inflightLimit,
|
|
369
|
+
oldestQueuedAgeMs: snapshot.totals.oldestQueuedAgeMs,
|
|
370
|
+
oldestActiveAgeMs: snapshot.totals.oldestActiveAgeMs,
|
|
371
|
+
});
|
|
372
|
+
for (const lane of snapshot.lanes)
|
|
373
|
+
record(lane.lane, lane);
|
|
374
|
+
}
|
|
375
|
+
/**
|
|
376
|
+
* One process-wide sampler provides transition/recovery logging and periodic
|
|
377
|
+
* summaries. It logs state changes rather than individual queue operations.
|
|
378
|
+
*/
|
|
379
|
+
export class BackpressureMonitor {
|
|
380
|
+
registry;
|
|
381
|
+
intervalMs;
|
|
382
|
+
summaryIntervalMs;
|
|
383
|
+
now;
|
|
384
|
+
emit;
|
|
385
|
+
logged = new Map();
|
|
386
|
+
timer = null;
|
|
387
|
+
constructor(options) {
|
|
388
|
+
this.registry = options.registry ?? backpressureRegistry;
|
|
389
|
+
this.intervalMs = Math.max(1_000, options.intervalMs ?? 5_000);
|
|
390
|
+
this.summaryIntervalMs = Math.max(this.intervalMs, options.summaryIntervalMs ?? 60_000);
|
|
391
|
+
this.now = options.now ?? Date.now;
|
|
392
|
+
this.emit = options.emit;
|
|
393
|
+
}
|
|
394
|
+
start() {
|
|
395
|
+
if (this.timer)
|
|
396
|
+
return;
|
|
397
|
+
this.sample();
|
|
398
|
+
this.timer = setInterval(() => this.sample(), this.intervalMs);
|
|
399
|
+
if (typeof this.timer.unref === 'function')
|
|
400
|
+
this.timer.unref();
|
|
401
|
+
}
|
|
402
|
+
stop() {
|
|
403
|
+
if (!this.timer)
|
|
404
|
+
return;
|
|
405
|
+
clearInterval(this.timer);
|
|
406
|
+
this.timer = null;
|
|
407
|
+
}
|
|
408
|
+
sample() {
|
|
409
|
+
const captured = this.registry.capture();
|
|
410
|
+
const now = this.now();
|
|
411
|
+
for (const scheduler of captured.schedulers) {
|
|
412
|
+
try {
|
|
413
|
+
recordBackpressureSnapshotMetrics(scheduler);
|
|
414
|
+
}
|
|
415
|
+
catch {
|
|
416
|
+
// The monitor must keep logging even if a metrics provider misbehaves.
|
|
417
|
+
}
|
|
418
|
+
const worstLaneState = scheduler.lanes.reduce((state, lane) => maxState(state, lane.state), 'healthy');
|
|
419
|
+
const samples = scheduler.lanes.map((lane) => ({
|
|
420
|
+
key: `${scheduler.scheduler}/${lane.lane}`,
|
|
421
|
+
state: lane.state,
|
|
422
|
+
lane,
|
|
423
|
+
}));
|
|
424
|
+
if (STATE_RANK[scheduler.state] > STATE_RANK[worstLaneState]) {
|
|
425
|
+
samples.push({
|
|
426
|
+
key: `${scheduler.scheduler}/all`,
|
|
427
|
+
state: scheduler.state,
|
|
428
|
+
lane: null,
|
|
429
|
+
});
|
|
430
|
+
}
|
|
431
|
+
for (const sample of samples) {
|
|
432
|
+
this.observeSample(sample.key, sample.state, scheduler, sample.lane, now);
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
observeSample(key, state, scheduler, lane, now) {
|
|
437
|
+
const previous = this.logged.get(key);
|
|
438
|
+
if (state === 'healthy') {
|
|
439
|
+
if (previous && previous.state !== 'healthy') {
|
|
440
|
+
this.safeEmit('info', this.message('recovered', scheduler, lane, state, previous.state), scheduler, lane);
|
|
441
|
+
}
|
|
442
|
+
this.logged.set(key, { state, lastLoggedAt: previous?.lastLoggedAt ?? now });
|
|
443
|
+
return;
|
|
444
|
+
}
|
|
445
|
+
const transition = !previous || previous.state !== state;
|
|
446
|
+
const summaryDue = !previous || now - previous.lastLoggedAt >= this.summaryIntervalMs;
|
|
447
|
+
if (transition || summaryDue) {
|
|
448
|
+
this.safeEmit('warn', this.message(transition ? 'transition' : 'summary', scheduler, lane, state, previous?.state), scheduler, lane);
|
|
449
|
+
this.logged.set(key, { state, lastLoggedAt: now });
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
message(event, scheduler, lane, state, previousState) {
|
|
453
|
+
const values = lane ?? {
|
|
454
|
+
lane: 'all',
|
|
455
|
+
queued: scheduler.totals.queued,
|
|
456
|
+
queueLimit: scheduler.totals.queueLimit,
|
|
457
|
+
inflight: scheduler.totals.inflight,
|
|
458
|
+
inflightLimit: scheduler.totals.inflightLimit,
|
|
459
|
+
oldestQueuedAgeMs: scheduler.totals.oldestQueuedAgeMs,
|
|
460
|
+
oldestActiveAgeMs: scheduler.totals.oldestActiveAgeMs,
|
|
461
|
+
rejectedTotal: scheduler.totals.rejectedTotal,
|
|
462
|
+
queuedOperations: scheduler.lanes.flatMap((item) => item.queuedOperations)
|
|
463
|
+
.sort((a, b) => b.oldestAgeMs - a.oldestAgeMs)
|
|
464
|
+
.slice(0, MAX_OPERATION_SUMMARIES),
|
|
465
|
+
activeOperations: scheduler.lanes.flatMap((item) => item.activeOperations)
|
|
466
|
+
.sort((a, b) => b.oldestAgeMs - a.oldestAgeMs)
|
|
467
|
+
.slice(0, MAX_OPERATION_SUMMARIES),
|
|
468
|
+
};
|
|
469
|
+
return `[backpressure] ${JSON.stringify({
|
|
470
|
+
event,
|
|
471
|
+
scheduler: scheduler.scheduler,
|
|
472
|
+
lane: values.lane,
|
|
473
|
+
state,
|
|
474
|
+
...(previousState ? { previousState } : {}),
|
|
475
|
+
queued: values.queued,
|
|
476
|
+
queueLimit: values.queueLimit,
|
|
477
|
+
inflight: values.inflight,
|
|
478
|
+
inflightLimit: values.inflightLimit,
|
|
479
|
+
oldestQueuedAgeMs: values.oldestQueuedAgeMs,
|
|
480
|
+
oldestActiveAgeMs: values.oldestActiveAgeMs,
|
|
481
|
+
rejectedTotal: values.rejectedTotal,
|
|
482
|
+
queuedOperations: values.queuedOperations,
|
|
483
|
+
activeOperations: values.activeOperations,
|
|
484
|
+
})}`;
|
|
485
|
+
}
|
|
486
|
+
safeEmit(level, message, scheduler, lane) {
|
|
487
|
+
try {
|
|
488
|
+
this.emit(level, message, scheduler, lane);
|
|
489
|
+
}
|
|
490
|
+
catch {
|
|
491
|
+
// Logging must never break scheduling or the monitor loop.
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
//# sourceMappingURL=backpressure-observability.js.map
|