@crewhaus/runtime-core 0.1.8 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/alert-watchdog.d.ts +138 -0
- package/dist/alert-watchdog.js +290 -0
- package/dist/incident-collector.d.ts +42 -0
- package/dist/incident-collector.js +125 -0
- package/dist/index.d.ts +286 -4
- package/dist/index.js +733 -30
- package/dist/janitor.d.ts +88 -0
- package/dist/janitor.js +287 -0
- package/dist/observability.d.ts +178 -2
- package/dist/observability.js +466 -2
- package/dist/project-memory.d.ts +17 -1
- package/dist/project-memory.js +42 -0
- package/dist/slo-monitor.d.ts +221 -0
- package/dist/slo-monitor.js +397 -0
- package/package.json +40 -39
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ops item 37 — the production-signal SLO monitor.
|
|
3
|
+
*
|
|
4
|
+
* A bus subscriber (wired into `attachDefaultSubscribers`, gated by CREWHAUS_SLO
|
|
5
|
+
* + a lowered `observability.slo` spec block) that folds live TraceEvents into
|
|
6
|
+
* ROLLING TIME WINDOWS and walks the declared mitigation ladder: `alert`
|
|
7
|
+
* (webhook/hook) → `pause-intake` (gateway/managed 429 `budget_exceeded` path) →
|
|
8
|
+
* `rollback` (auto-rollback the env pin via the deployment-controller). The
|
|
9
|
+
* `alert` rung fires immediately on the first breached evaluation; the
|
|
10
|
+
* DESTRUCTIVE rungs (`pause-intake`, `rollback`) fire ONLY on a SUSTAINED
|
|
11
|
+
* breach — the same metric breached for ≥ N consecutive evaluations spanning at
|
|
12
|
+
* least the declared window, never on a single transient tick (a blip that
|
|
13
|
+
* clears resets the streak). When the breach clears, a paused intake is resumed
|
|
14
|
+
* and the pause rung re-arms. Every rung is audit-logged through the same
|
|
15
|
+
* injected seams the alert watchdog uses, so runtime-core owns no
|
|
16
|
+
* deployment-controller / gateway / audit-log I/O — the caller (CLI / codegen)
|
|
17
|
+
* injects it.
|
|
18
|
+
*
|
|
19
|
+
* WHY A ROLLING WINDOW (vs the alert-watchdog's per-session snapshot): an SLO is
|
|
20
|
+
* a live, in-flight guarantee — "p95 TTFT under 1.4s over the last 5 minutes" —
|
|
21
|
+
* so it must fire DURING a run, not at session end. The watchdog grades a whole
|
|
22
|
+
* session against history AFTER the fact; the SLO monitor grades a moving window
|
|
23
|
+
* against a DECLARED target AS IT HAPPENS. The two are complementary and share
|
|
24
|
+
* the same event-folding vocabulary (turn durations, TTFT, cost, egress).
|
|
25
|
+
*
|
|
26
|
+
* Pure/unit-tested core: the window accumulator, the breach detector, and the
|
|
27
|
+
* ladder walk (with injected sinks) are side-effect-free apart from the trace
|
|
28
|
+
* event the monitor publishes.
|
|
29
|
+
*/
|
|
30
|
+
import type { RunContext } from "@crewhaus/run-context";
|
|
31
|
+
import type { TraceEvent, TraceEventBus, Unsubscribe } from "@crewhaus/trace-event-bus";
|
|
32
|
+
/** A mitigation-ladder rung, in the order the monitor walks it on a breach. */
|
|
33
|
+
export type SloMitigationRung = "alert" | "pause-intake" | "rollback";
|
|
34
|
+
/** The SLO targets the monitor evaluates. Every field is optional; an omitted
|
|
35
|
+
* target is never checked. Mirrors the lowered `IrSlo` (camelCase). */
|
|
36
|
+
export type SloTargets = {
|
|
37
|
+
readonly errorRate?: number;
|
|
38
|
+
readonly p95LatencyMs?: number;
|
|
39
|
+
readonly ttftMs?: number;
|
|
40
|
+
readonly costPerHourUsd?: number;
|
|
41
|
+
readonly egressBlockRate?: number;
|
|
42
|
+
/** Rolling window (ms) a breach must persist before the ladder fires. */
|
|
43
|
+
readonly windowMs?: number;
|
|
44
|
+
readonly mitigation: ReadonlyArray<SloMitigationRung>;
|
|
45
|
+
};
|
|
46
|
+
/** Default rolling window when the spec omits `window_seconds`: 5 minutes. */
|
|
47
|
+
export declare const DEFAULT_SLO_WINDOW_MS: number;
|
|
48
|
+
/** A single SLO breach found by {@link detectSloBreaches}. */
|
|
49
|
+
export type SloBreach = {
|
|
50
|
+
readonly metric: string;
|
|
51
|
+
readonly observed: number;
|
|
52
|
+
readonly target: number;
|
|
53
|
+
readonly detail: string;
|
|
54
|
+
};
|
|
55
|
+
/** The payload handed to the mitigation sink for each ladder rung executed. */
|
|
56
|
+
export type SloMitigationEvent = {
|
|
57
|
+
readonly sessionId: string;
|
|
58
|
+
readonly rung: SloMitigationRung;
|
|
59
|
+
readonly breach: SloBreach;
|
|
60
|
+
readonly windowMs: number;
|
|
61
|
+
};
|
|
62
|
+
/**
|
|
63
|
+
* Injected mitigation sink — the ladder rungs beyond the trace event + the
|
|
64
|
+
* always-on `alert_raised`-style delivery the alert watchdog already owns.
|
|
65
|
+
* Every callback is OPTIONAL and supplied by the caller so runtime-core stays
|
|
66
|
+
* free of deployment-controller / gateway / audit-log I/O:
|
|
67
|
+
* - `alert` → webhook/hook delivery + audit append (reuses the alert
|
|
68
|
+
* sink the CLI already builds);
|
|
69
|
+
* - `pauseIntake` → flip the gateway/managed daemon's intake gate so new
|
|
70
|
+
* requests get the 429 `budget_exceeded` path until cleared;
|
|
71
|
+
* - `rollback` → auto-rollback the env pin via the deployment-controller.
|
|
72
|
+
* A rung with no injected handler is a no-op (still audit-logged as attempted),
|
|
73
|
+
* so an SLO spec that asks for `rollback` on a shape with no controller wired
|
|
74
|
+
* degrades to alert-only rather than crashing.
|
|
75
|
+
*/
|
|
76
|
+
export type SloMitigationSink = {
|
|
77
|
+
alert?: (event: SloMitigationEvent) => Promise<void>;
|
|
78
|
+
pauseIntake?: (event: SloMitigationEvent) => Promise<void>;
|
|
79
|
+
/** RESUME intake — called when a paused breach clears so the durable gate is
|
|
80
|
+
* flipped back to `paused:false` and admission re-opens. Optional: a shape
|
|
81
|
+
* with no gate leaves this undefined and the pause simply stays until an
|
|
82
|
+
* operator clears it. */
|
|
83
|
+
resumeIntake?: (event: SloMitigationEvent) => Promise<void>;
|
|
84
|
+
rollback?: (event: SloMitigationEvent) => Promise<void>;
|
|
85
|
+
/** Audit every ATTEMPTED rung (even a no-op one) — one durable record. */
|
|
86
|
+
audit?: (event: SloMitigationEvent) => Promise<void>;
|
|
87
|
+
};
|
|
88
|
+
/**
|
|
89
|
+
* Accumulates a session's SLO signals into rolling windows keyed by wall-clock
|
|
90
|
+
* timestamp. `evaluate(now)` prunes everything older than `windowMs`, then
|
|
91
|
+
* computes the window's rate/percentile metrics. Mirrors the alert-watchdog's
|
|
92
|
+
* event→metric vocabulary (turn_end duration, TTFT from model_request→first
|
|
93
|
+
* model_stream_token, cost accrual, egress verdicts) so the monitor does not
|
|
94
|
+
* depend on the opt-in metrics-collector being attached.
|
|
95
|
+
*/
|
|
96
|
+
export declare class SloWindow {
|
|
97
|
+
private readonly windowMs;
|
|
98
|
+
private readonly turnLatencies;
|
|
99
|
+
private readonly ttfts;
|
|
100
|
+
private readonly costMicros;
|
|
101
|
+
private readonly modelCalls;
|
|
102
|
+
private readonly unrecoveredErrors;
|
|
103
|
+
private readonly externalCalls;
|
|
104
|
+
private readonly egressBlocks;
|
|
105
|
+
private readonly requestStarts;
|
|
106
|
+
private readonly ttftSeen;
|
|
107
|
+
/** Wall-clock time of the FIRST event ever folded — lets `evaluate` report how
|
|
108
|
+
* long the monitor has actually been observing, so a rate metric (cost/hour)
|
|
109
|
+
* extrapolated from a near-empty window that has only existed for a few
|
|
110
|
+
* seconds is not mistaken for a sustained burn. */
|
|
111
|
+
private firstFoldTs;
|
|
112
|
+
constructor(windowMs: number);
|
|
113
|
+
/** Fold one event into the window. `ev.timestamp` seeds the sample time. */
|
|
114
|
+
fold(ev: TraceEvent): void;
|
|
115
|
+
/**
|
|
116
|
+
* Compute the current window's SLO metrics at `now`, pruning samples older
|
|
117
|
+
* than `windowMs` first. Returns `undefined`-valued metrics as 0 counts /
|
|
118
|
+
* empty percentiles so the detector can decide "no data yet" per metric.
|
|
119
|
+
*/
|
|
120
|
+
evaluate(now: number): SloWindowMetrics;
|
|
121
|
+
}
|
|
122
|
+
export type SloWindowMetrics = {
|
|
123
|
+
readonly turnP95Ms: number;
|
|
124
|
+
readonly turnSamples: number;
|
|
125
|
+
readonly ttftP95Ms: number;
|
|
126
|
+
readonly ttftSamples: number;
|
|
127
|
+
readonly errorRate: number;
|
|
128
|
+
readonly modelCalls: number;
|
|
129
|
+
readonly costPerHourUsd: number;
|
|
130
|
+
/** Number of non-summary `cost_accrual` samples in the window. */
|
|
131
|
+
readonly costSamples: number;
|
|
132
|
+
/** Wall-clock span (ms) actually observed, capped at the rolling window. */
|
|
133
|
+
readonly windowElapsedMs: number;
|
|
134
|
+
readonly egressBlockRate: number;
|
|
135
|
+
readonly externalCalls: number;
|
|
136
|
+
};
|
|
137
|
+
/**
|
|
138
|
+
* Minimum samples a rate/percentile metric needs before it can breach — a
|
|
139
|
+
* single erroring call is a 100% error rate but not an SLO violation. Keeps the
|
|
140
|
+
* monitor from firing on cold-start noise. Cost uses its own (smaller) sample
|
|
141
|
+
* floor plus a min-elapsed-window floor (see {@link MIN_COST_SAMPLES} /
|
|
142
|
+
* {@link MIN_COST_ELAPSED_MS}) rather than {@link MIN_SLO_SAMPLES}, because a
|
|
143
|
+
* genuine burn spike can breach on fewer calls than a percentile needs.
|
|
144
|
+
*/
|
|
145
|
+
export declare const MIN_SLO_SAMPLES = 5;
|
|
146
|
+
/**
|
|
147
|
+
* Cost-per-hour (and any spend RATE) is an extrapolation, so it needs its own
|
|
148
|
+
* floor: a single cold-start turn ($0.20 over the run's first 3 seconds) must
|
|
149
|
+
* NOT project to $12/h and count as a burn breach. Require at least this many
|
|
150
|
+
* cost samples AND (via {@link MIN_COST_ELAPSED_MS}) a minimum observation span
|
|
151
|
+
* before the projection is trusted.
|
|
152
|
+
*/
|
|
153
|
+
export declare const MIN_COST_SAMPLES = 3;
|
|
154
|
+
/**
|
|
155
|
+
* Minimum wall-clock span the window must have observed before a spend-rate
|
|
156
|
+
* extrapolation is trusted (30s). Below this the numerator (spend) is divided
|
|
157
|
+
* by a tiny denominator (elapsed hours), inflating the projection wildly.
|
|
158
|
+
*/
|
|
159
|
+
export declare const MIN_COST_ELAPSED_MS = 30000;
|
|
160
|
+
/**
|
|
161
|
+
* Compare a window's metrics against the declared targets; list every breach.
|
|
162
|
+
* A "higher-is-worse" target breaches when observed > target AND the metric has
|
|
163
|
+
* enough samples to trust (see {@link MIN_SLO_SAMPLES}); cost-per-hour, being an
|
|
164
|
+
* extrapolation, requires its own {@link MIN_COST_SAMPLES} + {@link
|
|
165
|
+
* MIN_COST_ELAPSED_MS} floor so a 1–2-sample near-empty fresh window can never
|
|
166
|
+
* project to a false burn breach. A target the spec omitted is never checked.
|
|
167
|
+
*/
|
|
168
|
+
export declare function detectSloBreaches(metrics: SloWindowMetrics, targets: SloTargets): SloBreach[];
|
|
169
|
+
export type AttachedSloMonitor = {
|
|
170
|
+
/** Force an evaluation now (the periodic timer normally drives this). */
|
|
171
|
+
evaluate(): Promise<void>;
|
|
172
|
+
unsubscribe: Unsubscribe;
|
|
173
|
+
/** Stop the periodic evaluation timer. */
|
|
174
|
+
stop(): void;
|
|
175
|
+
};
|
|
176
|
+
export type AttachSloMonitorOptions = {
|
|
177
|
+
/** The lowered SLO targets from `observability.slo` (required — the monitor is
|
|
178
|
+
* a no-op without a spec block, mirroring the alert watchdog's env+spec gate). */
|
|
179
|
+
readonly targets?: SloTargets;
|
|
180
|
+
/** Injected mitigation delivery (see {@link SloMitigationSink}). */
|
|
181
|
+
readonly sink?: SloMitigationSink;
|
|
182
|
+
/** How often to evaluate the window, ms. Default: min(windowMs, 30s). */
|
|
183
|
+
readonly evalIntervalMs?: number;
|
|
184
|
+
/** Override the timer scheduler (tests). */
|
|
185
|
+
readonly setInterval?: (cb: () => void, ms: number) => unknown;
|
|
186
|
+
readonly clearInterval?: (handle: unknown) => void;
|
|
187
|
+
/** Override now() (tests). */
|
|
188
|
+
readonly now?: () => number;
|
|
189
|
+
};
|
|
190
|
+
/**
|
|
191
|
+
* Minimum number of CONSECUTIVE breached evaluations a metric must accumulate
|
|
192
|
+
* before a destructive rung fires. A breach must ALSO have persisted for at
|
|
193
|
+
* least the declared window (see {@link attachSloMonitor}); this second gate
|
|
194
|
+
* guards against a fast eval interval satisfying "N evaluations" in a couple of
|
|
195
|
+
* seconds. A single blip that clears on the next tick resets the counter.
|
|
196
|
+
*/
|
|
197
|
+
export declare const SUSTAINED_MIN_EVALS = 2;
|
|
198
|
+
/**
|
|
199
|
+
* Attach the SLO monitor when CREWHAUS_SLO is set AND a lowered `targets` block
|
|
200
|
+
* is supplied. Folds events into a {@link SloWindow}; a periodic timer evaluates
|
|
201
|
+
* the window and walks the declared mitigation ladder.
|
|
202
|
+
*
|
|
203
|
+
* SUSTAINED-BREACH GATING: the `alert` rung fires immediately on the first
|
|
204
|
+
* breached evaluation (an operator wants to hear about a spike as it starts).
|
|
205
|
+
* The DESTRUCTIVE rungs (`pause-intake`, `rollback`) fire ONLY after the SAME
|
|
206
|
+
* metric has been breached for at least {@link SUSTAINED_MIN_EVALS} consecutive
|
|
207
|
+
* evaluations AND that breach has persisted for at least the declared window —
|
|
208
|
+
* i.e. a real sustained violation, never a single transient tick. A metric
|
|
209
|
+
* whose breach clears on any evaluation resets its consecutive counter, so a
|
|
210
|
+
* blip cannot accumulate toward a rollback.
|
|
211
|
+
*
|
|
212
|
+
* RESUME: when NO metric is currently breached and this monitor had paused
|
|
213
|
+
* intake, it resumes intake (via the sink's `resumeIntake`) and re-arms the
|
|
214
|
+
* `pause-intake` rung so a future sustained breach can pause again. `alert` and
|
|
215
|
+
* `rollback` stay one-shot per session (re-alerting / re-rolling every tick is
|
|
216
|
+
* noise, and a rollback that stuck is intentionally sticky).
|
|
217
|
+
*
|
|
218
|
+
* Returns undefined (a no-op) when the env gate is off or no targets are
|
|
219
|
+
* declared, so it adds zero overhead by default.
|
|
220
|
+
*/
|
|
221
|
+
export declare function attachSloMonitor(bus: TraceEventBus, runContext: RunContext, env: NodeJS.ProcessEnv, options?: AttachSloMonitorOptions): AttachedSloMonitor | undefined;
|
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
import { percentile } from "./alert-watchdog";
|
|
2
|
+
/** Default rolling window when the spec omits `window_seconds`: 5 minutes. */
|
|
3
|
+
export const DEFAULT_SLO_WINDOW_MS = 5 * 60 * 1000;
|
|
4
|
+
/**
|
|
5
|
+
* Accumulates a session's SLO signals into rolling windows keyed by wall-clock
|
|
6
|
+
* timestamp. `evaluate(now)` prunes everything older than `windowMs`, then
|
|
7
|
+
* computes the window's rate/percentile metrics. Mirrors the alert-watchdog's
|
|
8
|
+
* event→metric vocabulary (turn_end duration, TTFT from model_request→first
|
|
9
|
+
* model_stream_token, cost accrual, egress verdicts) so the monitor does not
|
|
10
|
+
* depend on the opt-in metrics-collector being attached.
|
|
11
|
+
*/
|
|
12
|
+
export class SloWindow {
|
|
13
|
+
windowMs;
|
|
14
|
+
turnLatencies = [];
|
|
15
|
+
ttfts = [];
|
|
16
|
+
costMicros = [];
|
|
17
|
+
modelCalls = [];
|
|
18
|
+
unrecoveredErrors = [];
|
|
19
|
+
externalCalls = [];
|
|
20
|
+
egressBlocks = [];
|
|
21
|
+
requestStarts = new Map();
|
|
22
|
+
ttftSeen = new Set();
|
|
23
|
+
/** Wall-clock time of the FIRST event ever folded — lets `evaluate` report how
|
|
24
|
+
* long the monitor has actually been observing, so a rate metric (cost/hour)
|
|
25
|
+
* extrapolated from a near-empty window that has only existed for a few
|
|
26
|
+
* seconds is not mistaken for a sustained burn. */
|
|
27
|
+
firstFoldTs;
|
|
28
|
+
constructor(windowMs) {
|
|
29
|
+
this.windowMs = windowMs;
|
|
30
|
+
}
|
|
31
|
+
/** Fold one event into the window. `ev.timestamp` seeds the sample time. */
|
|
32
|
+
fold(ev) {
|
|
33
|
+
const t = Date.parse(ev.timestamp);
|
|
34
|
+
if (Number.isNaN(t))
|
|
35
|
+
return;
|
|
36
|
+
if (this.firstFoldTs === undefined || t < this.firstFoldTs)
|
|
37
|
+
this.firstFoldTs = t;
|
|
38
|
+
switch (ev.kind) {
|
|
39
|
+
case "turn_end":
|
|
40
|
+
this.turnLatencies.push({ ts: t, value: ev.durationMs });
|
|
41
|
+
return;
|
|
42
|
+
case "model_request":
|
|
43
|
+
this.requestStarts.set(ev.traceId, t);
|
|
44
|
+
this.ttftSeen.delete(ev.traceId);
|
|
45
|
+
return;
|
|
46
|
+
case "model_stream_token": {
|
|
47
|
+
if (this.ttftSeen.has(ev.traceId))
|
|
48
|
+
return;
|
|
49
|
+
const start = this.requestStarts.get(ev.traceId);
|
|
50
|
+
if (start === undefined)
|
|
51
|
+
return;
|
|
52
|
+
this.ttfts.push({ ts: t, value: Math.max(0, t - start) });
|
|
53
|
+
this.ttftSeen.add(ev.traceId);
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
case "model_response":
|
|
57
|
+
this.modelCalls.push({ ts: t, value: 1 });
|
|
58
|
+
this.requestStarts.delete(ev.traceId);
|
|
59
|
+
this.ttftSeen.delete(ev.traceId);
|
|
60
|
+
return;
|
|
61
|
+
case "error_recovered":
|
|
62
|
+
if (ev.action === "fail")
|
|
63
|
+
this.unrecoveredErrors.push({ ts: t, value: 1 });
|
|
64
|
+
return;
|
|
65
|
+
case "cost_accrual":
|
|
66
|
+
// Skip the FR-003 terminal aggregate (summary:true) — it double-counts.
|
|
67
|
+
if (ev.summary === true)
|
|
68
|
+
return;
|
|
69
|
+
this.costMicros.push({ ts: t, value: ev.costUsdMicros });
|
|
70
|
+
return;
|
|
71
|
+
case "permission_decision":
|
|
72
|
+
// No egress_* TraceEvent kind exists — derive the egress-block rate from
|
|
73
|
+
// the permission_decision egress outcomes. Every egress verdict (passed
|
|
74
|
+
// /warned/blocked) is one external-sink call attempt; the blocked subset
|
|
75
|
+
// is the numerator.
|
|
76
|
+
if (ev.outcome === "egress-passed" ||
|
|
77
|
+
ev.outcome === "egress-warned" ||
|
|
78
|
+
ev.outcome === "egress-blocked") {
|
|
79
|
+
this.externalCalls.push({ ts: t, value: 1 });
|
|
80
|
+
if (ev.outcome === "egress-blocked")
|
|
81
|
+
this.egressBlocks.push({ ts: t, value: 1 });
|
|
82
|
+
}
|
|
83
|
+
return;
|
|
84
|
+
default:
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Compute the current window's SLO metrics at `now`, pruning samples older
|
|
90
|
+
* than `windowMs` first. Returns `undefined`-valued metrics as 0 counts /
|
|
91
|
+
* empty percentiles so the detector can decide "no data yet" per metric.
|
|
92
|
+
*/
|
|
93
|
+
evaluate(now) {
|
|
94
|
+
const cutoff = now - this.windowMs;
|
|
95
|
+
const prune = (arr) => {
|
|
96
|
+
while (arr.length > 0 && (arr[0]?.ts ?? 0) < cutoff)
|
|
97
|
+
arr.shift();
|
|
98
|
+
};
|
|
99
|
+
prune(this.turnLatencies);
|
|
100
|
+
prune(this.ttfts);
|
|
101
|
+
prune(this.costMicros);
|
|
102
|
+
prune(this.modelCalls);
|
|
103
|
+
prune(this.unrecoveredErrors);
|
|
104
|
+
prune(this.externalCalls);
|
|
105
|
+
prune(this.egressBlocks);
|
|
106
|
+
const modelCalls = this.modelCalls.length;
|
|
107
|
+
const externalCalls = this.externalCalls.length;
|
|
108
|
+
const costUsd = this.costMicros.reduce((s, x) => s + x.value, 0) / 1_000_000;
|
|
109
|
+
// How long the monitor has actually been observing, capped at the rolling
|
|
110
|
+
// window (samples older than `windowMs` were pruned above). A cold-start run
|
|
111
|
+
// that has only existed for a few seconds reports a small elapsed span even
|
|
112
|
+
// though `windowMs` is 5 minutes.
|
|
113
|
+
const observedMs = this.firstFoldTs === undefined
|
|
114
|
+
? 0
|
|
115
|
+
: Math.min(this.windowMs, Math.max(0, now - this.firstFoldTs));
|
|
116
|
+
// Cost-per-hour is the observed spend extrapolated to a full hour. It is
|
|
117
|
+
// annualised over the ACTUAL elapsed observation span, NOT the nominal
|
|
118
|
+
// window — extrapolating $0.20 spent in the first 3 seconds of a run to a
|
|
119
|
+
// 5-minute window ($4.80/h) would fabricate a burn-rate breach. The detector
|
|
120
|
+
// additionally floors this with a min-sample + min-elapsed gate.
|
|
121
|
+
const extrapolationHours = observedMs / 3_600_000;
|
|
122
|
+
return {
|
|
123
|
+
turnP95Ms: percentile(this.turnLatencies.map((s) => s.value), 0.95),
|
|
124
|
+
turnSamples: this.turnLatencies.length,
|
|
125
|
+
ttftP95Ms: percentile(this.ttfts.map((s) => s.value), 0.95),
|
|
126
|
+
ttftSamples: this.ttfts.length,
|
|
127
|
+
errorRate: modelCalls > 0 ? this.unrecoveredErrors.length / modelCalls : 0,
|
|
128
|
+
modelCalls,
|
|
129
|
+
costPerHourUsd: extrapolationHours > 0 ? costUsd / extrapolationHours : 0,
|
|
130
|
+
costSamples: this.costMicros.length,
|
|
131
|
+
windowElapsedMs: observedMs,
|
|
132
|
+
egressBlockRate: externalCalls > 0 ? this.egressBlocks.length / externalCalls : 0,
|
|
133
|
+
externalCalls,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Minimum samples a rate/percentile metric needs before it can breach — a
|
|
139
|
+
* single erroring call is a 100% error rate but not an SLO violation. Keeps the
|
|
140
|
+
* monitor from firing on cold-start noise. Cost uses its own (smaller) sample
|
|
141
|
+
* floor plus a min-elapsed-window floor (see {@link MIN_COST_SAMPLES} /
|
|
142
|
+
* {@link MIN_COST_ELAPSED_MS}) rather than {@link MIN_SLO_SAMPLES}, because a
|
|
143
|
+
* genuine burn spike can breach on fewer calls than a percentile needs.
|
|
144
|
+
*/
|
|
145
|
+
export const MIN_SLO_SAMPLES = 5;
|
|
146
|
+
/**
|
|
147
|
+
* Cost-per-hour (and any spend RATE) is an extrapolation, so it needs its own
|
|
148
|
+
* floor: a single cold-start turn ($0.20 over the run's first 3 seconds) must
|
|
149
|
+
* NOT project to $12/h and count as a burn breach. Require at least this many
|
|
150
|
+
* cost samples AND (via {@link MIN_COST_ELAPSED_MS}) a minimum observation span
|
|
151
|
+
* before the projection is trusted.
|
|
152
|
+
*/
|
|
153
|
+
export const MIN_COST_SAMPLES = 3;
|
|
154
|
+
/**
|
|
155
|
+
* Minimum wall-clock span the window must have observed before a spend-rate
|
|
156
|
+
* extrapolation is trusted (30s). Below this the numerator (spend) is divided
|
|
157
|
+
* by a tiny denominator (elapsed hours), inflating the projection wildly.
|
|
158
|
+
*/
|
|
159
|
+
export const MIN_COST_ELAPSED_MS = 30_000;
|
|
160
|
+
// --------- breach detection ---------
|
|
161
|
+
/**
|
|
162
|
+
* Compare a window's metrics against the declared targets; list every breach.
|
|
163
|
+
* A "higher-is-worse" target breaches when observed > target AND the metric has
|
|
164
|
+
* enough samples to trust (see {@link MIN_SLO_SAMPLES}); cost-per-hour, being an
|
|
165
|
+
* extrapolation, requires its own {@link MIN_COST_SAMPLES} + {@link
|
|
166
|
+
* MIN_COST_ELAPSED_MS} floor so a 1–2-sample near-empty fresh window can never
|
|
167
|
+
* project to a false burn breach. A target the spec omitted is never checked.
|
|
168
|
+
*/
|
|
169
|
+
export function detectSloBreaches(metrics, targets) {
|
|
170
|
+
const breaches = [];
|
|
171
|
+
const add = (metric, observed, target, enoughSamples, fmt) => {
|
|
172
|
+
if (target === undefined || !enoughSamples)
|
|
173
|
+
return;
|
|
174
|
+
if (observed > target) {
|
|
175
|
+
breaches.push({
|
|
176
|
+
metric,
|
|
177
|
+
observed,
|
|
178
|
+
target,
|
|
179
|
+
detail: `${metric} ${fmt(observed)} exceeded SLO target ${fmt(target)}`,
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
};
|
|
183
|
+
const ms = (n) => `${Math.round(n)}ms`;
|
|
184
|
+
const rate = (n) => `${(n * 100).toFixed(1)}%`;
|
|
185
|
+
const usd = (n) => `$${n.toFixed(2)}/h`;
|
|
186
|
+
add("error_rate", metrics.errorRate, targets.errorRate, metrics.modelCalls >= MIN_SLO_SAMPLES, rate);
|
|
187
|
+
add("p95_latency_ms", metrics.turnP95Ms, targets.p95LatencyMs, metrics.turnSamples >= MIN_SLO_SAMPLES, ms);
|
|
188
|
+
add("ttft_ms", metrics.ttftP95Ms, targets.ttftMs, metrics.ttftSamples >= MIN_SLO_SAMPLES, ms);
|
|
189
|
+
add("cost_per_hour_usd", metrics.costPerHourUsd, targets.costPerHourUsd, metrics.costSamples >= MIN_COST_SAMPLES && metrics.windowElapsedMs >= MIN_COST_ELAPSED_MS, usd);
|
|
190
|
+
add("egress_block_rate", metrics.egressBlockRate, targets.egressBlockRate, metrics.externalCalls >= MIN_SLO_SAMPLES, rate);
|
|
191
|
+
return breaches;
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* The DESTRUCTIVE ladder rungs — the ones that change live production state.
|
|
195
|
+
* These fire only on a SUSTAINED breach (see {@link attachSloMonitor}); `alert`
|
|
196
|
+
* is the only immediate rung.
|
|
197
|
+
*/
|
|
198
|
+
const DESTRUCTIVE_RUNGS = new Set(["pause-intake", "rollback"]);
|
|
199
|
+
/**
|
|
200
|
+
* Minimum number of CONSECUTIVE breached evaluations a metric must accumulate
|
|
201
|
+
* before a destructive rung fires. A breach must ALSO have persisted for at
|
|
202
|
+
* least the declared window (see {@link attachSloMonitor}); this second gate
|
|
203
|
+
* guards against a fast eval interval satisfying "N evaluations" in a couple of
|
|
204
|
+
* seconds. A single blip that clears on the next tick resets the counter.
|
|
205
|
+
*/
|
|
206
|
+
export const SUSTAINED_MIN_EVALS = 2;
|
|
207
|
+
/**
|
|
208
|
+
* Attach the SLO monitor when CREWHAUS_SLO is set AND a lowered `targets` block
|
|
209
|
+
* is supplied. Folds events into a {@link SloWindow}; a periodic timer evaluates
|
|
210
|
+
* the window and walks the declared mitigation ladder.
|
|
211
|
+
*
|
|
212
|
+
* SUSTAINED-BREACH GATING: the `alert` rung fires immediately on the first
|
|
213
|
+
* breached evaluation (an operator wants to hear about a spike as it starts).
|
|
214
|
+
* The DESTRUCTIVE rungs (`pause-intake`, `rollback`) fire ONLY after the SAME
|
|
215
|
+
* metric has been breached for at least {@link SUSTAINED_MIN_EVALS} consecutive
|
|
216
|
+
* evaluations AND that breach has persisted for at least the declared window —
|
|
217
|
+
* i.e. a real sustained violation, never a single transient tick. A metric
|
|
218
|
+
* whose breach clears on any evaluation resets its consecutive counter, so a
|
|
219
|
+
* blip cannot accumulate toward a rollback.
|
|
220
|
+
*
|
|
221
|
+
* RESUME: when NO metric is currently breached and this monitor had paused
|
|
222
|
+
* intake, it resumes intake (via the sink's `resumeIntake`) and re-arms the
|
|
223
|
+
* `pause-intake` rung so a future sustained breach can pause again. `alert` and
|
|
224
|
+
* `rollback` stay one-shot per session (re-alerting / re-rolling every tick is
|
|
225
|
+
* noise, and a rollback that stuck is intentionally sticky).
|
|
226
|
+
*
|
|
227
|
+
* Returns undefined (a no-op) when the env gate is off or no targets are
|
|
228
|
+
* declared, so it adds zero overhead by default.
|
|
229
|
+
*/
|
|
230
|
+
export function attachSloMonitor(bus, runContext, env, options = {}) {
|
|
231
|
+
const gate = env["CREWHAUS_SLO"];
|
|
232
|
+
if (gate !== "1" && gate !== "true")
|
|
233
|
+
return undefined;
|
|
234
|
+
const targets = options.targets;
|
|
235
|
+
if (targets === undefined)
|
|
236
|
+
return undefined;
|
|
237
|
+
const windowMs = targets.windowMs ?? DEFAULT_SLO_WINDOW_MS;
|
|
238
|
+
const now = options.now ?? (() => Date.now());
|
|
239
|
+
const window = new SloWindow(windowMs);
|
|
240
|
+
const unsubscribe = bus.subscribe((event) => {
|
|
241
|
+
window.fold(event);
|
|
242
|
+
});
|
|
243
|
+
// One-shot guard for the sticky rungs (`alert`, `rollback`): once fired, they
|
|
244
|
+
// never re-fire this session (a rolled-back pin stays put; re-alerting every
|
|
245
|
+
// tick is noise). `pause-intake` is NOT one-shot — it re-arms on resume so a
|
|
246
|
+
// later breach can re-pause; its live state is `intakePaused` below.
|
|
247
|
+
const firedRungs = new Set();
|
|
248
|
+
// Whether THIS monitor currently holds intake paused (drives resume + re-arm).
|
|
249
|
+
let intakePaused = false;
|
|
250
|
+
// Per-metric consecutive-breach tracking: how many back-to-back evaluations a
|
|
251
|
+
// metric has been breached, and when the streak began. A metric absent from a
|
|
252
|
+
// given evaluation's breach list is cleared (a blip resets the streak).
|
|
253
|
+
const breachStreaks = new Map();
|
|
254
|
+
const sink = options.sink;
|
|
255
|
+
const publishRungEvent = (rung, breach) => {
|
|
256
|
+
const envelope = bus.envelope();
|
|
257
|
+
bus.publish({
|
|
258
|
+
...envelope,
|
|
259
|
+
kind: "alert_raised",
|
|
260
|
+
metric: `slo:${breach.metric}`,
|
|
261
|
+
observed: breach.observed,
|
|
262
|
+
threshold: breach.target,
|
|
263
|
+
baselineSessions: 0,
|
|
264
|
+
detail: `SLO mitigation [${rung}]: ${breach.detail} (window ${Math.round(windowMs / 1000)}s)`,
|
|
265
|
+
});
|
|
266
|
+
};
|
|
267
|
+
const runRung = async (rung, breach) => {
|
|
268
|
+
const event = { sessionId: runContext.sessionId, rung, breach, windowMs };
|
|
269
|
+
publishRungEvent(rung, breach);
|
|
270
|
+
// Audit the attempt (even a no-op rung), then run the injected handler.
|
|
271
|
+
if (sink?.audit !== undefined) {
|
|
272
|
+
await sink.audit(event).catch((err) => logSloError(runContext, "slo-audit", err));
|
|
273
|
+
}
|
|
274
|
+
const handler = rung === "alert" ? sink?.alert : rung === "pause-intake" ? sink?.pauseIntake : sink?.rollback;
|
|
275
|
+
if (handler !== undefined) {
|
|
276
|
+
await handler(event).catch((err) => logSloError(runContext, `slo-${rung}`, err));
|
|
277
|
+
}
|
|
278
|
+
};
|
|
279
|
+
const resumeIntake = async (breach) => {
|
|
280
|
+
intakePaused = false;
|
|
281
|
+
// Re-arm the pause rung so a future sustained breach can pause again.
|
|
282
|
+
firedRungs.delete("pause-intake");
|
|
283
|
+
const event = {
|
|
284
|
+
sessionId: runContext.sessionId,
|
|
285
|
+
rung: "pause-intake",
|
|
286
|
+
breach,
|
|
287
|
+
windowMs,
|
|
288
|
+
};
|
|
289
|
+
const envelope = bus.envelope();
|
|
290
|
+
bus.publish({
|
|
291
|
+
...envelope,
|
|
292
|
+
kind: "alert_raised",
|
|
293
|
+
metric: `slo:${breach.metric}`,
|
|
294
|
+
observed: breach.observed,
|
|
295
|
+
threshold: breach.target,
|
|
296
|
+
baselineSessions: 0,
|
|
297
|
+
detail: `SLO mitigation [pause-intake:resume]: ${breach.detail} cleared — intake resumed (window ${Math.round(windowMs / 1000)}s)`,
|
|
298
|
+
});
|
|
299
|
+
if (sink?.audit !== undefined) {
|
|
300
|
+
await sink.audit(event).catch((err) => logSloError(runContext, "slo-audit", err));
|
|
301
|
+
}
|
|
302
|
+
if (sink?.resumeIntake !== undefined) {
|
|
303
|
+
await sink.resumeIntake(event).catch((err) => logSloError(runContext, "slo-resume", err));
|
|
304
|
+
}
|
|
305
|
+
};
|
|
306
|
+
const evaluate = async () => {
|
|
307
|
+
const t = now();
|
|
308
|
+
const metrics = window.evaluate(t);
|
|
309
|
+
const breaches = detectSloBreaches(metrics, targets);
|
|
310
|
+
// Update the per-metric consecutive-breach streaks: bump the metrics that
|
|
311
|
+
// are breached this tick, drop the ones that cleared (a blip resets).
|
|
312
|
+
const breachedMetrics = new Set(breaches.map((b) => b.metric));
|
|
313
|
+
for (const metric of [...breachStreaks.keys()]) {
|
|
314
|
+
if (!breachedMetrics.has(metric))
|
|
315
|
+
breachStreaks.delete(metric);
|
|
316
|
+
}
|
|
317
|
+
for (const b of breaches) {
|
|
318
|
+
const prior = breachStreaks.get(b.metric);
|
|
319
|
+
if (prior === undefined)
|
|
320
|
+
breachStreaks.set(b.metric, { since: t, consecutive: 1 });
|
|
321
|
+
else
|
|
322
|
+
breachStreaks.set(b.metric, { since: prior.since, consecutive: prior.consecutive + 1 });
|
|
323
|
+
}
|
|
324
|
+
// No breach this tick: if this monitor is holding intake paused, the breach
|
|
325
|
+
// has cleared — resume + re-arm so a future breach can re-pause.
|
|
326
|
+
const first = breaches[0];
|
|
327
|
+
if (first === undefined) {
|
|
328
|
+
if (intakePaused) {
|
|
329
|
+
// The specific metric we paused on has cleared; report a synthetic
|
|
330
|
+
// cleared breach (the resume helper re-arms the pause rung + flips the
|
|
331
|
+
// durable gate back via the injected sink, if any).
|
|
332
|
+
await resumeIntake({
|
|
333
|
+
metric: "slo",
|
|
334
|
+
observed: 0,
|
|
335
|
+
target: 0,
|
|
336
|
+
detail: "all SLO metrics within target",
|
|
337
|
+
});
|
|
338
|
+
}
|
|
339
|
+
return;
|
|
340
|
+
}
|
|
341
|
+
// A metric's breach is SUSTAINED when it has been breached for at least
|
|
342
|
+
// SUSTAINED_MIN_EVALS consecutive evaluations AND has persisted for at least
|
|
343
|
+
// the declared window. Prefer the first breach (worst priority) that is
|
|
344
|
+
// sustained for destructive rungs; the alert rung uses the first breach.
|
|
345
|
+
const sustained = breaches.find((b) => {
|
|
346
|
+
const s = breachStreaks.get(b.metric);
|
|
347
|
+
return s !== undefined && s.consecutive >= SUSTAINED_MIN_EVALS && t - s.since >= windowMs;
|
|
348
|
+
});
|
|
349
|
+
// Walk the ladder in declared order. Destructive rungs require a sustained
|
|
350
|
+
// breach; `alert` fires on any current breach. Each rung is session-global,
|
|
351
|
+
// so we mitigate against a single breach (the sustained one for destructive
|
|
352
|
+
// rungs, else the first breach for alert).
|
|
353
|
+
for (const rung of targets.mitigation) {
|
|
354
|
+
if (DESTRUCTIVE_RUNGS.has(rung)) {
|
|
355
|
+
if (sustained === undefined)
|
|
356
|
+
continue; // not sustained yet — wait
|
|
357
|
+
if (rung === "pause-intake") {
|
|
358
|
+
if (intakePaused)
|
|
359
|
+
continue; // already paused; nothing to re-do
|
|
360
|
+
intakePaused = true;
|
|
361
|
+
await runRung(rung, sustained);
|
|
362
|
+
continue;
|
|
363
|
+
}
|
|
364
|
+
// rollback — one-shot per session.
|
|
365
|
+
if (firedRungs.has(rung))
|
|
366
|
+
continue;
|
|
367
|
+
firedRungs.add(rung);
|
|
368
|
+
await runRung(rung, sustained);
|
|
369
|
+
}
|
|
370
|
+
else {
|
|
371
|
+
// alert — immediate, one-shot per session.
|
|
372
|
+
if (firedRungs.has(rung))
|
|
373
|
+
continue;
|
|
374
|
+
firedRungs.add(rung);
|
|
375
|
+
await runRung(rung, first);
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
};
|
|
379
|
+
const intervalMs = options.evalIntervalMs ?? Math.min(windowMs, 30_000);
|
|
380
|
+
const schedule = options.setInterval ?? ((cb, ms) => setInterval(cb, ms));
|
|
381
|
+
const cancel = options.clearInterval ?? ((h) => clearInterval(h));
|
|
382
|
+
const timer = schedule(() => {
|
|
383
|
+
void evaluate().catch((err) => logSloError(runContext, "slo-eval", err));
|
|
384
|
+
}, intervalMs);
|
|
385
|
+
// Node timers keep the event loop alive; unref so a short CLI run can exit
|
|
386
|
+
// without waiting on the next tick (best-effort — a fake timer has no unref).
|
|
387
|
+
timer?.unref?.();
|
|
388
|
+
return {
|
|
389
|
+
evaluate,
|
|
390
|
+
unsubscribe,
|
|
391
|
+
stop: () => cancel(timer),
|
|
392
|
+
};
|
|
393
|
+
}
|
|
394
|
+
function logSloError(runContext, name, err) {
|
|
395
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
396
|
+
runContext.logger.error("slo_monitor.failed", { name, message });
|
|
397
|
+
}
|