@crewhaus/runtime-core 0.1.7 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,221 @@
1
+ /**
2
+ * Ops item 37 — the production-signal SLO monitor.
3
+ *
4
+ * A bus subscriber (wired into `attachDefaultSubscribers`, gated by CREWHAUS_SLO
5
+ * + a lowered `observability.slo` spec block) that folds live TraceEvents into
6
+ * ROLLING TIME WINDOWS and walks the declared mitigation ladder: `alert`
7
+ * (webhook/hook) → `pause-intake` (gateway/managed 429 `budget_exceeded` path) →
8
+ * `rollback` (auto-rollback the env pin via the deployment-controller). The
9
+ * `alert` rung fires immediately on the first breached evaluation; the
10
+ * DESTRUCTIVE rungs (`pause-intake`, `rollback`) fire ONLY on a SUSTAINED
11
+ * breach — the same metric breached for ≥ N consecutive evaluations spanning at
12
+ * least the declared window, never on a single transient tick (a blip that
13
+ * clears resets the streak). When the breach clears, a paused intake is resumed
14
+ * and the pause rung re-arms. Every rung is audit-logged through the same
15
+ * injected seams the alert watchdog uses, so runtime-core owns no
16
+ * deployment-controller / gateway / audit-log I/O — the caller (CLI / codegen)
17
+ * injects it.
18
+ *
19
+ * WHY A ROLLING WINDOW (vs the alert-watchdog's per-session snapshot): an SLO is
20
+ * a live, in-flight guarantee — "p95 TTFT under 1.4s over the last 5 minutes" —
21
+ * so it must fire DURING a run, not at session end. The watchdog grades a whole
22
+ * session against history AFTER the fact; the SLO monitor grades a moving window
23
+ * against a DECLARED target AS IT HAPPENS. The two are complementary and share
24
+ * the same event-folding vocabulary (turn durations, TTFT, cost, egress).
25
+ *
26
+ * Pure/unit-tested core: the window accumulator, the breach detector, and the
27
+ * ladder walk (with injected sinks) are side-effect-free apart from the trace
28
+ * event the monitor publishes.
29
+ */
30
+ import type { RunContext } from "@crewhaus/run-context";
31
+ import type { TraceEvent, TraceEventBus, Unsubscribe } from "@crewhaus/trace-event-bus";
32
+ /** A mitigation-ladder rung, in the order the monitor walks it on a breach. */
33
+ export type SloMitigationRung = "alert" | "pause-intake" | "rollback";
34
+ /** The SLO targets the monitor evaluates. Every field is optional; an omitted
35
+ * target is never checked. Mirrors the lowered `IrSlo` (camelCase). */
36
+ export type SloTargets = {
37
+ readonly errorRate?: number;
38
+ readonly p95LatencyMs?: number;
39
+ readonly ttftMs?: number;
40
+ readonly costPerHourUsd?: number;
41
+ readonly egressBlockRate?: number;
42
+ /** Rolling window (ms) a breach must persist before the ladder fires. */
43
+ readonly windowMs?: number;
44
+ readonly mitigation: ReadonlyArray<SloMitigationRung>;
45
+ };
46
+ /** Default rolling window when the spec omits `window_seconds`: 5 minutes. */
47
+ export declare const DEFAULT_SLO_WINDOW_MS: number;
48
+ /** A single SLO breach found by {@link detectSloBreaches}. */
49
+ export type SloBreach = {
50
+ readonly metric: string;
51
+ readonly observed: number;
52
+ readonly target: number;
53
+ readonly detail: string;
54
+ };
55
+ /** The payload handed to the mitigation sink for each ladder rung executed. */
56
+ export type SloMitigationEvent = {
57
+ readonly sessionId: string;
58
+ readonly rung: SloMitigationRung;
59
+ readonly breach: SloBreach;
60
+ readonly windowMs: number;
61
+ };
62
+ /**
63
+ * Injected mitigation sink — the ladder rungs beyond the trace event + the
64
+ * always-on `alert_raised`-style delivery the alert watchdog already owns.
65
+ * Every callback is OPTIONAL and supplied by the caller so runtime-core stays
66
+ * free of deployment-controller / gateway / audit-log I/O:
67
+ * - `alert` → webhook/hook delivery + audit append (reuses the alert
68
+ * sink the CLI already builds);
69
+ * - `pauseIntake` → flip the gateway/managed daemon's intake gate so new
70
+ * requests get the 429 `budget_exceeded` path until cleared;
71
+ * - `rollback` → auto-rollback the env pin via the deployment-controller.
72
+ * A rung with no injected handler is a no-op (still audit-logged as attempted),
73
+ * so an SLO spec that asks for `rollback` on a shape with no controller wired
74
+ * degrades to alert-only rather than crashing.
75
+ */
76
+ export type SloMitigationSink = {
77
+ alert?: (event: SloMitigationEvent) => Promise<void>;
78
+ pauseIntake?: (event: SloMitigationEvent) => Promise<void>;
79
+ /** RESUME intake — called when a paused breach clears so the durable gate is
80
+ * flipped back to `paused:false` and admission re-opens. Optional: a shape
81
+ * with no gate leaves this undefined and the pause simply stays until an
82
+ * operator clears it. */
83
+ resumeIntake?: (event: SloMitigationEvent) => Promise<void>;
84
+ rollback?: (event: SloMitigationEvent) => Promise<void>;
85
+ /** Audit every ATTEMPTED rung (even a no-op one) — one durable record. */
86
+ audit?: (event: SloMitigationEvent) => Promise<void>;
87
+ };
88
+ /**
89
+ * Accumulates a session's SLO signals into rolling windows keyed by wall-clock
90
+ * timestamp. `evaluate(now)` prunes everything older than `windowMs`, then
91
+ * computes the window's rate/percentile metrics. Mirrors the alert-watchdog's
92
+ * event→metric vocabulary (turn_end duration, TTFT from model_request→first
93
+ * model_stream_token, cost accrual, egress verdicts) so the monitor does not
94
+ * depend on the opt-in metrics-collector being attached.
95
+ */
96
+ export declare class SloWindow {
97
+ private readonly windowMs;
98
+ private readonly turnLatencies;
99
+ private readonly ttfts;
100
+ private readonly costMicros;
101
+ private readonly modelCalls;
102
+ private readonly unrecoveredErrors;
103
+ private readonly externalCalls;
104
+ private readonly egressBlocks;
105
+ private readonly requestStarts;
106
+ private readonly ttftSeen;
107
+ /** Wall-clock time of the FIRST event ever folded — lets `evaluate` report how
108
+ * long the monitor has actually been observing, so a rate metric (cost/hour)
109
+ * extrapolated from a near-empty window that has only existed for a few
110
+ * seconds is not mistaken for a sustained burn. */
111
+ private firstFoldTs;
112
+ constructor(windowMs: number);
113
+ /** Fold one event into the window. `ev.timestamp` seeds the sample time. */
114
+ fold(ev: TraceEvent): void;
115
+ /**
116
+ * Compute the current window's SLO metrics at `now`, pruning samples older
117
+ * than `windowMs` first. Returns `undefined`-valued metrics as 0 counts /
118
+ * empty percentiles so the detector can decide "no data yet" per metric.
119
+ */
120
+ evaluate(now: number): SloWindowMetrics;
121
+ }
122
+ export type SloWindowMetrics = {
123
+ readonly turnP95Ms: number;
124
+ readonly turnSamples: number;
125
+ readonly ttftP95Ms: number;
126
+ readonly ttftSamples: number;
127
+ readonly errorRate: number;
128
+ readonly modelCalls: number;
129
+ readonly costPerHourUsd: number;
130
+ /** Number of non-summary `cost_accrual` samples in the window. */
131
+ readonly costSamples: number;
132
+ /** Wall-clock span (ms) actually observed, capped at the rolling window. */
133
+ readonly windowElapsedMs: number;
134
+ readonly egressBlockRate: number;
135
+ readonly externalCalls: number;
136
+ };
137
+ /**
138
+ * Minimum samples a rate/percentile metric needs before it can breach — a
139
+ * single erroring call is a 100% error rate but not an SLO violation. Keeps the
140
+ * monitor from firing on cold-start noise. Cost uses its own (smaller) sample
141
+ * floor plus a min-elapsed-window floor (see {@link MIN_COST_SAMPLES} /
142
+ * {@link MIN_COST_ELAPSED_MS}) rather than {@link MIN_SLO_SAMPLES}, because a
143
+ * genuine burn spike can breach on fewer calls than a percentile needs.
144
+ */
145
+ export declare const MIN_SLO_SAMPLES = 5;
146
+ /**
147
+ * Cost-per-hour (and any spend RATE) is an extrapolation, so it needs its own
148
+ * floor: a single cold-start turn ($0.20 over the run's first 3 seconds) must
149
+ * NOT project to $12/h and count as a burn breach. Require at least this many
150
+ * cost samples AND (via {@link MIN_COST_ELAPSED_MS}) a minimum observation span
151
+ * before the projection is trusted.
152
+ */
153
+ export declare const MIN_COST_SAMPLES = 3;
154
+ /**
155
+ * Minimum wall-clock span the window must have observed before a spend-rate
156
+ * extrapolation is trusted (30s). Below this the numerator (spend) is divided
157
+ * by a tiny denominator (elapsed hours), inflating the projection wildly.
158
+ */
159
+ export declare const MIN_COST_ELAPSED_MS = 30000;
160
+ /**
161
+ * Compare a window's metrics against the declared targets; list every breach.
162
+ * A "higher-is-worse" target breaches when observed > target AND the metric has
163
+ * enough samples to trust (see {@link MIN_SLO_SAMPLES}); cost-per-hour, being an
164
+ * extrapolation, requires its own {@link MIN_COST_SAMPLES} + {@link
165
+ * MIN_COST_ELAPSED_MS} floor so a 1–2-sample near-empty fresh window can never
166
+ * project to a false burn breach. A target the spec omitted is never checked.
167
+ */
168
+ export declare function detectSloBreaches(metrics: SloWindowMetrics, targets: SloTargets): SloBreach[];
169
+ export type AttachedSloMonitor = {
170
+ /** Force an evaluation now (the periodic timer normally drives this). */
171
+ evaluate(): Promise<void>;
172
+ unsubscribe: Unsubscribe;
173
+ /** Stop the periodic evaluation timer. */
174
+ stop(): void;
175
+ };
176
+ export type AttachSloMonitorOptions = {
177
+ /** The lowered SLO targets from `observability.slo` (required — the monitor is
178
+ * a no-op without a spec block, mirroring the alert watchdog's env+spec gate). */
179
+ readonly targets?: SloTargets;
180
+ /** Injected mitigation delivery (see {@link SloMitigationSink}). */
181
+ readonly sink?: SloMitigationSink;
182
+ /** How often to evaluate the window, ms. Default: min(windowMs, 30s). */
183
+ readonly evalIntervalMs?: number;
184
+ /** Override the timer scheduler (tests). */
185
+ readonly setInterval?: (cb: () => void, ms: number) => unknown;
186
+ readonly clearInterval?: (handle: unknown) => void;
187
+ /** Override now() (tests). */
188
+ readonly now?: () => number;
189
+ };
190
+ /**
191
+ * Minimum number of CONSECUTIVE breached evaluations a metric must accumulate
192
+ * before a destructive rung fires. A breach must ALSO have persisted for at
193
+ * least the declared window (see {@link attachSloMonitor}); this second gate
194
+ * guards against a fast eval interval satisfying "N evaluations" in a couple of
195
+ * seconds. A single blip that clears on the next tick resets the counter.
196
+ */
197
+ export declare const SUSTAINED_MIN_EVALS = 2;
198
+ /**
199
+ * Attach the SLO monitor when CREWHAUS_SLO is set AND a lowered `targets` block
200
+ * is supplied. Folds events into a {@link SloWindow}; a periodic timer evaluates
201
+ * the window and walks the declared mitigation ladder.
202
+ *
203
+ * SUSTAINED-BREACH GATING: the `alert` rung fires immediately on the first
204
+ * breached evaluation (an operator wants to hear about a spike as it starts).
205
+ * The DESTRUCTIVE rungs (`pause-intake`, `rollback`) fire ONLY after the SAME
206
+ * metric has been breached for at least {@link SUSTAINED_MIN_EVALS} consecutive
207
+ * evaluations AND that breach has persisted for at least the declared window —
208
+ * i.e. a real sustained violation, never a single transient tick. A metric
209
+ * whose breach clears on any evaluation resets its consecutive counter, so a
210
+ * blip cannot accumulate toward a rollback.
211
+ *
212
+ * RESUME: when NO metric is currently breached and this monitor had paused
213
+ * intake, it resumes intake (via the sink's `resumeIntake`) and re-arms the
214
+ * `pause-intake` rung so a future sustained breach can pause again. `alert` and
215
+ * `rollback` stay one-shot per session (re-alerting / re-rolling every tick is
216
+ * noise, and a rollback that stuck is intentionally sticky).
217
+ *
218
+ * Returns undefined (a no-op) when the env gate is off or no targets are
219
+ * declared, so it adds zero overhead by default.
220
+ */
221
+ export declare function attachSloMonitor(bus: TraceEventBus, runContext: RunContext, env: NodeJS.ProcessEnv, options?: AttachSloMonitorOptions): AttachedSloMonitor | undefined;
@@ -0,0 +1,397 @@
1
+ import { percentile } from "./alert-watchdog";
2
+ /** Default rolling window when the spec omits `window_seconds`: 5 minutes. */
3
+ export const DEFAULT_SLO_WINDOW_MS = 5 * 60 * 1000;
4
+ /**
5
+ * Accumulates a session's SLO signals into rolling windows keyed by wall-clock
6
+ * timestamp. `evaluate(now)` prunes everything older than `windowMs`, then
7
+ * computes the window's rate/percentile metrics. Mirrors the alert-watchdog's
8
+ * event→metric vocabulary (turn_end duration, TTFT from model_request→first
9
+ * model_stream_token, cost accrual, egress verdicts) so the monitor does not
10
+ * depend on the opt-in metrics-collector being attached.
11
+ */
12
+ export class SloWindow {
13
+ windowMs;
14
+ turnLatencies = [];
15
+ ttfts = [];
16
+ costMicros = [];
17
+ modelCalls = [];
18
+ unrecoveredErrors = [];
19
+ externalCalls = [];
20
+ egressBlocks = [];
21
+ requestStarts = new Map();
22
+ ttftSeen = new Set();
23
+ /** Wall-clock time of the FIRST event ever folded — lets `evaluate` report how
24
+ * long the monitor has actually been observing, so a rate metric (cost/hour)
25
+ * extrapolated from a near-empty window that has only existed for a few
26
+ * seconds is not mistaken for a sustained burn. */
27
+ firstFoldTs;
28
+ constructor(windowMs) {
29
+ this.windowMs = windowMs;
30
+ }
31
+ /** Fold one event into the window. `ev.timestamp` seeds the sample time. */
32
+ fold(ev) {
33
+ const t = Date.parse(ev.timestamp);
34
+ if (Number.isNaN(t))
35
+ return;
36
+ if (this.firstFoldTs === undefined || t < this.firstFoldTs)
37
+ this.firstFoldTs = t;
38
+ switch (ev.kind) {
39
+ case "turn_end":
40
+ this.turnLatencies.push({ ts: t, value: ev.durationMs });
41
+ return;
42
+ case "model_request":
43
+ this.requestStarts.set(ev.traceId, t);
44
+ this.ttftSeen.delete(ev.traceId);
45
+ return;
46
+ case "model_stream_token": {
47
+ if (this.ttftSeen.has(ev.traceId))
48
+ return;
49
+ const start = this.requestStarts.get(ev.traceId);
50
+ if (start === undefined)
51
+ return;
52
+ this.ttfts.push({ ts: t, value: Math.max(0, t - start) });
53
+ this.ttftSeen.add(ev.traceId);
54
+ return;
55
+ }
56
+ case "model_response":
57
+ this.modelCalls.push({ ts: t, value: 1 });
58
+ this.requestStarts.delete(ev.traceId);
59
+ this.ttftSeen.delete(ev.traceId);
60
+ return;
61
+ case "error_recovered":
62
+ if (ev.action === "fail")
63
+ this.unrecoveredErrors.push({ ts: t, value: 1 });
64
+ return;
65
+ case "cost_accrual":
66
+ // Skip the FR-003 terminal aggregate (summary:true) — it double-counts.
67
+ if (ev.summary === true)
68
+ return;
69
+ this.costMicros.push({ ts: t, value: ev.costUsdMicros });
70
+ return;
71
+ case "permission_decision":
72
+ // No egress_* TraceEvent kind exists — derive the egress-block rate from
73
+ // the permission_decision egress outcomes. Every egress verdict (passed
74
+ // /warned/blocked) is one external-sink call attempt; the blocked subset
75
+ // is the numerator.
76
+ if (ev.outcome === "egress-passed" ||
77
+ ev.outcome === "egress-warned" ||
78
+ ev.outcome === "egress-blocked") {
79
+ this.externalCalls.push({ ts: t, value: 1 });
80
+ if (ev.outcome === "egress-blocked")
81
+ this.egressBlocks.push({ ts: t, value: 1 });
82
+ }
83
+ return;
84
+ default:
85
+ return;
86
+ }
87
+ }
88
+ /**
89
+ * Compute the current window's SLO metrics at `now`, pruning samples older
90
+ * than `windowMs` first. Returns `undefined`-valued metrics as 0 counts /
91
+ * empty percentiles so the detector can decide "no data yet" per metric.
92
+ */
93
+ evaluate(now) {
94
+ const cutoff = now - this.windowMs;
95
+ const prune = (arr) => {
96
+ while (arr.length > 0 && (arr[0]?.ts ?? 0) < cutoff)
97
+ arr.shift();
98
+ };
99
+ prune(this.turnLatencies);
100
+ prune(this.ttfts);
101
+ prune(this.costMicros);
102
+ prune(this.modelCalls);
103
+ prune(this.unrecoveredErrors);
104
+ prune(this.externalCalls);
105
+ prune(this.egressBlocks);
106
+ const modelCalls = this.modelCalls.length;
107
+ const externalCalls = this.externalCalls.length;
108
+ const costUsd = this.costMicros.reduce((s, x) => s + x.value, 0) / 1_000_000;
109
+ // How long the monitor has actually been observing, capped at the rolling
110
+ // window (samples older than `windowMs` were pruned above). A cold-start run
111
+ // that has only existed for a few seconds reports a small elapsed span even
112
+ // though `windowMs` is 5 minutes.
113
+ const observedMs = this.firstFoldTs === undefined
114
+ ? 0
115
+ : Math.min(this.windowMs, Math.max(0, now - this.firstFoldTs));
116
+ // Cost-per-hour is the observed spend extrapolated to a full hour. It is
117
+ // annualised over the ACTUAL elapsed observation span, NOT the nominal
118
+ // window — extrapolating $0.20 spent in the first 3 seconds of a run to a
119
+ // 5-minute window ($4.80/h) would fabricate a burn-rate breach. The detector
120
+ // additionally floors this with a min-sample + min-elapsed gate.
121
+ const extrapolationHours = observedMs / 3_600_000;
122
+ return {
123
+ turnP95Ms: percentile(this.turnLatencies.map((s) => s.value), 0.95),
124
+ turnSamples: this.turnLatencies.length,
125
+ ttftP95Ms: percentile(this.ttfts.map((s) => s.value), 0.95),
126
+ ttftSamples: this.ttfts.length,
127
+ errorRate: modelCalls > 0 ? this.unrecoveredErrors.length / modelCalls : 0,
128
+ modelCalls,
129
+ costPerHourUsd: extrapolationHours > 0 ? costUsd / extrapolationHours : 0,
130
+ costSamples: this.costMicros.length,
131
+ windowElapsedMs: observedMs,
132
+ egressBlockRate: externalCalls > 0 ? this.egressBlocks.length / externalCalls : 0,
133
+ externalCalls,
134
+ };
135
+ }
136
+ }
137
+ /**
138
+ * Minimum samples a rate/percentile metric needs before it can breach — a
139
+ * single erroring call is a 100% error rate but not an SLO violation. Keeps the
140
+ * monitor from firing on cold-start noise. Cost uses its own (smaller) sample
141
+ * floor plus a min-elapsed-window floor (see {@link MIN_COST_SAMPLES} /
142
+ * {@link MIN_COST_ELAPSED_MS}) rather than {@link MIN_SLO_SAMPLES}, because a
143
+ * genuine burn spike can breach on fewer calls than a percentile needs.
144
+ */
145
+ export const MIN_SLO_SAMPLES = 5;
146
+ /**
147
+ * Cost-per-hour (and any spend RATE) is an extrapolation, so it needs its own
148
+ * floor: a single cold-start turn ($0.20 over the run's first 3 seconds) must
149
+ * NOT project to $12/h and count as a burn breach. Require at least this many
150
+ * cost samples AND (via {@link MIN_COST_ELAPSED_MS}) a minimum observation span
151
+ * before the projection is trusted.
152
+ */
153
+ export const MIN_COST_SAMPLES = 3;
154
+ /**
155
+ * Minimum wall-clock span the window must have observed before a spend-rate
156
+ * extrapolation is trusted (30s). Below this the numerator (spend) is divided
157
+ * by a tiny denominator (elapsed hours), inflating the projection wildly.
158
+ */
159
+ export const MIN_COST_ELAPSED_MS = 30_000;
160
+ // --------- breach detection ---------
161
+ /**
162
+ * Compare a window's metrics against the declared targets; list every breach.
163
+ * A "higher-is-worse" target breaches when observed > target AND the metric has
164
+ * enough samples to trust (see {@link MIN_SLO_SAMPLES}); cost-per-hour, being an
165
+ * extrapolation, requires its own {@link MIN_COST_SAMPLES} + {@link
166
+ * MIN_COST_ELAPSED_MS} floor so a 1–2-sample near-empty fresh window can never
167
+ * project to a false burn breach. A target the spec omitted is never checked.
168
+ */
169
+ export function detectSloBreaches(metrics, targets) {
170
+ const breaches = [];
171
+ const add = (metric, observed, target, enoughSamples, fmt) => {
172
+ if (target === undefined || !enoughSamples)
173
+ return;
174
+ if (observed > target) {
175
+ breaches.push({
176
+ metric,
177
+ observed,
178
+ target,
179
+ detail: `${metric} ${fmt(observed)} exceeded SLO target ${fmt(target)}`,
180
+ });
181
+ }
182
+ };
183
+ const ms = (n) => `${Math.round(n)}ms`;
184
+ const rate = (n) => `${(n * 100).toFixed(1)}%`;
185
+ const usd = (n) => `$${n.toFixed(2)}/h`;
186
+ add("error_rate", metrics.errorRate, targets.errorRate, metrics.modelCalls >= MIN_SLO_SAMPLES, rate);
187
+ add("p95_latency_ms", metrics.turnP95Ms, targets.p95LatencyMs, metrics.turnSamples >= MIN_SLO_SAMPLES, ms);
188
+ add("ttft_ms", metrics.ttftP95Ms, targets.ttftMs, metrics.ttftSamples >= MIN_SLO_SAMPLES, ms);
189
+ add("cost_per_hour_usd", metrics.costPerHourUsd, targets.costPerHourUsd, metrics.costSamples >= MIN_COST_SAMPLES && metrics.windowElapsedMs >= MIN_COST_ELAPSED_MS, usd);
190
+ add("egress_block_rate", metrics.egressBlockRate, targets.egressBlockRate, metrics.externalCalls >= MIN_SLO_SAMPLES, rate);
191
+ return breaches;
192
+ }
193
+ /**
194
+ * The DESTRUCTIVE ladder rungs — the ones that change live production state.
195
+ * These fire only on a SUSTAINED breach (see {@link attachSloMonitor}); `alert`
196
+ * is the only immediate rung.
197
+ */
198
+ const DESTRUCTIVE_RUNGS = new Set(["pause-intake", "rollback"]);
199
+ /**
200
+ * Minimum number of CONSECUTIVE breached evaluations a metric must accumulate
201
+ * before a destructive rung fires. A breach must ALSO have persisted for at
202
+ * least the declared window (see {@link attachSloMonitor}); this second gate
203
+ * guards against a fast eval interval satisfying "N evaluations" in a couple of
204
+ * seconds. A single blip that clears on the next tick resets the counter.
205
+ */
206
+ export const SUSTAINED_MIN_EVALS = 2;
207
+ /**
208
+ * Attach the SLO monitor when CREWHAUS_SLO is set AND a lowered `targets` block
209
+ * is supplied. Folds events into a {@link SloWindow}; a periodic timer evaluates
210
+ * the window and walks the declared mitigation ladder.
211
+ *
212
+ * SUSTAINED-BREACH GATING: the `alert` rung fires immediately on the first
213
+ * breached evaluation (an operator wants to hear about a spike as it starts).
214
+ * The DESTRUCTIVE rungs (`pause-intake`, `rollback`) fire ONLY after the SAME
215
+ * metric has been breached for at least {@link SUSTAINED_MIN_EVALS} consecutive
216
+ * evaluations AND that breach has persisted for at least the declared window —
217
+ * i.e. a real sustained violation, never a single transient tick. A metric
218
+ * whose breach clears on any evaluation resets its consecutive counter, so a
219
+ * blip cannot accumulate toward a rollback.
220
+ *
221
+ * RESUME: when NO metric is currently breached and this monitor had paused
222
+ * intake, it resumes intake (via the sink's `resumeIntake`) and re-arms the
223
+ * `pause-intake` rung so a future sustained breach can pause again. `alert` and
224
+ * `rollback` stay one-shot per session (re-alerting / re-rolling every tick is
225
+ * noise, and a rollback that stuck is intentionally sticky).
226
+ *
227
+ * Returns undefined (a no-op) when the env gate is off or no targets are
228
+ * declared, so it adds zero overhead by default.
229
+ */
230
+ export function attachSloMonitor(bus, runContext, env, options = {}) {
231
+ const gate = env["CREWHAUS_SLO"];
232
+ if (gate !== "1" && gate !== "true")
233
+ return undefined;
234
+ const targets = options.targets;
235
+ if (targets === undefined)
236
+ return undefined;
237
+ const windowMs = targets.windowMs ?? DEFAULT_SLO_WINDOW_MS;
238
+ const now = options.now ?? (() => Date.now());
239
+ const window = new SloWindow(windowMs);
240
+ const unsubscribe = bus.subscribe((event) => {
241
+ window.fold(event);
242
+ });
243
+ // One-shot guard for the sticky rungs (`alert`, `rollback`): once fired, they
244
+ // never re-fire this session (a rolled-back pin stays put; re-alerting every
245
+ // tick is noise). `pause-intake` is NOT one-shot — it re-arms on resume so a
246
+ // later breach can re-pause; its live state is `intakePaused` below.
247
+ const firedRungs = new Set();
248
+ // Whether THIS monitor currently holds intake paused (drives resume + re-arm).
249
+ let intakePaused = false;
250
+ // Per-metric consecutive-breach tracking: how many back-to-back evaluations a
251
+ // metric has been breached, and when the streak began. A metric absent from a
252
+ // given evaluation's breach list is cleared (a blip resets the streak).
253
+ const breachStreaks = new Map();
254
+ const sink = options.sink;
255
+ const publishRungEvent = (rung, breach) => {
256
+ const envelope = bus.envelope();
257
+ bus.publish({
258
+ ...envelope,
259
+ kind: "alert_raised",
260
+ metric: `slo:${breach.metric}`,
261
+ observed: breach.observed,
262
+ threshold: breach.target,
263
+ baselineSessions: 0,
264
+ detail: `SLO mitigation [${rung}]: ${breach.detail} (window ${Math.round(windowMs / 1000)}s)`,
265
+ });
266
+ };
267
+ const runRung = async (rung, breach) => {
268
+ const event = { sessionId: runContext.sessionId, rung, breach, windowMs };
269
+ publishRungEvent(rung, breach);
270
+ // Audit the attempt (even a no-op rung), then run the injected handler.
271
+ if (sink?.audit !== undefined) {
272
+ await sink.audit(event).catch((err) => logSloError(runContext, "slo-audit", err));
273
+ }
274
+ const handler = rung === "alert" ? sink?.alert : rung === "pause-intake" ? sink?.pauseIntake : sink?.rollback;
275
+ if (handler !== undefined) {
276
+ await handler(event).catch((err) => logSloError(runContext, `slo-${rung}`, err));
277
+ }
278
+ };
279
+ const resumeIntake = async (breach) => {
280
+ intakePaused = false;
281
+ // Re-arm the pause rung so a future sustained breach can pause again.
282
+ firedRungs.delete("pause-intake");
283
+ const event = {
284
+ sessionId: runContext.sessionId,
285
+ rung: "pause-intake",
286
+ breach,
287
+ windowMs,
288
+ };
289
+ const envelope = bus.envelope();
290
+ bus.publish({
291
+ ...envelope,
292
+ kind: "alert_raised",
293
+ metric: `slo:${breach.metric}`,
294
+ observed: breach.observed,
295
+ threshold: breach.target,
296
+ baselineSessions: 0,
297
+ detail: `SLO mitigation [pause-intake:resume]: ${breach.detail} cleared — intake resumed (window ${Math.round(windowMs / 1000)}s)`,
298
+ });
299
+ if (sink?.audit !== undefined) {
300
+ await sink.audit(event).catch((err) => logSloError(runContext, "slo-audit", err));
301
+ }
302
+ if (sink?.resumeIntake !== undefined) {
303
+ await sink.resumeIntake(event).catch((err) => logSloError(runContext, "slo-resume", err));
304
+ }
305
+ };
306
+ const evaluate = async () => {
307
+ const t = now();
308
+ const metrics = window.evaluate(t);
309
+ const breaches = detectSloBreaches(metrics, targets);
310
+ // Update the per-metric consecutive-breach streaks: bump the metrics that
311
+ // are breached this tick, drop the ones that cleared (a blip resets).
312
+ const breachedMetrics = new Set(breaches.map((b) => b.metric));
313
+ for (const metric of [...breachStreaks.keys()]) {
314
+ if (!breachedMetrics.has(metric))
315
+ breachStreaks.delete(metric);
316
+ }
317
+ for (const b of breaches) {
318
+ const prior = breachStreaks.get(b.metric);
319
+ if (prior === undefined)
320
+ breachStreaks.set(b.metric, { since: t, consecutive: 1 });
321
+ else
322
+ breachStreaks.set(b.metric, { since: prior.since, consecutive: prior.consecutive + 1 });
323
+ }
324
+ // No breach this tick: if this monitor is holding intake paused, the breach
325
+ // has cleared — resume + re-arm so a future breach can re-pause.
326
+ const first = breaches[0];
327
+ if (first === undefined) {
328
+ if (intakePaused) {
329
+ // The specific metric we paused on has cleared; report a synthetic
330
+ // cleared breach (the resume helper re-arms the pause rung + flips the
331
+ // durable gate back via the injected sink, if any).
332
+ await resumeIntake({
333
+ metric: "slo",
334
+ observed: 0,
335
+ target: 0,
336
+ detail: "all SLO metrics within target",
337
+ });
338
+ }
339
+ return;
340
+ }
341
+ // A metric's breach is SUSTAINED when it has been breached for at least
342
+ // SUSTAINED_MIN_EVALS consecutive evaluations AND has persisted for at least
343
+ // the declared window. Prefer the first breach (worst priority) that is
344
+ // sustained for destructive rungs; the alert rung uses the first breach.
345
+ const sustained = breaches.find((b) => {
346
+ const s = breachStreaks.get(b.metric);
347
+ return s !== undefined && s.consecutive >= SUSTAINED_MIN_EVALS && t - s.since >= windowMs;
348
+ });
349
+ // Walk the ladder in declared order. Destructive rungs require a sustained
350
+ // breach; `alert` fires on any current breach. Each rung is session-global,
351
+ // so we mitigate against a single breach (the sustained one for destructive
352
+ // rungs, else the first breach for alert).
353
+ for (const rung of targets.mitigation) {
354
+ if (DESTRUCTIVE_RUNGS.has(rung)) {
355
+ if (sustained === undefined)
356
+ continue; // not sustained yet — wait
357
+ if (rung === "pause-intake") {
358
+ if (intakePaused)
359
+ continue; // already paused; nothing to re-do
360
+ intakePaused = true;
361
+ await runRung(rung, sustained);
362
+ continue;
363
+ }
364
+ // rollback — one-shot per session.
365
+ if (firedRungs.has(rung))
366
+ continue;
367
+ firedRungs.add(rung);
368
+ await runRung(rung, sustained);
369
+ }
370
+ else {
371
+ // alert — immediate, one-shot per session.
372
+ if (firedRungs.has(rung))
373
+ continue;
374
+ firedRungs.add(rung);
375
+ await runRung(rung, first);
376
+ }
377
+ }
378
+ };
379
+ const intervalMs = options.evalIntervalMs ?? Math.min(windowMs, 30_000);
380
+ const schedule = options.setInterval ?? ((cb, ms) => setInterval(cb, ms));
381
+ const cancel = options.clearInterval ?? ((h) => clearInterval(h));
382
+ const timer = schedule(() => {
383
+ void evaluate().catch((err) => logSloError(runContext, "slo-eval", err));
384
+ }, intervalMs);
385
+ // Node timers keep the event loop alive; unref so a short CLI run can exit
386
+ // without waiting on the next tick (best-effort — a fake timer has no unref).
387
+ timer?.unref?.();
388
+ return {
389
+ evaluate,
390
+ unsubscribe,
391
+ stop: () => cancel(timer),
392
+ };
393
+ }
394
+ function logSloError(runContext, name, err) {
395
+ const message = err instanceof Error ? err.message : String(err);
396
+ runContext.logger.error("slo_monitor.failed", { name, message });
397
+ }