@crewhaus/runtime-core 0.1.8 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,138 @@
1
+ import type { TraceEvent } from "@crewhaus/trace-event-bus";
2
+ /** Default location of the per-session metrics history, relative to cwd. */
3
+ export declare const DEFAULT_METRICS_DIR: string;
4
+ export declare const METRICS_FILENAME = "sessions.jsonl";
5
+ /** How many historical sessions we need before deriving a real baseline. */
6
+ export declare const MIN_BASELINE_SESSIONS = 5;
7
+ /** Headroom multiplier over the trailing p95 (a 50% cushion). */
8
+ export declare const HEADROOM_FACTOR = 1.5;
9
+ /** How many trailing sessions the baseline is derived from. */
10
+ export declare const BASELINE_WINDOW = 50;
11
+ /**
12
+ * F3 (ops pre-merge) — hard cap on how many lines `sessions.jsonl` is allowed
13
+ * to grow to. `deriveThresholds` only ever looks at the trailing
14
+ * {@link BASELINE_WINDOW} sessions, but `appendMetricsSnapshot` was
15
+ * append-only and `readMetricsHistory` re-read (and re-parsed) the ENTIRE
16
+ * file every session — unbounded growth for a file nothing ever reads past
17
+ * its tail. 4× the baseline window is generous headroom over what
18
+ * `deriveThresholds` consumes, with a 200-line floor so a small
19
+ * `BASELINE_WINDOW` still keeps a useful amount of history for operators
20
+ * eyeballing the file by hand.
21
+ */
22
+ export declare const MAX_METRICS_HISTORY_LINES: number;
23
+ /**
24
+ * The compact per-session snapshot persisted to `sessions.jsonl` and read back
25
+ * to derive baselines. Percentiles are computed from the per-session samples
26
+ * at session end so the stored line is small and directly comparable.
27
+ */
28
+ export type SessionMetricsSnapshot = {
29
+ readonly sessionId: string;
30
+ readonly ts: string;
31
+ /** Total user turns (turn_end events). */
32
+ readonly turns: number;
33
+ /** Total model calls (model_response events). */
34
+ readonly modelCalls: number;
35
+ /** Errors that were NOT recovered (error_recovered action "fail"). */
36
+ readonly unrecoveredErrors: number;
37
+ /** unrecoveredErrors / max(modelCalls,1) — the rate we alert on. */
38
+ readonly errorRate: number;
39
+ /** p95 of per-turn durations, seconds. */
40
+ readonly turnP95Seconds: number;
41
+ /** p95 of time-to-first-token, seconds. */
42
+ readonly ttftP95Seconds: number;
43
+ /** Total spend in USD-micros over the session. */
44
+ readonly costUsdMicros: number;
45
+ /** Cost burn rate: USD per minute of wall-clock session time. */
46
+ readonly costBurnUsdPerMin: number;
47
+ /** cost_accrual events that priced to $0 despite non-zero tokens (pricing miss). */
48
+ readonly pricingMisses: number;
49
+ /** Circuit-breaker transitions into "open". */
50
+ readonly circuitOpens: number;
51
+ /** Egress classifier block-tier verdicts. */
52
+ readonly egressBlocked: number;
53
+ /** Justification-gate / policy denials (permission_decision decision "deny"). */
54
+ readonly permissionDenials: number;
55
+ };
56
+ /** A single threshold breach found by {@link detectBreaches}. */
57
+ export type Breach = {
58
+ readonly metric: string;
59
+ readonly observed: number;
60
+ readonly threshold: number;
61
+ readonly detail: string;
62
+ };
63
+ /** Thresholds derived from history (or bootstrap defaults on a cold start). */
64
+ export type DerivedThresholds = {
65
+ readonly errorRate: number;
66
+ readonly turnP95Seconds: number;
67
+ readonly ttftP95Seconds: number;
68
+ readonly costBurnUsdPerMin: number;
69
+ readonly pricingMissRate: number;
70
+ readonly circuitOpens: number;
71
+ readonly egressBlocked: number;
72
+ /** How many historical sessions the thresholds were derived from (0 = bootstrap). */
73
+ readonly baselineSessions: number;
74
+ };
75
+ /**
76
+ * Accumulates a session's metric signals from bus events. Mirrors
77
+ * metrics-collector's event→metric derivation (turn_end duration, TTFT from
78
+ * model_request→first model_stream_token) so the watchdog does not depend on
79
+ * the opt-in metrics-collector being attached.
80
+ */
81
+ export declare class SessionMetricsAccumulator {
82
+ private turnDurations;
83
+ private ttfts;
84
+ private modelCalls;
85
+ private turns;
86
+ private unrecoveredErrors;
87
+ private costUsdMicros;
88
+ private pricingMisses;
89
+ private circuitOpens;
90
+ private egressBlocked;
91
+ private permissionDenials;
92
+ private firstTs;
93
+ private lastTs;
94
+ private readonly requestStarts;
95
+ private readonly ttftSeen;
96
+ fold(ev: TraceEvent): void;
97
+ /** Finalize into a persistable snapshot. `now` seeds the timestamp. */
98
+ snapshot(sessionId: string, now?: Date): SessionMetricsSnapshot;
99
+ }
100
+ /** Nearest-rank p95 (0-based index floor((p)*(n-1))). Empty → 0. */
101
+ export declare function percentile(values: ReadonlyArray<number>, p: number): number;
102
+ /**
103
+ * Derive thresholds from a session history. For the "higher-is-worse"
104
+ * continuous metrics (error rate, turn p95, TTFT p95, cost burn, pricing-miss
105
+ * rate) the threshold is `trailing-p95 × HEADROOM_FACTOR`. For the count
106
+ * metrics (circuit opens, egress blocks) the threshold is the trailing p95
107
+ * count (any session materially above its own historical worst is worth a
108
+ * look) with a floor of 1 so a single first-ever block still alerts.
109
+ *
110
+ * With fewer than {@link MIN_BASELINE_SESSIONS} historical rows, there is not
111
+ * enough signal to trust a percentile — conservative bootstrap defaults are
112
+ * used instead so a cold start neither alert-storms nor stays silent on an
113
+ * obviously broken session.
114
+ */
115
+ export declare function deriveThresholds(history: ReadonlyArray<SessionMetricsSnapshot>): DerivedThresholds;
116
+ /** Compare a session snapshot against derived thresholds; list every breach. */
117
+ export declare function detectBreaches(snapshot: SessionMetricsSnapshot, thresholds: DerivedThresholds): Breach[];
118
+ /**
119
+ * Read the persisted session history, oldest first. Torn lines are skipped.
120
+ *
121
+ * F3 — bounds itself to the trailing {@link MAX_METRICS_HISTORY_LINES} raw
122
+ * lines BEFORE parsing, so a pre-existing oversized file (e.g. one written by
123
+ * an older binary that predates the cap, or a file that grew past the cap
124
+ * some other way) is never fully materialized into memory just to derive a
125
+ * threshold from its tail. `appendMetricsSnapshot` keeps the file itself
126
+ * trimmed going forward; this is the reader's independent bound.
127
+ */
128
+ export declare function readMetricsHistory(metricsDir?: string): SessionMetricsSnapshot[];
129
+ /**
130
+ * Append one session snapshot to the history JSONL, creating dirs on first
131
+ * use, then trim the file to its trailing {@link MAX_METRICS_HISTORY_LINES}
132
+ * (F3 — the file otherwise grows forever even though `deriveThresholds` only
133
+ * ever looks at the trailing {@link BASELINE_WINDOW}). The trim is a
134
+ * temp-write + rename so a crash mid-trim never leaves a torn or truncated
135
+ * file in place of the real one; a torn LINE within the kept tail is still
136
+ * tolerated by `readMetricsHistory`.
137
+ */
138
+ export declare function appendMetricsSnapshot(snapshot: SessionMetricsSnapshot, metricsDir?: string): void;
@@ -0,0 +1,290 @@
1
+ /**
2
+ * Ops item 31 — alert watchdog with baseline-derived thresholds.
3
+ *
4
+ * A bus subscriber (wired into `attachDefaultSubscribers`, gated by
5
+ * CREWHAUS_ALERTS) that folds a run's TraceEvents into a small per-session
6
+ * metrics snapshot, then — at session end — compares that snapshot against
7
+ * thresholds DERIVED FROM HISTORY (the trailing p95 of prior sessions'
8
+ * persisted snapshots, ×a headroom factor) rather than hand-configured
9
+ * constants. On a breach it publishes an `alert_raised` trace event and hands
10
+ * the breach to injected sinks (audit append + settings.json `alert` hook /
11
+ * webhook — supplied by the caller so this module stays side-effect-free apart
12
+ * from the snapshot persistence it owns).
13
+ *
14
+ * WHY PERSIST A SNAPSHOT: nothing durably records per-session metric history
15
+ * today (metrics-collector is opt-in and its sinks are point-in-time). Without
16
+ * history there is nothing to derive a baseline from. So the watchdog appends
17
+ * one compact JSONL line per session under `.crewhaus/metrics/sessions.jsonl`
18
+ * (additive; readers tolerate torn lines). The FIRST few sessions have no
19
+ * baseline — the watchdog uses conservative bootstrap defaults until enough
20
+ * history accrues (`MIN_BASELINE_SESSIONS`), so a cold start never alert-storms.
21
+ *
22
+ * This file is the side-effect-free-ish core (the only I/O it owns is reading
23
+ * and appending its own snapshot JSONL); event folding, threshold derivation,
24
+ * and breach detection are pure and unit-tested.
25
+ */
26
+ import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync, } from "node:fs";
27
+ import { dirname, join } from "node:path";
28
+ /** Default location of the per-session metrics history, relative to cwd. */
29
+ export const DEFAULT_METRICS_DIR = join(".crewhaus", "metrics");
30
+ export const METRICS_FILENAME = "sessions.jsonl";
31
+ /** How many historical sessions we need before deriving a real baseline. */
32
+ export const MIN_BASELINE_SESSIONS = 5;
33
+ /** Headroom multiplier over the trailing p95 (a 50% cushion). */
34
+ export const HEADROOM_FACTOR = 1.5;
35
+ /** How many trailing sessions the baseline is derived from. */
36
+ export const BASELINE_WINDOW = 50;
37
+ /**
38
+ * F3 (ops pre-merge) — hard cap on how many lines `sessions.jsonl` is allowed
39
+ * to grow to. `deriveThresholds` only ever looks at the trailing
40
+ * {@link BASELINE_WINDOW} sessions, but `appendMetricsSnapshot` was
41
+ * append-only and `readMetricsHistory` re-read (and re-parsed) the ENTIRE
42
+ * file every session — unbounded growth for a file nothing ever reads past
43
+ * its tail. 4× the baseline window is generous headroom over what
44
+ * `deriveThresholds` consumes, with a 200-line floor so a small
45
+ * `BASELINE_WINDOW` still keeps a useful amount of history for operators
46
+ * eyeballing the file by hand.
47
+ */
48
+ export const MAX_METRICS_HISTORY_LINES = Math.max(BASELINE_WINDOW * 4, 200);
49
+ // --------- pure event folding ---------
50
+ /**
51
+ * Accumulates a session's metric signals from bus events. Mirrors
52
+ * metrics-collector's event→metric derivation (turn_end duration, TTFT from
53
+ * model_request→first model_stream_token) so the watchdog does not depend on
54
+ * the opt-in metrics-collector being attached.
55
+ */
56
+ export class SessionMetricsAccumulator {
57
+ turnDurations = [];
58
+ ttfts = [];
59
+ modelCalls = 0;
60
+ turns = 0;
61
+ unrecoveredErrors = 0;
62
+ costUsdMicros = 0;
63
+ pricingMisses = 0;
64
+ circuitOpens = 0;
65
+ egressBlocked = 0;
66
+ permissionDenials = 0;
67
+ firstTs;
68
+ lastTs;
69
+ requestStarts = new Map();
70
+ ttftSeen = new Set();
71
+ fold(ev) {
72
+ const t = Date.parse(ev.timestamp);
73
+ if (!Number.isNaN(t)) {
74
+ if (this.firstTs === undefined || t < this.firstTs)
75
+ this.firstTs = t;
76
+ if (this.lastTs === undefined || t > this.lastTs)
77
+ this.lastTs = t;
78
+ }
79
+ switch (ev.kind) {
80
+ case "turn_end":
81
+ this.turns += 1;
82
+ this.turnDurations.push(ev.durationMs / 1000);
83
+ return;
84
+ case "model_request":
85
+ this.requestStarts.set(ev.traceId, Date.parse(ev.timestamp));
86
+ this.ttftSeen.delete(ev.traceId);
87
+ return;
88
+ case "model_stream_token": {
89
+ if (this.ttftSeen.has(ev.traceId))
90
+ return;
91
+ const start = this.requestStarts.get(ev.traceId);
92
+ if (start === undefined)
93
+ return;
94
+ const tokenMs = Date.parse(ev.timestamp);
95
+ if (Number.isNaN(tokenMs))
96
+ return;
97
+ this.ttfts.push(Math.max(0, (tokenMs - start) / 1000));
98
+ this.ttftSeen.add(ev.traceId);
99
+ return;
100
+ }
101
+ case "model_response":
102
+ this.modelCalls += 1;
103
+ this.requestStarts.delete(ev.traceId);
104
+ this.ttftSeen.delete(ev.traceId);
105
+ return;
106
+ case "error_recovered":
107
+ // Only a terminal "fail" is an unrecovered error; retry/compact/etc.
108
+ // are the recovery engine doing its job.
109
+ if (ev.action === "fail")
110
+ this.unrecoveredErrors += 1;
111
+ return;
112
+ case "cost_accrual": {
113
+ // Skip the FR-003 terminal aggregate (summary:true) — it double-counts.
114
+ if (ev.summary === true)
115
+ return;
116
+ this.costUsdMicros += ev.costUsdMicros;
117
+ if (ev.costUsdMicros === 0 && ev.inputTokens + ev.outputTokens > 0) {
118
+ this.pricingMisses += 1;
119
+ }
120
+ return;
121
+ }
122
+ case "circuit_state_changed":
123
+ if (ev.toState === "open")
124
+ this.circuitOpens += 1;
125
+ return;
126
+ case "permission_decision":
127
+ if (ev.outcome === "egress-blocked")
128
+ this.egressBlocked += 1;
129
+ else if (ev.outcome === undefined && ev.decision === "deny")
130
+ this.permissionDenials += 1;
131
+ return;
132
+ default:
133
+ return;
134
+ }
135
+ }
136
+ /** Finalize into a persistable snapshot. `now` seeds the timestamp. */
137
+ snapshot(sessionId, now = new Date()) {
138
+ const wallMs = this.firstTs !== undefined && this.lastTs !== undefined ? this.lastTs - this.firstTs : 0;
139
+ const wallMin = wallMs / 60000;
140
+ const costUsd = this.costUsdMicros / 1_000_000;
141
+ return {
142
+ sessionId,
143
+ ts: now.toISOString(),
144
+ turns: this.turns,
145
+ modelCalls: this.modelCalls,
146
+ unrecoveredErrors: this.unrecoveredErrors,
147
+ errorRate: this.unrecoveredErrors / Math.max(this.modelCalls, 1),
148
+ turnP95Seconds: percentile(this.turnDurations, 0.95),
149
+ ttftP95Seconds: percentile(this.ttfts, 0.95),
150
+ costUsdMicros: this.costUsdMicros,
151
+ costBurnUsdPerMin: wallMin > 0 ? costUsd / wallMin : 0,
152
+ pricingMisses: this.pricingMisses,
153
+ circuitOpens: this.circuitOpens,
154
+ egressBlocked: this.egressBlocked,
155
+ permissionDenials: this.permissionDenials,
156
+ };
157
+ }
158
+ }
159
+ /** Nearest-rank p95 (0-based index floor((p)*(n-1))). Empty → 0. */
160
+ export function percentile(values, p) {
161
+ if (values.length === 0)
162
+ return 0;
163
+ const sorted = [...values].sort((a, b) => a - b);
164
+ const idx = Math.floor(p * (sorted.length - 1));
165
+ return sorted[idx] ?? 0;
166
+ }
167
+ // --------- threshold derivation + breach detection ---------
168
+ /**
169
+ * Derive thresholds from a session history. For the "higher-is-worse"
170
+ * continuous metrics (error rate, turn p95, TTFT p95, cost burn, pricing-miss
171
+ * rate) the threshold is `trailing-p95 × HEADROOM_FACTOR`. For the count
172
+ * metrics (circuit opens, egress blocks) the threshold is the trailing p95
173
+ * count (any session materially above its own historical worst is worth a
174
+ * look) with a floor of 1 so a single first-ever block still alerts.
175
+ *
176
+ * With fewer than {@link MIN_BASELINE_SESSIONS} historical rows, there is not
177
+ * enough signal to trust a percentile — conservative bootstrap defaults are
178
+ * used instead so a cold start neither alert-storms nor stays silent on an
179
+ * obviously broken session.
180
+ */
181
+ export function deriveThresholds(history) {
182
+ const window = history.slice(-BASELINE_WINDOW);
183
+ if (window.length < MIN_BASELINE_SESSIONS) {
184
+ return {
185
+ // Bootstrap defaults: loose enough not to fire on normal variance, tight
186
+ // enough to catch a session that is clearly off the rails.
187
+ errorRate: 0.5,
188
+ turnP95Seconds: 120,
189
+ ttftP95Seconds: 30,
190
+ costBurnUsdPerMin: 5,
191
+ pricingMissRate: 0.5,
192
+ circuitOpens: 1,
193
+ egressBlocked: 1,
194
+ baselineSessions: window.length,
195
+ };
196
+ }
197
+ const p95 = (pick) => percentile(window.map(pick), 0.95);
198
+ return {
199
+ errorRate: Math.max(p95((s) => s.errorRate) * HEADROOM_FACTOR, 0.05),
200
+ turnP95Seconds: p95((s) => s.turnP95Seconds) * HEADROOM_FACTOR,
201
+ ttftP95Seconds: p95((s) => s.ttftP95Seconds) * HEADROOM_FACTOR,
202
+ costBurnUsdPerMin: p95((s) => s.costBurnUsdPerMin) * HEADROOM_FACTOR,
203
+ pricingMissRate: Math.max(p95((s) => s.pricingMisses / Math.max(s.modelCalls, 1)), 0) * HEADROOM_FACTOR,
204
+ circuitOpens: Math.max(p95((s) => s.circuitOpens), 1),
205
+ egressBlocked: Math.max(p95((s) => s.egressBlocked), 1),
206
+ baselineSessions: window.length,
207
+ };
208
+ }
209
+ /** Compare a session snapshot against derived thresholds; list every breach. */
210
+ export function detectBreaches(snapshot, thresholds) {
211
+ const breaches = [];
212
+ const add = (metric, observed, threshold, unit) => {
213
+ if (observed > threshold) {
214
+ breaches.push({
215
+ metric,
216
+ observed,
217
+ threshold,
218
+ detail: `${metric} ${fmt(observed)}${unit} exceeded baseline threshold ${fmt(threshold)}${unit} (derived from ${thresholds.baselineSessions} session(s))`,
219
+ });
220
+ }
221
+ };
222
+ add("error_rate", snapshot.errorRate, thresholds.errorRate, "");
223
+ add("turn_p95_seconds", snapshot.turnP95Seconds, thresholds.turnP95Seconds, "s");
224
+ add("ttft_p95_seconds", snapshot.ttftP95Seconds, thresholds.ttftP95Seconds, "s");
225
+ add("cost_burn_usd_per_min", snapshot.costBurnUsdPerMin, thresholds.costBurnUsdPerMin, "$/min");
226
+ const missRate = snapshot.pricingMisses / Math.max(snapshot.modelCalls, 1);
227
+ add("pricing_miss_rate", missRate, thresholds.pricingMissRate, "");
228
+ add("circuit_opens", snapshot.circuitOpens, thresholds.circuitOpens, "");
229
+ add("egress_blocked", snapshot.egressBlocked, thresholds.egressBlocked, "");
230
+ return breaches;
231
+ }
232
+ function fmt(n) {
233
+ return Number.isInteger(n) ? String(n) : n.toFixed(3);
234
+ }
235
+ // --------- snapshot persistence (the only I/O this module owns) ---------
236
+ /**
237
+ * Read the persisted session history, oldest first. Torn lines are skipped.
238
+ *
239
+ * F3 — bounds itself to the trailing {@link MAX_METRICS_HISTORY_LINES} raw
240
+ * lines BEFORE parsing, so a pre-existing oversized file (e.g. one written by
241
+ * an older binary that predates the cap, or a file that grew past the cap
242
+ * some other way) is never fully materialized into memory just to derive a
243
+ * threshold from its tail. `appendMetricsSnapshot` keeps the file itself
244
+ * trimmed going forward; this is the reader's independent bound.
245
+ */
246
+ export function readMetricsHistory(metricsDir = DEFAULT_METRICS_DIR) {
247
+ const path = join(metricsDir, METRICS_FILENAME);
248
+ if (!existsSync(path))
249
+ return [];
250
+ // Filter blank lines (incl. the trailing "" from the file's final newline)
251
+ // BEFORE slicing to the trailing cap — slicing raw split() output would
252
+ // waste one slot of the cap on that trailing empty entry.
253
+ const lines = readFileSync(path, "utf-8")
254
+ .split("\n")
255
+ .filter((l) => l.trim() !== "");
256
+ const bounded = lines.slice(-MAX_METRICS_HISTORY_LINES);
257
+ const out = [];
258
+ for (const line of bounded) {
259
+ try {
260
+ out.push(JSON.parse(line.trim()));
261
+ }
262
+ catch {
263
+ // skip corrupt line — an append-only log must survive one torn write.
264
+ }
265
+ }
266
+ return out;
267
+ }
268
+ /**
269
+ * Append one session snapshot to the history JSONL, creating dirs on first
270
+ * use, then trim the file to its trailing {@link MAX_METRICS_HISTORY_LINES}
271
+ * (F3 — the file otherwise grows forever even though `deriveThresholds` only
272
+ * ever looks at the trailing {@link BASELINE_WINDOW}). The trim is a
273
+ * temp-write + rename so a crash mid-trim never leaves a torn or truncated
274
+ * file in place of the real one; a torn LINE within the kept tail is still
275
+ * tolerated by `readMetricsHistory`.
276
+ */
277
+ export function appendMetricsSnapshot(snapshot, metricsDir = DEFAULT_METRICS_DIR) {
278
+ const path = join(metricsDir, METRICS_FILENAME);
279
+ mkdirSync(dirname(path), { recursive: true });
280
+ appendFileSync(path, `${JSON.stringify(snapshot)}\n`);
281
+ const lines = readFileSync(path, "utf-8")
282
+ .split("\n")
283
+ .filter((l) => l.trim() !== "");
284
+ if (lines.length <= MAX_METRICS_HISTORY_LINES)
285
+ return;
286
+ const trimmed = lines.slice(-MAX_METRICS_HISTORY_LINES);
287
+ const tmpPath = `${path}.tmp-${process.pid}-${Date.now()}`;
288
+ writeFileSync(tmpPath, `${trimmed.join("\n")}\n`);
289
+ renameSync(tmpPath, path);
290
+ }
@@ -0,0 +1,42 @@
1
+ import type { RunContext } from "@crewhaus/run-context";
2
+ import type { TraceEvent, TraceEventBus, Unsubscribe } from "@crewhaus/trace-event-bus";
3
+ export type IncidentTriggerKind = "circuit_open" | "egress_blocked" | "justification_deny_storm";
4
+ /** How many judge-backed denials in the ring buffer constitute a "storm". */
5
+ export declare const DENY_STORM_THRESHOLD = 3;
6
+ export type IncidentCapture = {
7
+ readonly kind: IncidentTriggerKind;
8
+ readonly reason: string;
9
+ readonly triggeredBy: string;
10
+ };
11
+ /**
12
+ * Classify a live TraceEvent as an incident trigger. `ringDenials` is the
13
+ * count of judge-backed denials already in the ring buffer (NOT including
14
+ * `event`); a permission deny tips into a storm when `ringDenials + 1` reaches
15
+ * the threshold.
16
+ */
17
+ export declare function classifyIncidentTrigger(event: TraceEvent, ringDenials: number): IncidentCapture | undefined;
18
+ /** Compact an ISO timestamp into a filesystem-safe, sortable stem. */
19
+ export declare function incidentDirName(iso: string, kind: IncidentTriggerKind): string;
20
+ export type AttachIncidentCollectorOptions = {
21
+ /** Root of the incidents tree. Default: `.crewhaus/incidents`. */
22
+ readonly incidentsDir?: string;
23
+ /** Spec identity stamped into the raw capture. */
24
+ readonly spec?: {
25
+ readonly name: string;
26
+ readonly version?: string;
27
+ readonly hash?: string;
28
+ };
29
+ /** Override now() for deterministic tests. */
30
+ readonly now?: () => Date;
31
+ };
32
+ export type AttachedIncidentCollector = {
33
+ unsubscribe: Unsubscribe;
34
+ };
35
+ /**
36
+ * Attach the incident collector when CREWHAUS_INCIDENTS is set. On the FIRST
37
+ * trigger it writes a raw capture; later triggers in the same session are
38
+ * ignored (one bundle per session — the first failure is the story; the ring
39
+ * buffer already carries what followed). Returns undefined (no-op) when the
40
+ * env gate is off.
41
+ */
42
+ export declare function attachIncidentCollector(bus: TraceEventBus, runContext: RunContext, env: NodeJS.ProcessEnv, options?: AttachIncidentCollectorOptions): AttachedIncidentCollector | undefined;
@@ -0,0 +1,125 @@
1
+ /**
2
+ * Item 32 — runtime side of auto-assembled incident bundles. An env-gated
3
+ * (CREWHAUS_INCIDENTS) bus subscriber that watches for trigger events —
4
+ * circuit breaker → open, egress-blocked, a justification-deny storm — and, on
5
+ * the first trigger, snapshots the bus ring buffer (`bus.recent()`) plus the
6
+ * trigger metadata into `.crewhaus/incidents/<ts>-<kind>/` as a RAW capture
7
+ * (events.jsonl + incident.json). The rich, human-readable bundle (HTML,
8
+ * doctor output, audit-window join, cost summary) is assembled by
9
+ * `crewhaus incident collect --session <id>` — which reads this raw capture's
10
+ * sibling session log — so runtime-core stays free of the eval-report/doctor
11
+ * dependencies.
12
+ *
13
+ * Kept lean and dependency-light: the only I/O is the single mkdir + write of
14
+ * the raw capture. Trigger classification is pure and unit-tested; the writer
15
+ * is best-effort (a capture failure is logged, never thrown — an incident
16
+ * writer must not turn one failure into two).
17
+ */
18
+ import { mkdirSync, writeFileSync } from "node:fs";
19
+ import { join } from "node:path";
20
+ /** How many judge-backed denials in the ring buffer constitute a "storm". */
21
+ export const DENY_STORM_THRESHOLD = 3;
22
+ /**
23
+ * Classify a live TraceEvent as an incident trigger. `ringDenials` is the
24
+ * count of judge-backed denials already in the ring buffer (NOT including
25
+ * `event`); a permission deny tips into a storm when `ringDenials + 1` reaches
26
+ * the threshold.
27
+ */
28
+ export function classifyIncidentTrigger(event, ringDenials) {
29
+ switch (event.kind) {
30
+ case "circuit_state_changed":
31
+ if (event.toState === "open") {
32
+ return {
33
+ kind: "circuit_open",
34
+ reason: `circuit ${event.adapter} → open${event.reason ? `: ${event.reason}` : ""}`,
35
+ triggeredBy: "circuit_state_changed",
36
+ };
37
+ }
38
+ return undefined;
39
+ case "permission_decision":
40
+ if (event.outcome === "egress-blocked") {
41
+ return {
42
+ kind: "egress_blocked",
43
+ reason: `egress blocked on ${event.toolName}`,
44
+ triggeredBy: "permission_decision",
45
+ };
46
+ }
47
+ if (event.decision === "deny" && event.judgeModel !== undefined) {
48
+ if (ringDenials + 1 >= DENY_STORM_THRESHOLD) {
49
+ return {
50
+ kind: "justification_deny_storm",
51
+ reason: `justification-deny storm: ${ringDenials + 1} denials in the ring buffer`,
52
+ triggeredBy: "permission_decision",
53
+ };
54
+ }
55
+ }
56
+ return undefined;
57
+ default:
58
+ return undefined;
59
+ }
60
+ }
61
+ /** Is this ring event a judge-backed justification denial? */
62
+ function isJudgeDenial(ev) {
63
+ return ev.kind === "permission_decision" && ev.decision === "deny" && ev.judgeModel !== undefined;
64
+ }
65
+ /** Compact an ISO timestamp into a filesystem-safe, sortable stem. */
66
+ export function incidentDirName(iso, kind) {
67
+ const compact = iso
68
+ .replace(/[-:]/g, "")
69
+ .replace(/\.\d+Z$/, "")
70
+ .replace(/Z$/, "");
71
+ return `${compact}-${kind}`;
72
+ }
73
+ /**
74
+ * Attach the incident collector when CREWHAUS_INCIDENTS is set. On the FIRST
75
+ * trigger it writes a raw capture; later triggers in the same session are
76
+ * ignored (one bundle per session — the first failure is the story; the ring
77
+ * buffer already carries what followed). Returns undefined (no-op) when the
78
+ * env gate is off.
79
+ */
80
+ export function attachIncidentCollector(bus, runContext, env, options = {}) {
81
+ const gate = env["CREWHAUS_INCIDENTS"];
82
+ if (gate !== "1" && gate !== "true")
83
+ return undefined;
84
+ const incidentsDir = options.incidentsDir ?? join(".crewhaus", "incidents");
85
+ const now = options.now ?? (() => new Date());
86
+ let captured = false;
87
+ const unsubscribe = bus.subscribe((event) => {
88
+ if (captured)
89
+ return;
90
+ // Count prior judge denials in the ring buffer (excludes the live event —
91
+ // it is not yet buffered when a synchronous subscriber runs, but be safe).
92
+ const ringDenials = bus.recent().filter(isJudgeDenial).length;
93
+ const trigger = classifyIncidentTrigger(event, ringDenials);
94
+ if (trigger === undefined)
95
+ return;
96
+ captured = true;
97
+ try {
98
+ const ts = now().toISOString();
99
+ const dir = join(incidentsDir, incidentDirName(ts, trigger.kind));
100
+ mkdirSync(dir, { recursive: true });
101
+ const ring = bus.recent();
102
+ const capture = {
103
+ version: 1,
104
+ kind: trigger.kind,
105
+ reason: trigger.reason,
106
+ triggeredBy: trigger.triggeredBy,
107
+ sessionId: runContext.sessionId,
108
+ runId: runContext.runId,
109
+ incidentTs: ts,
110
+ spec: options.spec ?? null,
111
+ ringEventCount: ring.length,
112
+ };
113
+ writeFileSync(join(dir, "incident.json"), `${JSON.stringify(capture, null, 2)}\n`);
114
+ writeFileSync(join(dir, "events.jsonl"), ring.length === 0 ? "" : `${ring.map((e) => JSON.stringify(e)).join("\n")}\n`);
115
+ process.stderr.write(`[incident] ${trigger.kind} captured → ${dir} (run \`crewhaus incident collect --session ${runContext.sessionId}\` for the full bundle)\n`);
116
+ }
117
+ catch (err) {
118
+ // Best-effort: an incident writer must not turn one failure into two.
119
+ runContext.logger.error("incident.capture_failed", {
120
+ message: err instanceof Error ? err.message : String(err),
121
+ });
122
+ }
123
+ });
124
+ return { unsubscribe };
125
+ }