@crewhaus/runtime-core 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/alert-watchdog.d.ts +138 -0
- package/dist/alert-watchdog.js +290 -0
- package/dist/incident-collector.d.ts +42 -0
- package/dist/incident-collector.js +125 -0
- package/dist/index.d.ts +222 -4
- package/dist/index.js +512 -32
- package/dist/janitor.d.ts +88 -0
- package/dist/janitor.js +287 -0
- package/dist/observability.d.ts +178 -2
- package/dist/observability.js +466 -2
- package/dist/project-memory.d.ts +17 -1
- package/dist/project-memory.js +42 -0
- package/dist/slo-monitor.d.ts +221 -0
- package/dist/slo-monitor.js +397 -0
- package/package.json +39 -39
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import type { TraceEvent } from "@crewhaus/trace-event-bus";
|
|
2
|
+
/** Default location of the per-session metrics history, relative to cwd. */
|
|
3
|
+
export declare const DEFAULT_METRICS_DIR: string;
|
|
4
|
+
export declare const METRICS_FILENAME = "sessions.jsonl";
|
|
5
|
+
/** How many historical sessions we need before deriving a real baseline. */
|
|
6
|
+
export declare const MIN_BASELINE_SESSIONS = 5;
|
|
7
|
+
/** Headroom multiplier over the trailing p95 (a 50% cushion). */
|
|
8
|
+
export declare const HEADROOM_FACTOR = 1.5;
|
|
9
|
+
/** How many trailing sessions the baseline is derived from. */
|
|
10
|
+
export declare const BASELINE_WINDOW = 50;
|
|
11
|
+
/**
|
|
12
|
+
* F3 (ops pre-merge) — hard cap on how many lines `sessions.jsonl` is allowed
|
|
13
|
+
* to grow to. `deriveThresholds` only ever looks at the trailing
|
|
14
|
+
* {@link BASELINE_WINDOW} sessions, but `appendMetricsSnapshot` was
|
|
15
|
+
* append-only and `readMetricsHistory` re-read (and re-parsed) the ENTIRE
|
|
16
|
+
* file every session — unbounded growth for a file nothing ever reads past
|
|
17
|
+
* its tail. 4× the baseline window is generous headroom over what
|
|
18
|
+
* `deriveThresholds` consumes, with a 200-line floor so a small
|
|
19
|
+
* `BASELINE_WINDOW` still keeps a useful amount of history for operators
|
|
20
|
+
* eyeballing the file by hand.
|
|
21
|
+
*/
|
|
22
|
+
export declare const MAX_METRICS_HISTORY_LINES: number;
|
|
23
|
+
/**
|
|
24
|
+
* The compact per-session snapshot persisted to `sessions.jsonl` and read back
|
|
25
|
+
* to derive baselines. Percentiles are computed from the per-session samples
|
|
26
|
+
* at session end so the stored line is small and directly comparable.
|
|
27
|
+
*/
|
|
28
|
+
export type SessionMetricsSnapshot = {
|
|
29
|
+
readonly sessionId: string;
|
|
30
|
+
readonly ts: string;
|
|
31
|
+
/** Total user turns (turn_end events). */
|
|
32
|
+
readonly turns: number;
|
|
33
|
+
/** Total model calls (model_response events). */
|
|
34
|
+
readonly modelCalls: number;
|
|
35
|
+
/** Errors that were NOT recovered (error_recovered action "fail"). */
|
|
36
|
+
readonly unrecoveredErrors: number;
|
|
37
|
+
/** unrecoveredErrors / max(modelCalls,1) — the rate we alert on. */
|
|
38
|
+
readonly errorRate: number;
|
|
39
|
+
/** p95 of per-turn durations, seconds. */
|
|
40
|
+
readonly turnP95Seconds: number;
|
|
41
|
+
/** p95 of time-to-first-token, seconds. */
|
|
42
|
+
readonly ttftP95Seconds: number;
|
|
43
|
+
/** Total spend in USD-micros over the session. */
|
|
44
|
+
readonly costUsdMicros: number;
|
|
45
|
+
/** Cost burn rate: USD per minute of wall-clock session time. */
|
|
46
|
+
readonly costBurnUsdPerMin: number;
|
|
47
|
+
/** cost_accrual events that priced to $0 despite non-zero tokens (pricing miss). */
|
|
48
|
+
readonly pricingMisses: number;
|
|
49
|
+
/** Circuit-breaker transitions into "open". */
|
|
50
|
+
readonly circuitOpens: number;
|
|
51
|
+
/** Egress classifier block-tier verdicts. */
|
|
52
|
+
readonly egressBlocked: number;
|
|
53
|
+
/** Justification-gate / policy denials (permission_decision decision "deny"). */
|
|
54
|
+
readonly permissionDenials: number;
|
|
55
|
+
};
|
|
56
|
+
/** A single threshold breach found by {@link detectBreaches}. */
|
|
57
|
+
export type Breach = {
|
|
58
|
+
readonly metric: string;
|
|
59
|
+
readonly observed: number;
|
|
60
|
+
readonly threshold: number;
|
|
61
|
+
readonly detail: string;
|
|
62
|
+
};
|
|
63
|
+
/** Thresholds derived from history (or bootstrap defaults on a cold start). */
|
|
64
|
+
export type DerivedThresholds = {
|
|
65
|
+
readonly errorRate: number;
|
|
66
|
+
readonly turnP95Seconds: number;
|
|
67
|
+
readonly ttftP95Seconds: number;
|
|
68
|
+
readonly costBurnUsdPerMin: number;
|
|
69
|
+
readonly pricingMissRate: number;
|
|
70
|
+
readonly circuitOpens: number;
|
|
71
|
+
readonly egressBlocked: number;
|
|
72
|
+
/** How many historical sessions the thresholds were derived from (0 = bootstrap). */
|
|
73
|
+
readonly baselineSessions: number;
|
|
74
|
+
};
|
|
75
|
+
/**
|
|
76
|
+
* Accumulates a session's metric signals from bus events. Mirrors
|
|
77
|
+
* metrics-collector's event→metric derivation (turn_end duration, TTFT from
|
|
78
|
+
* model_request→first model_stream_token) so the watchdog does not depend on
|
|
79
|
+
* the opt-in metrics-collector being attached.
|
|
80
|
+
*/
|
|
81
|
+
export declare class SessionMetricsAccumulator {
|
|
82
|
+
private turnDurations;
|
|
83
|
+
private ttfts;
|
|
84
|
+
private modelCalls;
|
|
85
|
+
private turns;
|
|
86
|
+
private unrecoveredErrors;
|
|
87
|
+
private costUsdMicros;
|
|
88
|
+
private pricingMisses;
|
|
89
|
+
private circuitOpens;
|
|
90
|
+
private egressBlocked;
|
|
91
|
+
private permissionDenials;
|
|
92
|
+
private firstTs;
|
|
93
|
+
private lastTs;
|
|
94
|
+
private readonly requestStarts;
|
|
95
|
+
private readonly ttftSeen;
|
|
96
|
+
fold(ev: TraceEvent): void;
|
|
97
|
+
/** Finalize into a persistable snapshot. `now` seeds the timestamp. */
|
|
98
|
+
snapshot(sessionId: string, now?: Date): SessionMetricsSnapshot;
|
|
99
|
+
}
|
|
100
|
+
/** Nearest-rank p95 (0-based index floor((p)*(n-1))). Empty → 0. */
|
|
101
|
+
export declare function percentile(values: ReadonlyArray<number>, p: number): number;
|
|
102
|
+
/**
|
|
103
|
+
* Derive thresholds from a session history. For the "higher-is-worse"
|
|
104
|
+
* continuous metrics (error rate, turn p95, TTFT p95, cost burn, pricing-miss
|
|
105
|
+
* rate) the threshold is `trailing-p95 × HEADROOM_FACTOR`. For the count
|
|
106
|
+
* metrics (circuit opens, egress blocks) the threshold is the trailing p95
|
|
107
|
+
* count (any session materially above its own historical worst is worth a
|
|
108
|
+
* look) with a floor of 1 so a single first-ever block still alerts.
|
|
109
|
+
*
|
|
110
|
+
* With fewer than {@link MIN_BASELINE_SESSIONS} historical rows, there is not
|
|
111
|
+
* enough signal to trust a percentile — conservative bootstrap defaults are
|
|
112
|
+
* used instead so a cold start neither alert-storms nor stays silent on an
|
|
113
|
+
* obviously broken session.
|
|
114
|
+
*/
|
|
115
|
+
export declare function deriveThresholds(history: ReadonlyArray<SessionMetricsSnapshot>): DerivedThresholds;
|
|
116
|
+
/** Compare a session snapshot against derived thresholds; list every breach. */
|
|
117
|
+
export declare function detectBreaches(snapshot: SessionMetricsSnapshot, thresholds: DerivedThresholds): Breach[];
|
|
118
|
+
/**
|
|
119
|
+
* Read the persisted session history, oldest first. Torn lines are skipped.
|
|
120
|
+
*
|
|
121
|
+
* F3 — bounds itself to the trailing {@link MAX_METRICS_HISTORY_LINES} raw
|
|
122
|
+
* lines BEFORE parsing, so a pre-existing oversized file (e.g. one written by
|
|
123
|
+
* an older binary that predates the cap, or a file that grew past the cap
|
|
124
|
+
* some other way) is never fully materialized into memory just to derive a
|
|
125
|
+
* threshold from its tail. `appendMetricsSnapshot` keeps the file itself
|
|
126
|
+
* trimmed going forward; this is the reader's independent bound.
|
|
127
|
+
*/
|
|
128
|
+
export declare function readMetricsHistory(metricsDir?: string): SessionMetricsSnapshot[];
|
|
129
|
+
/**
|
|
130
|
+
* Append one session snapshot to the history JSONL, creating dirs on first
|
|
131
|
+
* use, then trim the file to its trailing {@link MAX_METRICS_HISTORY_LINES}
|
|
132
|
+
* (F3 — the file otherwise grows forever even though `deriveThresholds` only
|
|
133
|
+
* ever looks at the trailing {@link BASELINE_WINDOW}). The trim is a
|
|
134
|
+
* temp-write + rename so a crash mid-trim never leaves a torn or truncated
|
|
135
|
+
* file in place of the real one; a torn LINE within the kept tail is still
|
|
136
|
+
* tolerated by `readMetricsHistory`.
|
|
137
|
+
*/
|
|
138
|
+
export declare function appendMetricsSnapshot(snapshot: SessionMetricsSnapshot, metricsDir?: string): void;
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ops item 31 — alert watchdog with baseline-derived thresholds.
|
|
3
|
+
*
|
|
4
|
+
* A bus subscriber (wired into `attachDefaultSubscribers`, gated by
|
|
5
|
+
* CREWHAUS_ALERTS) that folds a run's TraceEvents into a small per-session
|
|
6
|
+
* metrics snapshot, then — at session end — compares that snapshot against
|
|
7
|
+
* thresholds DERIVED FROM HISTORY (the trailing p95 of prior sessions'
|
|
8
|
+
* persisted snapshots, ×a headroom factor) rather than hand-configured
|
|
9
|
+
* constants. On a breach it publishes an `alert_raised` trace event and hands
|
|
10
|
+
* the breach to injected sinks (audit append + settings.json `alert` hook /
|
|
11
|
+
* webhook — supplied by the caller so this module stays side-effect-free apart
|
|
12
|
+
* from the snapshot persistence it owns).
|
|
13
|
+
*
|
|
14
|
+
* WHY PERSIST A SNAPSHOT: nothing durably records per-session metric history
|
|
15
|
+
* today (metrics-collector is opt-in and its sinks are point-in-time). Without
|
|
16
|
+
* history there is nothing to derive a baseline from. So the watchdog appends
|
|
17
|
+
* one compact JSONL line per session under `.crewhaus/metrics/sessions.jsonl`
|
|
18
|
+
* (additive; readers tolerate torn lines). The FIRST few sessions have no
|
|
19
|
+
* baseline — the watchdog uses conservative bootstrap defaults until enough
|
|
20
|
+
* history accrues (`MIN_BASELINE_SESSIONS`), so a cold start never alert-storms.
|
|
21
|
+
*
|
|
22
|
+
* This file is the side-effect-free-ish core (the only I/O it owns is reading
|
|
23
|
+
* and appending its own snapshot JSONL); event folding, threshold derivation,
|
|
24
|
+
* and breach detection are pure and unit-tested.
|
|
25
|
+
*/
|
|
26
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync, } from "node:fs";
|
|
27
|
+
import { dirname, join } from "node:path";
|
|
28
|
+
/** Default location of the per-session metrics history, relative to cwd. */
|
|
29
|
+
export const DEFAULT_METRICS_DIR = join(".crewhaus", "metrics");
|
|
30
|
+
export const METRICS_FILENAME = "sessions.jsonl";
|
|
31
|
+
/** How many historical sessions we need before deriving a real baseline. */
|
|
32
|
+
export const MIN_BASELINE_SESSIONS = 5;
|
|
33
|
+
/** Headroom multiplier over the trailing p95 (a 50% cushion). */
|
|
34
|
+
export const HEADROOM_FACTOR = 1.5;
|
|
35
|
+
/** How many trailing sessions the baseline is derived from. */
|
|
36
|
+
export const BASELINE_WINDOW = 50;
|
|
37
|
+
/**
|
|
38
|
+
* F3 (ops pre-merge) — hard cap on how many lines `sessions.jsonl` is allowed
|
|
39
|
+
* to grow to. `deriveThresholds` only ever looks at the trailing
|
|
40
|
+
* {@link BASELINE_WINDOW} sessions, but `appendMetricsSnapshot` was
|
|
41
|
+
* append-only and `readMetricsHistory` re-read (and re-parsed) the ENTIRE
|
|
42
|
+
* file every session — unbounded growth for a file nothing ever reads past
|
|
43
|
+
* its tail. 4× the baseline window is generous headroom over what
|
|
44
|
+
* `deriveThresholds` consumes, with a 200-line floor so a small
|
|
45
|
+
* `BASELINE_WINDOW` still keeps a useful amount of history for operators
|
|
46
|
+
* eyeballing the file by hand.
|
|
47
|
+
*/
|
|
48
|
+
export const MAX_METRICS_HISTORY_LINES = Math.max(BASELINE_WINDOW * 4, 200);
|
|
49
|
+
// --------- pure event folding ---------
|
|
50
|
+
/**
|
|
51
|
+
* Accumulates a session's metric signals from bus events. Mirrors
|
|
52
|
+
* metrics-collector's event→metric derivation (turn_end duration, TTFT from
|
|
53
|
+
* model_request→first model_stream_token) so the watchdog does not depend on
|
|
54
|
+
* the opt-in metrics-collector being attached.
|
|
55
|
+
*/
|
|
56
|
+
export class SessionMetricsAccumulator {
|
|
57
|
+
turnDurations = [];
|
|
58
|
+
ttfts = [];
|
|
59
|
+
modelCalls = 0;
|
|
60
|
+
turns = 0;
|
|
61
|
+
unrecoveredErrors = 0;
|
|
62
|
+
costUsdMicros = 0;
|
|
63
|
+
pricingMisses = 0;
|
|
64
|
+
circuitOpens = 0;
|
|
65
|
+
egressBlocked = 0;
|
|
66
|
+
permissionDenials = 0;
|
|
67
|
+
firstTs;
|
|
68
|
+
lastTs;
|
|
69
|
+
requestStarts = new Map();
|
|
70
|
+
ttftSeen = new Set();
|
|
71
|
+
fold(ev) {
|
|
72
|
+
const t = Date.parse(ev.timestamp);
|
|
73
|
+
if (!Number.isNaN(t)) {
|
|
74
|
+
if (this.firstTs === undefined || t < this.firstTs)
|
|
75
|
+
this.firstTs = t;
|
|
76
|
+
if (this.lastTs === undefined || t > this.lastTs)
|
|
77
|
+
this.lastTs = t;
|
|
78
|
+
}
|
|
79
|
+
switch (ev.kind) {
|
|
80
|
+
case "turn_end":
|
|
81
|
+
this.turns += 1;
|
|
82
|
+
this.turnDurations.push(ev.durationMs / 1000);
|
|
83
|
+
return;
|
|
84
|
+
case "model_request":
|
|
85
|
+
this.requestStarts.set(ev.traceId, Date.parse(ev.timestamp));
|
|
86
|
+
this.ttftSeen.delete(ev.traceId);
|
|
87
|
+
return;
|
|
88
|
+
case "model_stream_token": {
|
|
89
|
+
if (this.ttftSeen.has(ev.traceId))
|
|
90
|
+
return;
|
|
91
|
+
const start = this.requestStarts.get(ev.traceId);
|
|
92
|
+
if (start === undefined)
|
|
93
|
+
return;
|
|
94
|
+
const tokenMs = Date.parse(ev.timestamp);
|
|
95
|
+
if (Number.isNaN(tokenMs))
|
|
96
|
+
return;
|
|
97
|
+
this.ttfts.push(Math.max(0, (tokenMs - start) / 1000));
|
|
98
|
+
this.ttftSeen.add(ev.traceId);
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
case "model_response":
|
|
102
|
+
this.modelCalls += 1;
|
|
103
|
+
this.requestStarts.delete(ev.traceId);
|
|
104
|
+
this.ttftSeen.delete(ev.traceId);
|
|
105
|
+
return;
|
|
106
|
+
case "error_recovered":
|
|
107
|
+
// Only a terminal "fail" is an unrecovered error; retry/compact/etc.
|
|
108
|
+
// are the recovery engine doing its job.
|
|
109
|
+
if (ev.action === "fail")
|
|
110
|
+
this.unrecoveredErrors += 1;
|
|
111
|
+
return;
|
|
112
|
+
case "cost_accrual": {
|
|
113
|
+
// Skip the FR-003 terminal aggregate (summary:true) — it double-counts.
|
|
114
|
+
if (ev.summary === true)
|
|
115
|
+
return;
|
|
116
|
+
this.costUsdMicros += ev.costUsdMicros;
|
|
117
|
+
if (ev.costUsdMicros === 0 && ev.inputTokens + ev.outputTokens > 0) {
|
|
118
|
+
this.pricingMisses += 1;
|
|
119
|
+
}
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
case "circuit_state_changed":
|
|
123
|
+
if (ev.toState === "open")
|
|
124
|
+
this.circuitOpens += 1;
|
|
125
|
+
return;
|
|
126
|
+
case "permission_decision":
|
|
127
|
+
if (ev.outcome === "egress-blocked")
|
|
128
|
+
this.egressBlocked += 1;
|
|
129
|
+
else if (ev.outcome === undefined && ev.decision === "deny")
|
|
130
|
+
this.permissionDenials += 1;
|
|
131
|
+
return;
|
|
132
|
+
default:
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
/** Finalize into a persistable snapshot. `now` seeds the timestamp. */
|
|
137
|
+
snapshot(sessionId, now = new Date()) {
|
|
138
|
+
const wallMs = this.firstTs !== undefined && this.lastTs !== undefined ? this.lastTs - this.firstTs : 0;
|
|
139
|
+
const wallMin = wallMs / 60000;
|
|
140
|
+
const costUsd = this.costUsdMicros / 1_000_000;
|
|
141
|
+
return {
|
|
142
|
+
sessionId,
|
|
143
|
+
ts: now.toISOString(),
|
|
144
|
+
turns: this.turns,
|
|
145
|
+
modelCalls: this.modelCalls,
|
|
146
|
+
unrecoveredErrors: this.unrecoveredErrors,
|
|
147
|
+
errorRate: this.unrecoveredErrors / Math.max(this.modelCalls, 1),
|
|
148
|
+
turnP95Seconds: percentile(this.turnDurations, 0.95),
|
|
149
|
+
ttftP95Seconds: percentile(this.ttfts, 0.95),
|
|
150
|
+
costUsdMicros: this.costUsdMicros,
|
|
151
|
+
costBurnUsdPerMin: wallMin > 0 ? costUsd / wallMin : 0,
|
|
152
|
+
pricingMisses: this.pricingMisses,
|
|
153
|
+
circuitOpens: this.circuitOpens,
|
|
154
|
+
egressBlocked: this.egressBlocked,
|
|
155
|
+
permissionDenials: this.permissionDenials,
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
/** Nearest-rank p95 (0-based index floor((p)*(n-1))). Empty → 0. */
|
|
160
|
+
export function percentile(values, p) {
|
|
161
|
+
if (values.length === 0)
|
|
162
|
+
return 0;
|
|
163
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
164
|
+
const idx = Math.floor(p * (sorted.length - 1));
|
|
165
|
+
return sorted[idx] ?? 0;
|
|
166
|
+
}
|
|
167
|
+
// --------- threshold derivation + breach detection ---------
|
|
168
|
+
/**
|
|
169
|
+
* Derive thresholds from a session history. For the "higher-is-worse"
|
|
170
|
+
* continuous metrics (error rate, turn p95, TTFT p95, cost burn, pricing-miss
|
|
171
|
+
* rate) the threshold is `trailing-p95 × HEADROOM_FACTOR`. For the count
|
|
172
|
+
* metrics (circuit opens, egress blocks) the threshold is the trailing p95
|
|
173
|
+
* count (any session materially above its own historical worst is worth a
|
|
174
|
+
* look) with a floor of 1 so a single first-ever block still alerts.
|
|
175
|
+
*
|
|
176
|
+
* With fewer than {@link MIN_BASELINE_SESSIONS} historical rows, there is not
|
|
177
|
+
* enough signal to trust a percentile — conservative bootstrap defaults are
|
|
178
|
+
* used instead so a cold start neither alert-storms nor stays silent on an
|
|
179
|
+
* obviously broken session.
|
|
180
|
+
*/
|
|
181
|
+
export function deriveThresholds(history) {
|
|
182
|
+
const window = history.slice(-BASELINE_WINDOW);
|
|
183
|
+
if (window.length < MIN_BASELINE_SESSIONS) {
|
|
184
|
+
return {
|
|
185
|
+
// Bootstrap defaults: loose enough not to fire on normal variance, tight
|
|
186
|
+
// enough to catch a session that is clearly off the rails.
|
|
187
|
+
errorRate: 0.5,
|
|
188
|
+
turnP95Seconds: 120,
|
|
189
|
+
ttftP95Seconds: 30,
|
|
190
|
+
costBurnUsdPerMin: 5,
|
|
191
|
+
pricingMissRate: 0.5,
|
|
192
|
+
circuitOpens: 1,
|
|
193
|
+
egressBlocked: 1,
|
|
194
|
+
baselineSessions: window.length,
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
const p95 = (pick) => percentile(window.map(pick), 0.95);
|
|
198
|
+
return {
|
|
199
|
+
errorRate: Math.max(p95((s) => s.errorRate) * HEADROOM_FACTOR, 0.05),
|
|
200
|
+
turnP95Seconds: p95((s) => s.turnP95Seconds) * HEADROOM_FACTOR,
|
|
201
|
+
ttftP95Seconds: p95((s) => s.ttftP95Seconds) * HEADROOM_FACTOR,
|
|
202
|
+
costBurnUsdPerMin: p95((s) => s.costBurnUsdPerMin) * HEADROOM_FACTOR,
|
|
203
|
+
pricingMissRate: Math.max(p95((s) => s.pricingMisses / Math.max(s.modelCalls, 1)), 0) * HEADROOM_FACTOR,
|
|
204
|
+
circuitOpens: Math.max(p95((s) => s.circuitOpens), 1),
|
|
205
|
+
egressBlocked: Math.max(p95((s) => s.egressBlocked), 1),
|
|
206
|
+
baselineSessions: window.length,
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
/** Compare a session snapshot against derived thresholds; list every breach. */
|
|
210
|
+
export function detectBreaches(snapshot, thresholds) {
|
|
211
|
+
const breaches = [];
|
|
212
|
+
const add = (metric, observed, threshold, unit) => {
|
|
213
|
+
if (observed > threshold) {
|
|
214
|
+
breaches.push({
|
|
215
|
+
metric,
|
|
216
|
+
observed,
|
|
217
|
+
threshold,
|
|
218
|
+
detail: `${metric} ${fmt(observed)}${unit} exceeded baseline threshold ${fmt(threshold)}${unit} (derived from ${thresholds.baselineSessions} session(s))`,
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
};
|
|
222
|
+
add("error_rate", snapshot.errorRate, thresholds.errorRate, "");
|
|
223
|
+
add("turn_p95_seconds", snapshot.turnP95Seconds, thresholds.turnP95Seconds, "s");
|
|
224
|
+
add("ttft_p95_seconds", snapshot.ttftP95Seconds, thresholds.ttftP95Seconds, "s");
|
|
225
|
+
add("cost_burn_usd_per_min", snapshot.costBurnUsdPerMin, thresholds.costBurnUsdPerMin, "$/min");
|
|
226
|
+
const missRate = snapshot.pricingMisses / Math.max(snapshot.modelCalls, 1);
|
|
227
|
+
add("pricing_miss_rate", missRate, thresholds.pricingMissRate, "");
|
|
228
|
+
add("circuit_opens", snapshot.circuitOpens, thresholds.circuitOpens, "");
|
|
229
|
+
add("egress_blocked", snapshot.egressBlocked, thresholds.egressBlocked, "");
|
|
230
|
+
return breaches;
|
|
231
|
+
}
|
|
232
|
+
function fmt(n) {
|
|
233
|
+
return Number.isInteger(n) ? String(n) : n.toFixed(3);
|
|
234
|
+
}
|
|
235
|
+
// --------- snapshot persistence (the only I/O this module owns) ---------
|
|
236
|
+
/**
|
|
237
|
+
* Read the persisted session history, oldest first. Torn lines are skipped.
|
|
238
|
+
*
|
|
239
|
+
* F3 — bounds itself to the trailing {@link MAX_METRICS_HISTORY_LINES} raw
|
|
240
|
+
* lines BEFORE parsing, so a pre-existing oversized file (e.g. one written by
|
|
241
|
+
* an older binary that predates the cap, or a file that grew past the cap
|
|
242
|
+
* some other way) is never fully materialized into memory just to derive a
|
|
243
|
+
* threshold from its tail. `appendMetricsSnapshot` keeps the file itself
|
|
244
|
+
* trimmed going forward; this is the reader's independent bound.
|
|
245
|
+
*/
|
|
246
|
+
export function readMetricsHistory(metricsDir = DEFAULT_METRICS_DIR) {
|
|
247
|
+
const path = join(metricsDir, METRICS_FILENAME);
|
|
248
|
+
if (!existsSync(path))
|
|
249
|
+
return [];
|
|
250
|
+
// Filter blank lines (incl. the trailing "" from the file's final newline)
|
|
251
|
+
// BEFORE slicing to the trailing cap — slicing raw split() output would
|
|
252
|
+
// waste one slot of the cap on that trailing empty entry.
|
|
253
|
+
const lines = readFileSync(path, "utf-8")
|
|
254
|
+
.split("\n")
|
|
255
|
+
.filter((l) => l.trim() !== "");
|
|
256
|
+
const bounded = lines.slice(-MAX_METRICS_HISTORY_LINES);
|
|
257
|
+
const out = [];
|
|
258
|
+
for (const line of bounded) {
|
|
259
|
+
try {
|
|
260
|
+
out.push(JSON.parse(line.trim()));
|
|
261
|
+
}
|
|
262
|
+
catch {
|
|
263
|
+
// skip corrupt line — an append-only log must survive one torn write.
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
return out;
|
|
267
|
+
}
|
|
268
|
+
/**
|
|
269
|
+
* Append one session snapshot to the history JSONL, creating dirs on first
|
|
270
|
+
* use, then trim the file to its trailing {@link MAX_METRICS_HISTORY_LINES}
|
|
271
|
+
* (F3 — the file otherwise grows forever even though `deriveThresholds` only
|
|
272
|
+
* ever looks at the trailing {@link BASELINE_WINDOW}). The trim is a
|
|
273
|
+
* temp-write + rename so a crash mid-trim never leaves a torn or truncated
|
|
274
|
+
* file in place of the real one; a torn LINE within the kept tail is still
|
|
275
|
+
* tolerated by `readMetricsHistory`.
|
|
276
|
+
*/
|
|
277
|
+
export function appendMetricsSnapshot(snapshot, metricsDir = DEFAULT_METRICS_DIR) {
|
|
278
|
+
const path = join(metricsDir, METRICS_FILENAME);
|
|
279
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
280
|
+
appendFileSync(path, `${JSON.stringify(snapshot)}\n`);
|
|
281
|
+
const lines = readFileSync(path, "utf-8")
|
|
282
|
+
.split("\n")
|
|
283
|
+
.filter((l) => l.trim() !== "");
|
|
284
|
+
if (lines.length <= MAX_METRICS_HISTORY_LINES)
|
|
285
|
+
return;
|
|
286
|
+
const trimmed = lines.slice(-MAX_METRICS_HISTORY_LINES);
|
|
287
|
+
const tmpPath = `${path}.tmp-${process.pid}-${Date.now()}`;
|
|
288
|
+
writeFileSync(tmpPath, `${trimmed.join("\n")}\n`);
|
|
289
|
+
renameSync(tmpPath, path);
|
|
290
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { RunContext } from "@crewhaus/run-context";
|
|
2
|
+
import type { TraceEvent, TraceEventBus, Unsubscribe } from "@crewhaus/trace-event-bus";
|
|
3
|
+
export type IncidentTriggerKind = "circuit_open" | "egress_blocked" | "justification_deny_storm";
|
|
4
|
+
/** How many judge-backed denials in the ring buffer constitute a "storm". */
|
|
5
|
+
export declare const DENY_STORM_THRESHOLD = 3;
|
|
6
|
+
export type IncidentCapture = {
|
|
7
|
+
readonly kind: IncidentTriggerKind;
|
|
8
|
+
readonly reason: string;
|
|
9
|
+
readonly triggeredBy: string;
|
|
10
|
+
};
|
|
11
|
+
/**
|
|
12
|
+
* Classify a live TraceEvent as an incident trigger. `ringDenials` is the
|
|
13
|
+
* count of judge-backed denials already in the ring buffer (NOT including
|
|
14
|
+
* `event`); a permission deny tips into a storm when `ringDenials + 1` reaches
|
|
15
|
+
* the threshold.
|
|
16
|
+
*/
|
|
17
|
+
export declare function classifyIncidentTrigger(event: TraceEvent, ringDenials: number): IncidentCapture | undefined;
|
|
18
|
+
/** Compact an ISO timestamp into a filesystem-safe, sortable stem. */
|
|
19
|
+
export declare function incidentDirName(iso: string, kind: IncidentTriggerKind): string;
|
|
20
|
+
export type AttachIncidentCollectorOptions = {
|
|
21
|
+
/** Root of the incidents tree. Default: `.crewhaus/incidents`. */
|
|
22
|
+
readonly incidentsDir?: string;
|
|
23
|
+
/** Spec identity stamped into the raw capture. */
|
|
24
|
+
readonly spec?: {
|
|
25
|
+
readonly name: string;
|
|
26
|
+
readonly version?: string;
|
|
27
|
+
readonly hash?: string;
|
|
28
|
+
};
|
|
29
|
+
/** Override now() for deterministic tests. */
|
|
30
|
+
readonly now?: () => Date;
|
|
31
|
+
};
|
|
32
|
+
export type AttachedIncidentCollector = {
|
|
33
|
+
unsubscribe: Unsubscribe;
|
|
34
|
+
};
|
|
35
|
+
/**
|
|
36
|
+
* Attach the incident collector when CREWHAUS_INCIDENTS is set. On the FIRST
|
|
37
|
+
* trigger it writes a raw capture; later triggers in the same session are
|
|
38
|
+
* ignored (one bundle per session — the first failure is the story; the ring
|
|
39
|
+
* buffer already carries what followed). Returns undefined (no-op) when the
|
|
40
|
+
* env gate is off.
|
|
41
|
+
*/
|
|
42
|
+
export declare function attachIncidentCollector(bus: TraceEventBus, runContext: RunContext, env: NodeJS.ProcessEnv, options?: AttachIncidentCollectorOptions): AttachedIncidentCollector | undefined;
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Item 32 — runtime side of auto-assembled incident bundles. An env-gated
|
|
3
|
+
* (CREWHAUS_INCIDENTS) bus subscriber that watches for trigger events —
|
|
4
|
+
* circuit breaker → open, egress-blocked, a justification-deny storm — and, on
|
|
5
|
+
* the first trigger, snapshots the bus ring buffer (`bus.recent()`) plus the
|
|
6
|
+
* trigger metadata into `.crewhaus/incidents/<ts>-<kind>/` as a RAW capture
|
|
7
|
+
* (events.jsonl + incident.json). The rich, human-readable bundle (HTML,
|
|
8
|
+
* doctor output, audit-window join, cost summary) is assembled by
|
|
9
|
+
* `crewhaus incident collect --session <id>` — which reads this raw capture's
|
|
10
|
+
* sibling session log — so runtime-core stays free of the eval-report/doctor
|
|
11
|
+
* dependencies.
|
|
12
|
+
*
|
|
13
|
+
* Kept lean and dependency-light: the only I/O is the single mkdir + write of
|
|
14
|
+
* the raw capture. Trigger classification is pure and unit-tested; the writer
|
|
15
|
+
* is best-effort (a capture failure is logged, never thrown — an incident
|
|
16
|
+
* writer must not turn one failure into two).
|
|
17
|
+
*/
|
|
18
|
+
import { mkdirSync, writeFileSync } from "node:fs";
|
|
19
|
+
import { join } from "node:path";
|
|
20
|
+
/** How many judge-backed denials in the ring buffer constitute a "storm". */
|
|
21
|
+
export const DENY_STORM_THRESHOLD = 3;
|
|
22
|
+
/**
|
|
23
|
+
* Classify a live TraceEvent as an incident trigger. `ringDenials` is the
|
|
24
|
+
* count of judge-backed denials already in the ring buffer (NOT including
|
|
25
|
+
* `event`); a permission deny tips into a storm when `ringDenials + 1` reaches
|
|
26
|
+
* the threshold.
|
|
27
|
+
*/
|
|
28
|
+
export function classifyIncidentTrigger(event, ringDenials) {
|
|
29
|
+
switch (event.kind) {
|
|
30
|
+
case "circuit_state_changed":
|
|
31
|
+
if (event.toState === "open") {
|
|
32
|
+
return {
|
|
33
|
+
kind: "circuit_open",
|
|
34
|
+
reason: `circuit ${event.adapter} → open${event.reason ? `: ${event.reason}` : ""}`,
|
|
35
|
+
triggeredBy: "circuit_state_changed",
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
return undefined;
|
|
39
|
+
case "permission_decision":
|
|
40
|
+
if (event.outcome === "egress-blocked") {
|
|
41
|
+
return {
|
|
42
|
+
kind: "egress_blocked",
|
|
43
|
+
reason: `egress blocked on ${event.toolName}`,
|
|
44
|
+
triggeredBy: "permission_decision",
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
if (event.decision === "deny" && event.judgeModel !== undefined) {
|
|
48
|
+
if (ringDenials + 1 >= DENY_STORM_THRESHOLD) {
|
|
49
|
+
return {
|
|
50
|
+
kind: "justification_deny_storm",
|
|
51
|
+
reason: `justification-deny storm: ${ringDenials + 1} denials in the ring buffer`,
|
|
52
|
+
triggeredBy: "permission_decision",
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return undefined;
|
|
57
|
+
default:
|
|
58
|
+
return undefined;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
/** Is this ring event a judge-backed justification denial? */
|
|
62
|
+
function isJudgeDenial(ev) {
|
|
63
|
+
return ev.kind === "permission_decision" && ev.decision === "deny" && ev.judgeModel !== undefined;
|
|
64
|
+
}
|
|
65
|
+
/** Compact an ISO timestamp into a filesystem-safe, sortable stem. */
|
|
66
|
+
export function incidentDirName(iso, kind) {
|
|
67
|
+
const compact = iso
|
|
68
|
+
.replace(/[-:]/g, "")
|
|
69
|
+
.replace(/\.\d+Z$/, "")
|
|
70
|
+
.replace(/Z$/, "");
|
|
71
|
+
return `${compact}-${kind}`;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Attach the incident collector when CREWHAUS_INCIDENTS is set. On the FIRST
|
|
75
|
+
* trigger it writes a raw capture; later triggers in the same session are
|
|
76
|
+
* ignored (one bundle per session — the first failure is the story; the ring
|
|
77
|
+
* buffer already carries what followed). Returns undefined (no-op) when the
|
|
78
|
+
* env gate is off.
|
|
79
|
+
*/
|
|
80
|
+
export function attachIncidentCollector(bus, runContext, env, options = {}) {
|
|
81
|
+
const gate = env["CREWHAUS_INCIDENTS"];
|
|
82
|
+
if (gate !== "1" && gate !== "true")
|
|
83
|
+
return undefined;
|
|
84
|
+
const incidentsDir = options.incidentsDir ?? join(".crewhaus", "incidents");
|
|
85
|
+
const now = options.now ?? (() => new Date());
|
|
86
|
+
let captured = false;
|
|
87
|
+
const unsubscribe = bus.subscribe((event) => {
|
|
88
|
+
if (captured)
|
|
89
|
+
return;
|
|
90
|
+
// Count prior judge denials in the ring buffer (excludes the live event —
|
|
91
|
+
// it is not yet buffered when a synchronous subscriber runs, but be safe).
|
|
92
|
+
const ringDenials = bus.recent().filter(isJudgeDenial).length;
|
|
93
|
+
const trigger = classifyIncidentTrigger(event, ringDenials);
|
|
94
|
+
if (trigger === undefined)
|
|
95
|
+
return;
|
|
96
|
+
captured = true;
|
|
97
|
+
try {
|
|
98
|
+
const ts = now().toISOString();
|
|
99
|
+
const dir = join(incidentsDir, incidentDirName(ts, trigger.kind));
|
|
100
|
+
mkdirSync(dir, { recursive: true });
|
|
101
|
+
const ring = bus.recent();
|
|
102
|
+
const capture = {
|
|
103
|
+
version: 1,
|
|
104
|
+
kind: trigger.kind,
|
|
105
|
+
reason: trigger.reason,
|
|
106
|
+
triggeredBy: trigger.triggeredBy,
|
|
107
|
+
sessionId: runContext.sessionId,
|
|
108
|
+
runId: runContext.runId,
|
|
109
|
+
incidentTs: ts,
|
|
110
|
+
spec: options.spec ?? null,
|
|
111
|
+
ringEventCount: ring.length,
|
|
112
|
+
};
|
|
113
|
+
writeFileSync(join(dir, "incident.json"), `${JSON.stringify(capture, null, 2)}\n`);
|
|
114
|
+
writeFileSync(join(dir, "events.jsonl"), ring.length === 0 ? "" : `${ring.map((e) => JSON.stringify(e)).join("\n")}\n`);
|
|
115
|
+
process.stderr.write(`[incident] ${trigger.kind} captured → ${dir} (run \`crewhaus incident collect --session ${runContext.sessionId}\` for the full bundle)\n`);
|
|
116
|
+
}
|
|
117
|
+
catch (err) {
|
|
118
|
+
// Best-effort: an incident writer must not turn one failure into two.
|
|
119
|
+
runContext.logger.error("incident.capture_failed", {
|
|
120
|
+
message: err instanceof Error ? err.message : String(err),
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
});
|
|
124
|
+
return { unsubscribe };
|
|
125
|
+
}
|