@crewhaus/runtime-core 0.5.8 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/alert-watchdog.d.ts +53 -0
- package/dist/alert-watchdog.js +88 -7
- package/dist/ephemeral.d.ts +6 -0
- package/dist/ephemeral.js +88 -0
- package/dist/index.d.ts +795 -21
- package/dist/index.js +2573 -208
- package/dist/model-latency.d.ts +43 -0
- package/dist/model-latency.js +61 -0
- package/dist/observability.d.ts +47 -2
- package/dist/observability.js +104 -3
- package/dist/pre-route.d.ts +149 -0
- package/dist/pre-route.js +271 -0
- package/dist/slo-monitor.d.ts +20 -0
- package/dist/slo-monitor.js +45 -1
- package/package.json +42 -40
package/dist/alert-watchdog.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { FLOOR_BLOCKED_ROUTE_REASON as ROUTER_FLOOR_BLOCKED_ROUTE_REASON } from "@crewhaus/model-router";
|
|
1
2
|
import type { TraceEvent } from "@crewhaus/trace-event-bus";
|
|
2
3
|
/** Default location of the per-session metrics history, relative to cwd. */
|
|
3
4
|
export declare const DEFAULT_METRICS_DIR: string;
|
|
@@ -20,6 +21,34 @@ export declare const BASELINE_WINDOW = 50;
|
|
|
20
21
|
* eyeballing the file by hand.
|
|
21
22
|
*/
|
|
22
23
|
export declare const MAX_METRICS_HISTORY_LINES: number;
|
|
24
|
+
/**
|
|
25
|
+
* 0.6.0 (design §7.10) — the `model_route.reason` the learned policy records
|
|
26
|
+
* when no non-floor arm is exploitable and it serves the floor arm instead.
|
|
27
|
+
* The watchdog, the SLO monitor and metrics-collector all key `floor_block_rate`
|
|
28
|
+
* on this string; the router publishes it verbatim and OWNS the constant
|
|
29
|
+
* (`@crewhaus/model-router`, PR 10) — re-exported here so every consumer
|
|
30
|
+
* imports one symbol instead of retyping the string.
|
|
31
|
+
*/
|
|
32
|
+
export declare const FLOOR_BLOCKED_ROUTE_REASON: typeof ROUTER_FLOOR_BLOCKED_ROUTE_REASON;
|
|
33
|
+
/**
|
|
34
|
+
* True for a `model_route` decision the quality floor forced onto the floor
|
|
35
|
+
* arm. The router's reason is the bare constant; runtime-core may append its
|
|
36
|
+
* own `; `-separated notes (an eligibility exclusion, `no-eligible-candidate`)
|
|
37
|
+
* after it, so the match accepts that suffix too.
|
|
38
|
+
*/
|
|
39
|
+
export declare function isFloorBlockedRoute(ev: {
|
|
40
|
+
readonly reason: string;
|
|
41
|
+
}): boolean;
|
|
42
|
+
/**
|
|
43
|
+
* 0.6.0 (design §8.4) — the events that count as ONE escalation, shared by the
|
|
44
|
+
* watchdog's `escalation_rate` and the SLO monitor's: a hybrid-strategy stage
|
|
45
|
+
* whose role is `escalation` counts when it STARTS (its `done`/`failed` twin is
|
|
46
|
+
* the same escalation), and a two-tier fast→default misroute recovery counts
|
|
47
|
+
* on its `model_tier_route`. `eval_graded.escalatedTo` is deliberately NOT a
|
|
48
|
+
* third source: under `on_fail: escalate` the strong re-run publishes its own
|
|
49
|
+
* escalation stage, and counting both would double every cascade escalation.
|
|
50
|
+
*/
|
|
51
|
+
export declare function isEscalationEvent(ev: TraceEvent): boolean;
|
|
23
52
|
/**
|
|
24
53
|
* The compact per-session snapshot persisted to `sessions.jsonl` and read back
|
|
25
54
|
* to derive baselines. Percentiles are computed from the per-session samples
|
|
@@ -52,6 +81,22 @@ export type SessionMetricsSnapshot = {
|
|
|
52
81
|
readonly egressBlocked: number;
|
|
53
82
|
/** Justification-gate / policy denials (permission_decision decision "deny"). */
|
|
54
83
|
readonly permissionDenials: number;
|
|
84
|
+
/** Escalations (see {@link isEscalationEvent}). */
|
|
85
|
+
readonly escalations?: number;
|
|
86
|
+
/** escalations / max(turns,1). */
|
|
87
|
+
readonly escalationRate?: number;
|
|
88
|
+
/** In-loop grades (`eval_graded`) plus judge-gate verdicts (`judge_verdict`). */
|
|
89
|
+
readonly judgeVerdicts?: number;
|
|
90
|
+
/** The failing subset of `judgeVerdicts`. */
|
|
91
|
+
readonly judgeFails?: number;
|
|
92
|
+
/** judgeFails / max(judgeVerdicts,1). */
|
|
93
|
+
readonly judgeFailRate?: number;
|
|
94
|
+
/** Pool routing decisions (`model_route`). */
|
|
95
|
+
readonly routeDecisions?: number;
|
|
96
|
+
/** Decisions the quality floor forced onto the floor arm ({@link isFloorBlockedRoute}). */
|
|
97
|
+
readonly floorBlocks?: number;
|
|
98
|
+
/** floorBlocks / max(routeDecisions,1). */
|
|
99
|
+
readonly floorBlockRate?: number;
|
|
55
100
|
};
|
|
56
101
|
/** A single threshold breach found by {@link detectBreaches}. */
|
|
57
102
|
export type Breach = {
|
|
@@ -69,6 +114,9 @@ export type DerivedThresholds = {
|
|
|
69
114
|
readonly pricingMissRate: number;
|
|
70
115
|
readonly circuitOpens: number;
|
|
71
116
|
readonly egressBlocked: number;
|
|
117
|
+
readonly escalationRate: number;
|
|
118
|
+
readonly judgeFailRate: number;
|
|
119
|
+
readonly floorBlockRate: number;
|
|
72
120
|
/** How many historical sessions the thresholds were derived from (0 = bootstrap). */
|
|
73
121
|
readonly baselineSessions: number;
|
|
74
122
|
};
|
|
@@ -89,6 +137,11 @@ export declare class SessionMetricsAccumulator {
|
|
|
89
137
|
private circuitOpens;
|
|
90
138
|
private egressBlocked;
|
|
91
139
|
private permissionDenials;
|
|
140
|
+
private escalations;
|
|
141
|
+
private judgeVerdicts;
|
|
142
|
+
private judgeFails;
|
|
143
|
+
private routeDecisions;
|
|
144
|
+
private floorBlocks;
|
|
92
145
|
private firstTs;
|
|
93
146
|
private lastTs;
|
|
94
147
|
private readonly requestStarts;
|
package/dist/alert-watchdog.js
CHANGED
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
*/
|
|
26
26
|
import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync, } from "node:fs";
|
|
27
27
|
import { dirname, join } from "node:path";
|
|
28
|
+
import { FLOOR_BLOCKED_ROUTE_REASON as ROUTER_FLOOR_BLOCKED_ROUTE_REASON } from "@crewhaus/model-router";
|
|
28
29
|
/** Default location of the per-session metrics history, relative to cwd. */
|
|
29
30
|
export const DEFAULT_METRICS_DIR = join(".crewhaus", "metrics");
|
|
30
31
|
export const METRICS_FILENAME = "sessions.jsonl";
|
|
@@ -46,6 +47,41 @@ export const BASELINE_WINDOW = 50;
|
|
|
46
47
|
* eyeballing the file by hand.
|
|
47
48
|
*/
|
|
48
49
|
export const MAX_METRICS_HISTORY_LINES = Math.max(BASELINE_WINDOW * 4, 200);
|
|
50
|
+
/**
|
|
51
|
+
* 0.6.0 (design §7.10) — the `model_route.reason` the learned policy records
|
|
52
|
+
* when no non-floor arm is exploitable and it serves the floor arm instead.
|
|
53
|
+
* The watchdog, the SLO monitor and metrics-collector all key `floor_block_rate`
|
|
54
|
+
* on this string; the router publishes it verbatim and OWNS the constant
|
|
55
|
+
* (`@crewhaus/model-router`, PR 10) — re-exported here so every consumer
|
|
56
|
+
* imports one symbol instead of retyping the string.
|
|
57
|
+
*/
|
|
58
|
+
export const FLOOR_BLOCKED_ROUTE_REASON = ROUTER_FLOOR_BLOCKED_ROUTE_REASON;
|
|
59
|
+
/**
|
|
60
|
+
* True for a `model_route` decision the quality floor forced onto the floor
|
|
61
|
+
* arm. The router's reason is the bare constant; runtime-core may append its
|
|
62
|
+
* own `; `-separated notes (an eligibility exclusion, `no-eligible-candidate`)
|
|
63
|
+
* after it, so the match accepts that suffix too.
|
|
64
|
+
*/
|
|
65
|
+
export function isFloorBlockedRoute(ev) {
|
|
66
|
+
return (ev.reason === FLOOR_BLOCKED_ROUTE_REASON ||
|
|
67
|
+
ev.reason.startsWith(`${FLOOR_BLOCKED_ROUTE_REASON}; `));
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* 0.6.0 (design §8.4) — the events that count as ONE escalation, shared by the
|
|
71
|
+
* watchdog's `escalation_rate` and the SLO monitor's: a hybrid-strategy stage
|
|
72
|
+
* whose role is `escalation` counts when it STARTS (its `done`/`failed` twin is
|
|
73
|
+
* the same escalation), and a two-tier fast→default misroute recovery counts
|
|
74
|
+
* on its `model_tier_route`. `eval_graded.escalatedTo` is deliberately NOT a
|
|
75
|
+
* third source: under `on_fail: escalate` the strong re-run publishes its own
|
|
76
|
+
* escalation stage, and counting both would double every cascade escalation.
|
|
77
|
+
*/
|
|
78
|
+
export function isEscalationEvent(ev) {
|
|
79
|
+
if (ev.kind === "model_stage")
|
|
80
|
+
return ev.role === "escalation" && ev.outcome === "started";
|
|
81
|
+
if (ev.kind === "model_tier_route")
|
|
82
|
+
return ev.escalated === true;
|
|
83
|
+
return false;
|
|
84
|
+
}
|
|
49
85
|
// --------- pure event folding ---------
|
|
50
86
|
/**
|
|
51
87
|
* Accumulates a session's metric signals from bus events. Mirrors
|
|
@@ -64,6 +100,11 @@ export class SessionMetricsAccumulator {
|
|
|
64
100
|
circuitOpens = 0;
|
|
65
101
|
egressBlocked = 0;
|
|
66
102
|
permissionDenials = 0;
|
|
103
|
+
escalations = 0;
|
|
104
|
+
judgeVerdicts = 0;
|
|
105
|
+
judgeFails = 0;
|
|
106
|
+
routeDecisions = 0;
|
|
107
|
+
floorBlocks = 0;
|
|
67
108
|
firstTs;
|
|
68
109
|
lastTs;
|
|
69
110
|
requestStarts = new Map();
|
|
@@ -132,14 +173,29 @@ export class SessionMetricsAccumulator {
|
|
|
132
173
|
else if (ev.outcome === undefined && ev.decision === "deny")
|
|
133
174
|
this.permissionDenials += 1;
|
|
134
175
|
return;
|
|
176
|
+
// ---- 0.6.0 (design §8.4) — hybrid-routing signals. ----
|
|
177
|
+
case "model_stage":
|
|
178
|
+
case "model_tier_route":
|
|
179
|
+
if (isEscalationEvent(ev))
|
|
180
|
+
this.escalations += 1;
|
|
181
|
+
return;
|
|
182
|
+
case "eval_graded":
|
|
183
|
+
case "judge_verdict":
|
|
184
|
+
// Every in-loop grade and judge-gate verdict is one judge verdict; the
|
|
185
|
+
// `fail` subset is the numerator of `judge_fail_rate`. (This closes the
|
|
186
|
+
// NEW-E-2 gap the old default branch documented: quality verdicts now
|
|
187
|
+
// do fold into the watchdog — as a FAIL RATE against the session's own
|
|
188
|
+
// history, not as a score threshold.) `response_rated` still stays out.
|
|
189
|
+
this.judgeVerdicts += 1;
|
|
190
|
+
if (ev.verdict === "fail")
|
|
191
|
+
this.judgeFails += 1;
|
|
192
|
+
return;
|
|
193
|
+
case "model_route":
|
|
194
|
+
this.routeDecisions += 1;
|
|
195
|
+
if (isFloorBlockedRoute(ev))
|
|
196
|
+
this.floorBlocks += 1;
|
|
197
|
+
return;
|
|
135
198
|
default:
|
|
136
|
-
// NEW-E-2 is PARTIAL by decision, not by omission: `eval_graded`,
|
|
137
|
-
// `judge_verdict` and `response_rated` fold into metrics-collector
|
|
138
|
-
// (crewhaus_eval_verdicts_total / _score / crewhaus_response_ratings_total)
|
|
139
|
-
// but deliberately NOT into this accumulator, so there is no
|
|
140
|
-
// "judge score dropped below baseline" breach yet. Adding one means
|
|
141
|
-
// new persisted snapshot fields + a `detectBreaches` rule; see the
|
|
142
|
-
// Wave 5 amendments in EVALS-CAMPAIGN-0.4.md before closing the gap.
|
|
143
199
|
return;
|
|
144
200
|
}
|
|
145
201
|
}
|
|
@@ -163,6 +219,14 @@ export class SessionMetricsAccumulator {
|
|
|
163
219
|
circuitOpens: this.circuitOpens,
|
|
164
220
|
egressBlocked: this.egressBlocked,
|
|
165
221
|
permissionDenials: this.permissionDenials,
|
|
222
|
+
escalations: this.escalations,
|
|
223
|
+
escalationRate: this.escalations / Math.max(this.turns, 1),
|
|
224
|
+
judgeVerdicts: this.judgeVerdicts,
|
|
225
|
+
judgeFails: this.judgeFails,
|
|
226
|
+
judgeFailRate: this.judgeFails / Math.max(this.judgeVerdicts, 1),
|
|
227
|
+
routeDecisions: this.routeDecisions,
|
|
228
|
+
floorBlocks: this.floorBlocks,
|
|
229
|
+
floorBlockRate: this.floorBlocks / Math.max(this.routeDecisions, 1),
|
|
166
230
|
};
|
|
167
231
|
}
|
|
168
232
|
}
|
|
@@ -201,6 +265,12 @@ export function deriveThresholds(history) {
|
|
|
201
265
|
pricingMissRate: 0.5,
|
|
202
266
|
circuitOpens: 1,
|
|
203
267
|
egressBlocked: 1,
|
|
268
|
+
// 0.6.0 — a cascade that escalates every other turn, a judge that fails
|
|
269
|
+
// half its grades, or a floor that blocks half the decisions is a
|
|
270
|
+
// hybrid setup that is not paying for itself; below that, normal.
|
|
271
|
+
escalationRate: 0.5,
|
|
272
|
+
judgeFailRate: 0.5,
|
|
273
|
+
floorBlockRate: 0.5,
|
|
204
274
|
baselineSessions: window.length,
|
|
205
275
|
};
|
|
206
276
|
}
|
|
@@ -213,6 +283,12 @@ export function deriveThresholds(history) {
|
|
|
213
283
|
pricingMissRate: Math.max(p95((s) => s.pricingMisses / Math.max(s.modelCalls, 1)), 0) * HEADROOM_FACTOR,
|
|
214
284
|
circuitOpens: Math.max(p95((s) => s.circuitOpens), 1),
|
|
215
285
|
egressBlocked: Math.max(p95((s) => s.egressBlocked), 1),
|
|
286
|
+
// 0.6.0 — rates over history with the same 5% floor `errorRate` has, so a
|
|
287
|
+
// baseline of zero escalations does not make the first one a breach on its
|
|
288
|
+
// own; an absent field (a 0.5.x snapshot line) reads as no activity.
|
|
289
|
+
escalationRate: Math.max(p95((s) => s.escalationRate ?? 0) * HEADROOM_FACTOR, 0.05),
|
|
290
|
+
judgeFailRate: Math.max(p95((s) => s.judgeFailRate ?? 0) * HEADROOM_FACTOR, 0.05),
|
|
291
|
+
floorBlockRate: Math.max(p95((s) => s.floorBlockRate ?? 0) * HEADROOM_FACTOR, 0.05),
|
|
216
292
|
baselineSessions: window.length,
|
|
217
293
|
};
|
|
218
294
|
}
|
|
@@ -237,6 +313,11 @@ export function detectBreaches(snapshot, thresholds) {
|
|
|
237
313
|
add("pricing_miss_rate", missRate, thresholds.pricingMissRate, "");
|
|
238
314
|
add("circuit_opens", snapshot.circuitOpens, thresholds.circuitOpens, "");
|
|
239
315
|
add("egress_blocked", snapshot.egressBlocked, thresholds.egressBlocked, "");
|
|
316
|
+
// 0.6.0 (design §8.4) — hybrid-routing rates. A session with no routing
|
|
317
|
+
// activity reports 0 (or an absent field) and can never breach.
|
|
318
|
+
add("escalation_rate", snapshot.escalationRate ?? 0, thresholds.escalationRate, "");
|
|
319
|
+
add("judge_fail_rate", snapshot.judgeFailRate ?? 0, thresholds.judgeFailRate, "");
|
|
320
|
+
add("floor_block_rate", snapshot.floorBlockRate ?? 0, thresholds.floorBlockRate, "");
|
|
240
321
|
return breaches;
|
|
241
322
|
}
|
|
242
323
|
function fmt(n) {
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { EventLog } from "@crewhaus/event-log";
|
|
2
|
+
import type { SessionStore } from "@crewhaus/session-store";
|
|
3
|
+
/** A `SessionStore` over a Map — the same id shape (`sess_<16 hex>`). */
|
|
4
|
+
export declare function createMemorySessionStore(now?: () => Date): SessionStore;
|
|
5
|
+
/** An `EventLog` over an array: appends are kept in order and replayable. */
|
|
6
|
+
export declare function createMemoryEventLog(now?: () => number): EventLog;
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 0.6.0 §16 Q6 (PR 9d) — the IN-MEMORY persistence twins behind
|
|
3
|
+
* `runChatLoop({ persistSession: false })`.
|
|
4
|
+
*
|
|
5
|
+
* A hybrid strategy runs nested single-turn side calls — consult, guide,
|
|
6
|
+
* shadow, committee members — each of which is a `runChatLoop` in its own
|
|
7
|
+
* right. Left to the defaults every one of them would create a session
|
|
8
|
+
* `.json` and an event-log `.jsonl` under the sessions dir (a Task child
|
|
9
|
+
* does exactly that), so a committee of three on a ten-step workflow would
|
|
10
|
+
* write thirty child sessions per run that no `--resume`, `sessions tail` or
|
|
11
|
+
* Hangar view ever reads: the side call's spend and stage transitions are
|
|
12
|
+
* re-published on the PARENT bus and persisted in the parent's log. These
|
|
13
|
+
* twins keep the loop's persistence contract (`create` / `update` /
|
|
14
|
+
* `append` / `close` all succeed, ids keep their shape) without touching the
|
|
15
|
+
* filesystem. `read()` replays what was appended so in-run readers behave.
|
|
16
|
+
*/
|
|
17
|
+
import { randomBytes } from "node:crypto";
|
|
18
|
+
/** A `SessionStore` over a Map — the same id shape (`sess_<16 hex>`). */
|
|
19
|
+
export function createMemorySessionStore(now = () => new Date()) {
|
|
20
|
+
const sessions = new Map();
|
|
21
|
+
return {
|
|
22
|
+
async create(opts) {
|
|
23
|
+
const id = opts.id ?? `sess_${randomBytes(8).toString("hex")}`;
|
|
24
|
+
const ts = now().toISOString();
|
|
25
|
+
const session = {
|
|
26
|
+
id,
|
|
27
|
+
createdAt: ts,
|
|
28
|
+
updatedAt: ts,
|
|
29
|
+
name: opts.name,
|
|
30
|
+
target: opts.target,
|
|
31
|
+
model: opts.model,
|
|
32
|
+
lastTurnIndex: 0,
|
|
33
|
+
};
|
|
34
|
+
sessions.set(id, session);
|
|
35
|
+
return { ...session };
|
|
36
|
+
},
|
|
37
|
+
async get(id) {
|
|
38
|
+
const s = sessions.get(id);
|
|
39
|
+
return s === undefined ? null : { ...s };
|
|
40
|
+
},
|
|
41
|
+
async list() {
|
|
42
|
+
return [...sessions.values()].map((s) => ({ ...s }));
|
|
43
|
+
},
|
|
44
|
+
async update(id, patch) {
|
|
45
|
+
const existing = sessions.get(id);
|
|
46
|
+
if (existing === undefined) {
|
|
47
|
+
throw new Error(`memory session store: unknown session "${id}"`);
|
|
48
|
+
}
|
|
49
|
+
const { pin, ...rest } = patch;
|
|
50
|
+
const { pin: existingPin, ...keep } = existing;
|
|
51
|
+
const nextPin = pin === null ? undefined : pin === undefined ? existingPin : pin;
|
|
52
|
+
const next = {
|
|
53
|
+
...keep,
|
|
54
|
+
...rest,
|
|
55
|
+
...(nextPin !== undefined ? { pin: nextPin } : {}),
|
|
56
|
+
updatedAt: now().toISOString(),
|
|
57
|
+
};
|
|
58
|
+
sessions.set(id, next);
|
|
59
|
+
return { ...next };
|
|
60
|
+
},
|
|
61
|
+
async delete(id) {
|
|
62
|
+
sessions.delete(id);
|
|
63
|
+
},
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
/** An `EventLog` over an array: appends are kept in order and replayable. */
|
|
67
|
+
export function createMemoryEventLog(now = () => Date.now()) {
|
|
68
|
+
const events = [];
|
|
69
|
+
let closed = false;
|
|
70
|
+
return {
|
|
71
|
+
async append(event) {
|
|
72
|
+
if (closed)
|
|
73
|
+
throw new Error("memory event log: append after close");
|
|
74
|
+
events.push({ ts: now(), version: 1, kind: event.kind, payload: event.payload });
|
|
75
|
+
},
|
|
76
|
+
read(opts = {}) {
|
|
77
|
+
const snapshot = events.filter((e) => (opts.since === undefined || e.ts >= opts.since) &&
|
|
78
|
+
(opts.until === undefined || e.ts <= opts.until));
|
|
79
|
+
return (async function* () {
|
|
80
|
+
for (const e of snapshot)
|
|
81
|
+
yield e;
|
|
82
|
+
})();
|
|
83
|
+
},
|
|
84
|
+
async close() {
|
|
85
|
+
closed = true;
|
|
86
|
+
},
|
|
87
|
+
};
|
|
88
|
+
}
|