crewhaus 0.1.7 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. package/README.md +10 -3
  2. package/dist/advice-apply.d.ts +182 -0
  3. package/dist/advice-apply.js +286 -0
  4. package/dist/advise-rules.d.ts +348 -0
  5. package/dist/advise-rules.js +905 -0
  6. package/dist/alert-sink.d.ts +48 -0
  7. package/dist/alert-sink.js +86 -0
  8. package/dist/approval-gate.d.ts +127 -0
  9. package/dist/approval-gate.js +254 -0
  10. package/dist/audit-verify.d.ts +69 -0
  11. package/dist/audit-verify.js +97 -0
  12. package/dist/autodistill.d.ts +113 -0
  13. package/dist/autodistill.js +256 -0
  14. package/dist/channel-provision.d.ts +360 -0
  15. package/dist/channel-provision.js +881 -0
  16. package/dist/ci-scaffold.d.ts +31 -0
  17. package/dist/ci-scaffold.js +343 -0
  18. package/dist/compile-check.d.ts +90 -0
  19. package/dist/compile-check.js +285 -0
  20. package/dist/compliance-schedule.d.ts +35 -0
  21. package/dist/compliance-schedule.js +36 -0
  22. package/dist/context-pressure.d.ts +80 -0
  23. package/dist/context-pressure.js +166 -0
  24. package/dist/dataset-mine.d.ts +172 -0
  25. package/dist/dataset-mine.js +403 -0
  26. package/dist/datasets.d.ts +124 -0
  27. package/dist/datasets.js +260 -0
  28. package/dist/deploy-canary.d.ts +83 -0
  29. package/dist/deploy-canary.js +87 -0
  30. package/dist/doctor-checks.d.ts +33 -0
  31. package/dist/doctor-checks.js +92 -0
  32. package/dist/doctor-detect.d.ts +108 -0
  33. package/dist/doctor-detect.js +214 -0
  34. package/dist/doctor-fix.d.ts +81 -0
  35. package/dist/doctor-fix.js +164 -0
  36. package/dist/egress-triage.d.ts +121 -0
  37. package/dist/egress-triage.js +261 -0
  38. package/dist/eval-bridge.d.ts +114 -0
  39. package/dist/eval-bridge.js +158 -0
  40. package/dist/eval-coverage.d.ts +140 -0
  41. package/dist/eval-coverage.js +428 -0
  42. package/dist/eval-history.d.ts +48 -0
  43. package/dist/eval-history.js +157 -0
  44. package/dist/eval-matrix.d.ts +80 -0
  45. package/dist/eval-matrix.js +182 -0
  46. package/dist/eval-sentinel.d.ts +65 -0
  47. package/dist/eval-sentinel.js +132 -0
  48. package/dist/faq.d.ts +68 -0
  49. package/dist/faq.js +168 -0
  50. package/dist/feedback.d.ts +246 -0
  51. package/dist/feedback.js +612 -0
  52. package/dist/fewshot.d.ts +83 -0
  53. package/dist/fewshot.js +158 -0
  54. package/dist/fleet.d.ts +207 -0
  55. package/dist/fleet.js +488 -0
  56. package/dist/flywheel.d.ts +193 -0
  57. package/dist/flywheel.js +519 -0
  58. package/dist/graders-suggest.d.ts +186 -0
  59. package/dist/graders-suggest.js +658 -0
  60. package/dist/incident.d.ts +99 -0
  61. package/dist/incident.js +217 -0
  62. package/dist/index.d.ts +9 -1
  63. package/dist/index.js +11727 -740
  64. package/dist/init-interactive.d.ts +105 -0
  65. package/dist/init-interactive.js +208 -0
  66. package/dist/intents.d.ts +105 -0
  67. package/dist/intents.js +292 -0
  68. package/dist/judge-calibrate.d.ts +137 -0
  69. package/dist/judge-calibrate.js +247 -0
  70. package/dist/justification-calibrate.d.ts +150 -0
  71. package/dist/justification-calibrate.js +262 -0
  72. package/dist/justification-gate.d.ts +27 -6
  73. package/dist/justification-gate.js +30 -6
  74. package/dist/knowledge-sync.d.ts +179 -0
  75. package/dist/knowledge-sync.js +551 -0
  76. package/dist/lessons.d.ts +87 -0
  77. package/dist/lessons.js +207 -0
  78. package/dist/lint.d.ts +127 -0
  79. package/dist/lint.js +226 -0
  80. package/dist/loadtest.d.ts +114 -0
  81. package/dist/loadtest.js +196 -0
  82. package/dist/marketplace-cli.d.ts +110 -0
  83. package/dist/marketplace-cli.js +250 -0
  84. package/dist/mcp-doctor.d.ts +121 -0
  85. package/dist/mcp-doctor.js +249 -0
  86. package/dist/model-scan.d.ts +116 -0
  87. package/dist/model-scan.js +226 -0
  88. package/dist/onchain-tune.d.ts +164 -0
  89. package/dist/onchain-tune.js +346 -0
  90. package/dist/permissions-suggest.d.ts +126 -0
  91. package/dist/permissions-suggest.js +333 -0
  92. package/dist/pii-tune.d.ts +107 -0
  93. package/dist/pii-tune.js +122 -0
  94. package/dist/propose.d.ts +117 -0
  95. package/dist/propose.js +184 -0
  96. package/dist/refresh-goldens.d.ts +82 -0
  97. package/dist/refresh-goldens.js +221 -0
  98. package/dist/regression-pin.d.ts +160 -0
  99. package/dist/regression-pin.js +281 -0
  100. package/dist/retention.d.ts +193 -0
  101. package/dist/retention.js +607 -0
  102. package/dist/retire.d.ts +118 -0
  103. package/dist/retire.js +291 -0
  104. package/dist/right-size.d.ts +100 -0
  105. package/dist/right-size.js +123 -0
  106. package/dist/scaffold-evals.d.ts +138 -0
  107. package/dist/scaffold-evals.js +410 -0
  108. package/dist/scope-audit-drift.d.ts +139 -0
  109. package/dist/scope-audit-drift.js +260 -0
  110. package/dist/security-corpus.d.ts +237 -0
  111. package/dist/security-corpus.js +516 -0
  112. package/dist/security-digest.d.ts +173 -0
  113. package/dist/security-digest.js +650 -0
  114. package/dist/sessions-index.d.ts +27 -0
  115. package/dist/sessions-index.js +51 -0
  116. package/dist/slo-doctor.d.ts +67 -0
  117. package/dist/slo-doctor.js +119 -0
  118. package/dist/slo-sink.d.ts +96 -0
  119. package/dist/slo-sink.js +107 -0
  120. package/dist/spec-changelog.d.ts +102 -0
  121. package/dist/spec-changelog.js +237 -0
  122. package/dist/state-backup.d.ts +223 -0
  123. package/dist/state-backup.js +648 -0
  124. package/dist/tools-cli.d.ts +170 -0
  125. package/dist/tools-cli.js +298 -0
  126. package/dist/triage.d.ts +202 -0
  127. package/dist/triage.js +403 -0
  128. package/dist/upgrade.d.ts +57 -0
  129. package/dist/upgrade.js +113 -0
  130. package/dist/version.d.ts +6 -0
  131. package/dist/version.js +27 -0
  132. package/dist/voice-eval.d.ts +138 -0
  133. package/dist/voice-eval.js +309 -0
  134. package/dist/watch.d.ts +58 -0
  135. package/dist/watch.js +97 -0
  136. package/package.json +89 -64
@@ -0,0 +1,138 @@
1
+ /**
2
+ * Item 65 — `crewhaus eval --voice`: replay recorded voice sessions through
3
+ * deterministic graders measuring latency, barge-in correctness, and transcript
4
+ * quality — WITHOUT live audio, a microphone, or provider credentials.
5
+ *
6
+ * Live realtime audio is out of scope for the eval loop (non-deterministic,
7
+ * credential-bound). Instead the replay reads persisted call-session logs — a
8
+ * JSONL of timestamped voice events, the same discriminated union
9
+ * `voice-runtime` emits (`session_created`, `transcript_partial/final`,
10
+ * `audio_chunk`, `tool_use`, `interrupt`, `disconnect`) plus the barge-in
11
+ * signal the `barge-in-controller` fires, each stamped with a `ts` (epoch ms)
12
+ * and, for user turns, a `role`. From that timeline the graders derive:
13
+ *
14
+ * - LATENCY: time-to-first-token (first user utterance end → first
15
+ * agent `transcript_partial`/`audio_chunk`), and per-turn
16
+ * response latency. Graded against declared budgets.
17
+ * - BARGE-IN: every `barge_in` signal must be followed by an agent
18
+ * `interrupt` (the agent yielded) within a window — a missed
19
+ * yield is a correctness failure.
20
+ * - TRANSCRIPT: each user turn should produce a non-empty agent transcript
21
+ * (ASR/response coverage) — an empty/absent response is a
22
+ * transcript-quality failure.
23
+ *
24
+ * Everything here is side-effect-free (the CLI entry file reads the replay
25
+ * JSONL files + writes the report). Deterministic: identical input timeline →
26
+ * identical metrics, so a fixture replays byte-stably in CI.
27
+ */
28
+ /** Thrown on a malformed replay log / bad flags. The CLI routes it through
29
+ * `die()`; tests assert on `.message`. */
30
+ export declare class VoiceEvalError extends Error {
31
+ readonly name = "VoiceEvalError";
32
+ }
33
+ /** One parsed replay-log line. `ts` is epoch ms; `role` marks user vs agent
34
+ * events (agent by default). Shape mirrors `voice-runtime`'s RealtimeEvent
35
+ * plus the `barge_in` controller signal. */
36
+ export type ReplayEvent = {
37
+ readonly ts: number;
38
+ readonly kind: "session_created" | "transcript_partial" | "transcript_final" | "audio_chunk" | "tool_use" | "interrupt" | "barge_in" | "disconnect" | "error";
39
+ readonly role?: "user" | "agent";
40
+ readonly text?: string;
41
+ /** For barge_in: how many consecutive speech frames triggered it. */
42
+ readonly speechFrames?: number;
43
+ };
44
+ /** Per-call thresholds; a voice spec's `voice.eval` block would carry these.
45
+ * Defaults are conservative realtime targets. */
46
+ export type VoiceThresholds = {
47
+ /** Max acceptable time-to-first-token (ms). Default 1200. */
48
+ readonly maxTtftMs: number;
49
+ /** Max acceptable per-turn response latency (ms). Default 2500. */
50
+ readonly maxTurnLatencyMs: number;
51
+ /** Max acceptable delay between a barge-in and the agent yielding (ms).
52
+ * Default 500. */
53
+ readonly maxBargeInYieldMs: number;
54
+ };
55
+ export declare const DEFAULT_VOICE_THRESHOLDS: VoiceThresholds;
56
+ /** A parsed replay session — the events sorted by ts + a stable id. */
57
+ export type ReplaySession = {
58
+ readonly sessionId: string;
59
+ readonly events: ReadonlyArray<ReplayEvent>;
60
+ };
61
+ /**
62
+ * Parse a replay-log JSONL blob into a sorted ReplaySession. Skips blank lines;
63
+ * throws `VoiceEvalError` on a line that parses but is not a well-formed event
64
+ * (bad kind / non-numeric ts) so a corrupt fixture is loud, not silently empty.
65
+ */
66
+ export declare function parseReplayLog(sessionId: string, jsonl: string): ReplaySession;
67
+ /** A per-turn latency record: the user utterance end → the agent's first
68
+ * response token, and to the agent's final transcript. */
69
+ export type TurnLatency = {
70
+ readonly turnIndex: number;
71
+ /** Time-to-first-token (ms): user turn end → first agent partial/audio. */
72
+ readonly ttftMs: number | undefined;
73
+ /** Full turn latency (ms): user turn end → agent transcript_final. */
74
+ readonly turnLatencyMs: number | undefined;
75
+ };
76
+ /** A barge-in outcome: whether the agent yielded (emitted `interrupt`) and how
77
+ * long it took. */
78
+ export type BargeInOutcome = {
79
+ readonly atMs: number;
80
+ readonly yielded: boolean;
81
+ readonly yieldLatencyMs: number | undefined;
82
+ };
83
+ /** The metrics distilled from one replay session. */
84
+ export type VoiceMetrics = {
85
+ readonly sessionId: string;
86
+ readonly turns: ReadonlyArray<TurnLatency>;
87
+ readonly bargeIns: ReadonlyArray<BargeInOutcome>;
88
+ /** Count of user turns that produced a non-empty agent transcript. */
89
+ readonly answeredTurns: number;
90
+ readonly totalUserTurns: number;
91
+ };
92
+ /**
93
+ * Distill latency / barge-in / transcript-coverage metrics from a replay
94
+ * session. A "user turn" opens on a user `transcript_final` (utterance end) and
95
+ * closes at the next user `transcript_final` (or session end). Deterministic.
96
+ */
97
+ export declare function computeVoiceMetrics(session: ReplaySession): VoiceMetrics;
98
+ /** A grade for one dimension of one session. Mirrors eval-grader's GradeResult. */
99
+ export type VoiceGrade = {
100
+ readonly passed: boolean;
101
+ readonly score: number;
102
+ readonly rationale: string;
103
+ };
104
+ /** LATENCY grader: every turn's TTFT ≤ maxTtftMs AND response ≤ maxTurnLatencyMs. */
105
+ export declare function gradeLatency(metrics: VoiceMetrics, thresholds: VoiceThresholds): VoiceGrade;
106
+ /** BARGE-IN grader: every barge-in must be followed by an agent yield within
107
+ * maxBargeInYieldMs. A session with no barge-ins passes vacuously. */
108
+ export declare function gradeBargeIn(metrics: VoiceMetrics, thresholds: VoiceThresholds): VoiceGrade;
109
+ /** TRANSCRIPT grader: every user turn should produce a non-empty agent
110
+ * response transcript. Coverage = answered / total. */
111
+ export declare function gradeTranscript(metrics: VoiceMetrics): VoiceGrade;
112
+ /** The three voice graders, by name — the registry-style pack. */
113
+ export declare const VOICE_GRADER_NAMES: readonly ["voice_latency", "voice_barge_in", "voice_transcript"];
114
+ /** A per-session voice eval result — the three grades + the raw metrics. */
115
+ export type VoiceSessionResult = {
116
+ readonly sessionId: string;
117
+ readonly metrics: VoiceMetrics;
118
+ readonly grades: {
119
+ readonly voice_latency: VoiceGrade;
120
+ readonly voice_barge_in: VoiceGrade;
121
+ readonly voice_transcript: VoiceGrade;
122
+ };
123
+ readonly passed: boolean;
124
+ };
125
+ /** Grade one replay session across all three dimensions. */
126
+ export declare function gradeVoiceSession(session: ReplaySession, thresholds?: VoiceThresholds): VoiceSessionResult;
127
+ /** Aggregate summary across a replay set. */
128
+ export type VoiceEvalSummary = {
129
+ readonly sessions: ReadonlyArray<VoiceSessionResult>;
130
+ readonly passRate: number;
131
+ readonly p50TtftMs: number | undefined;
132
+ readonly p95TtftMs: number | undefined;
133
+ readonly bargeInYieldRate: number;
134
+ };
135
+ /** Aggregate per-session results into a run summary. */
136
+ export declare function aggregateVoiceEval(results: ReadonlyArray<VoiceSessionResult>): VoiceEvalSummary;
137
+ /** Render a plain-text report of a voice eval run. */
138
+ export declare function renderVoiceReport(summary: VoiceEvalSummary): string;
@@ -0,0 +1,309 @@
1
+ /**
2
+ * Item 65 — `crewhaus eval --voice`: replay recorded voice sessions through
3
+ * deterministic graders measuring latency, barge-in correctness, and transcript
4
+ * quality — WITHOUT live audio, a microphone, or provider credentials.
5
+ *
6
+ * Live realtime audio is out of scope for the eval loop (non-deterministic,
7
+ * credential-bound). Instead the replay reads persisted call-session logs — a
8
+ * JSONL of timestamped voice events, the same discriminated union
9
+ * `voice-runtime` emits (`session_created`, `transcript_partial/final`,
10
+ * `audio_chunk`, `tool_use`, `interrupt`, `disconnect`) plus the barge-in
11
+ * signal the `barge-in-controller` fires, each stamped with a `ts` (epoch ms)
12
+ * and, for user turns, a `role`. From that timeline the graders derive:
13
+ *
14
+ * - LATENCY: time-to-first-token (first user utterance end → first
15
+ * agent `transcript_partial`/`audio_chunk`), and per-turn
16
+ * response latency. Graded against declared budgets.
17
+ * - BARGE-IN: every `barge_in` signal must be followed by an agent
18
+ * `interrupt` (the agent yielded) within a window — a missed
19
+ * yield is a correctness failure.
20
+ * - TRANSCRIPT: each user turn should produce a non-empty agent transcript
21
+ * (ASR/response coverage) — an empty/absent response is a
22
+ * transcript-quality failure.
23
+ *
24
+ * Everything here is side-effect-free (the CLI entry file reads the replay
25
+ * JSONL files + writes the report). Deterministic: identical input timeline →
26
+ * identical metrics, so a fixture replays byte-stably in CI.
27
+ */
28
+ /** Thrown on a malformed replay log / bad flags. The CLI routes it through
29
+ * `die()`; tests assert on `.message`. */
30
+ export class VoiceEvalError extends Error {
31
+ name = "VoiceEvalError";
32
+ }
33
+ export const DEFAULT_VOICE_THRESHOLDS = {
34
+ maxTtftMs: 1200,
35
+ maxTurnLatencyMs: 2500,
36
+ maxBargeInYieldMs: 500,
37
+ };
38
+ const VALID_KINDS = new Set([
39
+ "session_created",
40
+ "transcript_partial",
41
+ "transcript_final",
42
+ "audio_chunk",
43
+ "tool_use",
44
+ "interrupt",
45
+ "barge_in",
46
+ "disconnect",
47
+ "error",
48
+ ]);
49
+ /**
50
+ * Parse a replay-log JSONL blob into a sorted ReplaySession. Skips blank lines;
51
+ * throws `VoiceEvalError` on a line that parses but is not a well-formed event
52
+ * (bad kind / non-numeric ts) so a corrupt fixture is loud, not silently empty.
53
+ */
54
+ export function parseReplayLog(sessionId, jsonl) {
55
+ const events = [];
56
+ const lines = jsonl.split("\n");
57
+ for (let i = 0; i < lines.length; i++) {
58
+ const line = lines[i];
59
+ if (line === undefined || line.trim() === "")
60
+ continue;
61
+ let obj;
62
+ try {
63
+ obj = JSON.parse(line);
64
+ }
65
+ catch {
66
+ throw new VoiceEvalError(`replay ${sessionId}: line ${i + 1} is not valid JSON`);
67
+ }
68
+ if (obj === null || typeof obj !== "object") {
69
+ throw new VoiceEvalError(`replay ${sessionId}: line ${i + 1} is not an event object`);
70
+ }
71
+ const rec = obj;
72
+ // Accept the smoke-daemon envelope `{ kind: "voice_event", event: {...} }`
73
+ // by unwrapping, and the flat replay shape alike.
74
+ const inner = rec["kind"] === "voice_event" && typeof rec["event"] === "object" && rec["event"] !== null
75
+ ? { ...rec["event"], ts: rec["ts"] ?? undefined }
76
+ : rec;
77
+ const kind = inner["kind"];
78
+ if (typeof kind !== "string" || !VALID_KINDS.has(kind)) {
79
+ throw new VoiceEvalError(`replay ${sessionId}: line ${i + 1} has unknown event kind ${JSON.stringify(kind)}`);
80
+ }
81
+ const ts = inner["ts"];
82
+ if (typeof ts !== "number" || !Number.isFinite(ts)) {
83
+ throw new VoiceEvalError(`replay ${sessionId}: line ${i + 1} (${kind}) is missing a numeric ts (epoch ms)`);
84
+ }
85
+ const role = inner["role"];
86
+ const text = inner["text"];
87
+ const speechFrames = inner["speechFrames"];
88
+ events.push({
89
+ ts,
90
+ kind: kind,
91
+ ...(role === "user" || role === "agent" ? { role } : {}),
92
+ ...(typeof text === "string" ? { text } : {}),
93
+ ...(typeof speechFrames === "number" ? { speechFrames } : {}),
94
+ });
95
+ }
96
+ // Stable sort by ts (insertion order breaks ties — Array.sort is stable in
97
+ // modern engines, and we compare only ts so equal-ts events keep log order).
98
+ const sorted = [...events].sort((a, b) => a.ts - b.ts);
99
+ return { sessionId, events: sorted };
100
+ }
101
+ /**
102
+ * Distill latency / barge-in / transcript-coverage metrics from a replay
103
+ * session. A "user turn" opens on a user `transcript_final` (utterance end) and
104
+ * closes at the next user `transcript_final` (or session end). Deterministic.
105
+ */
106
+ export function computeVoiceMetrics(session) {
107
+ const turns = [];
108
+ const bargeIns = [];
109
+ let answeredTurns = 0;
110
+ let totalUserTurns = 0;
111
+ // Index user-turn boundaries.
112
+ const userTurnEnds = [];
113
+ for (const ev of session.events) {
114
+ if (ev.kind === "transcript_final" && ev.role === "user")
115
+ userTurnEnds.push(ev.ts);
116
+ }
117
+ for (let t = 0; t < userTurnEnds.length; t++) {
118
+ totalUserTurns += 1;
119
+ const start = userTurnEnds[t];
120
+ const nextStart = t + 1 < userTurnEnds.length ? userTurnEnds[t + 1] : Number.POSITIVE_INFINITY;
121
+ let ttftMs;
122
+ let turnLatencyMs;
123
+ let answered = false;
124
+ for (const ev of session.events) {
125
+ if (ev.ts <= start || ev.ts >= nextStart)
126
+ continue;
127
+ const isAgent = ev.role !== "user";
128
+ if (!isAgent)
129
+ continue;
130
+ if (ttftMs === undefined && (ev.kind === "transcript_partial" || ev.kind === "audio_chunk")) {
131
+ ttftMs = ev.ts - start;
132
+ }
133
+ if (ev.kind === "transcript_final") {
134
+ turnLatencyMs = ev.ts - start;
135
+ if (typeof ev.text === "string" && ev.text.trim() !== "")
136
+ answered = true;
137
+ }
138
+ }
139
+ if (answered)
140
+ answeredTurns += 1;
141
+ turns.push({ turnIndex: t, ttftMs, turnLatencyMs });
142
+ }
143
+ // Barge-in outcomes: each barge_in must be followed by an agent `interrupt`.
144
+ for (const ev of session.events) {
145
+ if (ev.kind !== "barge_in")
146
+ continue;
147
+ let yieldLatencyMs;
148
+ for (const other of session.events) {
149
+ if (other.ts < ev.ts)
150
+ continue;
151
+ if (other.kind === "interrupt") {
152
+ yieldLatencyMs = other.ts - ev.ts;
153
+ break;
154
+ }
155
+ }
156
+ bargeIns.push({
157
+ atMs: ev.ts,
158
+ yielded: yieldLatencyMs !== undefined,
159
+ yieldLatencyMs,
160
+ });
161
+ }
162
+ return { sessionId: session.sessionId, turns, bargeIns, answeredTurns, totalUserTurns };
163
+ }
164
+ /** LATENCY grader: every turn's TTFT ≤ maxTtftMs AND response ≤ maxTurnLatencyMs. */
165
+ export function gradeLatency(metrics, thresholds) {
166
+ if (metrics.turns.length === 0) {
167
+ return { passed: false, score: 0, rationale: "no user turns to measure latency" };
168
+ }
169
+ const violations = [];
170
+ let ok = 0;
171
+ for (const turn of metrics.turns) {
172
+ const ttftOk = turn.ttftMs !== undefined && turn.ttftMs <= thresholds.maxTtftMs;
173
+ const latOk = turn.turnLatencyMs !== undefined && turn.turnLatencyMs <= thresholds.maxTurnLatencyMs;
174
+ if (ttftOk && latOk) {
175
+ ok += 1;
176
+ }
177
+ else if (turn.ttftMs === undefined) {
178
+ violations.push(`turn ${turn.turnIndex}: no agent response (no TTFT)`);
179
+ }
180
+ else if (!ttftOk) {
181
+ violations.push(`turn ${turn.turnIndex}: TTFT ${turn.ttftMs}ms > ${thresholds.maxTtftMs}ms`);
182
+ }
183
+ else if (turn.turnLatencyMs === undefined) {
184
+ violations.push(`turn ${turn.turnIndex}: no final agent transcript (turn never completed)`);
185
+ }
186
+ else {
187
+ violations.push(`turn ${turn.turnIndex}: latency ${turn.turnLatencyMs}ms > ${thresholds.maxTurnLatencyMs}ms`);
188
+ }
189
+ }
190
+ const score = ok / metrics.turns.length;
191
+ return {
192
+ passed: violations.length === 0,
193
+ score,
194
+ rationale: violations.length === 0
195
+ ? `all ${metrics.turns.length} turns within latency budget`
196
+ : violations.join("; "),
197
+ };
198
+ }
199
+ /** BARGE-IN grader: every barge-in must be followed by an agent yield within
200
+ * maxBargeInYieldMs. A session with no barge-ins passes vacuously. */
201
+ export function gradeBargeIn(metrics, thresholds) {
202
+ if (metrics.bargeIns.length === 0) {
203
+ return { passed: true, score: 1, rationale: "no barge-ins in session (vacuously correct)" };
204
+ }
205
+ const violations = [];
206
+ let ok = 0;
207
+ for (const b of metrics.bargeIns) {
208
+ if (!b.yielded) {
209
+ violations.push(`barge-in @${b.atMs}ms: agent never yielded`);
210
+ }
211
+ else if (b.yieldLatencyMs > thresholds.maxBargeInYieldMs) {
212
+ violations.push(`barge-in @${b.atMs}ms: yielded after ${b.yieldLatencyMs}ms > ${thresholds.maxBargeInYieldMs}ms`);
213
+ }
214
+ else {
215
+ ok += 1;
216
+ }
217
+ }
218
+ return {
219
+ passed: violations.length === 0,
220
+ score: ok / metrics.bargeIns.length,
221
+ rationale: violations.length === 0
222
+ ? `all ${metrics.bargeIns.length} barge-ins yielded within ${thresholds.maxBargeInYieldMs}ms`
223
+ : violations.join("; "),
224
+ };
225
+ }
226
+ /** TRANSCRIPT grader: every user turn should produce a non-empty agent
227
+ * response transcript. Coverage = answered / total. */
228
+ export function gradeTranscript(metrics) {
229
+ if (metrics.totalUserTurns === 0) {
230
+ return { passed: false, score: 0, rationale: "no user turns in session" };
231
+ }
232
+ const coverage = metrics.answeredTurns / metrics.totalUserTurns;
233
+ return {
234
+ passed: metrics.answeredTurns === metrics.totalUserTurns,
235
+ score: coverage,
236
+ rationale: `${metrics.answeredTurns}/${metrics.totalUserTurns} user turns got a non-empty agent transcript`,
237
+ };
238
+ }
239
+ /** The three voice graders, by name — the registry-style pack. */
240
+ export const VOICE_GRADER_NAMES = ["voice_latency", "voice_barge_in", "voice_transcript"];
241
+ /** Grade one replay session across all three dimensions. */
242
+ export function gradeVoiceSession(session, thresholds = DEFAULT_VOICE_THRESHOLDS) {
243
+ const metrics = computeVoiceMetrics(session);
244
+ const latency = gradeLatency(metrics, thresholds);
245
+ const bargeIn = gradeBargeIn(metrics, thresholds);
246
+ const transcript = gradeTranscript(metrics);
247
+ return {
248
+ sessionId: session.sessionId,
249
+ metrics,
250
+ grades: { voice_latency: latency, voice_barge_in: bargeIn, voice_transcript: transcript },
251
+ passed: latency.passed && bargeIn.passed && transcript.passed,
252
+ };
253
+ }
254
+ function percentile(values, p) {
255
+ if (values.length === 0)
256
+ return undefined;
257
+ const sorted = [...values].sort((a, b) => a - b);
258
+ const idx = Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length));
259
+ return sorted[idx];
260
+ }
261
+ /** Aggregate per-session results into a run summary. */
262
+ export function aggregateVoiceEval(results) {
263
+ const passed = results.filter((r) => r.passed).length;
264
+ const ttfts = [];
265
+ let bargeInTotal = 0;
266
+ let bargeInYielded = 0;
267
+ for (const r of results) {
268
+ for (const turn of r.metrics.turns) {
269
+ if (turn.ttftMs !== undefined)
270
+ ttfts.push(turn.ttftMs);
271
+ }
272
+ for (const b of r.metrics.bargeIns) {
273
+ bargeInTotal += 1;
274
+ if (b.yielded)
275
+ bargeInYielded += 1;
276
+ }
277
+ }
278
+ return {
279
+ sessions: results,
280
+ passRate: results.length === 0 ? 0 : passed / results.length,
281
+ p50TtftMs: percentile(ttfts, 50),
282
+ p95TtftMs: percentile(ttfts, 95),
283
+ bargeInYieldRate: bargeInTotal === 0 ? 1 : bargeInYielded / bargeInTotal,
284
+ };
285
+ }
286
+ /** Render a plain-text report of a voice eval run. */
287
+ export function renderVoiceReport(summary) {
288
+ const lines = [];
289
+ lines.push(`voice eval: ${summary.sessions.length} session(s)`);
290
+ lines.push(` pass rate: ${(summary.passRate * 100).toFixed(0)}%`);
291
+ lines.push(` TTFT p50 / p95: ${fmtMs(summary.p50TtftMs)} / ${fmtMs(summary.p95TtftMs)}`);
292
+ lines.push(` barge-in yield rate: ${(summary.bargeInYieldRate * 100).toFixed(0)}%`);
293
+ for (const r of summary.sessions) {
294
+ lines.push(` ${r.passed ? "PASS" : "FAIL"} ${r.sessionId}`);
295
+ if (!r.grades.voice_latency.passed) {
296
+ lines.push(` latency: ${r.grades.voice_latency.rationale}`);
297
+ }
298
+ if (!r.grades.voice_barge_in.passed) {
299
+ lines.push(` barge-in: ${r.grades.voice_barge_in.rationale}`);
300
+ }
301
+ if (!r.grades.voice_transcript.passed) {
302
+ lines.push(` transcript: ${r.grades.voice_transcript.rationale}`);
303
+ }
304
+ }
305
+ return `${lines.join("\n")}\n`;
306
+ }
307
+ function fmtMs(ms) {
308
+ return ms === undefined ? "n/a" : `${ms}ms`;
309
+ }
@@ -0,0 +1,58 @@
1
+ /**
2
+ * Item 41 — `compile --watch` (and `run --watch`). Implements the watch mode
3
+ * the CLI header has listed as "future": watch `crewhaus.yaml` + the
4
+ * `.crewhaus/commands` and skills dirs, debounce bursts of fs events, and
5
+ * re-run a parse→lint→compile cycle on change, printing one green/red status
6
+ * per cycle. Ctrl-C-clean.
7
+ *
8
+ * The DEBOUNCE + cycle loop is factored out here as a pure controller driven by
9
+ * an INJECTED watcher + clock so it is unit-testable without real fs events or
10
+ * wall-clock waits. The CLI wrapper wires `node:fs.watch` and a real
11
+ * `setTimeout` behind the seams.
12
+ */
13
+ /** A watcher seam: register `onChange`, return a disposer. The CLI backs this
14
+ * with `fs.watch` over the spec + commands/skills dirs. */
15
+ export type Watcher = {
16
+ /** Subscribe to change events; every fs event calls `cb`. */
17
+ subscribe(cb: () => void): void;
18
+ /** Stop watching + release handles (called on Ctrl-C). */
19
+ close(): void;
20
+ };
21
+ /** A minimal timer seam so tests drive the debounce deterministically. */
22
+ export type TimerSeam = {
23
+ set(fn: () => void, ms: number): unknown;
24
+ clear(handle: unknown): void;
25
+ };
26
+ /** One cycle's result, printed as a status line. */
27
+ export type CycleOutcome = {
28
+ readonly green: boolean;
29
+ /** The status line to print (already colored/marked by the runner). */
30
+ readonly line: string;
31
+ };
32
+ export type WatchControllerOptions = {
33
+ readonly watcher: Watcher;
34
+ readonly timer: TimerSeam;
35
+ /** Debounce window: coalesce fs events arriving within this many ms. */
36
+ readonly debounceMs: number;
37
+ /** Run one parse→lint→compile cycle; returns the status to print. */
38
+ runCycle(): Promise<CycleOutcome>;
39
+ /** Sink for status lines (stdout in the CLI; a buffer in tests). */
40
+ print(line: string): void;
41
+ };
42
+ /**
43
+ * The debounced watch controller. A burst of fs events schedules exactly one
44
+ * cycle after `debounceMs` of quiet; events arriving mid-window reset the
45
+ * timer (classic trailing debounce). Cycles never overlap: an event during a
46
+ * running cycle marks the run "dirty" and re-triggers once it finishes, so the
47
+ * latest on-disk state always gets a final cycle.
48
+ *
49
+ * Returns a handle with `stop()` (used by the Ctrl-C handler) and, for tests,
50
+ * `pendingCycles()` so a test can await the controller settling.
51
+ */
52
+ export declare function createWatchController(opts: WatchControllerOptions): {
53
+ stop(): void;
54
+ /** Test seam: resolves once no cycle is running and none is queued. */
55
+ idle(): Promise<void>;
56
+ };
57
+ /** Compose a green/red status line for a cycle, timestamped. Pure formatter. */
58
+ export declare function formatCycleLine(green: boolean, detail: string, at?: Date): string;
package/dist/watch.js ADDED
@@ -0,0 +1,97 @@
1
+ /**
2
+ * Item 41 — `compile --watch` (and `run --watch`). Implements the watch mode
3
+ * the CLI header has listed as "future": watch `crewhaus.yaml` + the
4
+ * `.crewhaus/commands` and skills dirs, debounce bursts of fs events, and
5
+ * re-run a parse→lint→compile cycle on change, printing one green/red status
6
+ * per cycle. Ctrl-C-clean.
7
+ *
8
+ * The DEBOUNCE + cycle loop is factored out here as a pure controller driven by
9
+ * an INJECTED watcher + clock so it is unit-testable without real fs events or
10
+ * wall-clock waits. The CLI wrapper wires `node:fs.watch` and a real
11
+ * `setTimeout` behind the seams.
12
+ */
13
+ /**
14
+ * The debounced watch controller. A burst of fs events schedules exactly one
15
+ * cycle after `debounceMs` of quiet; events arriving mid-window reset the
16
+ * timer (classic trailing debounce). Cycles never overlap: an event during a
17
+ * running cycle marks the run "dirty" and re-triggers once it finishes, so the
18
+ * latest on-disk state always gets a final cycle.
19
+ *
20
+ * Returns a handle with `stop()` (used by the Ctrl-C handler) and, for tests,
21
+ * `pendingCycles()` so a test can await the controller settling.
22
+ */
23
+ export function createWatchController(opts) {
24
+ let timerHandle;
25
+ let running = false;
26
+ let dirtyWhileRunning = false;
27
+ let stopped = false;
28
+ const idleWaiters = [];
29
+ const settleIfIdle = () => {
30
+ if (!running && timerHandle === undefined && !dirtyWhileRunning) {
31
+ for (const w of idleWaiters.splice(0))
32
+ w();
33
+ }
34
+ };
35
+ const runOnce = async () => {
36
+ if (stopped)
37
+ return;
38
+ running = true;
39
+ try {
40
+ const outcome = await opts.runCycle();
41
+ opts.print(outcome.line);
42
+ }
43
+ catch (err) {
44
+ opts.print(`✗ watch cycle crashed: ${err.message}`);
45
+ }
46
+ finally {
47
+ running = false;
48
+ // An event landed mid-cycle — the printed result is already stale, so run
49
+ // one more cycle against the latest state.
50
+ if (dirtyWhileRunning && !stopped) {
51
+ dirtyWhileRunning = false;
52
+ void runOnce();
53
+ }
54
+ else {
55
+ settleIfIdle();
56
+ }
57
+ }
58
+ };
59
+ const onChange = () => {
60
+ if (stopped)
61
+ return;
62
+ if (running) {
63
+ // Don't interrupt a running cycle; remember to re-run after it finishes.
64
+ dirtyWhileRunning = true;
65
+ return;
66
+ }
67
+ // Trailing debounce: reset the timer on every event in the window.
68
+ if (timerHandle !== undefined)
69
+ opts.timer.clear(timerHandle);
70
+ timerHandle = opts.timer.set(() => {
71
+ timerHandle = undefined;
72
+ void runOnce();
73
+ }, opts.debounceMs);
74
+ };
75
+ opts.watcher.subscribe(onChange);
76
+ return {
77
+ stop: () => {
78
+ stopped = true;
79
+ if (timerHandle !== undefined) {
80
+ opts.timer.clear(timerHandle);
81
+ timerHandle = undefined;
82
+ }
83
+ opts.watcher.close();
84
+ settleIfIdle();
85
+ },
86
+ idle: () => new Promise((resolveIdle) => {
87
+ idleWaiters.push(resolveIdle);
88
+ settleIfIdle();
89
+ }),
90
+ };
91
+ }
92
+ /** Compose a green/red status line for a cycle, timestamped. Pure formatter. */
93
+ export function formatCycleLine(green, detail, at = new Date()) {
94
+ const ts = at.toTimeString().slice(0, 8);
95
+ const marker = green ? "✓" : "✗";
96
+ return `${marker} [${ts}] ${detail}`;
97
+ }