@ouro.bot/cli 0.1.0-alpha.583 → 0.1.0-alpha.584
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/changelog.json
CHANGED
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_note": "This changelog is maintained as part of the PR/version-bump workflow. Agent-curated, not auto-generated. Agents read this file directly via read_file to understand what changed between versions.",
|
|
3
3
|
"versions": [
|
|
4
|
+
{
|
|
5
|
+
"version": "0.1.0-alpha.584",
|
|
6
|
+
"changes": [
|
|
7
|
+
"Voice now has a transport-aware Realtime eval kernel that grades deterministic call timelines for first-audio latency, user-turn response latency, tool holding phrases, barge-in clearing/truncation, friend context, transcript continuity, and hangup control before live phone testing.",
|
|
8
|
+
"`npm run voice:eval` runs built-in no-human voice scenarios, including a healthy path and an expected known-bad latency canary, and emits the normal nerves events expected of executable sense entrypoints so future Voice transport work can prove synchronous behavior without requiring a human to answer calls."
|
|
9
|
+
]
|
|
10
|
+
},
|
|
4
11
|
{
|
|
5
12
|
"version": "0.1.0-alpha.583",
|
|
6
13
|
"changes": [
|
|
@@ -0,0 +1,384 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.gradeVoiceRealtimeEvalTimeline = gradeVoiceRealtimeEvalTimeline;
|
|
4
|
+
exports.buildVoiceRealtimeEvalHappyPath = buildVoiceRealtimeEvalHappyPath;
|
|
5
|
+
exports.runBuiltInVoiceRealtimeEvalSuite = runBuiltInVoiceRealtimeEvalSuite;
|
|
6
|
+
exports.summarizeVoiceRealtimeEvalSuite = summarizeVoiceRealtimeEvalSuite;
|
|
7
|
+
const runtime_1 = require("../../nerves/runtime");
|
|
8
|
+
function validateTimeline(scenarioId, events, expectation) {
|
|
9
|
+
const normalizedScenarioId = scenarioId.trim();
|
|
10
|
+
if (!normalizedScenarioId)
|
|
11
|
+
throw new Error("voice eval scenario id is empty");
|
|
12
|
+
if (events.length === 0)
|
|
13
|
+
throw new Error("voice eval timeline is empty");
|
|
14
|
+
const budgets = [
|
|
15
|
+
expectation.maxFirstAssistantAudioMs,
|
|
16
|
+
expectation.maxUserTurnResponseMs,
|
|
17
|
+
expectation.maxToolPresenceMs,
|
|
18
|
+
expectation.maxBargeInClearMs,
|
|
19
|
+
expectation.maxBargeInTruncateMs,
|
|
20
|
+
];
|
|
21
|
+
if (budgets.some((budget) => !Number.isFinite(budget) || budget <= 0)) {
|
|
22
|
+
throw new Error("voice eval latency budgets must be positive");
|
|
23
|
+
}
|
|
24
|
+
return normalizedScenarioId;
|
|
25
|
+
}
|
|
26
|
+
function sortedEvents(events) {
|
|
27
|
+
return [...events].sort((left, right) => left.atMs - right.atMs);
|
|
28
|
+
}
|
|
29
|
+
function firstEvent(events, type) {
|
|
30
|
+
return events.find((event) => event.type === type);
|
|
31
|
+
}
|
|
32
|
+
function allEvents(events, type) {
|
|
33
|
+
return events.filter((event) => event.type === type);
|
|
34
|
+
}
|
|
35
|
+
function lowerText(value) {
|
|
36
|
+
return value?.toLowerCase() ?? "";
|
|
37
|
+
}
|
|
38
|
+
function pushFinding(findings, finding) {
|
|
39
|
+
findings.push(finding);
|
|
40
|
+
}
|
|
41
|
+
function gradeFirstAudio(events, expectation, findings) {
|
|
42
|
+
const connected = firstEvent(events, "call.connected");
|
|
43
|
+
const firstAudio = firstEvent(events, "assistant.audio.started");
|
|
44
|
+
if (!connected || !firstAudio) {
|
|
45
|
+
pushFinding(findings, {
|
|
46
|
+
code: "first_audio_missing",
|
|
47
|
+
severity: "fail",
|
|
48
|
+
message: "Voice call did not produce assistant audio after connect.",
|
|
49
|
+
source: connected?.source ?? firstAudio?.source,
|
|
50
|
+
atMs: connected?.atMs ?? firstAudio?.atMs,
|
|
51
|
+
});
|
|
52
|
+
return undefined;
|
|
53
|
+
}
|
|
54
|
+
const ttfaMs = firstAudio.atMs - connected.atMs;
|
|
55
|
+
if (ttfaMs > expectation.maxFirstAssistantAudioMs) {
|
|
56
|
+
pushFinding(findings, {
|
|
57
|
+
code: "first_audio_late",
|
|
58
|
+
severity: "fail",
|
|
59
|
+
message: `First assistant audio started after ${ttfaMs}ms, over the ${expectation.maxFirstAssistantAudioMs}ms budget.`,
|
|
60
|
+
source: firstAudio.source,
|
|
61
|
+
atMs: firstAudio.atMs,
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
return ttfaMs;
|
|
65
|
+
}
|
|
66
|
+
function gradeFirstUserResponse(events, expectation, findings) {
|
|
67
|
+
const userTranscript = firstEvent(events, "user.transcript.done");
|
|
68
|
+
if (!userTranscript)
|
|
69
|
+
return undefined;
|
|
70
|
+
const response = events.find((event) => event.type === "response.requested"
|
|
71
|
+
&& event.atMs >= userTranscript.atMs
|
|
72
|
+
&& (!userTranscript.correlationId || event.correlationId === userTranscript.correlationId));
|
|
73
|
+
if (!response) {
|
|
74
|
+
pushFinding(findings, {
|
|
75
|
+
code: "user_response_missing",
|
|
76
|
+
severity: "fail",
|
|
77
|
+
message: "No voice response was requested after the caller transcript completed.",
|
|
78
|
+
source: userTranscript.source,
|
|
79
|
+
atMs: userTranscript.atMs,
|
|
80
|
+
});
|
|
81
|
+
return undefined;
|
|
82
|
+
}
|
|
83
|
+
const latencyMs = response.atMs - userTranscript.atMs;
|
|
84
|
+
if (latencyMs > expectation.maxUserTurnResponseMs) {
|
|
85
|
+
pushFinding(findings, {
|
|
86
|
+
code: "user_response_late",
|
|
87
|
+
severity: "fail",
|
|
88
|
+
message: `Voice response was requested after ${latencyMs}ms, over the ${expectation.maxUserTurnResponseMs}ms budget.`,
|
|
89
|
+
source: response.source,
|
|
90
|
+
atMs: response.atMs,
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
return latencyMs;
|
|
94
|
+
}
|
|
95
|
+
function gradeToolPresence(events, expectation, findings) {
|
|
96
|
+
const toolCall = firstEvent(events, "tool.call.started");
|
|
97
|
+
if (!toolCall)
|
|
98
|
+
return undefined;
|
|
99
|
+
const holding = events.find((event) => event.type === "tool.holding.started"
|
|
100
|
+
&& event.atMs >= toolCall.atMs
|
|
101
|
+
&& (!toolCall.correlationId || event.correlationId === toolCall.correlationId));
|
|
102
|
+
if (!holding) {
|
|
103
|
+
pushFinding(findings, {
|
|
104
|
+
code: "tool_presence_missing",
|
|
105
|
+
severity: "fail",
|
|
106
|
+
message: "Tool call did not produce a short voice holding phrase.",
|
|
107
|
+
source: toolCall.source,
|
|
108
|
+
atMs: toolCall.atMs,
|
|
109
|
+
});
|
|
110
|
+
return undefined;
|
|
111
|
+
}
|
|
112
|
+
const latencyMs = holding.atMs - toolCall.atMs;
|
|
113
|
+
if (latencyMs > expectation.maxToolPresenceMs) {
|
|
114
|
+
pushFinding(findings, {
|
|
115
|
+
code: "tool_presence_late",
|
|
116
|
+
severity: "fail",
|
|
117
|
+
message: `Tool holding phrase started after ${latencyMs}ms, over the ${expectation.maxToolPresenceMs}ms budget.`,
|
|
118
|
+
source: holding.source,
|
|
119
|
+
atMs: holding.atMs,
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
return latencyMs;
|
|
123
|
+
}
|
|
124
|
+
function gradeBargeIn(events, expectation, findings) {
|
|
125
|
+
const bargeIn = firstEvent(events, "barge_in.detected");
|
|
126
|
+
if (!bargeIn)
|
|
127
|
+
return {};
|
|
128
|
+
const clear = events.find((event) => event.type === "transport.playback_cleared" && event.atMs >= bargeIn.atMs);
|
|
129
|
+
const truncate = events.find((event) => event.type === "response.truncated" && event.atMs >= bargeIn.atMs);
|
|
130
|
+
const metrics = {};
|
|
131
|
+
if (!clear) {
|
|
132
|
+
pushFinding(findings, {
|
|
133
|
+
code: "barge_in_clear_missing",
|
|
134
|
+
severity: "fail",
|
|
135
|
+
message: "Caller barge-in did not clear transport playback.",
|
|
136
|
+
source: bargeIn.source,
|
|
137
|
+
atMs: bargeIn.atMs,
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
else {
|
|
141
|
+
metrics.firstBargeInClearMs = clear.atMs - bargeIn.atMs;
|
|
142
|
+
if (metrics.firstBargeInClearMs > expectation.maxBargeInClearMs) {
|
|
143
|
+
pushFinding(findings, {
|
|
144
|
+
code: "barge_in_clear_late",
|
|
145
|
+
severity: "fail",
|
|
146
|
+
message: `Barge-in playback clear took ${metrics.firstBargeInClearMs}ms, over the ${expectation.maxBargeInClearMs}ms budget.`,
|
|
147
|
+
source: clear.source,
|
|
148
|
+
atMs: clear.atMs,
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
if (!truncate) {
|
|
153
|
+
pushFinding(findings, {
|
|
154
|
+
code: "barge_in_truncate_missing",
|
|
155
|
+
severity: "fail",
|
|
156
|
+
message: "Caller barge-in did not truncate the active Realtime response.",
|
|
157
|
+
source: bargeIn.source,
|
|
158
|
+
atMs: bargeIn.atMs,
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
else {
|
|
162
|
+
metrics.firstBargeInTruncateMs = truncate.atMs - bargeIn.atMs;
|
|
163
|
+
if (metrics.firstBargeInTruncateMs > expectation.maxBargeInTruncateMs) {
|
|
164
|
+
pushFinding(findings, {
|
|
165
|
+
code: "barge_in_truncate_late",
|
|
166
|
+
severity: "fail",
|
|
167
|
+
message: `Barge-in response truncation took ${metrics.firstBargeInTruncateMs}ms, over the ${expectation.maxBargeInTruncateMs}ms budget.`,
|
|
168
|
+
source: truncate.source,
|
|
169
|
+
atMs: truncate.atMs,
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return metrics;
|
|
174
|
+
}
|
|
175
|
+
function gradeManualFloorControl(events, findings) {
|
|
176
|
+
const session = allEvents(events, "session.updated").find((event) => event.session?.turnDetection);
|
|
177
|
+
if (session?.session?.turnDetection?.createResponse === false
|
|
178
|
+
&& session.session.turnDetection.interruptResponse === false) {
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
pushFinding(findings, {
|
|
182
|
+
code: "manual_floor_control_missing",
|
|
183
|
+
severity: "fail",
|
|
184
|
+
message: "Realtime session did not disable provider auto-response and provider interruption.",
|
|
185
|
+
source: session?.source,
|
|
186
|
+
atMs: session?.atMs,
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
function gradeFriendContext(events, requirement, findings) {
|
|
190
|
+
const context = firstEvent(events, "voice.context.injected");
|
|
191
|
+
if (context?.friendId === requirement.friendId
|
|
192
|
+
&& context.sessionKey === requirement.sessionKey
|
|
193
|
+
&& lowerText(context.text).includes(requirement.marker.toLowerCase())) {
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
pushFinding(findings, {
|
|
197
|
+
code: "friend_context_mismatch",
|
|
198
|
+
severity: "fail",
|
|
199
|
+
message: "Voice context did not preserve the expected friend identity, trust marker, and stable session key.",
|
|
200
|
+
source: context?.source,
|
|
201
|
+
atMs: context?.atMs,
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
function gradeTranscripts(events, requirements, findings) {
|
|
205
|
+
for (const requirement of requirements) {
|
|
206
|
+
const type = requirement.role === "assistant"
|
|
207
|
+
? "assistant.transcript.done"
|
|
208
|
+
: "user.transcript.done";
|
|
209
|
+
const found = allEvents(events, type).some((event) => lowerText(event.text).includes(requirement.contains.toLowerCase()));
|
|
210
|
+
if (!found) {
|
|
211
|
+
pushFinding(findings, {
|
|
212
|
+
code: "transcript_missing",
|
|
213
|
+
severity: "fail",
|
|
214
|
+
message: `Missing ${requirement.role} transcript containing "${requirement.contains}".`,
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
function gradeHangup(events, findings) {
|
|
220
|
+
const hangup = firstEvent(events, "call.hangup.requested");
|
|
221
|
+
if (hangup)
|
|
222
|
+
return;
|
|
223
|
+
const ended = firstEvent(events, "call.ended");
|
|
224
|
+
pushFinding(findings, {
|
|
225
|
+
code: "hangup_missing",
|
|
226
|
+
severity: "fail",
|
|
227
|
+
message: "Voice eval expected an agent-controlled hangup request before call end.",
|
|
228
|
+
source: ended?.source,
|
|
229
|
+
atMs: ended?.atMs,
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
function gradeOverlappingResponses(events, findings) {
|
|
233
|
+
for (const response of allEvents(events, "response.requested")) {
|
|
234
|
+
const activeAudio = allEvents(events, "assistant.audio.started").find((started) => {
|
|
235
|
+
const done = events.find((event) => event.type === "assistant.audio.done" && event.atMs >= started.atMs);
|
|
236
|
+
return response.atMs > started.atMs && (!done || response.atMs < done.atMs);
|
|
237
|
+
});
|
|
238
|
+
if (activeAudio) {
|
|
239
|
+
pushFinding(findings, {
|
|
240
|
+
code: "response_overlap",
|
|
241
|
+
severity: "fail",
|
|
242
|
+
message: "Voice response was requested while assistant audio was still active.",
|
|
243
|
+
source: response.source,
|
|
244
|
+
atMs: response.atMs,
|
|
245
|
+
});
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
function collectTransportSources(events) {
|
|
251
|
+
return [...new Set(events.flatMap((event) => event.source ? [event.source.transport] : []))].sort();
|
|
252
|
+
}
|
|
253
|
+
function gradeVoiceRealtimeEvalTimeline(scenarioId, timeline, expectation) {
|
|
254
|
+
const normalizedScenarioId = validateTimeline(scenarioId, timeline, expectation);
|
|
255
|
+
const events = sortedEvents(timeline);
|
|
256
|
+
(0, runtime_1.emitNervesEvent)({
|
|
257
|
+
component: "senses",
|
|
258
|
+
event: "senses.voice_realtime_eval_start",
|
|
259
|
+
message: "starting Voice realtime eval timeline grading",
|
|
260
|
+
meta: { scenarioId: normalizedScenarioId, events: events.length },
|
|
261
|
+
});
|
|
262
|
+
const findings = [];
|
|
263
|
+
const metrics = {
|
|
264
|
+
ttfaMs: gradeFirstAudio(events, expectation, findings),
|
|
265
|
+
firstUserResponseMs: gradeFirstUserResponse(events, expectation, findings),
|
|
266
|
+
firstToolPresenceMs: gradeToolPresence(events, expectation, findings),
|
|
267
|
+
...gradeBargeIn(events, expectation, findings),
|
|
268
|
+
};
|
|
269
|
+
if (expectation.requireManualFloorControl)
|
|
270
|
+
gradeManualFloorControl(events, findings);
|
|
271
|
+
if (expectation.requireFriendContext)
|
|
272
|
+
gradeFriendContext(events, expectation.requireFriendContext, findings);
|
|
273
|
+
if (expectation.requiredTranscripts)
|
|
274
|
+
gradeTranscripts(events, expectation.requiredTranscripts, findings);
|
|
275
|
+
if (expectation.requireHangup)
|
|
276
|
+
gradeHangup(events, findings);
|
|
277
|
+
gradeOverlappingResponses(events, findings);
|
|
278
|
+
const report = {
|
|
279
|
+
scenarioId: normalizedScenarioId,
|
|
280
|
+
passed: findings.every((finding) => finding.severity !== "fail"),
|
|
281
|
+
findings,
|
|
282
|
+
metrics,
|
|
283
|
+
transportSources: collectTransportSources(events),
|
|
284
|
+
};
|
|
285
|
+
(0, runtime_1.emitNervesEvent)({
|
|
286
|
+
component: "senses",
|
|
287
|
+
event: "senses.voice_realtime_eval_end",
|
|
288
|
+
message: "finished Voice realtime eval timeline grading",
|
|
289
|
+
meta: { scenarioId: normalizedScenarioId, passed: report.passed, findings: findings.length },
|
|
290
|
+
});
|
|
291
|
+
return report;
|
|
292
|
+
}
|
|
293
|
+
function buildVoiceRealtimeEvalHappyPath() {
|
|
294
|
+
return [
|
|
295
|
+
{ type: "call.connected", atMs: 0, source: { transport: "openai-sip", id: "sip-call-1" } },
|
|
296
|
+
{
|
|
297
|
+
type: "voice.context.injected",
|
|
298
|
+
atMs: 80,
|
|
299
|
+
friendId: "friend-ari",
|
|
300
|
+
sessionKey: "twilio-phone-friend-ari-via-ouro",
|
|
301
|
+
text: "Resolved voice friend: Ari (friendId=friend-ari, trust=family).",
|
|
302
|
+
source: { transport: "voice-eval" },
|
|
303
|
+
},
|
|
304
|
+
{
|
|
305
|
+
type: "session.updated",
|
|
306
|
+
atMs: 100,
|
|
307
|
+
session: { turnDetection: { createResponse: false, interruptResponse: false } },
|
|
308
|
+
source: { transport: "openai-realtime-control", id: "ws-1" },
|
|
309
|
+
},
|
|
310
|
+
{ type: "response.requested", atMs: 120, correlationId: "greeting", source: { transport: "openai-realtime-control", id: "ws-1" } },
|
|
311
|
+
{ type: "assistant.audio.started", atMs: 720, correlationId: "greeting", source: { transport: "openai-sip", id: "sip-call-1" } },
|
|
312
|
+
{ type: "assistant.audio.done", atMs: 1_820, correlationId: "greeting", source: { transport: "openai-sip", id: "sip-call-1" } },
|
|
313
|
+
{
|
|
314
|
+
type: "assistant.transcript.done",
|
|
315
|
+
atMs: 1_840,
|
|
316
|
+
correlationId: "greeting",
|
|
317
|
+
text: "Hey Ari, I am checking the weather now.",
|
|
318
|
+
source: { transport: "openai-realtime-control", id: "ws-1" },
|
|
319
|
+
},
|
|
320
|
+
{
|
|
321
|
+
type: "user.transcript.done",
|
|
322
|
+
atMs: 2_200,
|
|
323
|
+
correlationId: "user-1",
|
|
324
|
+
text: "Can you check the weather and then hang up?",
|
|
325
|
+
source: { transport: "twilio-media-stream", id: "stream-1" },
|
|
326
|
+
},
|
|
327
|
+
{ type: "response.requested", atMs: 2_480, correlationId: "user-1", source: { transport: "openai-realtime-control", id: "ws-1" } },
|
|
328
|
+
{ type: "assistant.audio.started", atMs: 2_540, correlationId: "user-1", source: { transport: "openai-sip", id: "sip-call-1" } },
|
|
329
|
+
{ type: "assistant.audio.done", atMs: 2_820, correlationId: "user-1", source: { transport: "openai-sip", id: "sip-call-1" } },
|
|
330
|
+
{ type: "tool.call.started", atMs: 3_000, correlationId: "tool-1", toolName: "weather_lookup", source: { transport: "openai-realtime-control", id: "ws-1" } },
|
|
331
|
+
{ type: "tool.holding.started", atMs: 3_260, correlationId: "tool-1", text: "One sec, checking.", source: { transport: "openai-sip", id: "sip-call-1" } },
|
|
332
|
+
{ type: "tool.call.completed", atMs: 3_800, correlationId: "tool-1", toolName: "weather_lookup", source: { transport: "openai-realtime-control", id: "ws-1" } },
|
|
333
|
+
{ type: "barge_in.detected", atMs: 4_100, source: { transport: "twilio-media-stream", id: "stream-1" } },
|
|
334
|
+
{ type: "transport.playback_cleared", atMs: 4_140, source: { transport: "twilio-media-stream", id: "stream-1" } },
|
|
335
|
+
{ type: "response.truncated", atMs: 4_170, source: { transport: "openai-realtime-control", id: "ws-1" } },
|
|
336
|
+
{ type: "call.hangup.requested", atMs: 5_000, source: { transport: "openai-realtime-control", id: "ws-1" } },
|
|
337
|
+
{ type: "call.ended", atMs: 5_100, source: { transport: "openai-sip", id: "sip-call-1" } },
|
|
338
|
+
];
|
|
339
|
+
}
|
|
340
|
+
function builtInExpectation() {
|
|
341
|
+
return {
|
|
342
|
+
maxFirstAssistantAudioMs: 1_200,
|
|
343
|
+
maxUserTurnResponseMs: 900,
|
|
344
|
+
maxToolPresenceMs: 600,
|
|
345
|
+
maxBargeInClearMs: 120,
|
|
346
|
+
maxBargeInTruncateMs: 180,
|
|
347
|
+
requireManualFloorControl: true,
|
|
348
|
+
requireFriendContext: {
|
|
349
|
+
friendId: "friend-ari",
|
|
350
|
+
sessionKey: "twilio-phone-friend-ari-via-ouro",
|
|
351
|
+
marker: "trust=family",
|
|
352
|
+
},
|
|
353
|
+
requireHangup: true,
|
|
354
|
+
requiredTranscripts: [
|
|
355
|
+
{ role: "user", contains: "weather" },
|
|
356
|
+
{ role: "assistant", contains: "checking the weather" },
|
|
357
|
+
],
|
|
358
|
+
};
|
|
359
|
+
}
|
|
360
|
+
function buildKnownBadLatencyPath() {
|
|
361
|
+
return buildVoiceRealtimeEvalHappyPath().map((event) => {
|
|
362
|
+
if (event.type === "assistant.audio.started" && event.correlationId === "greeting")
|
|
363
|
+
return { ...event, atMs: 1_900 };
|
|
364
|
+
if (event.type === "response.requested" && event.correlationId === "user-1")
|
|
365
|
+
return { ...event, atMs: 3_500 };
|
|
366
|
+
return event;
|
|
367
|
+
});
|
|
368
|
+
}
|
|
369
|
+
function runBuiltInVoiceRealtimeEvalSuite() {
|
|
370
|
+
const expectation = builtInExpectation();
|
|
371
|
+
return [
|
|
372
|
+
gradeVoiceRealtimeEvalTimeline("voice-happy-path", buildVoiceRealtimeEvalHappyPath(), expectation),
|
|
373
|
+
gradeVoiceRealtimeEvalTimeline("voice-known-bad-latency", buildKnownBadLatencyPath(), expectation),
|
|
374
|
+
];
|
|
375
|
+
}
|
|
376
|
+
function summarizeVoiceRealtimeEvalSuite(reports) {
|
|
377
|
+
const failedScenarioIds = reports.filter((report) => !report.passed).map((report) => report.scenarioId);
|
|
378
|
+
return {
|
|
379
|
+
passed: reports.length - failedScenarioIds.length,
|
|
380
|
+
failed: failedScenarioIds.length,
|
|
381
|
+
total: reports.length,
|
|
382
|
+
failedScenarioIds,
|
|
383
|
+
};
|
|
384
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const realtime_eval_1 = require("./voice/realtime-eval");
|
|
4
|
+
const runtime_1 = require("../nerves/runtime");
|
|
5
|
+
(0, runtime_1.emitNervesEvent)({
|
|
6
|
+
component: "senses",
|
|
7
|
+
event: "senses.voice_realtime_eval_start",
|
|
8
|
+
message: "starting Voice realtime eval command",
|
|
9
|
+
meta: { scenarioId: "built-in-suite", events: 0 },
|
|
10
|
+
});
|
|
11
|
+
const reports = (0, realtime_eval_1.runBuiltInVoiceRealtimeEvalSuite)();
|
|
12
|
+
const summary = (0, realtime_eval_1.summarizeVoiceRealtimeEvalSuite)(reports);
|
|
13
|
+
const expectedKnownBadFailed = summary.failed === 1 && summary.failedScenarioIds[0] === "voice-known-bad-latency";
|
|
14
|
+
const happyPathPassed = reports.some((report) => report.scenarioId === "voice-happy-path" && report.passed);
|
|
15
|
+
(0, runtime_1.emitNervesEvent)({
|
|
16
|
+
component: "senses",
|
|
17
|
+
event: "senses.voice_realtime_eval_end",
|
|
18
|
+
message: "finished Voice realtime eval command",
|
|
19
|
+
meta: { scenarioId: "built-in-suite", passed: expectedKnownBadFailed && happyPathPassed, findings: summary.failed },
|
|
20
|
+
});
|
|
21
|
+
// eslint-disable-next-line no-console -- terminal UX: eval command summary
|
|
22
|
+
console.log(JSON.stringify({ summary, expectedKnownBadFailed, happyPathPassed }, null, 2));
|
|
23
|
+
if (!expectedKnownBadFailed || !happyPathPassed) {
|
|
24
|
+
process.exit(1);
|
|
25
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ouro.bot/cli",
|
|
3
|
-
"version": "0.1.0-alpha.
|
|
3
|
+
"version": "0.1.0-alpha.584",
|
|
4
4
|
"main": "dist/heart/daemon/ouro-entry.js",
|
|
5
5
|
"bin": {
|
|
6
6
|
"cli": "dist/heart/daemon/ouro-bot-entry.js",
|
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
"ouro": "tsc && node dist/heart/daemon/ouro-entry.js",
|
|
27
27
|
"teams": "tsc && node dist/senses/teams-entry.js --agent ouroboros",
|
|
28
28
|
"bluebubbles": "tsc && node dist/senses/bluebubbles/entry.js --agent ouroboros",
|
|
29
|
+
"voice:eval": "npm run build && node dist/senses/voice-realtime-eval-entry.js",
|
|
29
30
|
"test": "vitest run",
|
|
30
31
|
"test:integration": "npm run build && vitest run --config vitest.integration.config.ts",
|
|
31
32
|
"test:e2e:package": "npm run build && node scripts/package-e2e.cjs",
|