@ouro.bot/cli 0.1.0-alpha.583 → 0.1.0-alpha.585

package/changelog.json

@@ -1,6 +1,21 @@
 {
   "_note": "This changelog is maintained as part of the PR/version-bump workflow. Agent-curated, not auto-generated. Agents read this file directly via read_file to understand what changed between versions.",
   "versions": [
+    {
+      "version": "0.1.0-alpha.585",
+      "changes": [
+        "Voice realtime evals now support replayable trace JSON artifacts that preserve transport/source metadata, normalize SIP/Twilio/Realtime event names into the shared Voice eval timeline, validate causal ordering, and keep ignored provider-noise events visible without grading them.",
+        "`npm run voice:eval -- --trace <file>` grades one or more trace artifacts alongside the built-in happy/known-bad suite, treating expected-fail traces as canaries while failing on unexpected trace failures or unexpected passes.",
+        "Voice trace replay has golden fixtures for clean calls, barge-in, tool holding phrases, mid-turn hangup, delayed audio/transcript behavior, duplicate/late provider events, and redaction-safe summaries so phone regressions can be debugged without a human live call."
+      ]
+    },
+    {
+      "version": "0.1.0-alpha.584",
+      "changes": [
+        "Voice now has a transport-aware Realtime eval kernel that grades deterministic call timelines for first-audio latency, user-turn response latency, tool holding phrases, barge-in clearing/truncation, friend context, transcript continuity, and hangup control before live phone testing.",
+        "`npm run voice:eval` runs built-in no-human voice scenarios, including a healthy path and an expected known-bad latency canary, and emits the normal nerves events expected of executable sense entrypoints so future Voice transport work can prove synchronous behavior without requiring a human to answer calls."
+      ]
+    },
     {
       "version": "0.1.0-alpha.583",
       "changes": [

package/dist/senses/voice/index.js

@@ -26,3 +26,4 @@ __exportStar(require("./playback"), exports);
 __exportStar(require("./golden-path"), exports);
 __exportStar(require("./twilio-phone"), exports);
 __exportStar(require("./twilio-phone-runtime"), exports);
+__exportStar(require("./realtime-eval"), exports);

package/dist/senses/voice/realtime-eval.js

@@ -0,0 +1,393 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.gradeVoiceRealtimeEvalTimeline = gradeVoiceRealtimeEvalTimeline;
+exports.buildVoiceRealtimeEvalHappyPath = buildVoiceRealtimeEvalHappyPath;
+exports.buildVoiceRealtimeEvalDefaultExpectation = buildVoiceRealtimeEvalDefaultExpectation;
+exports.runBuiltInVoiceRealtimeEvalSuite = runBuiltInVoiceRealtimeEvalSuite;
+exports.summarizeVoiceRealtimeEvalSuite = summarizeVoiceRealtimeEvalSuite;
+const runtime_1 = require("../../nerves/runtime");
+function validateTimeline(scenarioId, events, expectation) {
+    const normalizedScenarioId = scenarioId.trim();
+    if (!normalizedScenarioId)
+        throw new Error("voice eval scenario id is empty");
+    if (events.length === 0)
+        throw new Error("voice eval timeline is empty");
+    const budgets = [
+        expectation.maxFirstAssistantAudioMs,
+        expectation.maxUserTurnResponseMs,
+        expectation.maxToolPresenceMs,
+        expectation.maxBargeInClearMs,
+        expectation.maxBargeInTruncateMs,
+    ];
+    if (budgets.some((budget) => !Number.isFinite(budget) || budget <= 0)) {
+        throw new Error("voice eval latency budgets must be positive");
+    }
+    return normalizedScenarioId;
+}
+function sortedEvents(events) {
+    return [...events].sort((left, right) => left.atMs - right.atMs);
+}
+function firstEvent(events, type) {
+    return events.find((event) => event.type === type);
+}
+function allEvents(events, type) {
+    return events.filter((event) => event.type === type);
+}
+function lowerText(value) {
+    return value?.toLowerCase() ?? "";
+}
+function pushFinding(findings, finding) {
+    findings.push(finding);
+}
+function gradeFirstAudio(events, expectation, findings) {
+    const connected = firstEvent(events, "call.connected");
+    const firstAudio = firstEvent(events, "assistant.audio.started");
+    if (!connected || !firstAudio) {
+        pushFinding(findings, {
+            code: "first_audio_missing",
+            severity: "fail",
+            message: "Voice call did not produce assistant audio after connect.",
+            source: connected?.source ?? firstAudio?.source,
+            atMs: connected?.atMs ?? firstAudio?.atMs,
+        });
+        return undefined;
+    }
+    const ttfaMs = firstAudio.atMs - connected.atMs;
+    if (ttfaMs > expectation.maxFirstAssistantAudioMs) {
+        pushFinding(findings, {
+            code: "first_audio_late",
+            severity: "fail",
+            message: `First assistant audio started after ${ttfaMs}ms, over the ${expectation.maxFirstAssistantAudioMs}ms budget.`,
+            source: firstAudio.source,
+            atMs: firstAudio.atMs,
+        });
+    }
+    return ttfaMs;
+}
+function gradeFirstUserResponse(events, expectation, findings) {
+    const userTranscript = firstEvent(events, "user.transcript.done");
+    if (!userTranscript)
+        return undefined;
+    const response = events.find((event) => event.type === "response.requested"
+        && event.atMs >= userTranscript.atMs
+        && (!userTranscript.correlationId || event.correlationId === userTranscript.correlationId));
+    if (!response) {
+        pushFinding(findings, {
+            code: "user_response_missing",
+            severity: "fail",
+            message: "No voice response was requested after the caller transcript completed.",
+            source: userTranscript.source,
+            atMs: userTranscript.atMs,
+        });
+        return undefined;
+    }
+    const latencyMs = response.atMs - userTranscript.atMs;
+    if (latencyMs > expectation.maxUserTurnResponseMs) {
+        pushFinding(findings, {
+            code: "user_response_late",
+            severity: "fail",
+            message: `Voice response was requested after ${latencyMs}ms, over the ${expectation.maxUserTurnResponseMs}ms budget.`,
+            source: response.source,
+            atMs: response.atMs,
+        });
+    }
+    return latencyMs;
+}
+function gradeToolPresence(events, expectation, findings) {
+    const toolCall = firstEvent(events, "tool.call.started");
+    if (!toolCall)
+        return undefined;
+    const holding = events.find((event) => event.type === "tool.holding.started"
+        && event.atMs >= toolCall.atMs
+        && (!toolCall.correlationId || event.correlationId === toolCall.correlationId));
+    if (!holding) {
+        pushFinding(findings, {
+            code: "tool_presence_missing",
+            severity: "fail",
+            message: "Tool call did not produce a short voice holding phrase.",
+            source: toolCall.source,
+            atMs: toolCall.atMs,
+        });
+        return undefined;
+    }
+    const latencyMs = holding.atMs - toolCall.atMs;
+    if (latencyMs > expectation.maxToolPresenceMs) {
+        pushFinding(findings, {
+            code: "tool_presence_late",
+            severity: "fail",
+            message: `Tool holding phrase started after ${latencyMs}ms, over the ${expectation.maxToolPresenceMs}ms budget.`,
+            source: holding.source,
+            atMs: holding.atMs,
+        });
+    }
+    return latencyMs;
+}
+function gradeBargeIn(events, expectation, findings) {
+    const bargeIn = firstEvent(events, "barge_in.detected");
+    if (!bargeIn)
+        return {};
+    const clear = events.find((event) => event.type === "transport.playback_cleared" && event.atMs >= bargeIn.atMs);
+    const truncate = events.find((event) => event.type === "response.truncated" && event.atMs >= bargeIn.atMs);
+    const metrics = {};
+    if (!clear) {
+        pushFinding(findings, {
+            code: "barge_in_clear_missing",
+            severity: "fail",
+            message: "Caller barge-in did not clear transport playback.",
+            source: bargeIn.source,
+            atMs: bargeIn.atMs,
+        });
+    }
+    else {
+        metrics.firstBargeInClearMs = clear.atMs - bargeIn.atMs;
+        if (metrics.firstBargeInClearMs > expectation.maxBargeInClearMs) {
+            pushFinding(findings, {
+                code: "barge_in_clear_late",
+                severity: "fail",
+                message: `Barge-in playback clear took ${metrics.firstBargeInClearMs}ms, over the ${expectation.maxBargeInClearMs}ms budget.`,
+                source: clear.source,
+                atMs: clear.atMs,
+            });
+        }
+    }
+    if (!truncate) {
+        pushFinding(findings, {
+            code: "barge_in_truncate_missing",
+            severity: "fail",
+            message: "Caller barge-in did not truncate the active Realtime response.",
+            source: bargeIn.source,
+            atMs: bargeIn.atMs,
+        });
+    }
+    else {
+        metrics.firstBargeInTruncateMs = truncate.atMs - bargeIn.atMs;
+        if (metrics.firstBargeInTruncateMs > expectation.maxBargeInTruncateMs) {
+            pushFinding(findings, {
+                code: "barge_in_truncate_late",
+                severity: "fail",
+                message: `Barge-in response truncation took ${metrics.firstBargeInTruncateMs}ms, over the ${expectation.maxBargeInTruncateMs}ms budget.`,
+                source: truncate.source,
+                atMs: truncate.atMs,
+            });
+        }
+    }
+    return metrics;
+}
+function gradeManualFloorControl(events, findings) {
+    const session = allEvents(events, "session.updated").find((event) => event.session?.turnDetection);
+    if (session?.session?.turnDetection?.createResponse === false
+        && session.session.turnDetection.interruptResponse === false) {
+        return;
+    }
+    pushFinding(findings, {
+        code: "manual_floor_control_missing",
+        severity: "fail",
+        message: "Realtime session did not disable provider auto-response and provider interruption.",
+        source: session?.source,
+        atMs: session?.atMs,
+    });
+}
+function gradeFriendContext(events, requirement, findings) {
+    const context = firstEvent(events, "voice.context.injected");
+    if (context?.friendId === requirement.friendId
+        && context.sessionKey === requirement.sessionKey
+        && lowerText(context.text).includes(requirement.marker.toLowerCase())) {
+        return;
+    }
+    pushFinding(findings, {
+        code: "friend_context_mismatch",
+        severity: "fail",
+        message: "Voice context did not preserve the expected friend identity, trust marker, and stable session key.",
+        source: context?.source,
+        atMs: context?.atMs,
+    });
+}
+function gradeTranscripts(events, requirements, findings) {
+    for (const requirement of requirements) {
+        const type = requirement.role === "assistant"
+            ? "assistant.transcript.done"
+            : "user.transcript.done";
+        const found = allEvents(events, type).some((event) => lowerText(event.text).includes(requirement.contains.toLowerCase()));
+        if (!found) {
+            pushFinding(findings, {
+                code: "transcript_missing",
+                severity: "fail",
+                message: `Missing ${requirement.role} transcript containing "${requirement.contains}".`,
+            });
+        }
+    }
+}
+function gradeHangup(events, findings) {
+    const hangup = firstEvent(events, "call.hangup.requested");
+    if (hangup)
+        return;
+    const ended = firstEvent(events, "call.ended");
+    pushFinding(findings, {
+        code: "hangup_missing",
+        severity: "fail",
+        message: "Voice eval expected an agent-controlled hangup request before call end.",
+        source: ended?.source,
+        atMs: ended?.atMs,
+    });
+}
+function gradeOverlappingResponses(events, findings) {
+    for (const response of allEvents(events, "response.requested")) {
+        const activeAudio = allEvents(events, "assistant.audio.started").find((started) => {
+            const done = events.find((event) => {
+                if (event.atMs < started.atMs)
+                    return false;
+                if (event.type === "response.truncated" || event.type === "call.hangup.requested")
+                    return true;
+                if (event.type !== "assistant.audio.done" && event.type !== "assistant.transcript.done")
+                    return false;
+                return !started.correlationId || !event.correlationId || event.correlationId === started.correlationId;
+            });
+            return response.atMs > started.atMs && (!done || response.atMs < done.atMs);
+        });
+        if (activeAudio) {
+            pushFinding(findings, {
+                code: "response_overlap",
+                severity: "fail",
+                message: "Voice response was requested while assistant audio was still active.",
+                source: response.source,
+                atMs: response.atMs,
+            });
+            return;
+        }
+    }
+}
+function collectTransportSources(events) {
+    return [...new Set(events.flatMap((event) => event.source ? [event.source.transport] : []))].sort();
+}
+function gradeVoiceRealtimeEvalTimeline(scenarioId, timeline, expectation) {
+    const normalizedScenarioId = validateTimeline(scenarioId, timeline, expectation);
+    const events = sortedEvents(timeline);
+    (0, runtime_1.emitNervesEvent)({
+        component: "senses",
+        event: "senses.voice_realtime_eval_start",
+        message: "starting Voice realtime eval timeline grading",
+        meta: { scenarioId: normalizedScenarioId, events: events.length },
+    });
+    const findings = [];
+    const metrics = {
+        ttfaMs: gradeFirstAudio(events, expectation, findings),
+        firstUserResponseMs: gradeFirstUserResponse(events, expectation, findings),
+        firstToolPresenceMs: gradeToolPresence(events, expectation, findings),
+        ...gradeBargeIn(events, expectation, findings),
+    };
+    if (expectation.requireManualFloorControl)
+        gradeManualFloorControl(events, findings);
+    if (expectation.requireFriendContext)
+        gradeFriendContext(events, expectation.requireFriendContext, findings);
+    if (expectation.requiredTranscripts)
+        gradeTranscripts(events, expectation.requiredTranscripts, findings);
+    if (expectation.requireHangup)
+        gradeHangup(events, findings);
+    gradeOverlappingResponses(events, findings);
+    const report = {
+        scenarioId: normalizedScenarioId,
+        passed: findings.every((finding) => finding.severity !== "fail"),
+        findings,
+        metrics,
+        transportSources: collectTransportSources(events),
+    };
+    (0, runtime_1.emitNervesEvent)({
+        component: "senses",
+        event: "senses.voice_realtime_eval_end",
+        message: "finished Voice realtime eval timeline grading",
+        meta: { scenarioId: normalizedScenarioId, passed: report.passed, findings: findings.length },
+    });
+    return report;
+}
+function buildVoiceRealtimeEvalHappyPath() {
+    return [
+        { type: "call.connected", atMs: 0, source: { transport: "openai-sip", id: "sip-call-1" } },
+        {
+            type: "voice.context.injected",
+            atMs: 80,
+            friendId: "friend-ari",
+            sessionKey: "twilio-phone-friend-ari-via-ouro",
+            text: "Resolved voice friend: Ari (friendId=friend-ari, trust=family).",
+            source: { transport: "voice-eval" },
+        },
+        {
+            type: "session.updated",
+            atMs: 100,
+            session: { turnDetection: { createResponse: false, interruptResponse: false } },
+            source: { transport: "openai-realtime-control", id: "ws-1" },
+        },
+        { type: "response.requested", atMs: 120, correlationId: "greeting", source: { transport: "openai-realtime-control", id: "ws-1" } },
+        { type: "assistant.audio.started", atMs: 720, correlationId: "greeting", source: { transport: "openai-sip", id: "sip-call-1" } },
+        { type: "assistant.audio.done", atMs: 1_820, correlationId: "greeting", source: { transport: "openai-sip", id: "sip-call-1" } },
+        {
+            type: "assistant.transcript.done",
+            atMs: 1_840,
+            correlationId: "greeting",
+            text: "Hey Ari, I am checking the weather now.",
+            source: { transport: "openai-realtime-control", id: "ws-1" },
+        },
+        {
+            type: "user.transcript.done",
+            atMs: 2_200,
+            correlationId: "user-1",
+            text: "Can you check the weather and then hang up?",
+            source: { transport: "twilio-media-stream", id: "stream-1" },
+        },
+        { type: "response.requested", atMs: 2_480, correlationId: "user-1", source: { transport: "openai-realtime-control", id: "ws-1" } },
+        { type: "assistant.audio.started", atMs: 2_540, correlationId: "user-1", source: { transport: "openai-sip", id: "sip-call-1" } },
+        { type: "assistant.audio.done", atMs: 2_820, correlationId: "user-1", source: { transport: "openai-sip", id: "sip-call-1" } },
+        { type: "tool.call.started", atMs: 3_000, correlationId: "tool-1", toolName: "weather_lookup", source: { transport: "openai-realtime-control", id: "ws-1" } },
+        { type: "tool.holding.started", atMs: 3_260, correlationId: "tool-1", text: "One sec, checking.", source: { transport: "openai-sip", id: "sip-call-1" } },
+        { type: "tool.call.completed", atMs: 3_800, correlationId: "tool-1", toolName: "weather_lookup", source: { transport: "openai-realtime-control", id: "ws-1" } },
+        { type: "barge_in.detected", atMs: 4_100, source: { transport: "twilio-media-stream", id: "stream-1" } },
+        { type: "transport.playback_cleared", atMs: 4_140, source: { transport: "twilio-media-stream", id: "stream-1" } },
+        { type: "response.truncated", atMs: 4_170, source: { transport: "openai-realtime-control", id: "ws-1" } },
+        { type: "call.hangup.requested", atMs: 5_000, source: { transport: "openai-realtime-control", id: "ws-1" } },
+        { type: "call.ended", atMs: 5_100, source: { transport: "openai-sip", id: "sip-call-1" } },
+    ];
+}
+function buildVoiceRealtimeEvalDefaultExpectation() {
+    return {
+        maxFirstAssistantAudioMs: 1_200,
+        maxUserTurnResponseMs: 900,
+        maxToolPresenceMs: 600,
+        maxBargeInClearMs: 120,
+        maxBargeInTruncateMs: 180,
+        requireManualFloorControl: true,
+        requireFriendContext: {
+            friendId: "friend-ari",
+            sessionKey: "twilio-phone-friend-ari-via-ouro",
+            marker: "trust=family",
+        },
+        requireHangup: true,
+        requiredTranscripts: [
+            { role: "user", contains: "weather" },
+            { role: "assistant", contains: "checking the weather" },
+        ],
+    };
+}
+function buildKnownBadLatencyPath() {
+    return buildVoiceRealtimeEvalHappyPath().map((event) => {
+        if (event.type === "assistant.audio.started" && event.correlationId === "greeting")
+            return { ...event, atMs: 1_900 };
+        if (event.type === "response.requested" && event.correlationId === "user-1")
+            return { ...event, atMs: 3_500 };
+        return event;
+    });
+}
+function runBuiltInVoiceRealtimeEvalSuite() {
+    const expectation = buildVoiceRealtimeEvalDefaultExpectation();
+    return [
+        gradeVoiceRealtimeEvalTimeline("voice-happy-path", buildVoiceRealtimeEvalHappyPath(), expectation),
+        gradeVoiceRealtimeEvalTimeline("voice-known-bad-latency", buildKnownBadLatencyPath(), expectation),
+    ];
+}
+function summarizeVoiceRealtimeEvalSuite(reports) {
+    const failedScenarioIds = reports.filter((report) => !report.passed).map((report) => report.scenarioId);
+    return {
+        passed: reports.length - failedScenarioIds.length,
+        failed: failedScenarioIds.length,
+        total: reports.length,
+        failedScenarioIds,
+    };
+}

package/dist/senses/voice/realtime-trace.js

@@ -0,0 +1,402 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseVoiceRealtimeEvalTraceArtifact = parseVoiceRealtimeEvalTraceArtifact;
+exports.loadVoiceRealtimeEvalTraceArtifact = loadVoiceRealtimeEvalTraceArtifact;
+exports.resolveVoiceRealtimeEvalTraceExpectation = resolveVoiceRealtimeEvalTraceExpectation;
+exports.traceArtifactToVoiceRealtimeEvalTimeline = traceArtifactToVoiceRealtimeEvalTimeline;
+exports.gradeVoiceRealtimeEvalTrace = gradeVoiceRealtimeEvalTrace;
+exports.formatVoiceRealtimeEvalTraceReport = formatVoiceRealtimeEvalTraceReport;
+const fs = __importStar(require("fs"));
+const runtime_1 = require("../../nerves/runtime");
+const realtime_eval_1 = require("./realtime-eval");
+const transports = new Set([
+    "browser-meeting",
+    "openai-realtime-control",
+    "openai-sip",
+    "twilio-media-stream",
+    "voice-eval",
+]);
+const normalizedEvents = new Set([
+    "assistant.audio.done",
+    "assistant.audio.started",
+    "assistant.transcript.done",
+    "barge_in.detected",
+    "call.connected",
+    "call.ended",
+    "call.hangup.requested",
+    "response.requested",
+    "response.truncated",
+    "session.updated",
+    "tool.call.completed",
+    "tool.call.started",
+    "tool.holding.started",
+    "transport.playback_cleared",
+    "user.transcript.done",
+    "voice.context.injected",
+]);
+const rawEventMap = new Map([
+    ["openai.realtime.call.hangup.sent", "call.hangup.requested"],
+    ["openai.realtime.conversation.item.truncate.sent", "response.truncated"],
+    ["openai.realtime.input_audio_buffer.speech_started", "barge_in.detected"],
+    ["openai.realtime.input_audio_transcription.completed", "user.transcript.done"],
+    ["openai.realtime.output_audio.delta", "assistant.audio.started"],
+    ["openai.realtime.output_audio.done", "assistant.audio.done"],
+    ["openai.realtime.output_audio_transcript.done", "assistant.transcript.done"],
+    ["openai.realtime.response.create.sent", "response.requested"],
+    ["openai.realtime.response.function_call_arguments.done", "tool.call.started"],
+    ["openai.realtime.session.updated", "session.updated"],
+    ["openai.realtime.tool.completed", "tool.call.completed"],
+    ["openai.sip.call.connected", "call.connected"],
+    ["openai.sip.call.ended", "call.ended"],
+    ["twilio.call.ended", "call.ended"],
+    ["twilio.media.clear.sent", "transport.playback_cleared"],
+    ["twilio.media.start", "call.connected"],
+    ["voice.hangup.requested", "call.hangup.requested"],
+    ["voice.tool_holding.started", "tool.holding.started"],
+]);
+const expectedOutcomes = new Set(["expected-fail", "fail", "pass"]);
+const expectationProfiles = new Set(["voice-phone-default"]);
+function objectRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value) ? value : undefined;
+}
+function label(prefix, detail) {
+    return prefix ? `${prefix}: ${detail}` : detail;
+}
+function requiredString(value, name, sourceLabel) {
+    if (typeof value !== "string" || value.trim() === "")
+        throw new Error(label(sourceLabel, `${name} must be a non-empty string`));
+    return value.trim();
+}
+function optionalString(value, name, sourceLabel) {
+    if (value === undefined)
+        return undefined;
+    if (typeof value !== "string")
+        throw new Error(label(sourceLabel, `${name} must be a string`));
+    return value;
+}
+function parseSource(value, sourceLabel) {
+    if (value === undefined)
+        return undefined;
+    const raw = objectRecord(value);
+    if (!raw)
+        throw new Error(label(sourceLabel, "source must be an object"));
+    const transport = raw.transport;
+    if (typeof transport !== "string" || !transports.has(transport)) {
+        throw new Error(label(sourceLabel, "source.transport is unsupported"));
+    }
+    const id = optionalString(raw.id, "source.id", sourceLabel);
+    return id === undefined ? { transport: transport } : { transport: transport, id };
+}
+function parseTurnDetection(value, sourceLabel) {
+    if (value === undefined)
+        return undefined;
+    const session = objectRecord(value);
+    if (!session)
+        throw new Error(label(sourceLabel, "session must be an object"));
+    const turnDetection = objectRecord(session.turnDetection);
+    if (!turnDetection)
+        return {};
+    const createResponse = turnDetection.createResponse;
+    const interruptResponse = turnDetection.interruptResponse;
+    if (createResponse !== undefined && typeof createResponse !== "boolean") {
+        throw new Error(label(sourceLabel, "session.turnDetection.createResponse must be boolean"));
+    }
+    if (interruptResponse !== undefined && typeof interruptResponse !== "boolean") {
+        throw new Error(label(sourceLabel, "session.turnDetection.interruptResponse must be boolean"));
+    }
+    return { turnDetection: { createResponse, interruptResponse } };
+}
+function validateExpectation(expectation, sourceLabel) {
+    const budgets = [
+        expectation.maxFirstAssistantAudioMs,
+        expectation.maxUserTurnResponseMs,
+        expectation.maxToolPresenceMs,
+        expectation.maxBargeInClearMs,
+        expectation.maxBargeInTruncateMs,
+    ];
+    if (budgets.some((budget) => typeof budget !== "number" || !Number.isFinite(budget) || budget <= 0)) {
+        throw new Error(label(sourceLabel, "expectation latency budgets must be positive finite numbers"));
+    }
+    return expectation;
+}
+function parseExpectation(value, sourceLabel) {
+    const raw = objectRecord(value);
+    if (!raw)
+        throw new Error(label(sourceLabel, "expectation must be an object"));
+    return validateExpectation(raw, sourceLabel);
+}
+function parseTraceEvent(value, index, sourceLabel) {
+    const eventLabel = `${sourceLabel} event[${index}]`;
+    const raw = objectRecord(value);
+    if (!raw)
+        throw new Error(label(eventLabel, "must be an object"));
+    const event = requiredString(raw.event, "event", eventLabel);
+    const ignored = raw.ignored === undefined ? undefined : raw.ignored;
+    if (ignored !== undefined && typeof ignored !== "boolean")
+        throw new Error(label(eventLabel, "ignored must be boolean"));
+    if (!ignored && !normalizedEvents.has(event) && !rawEventMap.has(event)) {
+        throw new Error(label(eventLabel, `unknown trace event ${event}`));
+    }
+    const atMs = raw.atMs;
+    if (typeof atMs !== "number" || !Number.isFinite(atMs)) {
+        throw new Error(label(eventLabel, "atMs must be a finite number"));
+    }
+    let role;
+    if (raw.role !== undefined) {
+        if (raw.role !== "assistant" && raw.role !== "user")
+            throw new Error(label(eventLabel, "role must be assistant or user"));
+        role = raw.role;
+    }
+    const parsed = {
+        atMs,
+        event,
+        source: parseSource(raw.source, eventLabel),
+        correlationId: optionalString(raw.correlationId, "correlationId", eventLabel),
+        text: optionalString(raw.text, "text", eventLabel),
+        role,
+        toolName: optionalString(raw.toolName, "toolName", eventLabel),
+        friendId: optionalString(raw.friendId, "friendId", eventLabel),
+        sessionKey: optionalString(raw.sessionKey, "sessionKey", eventLabel),
+        session: parseTurnDetection(raw.session, eventLabel),
+        ignored: ignored || undefined,
+        ignoreReason: optionalString(raw.ignoreReason, "ignoreReason", eventLabel),
+    };
+    if (parsed.ignored && !parsed.ignoreReason)
+        throw new Error(label(eventLabel, "ignored events require ignoreReason"));
+    return parsed;
+}
+function parseVoiceRealtimeEvalTraceArtifact(value, sourceLabel = "voice trace artifact") {
+    const raw = objectRecord(value);
+    if (!raw || raw.schemaVersion !== 1)
+        throw new Error(label(sourceLabel, "schemaVersion must be 1"));
+    const expectedOutcome = raw.expectedOutcome;
+    if (typeof expectedOutcome !== "string" || !expectedOutcomes.has(expectedOutcome)) {
+        throw new Error(label(sourceLabel, "expectedOutcome must be pass, fail, or expected-fail"));
+    }
+    const expectationProfile = raw.expectationProfile;
+    if (expectationProfile !== undefined && (typeof expectationProfile !== "string" || !expectationProfiles.has(expectationProfile))) {
+        throw new Error(label(sourceLabel, "expectationProfile is unsupported"));
+    }
+    const hasInlineExpectation = raw.expectation !== undefined;
+    if (hasInlineExpectation === (expectationProfile !== undefined)) {
+        throw new Error(label(sourceLabel, "provide exactly one of expectation or expectationProfile"));
+    }
+    if (!Array.isArray(raw.events) || raw.events.length === 0) {
+        throw new Error(label(sourceLabel, "events must contain at least one event"));
+    }
+    const redacted = raw.redacted;
+    if (redacted !== undefined && typeof redacted !== "boolean")
+        throw new Error(label(sourceLabel, "redacted must be boolean"));
+    return {
+        schemaVersion: 1,
+        traceId: requiredString(raw.traceId, "traceId", sourceLabel),
+        scenarioId: requiredString(raw.scenarioId, "scenarioId", sourceLabel),
+        expectedOutcome: expectedOutcome,
+        expectation: hasInlineExpectation ? parseExpectation(raw.expectation, sourceLabel) : undefined,
+        expectationProfile: expectationProfile,
+        redacted: redacted || undefined,
+        events: raw.events.map((event, index) => parseTraceEvent(event, index, sourceLabel)),
+    };
+}
+function loadVoiceRealtimeEvalTraceArtifact(filePath) {
+    let raw;
+    try {
+        raw = fs.readFileSync(filePath, "utf8");
+    }
+    catch {
+        throw new Error(`${filePath}: failed to read trace artifact`);
+    }
+    try {
+        return parseVoiceRealtimeEvalTraceArtifact(JSON.parse(raw), filePath);
+    }
+    catch (error) {
+        if (error instanceof SyntaxError)
+            throw new Error(`${filePath}: invalid JSON: ${error.message}`);
+        throw error;
+    }
+}
+function resolveVoiceRealtimeEvalTraceExpectation(artifact) {
+    if (artifact.expectationProfile === "voice-phone-default")
+        return (0, realtime_eval_1.buildVoiceRealtimeEvalDefaultExpectation)();
+    if (artifact.expectation)
+        return artifact.expectation;
+    throw new Error(`${artifact.traceId}: trace artifact has no expectation contract`);
+}
+function eventTypeFor(event) {
+    if (normalizedEvents.has(event.event))
+        return event.event;
+    const type = rawEventMap.get(event.event);
+    if (!type)
+        throw new Error(`unknown trace event ${event.event}`);
+    return type;
+}
+function toTimelineEvent(event, redacted) {
+    const type = eventTypeFor(event);
+    return {
+        type,
+        atMs: event.atMs,
+        source: event.source,
+        correlationId: event.correlationId,
+        text: redacted ? undefined : event.text,
+        role: event.role,
+        toolName: event.toolName,
+        friendId: event.friendId,
+        sessionKey: event.sessionKey,
+        session: event.session,
+    };
+}
+function sortedTimeline(events) {
+    return events
+        .map((event, index) => ({ event, index }))
+        .sort((left, right) => left.event.atMs - right.event.atMs || left.index - right.index)
+        .map(({ event }) => event);
+}
+function findFirst(events, type) {
+    return events.find((event) => event.type === type);
+}
+function validateCausalTimeline(artifact, timeline) {
+    for (let index = 0; index < artifact.events.length; index += 1) {
+        const event = artifact.events[index];
+        if (event.ignored)
+            continue;
+        if (event.atMs < 0) {
+            throw new Error(`${artifact.traceId} event[${index}] atMs must be a nonnegative finite number`);
+        }
+    }
+    const connected = findFirst(timeline, "call.connected");
+    const audio = findFirst(timeline, "assistant.audio.started");
+    if (connected && audio && audio.atMs < connected.atMs) {
+        throw new Error(`${artifact.traceId}: assistant audio started before call.connected`);
+    }
+    for (const response of timeline.filter((event) => event.type === "response.requested" && event.correlationId)) {
+        const transcript = timeline.find((event) => event.type === "user.transcript.done" && event.correlationId === response.correlationId);
+        if (transcript && response.atMs < transcript.atMs) {
+            throw new Error(`${artifact.traceId}: response.requested for ${response.correlationId} occurred before user.transcript.done`);
+        }
+    }
+    for (const completed of timeline.filter((event) => event.type === "tool.call.completed" && event.correlationId)) {
+        const started = timeline.find((event) => event.type === "tool.call.started" && event.correlationId === completed.correlationId);
+        if (started && completed.atMs < started.atMs) {
+            throw new Error(`${artifact.traceId}: tool.call.completed for ${completed.correlationId} occurred before tool.call.started`);
+        }
+    }
+    const expectation = resolveVoiceRealtimeEvalTraceExpectation(artifact);
+    if (expectation.requireHangup) {
+        const ended = findFirst(timeline, "call.ended");
+        const hangup = findFirst(timeline, "call.hangup.requested");
+        if (ended && hangup && ended.atMs < hangup.atMs) {
+            throw new Error(`${artifact.traceId}: call.ended occurred before call.hangup.requested`);
+        }
+    }
+}
+function traceArtifactToVoiceRealtimeEvalTimeline(artifact) {
+    const timeline = sortedTimeline(artifact.events
+        .filter((event) => !event.ignored)
+        .map((event) => toTimelineEvent(event, Boolean(artifact.redacted))));
+    validateCausalTimeline(artifact, timeline);
+    return timeline;
+}
+function expectedOutcomeMatched(expectedOutcome, passed) {
+    if (expectedOutcome === "pass")
+        return passed;
+    return !passed;
+}
+function gradeVoiceRealtimeEvalTrace(artifact) {
+    (0, runtime_1.emitNervesEvent)({
+        component: "senses",
+        event: "senses.voice_realtime_trace_replay_start",
+        message: "starting Voice realtime trace replay",
+        meta: { scenarioId: artifact.scenarioId, events: artifact.events.length },
+    });
+    const timeline = traceArtifactToVoiceRealtimeEvalTimeline(artifact);
+    const report = (0, realtime_eval_1.gradeVoiceRealtimeEvalTimeline)(artifact.scenarioId, timeline, resolveVoiceRealtimeEvalTraceExpectation(artifact));
+    const result = {
+        artifact,
+        traceId: artifact.traceId,
+        scenarioId: artifact.scenarioId,
+        expectedOutcome: artifact.expectedOutcome,
+        outcomeMatched: expectedOutcomeMatched(artifact.expectedOutcome, report.passed),
+        report,
+        timeline,
+        ignoredEvents: artifact.events.filter((event) => event.ignored),
+    };
+    (0, runtime_1.emitNervesEvent)({
+        component: "senses",
+        event: "senses.voice_realtime_trace_replay_end",
+        message: "finished Voice realtime trace replay",
+        meta: { scenarioId: artifact.scenarioId, passed: report.passed, findings: report.findings.length },
+    });
+    return result;
+}
+function textForSummary(result, event) {
+    if (result.artifact.redacted && (event.type === "assistant.transcript.done" ||
+        event.type === "user.transcript.done" ||
+        event.type === "voice.context.injected")) {
+        return " [redacted]";
+    }
+    if (event.text === undefined)
+        return "";
+    return ` "${event.text}"`;
+}
+function sourceForSummary(source) {
+    if (!source)
+        return "";
+    return source.id ? ` ${source.transport}/${source.id}` : ` ${source.transport}`;
+}
+function formatVoiceRealtimeEvalTraceReport(result) {
+    const lines = [
+        `trace ${result.traceId} scenario ${result.scenarioId}`,
+        `expected: ${result.expectedOutcome}; report passed: ${result.report.passed}; outcome matched: ${result.outcomeMatched}`,
+        `transports: ${result.report.transportSources.join(", ") || "none"}`,
+        `metrics: ${JSON.stringify(result.report.metrics)}`,
+    ];
+    if (result.report.findings.length > 0) {
+        lines.push("findings:");
+        for (const finding of result.report.findings) {
+            const message = result.artifact.redacted ? "[redacted]" : finding.message;
+            lines.push(`- ${finding.code}${finding.atMs === undefined ? "" : ` at ${finding.atMs}ms`}: ${message}`);
+        }
+    }
+    lines.push("events:");
+    for (const event of result.timeline) {
+        lines.push(`- ${event.atMs}ms ${event.type}${sourceForSummary(event.source)}${textForSummary(result, event)}`);
+    }
+    lines.push(`ignored provider events: ${result.ignoredEvents.length}`);
+    for (const event of result.ignoredEvents) {
+        lines.push(`- ${event.atMs}ms ${event.event}${sourceForSummary(event.source)}: ${event.ignoreReason}`);
+    }
+    return lines.join("\n");
+}

package/dist/senses/voice-realtime-eval-command.js

@@ -0,0 +1,99 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runVoiceRealtimeEvalCommand = runVoiceRealtimeEvalCommand;
+const runtime_1 = require("../nerves/runtime");
+const realtime_eval_1 = require("./voice/realtime-eval");
+const realtime_trace_1 = require("./voice/realtime-trace");
+function parseTraceArgs(argv) {
+    const tracePaths = [];
+    for (let index = 0; index < argv.length; index += 1) {
+        const arg = argv[index];
+        if (arg !== "--trace")
+            throw new Error(`unknown argument: ${arg}`);
+        const tracePath = argv[index + 1];
+        if (!tracePath)
+            throw new Error("--trace requires a file path");
+        tracePaths.push(tracePath);
+        index += 1;
+    }
+    return tracePaths;
+}
+function builtInPayload() {
+    const reports = (0, realtime_eval_1.runBuiltInVoiceRealtimeEvalSuite)();
+    const summary = (0, realtime_eval_1.summarizeVoiceRealtimeEvalSuite)(reports);
+    return {
+        summary,
+        expectedKnownBadFailed: summary.failed === 1 && summary.failedScenarioIds[0] === "voice-known-bad-latency",
+        happyPathPassed: reports.some((report) => report.scenarioId === "voice-happy-path" && report.passed),
+    };
+}
+function traceResultPayload(result) {
+    return {
+        traceId: result.traceId,
+        scenarioId: result.scenarioId,
+        expectedOutcome: result.expectedOutcome,
+        outcomeMatched: result.outcomeMatched,
+        report: result.report,
+        ignoredEvents: result.ignoredEvents.map((event) => ({
+            atMs: event.atMs,
+            event: event.event,
+            source: event.source,
+            ignoreReason: event.ignoreReason,
+        })),
+    };
+}
+function summarizeTraceResults(traces) {
+    const mismatchedScenarioIds = traces.filter((trace) => !trace.outcomeMatched).map((trace) => trace.scenarioId);
+    return {
+        matched: traces.length - mismatchedScenarioIds.length,
+        mismatched: mismatchedScenarioIds.length,
+        total: traces.length,
+        mismatchedScenarioIds,
+    };
+}
+function errorResult(error) {
+    return {
+        exitCode: 1,
+        payload: { error: String(error).replace(/^Error: /, "") },
+    };
+}
+function runVoiceRealtimeEvalCommand(argv) {
+    (0, runtime_1.emitNervesEvent)({
+        component: "senses",
+        event: "senses.voice_realtime_eval_command_start",
+        message: "starting Voice realtime eval command runner",
+        meta: { scenarioId: "voice-eval-command", events: argv.length },
+    });
+    try {
+        const tracePaths = parseTraceArgs(argv);
+        const payload = builtInPayload();
+        if (tracePaths.length > 0) {
+            const traces = tracePaths
+                .map((tracePath) => (0, realtime_trace_1.gradeVoiceRealtimeEvalTrace)((0, realtime_trace_1.loadVoiceRealtimeEvalTraceArtifact)(tracePath)))
+                .map(traceResultPayload);
+            payload.traces = traces;
+            payload.traceSummary = summarizeTraceResults(traces);
+        }
+        const builtInsPassed = Boolean(payload.expectedKnownBadFailed && payload.happyPathPassed);
+        const tracesPassed = payload.traceSummary ? payload.traceSummary.mismatched === 0 : true;
+        const result = { exitCode: builtInsPassed && tracesPassed ? 0 : 1, payload };
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.voice_realtime_eval_command_end",
+            message: "finished Voice realtime eval command runner",
+            meta: { scenarioId: "voice-eval-command", passed: result.exitCode === 0, findings: payload.traceSummary?.mismatched ?? 0 },
+        });
+        return result;
+    }
+    catch (error) {
+        const result = errorResult(error);
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.voice_realtime_eval_command_error",
+            message: "Voice realtime eval command runner failed",
+            meta: { scenarioId: "voice-eval-command", error: result.payload.error },
+            level: "error",
+        });
+        return result;
+    }
+}

package/dist/senses/voice-realtime-eval-entry.js

@@ -0,0 +1,21 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const runtime_1 = require("../nerves/runtime");
+const voice_realtime_eval_command_1 = require("./voice-realtime-eval-command");
+(0, runtime_1.emitNervesEvent)({
+    component: "senses",
+    event: "senses.voice_realtime_eval_start",
+    message: "starting Voice realtime eval command",
+    meta: { scenarioId: "built-in-suite", events: 0 },
+});
+const result = (0, voice_realtime_eval_command_1.runVoiceRealtimeEvalCommand)(process.argv.slice(2));
+(0, runtime_1.emitNervesEvent)({
+    component: "senses",
+    event: "senses.voice_realtime_eval_end",
+    message: "finished Voice realtime eval command",
+    meta: { scenarioId: "built-in-suite", passed: result.exitCode === 0, findings: result.payload.traceSummary?.mismatched ?? result.payload.summary?.failed ?? 0 },
+});
+// eslint-disable-next-line no-console -- terminal UX: eval command summary
+console.log(JSON.stringify(result.payload, null, 2));
+if (result.exitCode !== 0)
+    process.exit(result.exitCode);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ouro.bot/cli",
-  "version": "0.1.0-alpha.583",
+  "version": "0.1.0-alpha.585",
   "main": "dist/heart/daemon/ouro-entry.js",
   "bin": {
     "cli": "dist/heart/daemon/ouro-bot-entry.js",
@@ -26,6 +26,7 @@
     "ouro": "tsc && node dist/heart/daemon/ouro-entry.js",
     "teams": "tsc && node dist/senses/teams-entry.js --agent ouroboros",
     "bluebubbles": "tsc && node dist/senses/bluebubbles/entry.js --agent ouroboros",
+    "voice:eval": "npm run build && node dist/senses/voice-realtime-eval-entry.js",
     "test": "vitest run",
     "test:integration": "npm run build && vitest run --config vitest.integration.config.ts",
     "test:e2e:package": "npm run build && node scripts/package-e2e.cjs",