@ouro.bot/cli 0.1.0-alpha.584 → 0.1.0-alpha.585

package/changelog.json

@@ -1,6 +1,14 @@
 {
   "_note": "This changelog is maintained as part of the PR/version-bump workflow. Agent-curated, not auto-generated. Agents read this file directly via read_file to understand what changed between versions.",
   "versions": [
+    {
+      "version": "0.1.0-alpha.585",
+      "changes": [
+        "Voice realtime evals now support replayable trace JSON artifacts that preserve transport/source metadata, normalize SIP/Twilio/Realtime event names into the shared Voice eval timeline, validate causal ordering, and keep ignored provider-noise events visible without grading them.",
+        "`npm run voice:eval -- --trace <file>` grades one or more trace artifacts alongside the built-in happy/known-bad suite, treating expected-fail traces as canaries while failing on unexpected trace failures or unexpected passes.",
+        "Voice trace replay has golden fixtures for clean calls, barge-in, tool holding phrases, mid-turn hangup, delayed audio/transcript behavior, duplicate/late provider events, and redaction-safe summaries so phone regressions can be debugged without a human live call."
+      ]
+    },
     {
       "version": "0.1.0-alpha.584",
       "changes": [

package/dist/senses/voice/realtime-eval.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.gradeVoiceRealtimeEvalTimeline = gradeVoiceRealtimeEvalTimeline;
 exports.buildVoiceRealtimeEvalHappyPath = buildVoiceRealtimeEvalHappyPath;
+exports.buildVoiceRealtimeEvalDefaultExpectation = buildVoiceRealtimeEvalDefaultExpectation;
 exports.runBuiltInVoiceRealtimeEvalSuite = runBuiltInVoiceRealtimeEvalSuite;
 exports.summarizeVoiceRealtimeEvalSuite = summarizeVoiceRealtimeEvalSuite;
 const runtime_1 = require("../../nerves/runtime");
@@ -232,7 +233,15 @@ function gradeHangup(events, findings) {
 function gradeOverlappingResponses(events, findings) {
     for (const response of allEvents(events, "response.requested")) {
         const activeAudio = allEvents(events, "assistant.audio.started").find((started) => {
-            const done = events.find((event) => event.type === "assistant.audio.done" && event.atMs >= started.atMs);
+            const done = events.find((event) => {
+                if (event.atMs < started.atMs)
+                    return false;
+                if (event.type === "response.truncated" || event.type === "call.hangup.requested")
+                    return true;
+                if (event.type !== "assistant.audio.done" && event.type !== "assistant.transcript.done")
+                    return false;
+                return !started.correlationId || !event.correlationId || event.correlationId === started.correlationId;
+            });
             return response.atMs > started.atMs && (!done || response.atMs < done.atMs);
         });
         if (activeAudio) {
@@ -337,7 +346,7 @@ function buildVoiceRealtimeEvalHappyPath() {
         { type: "call.ended", atMs: 5_100, source: { transport: "openai-sip", id: "sip-call-1" } },
     ];
 }
-function builtInExpectation() {
+function buildVoiceRealtimeEvalDefaultExpectation() {
     return {
         maxFirstAssistantAudioMs: 1_200,
         maxUserTurnResponseMs: 900,
@@ -367,7 +376,7 @@ function buildKnownBadLatencyPath() {
     });
 }
 function runBuiltInVoiceRealtimeEvalSuite() {
-    const expectation = builtInExpectation();
+    const expectation = buildVoiceRealtimeEvalDefaultExpectation();
     return [
         gradeVoiceRealtimeEvalTimeline("voice-happy-path", buildVoiceRealtimeEvalHappyPath(), expectation),
         gradeVoiceRealtimeEvalTimeline("voice-known-bad-latency", buildKnownBadLatencyPath(), expectation),

package/dist/senses/voice/realtime-trace.js

@@ -0,0 +1,402 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseVoiceRealtimeEvalTraceArtifact = parseVoiceRealtimeEvalTraceArtifact;
+exports.loadVoiceRealtimeEvalTraceArtifact = loadVoiceRealtimeEvalTraceArtifact;
+exports.resolveVoiceRealtimeEvalTraceExpectation = resolveVoiceRealtimeEvalTraceExpectation;
+exports.traceArtifactToVoiceRealtimeEvalTimeline = traceArtifactToVoiceRealtimeEvalTimeline;
+exports.gradeVoiceRealtimeEvalTrace = gradeVoiceRealtimeEvalTrace;
+exports.formatVoiceRealtimeEvalTraceReport = formatVoiceRealtimeEvalTraceReport;
+const fs = __importStar(require("fs"));
+const runtime_1 = require("../../nerves/runtime");
+const realtime_eval_1 = require("./realtime-eval");
+const transports = new Set([
+    "browser-meeting",
+    "openai-realtime-control",
+    "openai-sip",
+    "twilio-media-stream",
+    "voice-eval",
+]);
+const normalizedEvents = new Set([
+    "assistant.audio.done",
+    "assistant.audio.started",
+    "assistant.transcript.done",
+    "barge_in.detected",
+    "call.connected",
+    "call.ended",
+    "call.hangup.requested",
+    "response.requested",
+    "response.truncated",
+    "session.updated",
+    "tool.call.completed",
+    "tool.call.started",
+    "tool.holding.started",
+    "transport.playback_cleared",
+    "user.transcript.done",
+    "voice.context.injected",
+]);
+const rawEventMap = new Map([
+    ["openai.realtime.call.hangup.sent", "call.hangup.requested"],
+    ["openai.realtime.conversation.item.truncate.sent", "response.truncated"],
+    ["openai.realtime.input_audio_buffer.speech_started", "barge_in.detected"],
+    ["openai.realtime.input_audio_transcription.completed", "user.transcript.done"],
+    ["openai.realtime.output_audio.delta", "assistant.audio.started"],
+    ["openai.realtime.output_audio.done", "assistant.audio.done"],
+    ["openai.realtime.output_audio_transcript.done", "assistant.transcript.done"],
+    ["openai.realtime.response.create.sent", "response.requested"],
+    ["openai.realtime.response.function_call_arguments.done", "tool.call.started"],
+    ["openai.realtime.session.updated", "session.updated"],
+    ["openai.realtime.tool.completed", "tool.call.completed"],
+    ["openai.sip.call.connected", "call.connected"],
+    ["openai.sip.call.ended", "call.ended"],
+    ["twilio.call.ended", "call.ended"],
+    ["twilio.media.clear.sent", "transport.playback_cleared"],
+    ["twilio.media.start", "call.connected"],
+    ["voice.hangup.requested", "call.hangup.requested"],
+    ["voice.tool_holding.started", "tool.holding.started"],
+]);
+const expectedOutcomes = new Set(["expected-fail", "fail", "pass"]);
+const expectationProfiles = new Set(["voice-phone-default"]);
+function objectRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value) ? value : undefined;
+}
+function label(prefix, detail) {
+    return prefix ? `${prefix}: ${detail}` : detail;
+}
+function requiredString(value, name, sourceLabel) {
+    if (typeof value !== "string" || value.trim() === "")
+        throw new Error(label(sourceLabel, `${name} must be a non-empty string`));
+    return value.trim();
+}
+function optionalString(value, name, sourceLabel) {
+    if (value === undefined)
+        return undefined;
+    if (typeof value !== "string")
+        throw new Error(label(sourceLabel, `${name} must be a string`));
+    return value;
+}
+function parseSource(value, sourceLabel) {
+    if (value === undefined)
+        return undefined;
+    const raw = objectRecord(value);
+    if (!raw)
+        throw new Error(label(sourceLabel, "source must be an object"));
+    const transport = raw.transport;
+    if (typeof transport !== "string" || !transports.has(transport)) {
+        throw new Error(label(sourceLabel, "source.transport is unsupported"));
+    }
+    const id = optionalString(raw.id, "source.id", sourceLabel);
+    return id === undefined ? { transport: transport } : { transport: transport, id };
+}
+function parseTurnDetection(value, sourceLabel) {
+    if (value === undefined)
+        return undefined;
+    const session = objectRecord(value);
+    if (!session)
+        throw new Error(label(sourceLabel, "session must be an object"));
+    const turnDetection = objectRecord(session.turnDetection);
+    if (!turnDetection)
+        return {};
+    const createResponse = turnDetection.createResponse;
+    const interruptResponse = turnDetection.interruptResponse;
+    if (createResponse !== undefined && typeof createResponse !== "boolean") {
+        throw new Error(label(sourceLabel, "session.turnDetection.createResponse must be boolean"));
+    }
+    if (interruptResponse !== undefined && typeof interruptResponse !== "boolean") {
+        throw new Error(label(sourceLabel, "session.turnDetection.interruptResponse must be boolean"));
+    }
+    return { turnDetection: { createResponse, interruptResponse } };
+}
+function validateExpectation(expectation, sourceLabel) {
+    const budgets = [
+        expectation.maxFirstAssistantAudioMs,
+        expectation.maxUserTurnResponseMs,
+        expectation.maxToolPresenceMs,
+        expectation.maxBargeInClearMs,
+        expectation.maxBargeInTruncateMs,
+    ];
+    if (budgets.some((budget) => typeof budget !== "number" || !Number.isFinite(budget) || budget <= 0)) {
+        throw new Error(label(sourceLabel, "expectation latency budgets must be positive finite numbers"));
+    }
+    return expectation;
+}
+function parseExpectation(value, sourceLabel) {
+    const raw = objectRecord(value);
+    if (!raw)
+        throw new Error(label(sourceLabel, "expectation must be an object"));
+    return validateExpectation(raw, sourceLabel);
+}
+function parseTraceEvent(value, index, sourceLabel) {
+    const eventLabel = `${sourceLabel} event[${index}]`;
+    const raw = objectRecord(value);
+    if (!raw)
+        throw new Error(label(eventLabel, "must be an object"));
+    const event = requiredString(raw.event, "event", eventLabel);
+    const ignored = raw.ignored === undefined ? undefined : raw.ignored;
+    if (ignored !== undefined && typeof ignored !== "boolean")
+        throw new Error(label(eventLabel, "ignored must be boolean"));
+    if (!ignored && !normalizedEvents.has(event) && !rawEventMap.has(event)) {
+        throw new Error(label(eventLabel, `unknown trace event ${event}`));
+    }
+    const atMs = raw.atMs;
+    if (typeof atMs !== "number" || !Number.isFinite(atMs)) {
+        throw new Error(label(eventLabel, "atMs must be a finite number"));
+    }
+    let role;
+    if (raw.role !== undefined) {
+        if (raw.role !== "assistant" && raw.role !== "user")
+            throw new Error(label(eventLabel, "role must be assistant or user"));
+        role = raw.role;
+    }
+    const parsed = {
+        atMs,
+        event,
+        source: parseSource(raw.source, eventLabel),
+        correlationId: optionalString(raw.correlationId, "correlationId", eventLabel),
+        text: optionalString(raw.text, "text", eventLabel),
+        role,
+        toolName: optionalString(raw.toolName, "toolName", eventLabel),
+        friendId: optionalString(raw.friendId, "friendId", eventLabel),
+        sessionKey: optionalString(raw.sessionKey, "sessionKey", eventLabel),
+        session: parseTurnDetection(raw.session, eventLabel),
+        ignored: ignored || undefined,
+        ignoreReason: optionalString(raw.ignoreReason, "ignoreReason", eventLabel),
+    };
+    if (parsed.ignored && !parsed.ignoreReason)
+        throw new Error(label(eventLabel, "ignored events require ignoreReason"));
+    return parsed;
+}
+function parseVoiceRealtimeEvalTraceArtifact(value, sourceLabel = "voice trace artifact") {
+    const raw = objectRecord(value);
+    if (!raw || raw.schemaVersion !== 1)
+        throw new Error(label(sourceLabel, "schemaVersion must be 1"));
+    const expectedOutcome = raw.expectedOutcome;
+    if (typeof expectedOutcome !== "string" || !expectedOutcomes.has(expectedOutcome)) {
+        throw new Error(label(sourceLabel, "expectedOutcome must be pass, fail, or expected-fail"));
+    }
+    const expectationProfile = raw.expectationProfile;
+    if (expectationProfile !== undefined && (typeof expectationProfile !== "string" || !expectationProfiles.has(expectationProfile))) {
+        throw new Error(label(sourceLabel, "expectationProfile is unsupported"));
+    }
+    const hasInlineExpectation = raw.expectation !== undefined;
+    if (hasInlineExpectation === (expectationProfile !== undefined)) {
+        throw new Error(label(sourceLabel, "provide exactly one of expectation or expectationProfile"));
+    }
+    if (!Array.isArray(raw.events) || raw.events.length === 0) {
+        throw new Error(label(sourceLabel, "events must contain at least one event"));
+    }
+    const redacted = raw.redacted;
+    if (redacted !== undefined && typeof redacted !== "boolean")
+        throw new Error(label(sourceLabel, "redacted must be boolean"));
+    return {
+        schemaVersion: 1,
+        traceId: requiredString(raw.traceId, "traceId", sourceLabel),
+        scenarioId: requiredString(raw.scenarioId, "scenarioId", sourceLabel),
+        expectedOutcome: expectedOutcome,
+        expectation: hasInlineExpectation ? parseExpectation(raw.expectation, sourceLabel) : undefined,
+        expectationProfile: expectationProfile,
+        redacted: redacted || undefined,
+        events: raw.events.map((event, index) => parseTraceEvent(event, index, sourceLabel)),
+    };
+}
+function loadVoiceRealtimeEvalTraceArtifact(filePath) {
+    let raw;
+    try {
+        raw = fs.readFileSync(filePath, "utf8");
+    }
+    catch {
+        throw new Error(`${filePath}: failed to read trace artifact`);
+    }
+    try {
+        return parseVoiceRealtimeEvalTraceArtifact(JSON.parse(raw), filePath);
+    }
+    catch (error) {
+        if (error instanceof SyntaxError)
+            throw new Error(`${filePath}: invalid JSON: ${error.message}`);
+        throw error;
+    }
+}
+function resolveVoiceRealtimeEvalTraceExpectation(artifact) {
+    if (artifact.expectationProfile === "voice-phone-default")
+        return (0, realtime_eval_1.buildVoiceRealtimeEvalDefaultExpectation)();
+    if (artifact.expectation)
+        return artifact.expectation;
+    throw new Error(`${artifact.traceId}: trace artifact has no expectation contract`);
+}
+function eventTypeFor(event) {
+    if (normalizedEvents.has(event.event))
+        return event.event;
+    const type = rawEventMap.get(event.event);
+    if (!type)
+        throw new Error(`unknown trace event ${event.event}`);
+    return type;
+}
+function toTimelineEvent(event, redacted) {
+    const type = eventTypeFor(event);
+    return {
+        type,
+        atMs: event.atMs,
+        source: event.source,
+        correlationId: event.correlationId,
+        text: redacted ? undefined : event.text,
+        role: event.role,
+        toolName: event.toolName,
+        friendId: event.friendId,
+        sessionKey: event.sessionKey,
+        session: event.session,
+    };
+}
+function sortedTimeline(events) {
+    return events
+        .map((event, index) => ({ event, index }))
+        .sort((left, right) => left.event.atMs - right.event.atMs || left.index - right.index)
+        .map(({ event }) => event);
+}
+function findFirst(events, type) {
+    return events.find((event) => event.type === type);
+}
+function validateCausalTimeline(artifact, timeline) {
+    for (let index = 0; index < artifact.events.length; index += 1) {
+        const event = artifact.events[index];
+        if (event.ignored)
+            continue;
+        if (event.atMs < 0) {
+            throw new Error(`${artifact.traceId} event[${index}] atMs must be a nonnegative finite number`);
+        }
+    }
+    const connected = findFirst(timeline, "call.connected");
+    const audio = findFirst(timeline, "assistant.audio.started");
+    if (connected && audio && audio.atMs < connected.atMs) {
+        throw new Error(`${artifact.traceId}: assistant audio started before call.connected`);
+    }
+    for (const response of timeline.filter((event) => event.type === "response.requested" && event.correlationId)) {
+        const transcript = timeline.find((event) => event.type === "user.transcript.done" && event.correlationId === response.correlationId);
+        if (transcript && response.atMs < transcript.atMs) {
+            throw new Error(`${artifact.traceId}: response.requested for ${response.correlationId} occurred before user.transcript.done`);
+        }
+    }
+    for (const completed of timeline.filter((event) => event.type === "tool.call.completed" && event.correlationId)) {
+        const started = timeline.find((event) => event.type === "tool.call.started" && event.correlationId === completed.correlationId);
+        if (started && completed.atMs < started.atMs) {
+            throw new Error(`${artifact.traceId}: tool.call.completed for ${completed.correlationId} occurred before tool.call.started`);
+        }
+    }
+    const expectation = resolveVoiceRealtimeEvalTraceExpectation(artifact);
+    if (expectation.requireHangup) {
+        const ended = findFirst(timeline, "call.ended");
+        const hangup = findFirst(timeline, "call.hangup.requested");
+        if (ended && hangup && ended.atMs < hangup.atMs) {
+            throw new Error(`${artifact.traceId}: call.ended occurred before call.hangup.requested`);
+        }
+    }
+}
+function traceArtifactToVoiceRealtimeEvalTimeline(artifact) {
+    const timeline = sortedTimeline(artifact.events
+        .filter((event) => !event.ignored)
+        .map((event) => toTimelineEvent(event, Boolean(artifact.redacted))));
+    validateCausalTimeline(artifact, timeline);
+    return timeline;
+}
+function expectedOutcomeMatched(expectedOutcome, passed) {
+    if (expectedOutcome === "pass")
+        return passed;
+    return !passed;
+}
+function gradeVoiceRealtimeEvalTrace(artifact) {
+    (0, runtime_1.emitNervesEvent)({
+        component: "senses",
+        event: "senses.voice_realtime_trace_replay_start",
+        message: "starting Voice realtime trace replay",
+        meta: { scenarioId: artifact.scenarioId, events: artifact.events.length },
+    });
+    const timeline = traceArtifactToVoiceRealtimeEvalTimeline(artifact);
+    const report = (0, realtime_eval_1.gradeVoiceRealtimeEvalTimeline)(artifact.scenarioId, timeline, resolveVoiceRealtimeEvalTraceExpectation(artifact));
+    const result = {
+        artifact,
+        traceId: artifact.traceId,
+        scenarioId: artifact.scenarioId,
+        expectedOutcome: artifact.expectedOutcome,
+        outcomeMatched: expectedOutcomeMatched(artifact.expectedOutcome, report.passed),
+        report,
+        timeline,
+        ignoredEvents: artifact.events.filter((event) => event.ignored),
+    };
+    (0, runtime_1.emitNervesEvent)({
+        component: "senses",
+        event: "senses.voice_realtime_trace_replay_end",
+        message: "finished Voice realtime trace replay",
+        meta: { scenarioId: artifact.scenarioId, passed: report.passed, findings: report.findings.length },
+    });
+    return result;
+}
+function textForSummary(result, event) {
+    if (result.artifact.redacted && (event.type === "assistant.transcript.done" ||
+        event.type === "user.transcript.done" ||
+        event.type === "voice.context.injected")) {
+        return " [redacted]";
+    }
+    if (event.text === undefined)
+        return "";
+    return ` "${event.text}"`;
+}
+function sourceForSummary(source) {
+    if (!source)
+        return "";
+    return source.id ? ` ${source.transport}/${source.id}` : ` ${source.transport}`;
+}
+function formatVoiceRealtimeEvalTraceReport(result) {
+    const lines = [
+        `trace ${result.traceId} scenario ${result.scenarioId}`,
+        `expected: ${result.expectedOutcome}; report passed: ${result.report.passed}; outcome matched: ${result.outcomeMatched}`,
+        `transports: ${result.report.transportSources.join(", ") || "none"}`,
+        `metrics: ${JSON.stringify(result.report.metrics)}`,
+    ];
+    if (result.report.findings.length > 0) {
+        lines.push("findings:");
+        for (const finding of result.report.findings) {
+            const message = result.artifact.redacted ? "[redacted]" : finding.message;
+            lines.push(`- ${finding.code}${finding.atMs === undefined ? "" : ` at ${finding.atMs}ms`}: ${message}`);
+        }
+    }
+    lines.push("events:");
+    for (const event of result.timeline) {
+        lines.push(`- ${event.atMs}ms ${event.type}${sourceForSummary(event.source)}${textForSummary(result, event)}`);
+    }
+    lines.push(`ignored provider events: ${result.ignoredEvents.length}`);
+    for (const event of result.ignoredEvents) {
+        lines.push(`- ${event.atMs}ms ${event.event}${sourceForSummary(event.source)}: ${event.ignoreReason}`);
+    }
+    return lines.join("\n");
+}

package/dist/senses/voice-realtime-eval-command.js

@@ -0,0 +1,99 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runVoiceRealtimeEvalCommand = runVoiceRealtimeEvalCommand;
+const runtime_1 = require("../nerves/runtime");
+const realtime_eval_1 = require("./voice/realtime-eval");
+const realtime_trace_1 = require("./voice/realtime-trace");
+function parseTraceArgs(argv) {
+    const tracePaths = [];
+    for (let index = 0; index < argv.length; index += 1) {
+        const arg = argv[index];
+        if (arg !== "--trace")
+            throw new Error(`unknown argument: ${arg}`);
+        const tracePath = argv[index + 1];
+        if (!tracePath)
+            throw new Error("--trace requires a file path");
+        tracePaths.push(tracePath);
+        index += 1;
+    }
+    return tracePaths;
+}
+function builtInPayload() {
+    const reports = (0, realtime_eval_1.runBuiltInVoiceRealtimeEvalSuite)();
+    const summary = (0, realtime_eval_1.summarizeVoiceRealtimeEvalSuite)(reports);
+    return {
+        summary,
+        expectedKnownBadFailed: summary.failed === 1 && summary.failedScenarioIds[0] === "voice-known-bad-latency",
+        happyPathPassed: reports.some((report) => report.scenarioId === "voice-happy-path" && report.passed),
+    };
+}
+function traceResultPayload(result) {
+    return {
+        traceId: result.traceId,
+        scenarioId: result.scenarioId,
+        expectedOutcome: result.expectedOutcome,
+        outcomeMatched: result.outcomeMatched,
+        report: result.report,
+        ignoredEvents: result.ignoredEvents.map((event) => ({
+            atMs: event.atMs,
+            event: event.event,
+            source: event.source,
+            ignoreReason: event.ignoreReason,
+        })),
+    };
+}
+function summarizeTraceResults(traces) {
+    const mismatchedScenarioIds = traces.filter((trace) => !trace.outcomeMatched).map((trace) => trace.scenarioId);
+    return {
+        matched: traces.length - mismatchedScenarioIds.length,
+        mismatched: mismatchedScenarioIds.length,
+        total: traces.length,
+        mismatchedScenarioIds,
+    };
+}
+function errorResult(error) {
+    return {
+        exitCode: 1,
+        payload: { error: String(error).replace(/^Error: /, "") },
+    };
+}
+function runVoiceRealtimeEvalCommand(argv) {
+    (0, runtime_1.emitNervesEvent)({
+        component: "senses",
+        event: "senses.voice_realtime_eval_command_start",
+        message: "starting Voice realtime eval command runner",
+        meta: { scenarioId: "voice-eval-command", events: argv.length },
+    });
+    try {
+        const tracePaths = parseTraceArgs(argv);
+        const payload = builtInPayload();
+        if (tracePaths.length > 0) {
+            const traces = tracePaths
+                .map((tracePath) => (0, realtime_trace_1.gradeVoiceRealtimeEvalTrace)((0, realtime_trace_1.loadVoiceRealtimeEvalTraceArtifact)(tracePath)))
+                .map(traceResultPayload);
+            payload.traces = traces;
+            payload.traceSummary = summarizeTraceResults(traces);
+        }
+        const builtInsPassed = Boolean(payload.expectedKnownBadFailed && payload.happyPathPassed);
+        const tracesPassed = payload.traceSummary ? payload.traceSummary.mismatched === 0 : true;
+        const result = { exitCode: builtInsPassed && tracesPassed ? 0 : 1, payload };
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.voice_realtime_eval_command_end",
+            message: "finished Voice realtime eval command runner",
+            meta: { scenarioId: "voice-eval-command", passed: result.exitCode === 0, findings: payload.traceSummary?.mismatched ?? 0 },
+        });
+        return result;
+    }
+    catch (error) {
+        const result = errorResult(error);
+        (0, runtime_1.emitNervesEvent)({
+            component: "senses",
+            event: "senses.voice_realtime_eval_command_error",
+            message: "Voice realtime eval command runner failed",
+            meta: { scenarioId: "voice-eval-command", error: result.payload.error },
+            level: "error",
+        });
+        return result;
+    }
+}

package/dist/senses/voice-realtime-eval-entry.js

@@ -1,25 +1,21 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-const realtime_eval_1 = require("./voice/realtime-eval");
 const runtime_1 = require("../nerves/runtime");
+const voice_realtime_eval_command_1 = require("./voice-realtime-eval-command");
 (0, runtime_1.emitNervesEvent)({
     component: "senses",
     event: "senses.voice_realtime_eval_start",
     message: "starting Voice realtime eval command",
     meta: { scenarioId: "built-in-suite", events: 0 },
 });
-const reports = (0, realtime_eval_1.runBuiltInVoiceRealtimeEvalSuite)();
-const summary = (0, realtime_eval_1.summarizeVoiceRealtimeEvalSuite)(reports);
-const expectedKnownBadFailed = summary.failed === 1 && summary.failedScenarioIds[0] === "voice-known-bad-latency";
-const happyPathPassed = reports.some((report) => report.scenarioId === "voice-happy-path" && report.passed);
+const result = (0, voice_realtime_eval_command_1.runVoiceRealtimeEvalCommand)(process.argv.slice(2));
 (0, runtime_1.emitNervesEvent)({
     component: "senses",
     event: "senses.voice_realtime_eval_end",
     message: "finished Voice realtime eval command",
-    meta: { scenarioId: "built-in-suite", passed: expectedKnownBadFailed && happyPathPassed, findings: summary.failed },
+    meta: { scenarioId: "built-in-suite", passed: result.exitCode === 0, findings: result.payload.traceSummary?.mismatched ?? result.payload.summary?.failed ?? 0 },
 });
 // eslint-disable-next-line no-console -- terminal UX: eval command summary
-console.log(JSON.stringify({ summary, expectedKnownBadFailed, happyPathPassed }, null, 2));
-if (!expectedKnownBadFailed || !happyPathPassed) {
-    process.exit(1);
-}
+console.log(JSON.stringify(result.payload, null, 2));
+if (result.exitCode !== 0)
+    process.exit(result.exitCode);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ouro.bot/cli",
-  "version": "0.1.0-alpha.584",
+  "version": "0.1.0-alpha.585",
   "main": "dist/heart/daemon/ouro-entry.js",
   "bin": {
     "cli": "dist/heart/daemon/ouro-bot-entry.js",