@absolutejs/voice 0.0.22-beta.42 → 0.0.22-beta.44

package/dist/evalRoutes.d.ts

@@ -28,17 +28,95 @@ export type VoiceEvalReport = {
     total: number;
     trend: VoiceEvalTrendBucket[];
 };
+export type VoiceEvalBaselineSummary = {
+    failed: number;
+    failedSessionIds: string[];
+    passRate: number;
+    passed: number;
+    total: number;
+};
+export type VoiceEvalBaselineComparison = {
+    baseline: VoiceEvalBaselineSummary;
+    checkedAt: number;
+    current: VoiceEvalBaselineSummary;
+    deltas: {
+        failed: number;
+        passRate: number;
+        passed: number;
+        total: number;
+    };
+    newFailedSessionIds: string[];
+    recoveredSessionIds: string[];
+    reasons: string[];
+    status: VoiceEvalStatus;
+};
+export type VoiceEvalBaselineComparisonOptions = {
+    failOnNewFailedSessions?: boolean;
+    maxFailedDelta?: number;
+    maxPassRateDrop?: number;
+};
+export type VoiceEvalBaselineStore = {
+    get: () => Promise<VoiceEvalReport | undefined>;
+    set: (report: VoiceEvalReport) => Promise<void>;
+};
+export type VoiceScenarioEvalDefinition = {
+    description?: string;
+    forbiddenHandoffActions?: string[];
+    forbiddenLifecycleTypes?: string[];
+    id: string;
+    label?: string;
+    maxProviderErrors?: number;
+    maxSessionErrors?: number;
+    minSessions?: number;
+    minTurns?: number;
+    requiredAssistantIncludes?: string[];
+    requiredDisposition?: string;
+    requiredHandoffActions?: string[];
+    requiredLifecycleTypes?: string[];
+    requiredPayloadPaths?: string[];
+    requiredTranscriptIncludes?: string[];
+    scenarioId?: string;
+};
+export type VoiceScenarioEvalSessionResult = {
+    eventCount: number;
+    issues: string[];
+    sessionId: string;
+    status: VoiceEvalStatus;
+};
+export type VoiceScenarioEvalResult = {
+    description?: string;
+    failed: number;
+    id: string;
+    issues: string[];
+    label: string;
+    matchedSessions: number;
+    passed: number;
+    sessions: VoiceScenarioEvalSessionResult[];
+    status: VoiceEvalStatus;
+};
+export type VoiceScenarioEvalReport = {
+    checkedAt: number;
+    failed: number;
+    passed: number;
+    scenarios: VoiceScenarioEvalResult[];
+    status: VoiceEvalStatus;
+    total: number;
+};
 export type VoiceEvalLink = {
     href: string;
     label: string;
 };
 export type VoiceEvalRoutesOptions = {
+    baseline?: VoiceEvalReport | (() => Promise<VoiceEvalReport | undefined>);
+    baselineComparison?: VoiceEvalBaselineComparisonOptions;
+    baselineStore?: VoiceEvalBaselineStore;
     events?: StoredVoiceTraceEvent[];
     headers?: HeadersInit;
     links?: VoiceEvalLink[];
     limit?: number;
     name?: string;
     path?: string;
+    scenarios?: VoiceScenarioEvalDefinition[];
     store?: VoiceTraceEventStore;
     thresholds?: VoiceQualityThresholds;
     title?: string;
@@ -49,10 +127,25 @@ export declare const runVoiceSessionEvals: (options?: {
     store?: VoiceTraceEventStore;
     thresholds?: VoiceQualityThresholds;
 }) => Promise<VoiceEvalReport>;
+export declare const runVoiceScenarioEvals: (options?: {
+    events?: StoredVoiceTraceEvent[];
+    scenarios?: VoiceScenarioEvalDefinition[];
+    store?: VoiceTraceEventStore;
+}) => Promise<VoiceScenarioEvalReport>;
+export declare const compareVoiceEvalBaseline: (currentReport: VoiceEvalReport, baselineReport: VoiceEvalReport, options?: VoiceEvalBaselineComparisonOptions) => VoiceEvalBaselineComparison;
+export declare const createVoiceFileEvalBaselineStore: (filePath: string) => VoiceEvalBaselineStore;
 export declare const renderVoiceEvalHTML: (report: VoiceEvalReport, options?: {
     links?: VoiceEvalLink[];
     title?: string;
 }) => string;
+export declare const renderVoiceEvalBaselineHTML: (comparison: VoiceEvalBaselineComparison, options?: {
+    links?: VoiceEvalLink[];
+    title?: string;
+}) => string;
+export declare const renderVoiceScenarioEvalHTML: (report: VoiceScenarioEvalReport, options?: {
+    links?: VoiceEvalLink[];
+    title?: string;
+}) => string;
 export declare const createVoiceEvalRoutes: (options: VoiceEvalRoutesOptions) => Elysia<"", {
     decorator: {};
     store: {};

package/dist/index.d.ts

@@ -2,7 +2,7 @@ export { voice } from './plugin';
 export { createVoiceAssistant, createVoiceExperiment, summarizeVoiceAssistantRuns } from './assistant';
 export { createVoiceAssistantHealthHTMLHandler, createVoiceAssistantHealthJSONHandler, createVoiceAssistantHealthRoutes, renderVoiceAssistantHealthHTML, summarizeVoiceAssistantHealth } from './assistantHealth';
 export { buildVoiceDiagnosticsMarkdown, createVoiceDiagnosticsRoutes, resolveVoiceDiagnosticsTraceFilter } from './diagnosticsRoutes';
-export { createVoiceEvalRoutes, renderVoiceEvalHTML, runVoiceSessionEvals } from './evalRoutes';
+export { compareVoiceEvalBaseline, createVoiceFileEvalBaselineStore, createVoiceEvalRoutes, renderVoiceEvalBaselineHTML, renderVoiceEvalHTML, renderVoiceScenarioEvalHTML, runVoiceScenarioEvals, runVoiceSessionEvals } from './evalRoutes';
 export { createVoiceSessionListRoutes, createVoiceSessionReplayHTMLHandler, createVoiceSessionReplayJSONHandler, createVoiceSessionReplayRoutes, createVoiceSessionsHTMLHandler, createVoiceSessionsJSONHandler, renderVoiceSessionsHTML, summarizeVoiceSessions, summarizeVoiceSessionReplay } from './sessionReplay';
 export { createVoiceAgent, createVoiceAgentSquad, createVoiceAgentTool } from './agent';
 export { createStoredVoiceCallReviewArtifact, createStoredVoiceExternalObjectMap, createStoredVoiceIntegrationEvent, createStoredVoiceOpsTask, createVoiceFileExternalObjectMapStore, createVoiceFileAssistantMemoryStore, createVoiceFileIntegrationEventStore, createVoiceFileReviewStore, createVoiceFileRuntimeStorage, createVoiceFileSessionStore, createVoiceFileTaskStore, createVoiceFileTraceSinkDeliveryStore, createVoiceFileTraceEventStore } from './fileStore';
@@ -40,7 +40,7 @@ export type { VoiceAssistant, VoiceAssistantArtifactPlan, VoiceAssistantExperime
 export type { VoiceAssistantHealthFailure, VoiceAssistantHealthHTMLHandlerOptions, VoiceAssistantHealthRoutesOptions, VoiceAssistantHealthSummary, VoiceAssistantHealthSummaryOptions } from './assistantHealth';
 export type { VoiceAssistantMemoryBinding, VoiceAssistantMemoryHandle, VoiceAssistantMemoryOptions, VoiceAssistantMemoryRecord, VoiceAssistantMemoryStore } from './assistantMemory';
 export type { VoiceDiagnosticsRoutesOptions } from './diagnosticsRoutes';
-export type { VoiceEvalLink, VoiceEvalReport, VoiceEvalRoutesOptions, VoiceEvalSessionReport, VoiceEvalStatus, VoiceEvalTrendBucket } from './evalRoutes';
+export type { VoiceEvalBaselineComparison, VoiceEvalBaselineComparisonOptions, VoiceEvalBaselineStore, VoiceEvalBaselineSummary, VoiceEvalLink, VoiceEvalReport, VoiceEvalRoutesOptions, VoiceEvalSessionReport, VoiceEvalStatus, VoiceEvalTrendBucket, VoiceScenarioEvalDefinition, VoiceScenarioEvalReport, VoiceScenarioEvalResult, VoiceScenarioEvalSessionResult } from './evalRoutes';
 export type { VoiceSessionListHTMLHandlerOptions, VoiceSessionListItem, VoiceSessionListOptions, VoiceSessionListRoutesOptions, VoiceSessionListStatus, VoiceSessionReplay, VoiceSessionReplayHTMLHandlerOptions, VoiceSessionReplayOptions, VoiceSessionReplayRoutesOptions, VoiceSessionReplayTurn } from './sessionReplay';
 export type { AnthropicVoiceAssistantModelOptions, GeminiVoiceAssistantModelOptions, OpenAIVoiceAssistantModelOptions, VoiceProviderRouterEvent, VoiceProviderRouterFallbackMode, VoiceProviderRouterHealthOptions, VoiceProviderRouterOptions, VoiceProviderRouterPolicy, VoiceProviderRouterProviderHealth, VoiceProviderRouterProviderProfile, VoiceJSONAssistantModelHandler, VoiceJSONAssistantModelOptions } from './modelAdapters';
 export type { VoiceProviderHealthStatus, VoiceProviderHealthSummary, VoiceProviderHealthSummaryOptions } from './providerHealth';

package/dist/index.js

@@ -7625,6 +7625,8 @@ var createVoiceDiagnosticsRoutes = (options) => {
 };
 // src/evalRoutes.ts
 import { Elysia as Elysia7 } from "elysia";
+import { mkdir } from "fs/promises";
+import { dirname } from "path";
 
 // src/qualityRoutes.ts
 import { Elysia as Elysia6 } from "elysia";
@@ -7974,6 +7976,25 @@ var createVoiceQualityRoutes = (options) => {
 
 // src/evalRoutes.ts
 var escapeHtml9 = (value) => value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;").replaceAll("'", "&#39;");
+var rate2 = (count, total) => count / Math.max(1, total);
+var normalizeSearchText = (value) => value.trim().toLowerCase();
+var getString6 = (value) => typeof value === "string" ? value : undefined;
+var getObject = (value) => value && typeof value === "object" && !Array.isArray(value) ? value : undefined;
+var getPathValue = (value, path) => {
+  let current = value;
+  for (const part of path.split(".").filter(Boolean)) {
+    const record = getObject(current);
+    if (!record || !(part in record)) {
+      return;
+    }
+    current = record[part];
+  }
+  return current;
+};
+var includesAll = (haystack, needles) => {
+  const normalized = normalizeSearchText(haystack);
+  return needles.filter((needle) => !normalized.includes(normalizeSearchText(needle)));
+};
 var sessionTime = (events) => {
   const sorted = filterVoiceTraceEvents(events);
   return {
@@ -8044,7 +8065,171 @@ var runVoiceSessionEvals = async (options = {}) => {
     trend: buildTrend(limitedSessions)
   };
 };
+var getSessionText = (events, type) => events.filter((event) => event.type === type).map((event) => getString6(event.payload.text)).filter((text) => Boolean(text?.trim())).join(`
+`);
+var countProviderErrors = (events) => events.filter((event) => event.type === "session.error" && (event.payload.providerStatus === "error" || typeof event.payload.provider === "string")).length;
+var evaluateScenarioSession = (scenario, sessionId, events) => {
+  const issues = [];
+  const committedText = getSessionText(events, "turn.committed");
+  const assistantText = getSessionText(events, "turn.assistant");
+  const lifecycleTypes = events.filter((event) => event.type === "call.lifecycle").map((event) => getString6(event.payload.type)).filter((type) => Boolean(type));
+  const dispositions = events.filter((event) => event.type === "call.lifecycle").map((event) => getString6(event.payload.disposition)).filter((disposition) => Boolean(disposition));
+  const handoffActions = events.filter((event) => event.type === "call.handoff").map((event) => getString6(event.payload.action)).filter((action) => Boolean(action));
+  const turnCount = events.filter((event) => event.type === "turn.committed").length;
+  const sessionErrorCount = events.filter((event) => event.type === "session.error").length;
+  const providerErrorCount = countProviderErrors(events);
+  for (const missing of includesAll(committedText, scenario.requiredTranscriptIncludes ?? [])) {
+    issues.push(`Missing transcript text: ${missing}`);
+  }
+  for (const missing of includesAll(assistantText, scenario.requiredAssistantIncludes ?? [])) {
+    issues.push(`Missing assistant text: ${missing}`);
+  }
+  for (const type of scenario.requiredLifecycleTypes ?? []) {
+    if (!lifecycleTypes.includes(type)) {
+      issues.push(`Missing lifecycle event: ${type}`);
+    }
+  }
+  for (const type of scenario.forbiddenLifecycleTypes ?? []) {
+    if (lifecycleTypes.includes(type)) {
+      issues.push(`Forbidden lifecycle event occurred: ${type}`);
+    }
+  }
+  for (const action of scenario.requiredHandoffActions ?? []) {
+    if (!handoffActions.includes(action)) {
+      issues.push(`Missing handoff action: ${action}`);
+    }
+  }
+  for (const action of scenario.forbiddenHandoffActions ?? []) {
+    if (handoffActions.includes(action)) {
+      issues.push(`Forbidden handoff action occurred: ${action}`);
+    }
+  }
+  if (scenario.requiredDisposition && !dispositions.includes(scenario.requiredDisposition)) {
+    issues.push(`Missing disposition: ${scenario.requiredDisposition}`);
+  }
+  if (scenario.minTurns !== undefined && turnCount < scenario.minTurns) {
+    issues.push(`Expected at least ${scenario.minTurns} turn(s), saw ${turnCount}.`);
+  }
+  if (scenario.maxSessionErrors !== undefined && sessionErrorCount > scenario.maxSessionErrors) {
+    issues.push(`Expected at most ${scenario.maxSessionErrors} session error(s), saw ${sessionErrorCount}.`);
+  }
+  if (scenario.maxProviderErrors !== undefined && providerErrorCount > scenario.maxProviderErrors) {
+    issues.push(`Expected at most ${scenario.maxProviderErrors} provider error(s), saw ${providerErrorCount}.`);
+  }
+  for (const path of scenario.requiredPayloadPaths ?? []) {
+    if (events.every((event) => getPathValue(event.payload, path) === undefined)) {
+      issues.push(`Missing payload path: ${path}`);
+    }
+  }
+  return {
+    eventCount: events.length,
+    issues,
+    sessionId,
+    status: issues.length > 0 ? "fail" : "pass"
+  };
+};
+var runVoiceScenarioEvals = async (options = {}) => {
+  const scenarios = options.scenarios ?? [];
+  const events = filterVoiceTraceEvents(options.events ?? await options.store?.list() ?? []);
+  const grouped = new Map;
+  for (const event of events) {
+    grouped.set(event.sessionId, [...grouped.get(event.sessionId) ?? [], event]);
+  }
+  const results = scenarios.map((scenario) => {
+    const sessions = [...grouped.entries()].filter(([, sessionEvents]) => scenario.scenarioId ? sessionEvents.some((event) => event.scenarioId === scenario.scenarioId) : true).map(([sessionId, sessionEvents]) => evaluateScenarioSession(scenario, sessionId, filterVoiceTraceEvents(sessionEvents))).sort((left, right) => left.sessionId.localeCompare(right.sessionId));
+    const issues = [];
+    const minSessions = scenario.minSessions ?? 1;
+    if (sessions.length < minSessions) {
+      issues.push(`Expected at least ${minSessions} matching session(s), saw ${sessions.length}.`);
+    }
+    const failed2 = sessions.filter((session) => session.status === "fail").length;
+    const passed2 = sessions.length - failed2;
+    return {
+      description: scenario.description,
+      failed: failed2,
+      id: scenario.id,
+      issues,
+      label: scenario.label ?? scenario.id,
+      matchedSessions: sessions.length,
+      passed: passed2,
+      sessions,
+      status: issues.length > 0 || failed2 > 0 ? "fail" : "pass"
+    };
+  });
+  const failed = results.filter((scenario) => scenario.status === "fail").length;
+  const passed = results.length - failed;
+  return {
+    checkedAt: Date.now(),
+    failed,
+    passed,
+    scenarios: results,
+    status: failed > 0 ? "fail" : "pass",
+    total: results.length
+  };
+};
+var summarizeEvalBaseline = (report) => {
+  const failedSessionIds = report.sessions.filter((session) => session.status === "fail").map((session) => session.sessionId).sort();
+  return {
+    failed: report.failed,
+    failedSessionIds,
+    passRate: rate2(report.passed, report.total),
+    passed: report.passed,
+    total: report.total
+  };
+};
+var compareVoiceEvalBaseline = (currentReport, baselineReport, options = {}) => {
+  const baseline = summarizeEvalBaseline(baselineReport);
+  const current = summarizeEvalBaseline(currentReport);
+  const maxFailedDelta = options.maxFailedDelta ?? 0;
+  const maxPassRateDrop = options.maxPassRateDrop ?? 0;
+  const failOnNewFailedSessions = options.failOnNewFailedSessions ?? true;
+  const baselineFailed = new Set(baseline.failedSessionIds);
+  const currentFailed = new Set(current.failedSessionIds);
+  const newFailedSessionIds = current.failedSessionIds.filter((sessionId) => !baselineFailed.has(sessionId));
+  const recoveredSessionIds = baseline.failedSessionIds.filter((sessionId) => !currentFailed.has(sessionId));
+  const deltas = {
+    failed: current.failed - baseline.failed,
+    passRate: current.passRate - baseline.passRate,
+    passed: current.passed - baseline.passed,
+    total: current.total - baseline.total
+  };
+  const reasons = [];
+  if (deltas.failed > maxFailedDelta) {
+    reasons.push(`Failed sessions increased by ${deltas.failed}, above allowed delta ${maxFailedDelta}.`);
+  }
+  if (deltas.passRate < -maxPassRateDrop) {
+    reasons.push(`Pass rate dropped by ${Math.abs(deltas.passRate).toFixed(4)}, above allowed drop ${maxPassRateDrop}.`);
+  }
+  if (failOnNewFailedSessions && newFailedSessionIds.length > 0) {
+    reasons.push(`${newFailedSessionIds.length} session(s) failed that were not failing in the baseline.`);
+  }
+  return {
+    baseline,
+    checkedAt: Date.now(),
+    current,
+    deltas,
+    newFailedSessionIds,
+    recoveredSessionIds,
+    reasons,
+    status: reasons.length > 0 ? "fail" : "pass"
+  };
+};
+var createVoiceFileEvalBaselineStore = (filePath) => ({
+  get: async () => {
+    const file = Bun.file(filePath);
+    if (!await file.exists()) {
+      return;
+    }
+    const text = await file.text();
+    return text.trim() ? JSON.parse(text) : undefined;
+  },
+  set: async (report) => {
+    await mkdir(dirname(filePath), { recursive: true });
+    await Bun.write(filePath, JSON.stringify(report, null, 2));
+  }
+});
 var formatTime = (value) => value === undefined ? "unknown" : new Date(value).toLocaleString();
+var formatPercent = (value) => `${(value * 100).toFixed(2)}%`;
 var renderVoiceEvalHTML = (report, options = {}) => {
   const title = options.title ?? "AbsoluteJS Voice Evals";
   const links = options.links?.length ? `<nav>${options.links.map((link) => `<a href="${escapeHtml9(link.href)}">${escapeHtml9(link.label)}</a>`).join("")}</nav>` : "";
@@ -8055,6 +8240,24 @@ var renderVoiceEvalHTML = (report, options = {}) => {
   }).join("") : '<tr><td colspan="7">No sessions found.</td></tr>';
   return `<!doctype html><html lang="en"><head><meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" /><title>${escapeHtml9(title)}</title><style>body{font-family:ui-sans-serif,system-ui,sans-serif;margin:2rem;background:#f8f7f2;color:#181713}main{max-width:1180px;margin:auto}nav{display:flex;gap:.5rem;flex-wrap:wrap;margin-bottom:1rem}nav a{background:#181713;border-radius:999px;color:white;padding:.35rem .7rem;text-decoration:none}.status{border-radius:999px;display:inline-flex;font-weight:800;padding:.35rem .75rem}.pass{color:#166534}.fail{color:#991b1b}.status.pass{background:#dcfce7}.status.fail{background:#fee2e2}.grid{display:grid;gap:1rem;grid-template-columns:repeat(auto-fit,minmax(160px,1fr));margin:1rem 0}.card{background:white;border:1px solid #e7e5e4;border-radius:1rem;padding:1rem}.card strong{display:block;font-size:2rem}table{border-collapse:collapse;background:white;width:100%;margin:1rem 0 2rem}td,th{border-bottom:1px solid #eee;padding:.75rem;text-align:left}tr.fail td{border-left:4px solid #dc2626}tr.pass td{border-left:4px solid #16a34a}</style></head><body><main>${links}<h1>${escapeHtml9(title)}</h1><p class="status ${report.status}">${report.status}</p><div class="grid"><article class="card"><span>Total</span><strong>${report.total}</strong></article><article class="card"><span>Passed</span><strong>${report.passed}</strong></article><article class="card"><span>Failed</span><strong>${report.failed}</strong></article></div><h2>Trend</h2><table><thead><tr><th>Day</th><th>Total</th><th>Passed</th><th>Failed</th></tr></thead><tbody>${trend}</tbody></table><h2>Session Eval Results</h2><table><thead><tr><th>Session</th><th>Status</th><th>Events</th><th>Turns</th><th>Errors</th><th>Last event</th><th>Failed metrics</th></tr></thead><tbody>${sessions}</tbody></table></main></body></html>`;
 };
+var renderVoiceEvalBaselineHTML = (comparison, options = {}) => {
+  const title = options.title ?? "AbsoluteJS Voice Eval Baseline";
+  const links = options.links?.length ? `<nav>${options.links.map((link) => `<a href="${escapeHtml9(link.href)}">${escapeHtml9(link.label)}</a>`).join("")}</nav>` : "";
+  const reasons = comparison.reasons.length ? comparison.reasons.map((reason) => `<li>${escapeHtml9(reason)}</li>`).join("") : "<li>No baseline regressions detected.</li>";
+  const newFailures = comparison.newFailedSessionIds.length ? comparison.newFailedSessionIds.map((id) => `<li>${escapeHtml9(id)}</li>`).join("") : "<li>none</li>";
+  const recovered = comparison.recoveredSessionIds.length ? comparison.recoveredSessionIds.map((id) => `<li>${escapeHtml9(id)}</li>`).join("") : "<li>none</li>";
+  return `<!doctype html><html lang="en"><head><meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" /><title>${escapeHtml9(title)}</title><style>body{font-family:ui-sans-serif,system-ui,sans-serif;margin:2rem;background:#f8f7f2;color:#181713}main{max-width:1000px;margin:auto}nav{display:flex;gap:.5rem;flex-wrap:wrap;margin-bottom:1rem}nav a{background:#181713;border-radius:999px;color:white;padding:.35rem .7rem;text-decoration:none}.status{border-radius:999px;display:inline-flex;font-weight:800;padding:.35rem .75rem}.pass{background:#dcfce7;color:#166534}.fail{background:#fee2e2;color:#991b1b}.grid{display:grid;gap:1rem;grid-template-columns:repeat(auto-fit,minmax(180px,1fr));margin:1rem 0}.card{background:white;border:1px solid #e7e5e4;border-radius:1rem;padding:1rem}.card strong{display:block;font-size:2rem}section{background:white;border:1px solid #e7e5e4;border-radius:1rem;margin:1rem 0;padding:1rem}</style></head><body><main>${links}<h1>${escapeHtml9(title)}</h1><p class="status ${comparison.status}">${comparison.status}</p><div class="grid"><article class="card"><span>Baseline pass rate</span><strong>${escapeHtml9(formatPercent(comparison.baseline.passRate))}</strong></article><article class="card"><span>Current pass rate</span><strong>${escapeHtml9(formatPercent(comparison.current.passRate))}</strong></article><article class="card"><span>Failed delta</span><strong>${comparison.deltas.failed}</strong></article><article class="card"><span>Pass rate delta</span><strong>${escapeHtml9(formatPercent(comparison.deltas.passRate))}</strong></article></div><section><h2>Regression Reasons</h2><ul>${reasons}</ul></section><section><h2>New Failed Sessions</h2><ul>${newFailures}</ul></section><section><h2>Recovered Sessions</h2><ul>${recovered}</ul></section></main></body></html>`;
+};
+var renderVoiceScenarioEvalHTML = (report, options = {}) => {
+  const title = options.title ?? "AbsoluteJS Voice Scenario Evals";
+  const links = options.links?.length ? `<nav>${options.links.map((link) => `<a href="${escapeHtml9(link.href)}">${escapeHtml9(link.label)}</a>`).join("")}</nav>` : "";
+  const scenarios = report.scenarios.length ? report.scenarios.map((scenario) => {
+    const scenarioIssues = scenario.issues.length ? `<ul>${scenario.issues.map((issue) => `<li>${escapeHtml9(issue)}</li>`).join("")}</ul>` : "";
+    const sessions = scenario.sessions.length ? scenario.sessions.map((session) => `<tr class="${session.status}"><td>${escapeHtml9(session.sessionId)}</td><td>${escapeHtml9(session.status)}</td><td>${session.eventCount}</td><td>${escapeHtml9(session.issues.join(", ") || "none")}</td></tr>`).join("") : '<tr><td colspan="4">No matching sessions.</td></tr>';
+    return `<section class="scenario ${scenario.status}"><h2>${escapeHtml9(scenario.label)}</h2>${scenario.description ? `<p>${escapeHtml9(scenario.description)}</p>` : ""}<p class="status ${scenario.status}">${scenario.status}</p><p>${scenario.passed} passed, ${scenario.failed} failed, ${scenario.matchedSessions} matched.</p>${scenarioIssues}<table><thead><tr><th>Session</th><th>Status</th><th>Events</th><th>Issues</th></tr></thead><tbody>${sessions}</tbody></table></section>`;
+  }).join("") : "<section><p>No scenarios configured.</p></section>";
+  return `<!doctype html><html lang="en"><head><meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" /><title>${escapeHtml9(title)}</title><style>body{font-family:ui-sans-serif,system-ui,sans-serif;margin:2rem;background:#f8f7f2;color:#181713}main{max-width:1180px;margin:auto}nav{display:flex;gap:.5rem;flex-wrap:wrap;margin-bottom:1rem}nav a{background:#181713;border-radius:999px;color:white;padding:.35rem .7rem;text-decoration:none}.status{border-radius:999px;display:inline-flex;font-weight:800;padding:.35rem .75rem}.status.pass{background:#dcfce7;color:#166534}.status.fail{background:#fee2e2;color:#991b1b}.grid{display:grid;gap:1rem;grid-template-columns:repeat(auto-fit,minmax(160px,1fr));margin:1rem 0}.card,section{background:white;border:1px solid #e7e5e4;border-radius:1rem;padding:1rem}.card strong{display:block;font-size:2rem}section{margin:1rem 0}table{border-collapse:collapse;width:100%;margin-top:1rem}td,th{border-bottom:1px solid #eee;padding:.75rem;text-align:left}tr.fail td{border-left:4px solid #dc2626}tr.pass td{border-left:4px solid #16a34a}</style></head><body><main>${links}<h1>${escapeHtml9(title)}</h1><p class="status ${report.status}">${report.status}</p><div class="grid"><article class="card"><span>Total</span><strong>${report.total}</strong></article><article class="card"><span>Passed</span><strong>${report.passed}</strong></article><article class="card"><span>Failed</span><strong>${report.failed}</strong></article></div>${scenarios}</main></body></html>`;
+};
 var createVoiceEvalRoutes = (options) => {
   const path = options.path ?? "/evals";
   const routes = new Elysia7({
@@ -8066,6 +8269,16 @@ var createVoiceEvalRoutes = (options) => {
     store: options.store,
     thresholds: options.thresholds
   });
+  const getBaseline = async () => typeof options.baseline === "function" ? options.baseline() : options.baseline ?? await options.baselineStore?.get();
+  const getBaselineComparison = async () => {
+    const [current, baseline] = await Promise.all([getReport(), getBaseline()]);
+    return baseline ? compareVoiceEvalBaseline(current, baseline, options.baselineComparison) : undefined;
+  };
+  const getScenarioReport = () => runVoiceScenarioEvals({
+    events: options.events,
+    scenarios: options.scenarios,
+    store: options.store
+  });
   routes.get(path, async () => {
     const report = await getReport();
     return new Response(renderVoiceEvalHTML(report, {
@@ -8086,11 +8299,78 @@ var createVoiceEvalRoutes = (options) => {
     }
     return report;
   });
+  routes.get(`${path}/baseline`, async ({ set }) => {
+    const comparison = await getBaselineComparison();
+    if (!comparison) {
+      set.status = 404;
+      return Response.json({ error: "No voice eval baseline found." });
+    }
+    return new Response(renderVoiceEvalBaselineHTML(comparison, {
+      links: options.links,
+      title: `${options.title ?? "AbsoluteJS Voice Evals"} Baseline`
+    }), {
+      headers: {
+        "Content-Type": "text/html; charset=utf-8",
+        ...options.headers
+      }
+    });
+  });
+  routes.get(`${path}/baseline/json`, async ({ set }) => {
+    const comparison = await getBaselineComparison();
+    if (!comparison) {
+      set.status = 404;
+      return { error: "No voice eval baseline found." };
+    }
+    return comparison;
+  });
+  routes.get(`${path}/baseline/status`, async ({ set }) => {
+    const comparison = await getBaselineComparison();
+    if (!comparison) {
+      set.status = 404;
+      return { error: "No voice eval baseline found." };
+    }
+    if (comparison.status === "fail") {
+      set.status = 503;
+    }
+    return comparison;
+  });
+  routes.post(`${path}/baseline`, async ({ set }) => {
+    if (!options.baselineStore) {
+      set.status = 501;
+      return { error: "No voice eval baseline store configured." };
+    }
+    const report = await getReport();
+    await options.baselineStore.set(report);
+    return {
+      baseline: report,
+      status: "saved"
+    };
+  });
+  routes.get(`${path}/scenarios`, async () => {
+    const report = await getScenarioReport();
+    return new Response(renderVoiceScenarioEvalHTML(report, {
+      links: options.links,
+      title: `${options.title ?? "AbsoluteJS Voice Evals"} Scenarios`
+    }), {
+      headers: {
+        "Content-Type": "text/html; charset=utf-8",
+        ...options.headers
+      }
+    });
+  });
+  routes.get(`${path}/scenarios/json`, async () => getScenarioReport());
+  routes.get(`${path}/scenarios/status`, async ({ set }) => {
+    const report = await getScenarioReport();
+    if (report.status === "fail") {
+      set.status = 503;
+    }
+    return report;
+  });
   return routes;
 };
 // src/sessionReplay.ts
 import { Elysia as Elysia8 } from "elysia";
-var getString6 = (value) => typeof value === "string" ? value : undefined;
+var getString7 = (value) => typeof value === "string" ? value : undefined;
 var escapeHtml10 = (value) => value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;").replaceAll("'", "&#39;");
 var increment3 = (record, key) => {
   record[key] = (record[key] ?? 0) + 1;
@@ -8120,14 +8400,14 @@ var buildReplayTurns = (events) => {
       case "turn.transcript":
         turn.transcripts.push({
           isFinal: event.payload.isFinal === true,
-          text: getString6(event.payload.text)
+          text: getString7(event.payload.text)
         });
         break;
       case "turn.committed":
-        turn.committedText = getString6(event.payload.text);
+        turn.committedText = getString7(event.payload.text);
         break;
       case "turn.assistant": {
-        const text = getString6(event.payload.text);
+        const text = getString7(event.payload.text);
         if (text) {
           turn.assistantReplies.push(text);
         }
@@ -8196,7 +8476,7 @@ var summarizeVoiceSessions = async (options = {}) => {
     let latestOutcome;
     let errorCount = 0;
     for (const event of sorted) {
-      const provider = getString6(event.payload.provider);
+      const provider = getString7(event.payload.provider);
       if (provider) {
         providers.add(provider);
       }
@@ -8204,7 +8484,7 @@ var summarizeVoiceSessions = async (options = {}) => {
         errorCount += 1;
         increment3(providerErrors, provider ?? "unknown");
       }
-      const outcome = getString6(event.payload.outcome);
+      const outcome = getString7(event.payload.outcome);
       if (outcome) {
         latestOutcome = outcome;
       }
@@ -8331,7 +8611,7 @@ var createVoiceSessionReplayRoutes = (options) => {
   return routes;
 };
 // src/fileStore.ts
-import { mkdir, readFile, readdir, rename, rm, writeFile } from "fs/promises";
+import { mkdir as mkdir2, readFile, readdir, rename, rm, writeFile } from "fs/promises";
 import { join } from "path";
 var listJsonFiles = async (directory) => {
   try {
@@ -8351,7 +8631,7 @@ var resolveFilePath = (directory, id) => join(directory, encodeStoreId(id));
 var createMemoryStoreId = (input) => `${input.assistantId}:${input.namespace}:${input.key}`;
 var readJsonFile = async (path) => JSON.parse(await readFile(path, "utf8"));
 var writeJsonFile = async (path, value, options) => {
-  await mkdir(options.directory, {
+  await mkdir2(options.directory, {
     recursive: true
   });
   const tempPath = `${path}.${crypto.randomUUID()}.tmp`;
@@ -9464,7 +9744,7 @@ import { Elysia as Elysia10 } from "elysia";
 // src/resilienceRoutes.ts
 import { Elysia as Elysia9 } from "elysia";
 var escapeHtml11 = (value) => value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;").replaceAll("'", "&#39;");
-var getString7 = (value) => typeof value === "string" ? value : undefined;
+var getString8 = (value) => typeof value === "string" ? value : undefined;
 var getNumber4 = (value) => typeof value === "number" && Number.isFinite(value) ? value : undefined;
 var getBoolean2 = (value) => value === true;
 var isProviderStatus2 = (value) => value === "error" || value === "fallback" || value === "success";
@@ -9474,23 +9754,23 @@ var listVoiceRoutingEvents = (events) => {
     if (event.type !== "session.error") {
       continue;
     }
-    const provider = getString7(event.payload.provider);
+    const provider = getString8(event.payload.provider);
     const providerStatus = isProviderStatus2(event.payload.providerStatus) ? event.payload.providerStatus : undefined;
     if (!provider || !providerStatus) {
       continue;
     }
-    const kind = getString7(event.payload.kind);
+    const kind = getString8(event.payload.kind);
     routingEvents.push({
       at: event.at,
       attempt: getNumber4(event.payload.attempt),
       elapsedMs: getNumber4(event.payload.elapsedMs),
-      error: getString7(event.payload.error),
-      fallbackProvider: getString7(event.payload.fallbackProvider),
+      error: getString8(event.payload.error),
+      fallbackProvider: getString8(event.payload.fallbackProvider),
       kind: kind === "stt" || kind === "tts" ? kind : "llm",
       latencyBudgetMs: getNumber4(event.payload.latencyBudgetMs),
-      operation: getString7(event.payload.operation),
+      operation: getString8(event.payload.operation),
       provider,
-      selectedProvider: getString7(event.payload.selectedProvider),
+      selectedProvider: getString8(event.payload.selectedProvider),
       sessionId: event.sessionId,
       status: providerStatus,
       timedOut: getBoolean2(event.payload.timedOut),
@@ -13032,6 +13312,7 @@ export {
   shapeTelephonyAssistantText,
   selectVoiceTraceEventsForPrune,
   runVoiceSessionEvals,
+  runVoiceScenarioEvals,
   resolveVoiceTraceRedactionOptions,
   resolveVoiceSTTRoutingStrategy,
   resolveVoiceRuntimePreset,
@@ -13049,12 +13330,14 @@ export {
   renderVoiceTraceMarkdown,
   renderVoiceTraceHTML,
   renderVoiceSessionsHTML,
+  renderVoiceScenarioEvalHTML,
   renderVoiceResilienceHTML,
   renderVoiceQualityHTML,
   renderVoiceProviderHealthHTML,
   renderVoiceOpsConsoleHTML,
   renderVoiceHandoffHealthHTML,
   renderVoiceEvalHTML,
+  renderVoiceEvalBaselineHTML,
   renderVoiceCallReviewMarkdown,
   renderVoiceCallReviewHTML,
   renderVoiceAssistantHealthHTML,
@@ -13176,6 +13459,7 @@ export {
   createVoiceFileReviewStore,
   createVoiceFileIntegrationEventStore,
   createVoiceFileExternalObjectMapStore,
+  createVoiceFileEvalBaselineStore,
   createVoiceFileAssistantMemoryStore,
   createVoiceExternalObjectMapId,
   createVoiceExternalObjectMap,
@@ -13213,6 +13497,7 @@ export {
   createAnthropicVoiceAssistantModel,
   conditionAudioChunk,
   completeVoiceOpsTask,
+  compareVoiceEvalBaseline,
   claimVoiceOpsTask,
   buildVoiceTraceReplay,
   buildVoiceOpsTaskFromSLABreach,

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@absolutejs/voice",
-	"version": "0.0.22-beta.42",
+	"version": "0.0.22-beta.44",
 	"description": "Voice primitives and Elysia plugin for AbsoluteJS",
 	"repository": {
 		"type": "git",