@absolutejs/voice 0.0.22-beta.42 → 0.0.22-beta.43

package/dist/evalRoutes.d.ts

@@ -28,11 +28,45 @@ export type VoiceEvalReport = {
     total: number;
     trend: VoiceEvalTrendBucket[];
 };
+export type VoiceEvalBaselineSummary = {
+    failed: number;
+    failedSessionIds: string[];
+    passRate: number;
+    passed: number;
+    total: number;
+};
+export type VoiceEvalBaselineComparison = {
+    baseline: VoiceEvalBaselineSummary;
+    checkedAt: number;
+    current: VoiceEvalBaselineSummary;
+    deltas: {
+        failed: number;
+        passRate: number;
+        passed: number;
+        total: number;
+    };
+    newFailedSessionIds: string[];
+    recoveredSessionIds: string[];
+    reasons: string[];
+    status: VoiceEvalStatus;
+};
+export type VoiceEvalBaselineComparisonOptions = {
+    failOnNewFailedSessions?: boolean;
+    maxFailedDelta?: number;
+    maxPassRateDrop?: number;
+};
+export type VoiceEvalBaselineStore = {
+    get: () => Promise<VoiceEvalReport | undefined>;
+    set: (report: VoiceEvalReport) => Promise<void>;
+};
 export type VoiceEvalLink = {
     href: string;
     label: string;
 };
 export type VoiceEvalRoutesOptions = {
+    baseline?: VoiceEvalReport | (() => Promise<VoiceEvalReport | undefined>);
+    baselineComparison?: VoiceEvalBaselineComparisonOptions;
+    baselineStore?: VoiceEvalBaselineStore;
     events?: StoredVoiceTraceEvent[];
     headers?: HeadersInit;
     links?: VoiceEvalLink[];
@@ -49,10 +83,16 @@ export declare const runVoiceSessionEvals: (options?: {
     store?: VoiceTraceEventStore;
     thresholds?: VoiceQualityThresholds;
 }) => Promise<VoiceEvalReport>;
+export declare const compareVoiceEvalBaseline: (currentReport: VoiceEvalReport, baselineReport: VoiceEvalReport, options?: VoiceEvalBaselineComparisonOptions) => VoiceEvalBaselineComparison;
+export declare const createVoiceFileEvalBaselineStore: (filePath: string) => VoiceEvalBaselineStore;
 export declare const renderVoiceEvalHTML: (report: VoiceEvalReport, options?: {
     links?: VoiceEvalLink[];
     title?: string;
 }) => string;
+export declare const renderVoiceEvalBaselineHTML: (comparison: VoiceEvalBaselineComparison, options?: {
+    links?: VoiceEvalLink[];
+    title?: string;
+}) => string;
 export declare const createVoiceEvalRoutes: (options: VoiceEvalRoutesOptions) => Elysia<"", {
     decorator: {};
     store: {};

package/dist/index.d.ts

@@ -2,7 +2,7 @@ export { voice } from './plugin';
 export { createVoiceAssistant, createVoiceExperiment, summarizeVoiceAssistantRuns } from './assistant';
 export { createVoiceAssistantHealthHTMLHandler, createVoiceAssistantHealthJSONHandler, createVoiceAssistantHealthRoutes, renderVoiceAssistantHealthHTML, summarizeVoiceAssistantHealth } from './assistantHealth';
 export { buildVoiceDiagnosticsMarkdown, createVoiceDiagnosticsRoutes, resolveVoiceDiagnosticsTraceFilter } from './diagnosticsRoutes';
-export { createVoiceEvalRoutes, renderVoiceEvalHTML, runVoiceSessionEvals } from './evalRoutes';
+export { compareVoiceEvalBaseline, createVoiceFileEvalBaselineStore, createVoiceEvalRoutes, renderVoiceEvalBaselineHTML, renderVoiceEvalHTML, runVoiceSessionEvals } from './evalRoutes';
 export { createVoiceSessionListRoutes, createVoiceSessionReplayHTMLHandler, createVoiceSessionReplayJSONHandler, createVoiceSessionReplayRoutes, createVoiceSessionsHTMLHandler, createVoiceSessionsJSONHandler, renderVoiceSessionsHTML, summarizeVoiceSessions, summarizeVoiceSessionReplay } from './sessionReplay';
 export { createVoiceAgent, createVoiceAgentSquad, createVoiceAgentTool } from './agent';
 export { createStoredVoiceCallReviewArtifact, createStoredVoiceExternalObjectMap, createStoredVoiceIntegrationEvent, createStoredVoiceOpsTask, createVoiceFileExternalObjectMapStore, createVoiceFileAssistantMemoryStore, createVoiceFileIntegrationEventStore, createVoiceFileReviewStore, createVoiceFileRuntimeStorage, createVoiceFileSessionStore, createVoiceFileTaskStore, createVoiceFileTraceSinkDeliveryStore, createVoiceFileTraceEventStore } from './fileStore';
@@ -40,7 +40,7 @@ export type { VoiceAssistant, VoiceAssistantArtifactPlan, VoiceAssistantExperime
 export type { VoiceAssistantHealthFailure, VoiceAssistantHealthHTMLHandlerOptions, VoiceAssistantHealthRoutesOptions, VoiceAssistantHealthSummary, VoiceAssistantHealthSummaryOptions } from './assistantHealth';
 export type { VoiceAssistantMemoryBinding, VoiceAssistantMemoryHandle, VoiceAssistantMemoryOptions, VoiceAssistantMemoryRecord, VoiceAssistantMemoryStore } from './assistantMemory';
 export type { VoiceDiagnosticsRoutesOptions } from './diagnosticsRoutes';
-export type { VoiceEvalLink, VoiceEvalReport, VoiceEvalRoutesOptions, VoiceEvalSessionReport, VoiceEvalStatus, VoiceEvalTrendBucket } from './evalRoutes';
+export type { VoiceEvalBaselineComparison, VoiceEvalBaselineComparisonOptions, VoiceEvalBaselineStore, VoiceEvalBaselineSummary, VoiceEvalLink, VoiceEvalReport, VoiceEvalRoutesOptions, VoiceEvalSessionReport, VoiceEvalStatus, VoiceEvalTrendBucket } from './evalRoutes';
 export type { VoiceSessionListHTMLHandlerOptions, VoiceSessionListItem, VoiceSessionListOptions, VoiceSessionListRoutesOptions, VoiceSessionListStatus, VoiceSessionReplay, VoiceSessionReplayHTMLHandlerOptions, VoiceSessionReplayOptions, VoiceSessionReplayRoutesOptions, VoiceSessionReplayTurn } from './sessionReplay';
 export type { AnthropicVoiceAssistantModelOptions, GeminiVoiceAssistantModelOptions, OpenAIVoiceAssistantModelOptions, VoiceProviderRouterEvent, VoiceProviderRouterFallbackMode, VoiceProviderRouterHealthOptions, VoiceProviderRouterOptions, VoiceProviderRouterPolicy, VoiceProviderRouterProviderHealth, VoiceProviderRouterProviderProfile, VoiceJSONAssistantModelHandler, VoiceJSONAssistantModelOptions } from './modelAdapters';
 export type { VoiceProviderHealthStatus, VoiceProviderHealthSummary, VoiceProviderHealthSummaryOptions } from './providerHealth';

package/dist/index.js

@@ -7625,6 +7625,8 @@ var createVoiceDiagnosticsRoutes = (options) => {
 };
 // src/evalRoutes.ts
 import { Elysia as Elysia7 } from "elysia";
+import { mkdir } from "fs/promises";
+import { dirname } from "path";
 
 // src/qualityRoutes.ts
 import { Elysia as Elysia6 } from "elysia";
@@ -7974,6 +7976,7 @@ var createVoiceQualityRoutes = (options) => {
 
 // src/evalRoutes.ts
 var escapeHtml9 = (value) => value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;").replaceAll("'", "&#39;");
+var rate2 = (count, total) => count / Math.max(1, total);
 var sessionTime = (events) => {
   const sorted = filterVoiceTraceEvents(events);
   return {
@@ -8044,7 +8047,69 @@ var runVoiceSessionEvals = async (options = {}) => {
     trend: buildTrend(limitedSessions)
   };
 };
+var summarizeEvalBaseline = (report) => {
+  const failedSessionIds = report.sessions.filter((session) => session.status === "fail").map((session) => session.sessionId).sort();
+  return {
+    failed: report.failed,
+    failedSessionIds,
+    passRate: rate2(report.passed, report.total),
+    passed: report.passed,
+    total: report.total
+  };
+};
+var compareVoiceEvalBaseline = (currentReport, baselineReport, options = {}) => {
+  const baseline = summarizeEvalBaseline(baselineReport);
+  const current = summarizeEvalBaseline(currentReport);
+  const maxFailedDelta = options.maxFailedDelta ?? 0;
+  const maxPassRateDrop = options.maxPassRateDrop ?? 0;
+  const failOnNewFailedSessions = options.failOnNewFailedSessions ?? true;
+  const baselineFailed = new Set(baseline.failedSessionIds);
+  const currentFailed = new Set(current.failedSessionIds);
+  const newFailedSessionIds = current.failedSessionIds.filter((sessionId) => !baselineFailed.has(sessionId));
+  const recoveredSessionIds = baseline.failedSessionIds.filter((sessionId) => !currentFailed.has(sessionId));
+  const deltas = {
+    failed: current.failed - baseline.failed,
+    passRate: current.passRate - baseline.passRate,
+    passed: current.passed - baseline.passed,
+    total: current.total - baseline.total
+  };
+  const reasons = [];
+  if (deltas.failed > maxFailedDelta) {
+    reasons.push(`Failed sessions increased by ${deltas.failed}, above allowed delta ${maxFailedDelta}.`);
+  }
+  if (deltas.passRate < -maxPassRateDrop) {
+    reasons.push(`Pass rate dropped by ${Math.abs(deltas.passRate).toFixed(4)}, above allowed drop ${maxPassRateDrop}.`);
+  }
+  if (failOnNewFailedSessions && newFailedSessionIds.length > 0) {
+    reasons.push(`${newFailedSessionIds.length} session(s) failed that were not failing in the baseline.`);
+  }
+  return {
+    baseline,
+    checkedAt: Date.now(),
+    current,
+    deltas,
+    newFailedSessionIds,
+    recoveredSessionIds,
+    reasons,
+    status: reasons.length > 0 ? "fail" : "pass"
+  };
+};
+var createVoiceFileEvalBaselineStore = (filePath) => ({
+  get: async () => {
+    const file = Bun.file(filePath);
+    if (!await file.exists()) {
+      return;
+    }
+    const text = await file.text();
+    return text.trim() ? JSON.parse(text) : undefined;
+  },
+  set: async (report) => {
+    await mkdir(dirname(filePath), { recursive: true });
+    await Bun.write(filePath, JSON.stringify(report, null, 2));
+  }
+});
 var formatTime = (value) => value === undefined ? "unknown" : new Date(value).toLocaleString();
+var formatPercent = (value) => `${(value * 100).toFixed(2)}%`;
 var renderVoiceEvalHTML = (report, options = {}) => {
   const title = options.title ?? "AbsoluteJS Voice Evals";
   const links = options.links?.length ? `<nav>${options.links.map((link) => `<a href="${escapeHtml9(link.href)}">${escapeHtml9(link.label)}</a>`).join("")}</nav>` : "";
@@ -8055,6 +8120,14 @@ var renderVoiceEvalHTML = (report, options = {}) => {
   }).join("") : '<tr><td colspan="7">No sessions found.</td></tr>';
   return `<!doctype html><html lang="en"><head><meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" /><title>${escapeHtml9(title)}</title><style>body{font-family:ui-sans-serif,system-ui,sans-serif;margin:2rem;background:#f8f7f2;color:#181713}main{max-width:1180px;margin:auto}nav{display:flex;gap:.5rem;flex-wrap:wrap;margin-bottom:1rem}nav a{background:#181713;border-radius:999px;color:white;padding:.35rem .7rem;text-decoration:none}.status{border-radius:999px;display:inline-flex;font-weight:800;padding:.35rem .75rem}.pass{color:#166534}.fail{color:#991b1b}.status.pass{background:#dcfce7}.status.fail{background:#fee2e2}.grid{display:grid;gap:1rem;grid-template-columns:repeat(auto-fit,minmax(160px,1fr));margin:1rem 0}.card{background:white;border:1px solid #e7e5e4;border-radius:1rem;padding:1rem}.card strong{display:block;font-size:2rem}table{border-collapse:collapse;background:white;width:100%;margin:1rem 0 2rem}td,th{border-bottom:1px solid #eee;padding:.75rem;text-align:left}tr.fail td{border-left:4px solid #dc2626}tr.pass td{border-left:4px solid #16a34a}</style></head><body><main>${links}<h1>${escapeHtml9(title)}</h1><p class="status ${report.status}">${report.status}</p><div class="grid"><article class="card"><span>Total</span><strong>${report.total}</strong></article><article class="card"><span>Passed</span><strong>${report.passed}</strong></article><article class="card"><span>Failed</span><strong>${report.failed}</strong></article></div><h2>Trend</h2><table><thead><tr><th>Day</th><th>Total</th><th>Passed</th><th>Failed</th></tr></thead><tbody>${trend}</tbody></table><h2>Session Eval Results</h2><table><thead><tr><th>Session</th><th>Status</th><th>Events</th><th>Turns</th><th>Errors</th><th>Last event</th><th>Failed metrics</th></tr></thead><tbody>${sessions}</tbody></table></main></body></html>`;
 };
+var renderVoiceEvalBaselineHTML = (comparison, options = {}) => {
+  const title = options.title ?? "AbsoluteJS Voice Eval Baseline";
+  const links = options.links?.length ? `<nav>${options.links.map((link) => `<a href="${escapeHtml9(link.href)}">${escapeHtml9(link.label)}</a>`).join("")}</nav>` : "";
+  const reasons = comparison.reasons.length ? comparison.reasons.map((reason) => `<li>${escapeHtml9(reason)}</li>`).join("") : "<li>No baseline regressions detected.</li>";
+  const newFailures = comparison.newFailedSessionIds.length ? comparison.newFailedSessionIds.map((id) => `<li>${escapeHtml9(id)}</li>`).join("") : "<li>none</li>";
+  const recovered = comparison.recoveredSessionIds.length ? comparison.recoveredSessionIds.map((id) => `<li>${escapeHtml9(id)}</li>`).join("") : "<li>none</li>";
+  return `<!doctype html><html lang="en"><head><meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" /><title>${escapeHtml9(title)}</title><style>body{font-family:ui-sans-serif,system-ui,sans-serif;margin:2rem;background:#f8f7f2;color:#181713}main{max-width:1000px;margin:auto}nav{display:flex;gap:.5rem;flex-wrap:wrap;margin-bottom:1rem}nav a{background:#181713;border-radius:999px;color:white;padding:.35rem .7rem;text-decoration:none}.status{border-radius:999px;display:inline-flex;font-weight:800;padding:.35rem .75rem}.pass{background:#dcfce7;color:#166534}.fail{background:#fee2e2;color:#991b1b}.grid{display:grid;gap:1rem;grid-template-columns:repeat(auto-fit,minmax(180px,1fr));margin:1rem 0}.card{background:white;border:1px solid #e7e5e4;border-radius:1rem;padding:1rem}.card strong{display:block;font-size:2rem}section{background:white;border:1px solid #e7e5e4;border-radius:1rem;margin:1rem 0;padding:1rem}</style></head><body><main>${links}<h1>${escapeHtml9(title)}</h1><p class="status ${comparison.status}">${comparison.status}</p><div class="grid"><article class="card"><span>Baseline pass rate</span><strong>${escapeHtml9(formatPercent(comparison.baseline.passRate))}</strong></article><article class="card"><span>Current pass rate</span><strong>${escapeHtml9(formatPercent(comparison.current.passRate))}</strong></article><article class="card"><span>Failed delta</span><strong>${comparison.deltas.failed}</strong></article><article class="card"><span>Pass rate delta</span><strong>${escapeHtml9(formatPercent(comparison.deltas.passRate))}</strong></article></div><section><h2>Regression Reasons</h2><ul>${reasons}</ul></section><section><h2>New Failed Sessions</h2><ul>${newFailures}</ul></section><section><h2>Recovered Sessions</h2><ul>${recovered}</ul></section></main></body></html>`;
+};
 var createVoiceEvalRoutes = (options) => {
   const path = options.path ?? "/evals";
   const routes = new Elysia7({
@@ -8066,6 +8139,11 @@ var createVoiceEvalRoutes = (options) => {
     store: options.store,
     thresholds: options.thresholds
   });
+  const getBaseline = async () => typeof options.baseline === "function" ? options.baseline() : options.baseline ?? await options.baselineStore?.get();
+  const getBaselineComparison = async () => {
+    const [current, baseline] = await Promise.all([getReport(), getBaseline()]);
+    return baseline ? compareVoiceEvalBaseline(current, baseline, options.baselineComparison) : undefined;
+  };
   routes.get(path, async () => {
     const report = await getReport();
     return new Response(renderVoiceEvalHTML(report, {
@@ -8086,6 +8164,53 @@ var createVoiceEvalRoutes = (options) => {
     }
     return report;
   });
+  routes.get(`${path}/baseline`, async ({ set }) => {
+    const comparison = await getBaselineComparison();
+    if (!comparison) {
+      set.status = 404;
+      return Response.json({ error: "No voice eval baseline found." });
+    }
+    return new Response(renderVoiceEvalBaselineHTML(comparison, {
+      links: options.links,
+      title: `${options.title ?? "AbsoluteJS Voice Evals"} Baseline`
+    }), {
+      headers: {
+        "Content-Type": "text/html; charset=utf-8",
+        ...options.headers
+      }
+    });
+  });
+  routes.get(`${path}/baseline/json`, async ({ set }) => {
+    const comparison = await getBaselineComparison();
+    if (!comparison) {
+      set.status = 404;
+      return { error: "No voice eval baseline found." };
+    }
+    return comparison;
+  });
+  routes.get(`${path}/baseline/status`, async ({ set }) => {
+    const comparison = await getBaselineComparison();
+    if (!comparison) {
+      set.status = 404;
+      return { error: "No voice eval baseline found." };
+    }
+    if (comparison.status === "fail") {
+      set.status = 503;
+    }
+    return comparison;
+  });
+  routes.post(`${path}/baseline`, async ({ set }) => {
+    if (!options.baselineStore) {
+      set.status = 501;
+      return { error: "No voice eval baseline store configured." };
+    }
+    const report = await getReport();
+    await options.baselineStore.set(report);
+    return {
+      baseline: report,
+      status: "saved"
+    };
+  });
   return routes;
 };
 // src/sessionReplay.ts
@@ -8331,7 +8456,7 @@ var createVoiceSessionReplayRoutes = (options) => {
   return routes;
 };
 // src/fileStore.ts
-import { mkdir, readFile, readdir, rename, rm, writeFile } from "fs/promises";
+import { mkdir as mkdir2, readFile, readdir, rename, rm, writeFile } from "fs/promises";
 import { join } from "path";
 var listJsonFiles = async (directory) => {
   try {
@@ -8351,7 +8476,7 @@ var resolveFilePath = (directory, id) => join(directory, encodeStoreId(id));
 var createMemoryStoreId = (input) => `${input.assistantId}:${input.namespace}:${input.key}`;
 var readJsonFile = async (path) => JSON.parse(await readFile(path, "utf8"));
 var writeJsonFile = async (path, value, options) => {
-  await mkdir(options.directory, {
+  await mkdir2(options.directory, {
     recursive: true
   });
   const tempPath = `${path}.${crypto.randomUUID()}.tmp`;
@@ -13055,6 +13180,7 @@ export {
   renderVoiceOpsConsoleHTML,
   renderVoiceHandoffHealthHTML,
   renderVoiceEvalHTML,
+  renderVoiceEvalBaselineHTML,
   renderVoiceCallReviewMarkdown,
   renderVoiceCallReviewHTML,
   renderVoiceAssistantHealthHTML,
@@ -13176,6 +13302,7 @@ export {
   createVoiceFileReviewStore,
   createVoiceFileIntegrationEventStore,
   createVoiceFileExternalObjectMapStore,
+  createVoiceFileEvalBaselineStore,
   createVoiceFileAssistantMemoryStore,
   createVoiceExternalObjectMapId,
   createVoiceExternalObjectMap,
@@ -13213,6 +13340,7 @@ export {
   createAnthropicVoiceAssistantModel,
   conditionAudioChunk,
   completeVoiceOpsTask,
+  compareVoiceEvalBaseline,
   claimVoiceOpsTask,
   buildVoiceTraceReplay,
   buildVoiceOpsTaskFromSLABreach,

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@absolutejs/voice",
-	"version": "0.0.22-beta.42",
+	"version": "0.0.22-beta.43",
 	"description": "Voice primitives and Elysia plugin for AbsoluteJS",
 	"repository": {
 		"type": "git",