@absolutejs/voice 0.0.22-beta.95 → 0.0.22-beta.97

package/dist/index.js

@@ -3596,7 +3596,7 @@ var createVoiceSession = (options) => {
   } : undefined;
   const appendTrace = async (input) => {
     await options.trace?.append({
-      at: Date.now(),
+      at: input.at ?? Date.now(),
       metadata: input.metadata,
       payload: input.payload,
       scenarioId: input.session?.scenarioId ?? options.scenarioId,
@@ -3605,6 +3605,13 @@ var createVoiceSession = (options) => {
       type: input.type
     });
   };
+  const appendTurnLatencyStage = async (input) => appendTrace({
+    at: input.at,
+    payload: { stage: input.stage },
+    session: input.session,
+    turnId: input.turnId,
+    type: "turn_latency.stage"
+  });
   const phraseHints = options.phraseHints ?? [];
   const lexicon = options.lexicon ?? [];
   let socket = options.socket;
@@ -4555,6 +4562,13 @@ var createVoiceSession = (options) => {
             turnId: activeTTSTurnId,
             type: "audio"
           });
+          if (activeTTSTurnId) {
+            await appendTurnLatencyStage({
+              at: receivedAt,
+              stage: "assistant_audio_received",
+              turnId: activeTTSTurnId
+            });
+          }
         });
       });
       openedSession.on("error", (event) => {
@@ -4613,6 +4627,7 @@ var createVoiceSession = (options) => {
       voicemail: committedOutput?.voicemail
     };
     if (output?.assistantText) {
+      const assistantTextStartedAt = Date.now();
       await writeSession((currentSession) => {
         setTurnResult(currentSession, turn.id, {
           assistantText: output.assistantText
@@ -4623,6 +4638,12 @@ var createVoiceSession = (options) => {
         turnId: turn.id,
         type: "assistant"
       });
+      await appendTurnLatencyStage({
+        at: assistantTextStartedAt,
+        session,
+        stage: "assistant_text_started",
+        turnId: turn.id
+      });
       await appendTrace({
         payload: {
           text: output.assistantText,
@@ -4637,7 +4658,18 @@ var createVoiceSession = (options) => {
         if (activeTTSSession) {
           const ttsStartedAt = Date.now();
           activeTTSTurnId = turn.id;
+          await appendTurnLatencyStage({
+            at: ttsStartedAt,
+            session,
+            stage: "tts_send_started",
+            turnId: turn.id
+          });
           await activeTTSSession.send(output.assistantText);
+          await appendTurnLatencyStage({
+            session,
+            stage: "tts_send_completed",
+            turnId: turn.id
+          });
           await appendTrace({
             payload: {
               elapsedMs: Date.now() - ttsStartedAt,
@@ -4834,6 +4866,30 @@ var createVoiceSession = (options) => {
       turnId: turn.id,
       type: "turn.cost"
     });
+    const firstTranscriptAt = turn.transcripts.map((transcript) => transcript.endedAtMs ?? transcript.startedAtMs).filter((value) => typeof value === "number").sort((left, right) => left - right)[0];
+    const finalTranscriptAt = turn.transcripts.filter((transcript) => transcript.isFinal).map((transcript) => transcript.endedAtMs ?? transcript.startedAtMs).filter((value) => typeof value === "number").sort((left, right) => left - right)[0];
+    if (firstTranscriptAt !== undefined) {
+      await appendTurnLatencyStage({
+        at: firstTranscriptAt,
+        session: updatedSession,
+        stage: "speech_detected",
+        turnId: turn.id
+      });
+    }
+    if (finalTranscriptAt !== undefined) {
+      await appendTurnLatencyStage({
+        at: finalTranscriptAt,
+        session: updatedSession,
+        stage: "final_transcript",
+        turnId: turn.id
+      });
+    }
+    await appendTurnLatencyStage({
+      at: turn.committedAt,
+      session: updatedSession,
+      stage: "turn_committed",
+      turnId: turn.id
+    });
     await send({
       turn,
       type: "turn"
@@ -9965,6 +10021,8 @@ var timelineLabel = (event) => {
       return `Error${getString9(event.payload.error) ? `: ${getString9(event.payload.error)}` : ""}`;
     case "turn.cost":
       return "Cost telemetry";
+    case "turn_latency.stage":
+      return `Latency ${getString9(event.payload.stage) ?? "stage"}`;
     case "workflow.contract":
       return `Workflow contract ${eventStatus(event) ?? ""}`.trim();
     default:
@@ -11174,17 +11232,44 @@ var DEFAULT_WARN_AFTER_MS = 1800;
 var DEFAULT_FAIL_AFTER_MS = 3200;
 var escapeHtml19 = (value) => value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;").replaceAll("'", "&#39;");
 var firstNumber2 = (values) => values.filter((value) => typeof value === "number").sort((left, right) => left - right)[0];
+var getString10 = (value) => typeof value === "string" && value.trim() ? value : undefined;
+var createTraceStageIndex = (events) => {
+  const index = new Map;
+  for (const event of events) {
+    if (event.type !== "turn_latency.stage" || !event.turnId) {
+      continue;
+    }
+    const stage = getString10(event.payload.stage);
+    if (!stage) {
+      continue;
+    }
+    const key = `${event.sessionId}:${event.turnId}`;
+    const stages = index.get(key) ?? new Map;
+    const previous = stages.get(stage);
+    if (previous === undefined || event.at < previous) {
+      stages.set(stage, event.at);
+    }
+    index.set(key, stages);
+  }
+  return index;
+};
 var summarizeTurn = (sessionId, turn, options) => {
-  const firstTranscriptAt = firstNumber2(turn.transcripts.map((transcript) => transcript.endedAtMs ?? transcript.startedAtMs));
-  const finalTranscriptAt = firstNumber2(turn.transcripts.filter((transcript) => transcript.isFinal).map((transcript) => transcript.endedAtMs ?? transcript.startedAtMs));
-  const commitAfterFirstMs = firstTranscriptAt === undefined ? undefined : Math.max(0, turn.committedAt - firstTranscriptAt);
-  const commitAfterFinalMs = finalTranscriptAt === undefined ? undefined : Math.max(0, turn.committedAt - finalTranscriptAt);
-  const assistantTextStartedAt = turn.assistantText ? turn.committedAt : undefined;
-  const totalMs = commitAfterFirstMs;
+  const traceStages = options.stageIndex?.get(`${sessionId}:${turn.id}`);
+  const firstTranscriptAt = traceStages?.get("speech_detected") ?? firstNumber2(turn.transcripts.map((transcript) => transcript.endedAtMs ?? transcript.startedAtMs));
+  const finalTranscriptAt = traceStages?.get("final_transcript") ?? firstNumber2(turn.transcripts.filter((transcript) => transcript.isFinal).map((transcript) => transcript.endedAtMs ?? transcript.startedAtMs));
+  const committedAt = traceStages?.get("turn_committed") ?? turn.committedAt;
+  const assistantTextStartedAt = traceStages?.get("assistant_text_started") ?? (turn.assistantText ? committedAt : undefined);
+  const ttsSendStartedAt = traceStages?.get("tts_send_started");
+  const ttsSendCompletedAt = traceStages?.get("tts_send_completed");
+  const assistantAudioReceivedAt = traceStages?.get("assistant_audio_received");
+  const commitAfterFirstMs = firstTranscriptAt === undefined ? undefined : Math.max(0, committedAt - firstTranscriptAt);
+  const commitAfterFinalMs = finalTranscriptAt === undefined ? undefined : Math.max(0, committedAt - finalTranscriptAt);
+  const totalEndAt = assistantAudioReceivedAt ?? assistantTextStartedAt ?? committedAt;
+  const totalMs = firstTranscriptAt === undefined ? commitAfterFirstMs : Math.max(0, totalEndAt - firstTranscriptAt);
   const status = totalMs === undefined ? "warn" : totalMs > options.failAfterMs ? "fail" : totalMs > options.warnAfterMs ? "warn" : "pass";
   return {
     assistantTextStartedAt,
-    committedAt: turn.committedAt,
+    committedAt,
     finalTranscriptAt,
     firstTranscriptAt,
     sessionId,
@@ -11193,7 +11278,19 @@ var summarizeTurn = (sessionId, turn, options) => {
       { label: "Final transcript to commit", valueMs: commitAfterFinalMs },
       {
         label: "Commit to assistant text",
-        valueMs: assistantTextStartedAt === undefined ? undefined : 0
+        valueMs: assistantTextStartedAt === undefined ? undefined : Math.max(0, assistantTextStartedAt - committedAt)
+      },
+      {
+        label: "Assistant text to TTS send",
+        valueMs: ttsSendStartedAt === undefined || assistantTextStartedAt === undefined ? undefined : Math.max(0, ttsSendStartedAt - assistantTextStartedAt)
+      },
+      {
+        label: "TTS send duration",
+        valueMs: ttsSendCompletedAt === undefined || ttsSendStartedAt === undefined ? undefined : Math.max(0, ttsSendCompletedAt - ttsSendStartedAt)
+      },
+      {
+        label: "TTS to first audio",
+        valueMs: assistantAudioReceivedAt === undefined || ttsSendCompletedAt === undefined ? undefined : Math.max(0, assistantAudioReceivedAt - ttsSendCompletedAt)
       }
     ],
     status,
@@ -11222,9 +11319,14 @@ var resolveSessions = async (options) => {
 };
 var summarizeVoiceTurnLatency = async (options) => {
   const sessions = await resolveSessions(options);
+  const traceEvents = options.traceStore ? await options.traceStore.list({
+    limit: 1000,
+    type: "turn_latency.stage"
+  }) : [];
+  const stageIndex = createTraceStageIndex(traceEvents);
   const warnAfterMs = options.warnAfterMs ?? DEFAULT_WARN_AFTER_MS;
   const failAfterMs = options.failAfterMs ?? DEFAULT_FAIL_AFTER_MS;
-  const turns = sessions.flatMap((session) => session.turns.map((turn) => summarizeTurn(session.id, turn, { failAfterMs, warnAfterMs }))).sort((left, right) => right.committedAt - left.committedAt);
+  const turns = sessions.flatMap((session) => session.turns.map((turn) => summarizeTurn(session.id, turn, { failAfterMs, stageIndex, warnAfterMs }))).sort((left, right) => right.committedAt - left.committedAt);
   const totals = turns.map((turn) => turn.totalMs).filter((value) => typeof value === "number");
   const failed = turns.filter((turn) => turn.status === "fail").length;
   const warnings = turns.filter((turn) => turn.status === "warn").length;

package/dist/testing/index.js

@@ -5216,7 +5216,7 @@ var createVoiceSession = (options) => {
   } : undefined;
   const appendTrace = async (input) => {
     await options.trace?.append({
-      at: Date.now(),
+      at: input.at ?? Date.now(),
       metadata: input.metadata,
       payload: input.payload,
       scenarioId: input.session?.scenarioId ?? options.scenarioId,
@@ -5225,6 +5225,13 @@ var createVoiceSession = (options) => {
       type: input.type
     });
   };
+  const appendTurnLatencyStage = async (input) => appendTrace({
+    at: input.at,
+    payload: { stage: input.stage },
+    session: input.session,
+    turnId: input.turnId,
+    type: "turn_latency.stage"
+  });
   const phraseHints = options.phraseHints ?? [];
   const lexicon = options.lexicon ?? [];
   let socket = options.socket;
@@ -6175,6 +6182,13 @@ var createVoiceSession = (options) => {
             turnId: activeTTSTurnId,
             type: "audio"
           });
+          if (activeTTSTurnId) {
+            await appendTurnLatencyStage({
+              at: receivedAt,
+              stage: "assistant_audio_received",
+              turnId: activeTTSTurnId
+            });
+          }
         });
       });
       openedSession.on("error", (event) => {
@@ -6233,6 +6247,7 @@ var createVoiceSession = (options) => {
       voicemail: committedOutput?.voicemail
     };
     if (output?.assistantText) {
+      const assistantTextStartedAt = Date.now();
       await writeSession((currentSession) => {
         setTurnResult(currentSession, turn.id, {
           assistantText: output.assistantText
@@ -6243,6 +6258,12 @@ var createVoiceSession = (options) => {
         turnId: turn.id,
         type: "assistant"
       });
+      await appendTurnLatencyStage({
+        at: assistantTextStartedAt,
+        session,
+        stage: "assistant_text_started",
+        turnId: turn.id
+      });
       await appendTrace({
         payload: {
           text: output.assistantText,
@@ -6257,7 +6278,18 @@ var createVoiceSession = (options) => {
         if (activeTTSSession) {
           const ttsStartedAt = Date.now();
           activeTTSTurnId = turn.id;
+          await appendTurnLatencyStage({
+            at: ttsStartedAt,
+            session,
+            stage: "tts_send_started",
+            turnId: turn.id
+          });
           await activeTTSSession.send(output.assistantText);
+          await appendTurnLatencyStage({
+            session,
+            stage: "tts_send_completed",
+            turnId: turn.id
+          });
           await appendTrace({
             payload: {
               elapsedMs: Date.now() - ttsStartedAt,
@@ -6454,6 +6486,30 @@ var createVoiceSession = (options) => {
       turnId: turn.id,
       type: "turn.cost"
     });
+    const firstTranscriptAt = turn.transcripts.map((transcript) => transcript.endedAtMs ?? transcript.startedAtMs).filter((value) => typeof value === "number").sort((left, right) => left - right)[0];
+    const finalTranscriptAt = turn.transcripts.filter((transcript) => transcript.isFinal).map((transcript) => transcript.endedAtMs ?? transcript.startedAtMs).filter((value) => typeof value === "number").sort((left, right) => left - right)[0];
+    if (firstTranscriptAt !== undefined) {
+      await appendTurnLatencyStage({
+        at: firstTranscriptAt,
+        session: updatedSession,
+        stage: "speech_detected",
+        turnId: turn.id
+      });
+    }
+    if (finalTranscriptAt !== undefined) {
+      await appendTurnLatencyStage({
+        at: finalTranscriptAt,
+        session: updatedSession,
+        stage: "final_transcript",
+        turnId: turn.id
+      });
+    }
+    await appendTurnLatencyStage({
+      at: turn.committedAt,
+      session: updatedSession,
+      stage: "turn_committed",
+      turnId: turn.id
+    });
     await send({
       turn,
       type: "turn"

package/dist/trace.d.ts

@@ -1,4 +1,4 @@
-export type VoiceTraceEventType = 'assistant.guardrail' | 'assistant.memory' | 'assistant.run' | 'agent.handoff' | 'agent.model' | 'agent.result' | 'agent.tool' | 'call.handoff' | 'call.lifecycle' | 'client.barge_in' | 'session.error' | 'turn.assistant' | 'turn.committed' | 'turn.cost' | 'turn.transcript' | 'workflow.contract';
+export type VoiceTraceEventType = 'assistant.guardrail' | 'assistant.memory' | 'assistant.run' | 'agent.handoff' | 'agent.model' | 'agent.result' | 'agent.tool' | 'call.handoff' | 'call.lifecycle' | 'client.barge_in' | 'session.error' | 'turn.assistant' | 'turn.committed' | 'turn.cost' | 'turn_latency.stage' | 'turn.transcript' | 'workflow.contract';
 export type VoiceTraceEvent<TPayload extends Record<string, unknown> = Record<string, unknown>> = {
     at: number;
     id?: string;

package/dist/turnLatency.d.ts

@@ -1,4 +1,5 @@
 import { Elysia } from 'elysia';
+import type { VoiceTraceEventStore } from './trace';
 import type { VoiceSessionRecord, VoiceSessionStore } from './types';
 export type VoiceTurnLatencyStatus = 'pass' | 'warn' | 'fail' | 'empty';
 export type VoiceTurnLatencyStage = {
@@ -32,6 +33,7 @@ export type VoiceTurnLatencyOptions<TSession extends VoiceSessionRecord = VoiceS
     sessionIds?: string[];
     sessions?: TSession[];
     store?: VoiceSessionStore<TSession>;
+    traceStore?: VoiceTraceEventStore;
     warnAfterMs?: number;
     failAfterMs?: number;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@absolutejs/voice",
-	"version": "0.0.22-beta.95",
+	"version": "0.0.22-beta.97",
 	"description": "Voice primitives and Elysia plugin for AbsoluteJS",
 	"repository": {
 		"type": "git",