@agent-native/toolkit 0.221.0-same-host-mcp-20261027-20261027081850 → 0.221.0

package/src/composer/realtime-voice-transcript.spec.ts

@@ -99,4 +99,40 @@ describe("realtime voice transcript registry", () => {
     visible = true;
     expect(registry.activeThreadId()).toBe("thread-1");
   });
+
+  it("queues a reply's work until its thread's sink takes it, finished over running", () => {
+    const registry = createRealtimeVoiceTranscriptRegistry();
+    const running = {
+      id: "call-1",
+      threadId: "thread-1",
+      calls: [
+        {
+          id: "call-1",
+          name: "web-search",
+          status: "running" as const,
+          startedAt: "2026-07-10T20:00:00.000Z",
+        },
+      ],
+    };
+    registry.publishWork(running);
+    registry.publishWork({
+      ...running,
+      completedAt: "2026-07-10T20:00:02.000Z",
+    });
+    registry.publishWork(running);
+    expect(registry.pendingCount()).toBe(1);
+
+    const appendWork = vi.fn(() => true);
+    registry.register({
+      threadId: "thread-1",
+      active: true,
+      append: vi.fn(() => true),
+      appendWork,
+    });
+    expect(appendWork).toHaveBeenCalledOnce();
+    expect(appendWork).toHaveBeenCalledWith(
+      expect.objectContaining({ completedAt: "2026-07-10T20:00:02.000Z" }),
+    );
+    expect(registry.pendingCount()).toBe(0);
+  });
 });

package/src/composer/realtime-voice-transcript.ts

@@ -10,17 +10,50 @@ export interface RealtimeVoiceTranscriptMessage {
   status?: "streaming" | "complete";
 }
 
+/** One tool call in a voice reply's work. */
+/** A tool name as a short label, such as "Edit design" for `edit-design`. */
+export function realtimeVoiceToolLabel(name: string): string {
+  const label = name.replace(/[_-]+/g, " ").trim();
+  return label ? label.charAt(0).toUpperCase() + label.slice(1) : name;
+}
+
+export interface RealtimeVoiceWorkCall {
+  /** The provider's call id. */
+  id: string;
+  name: string;
+  input?: Record<string, unknown>;
+  status: "running" | "completed" | "failed" | "cancelled";
+  output?: string;
+  startedAt: string;
+  completedAt?: string;
+}
+
+/**
+ * The tool calls one voice reply made, so the chat can show them as one work
+ * item. Each update is the whole reply so far under the same id.
+ */
+export interface RealtimeVoiceWorkUpdate {
+  id: string;
+  threadId: string;
+  calls: RealtimeVoiceWorkCall[];
+  /** Set once the reply has finished and every call in it has ended. */
+  completedAt?: string;
+}
+
 export interface RealtimeVoiceTranscriptSink {
   threadId: string;
   /** Read when a voice session starts, so a sink can report whether it is on screen. */
   active: boolean | (() => boolean);
   /** False keeps the message queued, newest text per id, until the sink re-registers. */
   append: (message: RealtimeVoiceTranscriptMessage) => boolean;
+  /** False keeps the update queued, newest per reply, until the sink re-registers. */
+  appendWork?: (update: RealtimeVoiceWorkUpdate) => boolean;
 }
 
 export interface RealtimeVoiceTranscriptRegistry {
   activeThreadId: () => string | undefined;
   publish: (message: RealtimeVoiceTranscriptMessage) => void;
+  publishWork: (update: RealtimeVoiceWorkUpdate) => void;
   register: (sink: RealtimeVoiceTranscriptSink) => () => void;
   pendingCount: () => number;
 }
@@ -31,10 +64,23 @@ export function createRealtimeVoiceTranscriptRegistry(): RealtimeVoiceTranscript
     RealtimeVoiceTranscriptSink & { order: number }
   >();
   const pending = new Map<string, RealtimeVoiceTranscriptMessage>();
+  const pendingWork = new Map<string, RealtimeVoiceWorkUpdate>();
   const delivered = new Set<string>();
   let order = 0;
 
   const flush = () => {
+    for (const [id, update] of pendingWork) {
+      let consumed = false;
+      for (const sink of sinks.values()) {
+        if (sink.threadId !== update.threadId || !sink.appendWork) continue;
+        try {
+          consumed = sink.appendWork(update) || consumed;
+        } catch {
+          // coercion-ok: a throwing sink leaves the update queued for the next flush.
+        }
+      }
+      if (consumed) pendingWork.delete(id);
+    }
     for (const [id, message] of pending) {
       let consumed = false;
       for (const sink of sinks.values()) {
@@ -86,13 +132,22 @@ export function createRealtimeVoiceTranscriptRegistry(): RealtimeVoiceTranscript
       });
       flush();
     },
+    publishWork(update) {
+      const id = update.id.trim();
+      const threadId = update.threadId.trim();
+      if (!id || !threadId) return;
+      // A finished reply outranks a late update from while it ran.
+      if (pendingWork.get(id)?.completedAt && !update.completedAt) return;
+      pendingWork.set(id, { ...update, id, threadId });
+      flush();
+    },
     register(sink) {
       const token = Symbol("realtime-voice-transcript-sink");
       sinks.set(token, { ...sink, order: ++order });
       flush();
       return () => sinks.delete(token);
     },
-    pendingCount: () => pending.size,
+    pendingCount: () => pending.size + pendingWork.size,
   };
 }
 

package/src/composer/realtime-voice-turn-metrics.spec.ts

@@ -0,0 +1,175 @@
+import { describe, expect, it, vi } from "vitest";
+
+import {
+  createRealtimeVoiceTurnMetrics,
+  REALTIME_VOICE_TURN_EVENT,
+} from "./realtime-voice-turn-metrics.js";
+
+function metrics() {
+  const emit = vi.fn();
+  const tracker = createRealtimeVoiceTurnMetrics(emit);
+  tracker.setContext({
+    protocol: "live",
+    intelligence: "balanced",
+    intelligenceSource: "default",
+  });
+  return { emit, tracker };
+}
+
+describe("realtime voice turn metrics", () => {
+  it("times speech end to first audio on the provider's session clock", () => {
+    const { emit, tracker } = metrics();
+    tracker.userTurnUpdated(1_000, "user-turn-1", 40_000);
+    tracker.userTurnUpdated(1_100, "user-turn-1", 40_200);
+    tracker.assistantAudioStarted(1_500, 41_100);
+    tracker.assistantAudioStarted(1_600, 41_400);
+    tracker.sessionEnded();
+
+    expect(emit).toHaveBeenCalledWith(REALTIME_VOICE_TURN_EVENT, {
+      protocol: "live",
+      intelligence: "balanced",
+      intelligence_source: "default",
+      turn_index: 0,
+      speech_to_audio_ms: 900,
+      latency_clock: "session",
+      tool_calls: 0,
+      write_tool_calls: 0,
+      failed_tool_calls: 0,
+      spoke_again_after_edit: null,
+      ms_after_edit: null,
+    });
+  });
+
+  it("flags speech within the window after an edit as a likely correction", () => {
+    const { emit, tracker } = metrics();
+    tracker.userSpeechStarted(0);
+    tracker.userSpeechEnded(1_000);
+    tracker.assistantAudioStarted(1_400);
+    tracker.toolCallStarted()(2_000, { status: "completed", readOnly: true });
+    tracker.toolCallStarted()(3_000, { status: "completed", readOnly: false });
+    tracker.toolCallStarted()(3_500, { status: "completed" });
+    tracker.userSpeechStarted(7_000);
+
+    expect(emit.mock.calls[0]![1]).toMatchObject({
+      speech_to_audio_ms: 400,
+      latency_clock: "browser",
+      tool_calls: 3,
+      write_tool_calls: 1,
+      spoke_again_after_edit: true,
+      ms_after_edit: 4_000,
+    });
+  });
+
+  it("does not count a reply long after the edit, or a failed edit", () => {
+    const { emit, tracker } = metrics();
+    tracker.userSpeechEnded(0);
+    tracker.toolCallStarted()(500, { status: "completed", readOnly: false });
+    tracker.userSpeechStarted(20_000);
+    tracker.userSpeechEnded(21_000);
+    tracker.toolCallStarted()(21_500, { status: "failed" });
+    tracker.userSpeechStarted(22_000);
+
+    expect(emit.mock.calls.map(([, properties]) => properties)).toMatchObject([
+      { turn_index: 0, spoke_again_after_edit: false, ms_after_edit: 19_500 },
+      {
+        turn_index: 1,
+        failed_tool_calls: 1,
+        write_tool_calls: 0,
+        spoke_again_after_edit: null,
+      },
+    ]);
+  });
+
+  it("counts neither an unclassified nor a cancelled call as an edit or a failure", () => {
+    const { emit, tracker } = metrics();
+    tracker.userSpeechEnded(0);
+    tracker.toolCallStarted()(500, { status: "completed" });
+    tracker.toolCallStarted()(600, { status: "cancelled" });
+    tracker.userSpeechStarted(1_000);
+
+    expect(emit.mock.calls[0]![1]).toMatchObject({
+      tool_calls: 2,
+      write_tool_calls: 0,
+      failed_tool_calls: 0,
+      spoke_again_after_edit: null,
+    });
+  });
+
+  it("keeps one turn while the user pauses with no reply yet", () => {
+    const { emit, tracker } = metrics();
+    tracker.userSpeechStarted(0);
+    tracker.userSpeechEnded(1_000);
+    tracker.userSpeechStarted(1_200);
+    tracker.userSpeechEnded(2_000);
+    tracker.assistantAudioStarted(2_300);
+    tracker.sessionEnded();
+
+    expect(emit).toHaveBeenCalledTimes(1);
+    expect(emit.mock.calls[0]![1]).toMatchObject({ speech_to_audio_ms: 300 });
+  });
+
+  it("ignores late text from a GPT-Live turn the assistant is already answering", () => {
+    const { emit, tracker } = metrics();
+    tracker.userTurnUpdated(1_000, "user-turn-1", 40_000);
+    tracker.assistantAudioStarted(1_400, 40_600);
+    tracker.userTurnUpdated(1_500, "user-turn-1", 40_100);
+    tracker.userTurnUpdated(9_000, "user-turn-2", 48_000);
+
+    expect(emit).toHaveBeenCalledTimes(1);
+    expect(emit.mock.calls[0]![1]).toMatchObject({
+      turn_index: 0,
+      speech_to_audio_ms: 600,
+      latency_clock: "session",
+    });
+  });
+
+  it("counts a tool result in the turn it started in, after an interruption", () => {
+    const { emit, tracker } = metrics();
+    tracker.userSpeechEnded(0);
+    const finishEdit = tracker.toolCallStarted();
+    // The user speaks again while the edit is still running.
+    tracker.userSpeechStarted(1_000);
+    tracker.userSpeechEnded(1_500);
+    expect(emit).not.toHaveBeenCalled();
+
+    finishEdit(2_000, { status: "completed", readOnly: false });
+    expect(emit).toHaveBeenCalledTimes(1);
+    // Speaking before the edit finished is not a reaction to it.
+    expect(emit.mock.calls[0]![1]).toMatchObject({
+      turn_index: 0,
+      tool_calls: 1,
+      write_tool_calls: 1,
+      spoke_again_after_edit: false,
+      ms_after_edit: null,
+    });
+
+    tracker.assistantResponded();
+    tracker.sessionEnded();
+    expect(emit.mock.calls[1]![1]).toMatchObject({
+      turn_index: 1,
+      tool_calls: 0,
+      write_tool_calls: 0,
+    });
+  });
+
+  it("measures a text-only reply without a first-audio time", () => {
+    const { emit, tracker } = metrics();
+    tracker.userSpeechEnded(0);
+    tracker.assistantResponded();
+    tracker.userSpeechStarted(5_000);
+
+    expect(emit.mock.calls[0]![1]).toMatchObject({
+      speech_to_audio_ms: null,
+      latency_clock: null,
+    });
+  });
+
+  it("skips a greeting nobody asked for and an empty session", () => {
+    const { emit, tracker } = metrics();
+    tracker.assistantAudioStarted(100);
+    tracker.sessionEnded();
+    tracker.sessionEnded();
+
+    expect(emit).not.toHaveBeenCalled();
+  });
+});

package/src/composer/realtime-voice-turn-metrics.ts

@@ -0,0 +1,192 @@
+export const REALTIME_VOICE_TURN_EVENT = "realtime_voice_turn";
+export const REALTIME_VOICE_CORRECTION_WINDOW_MS = 10_000;
+
+export interface RealtimeVoiceTurnMetricsContext {
+  protocol: "live" | "realtime";
+  intelligence: string;
+  intelligenceSource: "user" | "default";
+}
+
+/**
+ * Records how a tool call that started in a turn ended. Only a call classified
+ * `readOnly: false` counts as an edit; an unclassified call is neither.
+ */
+export type RealtimeVoiceTurnToolCallFinish = (
+  at: number,
+  result: { status: string; readOnly?: boolean | undefined },
+) => void;
+
+export interface RealtimeVoiceTurnMetrics {
+  setContext: (context: RealtimeVoiceTurnMetricsContext) => void;
+  /** Realtime: the provider heard the user start speaking. */
+  userSpeechStarted: (at: number) => void;
+  /** Realtime: the provider heard the user stop speaking. */
+  userSpeechEnded: (at: number) => void;
+  /**
+   * GPT-Live: a user turn's transcript grew. `turnId` names the transcript
+   * turn and `endMs` is its end on the session clock, so late text from a turn
+   * already answered never starts another.
+   */
+  userTurnUpdated: (at: number, turnId: string, endMs: number) => void;
+  /** The assistant replied in text, which carries no audio timing. */
+  assistantResponded: () => void;
+  /** `timelineMs` is the provider's session clock, when the event has one. */
+  assistantAudioStarted: (at: number, timelineMs?: number) => void;
+  /** The returned function records the call's outcome against this turn. */
+  toolCallStarted: () => RealtimeVoiceTurnToolCallFinish;
+  /** Reports the open turn; nobody can speak again after the session ends. */
+  sessionEnded: () => void;
+}
+
+interface Turn {
+  userTurnId?: string | undefined;
+  speechEndedAt?: number | undefined;
+  speechEndedTimelineMs?: number | undefined;
+  firstAudioAt?: number | undefined;
+  firstAudioTimelineMs?: number | undefined;
+  responding: boolean;
+  toolCalls: number;
+  pendingToolCalls: number;
+  writeToolCalls: number;
+  failedToolCalls: number;
+  lastWriteCompletedAt?: number | undefined;
+  /** Reported while a call still ran; it is sent once the last call ends. */
+  heldReport?: { index: number; nextSpeechAt: number | undefined } | undefined;
+}
+
+function emptyTurn(): Turn {
+  return {
+    responding: false,
+    toolCalls: 0,
+    pendingToolCalls: 0,
+    writeToolCalls: 0,
+    failedToolCalls: 0,
+  };
+}
+
+/**
+ * One event per voice turn: from the end of the user's speech to the next time
+ * they speak. It carries timings and counts only, never transcript text or
+ * tool arguments.
+ */
+export function createRealtimeVoiceTurnMetrics(
+  emit: (name: string, properties: Record<string, unknown>) => void,
+  correctionWindowMs = REALTIME_VOICE_CORRECTION_WINDOW_MS,
+): RealtimeVoiceTurnMetrics {
+  let context: RealtimeVoiceTurnMetricsContext | undefined;
+  let turn = emptyTurn();
+  let turnIndex = 0;
+
+  const send = (
+    current: Turn,
+    index: number,
+    nextSpeechAt: number | undefined,
+  ) => {
+    const { speechEndedTimelineMs, firstAudioTimelineMs } = current;
+    const sessionClock =
+      speechEndedTimelineMs !== undefined && firstAudioTimelineMs !== undefined;
+    const speechToAudioMs = sessionClock
+      ? firstAudioTimelineMs - speechEndedTimelineMs
+      : current.speechEndedAt !== undefined &&
+          current.firstAudioAt !== undefined
+        ? current.firstAudioAt - current.speechEndedAt
+        : null;
+    // Speech that began while the edit still ran did not answer the edit.
+    const msAfterEdit =
+      current.lastWriteCompletedAt !== undefined &&
+      nextSpeechAt !== undefined &&
+      nextSpeechAt >= current.lastWriteCompletedAt
+        ? Math.round(nextSpeechAt - current.lastWriteCompletedAt)
+        : null;
+    emit(REALTIME_VOICE_TURN_EVENT, {
+      protocol: context?.protocol ?? null,
+      intelligence: context?.intelligence ?? null,
+      intelligence_source: context?.intelligenceSource ?? null,
+      turn_index: index,
+      speech_to_audio_ms:
+        speechToAudioMs === null
+          ? null
+          : Math.max(0, Math.round(speechToAudioMs)),
+      latency_clock:
+        speechToAudioMs === null ? null : sessionClock ? "session" : "browser",
+      tool_calls: current.toolCalls,
+      write_tool_calls: current.writeToolCalls,
+      failed_tool_calls: current.failedToolCalls,
+      spoke_again_after_edit:
+        current.lastWriteCompletedAt === undefined
+          ? null
+          : msAfterEdit !== null && msAfterEdit <= correctionWindowMs,
+      ms_after_edit: msAfterEdit,
+    });
+  };
+
+  const report = (nextSpeechAt?: number) => {
+    const current = turn;
+    turn = emptyTurn();
+    if (current.speechEndedAt === undefined && current.toolCalls === 0) return;
+    const index = turnIndex++;
+    if (current.pendingToolCalls > 0) {
+      current.heldReport = { index, nextSpeechAt };
+      return;
+    }
+    send(current, index, nextSpeechAt);
+  };
+
+  return {
+    setContext(next) {
+      context = next;
+    },
+    userSpeechStarted(at) {
+      if (turn.responding) report(at);
+    },
+    userSpeechEnded(at) {
+      if (turn.responding) report(at);
+      turn.speechEndedAt = at;
+    },
+    userTurnUpdated(at, turnId, endMs) {
+      if (turn.userTurnId === turnId) {
+        if (turn.responding) return;
+      } else if (turn.responding) {
+        report(at);
+      }
+      turn.userTurnId = turnId;
+      turn.speechEndedAt = at;
+      turn.speechEndedTimelineMs = endMs;
+    },
+    assistantResponded() {
+      turn.responding = true;
+    },
+    assistantAudioStarted(at, timelineMs) {
+      turn.responding = true;
+      if (turn.firstAudioAt !== undefined) return;
+      turn.firstAudioAt = at;
+      turn.firstAudioTimelineMs = timelineMs;
+    },
+    toolCallStarted() {
+      const owner = turn;
+      owner.responding = true;
+      owner.toolCalls += 1;
+      owner.pendingToolCalls += 1;
+      let finished = false;
+      return (at, result) => {
+        if (finished) return;
+        finished = true;
+        owner.pendingToolCalls -= 1;
+        if (result.status === "failed") {
+          owner.failedToolCalls += 1;
+        } else if (result.status === "completed" && result.readOnly === false) {
+          owner.writeToolCalls += 1;
+          owner.lastWriteCompletedAt = at;
+        }
+        const held = owner.heldReport;
+        if (owner.pendingToolCalls > 0 || !held) return;
+        owner.heldReport = undefined;
+        send(owner, held.index, held.nextSpeechAt);
+      };
+    },
+    sessionEnded() {
+      report();
+      turnIndex = 0;
+    },
+  };
+}

package/src/composer/runtime-adapters.tsx

@@ -267,11 +267,13 @@ export interface ComposerRuntimeAdapters {
       text: string,
       context: VoiceContextPack | undefined,
     ) => string;
+    /** Sends a voice analytics event; properties never carry transcript text. */
+    trackEvent?: (name: string, properties: Record<string, unknown>) => void;
   };
 }
 
 const identityPath = (path: string) => path;
-const fallbackTranslate: ComposerTranslate = (key, options) => {
+export const fallbackTranslate: ComposerTranslate = (key, options) => {
   const template =
     typeof options?.defaultValue === "string" ? options.defaultValue : key;
 
@@ -352,6 +354,7 @@ type ResolvedComposerRuntimeAdapters = ComposerRuntimeAdapters &
           | "setAppState"
           | "subscribeSidebarState"
           | "applyContextReplacements"
+          | "trackEvent"
         >
       >;
   };
@@ -394,6 +397,7 @@ const fallbackAdapters: ResolvedComposerRuntimeAdapters = {
     setAppState: () => {},
     subscribeSidebarState: () => () => {},
     applyContextReplacements: applyVoiceContextReplacements,
+    trackEvent: () => {},
   },
 };