@kuralle-syrinx/realtime 4.2.0 → 4.4.0

package/src/realtime-bridge.ts

@@ -115,6 +115,7 @@ export class RealtimeBridge implements VoicePlugin {
   private turnAssistantText = "";
   /** Concatenated streamed transcript fragments — providers that emit deltas only, no final (Gemini Live). */
   private turnAssistantDeltas = "";
+  private pendingSpeechEndedAtMs: number | null = null;
   private sessionAbort: AbortController | null = null;
   private inflight: AbortController | undefined;
   private delegateTask: Promise<void> | null = null;
@@ -129,6 +130,16 @@ export class RealtimeBridge implements VoicePlugin {
     private readonly opts: RealtimeBridgeOptions = {},
   ) {}
 
+  injectContext(text: string): void {
+    // Provider history differs here: OpenAI retains system items, while Gemini drops
+    // system/developer history, so the adapter selects the provider-safe representation.
+    if (this.adapter.injectContext) {
+      this.adapter.injectContext(text);
+      return;
+    }
+    console.warn("RealtimeBridge: context injection requested but the adapter does not support it");
+  }
+
   async initialize(bus: PipelineBus, _cfg: PluginConfig): Promise<void> {
     this.bus = bus;
     this.sessionAbort = new AbortController();
@@ -216,6 +227,12 @@ export class RealtimeBridge implements VoicePlugin {
         } else if (!ev.final && ev.text) {
           // Streamed fragments already carry their own leading spaces — concatenate verbatim.
           this.turnAssistantDeltas += ev.text;
+          bus.push(Route.Main, {
+            kind: "llm.delta",
+            contextId: this.contextId,
+            timestampMs: Date.now(),
+            text: ev.text,
+          });
         }
         break;
       case "tool_call":
@@ -223,10 +240,23 @@ export class RealtimeBridge implements VoicePlugin {
         break;
       case "response_done":
         this.onResponseDone(bus);
+        // Meter the native front — previously native turns produced no usage at all.
+        if (ev.usage && this.contextId) {
+          bus.push(Route.Background, {
+            kind: "usage.recorded",
+            contextId: this.contextId,
+            timestampMs: Date.now(),
+            stage: "llm",
+            ...ev.usage,
+          });
+        }
         break;
       case "speech_started":
         this.onSpeechStarted(bus);
         break;
+      case "speech_stopped":
+        this.onSpeechStopped(bus);
+        break;
       case "resumption_handle": {
         // G4: persist-worthy native-resume handle (Gemini). Background route —
         // a durable host stores the latest and passes it back on reconnect.
@@ -338,6 +368,25 @@ export class RealtimeBridge implements VoicePlugin {
     this.inflight = controller;
     let answer = "";
     let grounded = false;
+    let blockedMessage: string | undefined;
+    let blockedPayload: unknown;
+    let passStartedMs = Date.now();
+    let passTtftRecorded = false;
+    const pushConversationMetric = (name: string, value: string, timestampMs: number): void => {
+      bus.push(Route.Main, {
+        kind: "metric.conversation",
+        contextId,
+        timestampMs,
+        name,
+        value,
+      });
+    };
+    const recordPassTtft = (timestampMs: number): void => {
+      if (passTtftRecorded) return;
+      passTtftRecorded = true;
+      pushConversationMetric("llm.pass_ttft_ms", String(timestampMs - passStartedMs), timestampMs);
+    };
+    pushConversationMetric("llm.call_started", "1", passStartedMs);
 
     // G2 observability: the query is on its way to the reasoner (Background route, R4).
     const queryStartedMs = Date.now();
@@ -361,10 +410,37 @@ export class RealtimeBridge implements VoicePlugin {
       })) {
         switch (part.type) {
           case "text-delta":
+            recordPassTtft(Date.now());
             answer += part.text;
             break;
+          case "tool-call":
+            recordPassTtft(Date.now());
+            break;
           case "tool-result":
             grounded = true;
+            passStartedMs = Date.now();
+            passTtftRecorded = false;
+            pushConversationMetric("llm.call_started", "1", passStartedMs);
+            break;
+          case "control": {
+            const controlMs = Date.now();
+            bus.push(Route.Background, {
+              kind: "delegate.result",
+              contextId,
+              timestampMs: controlMs,
+              query: userText,
+              answer,
+              durationMs: controlMs - queryStartedMs,
+              grounded,
+              toolId: ev.toolId,
+              toolName: ev.toolName,
+              control: { name: part.name, payload: part.payload },
+            });
+            break;
+          }
+          case "blocked":
+            blockedMessage = part.userFacingMessage;
+            blockedPayload = part.payload;
             break;
           case "finish":
             if (!answer && part.text) answer = part.text;
@@ -382,6 +458,7 @@ export class RealtimeBridge implements VoicePlugin {
             this.onError(bus, part.cause, true);
             return;
         }
+        if (blockedMessage !== undefined) break;
       }
     } catch (err) {
       if (isAbortError(err)) return;
@@ -392,6 +469,24 @@ export class RealtimeBridge implements VoicePlugin {
       if (this.inflight === controller) this.inflight = undefined;
     }
 
+    if (blockedMessage !== undefined) {
+      const blockedMs = Date.now();
+      bus.push(Route.Background, {
+        kind: "delegate.result",
+        contextId,
+        timestampMs: blockedMs,
+        query: userText,
+        answer: blockedMessage,
+        durationMs: blockedMs - queryStartedMs,
+        grounded,
+        toolId: ev.toolId,
+        toolName: ev.toolName,
+        blocked: { userFacingMessage: blockedMessage, payload: blockedPayload },
+      });
+      this.adapter.injectToolResult(ev.toolId, this.formatToolResult(blockedMessage));
+      return;
+    }
+
     if (answer.length === 0) {
       this.onError(bus, new Error("delegate produced no output"), false);
       return;
@@ -445,6 +540,20 @@ export class RealtimeBridge implements VoicePlugin {
     bus.push(Route.Critical, packet);
   }
 
+  private onSpeechStopped(bus: PipelineBus): void {
+    const timestampMs = Date.now();
+    if (!this.contextId) {
+      // Server VAD reports speech end before response.created mints the response context.
+      this.pendingSpeechEndedAtMs = timestampMs;
+      return;
+    }
+    bus.push(Route.Main, {
+      kind: "vad.speech_ended",
+      contextId: this.contextId,
+      timestampMs,
+    });
+  }
+
   private onResponseStarted(bus: PipelineBus): void {
     const previousContextId = this.contextId;
     this.contextId = crypto.randomUUID();
@@ -453,6 +562,14 @@ export class RealtimeBridge implements VoicePlugin {
     this.turnAssistantDeltas = "";
     this.playedMs = 0;
     this.audioRemainder = new Uint8Array(0);
+    if (this.pendingSpeechEndedAtMs !== null) {
+      bus.push(Route.Main, {
+        kind: "vad.speech_ended",
+        contextId: this.contextId,
+        timestampMs: this.pendingSpeechEndedAtMs,
+      });
+      this.pendingSpeechEndedAtMs = null;
+    }
     const packet: TurnChangePacket = {
       kind: "turn.change",
       contextId: this.contextId,
@@ -536,6 +653,8 @@ export class RealtimeBridge implements VoicePlugin {
       timestampMs,
       text: this.turnUserText,
       transcripts,
+      endpointingOwner: "timer",
+      endpointingReason: "end_of_speech",
     };
 
     if (this.opts.textOnly) {
@@ -558,19 +677,21 @@ export class RealtimeBridge implements VoicePlugin {
     // providers that only emit non-final deltas (Gemini Live).
     const assistantText = this.turnAssistantText.trim() || this.turnAssistantDeltas.trim();
     if (assistantText) {
-      const delta: LlmDeltaPacket = {
-        kind: "llm.delta",
-        contextId: this.contextId,
-        timestampMs,
-        text: assistantText,
-      };
       const done: LlmResponseDonePacket = {
         kind: "llm.done",
         contextId: this.contextId,
         timestampMs,
         text: assistantText,
       };
-      bus.push(Route.Main, delta, done);
+      if (this.turnAssistantDeltas.length === 0) {
+        bus.push(Route.Main, {
+          kind: "llm.delta",
+          contextId: this.contextId,
+          timestampMs,
+          text: assistantText,
+        });
+      }
+      bus.push(Route.Main, done);
     }
     bus.push(Route.Main, turnComplete, ttsEnd);
   }

package/src/edge-safety.test.ts

@@ -1,173 +0,0 @@
-// SPDX-License-Identifier: MIT
-//
-// R-14: regression gate — the realtime package must run on workerd without
-// Buffer, process, or node:crypto. Reintroducing any Node-only primitive in
-// src/ should fail this test.
-
-import { readFileSync, readdirSync } from "node:fs";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
-
-import { describe, expect, it } from "vitest";
-
-import {
-  PipelineBusImpl,
-  Route,
-  type TextToSpeechAudioPacket,
-  type TurnChangePacket,
-} from "@kuralle-syrinx/core";
-import type { ManagedSocket, SocketData, SocketFactory } from "@kuralle-syrinx/ws";
-
-import { bytesToBase64, fromOpenAIRealtime } from "./from-openai-realtime.js";
-import { RealtimeBridge } from "./realtime-bridge.js";
-
-const srcDir = path.dirname(fileURLToPath(import.meta.url));
-
-interface EdgeMockHarness {
-  readonly factory: SocketFactory;
-  readonly sent: string[];
-  inject(msg: Record<string, unknown>): void;
-}
-
-function createEdgeMockHarness(): EdgeMockHarness {
-  const sent: string[] = [];
-  let messageHandler: ((data: SocketData, isBinary: boolean) => void) | null = null;
-
-  const socket: ManagedSocket = {
-    get isOpen() {
-      return true;
-    },
-    send: (data: SocketData) => {
-      sent.push(typeof data === "string" ? data : "");
-    },
-    keepAlivePing: () => {},
-    verify: async () => true,
-    dispose: () => {},
-    onOpen: (handler) => {
-      queueMicrotask(() => handler());
-    },
-    onMessage: (handler) => {
-      messageHandler = handler;
-    },
-    onClose: () => {},
-    onError: () => {},
-  };
-
-  return {
-    factory: () => socket,
-    sent,
-    inject: (msg) => messageHandler?.(JSON.stringify(msg), false),
-  };
-}
-
-function pcmFromSamples(samples: readonly number[]): Uint8Array {
-  const pcm = Int16Array.from(samples);
-  return new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
-}
-
-async function waitFor(predicate: () => boolean, timeoutMs = 2000): Promise<void> {
-  const startedAt = Date.now();
-  while (Date.now() - startedAt < timeoutMs) {
-    if (predicate()) return;
-    await new Promise((resolve) => setTimeout(resolve, 5));
-  }
-  throw new Error("Timed out waiting for condition");
-}
-
-function collectSourceFiles(dir: string): string[] {
-  const out: string[] = [];
-  for (const entry of readdirSync(dir, { withFileTypes: true })) {
-    const full = path.join(dir, entry.name);
-    if (entry.isDirectory()) {
-      out.push(...collectSourceFiles(full));
-      continue;
-    }
-    if (entry.name.endsWith(".ts") && !entry.name.endsWith(".test.ts")) {
-      out.push(full);
-    }
-  }
-  return out;
-}
-
-describe("edge safety (R-14)", () => {
-  it("src/ contains no Buffer, process, or node:crypto imports", () => {
-    const forbidden = [
-      /\bfrom\s+["']node:crypto["']/,
-      /\brequire\s*\(\s*["']node:crypto["']\s*\)/,
-      /\bglobalThis\.Buffer\b/,
-      /\bglobalThis\.process\b/,
-      /\bprocess\.env\b/,
-      /\bBuffer\.from\b/,
-      /\bBuffer\.alloc\b/,
-    ];
-    const hits: string[] = [];
-    for (const file of collectSourceFiles(srcDir)) {
-      const text = readFileSync(file, "utf8");
-      for (const pattern of forbidden) {
-        if (pattern.test(text)) {
-          hits.push(`${path.relative(srcDir, file)}: ${pattern.source}`);
-        }
-      }
-    }
-    expect(hits).toEqual([]);
-  });
-
-  // NOTE: we do NOT delete globalThis.Buffer/process here — vitest's own worker needs `process`
-  // (its uncaughtException handler calls process.listeners), so deleting it crashes the runner.
-  // Edge-safety is enforced statically by the source-scan above; this test proves the audio path
-  // actually runs end-to-end using only the runtime-agnostic helpers.
-  it("fromOpenAIRealtime + RealtimeBridge round-trip audio via runtime-agnostic helpers", async () => {
-    const mock = createEdgeMockHarness();
-    const adapter = fromOpenAIRealtime({
-      apiKey: "edge-test-key",
-      socketFactory: mock.factory,
-      url: () => "wss://example.test/realtime?model=gpt-realtime-2",
-    });
-    const bridge = new RealtimeBridge(adapter);
-    const bus = new PipelineBusImpl();
-    const turnChanges: TurnChangePacket[] = [];
-    const ttsAudio: TextToSpeechAudioPacket[] = [];
-    bus.on("turn.change", (pkt) => { turnChanges.push(pkt as TurnChangePacket); });
-    bus.on("tts.audio", (pkt) => { ttsAudio.push(pkt as TextToSpeechAudioPacket); });
-
-    const started = bus.start();
-
-    const initTask = bridge.initialize(bus, {});
-    await waitFor(() => mock.sent.length > 0);
-    mock.inject({ type: "session.updated" });
-    await initTask;
-
-    bus.push(Route.Main, {
-      kind: "user.audio_received",
-      contextId: "transport-turn",
-      timestampMs: Date.now(),
-      audio: pcmFromSamples([100, 200, 300, 400]),
-    });
-
-    await waitFor(() =>
-      mock.sent.some((raw) => (JSON.parse(raw) as { type: string }).type === "input_audio_buffer.append"),
-    );
-
-    const providerPcm = pcmFromSamples(Array.from({ length: 960 }, (_, i) => i));
-    mock.inject({ type: "response.created" });
-    mock.inject({
-      type: "response.output_audio.delta",
-      delta: bytesToBase64(providerPcm),
-    });
-    mock.inject({ type: "response.done" });
-
-    await waitFor(() => ttsAudio.length > 0 && turnChanges.length > 0);
-
-    const contextId = turnChanges[0]!.contextId;
-    expect(contextId).toMatch(
-      /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/,
-    );
-    expect(ttsAudio[0]!.contextId).toBe(contextId);
-    expect(ttsAudio[0]!.sampleRateHz).toBe(16_000);
-    expect(ttsAudio[0]!.audio.byteLength).toBeGreaterThan(0);
-
-    await bridge.close();
-    bus.stop();
-    await started;
-  });
-});

package/src/from-gemini-live.test.ts

@@ -1,241 +0,0 @@
-// SPDX-License-Identifier: MIT
-
-import { afterEach, describe, expect, it, vi } from "vitest";
-
-import type { LiveServerMessage } from "@google/genai";
-
-import type { RealtimeEvent } from "./realtime-adapter.js";
-import { bytesToBase64 } from "./base64.js";
-import { fromGeminiLive } from "./from-gemini-live.js";
-
-const sendRealtimeInput = vi.fn();
-const sendToolResponse = vi.fn();
-const sendClientContent = vi.fn();
-const closeSession = vi.fn();
-
-let onopen: (() => void) | null = null;
-let onmessage: ((msg: LiveServerMessage) => void) | null = null;
-
-const liveConnect = vi.fn().mockImplementation(async ({ callbacks }: {
-  callbacks: {
-    onopen?: () => void;
-    onmessage?: (msg: LiveServerMessage) => void;
-  };
-}) => {
-  onopen = callbacks.onopen ?? null;
-  onmessage = callbacks.onmessage ?? null;
-  queueMicrotask(() => callbacks.onopen?.());
-  return {
-    sendRealtimeInput,
-    sendToolResponse,
-    sendClientContent,
-    close: closeSession,
-  };
-});
-
-vi.mock("@google/genai", () => ({
-  GoogleGenAI: vi.fn().mockImplementation(() => ({
-    live: { connect: liveConnect },
-  })),
-  Modality: { AUDIO: "AUDIO" },
-}));
-
-afterEach(() => {
-  sendRealtimeInput.mockClear();
-  sendToolResponse.mockClear();
-  sendClientContent.mockClear();
-  closeSession.mockClear();
-  liveConnect.mockClear();
-  onopen = null;
-  onmessage = null;
-});
-
-async function collectEvents(
-  events: AsyncIterable<RealtimeEvent>,
-  max = 12,
-): Promise<RealtimeEvent[]> {
-  const out: RealtimeEvent[] = [];
-  for await (const event of events) {
-    out.push(event);
-    if (out.length >= max) break;
-  }
-  return out;
-}
-
-function inject(msg: Partial<LiveServerMessage> & Record<string, unknown>): void {
-  if (!onmessage) throw new Error("mock session onmessage not wired");
-  onmessage(msg as LiveServerMessage);
-}
-
-describe("fromGeminiLive", () => {
-  it("emits client calls for open, audio, and tool result", async () => {
-    const adapter = fromGeminiLive({
-      apiKey: "test-key",
-      tools: [{
-        name: "consult_knowledge",
-        description: "Answer knowledge questions.",
-        parameters: { type: "object", properties: { query: { type: "string" } }, required: ["query"] },
-      }],
-    });
-
-    await adapter.open(new AbortController().signal);
-
-    expect(liveConnect).toHaveBeenCalledTimes(1);
-    const connectArg = liveConnect.mock.calls[0]![0] as {
-      model: string;
-      config: Record<string, unknown>;
-    };
-    expect(connectArg.model).toBe("gemini-3.1-flash-live-preview");
-    expect(connectArg.config["tools"]).toEqual([{
-      functionDeclarations: [{
-        name: "consult_knowledge",
-        description: "Answer knowledge questions.",
-        parametersJsonSchema: {
-          type: "object",
-          properties: { query: { type: "string" } },
-          required: ["query"],
-        },
-      }],
-    }]);
-
-    const pcm = new Uint8Array([0, 1, 2, 3]);
-    adapter.sendAudio(pcm);
-    expect(sendRealtimeInput).toHaveBeenCalledWith({
-      audio: {
-        data: bytesToBase64(pcm),
-        mimeType: "audio/pcm;rate=16000",
-      },
-    });
-
-    inject({
-      toolCall: {
-        functionCalls: [{
-          id: "call_abc",
-          name: "consult_knowledge",
-          args: { query: "late add" },
-        }],
-      },
-    });
-    adapter.injectToolResult("call_abc", "Late Add Petition required.");
-    expect(sendToolResponse).toHaveBeenCalledWith({
-      functionResponses: [{
-        id: "call_abc",
-        name: "consult_knowledge",
-        response: { result: "Late Add Petition required." },
-      }],
-    });
-
-    adapter.cancelResponse(420);
-    expect(sendRealtimeInput).toHaveBeenCalledTimes(1);
-  });
-
-  it("G4/WBS-4: native resume — always enables sessionResumption, passes a prior handle through, surfaces new handles", async () => {
-    const adapter = fromGeminiLive({ apiKey: "test-key", sessionResumptionHandle: "handle-prev" });
-    expect(adapter.caps.supportsNativeResume).toBe(true);
-
-    const eventsTask = collectEvents(adapter.events, 1);
-    await adapter.open(new AbortController().signal);
-
-    const connectArg = liveConnect.mock.calls[0]![0] as { config: Record<string, unknown> };
-    // Handle passthrough — the server restores the conversation; nothing is replayed
-    // client-side (sendClientContent untouched — R6: no double-apply).
-    expect(connectArg.config["sessionResumption"]).toEqual({ handle: "handle-prev" });
-    expect(sendClientContent).not.toHaveBeenCalled();
-
-    inject({ sessionResumptionUpdate: { newHandle: "handle-next", resumable: true } });
-    // Non-resumable updates carry no usable handle and must be ignored.
-    inject({ sessionResumptionUpdate: { newHandle: "", resumable: false } });
-
-    expect(await eventsTask).toEqual([{ type: "resumption_handle", handle: "handle-next" }]);
-    await adapter.close();
-  });
-
-  it("G4/WBS-4: enables handle issuance even without a prior handle", async () => {
-    const adapter = fromGeminiLive({ apiKey: "test-key" });
-    await adapter.open(new AbortController().signal);
-    const connectArg = liveConnect.mock.calls[0]![0] as { config: Record<string, unknown> };
-    expect(connectArg.config["sessionResumption"]).toEqual({});
-    await adapter.close();
-  });
-
-  it("sends a typed user turn via sendClientContent with turnComplete", async () => {
-    const adapter = fromGeminiLive({ apiKey: "test-key" });
-    await adapter.open(new AbortController().signal);
-
-    adapter.sendText!("when is the late-add deadline?");
-    expect(sendClientContent).toHaveBeenCalledWith({
-      turns: [{ role: "user", parts: [{ text: "when is the late-add deadline?" }] }],
-      turnComplete: true,
-    });
-  });
-
-  it("normalizes provider server messages into RealtimeEvent", async () => {
-    const adapter = fromGeminiLive({ apiKey: "test-key" });
-    const eventsTask = collectEvents(adapter.events, 7);
-    await adapter.open(new AbortController().signal);
-
-    const audioBytes = new Uint8Array([9, 10, 11, 12]);
-    inject({ setupComplete: {} });
-    inject({
-      serverContent: {
-        modelTurn: {
-          parts: [{
-            inlineData: {
-              data: bytesToBase64(audioBytes),
-              mimeType: "audio/pcm;rate=24000",
-            },
-          }],
-        },
-      },
-    });
-    inject({ serverContent: { interrupted: true } });
-    inject({
-      serverContent: {
-        inputTranscription: { text: "Can I add Biology?", finished: true },
-      },
-    });
-    inject({
-      serverContent: {
-        outputTranscription: { text: "Let me check that.", finished: true },
-      },
-    });
-    inject({
-      toolCall: {
-        functionCalls: [{
-          id: "call_123",
-          name: "consult_knowledge",
-          args: { query: "late add biology" },
-        }],
-      },
-    });
-    inject({ serverContent: { turnComplete: true } });
-
-    const events = await eventsTask;
-    expect(events.slice(0, 7)).toEqual([
-      { type: "response_started" },
-      { type: "audio", pcm16: audioBytes, sampleRateHz: 24000 },
-      { type: "speech_started" },
-      { type: "transcript", role: "user", text: "Can I add Biology?", final: true },
-      { type: "transcript", role: "assistant", text: "Let me check that.", final: true },
-      {
-        type: "tool_call",
-        toolId: "call_123",
-        toolName: "consult_knowledge",
-        args: { query: "late add biology" },
-      },
-      { type: "response_done" },
-    ]);
-  });
-
-  it("exposes Gemini Live capability flags", () => {
-    const adapter = fromGeminiLive({ apiKey: "test-key" });
-    expect(adapter.caps).toEqual({
-      inputSampleRateHz: 16_000,
-      outputSampleRateHz: 24_000,
-      supportsNativeResume: true,
-      supportsConcurrentToolAudio: false,
-      supportsTruncate: false,
-      emitsServerSpeechStarted: true,
-    });
-  });
-});