@kuralle-syrinx/realtime 4.1.0 → 4.3.0

package/src/realtime-bridge.ts

@@ -89,6 +89,22 @@ export interface RealtimeBridgeOptions {
     readonly toolName: string;
     readonly args: Record<string, unknown>;
   }) => string | undefined | Promise<string | undefined>;
+  /**
+   * Half-cascade text-only mode: stream assistant transcript into the cascade
+   * `llm.delta → segmenter → tts.text` path and ignore provider audio.
+   * REQUIRES a TTS plugin registered on the session bus (the bridge does not own the registry).
+   */
+  readonly textOnly?: boolean;
+  /**
+   * Syrinx-owned turn detection (REQ-6): on `eos.turn_complete`, call
+   * `adapter.requestResponse` so Syrinx endpointing/VAD/InteractionPolicy drives
+   * the provider response instead of server VAD.
+   *
+   * MUST be paired with `turnDetection: null` on the adapter — otherwise the
+   * provider's server VAD and Syrinx would BOTH trigger responses (double-turn).
+   * Native mode (`syrinxTurns` absent/false) is unchanged.
+   */
+  readonly syrinxTurns?: boolean;
 }
 
 export class RealtimeBridge implements VoicePlugin {
@@ -99,6 +115,7 @@ export class RealtimeBridge implements VoicePlugin {
   private turnAssistantText = "";
   /** Concatenated streamed transcript fragments — providers that emit deltas only, no final (Gemini Live). */
   private turnAssistantDeltas = "";
+  private pendingSpeechEndedAtMs: number | null = null;
   private sessionAbort: AbortController | null = null;
   private inflight: AbortController | undefined;
   private delegateTask: Promise<void> | null = null;
@@ -113,6 +130,16 @@ export class RealtimeBridge implements VoicePlugin {
     private readonly opts: RealtimeBridgeOptions = {},
   ) {}
 
+  injectContext(text: string): void {
+    // Provider history differs here: OpenAI retains system items, while Gemini drops
+    // system/developer history, so the adapter selects the provider-safe representation.
+    if (this.adapter.injectContext) {
+      this.adapter.injectContext(text);
+      return;
+    }
+    console.warn("RealtimeBridge: context injection requested but the adapter does not support it");
+  }
+
   async initialize(bus: PipelineBus, _cfg: PluginConfig): Promise<void> {
     this.bus = bus;
     this.sessionAbort = new AbortController();
@@ -138,6 +165,14 @@ export class RealtimeBridge implements VoicePlugin {
       }),
     );
 
+    if (this.opts.syrinxTurns) {
+      this.disposers.push(
+        bus.on("eos.turn_complete", () => {
+          this.adapter.requestResponse?.();
+        }),
+      );
+    }
+
     await this.adapter.open(this.sessionAbort.signal);
     void this.pump();
   }
@@ -176,11 +211,15 @@ export class RealtimeBridge implements VoicePlugin {
         this.onResponseStarted(bus);
         break;
       case "audio":
-        this.onAudio(bus, ev.pcm16, ev.sampleRateHz);
+        // textOnly: provider audio must not be used (REQ-4) — TTS plugin owns playout.
+        if (!this.opts.textOnly) this.onAudio(bus, ev.pcm16, ev.sampleRateHz);
         break;
       case "transcript":
         if (ev.role === "user") {
           if (ev.final) this.onFinalTranscript(bus, ev.text);
+        } else if (this.opts.textOnly) {
+          // Drive cascade TTS path as text arrives; do not buffer for display-only re-emit.
+          this.onTextOnlyAssistantTranscript(bus, ev.text, ev.final);
         } else if (ev.final && ev.text.trim()) {
           this.turnAssistantText = this.turnAssistantText
             ? `${this.turnAssistantText} ${ev.text.trim()}`
@@ -188,6 +227,12 @@ export class RealtimeBridge implements VoicePlugin {
         } else if (!ev.final && ev.text) {
           // Streamed fragments already carry their own leading spaces — concatenate verbatim.
           this.turnAssistantDeltas += ev.text;
+          bus.push(Route.Main, {
+            kind: "llm.delta",
+            contextId: this.contextId,
+            timestampMs: Date.now(),
+            text: ev.text,
+          });
         }
         break;
       case "tool_call":
@@ -195,10 +240,23 @@ export class RealtimeBridge implements VoicePlugin {
         break;
       case "response_done":
         this.onResponseDone(bus);
+        // Meter the native front — previously native turns produced no usage at all.
+        if (ev.usage && this.contextId) {
+          bus.push(Route.Background, {
+            kind: "usage.recorded",
+            contextId: this.contextId,
+            timestampMs: Date.now(),
+            stage: "llm",
+            ...ev.usage,
+          });
+        }
         break;
       case "speech_started":
         this.onSpeechStarted(bus);
         break;
+      case "speech_stopped":
+        this.onSpeechStopped(bus);
+        break;
       case "resumption_handle": {
         // G4: persist-worthy native-resume handle (Gemini). Background route —
         // a durable host stores the latest and passes it back on reconnect.
@@ -310,6 +368,25 @@ export class RealtimeBridge implements VoicePlugin {
     this.inflight = controller;
     let answer = "";
     let grounded = false;
+    let blockedMessage: string | undefined;
+    let blockedPayload: unknown;
+    let passStartedMs = Date.now();
+    let passTtftRecorded = false;
+    const pushConversationMetric = (name: string, value: string, timestampMs: number): void => {
+      bus.push(Route.Main, {
+        kind: "metric.conversation",
+        contextId,
+        timestampMs,
+        name,
+        value,
+      });
+    };
+    const recordPassTtft = (timestampMs: number): void => {
+      if (passTtftRecorded) return;
+      passTtftRecorded = true;
+      pushConversationMetric("llm.pass_ttft_ms", String(timestampMs - passStartedMs), timestampMs);
+    };
+    pushConversationMetric("llm.call_started", "1", passStartedMs);
 
     // G2 observability: the query is on its way to the reasoner (Background route, R4).
     const queryStartedMs = Date.now();
@@ -333,10 +410,37 @@ export class RealtimeBridge implements VoicePlugin {
       })) {
         switch (part.type) {
           case "text-delta":
+            recordPassTtft(Date.now());
             answer += part.text;
             break;
+          case "tool-call":
+            recordPassTtft(Date.now());
+            break;
           case "tool-result":
             grounded = true;
+            passStartedMs = Date.now();
+            passTtftRecorded = false;
+            pushConversationMetric("llm.call_started", "1", passStartedMs);
+            break;
+          case "control": {
+            const controlMs = Date.now();
+            bus.push(Route.Background, {
+              kind: "delegate.result",
+              contextId,
+              timestampMs: controlMs,
+              query: userText,
+              answer,
+              durationMs: controlMs - queryStartedMs,
+              grounded,
+              toolId: ev.toolId,
+              toolName: ev.toolName,
+              control: { name: part.name, payload: part.payload },
+            });
+            break;
+          }
+          case "blocked":
+            blockedMessage = part.userFacingMessage;
+            blockedPayload = part.payload;
             break;
           case "finish":
             if (!answer && part.text) answer = part.text;
@@ -354,6 +458,7 @@ export class RealtimeBridge implements VoicePlugin {
             this.onError(bus, part.cause, true);
             return;
         }
+        if (blockedMessage !== undefined) break;
       }
     } catch (err) {
       if (isAbortError(err)) return;
@@ -364,6 +469,24 @@ export class RealtimeBridge implements VoicePlugin {
       if (this.inflight === controller) this.inflight = undefined;
     }
 
+    if (blockedMessage !== undefined) {
+      const blockedMs = Date.now();
+      bus.push(Route.Background, {
+        kind: "delegate.result",
+        contextId,
+        timestampMs: blockedMs,
+        query: userText,
+        answer: blockedMessage,
+        durationMs: blockedMs - queryStartedMs,
+        grounded,
+        toolId: ev.toolId,
+        toolName: ev.toolName,
+        blocked: { userFacingMessage: blockedMessage, payload: blockedPayload },
+      });
+      this.adapter.injectToolResult(ev.toolId, this.formatToolResult(blockedMessage));
+      return;
+    }
+
     if (answer.length === 0) {
       this.onError(bus, new Error("delegate produced no output"), false);
       return;
@@ -417,6 +540,20 @@ export class RealtimeBridge implements VoicePlugin {
     bus.push(Route.Critical, packet);
   }
 
+  private onSpeechStopped(bus: PipelineBus): void {
+    const timestampMs = Date.now();
+    if (!this.contextId) {
+      // Server VAD reports speech end before response.created mints the response context.
+      this.pendingSpeechEndedAtMs = timestampMs;
+      return;
+    }
+    bus.push(Route.Main, {
+      kind: "vad.speech_ended",
+      contextId: this.contextId,
+      timestampMs,
+    });
+  }
+
   private onResponseStarted(bus: PipelineBus): void {
     const previousContextId = this.contextId;
     this.contextId = crypto.randomUUID();
@@ -425,6 +562,14 @@ export class RealtimeBridge implements VoicePlugin {
     this.turnAssistantDeltas = "";
     this.playedMs = 0;
     this.audioRemainder = new Uint8Array(0);
+    if (this.pendingSpeechEndedAtMs !== null) {
+      bus.push(Route.Main, {
+        kind: "vad.speech_ended",
+        contextId: this.contextId,
+        timestampMs: this.pendingSpeechEndedAtMs,
+      });
+      this.pendingSpeechEndedAtMs = null;
+    }
     const packet: TurnChangePacket = {
       kind: "turn.change",
       contextId: this.contextId,
@@ -455,10 +600,42 @@ export class RealtimeBridge implements VoicePlugin {
     bus.push(Route.Main, packet);
   }
 
+  /**
+   * textOnly: stream assistant transcript into the cascade LLM→segmenter→TTS path.
+   * Non-final deltas drive `llm.delta` immediately (no buffer). Final emits `llm.done` only —
+   * the C0 adapter's final carries the full accumulated text, so re-emitting as delta would
+   * duplicate every streamed fragment.
+   */
+  private onTextOnlyAssistantTranscript(bus: PipelineBus, text: string, final: boolean): void {
+    if (!this.contextId) return;
+    const timestampMs = Date.now();
+    if (!final && text) {
+      const delta: LlmDeltaPacket = {
+        kind: "llm.delta",
+        contextId: this.contextId,
+        timestampMs,
+        text,
+      };
+      bus.push(Route.Main, delta);
+      return;
+    }
+    if (final && text.trim()) {
+      const done: LlmResponseDonePacket = {
+        kind: "llm.done",
+        contextId: this.contextId,
+        timestampMs,
+        text,
+      };
+      bus.push(Route.Main, done);
+    }
+  }
+
   private onResponseDone(bus: PipelineBus): void {
     if (!this.contextId) return;
-    this.emitCoalescedAudio(bus, true);
-    this.audioRemainder = new Uint8Array(0);
+    if (!this.opts.textOnly) {
+      this.emitCoalescedAudio(bus, true);
+      this.audioRemainder = new Uint8Array(0);
+    }
 
     const timestampMs = Date.now();
     const transcripts: SttResultPacket[] = this.turnUserText
@@ -477,6 +654,16 @@ export class RealtimeBridge implements VoicePlugin {
       text: this.turnUserText,
       transcripts,
     };
+
+    if (this.opts.textOnly) {
+      // Already streamed llm.delta/llm.done above. TTS plugin owns tts.end (REQ-5) — forcing
+      // it here would cut playout / break barge-in heard-prefix timing.
+      // syrinxTurns: endpointing already emitted the user-turn eos.turn_complete that drove
+      // requestResponse; re-emitting here would re-subscribe-fire requestResponse → loop.
+      if (!this.opts.syrinxTurns) bus.push(Route.Main, turnComplete);
+      return;
+    }
+
     const ttsEnd: TextToSpeechEndPacket = {
       kind: "tts.end",
       contextId: this.contextId,
@@ -488,19 +675,21 @@ export class RealtimeBridge implements VoicePlugin {
     // providers that only emit non-final deltas (Gemini Live).
     const assistantText = this.turnAssistantText.trim() || this.turnAssistantDeltas.trim();
     if (assistantText) {
-      const delta: LlmDeltaPacket = {
-        kind: "llm.delta",
-        contextId: this.contextId,
-        timestampMs,
-        text: assistantText,
-      };
       const done: LlmResponseDonePacket = {
         kind: "llm.done",
         contextId: this.contextId,
         timestampMs,
         text: assistantText,
       };
-      bus.push(Route.Main, delta, done);
+      if (this.turnAssistantDeltas.length === 0) {
+        bus.push(Route.Main, {
+          kind: "llm.delta",
+          contextId: this.contextId,
+          timestampMs,
+          text: assistantText,
+        });
+      }
+      bus.push(Route.Main, done);
     }
     bus.push(Route.Main, turnComplete, ttsEnd);
   }

package/src/edge-safety.test.ts

@@ -1,173 +0,0 @@
-// SPDX-License-Identifier: MIT
-//
-// R-14: regression gate — the realtime package must run on workerd without
-// Buffer, process, or node:crypto. Reintroducing any Node-only primitive in
-// src/ should fail this test.
-
-import { readFileSync, readdirSync } from "node:fs";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
-
-import { describe, expect, it } from "vitest";
-
-import {
-  PipelineBusImpl,
-  Route,
-  type TextToSpeechAudioPacket,
-  type TurnChangePacket,
-} from "@kuralle-syrinx/core";
-import type { ManagedSocket, SocketData, SocketFactory } from "@kuralle-syrinx/ws";
-
-import { bytesToBase64, fromOpenAIRealtime } from "./from-openai-realtime.js";
-import { RealtimeBridge } from "./realtime-bridge.js";
-
-const srcDir = path.dirname(fileURLToPath(import.meta.url));
-
-interface EdgeMockHarness {
-  readonly factory: SocketFactory;
-  readonly sent: string[];
-  inject(msg: Record<string, unknown>): void;
-}
-
-function createEdgeMockHarness(): EdgeMockHarness {
-  const sent: string[] = [];
-  let messageHandler: ((data: SocketData, isBinary: boolean) => void) | null = null;
-
-  const socket: ManagedSocket = {
-    get isOpen() {
-      return true;
-    },
-    send: (data: SocketData) => {
-      sent.push(typeof data === "string" ? data : "");
-    },
-    keepAlivePing: () => {},
-    verify: async () => true,
-    dispose: () => {},
-    onOpen: (handler) => {
-      queueMicrotask(() => handler());
-    },
-    onMessage: (handler) => {
-      messageHandler = handler;
-    },
-    onClose: () => {},
-    onError: () => {},
-  };
-
-  return {
-    factory: () => socket,
-    sent,
-    inject: (msg) => messageHandler?.(JSON.stringify(msg), false),
-  };
-}
-
-function pcmFromSamples(samples: readonly number[]): Uint8Array {
-  const pcm = Int16Array.from(samples);
-  return new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
-}
-
-async function waitFor(predicate: () => boolean, timeoutMs = 2000): Promise<void> {
-  const startedAt = Date.now();
-  while (Date.now() - startedAt < timeoutMs) {
-    if (predicate()) return;
-    await new Promise((resolve) => setTimeout(resolve, 5));
-  }
-  throw new Error("Timed out waiting for condition");
-}
-
-function collectSourceFiles(dir: string): string[] {
-  const out: string[] = [];
-  for (const entry of readdirSync(dir, { withFileTypes: true })) {
-    const full = path.join(dir, entry.name);
-    if (entry.isDirectory()) {
-      out.push(...collectSourceFiles(full));
-      continue;
-    }
-    if (entry.name.endsWith(".ts") && !entry.name.endsWith(".test.ts")) {
-      out.push(full);
-    }
-  }
-  return out;
-}
-
-describe("edge safety (R-14)", () => {
-  it("src/ contains no Buffer, process, or node:crypto imports", () => {
-    const forbidden = [
-      /\bfrom\s+["']node:crypto["']/,
-      /\brequire\s*\(\s*["']node:crypto["']\s*\)/,
-      /\bglobalThis\.Buffer\b/,
-      /\bglobalThis\.process\b/,
-      /\bprocess\.env\b/,
-      /\bBuffer\.from\b/,
-      /\bBuffer\.alloc\b/,
-    ];
-    const hits: string[] = [];
-    for (const file of collectSourceFiles(srcDir)) {
-      const text = readFileSync(file, "utf8");
-      for (const pattern of forbidden) {
-        if (pattern.test(text)) {
-          hits.push(`${path.relative(srcDir, file)}: ${pattern.source}`);
-        }
-      }
-    }
-    expect(hits).toEqual([]);
-  });
-
-  // NOTE: we do NOT delete globalThis.Buffer/process here — vitest's own worker needs `process`
-  // (its uncaughtException handler calls process.listeners), so deleting it crashes the runner.
-  // Edge-safety is enforced statically by the source-scan above; this test proves the audio path
-  // actually runs end-to-end using only the runtime-agnostic helpers.
-  it("fromOpenAIRealtime + RealtimeBridge round-trip audio via runtime-agnostic helpers", async () => {
-    const mock = createEdgeMockHarness();
-    const adapter = fromOpenAIRealtime({
-      apiKey: "edge-test-key",
-      socketFactory: mock.factory,
-      url: () => "wss://example.test/realtime?model=gpt-realtime-2",
-    });
-    const bridge = new RealtimeBridge(adapter);
-    const bus = new PipelineBusImpl();
-    const turnChanges: TurnChangePacket[] = [];
-    const ttsAudio: TextToSpeechAudioPacket[] = [];
-    bus.on("turn.change", (pkt) => { turnChanges.push(pkt as TurnChangePacket); });
-    bus.on("tts.audio", (pkt) => { ttsAudio.push(pkt as TextToSpeechAudioPacket); });
-
-    const started = bus.start();
-
-    const initTask = bridge.initialize(bus, {});
-    await waitFor(() => mock.sent.length > 0);
-    mock.inject({ type: "session.updated" });
-    await initTask;
-
-    bus.push(Route.Main, {
-      kind: "user.audio_received",
-      contextId: "transport-turn",
-      timestampMs: Date.now(),
-      audio: pcmFromSamples([100, 200, 300, 400]),
-    });
-
-    await waitFor(() =>
-      mock.sent.some((raw) => (JSON.parse(raw) as { type: string }).type === "input_audio_buffer.append"),
-    );
-
-    const providerPcm = pcmFromSamples(Array.from({ length: 960 }, (_, i) => i));
-    mock.inject({ type: "response.created" });
-    mock.inject({
-      type: "response.output_audio.delta",
-      delta: bytesToBase64(providerPcm),
-    });
-    mock.inject({ type: "response.done" });
-
-    await waitFor(() => ttsAudio.length > 0 && turnChanges.length > 0);
-
-    const contextId = turnChanges[0]!.contextId;
-    expect(contextId).toMatch(
-      /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/,
-    );
-    expect(ttsAudio[0]!.contextId).toBe(contextId);
-    expect(ttsAudio[0]!.sampleRateHz).toBe(16_000);
-    expect(ttsAudio[0]!.audio.byteLength).toBeGreaterThan(0);
-
-    await bridge.close();
-    bus.stop();
-    await started;
-  });
-});