@kuralle-syrinx/browser-client 3.0.0 → 4.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kuralle-syrinx/browser-client",
-  "version": "3.0.0",
+  "version": "4.0.0",
   "private": false,
   "type": "module",
   "license": "MIT",
@@ -8,7 +8,7 @@
   "types": "./src/index.ts",
   "dependencies": {
     "@evan/opus": "1.0.3",
-    "@kuralle-syrinx/core": "3.0.0"
+    "@kuralle-syrinx/core": "4.0.0"
   },
   "devDependencies": {
     "typescript": "^5.7.0",

package/src/audio.ts

@@ -172,12 +172,37 @@ export class AudioJitterBuffer {
   private readonly sampleRateHz: number;
   private readonly contextIds = new Set<string>();
   private lastEnqueuedContextId: string | null = null;
+  // Per-context playout window on the AudioContext clock: the scheduledTime of the
+  // first frame (start) and the running end (start + all scheduled durations). Lets
+  // us report how many ms of a turn's audio the user has ACTUALLY heard — the
+  // authoritative signal for barge-in context truncation (B2).
+  private readonly playoutWindow = new Map<string, { startTime: number; endTime: number }>();
 
   /** True while assistant audio is scheduled or playing on the playout clock (not the generation clock). */
   get isPlayingOut(): boolean {
     return this.scheduledFrames.size > 0;
   }
 
+  /**
+   * Milliseconds of `contextId`'s audio that have actually played out of the
+   * speaker so far, clamped to what has been scheduled. This is the played-out
+   * clock the server needs to truncate assistant history to what was heard.
+   */
+  playedOutMs(contextId: string): number {
+    const window = this.playoutWindow.get(contextId);
+    if (!window) return 0;
+    const elapsedMs = (this.context.currentTime - window.startTime) * 1000;
+    const scheduledMs = (window.endTime - window.startTime) * 1000;
+    return Math.max(0, Math.min(elapsedMs, scheduledMs));
+  }
+
+  /** True once all of `contextId`'s scheduled audio has finished playing. */
+  isPlayoutComplete(contextId: string): boolean {
+    const window = this.playoutWindow.get(contextId);
+    if (!window) return false;
+    return this.context.currentTime >= window.endTime;
+  }
+
   /** ContextId of the most recently scheduled assistant audio while any audio is still playing out. */
   get activeContextId(): string | null {
     return this.scheduledFrames.size > 0 ? this.lastEnqueuedContextId : null;
@@ -226,6 +251,15 @@ export class AudioJitterBuffer {
       if (contextId) {
         this.contextIds.add(contextId);
         this.lastEnqueuedContextId = contextId;
+        const window = this.playoutWindow.get(contextId);
+        if (window) {
+          window.endTime = this.nextScheduledTime + audioBuffer.duration;
+        } else {
+          this.playoutWindow.set(contextId, {
+            startTime: this.nextScheduledTime,
+            endTime: this.nextScheduledTime + audioBuffer.duration,
+          });
+        }
       }
 
       // Schedule the audio
@@ -263,6 +297,7 @@ export class AudioJitterBuffer {
         }
       }
       this.contextIds.delete(contextId);
+      this.playoutWindow.delete(contextId);
       if (this.lastEnqueuedContextId === contextId) this.lastEnqueuedContextId = null;
       this.recomputeNextScheduledTime();
     } else {

package/src/index.test.ts

@@ -672,6 +672,39 @@ describe("SyrinxBrowserClient — metrics", () => {
   });
 });
 
+describe("SyrinxBrowserClient — tool-call cues (G3/WBS-3)", () => {
+  beforeEach(() => {
+    sockets = [];
+    globalThis.WebSocket = FakeWebSocket as unknown as typeof WebSocket;
+  });
+
+  afterEach(() => {
+    globalThis.WebSocket = originalWebSocket;
+  });
+
+  it("surfaces the four typed tool_call_* lifecycle messages", () => {
+    const client = makeClient();
+    const events = collectEvents(client);
+    client.connect();
+    const socket = sockets[0]!;
+    socket.dispatch("open", {});
+
+    const frames = [
+      { type: "tool_call_started", turnId: "turn-1", toolId: "t1", toolName: "consult_knowledge" },
+      { type: "tool_call_delayed", turnId: "turn-1", toolId: "t1", toolName: "consult_knowledge", afterMs: 2000 },
+      { type: "tool_call_complete", turnId: "turn-1", toolId: "t1", toolName: "consult_knowledge" },
+      { type: "tool_call_failed", turnId: "turn-1", toolId: "t1", toolName: "consult_knowledge" },
+    ];
+    for (const frame of frames) socket.dispatch("message", { data: JSON.stringify(frame) });
+
+    const cueMessages = events
+      .filter((event): event is Extract<SyrinxBrowserClientEvent, { type: "message" }> => event.type === "message")
+      .map((event) => event.message)
+      .filter((message) => message.type.startsWith("tool_call_"));
+    expect(cueMessages).toMatchObject(frames);
+  });
+});
+
 describe("local barge-in (client_interrupt)", () => {
   beforeEach(() => {
     sockets = [];

package/src/index.ts

@@ -79,6 +79,13 @@ export type SyrinxStudioMessage =
   | { readonly type: "agent_chunk"; readonly turnId?: string; readonly text: string }
   | { readonly type: "agent_tool_call"; readonly turnId?: string; readonly id?: string; readonly name: string; readonly args?: unknown }
   | { readonly type: "agent_tool_result"; readonly turnId?: string; readonly id?: string; readonly result?: unknown }
+  // G3 typed preamble/filler lifecycle (RFC bimodel-delegate-seam): the standard
+  // "thinking" cue. started = the front invoked a tool (arm an earcon/indicator);
+  // delayed = still working after afterMs; complete/failed = stop the cue.
+  | { readonly type: "tool_call_started"; readonly turnId?: string; readonly toolId?: string; readonly toolName?: string }
+  | { readonly type: "tool_call_delayed"; readonly turnId?: string; readonly toolId?: string; readonly toolName?: string; readonly afterMs?: number }
+  | { readonly type: "tool_call_complete"; readonly turnId?: string; readonly toolId?: string; readonly toolName?: string }
+  | { readonly type: "tool_call_failed"; readonly turnId?: string; readonly toolId?: string; readonly toolName?: string }
   | { readonly type: "agent_end"; readonly turnId?: string }
   | { readonly type: "agent_interrupted"; readonly turnId?: string; readonly reason?: string }
   | { readonly type: "audio_clear"; readonly turnId?: string; readonly reason?: string }
@@ -191,6 +198,8 @@ export class SyrinxBrowserClient {
   private openedAt = 0;
   private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
   private keepaliveTimer: ReturnType<typeof setInterval> | null = null;
+  private playoutProgressTimer: ReturnType<typeof setInterval> | null = null;
+  private lastProgressContextId: string | null = null;
   private jitterBuffer: AudioJitterBuffer | null = null;
   private outputSampleRateHz = 16000;
   private wireCodec: BrowserWireCodec = "pcm_s16le";
@@ -372,10 +381,12 @@ export class SyrinxBrowserClient {
       this.emit({ type: "open" });
     }
     this.startKeepalive();
+    this.startPlayoutProgressReporting();
   }
 
   private handleTransportClose(code: number, reason: string): void {
     this.stopKeepalive();
+    this.stopPlayoutProgressReporting();
     if (!this.cleanClose) {
       this.jitterBuffer?.clear();
     }
@@ -484,6 +495,43 @@ export class SyrinxBrowserClient {
     }
   }
 
+  // The browser is the playout clock on the WebSocket media path: report how many
+  // ms of the current assistant turn have actually reached the speaker, so the
+  // server can truncate conversation history to what the user HEARD on barge-in
+  // (B2) instead of falling back to a stale 0. ~200ms cadence matches the server
+  // emitter's throttle; a final complete:true is sent when a turn finishes playing.
+  private startPlayoutProgressReporting(): void {
+    this.stopPlayoutProgressReporting();
+    if (!this.jitterBuffer) return;
+    this.playoutProgressTimer = setInterval(() => {
+      const jitter = this.jitterBuffer;
+      if (!jitter || !this.transport.connected) return;
+      const contextId = jitter.activeContextId ?? this.lastProgressContextId;
+      if (!contextId) return;
+      const complete = jitter.isPlayoutComplete(contextId);
+      try {
+        this.sendJson({
+          type: "playout_progress",
+          contextId,
+          playedOutMs: Math.round(jitter.playedOutMs(contextId)),
+          complete,
+        });
+      } catch {
+        this.stopPlayoutProgressReporting();
+        return;
+      }
+      this.lastProgressContextId = complete ? null : contextId;
+    }, 200);
+  }
+
+  private stopPlayoutProgressReporting(): void {
+    if (this.playoutProgressTimer !== null) {
+      clearInterval(this.playoutProgressTimer);
+      this.playoutProgressTimer = null;
+    }
+    this.lastProgressContextId = null;
+  }
+
   private handleJsonMessage(data: unknown): void {
     if (typeof data !== "string") return;
     try {
@@ -719,6 +767,20 @@ function parseStudioMessage(value: unknown): SyrinxStudioMessage {
       result: value.result,
     };
   }
+  if (
+    type === "tool_call_started" ||
+    type === "tool_call_delayed" ||
+    type === "tool_call_complete" ||
+    type === "tool_call_failed"
+  ) {
+    return {
+      type,
+      turnId: optionalString(value.turnId, `${type}.turnId`),
+      toolId: optionalString(value.toolId, `${type}.toolId`),
+      toolName: optionalString(value.toolName, `${type}.toolName`),
+      ...(type === "tool_call_delayed" ? { afterMs: optionalNumber(value.afterMs, "tool_call_delayed.afterMs") } : {}),
+    };
+  }
   if (type === "agent_interrupted" || type === "audio_clear") {
     return {
       type,

package/src/jitter-buffer.test.ts

@@ -115,6 +115,29 @@ describe("AudioJitterBuffer", () => {
     expect(jitterBuffer.activeContextIds).toHaveLength(2);
   });
 
+  it("reports played-out ms clamped to the scheduled window (heard clock)", () => {
+    const frameData = new Int16Array(320); // 20ms at 16kHz
+    frameData.fill(1000);
+    mockContext.currentTime = 1.0;
+    jitterBuffer.enqueue(frameData.buffer, "ctx"); // scheduled at 1.1, ends 1.12
+
+    // Before the frame starts playing → nothing heard yet.
+    expect(jitterBuffer.playedOutMs("ctx")).toBe(0);
+    expect(jitterBuffer.isPlayoutComplete("ctx")).toBe(false);
+
+    // Halfway through the 20ms frame → ~10ms heard.
+    mockContext.currentTime = 1.11;
+    expect(jitterBuffer.playedOutMs("ctx")).toBeCloseTo(10, 0);
+
+    // Past the end → clamped to the full 20ms, and complete.
+    mockContext.currentTime = 1.2;
+    expect(jitterBuffer.playedOutMs("ctx")).toBeCloseTo(20, 0);
+    expect(jitterBuffer.isPlayoutComplete("ctx")).toBe(true);
+
+    // Unknown context reports 0.
+    expect(jitterBuffer.playedOutMs("nope")).toBe(0);
+  });
+
   it("clears specific context frames on context-specific clear", () => {
     const frameData = new Int16Array(320);
     frameData.fill(1000);