@cairnvibe/sdk 0.2.13 → 0.3.0

package/src/web-component.ts

@@ -25,6 +25,7 @@ import type { HistoryTurn as HistoryEntry, TourStep } from "@cairnvibe/core";
 import { collectVisible } from "./context-collector";
 import { findElement, highlightElement, logMiss, type MissContext } from "./element-ladder";
 import { executeVerbResponse } from "./verb-executor";
+import { createBargeInGate, createVadDetector } from "./vad";
 
 type Status = "idle" | "asking" | "recording" | "rt-connecting" | "rt-listening" | "rt-thinking" | "rt-speaking";
 
@@ -333,6 +334,15 @@ export class CairnWidgetElement extends HTMLElement {
   private rtMicMuted = false;
   private rtSpeakerMuted = false;
   private rtStarting = false; // closes the click-to-first-state-update gap so a rapid double-click can't open two sessions
+  // Generation of the most recent "final" this client has processed — see
+  // ServerMessage's own doc comment in realtime-server.ts (index.tsx
+  // carries the same fix, ported here) for the real, live-found race this
+  // closes: a locally-triggered barge-in can start a new turn before an
+  // earlier turn's own verb/audio, already in flight when the server
+  // processed the barge-in, actually arrives. isStaleRtMessage() drops
+  // anything older than this instead of applying it to whatever caption
+  // is now current.
+  private rtLastFinalGeneration = 0;
   private rtPlaybackCtx: AudioContext | null = null;
   private rtPlaybackGain: GainNode | null = null;
   private rtNextPlayTime = 0;
@@ -363,6 +373,16 @@ export class CairnWidgetElement extends HTMLElement {
     if (name === "persona" && this.panel) this.panel.setAttribute("aria-label", `${this.persona} help panel`);
   }
 
+  // Same real gap index.tsx's own unmount-safety-net closes, ported here:
+  // without this, removing the element from the DOM (an SPA route change,
+  // conditional rendering) while a realtime call is open left the
+  // WebSocket, the open mic stream, and both AudioContexts running
+  // orphaned — a zombie connection that keeps transcribing and replying
+  // in parallel with whatever comes next.
+  disconnectedCallback() {
+    if (this.rtSocket || this.rtCleanup) this.endRealtime();
+  }
+
   // --- attributes -----------------------------------------------------
   private get endpoint(): string { return this.getAttribute("endpoint") ?? "/api/copilot"; }
   private get speakEndpoint(): string | null { return this.getAttribute("speak-endpoint"); }
@@ -907,6 +927,13 @@ export class CairnWidgetElement extends HTMLElement {
   // Real-time voice conversation
   // ---------------------------------------------------------------------
 
+  /** See rtLastFinalGeneration's own doc comment. A message with no
+   * generation field at all is treated as current rather than dropped —
+   * additive/backward-compatible against a server predating this fix. */
+  private isStaleRtMessage(msg: { generation?: unknown }): boolean {
+    return typeof msg.generation === "number" && msg.generation < this.rtLastFinalGeneration;
+  }
+
   private async startRealtime() {
     // rtStarting closes the gap between click and the first status update
     // landing — without it a rapid double-click could race past the
@@ -946,6 +973,12 @@ export class CairnWidgetElement extends HTMLElement {
       const processor = audioCtx.createScriptProcessor(4096, 1, 1);
       const silence = audioCtx.createGain();
       silence.gain.value = 0;
+      const bargeInVad = createVadDetector();
+      // See index.tsx's own doc comment on its matching bargeInGate for
+      // the real, live-reported bug this closes (a single noise-burst
+      // VAD frame permanently cutting the agent off) and the production
+      // research (Pipecat/LiveKit/Vapi/Deepgram) it's grounded in.
+      const bargeInGate = createBargeInGate();
 
       // Only flips back to "listening" (and lets the mic resume sending)
       // once BOTH the server has said no more audio is coming for this
@@ -963,6 +996,7 @@ export class CairnWidgetElement extends HTMLElement {
           this.rtTourAudioDoneResolve = null;
           return;
         }
+        void audioCtx.resume().catch(() => {}); // don't wait up to 2s for the periodic health check if the browser already suspended capture
         this.setStatus("rt-listening");
         this.setCaption("");
       };
@@ -979,9 +1013,21 @@ export class CairnWidgetElement extends HTMLElement {
         this.rtThinkingWatchdog = setTimeout(() => {
           this.rtThinkingWatchdog = null;
           console.warn("[cairn] realtime turn timed out waiting on the server — resuming listening");
-          this.rtAudioDoneArriving = true;
-          this.setStatus("rt-listening");
-          this.setCaption("");
+          // Same real, live-found fix as index.tsx's own watchdog (see its
+          // doc comment) — this used to only reset LOCAL state, never
+          // telling the server anything, so a turn that was merely SLOW
+          // (e.g. retrying a rate-limited call across every configured
+          // key) kept running server-side and its reply arrived late,
+          // landing on whatever the user had moved on to instead of being
+          // recognized as stale. triggerBargeIn() sends the real barge_in
+          // signal, bumping the server's generation so that late reply
+          // gets correctly dropped by isStaleRtMessage when it arrives.
+          triggerBargeIn();
+          // triggerBargeIn() clears the caption but never touched the
+          // answer — without this, a timed-out turn gave the user
+          // literally nothing: no reply, no error, a silent reset that
+          // reads as "it heard me and did nothing."
+          this.setAnswer("That's taking longer than expected — try asking again.");
         }, 20000);
       };
 
@@ -1020,10 +1066,12 @@ export class CairnWidgetElement extends HTMLElement {
         // sent yet, and cut the agent off the instant the user starts
         // talking over it instead of making them wait for it to finish.
         if (this.status === "rt-speaking" && !this.touringActive) {
-          const rms = computeRms(e.inputBuffer.getChannelData(0));
-          if (rms > BARGE_IN_RMS_THRESHOLD) triggerBargeIn();
+          const frame = bargeInVad.process(e.inputBuffer.getChannelData(0));
+          const frameDurationMs = (e.inputBuffer.length / audioCtx.sampleRate) * 1000;
+          if (bargeInGate.update(frame, frameDurationMs)) triggerBargeIn();
           return;
         }
+        bargeInGate.reset(); // not currently interruptible — don't let stale progress carry into the next speaking phase
 
         if (this.status !== "rt-listening") return; // don't send our own mic while the agent is thinking/speaking
         const pcm = floatTo16BitPCM(downsampleTo16k(e.inputBuffer.getChannelData(0), audioCtx.sampleRate));
@@ -1033,7 +1081,31 @@ export class CairnWidgetElement extends HTMLElement {
       processor.connect(silence);
       silence.connect(audioCtx.destination);
 
+      // See index.tsx's own matching doc comment for the real, live-
+      // reported bug this closes: browsers can silently suspend an
+      // AudioContext with no active output (this capture context has
+      // none by design), after which onaudioprocess just stops firing —
+      // "Listening…" stays on screen while nothing is actually captured.
+      const micHealthCheck = setInterval(() => {
+        if (audioCtx.state !== "running") {
+          void audioCtx.resume().catch(() => {});
+        }
+        const track = stream.getAudioTracks()[0];
+        if (track && (track.readyState === "ended" || track.muted)) {
+          this.setAnswer("The microphone connection was lost — try starting the call again.");
+          this.endRealtime();
+        }
+      }, 2000);
+
+      const handleMicTrackEnded = () => {
+        this.setAnswer("The microphone connection was lost — try starting the call again.");
+        this.endRealtime();
+      };
+      stream.getAudioTracks().forEach((t) => t.addEventListener("ended", handleMicTrackEnded));
+
       this.rtCleanup = () => {
+        clearInterval(micHealthCheck);
+        stream.getAudioTracks().forEach((t) => t.removeEventListener("ended", handleMicTrackEnded));
         processor.disconnect();
         source.disconnect();
         stream.getTracks().forEach((t) => t.stop());
@@ -1056,17 +1128,21 @@ export class CairnWidgetElement extends HTMLElement {
         if (msg.type === "interim") {
           this.setCaption(msg.text);
         } else if (msg.type === "final") {
+          this.rtLastFinalGeneration = typeof msg.generation === "number" ? msg.generation : 0;
           this.setCaption(msg.text);
           this.setStatus("rt-thinking");
           armThinkingWatchdog();
         } else if (msg.type === "verb") {
+          if (this.isStaleRtMessage(msg)) return; // belongs to a turn a later "final" already superseded
           disarmThinkingWatchdog();
           this.handleVerb(msg.verb);
         } else if (msg.type === "speaking_start") {
+          if (this.isStaleRtMessage(msg)) return;
           disarmThinkingWatchdog();
           this.rtAudioDoneArriving = false;
           this.setStatus("rt-speaking");
         } else if (msg.type === "audio_chunk") {
+          if (this.isStaleRtMessage(msg)) return; // the literal "two speakers" case — a chunk from an abandoned turn, already in flight when the barge-in landed
           const ctx = this.rtPlaybackCtx;
           const gain = this.rtPlaybackGain;
           if (!ctx || !gain) return;
@@ -1101,6 +1177,7 @@ export class CairnWidgetElement extends HTMLElement {
             maybeResumeListening();
           };
         } else if (msg.type === "speaking_end" || msg.type === "turn_complete") {
+          if (this.isStaleRtMessage(msg)) return; // a newer turn's own speaking_end/turn_complete will arrive and resume listening correctly on its own
           // turn_complete covers a verb with nothing spoken — no audio_chunk
           // ever arrives for it, so rtScheduledSources is already empty and
           // maybeResumeListening() resumes immediately.
@@ -1202,18 +1279,6 @@ function summarizeVerbForHistory(raw: unknown): string {
 // identical to index.tsx's — same protocol, same math, ported directly)
 // ---------------------------------------------------------------------------
 
-// Heuristic energy gate for barge-in: real speech into a laptop/phone mic
-// typically sits well above this; normal room noise and the mic's own
-// noise floor typically sit below it. Same threshold as the React widget —
-// not independently recalibrated, since it's the same audio pipeline.
-const BARGE_IN_RMS_THRESHOLD = 0.02;
-
-function computeRms(channelData: Float32Array): number {
-  let sumSquares = 0;
-  for (let i = 0; i < channelData.length; i++) sumSquares += channelData[i] * channelData[i];
-  return Math.sqrt(sumSquares / channelData.length);
-}
-
 function downsampleTo16k(input: Float32Array, inputSampleRate: number): Float32Array {
   const targetRate = 16000;
   if (inputSampleRate === targetRate) return input;

package/src/webmcp-client.ts

@@ -8,12 +8,14 @@
 // no site has adopted it yet, so this is deliberately a no-op (empty list,
 // nothing to call) everywhere it isn't present, not a hard dependency.
 
-import type { WebMcpTool } from "@cairnvibe/core";
+import type { WebMcpRiskTier, WebMcpTool } from "@cairnvibe/core";
 
 interface ModelContextTool {
   name: string;
   description?: string;
   inputSchema?: Record<string, unknown>;
+  /** Architecture Pillar 6 — declared by the page's own tool registration, never invented by Cairn. See WebMcpToolSchema's own doc comment. */
+  riskTier?: WebMcpRiskTier;
 }
 
 interface ModelContext {
@@ -44,6 +46,11 @@ export async function discoverWebMcpTools(): Promise<WebMcpTool[]> {
       name: String(tool.name),
       description: String(tool.description ?? "").slice(0, MAX_DESCRIPTION_LENGTH),
       inputSchema: tool.inputSchema,
+      // Architecture Pillar 6 — passed through only when the page's own
+      // registration declared a real "confirm" tier; anything else
+      // (absent, or a value that isn't literally "confirm") stays
+      // undefined/"safe" — never invented, never widened by a typo.
+      riskTier: tool.riskTier === "confirm" ? "confirm" : undefined,
     }));
   } catch {
     // A page's own registerTool()/getTools() implementation throwing is
@@ -59,8 +66,22 @@ export async function discoverWebMcpTools(): Promise<WebMcpTool[]> {
  * exact request's own discoverWebMcpTools() call), never invented.
  * Returns a plain-text observation for the agent loop to reason about
  * next, the same shape a click/fill/read result already takes.
+ *
+ * Architecture Pillar 6 (the safety layer) — `confirmTool` is only ever
+ * consulted for a tool whose OWN registration declared `riskTier:
+ * "confirm"` (never something the model or this call site can widen) — a
+ * real-world-effect tool (a payment, a delete, anything hard to undo)
+ * that must get a genuine yes from the END USER before it runs, not just
+ * the model's own decision to call it. No `confirmTool` provided (a host
+ * app that hasn't wired up a confirmation UI) is treated as a decline,
+ * never as an implicit yes — the safe default when there's no real way
+ * to ask.
  */
-export async function executeWebMcpTool(name: string, args: Record<string, unknown> | undefined): Promise<{ ok: boolean; observation: string }> {
+export async function executeWebMcpTool(
+  name: string,
+  args: Record<string, unknown> | undefined,
+  confirmTool?: (tool: { name: string; description: string }) => Promise<boolean>,
+): Promise<{ ok: boolean; observation: string }> {
   const modelContext = getModelContext();
   if (!modelContext?.getTools || !modelContext.executeTool) {
     return { ok: false, observation: "This page no longer has that tool available." };
@@ -70,6 +91,13 @@ export async function executeWebMcpTool(name: string, args: Record<string, unkno
     const tool = Array.isArray(tools) ? tools.find((t) => t.name === name) : undefined;
     if (!tool) return { ok: false, observation: `No tool named "${name}" is available on this page right now.` };
 
+    if (tool.riskTier === "confirm") {
+      const confirmed = confirmTool ? await confirmTool({ name: tool.name, description: tool.description ?? "" }) : false;
+      if (!confirmed) {
+        return { ok: false, observation: "This action needs the user's real confirmation before it can run, and it wasn't confirmed." };
+      }
+    }
+
     const result = await modelContext.executeTool(tool, args ?? {});
     const observation = typeof result === "string" ? result : JSON.stringify(result ?? null);
     return { ok: true, observation: observation.slice(0, 2000) };