@runtypelabs/voice 0.2.3

package/dist/react.js

@@ -0,0 +1,495 @@
+"use client";
+
+// src/react.ts
+import { useEffect, useMemo, useRef, useSyncExternalStore } from "react";
+
+// src/pcm-player.ts
+var PCM_SAMPLE_RATE = 24e3;
+var PCM_WATERLINE_SAMPLES = 3600;
+var PCM_PLAYER_WORKLET = `
+class RuntypePcmPlayerProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super()
+    this.chunks = []
+    this.readOffset = 0
+    this.buffered = 0
+    this.waiting = true
+    // INVARIANT: Only report drained after end-of-stream, never during a jitter gap.
+    this.eosSeen = false
+    this.revision = 0
+    this.port.onmessage = (e) => {
+      const msg = e.data
+      this.revision = msg.revision
+      if (msg.type === 'push') {
+        this.eosSeen = false
+        this.chunks.push(msg.samples)
+        this.buffered += msg.samples.length
+        if (this.waiting && this.buffered >= ${PCM_WATERLINE_SAMPLES}) {
+          this.waiting = false
+        }
+      } else if (msg.type === 'eos') {
+        this.eosSeen = true
+        // INVARIANT: Short replies drain even below the waterline.
+        if (this.waiting && this.buffered > 0) this.waiting = false
+        // INVARIANT: An empty completed stream reports drained immediately.
+        if (this.buffered === 0) {
+          this.eosSeen = false
+          this.port.postMessage({ type: 'drained', revision: this.revision })
+        }
+      } else if (msg.type === 'clear') {
+        this.chunks = []
+        this.readOffset = 0
+        this.buffered = 0
+        this.waiting = true
+        this.eosSeen = false
+      }
+    }
+  }
+  process(inputs, outputs) {
+    const out = outputs[0][0]
+    if (!out || this.waiting) return true // outputs are pre-zeroed: silence
+    let i = 0
+    while (i < out.length && this.buffered > 0) {
+      const chunk = this.chunks[0]
+      out[i++] = chunk[this.readOffset++]
+      this.buffered--
+      if (this.readOffset >= chunk.length) {
+        this.chunks.shift()
+        this.readOffset = 0
+      }
+    }
+    if (this.buffered === 0) {
+      this.waiting = true // mid-reply underrun: re-buffer silently
+      if (this.eosSeen) {
+        this.eosSeen = false
+        this.port.postMessage({ type: 'drained', revision: this.revision })
+      }
+    }
+    return true
+  }
+}
+registerProcessor('runtype-pcm-player', RuntypePcmPlayerProcessor)
+`;
+async function createPcmPlayer(onDrained) {
+  let revision = 0;
+  const context = new AudioContext({ sampleRate: PCM_SAMPLE_RATE });
+  const moduleUrl = URL.createObjectURL(
+    new Blob([PCM_PLAYER_WORKLET], { type: "application/javascript" })
+  );
+  try {
+    if (context.state === "suspended") await context.resume();
+    await context.audioWorklet.addModule(moduleUrl);
+  } catch (err) {
+    context.close().catch(() => {
+    });
+    throw err;
+  } finally {
+    URL.revokeObjectURL(moduleUrl);
+  }
+  const node = new AudioWorkletNode(context, "runtype-pcm-player", {
+    numberOfInputs: 0,
+    numberOfOutputs: 1,
+    outputChannelCount: [1]
+  });
+  node.port.onmessage = (e) => {
+    if (e.data?.type === "drained" && e.data.revision === revision) onDrained();
+  };
+  node.connect(context.destination);
+  return {
+    async push(data) {
+      const pushRevision = revision;
+      const buffer = data instanceof Blob ? await data.arrayBuffer() : data;
+      if (pushRevision !== revision) return;
+      const samples = pcm16FrameToFloat32(buffer);
+      if (samples.length === 0) return;
+      node.port.postMessage({ type: "push", samples, revision }, [samples.buffer]);
+    },
+    endOfStream() {
+      node.port.postMessage({ type: "eos", revision });
+    },
+    clear() {
+      revision += 1;
+      node.port.postMessage({ type: "clear", revision });
+    },
+    close() {
+      revision += 1;
+      node.port.onmessage = null;
+      node.disconnect();
+      context.close().catch(() => {
+      });
+    }
+  };
+}
+function pcm16FrameToFloat32(buffer) {
+  const view = new DataView(buffer);
+  let offset = 0;
+  if (buffer.byteLength >= 44 && view.getUint32(0, false) === 1380533830) {
+    offset = 44;
+  }
+  const sampleCount = Math.floor((buffer.byteLength - offset) / 2);
+  const out = new Float32Array(sampleCount);
+  for (let i = 0; i < sampleCount; i++) {
+    out[i] = view.getInt16(offset + i * 2, true) / 32768;
+  }
+  return out;
+}
+
+// src/voice-client.ts
+var CAPTURE_SAMPLE_RATE = 16e3;
+var CAPTURE_BUFFER_SIZE = 4096;
+function initialSnapshot() {
+  return {
+    status: "idle",
+    transcript: [],
+    interimTranscript: null,
+    metrics: null,
+    audioLevel: 0,
+    isMuted: false,
+    error: null,
+    errorDetails: void 0,
+    interruptionMode: "none",
+    canCancel: false
+  };
+}
+var VoiceClient = class {
+  constructor(options) {
+    this.options = options;
+  }
+  options;
+  snapshot = initialSnapshot();
+  listeners = /* @__PURE__ */ new Set();
+  socket = null;
+  context = null;
+  stream = null;
+  source = null;
+  processor = null;
+  player = null;
+  generation = 0;
+  playbackRevision = 0;
+  pushChain = Promise.resolve();
+  awaitingClear = false;
+  stoppedResponse = false;
+  responseEnded = true;
+  hasPendingAudio = false;
+  getSnapshot = () => this.snapshot;
+  subscribe = (listener) => {
+    this.listeners.add(listener);
+    return () => {
+      this.listeners.delete(listener);
+    };
+  };
+  update(patch) {
+    const next = { ...this.snapshot, ...patch };
+    next.canCancel = !this.awaitingClear && next.interruptionMode !== "none" && (next.status === "speaking" || next.status === "thinking");
+    this.snapshot = next;
+    for (const listener of this.listeners) listener();
+  }
+  setStatus(status) {
+    this.update({ status });
+  }
+  /** Acquire microphone access and open a call. Invoke from a user gesture. Failures populate snapshot.error. */
+  startCall = async (tokenOverride) => {
+    if (this.snapshot.status !== "idle" && this.snapshot.status !== "error") return;
+    this.cleanup();
+    const generation = this.generation;
+    this.update({ ...initialSnapshot(), status: "connecting" });
+    try {
+      const token = tokenOverride ?? (typeof this.options.clientToken === "function" ? await this.options.clientToken() : this.options.clientToken);
+      if (generation !== this.generation) return;
+      if (!token) throw new Error("Voice token unavailable. Please retry.");
+      if (!this.options.agentId) throw new Error("Voice requires an agentId.");
+      const url = new URL(this.options.apiUrl ?? "https://api.runtype.com");
+      if (!["https:", "http:", "wss:", "ws:"].includes(url.protocol) || url.username || url.password) {
+        throw new Error("Voice requires an HTTP or WebSocket API base URL without credentials.");
+      }
+      url.protocol = url.protocol === "https:" || url.protocol === "wss:" ? "wss:" : "ws:";
+      const basePath = url.pathname.replace(/\/+$/, "");
+      url.pathname = `${basePath}/ws/agents/${encodeURIComponent(this.options.agentId)}/voice`;
+      url.search = "";
+      url.searchParams.set("voiceProtocol", "runtype-browser-v1");
+      url.hash = "";
+      const stream = await navigator.mediaDevices.getUserMedia({
+        audio: { sampleRate: CAPTURE_SAMPLE_RATE, channelCount: 1, echoCancellation: true }
+      });
+      if (generation !== this.generation) {
+        stream.getTracks().forEach((track) => track.stop());
+        return;
+      }
+      this.stream = stream;
+      const context = new AudioContext({ sampleRate: CAPTURE_SAMPLE_RATE });
+      this.context = context;
+      if (context.state === "suspended") await context.resume();
+      if (generation !== this.generation) return;
+      const player = await createPcmPlayer(() => {
+        if (generation !== this.generation) return;
+        this.hasPendingAudio = false;
+        if (this.snapshot.status === "speaking") this.setStatus("listening");
+      });
+      if (generation !== this.generation) {
+        player.close();
+        return;
+      }
+      this.player = player;
+      const socket = new WebSocket(url.toString(), ["runtype.bearer", token]);
+      socket.binaryType = "arraybuffer";
+      this.socket = socket;
+      socket.onopen = () => {
+        if (generation !== this.generation) return;
+        try {
+          this.startCapture(context, stream, socket, generation);
+          this.setStatus("listening");
+        } catch (error) {
+          this.fail(error);
+        }
+      };
+      socket.onmessage = (event) => {
+        if (generation === this.generation) this.handleMessage(event.data, player);
+      };
+      socket.onerror = () => {
+        if (generation === this.generation) this.fail(new Error("Voice connection failed"));
+      };
+      socket.onclose = (event) => {
+        if (generation !== this.generation) return;
+        if (event.code === 1e3) this.endCall();
+        else this.fail(new Error(`Connection closed: ${event.reason || "unknown reason"}`));
+      };
+    } catch (error) {
+      if (generation === this.generation) this.fail(error);
+    }
+  };
+  endCall = () => {
+    this.cleanup();
+    this.update({ status: "idle", interimTranscript: null, audioLevel: 0, isMuted: false });
+  };
+  toggleMute = () => {
+    this.update({ isMuted: !this.snapshot.isMuted, audioLevel: 0 });
+  };
+  cancelResponse = () => {
+    if (!this.snapshot.canCancel) return;
+    this.stopPlayback();
+  };
+  /** Stop the current reply explicitly, including local playback when interruptions are disabled. */
+  stopPlayback = () => {
+    if (this.awaitingClear || this.socket?.readyState !== WebSocket.OPEN || !["speaking", "thinking"].includes(this.snapshot.status))
+      return;
+    this.clearPlayback();
+    if (this.snapshot.interruptionMode === "none") {
+      this.stoppedResponse = !this.responseEnded;
+      this.setStatus("listening");
+      return;
+    }
+    this.awaitingClear = true;
+    this.setStatus("listening");
+    this.socket.send(JSON.stringify({ type: "cancel" }));
+  };
+  fail(error) {
+    this.cleanup();
+    const denied = error instanceof Error && (error.name === "NotAllowedError" || /Permission denied|NotAllowed/.test(error.message));
+    this.update({
+      status: "error",
+      errorDetails: void 0,
+      audioLevel: 0,
+      interimTranscript: null,
+      error: denied ? "Microphone access denied. Please check browser permissions." : error instanceof Error ? error.message : "Failed to start voice call"
+    });
+  }
+  cleanup() {
+    this.generation += 1;
+    if (this.processor) this.processor.onaudioprocess = null;
+    this.processor?.disconnect();
+    this.processor = null;
+    this.source?.disconnect();
+    this.source = null;
+    this.stream?.getTracks().forEach((track) => track.stop());
+    this.stream = null;
+    void this.context?.close().catch(() => {
+    });
+    this.context = null;
+    this.player?.close();
+    this.player = null;
+    const socket = this.socket;
+    this.socket = null;
+    if (socket) {
+      socket.onopen = socket.onmessage = socket.onerror = socket.onclose = null;
+      socket.close(1e3, "User ended call");
+    }
+    this.playbackRevision += 1;
+    this.pushChain = Promise.resolve();
+    this.hasPendingAudio = false;
+    this.awaitingClear = false;
+    this.stoppedResponse = false;
+    this.responseEnded = true;
+  }
+  clearPlayback() {
+    this.playbackRevision += 1;
+    this.pushChain = Promise.resolve();
+    this.player?.clear();
+    this.hasPendingAudio = false;
+  }
+  handleMessage(data, player) {
+    if (data instanceof ArrayBuffer || data instanceof Blob) {
+      if (this.awaitingClear || this.stoppedResponse) return;
+      this.responseEnded = false;
+      this.hasPendingAudio = true;
+      this.setStatus("speaking");
+      const revision = this.playbackRevision;
+      this.pushChain = this.pushChain.then(async () => {
+        if (revision !== this.playbackRevision) return;
+        try {
+          await player.push(data);
+        } catch (error) {
+          if (revision === this.playbackRevision) this.fail(error);
+        }
+      });
+      return;
+    }
+    if (typeof data !== "string") return;
+    let msg;
+    try {
+      const parsed = JSON.parse(data);
+      if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return;
+      msg = parsed;
+    } catch {
+      return;
+    }
+    switch (msg.type) {
+      case "session_config":
+        if (msg.interruptionMode === "none" || msg.interruptionMode === "cancel" || msg.interruptionMode === "barge-in") {
+          this.update({ interruptionMode: msg.interruptionMode });
+        }
+        break;
+      case "transcript_interim":
+        this.update({ interimTranscript: typeof msg.text === "string" ? msg.text || null : null });
+        break;
+      case "transcript_final":
+        if (typeof msg.text !== "string" || msg.role !== "user" && msg.role !== "assistant") break;
+        if (msg.role === "assistant" && (this.awaitingClear || this.stoppedResponse)) break;
+        this.responseEnded = false;
+        this.update({
+          interimTranscript: null,
+          transcript: [
+            ...this.snapshot.transcript,
+            {
+              role: msg.role,
+              content: msg.text,
+              timestamp: Date.now(),
+              ...typeof msg.turnId === "string" && msg.turnId ? { turnId: msg.turnId } : {}
+            }
+          ],
+          status: this.awaitingClear ? this.snapshot.status : msg.role === "user" ? "thinking" : "speaking"
+        });
+        break;
+      case "audio_end": {
+        this.responseEnded = true;
+        if (this.stoppedResponse) {
+          this.stoppedResponse = false;
+          break;
+        }
+        if (this.awaitingClear) break;
+        const revision = this.playbackRevision;
+        this.pushChain = this.pushChain.then(() => {
+          if (revision === this.playbackRevision) player.endOfStream();
+        });
+        if (!this.hasPendingAudio && this.snapshot.status === "speaking")
+          this.setStatus("listening");
+        break;
+      }
+      case "audio_clear":
+        this.clearPlayback();
+        this.awaitingClear = false;
+        this.stoppedResponse = false;
+        this.responseEnded = true;
+        this.setStatus("listening");
+        break;
+      case "metrics": {
+        const number = (value) => typeof value === "number" && Number.isFinite(value) ? value : void 0;
+        this.update({
+          metrics: {
+            llmMs: number(msg.llm_ms),
+            // @snake-case-ok: Existing voice wire contract.
+            ttsMs: number(msg.tts_ms),
+            // @snake-case-ok: Existing voice wire contract.
+            firstAudioMs: number(msg.first_audio_ms),
+            // @snake-case-ok: Existing voice wire contract.
+            totalMs: number(msg.total_ms)
+            // @snake-case-ok: Existing voice wire contract.
+          }
+        });
+        break;
+      }
+      case "error": {
+        const details = msg.details;
+        this.fail(
+          new Error(
+            typeof msg.error === "string" ? msg.error : typeof msg.message === "string" ? msg.message : "Voice error"
+          )
+        );
+        if (details && typeof details.code === "string" && details.code.startsWith("MCP_") && typeof details.serverName === "string" && typeof details.diagnosticId === "string") {
+          this.update({
+            errorDetails: {
+              code: details.code,
+              ...typeof details.serverId === "string" ? { serverId: details.serverId } : {},
+              serverName: details.serverName,
+              diagnosticId: details.diagnosticId
+            }
+          });
+        }
+        break;
+      }
+    }
+  }
+  startCapture(context, stream, socket, generation) {
+    const source = context.createMediaStreamSource(stream);
+    this.source = source;
+    const processor = context.createScriptProcessor(CAPTURE_BUFFER_SIZE, 1, 1);
+    this.processor = processor;
+    processor.onaudioprocess = (event) => {
+      if (generation !== this.generation || this.snapshot.isMuted) return;
+      const input = event.inputBuffer.getChannelData(0);
+      let sum = 0;
+      for (const sample of input) sum += sample * sample;
+      this.update({ audioLevel: Math.sqrt(sum / input.length) });
+      if (socket.readyState !== WebSocket.OPEN) return;
+      const pcm = new Int16Array(input.length);
+      if (this.snapshot.interruptionMode === "cancel" && (this.awaitingClear || this.snapshot.status === "thinking" || this.hasPendingAudio || this.snapshot.status === "speaking")) {
+        socket.send(pcm.buffer);
+        return;
+      }
+      for (let i = 0; i < input.length; i++) {
+        const sample = Math.max(-1, Math.min(1, input[i]));
+        pcm[i] = sample < 0 ? sample * 32768 : sample * 32767;
+      }
+      socket.send(pcm.buffer);
+    };
+    source.connect(processor);
+    processor.connect(context.destination);
+  }
+};
+
+// src/react.ts
+function useVoiceClient({ agentId, apiUrl, clientToken }) {
+  const tokenRef = useRef(clientToken);
+  useEffect(() => {
+    tokenRef.current = clientToken;
+  }, [clientToken]);
+  const client = useMemo(
+    () => new VoiceClient({
+      agentId,
+      apiUrl,
+      clientToken: () => typeof tokenRef.current === "function" ? tokenRef.current() : tokenRef.current
+    }),
+    [agentId, apiUrl]
+  );
+  const snapshot = useSyncExternalStore(client.subscribe, client.getSnapshot, client.getSnapshot);
+  useEffect(() => () => client.endCall(), [client]);
+  return {
+    ...snapshot,
+    startCall: client.startCall,
+    endCall: client.endCall,
+    toggleMute: client.toggleMute,
+    cancelResponse: client.cancelResponse
+  };
+}
+export {
+  useVoiceClient
+};

package/dist/types-9SfA7oHc.d.cts

@@ -0,0 +1,40 @@
+type VoiceStatus = 'idle' | 'connecting' | 'listening' | 'thinking' | 'speaking' | 'error';
+type InterruptionMode = 'none' | 'cancel' | 'barge-in';
+interface TranscriptEntry {
+    readonly role: 'user' | 'assistant';
+    readonly content: string;
+    readonly timestamp: number;
+    readonly turnId?: string;
+}
+interface VoiceMetrics {
+    readonly llmMs?: number;
+    readonly ttsMs?: number;
+    readonly firstAudioMs?: number;
+    readonly totalMs?: number;
+}
+interface VoiceSnapshot {
+    readonly status: VoiceStatus;
+    readonly transcript: readonly TranscriptEntry[];
+    readonly interimTranscript: string | null;
+    readonly metrics: VoiceMetrics | null;
+    readonly audioLevel: number;
+    readonly isMuted: boolean;
+    readonly error: string | null;
+    readonly errorDetails?: {
+        code: string;
+        serverId?: string;
+        serverName: string;
+        diagnosticId: string;
+    };
+    readonly interruptionMode: InterruptionMode;
+    readonly canCancel: boolean;
+}
+interface VoiceClientOptions {
+    agentId: string;
+    /** Browser client token, or a getter called once per call to obtain a fresh token. */
+    clientToken: string | (() => string | Promise<string>);
+    /** Absolute API base URL, including any proxy path prefix. Defaults to https://api.runtype.com. */
+    apiUrl?: string;
+}
+
+export type { InterruptionMode as I, TranscriptEntry as T, VoiceClientOptions as V, VoiceSnapshot as a, VoiceMetrics as b, VoiceStatus as c };

package/dist/types-9SfA7oHc.d.ts

@@ -0,0 +1,40 @@
+type VoiceStatus = 'idle' | 'connecting' | 'listening' | 'thinking' | 'speaking' | 'error';
+type InterruptionMode = 'none' | 'cancel' | 'barge-in';
+interface TranscriptEntry {
+    readonly role: 'user' | 'assistant';
+    readonly content: string;
+    readonly timestamp: number;
+    readonly turnId?: string;
+}
+interface VoiceMetrics {
+    readonly llmMs?: number;
+    readonly ttsMs?: number;
+    readonly firstAudioMs?: number;
+    readonly totalMs?: number;
+}
+interface VoiceSnapshot {
+    readonly status: VoiceStatus;
+    readonly transcript: readonly TranscriptEntry[];
+    readonly interimTranscript: string | null;
+    readonly metrics: VoiceMetrics | null;
+    readonly audioLevel: number;
+    readonly isMuted: boolean;
+    readonly error: string | null;
+    readonly errorDetails?: {
+        code: string;
+        serverId?: string;
+        serverName: string;
+        diagnosticId: string;
+    };
+    readonly interruptionMode: InterruptionMode;
+    readonly canCancel: boolean;
+}
+interface VoiceClientOptions {
+    agentId: string;
+    /** Browser client token, or a getter called once per call to obtain a fresh token. */
+    clientToken: string | (() => string | Promise<string>);
+    /** Absolute API base URL, including any proxy path prefix. Defaults to https://api.runtype.com. */
+    apiUrl?: string;
+}
+
+export type { InterruptionMode as I, TranscriptEntry as T, VoiceClientOptions as V, VoiceSnapshot as a, VoiceMetrics as b, VoiceStatus as c };

package/package.json

@@ -0,0 +1,73 @@
+{
+  "name": "@runtypelabs/voice",
+  "version": "0.2.3",
+  "description": "Runtype browser voice client with React and Persona adapters",
+  "type": "module",
+  "license": "MIT",
+  "sideEffects": false,
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    },
+    "./react": {
+      "types": "./dist/react.d.ts",
+      "import": "./dist/react.js",
+      "require": "./dist/react.cjs"
+    },
+    "./persona": {
+      "types": "./dist/persona.d.ts",
+      "import": "./dist/persona.js",
+      "require": "./dist/persona.cjs"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "test": "vitest run",
+    "typecheck": "tsgo --noEmit",
+    "lint": "eslint .",
+    "prepublishOnly": "pnpm build"
+  },
+  "peerDependencies": {
+    "react": ">=18 <20",
+    "@runtypelabs/persona": ">=4.22.0 <5"
+  },
+  "peerDependenciesMeta": {
+    "react": {
+      "optional": true
+    },
+    "@runtypelabs/persona": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@runtypelabs/persona": "4.22.0",
+    "@testing-library/react": "^16.3.2",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "jsdom": "^30.0.1",
+    "react": "^19",
+    "react-dom": "^19",
+    "tsup": "^8.0.0",
+    "typescript": "^6.0.3",
+    "vitest": "^4.1.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/runtypelabs/core",
+    "directory": "packages/voice"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}