@shvm/vani-client 0.0.2

package/dist/index.js

@@ -0,0 +1,651 @@
+import { useRef, useCallback, useEffect } from 'react';
+import { useMicVAD } from '@ricky0123/vad-react';
+import * as ort from 'onnxruntime-web';
+import { useActor } from '@xstate/react';
+import { fromCallback, setup, assign } from 'xstate';
+
+// src/shared/constants/models.ts
+var STT_MODELS = ["@cf/openai/whisper", "@cf/openai/whisper-tiny-en"];
+var LLM_MODELS = [
+  "@cf/meta/llama-3.1-8b-instruct",
+  "@cf/meta/llama-3-8b-instruct",
+  "@cf/meta/llama-2-7b-chat-int8",
+  "@cf/mistral/mistral-7b-instruct-v0.1"
+];
+var TTS_MODELS = ["@cf/deepgram/aura-2-en", "@cf/deepgram/aura-1"];
+var TTS_MODEL_VOICES = {
+  "@cf/deepgram/aura-2-en": ["asteria", "luna", "arcas", "athena", "helios", "orpheus", "perseus", "angus"],
+  "@cf/deepgram/aura-1": [
+    "asteria",
+    "luna",
+    "stella",
+    "athena",
+    "hera",
+    "orion",
+    "arcas",
+    "perseus",
+    "angus",
+    "orpheus",
+    "helios",
+    "zeus"
+  ]
+};
+
+// src/headless/adapters/blobUrl.ts
+function createBlobUrl(blob) {
+  try {
+    if (typeof URL === "undefined") return void 0;
+    if (typeof URL.createObjectURL !== "function") return void 0;
+    return URL.createObjectURL(blob);
+  } catch {
+    return void 0;
+  }
+}
+fromCallback(() => {
+  return () => {
+  };
+});
+fromCallback(() => {
+  return () => {
+  };
+});
+var clientMachine = setup({
+  types: {
+    context: {},
+    events: {}
+  },
+  actions: {
+    setStatusConfig: assign({
+      status: () => "connecting"
+    }),
+    setConnected: assign({
+      status: () => "idle",
+      history: ({ context }) => [
+        ...context.history,
+        {
+          id: Math.random().toString(36).slice(2),
+          type: "socket_event",
+          timestamp: Date.now(),
+          details: { status: "connected" }
+        }
+      ]
+    }),
+    setDisconnected: assign({
+      status: () => "disconnected",
+      history: ({ context }) => [
+        ...context.history,
+        {
+          id: Math.random().toString(36).slice(2),
+          type: "socket_event",
+          timestamp: Date.now(),
+          details: { status: "disconnected" }
+        }
+      ]
+    }),
+    setError: assign({
+      status: () => "error",
+      error: ({ event }) => event.type === "SET_ERROR" ? event.error : null,
+      history: ({ context, event }) => [
+        ...context.history,
+        {
+          id: Math.random().toString(36).slice(2),
+          type: "error",
+          timestamp: Date.now(),
+          details: { message: event.type === "SET_ERROR" ? event.error : "Unknown error" }
+        }
+      ]
+    }),
+    updateServerStatus: assign({
+      serverStatus: ({ context, event }) => event.type === "SERVER_STATE_CHANGE" ? event.status : context.serverStatus,
+      history: ({ context, event }) => {
+        if (event.type !== "SERVER_STATE_CHANGE") return context.history;
+        return [
+          ...context.history,
+          {
+            id: Math.random().toString(36).slice(2),
+            type: "state_change",
+            timestamp: Date.now(),
+            details: { from: context.serverStatus, to: event.status, source: "server" }
+          }
+        ];
+      }
+    }),
+    setPlaying: assign({
+      isPlaying: ({ event }) => event.type === "AUDIO_PLAYBACK_START"
+    }),
+    addMessage: assign({
+      transcript: ({ context, event }) => {
+        if (event.type !== "ADD_MESSAGE") return context.transcript;
+        return [
+          ...context.transcript,
+          {
+            id: Math.random().toString(36).slice(2),
+            role: event.role,
+            content: event.content,
+            timestamp: Date.now()
+          }
+        ];
+      },
+      history: ({ context, event }) => {
+        if (event.type !== "ADD_MESSAGE") return context.history;
+        return [
+          ...context.history,
+          {
+            id: Math.random().toString(36).slice(2),
+            type: "transcript",
+            timestamp: Date.now(),
+            details: { role: event.role, text: event.content }
+          }
+        ];
+      }
+    }),
+    logEvent: assign({
+      history: ({ context, event }) => {
+        if (event.type !== "LOG_EVENT") return context.history;
+        return [
+          ...context.history,
+          {
+            id: Math.random().toString(36).slice(2),
+            type: event.eventType,
+            timestamp: Date.now(),
+            details: event.details,
+            blobUrl: event.blob ? createBlobUrl(event.blob) : void 0
+          }
+        ];
+      }
+    }),
+    addToolCallStart: assign({
+      transcript: ({ context, event }) => {
+        if (event.type !== "TOOL_CALL_START") return context.transcript;
+        const newTranscript = [...context.transcript];
+        if (newTranscript.length === 0 || newTranscript[newTranscript.length - 1].role !== "assistant") {
+          newTranscript.push({
+            id: Math.random().toString(36).slice(2),
+            role: "assistant",
+            content: "",
+            timestamp: Date.now(),
+            toolCalls: []
+          });
+        }
+        const lastMsg = newTranscript[newTranscript.length - 1];
+        lastMsg.toolCalls = lastMsg.toolCalls || [];
+        lastMsg.toolCalls.push({ name: event.toolName, status: "calling" });
+        return newTranscript;
+      }
+    }),
+    addToolCallEnd: assign({
+      transcript: ({ context, event }) => {
+        if (event.type !== "TOOL_CALL_END") return context.transcript;
+        const newTranscript = [...context.transcript];
+        if (newTranscript.length > 0) {
+          const lastMsg = newTranscript[newTranscript.length - 1];
+          if (lastMsg.role === "assistant" && lastMsg.toolCalls) {
+            const activeTool = lastMsg.toolCalls.slice().reverse().find((t) => t.name === event.toolName && t.status === "calling");
+            if (activeTool) activeTool.status = "finished";
+          }
+        }
+        return newTranscript;
+      }
+    }),
+    clearError: assign({
+      error: null
+    })
+  },
+  guards: {
+    isServerThinkingOrSpeaking: ({ context, event }) => {
+      const status = event.type === "SERVER_STATE_CHANGE" ? event.status : context.serverStatus;
+      return status === "thinking" || status === "speaking";
+    }
+  }
+}).createMachine({
+  id: "client",
+  initial: "disconnected",
+  context: {
+    status: "disconnected",
+    serverStatus: "idle",
+    transcript: [],
+    history: [],
+    error: null,
+    isPlaying: false
+  },
+  on: {
+    LOG_EVENT: { actions: "logEvent" },
+    ADD_MESSAGE: { actions: ["addMessage", "clearError"] },
+    TOOL_CALL_START: { actions: "addToolCallStart" },
+    TOOL_CALL_END: { actions: "addToolCallEnd" },
+    SET_ERROR: { target: ".error", actions: "setError" },
+    DISCONNECT: { target: ".disconnected", actions: "setDisconnected" }
+  },
+  states: {
+    disconnected: {
+      on: {
+        CONNECT: { target: "connecting", actions: "setStatusConfig" }
+      }
+    },
+    connecting: {
+      on: {
+        CONNECTED: { target: "connected", actions: "setConnected" }
+      }
+    },
+    connected: {
+      initial: "idle",
+      states: {
+        idle: {
+          entry: assign({ status: "idle" }),
+          on: {
+            START_LISTENING: { target: "#client.listening", actions: "clearError" },
+            AUDIO_PLAYBACK_START: { target: "#client.speaking", actions: "setPlaying" },
+            SERVER_STATE_CHANGE: [
+              {
+                guard: "isServerThinkingOrSpeaking",
+                target: "processing",
+                actions: "updateServerStatus"
+              },
+              {
+                actions: "updateServerStatus"
+              }
+            ]
+          }
+        },
+        processing: {
+          entry: assign({ status: "processing" }),
+          after: {
+            2e4: { target: "idle", actions: assign({ error: "Server timed out. Interactions will reset." }) }
+          },
+          on: {
+            CANCEL: { target: "idle", actions: "clearError" },
+            START_LISTENING: { target: "#client.listening", actions: "clearError" },
+            AUDIO_PLAYBACK_START: { target: "#client.speaking", actions: "setPlaying" },
+            SERVER_STATE_CHANGE: [
+              {
+                guard: ({ event }) => event.status === "listening" || event.status === "idle",
+                target: "idle",
+                actions: "updateServerStatus"
+              },
+              {
+                actions: "updateServerStatus"
+              }
+            ]
+          }
+        }
+      }
+    },
+    listening: {
+      entry: assign({ status: "listening" }),
+      on: {
+        STOP_LISTENING: { target: "connected.processing" },
+        SERVER_STATE_CHANGE: { actions: "updateServerStatus" }
+      }
+    },
+    speaking: {
+      entry: assign({ status: "speaking" }),
+      on: {
+        AUDIO_PLAYBACK_END: {
+          target: "connected.idle",
+          actions: assign({ isPlaying: false })
+        },
+        SERVER_STATE_CHANGE: { actions: "updateServerStatus" }
+      }
+    },
+    error: {
+      on: {
+        CONNECT: { target: "connecting", actions: "setStatusConfig" },
+        START_LISTENING: { target: "listening", actions: "clearError" }
+      }
+    }
+  }
+});
+
+// src/headless/utils/webSocketUrl.ts
+var DEFAULT_VOICE_SERVER_URL = "https://shvm.in";
+function buildVoiceWebSocketUrl({
+  sessionId,
+  serverUrl,
+  wsPath,
+  getWebSocketUrlOverride
+}) {
+  if (getWebSocketUrlOverride) return getWebSocketUrlOverride(sessionId);
+  const wsPathValue = wsPath ? wsPath(sessionId) : `/ws/${sessionId}`;
+  const base = new URL(serverUrl ?? DEFAULT_VOICE_SERVER_URL);
+  const protocol = base.protocol === "https:" ? "wss:" : base.protocol === "http:" ? "ws:" : base.protocol;
+  base.protocol = protocol;
+  return new URL(wsPathValue, base).toString();
+}
+
+// src/headless/hooks/useVoiceSession.ts
+var ONNX_WASM_BASE_PATH = "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.22.0/dist/";
+var VAD_BASE_ASSET_PATH = "https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.29/dist/";
+var VAD_MODEL_URL = "https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.29/dist/silero_vad_v5.onnx";
+var ortConfigured = false;
+function ensureOrtConfig() {
+  if (ortConfigured) return;
+  if (typeof window !== "undefined") {
+    ort.env.wasm.wasmPaths = ONNX_WASM_BASE_PATH;
+    ort.env.wasm.proxy = false;
+  }
+  ortConfigured = true;
+}
+var VAD_SAMPLE_RATE = 16e3;
+function writeString(view, offset, text) {
+  for (let i = 0; i < text.length; i += 1) {
+    view.setUint8(offset + i, text.charCodeAt(i));
+  }
+}
+function encodeWav(audio, sampleRate) {
+  const buffer = new ArrayBuffer(44 + audio.length * 2);
+  const view = new DataView(buffer);
+  writeString(view, 0, "RIFF");
+  view.setUint32(4, 36 + audio.length * 2, true);
+  writeString(view, 8, "WAVE");
+  writeString(view, 12, "fmt ");
+  view.setUint32(16, 16, true);
+  view.setUint16(20, 1, true);
+  view.setUint16(22, 1, true);
+  view.setUint32(24, sampleRate, true);
+  view.setUint32(28, sampleRate * 2, true);
+  view.setUint16(32, 2, true);
+  view.setUint16(34, 16, true);
+  writeString(view, 36, "data");
+  view.setUint32(40, audio.length * 2, true);
+  let offset = 44;
+  for (let i = 0; i < audio.length; i += 1) {
+    const sample = Math.max(-1, Math.min(1, audio[i]));
+    view.setInt16(offset, sample < 0 ? sample * 32768 : sample * 32767, true);
+    offset += 2;
+  }
+  return buffer;
+}
+function useVoiceSession(props = {}) {
+  const {
+    onError,
+    onMessage,
+    onFeedback,
+    initialTranscript,
+    config,
+    serverUrl,
+    getWebSocketUrl: getWebSocketUrlOverride,
+    sessionId,
+    wsPath
+  } = props;
+  ensureOrtConfig();
+  const [snapshot, send, actorRef] = useActor(clientMachine);
+  const state = snapshot.context;
+  const wsRef = useRef(null);
+  const audioContextRef = useRef(null);
+  const audioQueueRef = useRef([]);
+  const isPlaybackLoopRunning = useRef(false);
+  const onErrorCallbackRef = useRef(onError);
+  const onMessageCallbackRef = useRef(onMessage);
+  const onFeedbackCallbackRef = useRef(onFeedback);
+  const configRef = useRef(config);
+  const turnActiveRef = useRef(false);
+  const lastVADErrorRef = useRef(null);
+  const hasSeededTranscriptRef = useRef(false);
+  const sessionIdRef = useRef(null);
+  if (sessionIdRef.current === null) {
+    sessionIdRef.current = sessionId ?? (typeof crypto !== "undefined" && "randomUUID" in crypto ? (
+      // @ts-ignore
+      crypto.randomUUID()
+    ) : "session-" + Math.floor(Math.random() * 1e4));
+  }
+  const buildWebSocketUrl = useCallback((activeSessionId) => {
+    return buildVoiceWebSocketUrl({
+      sessionId: activeSessionId,
+      serverUrl,
+      wsPath,
+      getWebSocketUrlOverride
+    });
+  }, [getWebSocketUrlOverride, serverUrl, wsPath]);
+  useEffect(() => {
+    onErrorCallbackRef.current = onError;
+    onMessageCallbackRef.current = onMessage;
+    onFeedbackCallbackRef.current = onFeedback;
+    configRef.current = config;
+  }, [onError, onMessage, onFeedback, config]);
+  useEffect(() => {
+    if (hasSeededTranscriptRef.current) return;
+    if (!initialTranscript || initialTranscript.length === 0) return;
+    if (actorRef.getSnapshot().context.transcript.length > 0) {
+      hasSeededTranscriptRef.current = true;
+      return;
+    }
+    initialTranscript.forEach((msg) => {
+      send({ type: "ADD_MESSAGE", role: msg.role, content: msg.content });
+    });
+    hasSeededTranscriptRef.current = true;
+  }, [actorRef, initialTranscript, send]);
+  const initAudio = async () => {
+    if (audioContextRef.current && audioContextRef.current.state !== "closed") {
+      if (audioContextRef.current.state === "suspended") {
+        await audioContextRef.current.resume();
+      }
+      return;
+    }
+    try {
+      audioContextRef.current = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 24e3 });
+      if (audioContextRef.current.state === "suspended") {
+        await audioContextRef.current.resume();
+      }
+    } catch (e) {
+      console.error("[Voice] Audio init error:", e);
+      const msg = "Audio initialization failed: " + e.message;
+      console.error("[Voice] Sending SET_ERROR (Audio Init)", msg);
+      send({ type: "SET_ERROR", error: msg });
+      throw e;
+    }
+  };
+  const handleSpeechStart = useCallback(() => {
+    const ws = wsRef.current;
+    if (!ws || ws.readyState !== WebSocket.OPEN) return;
+    if (turnActiveRef.current) return;
+    const currentContext = actorRef.getSnapshot().context;
+    const status = currentContext.status;
+    currentContext.error;
+    if (status === "speaking" || status === "processing") {
+      console.log("[Voice] Busy, rejecting speech input (strict turn-by-turn).");
+      return;
+    }
+    if (status !== "idle" && status !== "listening" && status !== "error") return;
+    turnActiveRef.current = true;
+    ws.send(JSON.stringify({
+      type: "start",
+      config: configRef.current
+    }));
+    send({ type: "START_LISTENING" });
+  }, [actorRef, send]);
+  const sendMessage = useCallback((text) => {
+    const ws = wsRef.current;
+    if (ws && ws.readyState === WebSocket.OPEN) {
+      ws.send(JSON.stringify({ type: "text.message", content: text }));
+      send({ type: "ADD_MESSAGE", role: "user", content: text });
+      send({ type: "SERVER_STATE_CHANGE", status: "speaking" });
+    }
+  }, [send]);
+  const handleSpeechEnd = useCallback(async (audio) => {
+    const ws = wsRef.current;
+    if (!turnActiveRef.current) return;
+    turnActiveRef.current = false;
+    if (!ws || ws.readyState !== WebSocket.OPEN) {
+      send({ type: "SERVER_STATE_CHANGE", status: "idle" });
+      return;
+    }
+    const wavBuffer = encodeWav(audio, VAD_SAMPLE_RATE);
+    const wavBlob = new Blob([wavBuffer], { type: "audio/wav" });
+    ws.send(wavBuffer);
+    send({ type: "LOG_EVENT", eventType: "audio_input", details: { size: wavBuffer.byteLength }, blob: wavBlob });
+    ws.send(JSON.stringify({ type: "stop" }));
+    send({ type: "STOP_LISTENING" });
+  }, [send]);
+  const handleVADMisfire = useCallback(() => {
+    if (!turnActiveRef.current) return;
+    turnActiveRef.current = false;
+    send({ type: "SERVER_STATE_CHANGE", status: "idle" });
+  }, [send]);
+  const vad = useMicVAD({
+    startOnLoad: false,
+    onSpeechStart: handleSpeechStart,
+    onSpeechEnd: handleSpeechEnd,
+    onVADMisfire: handleVADMisfire,
+    // @ts-expect-error
+    workletURL: VAD_BASE_ASSET_PATH + "vad.worklet.bundle.min.js",
+    modelURL: VAD_MODEL_URL,
+    onnxWASMBasePath: ONNX_WASM_BASE_PATH,
+    baseAssetPath: VAD_BASE_ASSET_PATH
+  });
+  useEffect(() => {
+    if (!vad.errored) return;
+    const message = typeof vad.errored === "string" ? vad.errored : vad.errored.message || "VAD failed to load";
+    if (lastVADErrorRef.current === message) return;
+    lastVADErrorRef.current = message;
+    console.error("[Voice] Sending SET_ERROR (VAD)", message);
+    send({ type: "SET_ERROR", error: message });
+  }, [vad.errored, send]);
+  useEffect(() => {
+    const shouldListen = state.status === "idle" || state.status === "listening";
+    if (shouldListen && !vad.listening && !vad.loading && !vad.errored) {
+      vad.start();
+    } else if (!shouldListen && vad.listening) {
+      vad.pause();
+    }
+  }, [state.status, vad.listening, vad.loading, vad.errored, vad.start, vad.pause]);
+  const connect = useCallback(() => {
+    if (wsRef.current) wsRef.current.close();
+    initAudio().catch((err) => console.warn("[Voice] Early audio init failed", err));
+    console.log("[Voice] Connect called");
+    send({ type: "CONNECT" });
+    const sessionId2 = sessionIdRef.current || "session-" + Math.floor(Math.random() * 1e4);
+    const url = buildWebSocketUrl(sessionId2);
+    const ws = new WebSocket(url);
+    wsRef.current = ws;
+    ws.onopen = () => {
+      send({ type: "CONNECTED" });
+      initAudio().catch(() => {
+      });
+    };
+    ws.onmessage = async (event) => {
+      if (event.data instanceof Blob) {
+        const buf = await event.data.arrayBuffer();
+        send({ type: "LOG_EVENT", eventType: "audio_output", details: { size: buf.byteLength }, blob: event.data });
+        queueAudio(buf);
+        return;
+      }
+      try {
+        const data = JSON.parse(event.data);
+        handleMessage(data);
+      } catch (e) {
+        console.error("[Voice] Parse error", e);
+      }
+    };
+    ws.onclose = (e) => {
+      console.log("[Voice] Closed", e.code);
+      send({ type: "DISCONNECT" });
+    };
+    ws.onerror = (e) => {
+      console.error("[Voice] WS Error", e);
+      send({ type: "SET_ERROR", error: "Connection failed: " + (e instanceof ErrorEvent ? e.message : "Unknown") });
+    };
+  }, [buildWebSocketUrl, send]);
+  const handleMessage = (data) => {
+    switch (data.type) {
+      case "state":
+        send({ type: "SERVER_STATE_CHANGE", status: data.value });
+        break;
+      case "transcript.final":
+        send({ type: "ADD_MESSAGE", role: "user", content: data.text });
+        onMessageCallbackRef.current?.({ role: "user", content: data.text });
+        break;
+      case "assistant.message":
+        send({ type: "ADD_MESSAGE", role: "assistant", content: data.message.content });
+        onMessageCallbackRef.current?.({ role: "assistant", content: data.message.content });
+        break;
+      case "assistant.partial":
+        break;
+      case "error":
+        console.error("[Voice] Sending SET_ERROR (Server)", data.reason);
+        send({ type: "SET_ERROR", error: data.reason });
+        break;
+      case "feedback":
+        onFeedbackCallbackRef.current?.(data.message);
+        break;
+      case "tool.call.start":
+        send({ type: "TOOL_CALL_START", toolName: data.toolName });
+        break;
+      case "tool.call.end":
+        send({ type: "TOOL_CALL_END", toolName: data.toolName });
+        break;
+    }
+  };
+  const queueAudio = (buffer) => {
+    audioQueueRef.current.push(buffer);
+    if (!isPlaybackLoopRunning.current) {
+      playQueue();
+    }
+  };
+  const playQueue = async () => {
+    isPlaybackLoopRunning.current = true;
+    while (audioQueueRef.current.length > 0) {
+      send({ type: "AUDIO_PLAYBACK_START" });
+      const buffer = audioQueueRef.current.shift();
+      if (!buffer) continue;
+      try {
+        const ctx = audioContextRef.current;
+        const decoded = await ctx.decodeAudioData(buffer.slice(0));
+        await new Promise((resolve) => {
+          const source = ctx.createBufferSource();
+          source.buffer = decoded;
+          source.connect(ctx.destination);
+          source.onended = () => resolve();
+          source.start(0);
+        });
+      } catch (e) {
+        console.warn("[Voice] Decode failed, trying fallback", e);
+        try {
+          const blob = new Blob([buffer], { type: "audio/wav" });
+          const url = URL.createObjectURL(blob);
+          const audio = new Audio(url);
+          await new Promise((resolve, reject) => {
+            audio.onended = () => {
+              URL.revokeObjectURL(url);
+              resolve();
+            };
+            audio.onerror = reject;
+            audio.play().catch(reject);
+          });
+        } catch (err) {
+          console.error("[Voice] Playback failed completely", err);
+        }
+      }
+    }
+    send({ type: "AUDIO_PLAYBACK_END" });
+    isPlaybackLoopRunning.current = false;
+  };
+  const disconnect = useCallback(() => {
+    if (wsRef.current) {
+      wsRef.current.close();
+      wsRef.current = null;
+    }
+    send({ type: "DISCONNECT" });
+  }, [send]);
+  const cancel = useCallback(() => {
+    const ws = wsRef.current;
+    if (ws && ws.readyState === WebSocket.OPEN) {
+      ws.send(JSON.stringify({ type: "reset" }));
+    }
+    send({ type: "CANCEL" });
+  }, [send]);
+  return {
+    ...state,
+    vadListening: vad.listening,
+    vadLoading: vad.loading,
+    vadErrored: vad.errored,
+    userSpeaking: vad.userSpeaking,
+    connect,
+    disconnect,
+    sendMessage,
+    cancel
+  };
+}
+
+export { LLM_MODELS, STT_MODELS, TTS_MODELS, TTS_MODEL_VOICES, clientMachine, createBlobUrl, useVoiceSession };
+//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map