@shogo-ai/voice 1.10.0 → 2.4.1

package/dist/react/index.cjs

@@ -43,6 +43,7 @@ __export(react_exports, {
   resolveOrganicParticlesConfig: () => resolveOrganicParticlesConfig,
   resolveOrganicSphereConfig: () => resolveOrganicSphereConfig,
   useChatConversation: () => useChatConversation,
+  useLiveVoiceConversation: () => useLiveVoiceConversation,
   useShogoChat: () => useShogoChat,
   useShogoVoice: () => useShogoVoice,
   useVoiceConversation: () => useVoiceConversation
@@ -244,6 +245,25 @@ function withProjectQuery(signedUrlPath, params) {
   return `${signedUrlPath}${sep}${tail.join("&")}`;
 }
 
+// src/react/mic-access.ts
+var MIC_BLOCKED_MESSAGE = "Microphone access is blocked. Allow it in your system or browser settings (on macOS: System Settings > Privacy & Security > Microphone), then try again.";
+var MIC_NOT_FOUND_MESSAGE = "No microphone was found. Connect one and try again.";
+async function requestMicStream(constraints = { audio: true }) {
+  const desktop = globalThis.shogoDesktop;
+  if (desktop?.isDesktop && desktop.ensureMicAccess) {
+    const access = await desktop.ensureMicAccess().catch(() => "granted");
+    if (access !== "granted") throw new Error(MIC_BLOCKED_MESSAGE);
+  }
+  try {
+    return await navigator.mediaDevices.getUserMedia(constraints);
+  } catch (err) {
+    const name = err?.name;
+    if (name === "NotAllowedError" || name === "SecurityError") throw new Error(MIC_BLOCKED_MESSAGE);
+    if (name === "NotFoundError" || name === "OverconstrainedError") throw new Error(MIC_NOT_FOUND_MESSAGE);
+    throw err;
+  }
+}
+
 // src/react/useVoiceConversation.ts
 function useVoiceConversation(options) {
   const {
@@ -359,7 +379,8 @@ function useVoiceConversation(options) {
         } catch {
         }
       }
-      await navigator.mediaDevices.getUserMedia({ audio: true });
+      const micStream = await requestMicStream({ audio: true });
+      micStream.getTracks().forEach((track) => track.stop());
       const data = await fetchSignedUrl({
         path: resolvedSignedUrlPath,
         fetchCredentials,
@@ -461,6 +482,322 @@ function useVoiceConversation(options) {
   };
 }
 
+// src/react/useLiveVoiceConversation.ts
+var import_react3 = require("react");
+function parseEvent(data) {
+  try {
+    const text = typeof data === "string" ? data : String(data);
+    const event = JSON.parse(text);
+    return event && typeof event === "object" ? event : null;
+  } catch {
+    return null;
+  }
+}
+function waitForIceGathering(pc) {
+  if (pc.iceGatheringState === "complete") return Promise.resolve();
+  return new Promise((resolve) => {
+    const onStateChange = () => {
+      if (pc.iceGatheringState === "complete") {
+        pc.removeEventListener("icegatheringstatechange", onStateChange);
+        resolve();
+      }
+    };
+    pc.addEventListener("icegatheringstatechange", onStateChange);
+    window.setTimeout(() => {
+      pc.removeEventListener("icegatheringstatechange", onStateChange);
+      resolve();
+    }, 2e3);
+  });
+}
+function splitContext(text) {
+  const chunks = [];
+  for (let offset = 0; offset < text.length; offset += 1800) {
+    chunks.push(text.slice(offset, offset + 1800));
+  }
+  return chunks.length ? chunks : [""];
+}
+var TRANSCRIPT_HISTORY_MAX = 30;
+function useLiveVoiceConversation(options) {
+  const { mintSession } = options;
+  const [status, setStatus] = (0, import_react3.useState)("disconnected");
+  const [isSpeaking, setIsSpeaking] = (0, import_react3.useState)(false);
+  const [isListening, setIsListening] = (0, import_react3.useState)(false);
+  const [isMuted, setIsMuted] = (0, import_react3.useState)(false);
+  const [sessionId, setSessionId] = (0, import_react3.useState)(null);
+  const optionsRef = (0, import_react3.useRef)(options);
+  optionsRef.current = options;
+  const pcRef = (0, import_react3.useRef)(null);
+  const dataChannelRef = (0, import_react3.useRef)(null);
+  const mediaStreamRef = (0, import_react3.useRef)(null);
+  const audioRef = (0, import_react3.useRef)(null);
+  const analyserRef = (0, import_react3.useRef)(null);
+  const frequencyRef = (0, import_react3.useRef)(null);
+  const audioContextRef = (0, import_react3.useRef)(null);
+  const speakingFrameRef = (0, import_react3.useRef)(null);
+  const inputTranscriptRef = (0, import_react3.useRef)("");
+  const outputTranscriptRef = (0, import_react3.useRef)("");
+  const historyRef = (0, import_react3.useRef)([]);
+  const closingRef = (0, import_react3.useRef)(false);
+  const mutedRef = (0, import_react3.useRef)(false);
+  const emit = (0, import_react3.useCallback)((source, message) => {
+    const text = message.trim();
+    if (!text) return;
+    const history = historyRef.current;
+    history.push({ role: source === "user" ? "user" : "assistant", text });
+    if (history.length > TRANSCRIPT_HISTORY_MAX) history.splice(0, history.length - TRANSCRIPT_HISTORY_MAX);
+    optionsRef.current.onMessage?.({ source, message: text });
+  }, []);
+  const flushTranscripts = (0, import_react3.useCallback)(() => {
+    const input = inputTranscriptRef.current.trim();
+    const output = outputTranscriptRef.current.trim();
+    inputTranscriptRef.current = "";
+    outputTranscriptRef.current = "";
+    if (input) emit("user", input);
+    if (output) emit("agent", output);
+  }, [emit]);
+  const sendEvent = (0, import_react3.useCallback)((event) => {
+    const channel = dataChannelRef.current;
+    if (channel?.readyState === "open") channel.send(JSON.stringify(event));
+  }, []);
+  const updateSpeaking = (0, import_react3.useCallback)(() => {
+    const analyser = analyserRef.current;
+    if (analyser) {
+      const data = frequencyRef.current ?? new Uint8Array(analyser.frequencyBinCount);
+      frequencyRef.current = data;
+      analyser.getByteFrequencyData(data);
+      let sum = 0;
+      for (const value of data) sum += value;
+      setIsSpeaking(data.length > 0 && sum / data.length > 8);
+    }
+    if (pcRef.current) speakingFrameRef.current = requestAnimationFrame(updateSpeaking);
+  }, []);
+  const handleEvent = (0, import_react3.useCallback)(async (event) => {
+    const nested = event.type === "response.event" && event.event ? event.event : event;
+    const type = nested.type ?? "";
+    if (type === "session.started" || type === "session.created") {
+      setStatus("connected");
+      setIsListening(!mutedRef.current);
+      const id = typeof nested.session?.id === "string" ? nested.session.id : null;
+      if (id) setSessionId(id);
+    }
+    if (type === "session.closed") {
+      flushTranscripts();
+      setStatus("disconnected");
+      setIsListening(false);
+    }
+    if (type === "error") {
+      optionsRef.current.onError?.(new Error(event.error?.message || "GPT-Live error"));
+    }
+    if (type === "session.input_transcript.delta" || type === "input_audio_transcription.delta") {
+      inputTranscriptRef.current += event.delta ?? "";
+    }
+    if (type === "session.output_transcript.delta" || type === "response.audio_transcript.delta") {
+      outputTranscriptRef.current += event.delta ?? "";
+    }
+    if (type === "session.input_transcript.done" || type === "input_audio_transcription.completed" || type === "session.output_transcript.done" || type === "response.audio_transcript.done" || type === "response.done" || type === "response.completed") {
+      flushTranscripts();
+    }
+    if (type === "session.delegation.created" && nested.delegation?.target !== "responses" && typeof nested.delegation?.id === "string") {
+      const onDelegation = optionsRef.current.onDelegation;
+      if (!onDelegation) return;
+      const delegationId = nested.delegation.id;
+      flushTranscripts();
+      let result;
+      try {
+        result = await onDelegation({ delegationId, transcript: [...historyRef.current] });
+      } catch (error) {
+        optionsRef.current.onError?.(error);
+        result = "That request failed before it could finish. Nothing was confirmed.";
+      }
+      const text = typeof result === "string" ? result.trim() : "";
+      if (!text) return;
+      for (const content of splitContext(text)) {
+        sendEvent({
+          type: "session.commentary.append",
+          delegation_id: delegationId,
+          content
+        });
+      }
+      return;
+    }
+    const item = nested.item;
+    if (type === "response.output_item.done" && item?.type === "function_call") {
+      const tool = optionsRef.current.clientTools?.[item.name ?? ""];
+      if (!tool || !item.call_id) return;
+      let args = {};
+      try {
+        const parsed = JSON.parse(item.arguments || "{}");
+        if (parsed && typeof parsed === "object") args = parsed;
+      } catch {
+      }
+      let output;
+      try {
+        output = String(await tool(args));
+      } catch (error) {
+        output = `Tool failed: ${error instanceof Error ? error.message : String(error)}`;
+      }
+      sendEvent({
+        type: "response.item.create",
+        item: {
+          type: "function_call_output",
+          call_id: item.call_id,
+          output
+        }
+      });
+      sendEvent({ type: "response.create" });
+    }
+  }, [flushTranscripts, sendEvent]);
+  const end = (0, import_react3.useCallback)(() => {
+    closingRef.current = true;
+    flushTranscripts();
+    sendEvent({ type: "session.close" });
+    dataChannelRef.current?.close();
+    pcRef.current?.close();
+    mediaStreamRef.current?.getTracks().forEach((track) => track.stop());
+    audioRef.current?.pause();
+    if (speakingFrameRef.current !== null) cancelAnimationFrame(speakingFrameRef.current);
+    speakingFrameRef.current = null;
+    pcRef.current = null;
+    dataChannelRef.current = null;
+    mediaStreamRef.current = null;
+    analyserRef.current = null;
+    frequencyRef.current = null;
+    audioContextRef.current?.close().catch(() => {
+    });
+    audioContextRef.current = null;
+    setIsSpeaking(false);
+    setIsListening(false);
+    setSessionId(null);
+    setStatus("disconnected");
+  }, [flushTranscripts, sendEvent]);
+  const startInternal = (0, import_react3.useCallback)(async () => {
+    if (status === "connected" || status === "connecting") end();
+    closingRef.current = false;
+    setStatus("connecting");
+    try {
+      const stream = await requestMicStream({ audio: true });
+      mediaStreamRef.current = stream;
+      const pc = new RTCPeerConnection();
+      pcRef.current = pc;
+      stream.getTracks().forEach((track) => pc.addTrack(track, stream));
+      const channel = pc.createDataChannel("oai-events");
+      dataChannelRef.current = channel;
+      channel.onopen = () => {
+        if (!closingRef.current) setIsListening(!mutedRef.current);
+      };
+      channel.onmessage = (message) => {
+        const event = parseEvent(message.data);
+        if (event) void handleEvent(event);
+      };
+      channel.onerror = (event) => optionsRef.current.onError?.(event);
+      pc.ontrack = (event) => {
+        const remoteStream = event.streams[0];
+        if (!remoteStream) return;
+        const audio = new Audio();
+        audio.autoplay = true;
+        audio.srcObject = remoteStream;
+        audioRef.current = audio;
+        void audio.play().catch(() => {
+        });
+        try {
+          const AudioContextImpl = window.AudioContext || window.webkitAudioContext;
+          if (AudioContextImpl) {
+            const context = new AudioContextImpl();
+            const source = context.createMediaStreamSource(remoteStream);
+            const analyser = context.createAnalyser();
+            analyser.fftSize = 256;
+            source.connect(analyser);
+            audioContextRef.current = context;
+            analyserRef.current = analyser;
+            frequencyRef.current = new Uint8Array(analyser.frequencyBinCount);
+            if (speakingFrameRef.current === null) {
+              speakingFrameRef.current = requestAnimationFrame(updateSpeaking);
+            }
+          }
+        } catch {
+        }
+      };
+      pc.onconnectionstatechange = () => {
+        if (["failed", "disconnected", "closed"].includes(pc.connectionState) && !closingRef.current) {
+          optionsRef.current.onError?.(new Error(`GPT-Live connection ${pc.connectionState}`));
+          end();
+        }
+      };
+      const offer = await pc.createOffer();
+      await pc.setLocalDescription(offer);
+      await waitForIceGathering(pc);
+      const localSdp = pc.localDescription?.sdp;
+      if (!localSdp) throw new Error("Failed to create a WebRTC offer");
+      const answer = await mintSession(localSdp);
+      await pc.setRemoteDescription({ type: "answer", sdp: answer.sdp });
+      setSessionId(answer.sessionId);
+      setStatus("connected");
+    } catch (error) {
+      end();
+      optionsRef.current.onError?.(error);
+      throw error;
+    }
+  }, [end, handleEvent, mintSession, status, updateSpeaking]);
+  const start = (0, import_react3.useCallback)(async () => {
+    await startInternal();
+  }, [startInternal]);
+  const restart = (0, import_react3.useCallback)(async () => {
+    end();
+    await new Promise((resolve) => window.setTimeout(resolve, 120));
+    await startInternal();
+  }, [end, startInternal]);
+  const setMuted = (0, import_react3.useCallback)((muted) => {
+    mutedRef.current = muted;
+    mediaStreamRef.current?.getAudioTracks().forEach((track) => {
+      track.enabled = !muted;
+    });
+    sendEvent({ type: muted ? "session.input_audio.mute" : "session.input_audio.unmute" });
+    setIsMuted(muted);
+    setIsListening(!muted && status === "connected");
+  }, [sendEvent, status]);
+  const sendContextualUpdate = (0, import_react3.useCallback)((text) => {
+    for (const content of splitContext(text)) {
+      sendEvent({
+        type: "session.thinking.append",
+        delegation_id: null,
+        content
+      });
+    }
+  }, [sendEvent]);
+  const sendUserMessage = (0, import_react3.useCallback)((text) => {
+    sendEvent({
+      type: "session.commentary.append",
+      delegation_id: null,
+      content: text
+    });
+  }, [sendEvent]);
+  const sendUserActivity = (0, import_react3.useCallback)(() => {
+    sendEvent({ type: "session.input_audio.commit" });
+  }, [sendEvent]);
+  const getOutputByteFrequencyData = (0, import_react3.useCallback)(() => {
+    return frequencyRef.current;
+  }, []);
+  const getTranscript = (0, import_react3.useCallback)(() => [...historyRef.current], []);
+  (0, import_react3.useEffect)(() => () => end(), [end]);
+  return {
+    start,
+    end,
+    restart,
+    status,
+    isSpeaking,
+    isListening,
+    isMuted,
+    setMuted,
+    getOutputByteFrequencyData,
+    sendContextualUpdate,
+    sendUserMessage,
+    sendUserActivity,
+    getTranscript,
+    conversationId: sessionId,
+    convaiConversationId: sessionId
+  };
+}
+
 // src/react/useShogoVoice.ts
 function useShogoVoice(opts = {}) {
   return useVoiceConversation({
@@ -472,8 +809,8 @@ function useShogoVoice(opts = {}) {
 }
 
 // src/react/useChatConversation.ts
-var import_react3 = require("react");
-var import_react4 = require("@ai-sdk/react");
+var import_react4 = require("react");
+var import_react5 = require("@ai-sdk/react");
 var import_ai = require("ai");
 
 // src/shared/chatUrl.ts
@@ -510,16 +847,16 @@ function useChatConversation(options = {}) {
     onError,
     dynamicVariables
   } = options;
-  const resolvedApi = (0, import_react3.useMemo)(
+  const resolvedApi = (0, import_react4.useMemo)(
     () => appendChatQuery(api, { projectId, conversationId, agentName }),
     [api, projectId, conversationId, agentName]
   );
-  const headers = (0, import_react3.useMemo)(() => {
+  const headers = (0, import_react4.useMemo)(() => {
     const h = {};
     if (shogoApiKey) h.authorization = `Bearer ${shogoApiKey}`;
     return h;
   }, [shogoApiKey]);
-  const body = (0, import_react3.useMemo)(() => {
+  const body = (0, import_react4.useMemo)(() => {
     const b = {};
     if (projectId) b.projectId = projectId;
     if (agentName) b.agentName = agentName;
@@ -530,7 +867,7 @@ function useChatConversation(options = {}) {
     }
     return b;
   }, [projectId, agentName, conversationId, tools, dynamicVariables]);
-  const transport = (0, import_react3.useMemo)(
+  const transport = (0, import_react4.useMemo)(
     () => new import_ai.DefaultChatTransport({
       api: resolvedApi,
       credentials: fetchCredentials,
@@ -539,11 +876,11 @@ function useChatConversation(options = {}) {
     }),
     [resolvedApi, fetchCredentials, headers, body]
   );
-  const toolsRef = (0, import_react3.useRef)(clientTools);
+  const toolsRef = (0, import_react4.useRef)(clientTools);
   toolsRef.current = clientTools;
-  const chatRef = (0, import_react3.useRef)(null);
+  const chatRef = (0, import_react4.useRef)(null);
   const resolvedId = id ?? (conversationId ? `shogo-chat:${conversationId}` : DEFAULT_CHAT_ID);
-  const chat = (0, import_react4.useChat)({
+  const chat = (0, import_react5.useChat)({
     id: resolvedId,
     transport,
     messages: initialMessages,
@@ -583,17 +920,17 @@ function useChatConversation(options = {}) {
     }
   });
   chatRef.current = chat;
-  const sendMessage = (0, import_react3.useCallback)(
+  const sendMessage = (0, import_react4.useCallback)(
     async (text) => {
       if (!text.trim()) return;
       await chatRef.current?.sendMessage({ text });
     },
     []
   );
-  const setMessages = (0, import_react3.useCallback)((messages) => {
+  const setMessages = (0, import_react4.useCallback)((messages) => {
     chatRef.current?.setMessages?.(messages);
   }, []);
-  const appendAssistantMessage = (0, import_react3.useCallback)((text) => {
+  const appendAssistantMessage = (0, import_react4.useCallback)((text) => {
     const next = {
       id: syntheticId("synthetic-assistant"),
       role: "assistant",
@@ -604,7 +941,7 @@ function useChatConversation(options = {}) {
     const nextList = [...current.messages, next];
     current.setMessages?.(nextList);
   }, []);
-  const appendUserMessage = (0, import_react3.useCallback)((text) => {
+  const appendUserMessage = (0, import_react4.useCallback)((text) => {
     const next = {
       id: syntheticId("synthetic-user"),
       role: "user",
@@ -637,14 +974,14 @@ function useShogoChat(opts = {}) {
 }
 
 // src/react/ShogoVoiceProvider.tsx
-var import_react5 = require("@elevenlabs/react");
+var import_react6 = require("@elevenlabs/react");
 var import_jsx_runtime = require("react/jsx-runtime");
 function ShogoVoiceProvider(props) {
-  return /* @__PURE__ */ (0, import_jsx_runtime.jsx)(import_react5.ConversationProvider, { children: props.children });
+  return /* @__PURE__ */ (0, import_jsx_runtime.jsx)(import_react6.ConversationProvider, { children: props.children });
 }
 
 // src/react/OrganicSphere.tsx
-var import_react6 = require("react");
+var import_react7 = require("react");
 var THREE = __toESM(require("three"), 1);
 
 // src/react/shaders/organicSphereShaders.ts
@@ -964,8 +1301,8 @@ function OrganicSphere({
   className,
   style
 }) {
-  const containerRef = (0, import_react6.useRef)(null);
-  const resolvedConfig = (0, import_react6.useMemo)(() => {
+  const containerRef = (0, import_react7.useRef)(null);
+  const resolvedConfig = (0, import_react7.useMemo)(() => {
     const base = resolveOrganicSphereConfig(configProp);
     const overrides = {};
     if (lightAColor !== void 0) overrides.lightAColor = lightAColor;
@@ -988,22 +1325,22 @@ function OrganicSphere({
     subdivisions,
     maxPixelRatio
   ]);
-  const configRef = (0, import_react6.useRef)(resolvedConfig);
-  (0, import_react6.useEffect)(() => {
+  const configRef = (0, import_react7.useRef)(resolvedConfig);
+  (0, import_react7.useEffect)(() => {
     configRef.current = resolvedConfig;
   }, [resolvedConfig]);
-  const getFrequencyDataRef = (0, import_react6.useRef)(getFrequencyData);
-  const activeRef = (0, import_react6.useRef)(active);
-  (0, import_react6.useEffect)(() => {
+  const getFrequencyDataRef = (0, import_react7.useRef)(getFrequencyData);
+  const activeRef = (0, import_react7.useRef)(active);
+  (0, import_react7.useEffect)(() => {
     getFrequencyDataRef.current = getFrequencyData;
   }, [getFrequencyData]);
-  (0, import_react6.useEffect)(() => {
+  (0, import_react7.useEffect)(() => {
     activeRef.current = active;
   }, [active]);
   const initSubdivisions = resolvedConfig.subdivisions;
   const initMaxPixelRatio = resolvedConfig.maxPixelRatio;
   const initBackgroundColor = resolvedConfig.backgroundColor;
-  (0, import_react6.useEffect)(() => {
+  (0, import_react7.useEffect)(() => {
     const container = containerRef.current;
     if (!container) return;
     const cfg0 = configRef.current;
@@ -1213,7 +1550,7 @@ function OrganicSphere({
 }
 
 // src/react/OrganicParticles.tsx
-var import_react7 = require("react");
+var import_react8 = require("react");
 var THREE2 = __toESM(require("three"), 1);
 
 // src/react/shaders/organicParticlesShaders.ts
@@ -1428,21 +1765,21 @@ function OrganicParticles({
   className,
   style
 }) {
-  const containerRef = (0, import_react7.useRef)(null);
-  const resolvedConfig = (0, import_react7.useMemo)(
+  const containerRef = (0, import_react8.useRef)(null);
+  const resolvedConfig = (0, import_react8.useMemo)(
     () => resolveOrganicParticlesConfig(configProp),
     [configProp]
   );
-  const configRef = (0, import_react7.useRef)(resolvedConfig);
-  (0, import_react7.useEffect)(() => {
+  const configRef = (0, import_react8.useRef)(resolvedConfig);
+  (0, import_react8.useEffect)(() => {
     configRef.current = resolvedConfig;
   }, [resolvedConfig]);
-  const getFrequencyDataRef = (0, import_react7.useRef)(getFrequencyData);
-  const activeRef = (0, import_react7.useRef)(active);
-  (0, import_react7.useEffect)(() => {
+  const getFrequencyDataRef = (0, import_react8.useRef)(getFrequencyData);
+  const activeRef = (0, import_react8.useRef)(active);
+  (0, import_react8.useEffect)(() => {
     getFrequencyDataRef.current = getFrequencyData;
   }, [getFrequencyData]);
-  (0, import_react7.useEffect)(() => {
+  (0, import_react8.useEffect)(() => {
     activeRef.current = active;
   }, [active]);
   const initCount = resolvedConfig.count;
@@ -1451,7 +1788,7 @@ function OrganicParticles({
   const initSeed = resolvedConfig.seed;
   const initMaxPixelRatio = resolvedConfig.maxPixelRatio;
   const initBackgroundColor = resolvedConfig.backgroundColor;
-  (0, import_react7.useEffect)(() => {
+  (0, import_react8.useEffect)(() => {
     const container = containerRef.current;
     if (!container) return;
     const cfg0 = configRef.current;
@@ -1663,6 +2000,7 @@ function OrganicParticles({
   resolveOrganicParticlesConfig,
   resolveOrganicSphereConfig,
   useChatConversation,
+  useLiveVoiceConversation,
   useShogoChat,
   useShogoVoice,
   useVoiceConversation