@shogo-ai/voice 1.10.0 → 2.4.1

package/dist/chunk-ME3SIZZU.js

@@ -0,0 +1,581 @@
+import {
+  appendTranscriptLine,
+  buildSessionPayload,
+  createMemoryAddTool,
+  createMemoryContextInjector,
+  createPostJson,
+  createTranscriptDisconnectHandler,
+  fetchSignedUrl,
+  withProjectQuery
+} from "./chunk-ZAW6F3YT.js";
+
+// src/react/useVoiceConversation.ts
+import { useCallback, useEffect, useRef, useState } from "react";
+import { useConversation } from "@elevenlabs/react";
+
+// src/react/mic-access.ts
+var MIC_BLOCKED_MESSAGE = "Microphone access is blocked. Allow it in your system or browser settings (on macOS: System Settings > Privacy & Security > Microphone), then try again.";
+var MIC_NOT_FOUND_MESSAGE = "No microphone was found. Connect one and try again.";
+async function requestMicStream(constraints = { audio: true }) {
+  const desktop = globalThis.shogoDesktop;
+  if (desktop?.isDesktop && desktop.ensureMicAccess) {
+    const access = await desktop.ensureMicAccess().catch(() => "granted");
+    if (access !== "granted") throw new Error(MIC_BLOCKED_MESSAGE);
+  }
+  try {
+    return await navigator.mediaDevices.getUserMedia(constraints);
+  } catch (err) {
+    const name = err?.name;
+    if (name === "NotAllowedError" || name === "SecurityError") throw new Error(MIC_BLOCKED_MESSAGE);
+    if (name === "NotFoundError" || name === "OverconstrainedError") throw new Error(MIC_NOT_FOUND_MESSAGE);
+    throw err;
+  }
+}
+
+// src/react/useVoiceConversation.ts
+function useVoiceConversation(options) {
+  const {
+    characterName,
+    signedUrlPath = "/api/voice/signed-url",
+    memoryAddPath = "/api/memory/add",
+    memoryRetrievePath = "/api/memory/retrieve",
+    transcriptIngestPath = "/api/memory/ingest",
+    autoInjectMemory = true,
+    includeMemoryTool = true,
+    clientTools = {},
+    onTranscript,
+    shogoApiKey,
+    projectId,
+    agentName,
+    fetchCredentials = shogoApiKey ? "omit" : "same-origin",
+    onError,
+    onMessage,
+    conversationId: optionConversationId,
+    dynamicVariables
+  } = options;
+  const authHeaders = useCallback(() => {
+    return shogoApiKey ? { authorization: `Bearer ${shogoApiKey}` } : {};
+  }, [shogoApiKey]);
+  const resolvedSignedUrlPath = withProjectQuery(signedUrlPath, { projectId, agentName });
+  const transcriptRef = useRef([]);
+  const lastInjectedRef = useRef("");
+  const weStartedSessionRef = useRef(false);
+  const [convaiConversationId, setConvaiConversationId] = useState(null);
+  const isRestartingRef = useRef(false);
+  const conversationRef = useRef(null);
+  const postJson = useCallback(
+    createPostJson({ authHeaders, fetchCredentials }),
+    [fetchCredentials, authHeaders]
+  );
+  const injectMemoryContext = useCallback(
+    createMemoryContextInjector({
+      postJson,
+      memoryRetrievePath,
+      conversationRef,
+      enabled: autoInjectMemory,
+      lastInjectedRef
+    }),
+    [autoInjectMemory, memoryRetrievePath, postJson]
+  );
+  const handleTranscriptOnDisconnect = useCallback(
+    createTranscriptDisconnectHandler({
+      isRestartingRef,
+      weStartedSessionRef,
+      transcriptRef,
+      onTranscript,
+      transcriptIngestPath,
+      postJson
+    }),
+    [onTranscript, postJson, transcriptIngestPath]
+  );
+  const builtInTools = includeMemoryTool ? { add_memory: createMemoryAddTool({ postJson, memoryAddPath }) } : {};
+  const mergedClientTools = { ...builtInTools, ...clientTools };
+  const conversation = useConversation({
+    clientTools: mergedClientTools,
+    onConnect: (info) => {
+      const id = info?.conversationId;
+      if (typeof id === "string" && id.length > 0) {
+        setConvaiConversationId(id);
+      }
+      if (!isRestartingRef.current) {
+        transcriptRef.current = [];
+        lastInjectedRef.current = "";
+      }
+    },
+    onDisconnect: () => {
+      handleTranscriptOnDisconnect();
+      if (!isRestartingRef.current) {
+        setConvaiConversationId(null);
+      }
+    },
+    onError: (e) => {
+      onError?.(e);
+    },
+    onMessage: (m) => {
+      const msg = m;
+      if (!msg?.message || !msg?.source) return;
+      appendTranscriptLine(msg, { transcriptRef, weStartedSessionRef });
+      onMessage?.({ source: msg.source, message: msg.message });
+      if (autoInjectMemory && msg.source === "user" && msg.message.trim().length >= 3 && weStartedSessionRef.current) {
+        void injectMemoryContext(msg.message);
+      }
+    }
+  });
+  const { status, isSpeaking, isListening, isMuted, setMuted, startSession, endSession } = conversation;
+  conversationRef.current = conversation;
+  useEffect(() => {
+    const onHide = () => {
+      if (!weStartedSessionRef.current || transcriptRef.current.length === 0) return;
+      const transcript = transcriptRef.current.join("\n").trim();
+      transcriptRef.current = [];
+      if (transcript.length < 20) return;
+      try {
+        const blob = new Blob([JSON.stringify({ transcript })], { type: "application/json" });
+        navigator.sendBeacon(transcriptIngestPath, blob);
+      } catch {
+      }
+    };
+    window.addEventListener("pagehide", onHide);
+    return () => window.removeEventListener("pagehide", onHide);
+  }, [transcriptIngestPath]);
+  const startInternal = useCallback(
+    async (opts) => {
+      if (status === "connected" || status === "connecting") {
+        try {
+          endSession();
+          await new Promise((r) => setTimeout(r, 120));
+        } catch {
+        }
+      }
+      const micStream = await requestMicStream({ audio: true });
+      micStream.getTracks().forEach((track) => track.stop());
+      const data = await fetchSignedUrl({
+        path: resolvedSignedUrlPath,
+        fetchCredentials,
+        authHeaders
+      });
+      const ctx = data.userContext || "No prior memories yet.";
+      weStartedSessionRef.current = true;
+      const sessionPayload = buildSessionPayload({
+        signedUrl: data.signedUrl,
+        characterName,
+        userContext: ctx,
+        agentPromptOverride: data.agentPromptOverride,
+        suppressFirstMessage: opts?.suppressFirstMessage,
+        conversationId: optionConversationId,
+        dynamicVariables
+      });
+      await startSession(sessionPayload);
+    },
+    [
+      status,
+      endSession,
+      resolvedSignedUrlPath,
+      fetchCredentials,
+      dynamicVariables,
+      startSession,
+      characterName,
+      authHeaders,
+      optionConversationId
+    ]
+  );
+  const start = useCallback(
+    async (options2) => {
+      await startInternal(options2);
+    },
+    [startInternal]
+  );
+  const end = useCallback(() => {
+    endSession();
+  }, [endSession]);
+  const restart = useCallback(
+    async (options2) => {
+      const suppress = options2?.suppressFirstMessage ?? true;
+      isRestartingRef.current = true;
+      try {
+        try {
+          endSession();
+        } catch {
+        }
+        await new Promise((r) => setTimeout(r, 120));
+        await startInternal({ suppressFirstMessage: suppress });
+      } finally {
+        isRestartingRef.current = false;
+      }
+    },
+    [endSession, startInternal]
+  );
+  const sendContextualUpdate = useCallback((text) => {
+    conversationRef.current?.sendContextualUpdate(text);
+  }, []);
+  const sendUserMessage = useCallback((text) => {
+    const ref = conversationRef.current;
+    if (!ref) return;
+    if (typeof ref.sendUserMessage === "function") {
+      ref.sendUserMessage(text);
+      return;
+    }
+    ref.sendContextualUpdate(text);
+    try {
+      ref.sendUserActivity?.();
+    } catch {
+    }
+  }, []);
+  const sendUserActivity = useCallback(() => {
+    conversationRef.current?.sendUserActivity?.();
+  }, []);
+  const getOutputByteFrequencyData = useCallback(() => {
+    try {
+      const fn = conversation.getOutputByteFrequencyData;
+      return fn ? fn() : null;
+    } catch {
+      return null;
+    }
+  }, [conversation]);
+  return {
+    start,
+    end,
+    restart,
+    status,
+    isSpeaking,
+    isListening,
+    isMuted,
+    setMuted,
+    getOutputByteFrequencyData,
+    sendContextualUpdate,
+    sendUserMessage,
+    sendUserActivity,
+    conversationId: optionConversationId ?? convaiConversationId,
+    convaiConversationId
+  };
+}
+
+// src/react/useLiveVoiceConversation.ts
+import { useCallback as useCallback2, useEffect as useEffect2, useRef as useRef2, useState as useState2 } from "react";
+function parseEvent(data) {
+  try {
+    const text = typeof data === "string" ? data : String(data);
+    const event = JSON.parse(text);
+    return event && typeof event === "object" ? event : null;
+  } catch {
+    return null;
+  }
+}
+function waitForIceGathering(pc) {
+  if (pc.iceGatheringState === "complete") return Promise.resolve();
+  return new Promise((resolve) => {
+    const onStateChange = () => {
+      if (pc.iceGatheringState === "complete") {
+        pc.removeEventListener("icegatheringstatechange", onStateChange);
+        resolve();
+      }
+    };
+    pc.addEventListener("icegatheringstatechange", onStateChange);
+    window.setTimeout(() => {
+      pc.removeEventListener("icegatheringstatechange", onStateChange);
+      resolve();
+    }, 2e3);
+  });
+}
+function splitContext(text) {
+  const chunks = [];
+  for (let offset = 0; offset < text.length; offset += 1800) {
+    chunks.push(text.slice(offset, offset + 1800));
+  }
+  return chunks.length ? chunks : [""];
+}
+var TRANSCRIPT_HISTORY_MAX = 30;
+function useLiveVoiceConversation(options) {
+  const { mintSession } = options;
+  const [status, setStatus] = useState2("disconnected");
+  const [isSpeaking, setIsSpeaking] = useState2(false);
+  const [isListening, setIsListening] = useState2(false);
+  const [isMuted, setIsMuted] = useState2(false);
+  const [sessionId, setSessionId] = useState2(null);
+  const optionsRef = useRef2(options);
+  optionsRef.current = options;
+  const pcRef = useRef2(null);
+  const dataChannelRef = useRef2(null);
+  const mediaStreamRef = useRef2(null);
+  const audioRef = useRef2(null);
+  const analyserRef = useRef2(null);
+  const frequencyRef = useRef2(null);
+  const audioContextRef = useRef2(null);
+  const speakingFrameRef = useRef2(null);
+  const inputTranscriptRef = useRef2("");
+  const outputTranscriptRef = useRef2("");
+  const historyRef = useRef2([]);
+  const closingRef = useRef2(false);
+  const mutedRef = useRef2(false);
+  const emit = useCallback2((source, message) => {
+    const text = message.trim();
+    if (!text) return;
+    const history = historyRef.current;
+    history.push({ role: source === "user" ? "user" : "assistant", text });
+    if (history.length > TRANSCRIPT_HISTORY_MAX) history.splice(0, history.length - TRANSCRIPT_HISTORY_MAX);
+    optionsRef.current.onMessage?.({ source, message: text });
+  }, []);
+  const flushTranscripts = useCallback2(() => {
+    const input = inputTranscriptRef.current.trim();
+    const output = outputTranscriptRef.current.trim();
+    inputTranscriptRef.current = "";
+    outputTranscriptRef.current = "";
+    if (input) emit("user", input);
+    if (output) emit("agent", output);
+  }, [emit]);
+  const sendEvent = useCallback2((event) => {
+    const channel = dataChannelRef.current;
+    if (channel?.readyState === "open") channel.send(JSON.stringify(event));
+  }, []);
+  const updateSpeaking = useCallback2(() => {
+    const analyser = analyserRef.current;
+    if (analyser) {
+      const data = frequencyRef.current ?? new Uint8Array(analyser.frequencyBinCount);
+      frequencyRef.current = data;
+      analyser.getByteFrequencyData(data);
+      let sum = 0;
+      for (const value of data) sum += value;
+      setIsSpeaking(data.length > 0 && sum / data.length > 8);
+    }
+    if (pcRef.current) speakingFrameRef.current = requestAnimationFrame(updateSpeaking);
+  }, []);
+  const handleEvent = useCallback2(async (event) => {
+    const nested = event.type === "response.event" && event.event ? event.event : event;
+    const type = nested.type ?? "";
+    if (type === "session.started" || type === "session.created") {
+      setStatus("connected");
+      setIsListening(!mutedRef.current);
+      const id = typeof nested.session?.id === "string" ? nested.session.id : null;
+      if (id) setSessionId(id);
+    }
+    if (type === "session.closed") {
+      flushTranscripts();
+      setStatus("disconnected");
+      setIsListening(false);
+    }
+    if (type === "error") {
+      optionsRef.current.onError?.(new Error(event.error?.message || "GPT-Live error"));
+    }
+    if (type === "session.input_transcript.delta" || type === "input_audio_transcription.delta") {
+      inputTranscriptRef.current += event.delta ?? "";
+    }
+    if (type === "session.output_transcript.delta" || type === "response.audio_transcript.delta") {
+      outputTranscriptRef.current += event.delta ?? "";
+    }
+    if (type === "session.input_transcript.done" || type === "input_audio_transcription.completed" || type === "session.output_transcript.done" || type === "response.audio_transcript.done" || type === "response.done" || type === "response.completed") {
+      flushTranscripts();
+    }
+    if (type === "session.delegation.created" && nested.delegation?.target !== "responses" && typeof nested.delegation?.id === "string") {
+      const onDelegation = optionsRef.current.onDelegation;
+      if (!onDelegation) return;
+      const delegationId = nested.delegation.id;
+      flushTranscripts();
+      let result;
+      try {
+        result = await onDelegation({ delegationId, transcript: [...historyRef.current] });
+      } catch (error) {
+        optionsRef.current.onError?.(error);
+        result = "That request failed before it could finish. Nothing was confirmed.";
+      }
+      const text = typeof result === "string" ? result.trim() : "";
+      if (!text) return;
+      for (const content of splitContext(text)) {
+        sendEvent({
+          type: "session.commentary.append",
+          delegation_id: delegationId,
+          content
+        });
+      }
+      return;
+    }
+    const item = nested.item;
+    if (type === "response.output_item.done" && item?.type === "function_call") {
+      const tool = optionsRef.current.clientTools?.[item.name ?? ""];
+      if (!tool || !item.call_id) return;
+      let args = {};
+      try {
+        const parsed = JSON.parse(item.arguments || "{}");
+        if (parsed && typeof parsed === "object") args = parsed;
+      } catch {
+      }
+      let output;
+      try {
+        output = String(await tool(args));
+      } catch (error) {
+        output = `Tool failed: ${error instanceof Error ? error.message : String(error)}`;
+      }
+      sendEvent({
+        type: "response.item.create",
+        item: {
+          type: "function_call_output",
+          call_id: item.call_id,
+          output
+        }
+      });
+      sendEvent({ type: "response.create" });
+    }
+  }, [flushTranscripts, sendEvent]);
+  const end = useCallback2(() => {
+    closingRef.current = true;
+    flushTranscripts();
+    sendEvent({ type: "session.close" });
+    dataChannelRef.current?.close();
+    pcRef.current?.close();
+    mediaStreamRef.current?.getTracks().forEach((track) => track.stop());
+    audioRef.current?.pause();
+    if (speakingFrameRef.current !== null) cancelAnimationFrame(speakingFrameRef.current);
+    speakingFrameRef.current = null;
+    pcRef.current = null;
+    dataChannelRef.current = null;
+    mediaStreamRef.current = null;
+    analyserRef.current = null;
+    frequencyRef.current = null;
+    audioContextRef.current?.close().catch(() => {
+    });
+    audioContextRef.current = null;
+    setIsSpeaking(false);
+    setIsListening(false);
+    setSessionId(null);
+    setStatus("disconnected");
+  }, [flushTranscripts, sendEvent]);
+  const startInternal = useCallback2(async () => {
+    if (status === "connected" || status === "connecting") end();
+    closingRef.current = false;
+    setStatus("connecting");
+    try {
+      const stream = await requestMicStream({ audio: true });
+      mediaStreamRef.current = stream;
+      const pc = new RTCPeerConnection();
+      pcRef.current = pc;
+      stream.getTracks().forEach((track) => pc.addTrack(track, stream));
+      const channel = pc.createDataChannel("oai-events");
+      dataChannelRef.current = channel;
+      channel.onopen = () => {
+        if (!closingRef.current) setIsListening(!mutedRef.current);
+      };
+      channel.onmessage = (message) => {
+        const event = parseEvent(message.data);
+        if (event) void handleEvent(event);
+      };
+      channel.onerror = (event) => optionsRef.current.onError?.(event);
+      pc.ontrack = (event) => {
+        const remoteStream = event.streams[0];
+        if (!remoteStream) return;
+        const audio = new Audio();
+        audio.autoplay = true;
+        audio.srcObject = remoteStream;
+        audioRef.current = audio;
+        void audio.play().catch(() => {
+        });
+        try {
+          const AudioContextImpl = window.AudioContext || window.webkitAudioContext;
+          if (AudioContextImpl) {
+            const context = new AudioContextImpl();
+            const source = context.createMediaStreamSource(remoteStream);
+            const analyser = context.createAnalyser();
+            analyser.fftSize = 256;
+            source.connect(analyser);
+            audioContextRef.current = context;
+            analyserRef.current = analyser;
+            frequencyRef.current = new Uint8Array(analyser.frequencyBinCount);
+            if (speakingFrameRef.current === null) {
+              speakingFrameRef.current = requestAnimationFrame(updateSpeaking);
+            }
+          }
+        } catch {
+        }
+      };
+      pc.onconnectionstatechange = () => {
+        if (["failed", "disconnected", "closed"].includes(pc.connectionState) && !closingRef.current) {
+          optionsRef.current.onError?.(new Error(`GPT-Live connection ${pc.connectionState}`));
+          end();
+        }
+      };
+      const offer = await pc.createOffer();
+      await pc.setLocalDescription(offer);
+      await waitForIceGathering(pc);
+      const localSdp = pc.localDescription?.sdp;
+      if (!localSdp) throw new Error("Failed to create a WebRTC offer");
+      const answer = await mintSession(localSdp);
+      await pc.setRemoteDescription({ type: "answer", sdp: answer.sdp });
+      setSessionId(answer.sessionId);
+      setStatus("connected");
+    } catch (error) {
+      end();
+      optionsRef.current.onError?.(error);
+      throw error;
+    }
+  }, [end, handleEvent, mintSession, status, updateSpeaking]);
+  const start = useCallback2(async () => {
+    await startInternal();
+  }, [startInternal]);
+  const restart = useCallback2(async () => {
+    end();
+    await new Promise((resolve) => window.setTimeout(resolve, 120));
+    await startInternal();
+  }, [end, startInternal]);
+  const setMuted = useCallback2((muted) => {
+    mutedRef.current = muted;
+    mediaStreamRef.current?.getAudioTracks().forEach((track) => {
+      track.enabled = !muted;
+    });
+    sendEvent({ type: muted ? "session.input_audio.mute" : "session.input_audio.unmute" });
+    setIsMuted(muted);
+    setIsListening(!muted && status === "connected");
+  }, [sendEvent, status]);
+  const sendContextualUpdate = useCallback2((text) => {
+    for (const content of splitContext(text)) {
+      sendEvent({
+        type: "session.thinking.append",
+        delegation_id: null,
+        content
+      });
+    }
+  }, [sendEvent]);
+  const sendUserMessage = useCallback2((text) => {
+    sendEvent({
+      type: "session.commentary.append",
+      delegation_id: null,
+      content: text
+    });
+  }, [sendEvent]);
+  const sendUserActivity = useCallback2(() => {
+    sendEvent({ type: "session.input_audio.commit" });
+  }, [sendEvent]);
+  const getOutputByteFrequencyData = useCallback2(() => {
+    return frequencyRef.current;
+  }, []);
+  const getTranscript = useCallback2(() => [...historyRef.current], []);
+  useEffect2(() => () => end(), [end]);
+  return {
+    start,
+    end,
+    restart,
+    status,
+    isSpeaking,
+    isListening,
+    isMuted,
+    setMuted,
+    getOutputByteFrequencyData,
+    sendContextualUpdate,
+    sendUserMessage,
+    sendUserActivity,
+    getTranscript,
+    conversationId: sessionId,
+    convaiConversationId: sessionId
+  };
+}
+
+// src/react/ShogoVoiceProvider.tsx
+import { ConversationProvider as ElevenLabsConversationProvider } from "@elevenlabs/react";
+import { jsx } from "react/jsx-runtime";
+function ShogoVoiceProvider(props) {
+  return /* @__PURE__ */ jsx(ElevenLabsConversationProvider, { children: props.children });
+}
+
+export {
+  useVoiceConversation,
+  useLiveVoiceConversation,
+  ShogoVoiceProvider
+};
+//# sourceMappingURL=chunk-ME3SIZZU.js.map

package/dist/chunk-TQOBJMEK.js

@@ -0,0 +1,17 @@
+// src/shared/chatUrl.ts
+function appendChatQuery(basePath, params) {
+  const tail = [];
+  for (const [k, v] of Object.entries(params)) {
+    if (typeof v === "string" && v.length > 0) {
+      tail.push(`${k}=${encodeURIComponent(v)}`);
+    }
+  }
+  if (tail.length === 0) return basePath;
+  const sep = basePath.includes("?") ? "&" : "?";
+  return `${basePath}${sep}${tail.join("&")}`;
+}
+
+export {
+  appendChatQuery
+};
+//# sourceMappingURL=chunk-TQOBJMEK.js.map

package/dist/{chunk-T3T4EBWY.js → chunk-UH75UVM2.js}

@@ -1,6 +1,6 @@
 import {
   createVoiceHandlers
-} from "./chunk-O362EJQR.js";
+} from "./chunk-CLF25CZC.js";
 
 // src/route/index.ts
 function defaultHandlers() {
@@ -12,6 +12,10 @@ function createVoiceRoute(config) {
     signedUrl: { GET: (req) => v.signedUrl(req) },
     ttsPreview: { POST: (req) => v.tts(req) },
     music: { POST: (req) => v.music(req) },
+    op: {
+      GET: (req, opId) => v.op(req, opId),
+      POST: (req, opId) => v.op(req, opId)
+    },
     agent: {
       POST: (req) => v.agent.create(req),
       PATCH: (req) => v.agent.patch(req),
@@ -25,4 +29,4 @@ export {
   defaultHandlers,
   createVoiceRoute
 };
-//# sourceMappingURL=chunk-T3T4EBWY.js.map
+//# sourceMappingURL=chunk-UH75UVM2.js.map