@runwayml/avatars-react 0.9.0 → 0.10.0-beta.1

package/dist/index.cjs

@@ -154,53 +154,22 @@ function useLatest(value) {
   }, [value]);
   return ref;
 }
-async function hasMediaDevice(kind, timeoutMs = 1e3) {
+
+// src/utils/parseClientEvent.ts
+function isAckMessage(args) {
+  return "status" in args && args.status === "event_sent";
+}
+function parseClientEvent(payload) {
   try {
-    const timeoutPromise = new Promise(
-      (resolve) => setTimeout(() => resolve(false), timeoutMs)
-    );
-    const checkPromise = navigator.mediaDevices.enumerateDevices().then((devices) => devices.some((device) => device.kind === kind));
-    return await Promise.race([checkPromise, timeoutPromise]);
+    const message = JSON.parse(new TextDecoder().decode(payload));
+    if (message?.type === "client_event" && typeof message.tool === "string" && message.args != null && typeof message.args === "object" && !isAckMessage(message.args)) {
+      return message;
+    }
+    return null;
   } catch {
-    return false;
+    return null;
   }
 }
-function useDeviceAvailability(requestAudio, requestVideo) {
-  const [state, setState] = react.useState({
-    audio: requestAudio,
-    // Optimistically assume devices exist
-    video: requestVideo
-  });
-  react.useEffect(() => {
-    let cancelled = false;
-    async function checkDevices() {
-      const [hasAudio, hasVideo] = await Promise.all([
-        requestAudio ? hasMediaDevice("audioinput") : Promise.resolve(false),
-        requestVideo ? hasMediaDevice("videoinput") : Promise.resolve(false)
-      ]);
-      if (!cancelled) {
-        setState({
-          audio: requestAudio && hasAudio,
-          video: requestVideo && hasVideo
-        });
-      }
-    }
-    checkDevices();
-    return () => {
-      cancelled = true;
-    };
-  }, [requestAudio, requestVideo]);
-  return state;
-}
-var MEDIA_DEVICE_ERROR_NAMES = /* @__PURE__ */ new Set([
-  "NotAllowedError",
-  "NotFoundError",
-  "NotReadableError",
-  "OverconstrainedError"
-]);
-function isMediaDeviceError(error) {
-  return MEDIA_DEVICE_ERROR_NAMES.has(error.name);
-}
 var DEFAULT_ROOM_OPTIONS = {
   adaptiveStream: false,
   dynacast: false
@@ -222,6 +191,7 @@ function mapConnectionState(connectionState) {
 var AvatarSessionContext = react.createContext(
   null
 );
+var MediaDeviceErrorContext = react.createContext(null);
 function AvatarSession({
   credentials,
   children,
@@ -229,16 +199,14 @@ function AvatarSession({
   video: requestVideo = true,
   onEnd,
   onError,
+  onClientEvent,
   initialScreenStream,
   __unstable_roomOptions
 }) {
   const errorRef = react.useRef(null);
-  const deviceAvailability = useDeviceAvailability(requestAudio, requestVideo);
   const handleError = (error) => {
     onError?.(error);
-    if (!isMediaDeviceError(error)) {
-      errorRef.current = error;
-    }
+    errorRef.current = error;
   };
   const roomOptions = {
     ...DEFAULT_ROOM_OPTIONS,
@@ -250,8 +218,8 @@ function AvatarSession({
       serverUrl: credentials.serverUrl,
       token: credentials.token,
       connect: true,
-      audio: deviceAvailability.audio,
-      video: deviceAvailability.video,
+      audio: false,
+      video: false,
       onDisconnected: () => onEnd?.(),
       onError: handleError,
       options: roomOptions,
@@ -263,7 +231,10 @@ function AvatarSession({
           AvatarSessionContextInner,
           {
             sessionId: credentials.sessionId,
+            requestAudio,
+            requestVideo,
             onEnd,
+            onClientEvent,
             errorRef,
             initialScreenStream,
             children
@@ -276,7 +247,10 @@ function AvatarSession({
 }
 function AvatarSessionContextInner({
   sessionId,
+  requestAudio,
+  requestVideo,
   onEnd,
+  onClientEvent,
   errorRef,
   initialScreenStream,
   children
@@ -285,6 +259,8 @@ function AvatarSessionContextInner({
   const connectionState = componentsReact.useConnectionState();
   const onEndRef = react.useRef(onEnd);
   onEndRef.current = onEnd;
+  const onClientEventRef = react.useRef(onClientEvent);
+  onClientEventRef.current = onClientEvent;
   const publishedRef = react.useRef(false);
   react.useEffect(() => {
     if (connectionState !== livekitClient.ConnectionState.Connected) return;
@@ -308,6 +284,76 @@ function AvatarSessionContextInner({
       });
     };
   }, [connectionState, initialScreenStream, room]);
+  const [micError, setMicError] = react.useState(null);
+  const [cameraError, setCameraError] = react.useState(null);
+  const mediaEnabledRef = react.useRef(false);
+  react.useEffect(() => {
+    if (connectionState !== livekitClient.ConnectionState.Connected) return;
+    if (mediaEnabledRef.current) return;
+    mediaEnabledRef.current = true;
+    async function enableMedia() {
+      if (requestAudio) {
+        try {
+          await room.localParticipant.setMicrophoneEnabled(true);
+        } catch (err) {
+          if (err instanceof Error) setMicError(err);
+        }
+      }
+      if (requestVideo) {
+        try {
+          await room.localParticipant.setCameraEnabled(true);
+        } catch (err) {
+          if (err instanceof Error) setCameraError(err);
+        }
+      }
+    }
+    enableMedia();
+  }, [connectionState, room, requestAudio, requestVideo]);
+  react.useEffect(() => {
+    function handleMediaDevicesError(error, kind) {
+      if (kind === "audioinput") {
+        setMicError(error);
+      } else if (kind === "videoinput") {
+        setCameraError(error);
+      }
+    }
+    room.on(livekitClient.RoomEvent.MediaDevicesError, handleMediaDevicesError);
+    return () => {
+      room.off(livekitClient.RoomEvent.MediaDevicesError, handleMediaDevicesError);
+    };
+  }, [room]);
+  const retryMic = react.useCallback(async () => {
+    try {
+      await room.localParticipant.setMicrophoneEnabled(true);
+      setMicError(null);
+    } catch (err) {
+      if (err instanceof Error) setMicError(err);
+    }
+  }, [room]);
+  const retryCamera = react.useCallback(async () => {
+    try {
+      await room.localParticipant.setCameraEnabled(true);
+      setCameraError(null);
+    } catch (err) {
+      if (err instanceof Error) setCameraError(err);
+    }
+  }, [room]);
+  const mediaDeviceErrors = react.useMemo(
+    () => ({ micError, cameraError, retryMic, retryCamera }),
+    [micError, cameraError, retryMic, retryCamera]
+  );
+  react.useEffect(() => {
+    function handleDataReceived(payload) {
+      const event = parseClientEvent(payload);
+      if (event) {
+        onClientEventRef.current?.(event);
+      }
+    }
+    room.on(livekitClient.RoomEvent.DataReceived, handleDataReceived);
+    return () => {
+      room.off(livekitClient.RoomEvent.DataReceived, handleDataReceived);
+    };
+  }, [room]);
   const end = react.useCallback(async () => {
     try {
       const encoder = new TextEncoder();
@@ -324,7 +370,7 @@ function AvatarSessionContextInner({
     error: errorRef.current,
     end
   };
-  return /* @__PURE__ */ jsxRuntime.jsx(AvatarSessionContext.Provider, { value: contextValue, children });
+  return /* @__PURE__ */ jsxRuntime.jsx(AvatarSessionContext.Provider, { value: contextValue, children: /* @__PURE__ */ jsxRuntime.jsx(MediaDeviceErrorContext.Provider, { value: mediaDeviceErrors, children }) });
 }
 function useAvatarSessionContext() {
   const context = react.useContext(AvatarSessionContext);
@@ -335,6 +381,9 @@ function useAvatarSessionContext() {
   }
   return context;
 }
+function useMediaDeviceErrorContext() {
+  return react.useContext(MediaDeviceErrorContext);
+}
 function useAvatar() {
   const remoteParticipants = componentsReact.useRemoteParticipants();
   const avatarParticipant = remoteParticipants[0] ?? null;
@@ -370,7 +419,10 @@ function useAvatarStatus() {
       return { status: "connecting" };
     case "active":
       if (hasVideo && videoTrackRef) {
-        return { status: "ready", videoTrackRef };
+        return {
+          status: "ready",
+          videoTrackRef
+        };
       }
       return { status: "waiting" };
     case "ending":
@@ -397,8 +449,16 @@ function AvatarVideo({ children, ...props }) {
     }
   );
 }
+var NOOP_ASYNC = async () => {
+};
 function useLocalMedia() {
   const { localParticipant } = componentsReact.useLocalParticipant();
+  const {
+    micError = null,
+    cameraError = null,
+    retryMic = NOOP_ASYNC,
+    retryCamera = NOOP_ASYNC
+  } = useMediaDeviceErrorContext() ?? {};
   const audioDevices = componentsReact.useMediaDevices({ kind: "audioinput" });
   const videoDevices = componentsReact.useMediaDevices({ kind: "videoinput" });
   const hasMic = audioDevices?.length > 0;
@@ -444,7 +504,11 @@ function useLocalMedia() {
     toggleMic,
     toggleCamera,
     toggleScreenShare,
-    localVideoTrackRef
+    localVideoTrackRef,
+    micError,
+    cameraError,
+    retryMic,
+    retryCamera
   };
 }
 function ControlBar({
@@ -462,7 +526,11 @@ function ControlBar({
     isScreenShareEnabled,
     toggleMic,
     toggleCamera,
-    toggleScreenShare
+    toggleScreenShare,
+    micError,
+    cameraError,
+    retryMic,
+    retryCamera
   } = useLocalMedia();
   const isActive = session.state === "active";
   const state = {
@@ -473,7 +541,11 @@ function ControlBar({
     toggleCamera,
     toggleScreenShare,
     endCall: session.end,
-    isActive
+    isActive,
+    micError,
+    cameraError,
+    retryMic,
+    retryCamera
   };
   if (children) {
     return /* @__PURE__ */ jsxRuntime.jsx(jsxRuntime.Fragment, { children: children(state) });
@@ -635,6 +707,7 @@ function AvatarCall({
   avatarImageUrl,
   onEnd,
   onError,
+  onClientEvent,
   children,
   initialScreenStream,
   __unstable_roomOptions,
@@ -688,6 +761,7 @@ function AvatarCall({
           video,
           onEnd,
           onError: handleSessionError,
+          onClientEvent,
           initialScreenStream,
           __unstable_roomOptions,
           children: children ?? defaultChildren
@@ -722,6 +796,78 @@ function ScreenShareVideo({
   }
   return /* @__PURE__ */ jsxRuntime.jsx("div", { ...props, "data-avatar-screen-share": "", "data-avatar-sharing": isSharing, children: screenShareTrackRef && componentsReact.isTrackReference(screenShareTrackRef) && /* @__PURE__ */ jsxRuntime.jsx(componentsReact.VideoTrack, { trackRef: screenShareTrackRef }) });
 }
+function useClientEvent(toolName, onEvent) {
+  const room = componentsReact.useRoomContext();
+  const [state, setState] = react.useState(null);
+  const onEventRef = react.useRef(onEvent);
+  onEventRef.current = onEvent;
+  react.useEffect(() => {
+    function handleDataReceived(payload) {
+      const event = parseClientEvent(payload);
+      if (event && event.tool === toolName) {
+        const args = event.args;
+        setState(args);
+        onEventRef.current?.(args);
+      }
+    }
+    room.on(livekitClient.RoomEvent.DataReceived, handleDataReceived);
+    return () => {
+      room.off(livekitClient.RoomEvent.DataReceived, handleDataReceived);
+    };
+  }, [room, toolName]);
+  return state;
+}
+function useClientEvents(handler) {
+  const room = componentsReact.useRoomContext();
+  const handlerRef = react.useRef(handler);
+  handlerRef.current = handler;
+  react.useEffect(() => {
+    function handleDataReceived(payload) {
+      const event = parseClientEvent(payload);
+      if (event) {
+        handlerRef.current(event);
+      }
+    }
+    room.on(livekitClient.RoomEvent.DataReceived, handleDataReceived);
+    return () => {
+      room.off(livekitClient.RoomEvent.DataReceived, handleDataReceived);
+    };
+  }, [room]);
+}
+function useTranscription(handler, options) {
+  const room = componentsReact.useRoomContext();
+  const handlerRef = react.useRef(handler);
+  handlerRef.current = handler;
+  const interimRef = react.useRef(options?.interim ?? false);
+  interimRef.current = options?.interim ?? false;
+  react.useEffect(() => {
+    function handleTranscription(segments, participant) {
+      const identity = participant?.identity ?? "unknown";
+      for (const segment of segments) {
+        if (!interimRef.current && !segment.final) continue;
+        handlerRef.current({
+          id: segment.id,
+          text: segment.text,
+          final: segment.final,
+          participantIdentity: identity
+        });
+      }
+    }
+    room.on(livekitClient.RoomEvent.TranscriptionReceived, handleTranscription);
+    return () => {
+      room.off(livekitClient.RoomEvent.TranscriptionReceived, handleTranscription);
+    };
+  }, [room]);
+}
+
+// src/tools.ts
+function clientTool(name, config) {
+  return {
+    type: "client_event",
+    name,
+    description: config.description
+  };
+}
 
 Object.defineProperty(exports, "AudioRenderer", {
   enumerable: true,
@@ -731,15 +877,23 @@ Object.defineProperty(exports, "VideoTrack", {
   enumerable: true,
   get: function () { return componentsReact.VideoTrack; }
 });
+Object.defineProperty(exports, "isTrackReference", {
+  enumerable: true,
+  get: function () { return componentsReact.isTrackReference; }
+});
 exports.AvatarCall = AvatarCall;
 exports.AvatarSession = AvatarSession;
 exports.AvatarVideo = AvatarVideo;
 exports.ControlBar = ControlBar;
 exports.ScreenShareVideo = ScreenShareVideo;
 exports.UserVideo = UserVideo;
+exports.clientTool = clientTool;
 exports.useAvatar = useAvatar;
 exports.useAvatarSession = useAvatarSession;
 exports.useAvatarStatus = useAvatarStatus;
+exports.useClientEvent = useClientEvent;
+exports.useClientEvents = useClientEvents;
 exports.useLocalMedia = useLocalMedia;
+exports.useTranscription = useTranscription;
 //# sourceMappingURL=index.cjs.map
 //# sourceMappingURL=index.cjs.map