@robinandeer/rtc-session-components 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 RunwayML
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,202 @@
+# @runwayml/avatar-react
+
+React SDK for real-time AI avatar interactions with GWM-1.
+
+## Installation
+
+```bash
+npm install @runwayml/avatar-react
+# or
+bun add @runwayml/avatar-react
+```
+
+## Quick Start
+
+```tsx
+import { AvatarProvider, useAvatar, AvatarCanvas } from '@runwayml/avatar-react';
+
+function App() {
+  return (
+    <AvatarProvider config={{ apiKey: process.env.RUNWAYML_API_KEY! }}>
+      <AvatarDemo />
+    </AvatarProvider>
+  );
+}
+
+function AvatarDemo() {
+  const { connect, disconnect, speak, isConnected, isConnecting } = useAvatar();
+
+  const handleConnect = async () => {
+    await connect({ modelId: 'gwm-1' });
+  };
+
+  return (
+    <div>
+      <AvatarCanvas fallback={<div>Not connected</div>} showFps />
+
+      {!isConnected ? (
+        <button onClick={handleConnect} disabled={isConnecting}>
+          {isConnecting ? 'Connecting...' : 'Connect'}
+        </button>
+      ) : (
+        <>
+          <button onClick={() => speak('Hello, world!')}>Say Hello</button>
+          <button onClick={disconnect}>Disconnect</button>
+        </>
+      )}
+    </div>
+  );
+}
+```
+
+## API Reference
+
+### Components
+
+#### `<AvatarProvider>`
+
+Provides the avatar client context to child components.
+
+```tsx
+<AvatarProvider config={{ apiKey: 'your-api-key', debug: true }}>{children}</AvatarProvider>
+```
+
+**Props:**
+
+- `config.apiKey` (required) - Your RunwayML API key
+- `config.baseUrl` - Custom API base URL
+- `config.wsUrl` - Custom WebSocket URL
+- `config.timeout` - Request timeout in ms (default: 30000)
+- `config.maxRetries` - Max retry attempts (default: 2)
+- `config.debug` - Enable debug logging (default: false)
+
+#### `<AvatarCanvas>`
+
+Canvas component that automatically renders avatar video frames.
+
+```tsx
+<AvatarCanvas showFps targetFps={30} fallback={<Placeholder />} style={{ width: '100%' }} />
+```
+
+### Hooks
+
+#### `useAvatar(options?)`
+
+Main hook for managing avatar sessions.
+
+```tsx
+const {
+  session,
+  connectionState,
+  avatarState,
+  isConnected,
+  isConnecting,
+  error,
+  connect,
+  disconnect,
+  speak,
+  sendAudio,
+  setEmotion,
+  setGaze,
+  interrupt,
+} = useAvatar({
+  autoConnect: false,
+  sessionConfig: { modelId: 'gwm-1' },
+  onConnected: () => console.log('Connected!'),
+  onError: (error) => console.error(error),
+});
+```
+
+#### `useAvatarVideo(options?)`
+
+Hook for handling avatar video frames.
+
+```tsx
+const { canvasRef, dimensions, fps, isPlaying, frameCount } = useAvatarVideo({
+  targetFps: 30,
+  autoPlay: true,
+  onFrame: (frame) => console.log('Frame received'),
+});
+```
+
+#### `useAvatarAudio(options?)`
+
+Hook for handling avatar audio output.
+
+```tsx
+const { isPlaying, isMuted, volume, setVolume, toggleMute } = useAvatarAudio({
+  autoPlay: true,
+  volume: 1,
+});
+```
+
+#### `useAvatarMicrophone(options?)`
+
+Hook for capturing and sending microphone audio.
+
+```tsx
+const { isActive, hasPermission, audioLevel, start, stop, toggleMute } = useAvatarMicrophone({
+  sampleRate: 48000,
+  echoCancellation: true,
+  noiseSuppression: true,
+});
+```
+
+### Types
+
+```typescript
+interface AvatarSessionConfig {
+  modelId: string;
+  resolution?: { width: number; height: number };
+  audio?: AudioConfig;
+  initialState?: Partial<AvatarState>;
+}
+
+interface AvatarState {
+  emotion: 'neutral' | 'happy' | 'sad' | 'surprised' | 'angry' | 'confused' | 'thinking';
+  isSpeaking: boolean;
+  isListening: boolean;
+  gazeDirection: { x: number; y: number };
+}
+
+type ConnectionState = 'disconnected' | 'connecting' | 'connected' | 'reconnecting' | 'error';
+```
+
+## Direct Client Usage
+
+For advanced use cases, you can use the client directly:
+
+```typescript
+import { createAvatarClient } from '@runwayml/avatar-react';
+
+const client = createAvatarClient({ apiKey: 'your-api-key' });
+
+await client.createSession({ modelId: 'gwm-1' });
+
+client.on('videoFrame', (event) => {
+  // Handle video frame
+});
+
+client.speak('Hello!');
+client.disconnect();
+```
+
+## Development
+
+```bash
+# Install dependencies
+bun install
+
+# Build
+bun run build
+
+# Type check
+bun run typecheck
+
+# Run tests
+bun test
+```
+
+## License
+
+MIT

package/dist/index.cjs

@@ -0,0 +1,401 @@
+'use strict';
+
+var react = require('react');
+var componentsReact = require('@livekit/components-react');
+var livekitClient = require('livekit-client');
+var jsxRuntime = require('react/jsx-runtime');
+
+// src/components/AvatarSession.tsx
+function mapConnectionState(connectionState) {
+  switch (connectionState) {
+    case livekitClient.ConnectionState.Connecting:
+      return "connecting";
+    case livekitClient.ConnectionState.Connected:
+      return "active";
+    case livekitClient.ConnectionState.Reconnecting:
+      return "connecting";
+    case livekitClient.ConnectionState.Disconnected:
+      return "ended";
+    default:
+      return "ended";
+  }
+}
+var AvatarSessionContext = react.createContext(null);
+function AvatarSession({
+  credentials,
+  children,
+  audio = true,
+  video = true,
+  onEnd,
+  onError
+}) {
+  const errorRef = react.useRef(null);
+  const handleError = (error) => {
+    errorRef.current = error;
+    onError?.(error);
+  };
+  return /* @__PURE__ */ jsxRuntime.jsxs(
+    componentsReact.LiveKitRoom,
+    {
+      serverUrl: credentials.livekitUrl,
+      token: credentials.token,
+      connect: true,
+      audio,
+      video,
+      onDisconnected: () => onEnd?.(),
+      onError: handleError,
+      options: {
+        adaptiveStream: true,
+        dynacast: true
+      },
+      children: [
+        /* @__PURE__ */ jsxRuntime.jsx(
+          AvatarSessionContextInner,
+          {
+            sessionId: credentials.sessionId,
+            onEnd,
+            errorRef,
+            children
+          }
+        ),
+        /* @__PURE__ */ jsxRuntime.jsx(componentsReact.RoomAudioRenderer, {})
+      ]
+    }
+  );
+}
+function AvatarSessionContextInner({
+  sessionId,
+  onEnd,
+  errorRef,
+  children
+}) {
+  const room = componentsReact.useRoomContext();
+  const connectionState = componentsReact.useConnectionState();
+  const onEndRef = react.useRef(onEnd);
+  onEndRef.current = onEnd;
+  const end = react.useCallback(async () => {
+    try {
+      const encoder = new TextEncoder();
+      const data = encoder.encode(JSON.stringify({ type: "END_CALL" }));
+      await room.localParticipant.publishData(data, { reliable: true });
+    } catch {
+    }
+    await room.disconnect();
+    onEndRef.current?.();
+  }, [room]);
+  const contextValue = {
+    state: mapConnectionState(connectionState),
+    sessionId,
+    error: errorRef.current,
+    end
+  };
+  return /* @__PURE__ */ jsxRuntime.jsx(AvatarSessionContext.Provider, { value: contextValue, children });
+}
+function useAvatarSessionContext() {
+  const context = react.useContext(AvatarSessionContext);
+  if (!context) {
+    throw new Error("useAvatarSessionContext must be used within an AvatarSession");
+  }
+  return context;
+}
+function useMaybeAvatarSessionContext() {
+  return react.useContext(AvatarSessionContext);
+}
+function useAvatar() {
+  const remoteParticipants = componentsReact.useRemoteParticipants();
+  const avatarParticipant = remoteParticipants[0] ?? null;
+  const avatarIdentity = avatarParticipant?.identity ?? null;
+  const [isSpeaking, setIsSpeaking] = react.useState(false);
+  react.useEffect(() => {
+    if (!avatarParticipant) {
+      setIsSpeaking(false);
+      return;
+    }
+    setIsSpeaking(avatarParticipant.isSpeaking);
+    const handleIsSpeakingChanged = (speaking) => {
+      setIsSpeaking(speaking);
+    };
+    avatarParticipant.on(livekitClient.ParticipantEvent.IsSpeakingChanged, handleIsSpeakingChanged);
+    return () => {
+      avatarParticipant.off(livekitClient.ParticipantEvent.IsSpeakingChanged, handleIsSpeakingChanged);
+    };
+  }, [avatarParticipant]);
+  const tracks = componentsReact.useTracks(
+    [
+      { source: livekitClient.Track.Source.Camera, withPlaceholder: true },
+      { source: livekitClient.Track.Source.Microphone, withPlaceholder: true }
+    ],
+    { onlySubscribed: true }
+  );
+  let videoTrackRef = null;
+  let audioTrackRef = null;
+  for (const trackRef of tracks) {
+    if (trackRef.participant.identity !== avatarIdentity) continue;
+    if (trackRef.source === livekitClient.Track.Source.Camera && !videoTrackRef) {
+      videoTrackRef = trackRef;
+    } else if (trackRef.source === livekitClient.Track.Source.Microphone && !audioTrackRef) {
+      audioTrackRef = trackRef;
+    }
+    if (videoTrackRef && audioTrackRef) break;
+  }
+  const hasVideo = videoTrackRef !== null && componentsReact.isTrackReference(videoTrackRef);
+  const hasAudio = audioTrackRef !== null && componentsReact.isTrackReference(audioTrackRef);
+  return {
+    participant: avatarParticipant,
+    videoTrackRef,
+    audioTrackRef,
+    isSpeaking,
+    hasVideo,
+    hasAudio
+  };
+}
+
+// src/hooks/useAvatarSession.ts
+function useAvatarSession() {
+  const context = useAvatarSessionContext();
+  return context;
+}
+function AvatarVideo({ children, ...props }) {
+  const session = useAvatarSession();
+  const { videoTrackRef, isSpeaking, hasVideo } = useAvatar();
+  const isConnecting = session.state === "connecting";
+  const state = {
+    hasVideo,
+    isConnecting,
+    isSpeaking,
+    trackRef: videoTrackRef
+  };
+  if (children) {
+    return /* @__PURE__ */ jsxRuntime.jsx(jsxRuntime.Fragment, { children: children(state) });
+  }
+  return /* @__PURE__ */ jsxRuntime.jsx(
+    "div",
+    {
+      ...props,
+      "data-has-video": hasVideo,
+      "data-connecting": isConnecting,
+      "data-speaking": isSpeaking,
+      children: hasVideo && videoTrackRef && componentsReact.isTrackReference(videoTrackRef) && /* @__PURE__ */ jsxRuntime.jsx(componentsReact.VideoTrack, { trackRef: videoTrackRef })
+    }
+  );
+}
+function useLocalMedia() {
+  const { localParticipant } = componentsReact.useLocalParticipant();
+  const audioDevices = componentsReact.useMediaDevices({ kind: "audioinput" });
+  const videoDevices = componentsReact.useMediaDevices({ kind: "videoinput" });
+  const hasMic = audioDevices.length > 0;
+  const hasCamera = videoDevices.length > 0;
+  const isMicEnabled = localParticipant?.isMicrophoneEnabled ?? false;
+  const isCameraEnabled = localParticipant?.isCameraEnabled ?? false;
+  const isScreenShareEnabled = localParticipant?.isScreenShareEnabled ?? false;
+  const isMicEnabledRef = react.useRef(isMicEnabled);
+  const isCameraEnabledRef = react.useRef(isCameraEnabled);
+  const isScreenShareEnabledRef = react.useRef(isScreenShareEnabled);
+  react.useEffect(() => {
+    isMicEnabledRef.current = isMicEnabled;
+  }, [isMicEnabled]);
+  react.useEffect(() => {
+    isCameraEnabledRef.current = isCameraEnabled;
+  }, [isCameraEnabled]);
+  react.useEffect(() => {
+    isScreenShareEnabledRef.current = isScreenShareEnabled;
+  }, [isScreenShareEnabled]);
+  const toggleMic = react.useCallback(() => {
+    localParticipant?.setMicrophoneEnabled(!isMicEnabledRef.current);
+  }, [localParticipant]);
+  const toggleCamera = react.useCallback(() => {
+    localParticipant?.setCameraEnabled(!isCameraEnabledRef.current);
+  }, [localParticipant]);
+  const toggleScreenShare = react.useCallback(() => {
+    localParticipant?.setScreenShareEnabled(!isScreenShareEnabledRef.current);
+  }, [localParticipant]);
+  const tracks = componentsReact.useTracks([{ source: livekitClient.Track.Source.Camera, withPlaceholder: true }], {
+    onlySubscribed: false
+  });
+  const localIdentity = localParticipant?.identity;
+  const localVideoTrackRef = tracks.find(
+    (trackRef) => trackRef.participant.identity === localIdentity && trackRef.source === livekitClient.Track.Source.Camera
+  ) ?? null;
+  return {
+    hasMic,
+    hasCamera,
+    isMicEnabled,
+    isCameraEnabled,
+    isScreenShareEnabled,
+    toggleMic,
+    toggleCamera,
+    toggleScreenShare,
+    localVideoTrackRef
+  };
+}
+function UserVideo({ children, mirror = true, ...props }) {
+  const { localVideoTrackRef, isCameraEnabled } = useLocalMedia();
+  const hasVideo = localVideoTrackRef !== null && componentsReact.isTrackReference(localVideoTrackRef);
+  const state = {
+    hasVideo,
+    isCameraEnabled,
+    trackRef: localVideoTrackRef
+  };
+  if (children) {
+    return /* @__PURE__ */ jsxRuntime.jsx(jsxRuntime.Fragment, { children: children(state) });
+  }
+  return /* @__PURE__ */ jsxRuntime.jsx(
+    "div",
+    {
+      ...props,
+      "data-has-video": hasVideo,
+      "data-camera-enabled": isCameraEnabled,
+      "data-mirror": mirror,
+      children: hasVideo && localVideoTrackRef && componentsReact.isTrackReference(localVideoTrackRef) && /* @__PURE__ */ jsxRuntime.jsx(componentsReact.VideoTrack, { trackRef: localVideoTrackRef })
+    }
+  );
+}
+function ControlBar({
+  children,
+  showMicrophone = true,
+  showCamera = true,
+  showScreenShare = false,
+  showEndCall = true,
+  ...props
+}) {
+  const session = useAvatarSession();
+  const { isMicEnabled, isCameraEnabled, toggleMic, toggleCamera } = useLocalMedia();
+  const isActive = session.state === "active";
+  const state = {
+    isMicEnabled,
+    isCameraEnabled,
+    toggleMic,
+    toggleCamera,
+    endCall: session.end,
+    isActive
+  };
+  if (children) {
+    return /* @__PURE__ */ jsxRuntime.jsx(jsxRuntime.Fragment, { children: children(state) });
+  }
+  if (!isActive) {
+    return null;
+  }
+  return /* @__PURE__ */ jsxRuntime.jsxs("div", { ...props, "data-active": isActive, children: [
+    showMicrophone && /* @__PURE__ */ jsxRuntime.jsx(
+      "button",
+      {
+        type: "button",
+        onClick: toggleMic,
+        "data-control": "microphone",
+        "data-enabled": isMicEnabled,
+        "aria-label": isMicEnabled ? "Mute microphone" : "Unmute microphone",
+        children: /* @__PURE__ */ jsxRuntime.jsx(MicrophoneIcon, {})
+      }
+    ),
+    showCamera && /* @__PURE__ */ jsxRuntime.jsx(
+      "button",
+      {
+        type: "button",
+        onClick: toggleCamera,
+        "data-control": "camera",
+        "data-enabled": isCameraEnabled,
+        "aria-label": isCameraEnabled ? "Turn off camera" : "Turn on camera",
+        children: /* @__PURE__ */ jsxRuntime.jsx(CameraIcon, {})
+      }
+    ),
+    showScreenShare && /* @__PURE__ */ jsxRuntime.jsx(componentsReact.TrackToggle, { source: livekitClient.Track.Source.ScreenShare, showIcon: false, "data-control": "screen-share", children: /* @__PURE__ */ jsxRuntime.jsx(ScreenShareIcon, {}) }),
+    showEndCall && /* @__PURE__ */ jsxRuntime.jsx("button", { type: "button", onClick: session.end, "data-control": "end-call", "aria-label": "End call", children: /* @__PURE__ */ jsxRuntime.jsx("span", { children: "Leave" }) })
+  ] });
+}
+function MicrophoneIcon() {
+  return /* @__PURE__ */ jsxRuntime.jsx("svg", { width: "20", height: "20", fill: "none", viewBox: "0 0 24 24", stroke: "currentColor", "aria-hidden": "true", children: /* @__PURE__ */ jsxRuntime.jsx(
+    "path",
+    {
+      strokeLinecap: "round",
+      strokeLinejoin: "round",
+      strokeWidth: 2,
+      d: "M19 11a7 7 0 01-7 7m0 0a7 7 0 01-7-7m7 7v4m0 0H8m4 0h4m-4-8a3 3 0 01-3-3V5a3 3 0 116 0v6a3 3 0 01-3 3z"
+    }
+  ) });
+}
+function CameraIcon() {
+  return /* @__PURE__ */ jsxRuntime.jsx("svg", { width: "20", height: "20", fill: "none", viewBox: "0 0 24 24", stroke: "currentColor", "aria-hidden": "true", children: /* @__PURE__ */ jsxRuntime.jsx(
+    "path",
+    {
+      strokeLinecap: "round",
+      strokeLinejoin: "round",
+      strokeWidth: 2,
+      d: "M15 10l4.553-2.276A1 1 0 0121 8.618v6.764a1 1 0 01-1.447.894L15 14M5 18h8a2 2 0 002-2V8a2 2 0 00-2-2H5a2 2 0 00-2 2v8a2 2 0 002 2z"
+    }
+  ) });
+}
+function ScreenShareIcon() {
+  return /* @__PURE__ */ jsxRuntime.jsx("svg", { width: "20", height: "20", fill: "none", viewBox: "0 0 24 24", stroke: "currentColor", "aria-hidden": "true", children: /* @__PURE__ */ jsxRuntime.jsx(
+    "path",
+    {
+      strokeLinecap: "round",
+      strokeLinejoin: "round",
+      strokeWidth: 2,
+      d: "M9.75 17L9 20l-1 1h8l-1-1-.75-3M3 13h18M5 17h14a2 2 0 002-2V5a2 2 0 00-2-2H5a2 2 0 00-2 2v10a2 2 0 002 2z"
+    }
+  ) });
+}
+function ScreenShareVideo({ children, ...props }) {
+  const { localParticipant } = componentsReact.useLocalParticipant();
+  const tracks = componentsReact.useTracks(
+    [{ source: livekitClient.Track.Source.ScreenShare, withPlaceholder: false }],
+    { onlySubscribed: false }
+  );
+  const localIdentity = localParticipant?.identity;
+  const screenShareTrackRef = tracks.find(
+    (trackRef) => trackRef.participant.identity === localIdentity && trackRef.source === livekitClient.Track.Source.ScreenShare
+  ) ?? null;
+  const isSharing = screenShareTrackRef !== null && componentsReact.isTrackReference(screenShareTrackRef);
+  const state = {
+    isSharing,
+    trackRef: screenShareTrackRef
+  };
+  if (children) {
+    return /* @__PURE__ */ jsxRuntime.jsx(jsxRuntime.Fragment, { children: children(state) });
+  }
+  if (!isSharing) {
+    return null;
+  }
+  return /* @__PURE__ */ jsxRuntime.jsx("div", { ...props, "data-sharing": isSharing, children: screenShareTrackRef && componentsReact.isTrackReference(screenShareTrackRef) && /* @__PURE__ */ jsxRuntime.jsx(componentsReact.VideoTrack, { trackRef: screenShareTrackRef }) });
+}
+
+// src/api/consume.ts
+var DEFAULT_BASE_URL = "https://api.runwayml.com/v1";
+async function consumeSession(options) {
+  const { sessionId, baseUrl = DEFAULT_BASE_URL } = options;
+  const response = await fetch(`${baseUrl}/realtime/sessions/${sessionId}/consume`, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json"
+    }
+  });
+  if (!response.ok) {
+    const errorText = await response.text();
+    throw new Error(`Failed to consume session: ${response.status} ${errorText}`);
+  }
+  const data = await response.json();
+  return {
+    sessionId,
+    livekitUrl: data.url,
+    token: data.token,
+    roomName: data.roomName
+  };
+}
+
+Object.defineProperty(exports, "AudioRenderer", {
+  enumerable: true,
+  get: function () { return componentsReact.RoomAudioRenderer; }
+});
+exports.AvatarSession = AvatarSession;
+exports.AvatarVideo = AvatarVideo;
+exports.ControlBar = ControlBar;
+exports.ScreenShareVideo = ScreenShareVideo;
+exports.UserVideo = UserVideo;
+exports.consumeSession = consumeSession;
+exports.useAvatar = useAvatar;
+exports.useAvatarSession = useAvatarSession;
+exports.useAvatarSessionContext = useAvatarSessionContext;
+exports.useLocalMedia = useLocalMedia;
+exports.useMaybeAvatarSessionContext = useMaybeAvatarSessionContext;
+//# sourceMappingURL=index.cjs.map
+//# sourceMappingURL=index.cjs.map