@runwayml/avatars-react 0.8.0 → 0.10.0-beta.0

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 import * as _livekit_components_react from '@livekit/components-react';
-import { TrackReferenceOrPlaceholder } from '@livekit/components-react';
-export { RoomAudioRenderer as AudioRenderer, VideoTrack } from '@livekit/components-react';
+import { TrackReference, TrackReferenceOrPlaceholder } from '@livekit/components-react';
+export { RoomAudioRenderer as AudioRenderer, VideoTrack, isTrackReference } from '@livekit/components-react';
 import * as react_jsx_runtime from 'react/jsx-runtime';
 import * as livekit_client from 'livekit-client';
 import { ComponentPropsWithoutRef, ReactNode } from 'react';
@@ -30,7 +30,7 @@ interface SessionCredentials {
 /**
  * Props for the AvatarSession component
  */
-interface AvatarSessionProps {
+interface AvatarSessionProps<E extends ClientEvent = ClientEvent> {
     /** Connection credentials from Runway API */
     credentials: SessionCredentials;
     /** Children to render inside the session */
@@ -43,6 +43,13 @@ interface AvatarSessionProps {
     onEnd?: () => void;
     /** Callback when an error occurs */
     onError?: (error: Error) => void;
+    /** Callback when a client event is received from the avatar */
+    onClientEvent?: ClientEventHandler<E>;
+    /**
+     * Pre-captured screen share stream (from getDisplayMedia).
+     * When provided, screen sharing activates automatically once the session connects.
+     */
+    initialScreenStream?: MediaStream;
     /**
      * Advanced LiveKit room options. Not part of the public API.
      * @internal
@@ -52,7 +59,7 @@ interface AvatarSessionProps {
 /**
  * Props for the AvatarCall component
  */
-interface AvatarCallProps extends Omit<React.ComponentPropsWithoutRef<'div'>, 'onError'> {
+interface AvatarCallProps<E extends ClientEvent = ClientEvent> extends Omit<React.ComponentPropsWithoutRef<'div'>, 'onError'> {
     /** The avatar ID to connect to */
     avatarId: string;
     /** Session ID (use with sessionKey - package will call consumeSession) */
@@ -77,8 +84,15 @@ interface AvatarCallProps extends Omit<React.ComponentPropsWithoutRef<'div'>, 'o
     onEnd?: () => void;
     /** Callback when an error occurs */
     onError?: (error: Error) => void;
+    /** Callback when a client event is received from the avatar */
+    onClientEvent?: ClientEventHandler<E>;
     /** Custom children - defaults to AvatarVideo + ControlBar if not provided */
     children?: React.ReactNode;
+    /**
+     * Pre-captured screen share stream (from getDisplayMedia).
+     * When provided, screen sharing activates automatically once the session connects.
+     */
+    initialScreenStream?: MediaStream;
     /**
      * Advanced LiveKit room options. Not part of the public API.
      * @internal
@@ -119,8 +133,51 @@ interface UseLocalMediaReturn {
     /** The local video track reference */
     localVideoTrackRef: _livekit_components_react.TrackReferenceOrPlaceholder | null;
 }
+/**
+ * Client event received from the avatar via the data channel.
+ * These are fire-and-forget events triggered by the avatar model.
+ *
+ * @typeParam T - The tool name (defaults to string for untyped usage)
+ * @typeParam A - The args type (defaults to Record<string, unknown>)
+ *
+ * @example
+ * ```typescript
+ * // Untyped usage
+ * const event: ClientEvent = { type: 'client_event', tool: 'show_caption', args: { text: 'Hello' } };
+ *
+ * // Typed usage with discriminated union
+ * type MyEvent = ClientEvent<'show_caption', { text: string }>;
+ * ```
+ */
+interface ClientEvent<T extends string = string, A = Record<string, unknown>> {
+    type: 'client_event';
+    tool: T;
+    args: A;
+}
+/**
+ * Handler function for client events
+ */
+type ClientEventHandler<E extends ClientEvent = ClientEvent> = (event: E) => void;
+/**
+ * A transcription segment received from the session.
+ * SDK-owned type wrapping the underlying transport's transcription data.
+ */
+interface TranscriptionEntry {
+    /** Unique segment identifier */
+    id: string;
+    /** Transcribed text */
+    text: string;
+    /** Whether this is a final (non-streaming) segment */
+    final: boolean;
+    /** Identity of the participant who spoke */
+    participantIdentity: string;
+}
+/**
+ * Handler function for transcription events
+ */
+type TranscriptionHandler = (entry: TranscriptionEntry) => void;
 
-declare function AvatarCall({ avatarId, sessionId, sessionKey, credentials: directCredentials, connectUrl, connect, baseUrl, audio, video, avatarImageUrl, onEnd, onError, children, __unstable_roomOptions, ...props }: AvatarCallProps): react_jsx_runtime.JSX.Element;
+declare function AvatarCall<E extends ClientEvent = ClientEvent>({ avatarId, sessionId, sessionKey, credentials: directCredentials, connectUrl, connect, baseUrl, audio, video, avatarImageUrl, onEnd, onError, onClientEvent, children, initialScreenStream, __unstable_roomOptions, ...props }: AvatarCallProps<E>): react_jsx_runtime.JSX.Element;
 
 /**
  * AvatarSession component - the main entry point for avatar sessions
@@ -128,7 +185,7 @@ declare function AvatarCall({ avatarId, sessionId, sessionKey, credentials: dire
  * Establishes a WebRTC connection and provides session state to children.
  * This is a headless component that renders minimal DOM.
  */
-declare function AvatarSession({ credentials, children, audio: requestAudio, video: requestVideo, onEnd, onError, __unstable_roomOptions, }: AvatarSessionProps): react_jsx_runtime.JSX.Element;
+declare function AvatarSession<E extends ClientEvent = ClientEvent>({ credentials, children, audio: requestAudio, video: requestVideo, onEnd, onError, onClientEvent, initialScreenStream, __unstable_roomOptions, }: AvatarSessionProps<E>): react_jsx_runtime.JSX.Element;
 
 /**
  * useAvatarStatus Hook
@@ -166,7 +223,7 @@ type AvatarStatus = {
     status: 'waiting';
 } | {
     status: 'ready';
-    videoTrackRef: TrackReferenceOrPlaceholder;
+    videoTrackRef: TrackReference;
 } | {
     status: 'ending';
 } | {
@@ -282,6 +339,70 @@ type UseAvatarSessionReturn = {
  */
 declare function useAvatarSession(): UseAvatarSessionReturn;
 
+type EventArgs<E extends ClientEvent, T extends E['tool']> = Extract<E, {
+    tool: T;
+}>['args'];
+/**
+ * Subscribe to a single client event type by tool name.
+ *
+ * Returns the latest args as React state (`null` before the first event),
+ * and optionally fires a callback on each event for side effects.
+ *
+ * Must be used within an AvatarSession or AvatarCall component.
+ *
+ * @example
+ * ```tsx
+ * // State only — returns latest args
+ * const score = useClientEvent<TriviaEvent, 'update_score'>('update_score');
+ * // score: { score: number; streak: number } | null
+ *
+ * // State + side effect
+ * const result = useClientEvent<TriviaEvent, 'reveal_answer'>('reveal_answer', (args) => {
+ *   if (args.correct) fireConfetti();
+ * });
+ *
+ * // Side effect only — ignore the return value
+ * useClientEvent<TriviaEvent, 'play_sound'>('play_sound', (args) => {
+ *   new Audio(SOUNDS[args.sound]).play();
+ * });
+ * ```
+ */
+declare function useClientEvent<E extends ClientEvent, T extends E['tool']>(toolName: T, onEvent?: (args: EventArgs<E, T>) => void): EventArgs<E, T> | null;
+
+/**
+ * Hook to listen for all client events from the avatar.
+ *
+ * Use this hook in child components to handle client events without prop drilling.
+ * Must be used within an AvatarSession or AvatarCall component.
+ *
+ * @typeParam E - The expected event type (defaults to ClientEvent for untyped usage)
+ *
+ * @example
+ * ```tsx
+ * // Untyped usage
+ * useClientEvents((event) => {
+ *   console.log('Received:', event.tool, event.args);
+ * });
+ *
+ * // Type-safe usage with discriminated union
+ * type MyEvents =
+ *   | ClientEvent<'show_caption', { text: string }>
+ *   | ClientEvent<'play_sound', { url: string }>;
+ *
+ * useClientEvents<MyEvents>((event) => {
+ *   switch (event.tool) {
+ *     case 'show_caption':
+ *       setCaption(event.args.text); // TypeScript knows this is string
+ *       break;
+ *     case 'play_sound':
+ *       new Audio(event.args.url).play();
+ *       break;
+ *   }
+ * });
+ * ```
+ */
+declare function useClientEvents<E extends ClientEvent = ClientEvent>(handler: ClientEventHandler<E>): void;
+
 /**
  * Hook for local media controls (mic, camera, screen share).
  *
@@ -291,4 +412,81 @@ declare function useAvatarSession(): UseAvatarSessionReturn;
  */
 declare function useLocalMedia(): UseLocalMediaReturn;
 
-export { AvatarCall, type AvatarCallProps, AvatarSession, type AvatarStatus, AvatarVideo, type AvatarVideoStatus, ControlBar, ScreenShareVideo, type SessionCredentials, type SessionState, UserVideo, useAvatar, useAvatarSession, useAvatarStatus, useLocalMedia };
+/**
+ * Hook to listen for transcription events from the session.
+ *
+ * Fires the handler for each transcription segment received. By default,
+ * only final segments are delivered. Pass `{ interim: true }` to also
+ * receive partial/streaming segments.
+ *
+ * Must be used within an AvatarSession or AvatarCall component.
+ *
+ * @example
+ * ```tsx
+ * useTranscription((entry) => {
+ *   console.log(`${entry.participantIdentity}: ${entry.text}`);
+ * });
+ *
+ * // Include interim (non-final) segments
+ * useTranscription((entry) => {
+ *   console.log(entry.final ? 'FINAL' : 'partial', entry.text);
+ * }, { interim: true });
+ * ```
+ */
+declare function useTranscription(handler: TranscriptionHandler, options?: {
+    interim?: boolean;
+}): void;
+
+/**
+ * A standalone client tool definition. Composable — combine into arrays
+ * and derive event types with `ClientEventsFrom`.
+ *
+ * At runtime this is just `{ type, name, description }`. The `Args` generic
+ * is phantom — it only exists at the TypeScript level for type narrowing.
+ */
+interface ClientToolDef<Name extends string = string, Args = unknown> {
+    readonly type: 'client_event';
+    readonly name: Name;
+    readonly description: string;
+    /** @internal phantom field — always `undefined` at runtime */
+    readonly _args?: Args;
+}
+/**
+ * Derive a discriminated union of ClientEvent types from an array of tools.
+ *
+ * @example
+ * ```typescript
+ * const tools = [showQuestion, playSound];
+ * type MyEvent = ClientEventsFrom<typeof tools>;
+ * ```
+ */
+type ClientEventsFrom<T extends ReadonlyArray<ClientToolDef>> = T[number] extends infer U ? U extends ClientToolDef<infer Name, infer Args> ? ClientEvent<Name, Args> : never : never;
+/**
+ * Define a single client tool.
+ *
+ * Returns a standalone object that can be composed into arrays and passed
+ * to `realtimeSessions.create({ tools })`.
+ *
+ * @example
+ * ```typescript
+ * const showQuestion = clientTool('show_question', {
+ *   description: 'Display a trivia question',
+ *   args: {} as { question: string; options: Array<string> },
+ * });
+ *
+ * const playSound = clientTool('play_sound', {
+ *   description: 'Play a sound effect',
+ *   args: {} as { sound: 'correct' | 'incorrect' },
+ * });
+ *
+ * // Combine and derive types
+ * const tools = [showQuestion, playSound];
+ * type MyEvent = ClientEventsFrom<typeof tools>;
+ * ```
+ */
+declare function clientTool<Name extends string, Args>(name: Name, config: {
+    description: string;
+    args: Args;
+}): ClientToolDef<Name, Args>;
+
+export { AvatarCall, type AvatarCallProps, AvatarSession, type AvatarStatus, AvatarVideo, type AvatarVideoStatus, type ClientEvent, type ClientEventHandler, type ClientEventsFrom, type ClientToolDef, ControlBar, ScreenShareVideo, type SessionCredentials, type SessionState, type TranscriptionEntry, type TranscriptionHandler, UserVideo, clientTool, useAvatar, useAvatarSession, useAvatarStatus, useClientEvent, useClientEvents, useLocalMedia, useTranscription };

package/dist/index.js

@@ -1,7 +1,7 @@
 import { LiveKitRoom, RoomAudioRenderer, useRoomContext, useConnectionState, useRemoteParticipants, useTracks, isTrackReference, VideoTrack, useLocalParticipant, useMediaDevices, TrackToggle } from '@livekit/components-react';
-export { RoomAudioRenderer as AudioRenderer, VideoTrack } from '@livekit/components-react';
-import { createContext, useRef, useCallback, useState, useEffect, useContext, useSyncExternalStore } from 'react';
-import { Track, ConnectionState } from 'livekit-client';
+export { RoomAudioRenderer as AudioRenderer, VideoTrack, isTrackReference } from '@livekit/components-react';
+import { createContext, useRef, useEffect, useCallback, useState, useContext, useSyncExternalStore } from 'react';
+import { ConnectionState, Track, RoomEvent } from 'livekit-client';
 import { jsxs, jsx, Fragment } from 'react/jsx-runtime';
 
 // src/api/config.ts
@@ -153,6 +153,22 @@ function useLatest(value) {
   }, [value]);
   return ref;
 }
+
+// src/utils/parseClientEvent.ts
+function isAckMessage(args) {
+  return "status" in args && args.status === "event_sent";
+}
+function parseClientEvent(payload) {
+  try {
+    const message = JSON.parse(new TextDecoder().decode(payload));
+    if (message?.type === "client_event" && typeof message.tool === "string" && message.args != null && typeof message.args === "object" && !isAckMessage(message.args)) {
+      return message;
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
 async function hasMediaDevice(kind, timeoutMs = 1e3) {
   try {
     const timeoutPromise = new Promise(
@@ -228,6 +244,8 @@ function AvatarSession({
   video: requestVideo = true,
   onEnd,
   onError,
+  onClientEvent,
+  initialScreenStream,
   __unstable_roomOptions
 }) {
   const errorRef = useRef(null);
@@ -262,7 +280,9 @@ function AvatarSession({
           {
             sessionId: credentials.sessionId,
             onEnd,
+            onClientEvent,
             errorRef,
+            initialScreenStream,
             children
           }
         ),
@@ -274,13 +294,52 @@ function AvatarSession({
 function AvatarSessionContextInner({
   sessionId,
   onEnd,
+  onClientEvent,
   errorRef,
+  initialScreenStream,
   children
 }) {
   const room = useRoomContext();
   const connectionState = useConnectionState();
   const onEndRef = useRef(onEnd);
   onEndRef.current = onEnd;
+  const onClientEventRef = useRef(onClientEvent);
+  onClientEventRef.current = onClientEvent;
+  const publishedRef = useRef(false);
+  useEffect(() => {
+    if (connectionState !== ConnectionState.Connected) return;
+    if (!initialScreenStream || publishedRef.current) return;
+    publishedRef.current = true;
+    const videoTrack = initialScreenStream.getVideoTracks()[0];
+    if (videoTrack) {
+      room.localParticipant.publishTrack(videoTrack, {
+        source: Track.Source.ScreenShare
+      });
+    }
+    const audioTrack = initialScreenStream.getAudioTracks()[0];
+    if (audioTrack) {
+      room.localParticipant.publishTrack(audioTrack, {
+        source: Track.Source.ScreenShareAudio
+      });
+    }
+    return () => {
+      initialScreenStream.getTracks().forEach((t) => {
+        t.stop();
+      });
+    };
+  }, [connectionState, initialScreenStream, room]);
+  useEffect(() => {
+    function handleDataReceived(payload) {
+      const event = parseClientEvent(payload);
+      if (event) {
+        onClientEventRef.current?.(event);
+      }
+    }
+    room.on(RoomEvent.DataReceived, handleDataReceived);
+    return () => {
+      room.off(RoomEvent.DataReceived, handleDataReceived);
+    };
+  }, [room]);
   const end = useCallback(async () => {
     try {
       const encoder = new TextEncoder();
@@ -343,7 +402,10 @@ function useAvatarStatus() {
       return { status: "connecting" };
     case "active":
       if (hasVideo && videoTrackRef) {
-        return { status: "ready", videoTrackRef };
+        return {
+          status: "ready",
+          videoTrackRef
+        };
       }
       return { status: "waiting" };
     case "ending":
@@ -608,7 +670,9 @@ function AvatarCall({
   avatarImageUrl,
   onEnd,
   onError,
+  onClientEvent,
   children,
+  initialScreenStream,
   __unstable_roomOptions,
   ...props
 }) {
@@ -660,6 +724,8 @@ function AvatarCall({
           video,
           onEnd,
           onError: handleSessionError,
+          onClientEvent,
+          initialScreenStream,
           __unstable_roomOptions,
           children: children ?? defaultChildren
         }
@@ -693,7 +759,79 @@ function ScreenShareVideo({
   }
   return /* @__PURE__ */ jsx("div", { ...props, "data-avatar-screen-share": "", "data-avatar-sharing": isSharing, children: screenShareTrackRef && isTrackReference(screenShareTrackRef) && /* @__PURE__ */ jsx(VideoTrack, { trackRef: screenShareTrackRef }) });
 }
+function useClientEvent(toolName, onEvent) {
+  const room = useRoomContext();
+  const [state, setState] = useState(null);
+  const onEventRef = useRef(onEvent);
+  onEventRef.current = onEvent;
+  useEffect(() => {
+    function handleDataReceived(payload) {
+      const event = parseClientEvent(payload);
+      if (event && event.tool === toolName) {
+        const args = event.args;
+        setState(args);
+        onEventRef.current?.(args);
+      }
+    }
+    room.on(RoomEvent.DataReceived, handleDataReceived);
+    return () => {
+      room.off(RoomEvent.DataReceived, handleDataReceived);
+    };
+  }, [room, toolName]);
+  return state;
+}
+function useClientEvents(handler) {
+  const room = useRoomContext();
+  const handlerRef = useRef(handler);
+  handlerRef.current = handler;
+  useEffect(() => {
+    function handleDataReceived(payload) {
+      const event = parseClientEvent(payload);
+      if (event) {
+        handlerRef.current(event);
+      }
+    }
+    room.on(RoomEvent.DataReceived, handleDataReceived);
+    return () => {
+      room.off(RoomEvent.DataReceived, handleDataReceived);
+    };
+  }, [room]);
+}
+function useTranscription(handler, options) {
+  const room = useRoomContext();
+  const handlerRef = useRef(handler);
+  handlerRef.current = handler;
+  const interimRef = useRef(options?.interim ?? false);
+  interimRef.current = options?.interim ?? false;
+  useEffect(() => {
+    function handleTranscription(segments, participant) {
+      const identity = participant?.identity ?? "unknown";
+      for (const segment of segments) {
+        if (!interimRef.current && !segment.final) continue;
+        handlerRef.current({
+          id: segment.id,
+          text: segment.text,
+          final: segment.final,
+          participantIdentity: identity
+        });
+      }
+    }
+    room.on(RoomEvent.TranscriptionReceived, handleTranscription);
+    return () => {
+      room.off(RoomEvent.TranscriptionReceived, handleTranscription);
+    };
+  }, [room]);
+}
+
+// src/tools.ts
+function clientTool(name, config) {
+  return {
+    type: "client_event",
+    name,
+    description: config.description
+  };
+}
 
-export { AvatarCall, AvatarSession, AvatarVideo, ControlBar, ScreenShareVideo, UserVideo, useAvatar, useAvatarSession, useAvatarStatus, useLocalMedia };
+export { AvatarCall, AvatarSession, AvatarVideo, ControlBar, ScreenShareVideo, UserVideo, clientTool, useAvatar, useAvatarSession, useAvatarStatus, useClientEvent, useClientEvents, useLocalMedia, useTranscription };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map