@runwayml/avatars-react 0.8.0 → 0.10.0-beta.0

package/README.md

@@ -56,6 +56,7 @@ The styles use CSS custom properties for easy customization:
 
 See [`examples/`](./examples) for complete working examples:
 - [`nextjs`](./examples/nextjs) - Next.js App Router
+- [`nextjs-client-events`](./examples/nextjs-client-events) - Client event tools (trivia game)
 - [`nextjs-server-actions`](./examples/nextjs-server-actions) - Next.js with Server Actions
 - [`react-router`](./examples/react-router) - React Router v7 framework mode
 - [`express`](./examples/express) - Express + Vite
@@ -196,9 +197,96 @@ Style connection states with CSS:
 />
 ```
 
+## Webcam & Screen Sharing
+
+The avatar can see your webcam feed or screen share, enabling visual interactions — show a plant for identification, [hold up a Pokémon card for trivia](https://x.com/technofantasyy/status/2031124673552097412), get [real-time coaching while you play a game](https://x.com/iamneubert/status/2031160102452081046), walk through a presentation, or ask for feedback on a design you're working on.
+
+**Compatibility:** Webcam and screen sharing are supported by all preset avatars and custom avatars that use a preset voice. Custom avatars with a custom voice do not support webcam or screen sharing.
+
+### Webcam
+
+The webcam is enabled by default. The `video` prop controls whether the camera activates on connect, and the `<UserVideo>` component renders the local camera feed:
+
+```tsx
+<AvatarCall avatarId="music-superstar" connectUrl="/api/avatar/connect">
+  <AvatarVideo />
+  <UserVideo />
+  <ControlBar />
+</AvatarCall>
+```
+
+To disable the webcam, set `video={false}`:
+
+```tsx
+<AvatarCall avatarId="music-superstar" connectUrl="/api/avatar/connect" video={false} />
+```
+
+### Screen Sharing
+
+Enable the screen share button by passing `showScreenShare` to `ControlBar`, and use `<ScreenShareVideo>` to display the shared content:
+
+```tsx
+<AvatarCall avatarId="music-superstar" connectUrl="/api/avatar/connect">
+  <AvatarVideo />
+  <ScreenShareVideo />
+  <ControlBar showScreenShare />
+</AvatarCall>
+```
+
+You can also start screen sharing automatically by passing a pre-captured stream via `initialScreenStream`. This is useful when you want to prompt the user for screen share permission before the session connects:
+
+```tsx
+function ScreenShareCall() {
+  const [stream, setStream] = useState<MediaStream | null>(null);
+
+  async function startWithScreenShare() {
+    const mediaStream = await navigator.mediaDevices.getDisplayMedia({ video: true });
+    setStream(mediaStream);
+  }
+
+  if (!stream) {
+    return <button onClick={startWithScreenShare}>Share Screen & Start Call</button>;
+  }
+
+  return (
+    <AvatarCall
+      avatarId="music-superstar"
+      connectUrl="/api/avatar/connect"
+      initialScreenStream={stream}
+    >
+      <AvatarVideo />
+      <ScreenShareVideo />
+      <ControlBar showScreenShare />
+    </AvatarCall>
+  );
+}
+```
+
+### Programmatic Control
+
+Use the `useLocalMedia` hook for full programmatic control over camera and screen sharing:
+
+```tsx
+function MediaControls() {
+  const {
+    isCameraEnabled,
+    isScreenShareEnabled,
+    toggleCamera,
+    toggleScreenShare,
+  } = useLocalMedia();
+
+  return (
+    <div>
+      <button onClick={toggleCamera}>{isCameraEnabled ? 'Hide Camera' : 'Show Camera'}</button>
+      <button onClick={toggleScreenShare}>{isScreenShareEnabled ? 'Stop Sharing' : 'Share Screen'}</button>
+    </div>
+  );
+}
+```
+
 ## Hooks
 
-Use hooks for custom components within an `AvatarCall` or `AvatarSession`:
+Use hooks for custom components within an `AvatarCall` or `AvatarSession`. Also available: `useClientEvent` and `useClientEvents` for [client events](#client-events), and `useTranscription` for real-time transcription.
 
 ### useAvatarSession
 
@@ -233,13 +321,14 @@ function CustomAvatar() {
 
 ### useLocalMedia
 
-Control local camera and microphone:
+Control local camera, microphone, and screen sharing:
 
 ```tsx
 function MediaControls() {
   const {
     isMicEnabled,
     isCameraEnabled,
+    isScreenShareEnabled,
     toggleMic,
     toggleCamera,
     toggleScreenShare,
@@ -249,11 +338,51 @@ function MediaControls() {
     <div>
       <button onClick={toggleMic}>{isMicEnabled ? 'Mute' : 'Unmute'}</button>
       <button onClick={toggleCamera}>{isCameraEnabled ? 'Hide' : 'Show'}</button>
+      <button onClick={toggleScreenShare}>{isScreenShareEnabled ? 'Stop Sharing' : 'Share Screen'}</button>
     </div>
   );
 }
 ```
 
+## Client Events
+
+Avatars can trigger UI events via tool calls sent over the data channel. Define tools, pass them when creating a session, and subscribe on the client:
+
+```ts
+// lib/tools.ts — shared between server and client
+import { clientTool, type ClientEventsFrom } from '@runwayml/avatars-react/api';
+
+export const showCaption = clientTool('show_caption', {
+  description: 'Display a caption overlay',
+  args: {} as { text: string },
+});
+
+export const tools = [showCaption];
+export type MyEvent = ClientEventsFrom<typeof tools>;
+```
+
+```ts
+// Server — pass tools when creating the session
+const { id } = await client.realtimeSessions.create({
+  model: 'gwm1_avatars',
+  avatar: { type: 'custom', avatarId: '...' },
+  tools,
+});
+```
+
+```tsx
+// Client — subscribe to events inside AvatarCall
+import { useClientEvent } from '@runwayml/avatars-react';
+import type { MyEvent } from '@/lib/tools';
+
+function CaptionOverlay() {
+  const caption = useClientEvent<MyEvent, 'show_caption'>('show_caption');
+  return caption ? <p>{caption.text}</p> : null;
+}
+```
+
+See the [`nextjs-client-events`](./examples/nextjs-client-events) example for a full working demo.
+
 ## Advanced: AvatarSession
 
 For full control over session management, use `AvatarSession` directly with pre-fetched credentials:
@@ -285,7 +414,7 @@ function AdvancedUsage({ credentials }) {
 | `AvatarSession` | Low-level wrapper that requires credentials |
 | `AvatarVideo` | Renders the remote avatar video |
 | `UserVideo` | Renders the local user's camera |
-| `ControlBar` | Media control buttons (mic, camera, end call) |
+| `ControlBar` | Media control buttons (mic, camera, screen share, end call) |
 | `ScreenShareVideo` | Renders screen share content |
 | `AudioRenderer` | Handles avatar audio playback |
 

package/dist/api.cjs

@@ -1,5 +1,14 @@
 'use strict';
 
+// src/tools.ts
+function clientTool(name, config) {
+  return {
+    type: "client_event",
+    name,
+    description: config.description
+  };
+}
+
 // src/api/config.ts
 var DEFAULT_BASE_URL = "https://api.dev.runwayml.com";
 
@@ -23,6 +32,7 @@ async function consumeSession(options) {
   return response.json();
 }
 
+exports.clientTool = clientTool;
 exports.consumeSession = consumeSession;
 //# sourceMappingURL=api.cjs.map
 //# sourceMappingURL=api.cjs.map

package/dist/api.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/api/config.ts","../src/api/consume.ts"],"names":[],"mappings":";;;AAAO,IAAM,gBAAA,GAAmB,8BAAA;;;ACGhC,eAAsB,eACpB,OAAA,EACiC;AACjC,EAAA,MAAM,EAAE,SAAA,EAAW,UAAA,EAAY,OAAA,GAAU,kBAAiB,GAAI,OAAA;AAE9D,EAAA,MAAM,GAAA,GAAM,CAAA,EAAG,OAAO,CAAA,sBAAA,EAAyB,SAAS,CAAA,QAAA,CAAA;AACxD,EAAA,MAAM,QAAA,GAAW,MAAM,KAAA,CAAM,GAAA,EAAK;AAAA,IAChC,MAAA,EAAQ,MAAA;AAAA,IACR,OAAA,EAAS;AAAA,MACP,cAAA,EAAgB,kBAAA;AAAA,MAChB,aAAA,EAAe,UAAU,UAAU,CAAA;AAAA;AACrC,GACD,CAAA;AAED,EAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AAChB,IAAA,MAAM,SAAA,GAAY,MAAM,QAAA,CAAS,IAAA,EAAK;AACtC,IAAA,MAAM,IAAI,KAAA;AAAA,MACR,CAAA,2BAAA,EAA8B,QAAA,CAAS,MAAM,CAAA,CAAA,EAAI,SAAS,CAAA;AAAA,KAC5D;AAAA,EACF;AAEA,EAAA,OAAO,SAAS,IAAA,EAAK;AACvB","file":"api.cjs","sourcesContent":["export const DEFAULT_BASE_URL = 'https://api.dev.runwayml.com';\n","import type { ConsumeSessionOptions, ConsumeSessionResponse } from '../types';\nimport { DEFAULT_BASE_URL } from './config';\n\nexport async function consumeSession(\n  options: ConsumeSessionOptions,\n): Promise<ConsumeSessionResponse> {\n  const { sessionId, sessionKey, baseUrl = DEFAULT_BASE_URL } = options;\n\n  const url = `${baseUrl}/v1/realtime_sessions/${sessionId}/consume`;\n  const response = await fetch(url, {\n    method: 'POST',\n    headers: {\n      'Content-Type': 'application/json',\n      Authorization: `Bearer ${sessionKey}`,\n    },\n  });\n\n  if (!response.ok) {\n    const errorText = await response.text();\n    throw new Error(\n      `Failed to consume session: ${response.status} ${errorText}`,\n    );\n  }\n\n  return response.json();\n}\n"]}
+{"version":3,"sources":["../src/tools.ts","../src/api/config.ts","../src/api/consume.ts"],"names":[],"mappings":";;;AAwDO,SAAS,UAAA,CACd,MACA,MAAA,EAC2B;AAC3B,EAAA,OAAO;AAAA,IACL,IAAA,EAAM,cAAA;AAAA,IACN,IAAA;AAAA,IACA,aAAa,MAAA,CAAO;AAAA,GACtB;AACF;;;ACjEO,IAAM,gBAAA,GAAmB,8BAAA;;;ACGhC,eAAsB,eACpB,OAAA,EACiC;AACjC,EAAA,MAAM,EAAE,SAAA,EAAW,UAAA,EAAY,OAAA,GAAU,kBAAiB,GAAI,OAAA;AAE9D,EAAA,MAAM,GAAA,GAAM,CAAA,EAAG,OAAO,CAAA,sBAAA,EAAyB,SAAS,CAAA,QAAA,CAAA;AACxD,EAAA,MAAM,QAAA,GAAW,MAAM,KAAA,CAAM,GAAA,EAAK;AAAA,IAChC,MAAA,EAAQ,MAAA;AAAA,IACR,OAAA,EAAS;AAAA,MACP,cAAA,EAAgB,kBAAA;AAAA,MAChB,aAAA,EAAe,UAAU,UAAU,CAAA;AAAA;AACrC,GACD,CAAA;AAED,EAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AAChB,IAAA,MAAM,SAAA,GAAY,MAAM,QAAA,CAAS,IAAA,EAAK;AACtC,IAAA,MAAM,IAAI,KAAA;AAAA,MACR,CAAA,2BAAA,EAA8B,QAAA,CAAS,MAAM,CAAA,CAAA,EAAI,SAAS,CAAA;AAAA,KAC5D;AAAA,EACF;AAEA,EAAA,OAAO,SAAS,IAAA,EAAK;AACvB","file":"api.cjs","sourcesContent":["import type { ClientEvent } from './types';\n\n/**\n * A standalone client tool definition. Composable — combine into arrays\n * and derive event types with `ClientEventsFrom`.\n *\n * At runtime this is just `{ type, name, description }`. The `Args` generic\n * is phantom — it only exists at the TypeScript level for type narrowing.\n */\nexport interface ClientToolDef<Name extends string = string, Args = unknown> {\n  readonly type: 'client_event';\n  readonly name: Name;\n  readonly description: string;\n  /** @internal phantom field — always `undefined` at runtime */\n  readonly _args?: Args;\n}\n\n/**\n * Derive a discriminated union of ClientEvent types from an array of tools.\n *\n * @example\n * ```typescript\n * const tools = [showQuestion, playSound];\n * type MyEvent = ClientEventsFrom<typeof tools>;\n * ```\n */\nexport type ClientEventsFrom<T extends ReadonlyArray<ClientToolDef>> =\n  T[number] extends infer U\n    ? U extends ClientToolDef<infer Name, infer Args>\n      ? ClientEvent<Name, Args>\n      : never\n    : never;\n\n/**\n * Define a single client tool.\n *\n * Returns a standalone object that can be composed into arrays and passed\n * to `realtimeSessions.create({ tools })`.\n *\n * @example\n * ```typescript\n * const showQuestion = clientTool('show_question', {\n *   description: 'Display a trivia question',\n *   args: {} as { question: string; options: Array<string> },\n * });\n *\n * const playSound = clientTool('play_sound', {\n *   description: 'Play a sound effect',\n *   args: {} as { sound: 'correct' | 'incorrect' },\n * });\n *\n * // Combine and derive types\n * const tools = [showQuestion, playSound];\n * type MyEvent = ClientEventsFrom<typeof tools>;\n * ```\n */\nexport function clientTool<Name extends string, Args>(\n  name: Name,\n  config: { description: string; args: Args },\n): ClientToolDef<Name, Args> {\n  return {\n    type: 'client_event',\n    name,\n    description: config.description,\n  };\n}\n","export const DEFAULT_BASE_URL = 'https://api.dev.runwayml.com';\n","import type { ConsumeSessionOptions, ConsumeSessionResponse } from '../types';\nimport { DEFAULT_BASE_URL } from './config';\n\nexport async function consumeSession(\n  options: ConsumeSessionOptions,\n): Promise<ConsumeSessionResponse> {\n  const { sessionId, sessionKey, baseUrl = DEFAULT_BASE_URL } = options;\n\n  const url = `${baseUrl}/v1/realtime_sessions/${sessionId}/consume`;\n  const response = await fetch(url, {\n    method: 'POST',\n    headers: {\n      'Content-Type': 'application/json',\n      Authorization: `Bearer ${sessionKey}`,\n    },\n  });\n\n  if (!response.ok) {\n    const errorText = await response.text();\n    throw new Error(\n      `Failed to consume session: ${response.status} ${errorText}`,\n    );\n  }\n\n  return response.json();\n}\n"]}

package/dist/api.d.cts

@@ -20,7 +20,80 @@ interface ConsumeSessionOptions {
     /** Optional base URL for the Runway API (defaults to production) */
     baseUrl?: string;
 }
+/**
+ * Client event received from the avatar via the data channel.
+ * These are fire-and-forget events triggered by the avatar model.
+ *
+ * @typeParam T - The tool name (defaults to string for untyped usage)
+ * @typeParam A - The args type (defaults to Record<string, unknown>)
+ *
+ * @example
+ * ```typescript
+ * // Untyped usage
+ * const event: ClientEvent = { type: 'client_event', tool: 'show_caption', args: { text: 'Hello' } };
+ *
+ * // Typed usage with discriminated union
+ * type MyEvent = ClientEvent<'show_caption', { text: string }>;
+ * ```
+ */
+interface ClientEvent<T extends string = string, A = Record<string, unknown>> {
+    type: 'client_event';
+    tool: T;
+    args: A;
+}
+
+/**
+ * A standalone client tool definition. Composable — combine into arrays
+ * and derive event types with `ClientEventsFrom`.
+ *
+ * At runtime this is just `{ type, name, description }`. The `Args` generic
+ * is phantom — it only exists at the TypeScript level for type narrowing.
+ */
+interface ClientToolDef<Name extends string = string, Args = unknown> {
+    readonly type: 'client_event';
+    readonly name: Name;
+    readonly description: string;
+    /** @internal phantom field — always `undefined` at runtime */
+    readonly _args?: Args;
+}
+/**
+ * Derive a discriminated union of ClientEvent types from an array of tools.
+ *
+ * @example
+ * ```typescript
+ * const tools = [showQuestion, playSound];
+ * type MyEvent = ClientEventsFrom<typeof tools>;
+ * ```
+ */
+type ClientEventsFrom<T extends ReadonlyArray<ClientToolDef>> = T[number] extends infer U ? U extends ClientToolDef<infer Name, infer Args> ? ClientEvent<Name, Args> : never : never;
+/**
+ * Define a single client tool.
+ *
+ * Returns a standalone object that can be composed into arrays and passed
+ * to `realtimeSessions.create({ tools })`.
+ *
+ * @example
+ * ```typescript
+ * const showQuestion = clientTool('show_question', {
+ *   description: 'Display a trivia question',
+ *   args: {} as { question: string; options: Array<string> },
+ * });
+ *
+ * const playSound = clientTool('play_sound', {
+ *   description: 'Play a sound effect',
+ *   args: {} as { sound: 'correct' | 'incorrect' },
+ * });
+ *
+ * // Combine and derive types
+ * const tools = [showQuestion, playSound];
+ * type MyEvent = ClientEventsFrom<typeof tools>;
+ * ```
+ */
+declare function clientTool<Name extends string, Args>(name: Name, config: {
+    description: string;
+    args: Args;
+}): ClientToolDef<Name, Args>;
 
 declare function consumeSession(options: ConsumeSessionOptions): Promise<ConsumeSessionResponse>;
 
-export { type ConsumeSessionOptions, type ConsumeSessionResponse, consumeSession };
+export { type ClientEvent, type ClientEventsFrom, type ClientToolDef, type ConsumeSessionOptions, type ConsumeSessionResponse, clientTool, consumeSession };

package/dist/api.d.ts

@@ -20,7 +20,80 @@ interface ConsumeSessionOptions {
     /** Optional base URL for the Runway API (defaults to production) */
     baseUrl?: string;
 }
+/**
+ * Client event received from the avatar via the data channel.
+ * These are fire-and-forget events triggered by the avatar model.
+ *
+ * @typeParam T - The tool name (defaults to string for untyped usage)
+ * @typeParam A - The args type (defaults to Record<string, unknown>)
+ *
+ * @example
+ * ```typescript
+ * // Untyped usage
+ * const event: ClientEvent = { type: 'client_event', tool: 'show_caption', args: { text: 'Hello' } };
+ *
+ * // Typed usage with discriminated union
+ * type MyEvent = ClientEvent<'show_caption', { text: string }>;
+ * ```
+ */
+interface ClientEvent<T extends string = string, A = Record<string, unknown>> {
+    type: 'client_event';
+    tool: T;
+    args: A;
+}
+
+/**
+ * A standalone client tool definition. Composable — combine into arrays
+ * and derive event types with `ClientEventsFrom`.
+ *
+ * At runtime this is just `{ type, name, description }`. The `Args` generic
+ * is phantom — it only exists at the TypeScript level for type narrowing.
+ */
+interface ClientToolDef<Name extends string = string, Args = unknown> {
+    readonly type: 'client_event';
+    readonly name: Name;
+    readonly description: string;
+    /** @internal phantom field — always `undefined` at runtime */
+    readonly _args?: Args;
+}
+/**
+ * Derive a discriminated union of ClientEvent types from an array of tools.
+ *
+ * @example
+ * ```typescript
+ * const tools = [showQuestion, playSound];
+ * type MyEvent = ClientEventsFrom<typeof tools>;
+ * ```
+ */
+type ClientEventsFrom<T extends ReadonlyArray<ClientToolDef>> = T[number] extends infer U ? U extends ClientToolDef<infer Name, infer Args> ? ClientEvent<Name, Args> : never : never;
+/**
+ * Define a single client tool.
+ *
+ * Returns a standalone object that can be composed into arrays and passed
+ * to `realtimeSessions.create({ tools })`.
+ *
+ * @example
+ * ```typescript
+ * const showQuestion = clientTool('show_question', {
+ *   description: 'Display a trivia question',
+ *   args: {} as { question: string; options: Array<string> },
+ * });
+ *
+ * const playSound = clientTool('play_sound', {
+ *   description: 'Play a sound effect',
+ *   args: {} as { sound: 'correct' | 'incorrect' },
+ * });
+ *
+ * // Combine and derive types
+ * const tools = [showQuestion, playSound];
+ * type MyEvent = ClientEventsFrom<typeof tools>;
+ * ```
+ */
+declare function clientTool<Name extends string, Args>(name: Name, config: {
+    description: string;
+    args: Args;
+}): ClientToolDef<Name, Args>;
 
 declare function consumeSession(options: ConsumeSessionOptions): Promise<ConsumeSessionResponse>;
 
-export { type ConsumeSessionOptions, type ConsumeSessionResponse, consumeSession };
+export { type ClientEvent, type ClientEventsFrom, type ClientToolDef, type ConsumeSessionOptions, type ConsumeSessionResponse, clientTool, consumeSession };

package/dist/api.js

@@ -1,3 +1,12 @@
+// src/tools.ts
+function clientTool(name, config) {
+  return {
+    type: "client_event",
+    name,
+    description: config.description
+  };
+}
+
 // src/api/config.ts
 var DEFAULT_BASE_URL = "https://api.dev.runwayml.com";
 
@@ -21,6 +30,6 @@ async function consumeSession(options) {
   return response.json();
 }
 
-export { consumeSession };
+export { clientTool, consumeSession };
 //# sourceMappingURL=api.js.map
 //# sourceMappingURL=api.js.map

package/dist/api.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/api/config.ts","../src/api/consume.ts"],"names":[],"mappings":";AAAO,IAAM,gBAAA,GAAmB,8BAAA;;;ACGhC,eAAsB,eACpB,OAAA,EACiC;AACjC,EAAA,MAAM,EAAE,SAAA,EAAW,UAAA,EAAY,OAAA,GAAU,kBAAiB,GAAI,OAAA;AAE9D,EAAA,MAAM,GAAA,GAAM,CAAA,EAAG,OAAO,CAAA,sBAAA,EAAyB,SAAS,CAAA,QAAA,CAAA;AACxD,EAAA,MAAM,QAAA,GAAW,MAAM,KAAA,CAAM,GAAA,EAAK;AAAA,IAChC,MAAA,EAAQ,MAAA;AAAA,IACR,OAAA,EAAS;AAAA,MACP,cAAA,EAAgB,kBAAA;AAAA,MAChB,aAAA,EAAe,UAAU,UAAU,CAAA;AAAA;AACrC,GACD,CAAA;AAED,EAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AAChB,IAAA,MAAM,SAAA,GAAY,MAAM,QAAA,CAAS,IAAA,EAAK;AACtC,IAAA,MAAM,IAAI,KAAA;AAAA,MACR,CAAA,2BAAA,EAA8B,QAAA,CAAS,MAAM,CAAA,CAAA,EAAI,SAAS,CAAA;AAAA,KAC5D;AAAA,EACF;AAEA,EAAA,OAAO,SAAS,IAAA,EAAK;AACvB","file":"api.js","sourcesContent":["export const DEFAULT_BASE_URL = 'https://api.dev.runwayml.com';\n","import type { ConsumeSessionOptions, ConsumeSessionResponse } from '../types';\nimport { DEFAULT_BASE_URL } from './config';\n\nexport async function consumeSession(\n  options: ConsumeSessionOptions,\n): Promise<ConsumeSessionResponse> {\n  const { sessionId, sessionKey, baseUrl = DEFAULT_BASE_URL } = options;\n\n  const url = `${baseUrl}/v1/realtime_sessions/${sessionId}/consume`;\n  const response = await fetch(url, {\n    method: 'POST',\n    headers: {\n      'Content-Type': 'application/json',\n      Authorization: `Bearer ${sessionKey}`,\n    },\n  });\n\n  if (!response.ok) {\n    const errorText = await response.text();\n    throw new Error(\n      `Failed to consume session: ${response.status} ${errorText}`,\n    );\n  }\n\n  return response.json();\n}\n"]}
+{"version":3,"sources":["../src/tools.ts","../src/api/config.ts","../src/api/consume.ts"],"names":[],"mappings":";AAwDO,SAAS,UAAA,CACd,MACA,MAAA,EAC2B;AAC3B,EAAA,OAAO;AAAA,IACL,IAAA,EAAM,cAAA;AAAA,IACN,IAAA;AAAA,IACA,aAAa,MAAA,CAAO;AAAA,GACtB;AACF;;;ACjEO,IAAM,gBAAA,GAAmB,8BAAA;;;ACGhC,eAAsB,eACpB,OAAA,EACiC;AACjC,EAAA,MAAM,EAAE,SAAA,EAAW,UAAA,EAAY,OAAA,GAAU,kBAAiB,GAAI,OAAA;AAE9D,EAAA,MAAM,GAAA,GAAM,CAAA,EAAG,OAAO,CAAA,sBAAA,EAAyB,SAAS,CAAA,QAAA,CAAA;AACxD,EAAA,MAAM,QAAA,GAAW,MAAM,KAAA,CAAM,GAAA,EAAK;AAAA,IAChC,MAAA,EAAQ,MAAA;AAAA,IACR,OAAA,EAAS;AAAA,MACP,cAAA,EAAgB,kBAAA;AAAA,MAChB,aAAA,EAAe,UAAU,UAAU,CAAA;AAAA;AACrC,GACD,CAAA;AAED,EAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AAChB,IAAA,MAAM,SAAA,GAAY,MAAM,QAAA,CAAS,IAAA,EAAK;AACtC,IAAA,MAAM,IAAI,KAAA;AAAA,MACR,CAAA,2BAAA,EAA8B,QAAA,CAAS,MAAM,CAAA,CAAA,EAAI,SAAS,CAAA;AAAA,KAC5D;AAAA,EACF;AAEA,EAAA,OAAO,SAAS,IAAA,EAAK;AACvB","file":"api.js","sourcesContent":["import type { ClientEvent } from './types';\n\n/**\n * A standalone client tool definition. Composable — combine into arrays\n * and derive event types with `ClientEventsFrom`.\n *\n * At runtime this is just `{ type, name, description }`. The `Args` generic\n * is phantom — it only exists at the TypeScript level for type narrowing.\n */\nexport interface ClientToolDef<Name extends string = string, Args = unknown> {\n  readonly type: 'client_event';\n  readonly name: Name;\n  readonly description: string;\n  /** @internal phantom field — always `undefined` at runtime */\n  readonly _args?: Args;\n}\n\n/**\n * Derive a discriminated union of ClientEvent types from an array of tools.\n *\n * @example\n * ```typescript\n * const tools = [showQuestion, playSound];\n * type MyEvent = ClientEventsFrom<typeof tools>;\n * ```\n */\nexport type ClientEventsFrom<T extends ReadonlyArray<ClientToolDef>> =\n  T[number] extends infer U\n    ? U extends ClientToolDef<infer Name, infer Args>\n      ? ClientEvent<Name, Args>\n      : never\n    : never;\n\n/**\n * Define a single client tool.\n *\n * Returns a standalone object that can be composed into arrays and passed\n * to `realtimeSessions.create({ tools })`.\n *\n * @example\n * ```typescript\n * const showQuestion = clientTool('show_question', {\n *   description: 'Display a trivia question',\n *   args: {} as { question: string; options: Array<string> },\n * });\n *\n * const playSound = clientTool('play_sound', {\n *   description: 'Play a sound effect',\n *   args: {} as { sound: 'correct' | 'incorrect' },\n * });\n *\n * // Combine and derive types\n * const tools = [showQuestion, playSound];\n * type MyEvent = ClientEventsFrom<typeof tools>;\n * ```\n */\nexport function clientTool<Name extends string, Args>(\n  name: Name,\n  config: { description: string; args: Args },\n): ClientToolDef<Name, Args> {\n  return {\n    type: 'client_event',\n    name,\n    description: config.description,\n  };\n}\n","export const DEFAULT_BASE_URL = 'https://api.dev.runwayml.com';\n","import type { ConsumeSessionOptions, ConsumeSessionResponse } from '../types';\nimport { DEFAULT_BASE_URL } from './config';\n\nexport async function consumeSession(\n  options: ConsumeSessionOptions,\n): Promise<ConsumeSessionResponse> {\n  const { sessionId, sessionKey, baseUrl = DEFAULT_BASE_URL } = options;\n\n  const url = `${baseUrl}/v1/realtime_sessions/${sessionId}/consume`;\n  const response = await fetch(url, {\n    method: 'POST',\n    headers: {\n      'Content-Type': 'application/json',\n      Authorization: `Bearer ${sessionKey}`,\n    },\n  });\n\n  if (!response.ok) {\n    const errorText = await response.text();\n    throw new Error(\n      `Failed to consume session: ${response.status} ${errorText}`,\n    );\n  }\n\n  return response.json();\n}\n"]}

package/dist/index.cjs

@@ -154,6 +154,22 @@ function useLatest(value) {
   }, [value]);
   return ref;
 }
+
+// src/utils/parseClientEvent.ts
+function isAckMessage(args) {
+  return "status" in args && args.status === "event_sent";
+}
+function parseClientEvent(payload) {
+  try {
+    const message = JSON.parse(new TextDecoder().decode(payload));
+    if (message?.type === "client_event" && typeof message.tool === "string" && message.args != null && typeof message.args === "object" && !isAckMessage(message.args)) {
+      return message;
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
 async function hasMediaDevice(kind, timeoutMs = 1e3) {
   try {
     const timeoutPromise = new Promise(
@@ -229,6 +245,8 @@ function AvatarSession({
   video: requestVideo = true,
   onEnd,
   onError,
+  onClientEvent,
+  initialScreenStream,
   __unstable_roomOptions
 }) {
   const errorRef = react.useRef(null);
@@ -263,7 +281,9 @@ function AvatarSession({
           {
             sessionId: credentials.sessionId,
             onEnd,
+            onClientEvent,
             errorRef,
+            initialScreenStream,
             children
           }
         ),
@@ -275,13 +295,52 @@ function AvatarSession({
 function AvatarSessionContextInner({
   sessionId,
   onEnd,
+  onClientEvent,
   errorRef,
+  initialScreenStream,
   children
 }) {
   const room = componentsReact.useRoomContext();
   const connectionState = componentsReact.useConnectionState();
   const onEndRef = react.useRef(onEnd);
   onEndRef.current = onEnd;
+  const onClientEventRef = react.useRef(onClientEvent);
+  onClientEventRef.current = onClientEvent;
+  const publishedRef = react.useRef(false);
+  react.useEffect(() => {
+    if (connectionState !== livekitClient.ConnectionState.Connected) return;
+    if (!initialScreenStream || publishedRef.current) return;
+    publishedRef.current = true;
+    const videoTrack = initialScreenStream.getVideoTracks()[0];
+    if (videoTrack) {
+      room.localParticipant.publishTrack(videoTrack, {
+        source: livekitClient.Track.Source.ScreenShare
+      });
+    }
+    const audioTrack = initialScreenStream.getAudioTracks()[0];
+    if (audioTrack) {
+      room.localParticipant.publishTrack(audioTrack, {
+        source: livekitClient.Track.Source.ScreenShareAudio
+      });
+    }
+    return () => {
+      initialScreenStream.getTracks().forEach((t) => {
+        t.stop();
+      });
+    };
+  }, [connectionState, initialScreenStream, room]);
+  react.useEffect(() => {
+    function handleDataReceived(payload) {
+      const event = parseClientEvent(payload);
+      if (event) {
+        onClientEventRef.current?.(event);
+      }
+    }
+    room.on(livekitClient.RoomEvent.DataReceived, handleDataReceived);
+    return () => {
+      room.off(livekitClient.RoomEvent.DataReceived, handleDataReceived);
+    };
+  }, [room]);
   const end = react.useCallback(async () => {
     try {
       const encoder = new TextEncoder();
@@ -344,7 +403,10 @@ function useAvatarStatus() {
       return { status: "connecting" };
     case "active":
       if (hasVideo && videoTrackRef) {
-        return { status: "ready", videoTrackRef };
+        return {
+          status: "ready",
+          videoTrackRef
+        };
       }
       return { status: "waiting" };
     case "ending":
@@ -609,7 +671,9 @@ function AvatarCall({
   avatarImageUrl,
   onEnd,
   onError,
+  onClientEvent,
   children,
+  initialScreenStream,
   __unstable_roomOptions,
   ...props
 }) {
@@ -661,6 +725,8 @@ function AvatarCall({
           video,
           onEnd,
           onError: handleSessionError,
+          onClientEvent,
+          initialScreenStream,
           __unstable_roomOptions,
           children: children ?? defaultChildren
         }
@@ -694,6 +760,78 @@ function ScreenShareVideo({
   }
   return /* @__PURE__ */ jsxRuntime.jsx("div", { ...props, "data-avatar-screen-share": "", "data-avatar-sharing": isSharing, children: screenShareTrackRef && componentsReact.isTrackReference(screenShareTrackRef) && /* @__PURE__ */ jsxRuntime.jsx(componentsReact.VideoTrack, { trackRef: screenShareTrackRef }) });
 }
+function useClientEvent(toolName, onEvent) {
+  const room = componentsReact.useRoomContext();
+  const [state, setState] = react.useState(null);
+  const onEventRef = react.useRef(onEvent);
+  onEventRef.current = onEvent;
+  react.useEffect(() => {
+    function handleDataReceived(payload) {
+      const event = parseClientEvent(payload);
+      if (event && event.tool === toolName) {
+        const args = event.args;
+        setState(args);
+        onEventRef.current?.(args);
+      }
+    }
+    room.on(livekitClient.RoomEvent.DataReceived, handleDataReceived);
+    return () => {
+      room.off(livekitClient.RoomEvent.DataReceived, handleDataReceived);
+    };
+  }, [room, toolName]);
+  return state;
+}
+function useClientEvents(handler) {
+  const room = componentsReact.useRoomContext();
+  const handlerRef = react.useRef(handler);
+  handlerRef.current = handler;
+  react.useEffect(() => {
+    function handleDataReceived(payload) {
+      const event = parseClientEvent(payload);
+      if (event) {
+        handlerRef.current(event);
+      }
+    }
+    room.on(livekitClient.RoomEvent.DataReceived, handleDataReceived);
+    return () => {
+      room.off(livekitClient.RoomEvent.DataReceived, handleDataReceived);
+    };
+  }, [room]);
+}
+function useTranscription(handler, options) {
+  const room = componentsReact.useRoomContext();
+  const handlerRef = react.useRef(handler);
+  handlerRef.current = handler;
+  const interimRef = react.useRef(options?.interim ?? false);
+  interimRef.current = options?.interim ?? false;
+  react.useEffect(() => {
+    function handleTranscription(segments, participant) {
+      const identity = participant?.identity ?? "unknown";
+      for (const segment of segments) {
+        if (!interimRef.current && !segment.final) continue;
+        handlerRef.current({
+          id: segment.id,
+          text: segment.text,
+          final: segment.final,
+          participantIdentity: identity
+        });
+      }
+    }
+    room.on(livekitClient.RoomEvent.TranscriptionReceived, handleTranscription);
+    return () => {
+      room.off(livekitClient.RoomEvent.TranscriptionReceived, handleTranscription);
+    };
+  }, [room]);
+}
+
+// src/tools.ts
+function clientTool(name, config) {
+  return {
+    type: "client_event",
+    name,
+    description: config.description
+  };
+}
 
 Object.defineProperty(exports, "AudioRenderer", {
   enumerable: true,
@@ -703,15 +841,23 @@ Object.defineProperty(exports, "VideoTrack", {
   enumerable: true,
   get: function () { return componentsReact.VideoTrack; }
 });
+Object.defineProperty(exports, "isTrackReference", {
+  enumerable: true,
+  get: function () { return componentsReact.isTrackReference; }
+});
 exports.AvatarCall = AvatarCall;
 exports.AvatarSession = AvatarSession;
 exports.AvatarVideo = AvatarVideo;
 exports.ControlBar = ControlBar;
 exports.ScreenShareVideo = ScreenShareVideo;
 exports.UserVideo = UserVideo;
+exports.clientTool = clientTool;
 exports.useAvatar = useAvatar;
 exports.useAvatarSession = useAvatarSession;
 exports.useAvatarStatus = useAvatarStatus;
+exports.useClientEvent = useClientEvent;
+exports.useClientEvents = useClientEvents;
 exports.useLocalMedia = useLocalMedia;
+exports.useTranscription = useTranscription;
 //# sourceMappingURL=index.cjs.map
 //# sourceMappingURL=index.cjs.map