@runwayml/avatars 0.16.0-beta.0

package/dist/index.d.cts

@@ -0,0 +1,534 @@
+/**
+ * Tiny typed event emitter. No Node.js polyfills, no dependencies.
+ *
+ * @example
+ * ```ts
+ * type Events = { count: [number]; ready: [] };
+ * const ee = new Emitter<Events>();
+ * ee.on('count', (n) => console.log(n));
+ * ee.emit('count', 42);
+ * ```
+ */
+type EventMap = Record<string, Array<unknown>>;
+type Handler<Args extends Array<unknown>> = (...args: Args) => void;
+declare class Emitter<E extends EventMap> {
+    private listeners;
+    on<K extends keyof E>(event: K, handler: Handler<E[K]>): this;
+    once<K extends keyof E>(event: K, handler: Handler<E[K]>): this;
+    off<K extends keyof E>(event: K, handler: Handler<E[K]>): this;
+    protected emit<K extends keyof E>(event: K, ...args: E[K]): void;
+    removeAllListeners(event?: keyof E): this;
+}
+
+/**
+ * @runwayml/avatars Types
+ *
+ * Core types for the avatar SDK. No React or framework dependencies.
+ */
+type SessionState = 'idle' | 'connecting' | 'reconnecting' | 'active' | 'ending' | 'ended' | 'error';
+interface ConsumeSessionResponse {
+    url: string;
+    token: string;
+    roomName: string;
+}
+interface SessionCredentials {
+    sessionId: string;
+    serverUrl: string;
+    token: string;
+    roomName: string;
+}
+interface SessionKeyResponse {
+    sessionId: string;
+    sessionKey: string;
+}
+type ConnectResponse = SessionCredentials | SessionKeyResponse;
+interface ConsumeSessionOptions {
+    sessionId: string;
+    sessionKey: string;
+    baseUrl?: string;
+}
+interface ClientEvent<T extends string = string, A = Record<string, unknown>> {
+    type: 'client_event';
+    tool: T;
+    args: A;
+}
+type ClientEventHandler<E extends ClientEvent = ClientEvent> = (event: E) => void;
+interface TranscriptionEntry {
+    id: string;
+    text: string;
+    final: boolean;
+    participantIdentity: string;
+    channel?: 'native' | 'custom';
+}
+type TranscriptionHandler = (entry: TranscriptionEntry) => void;
+type MicPermissionState = 'pending' | 'granted' | 'denied';
+type ConnectionQuality = 'excellent' | 'good' | 'poor' | 'lost' | 'unknown';
+type ActiveSpeaker = 'user' | 'avatar';
+declare const AvatarEvent: {
+    readonly StateChanged: "stateChanged";
+    readonly Transcript: "transcript";
+    readonly ClientEvent: "clientEvent";
+    readonly Error: "error";
+    readonly AvatarVideoReady: "avatarVideoReady";
+    readonly AvatarAudioReady: "avatarAudioReady";
+    readonly ScreenShareReady: "screenShareReady";
+    readonly LocalVideoReady: "localVideoReady";
+    readonly MediaChanged: "mediaChanged";
+    readonly UserSpeechStarted: "userSpeechStarted";
+    readonly UserSpeechEnded: "userSpeechEnded";
+    readonly AvatarSpeechStarted: "avatarSpeechStarted";
+    readonly AvatarSpeechEnded: "avatarSpeechEnded";
+    readonly ConnectionQualityChanged: "connectionQualityChanged";
+    readonly MicPermissionChanged: "micPermissionChanged";
+    readonly ActiveSpeakersChanged: "activeSpeakersChanged";
+};
+type AvatarEventMap = {
+    [AvatarEvent.StateChanged]: [state: SessionState];
+    [AvatarEvent.Transcript]: [entry: TranscriptionEntry];
+    [AvatarEvent.ClientEvent]: [event: ClientEvent];
+    [AvatarEvent.Error]: [error: Error];
+    [AvatarEvent.AvatarVideoReady]: [track: MediaStreamTrack];
+    [AvatarEvent.AvatarAudioReady]: [track: MediaStreamTrack];
+    [AvatarEvent.ScreenShareReady]: [track: MediaStreamTrack];
+    [AvatarEvent.LocalVideoReady]: [track: MediaStreamTrack];
+    [AvatarEvent.MediaChanged]: [];
+    [AvatarEvent.UserSpeechStarted]: [];
+    [AvatarEvent.UserSpeechEnded]: [];
+    [AvatarEvent.AvatarSpeechStarted]: [];
+    [AvatarEvent.AvatarSpeechEnded]: [];
+    [AvatarEvent.ConnectionQualityChanged]: [quality: ConnectionQuality];
+    [AvatarEvent.MicPermissionChanged]: [state: MicPermissionState];
+    [AvatarEvent.ActiveSpeakersChanged]: [speakers: ReadonlyArray<ActiveSpeaker>];
+};
+interface MediaController {
+    readonly isEnabled: boolean;
+    enable(): Promise<void>;
+    disable(): Promise<void>;
+    toggle(): Promise<void>;
+    setDevice(deviceId: string): Promise<void>;
+}
+interface ScreenShareController {
+    readonly isActive: boolean;
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    toggle(): Promise<void>;
+}
+interface TranscriptOptions {
+    interim?: boolean;
+    bufferSize?: number;
+}
+
+type TranscriptEvents = {
+    update: [entries: ReadonlyArray<TranscriptionEntry>];
+};
+/**
+ * Stateful transcript accumulator. Deduplicates by segment `id`, caps at
+ * `bufferSize`, and emits `update` on every change.
+ *
+ * Handles both LiveKit-style `{ segments: [...] }` and Runway flat-delta
+ * `{ type: "transcription", role, turn, text }` data-channel payloads,
+ * as well as native `TranscriptionReceived` segment arrays.
+ */
+declare class TranscriptAccumulator extends Emitter<TranscriptEvents> {
+    private readonly map;
+    private readonly flatAcc;
+    private readonly interim;
+    private readonly bufferSize;
+    private snapshot;
+    constructor(options?: TranscriptOptions);
+    get entries(): ReadonlyArray<TranscriptionEntry>;
+    /**
+     * Ingest a native transcription segment (from `RoomEvent.TranscriptionReceived`).
+     */
+    ingestNative(segments: ReadonlyArray<{
+        id: string;
+        text: string;
+        final: boolean;
+    }>, participantIdentity: string): void;
+    /**
+     * Ingest a raw data-channel payload (from `RoomEvent.DataReceived`).
+     * Handles both segment arrays and flat deltas.
+     */
+    ingestDataChannel(payload: Uint8Array, participantIdentity: string): void;
+    dispose(): void;
+    private flush;
+}
+
+/**
+ * A live avatar session. Returned by `streamTo` or `connect`.
+ *
+ * Wraps a LiveKit Room behind a clean event-driven API. No LiveKit
+ * types are exposed — tracks are standard `MediaStreamTrack` objects.
+ */
+declare class AvatarSession extends Emitter<AvatarEventMap> {
+    private room;
+    private _sessionId;
+    private _state;
+    private _error;
+    private _micEnabled;
+    private _cameraEnabled;
+    private _screenShareActive;
+    private attachedVideoElement;
+    private attachedAudioElement;
+    private avatarVideoTrack;
+    private avatarAudioTrack;
+    private _localVideoTrack;
+    private autoAudioElement;
+    private flatDeltaAcc;
+    private _userSpeaking;
+    private _avatarSpeaking;
+    private _connectedAt;
+    readonly mic: MediaController;
+    readonly camera: MediaController;
+    readonly screenShare: ScreenShareController;
+    constructor(sessionId: string);
+    get state(): SessionState;
+    get sessionId(): string;
+    get error(): Error | null;
+    get localVideoTrack(): MediaStreamTrack | null;
+    get duration(): number;
+    waitFor<K extends keyof AvatarEventMap>(event: K): Promise<AvatarEventMap[K][0]>;
+    streamTo(element: HTMLVideoElement): void;
+    stopStreaming(): void;
+    attachAudio(element: HTMLAudioElement): void;
+    detachAudio(): void;
+    end(): Promise<void>;
+    transcript(options?: TranscriptOptions): TranscriptAccumulator;
+    onClientEvent<T extends string, A>(toolOrName: {
+        name: T;
+    } | T, handler: (args: A) => void): () => void;
+    /** @internal Called by `streamTo` and `connect` entry points. */
+    _connect(serverUrl: string, token: string, options: {
+        audio?: boolean;
+        video?: boolean;
+    }): Promise<void>;
+    publishScreenShare(stream: MediaStream): Promise<void>;
+    private bindRoomEvents;
+    private reattachTracks;
+    private syncVideoElement;
+    private enableInitialMedia;
+    private setMic;
+    private setCamera;
+    private emitLocalVideoTrack;
+    private setScreenShare;
+    private switchDevice;
+    private autoPlayAudio;
+    private setError;
+    private setState;
+    private cleanup;
+}
+/**
+ * Start an avatar session and stream video to an element.
+ *
+ * Handles the consume call, LiveKit connection, and media setup
+ * in one call. Returns an `AvatarSession` for media controls,
+ * events, and ending the call.
+ *
+ * @example
+ * ```ts
+ * const credentials = await fetch('/api/connect').then(r => r.json());
+ * const session = await streamTo({ credentials, target: document.getElementById('avatar') });
+ *
+ * session.mic.toggle();
+ * session.end();
+ * ```
+ */
+declare function streamTo(options: {
+    credentials: {
+        sessionId: string;
+        sessionKey: string;
+        baseUrl?: string;
+    };
+    target: HTMLVideoElement;
+    audio?: boolean;
+    video?: boolean;
+}): Promise<AvatarSession>;
+/**
+ * Start an avatar session without a video element (headless).
+ *
+ * Use `session.streamTo(element)` later to attach video, or use
+ * this for audio-only sessions.
+ *
+ * @example
+ * ```ts
+ * const credentials = await fetch('/api/connect').then(r => r.json());
+ * const session = await connect({ credentials });
+ * ```
+ */
+declare function connect(options: {
+    credentials: {
+        sessionId: string;
+        sessionKey: string;
+        baseUrl?: string;
+    };
+    audio?: boolean;
+    video?: boolean;
+}): Promise<AvatarSession>;
+
+type AvatarErrorCode = 'CONSUME_FAILED' | 'CONNECTION_FAILED' | 'MEDIA_PERMISSION_DENIED' | 'MEDIA_DEVICE_ERROR' | 'SCREEN_SHARE_FAILED' | 'PUBLISH_FAILED' | 'UNKNOWN';
+declare class AvatarError extends Error {
+    readonly code: AvatarErrorCode;
+    readonly cause?: Error;
+    constructor(code: AvatarErrorCode, message: string, cause?: Error);
+}
+
+declare function consumeSession(options: ConsumeSessionOptions): Promise<ConsumeSessionResponse>;
+
+/**
+ * Standard Schema v1 — a tiny cross-library validation interface implemented
+ * by Zod, Valibot, ArkType, and others. Anything matching this shape plugs
+ * into `clientTool({ schema })`.
+ *
+ * @see https://standardschema.dev/
+ *
+ * We re-declare the spec here (instead of depending on `@standard-schema/spec`)
+ * so the SDK stays dependency-free and server-safe.
+ */
+/** The schema itself — what a `z.object(...)` etc. resolves to. */
+interface StandardSchemaV1<Input = unknown, Output = Input> {
+    readonly '~standard': {
+        readonly version: 1;
+        readonly vendor: string;
+        readonly validate: (value: unknown) => StandardSchemaResult<Output> | Promise<StandardSchemaResult<Output>>;
+        readonly types?: {
+            readonly input: Input;
+            readonly output: Output;
+        };
+    };
+}
+/** Result of calling `schema['~standard'].validate(value)`. */
+type StandardSchemaResult<Output> = {
+    readonly value: Output;
+    readonly issues?: undefined;
+} | {
+    readonly issues: ReadonlyArray<StandardSchemaIssue>;
+};
+/** A single validation failure — part of the error path. */
+interface StandardSchemaIssue {
+    readonly message: string;
+    readonly path?: ReadonlyArray<PropertyKey | {
+        readonly key: PropertyKey;
+    }>;
+}
+type InferSchemaInput<Schema extends StandardSchemaV1> = Schema extends StandardSchemaV1<infer Input, unknown> ? Input : never;
+type InferSchemaOutput<Schema extends StandardSchemaV1> = Schema extends StandardSchemaV1<unknown, infer Output> ? Output : never;
+
+/**
+ * A standalone client tool definition. Composable — combine into arrays
+ * and derive event types with `ClientEventsFrom`.
+ *
+ * At runtime this is just `{ type, name, description }` (exactly what the
+ * Runway session create payload expects). The `Args` generic is phantom —
+ * it only exists at the TypeScript level for type narrowing.
+ *
+ * When the tool is defined with a `schema` ([Standard Schema](https://standardschema.dev/)),
+ * the schema is attached internally (not serialized) for runtime validation
+ * and to infer `Args` from the schema output type.
+ */
+interface ClientToolDef<Name extends string = string, Args = unknown> {
+    readonly type: 'client_event';
+    readonly name: Name;
+    readonly description: string;
+    /** @internal phantom field — always `undefined` at runtime */
+    readonly _args?: Args;
+}
+type ClientToolArgs<Tool extends ClientToolDef> = Tool extends ClientToolDef<string, infer Args> ? Args : never;
+type ClientEventFromTool<Tool extends ClientToolDef> = Tool extends ClientToolDef<infer Name, infer Args> ? ClientEvent<Name, Args> : never;
+/**
+ * Derive a discriminated union of ClientEvent types from an array of tools.
+ *
+ * @example
+ * ```typescript
+ * const tools = [showQuestion, playSound];
+ * type MyEvent = ClientEventsFrom<typeof tools>;
+ * ```
+ */
+type ClientEventsFrom<T extends ReadonlyArray<ClientToolDef>> = T[number] extends infer U ? U extends ClientToolDef<infer Name, infer Args> ? ClientEvent<Name, Args> : never : never;
+/**
+ * Return the Standard Schema associated with a tool, if any.
+ * Useful when composing validation outside of the SDK hooks.
+ */
+declare function getClientToolSchema(tool: ClientToolDef): StandardSchemaV1 | undefined;
+/**
+ * Validate parsed client event args against a tool's schema.
+ *
+ * Returns the typed args on success, or `null` when the schema fails or
+ * when the schema would need to resolve asynchronously (Standard Schema
+ * allows async `validate`, but for fire-and-forget client events we only
+ * accept sync results here).
+ *
+ * Tools defined without a schema always validate successfully — the
+ * incoming args are returned as-is.
+ */
+declare function validateClientToolArgs<Tool extends ClientToolDef>(tool: Tool, args: unknown): ClientToolArgs<Tool> | null;
+/**
+ * Define a single client tool.
+ *
+ * Returns a standalone object that can be composed into arrays and passed
+ * to `realtimeSessions.create({ tools })`.
+ *
+ * Two forms are supported:
+ *
+ * 1. **Schema-driven** (recommended) — pass a
+ *    [Standard Schema](https://standardschema.dev/) (Zod, Valibot, ArkType, …)
+ *    to infer `args` types and enable runtime validation of incoming events:
+ *
+ *    ```typescript
+ *    import { z } from 'zod';
+ *    const showCaption = clientTool('show_caption', {
+ *      description: 'Display a caption',
+ *      schema: z.object({ text: z.string() }),
+ *    });
+ *    ```
+ *
+ * 2. **Type-only** — pass an `args` type via a cast when you don't need
+ *    runtime validation:
+ *
+ *    ```typescript
+ *    const showCaption = clientTool('show_caption', {
+ *      description: 'Display a caption',
+ *      args: {} as { text: string },
+ *    });
+ *    ```
+ *
+ * Combine tools and derive event types with `ClientEventsFrom`:
+ *
+ * ```typescript
+ * const tools = [showCaption, playSound];
+ * type MyEvent = ClientEventsFrom<typeof tools>;
+ * ```
+ */
+declare function clientTool<Name extends string, Schema extends StandardSchemaV1>(name: Name, config: {
+    description: string;
+    schema: Schema;
+}): ClientToolDef<Name, InferSchemaOutput<Schema>>;
+declare function clientTool<Name extends string, Args>(name: Name, config: {
+    description: string;
+    args: Args;
+}): ClientToolDef<Name, Args>;
+
+/**
+ * Parse and validate a data channel payload as a ClientEvent.
+ * Returns null for non-client-event messages, ack messages, or malformed payloads.
+ */
+declare function parseClientEvent(payload: Uint8Array): ClientEvent | null;
+
+interface FlatDelta {
+    role: string;
+    turn: number;
+    textDelta: string;
+}
+interface FlatDeltaTurn {
+    id: string;
+    text: string;
+    participantIdentity: string;
+}
+interface FlatDeltaEmission {
+    /** Prior turns on the same role that were just promoted to final by this delta. */
+    finalized: Array<FlatDeltaTurn>;
+    /** The turn this delta belongs to — still streaming, always interim. */
+    active: FlatDeltaTurn;
+}
+/**
+ * Accumulates Runway flat-delta transcription fragments into coherent turns.
+ *
+ * The flat-delta protocol sends `{ role, turn, textDelta }` messages without
+ * an explicit end-of-turn signal, so we infer the end: when a delta arrives
+ * on a higher turn for the same role, prior turns on that role are done and
+ * get emitted once as final. The active turn is always interim.
+ *
+ * Identity is remembered per-turn, so finalized priors are attributed to
+ * whoever originally produced them — not to whoever triggered the role
+ * switch.
+ */
+declare class FlatDeltaAccumulator {
+    private readonly turns;
+    private readonly finalized;
+    ingest(delta: FlatDelta, participantIdentity: string): FlatDeltaEmission;
+    reset(): void;
+}
+
+type DataChannelJSON = Record<string, unknown>;
+declare function tryDecodeJSON(payload: Uint8Array): DataChannelJSON | null;
+/**
+ * Parse LiveKit-style segment arrays: `{ segments: [{ id, text, final? }] }`
+ * Also handles wrapper shapes like `{ type: "transcription", segments: [...] }`
+ * and `{ data: { segments: [...] } }`.
+ */
+declare function tryParseSegmentArray(root: DataChannelJSON, participant?: {
+    identity: string;
+}): Array<TranscriptionEntry> | null;
+/**
+ * Parse Runway flat streaming deltas: `{ type: "transcription", role, turn, text }`
+ * These arrive one delta at a time and are accumulated per role+turn.
+ */
+declare function tryParseFlatDelta(root: DataChannelJSON): {
+    role: string;
+    turn: number;
+    textDelta: string;
+} | null;
+
+declare const toolDefs: readonly [ClientToolDef<"click", {
+    target: string;
+}>, ClientToolDef<"scroll_to", {
+    target: string;
+}>, ClientToolDef<"highlight", {
+    target: string;
+    duration?: number;
+}>];
+type PageActionEvent = ClientEventsFrom<typeof toolDefs>;
+/**
+ * Pre-built tool definitions with `parameters` arrays, ready to spread
+ * into `realtimeSessions.create({ tools })`.
+ *
+ * @example
+ * ```ts
+ * import { pageActionTools } from '@runwayml/avatars-react/api';
+ *
+ * await client.realtimeSessions.create({
+ *   model: 'gwm1_avatars',
+ *   avatar: { type: 'runway-preset', presetId: 'music-superstar' },
+ *   tools: [...pageActionTools, ...myCustomTools],
+ * });
+ * ```
+ */
+declare const pageActionTools: ({
+    parameters: {
+        name: string;
+        type: string;
+        description: string;
+    }[];
+    type: "client_event";
+    name: "click";
+    description: string;
+    _args?: {
+        target: string;
+    } | undefined;
+} | {
+    parameters: {
+        name: string;
+        type: string;
+        description: string;
+    }[];
+    type: "client_event";
+    name: "scroll_to";
+    description: string;
+    _args?: {
+        target: string;
+    } | undefined;
+} | {
+    parameters: {
+        name: string;
+        type: string;
+        description: string;
+    }[];
+    type: "client_event";
+    name: "highlight";
+    description: string;
+    _args?: {
+        target: string;
+        duration?: number;
+    } | undefined;
+})[];
+
+export { type ActiveSpeaker, AvatarError, type AvatarErrorCode, AvatarEvent, type AvatarEventMap, AvatarSession, type ClientEvent, type ClientEventFromTool, type ClientEventHandler, type ClientEventsFrom, type ClientToolArgs, type ClientToolDef, type ConnectResponse, type ConnectionQuality, type ConsumeSessionOptions, type ConsumeSessionResponse, FlatDeltaAccumulator, type InferSchemaInput, type InferSchemaOutput, type MediaController, type MicPermissionState, type PageActionEvent, type ScreenShareController, type SessionCredentials, type SessionKeyResponse, type SessionState, type StandardSchemaIssue, type StandardSchemaResult, type StandardSchemaV1, TranscriptAccumulator, type TranscriptOptions, type TranscriptionEntry, type TranscriptionHandler, clientTool, connect, consumeSession, getClientToolSchema, pageActionTools, parseClientEvent, streamTo, tryDecodeJSON, tryParseFlatDelta, tryParseSegmentArray, validateClientToolArgs };