@convai/web-sdk 1.8.0-beta.5 → 1.8.0-beta.6

package/dist/core/types.d.ts

@@ -115,6 +115,17 @@ export interface ServerResponseExtras {
     has_speak_tag?: boolean;
     text?: string;
     room_name?: string;
+    command_id?: string;
+    room_session_id?: string;
+    previous_membership_id?: string | null;
+    active_membership_id?: string | null;
+    previous_active_membership_id?: string | null;
+    route_epoch?: number;
+    roster_epoch?: number;
+    changed?: boolean;
+    added?: unknown[];
+    removed_membership_ids?: string[];
+    code?: string;
     supported_types?: string[];
     vision_buffer?: VisionBufferStatus;
     requested_respond_mode?: RespondMode;
@@ -267,6 +278,11 @@ export interface RespondModesConfig {
     trigger?: RespondMode;
     sceneMetadata?: RespondMode;
 }
+/** Client-requested action/model output protocol versions. */
+export interface ActionProtocolCapabilities {
+    actionProtocolVersion: 1 | 2;
+    modelOutputVersion?: 1 | 2;
+}
 export interface VisionSamplingWindowConfig {
     count: number;
     intervalMs: number;
@@ -401,8 +417,12 @@ export interface ConvaiConfig {
      *
      * May also carry a version selector suffix (`<uuid>-draft`, `<uuid>-latest`,
      * `<uuid>-1.2`); `characterVersion` is the clearer way to say the same thing.
+     *
+     * Optional only because a multi-character room supplies `characters`
+     * instead. Exactly one of the two is required, and a version selector is not
+     * accepted inside a roster entry.
      */
-    characterId: string;
+    characterId?: string;
     /**
      * Which version of the character to run (optional).
      *
@@ -431,6 +451,62 @@ export interface ConvaiConfig {
      * `https://api2-stg.convai.com` to author against staging.
      */
     characterApiUrl?: string;
+    /**
+     * Ordered roster of character instances for a multi-character room. Pass this
+     * *instead of* `characterId` -- exactly one of the two is valid.
+     *
+     * Every entry becomes a distinct membership with its own actor, LiveKit
+     * participant, voice, history and lifecycle. The same `characterId` may
+     * appear more than once; each copy is an independent clone addressed by its
+     * own `membershipId`. `characters[0]` is the initial active member.
+     *
+     * Roster rooms require the LiveKit transport, a nonblank `endUserId`, and the
+     * `MULTI_CHARACTER_V0` entitlement on the publisher.
+     *
+     * @example
+     * ```ts
+     * const client = new ConvaiClient({
+     *   apiKey: 'YOUR_API_KEY',
+     *   endUserId: 'player-42',
+     *   characters: [
+     *     { characterId: GUIDE },     // initial active member
+     *     { characterId: ASSESSOR },
+     *     { characterId: GUIDE },     // a second, independent copy of GUIDE
+     *   ],
+     * });
+     * ```
+     */
+    characters?: ConvaiCharacterSpec[];
+    /**
+     * Publisher-scoped room grouping and topology key (1-128 chars of letters,
+     * digits, hyphen or underscore). A matching create request reuses the
+     * existing room; reusing the key with a different ordered roster, creator or
+     * capacity is rejected with `ROSTER_FINGERPRINT_MISMATCH`. Reordering the
+     * same characters counts as different, because it changes the initial member.
+     *
+     * This does not raise human capacity on its own -- see `maxNumParticipants`.
+     */
+    sharedSessionKey?: string;
+    /**
+     * Idempotency key for one logical connect (1-128 chars, same charset as
+     * `sharedSessionKey`). Reuse it only when retrying with an identical payload:
+     * within the replay window a completed attempt returns its original response
+     * and token, while a changed payload or a still-running original is rejected.
+     */
+    connectAttemptId?: string;
+    /**
+     * Maximum number of distinct human speakers in a roster room. Characters do
+     * not count toward it. Defaults to the server's value of 1, which admits no
+     * second human -- raise it before expecting `joinRoom()` to succeed.
+     */
+    maxNumParticipants?: number;
+    /**
+     * Bounded retry when a roster room is still provisioning (409
+     * `ROSTER_PROVISIONING_IN_PROGRESS`). Defaults to four attempts starting at
+     * 500ms and doubling with jitter. Pass `false` to surface the error instead
+     * and retry in application code.
+     */
+    rosterProvisioningRetry?: RosterProvisioningRetry | false;
     /** Temporary character state of mind sent as `state_of_mind` on connect. */
     stateOfMind?: string | null;
     /**
@@ -490,6 +566,13 @@ export interface ConvaiConfig {
     visionInputConfig?: VisionInputConfig;
     /** Connect-time respond-mode defaults by modality. */
     respondModes?: RespondModesConfig;
+    /** Client-requested action/model output protocol versions. */
+    capabilities?: ActionProtocolCapabilities;
+    /** Descriptive objects visible in the scene at connection time. */
+    sceneDescription?: Array<{
+        name: string;
+        description: string;
+    }>;
     /**
      * WebRTC ICE transport policy for the LiveKit connection (default: "relay").
      * "relay" forces TURN-only (reliable against Convai's hosted LiveKit which
@@ -599,6 +682,12 @@ export interface ConvaiConfig {
             /** Object description or properties */
             description: string;
         }>;
+        /** Client-executed tools negotiated by action protocol v2. */
+        tools?: Array<{
+            name: string;
+            description: string;
+            inputSchema: Record<string, unknown>;
+        }>;
         /** Name of the object the character is currently focused on. Must match one of objects[].name. */
         current_attention_object?: string;
     };
@@ -652,6 +741,37 @@ export interface ConvaiConfig {
      * Enable debug mode for additional logging and diagnostics (default: false).
      */
     debug?: boolean;
+    /** Optional per-session response lifecycle contract override. */
+    useResponseLifecycleContract?: boolean;
+    /**
+     * Keep links in the character's text output (default: server-side, currently off).
+     *
+     * Core-service flattens the LLM's markdown before streaming `bot-llm-text`,
+     * which also drops link targets: `[Doc](https://…/a.pdf)` reaches the client
+     * as `Doc`. With this on, the session preserves `[text](url)`, `![alt](url)`
+     * and bare `http(s)://` URLs verbatim (trailing sentence punctuation stays
+     * outside the URL, non-http(s) schemes are still stripped) while bold,
+     * headings, tables and code are flattened as before. Speech is unaffected —
+     * URLs are never read aloud.
+     *
+     * Both widgets render the preserved form as clickable links, with inline
+     * previews for image and video URLs. If you render `chatMessages` yourself,
+     * expect markdown links in `content` once this is on.
+     *
+     * Sent as `preserve_links_in_output` on `/connect`; omitted when undefined so
+     * the server's own default applies. If you build the `/connect` request
+     * server-side for `connectWithConnectionData()`, add the field there.
+     *
+     * @example
+     * ```typescript
+     * const client = new ConvaiClient({
+     *   apiKey: 'your-api-key',
+     *   characterId: 'your-character-id',
+     *   preserveLinksInOutput: true,
+     * });
+     * ```
+     */
+    preserveLinksInOutput?: boolean;
     /**
      * Log decoded RTVI data messages to the browser console (default: true).
      * Set to false to silence incoming and outgoing RTVI message logs.
@@ -676,11 +796,254 @@ export interface SendUserTextMessageOptions {
     logicalTurnId?: string;
     /** User-selected temporary emotion to apply before processing the text. */
     stateOfMind?: string | null;
+    /** Latest request-scoped dynamic context to include in this same interaction. */
+    dynamicInfo?: DynamicInfo;
 }
 /**
  * The body returned by POST /connect. The embed receives this from a
  * customer-hosted proxy route rather than fetching it directly.
  */
+/**
+ * One ordered character instance requested for a multi-character room.
+ *
+ * Repeating a `characterId` is legal and intentional: each entry becomes an
+ * independent clone with its own membership, session, participant and voice.
+ */
+export interface ConvaiCharacterSpec {
+    /**
+     * Bare character UUID. Version selectors (`<uuid>-draft`, `<uuid>-1.2`) are
+     * accepted by single-character `characterId` but rejected inside a roster --
+     * the runtime types roster entries as plain UUIDs.
+     */
+    characterId: string;
+    /**
+     * Resume this instance's prior conversation history. Must be unique across
+     * the roster, and must belong to the same publisher, speaker and character.
+     */
+    characterSessionId?: string;
+}
+/**
+ * Dispatch state of one roster member, as reported by `/connect`.
+ *
+ * These describe provisioning only and never readiness. A member becomes
+ * usable when its own `bot-ready` arrives -- `dispatch_accepted` is not
+ * `bot-ready`. The union deliberately has no member that could be mistaken for
+ * a ready state; the separate `character-status` event carries those.
+ */
+export type CharacterProvisioningStatus = "dispatch_queued" | "dispatch_accepted" | "dispatch_failed";
+/** One concrete character instance in a roster room. */
+export interface ConvaiCharacterInstance {
+    /** Primary key for this instance. Address members by this, never by `characterId`. */
+    membershipId: string;
+    /** The authored character this instance runs. Not unique across a roster. */
+    characterId: string;
+    /** Backend session for support correlation. */
+    sessionId: string | null;
+    /** This instance's conversation/history session. */
+    characterSessionId: string | null;
+    /** Stable LiveKit identity, always `character:{membershipId}`. */
+    participantIdentity: string;
+    /** True for `characters[0]`, the initially active member. */
+    isInitial: boolean;
+    provisioningStatus: CharacterProvisioningStatus;
+    /**
+     * The character's display name, when the runtime sends one. Saves a separate
+     * `character/get` call just to label a member in the UI. Undocumented in the
+     * published contract but returned by the live API, so treat it as optional.
+     */
+    displayName: string | null;
+    /** Character description, when the runtime sends one. */
+    description: string | null;
+    /** Populated when `provisioningStatus` is `dispatch_failed`. */
+    failureCode: string | null;
+    /** True once this member's own `bot-ready` has arrived. */
+    isReady: boolean;
+}
+/**
+ * Server-owned snapshot of a multi-character room. Null for single-character
+ * sessions.
+ */
+export interface ConvaiRoomSession {
+    /** Durable room identity. Use it to join, and to correlate with support. */
+    roomSessionId: string;
+    roomName: string | null;
+    /** Trace id of the `/connect` that produced this room. Quote it in bug reports. */
+    requestTraceId: string | null;
+    /** Membership currently authorised to consume user turns. */
+    activeMembershipId: string | null;
+    /** Monotonic version of the active target; fences target switches. */
+    routeEpoch: number;
+    /** Monotonic version of roster composition; fences add/remove. */
+    rosterEpoch: number;
+    /** True when at least one member already failed dispatch. */
+    partialDispatch: boolean;
+    characters: ConvaiCharacterInstance[];
+}
+/** Result of a successful target switch. */
+export interface ActiveCharacterChange {
+    /** Membership that was active before, or null. */
+    previous: string | null;
+    /** Membership now authorised to consume user turns, or null when all gates are closed. */
+    active: string | null;
+    routeEpoch: number;
+}
+/** Result of a successful roster mutation. */
+export interface CharacterRosterChange {
+    added: ConvaiCharacterInstance[];
+    removed: ConvaiCharacterInstance[];
+    /** The roster after the change. */
+    characters: ConvaiCharacterInstance[];
+    rosterEpoch: number;
+    routeEpoch: number;
+    activeMembershipId: string | null;
+}
+/** Options for adding and removing roster members on a live room. */
+export interface UpdateCharacterRosterOptions {
+    /** Members to add. Repeating an existing characterId creates another clone. */
+    add?: ConvaiCharacterSpec[];
+    /** Membership ids to remove. Never characterIds -- clones make those ambiguous. */
+    remove?: string[];
+    /**
+     * Member to take over routing when the active one is removed. Must be a ready
+     * member that survives the change. Chosen automatically when omitted.
+     */
+    replacementTarget?: string;
+}
+/** Options for joining an existing multi-character room. */
+export interface JoinRoomOptions {
+    /** Preferred locator. Exactly one of this or `sharedSessionKey`. */
+    roomSessionId?: string;
+    /** Publisher-scoped alternative locator. */
+    sharedSessionKey?: string;
+    /** Required and nonblank; identifies the joining human. */
+    endUserId: string;
+    endUserMetadata?: Record<string, unknown>;
+    /** Idempotency key for retrying this join. */
+    connectAttemptId?: string;
+}
+/**
+ * Lifecycle transition of one roster member, after `/connect` has returned.
+ *
+ * `bot-ready` remains the readiness authority: do not enable interaction from a
+ * `ready` status alone. This event's real job is reporting a member that was
+ * added at runtime, or one whose dispatch failed in the background.
+ */
+export interface CharacterStatusEvent {
+    status: "starting" | "ready" | "failed";
+    membershipId: string;
+    characterId: string;
+    characterSessionId: string | null;
+    participantIdentity: string;
+    roomSessionId: string | null;
+    rosterEpoch: number | null;
+    /** Set when `status` is `failed`, e.g. `ACTOR_ASSIGNMENT_FAILED`. */
+    failureCode: string | null;
+}
+/**
+ * An audio track bound to the character instance that publishes it.
+ *
+ * In a multi-character room `membershipId` is always set -- a track that
+ * cannot be attributed to a roster member is dropped, never surfaced against
+ * the active character. In a single-character room it is null.
+ */
+export interface CharacterAudioTrackEvent {
+    membershipId: string | null;
+    characterId: string | null;
+    /** LiveKit participant identity; `character:{membershipId}` in a roster. */
+    participantIdentity: string;
+    track: MediaStreamTrack;
+}
+/**
+ * A finalized turn's usage/cost breakdown, streamed by the server after
+ * {@link IConvaiClient.toggleUsageUpdates} enables it. The shape is
+ * server-defined and informational; treat unknown fields as opaque.
+ */
+export type UsageUpdate = Record<string, unknown>;
+/**
+ * Every event the client emits, with its payload.
+ *
+ * `on()` is typed against this map, so a misspelt event name or a listener
+ * with the wrong signature fails to compile. Payloads the server defines as
+ * opaque blobs (blendshape frames, metrics, usage) are left as records rather
+ * than guessed at.
+ */
+export interface ConvaiEventMap {
+    connect: [];
+    connected: [];
+    disconnect: [reason: DisconnectReason | null];
+    disconnected: [];
+    error: [error: unknown];
+    stateChange: [state: ConvaiClientState];
+    botReady: [];
+    characterSessionId: [characterSessionId: string];
+    interactionCreated: [data: InteractionCreated];
+    message: [message: ChatMessage];
+    messagesChange: [messages: ChatMessage[]];
+    userTranscriptionChange: [transcription: string];
+    speakingChange: [isSpeaking: boolean];
+    listeningChange: [isListening: boolean];
+    botRespondingChange: [isResponding: boolean];
+    turnEnd: [turn: {
+        sessionId: number;
+        duration: number;
+        timestamp: number;
+    }];
+    conversationStart: [
+        start: {
+            sessionId: number;
+            userMessage: string;
+            timestamp: number;
+        }
+    ];
+    emotionChange: [emotion: {
+        emotion: string;
+        scale?: number;
+    } | null];
+    actionResponse: [data: ActionResponseEvent];
+    serverResponse: [response: ServerResponse];
+    botOutput: [data: Record<string, unknown>];
+    botTtsText: [data: {
+        text: string;
+    }];
+    botTtsStarted: [];
+    botTtsStopped: [];
+    botTurnCompleted: [data: unknown];
+    userMuteStarted: [];
+    userMuteStopped: [];
+    idleWarning: [data: {
+        remainingSeconds: number | null;
+    }];
+    llmNoResponse: [];
+    metrics: [data: Record<string, unknown>];
+    usageUpdate: [data: UsageUpdate];
+    blendshapes: [data: Record<string, unknown>];
+    blendshapeCancel: [data: Record<string, unknown>];
+    blendshapeStatsReceived: [stats: Record<string, unknown>];
+    botAudioTrack: [track: MediaStreamTrack];
+    audioStateChange: [enabled: boolean];
+    videoStateChange: [enabled: boolean];
+    screenShareStateChange: [active: boolean];
+    characterReady: [instance: ConvaiCharacterInstance | undefined];
+    characterStatus: [status: CharacterStatusEvent];
+    activeCharacterChange: [change: ActiveCharacterChange];
+    rosterChange: [change: CharacterRosterChange];
+    characterAudioTrack: [event: CharacterAudioTrackEvent];
+}
+/** A transcript line inserted locally by {@link IConvaiClient.appendMessage}. */
+export interface AppendMessageOptions {
+    sender: "user" | "bot";
+    content: string;
+    /** Attribute the line to a roster member, in a multi-character room. */
+    membershipId?: string;
+    characterId?: string;
+}
+/** Bounded retry for a roster room that is still provisioning. */
+export interface RosterProvisioningRetry {
+    /** Total attempts including the first. Default 4. */
+    attempts?: number;
+    /** Delay before the second attempt; doubles with jitter. Default 500. */
+    initialDelayMs?: number;
+}
 export interface ConnectionData {
     /** LiveKit room URL, or the WebSocket URL when the transport is websocket. */
     room_url: string;
@@ -714,6 +1077,14 @@ export interface ChatMessage {
      * - `llm-no-response`: LLM deliberately did not respond (abstain); `content` is always `""`
      */
     type: 'user' | 'convai' | 'emotion' | 'behavior-tree' | 'action' | 'user-transcription' | 'bot-llm-text' | 'bot-output' | 'bot-emotion' | 'user-llm-text' | 'interrupt-bot' | 'idle-warning' | 'llm-no-response';
+    /**
+     * Which character instance produced this message, in a multi-character room.
+     * Undefined in single-character sessions. Attribute UI to this, never to
+     * `characterId` -- two clones of one character share a `characterId`.
+     */
+    membershipId?: string;
+    /** The authored character behind {@link membershipId}. Not unique across a roster. */
+    characterId?: string;
     /** The text content of the message */
     content: string;
     /** ISO timestamp string of when the message was created */
@@ -819,6 +1190,16 @@ export interface ConvaiClientState {
      * @see DisconnectReason
      */
     disconnectReason: DisconnectReason | null;
+    /**
+     * Character instances in a multi-character room, in roster order. Empty for
+     * single-character sessions, so a check on `length` distinguishes the two.
+     */
+    characters: ConvaiCharacterInstance[];
+    /**
+     * Membership currently receiving user turns, or null. Only this member
+     * consumes interaction input.
+     */
+    activeMembershipId: string | null;
 }
 /**
  * Audio control interface for managing microphone
@@ -836,7 +1217,10 @@ export interface AudioControls {
     getAudioDevices: () => Promise<MediaDeviceInfo[]>;
     startAudioLevelMonitoring: () => void;
     stopAudioLevelMonitoring: () => void;
-    on: (event: string, callback: (...args: any[]) => void) => () => void;
+    on: {
+        <E extends keyof ConvaiEventMap>(event: E, callback: (...args: ConvaiEventMap[E]) => void): () => void;
+        (event: string, callback: (...args: any[]) => void): () => void;
+    };
     off: (event: string, callback: (...args: any[]) => void) => void;
 }
 /**
@@ -859,7 +1243,10 @@ export interface VideoControls {
     publishCanvas: (canvas: HTMLCanvasElement, options?: PublishCanvasOptions) => Promise<VisionSourceHandle>;
     publishVideoTrack: (track: MediaStreamTrack, options?: PublishVideoTrackOptions) => Promise<VisionSourceHandle>;
     unpublishVisionSource: (source?: VisionSourceHandle | MediaStreamTrack) => Promise<void>;
-    on: (event: string, callback: (...args: any[]) => void) => () => void;
+    on: {
+        <E extends keyof ConvaiEventMap>(event: E, callback: (...args: ConvaiEventMap[E]) => void): () => void;
+        (event: string, callback: (...args: any[]) => void): () => void;
+    };
     off: (event: string, callback: (...args: any[]) => void) => void;
 }
 /**
@@ -873,7 +1260,10 @@ export interface ScreenShareControls {
     toggleScreenShare: () => Promise<void>;
     enableScreenShareWithAudio: () => Promise<void>;
     getScreenShareTracks: () => Promise<any[]>;
-    on: (event: string, callback: (...args: any[]) => void) => () => void;
+    on: {
+        <E extends keyof ConvaiEventMap>(event: E, callback: (...args: ConvaiEventMap[E]) => void): () => void;
+        (event: string, callback: (...args: any[]) => void): () => void;
+    };
     off: (event: string, callback: (...args: any[]) => void) => void;
 }
 /**
@@ -1336,6 +1726,70 @@ export interface IConvaiClient {
      * await client.connectWithConnectionData(data)
      */
     connectWithConnectionData: (data: ConnectionData, config?: ConvaiConfig) => Promise<void>;
+    /**
+     * Join an existing multi-character room by `roomSessionId` or
+     * `sharedSessionKey`. The roster is server-owned, so no character topology is
+     * sent; joining never spawns, removes or repairs a character.
+     *
+     * @example
+     * ```ts
+     * await client.joinRoom({ roomSessionId: 'the-room-id', endUserId: 'player-99' })
+     * ```
+     */
+    joinRoom: (options: JoinRoomOptions) => Promise<void>;
+    /** Server-owned snapshot of a multi-character room (null when single-character) */
+    readonly roomSession: ConvaiRoomSession | null;
+    /** Character instances in roster order; empty for a single-character session */
+    readonly characters: ConvaiCharacterInstance[];
+    /** Membership currently receiving user turns, or null */
+    readonly activeMembershipId: string | null;
+    /**
+     * Insert a message into the transcript locally, without sending it to the
+     * character. Use it for client-side lines -- a greeting, a system notice,
+     * replayed history -- that must appear in the chat but must never reach
+     * the LLM or be billed as a turn. Not `sendUserTextMessage`: nothing is
+     * transmitted. Returns the message that was added.
+     *
+     * @example
+     * ```ts
+     * client.appendMessage({ sender: 'bot', content: 'Welcome back!' })
+     * ```
+     */
+    appendMessage: (message: AppendMessageOptions) => ChatMessage;
+    /**
+     * Opt in or out of `usageUpdate` events -- the server's per-turn usage and
+     * cost breakdown. Informational only; billing is unaffected.
+     *
+     * @example
+     * ```ts
+     * client.toggleUsageUpdates(true)
+     * client.on('usageUpdate', (u) => console.log(u))
+     * ```
+     */
+    toggleUsageUpdates: (enabled: boolean) => void;
+    /**
+     * Route subsequent user turns to a different character instance. Pass null to
+     * close every input gate so no member hears the user.
+     *
+     * Resolves on the server's acknowledgement. Retries once against the
+     * server's authoritative epoch when another participant switched first.
+     *
+     * @example
+     * ```ts
+     * await client.setInteractionTarget(assessor.membershipId)
+     * client.sendUserTextMessage('Now grade me.')
+     * ```
+     */
+    setInteractionTarget: (membershipId: string | null) => Promise<ActiveCharacterChange>;
+    /**
+     * Add or remove character instances on a live room, without reconnecting.
+     *
+     * @example
+     * ```ts
+     * await client.updateCharacterRoster({ add: [{ characterId: NARRATOR }] })
+     * ```
+     */
+    updateCharacterRoster: (options: UpdateCharacterRosterOptions) => Promise<CharacterRosterChange>;
     /** Disconnect from the current character session */
     disconnect: () => Promise<void>;
     /** Reconnect - disconnect and connect again using stored config */
@@ -1472,7 +1926,10 @@ export interface IConvaiClient {
      * @param callback Callback function
      * @returns Unsubscribe function
      */
-    on: (event: string, callback: (...args: any[]) => void) => () => void;
+    on: {
+        <E extends keyof ConvaiEventMap>(event: E, callback: (...args: ConvaiEventMap[E]) => void): () => void;
+        (event: string, callback: (...args: any[]) => void): () => void;
+    };
     /**
      * Unsubscribe from an event
      * @param event Event name