@snapcall/stream-ui 1.41.3-beta.1 → 1.41.3

package/dist/types.d.ts

@@ -1,5 +1,6 @@
 import * as mediasoupClient from "mediasoup-client";
 import { types } from "mediasoup-client";
+import { AIResponse as _AIResponse1 } from "streamerClient/LiveAIAgent/message";
 import * as protooClient from "protoo-client";
 type VideoResolution = 'qvga' | 'vga' | 'hd' | 'max';
 type PeerId = string;
@@ -34,6 +35,7 @@ interface Flow {
     name: string;
     is_default: number;
     config?: {
+        real_time_assistant?: boolean;
         description?: string;
         image_url?: string;
         output_language?: string;
@@ -262,6 +264,188 @@ declare class AudioRenderer {
     getSink(): string;
     play(): Promise<void>;
 }
+interface InputText {
+    type: 'input_text';
+    text: string;
+}
+interface InputImage {
+    type: 'input_image';
+    image_url?: string;
+}
+type ContentItem = InputText | InputImage;
+interface MessageItem {
+    id: string;
+    type: 'message';
+    status?: 'completed' | 'in_progress';
+    role: 'user' | 'assistant';
+    content: ContentItem[];
+}
+interface FunctionCallItem {
+    id: string;
+    type: 'function_call';
+    status: 'in_progress' | 'completed';
+    name: string;
+    call_id: string;
+    arguments: string;
+}
+interface FunctionCallOutputItem {
+    id: string;
+    type: 'function_call_output';
+    call_id: string;
+    output: string;
+}
+interface FunctionCallOutputItemCreate {
+    id: string;
+    type: 'function_call_output';
+    call_id: string;
+    output: string;
+}
+interface BaseEvent {
+    type: string;
+    event_id: string;
+}
+interface BaseSendMessage {
+    type: string;
+    event_id: string;
+}
+interface ResponseFunctionCallArgumentsDone extends BaseEvent {
+    type: 'response.function_call_arguments.done';
+    response_id: string;
+    item_id: string;
+    output_index: number;
+    call_id: string;
+    name: string;
+    arguments: string;
+}
+interface FunctionCallOutput extends BaseSendMessage {
+    item: FunctionCallOutputItem;
+}
+type CreateItem = MessageItem | FunctionCallItem | FunctionCallOutputItemCreate;
+interface ConversationItemCreate extends BaseSendMessage {
+    event_id: string;
+    type: 'conversation.item.create';
+    item: CreateItem;
+}
+interface ResponseCreate extends BaseSendMessage {
+    event_id: string;
+    type: 'response.create';
+    response: {
+        output_modalities: string[];
+        metadata: {
+            initial_event_id: string;
+        };
+    };
+}
+interface ResponseCancel extends BaseSendMessage {
+    event_id: string;
+    type: 'response.cancel';
+    response_id: string;
+}
+type SendMessage = ConversationItemCreate | ResponseCreate | ResponseCancel | FunctionCallOutputItem;
+declare class ToolOutputMessage {
+    id: string;
+    callId: string;
+    output: string;
+    arg: string;
+    name: string;
+    answer: boolean;
+    constructor(initMessage: ResponseFunctionCallArgumentsDone);
+    setOutput(output: string): void;
+    build(): FunctionCallOutput;
+}
+declare class UserMessage {
+    id: string;
+    status: 'pending' | 'sent' | 'added';
+    content: Array<InputText | InputImage>;
+    addContent(text: string): void;
+    addMediaContent(type: 'microphone' | 'camera' | 'screen', state: 'active' | 'inactive'): void;
+    addAppContent(type: 'button' | 'text' | 'event', text: string): void;
+    addInstructionContent(instruction: string): void;
+    addImageContent(url: string): Promise<void>;
+    build(): ConversationItemCreate;
+}
+declare class CancelMessage {
+    id: string;
+    type: 'response.cancel';
+    response_id: string;
+    constructor(response_id: string);
+    build(): SendMessage & {
+        event_id: string;
+    };
+}
+declare class AIResponse {
+    static internalIdCounter: number;
+    initiator: 'server' | 'client';
+    id: string;
+    responseId?: string;
+    terminated: boolean;
+    canceled: boolean;
+    constructor(initiator: 'server' | 'client');
+    setResponseId(response_id: string): void;
+    setTerminated(): void;
+    cancel(): CancelMessage | undefined;
+    build(): ResponseCreate;
+}
+interface AIAgentListener {
+    onStart: () => void;
+    onAssistantPartialMessage: (message: string) => void;
+    onAssistantMessage: (message: string) => void;
+    onAudioAssistant: (state: 'started' | 'stopped') => void;
+}
+declare class AIAgent {
+    responses: Record<string, AIResponse>;
+    lastDeltaMessage: Record<string, any> | undefined;
+    lastCompleteMessage: Record<string, any> | undefined;
+    constructor(listener: AIAgentListener);
+    mute(): void;
+    unmute(): void;
+    sendUserMessage(message: UserMessage): Promise<unknown> | undefined;
+    setTrack(track: MediaStreamTrack): void;
+    init({ instructions, voice, track, }: {
+        instructions: string;
+        voice: string;
+        track?: MediaStreamTrack;
+    }): Promise<void>;
+    stopResponse(): Promise<void>;
+    startResponse(origin: string): Promise<AIResponse | undefined>;
+    sendToolOutput(toolOutput: ToolOutputMessage): Promise<FunctionCallOutput>;
+    stop(): void;
+    dumpHistory(): void;
+}
+type AITextEvent = CustomEvent<{
+    text: string;
+}>;
+interface AIEvents {
+    aiTextMessage: AITextEvent['detail'];
+    aiDisplayMessage: AITextEvent;
+    aiStarted: AITextEvent;
+    aiStopped: AITextEvent;
+    aiUserTextMessage: AITextEvent;
+}
+declare class LiveAIAgent implements AIAgentListener {
+    static instance: LiveAIAgent;
+    agent: AIAgent;
+    constructor();
+    init(options: {
+        language: string;
+        voice: string;
+    }, streamInfo: StreamInfo, track?: MediaStreamTrack): Promise<void>;
+    setTrack(track: MediaStreamTrack): void;
+    addScriptedMessage(script: Scripted): Promise<unknown>;
+    addVideoImage(image: string): Promise<void>;
+    analyzeVideo(): Promise<void>;
+    analyzeImageResult(image: string): Promise<void>;
+    onStart(): Promise<void>;
+    sendInitialMessage(): Promise<void>;
+    startResponse(origin: string): Promise<_AIResponse1 | undefined> | undefined;
+    onAssistantPartialMessage(text: string): void;
+    onAssistantMessage(text: string): void;
+    onAudioAssistant(state: 'started' | 'stopped'): void;
+}
+interface Scripted {
+    message: UserMessage;
+    response: boolean;
+}
 interface Media {
     stream: MediaStream | undefined;
     track: MediaStreamTrack | undefined;
@@ -395,6 +579,7 @@ declare global {
     export namespace SnapCall {
         export { Permissions, Permission, MediaType };
         export { EventAnswer };
+        export { AITextEvent };
         export type SnapcallEvent<T> = CustomEvent<T>;
         export type PeersInfoEvent = SnapcallEvent<{
             videoRecordStarted: boolean;
@@ -515,7 +700,7 @@ declare global {
         };
     }
 }
-interface StreamerEventMap {
+interface StreamerEventMap extends AIEvents {
     audioLevel: SnapCall.AudioLevelEvent['detail'];
     localStartSpeak: SnapCall.BaseEvent['detail'];
     localStopSpeak: SnapCall.BaseEvent['detail'];
@@ -582,6 +767,8 @@ declare const StreamerEventTargetType: {
     prototype: StreamerEventTarget;
 };
 declare class StreamerClient extends StreamerEventTargetType implements AudioLevelListener, TransportMonitorListener {
+    liveAIAgent?: LiveAIAgent;
+    streamInfo?: StreamInfo;
     tracksHandler: MediaTracksHandler;
     peers: Map<PeerId, PeerInfo>;
     consumers: Map<ConsumerId, ConsumerData>;
@@ -646,6 +833,10 @@ declare class StreamerClient extends StreamerEventTargetType implements AudioLev
     enableMicrophone({ deviceId }?: {
         deviceId?: string;
     }): Promise<void>;
+    initLiveAIAgent(options: {
+        language: string;
+        voice: string;
+    }): LiveAIAgent;
     muteMicrophone(): void;
     unMuteMicrophone(): void;
     toggleMute(): Promise<{
@@ -722,6 +913,7 @@ declare class StreamerClient extends StreamerEventTargetType implements AudioLev
         };
     }>): Promise<CreateEventResult>;
     captureVideo(videoElement?: HTMLVideoElement): Promise<string>;
+    captureScreenShare(): Promise<string>;
     requestLocalVideo(element: HTMLVideoElement): {
         facingMode: string;
     };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@snapcall/stream-ui",
-  "version": "1.41.3-beta.1",
+  "version": "1.41.3",
   "description": "",
   "source": "src/index.tsx",
   "main": "dist/stream-ui.js",
@@ -59,7 +59,7 @@
     "bowser": "^2.11.0",
     "i18next": "^25.5.2",
     "inobounce": "^0.2.1",
-    "mediasoup-client": "^3.15.6",
+    "mediasoup-client": "^3.18.3",
     "protoo-client": "^4.0.6",
     "qrcode": "^1.5.4",
     "react": "^19.1.1",
@@ -101,7 +101,7 @@
     "classnames": "^2.5.1",
     "inobounce": "^0.2.1",
     "mixpanel-browser": "^2.67.0",
-    "protoo-client": "^4.0.6",
+    "protoo-client": "^4.0.7",
     "qrcode": "^1.5.4",
     "react-hook-form": "^7.59.0",
     "react-markdown": "^10.1.0",