@whereby.com/assistant-sdk 0.0.0-canary-20251002113535 → 0.0.0-canary-20251007140529

package/dist/index.mjs

@@ -2,16 +2,12 @@ import { WherebyClient } from '@whereby.com/core';
 import wrtc from '@roamhq/wrtc';
 import EventEmitter$1, { EventEmitter } from 'events';
 import { spawn } from 'child_process';
-import { PassThrough } from 'stream';
 import express from 'express';
 import assert from 'assert';
 import bodyParser from 'body-parser';
-import { networkInterfaces } from 'os';
-import * as dotenv from 'dotenv';
 
 const TRIGGER_EVENT_SUCCESS = "trigger_event_success";
 
-const AUDIO_STREAM_READY = "AUDIO_STREAM_READY";
 const ASSISTANT_JOINED_ROOM = "ASSISTANT_JOINED_ROOM";
 const ASSISTANT_LEFT_ROOM = "ASSISTANT_LEFT_ROOM";
 const PARTICIPANT_VIDEO_TRACK_ADDED = "PARTICIPANT_VIDEO_TRACK_ADDED";
@@ -19,48 +15,10 @@ const PARTICIPANT_VIDEO_TRACK_REMOVED = "PARTICIPANT_VIDEO_TRACK_REMOVED";
 const PARTICIPANT_AUDIO_TRACK_ADDED = "PARTICIPANT_AUDIO_TRACK_ADDED";
 const PARTICIPANT_AUDIO_TRACK_REMOVED = "PARTICIPANT_AUDIO_TRACK_REMOVED";
 
-/******************************************************************************
-Copyright (c) Microsoft Corporation.
-
-Permission to use, copy, modify, and/or distribute this software for any
-purpose with or without fee is hereby granted.
-
-THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
-REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
-AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
-INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
-LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
-OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-PERFORMANCE OF THIS SOFTWARE.
-***************************************************************************** */
-/* global Reflect, Promise, SuppressedError, Symbol, Iterator */
-
-
-function __awaiter(thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-}
-
-typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
-    var e = new Error(message);
-    return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
-};
-
-const { nonstandard: { RTCAudioSink }, } = wrtc;
-class AudioSource extends PassThrough {
-    constructor() {
-        super({
-            allowHalfOpen: true,
-            highWaterMark: 1 * 1024,
-        });
-    }
+const { nonstandard: { RTCAudioSink, RTCAudioSource }, } = wrtc;
+class AudioSource extends RTCAudioSource {
 }
-class AudioSink extends wrtc.nonstandard.RTCAudioSink {
+class AudioSink extends RTCAudioSink {
     constructor(track) {
         super(track);
         this._sink = new RTCAudioSink(track);
@@ -168,7 +126,7 @@ function createFfmpegMixer() {
      * @param ff        Child process handle from spawn("ffmpeg", ...)
      * @param slotCount Number of participant input slots (0..N-1 → fd 3..3+N-1)
      */
-    function startPacer(ff, slotCount, rtcAudioSource, onAudioStreamReady) {
+    function startPacer(ff, slotCount, rtcAudioSource) {
         if (stopPacerFn) {
             stopPacerFn();
             stopPacerFn = null;
@@ -186,8 +144,6 @@ function createFfmpegMixer() {
             frameQueue: [],
             nextDueMs: t0 + outputFrameMs,
             rtcAudioSource,
-            onAudioStreamReady,
-            didEmitReadyEvent: false,
         };
         const iv = setInterval(() => {
             const t = nowMs();
@@ -215,10 +171,6 @@ function createFfmpegMixer() {
             const state = outputPacerState;
             if (t >= state.nextDueMs) {
                 const samples = state.frameQueue.length > 0 ? state.frameQueue.shift() : new Int16Array(FRAME_10MS_SAMPLES); // silence
-                if (!state.didEmitReadyEvent) {
-                    state.onAudioStreamReady();
-                    state.didEmitReadyEvent = true;
-                }
                 state.rtcAudioSource.onData({
                     samples: samples,
                     sampleRate: STREAM_INPUT_SAMPLE_RATE_IN_HZ,
@@ -321,11 +273,11 @@ function createFfmpegMixer() {
      * The process will log its output to stderr.
      * @return The spawned FFmpeg process.
      */
-    function spawnFFmpegProcessDebug(rtcAudioSource, onAudioStreamReady) {
+    function spawnFFmpegProcessDebug(rtcAudioSource) {
         const stdio = ["ignore", "ignore", "pipe", ...Array(PARTICIPANT_SLOTS).fill("pipe")];
         const args = getFFmpegArgumentsDebug();
         const ffmpegProcess = spawn("ffmpeg", args, { stdio });
-        startPacer(ffmpegProcess, PARTICIPANT_SLOTS, rtcAudioSource, onAudioStreamReady);
+        startPacer(ffmpegProcess, PARTICIPANT_SLOTS, rtcAudioSource);
         ffmpegProcess.stderr.setEncoding("utf8");
         ffmpegProcess.stderr.on("data", (d) => console.error("[ffmpeg]", String(d).trim()));
         ffmpegProcess.on("error", () => console.error("FFmpeg process error (debug): is ffmpeg installed?"));
@@ -339,11 +291,11 @@ function createFfmpegMixer() {
      * @param rtcAudioSource The RTCAudioSource to which the mixed audio will be sent.
      * @return The spawned FFmpeg process.
      */
-    function spawnFFmpegProcess(rtcAudioSource, onAudioStreamReady) {
+    function spawnFFmpegProcess(rtcAudioSource) {
         const stdio = ["pipe", "pipe", "pipe", ...Array(PARTICIPANT_SLOTS).fill("pipe")];
         const args = getFFmpegArguments();
         const ffmpegProcess = spawn("ffmpeg", args, { stdio });
-        startPacer(ffmpegProcess, PARTICIPANT_SLOTS, rtcAudioSource, onAudioStreamReady);
+        startPacer(ffmpegProcess, PARTICIPANT_SLOTS, rtcAudioSource);
         ffmpegProcess.stderr.setEncoding("utf8");
         ffmpegProcess.stderr.on("data", (d) => console.error("[ffmpeg]", String(d).trim()));
         ffmpegProcess.on("error", () => console.error("FFmpeg process error: is ffmpeg installed?"));
@@ -441,7 +393,7 @@ function createFfmpegMixer() {
 }
 
 class AudioMixer extends EventEmitter {
-    constructor(onStreamReady) {
+    constructor() {
         super();
         this.ffmpegProcess = null;
         this.combinedAudioStream = null;
@@ -451,7 +403,6 @@ class AudioMixer extends EventEmitter {
         this.mixer = createFfmpegMixer();
         this.setupMediaStream();
         this.participantSlots = new Map(Array.from({ length: PARTICIPANT_SLOTS }, (_, i) => [i, ""]));
-        this.onStreamReady = onStreamReady;
     }
     setupMediaStream() {
         this.rtcAudioSource = new wrtc.nonstandard.RTCAudioSource();
@@ -467,7 +418,7 @@ class AudioMixer extends EventEmitter {
             return;
         }
         if (!this.ffmpegProcess && this.rtcAudioSource) {
-            this.ffmpegProcess = this.mixer.spawnFFmpegProcess(this.rtcAudioSource, this.onStreamReady);
+            this.ffmpegProcess = this.mixer.spawnFFmpegProcess(this.rtcAudioSource);
         }
         for (const p of participants)
             this.attachParticipantIfNeeded(p);
@@ -557,12 +508,28 @@ class AudioMixer extends EventEmitter {
     }
 }
 
+const { nonstandard: { RTCVideoSink, RTCVideoSource }, } = wrtc;
+class VideoSource extends RTCVideoSource {
+}
+class VideoSink extends RTCVideoSink {
+    constructor(track) {
+        super(track);
+        this._sink = new RTCVideoSink(track);
+    }
+    subscribe(cb) {
+        this._sink.onframe = cb;
+        return () => {
+            this._sink.onframe = undefined;
+        };
+    }
+}
+
 class Assistant extends EventEmitter$1 {
-    constructor({ assistantKey, startCombinedAudioStream = false, startLocalMedia = false }) {
+    constructor({ assistantKey }) {
         super();
-        this.mediaStream = null;
-        this.audioSource = null;
-        this.combinedStream = null;
+        this.localAudioSource = null;
+        this.localVideoSource = null;
+        this.combinedAudioSink = null;
         this.remoteMediaTracks = {};
         this.roomUrl = null;
         this.stateSubscriptions = [];
@@ -583,28 +550,34 @@ class Assistant extends EventEmitter$1 {
                 const tracks = stream.getTracks();
                 tracks.forEach((track) => {
                     if (!this.remoteMediaTracks[track.id]) {
-                        const eventName = track.kind === "video" ? PARTICIPANT_VIDEO_TRACK_ADDED : PARTICIPANT_AUDIO_TRACK_ADDED;
-                        this.emit(eventName, {
-                            participantId,
-                            stream,
-                            track,
-                        });
+                        if (track.kind === "video") {
+                            this.emit(PARTICIPANT_VIDEO_TRACK_ADDED, {
+                                participantId,
+                                trackId: track.id,
+                                data: new VideoSink(track),
+                            });
+                        }
+                        else {
+                            this.emit(PARTICIPANT_AUDIO_TRACK_ADDED, {
+                                participantId,
+                                trackId: track.id,
+                                data: new AudioSink(track),
+                            });
+                        }
                         this.remoteMediaTracks[track.id] = {
                             participantId,
-                            stream,
                             track,
                         };
                     }
                 });
                 return tracks;
             });
-            Object.values(this.remoteMediaTracks).forEach(({ participantId, stream, track }) => {
+            Object.values(this.remoteMediaTracks).forEach(({ participantId, track }) => {
                 if (!currentRemoteMediaTracks.includes(track)) {
                     const eventName = track.kind === "video" ? PARTICIPANT_VIDEO_TRACK_REMOVED : PARTICIPANT_AUDIO_TRACK_REMOVED;
                     this.emit(eventName, {
                         participantId,
-                        stream,
-                        track,
+                        trackId: track.id,
                     });
                     delete this.remoteMediaTracks[track.id];
                 }
@@ -614,135 +587,102 @@ class Assistant extends EventEmitter$1 {
         this.client = new WherebyClient();
         this.roomConnection = this.client.getRoomConnection();
         this.localMedia = this.client.getLocalMedia();
-        if (startLocalMedia) {
-            const outputAudioSource = new wrtc.nonstandard.RTCAudioSource();
-            const outputMediaStream = new wrtc.MediaStream([outputAudioSource.createTrack()]);
-            this.mediaStream = outputMediaStream;
-            this.audioSource = outputAudioSource;
-        }
-        if (startCombinedAudioStream) {
-            const handleStreamReady = () => {
-                if (!this.combinedStream) {
-                    console.warn("Combined stream is not available");
-                    return;
-                }
-                this.emit(AUDIO_STREAM_READY, {
-                    stream: this.combinedStream,
-                    track: this.combinedStream.getAudioTracks()[0],
-                });
-            };
-            const audioMixer = new AudioMixer(handleStreamReady);
-            this.combinedStream = audioMixer.getCombinedAudioStream();
-            this.stateSubscriptions.push(this.roomConnection.subscribeToRemoteParticipants(audioMixer.handleRemoteParticipants.bind(audioMixer)));
-        }
         this.stateSubscriptions.push(this.roomConnection.subscribeToConnectionStatus(this.handleConnectionStatusChange));
         this.stateSubscriptions.push(this.roomConnection.subscribeToRemoteParticipants(this.handleRemoteParticipantsTracksChange));
     }
     joinRoom(roomUrl) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (this.mediaStream) {
-                yield this.localMedia.startMedia(this.mediaStream);
-            }
-            this.roomUrl = roomUrl;
-            this.roomConnection.initialize({
-                localMediaOptions: {
-                    audio: false,
-                    video: false,
-                },
-                roomUrl,
-                isNodeSdk: true,
-                assistantKey: this.assistantKey,
-            });
-            return this.roomConnection.joinRoom();
+        this.roomUrl = roomUrl;
+        this.roomConnection.initialize({
+            localMediaOptions: {
+                audio: false,
+                video: false,
+            },
+            roomUrl,
+            isNodeSdk: true,
+            assistantKey: this.assistantKey,
         });
-    }
-    startLocalMedia() {
-        if (!this.mediaStream) {
-            const outputAudioSource = new wrtc.nonstandard.RTCAudioSource();
-            const outputMediaStream = new wrtc.MediaStream([outputAudioSource.createTrack()]);
-            this.mediaStream = outputMediaStream;
-            this.audioSource = outputAudioSource;
-        }
-        this.localMedia.startMedia(this.mediaStream);
-    }
-    getLocalMediaStream() {
-        return this.mediaStream;
-    }
-    getLocalAudioSource() {
-        return this.audioSource;
+        return this.roomConnection.joinRoom();
     }
     getRoomConnection() {
         return this.roomConnection;
     }
-    getCombinedAudioStream() {
-        return this.combinedStream;
-    }
-    getRemoteParticipants() {
-        return this.roomConnection.getState().remoteParticipants;
-    }
-    startCloudRecording() {
-        this.roomConnection.startCloudRecording();
-    }
-    stopCloudRecording() {
-        this.roomConnection.stopCloudRecording();
+    startLocalMedia() {
+        if (Boolean(this.localAudioSource) || Boolean(this.localVideoSource)) {
+            return;
+        }
+        this.localAudioSource = new AudioSource();
+        this.localVideoSource = new VideoSource();
+        const outputMediaStream = new wrtc.MediaStream([
+            this.localAudioSource.createTrack(),
+            this.localVideoSource.createTrack(),
+        ]);
+        this.localMedia.startMedia(outputMediaStream);
+        this.localMedia.toggleMicrophone(true);
+    }
+    stopLocalMedia() {
+        this.localMedia.stopMedia();
+        this.localAudioSource = null;
+        this.localVideoSource = null;
     }
-    sendChatMessage(message) {
-        this.roomConnection.sendChatMessage(message);
+    getLocalAudioSource() {
+        return this.localAudioSource;
     }
-    spotlightParticipant(participantId) {
-        this.roomConnection.spotlightParticipant(participantId);
+    getLocalVideoSource() {
+        return this.localVideoSource;
     }
-    removeSpotlight(participantId) {
-        this.roomConnection.removeSpotlight(participantId);
+    getLocalMedia() {
+        return this.localMedia;
     }
-    requestAudioEnable(participantId, enable) {
-        if (enable) {
-            this.roomConnection.askToSpeak(participantId);
-        }
-        else {
-            this.roomConnection.muteParticipants([participantId]);
+    getCombinedAudioSink() {
+        if (this.combinedAudioSink) {
+            return this.combinedAudioSink;
         }
-    }
-    requestVideoEnable(participantId, enable) {
-        if (enable) {
-            this.roomConnection.askToTurnOnCamera(participantId);
-        }
-        else {
-            this.roomConnection.turnOffParticipantCameras([participantId]);
+        const audioMixer = new AudioMixer();
+        const stream = audioMixer.getCombinedAudioStream();
+        const audioTracks = stream === null || stream === void 0 ? void 0 : stream.getAudioTracks();
+        if (audioTracks === null || audioTracks === void 0 ? void 0 : audioTracks.length) {
+            this.combinedAudioSink = new AudioSink(audioTracks[0]);
+            this.stateSubscriptions.push(this.roomConnection.subscribeToRemoteParticipants(audioMixer.handleRemoteParticipants.bind(audioMixer)));
+            return this.combinedAudioSink;
         }
-    }
-    acceptWaitingParticipant(participantId) {
-        this.roomConnection.acceptWaitingParticipant(participantId);
-    }
-    rejectWaitingParticipant(participantId) {
-        this.roomConnection.rejectWaitingParticipant(participantId);
-    }
-    subscribeToRemoteParticipants(callback) {
-        return this.roomConnection.subscribeToRemoteParticipants(callback);
-    }
-    subscribeToChatMessages(callback) {
-        return this.roomConnection.subscribeToChatMessages(callback);
+        return null;
     }
 }
 
-dotenv.config();
-const { IS_LOCAL = "false", BIND_INTERFACE = "en0" } = process.env;
+/******************************************************************************
+Copyright (c) Microsoft Corporation.
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.
+***************************************************************************** */
+/* global Reflect, Promise, SuppressedError, Symbol, Iterator */
+
+
+function __awaiter(thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+}
+
+typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
+    var e = new Error(message);
+    return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
+};
+
 function buildRoomUrl(roomPath, wherebySubdomain, baseDomain = "whereby.com") {
-    let wherebyDomain;
-    if (IS_LOCAL === "true") {
-        const ifaceAddrs = networkInterfaces()[BIND_INTERFACE];
-        if (!ifaceAddrs) {
-            throw new Error(`Unknown interface ${BIND_INTERFACE}`);
-        }
-        const [bindAddr] = ifaceAddrs.filter((iface) => iface.family === "IPv4");
-        if (!bindAddr) {
-            throw new Error(`No IPv4 address found for interface ${BIND_INTERFACE}`);
-        }
-        wherebyDomain = `${wherebySubdomain}-ip-${bindAddr.address.replace(/[.]/g, "-")}.hereby.dev:4443`;
-    }
-    else {
-        wherebyDomain = `${wherebySubdomain}.${baseDomain}`;
-    }
+    const wherebyDomain = `${wherebySubdomain}.${baseDomain}`;
     return `https://${wherebyDomain}${roomPath}`;
 }
 
@@ -801,4 +741,4 @@ class Trigger extends EventEmitter {
     }
 }
 
-export { ASSISTANT_JOINED_ROOM, ASSISTANT_LEFT_ROOM, AUDIO_STREAM_READY, Assistant, AudioSink, AudioSource, PARTICIPANT_AUDIO_TRACK_ADDED, PARTICIPANT_AUDIO_TRACK_REMOVED, PARTICIPANT_VIDEO_TRACK_ADDED, PARTICIPANT_VIDEO_TRACK_REMOVED, TRIGGER_EVENT_SUCCESS, Trigger };
+export { ASSISTANT_JOINED_ROOM, ASSISTANT_LEFT_ROOM, Assistant, AudioSink, AudioSource, PARTICIPANT_AUDIO_TRACK_ADDED, PARTICIPANT_AUDIO_TRACK_REMOVED, PARTICIPANT_VIDEO_TRACK_ADDED, PARTICIPANT_VIDEO_TRACK_REMOVED, TRIGGER_EVENT_SUCCESS, Trigger };