@unboundcx/video-sdk-client 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AudioMixer.js +235 -0
- package/README.md +400 -0
- package/VideoMeetingClient.js +1210 -0
- package/VideoProcessor.js +375 -0
- package/index.js +42 -0
- package/managers/ConnectionManager.js +243 -0
- package/managers/LocalMediaManager.js +1051 -0
- package/managers/MediasoupManager.js +789 -0
- package/managers/RemoteMediaManager.js +972 -0
- package/managers/StatsCollector.js +710 -0
- package/package.json +56 -0
- package/utils/EventEmitter.js +103 -0
- package/utils/Logger.js +114 -0
- package/utils/errors.js +136 -0
|
@@ -0,0 +1,972 @@
|
|
|
1
|
+
import { EventEmitter } from '../utils/EventEmitter.js';
|
|
2
|
+
import { Logger } from '../utils/Logger.js';
|
|
3
|
+
import { AudioMixer } from '../AudioMixer.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Manages remote participants and their media streams
|
|
7
|
+
*
|
|
8
|
+
* Events:
|
|
9
|
+
* - 'participant:added' - New remote participant
|
|
10
|
+
* - 'participant:removed' - Participant left
|
|
11
|
+
* - 'participant:updated' - Participant data changed
|
|
12
|
+
* - 'stream:added' - New remote stream
|
|
13
|
+
* - 'stream:removed' - Remote stream ended
|
|
14
|
+
*/
|
|
15
|
+
export class RemoteMediaManager extends EventEmitter {
|
|
16
|
+
/**
|
|
17
|
+
* @param {Object} options
|
|
18
|
+
* @param {MediasoupManager} options.mediasoup - Mediasoup manager instance
|
|
19
|
+
* @param {ConnectionManager} options.connection - Connection manager instance
|
|
20
|
+
* @param {boolean} options.debug - Enable debug logging
|
|
21
|
+
*/
|
|
22
|
+
constructor(options) {
|
|
23
|
+
super();
|
|
24
|
+
|
|
25
|
+
this.mediasoup = options.mediasoup;
|
|
26
|
+
this.connection = options.connection;
|
|
27
|
+
this.videoClient = options.videoClient; // Reference to parent VideoMeetingClient
|
|
28
|
+
this.logger = new Logger('SDK:RemoteMediaManager', options.debug);
|
|
29
|
+
|
|
30
|
+
// Map of participants: participantId -> participant data
|
|
31
|
+
this.participants = new Map();
|
|
32
|
+
|
|
33
|
+
// Map of streams: participantId -> { video, audio, screenShare }
|
|
34
|
+
this.streams = new Map();
|
|
35
|
+
|
|
36
|
+
// Map of consumers: consumerId -> { consumer, participantId, type }
|
|
37
|
+
this.consumers = new Map();
|
|
38
|
+
|
|
39
|
+
// Queue for producers that arrived before transports were ready
|
|
40
|
+
this.pendingProducers = [];
|
|
41
|
+
|
|
42
|
+
// Server event listeners will be set up after socket connects
|
|
43
|
+
this.listenersSetup = false;
|
|
44
|
+
|
|
45
|
+
// Map of video element observers: participantId -> { element, observer, lastWidth, lastHeight }
|
|
46
|
+
this.videoElementTracking = new Map();
|
|
47
|
+
|
|
48
|
+
// Map of local mute states for remote streams: `${participantId}:${type}` -> boolean
|
|
49
|
+
this.localMuteStates = new Map();
|
|
50
|
+
|
|
51
|
+
// Audio mixer for scalable audio (30+ participants)
|
|
52
|
+
this.audioMixer = new AudioMixer();
|
|
53
|
+
this.useAudioMixer = false; // Will be enabled automatically when participant count >= 30
|
|
54
|
+
this.MIXER_THRESHOLD = 30; // Threshold for switching to audio mixer
|
|
55
|
+
|
|
56
|
+
// Map of participant volumes: participantId -> volume (0.0 to 1.0)
|
|
57
|
+
this.participantVolumes = new Map();
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Setup listeners for server events (call after socket is connected)
|
|
62
|
+
* @private
|
|
63
|
+
*/
|
|
64
|
+
_setupServerListeners() {
|
|
65
|
+
if (this.listenersSetup) {
|
|
66
|
+
this.logger.warn('Server listeners already set up');
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
if (!this.connection.socket) {
|
|
71
|
+
this.logger.warn('Socket not connected yet, cannot set up listeners');
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
this.logger.info('Setting up server event listeners');
|
|
76
|
+
|
|
77
|
+
// All participants list (sent when you join or when participants change)
|
|
78
|
+
this.connection.onServerEvent('participant.all', (data) => {
|
|
79
|
+
this.logger.info('Received all participants');
|
|
80
|
+
if (data.participants && typeof data.participants === 'object') {
|
|
81
|
+
// participants is an object: { participantId: {...}, participantId2: {...} }
|
|
82
|
+
Object.entries(data.participants).forEach(([participantId, participant]) => {
|
|
83
|
+
// Skip ourselves
|
|
84
|
+
const myParticipantId = this.videoClient?.joinData?.participant?.id;
|
|
85
|
+
if (participantId === myParticipantId) {
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// Initialize this participant if not already tracked
|
|
90
|
+
if (!this.participants.has(participantId)) {
|
|
91
|
+
this._handleParticipantJoined({ id: participantId, ...participant });
|
|
92
|
+
}
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
// New participant joined
|
|
98
|
+
this.connection.onServerEvent('participant:joined', (data) => {
|
|
99
|
+
this.logger.info('Participant joined:', data.participant.id);
|
|
100
|
+
this._handleParticipantJoined(data.participant);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
// Participant left
|
|
104
|
+
this.connection.onServerEvent('participant:left', (data) => {
|
|
105
|
+
this.logger.info('Participant left:', data.participantId);
|
|
106
|
+
this._handleParticipantLeft(data.participantId);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
// Participant updated
|
|
110
|
+
this.connection.onServerEvent('participant:updated', (data) => {
|
|
111
|
+
this.logger.log('Participant updated:', data.participant.id);
|
|
112
|
+
this._handleParticipantUpdated(data.participant);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
// New producer created (someone started publishing)
|
|
116
|
+
this.connection.onServerEvent('media.producer.created', (data) => {
|
|
117
|
+
this.logger.info('Producer created:', data.producer.id, 'by', data.participant.id);
|
|
118
|
+
|
|
119
|
+
// Don't consume our own producers (loopback not needed)
|
|
120
|
+
const myParticipantId = this.videoClient?.joinData?.participant?.id;
|
|
121
|
+
if (myParticipantId && data.participant.id === myParticipantId) {
|
|
122
|
+
this.logger.info('Ignoring own producer - no loopback:', data.producer.id);
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
this._handleProducerCreated(data.producer, data.participant);
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
// Producer closed (someone stopped publishing)
|
|
130
|
+
this.connection.onServerEvent('media.producer.closed', (data) => {
|
|
131
|
+
this.logger.info('Producer closed:', data.producerId);
|
|
132
|
+
this._handleProducerClosed(data.producerId, data.participantId);
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
// Consumer closed (server closed our consumer for a remote producer)
|
|
136
|
+
this.connection.onServerEvent('participant.consumer.close', (data) => {
|
|
137
|
+
this.logger.info('Consumer closed by server:', data.consumer?.id, 'for participant:', data.participant?.id);
|
|
138
|
+
if (data.consumer?.id) {
|
|
139
|
+
this._handleConsumerClosed(data.consumer.id);
|
|
140
|
+
}
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
// Note: Mute/unmute events are handled via 'participant.update' events
|
|
144
|
+
// which are listened to in VideoMeetingClient and forwarded to the app
|
|
145
|
+
|
|
146
|
+
this.listenersSetup = true;
|
|
147
|
+
this.logger.info('Server event listeners set up successfully');
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Handle new participant joined
|
|
152
|
+
* @private
|
|
153
|
+
*/
|
|
154
|
+
_handleParticipantJoined(participant) {
|
|
155
|
+
// Store participant data
|
|
156
|
+
this.participants.set(participant.id, participant);
|
|
157
|
+
|
|
158
|
+
// Initialize streams object for this participant
|
|
159
|
+
this.streams.set(participant.id, {
|
|
160
|
+
video: null,
|
|
161
|
+
audio: null,
|
|
162
|
+
screenShare: null,
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
this.emit('participant:added', { participant });
|
|
166
|
+
|
|
167
|
+
// Check if we should enable audio mixer
|
|
168
|
+
this._checkAudioMixerThreshold();
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Handle participant left
|
|
173
|
+
* @private
|
|
174
|
+
*/
|
|
175
|
+
_handleParticipantLeft(participantId) {
|
|
176
|
+
// Remove participant
|
|
177
|
+
const participant = this.participants.get(participantId);
|
|
178
|
+
this.participants.delete(participantId);
|
|
179
|
+
|
|
180
|
+
// Clean up all streams for this participant
|
|
181
|
+
const streams = this.streams.get(participantId);
|
|
182
|
+
if (streams) {
|
|
183
|
+
Object.entries(streams).forEach(([type, stream]) => {
|
|
184
|
+
if (stream) {
|
|
185
|
+
this.emit('stream:removed', { participantId, type, stream });
|
|
186
|
+
}
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
this.streams.delete(participantId);
|
|
190
|
+
|
|
191
|
+
// Clean up consumers
|
|
192
|
+
for (const [consumerId, data] of this.consumers) {
|
|
193
|
+
if (data.participantId === participantId) {
|
|
194
|
+
data.consumer.close();
|
|
195
|
+
this.consumers.delete(consumerId);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// Remove from audio mixer if enabled
|
|
200
|
+
if (this.useAudioMixer) {
|
|
201
|
+
this.audioMixer.removeParticipant(participantId);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// Remove volume setting
|
|
205
|
+
this.participantVolumes.delete(participantId);
|
|
206
|
+
|
|
207
|
+
this.emit('participant:removed', { participantId, participant });
|
|
208
|
+
|
|
209
|
+
// Check if we should disable audio mixer
|
|
210
|
+
this._checkAudioMixerThreshold();
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Handle participant updated
|
|
215
|
+
* @private
|
|
216
|
+
*/
|
|
217
|
+
_handleParticipantUpdated(participant) {
|
|
218
|
+
// Update participant data
|
|
219
|
+
this.participants.set(participant.id, participant);
|
|
220
|
+
|
|
221
|
+
this.emit('participant:updated', { participant });
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* Handle new producer created by remote participant
|
|
226
|
+
* @private
|
|
227
|
+
*/
|
|
228
|
+
async _handleProducerCreated(producer, participant) {
|
|
229
|
+
try {
|
|
230
|
+
// Don't consume our own producers
|
|
231
|
+
const localParticipantId = this.connection.socketId;
|
|
232
|
+
if (participant.id === localParticipantId) {
|
|
233
|
+
this.logger.log('Ignoring own producer');
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// Wait for device and transports to be ready before consuming
|
|
238
|
+
// Transports are created when media.transports event arrives from server
|
|
239
|
+
if (!this.mediasoup.device.loaded || !this.mediasoup.recvTransport) {
|
|
240
|
+
this.logger.warn('Device or transport not ready yet, queuing producer:', producer.id);
|
|
241
|
+
this.pendingProducers.push({ producer, participant });
|
|
242
|
+
return;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
// Emit event to let UI know we're attempting to consume a stream
|
|
246
|
+
const streamType = producer.producerType || producer.type || 'unknown';
|
|
247
|
+
this.emit('stream:consuming', {
|
|
248
|
+
participantId: participant.id,
|
|
249
|
+
type: streamType,
|
|
250
|
+
producerId: producer.id,
|
|
251
|
+
});
|
|
252
|
+
|
|
253
|
+
// Consume the producer (with retry logic)
|
|
254
|
+
const consumer = await this.mediasoup.consume(
|
|
255
|
+
producer.id,
|
|
256
|
+
participant.id
|
|
257
|
+
);
|
|
258
|
+
|
|
259
|
+
// Store consumer
|
|
260
|
+
this.consumers.set(consumer.id, {
|
|
261
|
+
consumer,
|
|
262
|
+
participantId: participant.id,
|
|
263
|
+
producerId: producer.id,
|
|
264
|
+
type: producer.producerType || producer.type || consumer.kind,
|
|
265
|
+
});
|
|
266
|
+
|
|
267
|
+
// Get the media stream from the consumer
|
|
268
|
+
const stream = new MediaStream([consumer.track]);
|
|
269
|
+
|
|
270
|
+
// Store stream by type
|
|
271
|
+
const participantStreams = this.streams.get(participant.id);
|
|
272
|
+
if (participantStreams) {
|
|
273
|
+
const streamType = producer.producerType || producer.type || consumer.kind;
|
|
274
|
+
participantStreams[streamType] = stream;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
this.logger.info('RemoteMediaManager :: Stream Consumed ::', {
|
|
278
|
+
participantId: participant.id,
|
|
279
|
+
type: producer.producerType || producer.type || consumer.kind,
|
|
280
|
+
consumerId: consumer.id,
|
|
281
|
+
});
|
|
282
|
+
|
|
283
|
+
const finalStreamType = producer.producerType || producer.type || consumer.kind;
|
|
284
|
+
|
|
285
|
+
// Add audio stream to mixer if enabled and it's an audio stream
|
|
286
|
+
if (this.useAudioMixer && finalStreamType === 'audio') {
|
|
287
|
+
const volume = this.participantVolumes.get(participant.id) || 1.0;
|
|
288
|
+
this.audioMixer.addParticipant(participant.id, stream, volume);
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// Emit stream added event
|
|
292
|
+
this.emit('stream:added', {
|
|
293
|
+
participantId: participant.id,
|
|
294
|
+
type: finalStreamType,
|
|
295
|
+
stream,
|
|
296
|
+
consumer,
|
|
297
|
+
});
|
|
298
|
+
|
|
299
|
+
// Handle consumer close
|
|
300
|
+
consumer.on('transportclose', () => {
|
|
301
|
+
this.logger.warn('Consumer transport closed:', consumer.id);
|
|
302
|
+
this._handleConsumerClosed(consumer.id);
|
|
303
|
+
});
|
|
304
|
+
|
|
305
|
+
consumer.on('trackended', () => {
|
|
306
|
+
this.logger.warn('Consumer track ended:', consumer.id);
|
|
307
|
+
this._handleConsumerClosed(consumer.id);
|
|
308
|
+
});
|
|
309
|
+
|
|
310
|
+
} catch (error) {
|
|
311
|
+
const streamType = producer.producerType || producer.type || 'unknown';
|
|
312
|
+
this.logger.error('RemoteMediaManager :: Failed to Consume Producer ::', {
|
|
313
|
+
participantId: participant.id,
|
|
314
|
+
producerId: producer.id,
|
|
315
|
+
type: streamType,
|
|
316
|
+
error: error.message
|
|
317
|
+
});
|
|
318
|
+
|
|
319
|
+
// Emit failure event so UI can show error or retry later
|
|
320
|
+
this.emit('stream:consume-failed', {
|
|
321
|
+
participantId: participant.id,
|
|
322
|
+
type: streamType,
|
|
323
|
+
producerId: producer.id,
|
|
324
|
+
error: error.message,
|
|
325
|
+
canRetryManually: true
|
|
326
|
+
});
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/**
|
|
331
|
+
* Handle producer closed by remote participant
|
|
332
|
+
* @private
|
|
333
|
+
*/
|
|
334
|
+
_handleProducerClosed(producerId, participantId) {
|
|
335
|
+
// Find and close the consumer for this producer
|
|
336
|
+
for (const [consumerId, data] of this.consumers) {
|
|
337
|
+
if (data.producerId === producerId) {
|
|
338
|
+
this._handleConsumerClosed(consumerId);
|
|
339
|
+
break;
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/**
|
|
345
|
+
* Handle consumer closed
|
|
346
|
+
* @private
|
|
347
|
+
*/
|
|
348
|
+
_handleConsumerClosed(consumerId) {
|
|
349
|
+
const consumerData = this.consumers.get(consumerId);
|
|
350
|
+
if (!consumerData) return;
|
|
351
|
+
|
|
352
|
+
const { consumer, participantId, type } = consumerData;
|
|
353
|
+
|
|
354
|
+
// Close consumer
|
|
355
|
+
consumer.close();
|
|
356
|
+
this.consumers.delete(consumerId);
|
|
357
|
+
|
|
358
|
+
// Remove stream
|
|
359
|
+
const participantStreams = this.streams.get(participantId);
|
|
360
|
+
if (participantStreams && participantStreams[type]) {
|
|
361
|
+
const stream = participantStreams[type];
|
|
362
|
+
participantStreams[type] = null;
|
|
363
|
+
|
|
364
|
+
this.emit('stream:removed', { participantId, type, stream });
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
this.logger.info('Consumer closed:', consumerId);
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
/**
|
|
371
|
+
* Get participant by ID
|
|
372
|
+
* @param {string} participantId - Participant ID
|
|
373
|
+
* @returns {Object|null} Participant data
|
|
374
|
+
*/
|
|
375
|
+
getParticipant(participantId) {
|
|
376
|
+
return this.participants.get(participantId) || null;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* Get all participants
|
|
381
|
+
* @returns {Array<Object>} Array of participant data
|
|
382
|
+
*/
|
|
383
|
+
getAllParticipants() {
|
|
384
|
+
return Array.from(this.participants.values());
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/**
|
|
388
|
+
* Get stream for a participant
|
|
389
|
+
* @param {string} participantId - Participant ID
|
|
390
|
+
* @param {string} type - Stream type (video, audio, screenShare)
|
|
391
|
+
* @returns {MediaStream|null}
|
|
392
|
+
*/
|
|
393
|
+
getStream(participantId, type) {
|
|
394
|
+
const streams = this.streams.get(participantId);
|
|
395
|
+
return streams ? streams[type] : null;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/**
|
|
399
|
+
* Get all streams for a participant
|
|
400
|
+
* @param {string} participantId - Participant ID
|
|
401
|
+
* @returns {Object|null} Object with video, audio, screenShare streams
|
|
402
|
+
*/
|
|
403
|
+
getStreams(participantId) {
|
|
404
|
+
return this.streams.get(participantId) || null;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* Check if participant has active video
|
|
409
|
+
* @param {string} participantId - Participant ID
|
|
410
|
+
* @returns {boolean}
|
|
411
|
+
*/
|
|
412
|
+
hasVideo(participantId) {
|
|
413
|
+
const streams = this.streams.get(participantId);
|
|
414
|
+
return !!(streams && streams.video);
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/**
|
|
418
|
+
* Check if participant has active audio
|
|
419
|
+
* @param {string} participantId - Participant ID
|
|
420
|
+
* @returns {boolean}
|
|
421
|
+
*/
|
|
422
|
+
hasAudio(participantId) {
|
|
423
|
+
const streams = this.streams.get(participantId);
|
|
424
|
+
return !!(streams && streams.audio);
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
/**
|
|
428
|
+
* Check if participant is sharing screen
|
|
429
|
+
* @param {string} participantId - Participant ID
|
|
430
|
+
* @returns {boolean}
|
|
431
|
+
*/
|
|
432
|
+
hasScreenShare(participantId) {
|
|
433
|
+
const streams = this.streams.get(participantId);
|
|
434
|
+
return !!(streams && streams.screenShare);
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
/**
|
|
438
|
+
* Get participant count
|
|
439
|
+
* @returns {number}
|
|
440
|
+
*/
|
|
441
|
+
get participantCount() {
|
|
442
|
+
return this.participants.size;
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
/**
|
|
446
|
+
* Process pending producers that arrived before transports were ready
|
|
447
|
+
* This should be called after transports are created
|
|
448
|
+
*/
|
|
449
|
+
async processPendingProducers() {
|
|
450
|
+
if (this.pendingProducers.length === 0) {
|
|
451
|
+
return;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
this.logger.info(`Processing ${this.pendingProducers.length} pending producers`);
|
|
455
|
+
|
|
456
|
+
// Process all pending producers
|
|
457
|
+
const pending = [...this.pendingProducers];
|
|
458
|
+
this.pendingProducers = [];
|
|
459
|
+
|
|
460
|
+
for (const { producer, participant } of pending) {
|
|
461
|
+
try {
|
|
462
|
+
await this._handleProducerCreated(producer, participant);
|
|
463
|
+
} catch (error) {
|
|
464
|
+
this.logger.error('Failed to process pending producer:', producer.id, error);
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
/**
|
|
470
|
+
* Calculate optimal spatial layer based on video element dimensions
|
|
471
|
+
* @private
|
|
472
|
+
* @param {number} width - Video element width in pixels
|
|
473
|
+
* @param {number} height - Video element height in pixels
|
|
474
|
+
* @returns {number} - Optimal spatial layer (0-2)
|
|
475
|
+
*/
|
|
476
|
+
_calculateOptimalSpatialLayer(width, height) {
|
|
477
|
+
// Use the larger dimension to determine quality
|
|
478
|
+
const maxDimension = Math.max(width, height);
|
|
479
|
+
|
|
480
|
+
// Spatial layer mapping (assumes standard simulcast layers):
|
|
481
|
+
// Layer 0: 320x180 - for thumbnails < 240px
|
|
482
|
+
// Layer 1: 960x540 - for medium tiles 240px - 540px
|
|
483
|
+
// Layer 2: 1920x1080 - for large tiles/fullscreen > 540px
|
|
484
|
+
|
|
485
|
+
if (maxDimension <= 240) {
|
|
486
|
+
return 0; // Low quality for small thumbnails
|
|
487
|
+
} else if (maxDimension <= 540) {
|
|
488
|
+
return 1; // Medium quality for smaller grid tiles
|
|
489
|
+
} else {
|
|
490
|
+
return 2; // High quality (1080p) for anything larger
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
/**
|
|
495
|
+
* Track a video element and automatically adjust consumer layers on resize
|
|
496
|
+
* @param {string} participantId - Participant ID
|
|
497
|
+
* @param {HTMLVideoElement} videoElement - Video element to track
|
|
498
|
+
*/
|
|
499
|
+
trackVideoElement(participantId, videoElement) {
|
|
500
|
+
if (!videoElement) {
|
|
501
|
+
this.logger.warn('Cannot track null video element for participant:', participantId);
|
|
502
|
+
return;
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
// Clean up any existing tracking for this participant
|
|
506
|
+
this.untrackVideoElement(participantId);
|
|
507
|
+
|
|
508
|
+
// Get the video consumer for this participant
|
|
509
|
+
const videoConsumer = this._getVideoConsumerForParticipant(participantId);
|
|
510
|
+
if (!videoConsumer) {
|
|
511
|
+
this.logger.warn('No video consumer found for participant:', participantId);
|
|
512
|
+
return;
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
// Create ResizeObserver to track element size changes
|
|
516
|
+
const observer = new ResizeObserver((entries) => {
|
|
517
|
+
for (const entry of entries) {
|
|
518
|
+
const { width, height } = entry.contentRect;
|
|
519
|
+
|
|
520
|
+
// Only update if size changed significantly (avoid tiny changes)
|
|
521
|
+
const tracking = this.videoElementTracking.get(participantId);
|
|
522
|
+
if (tracking) {
|
|
523
|
+
const widthDiff = Math.abs(width - tracking.lastWidth);
|
|
524
|
+
const heightDiff = Math.abs(height - tracking.lastHeight);
|
|
525
|
+
|
|
526
|
+
// Threshold: 50px change to avoid excessive updates
|
|
527
|
+
if (widthDiff < 50 && heightDiff < 50) {
|
|
528
|
+
return;
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
tracking.lastWidth = width;
|
|
532
|
+
tracking.lastHeight = height;
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
// Calculate optimal layer
|
|
536
|
+
const optimalLayer = this._calculateOptimalSpatialLayer(width, height);
|
|
537
|
+
|
|
538
|
+
// Update consumer preferred layers
|
|
539
|
+
this._updateConsumerLayers(videoConsumer, optimalLayer);
|
|
540
|
+
|
|
541
|
+
this.logger.log('Video element resized', {
|
|
542
|
+
participantId,
|
|
543
|
+
width: Math.round(width),
|
|
544
|
+
height: Math.round(height),
|
|
545
|
+
spatialLayer: optimalLayer,
|
|
546
|
+
});
|
|
547
|
+
}
|
|
548
|
+
});
|
|
549
|
+
|
|
550
|
+
// Start observing
|
|
551
|
+
observer.observe(videoElement);
|
|
552
|
+
|
|
553
|
+
// Store tracking info
|
|
554
|
+
this.videoElementTracking.set(participantId, {
|
|
555
|
+
element: videoElement,
|
|
556
|
+
observer,
|
|
557
|
+
lastWidth: videoElement.clientWidth,
|
|
558
|
+
lastHeight: videoElement.clientHeight,
|
|
559
|
+
});
|
|
560
|
+
|
|
561
|
+
// Set initial layer based on current size
|
|
562
|
+
const initialLayer = this._calculateOptimalSpatialLayer(
|
|
563
|
+
videoElement.clientWidth,
|
|
564
|
+
videoElement.clientHeight
|
|
565
|
+
);
|
|
566
|
+
this._updateConsumerLayers(videoConsumer, initialLayer);
|
|
567
|
+
|
|
568
|
+
this.logger.info('Started tracking video element', {
|
|
569
|
+
participantId,
|
|
570
|
+
width: videoElement.clientWidth,
|
|
571
|
+
height: videoElement.clientHeight,
|
|
572
|
+
initialLayer,
|
|
573
|
+
});
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
/**
|
|
577
|
+
* Stop tracking a video element
|
|
578
|
+
* @param {string} participantId - Participant ID
|
|
579
|
+
*/
|
|
580
|
+
untrackVideoElement(participantId) {
|
|
581
|
+
const tracking = this.videoElementTracking.get(participantId);
|
|
582
|
+
if (tracking) {
|
|
583
|
+
tracking.observer.disconnect();
|
|
584
|
+
this.videoElementTracking.delete(participantId);
|
|
585
|
+
this.logger.log('Stopped tracking video element for participant:', participantId);
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
/**
|
|
590
|
+
* Get video consumer for a participant
|
|
591
|
+
* @private
|
|
592
|
+
* @param {string} participantId - Participant ID
|
|
593
|
+
* @returns {Object|null} Consumer object
|
|
594
|
+
*/
|
|
595
|
+
_getVideoConsumerForParticipant(participantId) {
|
|
596
|
+
for (const [consumerId, data] of this.consumers) {
|
|
597
|
+
if (data.participantId === participantId && data.type === 'video') {
|
|
598
|
+
return data.consumer;
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
return null;
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
/**
|
|
605
|
+
* Update consumer preferred layers
|
|
606
|
+
* @private
|
|
607
|
+
* @param {Object} consumer - Mediasoup consumer
|
|
608
|
+
* @param {number} spatialLayer - Desired spatial layer (0-2)
|
|
609
|
+
*/
|
|
610
|
+
async _updateConsumerLayers(consumer, spatialLayer) {
|
|
611
|
+
if (!consumer || typeof consumer.setPreferredLayers !== 'function') {
|
|
612
|
+
// This is normal for audio consumers - they don't support simulcast
|
|
613
|
+
this.logger.log('RemoteMediaManager :: Skip Layer Update :: Consumer does not support simulcast layers (likely audio)');
|
|
614
|
+
return;
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
// Check if the consumer has multiple layers available (simulcast enabled)
|
|
618
|
+
// If there's only one layer, no point in setting preferred layers
|
|
619
|
+
if (consumer.rtpParameters?.encodings?.length <= 1) {
|
|
620
|
+
this.logger.log('RemoteMediaManager :: Skip Layer Update :: Consumer has single layer (simulcast not enabled)');
|
|
621
|
+
return;
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
try {
|
|
625
|
+
// Set preferred layers: spatialLayer and temporalLayer
|
|
626
|
+
// temporalLayer 2 = highest frame rate for the spatial layer
|
|
627
|
+
await consumer.setPreferredLayers({
|
|
628
|
+
spatialLayer: spatialLayer,
|
|
629
|
+
temporalLayer: 2,
|
|
630
|
+
});
|
|
631
|
+
|
|
632
|
+
this.logger.log('RemoteMediaManager :: Updated Consumer Layers ::', {
|
|
633
|
+
consumerId: consumer.id,
|
|
634
|
+
spatialLayer,
|
|
635
|
+
temporalLayer: 2,
|
|
636
|
+
});
|
|
637
|
+
} catch (error) {
|
|
638
|
+
this.logger.error('RemoteMediaManager :: Failed to Update Consumer Layers ::', error);
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
/**
|
|
643
|
+
* Clean up all remote media
|
|
644
|
+
*/
|
|
645
|
+
async cleanup() {
|
|
646
|
+
this.logger.info('Cleaning up remote media...');
|
|
647
|
+
|
|
648
|
+
// Stop tracking all video elements
|
|
649
|
+
for (const participantId of this.videoElementTracking.keys()) {
|
|
650
|
+
this.untrackVideoElement(participantId);
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
// Close all consumers
|
|
654
|
+
for (const [consumerId, data] of this.consumers) {
|
|
655
|
+
try {
|
|
656
|
+
data.consumer.close();
|
|
657
|
+
} catch (error) {
|
|
658
|
+
this.logger.error('Error closing consumer:', consumerId, error);
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
this.consumers.clear();
|
|
663
|
+
this.participants.clear();
|
|
664
|
+
this.streams.clear();
|
|
665
|
+
this.localMuteStates.clear();
|
|
666
|
+
|
|
667
|
+
this.logger.info('Remote media cleanup complete');
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
/**
|
|
671
|
+
* Locally mute a remote stream (for the local user only, doesn't affect other viewers)
|
|
672
|
+
* @param {string} participantId - Participant ID
|
|
673
|
+
* @param {string} type - Stream type (audio, screenShareAudio)
|
|
674
|
+
* @returns {boolean} Success
|
|
675
|
+
*/
|
|
676
|
+
localMuteRemoteStream(participantId, type) {
|
|
677
|
+
const key = `${participantId}:${type}`;
|
|
678
|
+
const streams = this.streams.get(participantId);
|
|
679
|
+
|
|
680
|
+
if (!streams) {
|
|
681
|
+
this.logger.warn('No streams found for participant:', participantId);
|
|
682
|
+
return false;
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
// Get the stream for the specified type
|
|
686
|
+
const stream = streams[type];
|
|
687
|
+
|
|
688
|
+
if (!stream) {
|
|
689
|
+
this.logger.warn('Stream not found:', type, 'for participant:', participantId);
|
|
690
|
+
return false;
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
// Mute audio tracks
|
|
694
|
+
const audioTracks = stream.getAudioTracks();
|
|
695
|
+
if (audioTracks.length === 0) {
|
|
696
|
+
this.logger.warn('No audio tracks found in stream:', type);
|
|
697
|
+
return false;
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
audioTracks.forEach(track => {
|
|
701
|
+
track.enabled = false;
|
|
702
|
+
});
|
|
703
|
+
|
|
704
|
+
this.localMuteStates.set(key, true);
|
|
705
|
+
this.logger.info('Locally muted remote stream:', participantId, type);
|
|
706
|
+
|
|
707
|
+
this.emit('local-mute-changed', { participantId, type, muted: true });
|
|
708
|
+
|
|
709
|
+
return true;
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
/**
|
|
713
|
+
* Locally unmute a remote stream (for the local user only)
|
|
714
|
+
* @param {string} participantId - Participant ID
|
|
715
|
+
* @param {string} type - Stream type (audio, screenShareAudio)
|
|
716
|
+
* @returns {boolean} Success
|
|
717
|
+
*/
|
|
718
|
+
localUnmuteRemoteStream(participantId, type) {
|
|
719
|
+
const key = `${participantId}:${type}`;
|
|
720
|
+
const streams = this.streams.get(participantId);
|
|
721
|
+
|
|
722
|
+
if (!streams) {
|
|
723
|
+
this.logger.warn('No streams found for participant:', participantId);
|
|
724
|
+
return false;
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
// Get the stream for the specified type
|
|
728
|
+
const stream = streams[type];
|
|
729
|
+
|
|
730
|
+
if (!stream) {
|
|
731
|
+
this.logger.warn('Stream not found:', type, 'for participant:', participantId);
|
|
732
|
+
return false;
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
// Unmute audio tracks
|
|
736
|
+
const audioTracks = stream.getAudioTracks();
|
|
737
|
+
if (audioTracks.length === 0) {
|
|
738
|
+
this.logger.warn('No audio tracks found in stream:', type);
|
|
739
|
+
return false;
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
audioTracks.forEach(track => {
|
|
743
|
+
track.enabled = true;
|
|
744
|
+
});
|
|
745
|
+
|
|
746
|
+
this.localMuteStates.set(key, false);
|
|
747
|
+
this.logger.info('Locally unmuted remote stream:', participantId, type);
|
|
748
|
+
|
|
749
|
+
this.emit('local-mute-changed', { participantId, type, muted: false });
|
|
750
|
+
|
|
751
|
+
return true;
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
/**
|
|
755
|
+
* Toggle local mute state for a remote stream
|
|
756
|
+
* @param {string} participantId - Participant ID
|
|
757
|
+
* @param {string} type - Stream type (audio, screenShareAudio)
|
|
758
|
+
* @returns {boolean} New mute state
|
|
759
|
+
*/
|
|
760
|
+
toggleLocalMuteRemoteStream(participantId, type) {
|
|
761
|
+
const key = `${participantId}:${type}`;
|
|
762
|
+
const currentlyMuted = this.localMuteStates.get(key) || false;
|
|
763
|
+
|
|
764
|
+
if (currentlyMuted) {
|
|
765
|
+
this.localUnmuteRemoteStream(participantId, type);
|
|
766
|
+
} else {
|
|
767
|
+
this.localMuteRemoteStream(participantId, type);
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
return this.localMuteStates.get(key) || false;
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
/**
|
|
774
|
+
* Get local mute state for a remote stream
|
|
775
|
+
* @param {string} participantId - Participant ID
|
|
776
|
+
* @param {string} type - Stream type
|
|
777
|
+
* @returns {boolean} Mute state
|
|
778
|
+
*/
|
|
779
|
+
isLocallyMuted(participantId, type) {
|
|
780
|
+
const key = `${participantId}:${type}`;
|
|
781
|
+
return this.localMuteStates.get(key) || false;
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
/**
|
|
785
|
+
* Set volume for a specific participant (0.0 to 1.0)
|
|
786
|
+
* Works with both audio elements and audio mixer
|
|
787
|
+
* @param {string} participantId - Participant ID
|
|
788
|
+
* @param {number} volume - Volume level (0.0 to 1.0)
|
|
789
|
+
* @returns {boolean} Success status
|
|
790
|
+
*/
|
|
791
|
+
setParticipantVolume(participantId, volume) {
|
|
792
|
+
// Clamp volume
|
|
793
|
+
const clampedVolume = Math.max(0, Math.min(1, volume));
|
|
794
|
+
|
|
795
|
+
console.log('SDK :: RemoteMediaManager :: setParticipantVolume:', {
|
|
796
|
+
participantId,
|
|
797
|
+
originalVolume: volume,
|
|
798
|
+
clampedVolume,
|
|
799
|
+
useAudioMixer: this.useAudioMixer,
|
|
800
|
+
mixerInitialized: this.audioMixer?.isInitialized
|
|
801
|
+
});
|
|
802
|
+
|
|
803
|
+
// Store volume setting
|
|
804
|
+
this.participantVolumes.set(participantId, clampedVolume);
|
|
805
|
+
|
|
806
|
+
// Apply to audio mixer if enabled
|
|
807
|
+
if (this.useAudioMixer && this.audioMixer.isInitialized) {
|
|
808
|
+
console.log('SDK :: RemoteMediaManager :: Using AudioMixer for volume control');
|
|
809
|
+
return this.audioMixer.setParticipantVolume(participantId, clampedVolume);
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
// If not using mixer, volume control happens at app level (audio elements)
|
|
813
|
+
// Emit event so app can update audio element volume
|
|
814
|
+
console.log('SDK :: RemoteMediaManager :: Emitting participant:volume-changed event');
|
|
815
|
+
this.emit('participant:volume-changed', {
|
|
816
|
+
participantId,
|
|
817
|
+
volume: clampedVolume,
|
|
818
|
+
});
|
|
819
|
+
|
|
820
|
+
return true;
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
/**
|
|
824
|
+
* Get volume for a specific participant
|
|
825
|
+
* @param {string} participantId - Participant ID
|
|
826
|
+
* @returns {number} Volume level (0.0 to 1.0)
|
|
827
|
+
*/
|
|
828
|
+
getParticipantVolume(participantId) {
|
|
829
|
+
if (this.useAudioMixer && this.audioMixer.isInitialized) {
|
|
830
|
+
const mixerVolume = this.audioMixer.getParticipantVolume(participantId);
|
|
831
|
+
if (mixerVolume !== null) {
|
|
832
|
+
return mixerVolume;
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
// Fix: Use ?? instead of || to allow 0 as valid volume (0 is falsy but valid)
|
|
836
|
+
const volume = this.participantVolumes.get(participantId);
|
|
837
|
+
return volume !== undefined ? volume : 1.0;
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
/**
|
|
841
|
+
* Retry consuming a failed producer
|
|
842
|
+
* Public method to allow manual retry from UI
|
|
843
|
+
* @param {string} producerId - Producer ID to retry
|
|
844
|
+
* @param {string} participantId - Participant ID
|
|
845
|
+
* @returns {Promise<boolean>} Success status
|
|
846
|
+
*/
|
|
847
|
+
async retryConsumeProducer(producerId, participantId) {
|
|
848
|
+
this.logger.info('RemoteMediaManager :: Manually Retrying Producer ::', {
|
|
849
|
+
producerId,
|
|
850
|
+
participantId
|
|
851
|
+
});
|
|
852
|
+
|
|
853
|
+
try {
|
|
854
|
+
// Find the producer in pending list or create a minimal producer object
|
|
855
|
+
const pendingProducer = this.pendingProducers.find(
|
|
856
|
+
p => p.producer.id === producerId && p.participant.id === participantId
|
|
857
|
+
);
|
|
858
|
+
|
|
859
|
+
if (pendingProducer) {
|
|
860
|
+
// Remove from pending and retry
|
|
861
|
+
this.pendingProducers = this.pendingProducers.filter(
|
|
862
|
+
p => !(p.producer.id === producerId && p.participant.id === participantId)
|
|
863
|
+
);
|
|
864
|
+
await this._handleProducerCreated(pendingProducer.producer, pendingProducer.participant);
|
|
865
|
+
} else {
|
|
866
|
+
// Create minimal producer object for retry
|
|
867
|
+
const participant = this.participants.get(participantId);
|
|
868
|
+
if (!participant) {
|
|
869
|
+
throw new Error('Participant not found');
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
await this._handleProducerCreated({ id: producerId }, participant);
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
return true;
|
|
876
|
+
} catch (error) {
|
|
877
|
+
this.logger.error('RemoteMediaManager :: Manual Retry Failed ::', error);
|
|
878
|
+
return false;
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
/**
|
|
883
|
+
* Check if audio mixer should be enabled based on participant count
|
|
884
|
+
* @private
|
|
885
|
+
*/
|
|
886
|
+
async _checkAudioMixerThreshold() {
|
|
887
|
+
const participantCount = this.participants.size;
|
|
888
|
+
|
|
889
|
+
// Should we enable the mixer?
|
|
890
|
+
const shouldUseMixer = participantCount >= this.MIXER_THRESHOLD;
|
|
891
|
+
|
|
892
|
+
// Already in correct state
|
|
893
|
+
if (shouldUseMixer === this.useAudioMixer) {
|
|
894
|
+
return;
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
if (shouldUseMixer && !this.useAudioMixer) {
|
|
898
|
+
// Enable mixer
|
|
899
|
+
this.logger.info(`Enabling audio mixer (${participantCount} participants >= ${this.MIXER_THRESHOLD})`);
|
|
900
|
+
await this._enableAudioMixer();
|
|
901
|
+
} else if (!shouldUseMixer && this.useAudioMixer) {
|
|
902
|
+
// Disable mixer
|
|
903
|
+
this.logger.info(`Disabling audio mixer (${participantCount} participants < ${this.MIXER_THRESHOLD})`);
|
|
904
|
+
await this._disableAudioMixer();
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
/**
|
|
909
|
+
* Enable audio mixer
|
|
910
|
+
* @private
|
|
911
|
+
*/
|
|
912
|
+
async _enableAudioMixer() {
|
|
913
|
+
try {
|
|
914
|
+
// Initialize mixer if not already
|
|
915
|
+
if (!this.audioMixer.isInitialized) {
|
|
916
|
+
await this.audioMixer.initialize();
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
// Resume context (may be suspended due to autoplay policy)
|
|
920
|
+
await this.audioMixer.resume();
|
|
921
|
+
|
|
922
|
+
// Add all current audio streams to mixer
|
|
923
|
+
for (const [participantId, streams] of this.streams) {
|
|
924
|
+
if (streams.audio) {
|
|
925
|
+
const volume = this.participantVolumes.get(participantId) || 1.0;
|
|
926
|
+
this.audioMixer.addParticipant(participantId, streams.audio, volume);
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
this.useAudioMixer = true;
|
|
931
|
+
|
|
932
|
+
// Emit event so app knows to stop using audio elements
|
|
933
|
+
this.emit('audio-mixer:enabled', {
|
|
934
|
+
participantCount: this.participants.size,
|
|
935
|
+
});
|
|
936
|
+
|
|
937
|
+
this.logger.info('Audio mixer enabled');
|
|
938
|
+
} catch (err) {
|
|
939
|
+
this.logger.error('Failed to enable audio mixer:', err);
|
|
940
|
+
}
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
/**
|
|
944
|
+
* Disable audio mixer
|
|
945
|
+
* @private
|
|
946
|
+
*/
|
|
947
|
+
async _disableAudioMixer() {
|
|
948
|
+
try {
|
|
949
|
+
// Clear all participants from mixer
|
|
950
|
+
this.audioMixer.clear();
|
|
951
|
+
|
|
952
|
+
this.useAudioMixer = false;
|
|
953
|
+
|
|
954
|
+
// Emit event so app knows to use audio elements again
|
|
955
|
+
this.emit('audio-mixer:disabled', {
|
|
956
|
+
participantCount: this.participants.size,
|
|
957
|
+
});
|
|
958
|
+
|
|
959
|
+
this.logger.info('Audio mixer disabled');
|
|
960
|
+
} catch (err) {
|
|
961
|
+
this.logger.error('Failed to disable audio mixer:', err);
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
|
|
965
|
+
/**
|
|
966
|
+
* Get whether audio mixer is currently active
|
|
967
|
+
* @returns {boolean}
|
|
968
|
+
*/
|
|
969
|
+
isAudioMixerEnabled() {
|
|
970
|
+
return this.useAudioMixer;
|
|
971
|
+
}
|
|
972
|
+
}
|