@voltkit-rtc/client 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/CHANGELOG.md +110 -0
  2. package/README.md +196 -24
  3. package/dist/cjs/chat.d.ts +2 -0
  4. package/dist/cjs/chat.d.ts.map +1 -1
  5. package/dist/cjs/chat.js +23 -4
  6. package/dist/cjs/chat.js.map +1 -1
  7. package/dist/cjs/device.d.ts +85 -0
  8. package/dist/cjs/device.d.ts.map +1 -0
  9. package/dist/cjs/device.js +134 -0
  10. package/dist/cjs/device.js.map +1 -0
  11. package/dist/cjs/index.d.ts +212 -6
  12. package/dist/cjs/index.d.ts.map +1 -1
  13. package/dist/cjs/index.js +822 -52
  14. package/dist/cjs/index.js.map +1 -1
  15. package/dist/cjs/music.d.ts +1 -1
  16. package/dist/cjs/music.d.ts.map +1 -1
  17. package/dist/cjs/quality.d.ts +43 -16
  18. package/dist/cjs/quality.d.ts.map +1 -1
  19. package/dist/cjs/quality.js +81 -31
  20. package/dist/cjs/quality.js.map +1 -1
  21. package/dist/cjs/stats.d.ts +151 -0
  22. package/dist/cjs/stats.d.ts.map +1 -0
  23. package/dist/cjs/stats.js +243 -0
  24. package/dist/cjs/stats.js.map +1 -0
  25. package/dist/esm/chat.d.ts +2 -0
  26. package/dist/esm/chat.d.ts.map +1 -1
  27. package/dist/esm/chat.js +23 -4
  28. package/dist/esm/chat.js.map +1 -1
  29. package/dist/esm/device.d.ts +85 -0
  30. package/dist/esm/device.d.ts.map +1 -0
  31. package/dist/esm/device.js +122 -0
  32. package/dist/esm/device.js.map +1 -0
  33. package/dist/esm/index.d.ts +212 -6
  34. package/dist/esm/index.d.ts.map +1 -1
  35. package/dist/esm/index.js +823 -53
  36. package/dist/esm/index.js.map +1 -1
  37. package/dist/esm/music.d.ts +1 -1
  38. package/dist/esm/music.d.ts.map +1 -1
  39. package/dist/esm/quality.d.ts +43 -16
  40. package/dist/esm/quality.d.ts.map +1 -1
  41. package/dist/esm/quality.js +76 -31
  42. package/dist/esm/quality.js.map +1 -1
  43. package/dist/esm/stats.d.ts +151 -0
  44. package/dist/esm/stats.d.ts.map +1 -0
  45. package/dist/esm/stats.js +232 -0
  46. package/dist/esm/stats.js.map +1 -0
  47. package/package.json +1 -1
  48. package/src/chat.ts +21 -4
  49. package/src/device.ts +168 -0
  50. package/src/index.ts +879 -58
  51. package/src/music.ts +1 -1
  52. package/src/quality.ts +96 -32
  53. package/src/stats.ts +328 -0
package/dist/cjs/index.js CHANGED
@@ -9,8 +9,10 @@ exports.StreamClient = exports.RpcError = exports.VIDEO_QUALITIES = exports.vide
9
9
  * setup, consumer resume ordering and reconnection, each slightly wrong.
10
10
  */
11
11
  const mediasoup_client_1 = require("mediasoup-client");
12
+ const device_js_1 = require("./device.js");
12
13
  const music_js_1 = require("./music.js");
13
14
  const quality_js_1 = require("./quality.js");
15
+ const stats_js_1 = require("./stats.js");
14
16
  var broadcast_js_1 = require("./broadcast.js");
15
17
  Object.defineProperty(exports, "BroadcastPublisher", { enumerable: true, get: function () { return broadcast_js_1.BroadcastPublisher; } });
16
18
  Object.defineProperty(exports, "WhipError", { enumerable: true, get: function () { return broadcast_js_1.WhipError; } });
@@ -64,6 +66,14 @@ const CLOSE_REPLACED = 4001;
64
66
  /** The server ended the session on purpose and said why first: do not reconnect. */
65
67
  const CLOSE_ENDED = 4003;
66
68
  const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
69
+ const isLayer = (value) => Number.isInteger(value) && value >= 0;
70
+ /**
71
+ * Where a new camera stream's bandwidth estimate starts, in kbps. Chrome
72
+ * otherwise starts low and takes many seconds to climb to the top layer.
73
+ */
74
+ const VIDEO_START_BITRATE_KBPS = 2000;
75
+ /** Why a publish gave up: unpublish() ran while it was under way, and unpublish wins. */
76
+ const unpublishedWhile = (what) => `the ${what} was unpublished while it was being published`;
67
77
  class StreamClient {
68
78
  wsUrl;
69
79
  ws = null;
@@ -94,11 +104,47 @@ class StreamClient {
94
104
  /** In-app music being published, kept across reconnects so it can be published again. */
95
105
  music = null;
96
106
  musicProducerId = null;
107
+ screenStream = null;
108
+ screenMode = 'detail';
109
+ screenAudioEnabled = true;
110
+ screenGeneration = 0;
111
+ screenPending = false;
112
+ screenStarted = false;
113
+ screenEndedListener = null;
114
+ screenProducerIds = new Set();
115
+ remoteVideoLayers = new Map();
116
+ receiveQualities = new Map();
117
+ cameraFrameRate = 30;
118
+ frameRateFallback = false;
119
+ frameRateLowSamples = 0;
120
+ frameRateWarmupSamples = 2;
97
121
  /** What each remote producer carries, so a pause can say "music" rather than just "audio". */
98
122
  remoteSources = new Map();
123
+ /**
124
+ * The layers asked for each remote video, by producerId. A reconnect
125
+ * consumes everything again, and the new consumer would otherwise start at
126
+ * the full picture.
127
+ */
128
+ preferredLayers = new Map();
99
129
  closedByUs = false;
100
130
  reconnecting = false;
101
131
  reconnectAttempt = 0;
132
+ /** Set once a connect() has joined and set up. Only such a session is reconnected. */
133
+ joinedOnce = false;
134
+ /** The camera producer's layers, to tell whether a new track fits them. */
135
+ cameraEncodings = null;
136
+ // Changes to the camera producer run one at a time. Two overlapping swaps
137
+ // (an effect toggled on and off quickly) would otherwise both close the
138
+ // same producer and both produce a new one.
139
+ cameraQueue = Promise.resolve();
140
+ // Bumped by unpublish(). A producer still being made when it runs (camera,
141
+ // microphone or music) belongs to the publish it ended: it is closed, not
142
+ // kept.
143
+ publishGeneration = 0;
144
+ codecFallbackLogged = false;
145
+ sendStatsBaseline = new Map();
146
+ recvStatsBaseline = new Map();
147
+ limitWatch = new stats_js_1.VideoLimitWatch((baseline) => this.sampleCamera(baseline), (event) => this.emit('videoLimited', event));
102
148
  constructor(wsUrl, token) {
103
149
  this.wsUrl = wsUrl;
104
150
  this.token = token;
@@ -110,7 +156,17 @@ class StreamClient {
110
156
  this.listeners.set(event, set);
111
157
  }
112
158
  set.add(fn);
113
- return () => set.delete(fn);
159
+ // videoLimited costs a getStats every 2 s, so it runs only while heard.
160
+ if (event === 'videoLimited')
161
+ this.syncLimitWatch();
162
+ return () => {
163
+ set.delete(fn);
164
+ if (event === 'videoLimited')
165
+ this.syncLimitWatch();
166
+ };
167
+ }
168
+ listenerCount(event) {
169
+ return this.listeners.get(event)?.size ?? 0;
114
170
  }
115
171
  emit(event, ...args) {
116
172
  for (const fn of this.listeners.get(event) ?? []) {
@@ -124,12 +180,52 @@ class StreamClient {
124
180
  }
125
181
  // ------------------------------------------------------------- connection
126
182
  async connect() {
127
- await this.openSocket();
128
- const result = (await this.rpc('join', { token: this.token }));
129
- this.joinResult = result;
130
- this.device = new mediasoup_client_1.Device();
131
- await this.device.load({ routerRtpCapabilities: result.routerRtpCapabilities });
183
+ let result;
184
+ try {
185
+ await this.openSocket();
186
+ result = (await this.rpc('join', { token: this.token }));
187
+ this.joinResult = result;
188
+ this.device = this.createDevice();
189
+ await this.device.load({ routerRtpCapabilities: result.routerRtpCapabilities });
190
+ }
191
+ catch (err) {
192
+ // A session that never got going: the app has the error. Left open,
193
+ // the socket would be closed by the server's join deadline instead, and
194
+ // that close would start reconnecting a session that never began.
195
+ if (!this.joinedOnce)
196
+ this.detachSocket();
197
+ throw err;
198
+ }
199
+ // Joined and set up: the next drop gets every reconnect attempt again.
200
+ // Not when the socket opens -- a join refused with a code worth retrying
201
+ // (no-capacity) never used them up, and every client of a full cluster
202
+ // retried about twice a second, forever.
203
+ this.reconnectAttempt = 0;
204
+ this.joinedOnce = true;
205
+ // A role or mute can change while the socket is away. The fresh roster
206
+ // decides what may return; reconnecting never turns a blocked device on.
207
+ if (result.role !== 'host' && result.role !== 'cohost' && (this.wasPublishing || this.screenPending)) {
208
+ await this.unpublish();
209
+ }
210
+ const own = result.participants.find((p) => p.identity === result.identity);
211
+ if (own?.muted)
212
+ this.applyLocalModeration('audio');
213
+ if (own?.videoBlocked)
214
+ this.applyLocalModeration('video');
132
215
  this.emit('joined', result);
216
+ // A video that ended while this client was away sends it no
217
+ // producerClosed: forget the layers asked for it.
218
+ const present = new Set(result.producers.map((p) => p.producerId));
219
+ for (const producerId of this.preferredLayers.keys()) {
220
+ if (!present.has(producerId)) {
221
+ this.preferredLayers.delete(producerId);
222
+ this.receiveQualities.delete(producerId);
223
+ this.remoteVideoLayers.delete(producerId);
224
+ }
225
+ }
226
+ for (const producerId of this.remoteVideoLayers.keys())
227
+ if (!present.has(producerId))
228
+ this.remoteVideoLayers.delete(producerId);
133
229
  // Subscribe to everything already publishing before we announce readiness.
134
230
  for (const p of result.producers) {
135
231
  await this.consume(p).catch((err) => this.emit('error', err));
@@ -142,6 +238,10 @@ class StreamClient {
142
238
  }
143
239
  return result;
144
240
  }
241
+ /** A new mediasoup Device for each join. Its own method so tests, which have no WebRTC, can stand one in. */
242
+ createDevice() {
243
+ return new mediasoup_client_1.Device();
244
+ }
145
245
  openSocket() {
146
246
  // Abandon any previous socket first. Its close handler checks identity
147
247
  // against `this.ws`, so once it is detached it can no longer trigger a
@@ -153,7 +253,6 @@ class StreamClient {
153
253
  const ws = new WebSocket(url.toString());
154
254
  this.ws = ws;
155
255
  const onOpen = () => {
156
- this.reconnectAttempt = 0;
157
256
  this.emit('connected');
158
257
  resolve();
159
258
  };
@@ -179,6 +278,9 @@ class StreamClient {
179
278
  // displace it, and it would displace us right back, forever.
180
279
  this.emit('error', new Error('this identity connected from somewhere else'));
181
280
  this.closedByUs = true;
281
+ // The producers outlive this close until the app calls close(); the
282
+ // videoLimited sampling must not, or it reports after `closed`.
283
+ this.limitWatch.reset();
182
284
  this.emit('closed', 'replaced');
183
285
  return;
184
286
  }
@@ -186,9 +288,14 @@ class StreamClient {
186
288
  // Normally roomClosed has already ended the session; this covers a
187
289
  // close that arrived without it.
188
290
  this.closedByUs = true;
291
+ this.limitWatch.reset();
189
292
  this.emit('closed', 'room-closed');
190
293
  return;
191
294
  }
295
+ // Before a first successful join the app is still waiting on
296
+ // connect(), which reports the failure; there is nothing to restore.
297
+ if (!this.joinedOnce)
298
+ return;
192
299
  void this.scheduleReconnect();
193
300
  });
194
301
  });
@@ -232,8 +339,13 @@ class StreamClient {
232
339
  return;
233
340
  try {
234
341
  this.teardownMedia();
235
- await this.connect();
236
- return; // connect() resets the attempt counter once the socket opens
342
+ await this.connect(); // which resets the attempt counter once joined
343
+ // The socket can drop again while the session is being restored,
344
+ // and its close handler then finds this loop still running and
345
+ // leaves it to us. Returning would leave the client disconnected
346
+ // for good, with no `closed`.
347
+ if (this.ws)
348
+ return;
237
349
  }
238
350
  catch (err) {
239
351
  this.emit('error', err);
@@ -257,8 +369,17 @@ class StreamClient {
257
369
  // others have already reported themselves and set closedByUs.
258
370
  const wasOpen = !this.closedByUs;
259
371
  this.closedByUs = true;
372
+ this.publishGeneration += 1;
260
373
  this.wasPublishing = false;
261
374
  this.teardownMedia();
375
+ this.preferredLayers.clear();
376
+ this.receiveQualities.clear();
377
+ this.remoteVideoLayers.clear();
378
+ this.releaseScreen();
379
+ this.music?.close();
380
+ this.music = null;
381
+ // Nothing is emitted after closing, so not even a clearing videoLimited.
382
+ this.limitWatch.reset();
262
383
  // Producers no longer stop their tracks, so releasing the camera and
263
384
  // microphone is ours to do -- otherwise the device light stays on.
264
385
  this.localStream?.getTracks().forEach((t) => t.stop());
@@ -277,6 +398,11 @@ class StreamClient {
277
398
  p.close();
278
399
  this.producers.clear();
279
400
  this.musicProducerId = null;
401
+ this.screenProducerIds.clear();
402
+ this.cameraEncodings = null;
403
+ // Paused, not reset: a reconnect's republish starts it again, and a
404
+ // limitation already reported is not reported twice.
405
+ this.syncLimitWatch();
280
406
  this.sendTransport?.close();
281
407
  this.recvTransport?.close();
282
408
  this.sendTransport = null;
@@ -345,6 +471,9 @@ class StreamClient {
345
471
  break;
346
472
  case 'producerClosed': {
347
473
  const { producerId, identity } = data;
474
+ this.preferredLayers.delete(producerId);
475
+ this.receiveQualities.delete(producerId);
476
+ this.remoteVideoLayers.delete(producerId);
348
477
  const consumer = this.consumers.get(producerId);
349
478
  if (!consumer)
350
479
  break; // never consumed it, or already cleaned up
@@ -368,16 +497,29 @@ class StreamClient {
368
497
  case 'participantJoined':
369
498
  this.emit('participantJoined', data);
370
499
  break;
500
+ case 'participantMuted': {
501
+ const wire = data;
502
+ const e = { identity: wire.identity, kind: wire.kind, paused: wire.paused ?? wire.muted ?? false };
503
+ if (this.joinResult)
504
+ this.joinResult.participants = this.joinResult.participants.map((p) => p.identity !== e.identity ? p : { ...p,
505
+ ...(!e.kind || e.kind === 'audio' ? { muted: e.paused } : {}),
506
+ ...(!e.kind || e.kind === 'video' ? { videoBlocked: e.paused } : {}),
507
+ });
508
+ this.emit('participantMuted', e);
509
+ break;
510
+ }
371
511
  case 'participantLeft':
372
512
  this.emit('participantLeft', data);
373
513
  break;
374
514
  case 'roleChanged': {
375
515
  const e = data;
516
+ if (e.identity === this.joinResult?.identity)
517
+ this.joinResult.role = e.role;
376
518
  this.emit('roleChanged', e);
377
519
  // Demoted: the server has already closed our producers. Release the
378
520
  // camera and microphone too, instead of capturing for nobody.
379
521
  const canPublish = e.role === 'host' || e.role === 'cohost';
380
- if (e.identity === this.joinResult?.identity && !canPublish && this.producers.size > 0) {
522
+ if (e.identity === this.joinResult?.identity && !canPublish && (this.wasPublishing || this.screenPending)) {
381
523
  void this.unpublish().catch(() => undefined);
382
524
  }
383
525
  break;
@@ -392,10 +534,7 @@ class StreamClient {
392
534
  // changes nothing here -- switching back on is the user's choice.
393
535
  const e = data;
394
536
  if (e.paused) {
395
- for (const p of this.producers.values()) {
396
- if (!e.kind || p.kind === e.kind)
397
- p.pause();
398
- }
537
+ this.applyLocalModeration(e.kind);
399
538
  this.emitLocalMedia();
400
539
  }
401
540
  this.emit('forceMuted', e);
@@ -483,7 +622,7 @@ class StreamClient {
483
622
  .catch(fail);
484
623
  });
485
624
  transport.on('produce', ({ kind, rtpParameters, appData }, ok, fail) => {
486
- this.rpc('produce', { transportId: transport.id, kind, rtpParameters, appData, source: appData.source })
625
+ this.rpc('produce', { transportId: transport.id, kind, rtpParameters, appData, source: appData.source, videoLayers: appData.videoLayers })
487
626
  .then((res) => ok({ id: res.producerId }))
488
627
  .catch(fail);
489
628
  });
@@ -524,15 +663,26 @@ class StreamClient {
524
663
  *
525
664
  * `options.quality` picks 1080p, 720p or 480p (see `videoPolicy` for what
526
665
  * the project allows). Capture at least that size for the full picture.
666
+ * `options.layers`, `codec` and `degradationPreference` tune how the video
667
+ * is encoded; the defaults suit both phones and computers. The options are
668
+ * kept for a reconnect's republish.
527
669
  */
528
670
  async publish(stream, constraints, options) {
671
+ const generation = this.publishGeneration;
529
672
  const transport = await this.ensureSendTransport();
673
+ // unpublish() ran first: this publish is over before it began, and must
674
+ // leave nothing behind that a reconnect would publish again.
675
+ if (generation !== this.publishGeneration)
676
+ throw new Error(unpublishedWhile('stream'));
530
677
  this.localStream = stream;
531
678
  this.wasPublishing = true;
532
679
  if (constraints)
533
680
  this.publishConstraints = constraints;
534
- if (options)
681
+ if (options) {
535
682
  this.publishOptions = { ...options };
683
+ this.frameRateFallback = false;
684
+ this.frameRateLowSamples = 0;
685
+ }
536
686
  const audio = stream.getAudioTracks()[0];
537
687
  if (audio) {
538
688
  const producer = await transport.produce({
@@ -543,13 +693,18 @@ class StreamClient {
543
693
  codecOptions: { opusStereo: true, opusDtx: true, opusFec: true },
544
694
  appData: { source: 'mic' },
545
695
  });
546
- this.producers.set(producer.id, producer);
696
+ await this.keepProducer(producer, generation, 'microphone');
697
+ // A microphone handed over switched off, or muted before a reconnect.
698
+ await this.pauseOnServer(producer);
547
699
  }
548
700
  const video = stream.getVideoTracks()[0];
549
- if (video)
550
- await this.produceCamera(transport, video);
701
+ if (video) {
702
+ // The track is read when the camera's turn comes: one handed to
703
+ // replaceVideoTrack while this publish was under way is the one to send.
704
+ await this.cameraTask(() => this.produceCamera(transport, this.localStream?.getVideoTracks()[0] ?? video, generation));
705
+ }
551
706
  }
552
- /** What this project may publish. Known once joined; 720p before that. */
707
+ /** What this project may publish. Known once joined; 1080p before that. */
553
708
  get videoPolicy() {
554
709
  return this.joinResult?.video ?? quality_js_1.DEFAULT_VIDEO_POLICY;
555
710
  }
@@ -565,36 +720,401 @@ class StreamClient {
565
720
  */
566
721
  async setVideoQuality(quality) {
567
722
  this.publishOptions = { ...this.publishOptions, quality };
568
- const current = [...this.producers.values()].find((p) => p.kind === 'video' && p.appData.source === 'camera');
569
- const track = current?.track;
570
- if (!current || !track || track.readyState !== 'live')
571
- return this.videoQuality;
723
+ await this.cameraTask(async () => {
724
+ const current = this.cameraProducer();
725
+ const track = current?.track;
726
+ if (!current || !track || track.readyState !== 'live')
727
+ return;
728
+ await this.reproduceCamera(current, track);
729
+ });
730
+ return this.videoQuality;
731
+ }
732
+ /** Change the camera frame rate without stopping screen, microphone or music. */
733
+ async setVideoFrameRate(frameRate) {
734
+ if (frameRate !== 30 && frameRate !== 60)
735
+ throw new RangeError('setVideoFrameRate: expected 30 or 60');
736
+ this.publishOptions = { ...this.publishOptions, frameRate };
737
+ this.frameRateFallback = false;
738
+ this.frameRateLowSamples = 0;
739
+ await this.cameraTask(async () => {
740
+ const current = this.cameraProducer();
741
+ const track = current?.track;
742
+ if (!current || !track || track.readyState !== 'live')
743
+ return;
744
+ try {
745
+ await track.applyConstraints?.({ frameRate: { ideal: frameRate, max: frameRate } });
746
+ }
747
+ catch { /* Existing capture settings decide whether 60 fps is possible. */ }
748
+ await this.reproduceCamera(current, track);
749
+ });
750
+ return this.cameraProducer() ? this.cameraFrameRate : this.videoPolicy.maxFrameRate === 30 ? 30 : frameRate;
751
+ }
752
+ /**
753
+ * Send a different video track from now on: a canvas with effects drawn on
754
+ * it, the raw camera again, another camera. When the new track needs the
755
+ * same dimensions and layers as the current one, viewers see no pause, and the layers
756
+ * take the new picture's bitrates. Otherwise the camera producer is
757
+ * replaced, as `setVideoQuality` does. Either way a paused camera stays
758
+ * paused.
759
+ *
760
+ * Your own stream is left as it is: the client keeps a stream of its own
761
+ * (`local`, announced by `localStreamChanged`) with the new track and your
762
+ * microphone, and a reconnect republishes that. The track you replaced is
763
+ * not stopped; it is yours again, switched back on if pausing the camera
764
+ * had switched it off. Rejects when no camera is being published.
765
+ */
766
+ async replaceVideoTrack(track) {
767
+ if (!track || track.kind !== 'video')
768
+ throw new TypeError('replaceVideoTrack needs a video track');
769
+ if (track.readyState === 'ended')
770
+ throw new Error('replaceVideoTrack: that track has ended');
771
+ await this.cameraTask(async () => {
772
+ const current = this.cameraProducer();
773
+ // Between a dropped connection and the republish there is no producer,
774
+ // but the camera is still being published: the republish sends this.
775
+ const reconnecting = !current && this.wasPublishing && (this.localStream?.getVideoTracks().length ?? 0) > 0;
776
+ if (!current && !reconnecting)
777
+ throw new Error('replaceVideoTrack: no camera is being published');
778
+ if (!current) {
779
+ // The republish reads the camera's state off the track it sends, and
780
+ // the track being replaced is switched off exactly when the camera is
781
+ // paused. Hand that state on, or a camera that was off comes back on;
782
+ // and the old track back switched on, as below.
783
+ const replaced = this.localStream?.getVideoTracks()[0];
784
+ if (replaced && replaced !== track) {
785
+ track.enabled = replaced.enabled;
786
+ replaced.enabled = true;
787
+ }
788
+ }
789
+ this.adoptVideoTrack(track);
790
+ if (!current || current.track === track)
791
+ return;
792
+ const previous = current.track;
793
+ const paused = current.paused;
794
+ try {
795
+ const { width = 0, height = 0 } = track.getSettings();
796
+ const next = (0, quality_js_1.videoEncodings)(width, height, this.videoQuality, this.layerCount(), this.frameRateFor(track));
797
+ const previousSize = current.track?.getSettings();
798
+ if (this.cameraEncodings && (0, quality_js_1.sameLayout)(this.cameraEncodings, next) &&
799
+ previousSize?.width === width && previousSize?.height === height) {
800
+ await current.replaceTrack({ track });
801
+ await this.applyLayerBitrates(current, next);
802
+ return;
803
+ }
804
+ await this.reproduceCamera(current, track);
805
+ }
806
+ finally {
807
+ // Pausing switched the old track off (mediasoup-client pauses that
808
+ // way), and nothing switches it back on once it is replaced. It is the
809
+ // app's again, often the camera its canvas draws from: handed back
810
+ // off, that canvas stays black after the camera is resumed.
811
+ if (paused && previous && previous !== this.cameraProducer()?.track)
812
+ previous.enabled = true;
813
+ }
814
+ });
815
+ }
816
+ /**
817
+ * Give a camera producer whose track was swapped in place the bitrates of
818
+ * its new picture. The layout matched, but a bigger picture at the same
819
+ * scales (a 1280×720 canvas for a 960×540 camera) would otherwise stay
820
+ * capped at the smaller one's. Best effort: a browser that refuses keeps
821
+ * the old caps.
822
+ */
823
+ async applyLayerBitrates(producer, encodings) {
824
+ const before = this.cameraEncodings;
825
+ if (before && before.every((encoding, i) => encoding.maxBitrate === encodings[i]?.maxBitrate)) {
826
+ this.cameraEncodings = encodings;
827
+ return;
828
+ }
829
+ const sender = producer.rtpSender;
830
+ if (!sender)
831
+ return;
832
+ try {
833
+ const parameters = sender.getParameters();
834
+ // One entry per layer, smallest first, as they were produced. A browser
835
+ // that reports them some other way is left with its own.
836
+ if (parameters.encodings.length !== encodings.length)
837
+ return;
838
+ parameters.encodings.forEach((encoding, i) => {
839
+ encoding.maxBitrate = encodings[i].maxBitrate;
840
+ });
841
+ await sender.setParameters(parameters);
842
+ this.cameraEncodings = encodings;
843
+ }
844
+ catch {
845
+ // Unsupported here; the layers keep the caps they were produced with.
846
+ }
847
+ }
848
+ /**
849
+ * What your camera video is sending now, layer by layer, or null when no
850
+ * camera is being published. The top-level fields describe the biggest
851
+ * layer going out. `qualityLimitation` says why it is less than asked:
852
+ * `cpu` is this device, `bandwidth` the upload, `none` neither (then check
853
+ * the camera and the light). Bitrates are averaged since the previous call,
854
+ * so call it every second or two; the first call reports 0.
855
+ */
856
+ async getVideoStats() {
857
+ return this.collectVideoStats(this.sendStatsBaseline);
858
+ }
859
+ /**
860
+ * What this client receives of a remote video (by its `producerId`, from
861
+ * the `track` event), or null when it is not receiving that video. Bitrates
862
+ * are averaged since the previous call for it; the first call reports 0.
863
+ */
864
+ async getReceiveStats(producerId) {
865
+ const consumer = this.consumers.get(producerId);
866
+ if (!consumer || consumer.kind !== 'video')
867
+ return null;
868
+ let report;
869
+ try {
870
+ report = await consumer.getStats();
871
+ }
872
+ catch (err) {
873
+ if (consumer.closed)
874
+ return null;
875
+ throw err;
876
+ }
877
+ const live = new Set([...this.consumers.values()].map((c) => c.id));
878
+ (0, stats_js_1.pruneBaseline)(this.recvStatsBaseline, (key) => live.has(key));
879
+ return (0, stats_js_1.parseReceiveStats)(report.values(), { producerId, baseline: this.recvStatsBaseline, key: consumer.id });
880
+ }
881
+ /**
882
+ * Receive a smaller copy of someone's video (by its `producerId`, from the
883
+ * `track` event), for a tile that does not need the full picture. Spatial
884
+ * layer 0 is the smallest copy the publisher sends: a computer sends three
885
+ * (at 720p 320×180, 640×360 and 1280×720), a phone two (640×360 and
886
+ * 1280×720), and a layer above the biggest means the biggest. Leave
887
+ * `temporalLayer` out to keep the full frame rate. It is the most this
888
+ * client is sent: a connection that cannot carry it still gets less. The
889
+ * choice is kept across reconnects and forgotten when the track ends.
890
+ * Rejects for a producer whose video this client is not receiving.
891
+ */
892
+ async setPreferredLayers(producerId, spatialLayer, temporalLayer) {
893
+ if (!isLayer(spatialLayer) || (temporalLayer !== undefined && !isLayer(temporalLayer))) {
894
+ throw new RangeError('setPreferredLayers: layers are whole numbers from 0');
895
+ }
896
+ const consumer = this.consumers.get(producerId);
897
+ if (!consumer || consumer.kind !== 'video') {
898
+ throw new Error(`setPreferredLayers: this client is not receiving a video with producerId ${producerId}`);
899
+ }
900
+ const layers = temporalLayer === undefined ? { spatialLayer } : { spatialLayer, temporalLayer };
901
+ this.receiveQualities.delete(producerId);
902
+ // Kept first: a reconnect that interrupts the request still applies it.
903
+ this.preferredLayers.set(producerId, layers);
904
+ await this.rpc('setPreferredLayers', { consumerId: consumer.id, ...layers });
905
+ }
906
+ async collectVideoStats(baseline) {
907
+ const producer = this.cameraProducer();
908
+ if (!producer)
909
+ return null;
910
+ let report;
911
+ try {
912
+ report = await producer.getStats();
913
+ }
914
+ catch (err) {
915
+ if (producer.closed)
916
+ return null;
917
+ throw err;
918
+ }
919
+ const prefix = `${producer.id}:`;
920
+ (0, stats_js_1.pruneBaseline)(baseline, (key) => key.startsWith(prefix));
921
+ return (0, stats_js_1.parseSendStats)(report.values(), { quality: this.videoQuality, baseline, keyPrefix: prefix });
922
+ }
923
+ /** Friendly receive ceiling using this producer's real layer sizes; auto restores adaptation. */
924
+ async setReceiveQuality(producerId, quality) {
925
+ if (!['auto', '1080p', '720p', '480p'].includes(quality))
926
+ throw new RangeError('setReceiveQuality: unsupported quality');
927
+ const consumer = this.consumers.get(producerId);
928
+ if (!consumer || consumer.kind !== 'video')
929
+ throw new Error('setReceiveQuality: this client is not receiving that video');
930
+ let layers;
931
+ if (quality === 'auto')
932
+ layers = { spatialLayer: -1, temporalLayer: -1 };
933
+ else {
934
+ try {
935
+ layers = { spatialLayer: (0, quality_js_1.receiveSpatialLayer)(this.remoteVideoLayers.get(producerId) ?? [], quality) };
936
+ }
937
+ catch {
938
+ throw new RpcError('quality-unavailable', 'this publisher does not describe its video layers; use Auto');
939
+ }
940
+ }
941
+ this.receiveQualities.set(producerId, quality);
942
+ this.preferredLayers.set(producerId, layers);
943
+ await this.rpc('setPreferredLayers', { consumerId: consumer.id, ...layers });
944
+ }
945
+ cameraProducer() {
946
+ for (const p of this.producers.values()) {
947
+ if (p.kind === 'video' && p.appData.source === 'camera')
948
+ return p;
949
+ }
950
+ return undefined;
951
+ }
952
+ cameraTask(task) {
953
+ const run = this.cameraQueue.then(task, task);
954
+ this.cameraQueue = run.catch(() => undefined);
955
+ return run;
956
+ }
957
+ /** Swap the video in the client's own stream, never the app's. */
958
+ adoptVideoTrack(track) {
959
+ const previous = this.localStream;
960
+ if (previous && previous.getVideoTracks()[0] === track)
961
+ return;
962
+ const others = previous ? previous.getTracks().filter((t) => t.kind !== 'video') : [];
963
+ this.localStream = new MediaStream([...others, track]);
964
+ this.emit('localStreamChanged', this.localStream);
965
+ }
966
+ /** Close the camera producer and produce `track` in its place, paused on both ends if it was. */
967
+ async reproduceCamera(current, track) {
968
+ const generation = this.publishGeneration;
572
969
  const paused = current.paused;
573
970
  current.close();
574
971
  this.producers.delete(current.id);
972
+ this.cameraEncodings = null;
973
+ this.syncLimitWatch();
575
974
  await this.rpc('closeProducer', { producerId: current.id }).catch(() => undefined);
576
- const producer = await this.produceCamera(await this.ensureSendTransport(), track);
577
- if (paused) {
578
- producer.pause();
579
- await this.rpc('pauseProducer', { producerId: producer.id, paused: true });
580
- }
581
- return this.videoQuality;
975
+ return this.produceCamera(await this.ensureSendTransport(), track, generation, paused);
976
+ }
977
+ /** `options.layers` when valid, else 2 on phones and tablets and 3 elsewhere. */
978
+ layerCount() {
979
+ const asked = this.publishOptions.layers;
980
+ return asked === 1 || asked === 2 || asked === 3 ? asked : (0, device_js_1.defaultLayerCount)();
582
981
  }
583
- async produceCamera(transport, track) {
982
+ /**
983
+ * Produce the camera from `track`, paused on both ends when `paused`. By
984
+ * default the track says: one handed over switched off is a camera that is
985
+ * off, and so is one a pause switched off before a reconnect.
986
+ */
987
+ async produceCamera(transport, track, generation, paused = !track.enabled) {
988
+ if (generation !== this.publishGeneration)
989
+ throw new Error(unpublishedWhile('camera'));
990
+ // mediasoup-client starts a producer paused exactly when its track is
991
+ // off. Setting the track to the camera's state first makes the producer
992
+ // start in it, and a camera that is off never sends a frame of the new
993
+ // track; the server is told below.
994
+ track.enabled = !paused;
584
995
  // Simulcast from day one: it is what lets one node serve far more viewers.
585
- // The layers follow the picture's real size and the project's quality, so
586
- // the browser sends every layer asked for (see videoEncodings).
996
+ // The layers follow the picture's real size, the project's quality and
997
+ // the device, so the browser sends every layer asked for (see
998
+ // videoEncodings).
587
999
  const { width = 0, height = 0 } = track.getSettings();
588
- const producer = await transport.produce({
1000
+ const quality = this.videoQuality;
1001
+ this.cameraFrameRate = this.frameRateFor(track);
1002
+ const encodings = (0, quality_js_1.videoEncodings)(width, height, quality, this.layerCount(), this.cameraFrameRate);
1003
+ let codecs;
1004
+ try {
1005
+ codecs = this.device?.sendRtpCapabilities.codecs;
1006
+ }
1007
+ catch {
1008
+ codecs = undefined;
1009
+ }
1010
+ const chosen = await this.cameraCodec(codecs, encodings, width, height, quality);
1011
+ const produce = (codec) => transport.produce({
589
1012
  track,
590
1013
  stopTracks: false,
591
- encodings: (0, quality_js_1.videoEncodings)(width, height, this.videoQuality),
592
- codecOptions: { videoGoogleStartBitrate: 1000 },
593
- appData: { source: 'camera' },
1014
+ encodings,
1015
+ codecOptions: { videoGoogleStartBitrate: VIDEO_START_BITRATE_KBPS },
1016
+ ...(codec ? { codec } : {}),
1017
+ appData: { source: 'camera', videoLayers: (0, quality_js_1.describeVideoLayers)(width, height, encodings, quality) },
594
1018
  });
595
- this.producers.set(producer.id, producer);
1019
+ let producer;
1020
+ try {
1021
+ producer = await produce(chosen.capability);
1022
+ }
1023
+ catch (err) {
1024
+ // Never fail a publish over the codec: H.264 refused is VP8 instead.
1025
+ // A closed transport (a reconnect under way) fails either way.
1026
+ if (chosen.codec !== 'h264' || transport.closed)
1027
+ throw err;
1028
+ this.reportCodecFallback(`H.264 was refused (${err.message}): publishing VP8 instead`);
1029
+ producer = await produce((0, device_js_1.findCodec)(codecs, 'vp8'));
1030
+ }
1031
+ // unpublish() may have run while this was being made (a publish, a
1032
+ // quality change or a track swap under way).
1033
+ await this.keepProducer(producer, generation, 'camera');
1034
+ this.cameraEncodings = encodings;
1035
+ this.frameRateLowSamples = 0;
1036
+ this.frameRateWarmupSamples = 2;
1037
+ await this.pauseOnServer(producer);
1038
+ await this.applyDegradationPreference(producer);
1039
+ this.syncLimitWatch();
596
1040
  return producer;
597
1041
  }
1042
+ /**
1043
+ * Keep a producer just made -- unless unpublish() ran while it was being
1044
+ * made. Kept, it would publish again what was just unpublished, so it is
1045
+ * closed instead, on both ends.
1046
+ */
1047
+ async keepProducer(producer, generation, what) {
1048
+ if (generation !== this.publishGeneration) {
1049
+ producer.close();
1050
+ await this.rpc('closeProducer', { producerId: producer.id }).catch(() => undefined);
1051
+ throw new Error(unpublishedWhile(what));
1052
+ }
1053
+ this.producers.set(producer.id, producer);
1054
+ }
1055
+ /**
1056
+ * mediasoup-client makes a producer paused when its track is off, and the
1057
+ * server makes every producer live. Unless the server is told, viewers get
1058
+ * black video or silence from a camera or microphone they are told is on.
1059
+ */
1060
+ async pauseOnServer(producer) {
1061
+ if (producer.paused)
1062
+ await this.rpc('pauseProducer', { producerId: producer.id, paused: true });
1063
+ }
1064
+ /**
1065
+ * H.264 or VP8 for the camera (see chooseCodec), with the device's own
1066
+ * capability entry for it: produce() must only ever get one of those.
1067
+ */
1068
+ async cameraCodec(codecs, encodings, width, height, quality) {
1069
+ const requested = (0, device_js_1.normalizeCodec)(this.publishOptions.codec);
1070
+ const env = (0, device_js_1.currentEnv)();
1071
+ const phoneOrTablet = (0, device_js_1.isPhoneOrTablet)(env);
1072
+ const h264 = (0, device_js_1.findCodec)(codecs, 'h264');
1073
+ let h264Hardware = null;
1074
+ if (requested === 'auto' && phoneOrTablet && h264) {
1075
+ const top = (0, quality_js_1.topLayerSize)(width, height, quality);
1076
+ h264Hardware = await (0, device_js_1.h264HardwareEncode)({ width: top.width, height: top.height, bitrate: encodings[encodings.length - 1].maxBitrate });
1077
+ }
1078
+ const choice = (0, device_js_1.chooseCodec)({ requested, phoneOrTablet, appleMobile: (0, device_js_1.isAppleMobile)(env), h264Available: Boolean(h264), h264Hardware });
1079
+ if (choice.warning)
1080
+ this.reportCodecFallback(choice.warning);
1081
+ else if (requested === 'auto' && phoneOrTablet && !h264)
1082
+ this.reportCodecFallback('this browser cannot send H.264 to this server: publishing VP8');
1083
+ if (choice.codec === 'h264' && h264)
1084
+ return { codec: 'h264', capability: h264 };
1085
+ return { codec: 'vp8', capability: (0, device_js_1.findCodec)(codecs, 'vp8') };
1086
+ }
1087
+ /** A forced H.264 that could not be used is the app's to hear about; 'auto' just notes it, once. */
1088
+ reportCodecFallback(message) {
1089
+ if ((0, device_js_1.normalizeCodec)(this.publishOptions.codec) === 'h264') {
1090
+ this.emit('error', new Error(message));
1091
+ }
1092
+ else if (!this.codecFallbackLogged) {
1093
+ this.codecFallbackLogged = true;
1094
+ console.warn(`[voltkit-rtc] ${message}`);
1095
+ }
1096
+ }
1097
+ /** Best effort: a browser that ignores or refuses it keeps its own default. */
1098
+ async applyDegradationPreference(producer) {
1099
+ const preference = this.publishOptions.degradationPreference;
1100
+ const sender = producer.rtpSender;
1101
+ if (!sender)
1102
+ return;
1103
+ if (preference !== 'maintain-resolution' && preference !== 'maintain-framerate' && preference !== 'balanced')
1104
+ return;
1105
+ try {
1106
+ const parameters = sender.getParameters();
1107
+ parameters.degradationPreference = preference;
1108
+ await sender.setParameters(parameters);
1109
+ }
1110
+ catch {
1111
+ // Unsupported here; the encoder keeps the browser's own trade-off.
1112
+ }
1113
+ }
1114
+ /** videoLimited sampling runs while a camera is published and someone listens. */
1115
+ syncLimitWatch() {
1116
+ this.limitWatch.set(!this.closedByUs && this.cameraProducer() !== undefined && (this.listenerCount('videoLimited') > 0 || this.cameraFrameRate === 60));
1117
+ }
598
1118
  /**
599
1119
  * Re-establish this client's broadcast after a reconnect.
600
1120
  *
@@ -603,10 +1123,22 @@ class StreamClient {
603
1123
  * when the tracks have genuinely ended.
604
1124
  */
605
1125
  async republish() {
1126
+ // unpublish() can run while this is under way, and wins: nothing it
1127
+ // stopped is published again.
1128
+ const generation = this.publishGeneration;
606
1129
  // Music is not a device and cannot be "gone": publish it again whatever
607
1130
  // happens to the camera and microphone below.
1131
+ const screen = this.screenStream;
1132
+ if (screen)
1133
+ await this.restoreScreen().catch(async (err) => {
1134
+ if (this.screenStream === screen)
1135
+ await this.stopScreen('stopped');
1136
+ this.emit('error', err);
1137
+ });
608
1138
  if (this.music)
609
- await this.produceMusic(this.music).catch((err) => this.emit('error', err));
1139
+ await this.produceMusic(this.music, generation).catch((err) => this.emit('error', err));
1140
+ if (generation !== this.publishGeneration)
1141
+ return;
610
1142
  let stream = this.localStream;
611
1143
  if (!stream && !this.publishConstraints)
612
1144
  return; // music was all that was being published
@@ -616,7 +1148,20 @@ class StreamClient {
616
1148
  this.emit('error', new Error('cannot restore the broadcast: the devices are gone'));
617
1149
  return;
618
1150
  }
1151
+ const ended = stream;
619
1152
  stream = await navigator.mediaDevices.getUserMedia(this.publishConstraints);
1153
+ if (generation !== this.publishGeneration) {
1154
+ for (const track of stream.getTracks())
1155
+ track.stop();
1156
+ return;
1157
+ }
1158
+ // New devices come back switched on. A camera or microphone the user
1159
+ // had switched off stays off: a dropped connection is no reason to
1160
+ // switch it back on.
1161
+ for (const track of stream.getTracks()) {
1162
+ if (ended?.getTracks().some((t) => t.kind === track.kind && !t.enabled))
1163
+ track.enabled = false;
1164
+ }
620
1165
  this.localStream = stream;
621
1166
  this.emit('localStreamChanged', stream);
622
1167
  }
@@ -624,6 +1169,170 @@ class StreamClient {
624
1169
  return;
625
1170
  await this.publish(stream);
626
1171
  }
1172
+ // ------------------------------------------------------------- screen sharing
1173
+ /** Publish a screen beside the camera. Call from a click so capture permission can open. */
1174
+ async publishScreen(options = {}) {
1175
+ if (this.joinResult?.role !== 'host' && this.joinResult?.role !== 'cohost')
1176
+ throw new RpcError('forbidden', 'only hosts and cohosts can share a screen');
1177
+ if (this.screenStream || this.screenPending)
1178
+ throw new RpcError('screen-share-active', 'a screen share is already active');
1179
+ if (options.mode !== undefined && options.mode !== 'detail' && options.mode !== 'motion')
1180
+ throw new TypeError('publishScreen: mode must be detail or motion');
1181
+ const mode = options.mode ?? 'detail';
1182
+ if (!options.stream && (typeof navigator === 'undefined' || !navigator.mediaDevices?.getDisplayMedia)) {
1183
+ throw new RpcError('screen-share-unsupported', 'screen capture is not supported by this browser; use a supported desktop browser');
1184
+ }
1185
+ const generation = this.screenGeneration;
1186
+ this.screenPending = true;
1187
+ let stream;
1188
+ try {
1189
+ stream = options.stream ?? await navigator.mediaDevices.getDisplayMedia({
1190
+ video: { width: { ideal: 1920, max: 1920 }, height: { ideal: 1080, max: 1080 }, frameRate: { ideal: mode === 'motion' ? 30 : 15, max: mode === 'motion' ? 30 : 15 } },
1191
+ audio: options.audio ?? false,
1192
+ });
1193
+ const video = stream.getVideoTracks()[0];
1194
+ if (!video || video.readyState !== 'live')
1195
+ throw new RpcError('screen-share-no-video', 'screen capture did not provide a live video track');
1196
+ if (generation !== this.screenGeneration || this.closedByUs)
1197
+ throw new Error(unpublishedWhile('screen'));
1198
+ this.screenStream = stream;
1199
+ this.screenMode = mode;
1200
+ this.screenAudioEnabled = options.audio !== false;
1201
+ video.contentHint = mode;
1202
+ const onEnded = () => { void this.stopScreen('track-ended').catch((err) => this.emit('error', err)); };
1203
+ this.screenEndedListener = onEnded;
1204
+ video.addEventListener('ended', onEnded, { once: true });
1205
+ await this.restoreScreen();
1206
+ if (generation !== this.screenGeneration || video.readyState !== 'live')
1207
+ throw new Error(unpublishedWhile('screen'));
1208
+ this.screenStarted = true;
1209
+ this.wasPublishing = true;
1210
+ this.emit('screenShareStarted', { stream, mode });
1211
+ return stream;
1212
+ }
1213
+ catch (error) {
1214
+ if (generation === this.screenGeneration)
1215
+ await this.stopScreen('stopped');
1216
+ stream?.getTracks().forEach((track) => track.stop());
1217
+ throw error;
1218
+ }
1219
+ finally {
1220
+ if (generation === this.screenGeneration)
1221
+ this.screenPending = false;
1222
+ }
1223
+ }
1224
+ /** Stop only the screen and its tab/system audio; keep camera, microphone and music. */
1225
+ async unpublishScreen() {
1226
+ await this.stopScreen('stopped');
1227
+ }
1228
+ releaseScreen() {
1229
+ this.screenGeneration += 1;
1230
+ this.screenPending = false;
1231
+ const stream = this.screenStream;
1232
+ if (this.screenEndedListener)
1233
+ stream?.getVideoTracks()[0]?.removeEventListener('ended', this.screenEndedListener);
1234
+ this.screenEndedListener = null;
1235
+ this.screenStream = null;
1236
+ this.screenStarted = false;
1237
+ stream?.getTracks().forEach((track) => track.stop());
1238
+ }
1239
+ async stopScreen(reason) {
1240
+ const started = this.screenStarted;
1241
+ const ids = [...this.screenProducerIds];
1242
+ this.screenProducerIds.clear();
1243
+ this.releaseScreen();
1244
+ for (const id of ids) {
1245
+ this.producers.get(id)?.close();
1246
+ this.producers.delete(id);
1247
+ }
1248
+ if (!this.localStream && !this.music)
1249
+ this.wasPublishing = false;
1250
+ if (started && !this.closedByUs)
1251
+ this.emit('screenShareEnded', { reason });
1252
+ await Promise.all(ids.map((producerId) => this.rpc('closeProducer', { producerId }).catch(() => undefined)));
1253
+ }
1254
+ async restoreScreen() {
1255
+ const stream = this.screenStream;
1256
+ const video = stream?.getVideoTracks()[0];
1257
+ if (!stream || !video)
1258
+ return;
1259
+ if (video.readyState !== 'live') {
1260
+ await this.stopScreen('track-ended');
1261
+ return;
1262
+ }
1263
+ const generation = this.screenGeneration;
1264
+ const publishing = this.publishGeneration;
1265
+ const transport = await this.ensureSendTransport();
1266
+ if (generation !== this.screenGeneration)
1267
+ return;
1268
+ const { width = 0, height = 0 } = video.getSettings();
1269
+ const encodings = (0, quality_js_1.screenEncodings)(width, height, this.screenMode);
1270
+ // A project's explicit safety restriction also applies to screen video.
1271
+ const cap = (0, quality_js_1.topLayerSize)(width, height, this.videoPolicy.maxQuality);
1272
+ const screenScale = encodings.at(-1).scaleResolutionDownBy;
1273
+ if (cap.scale > screenScale)
1274
+ encodings.forEach((e) => { e.scaleResolutionDownBy *= cap.scale / screenScale; });
1275
+ const keep = async (producer) => {
1276
+ if (generation !== this.screenGeneration || publishing !== this.publishGeneration || video.readyState !== 'live') {
1277
+ producer.close();
1278
+ await this.rpc('closeProducer', { producerId: producer.id }).catch(() => undefined);
1279
+ throw new Error(unpublishedWhile('screen'));
1280
+ }
1281
+ this.producers.set(producer.id, producer);
1282
+ this.screenProducerIds.add(producer.id);
1283
+ await this.pauseOnServer(producer);
1284
+ };
1285
+ await keep(await transport.produce({
1286
+ track: video, stopTracks: false, encodings,
1287
+ codecOptions: { videoGoogleStartBitrate: VIDEO_START_BITRATE_KBPS },
1288
+ appData: { source: 'screen', videoLayers: (0, quality_js_1.describeVideoLayers)(width, height, encodings) },
1289
+ }));
1290
+ const audio = stream.getAudioTracks().find((track) => track.readyState === 'live');
1291
+ if (audio && this.screenAudioEnabled && generation === this.screenGeneration)
1292
+ await keep(await transport.produce({
1293
+ track: audio, stopTracks: false,
1294
+ codecOptions: { opusStereo: true, opusDtx: false, opusFec: true, opusMaxAverageBitrate: 128000 },
1295
+ appData: { source: 'screen-audio' },
1296
+ }));
1297
+ }
1298
+ frameRateFor(track) {
1299
+ if (this.publishOptions.frameRate !== 60 || this.frameRateFallback)
1300
+ return 30;
1301
+ const captured = track.getSettings().frameRate;
1302
+ if (this.videoPolicy.maxFrameRate === 30 || (typeof captured === 'number' && captured < 50)) {
1303
+ this.frameRateFallback = true;
1304
+ this.emit('error', new RpcError('frame-rate-fallback', '60 fps is unavailable from this camera or project; publishing at 30 fps'));
1305
+ return 30;
1306
+ }
1307
+ return 60;
1308
+ }
1309
+ async sampleCamera(baseline) {
1310
+ const stats = await this.collectVideoStats(baseline);
1311
+ if (this.cameraFrameRate !== 60 || this.frameRateFallback || !stats)
1312
+ return stats;
1313
+ if (this.cameraProducer()?.paused)
1314
+ return stats;
1315
+ // The lower simulcast copies intentionally run at 30 fps. During the
1316
+ // initial bandwidth ramp they may be the only active copies, which does
1317
+ // not establish that the camera cannot sustain 60. Match native warm-up:
1318
+ // two 2-second samples, then require three consecutive limited samples.
1319
+ if (this.frameRateWarmupSamples-- > 0)
1320
+ return stats;
1321
+ const limited = stats.layers.some((layer) => layer.active && (layer.qualityLimitation === 'cpu' || layer.qualityLimitation === 'bandwidth'));
1322
+ this.frameRateLowSamples = limited && stats.fps > 0 && stats.fps < 50 ? this.frameRateLowSamples + 1 : 0;
1323
+ if (this.frameRateLowSamples >= 3) {
1324
+ this.frameRateFallback = true;
1325
+ void this.cameraTask(async () => {
1326
+ const producer = this.cameraProducer();
1327
+ if (!producer?.track || producer.closed)
1328
+ return;
1329
+ await producer.track.applyConstraints?.({ frameRate: { ideal: 30, max: 30 } }).catch(() => undefined);
1330
+ await this.reproduceCamera(producer, producer.track);
1331
+ this.emit('error', new RpcError('frame-rate-fallback', 'this device or upload cannot sustain 60 fps; publishing at 30 fps'));
1332
+ }).catch((error) => this.emit('error', error));
1333
+ }
1334
+ return stats;
1335
+ }
627
1336
  // ------------------------------------------------------------------ music
628
1337
  /**
629
1338
  * Publish music playing inside the page, as its own track.
@@ -636,11 +1345,17 @@ class StreamClient {
636
1345
  * the user has not interacted with.
637
1346
  */
638
1347
  async publishMusic(input, options) {
1348
+ const generation = this.publishGeneration;
639
1349
  if (this.music)
640
1350
  await this.stopMusic();
641
1351
  const music = new music_js_1.MusicSource(input, options);
642
1352
  try {
643
- await this.produceMusic(music);
1353
+ await this.produceMusic(music, generation);
1354
+ // Checked again here: an unpublish() after the producer was kept has
1355
+ // already closed it, and the music must not be remembered for a
1356
+ // reconnect to publish again.
1357
+ if (generation !== this.publishGeneration)
1358
+ throw new Error(unpublishedWhile('music'));
644
1359
  }
645
1360
  catch (err) {
646
1361
  music.close();
@@ -650,8 +1365,10 @@ class StreamClient {
650
1365
  this.wasPublishing = true;
651
1366
  this.emitLocalMedia();
652
1367
  }
653
- async produceMusic(music) {
1368
+ async produceMusic(music, generation) {
654
1369
  const transport = await this.ensureSendTransport();
1370
+ if (generation !== this.publishGeneration)
1371
+ throw new Error(unpublishedWhile('music'));
655
1372
  const producer = await transport.produce({
656
1373
  track: music.track,
657
1374
  stopTracks: false,
@@ -660,8 +1377,10 @@ class StreamClient {
660
1377
  codecOptions: { opusStereo: true, opusDtx: false, opusFec: true, opusMaxAverageBitrate: 128_000 },
661
1378
  appData: { source: 'music' },
662
1379
  });
663
- this.producers.set(producer.id, producer);
1380
+ await this.keepProducer(producer, generation, 'music');
664
1381
  this.musicProducerId = producer.id;
1382
+ // Music switched off before a reconnect comes back paused here.
1383
+ await this.pauseOnServer(producer);
665
1384
  }
666
1385
  /** True while music is published, whether or not it is switched on. */
667
1386
  get musicPublishing() {
@@ -724,6 +1443,24 @@ class StreamClient {
724
1443
  }));
725
1444
  const consumer = await transport.consume(params);
726
1445
  this.consumers.set(remote.producerId, consumer);
1446
+ this.remoteVideoLayers.set(remote.producerId, remote.videoLayers ?? remote.appData?.videoLayers ?? params.videoLayers ?? []);
1447
+ // Layers asked for before a reconnect: the new consumer gets them before
1448
+ // its first frame instead of starting at the full picture. Not getting
1449
+ // them costs data, not the track, so a refusal is only reported.
1450
+ let layers = this.preferredLayers.get(remote.producerId);
1451
+ const receiveQuality = this.receiveQualities.get(remote.producerId);
1452
+ if (receiveQuality && receiveQuality !== 'auto') {
1453
+ try {
1454
+ layers = { spatialLayer: (0, quality_js_1.receiveSpatialLayer)(this.remoteVideoLayers.get(remote.producerId) ?? [], receiveQuality) };
1455
+ }
1456
+ catch {
1457
+ layers = undefined;
1458
+ this.emit('error', new RpcError('quality-unavailable', 'this publisher no longer describes its video layers; using Auto'));
1459
+ }
1460
+ }
1461
+ if (layers) {
1462
+ await this.rpc('setPreferredLayers', { consumerId: consumer.id, ...layers }).catch((err) => this.emit('error', err));
1463
+ }
727
1464
  // The server created it paused. Resume only now that our end exists,
728
1465
  // otherwise the first frames arrive before there is anything to decode them.
729
1466
  await this.rpc('resumeConsumer', { consumerId: consumer.id });
@@ -752,7 +1489,7 @@ class StreamClient {
752
1489
  }
753
1490
  trackEnabled(kind) {
754
1491
  for (const p of this.producers.values()) {
755
- if (p.kind === kind && p.id !== this.musicProducerId)
1492
+ if (p.kind === kind && p.appData.source === defaultSource(kind))
756
1493
  return !p.paused;
757
1494
  }
758
1495
  return false;
@@ -768,43 +1505,76 @@ class StreamClient {
768
1505
  async setCamEnabled(enabled) {
769
1506
  await this.setKindEnabled('video', enabled);
770
1507
  }
1508
+ applyLocalModeration(kind) {
1509
+ for (const producer of this.producers.values()) {
1510
+ if (!kind || producer.kind === kind)
1511
+ producer.pause();
1512
+ }
1513
+ // During reconnect there are no producers, but the retained tracks still
1514
+ // capture. Keep their muted state for republish as well.
1515
+ for (const stream of [this.localStream, this.screenStream]) {
1516
+ for (const track of stream?.getTracks() ?? [])
1517
+ if (!kind || track.kind === kind)
1518
+ track.enabled = false;
1519
+ }
1520
+ if (!kind || kind === 'audio')
1521
+ this.music?.setEnabled(false);
1522
+ }
771
1523
  async setKindEnabled(kind, enabled) {
772
1524
  for (const producer of this.producers.values()) {
773
1525
  // The music has its own switch. Muting the microphone must not stop the song.
774
- if (producer.kind !== kind || producer.id === this.musicProducerId)
1526
+ if (producer.kind !== kind || producer.appData.source !== defaultSource(kind))
775
1527
  continue;
776
- if (enabled)
777
- producer.resume();
778
- else
1528
+ if (!enabled)
779
1529
  producer.pause();
780
1530
  await this.rpc('pauseProducer', { producerId: producer.id, paused: !enabled });
1531
+ if (enabled)
1532
+ producer.resume();
781
1533
  }
782
1534
  this.emitLocalMedia();
783
1535
  }
784
1536
  /** Stop publishing entirely and release the local devices. */
785
1537
  async unpublish() {
1538
+ // All of it takes effect now; only telling the server is waited for. So
1539
+ // publish() and unpublish() act in the order they are called: a publish
1540
+ // called while this still waits is not undone by it, and one already
1541
+ // under way stops at its next step (keepProducer).
1542
+ this.publishGeneration += 1;
1543
+ const screenWasStarted = this.screenStarted;
1544
+ this.releaseScreen();
1545
+ if (screenWasStarted)
1546
+ this.emit('screenShareEnded', { reason: 'stopped' });
786
1547
  const ids = [...this.producers.keys()];
787
1548
  for (const producer of this.producers.values())
788
1549
  producer.close();
789
1550
  this.producers.clear();
790
- for (const id of ids) {
791
- await this.rpc('closeProducer', { producerId: id }).catch(() => undefined);
792
- }
1551
+ this.cameraEncodings = null;
1552
+ // Start over for the next publish, and clear a limitation still showing:
1553
+ // nothing is being held back once nothing is being sent.
1554
+ const cleared = this.limitWatch.reset();
1555
+ if (cleared)
1556
+ this.emit('videoLimited', cleared);
793
1557
  this.localStream?.getTracks().forEach((t) => t.stop());
794
1558
  this.localStream = null;
1559
+ // Forgotten with the stream: kept, a reconnect would ask for the camera
1560
+ // and microphone again and publish them, after the app unpublished them.
1561
+ this.publishConstraints = null;
795
1562
  this.music?.close();
796
1563
  this.music = null;
797
1564
  this.musicProducerId = null;
798
1565
  this.wasPublishing = false;
799
1566
  this.emit('localStreamChanged', null);
800
1567
  this.emit('localMediaChanged', { micEnabled: false, camEnabled: false, musicEnabled: false });
1568
+ for (const id of ids) {
1569
+ await this.rpc('closeProducer', { producerId: id }).catch(() => undefined);
1570
+ }
801
1571
  }
802
1572
  get local() {
803
1573
  return this.localStream;
804
1574
  }
805
1575
  // ------------------------------------------------------------- moderation
806
- raiseHand() { return this.rpc('raiseHand'); }
807
- lowerHand() { return this.rpc('lowerHand'); }
1576
+ raiseHand(metadata) { return this.rpc('raiseHand', metadata ? { metadata } : {}); }
1577
+ lowerHand(identity) { return this.rpc('lowerHand', identity ? { identity } : {}); }
808
1578
  muteParticipant(identity) { return this.rpc('requestMute', { identity }); }
809
1579
  /**
810
1580
  * Moderating roles: switch someone's microphone (`audio`), camera (`video`)