livekit-client 2.22.3 → 2.22.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (214) hide show
  1. package/README.md +35 -0
  2. package/dist/livekit-client.e2ee.worker.js +1 -1
  3. package/dist/livekit-client.e2ee.worker.js.map +1 -1
  4. package/dist/livekit-client.e2ee.worker.mjs +2 -1
  5. package/dist/livekit-client.e2ee.worker.mjs.map +1 -1
  6. package/dist/livekit-client.esm.mjs +602 -247
  7. package/dist/livekit-client.esm.mjs.map +1 -1
  8. package/dist/livekit-client.fm.worker.js +1 -1
  9. package/dist/livekit-client.fm.worker.js.map +1 -1
  10. package/dist/livekit-client.fm.worker.mjs +2 -1
  11. package/dist/livekit-client.fm.worker.mjs.map +1 -1
  12. package/dist/livekit-client.umd.js +1 -1
  13. package/dist/livekit-client.umd.js.map +1 -1
  14. package/dist/src/api/SignalClient.d.ts +4 -2
  15. package/dist/src/api/SignalClient.d.ts.map +1 -1
  16. package/dist/src/options.d.ts +14 -0
  17. package/dist/src/options.d.ts.map +1 -1
  18. package/dist/src/room/PCTransport.d.ts +78 -7
  19. package/dist/src/room/PCTransport.d.ts.map +1 -1
  20. package/dist/src/room/PCTransportManager.d.ts.map +1 -1
  21. package/dist/src/room/RTCEngine.d.ts +8 -0
  22. package/dist/src/room/RTCEngine.d.ts.map +1 -1
  23. package/dist/src/room/Room.d.ts +1 -1
  24. package/dist/src/room/Room.d.ts.map +1 -1
  25. package/dist/src/room/defaults.d.ts.map +1 -1
  26. package/dist/src/room/errors.d.ts +18 -0
  27. package/dist/src/room/errors.d.ts.map +1 -1
  28. package/dist/src/room/events.d.ts +3 -2
  29. package/dist/src/room/events.d.ts.map +1 -1
  30. package/dist/src/room/participant/publishUtils.d.ts.map +1 -1
  31. package/dist/src/room/rpc/client/RpcClientManager.d.ts.map +1 -1
  32. package/dist/src/room/track/LocalTrack.d.ts.map +1 -1
  33. package/dist/src/room/track/LocalVideoTrack.d.ts +2 -0
  34. package/dist/src/room/track/LocalVideoTrack.d.ts.map +1 -1
  35. package/dist/src/room/track/utils.d.ts +13 -0
  36. package/dist/src/room/track/utils.d.ts.map +1 -1
  37. package/dist/src/room/utils.d.ts +32 -0
  38. package/dist/src/room/utils.d.ts.map +1 -1
  39. package/dist/src/test/compressionStreamStubs.d.ts +12 -0
  40. package/dist/src/test/compressionStreamStubs.d.ts.map +1 -0
  41. package/dist/ts4.2/api/SignalClient.d.ts +187 -185
  42. package/dist/ts4.2/api/SignalClientStateMachine.d.ts +85 -85
  43. package/dist/ts4.2/api/WebSocketStream.d.ts +33 -33
  44. package/dist/ts4.2/api/utils.d.ts +9 -9
  45. package/dist/ts4.2/connectionHelper/ConnectionCheck.d.ts +30 -30
  46. package/dist/ts4.2/connectionHelper/checks/Checker.d.ts +62 -62
  47. package/dist/ts4.2/connectionHelper/checks/cloudRegion.d.ts +18 -18
  48. package/dist/ts4.2/connectionHelper/checks/connectionProtocol.d.ts +20 -20
  49. package/dist/ts4.2/connectionHelper/checks/publishAudio.d.ts +6 -6
  50. package/dist/ts4.2/connectionHelper/checks/publishVideo.d.ts +7 -7
  51. package/dist/ts4.2/connectionHelper/checks/reconnect.d.ts +6 -6
  52. package/dist/ts4.2/connectionHelper/checks/turn.d.ts +6 -6
  53. package/dist/ts4.2/connectionHelper/checks/webrtc.d.ts +6 -6
  54. package/dist/ts4.2/connectionHelper/checks/websocket.d.ts +6 -6
  55. package/dist/ts4.2/e2ee/E2eeManager.d.ts +111 -111
  56. package/dist/ts4.2/e2ee/KeyProvider.d.ts +54 -54
  57. package/dist/ts4.2/e2ee/constants.d.ts +21 -21
  58. package/dist/ts4.2/e2ee/errors.d.ts +12 -12
  59. package/dist/ts4.2/e2ee/events.d.ts +39 -39
  60. package/dist/ts4.2/e2ee/index.d.ts +6 -6
  61. package/dist/ts4.2/e2ee/types.d.ts +231 -231
  62. package/dist/ts4.2/e2ee/utils.d.ts +32 -32
  63. package/dist/ts4.2/e2ee/worker/DataCryptor.d.ts +16 -16
  64. package/dist/ts4.2/e2ee/worker/ErrorRateLimiter.d.ts +21 -21
  65. package/dist/ts4.2/e2ee/worker/FrameCryptor.d.ts +209 -209
  66. package/dist/ts4.2/e2ee/worker/ParticipantKeyHandler.d.ts +81 -84
  67. package/dist/ts4.2/e2ee/worker/e2ee.worker.d.ts +2 -2
  68. package/dist/ts4.2/e2ee/worker/naluUtils.d.ts +28 -28
  69. package/dist/ts4.2/e2ee/worker/sifPayload.d.ts +23 -23
  70. package/dist/ts4.2/frameMetadata/FrameMetadataManager.d.ts +53 -53
  71. package/dist/ts4.2/frameMetadata/frameMetadata.d.ts +33 -33
  72. package/dist/ts4.2/frameMetadata/types.d.ts +62 -62
  73. package/dist/ts4.2/frameMetadata/utils.d.ts +9 -9
  74. package/dist/ts4.2/frameMetadata/worker/frameMetadata.worker.d.ts +2 -2
  75. package/dist/ts4.2/index.d.ts +76 -76
  76. package/dist/ts4.2/logger.d.ts +66 -66
  77. package/dist/ts4.2/options.d.ts +163 -149
  78. package/dist/ts4.2/room/BackOffStrategy.d.ts +27 -27
  79. package/dist/ts4.2/room/DefaultReconnectPolicy.d.ts +8 -8
  80. package/dist/ts4.2/room/DeviceManager.d.ts +12 -12
  81. package/dist/ts4.2/room/PCTransport.d.ts +246 -175
  82. package/dist/ts4.2/room/PCTransportManager.d.ts +77 -77
  83. package/dist/ts4.2/room/RTCEngine.d.ts +269 -262
  84. package/dist/ts4.2/room/ReconnectPolicy.d.ts +24 -24
  85. package/dist/ts4.2/room/RegionUrlProvider.d.ts +36 -36
  86. package/dist/ts4.2/room/Room.d.ts +354 -365
  87. package/dist/ts4.2/room/attribute-typings.d.ts +35 -35
  88. package/dist/ts4.2/room/data-channel/DataChannelManager.d.ts +62 -62
  89. package/dist/ts4.2/room/data-channel/FlowControlledDataChannel.d.ts +94 -95
  90. package/dist/ts4.2/room/data-channel/LossyDataChannel.d.ts +41 -41
  91. package/dist/ts4.2/room/data-channel/ReliableDataChannel.d.ts +57 -57
  92. package/dist/ts4.2/room/data-channel/types.d.ts +12 -12
  93. package/dist/ts4.2/room/data-stream/compression.d.ts +49 -49
  94. package/dist/ts4.2/room/data-stream/constants.d.ts +18 -18
  95. package/dist/ts4.2/room/data-stream/incoming/IncomingDataStreamManager.d.ts +29 -29
  96. package/dist/ts4.2/room/data-stream/incoming/StreamReader.d.ts +80 -80
  97. package/dist/ts4.2/room/data-stream/outgoing/OutgoingDataStreamManager.d.ts +73 -73
  98. package/dist/ts4.2/room/data-stream/outgoing/StreamWriter.d.ts +16 -16
  99. package/dist/ts4.2/room/data-stream/outgoing/header-utils.d.ts +18 -18
  100. package/dist/ts4.2/room/data-track/LocalDataTrack.d.ts +79 -80
  101. package/dist/ts4.2/room/data-track/RemoteDataTrack.d.ts +50 -50
  102. package/dist/ts4.2/room/data-track/depacketizer.d.ts +59 -59
  103. package/dist/ts4.2/room/data-track/frame.d.ts +22 -22
  104. package/dist/ts4.2/room/data-track/handle.d.ts +27 -27
  105. package/dist/ts4.2/room/data-track/incoming/IncomingDataTrackManager.d.ts +124 -124
  106. package/dist/ts4.2/room/data-track/incoming/errors.d.ts +24 -24
  107. package/dist/ts4.2/room/data-track/incoming/pipeline.d.ts +41 -41
  108. package/dist/ts4.2/room/data-track/incoming/types.d.ts +20 -20
  109. package/dist/ts4.2/room/data-track/outgoing/OutgoingDataTrackManager.d.ts +126 -126
  110. package/dist/ts4.2/room/data-track/outgoing/errors.d.ts +74 -74
  111. package/dist/ts4.2/room/data-track/outgoing/pipeline.d.ts +23 -23
  112. package/dist/ts4.2/room/data-track/outgoing/types.d.ts +48 -48
  113. package/dist/ts4.2/room/data-track/packet/constants.d.ts +20 -20
  114. package/dist/ts4.2/room/data-track/packet/errors.d.ts +40 -40
  115. package/dist/ts4.2/room/data-track/packet/extensions.d.ts +69 -69
  116. package/dist/ts4.2/room/data-track/packet/index.d.ts +98 -98
  117. package/dist/ts4.2/room/data-track/packet/serializable.d.ts +13 -13
  118. package/dist/ts4.2/room/data-track/packetizer.d.ts +43 -43
  119. package/dist/ts4.2/room/data-track/track-interfaces.d.ts +23 -23
  120. package/dist/ts4.2/room/data-track/types.d.ts +22 -22
  121. package/dist/ts4.2/room/data-track/utils.d.ts +60 -60
  122. package/dist/ts4.2/room/debounce.d.ts +11 -11
  123. package/dist/ts4.2/room/defaults.d.ts +9 -9
  124. package/dist/ts4.2/room/errors.d.ts +180 -162
  125. package/dist/ts4.2/room/events.d.ts +638 -637
  126. package/dist/ts4.2/room/participant/LocalParticipant.d.ts +287 -287
  127. package/dist/ts4.2/room/participant/Participant.d.ts +142 -144
  128. package/dist/ts4.2/room/participant/ParticipantTrackPermission.d.ts +26 -26
  129. package/dist/ts4.2/room/participant/RemoteParticipant.d.ts +85 -86
  130. package/dist/ts4.2/room/participant/publishUtils.d.ts +53 -53
  131. package/dist/ts4.2/room/rpc/client/RpcClientManager.d.ts +43 -43
  132. package/dist/ts4.2/room/rpc/client/events.d.ts +8 -8
  133. package/dist/ts4.2/room/rpc/index.d.ts +7 -7
  134. package/dist/ts4.2/room/rpc/server/RpcServerManager.d.ts +44 -44
  135. package/dist/ts4.2/room/rpc/server/events.d.ts +8 -8
  136. package/dist/ts4.2/room/rpc/utils.d.ts +131 -131
  137. package/dist/ts4.2/room/stats.d.ts +71 -71
  138. package/dist/ts4.2/room/statsSummary.d.ts +13 -13
  139. package/dist/ts4.2/room/timers.d.ts +12 -12
  140. package/dist/ts4.2/room/token-source/TokenSource.d.ts +82 -82
  141. package/dist/ts4.2/room/token-source/types.d.ts +70 -70
  142. package/dist/ts4.2/room/token-source/utils.d.ts +8 -8
  143. package/dist/ts4.2/room/track/FrameMetadataExtractor.d.ts +19 -19
  144. package/dist/ts4.2/room/track/LocalAudioTrack.d.ts +39 -42
  145. package/dist/ts4.2/room/track/LocalTrack.d.ts +125 -125
  146. package/dist/ts4.2/room/track/LocalTrackPublication.d.ts +42 -42
  147. package/dist/ts4.2/room/track/LocalVideoTrack.d.ts +102 -100
  148. package/dist/ts4.2/room/track/RemoteAudioTrack.d.ts +55 -55
  149. package/dist/ts4.2/room/track/RemoteTrack.d.ts +38 -38
  150. package/dist/ts4.2/room/track/RemoteTrackPublication.d.ts +72 -75
  151. package/dist/ts4.2/room/track/RemoteVideoTrack.d.ts +73 -76
  152. package/dist/ts4.2/room/track/Track.d.ts +156 -166
  153. package/dist/ts4.2/room/track/TrackPublication.d.ts +75 -81
  154. package/dist/ts4.2/room/track/create.d.ts +25 -25
  155. package/dist/ts4.2/room/track/facingMode.d.ts +41 -41
  156. package/dist/ts4.2/room/track/options.d.ts +403 -403
  157. package/dist/ts4.2/room/track/processor/types.d.ts +31 -31
  158. package/dist/ts4.2/room/track/record.d.ts +13 -13
  159. package/dist/ts4.2/room/track/types.d.ts +28 -28
  160. package/dist/ts4.2/room/track/utils.d.ts +60 -47
  161. package/dist/ts4.2/room/types.d.ts +127 -127
  162. package/dist/ts4.2/room/utils.d.ts +246 -214
  163. package/dist/ts4.2/test/MockMediaStreamTrack.d.ts +26 -26
  164. package/dist/ts4.2/test/compressionStreamStubs.d.ts +12 -0
  165. package/dist/ts4.2/test/mocks.d.ts +12 -12
  166. package/dist/ts4.2/test/promiseState.d.ts +12 -12
  167. package/dist/ts4.2/test/signalToken.d.ts +26 -26
  168. package/dist/ts4.2/type-polyfills/non-shared-typed-arrays.d.ts +2 -2
  169. package/dist/ts4.2/utils/AsyncQueue.d.ts +23 -23
  170. package/dist/ts4.2/utils/TypedPromise.d.ts +17 -17
  171. package/dist/ts4.2/utils/abort-signal-polyfill.d.ts +13 -13
  172. package/dist/ts4.2/utils/browserParser.d.ts +13 -13
  173. package/dist/ts4.2/utils/camelToSnakeCase.d.ts +8 -8
  174. package/dist/ts4.2/utils/cloneDeep.d.ts +2 -2
  175. package/dist/ts4.2/utils/dataPacketBuffer.d.ts +34 -34
  176. package/dist/ts4.2/utils/deferrable-map.d.ts +32 -32
  177. package/dist/ts4.2/utils/machineInspector.d.ts +54 -54
  178. package/dist/ts4.2/utils/serializer.d.ts +48 -48
  179. package/dist/ts4.2/utils/subscribeToEvents.d.ts +15 -15
  180. package/dist/ts4.2/utils/ttlmap.d.ts +20 -20
  181. package/dist/ts4.2/version.d.ts +15 -15
  182. package/package.json +5 -4
  183. package/src/api/SignalClient.e2e.test.ts +1 -3
  184. package/src/api/SignalClient.test.ts +160 -10
  185. package/src/api/SignalClient.ts +75 -54
  186. package/src/options.ts +16 -0
  187. package/src/room/PCTransport.test.ts +183 -7
  188. package/src/room/PCTransport.ts +197 -36
  189. package/src/room/PCTransportManager.ts +3 -1
  190. package/src/room/RTCEngine.test.ts +84 -0
  191. package/src/room/RTCEngine.ts +24 -5
  192. package/src/room/Room.test.ts +32 -6
  193. package/src/room/Room.ts +26 -6
  194. package/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts +16 -20
  195. package/src/room/data-stream/incoming/IncomingDataStreamManager.ts +5 -5
  196. package/src/room/data-stream/outgoing/OutgoingDataStreamManager.test.ts +60 -39
  197. package/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +4 -4
  198. package/src/room/defaults.ts +1 -0
  199. package/src/room/errors.test.ts +36 -0
  200. package/src/room/errors.ts +27 -0
  201. package/src/room/events.ts +2 -1
  202. package/src/room/participant/publishUtils.ts +0 -19
  203. package/src/room/rpc/client/RpcClientManager.test.ts +51 -0
  204. package/src/room/rpc/client/RpcClientManager.ts +32 -15
  205. package/src/room/track/LocalTrack.ts +18 -7
  206. package/src/room/track/LocalVideoTrack.ts +10 -3
  207. package/src/room/track/RemoteAudioTrack.ts +3 -3
  208. package/src/room/track/RemoteVideoTrack.test.ts +210 -0
  209. package/src/room/track/RemoteVideoTrack.ts +29 -3
  210. package/src/room/track/utils.test.ts +201 -2
  211. package/src/room/track/utils.ts +88 -0
  212. package/src/room/utils.test.ts +124 -0
  213. package/src/room/utils.ts +70 -0
  214. package/src/test/compressionStreamStubs.ts +53 -0
@@ -255,6 +255,57 @@ describe('RpcClientManager', () => {
255
255
  }
256
256
  });
257
257
 
258
+ it('cleans up a failed v2 publish without a later ack timeout', async () => {
259
+ vi.useFakeTimers();
260
+
261
+ try {
262
+ sendTextMock.mockRejectedValue(new Error('engine closed'));
263
+ await expect(
264
+ rpcClientManager.performRpc({
265
+ destinationIdentity: 'remote-identity',
266
+ method: 'test-method',
267
+ payload: 'test-payload',
268
+ }),
269
+ ).rejects.toThrow('engine closed');
270
+
271
+ expect(vi.getTimerCount()).toBe(0);
272
+ expect((rpcClientManager as any).pendingAcks.size).toBe(0);
273
+ expect((rpcClientManager as any).pendingResponses.size).toBe(0);
274
+ await vi.advanceTimersByTimeAsync(7001);
275
+ // An unhandled rejection from completionFuture would also fail Vitest.
276
+ } finally {
277
+ vi.useRealTimers();
278
+ }
279
+ });
280
+
281
+ it('handles an ack timeout while a v2 publish is still pending', async () => {
282
+ vi.useFakeTimers();
283
+
284
+ try {
285
+ let resolveSend!: () => void;
286
+ sendTextMock.mockReturnValue(
287
+ new Promise<void>((resolve) => {
288
+ resolveSend = resolve;
289
+ }),
290
+ );
291
+ const performRpcPromise = rpcClientManager.performRpc({
292
+ destinationIdentity: 'remote-identity',
293
+ method: 'test-method',
294
+ payload: 'test-payload',
295
+ });
296
+
297
+ await vi.advanceTimersByTimeAsync(7001);
298
+ resolveSend();
299
+ const [, completionPromise] = await performRpcPromise;
300
+ await expect(completionPromise).rejects.toThrow(/Connection timeout/i);
301
+ expect(vi.getTimerCount()).toBe(0);
302
+ expect((rpcClientManager as any).pendingAcks.size).toBe(0);
303
+ expect((rpcClientManager as any).pendingResponses.size).toBe(0);
304
+ } finally {
305
+ vi.useRealTimers();
306
+ }
307
+ });
308
+
258
309
  it('should handle a v2 RPC error response', async () => {
259
310
  const errorCode = 101;
260
311
  const errorMessage = 'Test error message';
@@ -108,22 +108,13 @@ export default class RpcClientManager extends (EventEmitter as new () => TypedEm
108
108
  participantIdentity: destinationIdentity,
109
109
  });
110
110
 
111
- await this.publishRpcRequest(
112
- destinationIdentity,
113
- id,
114
- method,
115
- payload,
116
- effectiveTimeoutMs,
117
- remoteClientProtocol,
118
- );
119
-
120
- responseTimeoutId = setTimeout(() => {
121
- this.pendingResponses.delete(id);
122
- completionFuture.reject?.(RpcError.builtIn('RESPONSE_TIMEOUT'));
123
- }, responseTimeoutMs);
124
-
111
+ // Observe the future before publishing: a slow send can outlast the ack timer.
112
+ // Keep the returned promise rejected for callers, while marking its rejection
113
+ // handled until performRpc has finished publishing and can return it.
125
114
  const completionPromise = completionFuture.promise.finally(() => {
126
- clearTimeout(responseTimeoutId);
115
+ if (responseTimeoutId !== null) {
116
+ clearTimeout(responseTimeoutId);
117
+ }
127
118
 
128
119
  if (this.pendingAcks.has(id)) {
129
120
  this.log.warn('RPC response received before ack', id);
@@ -131,6 +122,32 @@ export default class RpcClientManager extends (EventEmitter as new () => TypedEm
131
122
  clearTimeout(ackTimeoutId);
132
123
  }
133
124
  });
125
+ completionPromise.catch(() => {});
126
+
127
+ try {
128
+ await this.publishRpcRequest(
129
+ destinationIdentity,
130
+ id,
131
+ method,
132
+ payload,
133
+ effectiveTimeoutMs,
134
+ remoteClientProtocol,
135
+ );
136
+ } catch (e) {
137
+ // The caller receives the publish error. Do not leave an ack timer that
138
+ // later rejects a future the caller never received.
139
+ clearTimeout(ackTimeoutId);
140
+ this.pendingAcks.delete(id);
141
+ this.pendingResponses.delete(id);
142
+ throw e;
143
+ }
144
+
145
+ if (this.pendingResponses.has(id)) {
146
+ responseTimeoutId = setTimeout(() => {
147
+ this.pendingResponses.delete(id);
148
+ completionFuture.reject?.(RpcError.builtIn('RESPONSE_TIMEOUT'));
149
+ }, responseTimeoutMs);
150
+ }
134
151
 
135
152
  return [id, completionPromise];
136
153
  }
@@ -5,12 +5,13 @@ import { debounce } from '../debounce';
5
5
  import { DeviceUnsupportedError, TrackInvalidError } from '../errors';
6
6
  import { TrackEvent } from '../events';
7
7
  import type { LoggerOptions } from '../types';
8
- import { compareVersions, isMobile, sleep, unwrapConstraint } from '../utils';
8
+ import { compareVersions, isAppleMobile, isMobile, sleep, unwrapConstraint } from '../utils';
9
9
  import { Track, attachToElement, detachTrack } from './Track';
10
10
  import type { VideoCodec } from './options';
11
11
  import type { TrackProcessor } from './processor/types';
12
12
  import { LocalTrackRecorder, isRecordingSupported } from './record';
13
13
  import type { ReplaceTrackOptions } from './types';
14
+ import { waitForFirstVideoFrame } from './utils';
14
15
 
15
16
  const DEFAULT_DIMENSIONS_TIMEOUT = 1000;
16
17
  const PRE_CONNECT_BUFFER_TIMEOUT = 10_000;
@@ -224,18 +225,28 @@ export default abstract class LocalTrack<
224
225
  throw new Error('cannot get dimensions for audio tracks');
225
226
  }
226
227
 
227
- if (getBrowser()?.os === 'iOS') {
228
- // browsers report wrong initial resolution on iOS.
229
- // when slightly delaying the call to .getSettings(), the correct resolution is being reported
230
- await sleep(10);
228
+ const started = Date.now();
229
+
230
+ if (isAppleMobile()) {
231
+ // iOS and iPadOS keep reporting the camera's sensor frame from getSettings() until the
232
+ // camera has delivered its first picture, so a portrait capture initially reads as
233
+ // landscape. Wait for that first frame instead of guessing at a delay.
234
+ // Gated to the Apple capture pipeline rather than run everywhere: elsewhere getSettings() is
235
+ // already right and the first frame can be hundreds of milliseconds out (measured ~500ms on
236
+ // macOS Safari), which would be pure added publish latency.
237
+ // https://github.com/livekit/client-sdk-js/issues/2099
238
+ // https://github.com/livekit/client-sdk-js/issues/2107
239
+ await waitForFirstVideoFrame(this._mediaStreamTrack, timeout, this.attachedElements);
231
240
  }
232
241
 
233
- const started = Date.now();
234
- while (Date.now() - started < timeout) {
242
+ while (true) {
235
243
  const dims = this.dimensions;
236
244
  if (dims) {
237
245
  return dims;
238
246
  }
247
+ if (Date.now() - started >= timeout) {
248
+ break;
249
+ }
239
250
  await sleep(50);
240
251
  }
241
252
  throw new TrackInvalidError('unable to get track dimensions after timeout');
@@ -482,12 +482,19 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
482
482
  // browser will reenable disabled codec/layers after new codec has been published,
483
483
  // so refresh subscribedCodecs after publish a new codec
484
484
  setTimeout(() => {
485
- if (this.subscribedCodecs) {
486
- this.setPublishingCodecs(this.subscribedCodecs);
487
- }
485
+ this.refreshSubscribedCodecs();
488
486
  }, refreshSubscribedCodecAfterNewCodec);
489
487
  }
490
488
 
489
+ /** @interal */
490
+ async refreshSubscribedCodecs() {
491
+ if (this.subscribedCodecs) {
492
+ return this.setPublishingCodecs(this.subscribedCodecs).catch((e) =>
493
+ this.log.warn(`failed to refresh codec for track`, { ...this.logContext, error: e }),
494
+ );
495
+ }
496
+ }
497
+
491
498
  /**
492
499
  * @internal
493
500
  * Sets codecs that should be publishing, returns new codecs that have not yet
@@ -60,7 +60,7 @@ export default class RemoteAudioTrack extends RemoteTrack<Track.Kind.Audio> {
60
60
  * gets the volume of attached audio elements (loudest)
61
61
  */
62
62
  getVolume(): number {
63
- if (this.elementVolume) {
63
+ if (this.elementVolume !== undefined) {
64
64
  return this.elementVolume;
65
65
  }
66
66
  if (isReactNative()) {
@@ -115,7 +115,7 @@ export default class RemoteAudioTrack extends RemoteTrack<Track.Kind.Audio> {
115
115
  element.muted = true;
116
116
  }
117
117
 
118
- if (this.elementVolume) {
118
+ if (this.elementVolume !== undefined) {
119
119
  // make sure volume setting is being applied to the newly attached element
120
120
  this.setVolume(this.elementVolume);
121
121
  }
@@ -191,7 +191,7 @@ export default class RemoteAudioTrack extends RemoteTrack<Track.Kind.Audio> {
191
191
  lastNode.connect(this.gainNode);
192
192
  this.gainNode.connect(context.destination);
193
193
 
194
- if (this.elementVolume) {
194
+ if (this.elementVolume !== undefined) {
195
195
  this.gainNode.gain.setTargetAtTime(this.elementVolume, 0, 0.1);
196
196
  }
197
197
 
@@ -98,6 +98,216 @@ describe('RemoteVideoTrack', () => {
98
98
  });
99
99
  });
100
100
 
101
+ describe('Document Picture-in-Picture detection', () => {
102
+ /**
103
+ * happy-dom's MediaStream implements everything except getTracks(), which Track.attach()
104
+ * reads. Subclassing keeps it assignable to the element's srcObject, which type checks the
105
+ * value against happy-dom's own MediaStream.
106
+ */
107
+ class TestMediaStream extends MediaStream {
108
+ getTracks() {
109
+ return [...this.getAudioTracks(), ...this.getVideoTracks()];
110
+ }
111
+ }
112
+
113
+ /** Stands in for window.documentPictureInPicture, which happy-dom does not implement. */
114
+ class FakeDocumentPictureInPicture extends EventTarget {
115
+ window?: Window;
116
+ }
117
+
118
+ let documentPiP: FakeDocumentPictureInPicture;
119
+
120
+ /** Gives an element the layout box that isElementInViewport reads. */
121
+ function placeElement(
122
+ el: HTMLElement,
123
+ { top, left, width, height }: { top: number; left: number; width: number; height: number },
124
+ ) {
125
+ Object.defineProperties(el, {
126
+ offsetTop: { value: top, configurable: true },
127
+ offsetLeft: { value: left, configurable: true },
128
+ offsetWidth: { value: width, configurable: true },
129
+ offsetHeight: { value: height, configurable: true },
130
+ offsetParent: { value: null, configurable: true },
131
+ });
132
+ }
133
+
134
+ function createVideo(
135
+ ownerDocument: Document,
136
+ box = { top: 40, left: 40, width: 320, height: 180 },
137
+ ) {
138
+ const video = ownerDocument.createElement('video');
139
+ ownerDocument.body.appendChild(video);
140
+ placeElement(video, box);
141
+ return video;
142
+ }
143
+
144
+ /**
145
+ * Creates a frame in the given document and returns its document. happy-dom implements
146
+ * neither frameElement nor a correct top on a frame's window, so both are wired up here.
147
+ */
148
+ function createFrame(parentDocument: Document, topWindow: Window) {
149
+ const frame = parentDocument.createElement('iframe');
150
+ parentDocument.body.appendChild(frame);
151
+ const frameDocument = frame.contentDocument!;
152
+ Object.defineProperties(frameDocument.defaultView!, {
153
+ frameElement: { value: frame, configurable: true },
154
+ top: { value: topWindow, configurable: true },
155
+ });
156
+ return frameDocument;
157
+ }
158
+
159
+ /**
160
+ * Makes a frame document look like one reached through a cross-origin frame: the containing
161
+ * frame belongs to another origin, so frameElement reads null while top stays readable.
162
+ */
163
+ function hideContainingFrame(frameDocument: Document) {
164
+ Object.defineProperty(frameDocument.defaultView!, 'frameElement', {
165
+ value: null,
166
+ configurable: true,
167
+ });
168
+ return frameDocument;
169
+ }
170
+
171
+ /** Opens a Document PiP window and fires the 'enter' event the SDK listens for. */
172
+ function openPiPWindow({ width, height } = { width: 1280, height: 820 }) {
173
+ const pipDocument = document.implementation.createHTMLDocument('pip');
174
+ documentPiP.window = Object.assign(new EventTarget(), {
175
+ document: pipDocument,
176
+ innerWidth: width,
177
+ innerHeight: height,
178
+ pageXOffset: 0,
179
+ pageYOffset: 0,
180
+ }) as unknown as Window;
181
+ documentPiP.dispatchEvent(new Event('enter'));
182
+ return pipDocument;
183
+ }
184
+
185
+ /** Attaches a fresh track to the element and records every VisibilityChanged emission. */
186
+ function attachTrack(video: HTMLElement) {
187
+ const track = new RemoteVideoTrack(new MockMediaStreamTrack(), 'sid', undefined!, {});
188
+ const events: boolean[] = [];
189
+ track.on(TrackEvent.VisibilityChanged, (visible) => events.push(visible));
190
+ track.attach(video as HTMLMediaElement);
191
+ return events;
192
+ }
193
+
194
+ /** Flushes the microtask + animation frame that onEnterPiP defers its re-check behind. */
195
+ async function flushDeferredPiPCheck() {
196
+ await vi.advanceTimersByTimeAsync(50);
197
+ }
198
+
199
+ function setOpenerHidden() {
200
+ Object.defineProperty(document, 'visibilityState', { value: 'hidden', configurable: true });
201
+ }
202
+
203
+ beforeEach(() => {
204
+ vi.stubGlobal('MediaStream', TestMediaStream);
205
+ documentPiP = new FakeDocumentPictureInPicture();
206
+ (window as any).documentPictureInPicture = documentPiP;
207
+ // The opener tab is backgrounded in every case, so pauseVideoInBackground (on by default)
208
+ // reports the element hidden unless it is genuinely in PiP.
209
+ setOpenerHidden();
210
+ });
211
+
212
+ afterEach(() => {
213
+ vi.unstubAllGlobals();
214
+ delete (window as any).documentPictureInPicture;
215
+ Object.defineProperty(document, 'visibilityState', { value: 'visible', configurable: true });
216
+ document.body.innerHTML = '';
217
+ });
218
+
219
+ it('does not report an opener-document element as being in PiP', () => {
220
+ openPiPWindow();
221
+ const video = createVideo(document);
222
+
223
+ // The top-window check must not open a hole here: the opener's top is the opener itself.
224
+ expect(window.top).toBe(window);
225
+ expect(window.top).not.toBe(documentPiP.window);
226
+ // The element sits at (40, 40) of the OPENER page, which falls inside the PiP window's
227
+ // 1280x820 box. It is not in PiP, so the background pause applies.
228
+ expect(attachTrack(video)).toEqual([false]);
229
+ });
230
+
231
+ it('does not report an element in an opener frame as being in PiP', () => {
232
+ openPiPWindow();
233
+ const openerFrame = createFrame(document, window);
234
+ const video = createVideo(openerFrame);
235
+
236
+ // An opener frame tops out at the opener, so neither the top check nor the walk matches.
237
+ expect(openerFrame.defaultView!.top).toBe(window);
238
+ expect(attachTrack(video)).toEqual([false]);
239
+ });
240
+
241
+ it('reports an element moved into the PiP document after the enter event', async () => {
242
+ const video = createVideo(document);
243
+ const events = attachTrack(video);
244
+ expect(events).toEqual([false]);
245
+
246
+ const pipDocument = openPiPWindow();
247
+ // The SDK defers its re-check behind a microtask and an animation frame precisely so the
248
+ // embedder can move its subtree across after the enter event fires.
249
+ pipDocument.body.appendChild(video);
250
+ expect(video.ownerDocument).toBe(pipDocument);
251
+ await flushDeferredPiPCheck();
252
+
253
+ expect(events).toEqual([false, true]);
254
+ });
255
+
256
+ it('leaves an unrelated opener element hidden when another element enters PiP', async () => {
257
+ const openerVideo = createVideo(document);
258
+ const movedVideo = createVideo(document);
259
+ const openerEvents = attachTrack(openerVideo);
260
+ const movedEvents = attachTrack(movedVideo);
261
+
262
+ const pipDocument = openPiPWindow();
263
+ pipDocument.body.appendChild(movedVideo);
264
+ await flushDeferredPiPCheck();
265
+
266
+ expect(movedEvents).toEqual([false, true]);
267
+ expect(openerEvents).toEqual([false]);
268
+ });
269
+
270
+ it('reports an element inside a frame nested in the PiP document', () => {
271
+ const pipDocument = openPiPWindow();
272
+ const video = createVideo(createFrame(pipDocument, documentPiP.window!), {
273
+ top: 10,
274
+ left: 10,
275
+ width: 320,
276
+ height: 180,
277
+ });
278
+
279
+ expect(attachTrack(video)).toEqual([true]);
280
+ });
281
+
282
+ it('reports an element behind a cross-origin frame nested in the PiP window', () => {
283
+ const pipDocument = openPiPWindow();
284
+ const pipWindow = documentPiP.window!;
285
+ // PiP document (origin A) -> frame (origin B) -> frame (origin A). The opener can reach
286
+ // the innermost document, but its containing frame belongs to B, so frameElement is null
287
+ // there and the same-origin walk cannot get back to the PiP document.
288
+ const crossOriginFrame = createFrame(pipDocument, pipWindow);
289
+ const innerFrame = hideContainingFrame(createFrame(crossOriginFrame, pipWindow));
290
+ const video = createVideo(innerFrame, { top: 10, left: 10, width: 320, height: 180 });
291
+
292
+ expect(innerFrame.defaultView!.frameElement).toBeNull();
293
+ expect(innerFrame.defaultView!.top).toBe(pipWindow);
294
+ expect(attachTrack(video)).toEqual([true]);
295
+ });
296
+
297
+ it('does not report an element scrolled out of the PiP window viewport', () => {
298
+ const pipDocument = openPiPWindow();
299
+ const video = createVideo(pipDocument, { top: 2000, left: 40, width: 320, height: 180 });
300
+
301
+ expect(attachTrack(video)).toEqual([false]);
302
+ });
303
+
304
+ it('does not report an element as being in PiP when no PiP window is open', () => {
305
+ const video = createVideo(document);
306
+
307
+ expect(attachTrack(video)).toEqual([false]);
308
+ });
309
+ });
310
+
101
311
  class MockElementInfo implements ElementInfo {
102
312
  element: object = {};
103
313
 
@@ -435,9 +435,35 @@ class HTMLElementInfo implements ElementInfo {
435
435
  function isElementInPiP(el: HTMLElement) {
436
436
  // Simple video PiP
437
437
  if (document.pictureInPictureElement === el) return true;
438
- // Document PiP
439
- if (window.documentPictureInPicture?.window)
440
- return isElementInViewport(el, window.documentPictureInPicture?.window);
438
+ // Document PiP: the element has to actually live in the PiP window, otherwise an
439
+ // opener-document element whose coordinates happen to fall inside the PiP window's
440
+ // bounds is reported as being in PiP.
441
+ const pipWindow = window.documentPictureInPicture?.window;
442
+ if (pipWindow) {
443
+ return isDocumentInPiPWindow(el.ownerDocument, pipWindow) && isElementInViewport(el, pipWindow);
444
+ }
445
+ return false;
446
+ }
447
+
448
+ // Whether the document is the Document PiP window's own document, or the document of a frame
449
+ // nested inside it. The spec allows frames in a PiP window, so an element in one is in PiP:
450
+ // https://wicg.github.io/document-picture-in-picture/#iframes
451
+ function isDocumentInPiPWindow(doc: Document, pipWindow: Window) {
452
+ // A cross-origin frame anywhere between the element and the PiP window hides frameElement
453
+ // from the walk below, but top stays readable across origins and identifies the top-level
454
+ // window the element belongs to. For an opener element that is the opener, not the PiP window.
455
+ if (doc.defaultView?.top === pipWindow) return true;
456
+ let current: Document | null | undefined = doc;
457
+ while (current) {
458
+ if (current === pipWindow.document) return true;
459
+ try {
460
+ // Reading frameElement across a cross-origin boundary throws, which means we cannot
461
+ // establish the ancestry and must not claim the element is in PiP.
462
+ current = current.defaultView?.frameElement?.ownerDocument;
463
+ } catch {
464
+ return false;
465
+ }
466
+ }
441
467
  return false;
442
468
  }
443
469
 
@@ -1,7 +1,12 @@
1
- import { describe, expect, it } from 'vitest';
1
+ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
2
2
  import { audioDefaults, videoDefaults } from '../defaults';
3
3
  import { type AudioCaptureOptions, VideoPresets } from './options';
4
- import { constraintsForOptions, diffAttributes, mergeDefaultOptions } from './utils';
4
+ import {
5
+ constraintsForOptions,
6
+ diffAttributes,
7
+ mergeDefaultOptions,
8
+ waitForFirstVideoFrame,
9
+ } from './utils';
5
10
 
6
11
  describe('mergeDefaultOptions', () => {
7
12
  it('does not enable undefined options', () => {
@@ -135,3 +140,197 @@ describe('diffAttributes', () => {
135
140
  expect(diff.a).toBe('value');
136
141
  });
137
142
  });
143
+
144
+ class FakeVideoTrack extends EventTarget {
145
+ kind = 'video';
146
+
147
+ id = 'fake-video-track';
148
+
149
+ label = 'fake';
150
+
151
+ enabled = true;
152
+
153
+ muted = false;
154
+
155
+ readyState: MediaStreamTrackState = 'live';
156
+
157
+ settings: MediaTrackSettings;
158
+
159
+ constructor(settings: MediaTrackSettings = { width: 1280, height: 720 }) {
160
+ super();
161
+ this.settings = settings;
162
+ }
163
+
164
+ getSettings() {
165
+ return this.settings;
166
+ }
167
+
168
+ getConstraints() {
169
+ return {};
170
+ }
171
+ }
172
+
173
+ const asMediaStreamTrack = (track: FakeVideoTrack) => track as unknown as MediaStreamTrack;
174
+
175
+ describe('waitForFirstVideoFrame', () => {
176
+ let frameCallbacks: Map<number, VideoFrameRequestCallback>;
177
+ let cancelledHandles: number[];
178
+ let createdVideoElements: HTMLVideoElement[];
179
+
180
+ /** happy-dom has no requestVideoFrameCallback, install a controllable one */
181
+ function enableFrameCallbacks() {
182
+ let nextHandle = 1;
183
+ HTMLVideoElement.prototype.requestVideoFrameCallback = function requestVideoFrameCallback(
184
+ callback: VideoFrameRequestCallback,
185
+ ) {
186
+ const handle = nextHandle;
187
+ nextHandle += 1;
188
+ frameCallbacks.set(handle, callback);
189
+ return handle;
190
+ };
191
+ HTMLVideoElement.prototype.cancelVideoFrameCallback = function cancelVideoFrameCallback(
192
+ handle: number,
193
+ ) {
194
+ cancelledHandles.push(handle);
195
+ frameCallbacks.delete(handle);
196
+ };
197
+ }
198
+
199
+ function deliverFrame() {
200
+ Array.from(frameCallbacks.values()).forEach((callback) =>
201
+ callback(0, {} as VideoFrameCallbackMetadata),
202
+ );
203
+ }
204
+
205
+ beforeEach(() => {
206
+ frameCallbacks = new Map();
207
+ cancelledHandles = [];
208
+ createdVideoElements = [];
209
+ const createElement = document.createElement.bind(document);
210
+ vi.spyOn(document, 'createElement').mockImplementation(((
211
+ tagName: string,
212
+ options?: ElementCreationOptions,
213
+ ) => {
214
+ const element = createElement(tagName, options);
215
+ if (tagName === 'video') {
216
+ createdVideoElements.push(element as HTMLVideoElement);
217
+ }
218
+ return element;
219
+ }) as typeof document.createElement);
220
+ });
221
+
222
+ afterEach(() => {
223
+ vi.restoreAllMocks();
224
+ // @ts-expect-error removing the stand-in we installed above
225
+ delete HTMLVideoElement.prototype.requestVideoFrameCallback;
226
+ // @ts-expect-error removing the stand-in we installed above
227
+ delete HTMLVideoElement.prototype.cancelVideoFrameCallback;
228
+ });
229
+
230
+ it('resolves once a frame has been delivered', async () => {
231
+ enableFrameCallbacks();
232
+ const pending = waitForFirstVideoFrame(asMediaStreamTrack(new FakeVideoTrack()), 1000);
233
+ expect(createdVideoElements).toHaveLength(1);
234
+
235
+ deliverFrame();
236
+
237
+ await expect(pending).resolves.toBe(true);
238
+ });
239
+
240
+ it('gives up once the timeout elapses', async () => {
241
+ enableFrameCallbacks();
242
+
243
+ await expect(
244
+ waitForFirstVideoFrame(asMediaStreamTrack(new FakeVideoTrack()), 20),
245
+ ).resolves.toBe(false);
246
+ });
247
+
248
+ it.each([
249
+ ['ended', { readyState: 'ended' as MediaStreamTrackState }],
250
+ ['muted', { muted: true }],
251
+ ['disabled', { enabled: false }],
252
+ ])('does not wait on a track that is %s', async (_label, overrides) => {
253
+ enableFrameCallbacks();
254
+ const track = Object.assign(new FakeVideoTrack(), overrides);
255
+
256
+ await expect(waitForFirstVideoFrame(asMediaStreamTrack(track), 10_000)).resolves.toBe(false);
257
+ expect(createdVideoElements).toHaveLength(0);
258
+ });
259
+
260
+ it('falls back to loadeddata when requestVideoFrameCallback is unavailable', async () => {
261
+ const pending = waitForFirstVideoFrame(asMediaStreamTrack(new FakeVideoTrack()), 1000);
262
+
263
+ createdVideoElements[0].dispatchEvent(new Event('loadeddata'));
264
+
265
+ await expect(pending).resolves.toBe(true);
266
+ });
267
+
268
+ it('keeps waiting when play() is rejected', async () => {
269
+ enableFrameCallbacks();
270
+ vi.spyOn(HTMLMediaElement.prototype, 'play').mockRejectedValue(new Error('NotAllowedError'));
271
+ const pending = waitForFirstVideoFrame(asMediaStreamTrack(new FakeVideoTrack()), 1000);
272
+
273
+ deliverFrame();
274
+
275
+ await expect(pending).resolves.toBe(true);
276
+ });
277
+
278
+ it('reuses an element that already renders the track', async () => {
279
+ enableFrameCallbacks();
280
+ const track = new FakeVideoTrack();
281
+ const attached = document.createElement('video');
282
+ attached.srcObject = new MediaStream([asMediaStreamTrack(track)]);
283
+ createdVideoElements.length = 0;
284
+
285
+ const pending = waitForFirstVideoFrame(asMediaStreamTrack(track), 1000, [attached]);
286
+ expect(createdVideoElements).toHaveLength(0);
287
+ deliverFrame();
288
+
289
+ await expect(pending).resolves.toBe(true);
290
+ // the caller owns that element, it must be left rendering
291
+ expect(attached.srcObject).not.toBeNull();
292
+ });
293
+
294
+ it('reuses an element that has not decoded a frame yet without requestVideoFrameCallback', async () => {
295
+ const track = new FakeVideoTrack();
296
+ const attached = document.createElement('video');
297
+ attached.srcObject = new MediaStream([asMediaStreamTrack(track)]);
298
+ Object.defineProperty(attached, 'readyState', { value: 1, configurable: true });
299
+ createdVideoElements.length = 0;
300
+
301
+ const pending = waitForFirstVideoFrame(asMediaStreamTrack(track), 1000, [attached]);
302
+ expect(createdVideoElements).toHaveLength(0);
303
+ // `loadeddata` has not fired on that element yet, so listening for it still works
304
+ attached.dispatchEvent(new Event('loadeddata'));
305
+
306
+ await expect(pending).resolves.toBe(true);
307
+ });
308
+
309
+ it('declines an already loaded element without requestVideoFrameCallback', async () => {
310
+ const track = new FakeVideoTrack();
311
+ const attached = document.createElement('video');
312
+ attached.srcObject = new MediaStream([asMediaStreamTrack(track)]);
313
+ Object.defineProperty(attached, 'readyState', { value: 2, configurable: true });
314
+ createdVideoElements.length = 0;
315
+
316
+ const pending = waitForFirstVideoFrame(asMediaStreamTrack(track), 1000, [attached]);
317
+ // `loadeddata` already fired on that element and is never replayed, so we need our own
318
+ expect(createdVideoElements).toHaveLength(1);
319
+ createdVideoElements[0].dispatchEvent(new Event('loadeddata'));
320
+
321
+ await expect(pending).resolves.toBe(true);
322
+ });
323
+
324
+ it('tears down the element it created', async () => {
325
+ enableFrameCallbacks();
326
+ const pause = vi.spyOn(HTMLMediaElement.prototype, 'pause');
327
+ const pending = waitForFirstVideoFrame(asMediaStreamTrack(new FakeVideoTrack()), 1000);
328
+
329
+ deliverFrame();
330
+ await pending;
331
+
332
+ expect(createdVideoElements[0].srcObject).toBeNull();
333
+ expect(pause).toHaveBeenCalled();
334
+ expect(cancelledHandles).toHaveLength(1);
335
+ });
336
+ });