mediasfu-shared 1.2.3 → 1.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/CHANGELOG.md +4 -0
  2. package/dist/audioProcessing-0z5KlcmO.js +35 -0
  3. package/dist/audioProcessing-0z5KlcmO.js.map +1 -0
  4. package/dist/audioProcessing-BpbrdCk-.cjs +34 -0
  5. package/dist/audioProcessing-BpbrdCk-.cjs.map +1 -0
  6. package/dist/consumers/audioProcessing.d.ts +8 -0
  7. package/dist/consumers/audioProcessing.d.ts.map +1 -0
  8. package/dist/consumers/createSendTransport.d.ts.map +1 -1
  9. package/dist/consumers/gridLayout/addVideosGrid.engine.d.ts +6 -0
  10. package/dist/consumers/gridLayout/addVideosGrid.engine.d.ts.map +1 -1
  11. package/dist/consumers/index.cjs +3 -2
  12. package/dist/consumers/index.cjs.map +1 -1
  13. package/dist/consumers/index.js +14 -13
  14. package/dist/consumers/signalNewConsumerTransport.d.ts.map +1 -1
  15. package/dist/consumers/switchUserAudio.d.ts +3 -1
  16. package/dist/consumers/switchUserAudio.d.ts.map +1 -1
  17. package/dist/consumers/transportRecovery.d.ts +8 -0
  18. package/dist/consumers/transportRecovery.d.ts.map +1 -0
  19. package/dist/{getParticipantMedia-AwwtMCTz.cjs → getParticipantMedia-CP4hE5Rg.cjs} +9 -4
  20. package/dist/getParticipantMedia-CP4hE5Rg.cjs.map +1 -0
  21. package/dist/{getParticipantMedia-CyaOk2oD.js → getParticipantMedia-CvXdDj03.js} +9 -4
  22. package/dist/getParticipantMedia-CvXdDj03.js.map +1 -0
  23. package/dist/index.cjs +106 -22
  24. package/dist/index.cjs.map +1 -1
  25. package/dist/index.d.ts +2 -0
  26. package/dist/index.d.ts.map +1 -1
  27. package/dist/index.js +173 -89
  28. package/dist/index.js.map +1 -1
  29. package/dist/index.native.cjs +8 -7
  30. package/dist/index.native.cjs.map +1 -1
  31. package/dist/index.native.js +19 -18
  32. package/dist/{joinLocalRoom-Bu3gz27d.js → joinLocalRoom-Cx1y3Ojj.js} +4 -4
  33. package/dist/{joinLocalRoom-Bu3gz27d.js.map → joinLocalRoom-Cx1y3Ojj.js.map} +1 -1
  34. package/dist/{joinLocalRoom-Cii5D5IG.cjs → joinLocalRoom-DNJ2zb8W.cjs} +10 -10
  35. package/dist/{joinLocalRoom-Cii5D5IG.cjs.map → joinLocalRoom-DNJ2zb8W.cjs.map} +1 -1
  36. package/dist/{joinRoomOnMediaSFU-y6vJ1OBo.js → joinRoomOnMediaSFU-BnAWLciF.js} +9 -3
  37. package/dist/joinRoomOnMediaSFU-BnAWLciF.js.map +1 -0
  38. package/dist/{joinRoomOnMediaSFU-CC8RVJgr.cjs → joinRoomOnMediaSFU-D2C7q1GK.cjs} +9 -3
  39. package/dist/joinRoomOnMediaSFU-D2C7q1GK.cjs.map +1 -0
  40. package/dist/methods/index.cjs +3 -3
  41. package/dist/methods/index.js +3 -3
  42. package/dist/methods/stream/clickAudio.d.ts +3 -1
  43. package/dist/methods/stream/clickAudio.d.ts.map +1 -1
  44. package/dist/methods/stream/switchAudio.d.ts +3 -1
  45. package/dist/methods/stream/switchAudio.d.ts.map +1 -1
  46. package/dist/methods/utils/headless/virtualBackground.d.ts +3 -1
  47. package/dist/methods/utils/headless/virtualBackground.d.ts.map +1 -1
  48. package/dist/methods/utils/virtualBackgroundFrameLoop.d.ts +10 -0
  49. package/dist/methods/utils/virtualBackgroundFrameLoop.d.ts.map +1 -0
  50. package/dist/{translationConsumerSwitch-C20MfNNq.cjs → translationConsumerSwitch-BAsQUpH6.cjs} +9 -1
  51. package/dist/translationConsumerSwitch-BAsQUpH6.cjs.map +1 -0
  52. package/dist/{translationConsumerSwitch-zKuS5wgH.js → translationConsumerSwitch-BUODiT6o.js} +20 -12
  53. package/dist/translationConsumerSwitch-BUODiT6o.js.map +1 -0
  54. package/dist/types/index.cjs +3 -3
  55. package/dist/types/index.js +3 -3
  56. package/dist/types/shared-base-types.d.ts +12 -0
  57. package/dist/types/shared-base-types.d.ts.map +1 -1
  58. package/dist/{updateParticipantAudioDecibels-QZyahisU.cjs → updateParticipantAudioDecibels-BJ2PxJJD.cjs} +203 -46
  59. package/dist/updateParticipantAudioDecibels-BJ2PxJJD.cjs.map +1 -0
  60. package/dist/{updateParticipantAudioDecibels-CnEXJBmc.js → updateParticipantAudioDecibels-DiJ_LFfH.js} +200 -43
  61. package/dist/updateParticipantAudioDecibels-DiJ_LFfH.js.map +1 -0
  62. package/package.json +1 -1
  63. package/src/consumers/audioProcessing.ts +26 -0
  64. package/src/consumers/createSendTransport.ts +5 -20
  65. package/src/consumers/gridLayout/addVideosGrid.engine.ts +13 -0
  66. package/src/consumers/signalNewConsumerTransport.ts +7 -25
  67. package/src/consumers/switchUserAudio.ts +18 -5
  68. package/src/consumers/transportRecovery.ts +170 -0
  69. package/src/index.ts +2 -0
  70. package/src/methods/stream/clickAudio.ts +13 -5
  71. package/src/methods/stream/switchAudio.ts +10 -1
  72. package/src/methods/utils/headless/virtualBackground.ts +21 -13
  73. package/src/methods/utils/virtualBackgroundFrameLoop.ts +69 -0
  74. package/src/types/shared-base-types.ts +3 -0
  75. package/dist/getParticipantMedia-AwwtMCTz.cjs.map +0 -1
  76. package/dist/getParticipantMedia-CyaOk2oD.js.map +0 -1
  77. package/dist/joinRoomOnMediaSFU-CC8RVJgr.cjs.map +0 -1
  78. package/dist/joinRoomOnMediaSFU-y6vJ1OBo.js.map +0 -1
  79. package/dist/translationConsumerSwitch-C20MfNNq.cjs.map +0 -1
  80. package/dist/translationConsumerSwitch-zKuS5wgH.js.map +0 -1
  81. package/dist/updateParticipantAudioDecibels-CnEXJBmc.js.map +0 -1
  82. package/dist/updateParticipantAudioDecibels-QZyahisU.cjs.map +0 -1
  83. package/dist/validateAlphanumeric-DKn5BsUP.cjs +0 -15
  84. package/dist/validateAlphanumeric-DKn5BsUP.cjs.map +0 -1
  85. package/dist/validateAlphanumeric-Dk1JB80W.js +0 -16
  86. package/dist/validateAlphanumeric-Dk1JB80W.js.map +0 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mediasfu-shared",
3
- "version": "1.2.3",
3
+ "version": "1.2.5",
4
4
  "description": "mediasfu-shared – framework-agnostic WebRTC runtime for MediaSFU. Room helpers, mediasoup signaling, socket management, media state, and TypeScript types for React, Vue, Angular, Svelte, and plain TS.",
5
5
  "main": "dist/index.cjs",
6
6
  "module": "dist/index.js",
@@ -0,0 +1,26 @@
1
+ export type AudioProcessingOptions = {
2
+ echoCancellation?: boolean;
3
+ noiseSuppression?: boolean;
4
+ autoGainControl?: boolean;
5
+ };
6
+
7
+ export function audioProcessingConstraints(options?: AudioProcessingOptions): MediaTrackConstraints {
8
+ const result: MediaTrackConstraints = {};
9
+ if (options === undefined) return result;
10
+ if (!options || typeof options !== 'object' || Array.isArray(options)) throw new TypeError('Invalid audioProcessing');
11
+ for (const key of Object.keys(options)) {
12
+ if (!['echoCancellation', 'noiseSuppression', 'autoGainControl'].includes(key)) throw new TypeError('Unsupported audioProcessing option');
13
+ const value = options[key as keyof AudioProcessingOptions];
14
+ if (value !== undefined && typeof value !== 'boolean') throw new TypeError('Audio processing options must be boolean');
15
+ if (value !== undefined) result[key as keyof AudioProcessingOptions] = value;
16
+ }
17
+ return result;
18
+ }
19
+
20
+ // applyConstraints replaces the constraint set: merge to retain device and
21
+ // capture choices unrelated to the three processing preferences.
22
+ export async function applyAudioProcessing(track: MediaStreamTrack, options?: AudioProcessingOptions): Promise<void> {
23
+ const selected = audioProcessingConstraints(options);
24
+ if (!Object.keys(selected).length) return;
25
+ await track.applyConstraints({ ...track.getConstraints(), ...selected });
26
+ }
@@ -1,4 +1,5 @@
1
1
  import type { Device, Transport, DtlsParameters } from 'mediasoup-client/types';
2
+ import { attachTransportRecovery, clearFailedSendState } from './transportRecovery';
2
3
  import { Socket } from "socket.io-client";
3
4
  import {
4
5
  ConnectSendTransportParameters,
@@ -151,12 +152,8 @@ const createLocalSendTransport = async ({
151
152
  }
152
153
  );
153
154
 
154
- createdLocalTransport.on("connectionstatechange", (state: string) => {
155
- if (state === "failed") {
156
- console.error("Local transport connection failed.");
157
- createdLocalTransport.close();
158
- }
159
- });
155
+ attachTransportRecovery(createdLocalTransport, localSocket, () =>
156
+ clearFailedSendState(parameters, createdLocalTransport, true));
160
157
 
161
158
  // Mark local transport as created
162
159
  localTransportCreated = true;
@@ -330,20 +327,8 @@ export const createSendTransport: CreateSendTransportType = async ({
330
327
  );
331
328
 
332
329
  // Handle 'connectionstatechange' event
333
- createdProducerTransport.on("connectionstatechange", async (state: string) => {
334
- switch (state) {
335
- case "connecting":
336
- break;
337
- case "connected":
338
- break;
339
- case "failed":
340
- console.log("Transport connection failed.");
341
- createdProducerTransport.close();
342
- break;
343
- default:
344
- break;
345
- }
346
- });
330
+ attachTransportRecovery(createdProducerTransport, socket, () =>
331
+ clearFailedSendState(parameters, createdProducerTransport));
347
332
 
348
333
  // Update transport creation state
349
334
  transportCreated = true;
@@ -14,6 +14,19 @@ export interface AddVideosGridPlan<T> {
14
14
  altEntries: GridPlanEntry<T>[];
15
15
  }
16
16
 
17
+ /** A lone side-panel tile must remain fully visible during screen sharing. */
18
+ export function resolveSidePanelForceFullDisplay({
19
+ forceFullDisplay,
20
+ screenShareActive,
21
+ itemCount,
22
+ }: {
23
+ forceFullDisplay: boolean;
24
+ screenShareActive: boolean;
25
+ itemCount: number;
26
+ }): boolean {
27
+ return forceFullDisplay && !(screenShareActive && itemCount < 2);
28
+ }
29
+
17
30
  /**
18
31
  * Pure planning engine for addVideosGrid.
19
32
  * Produces deterministic main/alt stream iteration plans without rendering concerns.
@@ -1,4 +1,5 @@
1
1
  import { Socket } from "socket.io-client";
2
+ import { attachTransportRecovery } from './transportRecovery';
2
3
  import { ReorderStreamsParameters, ReorderStreamsType, ConnectRecvTransportType,
3
4
  ConnectRecvTransportParameters, CreateWebRTCTransportResponse } from "../types/types";
4
5
  import type { Device, DtlsParameters } from 'mediasoup-client/types';
@@ -128,31 +129,12 @@ export const signalNewConsumerTransport = async ({
128
129
  );
129
130
 
130
131
  // Listen for connection state change
131
- consumerTransport.on("connectionstatechange", async (state: string) => {
132
- switch (state) {
133
- case "connecting":
134
- // Handle connecting state
135
- break;
136
-
137
- case "connected":
138
- // Handle connected state
139
- break;
140
-
141
- case "failed":
142
- // Handle failed state
143
- consumerTransport.close();
144
-
145
- // Reorder streams based on conditions
146
- if (lock_screen) {
147
- await reorderStreams({ add: true, parameters });
148
- } else {
149
- await reorderStreams({ add: false, parameters });
150
- }
151
- break;
152
-
153
- default:
154
- break;
155
- }
132
+ attachTransportRecovery(consumerTransport, nsock, () => {
133
+ const live: any = parameters.getCurrentParams?.() ?? parameters.getUpdatedAllParams?.() ?? parameters;
134
+ live.updateConsumingTransports?.((live.consumingTransports || []).filter((id: string) => id !== remoteProducerId));
135
+ live.updateConsumerTransports?.((live.consumerTransports || []).filter((entry: any) => entry.consumerTransport !== consumerTransport));
136
+ live.showAlert?.({ message: 'A remote media connection could not recover. Rejoin to restore it.', type: 'danger', duration: 10000 });
137
+ void Promise.resolve(reorderStreams({ add: Boolean(lock_screen), parameters })).catch(() => {});
156
138
  });
157
139
 
158
140
  // Connect the receiving transport
@@ -17,7 +17,10 @@ export interface SwitchUserAudioParameters extends StreamSuccessAudioSwitchParam
17
17
  [key: string]: any;
18
18
  }
19
19
 
20
+ import { AudioProcessingOptions, audioProcessingConstraints } from './audioProcessing';
21
+
20
22
  export interface SwitchUserAudioOptions {
23
+ audioProcessing?: AudioProcessingOptions;
21
24
  audioPreference: string;
22
25
  parameters: SwitchUserAudioParameters;
23
26
  }
@@ -63,7 +66,7 @@ export type SwitchUserAudioType = (options: SwitchUserAudioOptions) => Promise<v
63
66
  */
64
67
 
65
68
 
66
- export async function switchUserAudio({ audioPreference, parameters }: SwitchUserAudioOptions): Promise<void> {
69
+ export async function switchUserAudio({ audioPreference, parameters, audioProcessing }: SwitchUserAudioOptions): Promise<void> {
67
70
  const {
68
71
  mediaDevices,
69
72
  prevAudioInputDevice,
@@ -94,12 +97,23 @@ export async function switchUserAudio({ audioPreference, parameters }: SwitchUse
94
97
  }
95
98
  }
96
99
 
100
+ // Preserve the microphone's processing policy, not its old deviceId.
101
+ // Omit unknown values so the browser retains its normal defaults.
102
+ const track = parameters.localStreamAudio?.getAudioTracks?.()[0]
103
+ ?? parameters.localStream?.getAudioTracks?.()[0]
104
+ ?? parameters.audioProducer?.track;
105
+ const previousConstraints = track?.getConstraints?.() ?? {};
106
+ const previousSettings = track?.getSettings?.() ?? {};
107
+ const processing: MediaTrackConstraints = {};
108
+ for (const key of ['echoCancellation', 'noiseSuppression', 'autoGainControl'] as const) {
109
+ const value = previousConstraints[key] ?? previousSettings[key];
110
+ if (value !== undefined) processing[key] = value;
111
+ }
97
112
  const mediaConstraints: MediaStreamConstraints = {
98
113
  audio: {
99
114
  deviceId: { exact: audioPreference },
100
- echoCancellation: false,
101
- noiseSuppression: false,
102
- autoGainControl: false,
115
+ ...processing,
116
+ ...audioProcessingConstraints(audioProcessing),
103
117
  },
104
118
  video: false,
105
119
  };
@@ -137,4 +151,3 @@ export async function switchUserAudio({ audioPreference, parameters }: SwitchUse
137
151
 
138
152
  }
139
153
  }
140
-
@@ -0,0 +1,170 @@
1
+ /**
2
+ * Bounded ICE recovery shared by sending and receiving transports.
3
+ * Keeps tracks/producers alive while ICE is restarted on both endpoints.
4
+ */
5
+ export function attachTransportRecovery(transport: any, socket: any, onFailure: () => void) {
6
+ let timer: ReturnType<typeof setTimeout> | undefined;
7
+ let disposed = false;
8
+ let recovering = false;
9
+ let attempts = 0;
10
+ const socketId = socket.id;
11
+ const listeners: Array<() => void> = [];
12
+ const report = (state: string) => {
13
+ transport.appData = { ...transport.appData, recoveryState: state };
14
+ transport.observer?.emit('recoverychange', state);
15
+ };
16
+ const clearTimer = () => { if (timer) clearTimeout(timer); timer = undefined; };
17
+ const current = () => !disposed && !transport.closed && socket.connected !== false && socket.id === socketId;
18
+ const dispose = () => {
19
+ disposed = true;
20
+ clearTimer();
21
+ listeners.splice(0).forEach(cancel => cancel());
22
+ socket.off?.('disconnect', dispose);
23
+ transport.off?.('connectionstatechange', changed);
24
+ transport.observer?.off?.('close', dispose);
25
+ };
26
+ const failed = () => {
27
+ if (disposed) return;
28
+ report('failed');
29
+ try { onFailure(); } catch (_) { /* UI callbacks must not prevent teardown. */ }
30
+ try { transport.close(); } finally { dispose(); }
31
+ };
32
+ const waitConnected = () => new Promise<void>((resolve, reject) => {
33
+ let settled = false;
34
+ const finish = (error?: Error) => {
35
+ if (settled) return;
36
+ settled = true;
37
+ clearTimeout(deadline);
38
+ transport.off?.('connectionstatechange', connected);
39
+ const i = listeners.indexOf(cancel);
40
+ if (i >= 0) listeners.splice(i, 1);
41
+ error ? reject(error) : resolve();
42
+ };
43
+ const connected = (state: string) => { if (state === 'connected') finish(); };
44
+ const cancel = () => finish(new Error('Recovery cancelled'));
45
+ const deadline = setTimeout(() => finish(new Error('ICE recovery timed out')), 10000);
46
+ listeners.push(cancel);
47
+ transport.on('connectionstatechange', connected);
48
+ if (transport.connectionState === 'connected') finish();
49
+ });
50
+ const restart = async () => {
51
+ if (!current() || recovering) return;
52
+ recovering = true;
53
+ report('recovering');
54
+ try {
55
+ while (current() && attempts < 2) {
56
+ attempts++;
57
+ try {
58
+ const iceParameters = await new Promise<any>((resolve, reject) => {
59
+ let settled = false;
60
+ const finish = (error?: Error, value?: any) => {
61
+ if (settled) return;
62
+ settled = true;
63
+ clearTimeout(deadline);
64
+ const i = listeners.indexOf(cancel);
65
+ if (i >= 0) listeners.splice(i, 1);
66
+ error ? reject(error) : resolve(value);
67
+ };
68
+ const cancel = () => finish(new Error('Recovery cancelled'));
69
+ const deadline = setTimeout(() => finish(new Error('ICE restart acknowledgement timed out')), 8000);
70
+ listeners.push(cancel);
71
+ socket.emit('transport-restart-ice', { transportId: transport.id }, (response: any) => {
72
+ if (!response?.iceParameters || response.error) finish(new Error(response?.error || 'Invalid ICE response'));
73
+ else finish(undefined, response.iceParameters);
74
+ });
75
+ });
76
+ if (!current()) return;
77
+ await new Promise<void>((resolve, reject) => {
78
+ let settled = false;
79
+ const finish = (error?: Error) => {
80
+ if (settled) return;
81
+ settled = true;
82
+ clearTimeout(deadline);
83
+ const i = listeners.indexOf(cancel);
84
+ if (i >= 0) listeners.splice(i, 1);
85
+ error ? reject(error) : resolve();
86
+ };
87
+ const cancel = () => finish(new Error('Recovery cancelled'));
88
+ const deadline = setTimeout(() => finish(new Error('Local ICE restart timed out')), 8000);
89
+ listeners.push(cancel);
90
+ Promise.resolve().then(() => {
91
+ if (!current()) throw new Error('Recovery cancelled');
92
+ return transport.restartIce({ iceParameters });
93
+ }).then(() => finish(), error => finish(error));
94
+ });
95
+ if (!current()) return;
96
+ await waitConnected();
97
+ if (!current()) return;
98
+ attempts = 0;
99
+ report('connected');
100
+ return;
101
+ } catch (_) {
102
+ if (!current()) return;
103
+ if (transport.connectionState === 'connected') {
104
+ attempts = 0;
105
+ report('connected');
106
+ return;
107
+ }
108
+ if (attempts < 2) {
109
+ // Respect the server's two-second restart throttle, including
110
+ // immediate acknowledgement failures. Teardown cancels the wait.
111
+ await new Promise<void>(resolve => {
112
+ const cancel = () => {
113
+ clearTimeout(delay);
114
+ const i = listeners.indexOf(cancel);
115
+ if (i >= 0) listeners.splice(i, 1);
116
+ resolve();
117
+ };
118
+ const delay = setTimeout(cancel, 2100);
119
+ listeners.push(cancel);
120
+ });
121
+ }
122
+ }
123
+ }
124
+ if (current()) failed();
125
+ } finally {
126
+ recovering = false;
127
+ }
128
+ };
129
+ const changed = (state: string) => {
130
+ if (disposed) return;
131
+ if (state === 'connected') {
132
+ clearTimer();
133
+ if (!recovering) { attempts = 0; report('connected'); }
134
+ } else if (state === 'disconnected') {
135
+ if (recovering || timer) return;
136
+ report('waiting');
137
+ timer = setTimeout(() => { timer = undefined; void restart(); }, 3000);
138
+ } else if (state === 'failed') {
139
+ clearTimer();
140
+ void restart();
141
+ } else if (state === 'closed') dispose();
142
+ };
143
+ socket.on?.('disconnect', dispose);
144
+ transport.on('connectionstatechange', changed);
145
+ transport.observer?.once('close', dispose);
146
+ return dispose;
147
+ }
148
+
149
+ /** Clear publishing indicators only if this is still the active transport. */
150
+ export function clearFailedSendState(parameters: any, transport: any, local = false) {
151
+ const live = parameters.getCurrentParams?.() ?? parameters.getUpdatedAllParams?.() ?? parameters;
152
+ if ((local ? live.localProducerTransport : live.producerTransport) !== transport) return;
153
+ if (local) {
154
+ live.updateLocalProducerTransport?.(null);
155
+ live.updateLocalTransportCreated?.(false);
156
+ return;
157
+ }
158
+ for (const kind of ['Audio', 'Video', 'Screen']) {
159
+ const key = kind.toLowerCase();
160
+ live['update' + kind + 'Producer']?.(null);
161
+ live['updateTransportCreated' + kind]?.(false);
162
+ live['update' + kind + 'AlreadyOn']?.(false);
163
+ // Notify room membership using the existing pause protocol, preserving
164
+ // users' local tracks for an explicit retry.
165
+ live.socket?.emit('pauseProducerMedia', { mediaTag: key, roomName: live.roomName });
166
+ }
167
+ live.updateProducerTransport?.(null);
168
+ live.updateTransportCreated?.(false);
169
+ live.showAlert?.({ message: 'Media connection could not recover. Rejoin the room to restore audio and video.', type: 'danger', duration: 10000 });
170
+ }
package/src/index.ts CHANGED
@@ -117,6 +117,7 @@ export * from './methods/stream/switchAudio';
117
117
  export * from './methods/stream/switchVideo';
118
118
  export * from './methods/stream/switchVideoAlt';
119
119
  export * from './methods/stream/clickAudio';
120
+ export type { AudioProcessingOptions } from './consumers/audioProcessing';
120
121
  export * from './methods/stream/clickChat';
121
122
  export * from './methods/stream/clickScreenShare';
122
123
  export * from './methods/utils/liveSubtitle';
@@ -230,3 +231,4 @@ export * from './methods/utils/headless';
230
231
  export * from './methods/utils/getParticipantMedia';
231
232
  // Shared web compositor used by headless virtual-background processing.
232
233
  export * from './methods/utils/virtualBackgroundCompositor';
234
+ export * from './methods/utils/virtualBackgroundFrameLoop';
@@ -72,7 +72,10 @@ export interface ClickAudioParameters
72
72
  [key: string]: any;
73
73
  }
74
74
 
75
+ import { AudioProcessingOptions, audioProcessingConstraints, applyAudioProcessing } from '../../consumers/audioProcessing';
76
+
75
77
  export interface ClickAudioOptions {
78
+ audioProcessing?: AudioProcessingOptions;
76
79
  parameters: ClickAudioParameters;
77
80
  }
78
81
 
@@ -88,7 +91,7 @@ export type ClickAudioType = (options: ClickAudioOptions) => Promise<void>;
88
91
  * @param options Function options containing the full runtime parameter bag.
89
92
  * @returns A promise that resolves after the microphone action has been processed.
90
93
  */
91
- export const clickAudio = async ({ parameters }: ClickAudioOptions): Promise<void> => {
94
+ export const clickAudio = async ({ parameters, audioProcessing }: ClickAudioOptions): Promise<void> => {
92
95
  let {
93
96
  checkMediaPermission,
94
97
  hasAudioPermission,
@@ -307,6 +310,7 @@ export const clickAudio = async ({ parameters }: ClickAudioOptions): Promise<voi
307
310
 
308
311
  case 0:
309
312
  if (audioPaused) {
313
+ await applyAudioProcessing(localStream!.getAudioTracks()[0], audioProcessing);
310
314
  if (localStream && localStream.getAudioTracks().length > 0) {
311
315
  localStream.getAudioTracks()[0].enabled = true;
312
316
  }
@@ -355,9 +359,13 @@ export const clickAudio = async ({ parameters }: ClickAudioOptions): Promise<voi
355
359
  }
356
360
  }
357
361
 
358
- const mediaConstraints = userDefaultAudioInputDevice
359
- ? { audio: { deviceId: userDefaultAudioInputDevice }, video: false }
360
- : { audio: true, video: false };
362
+ const processing = audioProcessingConstraints(audioProcessing);
363
+ const mediaConstraints = {
364
+ audio: userDefaultAudioInputDevice || Object.keys(processing).length
365
+ ? { ...(userDefaultAudioInputDevice ? { deviceId: userDefaultAudioInputDevice } : {}), ...processing }
366
+ : true,
367
+ video: false,
368
+ };
361
369
 
362
370
  try {
363
371
  const stream = await mediaDevices.getUserMedia(mediaConstraints);
@@ -377,4 +385,4 @@ export const clickAudio = async ({ parameters }: ClickAudioOptions): Promise<voi
377
385
  default:
378
386
  }
379
387
  }
380
- };
388
+ };
@@ -14,7 +14,10 @@ export interface SwitchAudioParameters extends SwitchUserAudioParameters {
14
14
  [key: string]: any
15
15
  }
16
16
 
17
+ import { AudioProcessingOptions, applyAudioProcessing } from '../../consumers/audioProcessing';
18
+
17
19
  export interface SwitchAudioOptions {
20
+ audioProcessing?: AudioProcessingOptions;
18
21
  audioPreference: string
19
22
  parameters: SwitchAudioParameters
20
23
  }
@@ -47,6 +50,7 @@ export type SwitchAudioType = (options: SwitchAudioOptions) => Promise<void>
47
50
 
48
51
  export const switchAudio = async ({
49
52
  audioPreference,
53
+ audioProcessing,
50
54
  parameters
51
55
  }: SwitchAudioOptions): Promise<void> => {
52
56
  let {
@@ -67,7 +71,12 @@ export const switchAudio = async ({
67
71
  updateUserDefaultAudioInputDevice(userDefaultAudioInputDevice)
68
72
 
69
73
  if (defAudioID) {
70
- await switchUserAudio({ audioPreference, parameters })
74
+ await switchUserAudio({ audioPreference, parameters, audioProcessing })
71
75
  }
76
+ } else if (audioProcessing !== undefined) {
77
+ const track = parameters.localStreamAudio?.getAudioTracks?.()[0]
78
+ ?? parameters.localStream?.getAudioTracks?.()[0]
79
+ ?? parameters.audioProducer?.track;
80
+ if (track) await applyAudioProcessing(track, audioProcessing);
72
81
  }
73
82
  }
@@ -7,6 +7,7 @@ import {
7
7
  compositeVirtualBackgroundFrame,
8
8
  DEFAULT_BACKGROUND_BLUR_PIXELS,
9
9
  } from '../virtualBackgroundCompositor';
10
+ import { startVirtualBackgroundFrameLoop } from '../virtualBackgroundFrameLoop';
10
11
 
11
12
  const MEDIAPIPE_CDN = 'https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation';
12
13
 
@@ -65,6 +66,8 @@ export interface ApplyVirtualBackgroundOptions extends HeadlessOptions {
65
66
  * is greater than zero it takes precedence over `image`. Defaults to 0.
66
67
  */
67
68
  blurPixels?: number;
69
+ /** Best-effort hidden-tab processing for blur and image backgrounds. Defaults to true. */
70
+ keepProcessingWhenHidden?: boolean;
68
71
  }
69
72
 
70
73
  export type ApplyVirtualBackgroundType = (
@@ -107,6 +110,7 @@ export async function applyVirtualBackground({
107
110
  publish = true,
108
111
  assetPath = MEDIAPIPE_CDN,
109
112
  blurPixels = 0,
113
+ keepProcessingWhenHidden = true,
110
114
  }: ApplyVirtualBackgroundOptions): Promise<HeadlessActionResult & { stream: MediaStream | null }> {
111
115
  const live = getCurrentParams({ parameters });
112
116
  if (live.audioOnlyRoom) {
@@ -189,27 +193,31 @@ export async function applyVirtualBackground({
189
193
  });
190
194
 
191
195
  let stopped = false;
192
- let frameHandle = 0;
193
- const pump = async () => {
194
- if (stopped) return;
195
- try {
196
- if (video.readyState >= 2) await segmentation.send({ image: video });
197
- } catch {
198
- // Ignore a single failed frame; the next one is already queued.
199
- }
200
- frameHandle = window.requestAnimationFrame(() => { void pump(); });
201
- };
202
- void pump();
196
+ const stopFrameLoop = startVirtualBackgroundFrameLoop({
197
+ owner: segmentation,
198
+ keepProcessingWhenHidden,
199
+ shouldContinue: () => !stopped && processingTrack.readyState === 'live',
200
+ processFrame: () => video.readyState >= 2
201
+ ? segmentation.send({ image: video })
202
+ : undefined,
203
+ });
203
204
 
204
- const processed: MediaStream = (canvas as any).captureStream(frameRate);
205
+ let processed: MediaStream;
206
+ try {
207
+ processed = (canvas as any).captureStream(frameRate);
208
+ } catch (error) {
209
+ stopFrameLoop();
210
+ throw error;
211
+ }
205
212
  if (!processed?.getVideoTracks?.().length) {
213
+ stopFrameLoop();
206
214
  throw new Error('The processed background stream produced no video track.');
207
215
  }
208
216
 
209
217
  const stop = () => {
210
218
  if (stopped) return;
211
219
  stopped = true;
212
- window.cancelAnimationFrame(frameHandle);
220
+ stopFrameLoop();
213
221
  try { segmentation.close(); } catch { /* already closed */ }
214
222
  processed.getTracks().forEach((track) => track.stop());
215
223
  processingTrack.stop();
@@ -0,0 +1,69 @@
1
+ export interface VirtualBackgroundFrameLoopOptions {
2
+ owner: object;
3
+ processFrame: () => Promise<unknown> | unknown;
4
+ shouldContinue: () => boolean;
5
+ /** Best effort: background tabs may still throttle timers and camera capture. */
6
+ keepProcessingWhenHidden?: boolean;
7
+ }
8
+
9
+ const activeLoops = new WeakMap<object, () => void>();
10
+ const activeFrames = new WeakMap<object, Promise<unknown>>();
11
+
12
+ /** Run at most one segmentation frame at a time, including in hidden tabs. */
13
+ export function startVirtualBackgroundFrameLoop({
14
+ owner,
15
+ processFrame,
16
+ shouldContinue,
17
+ keepProcessingWhenHidden = true,
18
+ }: VirtualBackgroundFrameLoopOptions): () => void {
19
+ activeLoops.get(owner)?.();
20
+ let stopped = false;
21
+ let inFlight = false;
22
+ let animationFrameId: number | null = null;
23
+ let timeoutId: ReturnType<typeof setTimeout> | null = null;
24
+
25
+ const cancelScheduledFrame = () => {
26
+ if (animationFrameId !== null) cancelAnimationFrame(animationFrameId);
27
+ if (timeoutId !== null) clearTimeout(timeoutId);
28
+ animationFrameId = null;
29
+ timeoutId = null;
30
+ };
31
+ const stop = () => {
32
+ if (stopped) return;
33
+ stopped = true;
34
+ cancelScheduledFrame();
35
+ document.removeEventListener('visibilitychange', onVisibilityChange);
36
+ if (activeLoops.get(owner) === stop) activeLoops.delete(owner);
37
+ };
38
+ const schedule = () => {
39
+ if (stopped || inFlight) return;
40
+ if (!shouldContinue()) { stop(); return; }
41
+ if (document.hidden && keepProcessingWhenHidden) {
42
+ timeoutId = setTimeout(() => { void runFrame(); }, 200);
43
+ } else {
44
+ animationFrameId = requestAnimationFrame(() => { void runFrame(); });
45
+ }
46
+ };
47
+ const runFrame = async () => {
48
+ animationFrameId = null;
49
+ timeoutId = null;
50
+ if (stopped || inFlight) return;
51
+ if (!shouldContinue()) { stop(); return; }
52
+ inFlight = true;
53
+ try {
54
+ const previousFrame = activeFrames.get(owner);
55
+ if (previousFrame) await previousFrame.catch(() => undefined);
56
+ if (stopped || !shouldContinue()) { stop(); return; }
57
+ const currentFrame = Promise.resolve(processFrame());
58
+ activeFrames.set(owner, currentFrame);
59
+ try { await currentFrame; }
60
+ finally { if (activeFrames.get(owner) === currentFrame) activeFrames.delete(owner); }
61
+ } catch { /* A dropped frame is recoverable. */ }
62
+ finally { inFlight = false; schedule(); }
63
+ };
64
+ const onVisibilityChange = () => { cancelScheduledFrame(); schedule(); };
65
+ activeLoops.set(owner, stop);
66
+ document.addEventListener('visibilitychange', onVisibilityChange);
67
+ void runFrame();
68
+ return stop;
69
+ }
@@ -240,6 +240,7 @@ export type SeedData = {
240
240
  };
241
241
 
242
242
  export interface MeetingRoomParams {
243
+ backendAudioDenoise?: { enabled: boolean; profile?: 'arnndn' }; // Optional; omitted uses the account preset.
243
244
  itemPageLimit: number;
244
245
  mediaType: 'audio' | 'video';
245
246
  addCoHost: boolean;
@@ -316,6 +317,7 @@ export type ScreenParamsType = {
316
317
  };
317
318
 
318
319
  export interface CreateRoomOptions {
320
+ backendAudioDenoise?: { enabled: boolean; profile?: 'arnndn' };
319
321
  action: 'create' | 'join';
320
322
  meetingID: string;
321
323
  duration: number;
@@ -339,6 +341,7 @@ export interface CreateRoomOptions {
339
341
  }
340
342
 
341
343
  export interface CreateMediaSFURoomOptions {
344
+ backendAudioDenoise?: { enabled: boolean; profile?: 'arnndn' };
342
345
  action: 'create';
343
346
  duration: number;
344
347
  capacity: number;