@unwanted/matrix-sdk-mini 34.12.0-1 → 34.12.0-2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (185) hide show
  1. package/git-revision.txt +1 -1
  2. package/lib/@types/event.d.ts +0 -19
  3. package/lib/@types/event.d.ts.map +1 -1
  4. package/lib/@types/event.js.map +1 -1
  5. package/lib/client.d.ts +2 -50
  6. package/lib/client.d.ts.map +1 -1
  7. package/lib/client.js +391 -501
  8. package/lib/client.js.map +1 -1
  9. package/lib/embedded.d.ts.map +1 -1
  10. package/lib/embedded.js +0 -1
  11. package/lib/embedded.js.map +1 -1
  12. package/lib/matrix.d.ts +0 -6
  13. package/lib/matrix.d.ts.map +1 -1
  14. package/lib/matrix.js +1 -5
  15. package/lib/matrix.js.map +1 -1
  16. package/package.json +1 -1
  17. package/src/@types/event.ts +2 -36
  18. package/src/client.ts +1 -150
  19. package/src/embedded.ts +0 -2
  20. package/src/matrix.ts +0 -13
  21. package/lib/matrixrtc/CallMembership.d.ts +0 -66
  22. package/lib/matrixrtc/CallMembership.d.ts.map +0 -1
  23. package/lib/matrixrtc/CallMembership.js +0 -197
  24. package/lib/matrixrtc/CallMembership.js.map +0 -1
  25. package/lib/matrixrtc/LivekitFocus.d.ts +0 -16
  26. package/lib/matrixrtc/LivekitFocus.d.ts.map +0 -1
  27. package/lib/matrixrtc/LivekitFocus.js +0 -20
  28. package/lib/matrixrtc/LivekitFocus.js.map +0 -1
  29. package/lib/matrixrtc/MatrixRTCSession.d.ts +0 -295
  30. package/lib/matrixrtc/MatrixRTCSession.d.ts.map +0 -1
  31. package/lib/matrixrtc/MatrixRTCSession.js +0 -1043
  32. package/lib/matrixrtc/MatrixRTCSession.js.map +0 -1
  33. package/lib/matrixrtc/MatrixRTCSessionManager.d.ts +0 -40
  34. package/lib/matrixrtc/MatrixRTCSessionManager.d.ts.map +0 -1
  35. package/lib/matrixrtc/MatrixRTCSessionManager.js +0 -146
  36. package/lib/matrixrtc/MatrixRTCSessionManager.js.map +0 -1
  37. package/lib/matrixrtc/focus.d.ts +0 -10
  38. package/lib/matrixrtc/focus.d.ts.map +0 -1
  39. package/lib/matrixrtc/focus.js +0 -1
  40. package/lib/matrixrtc/focus.js.map +0 -1
  41. package/lib/matrixrtc/index.d.ts +0 -7
  42. package/lib/matrixrtc/index.d.ts.map +0 -1
  43. package/lib/matrixrtc/index.js +0 -21
  44. package/lib/matrixrtc/index.js.map +0 -1
  45. package/lib/matrixrtc/types.d.ts +0 -19
  46. package/lib/matrixrtc/types.d.ts.map +0 -1
  47. package/lib/matrixrtc/types.js +0 -1
  48. package/lib/matrixrtc/types.js.map +0 -1
  49. package/lib/webrtc/audioContext.d.ts +0 -15
  50. package/lib/webrtc/audioContext.d.ts.map +0 -1
  51. package/lib/webrtc/audioContext.js +0 -46
  52. package/lib/webrtc/audioContext.js.map +0 -1
  53. package/lib/webrtc/call.d.ts +0 -560
  54. package/lib/webrtc/call.d.ts.map +0 -1
  55. package/lib/webrtc/call.js +0 -2541
  56. package/lib/webrtc/call.js.map +0 -1
  57. package/lib/webrtc/callEventHandler.d.ts +0 -37
  58. package/lib/webrtc/callEventHandler.d.ts.map +0 -1
  59. package/lib/webrtc/callEventHandler.js +0 -344
  60. package/lib/webrtc/callEventHandler.js.map +0 -1
  61. package/lib/webrtc/callEventTypes.d.ts +0 -73
  62. package/lib/webrtc/callEventTypes.d.ts.map +0 -1
  63. package/lib/webrtc/callEventTypes.js +0 -13
  64. package/lib/webrtc/callEventTypes.js.map +0 -1
  65. package/lib/webrtc/callFeed.d.ts +0 -128
  66. package/lib/webrtc/callFeed.d.ts.map +0 -1
  67. package/lib/webrtc/callFeed.js +0 -289
  68. package/lib/webrtc/callFeed.js.map +0 -1
  69. package/lib/webrtc/groupCall.d.ts +0 -323
  70. package/lib/webrtc/groupCall.d.ts.map +0 -1
  71. package/lib/webrtc/groupCall.js +0 -1337
  72. package/lib/webrtc/groupCall.js.map +0 -1
  73. package/lib/webrtc/groupCallEventHandler.d.ts +0 -31
  74. package/lib/webrtc/groupCallEventHandler.d.ts.map +0 -1
  75. package/lib/webrtc/groupCallEventHandler.js +0 -178
  76. package/lib/webrtc/groupCallEventHandler.js.map +0 -1
  77. package/lib/webrtc/mediaHandler.d.ts +0 -89
  78. package/lib/webrtc/mediaHandler.d.ts.map +0 -1
  79. package/lib/webrtc/mediaHandler.js +0 -437
  80. package/lib/webrtc/mediaHandler.js.map +0 -1
  81. package/lib/webrtc/stats/callFeedStatsReporter.d.ts +0 -8
  82. package/lib/webrtc/stats/callFeedStatsReporter.d.ts.map +0 -1
  83. package/lib/webrtc/stats/callFeedStatsReporter.js +0 -82
  84. package/lib/webrtc/stats/callFeedStatsReporter.js.map +0 -1
  85. package/lib/webrtc/stats/callStatsReportGatherer.d.ts +0 -25
  86. package/lib/webrtc/stats/callStatsReportGatherer.d.ts.map +0 -1
  87. package/lib/webrtc/stats/callStatsReportGatherer.js +0 -199
  88. package/lib/webrtc/stats/callStatsReportGatherer.js.map +0 -1
  89. package/lib/webrtc/stats/callStatsReportSummary.d.ts +0 -17
  90. package/lib/webrtc/stats/callStatsReportSummary.d.ts.map +0 -1
  91. package/lib/webrtc/stats/callStatsReportSummary.js +0 -1
  92. package/lib/webrtc/stats/callStatsReportSummary.js.map +0 -1
  93. package/lib/webrtc/stats/connectionStats.d.ts +0 -28
  94. package/lib/webrtc/stats/connectionStats.d.ts.map +0 -1
  95. package/lib/webrtc/stats/connectionStats.js +0 -26
  96. package/lib/webrtc/stats/connectionStats.js.map +0 -1
  97. package/lib/webrtc/stats/connectionStatsBuilder.d.ts +0 -5
  98. package/lib/webrtc/stats/connectionStatsBuilder.d.ts.map +0 -1
  99. package/lib/webrtc/stats/connectionStatsBuilder.js +0 -27
  100. package/lib/webrtc/stats/connectionStatsBuilder.js.map +0 -1
  101. package/lib/webrtc/stats/connectionStatsReportBuilder.d.ts +0 -7
  102. package/lib/webrtc/stats/connectionStatsReportBuilder.d.ts.map +0 -1
  103. package/lib/webrtc/stats/connectionStatsReportBuilder.js +0 -121
  104. package/lib/webrtc/stats/connectionStatsReportBuilder.js.map +0 -1
  105. package/lib/webrtc/stats/groupCallStats.d.ts +0 -22
  106. package/lib/webrtc/stats/groupCallStats.d.ts.map +0 -1
  107. package/lib/webrtc/stats/groupCallStats.js +0 -78
  108. package/lib/webrtc/stats/groupCallStats.js.map +0 -1
  109. package/lib/webrtc/stats/media/mediaSsrcHandler.d.ts +0 -10
  110. package/lib/webrtc/stats/media/mediaSsrcHandler.d.ts.map +0 -1
  111. package/lib/webrtc/stats/media/mediaSsrcHandler.js +0 -57
  112. package/lib/webrtc/stats/media/mediaSsrcHandler.js.map +0 -1
  113. package/lib/webrtc/stats/media/mediaTrackHandler.d.ts +0 -12
  114. package/lib/webrtc/stats/media/mediaTrackHandler.d.ts.map +0 -1
  115. package/lib/webrtc/stats/media/mediaTrackHandler.js +0 -62
  116. package/lib/webrtc/stats/media/mediaTrackHandler.js.map +0 -1
  117. package/lib/webrtc/stats/media/mediaTrackStats.d.ts +0 -86
  118. package/lib/webrtc/stats/media/mediaTrackStats.d.ts.map +0 -1
  119. package/lib/webrtc/stats/media/mediaTrackStats.js +0 -142
  120. package/lib/webrtc/stats/media/mediaTrackStats.js.map +0 -1
  121. package/lib/webrtc/stats/media/mediaTrackStatsHandler.d.ts +0 -22
  122. package/lib/webrtc/stats/media/mediaTrackStatsHandler.d.ts.map +0 -1
  123. package/lib/webrtc/stats/media/mediaTrackStatsHandler.js +0 -76
  124. package/lib/webrtc/stats/media/mediaTrackStatsHandler.js.map +0 -1
  125. package/lib/webrtc/stats/statsReport.d.ts +0 -99
  126. package/lib/webrtc/stats/statsReport.d.ts.map +0 -1
  127. package/lib/webrtc/stats/statsReport.js +0 -32
  128. package/lib/webrtc/stats/statsReport.js.map +0 -1
  129. package/lib/webrtc/stats/statsReportEmitter.d.ts +0 -15
  130. package/lib/webrtc/stats/statsReportEmitter.d.ts.map +0 -1
  131. package/lib/webrtc/stats/statsReportEmitter.js +0 -33
  132. package/lib/webrtc/stats/statsReportEmitter.js.map +0 -1
  133. package/lib/webrtc/stats/summaryStatsReportGatherer.d.ts +0 -16
  134. package/lib/webrtc/stats/summaryStatsReportGatherer.d.ts.map +0 -1
  135. package/lib/webrtc/stats/summaryStatsReportGatherer.js +0 -116
  136. package/lib/webrtc/stats/summaryStatsReportGatherer.js.map +0 -1
  137. package/lib/webrtc/stats/trackStatsBuilder.d.ts +0 -19
  138. package/lib/webrtc/stats/trackStatsBuilder.d.ts.map +0 -1
  139. package/lib/webrtc/stats/trackStatsBuilder.js +0 -168
  140. package/lib/webrtc/stats/trackStatsBuilder.js.map +0 -1
  141. package/lib/webrtc/stats/transportStats.d.ts +0 -11
  142. package/lib/webrtc/stats/transportStats.d.ts.map +0 -1
  143. package/lib/webrtc/stats/transportStats.js +0 -1
  144. package/lib/webrtc/stats/transportStats.js.map +0 -1
  145. package/lib/webrtc/stats/transportStatsBuilder.d.ts +0 -5
  146. package/lib/webrtc/stats/transportStatsBuilder.d.ts.map +0 -1
  147. package/lib/webrtc/stats/transportStatsBuilder.js +0 -34
  148. package/lib/webrtc/stats/transportStatsBuilder.js.map +0 -1
  149. package/lib/webrtc/stats/valueFormatter.d.ts +0 -4
  150. package/lib/webrtc/stats/valueFormatter.d.ts.map +0 -1
  151. package/lib/webrtc/stats/valueFormatter.js +0 -25
  152. package/lib/webrtc/stats/valueFormatter.js.map +0 -1
  153. package/src/matrixrtc/CallMembership.ts +0 -247
  154. package/src/matrixrtc/LivekitFocus.ts +0 -39
  155. package/src/matrixrtc/MatrixRTCSession.ts +0 -1319
  156. package/src/matrixrtc/MatrixRTCSessionManager.ts +0 -166
  157. package/src/matrixrtc/focus.ts +0 -25
  158. package/src/matrixrtc/index.ts +0 -22
  159. package/src/matrixrtc/types.ts +0 -36
  160. package/src/webrtc/audioContext.ts +0 -44
  161. package/src/webrtc/call.ts +0 -3074
  162. package/src/webrtc/callEventHandler.ts +0 -425
  163. package/src/webrtc/callEventTypes.ts +0 -93
  164. package/src/webrtc/callFeed.ts +0 -364
  165. package/src/webrtc/groupCall.ts +0 -1735
  166. package/src/webrtc/groupCallEventHandler.ts +0 -234
  167. package/src/webrtc/mediaHandler.ts +0 -484
  168. package/src/webrtc/stats/callFeedStatsReporter.ts +0 -94
  169. package/src/webrtc/stats/callStatsReportGatherer.ts +0 -219
  170. package/src/webrtc/stats/callStatsReportSummary.ts +0 -30
  171. package/src/webrtc/stats/connectionStats.ts +0 -47
  172. package/src/webrtc/stats/connectionStatsBuilder.ts +0 -28
  173. package/src/webrtc/stats/connectionStatsReportBuilder.ts +0 -140
  174. package/src/webrtc/stats/groupCallStats.ts +0 -93
  175. package/src/webrtc/stats/media/mediaSsrcHandler.ts +0 -57
  176. package/src/webrtc/stats/media/mediaTrackHandler.ts +0 -76
  177. package/src/webrtc/stats/media/mediaTrackStats.ts +0 -176
  178. package/src/webrtc/stats/media/mediaTrackStatsHandler.ts +0 -90
  179. package/src/webrtc/stats/statsReport.ts +0 -133
  180. package/src/webrtc/stats/statsReportEmitter.ts +0 -49
  181. package/src/webrtc/stats/summaryStatsReportGatherer.ts +0 -148
  182. package/src/webrtc/stats/trackStatsBuilder.ts +0 -207
  183. package/src/webrtc/stats/transportStats.ts +0 -26
  184. package/src/webrtc/stats/transportStatsBuilder.ts +0 -48
  185. package/src/webrtc/stats/valueFormatter.ts +0 -27
@@ -1,3074 +0,0 @@
1
- /*
2
- Copyright 2015, 2016 OpenMarket Ltd
3
- Copyright 2017 New Vector Ltd
4
- Copyright 2019, 2020 The Matrix.org Foundation C.I.C.
5
- Copyright 2021 - 2022 Šimon Brandner <simon.bra.ag@gmail.com>
6
-
7
- Licensed under the Apache License, Version 2.0 (the "License");
8
- you may not use this file except in compliance with the License.
9
- You may obtain a copy of the License at
10
-
11
- http://www.apache.org/licenses/LICENSE-2.0
12
-
13
- Unless required by applicable law or agreed to in writing, software
14
- distributed under the License is distributed on an "AS IS" BASIS,
15
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
- See the License for the specific language governing permissions and
17
- limitations under the License.
18
- */
19
-
20
- /**
21
- * This is an internal module. See {@link createNewMatrixCall} for the public API.
22
- */
23
-
24
- import { v4 as uuidv4 } from "uuid";
25
- import { parse as parseSdp, write as writeSdp } from "sdp-transform";
26
-
27
- import { logger } from "../logger.ts";
28
- import { checkObjectHasKeys, isNullOrUndefined, recursivelyAssign } from "../utils.ts";
29
- import { MatrixEvent } from "../models/event.ts";
30
- import { EventType, TimelineEvents, ToDeviceMessageId } from "../@types/event.ts";
31
- import { RoomMember } from "../models/room-member.ts";
32
- import { randomString } from "../randomstring.ts";
33
- import {
34
- MCallReplacesEvent,
35
- MCallAnswer,
36
- MCallInviteNegotiate,
37
- CallCapabilities,
38
- SDPStreamMetadataPurpose,
39
- SDPStreamMetadata,
40
- SDPStreamMetadataKey,
41
- MCallSDPStreamMetadataChanged,
42
- MCallSelectAnswer,
43
- MCAllAssertedIdentity,
44
- MCallCandidates,
45
- MCallBase,
46
- MCallHangupReject,
47
- } from "./callEventTypes.ts";
48
- import { CallFeed } from "./callFeed.ts";
49
- import { MatrixClient } from "../client.ts";
50
- import { EventEmitterEvents, TypedEventEmitter } from "../models/typed-event-emitter.ts";
51
- import { DeviceInfo } from "../crypto/deviceinfo.ts";
52
- import { GroupCallUnknownDeviceError } from "./groupCall.ts";
53
- import { IScreensharingOpts } from "./mediaHandler.ts";
54
- import { MatrixError } from "../http-api/index.ts";
55
- import { GroupCallStats } from "./stats/groupCallStats.ts";
56
-
57
- interface CallOpts {
58
- // The room ID for this call.
59
- roomId: string;
60
- invitee?: string;
61
- // The Matrix Client instance to send events to.
62
- client: MatrixClient;
63
- /**
64
- * Whether relay through TURN should be forced.
65
- * @deprecated use opts.forceTURN when creating the matrix client
66
- * since it's only possible to set this option on outbound calls.
67
- */
68
- forceTURN?: boolean;
69
- // A list of TURN servers.
70
- turnServers?: Array<TurnServer>;
71
- opponentDeviceId?: string;
72
- opponentSessionId?: string;
73
- groupCallId?: string;
74
- }
75
-
76
- interface TurnServer {
77
- urls: Array<string>;
78
- username?: string;
79
- password?: string;
80
- ttl?: number;
81
- }
82
-
83
- interface AssertedIdentity {
84
- id: string;
85
- displayName: string;
86
- }
87
-
88
- enum MediaType {
89
- AUDIO = "audio",
90
- VIDEO = "video",
91
- }
92
-
93
- enum CodecName {
94
- OPUS = "opus",
95
- // add more as needed
96
- }
97
-
98
- // Used internally to specify modifications to codec parameters in SDP
99
- interface CodecParamsMod {
100
- mediaType: MediaType;
101
- codec: CodecName;
102
- enableDtx?: boolean; // true to enable discontinuous transmission, false to disable, undefined to leave as-is
103
- maxAverageBitrate?: number; // sets the max average bitrate, or undefined to leave as-is
104
- }
105
-
106
- export enum CallState {
107
- Fledgling = "fledgling",
108
- InviteSent = "invite_sent",
109
- WaitLocalMedia = "wait_local_media",
110
- CreateOffer = "create_offer",
111
- CreateAnswer = "create_answer",
112
- Connecting = "connecting",
113
- Connected = "connected",
114
- Ringing = "ringing",
115
- Ended = "ended",
116
- }
117
-
118
- export enum CallType {
119
- Voice = "voice",
120
- Video = "video",
121
- }
122
-
123
- export enum CallDirection {
124
- Inbound = "inbound",
125
- Outbound = "outbound",
126
- }
127
-
128
- export enum CallParty {
129
- Local = "local",
130
- Remote = "remote",
131
- }
132
-
133
- export enum CallEvent {
134
- Hangup = "hangup",
135
- State = "state",
136
- Error = "error",
137
- Replaced = "replaced",
138
-
139
- // The value of isLocalOnHold() has changed
140
- LocalHoldUnhold = "local_hold_unhold",
141
- // The value of isRemoteOnHold() has changed
142
- RemoteHoldUnhold = "remote_hold_unhold",
143
- // backwards compat alias for LocalHoldUnhold: remove in a major version bump
144
- HoldUnhold = "hold_unhold",
145
- // Feeds have changed
146
- FeedsChanged = "feeds_changed",
147
-
148
- AssertedIdentityChanged = "asserted_identity_changed",
149
-
150
- LengthChanged = "length_changed",
151
-
152
- DataChannel = "datachannel",
153
-
154
- SendVoipEvent = "send_voip_event",
155
-
156
- // When the call instantiates its peer connection
157
- // For apps that want to access the underlying peer connection, eg for debugging
158
- PeerConnectionCreated = "peer_connection_created",
159
- }
160
-
161
- export enum CallErrorCode {
162
- /** The user chose to end the call */
163
- UserHangup = "user_hangup",
164
-
165
- /** An error code when the local client failed to create an offer. */
166
- LocalOfferFailed = "local_offer_failed",
167
- /**
168
- * An error code when there is no local mic/camera to use. This may be because
169
- * the hardware isn't plugged in, or the user has explicitly denied access.
170
- */
171
- NoUserMedia = "no_user_media",
172
-
173
- /**
174
- * Error code used when a call event failed to send
175
- * because unknown devices were present in the room
176
- */
177
- UnknownDevices = "unknown_devices",
178
-
179
- /**
180
- * Error code used when we fail to send the invite
181
- * for some reason other than there being unknown devices
182
- */
183
- SendInvite = "send_invite",
184
-
185
- /**
186
- * An answer could not be created
187
- */
188
- CreateAnswer = "create_answer",
189
-
190
- /**
191
- * An offer could not be created
192
- */
193
- CreateOffer = "create_offer",
194
-
195
- /**
196
- * Error code used when we fail to send the answer
197
- * for some reason other than there being unknown devices
198
- */
199
- SendAnswer = "send_answer",
200
-
201
- /**
202
- * The session description from the other side could not be set
203
- */
204
- SetRemoteDescription = "set_remote_description",
205
-
206
- /**
207
- * The session description from this side could not be set
208
- */
209
- SetLocalDescription = "set_local_description",
210
-
211
- /**
212
- * A different device answered the call
213
- */
214
- AnsweredElsewhere = "answered_elsewhere",
215
-
216
- /**
217
- * No media connection could be established to the other party
218
- */
219
- IceFailed = "ice_failed",
220
-
221
- /**
222
- * The invite timed out whilst waiting for an answer
223
- */
224
- InviteTimeout = "invite_timeout",
225
-
226
- /**
227
- * The call was replaced by another call
228
- */
229
- Replaced = "replaced",
230
-
231
- /**
232
- * Signalling for the call could not be sent (other than the initial invite)
233
- */
234
- SignallingFailed = "signalling_timeout",
235
-
236
- /**
237
- * The remote party is busy
238
- */
239
- UserBusy = "user_busy",
240
-
241
- /**
242
- * We transferred the call off to somewhere else
243
- */
244
- Transferred = "transferred",
245
-
246
- /**
247
- * A call from the same user was found with a new session id
248
- */
249
- NewSession = "new_session",
250
- }
251
-
252
- /**
253
- * The version field that we set in m.call.* events
254
- */
255
- const VOIP_PROTO_VERSION = "1";
256
-
257
- /** The fallback ICE server to use for STUN or TURN protocols. */
258
- export const FALLBACK_ICE_SERVER = "stun:turn.matrix.org";
259
-
260
- /** The length of time a call can be ringing for. */
261
- const CALL_TIMEOUT_MS = 60 * 1000; // ms
262
- /** The time after which we increment callLength */
263
- const CALL_LENGTH_INTERVAL = 1000; // ms
264
- /** The time after which we end the call, if ICE got disconnected */
265
- const ICE_DISCONNECTED_TIMEOUT = 30 * 1000; // ms
266
- /** The time after which we try a ICE restart, if ICE got disconnected */
267
- const ICE_RECONNECTING_TIMEOUT = 2 * 1000; // ms
268
- export class CallError extends Error {
269
- public readonly code: string;
270
-
271
- public constructor(code: CallErrorCode, msg: string, err: Error) {
272
- // Still don't think there's any way to have proper nested errors
273
- super(msg + ": " + err);
274
-
275
- this.code = code;
276
- }
277
- }
278
-
279
- export function genCallID(): string {
280
- return Date.now().toString() + randomString(16);
281
- }
282
-
283
- function getCodecParamMods(isPtt: boolean): CodecParamsMod[] {
284
- const mods = [
285
- {
286
- mediaType: "audio",
287
- codec: "opus",
288
- enableDtx: true,
289
- maxAverageBitrate: isPtt ? 12000 : undefined,
290
- },
291
- ] as CodecParamsMod[];
292
-
293
- return mods;
294
- }
295
-
296
- type CallEventType =
297
- | EventType.CallReplaces
298
- | EventType.CallAnswer
299
- | EventType.CallSelectAnswer
300
- | EventType.CallNegotiate
301
- | EventType.CallInvite
302
- | EventType.CallCandidates
303
- | EventType.CallHangup
304
- | EventType.CallReject
305
- | EventType.CallSDPStreamMetadataChangedPrefix;
306
-
307
- export interface VoipEvent {
308
- type: "toDevice" | "sendEvent";
309
- eventType: string;
310
- userId?: string;
311
- opponentDeviceId?: string;
312
- roomId?: string;
313
- content: TimelineEvents[CallEventType];
314
- }
315
-
316
- /**
317
- * These now all have the call object as an argument. Why? Well, to know which call a given event is
318
- * about you have three options:
319
- * 1. Use a closure as the callback that remembers what call it's listening to. This can be
320
- * a pain because you need to pass the listener function again when you remove the listener,
321
- * which might be somewhere else.
322
- * 2. Use not-very-well-known fact that EventEmitter sets 'this' to the emitter object in the
323
- * callback. This doesn't really play well with modern Typescript and eslint and doesn't work
324
- * with our pattern of re-emitting events.
325
- * 3. Pass the object in question as an argument to the callback.
326
- *
327
- * Now that we have group calls which have to deal with multiple call objects, this will
328
- * become more important, and I think methods 1 and 2 are just going to cause issues.
329
- */
330
- export type CallEventHandlerMap = {
331
- [CallEvent.DataChannel]: (channel: RTCDataChannel, call: MatrixCall) => void;
332
- [CallEvent.FeedsChanged]: (feeds: CallFeed[], call: MatrixCall) => void;
333
- [CallEvent.Replaced]: (newCall: MatrixCall, oldCall: MatrixCall) => void;
334
- [CallEvent.Error]: (error: CallError, call: MatrixCall) => void;
335
- [CallEvent.RemoteHoldUnhold]: (onHold: boolean, call: MatrixCall) => void;
336
- [CallEvent.LocalHoldUnhold]: (onHold: boolean, call: MatrixCall) => void;
337
- [CallEvent.LengthChanged]: (length: number, call: MatrixCall) => void;
338
- [CallEvent.State]: (state: CallState, oldState: CallState, call: MatrixCall) => void;
339
- [CallEvent.Hangup]: (call: MatrixCall) => void;
340
- [CallEvent.AssertedIdentityChanged]: (call: MatrixCall) => void;
341
- /* @deprecated */
342
- [CallEvent.HoldUnhold]: (onHold: boolean) => void;
343
- [CallEvent.SendVoipEvent]: (event: VoipEvent, call: MatrixCall) => void;
344
- [CallEvent.PeerConnectionCreated]: (peerConn: RTCPeerConnection, call: MatrixCall) => void;
345
- };
346
-
347
- // The key of the transceiver map (purpose + media type, separated by ':')
348
- type TransceiverKey = string;
349
-
350
- // generates keys for the map of transceivers
351
- // kind is unfortunately a string rather than MediaType as this is the type of
352
- // track.kind
353
- function getTransceiverKey(purpose: SDPStreamMetadataPurpose, kind: TransceiverKey): string {
354
- return purpose + ":" + kind;
355
- }
356
-
357
- export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap> {
358
- public roomId: string;
359
- public callId: string;
360
- public invitee?: string;
361
- public hangupParty?: CallParty;
362
- public hangupReason?: string;
363
- public direction?: CallDirection;
364
- public ourPartyId: string;
365
- public peerConn?: RTCPeerConnection;
366
- public toDeviceSeq = 0;
367
-
368
- // whether this call should have push-to-talk semantics
369
- // This should be set by the consumer on incoming & outgoing calls.
370
- public isPtt = false;
371
-
372
- private _state = CallState.Fledgling;
373
- private readonly client: MatrixClient;
374
- private readonly forceTURN?: boolean;
375
- private readonly turnServers: Array<TurnServer>;
376
- // A queue for candidates waiting to go out.
377
- // We try to amalgamate candidates into a single candidate message where
378
- // possible
379
- private candidateSendQueue: Array<RTCIceCandidate> = [];
380
- private candidateSendTries = 0;
381
- private candidatesEnded = false;
382
- private feeds: Array<CallFeed> = [];
383
-
384
- // our transceivers for each purpose and type of media
385
- private transceivers = new Map<TransceiverKey, RTCRtpTransceiver>();
386
-
387
- private inviteOrAnswerSent = false;
388
- private waitForLocalAVStream = false;
389
- private successor?: MatrixCall;
390
- private opponentMember?: RoomMember;
391
- private opponentVersion?: number | string;
392
- // The party ID of the other side: undefined if we haven't chosen a partner
393
- // yet, null if we have but they didn't send a party ID.
394
- private opponentPartyId: string | null | undefined;
395
- private opponentCaps?: CallCapabilities;
396
- private iceDisconnectedTimeout?: ReturnType<typeof setTimeout>;
397
- private iceReconnectionTimeOut?: ReturnType<typeof setTimeout> | undefined;
398
- private inviteTimeout?: ReturnType<typeof setTimeout>;
399
- private readonly removeTrackListeners = new Map<MediaStream, () => void>();
400
-
401
- // The logic of when & if a call is on hold is nontrivial and explained in is*OnHold
402
- // This flag represents whether we want the other party to be on hold
403
- private remoteOnHold = false;
404
-
405
- // the stats for the call at the point it ended. We can't get these after we
406
- // tear the call down, so we just grab a snapshot before we stop the call.
407
- // The typescript definitions have this type as 'any' :(
408
- private callStatsAtEnd?: any[];
409
-
410
- // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example
411
- private makingOffer = false;
412
- private ignoreOffer = false;
413
- private isSettingRemoteAnswerPending = false;
414
-
415
- private responsePromiseChain?: Promise<void>;
416
-
417
- // If candidates arrive before we've picked an opponent (which, in particular,
418
- // will happen if the opponent sends candidates eagerly before the user answers
419
- // the call) we buffer them up here so we can then add the ones from the party we pick
420
- private remoteCandidateBuffer = new Map<string, MCallCandidates["candidates"]>();
421
-
422
- private remoteAssertedIdentity?: AssertedIdentity;
423
- private remoteSDPStreamMetadata?: SDPStreamMetadata;
424
-
425
- private callLengthInterval?: ReturnType<typeof setInterval>;
426
- private callStartTime?: number;
427
-
428
- private opponentDeviceId?: string;
429
- private opponentDeviceInfo?: DeviceInfo;
430
- private opponentSessionId?: string;
431
- public groupCallId?: string;
432
-
433
- // Used to keep the timer for the delay before actually stopping our
434
- // video track after muting (see setLocalVideoMuted)
435
- private stopVideoTrackTimer?: ReturnType<typeof setTimeout>;
436
- // Used to allow connection without Video and Audio. To establish a webrtc connection without media a Data channel is
437
- // needed At the moment this property is true if we allow MatrixClient with isVoipWithNoMediaAllowed = true
438
- private readonly isOnlyDataChannelAllowed: boolean;
439
- private stats: GroupCallStats | undefined;
440
-
441
- /**
442
- * Construct a new Matrix Call.
443
- * @param opts - Config options.
444
- */
445
- public constructor(opts: CallOpts) {
446
- super();
447
-
448
- this.roomId = opts.roomId;
449
- this.invitee = opts.invitee;
450
- this.client = opts.client;
451
-
452
- if (!this.client.deviceId) throw new Error("Client must have a device ID to start calls");
453
-
454
- this.forceTURN = opts.forceTURN ?? false;
455
- this.ourPartyId = this.client.deviceId;
456
- this.opponentDeviceId = opts.opponentDeviceId;
457
- this.opponentSessionId = opts.opponentSessionId;
458
- this.groupCallId = opts.groupCallId;
459
- // Array of Objects with urls, username, credential keys
460
- this.turnServers = opts.turnServers || [];
461
- if (this.turnServers.length === 0 && this.client.isFallbackICEServerAllowed()) {
462
- this.turnServers.push({
463
- urls: [FALLBACK_ICE_SERVER],
464
- });
465
- }
466
- for (const server of this.turnServers) {
467
- checkObjectHasKeys(server, ["urls"]);
468
- }
469
- this.callId = genCallID();
470
- // If the Client provides calls without audio and video we need a datachannel for a webrtc connection
471
- this.isOnlyDataChannelAllowed = this.client.isVoipWithNoMediaAllowed;
472
- }
473
-
474
- /**
475
- * Place a voice call to this room.
476
- * @throws If you have not specified a listener for 'error' events.
477
- */
478
- public async placeVoiceCall(): Promise<void> {
479
- await this.placeCall(true, false);
480
- }
481
-
482
- /**
483
- * Place a video call to this room.
484
- * @throws If you have not specified a listener for 'error' events.
485
- */
486
- public async placeVideoCall(): Promise<void> {
487
- await this.placeCall(true, true);
488
- }
489
-
490
- /**
491
- * Create a datachannel using this call's peer connection.
492
- * @param label - A human readable label for this datachannel
493
- * @param options - An object providing configuration options for the data channel.
494
- */
495
- public createDataChannel(label: string, options: RTCDataChannelInit | undefined): RTCDataChannel {
496
- const dataChannel = this.peerConn!.createDataChannel(label, options);
497
- this.emit(CallEvent.DataChannel, dataChannel, this);
498
- return dataChannel;
499
- }
500
-
501
- public getOpponentMember(): RoomMember | undefined {
502
- return this.opponentMember;
503
- }
504
-
505
- public getOpponentDeviceId(): string | undefined {
506
- return this.opponentDeviceId;
507
- }
508
-
509
- public getOpponentSessionId(): string | undefined {
510
- return this.opponentSessionId;
511
- }
512
-
513
- public opponentCanBeTransferred(): boolean {
514
- return Boolean(this.opponentCaps && this.opponentCaps["m.call.transferee"]);
515
- }
516
-
517
- public opponentSupportsDTMF(): boolean {
518
- return Boolean(this.opponentCaps && this.opponentCaps["m.call.dtmf"]);
519
- }
520
-
521
- public getRemoteAssertedIdentity(): AssertedIdentity | undefined {
522
- return this.remoteAssertedIdentity;
523
- }
524
-
525
- public get state(): CallState {
526
- return this._state;
527
- }
528
-
529
- private set state(state: CallState) {
530
- const oldState = this._state;
531
- this._state = state;
532
- this.emit(CallEvent.State, state, oldState, this);
533
- }
534
-
535
- public get type(): CallType {
536
- // we may want to look for a video receiver here rather than a track to match the
537
- // sender behaviour, although in practice they should be the same thing
538
- return this.hasUserMediaVideoSender || this.hasRemoteUserMediaVideoTrack ? CallType.Video : CallType.Voice;
539
- }
540
-
541
- public get hasLocalUserMediaVideoTrack(): boolean {
542
- return !!this.localUsermediaStream?.getVideoTracks().length;
543
- }
544
-
545
- public get hasRemoteUserMediaVideoTrack(): boolean {
546
- return this.getRemoteFeeds().some((feed) => {
547
- return feed.purpose === SDPStreamMetadataPurpose.Usermedia && feed.stream?.getVideoTracks().length;
548
- });
549
- }
550
-
551
- public get hasLocalUserMediaAudioTrack(): boolean {
552
- return !!this.localUsermediaStream?.getAudioTracks().length;
553
- }
554
-
555
- public get hasRemoteUserMediaAudioTrack(): boolean {
556
- return this.getRemoteFeeds().some((feed) => {
557
- return feed.purpose === SDPStreamMetadataPurpose.Usermedia && !!feed.stream?.getAudioTracks().length;
558
- });
559
- }
560
-
561
- private get hasUserMediaAudioSender(): boolean {
562
- return Boolean(this.transceivers.get(getTransceiverKey(SDPStreamMetadataPurpose.Usermedia, "audio"))?.sender);
563
- }
564
-
565
- private get hasUserMediaVideoSender(): boolean {
566
- return Boolean(this.transceivers.get(getTransceiverKey(SDPStreamMetadataPurpose.Usermedia, "video"))?.sender);
567
- }
568
-
569
- public get localUsermediaFeed(): CallFeed | undefined {
570
- return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia);
571
- }
572
-
573
- public get localScreensharingFeed(): CallFeed | undefined {
574
- return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare);
575
- }
576
-
577
- public get localUsermediaStream(): MediaStream | undefined {
578
- return this.localUsermediaFeed?.stream;
579
- }
580
-
581
- public get localScreensharingStream(): MediaStream | undefined {
582
- return this.localScreensharingFeed?.stream;
583
- }
584
-
585
- public get remoteUsermediaFeed(): CallFeed | undefined {
586
- return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia);
587
- }
588
-
589
- public get remoteScreensharingFeed(): CallFeed | undefined {
590
- return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare);
591
- }
592
-
593
- public get remoteUsermediaStream(): MediaStream | undefined {
594
- return this.remoteUsermediaFeed?.stream;
595
- }
596
-
597
- public get remoteScreensharingStream(): MediaStream | undefined {
598
- return this.remoteScreensharingFeed?.stream;
599
- }
600
-
601
- private getFeedByStreamId(streamId: string): CallFeed | undefined {
602
- return this.getFeeds().find((feed) => feed.stream.id === streamId);
603
- }
604
-
605
- /**
606
- * Returns an array of all CallFeeds
607
- * @returns CallFeeds
608
- */
609
- public getFeeds(): Array<CallFeed> {
610
- return this.feeds;
611
- }
612
-
613
- /**
614
- * Returns an array of all local CallFeeds
615
- * @returns local CallFeeds
616
- */
617
- public getLocalFeeds(): Array<CallFeed> {
618
- return this.feeds.filter((feed) => feed.isLocal());
619
- }
620
-
621
- /**
622
- * Returns an array of all remote CallFeeds
623
- * @returns remote CallFeeds
624
- */
625
- public getRemoteFeeds(): Array<CallFeed> {
626
- return this.feeds.filter((feed) => !feed.isLocal());
627
- }
628
-
629
- private async initOpponentCrypto(): Promise<void> {
630
- if (!this.opponentDeviceId) return;
631
- if (!this.client.getUseE2eForGroupCall()) return;
632
- // It's possible to want E2EE and yet not have the means to manage E2EE
633
- // ourselves (for example if the client is a RoomWidgetClient)
634
- if (!this.client.isCryptoEnabled()) {
635
- // All we know is the device ID
636
- this.opponentDeviceInfo = new DeviceInfo(this.opponentDeviceId);
637
- return;
638
- }
639
- // if we've got to this point, we do want to init crypto, so throw if we can't
640
- if (!this.client.crypto) throw new Error("Crypto is not initialised.");
641
-
642
- const userId = this.invitee || this.getOpponentMember()?.userId;
643
-
644
- if (!userId) throw new Error("Couldn't find opponent user ID to init crypto");
645
-
646
- const deviceInfoMap = await this.client.crypto.deviceList.downloadKeys([userId], false);
647
- this.opponentDeviceInfo = deviceInfoMap.get(userId)?.get(this.opponentDeviceId);
648
- if (this.opponentDeviceInfo === undefined) {
649
- throw new GroupCallUnknownDeviceError(userId);
650
- }
651
- }
652
-
653
- /**
654
- * Generates and returns localSDPStreamMetadata
655
- * @returns localSDPStreamMetadata
656
- */
657
- private getLocalSDPStreamMetadata(updateStreamIds = false): SDPStreamMetadata {
658
- const metadata: SDPStreamMetadata = {};
659
- for (const localFeed of this.getLocalFeeds()) {
660
- if (updateStreamIds) {
661
- localFeed.sdpMetadataStreamId = localFeed.stream.id;
662
- }
663
-
664
- metadata[localFeed.sdpMetadataStreamId] = {
665
- purpose: localFeed.purpose,
666
- audio_muted: localFeed.isAudioMuted(),
667
- video_muted: localFeed.isVideoMuted(),
668
- };
669
- }
670
- return metadata;
671
- }
672
-
673
- /**
674
- * Returns true if there are no incoming feeds,
675
- * otherwise returns false
676
- * @returns no incoming feeds
677
- */
678
- public noIncomingFeeds(): boolean {
679
- return !this.feeds.some((feed) => !feed.isLocal());
680
- }
681
-
682
- private pushRemoteFeed(stream: MediaStream): void {
683
- // Fallback to old behavior if the other side doesn't support SDPStreamMetadata
684
- if (!this.opponentSupportsSDPStreamMetadata()) {
685
- this.pushRemoteFeedWithoutMetadata(stream);
686
- return;
687
- }
688
-
689
- const userId = this.getOpponentMember()!.userId;
690
- const purpose = this.remoteSDPStreamMetadata![stream.id].purpose;
691
- const audioMuted = this.remoteSDPStreamMetadata![stream.id].audio_muted;
692
- const videoMuted = this.remoteSDPStreamMetadata![stream.id].video_muted;
693
-
694
- if (!purpose) {
695
- logger.warn(
696
- `Call ${this.callId} pushRemoteFeed() ignoring stream because we didn't get any metadata about it (streamId=${stream.id})`,
697
- );
698
- return;
699
- }
700
-
701
- if (this.getFeedByStreamId(stream.id)) {
702
- logger.warn(
703
- `Call ${this.callId} pushRemoteFeed() ignoring stream because we already have a feed for it (streamId=${stream.id})`,
704
- );
705
- return;
706
- }
707
-
708
- this.feeds.push(
709
- new CallFeed({
710
- client: this.client,
711
- call: this,
712
- roomId: this.roomId,
713
- userId,
714
- deviceId: this.getOpponentDeviceId(),
715
- stream,
716
- purpose,
717
- audioMuted,
718
- videoMuted,
719
- }),
720
- );
721
-
722
- this.emit(CallEvent.FeedsChanged, this.feeds, this);
723
-
724
- logger.info(
725
- `Call ${this.callId} pushRemoteFeed() pushed stream (streamId=${stream.id}, active=${stream.active}, purpose=${purpose})`,
726
- );
727
- }
728
-
729
- /**
730
- * This method is used ONLY if the other client doesn't support sending SDPStreamMetadata
731
- */
732
- private pushRemoteFeedWithoutMetadata(stream: MediaStream): void {
733
- const userId = this.getOpponentMember()!.userId;
734
- // We can guess the purpose here since the other client can only send one stream
735
- const purpose = SDPStreamMetadataPurpose.Usermedia;
736
- const oldRemoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream;
737
-
738
- // Note that we check by ID and always set the remote stream: Chrome appears
739
- // to make new stream objects when transceiver directionality is changed and the 'active'
740
- // status of streams change - Dave
741
- // If we already have a stream, check this stream has the same id
742
- if (oldRemoteStream && stream.id !== oldRemoteStream.id) {
743
- logger.warn(
744
- `Call ${this.callId} pushRemoteFeedWithoutMetadata() ignoring new stream because we already have stream (streamId=${stream.id})`,
745
- );
746
- return;
747
- }
748
-
749
- if (this.getFeedByStreamId(stream.id)) {
750
- logger.warn(
751
- `Call ${this.callId} pushRemoteFeedWithoutMetadata() ignoring stream because we already have a feed for it (streamId=${stream.id})`,
752
- );
753
- return;
754
- }
755
-
756
- this.feeds.push(
757
- new CallFeed({
758
- client: this.client,
759
- call: this,
760
- roomId: this.roomId,
761
- audioMuted: false,
762
- videoMuted: false,
763
- userId,
764
- deviceId: this.getOpponentDeviceId(),
765
- stream,
766
- purpose,
767
- }),
768
- );
769
-
770
- this.emit(CallEvent.FeedsChanged, this.feeds, this);
771
-
772
- logger.info(
773
- `Call ${this.callId} pushRemoteFeedWithoutMetadata() pushed stream (streamId=${stream.id}, active=${stream.active})`,
774
- );
775
- }
776
-
777
- private pushNewLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void {
778
- const userId = this.client.getUserId()!;
779
-
780
- // Tracks don't always start off enabled, eg. chrome will give a disabled
781
- // audio track if you ask for user media audio and already had one that
782
- // you'd set to disabled (presumably because it clones them internally).
783
- setTracksEnabled(stream.getAudioTracks(), true);
784
- setTracksEnabled(stream.getVideoTracks(), true);
785
-
786
- if (this.getFeedByStreamId(stream.id)) {
787
- logger.warn(
788
- `Call ${this.callId} pushNewLocalFeed() ignoring stream because we already have a feed for it (streamId=${stream.id})`,
789
- );
790
- return;
791
- }
792
-
793
- this.pushLocalFeed(
794
- new CallFeed({
795
- client: this.client,
796
- roomId: this.roomId,
797
- audioMuted: false,
798
- videoMuted: false,
799
- userId,
800
- deviceId: this.getOpponentDeviceId(),
801
- stream,
802
- purpose,
803
- }),
804
- addToPeerConnection,
805
- );
806
- }
807
-
808
- /**
809
- * Pushes supplied feed to the call
810
- * @param callFeed - to push
811
- * @param addToPeerConnection - whether to add the tracks to the peer connection
812
- */
813
- public pushLocalFeed(callFeed: CallFeed, addToPeerConnection = true): void {
814
- if (this.feeds.some((feed) => callFeed.stream.id === feed.stream.id)) {
815
- logger.info(
816
- `Call ${this.callId} pushLocalFeed() ignoring duplicate local stream (streamId=${callFeed.stream.id})`,
817
- );
818
- return;
819
- }
820
-
821
- this.feeds.push(callFeed);
822
-
823
- if (addToPeerConnection) {
824
- for (const track of callFeed.stream.getTracks()) {
825
- logger.info(
826
- `Call ${this.callId} pushLocalFeed() adding track to peer connection (id=${track.id}, kind=${track.kind}, streamId=${callFeed.stream.id}, streamPurpose=${callFeed.purpose}, enabled=${track.enabled})`,
827
- );
828
-
829
- const tKey = getTransceiverKey(callFeed.purpose, track.kind);
830
- if (this.transceivers.has(tKey)) {
831
- // we already have a sender, so we re-use it. We try to re-use transceivers as much
832
- // as possible because they can't be removed once added, so otherwise they just
833
- // accumulate which makes the SDP very large very quickly: in fact it only takes
834
- // about 6 video tracks to exceed the maximum size of an Olm-encrypted
835
- // Matrix event.
836
- const transceiver = this.transceivers.get(tKey)!;
837
-
838
- transceiver.sender.replaceTrack(track);
839
- // set the direction to indicate we're going to start sending again
840
- // (this will trigger the re-negotiation)
841
- transceiver.direction = transceiver.direction === "inactive" ? "sendonly" : "sendrecv";
842
- } else {
843
- // create a new one. We need to use addTrack rather addTransceiver for this because firefox
844
- // doesn't yet implement RTCRTPSender.setStreams()
845
- // (https://bugzilla.mozilla.org/show_bug.cgi?id=1510802) so we'd have no way to group the
846
- // two tracks together into a stream.
847
- const newSender = this.peerConn!.addTrack(track, callFeed.stream);
848
-
849
- // now go & fish for the new transceiver
850
- const newTransceiver = this.peerConn!.getTransceivers().find((t) => t.sender === newSender);
851
- if (newTransceiver) {
852
- this.transceivers.set(tKey, newTransceiver);
853
- } else {
854
- logger.warn(
855
- `Call ${this.callId} pushLocalFeed() didn't find a matching transceiver after adding track!`,
856
- );
857
- }
858
- }
859
- }
860
- }
861
-
862
- logger.info(
863
- `Call ${this.callId} pushLocalFeed() pushed stream (id=${callFeed.stream.id}, active=${callFeed.stream.active}, purpose=${callFeed.purpose})`,
864
- );
865
-
866
- this.emit(CallEvent.FeedsChanged, this.feeds, this);
867
- }
868
-
869
- /**
870
- * Removes local call feed from the call and its tracks from the peer
871
- * connection
872
- * @param callFeed - to remove
873
- */
874
- public removeLocalFeed(callFeed: CallFeed): void {
875
- const audioTransceiverKey = getTransceiverKey(callFeed.purpose, "audio");
876
- const videoTransceiverKey = getTransceiverKey(callFeed.purpose, "video");
877
-
878
- for (const transceiverKey of [audioTransceiverKey, videoTransceiverKey]) {
879
- // this is slightly mixing the track and transceiver API but is basically just shorthand.
880
- // There is no way to actually remove a transceiver, so this just sets it to inactive
881
- // (or recvonly) and replaces the source with nothing.
882
- if (this.transceivers.has(transceiverKey)) {
883
- const transceiver = this.transceivers.get(transceiverKey)!;
884
- if (transceiver.sender) this.peerConn!.removeTrack(transceiver.sender);
885
- }
886
- }
887
-
888
- if (callFeed.purpose === SDPStreamMetadataPurpose.Screenshare) {
889
- this.client.getMediaHandler().stopScreensharingStream(callFeed.stream);
890
- }
891
-
892
- this.deleteFeed(callFeed);
893
- }
894
-
895
- private deleteAllFeeds(): void {
896
- for (const feed of this.feeds) {
897
- if (!feed.isLocal() || !this.groupCallId) {
898
- feed.dispose();
899
- }
900
- }
901
-
902
- this.feeds = [];
903
- this.emit(CallEvent.FeedsChanged, this.feeds, this);
904
- }
905
-
906
- private deleteFeedByStream(stream: MediaStream): void {
907
- const feed = this.getFeedByStreamId(stream.id);
908
- if (!feed) {
909
- logger.warn(
910
- `Call ${this.callId} deleteFeedByStream() didn't find the feed to delete (streamId=${stream.id})`,
911
- );
912
- return;
913
- }
914
- this.deleteFeed(feed);
915
- }
916
-
917
- private deleteFeed(feed: CallFeed): void {
918
- feed.dispose();
919
- this.feeds.splice(this.feeds.indexOf(feed), 1);
920
- this.emit(CallEvent.FeedsChanged, this.feeds, this);
921
- }
922
-
923
- // The typescript definitions have this type as 'any' :(
924
- public async getCurrentCallStats(): Promise<any[] | undefined> {
925
- if (this.callHasEnded()) {
926
- return this.callStatsAtEnd;
927
- }
928
-
929
- return this.collectCallStats();
930
- }
931
-
932
- private async collectCallStats(): Promise<any[] | undefined> {
933
- // This happens when the call fails before it starts.
934
- // For example when we fail to get capture sources
935
- if (!this.peerConn) return;
936
-
937
- const statsReport = await this.peerConn.getStats();
938
- const stats: any[] = [];
939
- statsReport.forEach((item) => {
940
- stats.push(item);
941
- });
942
-
943
- return stats;
944
- }
945
-
946
- /**
947
- * Configure this call from an invite event. Used by MatrixClient.
948
- * @param event - The m.call.invite event
949
- */
950
- public async initWithInvite(event: MatrixEvent): Promise<void> {
951
- const invite = event.getContent<MCallInviteNegotiate>();
952
- this.direction = CallDirection.Inbound;
953
-
954
- // make sure we have valid turn creds. Unless something's gone wrong, it should
955
- // poll and keep the credentials valid so this should be instant.
956
- const haveTurnCreds = await this.client.checkTurnServers();
957
- if (!haveTurnCreds) {
958
- logger.warn(
959
- `Call ${this.callId} initWithInvite() failed to get TURN credentials! Proceeding with call anyway...`,
960
- );
961
- }
962
-
963
- const sdpStreamMetadata = invite[SDPStreamMetadataKey];
964
- if (sdpStreamMetadata) {
965
- this.updateRemoteSDPStreamMetadata(sdpStreamMetadata);
966
- } else {
967
- logger.debug(
968
- `Call ${this.callId} initWithInvite() did not get any SDPStreamMetadata! Can not send/receive multiple streams`,
969
- );
970
- }
971
-
972
- this.peerConn = this.createPeerConnection();
973
- this.emit(CallEvent.PeerConnectionCreated, this.peerConn, this);
974
- // we must set the party ID before await-ing on anything: the call event
975
- // handler will start giving us more call events (eg. candidates) so if
976
- // we haven't set the party ID, we'll ignore them.
977
- this.chooseOpponent(event);
978
- await this.initOpponentCrypto();
979
- try {
980
- await this.peerConn.setRemoteDescription(invite.offer);
981
- logger.debug(`Call ${this.callId} initWithInvite() set remote description: ${invite.offer.type}`);
982
- await this.addBufferedIceCandidates();
983
- } catch (e) {
984
- logger.debug(`Call ${this.callId} initWithInvite() failed to set remote description`, e);
985
- this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false);
986
- return;
987
- }
988
-
989
- const remoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream;
990
-
991
- // According to previous comments in this file, firefox at some point did not
992
- // add streams until media started arriving on them. Testing latest firefox
993
- // (81 at time of writing), this is no longer a problem, so let's do it the correct way.
994
- //
995
- // For example in case of no media webrtc connections like screen share only call we have to allow webrtc
996
- // connections without remote media. In this case we always use a data channel. At the moment we allow as well
997
- // only data channel as media in the WebRTC connection with this setup here.
998
- if (!this.isOnlyDataChannelAllowed && (!remoteStream || remoteStream.getTracks().length === 0)) {
999
- logger.error(
1000
- `Call ${this.callId} initWithInvite() no remote stream or no tracks after setting remote description!`,
1001
- );
1002
- this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false);
1003
- return;
1004
- }
1005
-
1006
- this.state = CallState.Ringing;
1007
-
1008
- if (event.getLocalAge()) {
1009
- // Time out the call if it's ringing for too long
1010
- const ringingTimer = setTimeout(() => {
1011
- if (this.state == CallState.Ringing) {
1012
- logger.debug(`Call ${this.callId} initWithInvite() invite has expired. Hanging up.`);
1013
- this.hangupParty = CallParty.Remote; // effectively
1014
- this.state = CallState.Ended;
1015
- this.stopAllMedia();
1016
- if (this.peerConn!.signalingState != "closed") {
1017
- this.peerConn!.close();
1018
- }
1019
- this.stats?.removeStatsReportGatherer(this.callId);
1020
- this.emit(CallEvent.Hangup, this);
1021
- }
1022
- }, invite.lifetime - event.getLocalAge());
1023
-
1024
- const onState = (state: CallState): void => {
1025
- if (state !== CallState.Ringing) {
1026
- clearTimeout(ringingTimer);
1027
- this.off(CallEvent.State, onState);
1028
- }
1029
- };
1030
- this.on(CallEvent.State, onState);
1031
- }
1032
- }
1033
-
1034
- /**
1035
- * Configure this call from a hangup or reject event. Used by MatrixClient.
1036
- * @param event - The m.call.hangup event
1037
- */
1038
- public initWithHangup(event: MatrixEvent): void {
1039
- // perverse as it may seem, sometimes we want to instantiate a call with a
1040
- // hangup message (because when getting the state of the room on load, events
1041
- // come in reverse order and we want to remember that a call has been hung up)
1042
- this.state = CallState.Ended;
1043
- }
1044
-
1045
- private shouldAnswerWithMediaType(
1046
- wantedValue: boolean | undefined,
1047
- valueOfTheOtherSide: boolean,
1048
- type: "audio" | "video",
1049
- ): boolean {
1050
- if (wantedValue && !valueOfTheOtherSide) {
1051
- // TODO: Figure out how to do this
1052
- logger.warn(
1053
- `Call ${this.callId} shouldAnswerWithMediaType() unable to answer with ${type} because the other side isn't sending it either.`,
1054
- );
1055
- return false;
1056
- } else if (
1057
- !isNullOrUndefined(wantedValue) &&
1058
- wantedValue !== valueOfTheOtherSide &&
1059
- !this.opponentSupportsSDPStreamMetadata()
1060
- ) {
1061
- logger.warn(
1062
- `Call ${this.callId} shouldAnswerWithMediaType() unable to answer with ${type}=${wantedValue} because the other side doesn't support it. Answering with ${type}=${valueOfTheOtherSide}.`,
1063
- );
1064
- return valueOfTheOtherSide!;
1065
- }
1066
- return wantedValue ?? valueOfTheOtherSide!;
1067
- }
1068
-
1069
- /**
1070
- * Answer a call.
1071
- */
1072
- public async answer(audio?: boolean, video?: boolean): Promise<void> {
1073
- if (this.inviteOrAnswerSent) return;
1074
- // TODO: Figure out how to do this
1075
- if (audio === false && video === false) throw new Error("You CANNOT answer a call without media");
1076
-
1077
- if (!this.localUsermediaStream && !this.waitForLocalAVStream) {
1078
- const prevState = this.state;
1079
- const answerWithAudio = this.shouldAnswerWithMediaType(audio, this.hasRemoteUserMediaAudioTrack, "audio");
1080
- const answerWithVideo = this.shouldAnswerWithMediaType(video, this.hasRemoteUserMediaVideoTrack, "video");
1081
-
1082
- this.state = CallState.WaitLocalMedia;
1083
- this.waitForLocalAVStream = true;
1084
-
1085
- try {
1086
- const stream = await this.client.getMediaHandler().getUserMediaStream(answerWithAudio, answerWithVideo);
1087
- this.waitForLocalAVStream = false;
1088
- const usermediaFeed = new CallFeed({
1089
- client: this.client,
1090
- roomId: this.roomId,
1091
- userId: this.client.getUserId()!,
1092
- deviceId: this.client.getDeviceId() ?? undefined,
1093
- stream,
1094
- purpose: SDPStreamMetadataPurpose.Usermedia,
1095
- audioMuted: false,
1096
- videoMuted: false,
1097
- });
1098
-
1099
- const feeds = [usermediaFeed];
1100
-
1101
- if (this.localScreensharingFeed) {
1102
- feeds.push(this.localScreensharingFeed);
1103
- }
1104
-
1105
- this.answerWithCallFeeds(feeds);
1106
- } catch (e) {
1107
- if (answerWithVideo) {
1108
- // Try to answer without video
1109
- logger.warn(
1110
- `Call ${this.callId} answer() failed to getUserMedia(), trying to getUserMedia() without video`,
1111
- );
1112
- this.state = prevState;
1113
- this.waitForLocalAVStream = false;
1114
- await this.answer(answerWithAudio, false);
1115
- } else {
1116
- this.getUserMediaFailed(<Error>e);
1117
- return;
1118
- }
1119
- }
1120
- } else if (this.waitForLocalAVStream) {
1121
- this.state = CallState.WaitLocalMedia;
1122
- }
1123
- }
1124
-
1125
- public answerWithCallFeeds(callFeeds: CallFeed[]): void {
1126
- if (this.inviteOrAnswerSent) return;
1127
-
1128
- this.queueGotCallFeedsForAnswer(callFeeds);
1129
- }
1130
-
1131
- /**
1132
- * Replace this call with a new call, e.g. for glare resolution. Used by
1133
- * MatrixClient.
1134
- * @param newCall - The new call.
1135
- */
1136
- public replacedBy(newCall: MatrixCall): void {
1137
- logger.debug(`Call ${this.callId} replacedBy() running (newCallId=${newCall.callId})`);
1138
- if (this.state === CallState.WaitLocalMedia) {
1139
- logger.debug(
1140
- `Call ${this.callId} replacedBy() telling new call to wait for local media (newCallId=${newCall.callId})`,
1141
- );
1142
- newCall.waitForLocalAVStream = true;
1143
- } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) {
1144
- if (newCall.direction === CallDirection.Outbound) {
1145
- newCall.queueGotCallFeedsForAnswer([]);
1146
- } else {
1147
- logger.debug(
1148
- `Call ${this.callId} replacedBy() handing local stream to new call(newCallId=${newCall.callId})`,
1149
- );
1150
- newCall.queueGotCallFeedsForAnswer(this.getLocalFeeds().map((feed) => feed.clone()));
1151
- }
1152
- }
1153
- this.successor = newCall;
1154
- this.emit(CallEvent.Replaced, newCall, this);
1155
- this.hangup(CallErrorCode.Replaced, true);
1156
- }
1157
-
1158
- /**
1159
- * Hangup a call.
1160
- * @param reason - The reason why the call is being hung up.
1161
- * @param suppressEvent - True to suppress emitting an event.
1162
- */
1163
- public hangup(reason: CallErrorCode, suppressEvent: boolean): void {
1164
- if (this.callHasEnded()) return;
1165
-
1166
- logger.debug(`Call ${this.callId} hangup() ending call (reason=${reason})`);
1167
- this.terminate(CallParty.Local, reason, !suppressEvent);
1168
- // We don't want to send hangup here if we didn't even get to sending an invite
1169
- if ([CallState.Fledgling, CallState.WaitLocalMedia].includes(this.state)) return;
1170
- const content: Omit<MCallHangupReject, "version" | "call_id" | "party_id" | "conf_id"> = {};
1171
- // Don't send UserHangup reason to older clients
1172
- if ((this.opponentVersion && this.opponentVersion !== 0) || reason !== CallErrorCode.UserHangup) {
1173
- content["reason"] = reason;
1174
- }
1175
- this.sendVoipEvent(EventType.CallHangup, content);
1176
- }
1177
-
1178
- /**
1179
- * Reject a call
1180
- * This used to be done by calling hangup, but is a separate method and protocol
1181
- * event as of MSC2746.
1182
- */
1183
- public reject(): void {
1184
- if (this.state !== CallState.Ringing) {
1185
- throw Error("Call must be in 'ringing' state to reject!");
1186
- }
1187
-
1188
- if (this.opponentVersion === 0) {
1189
- logger.info(
1190
- `Call ${this.callId} reject() opponent version is less than 1: sending hangup instead of reject (opponentVersion=${this.opponentVersion})`,
1191
- );
1192
- this.hangup(CallErrorCode.UserHangup, true);
1193
- return;
1194
- }
1195
-
1196
- logger.debug("Rejecting call: " + this.callId);
1197
- this.terminate(CallParty.Local, CallErrorCode.UserHangup, true);
1198
- this.sendVoipEvent(EventType.CallReject, {});
1199
- }
1200
-
1201
- /**
1202
- * Adds an audio and/or video track - upgrades the call
1203
- * @param audio - should add an audio track
1204
- * @param video - should add an video track
1205
- */
1206
- private async upgradeCall(audio: boolean, video: boolean): Promise<void> {
1207
- // We don't do call downgrades
1208
- if (!audio && !video) return;
1209
- if (!this.opponentSupportsSDPStreamMetadata()) return;
1210
-
1211
- try {
1212
- logger.debug(`Call ${this.callId} upgradeCall() upgrading call (audio=${audio}, video=${video})`);
1213
- const getAudio = audio || this.hasLocalUserMediaAudioTrack;
1214
- const getVideo = video || this.hasLocalUserMediaVideoTrack;
1215
-
1216
- // updateLocalUsermediaStream() will take the tracks, use them as
1217
- // replacement and throw the stream away, so it isn't reusable
1218
- const stream = await this.client.getMediaHandler().getUserMediaStream(getAudio, getVideo, false);
1219
- await this.updateLocalUsermediaStream(stream, audio, video);
1220
- } catch (error) {
1221
- logger.error(`Call ${this.callId} upgradeCall() failed to upgrade the call`, error);
1222
- this.emit(
1223
- CallEvent.Error,
1224
- new CallError(CallErrorCode.NoUserMedia, "Failed to get camera access: ", <Error>error),
1225
- this,
1226
- );
1227
- }
1228
- }
1229
-
1230
- /**
1231
- * Returns true if this.remoteSDPStreamMetadata is defined, otherwise returns false
1232
- * @returns can screenshare
1233
- */
1234
- public opponentSupportsSDPStreamMetadata(): boolean {
1235
- return Boolean(this.remoteSDPStreamMetadata);
1236
- }
1237
-
1238
- /**
1239
- * If there is a screensharing stream returns true, otherwise returns false
1240
- * @returns is screensharing
1241
- */
1242
- public isScreensharing(): boolean {
1243
- return Boolean(this.localScreensharingStream);
1244
- }
1245
-
1246
- /**
1247
- * Starts/stops screensharing
1248
- * @param enabled - the desired screensharing state
1249
- * @param opts - screen sharing options
1250
- * @returns new screensharing state
1251
- */
1252
- public async setScreensharingEnabled(enabled: boolean, opts?: IScreensharingOpts): Promise<boolean> {
1253
- // Skip if there is nothing to do
1254
- if (enabled && this.isScreensharing()) {
1255
- logger.warn(
1256
- `Call ${this.callId} setScreensharingEnabled() there is already a screensharing stream - there is nothing to do!`,
1257
- );
1258
- return true;
1259
- } else if (!enabled && !this.isScreensharing()) {
1260
- logger.warn(
1261
- `Call ${this.callId} setScreensharingEnabled() there already isn't a screensharing stream - there is nothing to do!`,
1262
- );
1263
- return false;
1264
- }
1265
-
1266
- // Fallback to replaceTrack()
1267
- if (!this.opponentSupportsSDPStreamMetadata()) {
1268
- return this.setScreensharingEnabledWithoutMetadataSupport(enabled, opts);
1269
- }
1270
-
1271
- logger.debug(`Call ${this.callId} setScreensharingEnabled() running (enabled=${enabled})`);
1272
- if (enabled) {
1273
- try {
1274
- const stream = await this.client.getMediaHandler().getScreensharingStream(opts);
1275
- if (!stream) return false;
1276
- this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare);
1277
- return true;
1278
- } catch (err) {
1279
- logger.error(`Call ${this.callId} setScreensharingEnabled() failed to get screen-sharing stream:`, err);
1280
- return false;
1281
- }
1282
- } else {
1283
- const audioTransceiver = this.transceivers.get(
1284
- getTransceiverKey(SDPStreamMetadataPurpose.Screenshare, "audio"),
1285
- );
1286
- const videoTransceiver = this.transceivers.get(
1287
- getTransceiverKey(SDPStreamMetadataPurpose.Screenshare, "video"),
1288
- );
1289
-
1290
- for (const transceiver of [audioTransceiver, videoTransceiver]) {
1291
- // this is slightly mixing the track and transceiver API but is basically just shorthand
1292
- // for removing the sender.
1293
- if (transceiver && transceiver.sender) this.peerConn!.removeTrack(transceiver.sender);
1294
- }
1295
-
1296
- this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream!);
1297
- this.deleteFeedByStream(this.localScreensharingStream!);
1298
- return false;
1299
- }
1300
- }
1301
-
1302
- /**
1303
- * Starts/stops screensharing
1304
- * Should be used ONLY if the opponent doesn't support SDPStreamMetadata
1305
- * @param enabled - the desired screensharing state
1306
- * @param opts - screen sharing options
1307
- * @returns new screensharing state
1308
- */
1309
- private async setScreensharingEnabledWithoutMetadataSupport(
1310
- enabled: boolean,
1311
- opts?: IScreensharingOpts,
1312
- ): Promise<boolean> {
1313
- logger.debug(
1314
- `Call ${this.callId} setScreensharingEnabledWithoutMetadataSupport() running (enabled=${enabled})`,
1315
- );
1316
- if (enabled) {
1317
- try {
1318
- const stream = await this.client.getMediaHandler().getScreensharingStream(opts);
1319
- if (!stream) return false;
1320
-
1321
- const track = stream.getTracks().find((track) => track.kind === "video");
1322
-
1323
- const sender = this.transceivers.get(
1324
- getTransceiverKey(SDPStreamMetadataPurpose.Usermedia, "video"),
1325
- )?.sender;
1326
-
1327
- sender?.replaceTrack(track ?? null);
1328
-
1329
- this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false);
1330
-
1331
- return true;
1332
- } catch (err) {
1333
- logger.error(
1334
- `Call ${this.callId} setScreensharingEnabledWithoutMetadataSupport() failed to get screen-sharing stream:`,
1335
- err,
1336
- );
1337
- return false;
1338
- }
1339
- } else {
1340
- const track = this.localUsermediaStream?.getTracks().find((track) => track.kind === "video");
1341
- const sender = this.transceivers.get(
1342
- getTransceiverKey(SDPStreamMetadataPurpose.Usermedia, "video"),
1343
- )?.sender;
1344
- sender?.replaceTrack(track ?? null);
1345
-
1346
- this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream!);
1347
- this.deleteFeedByStream(this.localScreensharingStream!);
1348
-
1349
- return false;
1350
- }
1351
- }
1352
-
1353
- /**
1354
- * Replaces/adds the tracks from the passed stream to the localUsermediaStream
1355
- * @param stream - to use a replacement for the local usermedia stream
1356
- */
1357
- public async updateLocalUsermediaStream(
1358
- stream: MediaStream,
1359
- forceAudio = false,
1360
- forceVideo = false,
1361
- ): Promise<void> {
1362
- const callFeed = this.localUsermediaFeed!;
1363
- const audioEnabled = forceAudio || (!callFeed.isAudioMuted() && !this.remoteOnHold);
1364
- const videoEnabled = forceVideo || (!callFeed.isVideoMuted() && !this.remoteOnHold);
1365
- logger.log(
1366
- `Call ${this.callId} updateLocalUsermediaStream() running (streamId=${stream.id}, audio=${audioEnabled}, video=${videoEnabled})`,
1367
- );
1368
- setTracksEnabled(stream.getAudioTracks(), audioEnabled);
1369
- setTracksEnabled(stream.getVideoTracks(), videoEnabled);
1370
-
1371
- // We want to keep the same stream id, so we replace the tracks rather
1372
- // than the whole stream.
1373
-
1374
- // Firstly, we replace the tracks in our localUsermediaStream.
1375
- for (const track of this.localUsermediaStream!.getTracks()) {
1376
- this.localUsermediaStream!.removeTrack(track);
1377
- track.stop();
1378
- }
1379
- for (const track of stream.getTracks()) {
1380
- this.localUsermediaStream!.addTrack(track);
1381
- }
1382
-
1383
- // Then replace the old tracks, if possible.
1384
- for (const track of stream.getTracks()) {
1385
- const tKey = getTransceiverKey(SDPStreamMetadataPurpose.Usermedia, track.kind);
1386
-
1387
- const transceiver = this.transceivers.get(tKey);
1388
- const oldSender = transceiver?.sender;
1389
- let added = false;
1390
- if (oldSender) {
1391
- try {
1392
- logger.info(
1393
- `Call ${this.callId} updateLocalUsermediaStream() replacing track (id=${track.id}, kind=${track.kind}, streamId=${stream.id}, streamPurpose=${callFeed.purpose})`,
1394
- );
1395
- await oldSender.replaceTrack(track);
1396
- // Set the direction to indicate we're going to be sending.
1397
- // This is only necessary in the cases where we're upgrading
1398
- // the call to video after downgrading it.
1399
- transceiver.direction = transceiver.direction === "inactive" ? "sendonly" : "sendrecv";
1400
- added = true;
1401
- } catch (error) {
1402
- logger.warn(
1403
- `Call ${this.callId} updateLocalUsermediaStream() replaceTrack failed: adding new transceiver instead`,
1404
- error,
1405
- );
1406
- }
1407
- }
1408
-
1409
- if (!added) {
1410
- logger.info(
1411
- `Call ${this.callId} updateLocalUsermediaStream() adding track to peer connection (id=${track.id}, kind=${track.kind}, streamId=${stream.id}, streamPurpose=${callFeed.purpose})`,
1412
- );
1413
-
1414
- const newSender = this.peerConn!.addTrack(track, this.localUsermediaStream!);
1415
- const newTransceiver = this.peerConn!.getTransceivers().find((t) => t.sender === newSender);
1416
- if (newTransceiver) {
1417
- this.transceivers.set(tKey, newTransceiver);
1418
- } else {
1419
- logger.warn(
1420
- `Call ${this.callId} updateLocalUsermediaStream() couldn't find matching transceiver for newly added track!`,
1421
- );
1422
- }
1423
- }
1424
- }
1425
- }
1426
-
1427
- /**
1428
- * Set whether our outbound video should be muted or not.
1429
- * @param muted - True to mute the outbound video.
1430
- * @returns the new mute state
1431
- */
1432
- public async setLocalVideoMuted(muted: boolean): Promise<boolean> {
1433
- logger.log(`Call ${this.callId} setLocalVideoMuted() running ${muted}`);
1434
-
1435
- // if we were still thinking about stopping and removing the video
1436
- // track: don't, because we want it back.
1437
- if (!muted && this.stopVideoTrackTimer !== undefined) {
1438
- clearTimeout(this.stopVideoTrackTimer);
1439
- this.stopVideoTrackTimer = undefined;
1440
- }
1441
-
1442
- if (!(await this.client.getMediaHandler().hasVideoDevice())) {
1443
- return this.isLocalVideoMuted();
1444
- }
1445
-
1446
- if (!this.hasUserMediaVideoSender && !muted) {
1447
- this.localUsermediaFeed?.setAudioVideoMuted(null, muted);
1448
- await this.upgradeCall(false, true);
1449
- return this.isLocalVideoMuted();
1450
- }
1451
-
1452
- // we may not have a video track - if not, re-request usermedia
1453
- if (!muted && this.localUsermediaStream!.getVideoTracks().length === 0) {
1454
- const stream = await this.client.getMediaHandler().getUserMediaStream(true, true);
1455
- await this.updateLocalUsermediaStream(stream);
1456
- }
1457
-
1458
- this.localUsermediaFeed?.setAudioVideoMuted(null, muted);
1459
-
1460
- this.updateMuteStatus();
1461
- await this.sendMetadataUpdate();
1462
-
1463
- // if we're muting video, set a timeout to stop & remove the video track so we release
1464
- // the camera. We wait a short time to do this because when we disable a track, WebRTC
1465
- // will send black video for it. If we just stop and remove it straight away, the video
1466
- // will just freeze which means that when we unmute video, the other side will briefly
1467
- // get a static frame of us from before we muted. This way, the still frame is just black.
1468
- // A very small delay is not always enough so the theory here is that it needs to be long
1469
- // enough for WebRTC to encode a frame: 120ms should be long enough even if we're only
1470
- // doing 10fps.
1471
- if (muted) {
1472
- this.stopVideoTrackTimer = setTimeout(() => {
1473
- for (const t of this.localUsermediaStream!.getVideoTracks()) {
1474
- t.stop();
1475
- this.localUsermediaStream!.removeTrack(t);
1476
- }
1477
- }, 120);
1478
- }
1479
-
1480
- return this.isLocalVideoMuted();
1481
- }
1482
-
1483
- /**
1484
- * Check if local video is muted.
1485
- *
1486
- * If there are multiple video tracks, <i>all</i> of the tracks need to be muted
1487
- * for this to return true. This means if there are no video tracks, this will
1488
- * return true.
1489
- * @returns True if the local preview video is muted, else false
1490
- * (including if the call is not set up yet).
1491
- */
1492
- public isLocalVideoMuted(): boolean {
1493
- return this.localUsermediaFeed?.isVideoMuted() ?? false;
1494
- }
1495
-
1496
- /**
1497
- * Set whether the microphone should be muted or not.
1498
- * @param muted - True to mute the mic.
1499
- * @returns the new mute state
1500
- */
1501
- public async setMicrophoneMuted(muted: boolean): Promise<boolean> {
1502
- logger.log(`Call ${this.callId} setMicrophoneMuted() running ${muted}`);
1503
- if (!(await this.client.getMediaHandler().hasAudioDevice())) {
1504
- return this.isMicrophoneMuted();
1505
- }
1506
-
1507
- if (!muted && (!this.hasUserMediaAudioSender || !this.hasLocalUserMediaAudioTrack)) {
1508
- await this.upgradeCall(true, false);
1509
- return this.isMicrophoneMuted();
1510
- }
1511
- this.localUsermediaFeed?.setAudioVideoMuted(muted, null);
1512
- this.updateMuteStatus();
1513
- await this.sendMetadataUpdate();
1514
- return this.isMicrophoneMuted();
1515
- }
1516
-
1517
- /**
1518
- * Check if the microphone is muted.
1519
- *
1520
- * If there are multiple audio tracks, <i>all</i> of the tracks need to be muted
1521
- * for this to return true. This means if there are no audio tracks, this will
1522
- * return true.
1523
- * @returns True if the mic is muted, else false (including if the call
1524
- * is not set up yet).
1525
- */
1526
- public isMicrophoneMuted(): boolean {
1527
- return this.localUsermediaFeed?.isAudioMuted() ?? false;
1528
- }
1529
-
1530
- /**
1531
- * @returns true if we have put the party on the other side of the call on hold
1532
- * (that is, we are signalling to them that we are not listening)
1533
- */
1534
- public isRemoteOnHold(): boolean {
1535
- return this.remoteOnHold;
1536
- }
1537
-
1538
- public setRemoteOnHold(onHold: boolean): void {
1539
- if (this.isRemoteOnHold() === onHold) return;
1540
- this.remoteOnHold = onHold;
1541
-
1542
- for (const transceiver of this.peerConn!.getTransceivers()) {
1543
- // We don't send hold music or anything so we're not actually
1544
- // sending anything, but sendrecv is fairly standard for hold and
1545
- // it makes it a lot easier to figure out who's put who on hold.
1546
- transceiver.direction = onHold ? "sendonly" : "sendrecv";
1547
- }
1548
- this.updateMuteStatus();
1549
- this.sendMetadataUpdate();
1550
-
1551
- this.emit(CallEvent.RemoteHoldUnhold, this.remoteOnHold, this);
1552
- }
1553
-
1554
- /**
1555
- * Indicates whether we are 'on hold' to the remote party (ie. if true,
1556
- * they cannot hear us).
1557
- * @returns true if the other party has put us on hold
1558
- */
1559
- public isLocalOnHold(): boolean {
1560
- if (this.state !== CallState.Connected) return false;
1561
-
1562
- let callOnHold = true;
1563
-
1564
- // We consider a call to be on hold only if *all* the tracks are on hold
1565
- // (is this the right thing to do?)
1566
- for (const transceiver of this.peerConn!.getTransceivers()) {
1567
- const trackOnHold = ["inactive", "recvonly"].includes(transceiver.currentDirection!);
1568
-
1569
- if (!trackOnHold) callOnHold = false;
1570
- }
1571
-
1572
- return callOnHold;
1573
- }
1574
-
1575
- /**
1576
- * Sends a DTMF digit to the other party
1577
- * @param digit - The digit (nb. string - '#' and '*' are dtmf too)
1578
- */
1579
- public sendDtmfDigit(digit: string): void {
1580
- for (const sender of this.peerConn!.getSenders()) {
1581
- if (sender.track?.kind === "audio" && sender.dtmf) {
1582
- sender.dtmf.insertDTMF(digit);
1583
- return;
1584
- }
1585
- }
1586
-
1587
- throw new Error("Unable to find a track to send DTMF on");
1588
- }
1589
-
1590
- private updateMuteStatus(): void {
1591
- const micShouldBeMuted = this.isMicrophoneMuted() || this.remoteOnHold;
1592
- const vidShouldBeMuted = this.isLocalVideoMuted() || this.remoteOnHold;
1593
-
1594
- logger.log(
1595
- `Call ${this.callId} updateMuteStatus stream ${
1596
- this.localUsermediaStream!.id
1597
- } micShouldBeMuted ${micShouldBeMuted} vidShouldBeMuted ${vidShouldBeMuted}`,
1598
- );
1599
-
1600
- setTracksEnabled(this.localUsermediaStream!.getAudioTracks(), !micShouldBeMuted);
1601
- setTracksEnabled(this.localUsermediaStream!.getVideoTracks(), !vidShouldBeMuted);
1602
- }
1603
-
1604
- public async sendMetadataUpdate(): Promise<void> {
1605
- await this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, {
1606
- [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(),
1607
- });
1608
- }
1609
-
1610
- private gotCallFeedsForInvite(callFeeds: CallFeed[], requestScreenshareFeed = false): void {
1611
- if (this.successor) {
1612
- this.successor.queueGotCallFeedsForAnswer(callFeeds);
1613
- return;
1614
- }
1615
- if (this.callHasEnded()) {
1616
- this.stopAllMedia();
1617
- return;
1618
- }
1619
-
1620
- for (const feed of callFeeds) {
1621
- this.pushLocalFeed(feed);
1622
- }
1623
-
1624
- if (requestScreenshareFeed) {
1625
- this.peerConn!.addTransceiver("video", {
1626
- direction: "recvonly",
1627
- });
1628
- }
1629
-
1630
- this.state = CallState.CreateOffer;
1631
-
1632
- logger.debug(`Call ${this.callId} gotUserMediaForInvite() run`);
1633
- // Now we wait for the negotiationneeded event
1634
- }
1635
-
1636
- private async sendAnswer(): Promise<void> {
1637
- const answerContent = {
1638
- answer: {
1639
- sdp: this.peerConn!.localDescription!.sdp,
1640
- // type is now deprecated as of Matrix VoIP v1, but
1641
- // required to still be sent for backwards compat
1642
- type: this.peerConn!.localDescription!.type,
1643
- },
1644
- [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true),
1645
- } as MCallAnswer;
1646
-
1647
- answerContent.capabilities = {
1648
- "m.call.transferee": this.client.supportsCallTransfer,
1649
- "m.call.dtmf": false,
1650
- };
1651
-
1652
- // We have just taken the local description from the peerConn which will
1653
- // contain all the local candidates added so far, so we can discard any candidates
1654
- // we had queued up because they'll be in the answer.
1655
- const discardCount = this.discardDuplicateCandidates();
1656
- logger.info(
1657
- `Call ${this.callId} sendAnswer() discarding ${discardCount} candidates that will be sent in answer`,
1658
- );
1659
-
1660
- try {
1661
- await this.sendVoipEvent(EventType.CallAnswer, answerContent);
1662
- // If this isn't the first time we've tried to send the answer,
1663
- // we may have candidates queued up, so send them now.
1664
- this.inviteOrAnswerSent = true;
1665
- } catch (error) {
1666
- // We've failed to answer: back to the ringing state
1667
- this.state = CallState.Ringing;
1668
- if (error instanceof MatrixError && error.event) this.client.cancelPendingEvent(error.event);
1669
-
1670
- let code = CallErrorCode.SendAnswer;
1671
- let message = "Failed to send answer";
1672
- if ((<Error>error).name == "UnknownDeviceError") {
1673
- code = CallErrorCode.UnknownDevices;
1674
- message = "Unknown devices present in the room";
1675
- }
1676
- this.emit(CallEvent.Error, new CallError(code, message, <Error>error), this);
1677
- throw error;
1678
- }
1679
-
1680
- // error handler re-throws so this won't happen on error, but
1681
- // we don't want the same error handling on the candidate queue
1682
- this.sendCandidateQueue();
1683
- }
1684
-
1685
- private queueGotCallFeedsForAnswer(callFeeds: CallFeed[]): void {
1686
- // Ensure only one negotiate/answer event is being processed at a time.
1687
- if (this.responsePromiseChain) {
1688
- this.responsePromiseChain = this.responsePromiseChain.then(() => this.gotCallFeedsForAnswer(callFeeds));
1689
- } else {
1690
- this.responsePromiseChain = this.gotCallFeedsForAnswer(callFeeds);
1691
- }
1692
- }
1693
-
1694
- // Enables DTX (discontinuous transmission) on the given session to reduce
1695
- // bandwidth when transmitting silence
1696
- private mungeSdp(description: RTCSessionDescriptionInit, mods: CodecParamsMod[]): void {
1697
- // The only way to enable DTX at this time is through SDP munging
1698
- const sdp = parseSdp(description.sdp!);
1699
-
1700
- sdp.media.forEach((media) => {
1701
- const payloadTypeToCodecMap = new Map<number, string>();
1702
- const codecToPayloadTypeMap = new Map<string, number>();
1703
- for (const rtp of media.rtp) {
1704
- payloadTypeToCodecMap.set(rtp.payload, rtp.codec);
1705
- codecToPayloadTypeMap.set(rtp.codec, rtp.payload);
1706
- }
1707
-
1708
- for (const mod of mods) {
1709
- if (mod.mediaType !== media.type) continue;
1710
-
1711
- if (!codecToPayloadTypeMap.has(mod.codec)) {
1712
- logger.info(
1713
- `Call ${this.callId} mungeSdp() ignoring SDP modifications for ${mod.codec} as it's not present.`,
1714
- );
1715
- continue;
1716
- }
1717
-
1718
- const extraConfig: string[] = [];
1719
- if (mod.enableDtx !== undefined) {
1720
- extraConfig.push(`usedtx=${mod.enableDtx ? "1" : "0"}`);
1721
- }
1722
- if (mod.maxAverageBitrate !== undefined) {
1723
- extraConfig.push(`maxaveragebitrate=${mod.maxAverageBitrate}`);
1724
- }
1725
-
1726
- let found = false;
1727
- for (const fmtp of media.fmtp) {
1728
- if (payloadTypeToCodecMap.get(fmtp.payload) === mod.codec) {
1729
- found = true;
1730
- fmtp.config += ";" + extraConfig.join(";");
1731
- }
1732
- }
1733
- if (!found) {
1734
- media.fmtp.push({
1735
- payload: codecToPayloadTypeMap.get(mod.codec)!,
1736
- config: extraConfig.join(";"),
1737
- });
1738
- }
1739
- }
1740
- });
1741
- description.sdp = writeSdp(sdp);
1742
- }
1743
-
1744
- private async createOffer(): Promise<RTCSessionDescriptionInit> {
1745
- const offer = await this.peerConn!.createOffer();
1746
- this.mungeSdp(offer, getCodecParamMods(this.isPtt));
1747
- return offer;
1748
- }
1749
-
1750
- private async createAnswer(): Promise<RTCSessionDescriptionInit> {
1751
- const answer = await this.peerConn!.createAnswer();
1752
- this.mungeSdp(answer, getCodecParamMods(this.isPtt));
1753
- return answer;
1754
- }
1755
-
1756
- private async gotCallFeedsForAnswer(callFeeds: CallFeed[]): Promise<void> {
1757
- if (this.callHasEnded()) return;
1758
-
1759
- this.waitForLocalAVStream = false;
1760
-
1761
- for (const feed of callFeeds) {
1762
- this.pushLocalFeed(feed);
1763
- }
1764
-
1765
- this.state = CallState.CreateAnswer;
1766
-
1767
- let answer: RTCSessionDescriptionInit;
1768
- try {
1769
- this.getRidOfRTXCodecs();
1770
- answer = await this.createAnswer();
1771
- } catch (err) {
1772
- logger.debug(`Call ${this.callId} gotCallFeedsForAnswer() failed to create answer: `, err);
1773
- this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true);
1774
- return;
1775
- }
1776
-
1777
- try {
1778
- await this.peerConn!.setLocalDescription(answer);
1779
-
1780
- // make sure we're still going
1781
- if (this.callHasEnded()) return;
1782
-
1783
- this.state = CallState.Connecting;
1784
-
1785
- // Allow a short time for initial candidates to be gathered
1786
- await new Promise((resolve) => {
1787
- setTimeout(resolve, 200);
1788
- });
1789
-
1790
- // make sure the call hasn't ended before we continue
1791
- if (this.callHasEnded()) return;
1792
-
1793
- this.sendAnswer();
1794
- } catch (err) {
1795
- logger.debug(`Call ${this.callId} gotCallFeedsForAnswer() error setting local description!`, err);
1796
- this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true);
1797
- return;
1798
- }
1799
- }
1800
-
1801
- /**
1802
- * Internal
1803
- */
1804
- private gotLocalIceCandidate = (event: RTCPeerConnectionIceEvent): void => {
1805
- if (event.candidate) {
1806
- if (this.candidatesEnded) {
1807
- logger.warn(`Call ${this.callId} gotLocalIceCandidate() got candidate after candidates have ended!`);
1808
- }
1809
-
1810
- logger.debug(`Call ${this.callId} got local ICE ${event.candidate.sdpMid} ${event.candidate.candidate}`);
1811
-
1812
- if (this.callHasEnded()) return;
1813
-
1814
- // As with the offer, note we need to make a copy of this object, not
1815
- // pass the original: that broke in Chrome ~m43.
1816
- if (event.candidate.candidate === "") {
1817
- this.queueCandidate(null);
1818
- } else {
1819
- this.queueCandidate(event.candidate);
1820
- }
1821
- }
1822
- };
1823
-
1824
- private onIceGatheringStateChange = (event: Event): void => {
1825
- logger.debug(
1826
- `Call ${this.callId} onIceGatheringStateChange() ice gathering state changed to ${
1827
- this.peerConn!.iceGatheringState
1828
- }`,
1829
- );
1830
- if (this.peerConn?.iceGatheringState === "complete") {
1831
- this.queueCandidate(null); // We should leave it to WebRTC to announce the end
1832
- logger.debug(
1833
- `Call ${this.callId} onIceGatheringStateChange() ice gathering state complete, set candidates have ended`,
1834
- );
1835
- }
1836
- };
1837
-
1838
- public async onRemoteIceCandidatesReceived(ev: MatrixEvent): Promise<void> {
1839
- if (this.callHasEnded()) {
1840
- //debuglog("Ignoring remote ICE candidate because call has ended");
1841
- return;
1842
- }
1843
-
1844
- const content = ev.getContent<MCallCandidates>();
1845
- const candidates = content.candidates;
1846
- if (!candidates) {
1847
- logger.info(
1848
- `Call ${this.callId} onRemoteIceCandidatesReceived() ignoring candidates event with no candidates!`,
1849
- );
1850
- return;
1851
- }
1852
-
1853
- const fromPartyId = content.version === 0 ? null : content.party_id || null;
1854
-
1855
- if (this.opponentPartyId === undefined) {
1856
- // we haven't picked an opponent yet so save the candidates
1857
- if (fromPartyId) {
1858
- logger.info(
1859
- `Call ${this.callId} onRemoteIceCandidatesReceived() buffering ${candidates.length} candidates until we pick an opponent`,
1860
- );
1861
- const bufferedCandidates = this.remoteCandidateBuffer.get(fromPartyId) || [];
1862
- bufferedCandidates.push(...candidates);
1863
- this.remoteCandidateBuffer.set(fromPartyId, bufferedCandidates);
1864
- }
1865
- return;
1866
- }
1867
-
1868
- if (!this.partyIdMatches(content)) {
1869
- logger.info(
1870
- `Call ${this.callId} onRemoteIceCandidatesReceived() ignoring candidates from party ID ${content.party_id}: we have chosen party ID ${this.opponentPartyId}`,
1871
- );
1872
-
1873
- return;
1874
- }
1875
-
1876
- await this.addIceCandidates(candidates);
1877
- }
1878
-
1879
- /**
1880
- * Used by MatrixClient.
1881
- */
1882
- public async onAnswerReceived(event: MatrixEvent): Promise<void> {
1883
- const content = event.getContent<MCallAnswer>();
1884
- logger.debug(`Call ${this.callId} onAnswerReceived() running (hangupParty=${content.party_id})`);
1885
-
1886
- if (this.callHasEnded()) {
1887
- logger.debug(`Call ${this.callId} onAnswerReceived() ignoring answer because call has ended`);
1888
- return;
1889
- }
1890
-
1891
- if (this.opponentPartyId !== undefined) {
1892
- logger.info(
1893
- `Call ${this.callId} onAnswerReceived() ignoring answer from party ID ${content.party_id}: we already have an answer/reject from ${this.opponentPartyId}`,
1894
- );
1895
- return;
1896
- }
1897
-
1898
- this.chooseOpponent(event);
1899
- await this.addBufferedIceCandidates();
1900
-
1901
- this.state = CallState.Connecting;
1902
-
1903
- const sdpStreamMetadata = content[SDPStreamMetadataKey];
1904
- if (sdpStreamMetadata) {
1905
- this.updateRemoteSDPStreamMetadata(sdpStreamMetadata);
1906
- } else {
1907
- logger.warn(
1908
- `Call ${this.callId} onAnswerReceived() did not get any SDPStreamMetadata! Can not send/receive multiple streams`,
1909
- );
1910
- }
1911
-
1912
- try {
1913
- this.isSettingRemoteAnswerPending = true;
1914
- await this.peerConn!.setRemoteDescription(content.answer);
1915
- this.isSettingRemoteAnswerPending = false;
1916
- logger.debug(`Call ${this.callId} onAnswerReceived() set remote description: ${content.answer.type}`);
1917
- } catch (e) {
1918
- this.isSettingRemoteAnswerPending = false;
1919
- logger.debug(`Call ${this.callId} onAnswerReceived() failed to set remote description`, e);
1920
- this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false);
1921
- return;
1922
- }
1923
-
1924
- // If the answer we selected has a party_id, send a select_answer event
1925
- // We do this after setting the remote description since otherwise we'd block
1926
- // call setup on it
1927
- if (this.opponentPartyId !== null) {
1928
- try {
1929
- await this.sendVoipEvent(EventType.CallSelectAnswer, {
1930
- selected_party_id: this.opponentPartyId!,
1931
- });
1932
- } catch (err) {
1933
- // This isn't fatal, and will just mean that if another party has raced to answer
1934
- // the call, they won't know they got rejected, so we carry on & don't retry.
1935
- logger.warn(`Call ${this.callId} onAnswerReceived() failed to send select_answer event`, err);
1936
- }
1937
- }
1938
- }
1939
-
1940
- public async onSelectAnswerReceived(event: MatrixEvent): Promise<void> {
1941
- if (this.direction !== CallDirection.Inbound) {
1942
- logger.warn(
1943
- `Call ${this.callId} onSelectAnswerReceived() got select_answer for an outbound call: ignoring`,
1944
- );
1945
- return;
1946
- }
1947
-
1948
- const selectedPartyId = event.getContent<MCallSelectAnswer>().selected_party_id;
1949
-
1950
- if (selectedPartyId === undefined || selectedPartyId === null) {
1951
- logger.warn(
1952
- `Call ${this.callId} onSelectAnswerReceived() got nonsensical select_answer with null/undefined selected_party_id: ignoring`,
1953
- );
1954
- return;
1955
- }
1956
-
1957
- if (selectedPartyId !== this.ourPartyId) {
1958
- logger.info(
1959
- `Call ${this.callId} onSelectAnswerReceived() got select_answer for party ID ${selectedPartyId}: we are party ID ${this.ourPartyId}.`,
1960
- );
1961
- // The other party has picked somebody else's answer
1962
- await this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true);
1963
- }
1964
- }
1965
-
1966
- public async onNegotiateReceived(event: MatrixEvent): Promise<void> {
1967
- const content = event.getContent<MCallInviteNegotiate>();
1968
- const description = content.description;
1969
- if (!description || !description.sdp || !description.type) {
1970
- logger.info(`Call ${this.callId} onNegotiateReceived() ignoring invalid m.call.negotiate event`);
1971
- return;
1972
- }
1973
- // Politeness always follows the direction of the call: in a glare situation,
1974
- // we pick either the inbound or outbound call, so one side will always be
1975
- // inbound and one outbound
1976
- const polite = this.direction === CallDirection.Inbound;
1977
-
1978
- // Here we follow the perfect negotiation logic from
1979
- // https://w3c.github.io/webrtc-pc/#perfect-negotiation-example
1980
- const readyForOffer =
1981
- !this.makingOffer && (this.peerConn!.signalingState === "stable" || this.isSettingRemoteAnswerPending);
1982
-
1983
- const offerCollision = description.type === "offer" && !readyForOffer;
1984
-
1985
- this.ignoreOffer = !polite && offerCollision;
1986
- if (this.ignoreOffer) {
1987
- logger.info(
1988
- `Call ${this.callId} onNegotiateReceived() ignoring colliding negotiate event because we're impolite`,
1989
- );
1990
- return;
1991
- }
1992
-
1993
- const prevLocalOnHold = this.isLocalOnHold();
1994
-
1995
- const sdpStreamMetadata = content[SDPStreamMetadataKey];
1996
- if (sdpStreamMetadata) {
1997
- this.updateRemoteSDPStreamMetadata(sdpStreamMetadata);
1998
- } else {
1999
- logger.warn(
2000
- `Call ${this.callId} onNegotiateReceived() received negotiation event without SDPStreamMetadata!`,
2001
- );
2002
- }
2003
-
2004
- try {
2005
- this.isSettingRemoteAnswerPending = description.type == "answer";
2006
- await this.peerConn!.setRemoteDescription(description); // SRD rolls back as needed
2007
- this.isSettingRemoteAnswerPending = false;
2008
-
2009
- logger.debug(`Call ${this.callId} onNegotiateReceived() set remote description: ${description.type}`);
2010
-
2011
- if (description.type === "offer") {
2012
- let answer: RTCSessionDescriptionInit;
2013
- try {
2014
- this.getRidOfRTXCodecs();
2015
- answer = await this.createAnswer();
2016
- } catch (err) {
2017
- logger.debug(`Call ${this.callId} onNegotiateReceived() failed to create answer: `, err);
2018
- this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true);
2019
- return;
2020
- }
2021
-
2022
- await this.peerConn!.setLocalDescription(answer);
2023
- logger.debug(`Call ${this.callId} onNegotiateReceived() create an answer`);
2024
-
2025
- this.sendVoipEvent(EventType.CallNegotiate, {
2026
- lifetime: CALL_TIMEOUT_MS,
2027
- description: this.peerConn!.localDescription?.toJSON() as RTCSessionDescription,
2028
- [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true),
2029
- });
2030
- }
2031
- } catch (err) {
2032
- this.isSettingRemoteAnswerPending = false;
2033
- logger.warn(`Call ${this.callId} onNegotiateReceived() failed to complete negotiation`, err);
2034
- }
2035
-
2036
- const newLocalOnHold = this.isLocalOnHold();
2037
- if (prevLocalOnHold !== newLocalOnHold) {
2038
- this.emit(CallEvent.LocalHoldUnhold, newLocalOnHold, this);
2039
- // also this one for backwards compat
2040
- this.emit(CallEvent.HoldUnhold, newLocalOnHold);
2041
- }
2042
- }
2043
-
2044
- private updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void {
2045
- this.remoteSDPStreamMetadata = recursivelyAssign(this.remoteSDPStreamMetadata || {}, metadata, true);
2046
- for (const feed of this.getRemoteFeeds()) {
2047
- const streamId = feed.stream.id;
2048
- const metadata = this.remoteSDPStreamMetadata![streamId];
2049
-
2050
- feed.setAudioVideoMuted(metadata?.audio_muted, metadata?.video_muted);
2051
- feed.purpose = this.remoteSDPStreamMetadata![streamId]?.purpose;
2052
- }
2053
- }
2054
-
2055
- public onSDPStreamMetadataChangedReceived(event: MatrixEvent): void {
2056
- const content = event.getContent<MCallSDPStreamMetadataChanged>();
2057
- const metadata = content[SDPStreamMetadataKey];
2058
- this.updateRemoteSDPStreamMetadata(metadata);
2059
- }
2060
-
2061
- public async onAssertedIdentityReceived(event: MatrixEvent): Promise<void> {
2062
- const content = event.getContent<MCAllAssertedIdentity>();
2063
- if (!content.asserted_identity) return;
2064
-
2065
- this.remoteAssertedIdentity = {
2066
- id: content.asserted_identity.id,
2067
- displayName: content.asserted_identity.display_name,
2068
- };
2069
- this.emit(CallEvent.AssertedIdentityChanged, this);
2070
- }
2071
-
2072
- public callHasEnded(): boolean {
2073
- // This exists as workaround to typescript trying to be clever and erroring
2074
- // when putting if (this.state === CallState.Ended) return; twice in the same
2075
- // function, even though that function is async.
2076
- return this.state === CallState.Ended;
2077
- }
2078
-
2079
- private queueGotLocalOffer(): void {
2080
- // Ensure only one negotiate/answer event is being processed at a time.
2081
- if (this.responsePromiseChain) {
2082
- this.responsePromiseChain = this.responsePromiseChain.then(() => this.wrappedGotLocalOffer());
2083
- } else {
2084
- this.responsePromiseChain = this.wrappedGotLocalOffer();
2085
- }
2086
- }
2087
-
2088
- private async wrappedGotLocalOffer(): Promise<void> {
2089
- this.makingOffer = true;
2090
- try {
2091
- // XXX: in what situations do we believe gotLocalOffer actually throws? It appears
2092
- // to handle most of its exceptions itself and terminate the call. I'm not entirely
2093
- // sure it would ever throw, so I can't add a test for these lines.
2094
- // Also the tense is different between "gotLocalOffer" and "getLocalOfferFailed" so
2095
- // it's not entirely clear whether getLocalOfferFailed is just misnamed or whether
2096
- // they've been cross-polinated somehow at some point.
2097
- await this.gotLocalOffer();
2098
- } catch (e) {
2099
- this.getLocalOfferFailed(e as Error);
2100
- return;
2101
- } finally {
2102
- this.makingOffer = false;
2103
- }
2104
- }
2105
-
2106
- private async gotLocalOffer(): Promise<void> {
2107
- logger.debug(`Call ${this.callId} gotLocalOffer() running`);
2108
-
2109
- if (this.callHasEnded()) {
2110
- logger.debug(
2111
- `Call ${this.callId} gotLocalOffer() ignoring newly created offer because the call has ended"`,
2112
- );
2113
- return;
2114
- }
2115
-
2116
- let offer: RTCSessionDescriptionInit;
2117
- try {
2118
- this.getRidOfRTXCodecs();
2119
- offer = await this.createOffer();
2120
- } catch (err) {
2121
- logger.debug(`Call ${this.callId} gotLocalOffer() failed to create offer: `, err);
2122
- this.terminate(CallParty.Local, CallErrorCode.CreateOffer, true);
2123
- return;
2124
- }
2125
-
2126
- try {
2127
- await this.peerConn!.setLocalDescription(offer);
2128
- } catch (err) {
2129
- logger.debug(`Call ${this.callId} gotLocalOffer() error setting local description!`, err);
2130
- this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true);
2131
- return;
2132
- }
2133
-
2134
- if (this.peerConn!.iceGatheringState === "gathering") {
2135
- // Allow a short time for initial candidates to be gathered
2136
- await new Promise((resolve) => {
2137
- setTimeout(resolve, 200);
2138
- });
2139
- }
2140
-
2141
- if (this.callHasEnded()) return;
2142
-
2143
- const eventType = this.state === CallState.CreateOffer ? EventType.CallInvite : EventType.CallNegotiate;
2144
-
2145
- const content = {
2146
- lifetime: CALL_TIMEOUT_MS,
2147
- } as MCallInviteNegotiate;
2148
-
2149
- if (eventType === EventType.CallInvite && this.invitee) {
2150
- content.invitee = this.invitee;
2151
- }
2152
-
2153
- // clunky because TypeScript can't follow the types through if we use an expression as the key
2154
- if (this.state === CallState.CreateOffer) {
2155
- content.offer = this.peerConn!.localDescription?.toJSON() as RTCSessionDescription;
2156
- } else {
2157
- content.description = this.peerConn!.localDescription?.toJSON() as RTCSessionDescription;
2158
- }
2159
-
2160
- content.capabilities = {
2161
- "m.call.transferee": this.client.supportsCallTransfer,
2162
- "m.call.dtmf": false,
2163
- };
2164
-
2165
- content[SDPStreamMetadataKey] = this.getLocalSDPStreamMetadata(true);
2166
-
2167
- // Get rid of any candidates waiting to be sent: they'll be included in the local
2168
- // description we just got and will send in the offer.
2169
- const discardCount = this.discardDuplicateCandidates();
2170
- logger.info(
2171
- `Call ${this.callId} gotLocalOffer() discarding ${discardCount} candidates that will be sent in offer`,
2172
- );
2173
-
2174
- try {
2175
- await this.sendVoipEvent(eventType, content);
2176
- } catch (error) {
2177
- logger.error(`Call ${this.callId} gotLocalOffer() failed to send invite`, error);
2178
- if (error instanceof MatrixError && error.event) this.client.cancelPendingEvent(error.event);
2179
-
2180
- let code = CallErrorCode.SignallingFailed;
2181
- let message = "Signalling failed";
2182
- if (this.state === CallState.CreateOffer) {
2183
- code = CallErrorCode.SendInvite;
2184
- message = "Failed to send invite";
2185
- }
2186
- if ((<Error>error).name == "UnknownDeviceError") {
2187
- code = CallErrorCode.UnknownDevices;
2188
- message = "Unknown devices present in the room";
2189
- }
2190
-
2191
- this.emit(CallEvent.Error, new CallError(code, message, <Error>error), this);
2192
- this.terminate(CallParty.Local, code, false);
2193
-
2194
- // no need to carry on & send the candidate queue, but we also
2195
- // don't want to rethrow the error
2196
- return;
2197
- }
2198
-
2199
- this.sendCandidateQueue();
2200
- if (this.state === CallState.CreateOffer) {
2201
- this.inviteOrAnswerSent = true;
2202
- this.state = CallState.InviteSent;
2203
- this.inviteTimeout = setTimeout(() => {
2204
- this.inviteTimeout = undefined;
2205
- if (this.state === CallState.InviteSent) {
2206
- this.hangup(CallErrorCode.InviteTimeout, false);
2207
- }
2208
- }, CALL_TIMEOUT_MS);
2209
- }
2210
- }
2211
-
2212
- private getLocalOfferFailed = (err: Error): void => {
2213
- logger.error(`Call ${this.callId} getLocalOfferFailed() running`, err);
2214
-
2215
- this.emit(
2216
- CallEvent.Error,
2217
- new CallError(CallErrorCode.LocalOfferFailed, "Failed to get local offer!", err),
2218
- this,
2219
- );
2220
- this.terminate(CallParty.Local, CallErrorCode.LocalOfferFailed, false);
2221
- };
2222
-
2223
- private getUserMediaFailed = (err: Error): void => {
2224
- if (this.successor) {
2225
- this.successor.getUserMediaFailed(err);
2226
- return;
2227
- }
2228
-
2229
- logger.warn(`Call ${this.callId} getUserMediaFailed() failed to get user media - ending call`, err);
2230
-
2231
- this.emit(
2232
- CallEvent.Error,
2233
- new CallError(
2234
- CallErrorCode.NoUserMedia,
2235
- "Couldn't start capturing media! Is your microphone set up and does this app have permission?",
2236
- err,
2237
- ),
2238
- this,
2239
- );
2240
- this.terminate(CallParty.Local, CallErrorCode.NoUserMedia, false);
2241
- };
2242
-
2243
- private placeCallFailed = (err: Error): void => {
2244
- if (this.successor) {
2245
- this.successor.placeCallFailed(err);
2246
- return;
2247
- }
2248
-
2249
- logger.warn(`Call ${this.callId} placeCallWithCallFeeds() failed - ending call`, err);
2250
-
2251
- this.emit(
2252
- CallEvent.Error,
2253
- new CallError(CallErrorCode.IceFailed, "Couldn't start call! Invalid ICE server configuration.", err),
2254
- this,
2255
- );
2256
- this.terminate(CallParty.Local, CallErrorCode.IceFailed, false);
2257
- };
2258
-
2259
- private onIceConnectionStateChanged = (): void => {
2260
- if (this.callHasEnded()) {
2261
- return; // because ICE can still complete as we're ending the call
2262
- }
2263
- logger.debug(
2264
- `Call ${this.callId} onIceConnectionStateChanged() running (state=${this.peerConn?.iceConnectionState}, conn=${this.peerConn?.connectionState})`,
2265
- );
2266
-
2267
- // ideally we'd consider the call to be connected when we get media but
2268
- // chrome doesn't implement any of the 'onstarted' events yet
2269
- if (["connected", "completed"].includes(this.peerConn?.iceConnectionState ?? "")) {
2270
- clearTimeout(this.iceDisconnectedTimeout);
2271
- this.iceDisconnectedTimeout = undefined;
2272
- if (this.iceReconnectionTimeOut) {
2273
- clearTimeout(this.iceReconnectionTimeOut);
2274
- }
2275
- this.state = CallState.Connected;
2276
-
2277
- if (!this.callLengthInterval && !this.callStartTime) {
2278
- this.callStartTime = Date.now();
2279
-
2280
- this.callLengthInterval = setInterval(() => {
2281
- this.emit(CallEvent.LengthChanged, Math.round((Date.now() - this.callStartTime!) / 1000), this);
2282
- }, CALL_LENGTH_INTERVAL);
2283
- }
2284
- } else if (this.peerConn?.iceConnectionState == "failed") {
2285
- this.candidatesEnded = false;
2286
- // Firefox for Android does not yet have support for restartIce()
2287
- // (the types say it's always defined though, so we have to cast
2288
- // to prevent typescript from warning).
2289
- if (this.peerConn?.restartIce as (() => void) | null) {
2290
- this.candidatesEnded = false;
2291
- logger.debug(
2292
- `Call ${this.callId} onIceConnectionStateChanged() ice restart (state=${this.peerConn?.iceConnectionState})`,
2293
- );
2294
- this.peerConn!.restartIce();
2295
- } else {
2296
- logger.info(
2297
- `Call ${this.callId} onIceConnectionStateChanged() hanging up call (ICE failed and no ICE restart method)`,
2298
- );
2299
- this.hangup(CallErrorCode.IceFailed, false);
2300
- }
2301
- } else if (this.peerConn?.iceConnectionState == "disconnected") {
2302
- this.candidatesEnded = false;
2303
- this.iceReconnectionTimeOut = setTimeout((): void => {
2304
- logger.info(
2305
- `Call ${this.callId} onIceConnectionStateChanged() ICE restarting because of ICE disconnected, (state=${this.peerConn?.iceConnectionState}, conn=${this.peerConn?.connectionState})`,
2306
- );
2307
- if (this.peerConn?.restartIce as (() => void) | null) {
2308
- this.candidatesEnded = false;
2309
- this.peerConn!.restartIce();
2310
- }
2311
- this.iceReconnectionTimeOut = undefined;
2312
- }, ICE_RECONNECTING_TIMEOUT);
2313
-
2314
- this.iceDisconnectedTimeout = setTimeout((): void => {
2315
- logger.info(
2316
- `Call ${this.callId} onIceConnectionStateChanged() hanging up call (ICE disconnected for too long)`,
2317
- );
2318
- this.hangup(CallErrorCode.IceFailed, false);
2319
- }, ICE_DISCONNECTED_TIMEOUT);
2320
- this.state = CallState.Connecting;
2321
- }
2322
-
2323
- // In PTT mode, override feed status to muted when we lose connection to
2324
- // the peer, since we don't want to block the line if they're not saying anything.
2325
- // Experimenting in Chrome, this happens after 5 or 6 seconds, which is probably
2326
- // fast enough.
2327
- if (this.isPtt && ["failed", "disconnected"].includes(this.peerConn!.iceConnectionState)) {
2328
- for (const feed of this.getRemoteFeeds()) {
2329
- feed.setAudioVideoMuted(true, true);
2330
- }
2331
- }
2332
- };
2333
-
2334
- private onSignallingStateChanged = (): void => {
2335
- logger.debug(`Call ${this.callId} onSignallingStateChanged() running (state=${this.peerConn?.signalingState})`);
2336
- };
2337
-
2338
- private onTrack = (ev: RTCTrackEvent): void => {
2339
- if (ev.streams.length === 0) {
2340
- logger.warn(
2341
- `Call ${this.callId} onTrack() called with streamless track streamless (kind=${ev.track.kind})`,
2342
- );
2343
- return;
2344
- }
2345
-
2346
- const stream = ev.streams[0];
2347
- this.pushRemoteFeed(stream);
2348
-
2349
- if (!this.removeTrackListeners.has(stream)) {
2350
- const onRemoveTrack = (): void => {
2351
- if (stream.getTracks().length === 0) {
2352
- logger.info(`Call ${this.callId} onTrack() removing track (streamId=${stream.id})`);
2353
- this.deleteFeedByStream(stream);
2354
- stream.removeEventListener("removetrack", onRemoveTrack);
2355
- this.removeTrackListeners.delete(stream);
2356
- }
2357
- };
2358
- stream.addEventListener("removetrack", onRemoveTrack);
2359
- this.removeTrackListeners.set(stream, onRemoveTrack);
2360
- }
2361
- };
2362
-
2363
- private onDataChannel = (ev: RTCDataChannelEvent): void => {
2364
- this.emit(CallEvent.DataChannel, ev.channel, this);
2365
- };
2366
-
2367
- /**
2368
- * This method removes all video/rtx codecs from screensharing video
2369
- * transceivers. This is necessary since they can cause problems. Without
2370
- * this the following steps should produce an error:
2371
- * Chromium calls Firefox
2372
- * Firefox answers
2373
- * Firefox starts screen-sharing
2374
- * Chromium starts screen-sharing
2375
- * Call crashes for Chromium with:
2376
- * [96685:23:0518/162603.933321:ERROR:webrtc_video_engine.cc(3296)] RTX codec (PT=97) mapped to PT=96 which is not in the codec list.
2377
- * [96685:23:0518/162603.933377:ERROR:webrtc_video_engine.cc(1171)] GetChangedRecvParameters called without any video codecs.
2378
- * [96685:23:0518/162603.933430:ERROR:sdp_offer_answer.cc(4302)] Failed to set local video description recv parameters for m-section with mid='2'. (INVALID_PARAMETER)
2379
- */
2380
- private getRidOfRTXCodecs(): void {
2381
- // RTCRtpReceiver.getCapabilities and RTCRtpSender.getCapabilities don't seem to be supported on FF before v113
2382
- if (!RTCRtpReceiver.getCapabilities || !RTCRtpSender.getCapabilities) return;
2383
-
2384
- const screenshareVideoTransceiver = this.transceivers.get(
2385
- getTransceiverKey(SDPStreamMetadataPurpose.Screenshare, "video"),
2386
- );
2387
-
2388
- // setCodecPreferences isn't supported on FF (as of v113)
2389
- if (!screenshareVideoTransceiver || !screenshareVideoTransceiver.setCodecPreferences) return;
2390
-
2391
- const recvCodecs = RTCRtpReceiver.getCapabilities("video")!.codecs;
2392
- const sendCodecs = RTCRtpSender.getCapabilities("video")!.codecs;
2393
- const codecs = [];
2394
-
2395
- for (const codec of [...recvCodecs, ...sendCodecs]) {
2396
- if (codec.mimeType !== "video/rtx") {
2397
- codecs.push(codec);
2398
- try {
2399
- screenshareVideoTransceiver.setCodecPreferences(codecs);
2400
- } catch (e) {
2401
- // Specifically, Chrome around version 125 and Electron 30 (which is Chromium 124) return an H.264 codec in
2402
- // the sender's capabilities but throw when you try to set it. Hence... this mess.
2403
- // Specifically, that codec is:
2404
- // {
2405
- // clockRate: 90000,
2406
- // mimeType: "video/H264",
2407
- // sdpFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640034",
2408
- // }
2409
- logger.info(
2410
- "Working around buggy WebRTC impl: claimed to support codec but threw when setting codec preferences",
2411
- codec,
2412
- e,
2413
- );
2414
- codecs.pop();
2415
- }
2416
- }
2417
- }
2418
- }
2419
-
2420
- private onNegotiationNeeded = async (): Promise<void> => {
2421
- logger.info(`Call ${this.callId} onNegotiationNeeded() negotiation is needed!`);
2422
-
2423
- if (this.state !== CallState.CreateOffer && this.opponentVersion === 0) {
2424
- logger.info(
2425
- `Call ${this.callId} onNegotiationNeeded() opponent does not support renegotiation: ignoring negotiationneeded event`,
2426
- );
2427
- return;
2428
- }
2429
-
2430
- this.queueGotLocalOffer();
2431
- };
2432
-
2433
- public onHangupReceived = (msg: MCallHangupReject): void => {
2434
- logger.debug(`Call ${this.callId} onHangupReceived() running`);
2435
-
2436
- // party ID must match (our chosen partner hanging up the call) or be undefined (we haven't chosen
2437
- // a partner yet but we're treating the hangup as a reject as per VoIP v0)
2438
- if (this.partyIdMatches(msg) || this.state === CallState.Ringing) {
2439
- // default reason is user_hangup
2440
- this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true);
2441
- } else {
2442
- logger.info(
2443
- `Call ${this.callId} onHangupReceived() ignoring message from party ID ${msg.party_id}: our partner is ${this.opponentPartyId}`,
2444
- );
2445
- }
2446
- };
2447
-
2448
- public onRejectReceived = (msg: MCallHangupReject): void => {
2449
- logger.debug(`Call ${this.callId} onRejectReceived() running`);
2450
-
2451
- // No need to check party_id for reject because if we'd received either
2452
- // an answer or reject, we wouldn't be in state InviteSent
2453
-
2454
- const shouldTerminate =
2455
- // reject events also end the call if it's ringing: it's another of
2456
- // our devices rejecting the call.
2457
- [CallState.InviteSent, CallState.Ringing].includes(this.state) ||
2458
- // also if we're in the init state and it's an inbound call, since
2459
- // this means we just haven't entered the ringing state yet
2460
- (this.state === CallState.Fledgling && this.direction === CallDirection.Inbound);
2461
-
2462
- if (shouldTerminate) {
2463
- this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true);
2464
- } else {
2465
- logger.debug(`Call ${this.callId} onRejectReceived() called in wrong state (state=${this.state})`);
2466
- }
2467
- };
2468
-
2469
- public onAnsweredElsewhere = (msg: MCallAnswer): void => {
2470
- logger.debug(`Call ${this.callId} onAnsweredElsewhere() running`);
2471
- this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true);
2472
- };
2473
-
2474
- /**
2475
- * @internal
2476
- */
2477
- private async sendVoipEvent<K extends keyof Pick<TimelineEvents, CallEventType>>(
2478
- eventType: K,
2479
- content: Omit<TimelineEvents[K], "version" | "call_id" | "party_id" | "conf_id">,
2480
- ): Promise<void> {
2481
- const realContent = {
2482
- ...content,
2483
- version: VOIP_PROTO_VERSION,
2484
- call_id: this.callId,
2485
- party_id: this.ourPartyId,
2486
- conf_id: this.groupCallId,
2487
- } as TimelineEvents[K];
2488
-
2489
- if (this.opponentDeviceId) {
2490
- const toDeviceSeq = this.toDeviceSeq++;
2491
- const content = {
2492
- ...realContent,
2493
- device_id: this.client.deviceId,
2494
- sender_session_id: this.client.getSessionId(),
2495
- dest_session_id: this.opponentSessionId,
2496
- seq: toDeviceSeq,
2497
- [ToDeviceMessageId]: uuidv4(),
2498
- };
2499
-
2500
- this.emit(
2501
- CallEvent.SendVoipEvent,
2502
- {
2503
- type: "toDevice",
2504
- eventType,
2505
- userId: this.invitee || this.getOpponentMember()?.userId,
2506
- opponentDeviceId: this.opponentDeviceId,
2507
- content,
2508
- },
2509
- this,
2510
- );
2511
-
2512
- const userId = this.invitee || this.getOpponentMember()!.userId;
2513
- if (this.client.getUseE2eForGroupCall()) {
2514
- if (!this.opponentDeviceInfo) {
2515
- logger.warn(`Call ${this.callId} sendVoipEvent() failed: we do not have opponentDeviceInfo`);
2516
- return;
2517
- }
2518
-
2519
- await this.client.encryptAndSendToDevices(
2520
- [
2521
- {
2522
- userId,
2523
- deviceInfo: this.opponentDeviceInfo,
2524
- },
2525
- ],
2526
- {
2527
- type: eventType,
2528
- content,
2529
- },
2530
- );
2531
- } else {
2532
- await this.client.sendToDevice(
2533
- eventType,
2534
- new Map<string, any>([[userId, new Map([[this.opponentDeviceId, content]])]]),
2535
- );
2536
- }
2537
- } else {
2538
- this.emit(
2539
- CallEvent.SendVoipEvent,
2540
- {
2541
- type: "sendEvent",
2542
- eventType,
2543
- roomId: this.roomId,
2544
- content: realContent,
2545
- userId: this.invitee || this.getOpponentMember()?.userId,
2546
- },
2547
- this,
2548
- );
2549
-
2550
- await this.client.sendEvent(this.roomId!, eventType, realContent);
2551
- }
2552
- }
2553
-
2554
- /**
2555
- * Queue a candidate to be sent
2556
- * @param content - The candidate to queue up, or null if candidates have finished being generated
2557
- * and end-of-candidates should be signalled
2558
- */
2559
- private queueCandidate(content: RTCIceCandidate | null): void {
2560
- // We partially de-trickle candidates by waiting for `delay` before sending them
2561
- // amalgamated, in order to avoid sending too many m.call.candidates events and hitting
2562
- // rate limits in Matrix.
2563
- // In practice, it'd be better to remove rate limits for m.call.*
2564
-
2565
- // N.B. this deliberately lets you queue and send blank candidates, which MSC2746
2566
- // currently proposes as the way to indicate that candidate gathering is complete.
2567
- // This will hopefully be changed to an explicit rather than implicit notification
2568
- // shortly.
2569
- if (content) {
2570
- this.candidateSendQueue.push(content);
2571
- } else {
2572
- this.candidatesEnded = true;
2573
- }
2574
-
2575
- // Don't send the ICE candidates yet if the call is in the ringing state: this
2576
- // means we tried to pick (ie. started generating candidates) and then failed to
2577
- // send the answer and went back to the ringing state. Queue up the candidates
2578
- // to send if we successfully send the answer.
2579
- // Equally don't send if we haven't yet sent the answer because we can send the
2580
- // first batch of candidates along with the answer
2581
- if (this.state === CallState.Ringing || !this.inviteOrAnswerSent) return;
2582
-
2583
- // MSC2746 recommends these values (can be quite long when calling because the
2584
- // callee will need a while to answer the call)
2585
- const delay = this.direction === CallDirection.Inbound ? 500 : 2000;
2586
-
2587
- if (this.candidateSendTries === 0) {
2588
- setTimeout(() => {
2589
- this.sendCandidateQueue();
2590
- }, delay);
2591
- }
2592
- }
2593
-
2594
- // Discard all non-end-of-candidates messages
2595
- // Return the number of candidate messages that were discarded.
2596
- // Call this method before sending an invite or answer message
2597
- private discardDuplicateCandidates(): number {
2598
- let discardCount = 0;
2599
- const newQueue: RTCIceCandidate[] = [];
2600
-
2601
- for (let i = 0; i < this.candidateSendQueue.length; i++) {
2602
- const candidate = this.candidateSendQueue[i];
2603
- if (candidate.candidate === "") {
2604
- newQueue.push(candidate);
2605
- } else {
2606
- discardCount++;
2607
- }
2608
- }
2609
-
2610
- this.candidateSendQueue = newQueue;
2611
-
2612
- return discardCount;
2613
- }
2614
-
2615
- /*
2616
- * Transfers this call to another user
2617
- */
2618
- public async transfer(targetUserId: string): Promise<void> {
2619
- // Fetch the target user's global profile info: their room avatar / displayname
2620
- // could be different in whatever room we share with them.
2621
- const profileInfo = await this.client.getProfileInfo(targetUserId);
2622
-
2623
- const replacementId = genCallID();
2624
-
2625
- const body = {
2626
- replacement_id: genCallID(),
2627
- target_user: {
2628
- id: targetUserId,
2629
- display_name: profileInfo.displayname,
2630
- avatar_url: profileInfo.avatar_url,
2631
- },
2632
- create_call: replacementId,
2633
- } as MCallReplacesEvent;
2634
-
2635
- await this.sendVoipEvent(EventType.CallReplaces, body);
2636
-
2637
- await this.terminate(CallParty.Local, CallErrorCode.Transferred, true);
2638
- }
2639
-
2640
- /*
2641
- * Transfers this call to the target call, effectively 'joining' the
2642
- * two calls (so the remote parties on each call are connected together).
2643
- */
2644
- public async transferToCall(transferTargetCall: MatrixCall): Promise<void> {
2645
- const targetUserId = transferTargetCall.getOpponentMember()?.userId;
2646
- const targetProfileInfo = targetUserId ? await this.client.getProfileInfo(targetUserId) : undefined;
2647
- const opponentUserId = this.getOpponentMember()?.userId;
2648
- const transfereeProfileInfo = opponentUserId ? await this.client.getProfileInfo(opponentUserId) : undefined;
2649
-
2650
- const newCallId = genCallID();
2651
-
2652
- const bodyToTransferTarget = {
2653
- // the replacements on each side have their own ID, and it's distinct from the
2654
- // ID of the new call (but we can use the same function to generate it)
2655
- replacement_id: genCallID(),
2656
- target_user: {
2657
- id: opponentUserId,
2658
- display_name: transfereeProfileInfo?.displayname,
2659
- avatar_url: transfereeProfileInfo?.avatar_url,
2660
- },
2661
- await_call: newCallId,
2662
- } as MCallReplacesEvent;
2663
-
2664
- await transferTargetCall.sendVoipEvent(EventType.CallReplaces, bodyToTransferTarget);
2665
-
2666
- const bodyToTransferee = {
2667
- replacement_id: genCallID(),
2668
- target_user: {
2669
- id: targetUserId,
2670
- display_name: targetProfileInfo?.displayname,
2671
- avatar_url: targetProfileInfo?.avatar_url,
2672
- },
2673
- create_call: newCallId,
2674
- } as MCallReplacesEvent;
2675
-
2676
- await this.sendVoipEvent(EventType.CallReplaces, bodyToTransferee);
2677
-
2678
- await this.terminate(CallParty.Local, CallErrorCode.Transferred, true);
2679
- await transferTargetCall.terminate(CallParty.Local, CallErrorCode.Transferred, true);
2680
- }
2681
-
2682
- private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise<void> {
2683
- if (this.callHasEnded()) return;
2684
-
2685
- this.hangupParty = hangupParty;
2686
- this.hangupReason = hangupReason;
2687
- this.state = CallState.Ended;
2688
-
2689
- if (this.inviteTimeout) {
2690
- clearTimeout(this.inviteTimeout);
2691
- this.inviteTimeout = undefined;
2692
- }
2693
- if (this.iceDisconnectedTimeout !== undefined) {
2694
- clearTimeout(this.iceDisconnectedTimeout);
2695
- this.iceDisconnectedTimeout = undefined;
2696
- }
2697
- if (this.callLengthInterval) {
2698
- clearInterval(this.callLengthInterval);
2699
- this.callLengthInterval = undefined;
2700
- }
2701
- if (this.stopVideoTrackTimer !== undefined) {
2702
- clearTimeout(this.stopVideoTrackTimer);
2703
- this.stopVideoTrackTimer = undefined;
2704
- }
2705
-
2706
- for (const [stream, listener] of this.removeTrackListeners) {
2707
- stream.removeEventListener("removetrack", listener);
2708
- }
2709
- this.removeTrackListeners.clear();
2710
-
2711
- this.callStatsAtEnd = await this.collectCallStats();
2712
-
2713
- // Order is important here: first we stopAllMedia() and only then we can deleteAllFeeds()
2714
- this.stopAllMedia();
2715
- this.deleteAllFeeds();
2716
-
2717
- if (this.peerConn && this.peerConn.signalingState !== "closed") {
2718
- this.peerConn.close();
2719
- }
2720
- this.stats?.removeStatsReportGatherer(this.callId);
2721
-
2722
- if (shouldEmit) {
2723
- this.emit(CallEvent.Hangup, this);
2724
- }
2725
-
2726
- this.client.callEventHandler!.calls.delete(this.callId);
2727
- }
2728
-
2729
- private stopAllMedia(): void {
2730
- logger.debug(`Call ${this.callId} stopAllMedia() running`);
2731
-
2732
- for (const feed of this.feeds) {
2733
- // Slightly awkward as local feed need to go via the correct method on
2734
- // the MediaHandler so they get removed from MediaHandler (remote tracks
2735
- // don't)
2736
- // NB. We clone local streams when passing them to individual calls in a group
2737
- // call, so we can (and should) stop the clones once we no longer need them:
2738
- // the other clones will continue fine.
2739
- if (feed.isLocal() && feed.purpose === SDPStreamMetadataPurpose.Usermedia) {
2740
- this.client.getMediaHandler().stopUserMediaStream(feed.stream);
2741
- } else if (feed.isLocal() && feed.purpose === SDPStreamMetadataPurpose.Screenshare) {
2742
- this.client.getMediaHandler().stopScreensharingStream(feed.stream);
2743
- } else if (!feed.isLocal()) {
2744
- logger.debug(`Call ${this.callId} stopAllMedia() stopping stream (streamId=${feed.stream.id})`);
2745
- for (const track of feed.stream.getTracks()) {
2746
- track.stop();
2747
- }
2748
- }
2749
- }
2750
- }
2751
-
2752
- private checkForErrorListener(): void {
2753
- if (this.listeners(EventEmitterEvents.Error).length === 0) {
2754
- throw new Error("You MUST attach an error listener using call.on('error', function() {})");
2755
- }
2756
- }
2757
-
2758
- private async sendCandidateQueue(): Promise<void> {
2759
- if (this.candidateSendQueue.length === 0 || this.callHasEnded()) {
2760
- return;
2761
- }
2762
-
2763
- const candidates = this.candidateSendQueue;
2764
- this.candidateSendQueue = [];
2765
- ++this.candidateSendTries;
2766
- const content: Pick<MCallCandidates, "candidates"> = {
2767
- candidates: candidates.map((candidate) => candidate.toJSON()),
2768
- };
2769
- if (this.candidatesEnded) {
2770
- // If there are no more candidates, signal this by adding an empty string candidate
2771
- content.candidates.push({
2772
- candidate: "",
2773
- });
2774
- }
2775
- logger.debug(`Call ${this.callId} sendCandidateQueue() attempting to send ${candidates.length} candidates`);
2776
- try {
2777
- await this.sendVoipEvent(EventType.CallCandidates, content);
2778
- // reset our retry count if we have successfully sent our candidates
2779
- // otherwise queueCandidate() will refuse to try to flush the queue
2780
- this.candidateSendTries = 0;
2781
-
2782
- // Try to send candidates again just in case we received more candidates while sending.
2783
- this.sendCandidateQueue();
2784
- } catch (error) {
2785
- // don't retry this event: we'll send another one later as we might
2786
- // have more candidates by then.
2787
- if (error instanceof MatrixError && error.event) this.client.cancelPendingEvent(error.event);
2788
-
2789
- // put all the candidates we failed to send back in the queue
2790
- this.candidateSendQueue.push(...candidates);
2791
-
2792
- if (this.candidateSendTries > 5) {
2793
- logger.debug(
2794
- `Call ${this.callId} sendCandidateQueue() failed to send candidates on attempt ${this.candidateSendTries}. Giving up on this call.`,
2795
- error,
2796
- );
2797
-
2798
- const code = CallErrorCode.SignallingFailed;
2799
- const message = "Signalling failed";
2800
-
2801
- this.emit(CallEvent.Error, new CallError(code, message, <Error>error), this);
2802
- this.hangup(code, false);
2803
-
2804
- return;
2805
- }
2806
-
2807
- const delayMs = 500 * Math.pow(2, this.candidateSendTries);
2808
- ++this.candidateSendTries;
2809
- logger.debug(
2810
- `Call ${this.callId} sendCandidateQueue() failed to send candidates. Retrying in ${delayMs}ms`,
2811
- error,
2812
- );
2813
- setTimeout(() => {
2814
- this.sendCandidateQueue();
2815
- }, delayMs);
2816
- }
2817
- }
2818
-
2819
- /**
2820
- * Place a call to this room.
2821
- * @throws if you have not specified a listener for 'error' events.
2822
- * @throws if have passed audio=false.
2823
- */
2824
- public async placeCall(audio: boolean, video: boolean): Promise<void> {
2825
- if (!audio) {
2826
- throw new Error("You CANNOT start a call without audio");
2827
- }
2828
- this.state = CallState.WaitLocalMedia;
2829
-
2830
- let callFeed: CallFeed;
2831
- try {
2832
- const stream = await this.client.getMediaHandler().getUserMediaStream(audio, video);
2833
-
2834
- // make sure all the tracks are enabled (same as pushNewLocalFeed -
2835
- // we probably ought to just have one code path for adding streams)
2836
- setTracksEnabled(stream.getAudioTracks(), true);
2837
- setTracksEnabled(stream.getVideoTracks(), true);
2838
-
2839
- callFeed = new CallFeed({
2840
- client: this.client,
2841
- roomId: this.roomId,
2842
- userId: this.client.getUserId()!,
2843
- deviceId: this.client.getDeviceId() ?? undefined,
2844
- stream,
2845
- purpose: SDPStreamMetadataPurpose.Usermedia,
2846
- audioMuted: false,
2847
- videoMuted: false,
2848
- });
2849
- } catch (e) {
2850
- this.getUserMediaFailed(<Error>e);
2851
- return;
2852
- }
2853
-
2854
- try {
2855
- await this.placeCallWithCallFeeds([callFeed]);
2856
- } catch (e) {
2857
- this.placeCallFailed(<Error>e);
2858
- return;
2859
- }
2860
- }
2861
-
2862
- /**
2863
- * Place a call to this room with call feed.
2864
- * @param callFeeds - to use
2865
- * @throws if you have not specified a listener for 'error' events.
2866
- * @throws if have passed audio=false.
2867
- */
2868
- public async placeCallWithCallFeeds(callFeeds: CallFeed[], requestScreenshareFeed = false): Promise<void> {
2869
- this.checkForErrorListener();
2870
- this.direction = CallDirection.Outbound;
2871
-
2872
- await this.initOpponentCrypto();
2873
-
2874
- // XXX Find a better way to do this
2875
- this.client.callEventHandler!.calls.set(this.callId, this);
2876
-
2877
- // make sure we have valid turn creds. Unless something's gone wrong, it should
2878
- // poll and keep the credentials valid so this should be instant.
2879
- const haveTurnCreds = await this.client.checkTurnServers();
2880
- if (!haveTurnCreds) {
2881
- logger.warn(
2882
- `Call ${this.callId} placeCallWithCallFeeds() failed to get TURN credentials! Proceeding with call anyway...`,
2883
- );
2884
- }
2885
-
2886
- // create the peer connection now so it can be gathering candidates while we get user
2887
- // media (assuming a candidate pool size is configured)
2888
- this.peerConn = this.createPeerConnection();
2889
- this.emit(CallEvent.PeerConnectionCreated, this.peerConn, this);
2890
- this.gotCallFeedsForInvite(callFeeds, requestScreenshareFeed);
2891
- }
2892
-
2893
- private createPeerConnection(): RTCPeerConnection {
2894
- const pc = new window.RTCPeerConnection({
2895
- iceTransportPolicy: this.forceTURN ? "relay" : undefined,
2896
- iceServers: this.turnServers.length ? this.turnServers : undefined,
2897
- iceCandidatePoolSize: this.client.iceCandidatePoolSize,
2898
- bundlePolicy: "max-bundle",
2899
- });
2900
-
2901
- // 'connectionstatechange' would be better, but firefox doesn't implement that.
2902
- pc.addEventListener("iceconnectionstatechange", this.onIceConnectionStateChanged);
2903
- pc.addEventListener("signalingstatechange", this.onSignallingStateChanged);
2904
- pc.addEventListener("icecandidate", this.gotLocalIceCandidate);
2905
- pc.addEventListener("icegatheringstatechange", this.onIceGatheringStateChange);
2906
- pc.addEventListener("track", this.onTrack);
2907
- pc.addEventListener("negotiationneeded", this.onNegotiationNeeded);
2908
- pc.addEventListener("datachannel", this.onDataChannel);
2909
-
2910
- const opponentMember: RoomMember | undefined = this.getOpponentMember();
2911
- const opponentMemberId = opponentMember ? opponentMember.userId : "unknown";
2912
- this.stats?.addStatsReportGatherer(this.callId, opponentMemberId, pc);
2913
- return pc;
2914
- }
2915
-
2916
- private partyIdMatches(msg: MCallBase): boolean {
2917
- // They must either match or both be absent (in which case opponentPartyId will be null)
2918
- // Also we ignore party IDs on the invite/offer if the version is 0, so we must do the same
2919
- // here and use null if the version is 0 (woe betide any opponent sending messages in the
2920
- // same call with different versions)
2921
- const msgPartyId = msg.version === 0 ? null : msg.party_id || null;
2922
- return msgPartyId === this.opponentPartyId;
2923
- }
2924
-
2925
- // Commits to an opponent for the call
2926
- // ev: An invite or answer event
2927
- private chooseOpponent(ev: MatrixEvent): void {
2928
- // I choo-choo-choose you
2929
- const msg = ev.getContent<MCallInviteNegotiate | MCallAnswer>();
2930
-
2931
- logger.debug(`Call ${this.callId} chooseOpponent() running (partyId=${msg.party_id})`);
2932
-
2933
- this.opponentVersion = msg.version;
2934
- if (this.opponentVersion === 0) {
2935
- // set to null to indicate that we've chosen an opponent, but because
2936
- // they're v0 they have no party ID (even if they sent one, we're ignoring it)
2937
- this.opponentPartyId = null;
2938
- } else {
2939
- // set to their party ID, or if they're naughty and didn't send one despite
2940
- // not being v0, set it to null to indicate we picked an opponent with no
2941
- // party ID
2942
- this.opponentPartyId = msg.party_id || null;
2943
- }
2944
- this.opponentCaps = msg.capabilities || ({} as CallCapabilities);
2945
- this.opponentMember = this.client.getRoom(this.roomId)!.getMember(ev.getSender()!) ?? undefined;
2946
- if (this.opponentMember) {
2947
- this.stats?.updateOpponentMember(this.callId, this.opponentMember.userId);
2948
- }
2949
- }
2950
-
2951
- private async addBufferedIceCandidates(): Promise<void> {
2952
- const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId!);
2953
- if (bufferedCandidates) {
2954
- logger.info(
2955
- `Call ${this.callId} addBufferedIceCandidates() adding ${bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`,
2956
- );
2957
- await this.addIceCandidates(bufferedCandidates);
2958
- }
2959
- this.remoteCandidateBuffer.clear();
2960
- }
2961
-
2962
- private async addIceCandidates(candidates: RTCIceCandidate[] | MCallCandidates["candidates"]): Promise<void> {
2963
- for (const candidate of candidates) {
2964
- if (
2965
- (candidate.sdpMid === null || candidate.sdpMid === undefined) &&
2966
- (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined)
2967
- ) {
2968
- logger.debug(`Call ${this.callId} addIceCandidates() got remote ICE end-of-candidates`);
2969
- } else {
2970
- logger.debug(
2971
- `Call ${this.callId} addIceCandidates() got remote ICE candidate (sdpMid=${candidate.sdpMid}, candidate=${candidate.candidate})`,
2972
- );
2973
- }
2974
-
2975
- try {
2976
- await this.peerConn!.addIceCandidate(candidate);
2977
- } catch (err) {
2978
- if (!this.ignoreOffer) {
2979
- logger.info(`Call ${this.callId} addIceCandidates() failed to add remote ICE candidate`, err);
2980
- } else {
2981
- logger.debug(
2982
- `Call ${this.callId} addIceCandidates() failed to add remote ICE candidate because ignoring offer`,
2983
- err,
2984
- );
2985
- }
2986
- }
2987
- }
2988
- }
2989
-
2990
- public get hasPeerConnection(): boolean {
2991
- return Boolean(this.peerConn);
2992
- }
2993
-
2994
- public initStats(stats: GroupCallStats, peerId = "unknown"): void {
2995
- this.stats = stats;
2996
- this.stats.start();
2997
- }
2998
- }
2999
-
3000
- export function setTracksEnabled(tracks: Array<MediaStreamTrack>, enabled: boolean): void {
3001
- for (const track of tracks) {
3002
- track.enabled = enabled;
3003
- }
3004
- }
3005
-
3006
- export function supportsMatrixCall(): boolean {
3007
- // typeof prevents Node from erroring on an undefined reference
3008
- if (typeof window === "undefined" || typeof document === "undefined") {
3009
- // NB. We don't log here as apps try to create a call object as a test for
3010
- // whether calls are supported, so we shouldn't fill the logs up.
3011
- return false;
3012
- }
3013
-
3014
- // Firefox throws on so little as accessing the RTCPeerConnection when operating in a secure mode.
3015
- // There's some information at https://bugzilla.mozilla.org/show_bug.cgi?id=1542616 though the concern
3016
- // is that the browser throwing a SecurityError will brick the client creation process.
3017
- try {
3018
- const supported = Boolean(
3019
- window.RTCPeerConnection ||
3020
- window.RTCSessionDescription ||
3021
- window.RTCIceCandidate ||
3022
- navigator.mediaDevices,
3023
- );
3024
- if (!supported) {
3025
- /* istanbul ignore if */ // Adds a lot of noise to test runs, so disable logging there.
3026
- if (process.env.NODE_ENV !== "test") {
3027
- logger.error("WebRTC is not supported in this browser / environment");
3028
- }
3029
- return false;
3030
- }
3031
- } catch (e) {
3032
- logger.error("Exception thrown when trying to access WebRTC", e);
3033
- return false;
3034
- }
3035
-
3036
- return true;
3037
- }
3038
-
3039
- /**
3040
- * DEPRECATED
3041
- * Use client.createCall()
3042
- *
3043
- * Create a new Matrix call for the browser.
3044
- * @param client - The client instance to use.
3045
- * @param roomId - The room the call is in.
3046
- * @param options - DEPRECATED optional options map.
3047
- * @returns the call or null if the browser doesn't support calling.
3048
- */
3049
- export function createNewMatrixCall(
3050
- client: MatrixClient,
3051
- roomId: string,
3052
- options?: Pick<CallOpts, "forceTURN" | "invitee" | "opponentDeviceId" | "opponentSessionId" | "groupCallId">,
3053
- ): MatrixCall | null {
3054
- if (!supportsMatrixCall()) return null;
3055
-
3056
- const optionsForceTURN = options ? options.forceTURN : false;
3057
-
3058
- const opts: CallOpts = {
3059
- client: client,
3060
- roomId: roomId,
3061
- invitee: options?.invitee,
3062
- turnServers: client.getTurnServers(),
3063
- // call level options
3064
- forceTURN: client.forceTURN || optionsForceTURN,
3065
- opponentDeviceId: options?.opponentDeviceId,
3066
- opponentSessionId: options?.opponentSessionId,
3067
- groupCallId: options?.groupCallId,
3068
- };
3069
- const call = new MatrixCall(opts);
3070
-
3071
- client.reEmitter.reEmit(call, Object.values(CallEvent));
3072
-
3073
- return call;
3074
- }