livekit-client 2.1.0 → 2.1.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (54) hide show
  1. package/dist/livekit-client.esm.mjs +2212 -2100
  2. package/dist/livekit-client.esm.mjs.map +1 -1
  3. package/dist/livekit-client.umd.js +1 -1
  4. package/dist/livekit-client.umd.js.map +1 -1
  5. package/dist/src/index.d.ts +1 -1
  6. package/dist/src/index.d.ts.map +1 -1
  7. package/dist/src/room/PCTransport.d.ts.map +1 -1
  8. package/dist/src/room/RTCEngine.d.ts +4 -4
  9. package/dist/src/room/RTCEngine.d.ts.map +1 -1
  10. package/dist/src/room/Room.d.ts +5 -2
  11. package/dist/src/room/Room.d.ts.map +1 -1
  12. package/dist/src/room/events.d.ts +20 -1
  13. package/dist/src/room/events.d.ts.map +1 -1
  14. package/dist/src/room/participant/LocalParticipant.d.ts.map +1 -1
  15. package/dist/src/room/participant/Participant.d.ts +2 -1
  16. package/dist/src/room/participant/Participant.d.ts.map +1 -1
  17. package/dist/src/room/track/RemoteTrack.d.ts +1 -0
  18. package/dist/src/room/track/RemoteTrack.d.ts.map +1 -1
  19. package/dist/src/room/track/Track.d.ts +4 -0
  20. package/dist/src/room/track/Track.d.ts.map +1 -1
  21. package/dist/src/room/track/TrackPublication.d.ts +3 -1
  22. package/dist/src/room/track/TrackPublication.d.ts.map +1 -1
  23. package/dist/src/room/types.d.ts +8 -0
  24. package/dist/src/room/types.d.ts.map +1 -1
  25. package/dist/src/room/utils.d.ts +4 -2
  26. package/dist/src/room/utils.d.ts.map +1 -1
  27. package/dist/src/utils/browserParser.d.ts +1 -0
  28. package/dist/src/utils/browserParser.d.ts.map +1 -1
  29. package/dist/ts4.2/src/index.d.ts +1 -1
  30. package/dist/ts4.2/src/room/RTCEngine.d.ts +4 -4
  31. package/dist/ts4.2/src/room/Room.d.ts +5 -2
  32. package/dist/ts4.2/src/room/events.d.ts +20 -1
  33. package/dist/ts4.2/src/room/participant/Participant.d.ts +2 -1
  34. package/dist/ts4.2/src/room/track/RemoteTrack.d.ts +1 -0
  35. package/dist/ts4.2/src/room/track/Track.d.ts +4 -0
  36. package/dist/ts4.2/src/room/track/TrackPublication.d.ts +3 -1
  37. package/dist/ts4.2/src/room/types.d.ts +8 -0
  38. package/dist/ts4.2/src/room/utils.d.ts +4 -2
  39. package/dist/ts4.2/src/utils/browserParser.d.ts +1 -0
  40. package/package.json +8 -8
  41. package/src/index.ts +1 -1
  42. package/src/room/PCTransport.ts +0 -2
  43. package/src/room/RTCEngine.ts +11 -61
  44. package/src/room/Room.ts +27 -1
  45. package/src/room/events.ts +23 -0
  46. package/src/room/participant/LocalParticipant.ts +3 -5
  47. package/src/room/participant/Participant.ts +5 -1
  48. package/src/room/track/RemoteTrack.ts +13 -0
  49. package/src/room/track/Track.ts +9 -0
  50. package/src/room/track/TrackPublication.ts +3 -1
  51. package/src/room/types.ts +9 -0
  52. package/src/room/utils.ts +46 -29
  53. package/src/utils/browserParser.test.ts +4 -0
  54. package/src/utils/browserParser.ts +11 -1
@@ -6,7 +6,7 @@ import type RemoteTrack from '../track/RemoteTrack';
6
6
  import type RemoteTrackPublication from '../track/RemoteTrackPublication';
7
7
  import { Track } from '../track/Track';
8
8
  import type { TrackPublication } from '../track/TrackPublication';
9
- import type { LoggerOptions } from '../types';
9
+ import type { LoggerOptions, TranscriptionSegment } from '../types';
10
10
  export declare enum ConnectionQuality {
11
11
  Excellent = "excellent",
12
12
  Good = "good",
@@ -99,6 +99,7 @@ export type ParticipantEventCallbacks = {
99
99
  participantMetadataChanged: (prevMetadata: string | undefined, participant?: any) => void;
100
100
  participantNameChanged: (name: string) => void;
101
101
  dataReceived: (payload: Uint8Array, kind: DataPacket_Kind) => void;
102
+ transcriptionReceived: (transcription: TranscriptionSegment[], publication?: TrackPublication) => void;
102
103
  isSpeakingChanged: (speaking: boolean) => void;
103
104
  connectionQualityChanged: (connectionQuality: ConnectionQuality) => void;
104
105
  trackStreamStateChanged: (publication: RemoteTrackPublication, streamState: Track.StreamState) => void;
@@ -19,5 +19,6 @@ export default abstract class RemoteTrack<TrackKind extends Track.Kind = Track.K
19
19
  getRTCStatsReport(): Promise<RTCStatsReport | undefined>;
20
20
  startMonitor(): void;
21
21
  protected abstract monitorReceiver(): void;
22
+ registerTimeSyncUpdate(): void;
22
23
  }
23
24
  //# sourceMappingURL=RemoteTrack.d.ts.map
@@ -28,11 +28,14 @@ export declare abstract class Track<TrackKind extends Track.Kind = Track.Kind> e
28
28
  * has been paused by congestion controller
29
29
  */
30
30
  streamState: Track.StreamState;
31
+ /** @internal */
32
+ rtpTimestamp: number | undefined;
31
33
  protected _mediaStreamTrack: MediaStreamTrack;
32
34
  protected _mediaStreamID: string;
33
35
  protected isInBackground: boolean;
34
36
  private backgroundTimeout;
35
37
  private loggerContextCb;
38
+ protected timeSyncHandle: number | undefined;
36
39
  protected _currentBitrate: number;
37
40
  protected monitorInterval?: ReturnType<typeof setInterval>;
38
41
  protected log: StructuredLogger;
@@ -137,6 +140,7 @@ export type TrackEventCallbacks = {
137
140
  upstreamResumed: (track: any) => void;
138
141
  trackProcessorUpdate: (processor?: TrackProcessor<Track.Kind, any>) => void;
139
142
  audioTrackFeatureUpdate: (track: any, feature: AudioTrackFeature, enabled: boolean) => void;
143
+ timeSyncUpdate: (timestamp: number) => void;
140
144
  };
141
145
  export {};
142
146
  //# sourceMappingURL=Track.d.ts.map
@@ -1,7 +1,7 @@
1
1
  import { Encryption_Type } from '@livekit/protocol';
2
2
  import type { SubscriptionError, TrackInfo, UpdateSubscription, UpdateTrackSettings } from '@livekit/protocol';
3
3
  import type TypedEventEmitter from 'typed-emitter';
4
- import type { LoggerOptions } from '../types';
4
+ import type { LoggerOptions, TranscriptionSegment } from '../types';
5
5
  import LocalAudioTrack from './LocalAudioTrack';
6
6
  import LocalVideoTrack from './LocalVideoTrack';
7
7
  import RemoteAudioTrack from './RemoteAudioTrack';
@@ -72,6 +72,8 @@ export type PublicationEventCallbacks = {
72
72
  unsubscribed: (track: RemoteTrack) => void;
73
73
  subscriptionStatusChanged: (status: TrackPublication.SubscriptionStatus, prevStatus: TrackPublication.SubscriptionStatus) => void;
74
74
  subscriptionFailed: (error: SubscriptionError) => void;
75
+ transcriptionReceived: (transcription: TranscriptionSegment[]) => void;
76
+ timeSyncUpdate: (timestamp: number) => void;
75
77
  };
76
78
  export {};
77
79
  //# sourceMappingURL=TrackPublication.d.ts.map
@@ -35,4 +35,12 @@ export type LoggerOptions = {
35
35
  loggerName?: string;
36
36
  loggerContextCb?: () => Record<string, unknown>;
37
37
  };
38
+ export interface TranscriptionSegment {
39
+ id: string;
40
+ text: string;
41
+ language: string;
42
+ startTime: number;
43
+ endTime: number;
44
+ final: boolean;
45
+ }
38
46
  //# sourceMappingURL=types.d.ts.map
@@ -1,7 +1,8 @@
1
- import { ClientInfo } from '@livekit/protocol';
1
+ import { ClientInfo, Transcription as TranscriptionModel } from '@livekit/protocol';
2
2
  import type LocalAudioTrack from './track/LocalAudioTrack';
3
3
  import type RemoteAudioTrack from './track/RemoteAudioTrack';
4
4
  import { VideoCodec } from './track/options';
5
+ import type { TranscriptionSegment } from './types';
5
6
  export declare const ddExtensionURI = "https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension";
6
7
  export declare function unpackStreamId(packed: string): string[];
7
8
  export declare function sleep(duration: number): Promise<void>;
@@ -15,13 +16,13 @@ export declare function supportsAV1(): boolean;
15
16
  export declare function supportsVP9(): boolean;
16
17
  export declare function isSVCCodec(codec?: string): boolean;
17
18
  export declare function supportsSetSinkId(elm?: HTMLMediaElement): boolean;
18
- export declare function supportsSetCodecPreferences(transceiver: RTCRtpTransceiver): boolean;
19
19
  export declare function isBrowserSupported(): boolean;
20
20
  export declare function isFireFox(): boolean;
21
21
  export declare function isChromiumBased(): boolean;
22
22
  export declare function isSafari(): boolean;
23
23
  export declare function isSafari17(): boolean;
24
24
  export declare function isMobile(): boolean;
25
+ export declare function isE2EESimulcastSupported(): boolean | undefined;
25
26
  export declare function isWeb(): boolean;
26
27
  export declare function isReactNative(): boolean;
27
28
  export declare function isCloud(serverUrl: URL): boolean;
@@ -93,4 +94,5 @@ export declare function isVideoCodec(maybeCodec: string): maybeCodec is VideoCod
93
94
  export declare function unwrapConstraint(constraint: ConstrainDOMString): string;
94
95
  export declare function toWebsocketUrl(url: string): string;
95
96
  export declare function toHttpUrl(url: string): string;
97
+ export declare function extractTranscriptionSegments(transcription: TranscriptionModel): TranscriptionSegment[];
96
98
  //# sourceMappingURL=utils.d.ts.map
@@ -4,6 +4,7 @@ export type BrowserDetails = {
4
4
  name: DetectableBrowser;
5
5
  version: string;
6
6
  os?: DetectableOS;
7
+ osVersion?: string;
7
8
  };
8
9
  /**
9
10
  * @internal
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "livekit-client",
3
- "version": "2.1.0",
3
+ "version": "2.1.2",
4
4
  "description": "JavaScript/TypeScript client SDK for LiveKit",
5
5
  "main": "./dist/livekit-client.umd.js",
6
6
  "unpkg": "./dist/livekit-client.umd.js",
@@ -36,7 +36,7 @@
36
36
  "author": "David Zhao <david@davidzhao.com>",
37
37
  "license": "Apache-2.0",
38
38
  "dependencies": {
39
- "@livekit/protocol": "1.13.0",
39
+ "@livekit/protocol": "1.15.0",
40
40
  "events": "^3.3.0",
41
41
  "loglevel": "^1.8.0",
42
42
  "sdp-transform": "^2.14.1",
@@ -46,8 +46,8 @@
46
46
  "webrtc-adapter": "^8.1.1"
47
47
  },
48
48
  "devDependencies": {
49
- "@babel/core": "7.24.4",
50
- "@babel/preset-env": "7.24.4",
49
+ "@babel/core": "7.24.5",
50
+ "@babel/preset-env": "7.24.5",
51
51
  "@bufbuild/protoc-gen-es": "^1.3.0",
52
52
  "@changesets/cli": "2.27.1",
53
53
  "@livekit/changesets-changelog-github": "^0.0.4",
@@ -73,15 +73,15 @@
73
73
  "gh-pages": "6.1.1",
74
74
  "jsdom": "^24.0.0",
75
75
  "prettier": "^3.0.0",
76
- "rollup": "4.14.0",
76
+ "rollup": "4.17.2",
77
77
  "rollup-plugin-delete": "^2.0.0",
78
78
  "rollup-plugin-re": "1.0.7",
79
79
  "rollup-plugin-typescript2": "0.36.0",
80
80
  "size-limit": "^8.2.4",
81
- "typedoc": "0.25.12",
81
+ "typedoc": "0.25.13",
82
82
  "typedoc-plugin-no-inherit": "1.4.0",
83
- "typescript": "5.4.3",
84
- "vite": "5.0.13",
83
+ "typescript": "5.4.5",
84
+ "vite": "5.2.10",
85
85
  "vitest": "^1.0.0"
86
86
  },
87
87
  "scripts": {
package/src/index.ts CHANGED
@@ -44,7 +44,7 @@ export { facingModeFromDeviceLabel, facingModeFromLocalTrack } from './room/trac
44
44
  export * from './room/track/options';
45
45
  export * from './room/track/processor/types';
46
46
  export * from './room/track/types';
47
- export type { DataPublishOptions, SimulationScenario } from './room/types';
47
+ export type { DataPublishOptions, SimulationScenario, TranscriptionSegment } from './room/types';
48
48
  export * from './version';
49
49
  export {
50
50
  ConnectionQuality,
@@ -474,8 +474,6 @@ export default class PCTransport extends EventEmitter {
474
474
  await this.pc.setLocalDescription(sd);
475
475
  }
476
476
  } catch (e) {
477
- // this error cannot always be caught.
478
- // If the local description has a setCodecPreferences error, this error will be uncaught
479
477
  let msg = 'unknown error';
480
478
  if (e instanceof Error) {
481
479
  msg = e.message;
@@ -23,6 +23,7 @@ import {
23
23
  TrackInfo,
24
24
  type TrackPublishedResponse,
25
25
  TrackUnpublishedResponse,
26
+ Transcription,
26
27
  UpdateSubscription,
27
28
  UserPacket,
28
29
  } from '@livekit/protocol';
@@ -53,22 +54,14 @@ import { EngineEvent } from './events';
53
54
  import CriticalTimers from './timers';
54
55
  import type LocalTrack from './track/LocalTrack';
55
56
  import type LocalTrackPublication from './track/LocalTrackPublication';
56
- import type LocalVideoTrack from './track/LocalVideoTrack';
57
+ import LocalVideoTrack from './track/LocalVideoTrack';
57
58
  import type { SimulcastTrackInfo } from './track/LocalVideoTrack';
58
59
  import type RemoteTrackPublication from './track/RemoteTrackPublication';
59
- import { Track } from './track/Track';
60
+ import type { Track } from './track/Track';
60
61
  import type { TrackPublishOptions, VideoCodec } from './track/options';
61
62
  import { getTrackPublicationInfo } from './track/utils';
62
63
  import type { LoggerOptions } from './types';
63
- import {
64
- Mutex,
65
- isVideoCodec,
66
- isWeb,
67
- sleep,
68
- supportsAddTrack,
69
- supportsSetCodecPreferences,
70
- supportsTransceiver,
71
- } from './utils';
64
+ import { Mutex, isVideoCodec, isWeb, sleep, supportsAddTrack, supportsTransceiver } from './utils';
72
65
 
73
66
  const lossyDataChannel = '_lossy';
74
67
  const reliableDataChannel = '_reliable';
@@ -642,6 +635,8 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
642
635
  this.emit(EngineEvent.ActiveSpeakersUpdate, dp.value.value.speakers);
643
636
  } else if (dp.value?.case === 'user') {
644
637
  this.emit(EngineEvent.DataPacketReceived, dp.value.value, dp.kind);
638
+ } else if (dp.value?.case === 'transcription') {
639
+ this.emit(EngineEvent.TranscriptionReceived, dp.value.value);
645
640
  }
646
641
  } finally {
647
642
  unlock();
@@ -671,51 +666,6 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
671
666
  this.updateAndEmitDCBufferStatus(channelKind);
672
667
  };
673
668
 
674
- private setPreferredCodec(
675
- transceiver: RTCRtpTransceiver,
676
- kind: Track.Kind,
677
- videoCodec: VideoCodec,
678
- ) {
679
- if (!('getCapabilities' in RTCRtpReceiver)) {
680
- return;
681
- }
682
- // when setting codec preferences, the capabilites need to be read from the RTCRtpReceiver
683
- const cap = RTCRtpReceiver.getCapabilities(kind);
684
- if (!cap) return;
685
- this.log.debug('get receiver capabilities', { ...this.logContext, cap });
686
- const matched: RTCRtpCodecCapability[] = [];
687
- const partialMatched: RTCRtpCodecCapability[] = [];
688
- const unmatched: RTCRtpCodecCapability[] = [];
689
- cap.codecs.forEach((c) => {
690
- const codec = c.mimeType.toLowerCase();
691
- if (codec === 'audio/opus') {
692
- matched.push(c);
693
- return;
694
- }
695
- const matchesVideoCodec = codec === `video/${videoCodec}`;
696
- if (!matchesVideoCodec) {
697
- unmatched.push(c);
698
- return;
699
- }
700
- // for h264 codecs that have sdpFmtpLine available, use only if the
701
- // profile-level-id is 42e01f for cross-browser compatibility
702
- if (videoCodec === 'h264') {
703
- if (c.sdpFmtpLine && c.sdpFmtpLine.includes('profile-level-id=42e01f')) {
704
- matched.push(c);
705
- } else {
706
- partialMatched.push(c);
707
- }
708
- return;
709
- }
710
-
711
- matched.push(c);
712
- });
713
-
714
- if (supportsSetCodecPreferences(transceiver)) {
715
- transceiver.setCodecPreferences(matched.concat(partialMatched, unmatched));
716
- }
717
- }
718
-
719
669
  async createSender(
720
670
  track: LocalTrack,
721
671
  opts: TrackPublishOptions,
@@ -766,6 +716,10 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
766
716
  streams.push(track.mediaStream);
767
717
  }
768
718
 
719
+ if (track instanceof LocalVideoTrack) {
720
+ track.codec = opts.videoCodec;
721
+ }
722
+
769
723
  const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly', streams };
770
724
  if (encodings) {
771
725
  transceiverInit.sendEncodings = encodings;
@@ -776,10 +730,6 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
776
730
  transceiverInit,
777
731
  );
778
732
 
779
- if (track.kind === Track.Kind.Video && opts.videoCodec) {
780
- this.setPreferredCodec(transceiver, track.kind, opts.videoCodec);
781
- track.codec = opts.videoCodec;
782
- }
783
733
  return transceiver.sender;
784
734
  }
785
735
 
@@ -804,7 +754,6 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
804
754
  if (!opts.videoCodec) {
805
755
  return;
806
756
  }
807
- this.setPreferredCodec(transceiver, track.kind, opts.videoCodec);
808
757
  track.setSimulcastTrackSender(opts.videoCodec, transceiver.sender);
809
758
  return transceiver.sender;
810
759
  }
@@ -1411,6 +1360,7 @@ export type EngineEventCallbacks = {
1411
1360
  ) => void;
1412
1361
  activeSpeakersUpdate: (speakers: Array<SpeakerInfo>) => void;
1413
1362
  dataPacketReceived: (userPacket: UserPacket, kind: DataPacket_Kind) => void;
1363
+ transcriptionReceived: (transcription: Transcription) => void;
1414
1364
  transportsCreated: (publisher: PCTransport, subscriber: PCTransport) => void;
1415
1365
  /** @internal */
1416
1366
  trackSenderAdded: (track: Track, sender: RTCRtpSender) => void;
package/src/room/Room.ts CHANGED
@@ -18,6 +18,8 @@ import {
18
18
  TrackInfo,
19
19
  TrackSource,
20
20
  TrackType,
21
+ Transcription as TranscriptionModel,
22
+ TranscriptionSegment as TranscriptionSegmentModel,
21
23
  UserPacket,
22
24
  protoInt64,
23
25
  } from '@livekit/protocol';
@@ -61,11 +63,12 @@ import type { TrackPublication } from './track/TrackPublication';
61
63
  import type { TrackProcessor } from './track/processor/types';
62
64
  import type { AdaptiveStreamSettings } from './track/types';
63
65
  import { getNewAudioContext, sourceToKind } from './track/utils';
64
- import type { SimulationOptions, SimulationScenario } from './types';
66
+ import type { SimulationOptions, SimulationScenario, TranscriptionSegment } from './types';
65
67
  import {
66
68
  Future,
67
69
  Mutex,
68
70
  createDummyVideoStreamTrack,
71
+ extractTranscriptionSegments,
69
72
  getEmptyAudioStreamTrack,
70
73
  isBrowserSupported,
71
74
  isCloud,
@@ -330,6 +333,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
330
333
  })
331
334
  .on(EngineEvent.ActiveSpeakersUpdate, this.handleActiveSpeakersUpdate)
332
335
  .on(EngineEvent.DataPacketReceived, this.handleDataPacket)
336
+ .on(EngineEvent.TranscriptionReceived, this.handleTranscription)
333
337
  .on(EngineEvent.Resuming, () => {
334
338
  this.clearConnectionReconcile();
335
339
  this.isResuming = true;
@@ -1471,6 +1475,23 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
1471
1475
  participant?.emit(ParticipantEvent.DataReceived, userPacket.payload, kind);
1472
1476
  };
1473
1477
 
1478
+ bufferedSegments: Map<string, TranscriptionSegmentModel> = new Map();
1479
+
1480
+ private handleTranscription = (transcription: TranscriptionModel) => {
1481
+ // find the participant
1482
+ const participant =
1483
+ transcription.participantIdentity === this.localParticipant.identity
1484
+ ? this.localParticipant
1485
+ : this.remoteParticipants.get(transcription.participantIdentity);
1486
+ const publication = participant?.trackPublications.get(transcription.trackId);
1487
+
1488
+ const segments = extractTranscriptionSegments(transcription);
1489
+
1490
+ publication?.emit(TrackEvent.TranscriptionReceived, segments);
1491
+ participant?.emit(ParticipantEvent.TranscriptionReceived, segments, publication);
1492
+ this.emit(RoomEvent.TranscriptionReceived, segments, participant, publication);
1493
+ };
1494
+
1474
1495
  private handleAudioPlaybackStarted = () => {
1475
1496
  if (this.canPlaybackAudio) {
1476
1497
  return;
@@ -2071,6 +2092,11 @@ export type RoomEventCallbacks = {
2071
2092
  kind?: DataPacket_Kind,
2072
2093
  topic?: string,
2073
2094
  ) => void;
2095
+ transcriptionReceived: (
2096
+ transcription: TranscriptionSegment[],
2097
+ participant?: Participant,
2098
+ publication?: TrackPublication,
2099
+ ) => void;
2074
2100
  connectionQualityChanged: (quality: ConnectionQuality, participant: Participant) => void;
2075
2101
  mediaDevicesError: (error: Error) => void;
2076
2102
  trackStreamStateChanged: (
@@ -197,6 +197,12 @@ export enum RoomEvent {
197
197
  */
198
198
  DataReceived = 'dataReceived',
199
199
 
200
+ /**
201
+ * Transcription received from a participant's track.
202
+ * @beta
203
+ */
204
+ TranscriptionReceived = 'transcriptionReceived',
205
+
200
206
  /**
201
207
  * Connection quality was changed for a Participant. It'll receive updates
202
208
  * from the local participant, as well as any [[RemoteParticipant]]s that we are
@@ -402,6 +408,12 @@ export enum ParticipantEvent {
402
408
  */
403
409
  DataReceived = 'dataReceived',
404
410
 
411
+ /**
412
+ * Transcription received from this participant as data source.
413
+ * @beta
414
+ */
415
+ TranscriptionReceived = 'transcriptionReceived',
416
+
405
417
  /**
406
418
  * Has speaking status changed for the current participant
407
419
  *
@@ -479,6 +491,7 @@ export enum EngineEvent {
479
491
  MediaTrackAdded = 'mediaTrackAdded',
480
492
  ActiveSpeakersUpdate = 'activeSpeakersUpdate',
481
493
  DataPacketReceived = 'dataPacketReceived',
494
+ TranscriptionReceived = 'transcriptionReceived',
482
495
  RTPVideoMapUpdate = 'rtpVideoMapUpdate',
483
496
  DCBufferStatusChanged = 'dcBufferStatusChanged',
484
497
  ParticipantUpdate = 'participantUpdate',
@@ -562,4 +575,14 @@ export enum TrackEvent {
562
575
  * @internal
563
576
  */
564
577
  AudioTrackFeatureUpdate = 'audioTrackFeatureUpdate',
578
+
579
+ /**
580
+ * @beta
581
+ */
582
+ TranscriptionReceived = 'transcriptionReceived',
583
+
584
+ /**
585
+ * @experimental
586
+ */
587
+ TimeSyncUpdate = 'timeSyncUpdate',
565
588
  }
@@ -40,9 +40,9 @@ import {
40
40
  import type { DataPublishOptions } from '../types';
41
41
  import {
42
42
  Future,
43
+ isE2EESimulcastSupported,
43
44
  isFireFox,
44
45
  isSVCCodec,
45
- isSafari,
46
46
  isSafari17,
47
47
  isWeb,
48
48
  supportsAV1,
@@ -621,10 +621,9 @@ export default class LocalParticipant extends Participant {
621
621
  ...options,
622
622
  };
623
623
 
624
- // disable simulcast if e2ee is set on safari
625
- if (isSafari() && this.roomOptions.e2ee) {
624
+ if (!isE2EESimulcastSupported() && this.roomOptions.e2ee) {
626
625
  this.log.info(
627
- `End-to-end encryption is set up, simulcast publishing will be disabled on Safari`,
626
+ `End-to-end encryption is set up, simulcast publishing will be disabled on Safari versions and iOS browsers running iOS < v17.2`,
628
627
  {
629
628
  ...this.logContext,
630
629
  },
@@ -827,7 +826,6 @@ export default class LocalParticipant extends Participant {
827
826
  ...getLogContextFromTrack(track),
828
827
  codec: updatedCodec,
829
828
  });
830
- /* @ts-ignore */
831
829
  opts.videoCodec = updatedCodec;
832
830
 
833
831
  // recompute encodings since bitrates/etc could have changed
@@ -16,7 +16,7 @@ import type RemoteTrack from '../track/RemoteTrack';
16
16
  import type RemoteTrackPublication from '../track/RemoteTrackPublication';
17
17
  import { Track } from '../track/Track';
18
18
  import type { TrackPublication } from '../track/TrackPublication';
19
- import type { LoggerOptions } from '../types';
19
+ import type { LoggerOptions, TranscriptionSegment } from '../types';
20
20
 
21
21
  export enum ConnectionQuality {
22
22
  Excellent = 'excellent',
@@ -329,6 +329,10 @@ export type ParticipantEventCallbacks = {
329
329
  participantMetadataChanged: (prevMetadata: string | undefined, participant?: any) => void;
330
330
  participantNameChanged: (name: string) => void;
331
331
  dataReceived: (payload: Uint8Array, kind: DataPacket_Kind) => void;
332
+ transcriptionReceived: (
333
+ transcription: TranscriptionSegment[],
334
+ publication?: TrackPublication,
335
+ ) => void;
332
336
  isSpeakingChanged: (speaking: boolean) => void;
333
337
  connectionQualityChanged: (connectionQuality: ConnectionQuality) => void;
334
338
  trackStreamStateChanged: (
@@ -77,7 +77,20 @@ export default abstract class RemoteTrack<
77
77
  if (!this.monitorInterval) {
78
78
  this.monitorInterval = setInterval(() => this.monitorReceiver(), monitorFrequency);
79
79
  }
80
+ this.registerTimeSyncUpdate();
80
81
  }
81
82
 
82
83
  protected abstract monitorReceiver(): void;
84
+
85
+ registerTimeSyncUpdate() {
86
+ const loop = () => {
87
+ this.timeSyncHandle = requestAnimationFrame(() => loop());
88
+ const newTime = this.receiver?.getSynchronizationSources()[0]?.rtpTimestamp;
89
+ if (newTime && this.rtpTimestamp !== newTime) {
90
+ this.emit(TrackEvent.TimeSyncUpdate, newTime);
91
+ this.rtpTimestamp = newTime;
92
+ }
93
+ };
94
+ loop();
95
+ }
83
96
  }
@@ -53,6 +53,9 @@ export abstract class Track<
53
53
  */
54
54
  streamState: Track.StreamState = Track.StreamState.Active;
55
55
 
56
+ /** @internal */
57
+ rtpTimestamp: number | undefined;
58
+
56
59
  protected _mediaStreamTrack: MediaStreamTrack;
57
60
 
58
61
  protected _mediaStreamID: string;
@@ -63,6 +66,8 @@ export abstract class Track<
63
66
 
64
67
  private loggerContextCb: LoggerOptions['loggerContextCb'];
65
68
 
69
+ protected timeSyncHandle: number | undefined;
70
+
66
71
  protected _currentBitrate: number = 0;
67
72
 
68
73
  protected monitorInterval?: ReturnType<typeof setInterval>;
@@ -255,6 +260,9 @@ export abstract class Track<
255
260
  if (this.monitorInterval) {
256
261
  clearInterval(this.monitorInterval);
257
262
  }
263
+ if (this.timeSyncHandle) {
264
+ cancelAnimationFrame(this.timeSyncHandle);
265
+ }
258
266
  }
259
267
 
260
268
  /** @internal */
@@ -517,4 +525,5 @@ export type TrackEventCallbacks = {
517
525
  upstreamResumed: (track: any) => void;
518
526
  trackProcessorUpdate: (processor?: TrackProcessor<Track.Kind, any>) => void;
519
527
  audioTrackFeatureUpdate: (track: any, feature: AudioTrackFeature, enabled: boolean) => void;
528
+ timeSyncUpdate: (timestamp: number) => void;
520
529
  };
@@ -9,7 +9,7 @@ import { EventEmitter } from 'events';
9
9
  import type TypedEventEmitter from 'typed-emitter';
10
10
  import log, { LoggerNames, getLogger } from '../../logger';
11
11
  import { TrackEvent } from '../events';
12
- import type { LoggerOptions } from '../types';
12
+ import type { LoggerOptions, TranscriptionSegment } from '../types';
13
13
  import LocalAudioTrack from './LocalAudioTrack';
14
14
  import LocalVideoTrack from './LocalVideoTrack';
15
15
  import RemoteAudioTrack from './RemoteAudioTrack';
@@ -174,4 +174,6 @@ export type PublicationEventCallbacks = {
174
174
  prevStatus: TrackPublication.SubscriptionStatus,
175
175
  ) => void;
176
176
  subscriptionFailed: (error: SubscriptionError) => void;
177
+ transcriptionReceived: (transcription: TranscriptionSegment[]) => void;
178
+ timeSyncUpdate: (timestamp: number) => void;
177
179
  };
package/src/room/types.ts CHANGED
@@ -55,3 +55,12 @@ export type LoggerOptions = {
55
55
  loggerName?: string;
56
56
  loggerContextCb?: () => Record<string, unknown>;
57
57
  };
58
+
59
+ export interface TranscriptionSegment {
60
+ id: string;
61
+ text: string;
62
+ language: string;
63
+ startTime: number;
64
+ endTime: number;
65
+ final: boolean;
66
+ }