livekit-client 2.1.0 → 2.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/dist/livekit-client.esm.mjs +2212 -2100
  2. package/dist/livekit-client.esm.mjs.map +1 -1
  3. package/dist/livekit-client.umd.js +1 -1
  4. package/dist/livekit-client.umd.js.map +1 -1
  5. package/dist/src/index.d.ts +1 -1
  6. package/dist/src/index.d.ts.map +1 -1
  7. package/dist/src/room/PCTransport.d.ts.map +1 -1
  8. package/dist/src/room/RTCEngine.d.ts +4 -4
  9. package/dist/src/room/RTCEngine.d.ts.map +1 -1
  10. package/dist/src/room/Room.d.ts +5 -2
  11. package/dist/src/room/Room.d.ts.map +1 -1
  12. package/dist/src/room/events.d.ts +20 -1
  13. package/dist/src/room/events.d.ts.map +1 -1
  14. package/dist/src/room/participant/LocalParticipant.d.ts.map +1 -1
  15. package/dist/src/room/participant/Participant.d.ts +2 -1
  16. package/dist/src/room/participant/Participant.d.ts.map +1 -1
  17. package/dist/src/room/track/RemoteTrack.d.ts +1 -0
  18. package/dist/src/room/track/RemoteTrack.d.ts.map +1 -1
  19. package/dist/src/room/track/Track.d.ts +4 -0
  20. package/dist/src/room/track/Track.d.ts.map +1 -1
  21. package/dist/src/room/track/TrackPublication.d.ts +3 -1
  22. package/dist/src/room/track/TrackPublication.d.ts.map +1 -1
  23. package/dist/src/room/types.d.ts +8 -0
  24. package/dist/src/room/types.d.ts.map +1 -1
  25. package/dist/src/room/utils.d.ts +4 -2
  26. package/dist/src/room/utils.d.ts.map +1 -1
  27. package/dist/src/utils/browserParser.d.ts +1 -0
  28. package/dist/src/utils/browserParser.d.ts.map +1 -1
  29. package/dist/ts4.2/src/index.d.ts +1 -1
  30. package/dist/ts4.2/src/room/RTCEngine.d.ts +4 -4
  31. package/dist/ts4.2/src/room/Room.d.ts +5 -2
  32. package/dist/ts4.2/src/room/events.d.ts +20 -1
  33. package/dist/ts4.2/src/room/participant/Participant.d.ts +2 -1
  34. package/dist/ts4.2/src/room/track/RemoteTrack.d.ts +1 -0
  35. package/dist/ts4.2/src/room/track/Track.d.ts +4 -0
  36. package/dist/ts4.2/src/room/track/TrackPublication.d.ts +3 -1
  37. package/dist/ts4.2/src/room/types.d.ts +8 -0
  38. package/dist/ts4.2/src/room/utils.d.ts +4 -2
  39. package/dist/ts4.2/src/utils/browserParser.d.ts +1 -0
  40. package/package.json +8 -8
  41. package/src/index.ts +1 -1
  42. package/src/room/PCTransport.ts +0 -2
  43. package/src/room/RTCEngine.ts +11 -61
  44. package/src/room/Room.ts +27 -1
  45. package/src/room/events.ts +23 -0
  46. package/src/room/participant/LocalParticipant.ts +3 -5
  47. package/src/room/participant/Participant.ts +5 -1
  48. package/src/room/track/RemoteTrack.ts +13 -0
  49. package/src/room/track/Track.ts +9 -0
  50. package/src/room/track/TrackPublication.ts +3 -1
  51. package/src/room/types.ts +9 -0
  52. package/src/room/utils.ts +46 -29
  53. package/src/utils/browserParser.test.ts +4 -0
  54. package/src/utils/browserParser.ts +11 -1
@@ -6,7 +6,7 @@ import type RemoteTrack from '../track/RemoteTrack';
6
6
  import type RemoteTrackPublication from '../track/RemoteTrackPublication';
7
7
  import { Track } from '../track/Track';
8
8
  import type { TrackPublication } from '../track/TrackPublication';
9
- import type { LoggerOptions } from '../types';
9
+ import type { LoggerOptions, TranscriptionSegment } from '../types';
10
10
  export declare enum ConnectionQuality {
11
11
  Excellent = "excellent",
12
12
  Good = "good",
@@ -99,6 +99,7 @@ export type ParticipantEventCallbacks = {
99
99
  participantMetadataChanged: (prevMetadata: string | undefined, participant?: any) => void;
100
100
  participantNameChanged: (name: string) => void;
101
101
  dataReceived: (payload: Uint8Array, kind: DataPacket_Kind) => void;
102
+ transcriptionReceived: (transcription: TranscriptionSegment[], publication?: TrackPublication) => void;
102
103
  isSpeakingChanged: (speaking: boolean) => void;
103
104
  connectionQualityChanged: (connectionQuality: ConnectionQuality) => void;
104
105
  trackStreamStateChanged: (publication: RemoteTrackPublication, streamState: Track.StreamState) => void;
@@ -19,5 +19,6 @@ export default abstract class RemoteTrack<TrackKind extends Track.Kind = Track.K
19
19
  getRTCStatsReport(): Promise<RTCStatsReport | undefined>;
20
20
  startMonitor(): void;
21
21
  protected abstract monitorReceiver(): void;
22
+ registerTimeSyncUpdate(): void;
22
23
  }
23
24
  //# sourceMappingURL=RemoteTrack.d.ts.map
@@ -28,11 +28,14 @@ export declare abstract class Track<TrackKind extends Track.Kind = Track.Kind> e
28
28
  * has been paused by congestion controller
29
29
  */
30
30
  streamState: Track.StreamState;
31
+ /** @internal */
32
+ rtpTimestamp: number | undefined;
31
33
  protected _mediaStreamTrack: MediaStreamTrack;
32
34
  protected _mediaStreamID: string;
33
35
  protected isInBackground: boolean;
34
36
  private backgroundTimeout;
35
37
  private loggerContextCb;
38
+ protected timeSyncHandle: number | undefined;
36
39
  protected _currentBitrate: number;
37
40
  protected monitorInterval?: ReturnType<typeof setInterval>;
38
41
  protected log: StructuredLogger;
@@ -137,6 +140,7 @@ export type TrackEventCallbacks = {
137
140
  upstreamResumed: (track: any) => void;
138
141
  trackProcessorUpdate: (processor?: TrackProcessor<Track.Kind, any>) => void;
139
142
  audioTrackFeatureUpdate: (track: any, feature: AudioTrackFeature, enabled: boolean) => void;
143
+ timeSyncUpdate: (timestamp: number) => void;
140
144
  };
141
145
  export {};
142
146
  //# sourceMappingURL=Track.d.ts.map
@@ -1,7 +1,7 @@
1
1
  import { Encryption_Type } from '@livekit/protocol';
2
2
  import type { SubscriptionError, TrackInfo, UpdateSubscription, UpdateTrackSettings } from '@livekit/protocol';
3
3
  import type TypedEventEmitter from 'typed-emitter';
4
- import type { LoggerOptions } from '../types';
4
+ import type { LoggerOptions, TranscriptionSegment } from '../types';
5
5
  import LocalAudioTrack from './LocalAudioTrack';
6
6
  import LocalVideoTrack from './LocalVideoTrack';
7
7
  import RemoteAudioTrack from './RemoteAudioTrack';
@@ -72,6 +72,8 @@ export type PublicationEventCallbacks = {
72
72
  unsubscribed: (track: RemoteTrack) => void;
73
73
  subscriptionStatusChanged: (status: TrackPublication.SubscriptionStatus, prevStatus: TrackPublication.SubscriptionStatus) => void;
74
74
  subscriptionFailed: (error: SubscriptionError) => void;
75
+ transcriptionReceived: (transcription: TranscriptionSegment[]) => void;
76
+ timeSyncUpdate: (timestamp: number) => void;
75
77
  };
76
78
  export {};
77
79
  //# sourceMappingURL=TrackPublication.d.ts.map
@@ -35,4 +35,12 @@ export type LoggerOptions = {
35
35
  loggerName?: string;
36
36
  loggerContextCb?: () => Record<string, unknown>;
37
37
  };
38
+ export interface TranscriptionSegment {
39
+ id: string;
40
+ text: string;
41
+ language: string;
42
+ startTime: number;
43
+ endTime: number;
44
+ final: boolean;
45
+ }
38
46
  //# sourceMappingURL=types.d.ts.map
@@ -1,7 +1,8 @@
1
- import { ClientInfo } from '@livekit/protocol';
1
+ import { ClientInfo, Transcription as TranscriptionModel } from '@livekit/protocol';
2
2
  import type LocalAudioTrack from './track/LocalAudioTrack';
3
3
  import type RemoteAudioTrack from './track/RemoteAudioTrack';
4
4
  import { VideoCodec } from './track/options';
5
+ import type { TranscriptionSegment } from './types';
5
6
  export declare const ddExtensionURI = "https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension";
6
7
  export declare function unpackStreamId(packed: string): string[];
7
8
  export declare function sleep(duration: number): Promise<void>;
@@ -15,13 +16,13 @@ export declare function supportsAV1(): boolean;
15
16
  export declare function supportsVP9(): boolean;
16
17
  export declare function isSVCCodec(codec?: string): boolean;
17
18
  export declare function supportsSetSinkId(elm?: HTMLMediaElement): boolean;
18
- export declare function supportsSetCodecPreferences(transceiver: RTCRtpTransceiver): boolean;
19
19
  export declare function isBrowserSupported(): boolean;
20
20
  export declare function isFireFox(): boolean;
21
21
  export declare function isChromiumBased(): boolean;
22
22
  export declare function isSafari(): boolean;
23
23
  export declare function isSafari17(): boolean;
24
24
  export declare function isMobile(): boolean;
25
+ export declare function isE2EESimulcastSupported(): boolean | undefined;
25
26
  export declare function isWeb(): boolean;
26
27
  export declare function isReactNative(): boolean;
27
28
  export declare function isCloud(serverUrl: URL): boolean;
@@ -93,4 +94,5 @@ export declare function isVideoCodec(maybeCodec: string): maybeCodec is VideoCod
93
94
  export declare function unwrapConstraint(constraint: ConstrainDOMString): string;
94
95
  export declare function toWebsocketUrl(url: string): string;
95
96
  export declare function toHttpUrl(url: string): string;
97
+ export declare function extractTranscriptionSegments(transcription: TranscriptionModel): TranscriptionSegment[];
96
98
  //# sourceMappingURL=utils.d.ts.map
@@ -4,6 +4,7 @@ export type BrowserDetails = {
4
4
  name: DetectableBrowser;
5
5
  version: string;
6
6
  os?: DetectableOS;
7
+ osVersion?: string;
7
8
  };
8
9
  /**
9
10
  * @internal
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "livekit-client",
3
- "version": "2.1.0",
3
+ "version": "2.1.2",
4
4
  "description": "JavaScript/TypeScript client SDK for LiveKit",
5
5
  "main": "./dist/livekit-client.umd.js",
6
6
  "unpkg": "./dist/livekit-client.umd.js",
@@ -36,7 +36,7 @@
36
36
  "author": "David Zhao <david@davidzhao.com>",
37
37
  "license": "Apache-2.0",
38
38
  "dependencies": {
39
- "@livekit/protocol": "1.13.0",
39
+ "@livekit/protocol": "1.15.0",
40
40
  "events": "^3.3.0",
41
41
  "loglevel": "^1.8.0",
42
42
  "sdp-transform": "^2.14.1",
@@ -46,8 +46,8 @@
46
46
  "webrtc-adapter": "^8.1.1"
47
47
  },
48
48
  "devDependencies": {
49
- "@babel/core": "7.24.4",
50
- "@babel/preset-env": "7.24.4",
49
+ "@babel/core": "7.24.5",
50
+ "@babel/preset-env": "7.24.5",
51
51
  "@bufbuild/protoc-gen-es": "^1.3.0",
52
52
  "@changesets/cli": "2.27.1",
53
53
  "@livekit/changesets-changelog-github": "^0.0.4",
@@ -73,15 +73,15 @@
73
73
  "gh-pages": "6.1.1",
74
74
  "jsdom": "^24.0.0",
75
75
  "prettier": "^3.0.0",
76
- "rollup": "4.14.0",
76
+ "rollup": "4.17.2",
77
77
  "rollup-plugin-delete": "^2.0.0",
78
78
  "rollup-plugin-re": "1.0.7",
79
79
  "rollup-plugin-typescript2": "0.36.0",
80
80
  "size-limit": "^8.2.4",
81
- "typedoc": "0.25.12",
81
+ "typedoc": "0.25.13",
82
82
  "typedoc-plugin-no-inherit": "1.4.0",
83
- "typescript": "5.4.3",
84
- "vite": "5.0.13",
83
+ "typescript": "5.4.5",
84
+ "vite": "5.2.10",
85
85
  "vitest": "^1.0.0"
86
86
  },
87
87
  "scripts": {
package/src/index.ts CHANGED
@@ -44,7 +44,7 @@ export { facingModeFromDeviceLabel, facingModeFromLocalTrack } from './room/trac
44
44
  export * from './room/track/options';
45
45
  export * from './room/track/processor/types';
46
46
  export * from './room/track/types';
47
- export type { DataPublishOptions, SimulationScenario } from './room/types';
47
+ export type { DataPublishOptions, SimulationScenario, TranscriptionSegment } from './room/types';
48
48
  export * from './version';
49
49
  export {
50
50
  ConnectionQuality,
@@ -474,8 +474,6 @@ export default class PCTransport extends EventEmitter {
474
474
  await this.pc.setLocalDescription(sd);
475
475
  }
476
476
  } catch (e) {
477
- // this error cannot always be caught.
478
- // If the local description has a setCodecPreferences error, this error will be uncaught
479
477
  let msg = 'unknown error';
480
478
  if (e instanceof Error) {
481
479
  msg = e.message;
@@ -23,6 +23,7 @@ import {
23
23
  TrackInfo,
24
24
  type TrackPublishedResponse,
25
25
  TrackUnpublishedResponse,
26
+ Transcription,
26
27
  UpdateSubscription,
27
28
  UserPacket,
28
29
  } from '@livekit/protocol';
@@ -53,22 +54,14 @@ import { EngineEvent } from './events';
53
54
  import CriticalTimers from './timers';
54
55
  import type LocalTrack from './track/LocalTrack';
55
56
  import type LocalTrackPublication from './track/LocalTrackPublication';
56
- import type LocalVideoTrack from './track/LocalVideoTrack';
57
+ import LocalVideoTrack from './track/LocalVideoTrack';
57
58
  import type { SimulcastTrackInfo } from './track/LocalVideoTrack';
58
59
  import type RemoteTrackPublication from './track/RemoteTrackPublication';
59
- import { Track } from './track/Track';
60
+ import type { Track } from './track/Track';
60
61
  import type { TrackPublishOptions, VideoCodec } from './track/options';
61
62
  import { getTrackPublicationInfo } from './track/utils';
62
63
  import type { LoggerOptions } from './types';
63
- import {
64
- Mutex,
65
- isVideoCodec,
66
- isWeb,
67
- sleep,
68
- supportsAddTrack,
69
- supportsSetCodecPreferences,
70
- supportsTransceiver,
71
- } from './utils';
64
+ import { Mutex, isVideoCodec, isWeb, sleep, supportsAddTrack, supportsTransceiver } from './utils';
72
65
 
73
66
  const lossyDataChannel = '_lossy';
74
67
  const reliableDataChannel = '_reliable';
@@ -642,6 +635,8 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
642
635
  this.emit(EngineEvent.ActiveSpeakersUpdate, dp.value.value.speakers);
643
636
  } else if (dp.value?.case === 'user') {
644
637
  this.emit(EngineEvent.DataPacketReceived, dp.value.value, dp.kind);
638
+ } else if (dp.value?.case === 'transcription') {
639
+ this.emit(EngineEvent.TranscriptionReceived, dp.value.value);
645
640
  }
646
641
  } finally {
647
642
  unlock();
@@ -671,51 +666,6 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
671
666
  this.updateAndEmitDCBufferStatus(channelKind);
672
667
  };
673
668
 
674
- private setPreferredCodec(
675
- transceiver: RTCRtpTransceiver,
676
- kind: Track.Kind,
677
- videoCodec: VideoCodec,
678
- ) {
679
- if (!('getCapabilities' in RTCRtpReceiver)) {
680
- return;
681
- }
682
- // when setting codec preferences, the capabilites need to be read from the RTCRtpReceiver
683
- const cap = RTCRtpReceiver.getCapabilities(kind);
684
- if (!cap) return;
685
- this.log.debug('get receiver capabilities', { ...this.logContext, cap });
686
- const matched: RTCRtpCodecCapability[] = [];
687
- const partialMatched: RTCRtpCodecCapability[] = [];
688
- const unmatched: RTCRtpCodecCapability[] = [];
689
- cap.codecs.forEach((c) => {
690
- const codec = c.mimeType.toLowerCase();
691
- if (codec === 'audio/opus') {
692
- matched.push(c);
693
- return;
694
- }
695
- const matchesVideoCodec = codec === `video/${videoCodec}`;
696
- if (!matchesVideoCodec) {
697
- unmatched.push(c);
698
- return;
699
- }
700
- // for h264 codecs that have sdpFmtpLine available, use only if the
701
- // profile-level-id is 42e01f for cross-browser compatibility
702
- if (videoCodec === 'h264') {
703
- if (c.sdpFmtpLine && c.sdpFmtpLine.includes('profile-level-id=42e01f')) {
704
- matched.push(c);
705
- } else {
706
- partialMatched.push(c);
707
- }
708
- return;
709
- }
710
-
711
- matched.push(c);
712
- });
713
-
714
- if (supportsSetCodecPreferences(transceiver)) {
715
- transceiver.setCodecPreferences(matched.concat(partialMatched, unmatched));
716
- }
717
- }
718
-
719
669
  async createSender(
720
670
  track: LocalTrack,
721
671
  opts: TrackPublishOptions,
@@ -766,6 +716,10 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
766
716
  streams.push(track.mediaStream);
767
717
  }
768
718
 
719
+ if (track instanceof LocalVideoTrack) {
720
+ track.codec = opts.videoCodec;
721
+ }
722
+
769
723
  const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly', streams };
770
724
  if (encodings) {
771
725
  transceiverInit.sendEncodings = encodings;
@@ -776,10 +730,6 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
776
730
  transceiverInit,
777
731
  );
778
732
 
779
- if (track.kind === Track.Kind.Video && opts.videoCodec) {
780
- this.setPreferredCodec(transceiver, track.kind, opts.videoCodec);
781
- track.codec = opts.videoCodec;
782
- }
783
733
  return transceiver.sender;
784
734
  }
785
735
 
@@ -804,7 +754,6 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
804
754
  if (!opts.videoCodec) {
805
755
  return;
806
756
  }
807
- this.setPreferredCodec(transceiver, track.kind, opts.videoCodec);
808
757
  track.setSimulcastTrackSender(opts.videoCodec, transceiver.sender);
809
758
  return transceiver.sender;
810
759
  }
@@ -1411,6 +1360,7 @@ export type EngineEventCallbacks = {
1411
1360
  ) => void;
1412
1361
  activeSpeakersUpdate: (speakers: Array<SpeakerInfo>) => void;
1413
1362
  dataPacketReceived: (userPacket: UserPacket, kind: DataPacket_Kind) => void;
1363
+ transcriptionReceived: (transcription: Transcription) => void;
1414
1364
  transportsCreated: (publisher: PCTransport, subscriber: PCTransport) => void;
1415
1365
  /** @internal */
1416
1366
  trackSenderAdded: (track: Track, sender: RTCRtpSender) => void;
package/src/room/Room.ts CHANGED
@@ -18,6 +18,8 @@ import {
18
18
  TrackInfo,
19
19
  TrackSource,
20
20
  TrackType,
21
+ Transcription as TranscriptionModel,
22
+ TranscriptionSegment as TranscriptionSegmentModel,
21
23
  UserPacket,
22
24
  protoInt64,
23
25
  } from '@livekit/protocol';
@@ -61,11 +63,12 @@ import type { TrackPublication } from './track/TrackPublication';
61
63
  import type { TrackProcessor } from './track/processor/types';
62
64
  import type { AdaptiveStreamSettings } from './track/types';
63
65
  import { getNewAudioContext, sourceToKind } from './track/utils';
64
- import type { SimulationOptions, SimulationScenario } from './types';
66
+ import type { SimulationOptions, SimulationScenario, TranscriptionSegment } from './types';
65
67
  import {
66
68
  Future,
67
69
  Mutex,
68
70
  createDummyVideoStreamTrack,
71
+ extractTranscriptionSegments,
69
72
  getEmptyAudioStreamTrack,
70
73
  isBrowserSupported,
71
74
  isCloud,
@@ -330,6 +333,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
330
333
  })
331
334
  .on(EngineEvent.ActiveSpeakersUpdate, this.handleActiveSpeakersUpdate)
332
335
  .on(EngineEvent.DataPacketReceived, this.handleDataPacket)
336
+ .on(EngineEvent.TranscriptionReceived, this.handleTranscription)
333
337
  .on(EngineEvent.Resuming, () => {
334
338
  this.clearConnectionReconcile();
335
339
  this.isResuming = true;
@@ -1471,6 +1475,23 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
1471
1475
  participant?.emit(ParticipantEvent.DataReceived, userPacket.payload, kind);
1472
1476
  };
1473
1477
 
1478
+ bufferedSegments: Map<string, TranscriptionSegmentModel> = new Map();
1479
+
1480
+ private handleTranscription = (transcription: TranscriptionModel) => {
1481
+ // find the participant
1482
+ const participant =
1483
+ transcription.participantIdentity === this.localParticipant.identity
1484
+ ? this.localParticipant
1485
+ : this.remoteParticipants.get(transcription.participantIdentity);
1486
+ const publication = participant?.trackPublications.get(transcription.trackId);
1487
+
1488
+ const segments = extractTranscriptionSegments(transcription);
1489
+
1490
+ publication?.emit(TrackEvent.TranscriptionReceived, segments);
1491
+ participant?.emit(ParticipantEvent.TranscriptionReceived, segments, publication);
1492
+ this.emit(RoomEvent.TranscriptionReceived, segments, participant, publication);
1493
+ };
1494
+
1474
1495
  private handleAudioPlaybackStarted = () => {
1475
1496
  if (this.canPlaybackAudio) {
1476
1497
  return;
@@ -2071,6 +2092,11 @@ export type RoomEventCallbacks = {
2071
2092
  kind?: DataPacket_Kind,
2072
2093
  topic?: string,
2073
2094
  ) => void;
2095
+ transcriptionReceived: (
2096
+ transcription: TranscriptionSegment[],
2097
+ participant?: Participant,
2098
+ publication?: TrackPublication,
2099
+ ) => void;
2074
2100
  connectionQualityChanged: (quality: ConnectionQuality, participant: Participant) => void;
2075
2101
  mediaDevicesError: (error: Error) => void;
2076
2102
  trackStreamStateChanged: (
@@ -197,6 +197,12 @@ export enum RoomEvent {
197
197
  */
198
198
  DataReceived = 'dataReceived',
199
199
 
200
+ /**
201
+ * Transcription received from a participant's track.
202
+ * @beta
203
+ */
204
+ TranscriptionReceived = 'transcriptionReceived',
205
+
200
206
  /**
201
207
  * Connection quality was changed for a Participant. It'll receive updates
202
208
  * from the local participant, as well as any [[RemoteParticipant]]s that we are
@@ -402,6 +408,12 @@ export enum ParticipantEvent {
402
408
  */
403
409
  DataReceived = 'dataReceived',
404
410
 
411
+ /**
412
+ * Transcription received from this participant as data source.
413
+ * @beta
414
+ */
415
+ TranscriptionReceived = 'transcriptionReceived',
416
+
405
417
  /**
406
418
  * Has speaking status changed for the current participant
407
419
  *
@@ -479,6 +491,7 @@ export enum EngineEvent {
479
491
  MediaTrackAdded = 'mediaTrackAdded',
480
492
  ActiveSpeakersUpdate = 'activeSpeakersUpdate',
481
493
  DataPacketReceived = 'dataPacketReceived',
494
+ TranscriptionReceived = 'transcriptionReceived',
482
495
  RTPVideoMapUpdate = 'rtpVideoMapUpdate',
483
496
  DCBufferStatusChanged = 'dcBufferStatusChanged',
484
497
  ParticipantUpdate = 'participantUpdate',
@@ -562,4 +575,14 @@ export enum TrackEvent {
562
575
  * @internal
563
576
  */
564
577
  AudioTrackFeatureUpdate = 'audioTrackFeatureUpdate',
578
+
579
+ /**
580
+ * @beta
581
+ */
582
+ TranscriptionReceived = 'transcriptionReceived',
583
+
584
+ /**
585
+ * @experimental
586
+ */
587
+ TimeSyncUpdate = 'timeSyncUpdate',
565
588
  }
@@ -40,9 +40,9 @@ import {
40
40
  import type { DataPublishOptions } from '../types';
41
41
  import {
42
42
  Future,
43
+ isE2EESimulcastSupported,
43
44
  isFireFox,
44
45
  isSVCCodec,
45
- isSafari,
46
46
  isSafari17,
47
47
  isWeb,
48
48
  supportsAV1,
@@ -621,10 +621,9 @@ export default class LocalParticipant extends Participant {
621
621
  ...options,
622
622
  };
623
623
 
624
- // disable simulcast if e2ee is set on safari
625
- if (isSafari() && this.roomOptions.e2ee) {
624
+ if (!isE2EESimulcastSupported() && this.roomOptions.e2ee) {
626
625
  this.log.info(
627
- `End-to-end encryption is set up, simulcast publishing will be disabled on Safari`,
626
+ `End-to-end encryption is set up, simulcast publishing will be disabled on Safari versions and iOS browsers running iOS < v17.2`,
628
627
  {
629
628
  ...this.logContext,
630
629
  },
@@ -827,7 +826,6 @@ export default class LocalParticipant extends Participant {
827
826
  ...getLogContextFromTrack(track),
828
827
  codec: updatedCodec,
829
828
  });
830
- /* @ts-ignore */
831
829
  opts.videoCodec = updatedCodec;
832
830
 
833
831
  // recompute encodings since bitrates/etc could have changed
@@ -16,7 +16,7 @@ import type RemoteTrack from '../track/RemoteTrack';
16
16
  import type RemoteTrackPublication from '../track/RemoteTrackPublication';
17
17
  import { Track } from '../track/Track';
18
18
  import type { TrackPublication } from '../track/TrackPublication';
19
- import type { LoggerOptions } from '../types';
19
+ import type { LoggerOptions, TranscriptionSegment } from '../types';
20
20
 
21
21
  export enum ConnectionQuality {
22
22
  Excellent = 'excellent',
@@ -329,6 +329,10 @@ export type ParticipantEventCallbacks = {
329
329
  participantMetadataChanged: (prevMetadata: string | undefined, participant?: any) => void;
330
330
  participantNameChanged: (name: string) => void;
331
331
  dataReceived: (payload: Uint8Array, kind: DataPacket_Kind) => void;
332
+ transcriptionReceived: (
333
+ transcription: TranscriptionSegment[],
334
+ publication?: TrackPublication,
335
+ ) => void;
332
336
  isSpeakingChanged: (speaking: boolean) => void;
333
337
  connectionQualityChanged: (connectionQuality: ConnectionQuality) => void;
334
338
  trackStreamStateChanged: (
@@ -77,7 +77,20 @@ export default abstract class RemoteTrack<
77
77
  if (!this.monitorInterval) {
78
78
  this.monitorInterval = setInterval(() => this.monitorReceiver(), monitorFrequency);
79
79
  }
80
+ this.registerTimeSyncUpdate();
80
81
  }
81
82
 
82
83
  protected abstract monitorReceiver(): void;
84
+
85
+ registerTimeSyncUpdate() {
86
+ const loop = () => {
87
+ this.timeSyncHandle = requestAnimationFrame(() => loop());
88
+ const newTime = this.receiver?.getSynchronizationSources()[0]?.rtpTimestamp;
89
+ if (newTime && this.rtpTimestamp !== newTime) {
90
+ this.emit(TrackEvent.TimeSyncUpdate, newTime);
91
+ this.rtpTimestamp = newTime;
92
+ }
93
+ };
94
+ loop();
95
+ }
83
96
  }
@@ -53,6 +53,9 @@ export abstract class Track<
53
53
  */
54
54
  streamState: Track.StreamState = Track.StreamState.Active;
55
55
 
56
+ /** @internal */
57
+ rtpTimestamp: number | undefined;
58
+
56
59
  protected _mediaStreamTrack: MediaStreamTrack;
57
60
 
58
61
  protected _mediaStreamID: string;
@@ -63,6 +66,8 @@ export abstract class Track<
63
66
 
64
67
  private loggerContextCb: LoggerOptions['loggerContextCb'];
65
68
 
69
+ protected timeSyncHandle: number | undefined;
70
+
66
71
  protected _currentBitrate: number = 0;
67
72
 
68
73
  protected monitorInterval?: ReturnType<typeof setInterval>;
@@ -255,6 +260,9 @@ export abstract class Track<
255
260
  if (this.monitorInterval) {
256
261
  clearInterval(this.monitorInterval);
257
262
  }
263
+ if (this.timeSyncHandle) {
264
+ cancelAnimationFrame(this.timeSyncHandle);
265
+ }
258
266
  }
259
267
 
260
268
  /** @internal */
@@ -517,4 +525,5 @@ export type TrackEventCallbacks = {
517
525
  upstreamResumed: (track: any) => void;
518
526
  trackProcessorUpdate: (processor?: TrackProcessor<Track.Kind, any>) => void;
519
527
  audioTrackFeatureUpdate: (track: any, feature: AudioTrackFeature, enabled: boolean) => void;
528
+ timeSyncUpdate: (timestamp: number) => void;
520
529
  };
@@ -9,7 +9,7 @@ import { EventEmitter } from 'events';
9
9
  import type TypedEventEmitter from 'typed-emitter';
10
10
  import log, { LoggerNames, getLogger } from '../../logger';
11
11
  import { TrackEvent } from '../events';
12
- import type { LoggerOptions } from '../types';
12
+ import type { LoggerOptions, TranscriptionSegment } from '../types';
13
13
  import LocalAudioTrack from './LocalAudioTrack';
14
14
  import LocalVideoTrack from './LocalVideoTrack';
15
15
  import RemoteAudioTrack from './RemoteAudioTrack';
@@ -174,4 +174,6 @@ export type PublicationEventCallbacks = {
174
174
  prevStatus: TrackPublication.SubscriptionStatus,
175
175
  ) => void;
176
176
  subscriptionFailed: (error: SubscriptionError) => void;
177
+ transcriptionReceived: (transcription: TranscriptionSegment[]) => void;
178
+ timeSyncUpdate: (timestamp: number) => void;
177
179
  };
package/src/room/types.ts CHANGED
@@ -55,3 +55,12 @@ export type LoggerOptions = {
55
55
  loggerName?: string;
56
56
  loggerContextCb?: () => Record<string, unknown>;
57
57
  };
58
+
59
+ export interface TranscriptionSegment {
60
+ id: string;
61
+ text: string;
62
+ language: string;
63
+ startTime: number;
64
+ endTime: number;
65
+ final: boolean;
66
+ }