livekit-client 2.1.0 → 2.1.2
Sign up to get free protection for your applications and to get access to all the features.
- package/dist/livekit-client.esm.mjs +2212 -2100
- package/dist/livekit-client.esm.mjs.map +1 -1
- package/dist/livekit-client.umd.js +1 -1
- package/dist/livekit-client.umd.js.map +1 -1
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/room/PCTransport.d.ts.map +1 -1
- package/dist/src/room/RTCEngine.d.ts +4 -4
- package/dist/src/room/RTCEngine.d.ts.map +1 -1
- package/dist/src/room/Room.d.ts +5 -2
- package/dist/src/room/Room.d.ts.map +1 -1
- package/dist/src/room/events.d.ts +20 -1
- package/dist/src/room/events.d.ts.map +1 -1
- package/dist/src/room/participant/LocalParticipant.d.ts.map +1 -1
- package/dist/src/room/participant/Participant.d.ts +2 -1
- package/dist/src/room/participant/Participant.d.ts.map +1 -1
- package/dist/src/room/track/RemoteTrack.d.ts +1 -0
- package/dist/src/room/track/RemoteTrack.d.ts.map +1 -1
- package/dist/src/room/track/Track.d.ts +4 -0
- package/dist/src/room/track/Track.d.ts.map +1 -1
- package/dist/src/room/track/TrackPublication.d.ts +3 -1
- package/dist/src/room/track/TrackPublication.d.ts.map +1 -1
- package/dist/src/room/types.d.ts +8 -0
- package/dist/src/room/types.d.ts.map +1 -1
- package/dist/src/room/utils.d.ts +4 -2
- package/dist/src/room/utils.d.ts.map +1 -1
- package/dist/src/utils/browserParser.d.ts +1 -0
- package/dist/src/utils/browserParser.d.ts.map +1 -1
- package/dist/ts4.2/src/index.d.ts +1 -1
- package/dist/ts4.2/src/room/RTCEngine.d.ts +4 -4
- package/dist/ts4.2/src/room/Room.d.ts +5 -2
- package/dist/ts4.2/src/room/events.d.ts +20 -1
- package/dist/ts4.2/src/room/participant/Participant.d.ts +2 -1
- package/dist/ts4.2/src/room/track/RemoteTrack.d.ts +1 -0
- package/dist/ts4.2/src/room/track/Track.d.ts +4 -0
- package/dist/ts4.2/src/room/track/TrackPublication.d.ts +3 -1
- package/dist/ts4.2/src/room/types.d.ts +8 -0
- package/dist/ts4.2/src/room/utils.d.ts +4 -2
- package/dist/ts4.2/src/utils/browserParser.d.ts +1 -0
- package/package.json +8 -8
- package/src/index.ts +1 -1
- package/src/room/PCTransport.ts +0 -2
- package/src/room/RTCEngine.ts +11 -61
- package/src/room/Room.ts +27 -1
- package/src/room/events.ts +23 -0
- package/src/room/participant/LocalParticipant.ts +3 -5
- package/src/room/participant/Participant.ts +5 -1
- package/src/room/track/RemoteTrack.ts +13 -0
- package/src/room/track/Track.ts +9 -0
- package/src/room/track/TrackPublication.ts +3 -1
- package/src/room/types.ts +9 -0
- package/src/room/utils.ts +46 -29
- package/src/utils/browserParser.test.ts +4 -0
- package/src/utils/browserParser.ts +11 -1
@@ -6,7 +6,7 @@ import type RemoteTrack from '../track/RemoteTrack';
|
|
6
6
|
import type RemoteTrackPublication from '../track/RemoteTrackPublication';
|
7
7
|
import { Track } from '../track/Track';
|
8
8
|
import type { TrackPublication } from '../track/TrackPublication';
|
9
|
-
import type { LoggerOptions } from '../types';
|
9
|
+
import type { LoggerOptions, TranscriptionSegment } from '../types';
|
10
10
|
export declare enum ConnectionQuality {
|
11
11
|
Excellent = "excellent",
|
12
12
|
Good = "good",
|
@@ -99,6 +99,7 @@ export type ParticipantEventCallbacks = {
|
|
99
99
|
participantMetadataChanged: (prevMetadata: string | undefined, participant?: any) => void;
|
100
100
|
participantNameChanged: (name: string) => void;
|
101
101
|
dataReceived: (payload: Uint8Array, kind: DataPacket_Kind) => void;
|
102
|
+
transcriptionReceived: (transcription: TranscriptionSegment[], publication?: TrackPublication) => void;
|
102
103
|
isSpeakingChanged: (speaking: boolean) => void;
|
103
104
|
connectionQualityChanged: (connectionQuality: ConnectionQuality) => void;
|
104
105
|
trackStreamStateChanged: (publication: RemoteTrackPublication, streamState: Track.StreamState) => void;
|
@@ -19,5 +19,6 @@ export default abstract class RemoteTrack<TrackKind extends Track.Kind = Track.K
|
|
19
19
|
getRTCStatsReport(): Promise<RTCStatsReport | undefined>;
|
20
20
|
startMonitor(): void;
|
21
21
|
protected abstract monitorReceiver(): void;
|
22
|
+
registerTimeSyncUpdate(): void;
|
22
23
|
}
|
23
24
|
//# sourceMappingURL=RemoteTrack.d.ts.map
|
@@ -28,11 +28,14 @@ export declare abstract class Track<TrackKind extends Track.Kind = Track.Kind> e
|
|
28
28
|
* has been paused by congestion controller
|
29
29
|
*/
|
30
30
|
streamState: Track.StreamState;
|
31
|
+
/** @internal */
|
32
|
+
rtpTimestamp: number | undefined;
|
31
33
|
protected _mediaStreamTrack: MediaStreamTrack;
|
32
34
|
protected _mediaStreamID: string;
|
33
35
|
protected isInBackground: boolean;
|
34
36
|
private backgroundTimeout;
|
35
37
|
private loggerContextCb;
|
38
|
+
protected timeSyncHandle: number | undefined;
|
36
39
|
protected _currentBitrate: number;
|
37
40
|
protected monitorInterval?: ReturnType<typeof setInterval>;
|
38
41
|
protected log: StructuredLogger;
|
@@ -137,6 +140,7 @@ export type TrackEventCallbacks = {
|
|
137
140
|
upstreamResumed: (track: any) => void;
|
138
141
|
trackProcessorUpdate: (processor?: TrackProcessor<Track.Kind, any>) => void;
|
139
142
|
audioTrackFeatureUpdate: (track: any, feature: AudioTrackFeature, enabled: boolean) => void;
|
143
|
+
timeSyncUpdate: (timestamp: number) => void;
|
140
144
|
};
|
141
145
|
export {};
|
142
146
|
//# sourceMappingURL=Track.d.ts.map
|
@@ -1,7 +1,7 @@
|
|
1
1
|
import { Encryption_Type } from '@livekit/protocol';
|
2
2
|
import type { SubscriptionError, TrackInfo, UpdateSubscription, UpdateTrackSettings } from '@livekit/protocol';
|
3
3
|
import type TypedEventEmitter from 'typed-emitter';
|
4
|
-
import type { LoggerOptions } from '../types';
|
4
|
+
import type { LoggerOptions, TranscriptionSegment } from '../types';
|
5
5
|
import LocalAudioTrack from './LocalAudioTrack';
|
6
6
|
import LocalVideoTrack from './LocalVideoTrack';
|
7
7
|
import RemoteAudioTrack from './RemoteAudioTrack';
|
@@ -72,6 +72,8 @@ export type PublicationEventCallbacks = {
|
|
72
72
|
unsubscribed: (track: RemoteTrack) => void;
|
73
73
|
subscriptionStatusChanged: (status: TrackPublication.SubscriptionStatus, prevStatus: TrackPublication.SubscriptionStatus) => void;
|
74
74
|
subscriptionFailed: (error: SubscriptionError) => void;
|
75
|
+
transcriptionReceived: (transcription: TranscriptionSegment[]) => void;
|
76
|
+
timeSyncUpdate: (timestamp: number) => void;
|
75
77
|
};
|
76
78
|
export {};
|
77
79
|
//# sourceMappingURL=TrackPublication.d.ts.map
|
@@ -35,4 +35,12 @@ export type LoggerOptions = {
|
|
35
35
|
loggerName?: string;
|
36
36
|
loggerContextCb?: () => Record<string, unknown>;
|
37
37
|
};
|
38
|
+
export interface TranscriptionSegment {
|
39
|
+
id: string;
|
40
|
+
text: string;
|
41
|
+
language: string;
|
42
|
+
startTime: number;
|
43
|
+
endTime: number;
|
44
|
+
final: boolean;
|
45
|
+
}
|
38
46
|
//# sourceMappingURL=types.d.ts.map
|
@@ -1,7 +1,8 @@
|
|
1
|
-
import { ClientInfo } from '@livekit/protocol';
|
1
|
+
import { ClientInfo, Transcription as TranscriptionModel } from '@livekit/protocol';
|
2
2
|
import type LocalAudioTrack from './track/LocalAudioTrack';
|
3
3
|
import type RemoteAudioTrack from './track/RemoteAudioTrack';
|
4
4
|
import { VideoCodec } from './track/options';
|
5
|
+
import type { TranscriptionSegment } from './types';
|
5
6
|
export declare const ddExtensionURI = "https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension";
|
6
7
|
export declare function unpackStreamId(packed: string): string[];
|
7
8
|
export declare function sleep(duration: number): Promise<void>;
|
@@ -15,13 +16,13 @@ export declare function supportsAV1(): boolean;
|
|
15
16
|
export declare function supportsVP9(): boolean;
|
16
17
|
export declare function isSVCCodec(codec?: string): boolean;
|
17
18
|
export declare function supportsSetSinkId(elm?: HTMLMediaElement): boolean;
|
18
|
-
export declare function supportsSetCodecPreferences(transceiver: RTCRtpTransceiver): boolean;
|
19
19
|
export declare function isBrowserSupported(): boolean;
|
20
20
|
export declare function isFireFox(): boolean;
|
21
21
|
export declare function isChromiumBased(): boolean;
|
22
22
|
export declare function isSafari(): boolean;
|
23
23
|
export declare function isSafari17(): boolean;
|
24
24
|
export declare function isMobile(): boolean;
|
25
|
+
export declare function isE2EESimulcastSupported(): boolean | undefined;
|
25
26
|
export declare function isWeb(): boolean;
|
26
27
|
export declare function isReactNative(): boolean;
|
27
28
|
export declare function isCloud(serverUrl: URL): boolean;
|
@@ -93,4 +94,5 @@ export declare function isVideoCodec(maybeCodec: string): maybeCodec is VideoCod
|
|
93
94
|
export declare function unwrapConstraint(constraint: ConstrainDOMString): string;
|
94
95
|
export declare function toWebsocketUrl(url: string): string;
|
95
96
|
export declare function toHttpUrl(url: string): string;
|
97
|
+
export declare function extractTranscriptionSegments(transcription: TranscriptionModel): TranscriptionSegment[];
|
96
98
|
//# sourceMappingURL=utils.d.ts.map
|
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "livekit-client",
|
3
|
-
"version": "2.1.
|
3
|
+
"version": "2.1.2",
|
4
4
|
"description": "JavaScript/TypeScript client SDK for LiveKit",
|
5
5
|
"main": "./dist/livekit-client.umd.js",
|
6
6
|
"unpkg": "./dist/livekit-client.umd.js",
|
@@ -36,7 +36,7 @@
|
|
36
36
|
"author": "David Zhao <david@davidzhao.com>",
|
37
37
|
"license": "Apache-2.0",
|
38
38
|
"dependencies": {
|
39
|
-
"@livekit/protocol": "1.
|
39
|
+
"@livekit/protocol": "1.15.0",
|
40
40
|
"events": "^3.3.0",
|
41
41
|
"loglevel": "^1.8.0",
|
42
42
|
"sdp-transform": "^2.14.1",
|
@@ -46,8 +46,8 @@
|
|
46
46
|
"webrtc-adapter": "^8.1.1"
|
47
47
|
},
|
48
48
|
"devDependencies": {
|
49
|
-
"@babel/core": "7.24.
|
50
|
-
"@babel/preset-env": "7.24.
|
49
|
+
"@babel/core": "7.24.5",
|
50
|
+
"@babel/preset-env": "7.24.5",
|
51
51
|
"@bufbuild/protoc-gen-es": "^1.3.0",
|
52
52
|
"@changesets/cli": "2.27.1",
|
53
53
|
"@livekit/changesets-changelog-github": "^0.0.4",
|
@@ -73,15 +73,15 @@
|
|
73
73
|
"gh-pages": "6.1.1",
|
74
74
|
"jsdom": "^24.0.0",
|
75
75
|
"prettier": "^3.0.0",
|
76
|
-
"rollup": "4.
|
76
|
+
"rollup": "4.17.2",
|
77
77
|
"rollup-plugin-delete": "^2.0.0",
|
78
78
|
"rollup-plugin-re": "1.0.7",
|
79
79
|
"rollup-plugin-typescript2": "0.36.0",
|
80
80
|
"size-limit": "^8.2.4",
|
81
|
-
"typedoc": "0.25.
|
81
|
+
"typedoc": "0.25.13",
|
82
82
|
"typedoc-plugin-no-inherit": "1.4.0",
|
83
|
-
"typescript": "5.4.
|
84
|
-
"vite": "5.
|
83
|
+
"typescript": "5.4.5",
|
84
|
+
"vite": "5.2.10",
|
85
85
|
"vitest": "^1.0.0"
|
86
86
|
},
|
87
87
|
"scripts": {
|
package/src/index.ts
CHANGED
@@ -44,7 +44,7 @@ export { facingModeFromDeviceLabel, facingModeFromLocalTrack } from './room/trac
|
|
44
44
|
export * from './room/track/options';
|
45
45
|
export * from './room/track/processor/types';
|
46
46
|
export * from './room/track/types';
|
47
|
-
export type { DataPublishOptions, SimulationScenario } from './room/types';
|
47
|
+
export type { DataPublishOptions, SimulationScenario, TranscriptionSegment } from './room/types';
|
48
48
|
export * from './version';
|
49
49
|
export {
|
50
50
|
ConnectionQuality,
|
package/src/room/PCTransport.ts
CHANGED
@@ -474,8 +474,6 @@ export default class PCTransport extends EventEmitter {
|
|
474
474
|
await this.pc.setLocalDescription(sd);
|
475
475
|
}
|
476
476
|
} catch (e) {
|
477
|
-
// this error cannot always be caught.
|
478
|
-
// If the local description has a setCodecPreferences error, this error will be uncaught
|
479
477
|
let msg = 'unknown error';
|
480
478
|
if (e instanceof Error) {
|
481
479
|
msg = e.message;
|
package/src/room/RTCEngine.ts
CHANGED
@@ -23,6 +23,7 @@ import {
|
|
23
23
|
TrackInfo,
|
24
24
|
type TrackPublishedResponse,
|
25
25
|
TrackUnpublishedResponse,
|
26
|
+
Transcription,
|
26
27
|
UpdateSubscription,
|
27
28
|
UserPacket,
|
28
29
|
} from '@livekit/protocol';
|
@@ -53,22 +54,14 @@ import { EngineEvent } from './events';
|
|
53
54
|
import CriticalTimers from './timers';
|
54
55
|
import type LocalTrack from './track/LocalTrack';
|
55
56
|
import type LocalTrackPublication from './track/LocalTrackPublication';
|
56
|
-
import
|
57
|
+
import LocalVideoTrack from './track/LocalVideoTrack';
|
57
58
|
import type { SimulcastTrackInfo } from './track/LocalVideoTrack';
|
58
59
|
import type RemoteTrackPublication from './track/RemoteTrackPublication';
|
59
|
-
import { Track } from './track/Track';
|
60
|
+
import type { Track } from './track/Track';
|
60
61
|
import type { TrackPublishOptions, VideoCodec } from './track/options';
|
61
62
|
import { getTrackPublicationInfo } from './track/utils';
|
62
63
|
import type { LoggerOptions } from './types';
|
63
|
-
import {
|
64
|
-
Mutex,
|
65
|
-
isVideoCodec,
|
66
|
-
isWeb,
|
67
|
-
sleep,
|
68
|
-
supportsAddTrack,
|
69
|
-
supportsSetCodecPreferences,
|
70
|
-
supportsTransceiver,
|
71
|
-
} from './utils';
|
64
|
+
import { Mutex, isVideoCodec, isWeb, sleep, supportsAddTrack, supportsTransceiver } from './utils';
|
72
65
|
|
73
66
|
const lossyDataChannel = '_lossy';
|
74
67
|
const reliableDataChannel = '_reliable';
|
@@ -642,6 +635,8 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
|
|
642
635
|
this.emit(EngineEvent.ActiveSpeakersUpdate, dp.value.value.speakers);
|
643
636
|
} else if (dp.value?.case === 'user') {
|
644
637
|
this.emit(EngineEvent.DataPacketReceived, dp.value.value, dp.kind);
|
638
|
+
} else if (dp.value?.case === 'transcription') {
|
639
|
+
this.emit(EngineEvent.TranscriptionReceived, dp.value.value);
|
645
640
|
}
|
646
641
|
} finally {
|
647
642
|
unlock();
|
@@ -671,51 +666,6 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
|
|
671
666
|
this.updateAndEmitDCBufferStatus(channelKind);
|
672
667
|
};
|
673
668
|
|
674
|
-
private setPreferredCodec(
|
675
|
-
transceiver: RTCRtpTransceiver,
|
676
|
-
kind: Track.Kind,
|
677
|
-
videoCodec: VideoCodec,
|
678
|
-
) {
|
679
|
-
if (!('getCapabilities' in RTCRtpReceiver)) {
|
680
|
-
return;
|
681
|
-
}
|
682
|
-
// when setting codec preferences, the capabilites need to be read from the RTCRtpReceiver
|
683
|
-
const cap = RTCRtpReceiver.getCapabilities(kind);
|
684
|
-
if (!cap) return;
|
685
|
-
this.log.debug('get receiver capabilities', { ...this.logContext, cap });
|
686
|
-
const matched: RTCRtpCodecCapability[] = [];
|
687
|
-
const partialMatched: RTCRtpCodecCapability[] = [];
|
688
|
-
const unmatched: RTCRtpCodecCapability[] = [];
|
689
|
-
cap.codecs.forEach((c) => {
|
690
|
-
const codec = c.mimeType.toLowerCase();
|
691
|
-
if (codec === 'audio/opus') {
|
692
|
-
matched.push(c);
|
693
|
-
return;
|
694
|
-
}
|
695
|
-
const matchesVideoCodec = codec === `video/${videoCodec}`;
|
696
|
-
if (!matchesVideoCodec) {
|
697
|
-
unmatched.push(c);
|
698
|
-
return;
|
699
|
-
}
|
700
|
-
// for h264 codecs that have sdpFmtpLine available, use only if the
|
701
|
-
// profile-level-id is 42e01f for cross-browser compatibility
|
702
|
-
if (videoCodec === 'h264') {
|
703
|
-
if (c.sdpFmtpLine && c.sdpFmtpLine.includes('profile-level-id=42e01f')) {
|
704
|
-
matched.push(c);
|
705
|
-
} else {
|
706
|
-
partialMatched.push(c);
|
707
|
-
}
|
708
|
-
return;
|
709
|
-
}
|
710
|
-
|
711
|
-
matched.push(c);
|
712
|
-
});
|
713
|
-
|
714
|
-
if (supportsSetCodecPreferences(transceiver)) {
|
715
|
-
transceiver.setCodecPreferences(matched.concat(partialMatched, unmatched));
|
716
|
-
}
|
717
|
-
}
|
718
|
-
|
719
669
|
async createSender(
|
720
670
|
track: LocalTrack,
|
721
671
|
opts: TrackPublishOptions,
|
@@ -766,6 +716,10 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
|
|
766
716
|
streams.push(track.mediaStream);
|
767
717
|
}
|
768
718
|
|
719
|
+
if (track instanceof LocalVideoTrack) {
|
720
|
+
track.codec = opts.videoCodec;
|
721
|
+
}
|
722
|
+
|
769
723
|
const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly', streams };
|
770
724
|
if (encodings) {
|
771
725
|
transceiverInit.sendEncodings = encodings;
|
@@ -776,10 +730,6 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
|
|
776
730
|
transceiverInit,
|
777
731
|
);
|
778
732
|
|
779
|
-
if (track.kind === Track.Kind.Video && opts.videoCodec) {
|
780
|
-
this.setPreferredCodec(transceiver, track.kind, opts.videoCodec);
|
781
|
-
track.codec = opts.videoCodec;
|
782
|
-
}
|
783
733
|
return transceiver.sender;
|
784
734
|
}
|
785
735
|
|
@@ -804,7 +754,6 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
|
|
804
754
|
if (!opts.videoCodec) {
|
805
755
|
return;
|
806
756
|
}
|
807
|
-
this.setPreferredCodec(transceiver, track.kind, opts.videoCodec);
|
808
757
|
track.setSimulcastTrackSender(opts.videoCodec, transceiver.sender);
|
809
758
|
return transceiver.sender;
|
810
759
|
}
|
@@ -1411,6 +1360,7 @@ export type EngineEventCallbacks = {
|
|
1411
1360
|
) => void;
|
1412
1361
|
activeSpeakersUpdate: (speakers: Array<SpeakerInfo>) => void;
|
1413
1362
|
dataPacketReceived: (userPacket: UserPacket, kind: DataPacket_Kind) => void;
|
1363
|
+
transcriptionReceived: (transcription: Transcription) => void;
|
1414
1364
|
transportsCreated: (publisher: PCTransport, subscriber: PCTransport) => void;
|
1415
1365
|
/** @internal */
|
1416
1366
|
trackSenderAdded: (track: Track, sender: RTCRtpSender) => void;
|
package/src/room/Room.ts
CHANGED
@@ -18,6 +18,8 @@ import {
|
|
18
18
|
TrackInfo,
|
19
19
|
TrackSource,
|
20
20
|
TrackType,
|
21
|
+
Transcription as TranscriptionModel,
|
22
|
+
TranscriptionSegment as TranscriptionSegmentModel,
|
21
23
|
UserPacket,
|
22
24
|
protoInt64,
|
23
25
|
} from '@livekit/protocol';
|
@@ -61,11 +63,12 @@ import type { TrackPublication } from './track/TrackPublication';
|
|
61
63
|
import type { TrackProcessor } from './track/processor/types';
|
62
64
|
import type { AdaptiveStreamSettings } from './track/types';
|
63
65
|
import { getNewAudioContext, sourceToKind } from './track/utils';
|
64
|
-
import type { SimulationOptions, SimulationScenario } from './types';
|
66
|
+
import type { SimulationOptions, SimulationScenario, TranscriptionSegment } from './types';
|
65
67
|
import {
|
66
68
|
Future,
|
67
69
|
Mutex,
|
68
70
|
createDummyVideoStreamTrack,
|
71
|
+
extractTranscriptionSegments,
|
69
72
|
getEmptyAudioStreamTrack,
|
70
73
|
isBrowserSupported,
|
71
74
|
isCloud,
|
@@ -330,6 +333,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
|
|
330
333
|
})
|
331
334
|
.on(EngineEvent.ActiveSpeakersUpdate, this.handleActiveSpeakersUpdate)
|
332
335
|
.on(EngineEvent.DataPacketReceived, this.handleDataPacket)
|
336
|
+
.on(EngineEvent.TranscriptionReceived, this.handleTranscription)
|
333
337
|
.on(EngineEvent.Resuming, () => {
|
334
338
|
this.clearConnectionReconcile();
|
335
339
|
this.isResuming = true;
|
@@ -1471,6 +1475,23 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
|
|
1471
1475
|
participant?.emit(ParticipantEvent.DataReceived, userPacket.payload, kind);
|
1472
1476
|
};
|
1473
1477
|
|
1478
|
+
bufferedSegments: Map<string, TranscriptionSegmentModel> = new Map();
|
1479
|
+
|
1480
|
+
private handleTranscription = (transcription: TranscriptionModel) => {
|
1481
|
+
// find the participant
|
1482
|
+
const participant =
|
1483
|
+
transcription.participantIdentity === this.localParticipant.identity
|
1484
|
+
? this.localParticipant
|
1485
|
+
: this.remoteParticipants.get(transcription.participantIdentity);
|
1486
|
+
const publication = participant?.trackPublications.get(transcription.trackId);
|
1487
|
+
|
1488
|
+
const segments = extractTranscriptionSegments(transcription);
|
1489
|
+
|
1490
|
+
publication?.emit(TrackEvent.TranscriptionReceived, segments);
|
1491
|
+
participant?.emit(ParticipantEvent.TranscriptionReceived, segments, publication);
|
1492
|
+
this.emit(RoomEvent.TranscriptionReceived, segments, participant, publication);
|
1493
|
+
};
|
1494
|
+
|
1474
1495
|
private handleAudioPlaybackStarted = () => {
|
1475
1496
|
if (this.canPlaybackAudio) {
|
1476
1497
|
return;
|
@@ -2071,6 +2092,11 @@ export type RoomEventCallbacks = {
|
|
2071
2092
|
kind?: DataPacket_Kind,
|
2072
2093
|
topic?: string,
|
2073
2094
|
) => void;
|
2095
|
+
transcriptionReceived: (
|
2096
|
+
transcription: TranscriptionSegment[],
|
2097
|
+
participant?: Participant,
|
2098
|
+
publication?: TrackPublication,
|
2099
|
+
) => void;
|
2074
2100
|
connectionQualityChanged: (quality: ConnectionQuality, participant: Participant) => void;
|
2075
2101
|
mediaDevicesError: (error: Error) => void;
|
2076
2102
|
trackStreamStateChanged: (
|
package/src/room/events.ts
CHANGED
@@ -197,6 +197,12 @@ export enum RoomEvent {
|
|
197
197
|
*/
|
198
198
|
DataReceived = 'dataReceived',
|
199
199
|
|
200
|
+
/**
|
201
|
+
* Transcription received from a participant's track.
|
202
|
+
* @beta
|
203
|
+
*/
|
204
|
+
TranscriptionReceived = 'transcriptionReceived',
|
205
|
+
|
200
206
|
/**
|
201
207
|
* Connection quality was changed for a Participant. It'll receive updates
|
202
208
|
* from the local participant, as well as any [[RemoteParticipant]]s that we are
|
@@ -402,6 +408,12 @@ export enum ParticipantEvent {
|
|
402
408
|
*/
|
403
409
|
DataReceived = 'dataReceived',
|
404
410
|
|
411
|
+
/**
|
412
|
+
* Transcription received from this participant as data source.
|
413
|
+
* @beta
|
414
|
+
*/
|
415
|
+
TranscriptionReceived = 'transcriptionReceived',
|
416
|
+
|
405
417
|
/**
|
406
418
|
* Has speaking status changed for the current participant
|
407
419
|
*
|
@@ -479,6 +491,7 @@ export enum EngineEvent {
|
|
479
491
|
MediaTrackAdded = 'mediaTrackAdded',
|
480
492
|
ActiveSpeakersUpdate = 'activeSpeakersUpdate',
|
481
493
|
DataPacketReceived = 'dataPacketReceived',
|
494
|
+
TranscriptionReceived = 'transcriptionReceived',
|
482
495
|
RTPVideoMapUpdate = 'rtpVideoMapUpdate',
|
483
496
|
DCBufferStatusChanged = 'dcBufferStatusChanged',
|
484
497
|
ParticipantUpdate = 'participantUpdate',
|
@@ -562,4 +575,14 @@ export enum TrackEvent {
|
|
562
575
|
* @internal
|
563
576
|
*/
|
564
577
|
AudioTrackFeatureUpdate = 'audioTrackFeatureUpdate',
|
578
|
+
|
579
|
+
/**
|
580
|
+
* @beta
|
581
|
+
*/
|
582
|
+
TranscriptionReceived = 'transcriptionReceived',
|
583
|
+
|
584
|
+
/**
|
585
|
+
* @experimental
|
586
|
+
*/
|
587
|
+
TimeSyncUpdate = 'timeSyncUpdate',
|
565
588
|
}
|
@@ -40,9 +40,9 @@ import {
|
|
40
40
|
import type { DataPublishOptions } from '../types';
|
41
41
|
import {
|
42
42
|
Future,
|
43
|
+
isE2EESimulcastSupported,
|
43
44
|
isFireFox,
|
44
45
|
isSVCCodec,
|
45
|
-
isSafari,
|
46
46
|
isSafari17,
|
47
47
|
isWeb,
|
48
48
|
supportsAV1,
|
@@ -621,10 +621,9 @@ export default class LocalParticipant extends Participant {
|
|
621
621
|
...options,
|
622
622
|
};
|
623
623
|
|
624
|
-
|
625
|
-
if (isSafari() && this.roomOptions.e2ee) {
|
624
|
+
if (!isE2EESimulcastSupported() && this.roomOptions.e2ee) {
|
626
625
|
this.log.info(
|
627
|
-
`End-to-end encryption is set up, simulcast publishing will be disabled on Safari`,
|
626
|
+
`End-to-end encryption is set up, simulcast publishing will be disabled on Safari versions and iOS browsers running iOS < v17.2`,
|
628
627
|
{
|
629
628
|
...this.logContext,
|
630
629
|
},
|
@@ -827,7 +826,6 @@ export default class LocalParticipant extends Participant {
|
|
827
826
|
...getLogContextFromTrack(track),
|
828
827
|
codec: updatedCodec,
|
829
828
|
});
|
830
|
-
/* @ts-ignore */
|
831
829
|
opts.videoCodec = updatedCodec;
|
832
830
|
|
833
831
|
// recompute encodings since bitrates/etc could have changed
|
@@ -16,7 +16,7 @@ import type RemoteTrack from '../track/RemoteTrack';
|
|
16
16
|
import type RemoteTrackPublication from '../track/RemoteTrackPublication';
|
17
17
|
import { Track } from '../track/Track';
|
18
18
|
import type { TrackPublication } from '../track/TrackPublication';
|
19
|
-
import type { LoggerOptions } from '../types';
|
19
|
+
import type { LoggerOptions, TranscriptionSegment } from '../types';
|
20
20
|
|
21
21
|
export enum ConnectionQuality {
|
22
22
|
Excellent = 'excellent',
|
@@ -329,6 +329,10 @@ export type ParticipantEventCallbacks = {
|
|
329
329
|
participantMetadataChanged: (prevMetadata: string | undefined, participant?: any) => void;
|
330
330
|
participantNameChanged: (name: string) => void;
|
331
331
|
dataReceived: (payload: Uint8Array, kind: DataPacket_Kind) => void;
|
332
|
+
transcriptionReceived: (
|
333
|
+
transcription: TranscriptionSegment[],
|
334
|
+
publication?: TrackPublication,
|
335
|
+
) => void;
|
332
336
|
isSpeakingChanged: (speaking: boolean) => void;
|
333
337
|
connectionQualityChanged: (connectionQuality: ConnectionQuality) => void;
|
334
338
|
trackStreamStateChanged: (
|
@@ -77,7 +77,20 @@ export default abstract class RemoteTrack<
|
|
77
77
|
if (!this.monitorInterval) {
|
78
78
|
this.monitorInterval = setInterval(() => this.monitorReceiver(), monitorFrequency);
|
79
79
|
}
|
80
|
+
this.registerTimeSyncUpdate();
|
80
81
|
}
|
81
82
|
|
82
83
|
protected abstract monitorReceiver(): void;
|
84
|
+
|
85
|
+
registerTimeSyncUpdate() {
|
86
|
+
const loop = () => {
|
87
|
+
this.timeSyncHandle = requestAnimationFrame(() => loop());
|
88
|
+
const newTime = this.receiver?.getSynchronizationSources()[0]?.rtpTimestamp;
|
89
|
+
if (newTime && this.rtpTimestamp !== newTime) {
|
90
|
+
this.emit(TrackEvent.TimeSyncUpdate, newTime);
|
91
|
+
this.rtpTimestamp = newTime;
|
92
|
+
}
|
93
|
+
};
|
94
|
+
loop();
|
95
|
+
}
|
83
96
|
}
|
package/src/room/track/Track.ts
CHANGED
@@ -53,6 +53,9 @@ export abstract class Track<
|
|
53
53
|
*/
|
54
54
|
streamState: Track.StreamState = Track.StreamState.Active;
|
55
55
|
|
56
|
+
/** @internal */
|
57
|
+
rtpTimestamp: number | undefined;
|
58
|
+
|
56
59
|
protected _mediaStreamTrack: MediaStreamTrack;
|
57
60
|
|
58
61
|
protected _mediaStreamID: string;
|
@@ -63,6 +66,8 @@ export abstract class Track<
|
|
63
66
|
|
64
67
|
private loggerContextCb: LoggerOptions['loggerContextCb'];
|
65
68
|
|
69
|
+
protected timeSyncHandle: number | undefined;
|
70
|
+
|
66
71
|
protected _currentBitrate: number = 0;
|
67
72
|
|
68
73
|
protected monitorInterval?: ReturnType<typeof setInterval>;
|
@@ -255,6 +260,9 @@ export abstract class Track<
|
|
255
260
|
if (this.monitorInterval) {
|
256
261
|
clearInterval(this.monitorInterval);
|
257
262
|
}
|
263
|
+
if (this.timeSyncHandle) {
|
264
|
+
cancelAnimationFrame(this.timeSyncHandle);
|
265
|
+
}
|
258
266
|
}
|
259
267
|
|
260
268
|
/** @internal */
|
@@ -517,4 +525,5 @@ export type TrackEventCallbacks = {
|
|
517
525
|
upstreamResumed: (track: any) => void;
|
518
526
|
trackProcessorUpdate: (processor?: TrackProcessor<Track.Kind, any>) => void;
|
519
527
|
audioTrackFeatureUpdate: (track: any, feature: AudioTrackFeature, enabled: boolean) => void;
|
528
|
+
timeSyncUpdate: (timestamp: number) => void;
|
520
529
|
};
|
@@ -9,7 +9,7 @@ import { EventEmitter } from 'events';
|
|
9
9
|
import type TypedEventEmitter from 'typed-emitter';
|
10
10
|
import log, { LoggerNames, getLogger } from '../../logger';
|
11
11
|
import { TrackEvent } from '../events';
|
12
|
-
import type { LoggerOptions } from '../types';
|
12
|
+
import type { LoggerOptions, TranscriptionSegment } from '../types';
|
13
13
|
import LocalAudioTrack from './LocalAudioTrack';
|
14
14
|
import LocalVideoTrack from './LocalVideoTrack';
|
15
15
|
import RemoteAudioTrack from './RemoteAudioTrack';
|
@@ -174,4 +174,6 @@ export type PublicationEventCallbacks = {
|
|
174
174
|
prevStatus: TrackPublication.SubscriptionStatus,
|
175
175
|
) => void;
|
176
176
|
subscriptionFailed: (error: SubscriptionError) => void;
|
177
|
+
transcriptionReceived: (transcription: TranscriptionSegment[]) => void;
|
178
|
+
timeSyncUpdate: (timestamp: number) => void;
|
177
179
|
};
|
package/src/room/types.ts
CHANGED
@@ -55,3 +55,12 @@ export type LoggerOptions = {
|
|
55
55
|
loggerName?: string;
|
56
56
|
loggerContextCb?: () => Record<string, unknown>;
|
57
57
|
};
|
58
|
+
|
59
|
+
export interface TranscriptionSegment {
|
60
|
+
id: string;
|
61
|
+
text: string;
|
62
|
+
language: string;
|
63
|
+
startTime: number;
|
64
|
+
endTime: number;
|
65
|
+
final: boolean;
|
66
|
+
}
|