livekit-client 2.1.0 → 2.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/livekit-client.esm.mjs +2212 -2100
- package/dist/livekit-client.esm.mjs.map +1 -1
- package/dist/livekit-client.umd.js +1 -1
- package/dist/livekit-client.umd.js.map +1 -1
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/room/PCTransport.d.ts.map +1 -1
- package/dist/src/room/RTCEngine.d.ts +4 -4
- package/dist/src/room/RTCEngine.d.ts.map +1 -1
- package/dist/src/room/Room.d.ts +5 -2
- package/dist/src/room/Room.d.ts.map +1 -1
- package/dist/src/room/events.d.ts +20 -1
- package/dist/src/room/events.d.ts.map +1 -1
- package/dist/src/room/participant/LocalParticipant.d.ts.map +1 -1
- package/dist/src/room/participant/Participant.d.ts +2 -1
- package/dist/src/room/participant/Participant.d.ts.map +1 -1
- package/dist/src/room/track/RemoteTrack.d.ts +1 -0
- package/dist/src/room/track/RemoteTrack.d.ts.map +1 -1
- package/dist/src/room/track/Track.d.ts +4 -0
- package/dist/src/room/track/Track.d.ts.map +1 -1
- package/dist/src/room/track/TrackPublication.d.ts +3 -1
- package/dist/src/room/track/TrackPublication.d.ts.map +1 -1
- package/dist/src/room/types.d.ts +8 -0
- package/dist/src/room/types.d.ts.map +1 -1
- package/dist/src/room/utils.d.ts +4 -2
- package/dist/src/room/utils.d.ts.map +1 -1
- package/dist/src/utils/browserParser.d.ts +1 -0
- package/dist/src/utils/browserParser.d.ts.map +1 -1
- package/dist/ts4.2/src/index.d.ts +1 -1
- package/dist/ts4.2/src/room/RTCEngine.d.ts +4 -4
- package/dist/ts4.2/src/room/Room.d.ts +5 -2
- package/dist/ts4.2/src/room/events.d.ts +20 -1
- package/dist/ts4.2/src/room/participant/Participant.d.ts +2 -1
- package/dist/ts4.2/src/room/track/RemoteTrack.d.ts +1 -0
- package/dist/ts4.2/src/room/track/Track.d.ts +4 -0
- package/dist/ts4.2/src/room/track/TrackPublication.d.ts +3 -1
- package/dist/ts4.2/src/room/types.d.ts +8 -0
- package/dist/ts4.2/src/room/utils.d.ts +4 -2
- package/dist/ts4.2/src/utils/browserParser.d.ts +1 -0
- package/package.json +8 -8
- package/src/index.ts +1 -1
- package/src/room/PCTransport.ts +0 -2
- package/src/room/RTCEngine.ts +11 -61
- package/src/room/Room.ts +27 -1
- package/src/room/events.ts +23 -0
- package/src/room/participant/LocalParticipant.ts +3 -5
- package/src/room/participant/Participant.ts +5 -1
- package/src/room/track/RemoteTrack.ts +13 -0
- package/src/room/track/Track.ts +9 -0
- package/src/room/track/TrackPublication.ts +3 -1
- package/src/room/types.ts +9 -0
- package/src/room/utils.ts +46 -29
- package/src/utils/browserParser.test.ts +4 -0
- package/src/utils/browserParser.ts +11 -1
@@ -6,7 +6,7 @@ import type RemoteTrack from '../track/RemoteTrack';
|
|
6
6
|
import type RemoteTrackPublication from '../track/RemoteTrackPublication';
|
7
7
|
import { Track } from '../track/Track';
|
8
8
|
import type { TrackPublication } from '../track/TrackPublication';
|
9
|
-
import type { LoggerOptions } from '../types';
|
9
|
+
import type { LoggerOptions, TranscriptionSegment } from '../types';
|
10
10
|
export declare enum ConnectionQuality {
|
11
11
|
Excellent = "excellent",
|
12
12
|
Good = "good",
|
@@ -99,6 +99,7 @@ export type ParticipantEventCallbacks = {
|
|
99
99
|
participantMetadataChanged: (prevMetadata: string | undefined, participant?: any) => void;
|
100
100
|
participantNameChanged: (name: string) => void;
|
101
101
|
dataReceived: (payload: Uint8Array, kind: DataPacket_Kind) => void;
|
102
|
+
transcriptionReceived: (transcription: TranscriptionSegment[], publication?: TrackPublication) => void;
|
102
103
|
isSpeakingChanged: (speaking: boolean) => void;
|
103
104
|
connectionQualityChanged: (connectionQuality: ConnectionQuality) => void;
|
104
105
|
trackStreamStateChanged: (publication: RemoteTrackPublication, streamState: Track.StreamState) => void;
|
@@ -19,5 +19,6 @@ export default abstract class RemoteTrack<TrackKind extends Track.Kind = Track.K
|
|
19
19
|
getRTCStatsReport(): Promise<RTCStatsReport | undefined>;
|
20
20
|
startMonitor(): void;
|
21
21
|
protected abstract monitorReceiver(): void;
|
22
|
+
registerTimeSyncUpdate(): void;
|
22
23
|
}
|
23
24
|
//# sourceMappingURL=RemoteTrack.d.ts.map
|
@@ -28,11 +28,14 @@ export declare abstract class Track<TrackKind extends Track.Kind = Track.Kind> e
|
|
28
28
|
* has been paused by congestion controller
|
29
29
|
*/
|
30
30
|
streamState: Track.StreamState;
|
31
|
+
/** @internal */
|
32
|
+
rtpTimestamp: number | undefined;
|
31
33
|
protected _mediaStreamTrack: MediaStreamTrack;
|
32
34
|
protected _mediaStreamID: string;
|
33
35
|
protected isInBackground: boolean;
|
34
36
|
private backgroundTimeout;
|
35
37
|
private loggerContextCb;
|
38
|
+
protected timeSyncHandle: number | undefined;
|
36
39
|
protected _currentBitrate: number;
|
37
40
|
protected monitorInterval?: ReturnType<typeof setInterval>;
|
38
41
|
protected log: StructuredLogger;
|
@@ -137,6 +140,7 @@ export type TrackEventCallbacks = {
|
|
137
140
|
upstreamResumed: (track: any) => void;
|
138
141
|
trackProcessorUpdate: (processor?: TrackProcessor<Track.Kind, any>) => void;
|
139
142
|
audioTrackFeatureUpdate: (track: any, feature: AudioTrackFeature, enabled: boolean) => void;
|
143
|
+
timeSyncUpdate: (timestamp: number) => void;
|
140
144
|
};
|
141
145
|
export {};
|
142
146
|
//# sourceMappingURL=Track.d.ts.map
|
@@ -1,7 +1,7 @@
|
|
1
1
|
import { Encryption_Type } from '@livekit/protocol';
|
2
2
|
import type { SubscriptionError, TrackInfo, UpdateSubscription, UpdateTrackSettings } from '@livekit/protocol';
|
3
3
|
import type TypedEventEmitter from 'typed-emitter';
|
4
|
-
import type { LoggerOptions } from '../types';
|
4
|
+
import type { LoggerOptions, TranscriptionSegment } from '../types';
|
5
5
|
import LocalAudioTrack from './LocalAudioTrack';
|
6
6
|
import LocalVideoTrack from './LocalVideoTrack';
|
7
7
|
import RemoteAudioTrack from './RemoteAudioTrack';
|
@@ -72,6 +72,8 @@ export type PublicationEventCallbacks = {
|
|
72
72
|
unsubscribed: (track: RemoteTrack) => void;
|
73
73
|
subscriptionStatusChanged: (status: TrackPublication.SubscriptionStatus, prevStatus: TrackPublication.SubscriptionStatus) => void;
|
74
74
|
subscriptionFailed: (error: SubscriptionError) => void;
|
75
|
+
transcriptionReceived: (transcription: TranscriptionSegment[]) => void;
|
76
|
+
timeSyncUpdate: (timestamp: number) => void;
|
75
77
|
};
|
76
78
|
export {};
|
77
79
|
//# sourceMappingURL=TrackPublication.d.ts.map
|
@@ -35,4 +35,12 @@ export type LoggerOptions = {
|
|
35
35
|
loggerName?: string;
|
36
36
|
loggerContextCb?: () => Record<string, unknown>;
|
37
37
|
};
|
38
|
+
export interface TranscriptionSegment {
|
39
|
+
id: string;
|
40
|
+
text: string;
|
41
|
+
language: string;
|
42
|
+
startTime: number;
|
43
|
+
endTime: number;
|
44
|
+
final: boolean;
|
45
|
+
}
|
38
46
|
//# sourceMappingURL=types.d.ts.map
|
@@ -1,7 +1,8 @@
|
|
1
|
-
import { ClientInfo } from '@livekit/protocol';
|
1
|
+
import { ClientInfo, Transcription as TranscriptionModel } from '@livekit/protocol';
|
2
2
|
import type LocalAudioTrack from './track/LocalAudioTrack';
|
3
3
|
import type RemoteAudioTrack from './track/RemoteAudioTrack';
|
4
4
|
import { VideoCodec } from './track/options';
|
5
|
+
import type { TranscriptionSegment } from './types';
|
5
6
|
export declare const ddExtensionURI = "https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension";
|
6
7
|
export declare function unpackStreamId(packed: string): string[];
|
7
8
|
export declare function sleep(duration: number): Promise<void>;
|
@@ -15,13 +16,13 @@ export declare function supportsAV1(): boolean;
|
|
15
16
|
export declare function supportsVP9(): boolean;
|
16
17
|
export declare function isSVCCodec(codec?: string): boolean;
|
17
18
|
export declare function supportsSetSinkId(elm?: HTMLMediaElement): boolean;
|
18
|
-
export declare function supportsSetCodecPreferences(transceiver: RTCRtpTransceiver): boolean;
|
19
19
|
export declare function isBrowserSupported(): boolean;
|
20
20
|
export declare function isFireFox(): boolean;
|
21
21
|
export declare function isChromiumBased(): boolean;
|
22
22
|
export declare function isSafari(): boolean;
|
23
23
|
export declare function isSafari17(): boolean;
|
24
24
|
export declare function isMobile(): boolean;
|
25
|
+
export declare function isE2EESimulcastSupported(): boolean | undefined;
|
25
26
|
export declare function isWeb(): boolean;
|
26
27
|
export declare function isReactNative(): boolean;
|
27
28
|
export declare function isCloud(serverUrl: URL): boolean;
|
@@ -93,4 +94,5 @@ export declare function isVideoCodec(maybeCodec: string): maybeCodec is VideoCod
|
|
93
94
|
export declare function unwrapConstraint(constraint: ConstrainDOMString): string;
|
94
95
|
export declare function toWebsocketUrl(url: string): string;
|
95
96
|
export declare function toHttpUrl(url: string): string;
|
97
|
+
export declare function extractTranscriptionSegments(transcription: TranscriptionModel): TranscriptionSegment[];
|
96
98
|
//# sourceMappingURL=utils.d.ts.map
|
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "livekit-client",
|
3
|
-
"version": "2.1.
|
3
|
+
"version": "2.1.2",
|
4
4
|
"description": "JavaScript/TypeScript client SDK for LiveKit",
|
5
5
|
"main": "./dist/livekit-client.umd.js",
|
6
6
|
"unpkg": "./dist/livekit-client.umd.js",
|
@@ -36,7 +36,7 @@
|
|
36
36
|
"author": "David Zhao <david@davidzhao.com>",
|
37
37
|
"license": "Apache-2.0",
|
38
38
|
"dependencies": {
|
39
|
-
"@livekit/protocol": "1.
|
39
|
+
"@livekit/protocol": "1.15.0",
|
40
40
|
"events": "^3.3.0",
|
41
41
|
"loglevel": "^1.8.0",
|
42
42
|
"sdp-transform": "^2.14.1",
|
@@ -46,8 +46,8 @@
|
|
46
46
|
"webrtc-adapter": "^8.1.1"
|
47
47
|
},
|
48
48
|
"devDependencies": {
|
49
|
-
"@babel/core": "7.24.
|
50
|
-
"@babel/preset-env": "7.24.
|
49
|
+
"@babel/core": "7.24.5",
|
50
|
+
"@babel/preset-env": "7.24.5",
|
51
51
|
"@bufbuild/protoc-gen-es": "^1.3.0",
|
52
52
|
"@changesets/cli": "2.27.1",
|
53
53
|
"@livekit/changesets-changelog-github": "^0.0.4",
|
@@ -73,15 +73,15 @@
|
|
73
73
|
"gh-pages": "6.1.1",
|
74
74
|
"jsdom": "^24.0.0",
|
75
75
|
"prettier": "^3.0.0",
|
76
|
-
"rollup": "4.
|
76
|
+
"rollup": "4.17.2",
|
77
77
|
"rollup-plugin-delete": "^2.0.0",
|
78
78
|
"rollup-plugin-re": "1.0.7",
|
79
79
|
"rollup-plugin-typescript2": "0.36.0",
|
80
80
|
"size-limit": "^8.2.4",
|
81
|
-
"typedoc": "0.25.
|
81
|
+
"typedoc": "0.25.13",
|
82
82
|
"typedoc-plugin-no-inherit": "1.4.0",
|
83
|
-
"typescript": "5.4.
|
84
|
-
"vite": "5.
|
83
|
+
"typescript": "5.4.5",
|
84
|
+
"vite": "5.2.10",
|
85
85
|
"vitest": "^1.0.0"
|
86
86
|
},
|
87
87
|
"scripts": {
|
package/src/index.ts
CHANGED
@@ -44,7 +44,7 @@ export { facingModeFromDeviceLabel, facingModeFromLocalTrack } from './room/trac
|
|
44
44
|
export * from './room/track/options';
|
45
45
|
export * from './room/track/processor/types';
|
46
46
|
export * from './room/track/types';
|
47
|
-
export type { DataPublishOptions, SimulationScenario } from './room/types';
|
47
|
+
export type { DataPublishOptions, SimulationScenario, TranscriptionSegment } from './room/types';
|
48
48
|
export * from './version';
|
49
49
|
export {
|
50
50
|
ConnectionQuality,
|
package/src/room/PCTransport.ts
CHANGED
@@ -474,8 +474,6 @@ export default class PCTransport extends EventEmitter {
|
|
474
474
|
await this.pc.setLocalDescription(sd);
|
475
475
|
}
|
476
476
|
} catch (e) {
|
477
|
-
// this error cannot always be caught.
|
478
|
-
// If the local description has a setCodecPreferences error, this error will be uncaught
|
479
477
|
let msg = 'unknown error';
|
480
478
|
if (e instanceof Error) {
|
481
479
|
msg = e.message;
|
package/src/room/RTCEngine.ts
CHANGED
@@ -23,6 +23,7 @@ import {
|
|
23
23
|
TrackInfo,
|
24
24
|
type TrackPublishedResponse,
|
25
25
|
TrackUnpublishedResponse,
|
26
|
+
Transcription,
|
26
27
|
UpdateSubscription,
|
27
28
|
UserPacket,
|
28
29
|
} from '@livekit/protocol';
|
@@ -53,22 +54,14 @@ import { EngineEvent } from './events';
|
|
53
54
|
import CriticalTimers from './timers';
|
54
55
|
import type LocalTrack from './track/LocalTrack';
|
55
56
|
import type LocalTrackPublication from './track/LocalTrackPublication';
|
56
|
-
import
|
57
|
+
import LocalVideoTrack from './track/LocalVideoTrack';
|
57
58
|
import type { SimulcastTrackInfo } from './track/LocalVideoTrack';
|
58
59
|
import type RemoteTrackPublication from './track/RemoteTrackPublication';
|
59
|
-
import { Track } from './track/Track';
|
60
|
+
import type { Track } from './track/Track';
|
60
61
|
import type { TrackPublishOptions, VideoCodec } from './track/options';
|
61
62
|
import { getTrackPublicationInfo } from './track/utils';
|
62
63
|
import type { LoggerOptions } from './types';
|
63
|
-
import {
|
64
|
-
Mutex,
|
65
|
-
isVideoCodec,
|
66
|
-
isWeb,
|
67
|
-
sleep,
|
68
|
-
supportsAddTrack,
|
69
|
-
supportsSetCodecPreferences,
|
70
|
-
supportsTransceiver,
|
71
|
-
} from './utils';
|
64
|
+
import { Mutex, isVideoCodec, isWeb, sleep, supportsAddTrack, supportsTransceiver } from './utils';
|
72
65
|
|
73
66
|
const lossyDataChannel = '_lossy';
|
74
67
|
const reliableDataChannel = '_reliable';
|
@@ -642,6 +635,8 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
|
|
642
635
|
this.emit(EngineEvent.ActiveSpeakersUpdate, dp.value.value.speakers);
|
643
636
|
} else if (dp.value?.case === 'user') {
|
644
637
|
this.emit(EngineEvent.DataPacketReceived, dp.value.value, dp.kind);
|
638
|
+
} else if (dp.value?.case === 'transcription') {
|
639
|
+
this.emit(EngineEvent.TranscriptionReceived, dp.value.value);
|
645
640
|
}
|
646
641
|
} finally {
|
647
642
|
unlock();
|
@@ -671,51 +666,6 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
|
|
671
666
|
this.updateAndEmitDCBufferStatus(channelKind);
|
672
667
|
};
|
673
668
|
|
674
|
-
private setPreferredCodec(
|
675
|
-
transceiver: RTCRtpTransceiver,
|
676
|
-
kind: Track.Kind,
|
677
|
-
videoCodec: VideoCodec,
|
678
|
-
) {
|
679
|
-
if (!('getCapabilities' in RTCRtpReceiver)) {
|
680
|
-
return;
|
681
|
-
}
|
682
|
-
// when setting codec preferences, the capabilites need to be read from the RTCRtpReceiver
|
683
|
-
const cap = RTCRtpReceiver.getCapabilities(kind);
|
684
|
-
if (!cap) return;
|
685
|
-
this.log.debug('get receiver capabilities', { ...this.logContext, cap });
|
686
|
-
const matched: RTCRtpCodecCapability[] = [];
|
687
|
-
const partialMatched: RTCRtpCodecCapability[] = [];
|
688
|
-
const unmatched: RTCRtpCodecCapability[] = [];
|
689
|
-
cap.codecs.forEach((c) => {
|
690
|
-
const codec = c.mimeType.toLowerCase();
|
691
|
-
if (codec === 'audio/opus') {
|
692
|
-
matched.push(c);
|
693
|
-
return;
|
694
|
-
}
|
695
|
-
const matchesVideoCodec = codec === `video/${videoCodec}`;
|
696
|
-
if (!matchesVideoCodec) {
|
697
|
-
unmatched.push(c);
|
698
|
-
return;
|
699
|
-
}
|
700
|
-
// for h264 codecs that have sdpFmtpLine available, use only if the
|
701
|
-
// profile-level-id is 42e01f for cross-browser compatibility
|
702
|
-
if (videoCodec === 'h264') {
|
703
|
-
if (c.sdpFmtpLine && c.sdpFmtpLine.includes('profile-level-id=42e01f')) {
|
704
|
-
matched.push(c);
|
705
|
-
} else {
|
706
|
-
partialMatched.push(c);
|
707
|
-
}
|
708
|
-
return;
|
709
|
-
}
|
710
|
-
|
711
|
-
matched.push(c);
|
712
|
-
});
|
713
|
-
|
714
|
-
if (supportsSetCodecPreferences(transceiver)) {
|
715
|
-
transceiver.setCodecPreferences(matched.concat(partialMatched, unmatched));
|
716
|
-
}
|
717
|
-
}
|
718
|
-
|
719
669
|
async createSender(
|
720
670
|
track: LocalTrack,
|
721
671
|
opts: TrackPublishOptions,
|
@@ -766,6 +716,10 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
|
|
766
716
|
streams.push(track.mediaStream);
|
767
717
|
}
|
768
718
|
|
719
|
+
if (track instanceof LocalVideoTrack) {
|
720
|
+
track.codec = opts.videoCodec;
|
721
|
+
}
|
722
|
+
|
769
723
|
const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly', streams };
|
770
724
|
if (encodings) {
|
771
725
|
transceiverInit.sendEncodings = encodings;
|
@@ -776,10 +730,6 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
|
|
776
730
|
transceiverInit,
|
777
731
|
);
|
778
732
|
|
779
|
-
if (track.kind === Track.Kind.Video && opts.videoCodec) {
|
780
|
-
this.setPreferredCodec(transceiver, track.kind, opts.videoCodec);
|
781
|
-
track.codec = opts.videoCodec;
|
782
|
-
}
|
783
733
|
return transceiver.sender;
|
784
734
|
}
|
785
735
|
|
@@ -804,7 +754,6 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
|
|
804
754
|
if (!opts.videoCodec) {
|
805
755
|
return;
|
806
756
|
}
|
807
|
-
this.setPreferredCodec(transceiver, track.kind, opts.videoCodec);
|
808
757
|
track.setSimulcastTrackSender(opts.videoCodec, transceiver.sender);
|
809
758
|
return transceiver.sender;
|
810
759
|
}
|
@@ -1411,6 +1360,7 @@ export type EngineEventCallbacks = {
|
|
1411
1360
|
) => void;
|
1412
1361
|
activeSpeakersUpdate: (speakers: Array<SpeakerInfo>) => void;
|
1413
1362
|
dataPacketReceived: (userPacket: UserPacket, kind: DataPacket_Kind) => void;
|
1363
|
+
transcriptionReceived: (transcription: Transcription) => void;
|
1414
1364
|
transportsCreated: (publisher: PCTransport, subscriber: PCTransport) => void;
|
1415
1365
|
/** @internal */
|
1416
1366
|
trackSenderAdded: (track: Track, sender: RTCRtpSender) => void;
|
package/src/room/Room.ts
CHANGED
@@ -18,6 +18,8 @@ import {
|
|
18
18
|
TrackInfo,
|
19
19
|
TrackSource,
|
20
20
|
TrackType,
|
21
|
+
Transcription as TranscriptionModel,
|
22
|
+
TranscriptionSegment as TranscriptionSegmentModel,
|
21
23
|
UserPacket,
|
22
24
|
protoInt64,
|
23
25
|
} from '@livekit/protocol';
|
@@ -61,11 +63,12 @@ import type { TrackPublication } from './track/TrackPublication';
|
|
61
63
|
import type { TrackProcessor } from './track/processor/types';
|
62
64
|
import type { AdaptiveStreamSettings } from './track/types';
|
63
65
|
import { getNewAudioContext, sourceToKind } from './track/utils';
|
64
|
-
import type { SimulationOptions, SimulationScenario } from './types';
|
66
|
+
import type { SimulationOptions, SimulationScenario, TranscriptionSegment } from './types';
|
65
67
|
import {
|
66
68
|
Future,
|
67
69
|
Mutex,
|
68
70
|
createDummyVideoStreamTrack,
|
71
|
+
extractTranscriptionSegments,
|
69
72
|
getEmptyAudioStreamTrack,
|
70
73
|
isBrowserSupported,
|
71
74
|
isCloud,
|
@@ -330,6 +333,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
|
|
330
333
|
})
|
331
334
|
.on(EngineEvent.ActiveSpeakersUpdate, this.handleActiveSpeakersUpdate)
|
332
335
|
.on(EngineEvent.DataPacketReceived, this.handleDataPacket)
|
336
|
+
.on(EngineEvent.TranscriptionReceived, this.handleTranscription)
|
333
337
|
.on(EngineEvent.Resuming, () => {
|
334
338
|
this.clearConnectionReconcile();
|
335
339
|
this.isResuming = true;
|
@@ -1471,6 +1475,23 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
|
|
1471
1475
|
participant?.emit(ParticipantEvent.DataReceived, userPacket.payload, kind);
|
1472
1476
|
};
|
1473
1477
|
|
1478
|
+
bufferedSegments: Map<string, TranscriptionSegmentModel> = new Map();
|
1479
|
+
|
1480
|
+
private handleTranscription = (transcription: TranscriptionModel) => {
|
1481
|
+
// find the participant
|
1482
|
+
const participant =
|
1483
|
+
transcription.participantIdentity === this.localParticipant.identity
|
1484
|
+
? this.localParticipant
|
1485
|
+
: this.remoteParticipants.get(transcription.participantIdentity);
|
1486
|
+
const publication = participant?.trackPublications.get(transcription.trackId);
|
1487
|
+
|
1488
|
+
const segments = extractTranscriptionSegments(transcription);
|
1489
|
+
|
1490
|
+
publication?.emit(TrackEvent.TranscriptionReceived, segments);
|
1491
|
+
participant?.emit(ParticipantEvent.TranscriptionReceived, segments, publication);
|
1492
|
+
this.emit(RoomEvent.TranscriptionReceived, segments, participant, publication);
|
1493
|
+
};
|
1494
|
+
|
1474
1495
|
private handleAudioPlaybackStarted = () => {
|
1475
1496
|
if (this.canPlaybackAudio) {
|
1476
1497
|
return;
|
@@ -2071,6 +2092,11 @@ export type RoomEventCallbacks = {
|
|
2071
2092
|
kind?: DataPacket_Kind,
|
2072
2093
|
topic?: string,
|
2073
2094
|
) => void;
|
2095
|
+
transcriptionReceived: (
|
2096
|
+
transcription: TranscriptionSegment[],
|
2097
|
+
participant?: Participant,
|
2098
|
+
publication?: TrackPublication,
|
2099
|
+
) => void;
|
2074
2100
|
connectionQualityChanged: (quality: ConnectionQuality, participant: Participant) => void;
|
2075
2101
|
mediaDevicesError: (error: Error) => void;
|
2076
2102
|
trackStreamStateChanged: (
|
package/src/room/events.ts
CHANGED
@@ -197,6 +197,12 @@ export enum RoomEvent {
|
|
197
197
|
*/
|
198
198
|
DataReceived = 'dataReceived',
|
199
199
|
|
200
|
+
/**
|
201
|
+
* Transcription received from a participant's track.
|
202
|
+
* @beta
|
203
|
+
*/
|
204
|
+
TranscriptionReceived = 'transcriptionReceived',
|
205
|
+
|
200
206
|
/**
|
201
207
|
* Connection quality was changed for a Participant. It'll receive updates
|
202
208
|
* from the local participant, as well as any [[RemoteParticipant]]s that we are
|
@@ -402,6 +408,12 @@ export enum ParticipantEvent {
|
|
402
408
|
*/
|
403
409
|
DataReceived = 'dataReceived',
|
404
410
|
|
411
|
+
/**
|
412
|
+
* Transcription received from this participant as data source.
|
413
|
+
* @beta
|
414
|
+
*/
|
415
|
+
TranscriptionReceived = 'transcriptionReceived',
|
416
|
+
|
405
417
|
/**
|
406
418
|
* Has speaking status changed for the current participant
|
407
419
|
*
|
@@ -479,6 +491,7 @@ export enum EngineEvent {
|
|
479
491
|
MediaTrackAdded = 'mediaTrackAdded',
|
480
492
|
ActiveSpeakersUpdate = 'activeSpeakersUpdate',
|
481
493
|
DataPacketReceived = 'dataPacketReceived',
|
494
|
+
TranscriptionReceived = 'transcriptionReceived',
|
482
495
|
RTPVideoMapUpdate = 'rtpVideoMapUpdate',
|
483
496
|
DCBufferStatusChanged = 'dcBufferStatusChanged',
|
484
497
|
ParticipantUpdate = 'participantUpdate',
|
@@ -562,4 +575,14 @@ export enum TrackEvent {
|
|
562
575
|
* @internal
|
563
576
|
*/
|
564
577
|
AudioTrackFeatureUpdate = 'audioTrackFeatureUpdate',
|
578
|
+
|
579
|
+
/**
|
580
|
+
* @beta
|
581
|
+
*/
|
582
|
+
TranscriptionReceived = 'transcriptionReceived',
|
583
|
+
|
584
|
+
/**
|
585
|
+
* @experimental
|
586
|
+
*/
|
587
|
+
TimeSyncUpdate = 'timeSyncUpdate',
|
565
588
|
}
|
@@ -40,9 +40,9 @@ import {
|
|
40
40
|
import type { DataPublishOptions } from '../types';
|
41
41
|
import {
|
42
42
|
Future,
|
43
|
+
isE2EESimulcastSupported,
|
43
44
|
isFireFox,
|
44
45
|
isSVCCodec,
|
45
|
-
isSafari,
|
46
46
|
isSafari17,
|
47
47
|
isWeb,
|
48
48
|
supportsAV1,
|
@@ -621,10 +621,9 @@ export default class LocalParticipant extends Participant {
|
|
621
621
|
...options,
|
622
622
|
};
|
623
623
|
|
624
|
-
|
625
|
-
if (isSafari() && this.roomOptions.e2ee) {
|
624
|
+
if (!isE2EESimulcastSupported() && this.roomOptions.e2ee) {
|
626
625
|
this.log.info(
|
627
|
-
`End-to-end encryption is set up, simulcast publishing will be disabled on Safari`,
|
626
|
+
`End-to-end encryption is set up, simulcast publishing will be disabled on Safari versions and iOS browsers running iOS < v17.2`,
|
628
627
|
{
|
629
628
|
...this.logContext,
|
630
629
|
},
|
@@ -827,7 +826,6 @@ export default class LocalParticipant extends Participant {
|
|
827
826
|
...getLogContextFromTrack(track),
|
828
827
|
codec: updatedCodec,
|
829
828
|
});
|
830
|
-
/* @ts-ignore */
|
831
829
|
opts.videoCodec = updatedCodec;
|
832
830
|
|
833
831
|
// recompute encodings since bitrates/etc could have changed
|
@@ -16,7 +16,7 @@ import type RemoteTrack from '../track/RemoteTrack';
|
|
16
16
|
import type RemoteTrackPublication from '../track/RemoteTrackPublication';
|
17
17
|
import { Track } from '../track/Track';
|
18
18
|
import type { TrackPublication } from '../track/TrackPublication';
|
19
|
-
import type { LoggerOptions } from '../types';
|
19
|
+
import type { LoggerOptions, TranscriptionSegment } from '../types';
|
20
20
|
|
21
21
|
export enum ConnectionQuality {
|
22
22
|
Excellent = 'excellent',
|
@@ -329,6 +329,10 @@ export type ParticipantEventCallbacks = {
|
|
329
329
|
participantMetadataChanged: (prevMetadata: string | undefined, participant?: any) => void;
|
330
330
|
participantNameChanged: (name: string) => void;
|
331
331
|
dataReceived: (payload: Uint8Array, kind: DataPacket_Kind) => void;
|
332
|
+
transcriptionReceived: (
|
333
|
+
transcription: TranscriptionSegment[],
|
334
|
+
publication?: TrackPublication,
|
335
|
+
) => void;
|
332
336
|
isSpeakingChanged: (speaking: boolean) => void;
|
333
337
|
connectionQualityChanged: (connectionQuality: ConnectionQuality) => void;
|
334
338
|
trackStreamStateChanged: (
|
@@ -77,7 +77,20 @@ export default abstract class RemoteTrack<
|
|
77
77
|
if (!this.monitorInterval) {
|
78
78
|
this.monitorInterval = setInterval(() => this.monitorReceiver(), monitorFrequency);
|
79
79
|
}
|
80
|
+
this.registerTimeSyncUpdate();
|
80
81
|
}
|
81
82
|
|
82
83
|
protected abstract monitorReceiver(): void;
|
84
|
+
|
85
|
+
registerTimeSyncUpdate() {
|
86
|
+
const loop = () => {
|
87
|
+
this.timeSyncHandle = requestAnimationFrame(() => loop());
|
88
|
+
const newTime = this.receiver?.getSynchronizationSources()[0]?.rtpTimestamp;
|
89
|
+
if (newTime && this.rtpTimestamp !== newTime) {
|
90
|
+
this.emit(TrackEvent.TimeSyncUpdate, newTime);
|
91
|
+
this.rtpTimestamp = newTime;
|
92
|
+
}
|
93
|
+
};
|
94
|
+
loop();
|
95
|
+
}
|
83
96
|
}
|
package/src/room/track/Track.ts
CHANGED
@@ -53,6 +53,9 @@ export abstract class Track<
|
|
53
53
|
*/
|
54
54
|
streamState: Track.StreamState = Track.StreamState.Active;
|
55
55
|
|
56
|
+
/** @internal */
|
57
|
+
rtpTimestamp: number | undefined;
|
58
|
+
|
56
59
|
protected _mediaStreamTrack: MediaStreamTrack;
|
57
60
|
|
58
61
|
protected _mediaStreamID: string;
|
@@ -63,6 +66,8 @@ export abstract class Track<
|
|
63
66
|
|
64
67
|
private loggerContextCb: LoggerOptions['loggerContextCb'];
|
65
68
|
|
69
|
+
protected timeSyncHandle: number | undefined;
|
70
|
+
|
66
71
|
protected _currentBitrate: number = 0;
|
67
72
|
|
68
73
|
protected monitorInterval?: ReturnType<typeof setInterval>;
|
@@ -255,6 +260,9 @@ export abstract class Track<
|
|
255
260
|
if (this.monitorInterval) {
|
256
261
|
clearInterval(this.monitorInterval);
|
257
262
|
}
|
263
|
+
if (this.timeSyncHandle) {
|
264
|
+
cancelAnimationFrame(this.timeSyncHandle);
|
265
|
+
}
|
258
266
|
}
|
259
267
|
|
260
268
|
/** @internal */
|
@@ -517,4 +525,5 @@ export type TrackEventCallbacks = {
|
|
517
525
|
upstreamResumed: (track: any) => void;
|
518
526
|
trackProcessorUpdate: (processor?: TrackProcessor<Track.Kind, any>) => void;
|
519
527
|
audioTrackFeatureUpdate: (track: any, feature: AudioTrackFeature, enabled: boolean) => void;
|
528
|
+
timeSyncUpdate: (timestamp: number) => void;
|
520
529
|
};
|
@@ -9,7 +9,7 @@ import { EventEmitter } from 'events';
|
|
9
9
|
import type TypedEventEmitter from 'typed-emitter';
|
10
10
|
import log, { LoggerNames, getLogger } from '../../logger';
|
11
11
|
import { TrackEvent } from '../events';
|
12
|
-
import type { LoggerOptions } from '../types';
|
12
|
+
import type { LoggerOptions, TranscriptionSegment } from '../types';
|
13
13
|
import LocalAudioTrack from './LocalAudioTrack';
|
14
14
|
import LocalVideoTrack from './LocalVideoTrack';
|
15
15
|
import RemoteAudioTrack from './RemoteAudioTrack';
|
@@ -174,4 +174,6 @@ export type PublicationEventCallbacks = {
|
|
174
174
|
prevStatus: TrackPublication.SubscriptionStatus,
|
175
175
|
) => void;
|
176
176
|
subscriptionFailed: (error: SubscriptionError) => void;
|
177
|
+
transcriptionReceived: (transcription: TranscriptionSegment[]) => void;
|
178
|
+
timeSyncUpdate: (timestamp: number) => void;
|
177
179
|
};
|
package/src/room/types.ts
CHANGED
@@ -55,3 +55,12 @@ export type LoggerOptions = {
|
|
55
55
|
loggerName?: string;
|
56
56
|
loggerContextCb?: () => Record<string, unknown>;
|
57
57
|
};
|
58
|
+
|
59
|
+
export interface TranscriptionSegment {
|
60
|
+
id: string;
|
61
|
+
text: string;
|
62
|
+
language: string;
|
63
|
+
startTime: number;
|
64
|
+
endTime: number;
|
65
|
+
final: boolean;
|
66
|
+
}
|