livekit-client 2.1.1 → 2.1.3

package/dist/ts4.2/src/room/track/TrackPublication.d.ts

@@ -1,7 +1,7 @@
 import { Encryption_Type } from '@livekit/protocol';
 import type { SubscriptionError, TrackInfo, UpdateSubscription, UpdateTrackSettings } from '@livekit/protocol';
 import type TypedEventEmitter from 'typed-emitter';
-import type { LoggerOptions } from '../types';
+import type { LoggerOptions, TranscriptionSegment } from '../types';
 import LocalAudioTrack from './LocalAudioTrack';
 import LocalVideoTrack from './LocalVideoTrack';
 import RemoteAudioTrack from './RemoteAudioTrack';
@@ -72,6 +72,8 @@ export type PublicationEventCallbacks = {
     unsubscribed: (track: RemoteTrack) => void;
     subscriptionStatusChanged: (status: TrackPublication.SubscriptionStatus, prevStatus: TrackPublication.SubscriptionStatus) => void;
     subscriptionFailed: (error: SubscriptionError) => void;
+    transcriptionReceived: (transcription: TranscriptionSegment[]) => void;
+    timeSyncUpdate: (timestamp: number) => void;
 };
 export {};
 //# sourceMappingURL=TrackPublication.d.ts.map

package/dist/ts4.2/src/room/types.d.ts

@@ -35,4 +35,12 @@ export type LoggerOptions = {
     loggerName?: string;
     loggerContextCb?: () => Record<string, unknown>;
 };
+export interface TranscriptionSegment {
+    id: string;
+    text: string;
+    language: string;
+    startTime: number;
+    endTime: number;
+    final: boolean;
+}
 //# sourceMappingURL=types.d.ts.map

package/dist/ts4.2/src/room/utils.d.ts

@@ -1,7 +1,8 @@
-import { ClientInfo } from '@livekit/protocol';
+import { ClientInfo, Transcription as TranscriptionModel } from '@livekit/protocol';
 import type LocalAudioTrack from './track/LocalAudioTrack';
 import type RemoteAudioTrack from './track/RemoteAudioTrack';
 import { VideoCodec } from './track/options';
+import type { TranscriptionSegment } from './types';
 export declare const ddExtensionURI = "https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension";
 export declare function unpackStreamId(packed: string): string[];
 export declare function sleep(duration: number): Promise<void>;
@@ -93,4 +94,5 @@ export declare function isVideoCodec(maybeCodec: string): maybeCodec is VideoCod
 export declare function unwrapConstraint(constraint: ConstrainDOMString): string;
 export declare function toWebsocketUrl(url: string): string;
 export declare function toHttpUrl(url: string): string;
+export declare function extractTranscriptionSegments(transcription: TranscriptionModel): TranscriptionSegment[];
 //# sourceMappingURL=utils.d.ts.map

package/dist/ts4.2/src/version.d.ts

@@ -1,3 +1,3 @@
 export declare const version: string;
-export declare const protocolVersion = 12;
+export declare const protocolVersion = 13;
 //# sourceMappingURL=version.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "livekit-client",
-  "version": "2.1.1",
+  "version": "2.1.3",
   "description": "JavaScript/TypeScript client SDK for LiveKit",
   "main": "./dist/livekit-client.umd.js",
   "unpkg": "./dist/livekit-client.umd.js",
@@ -36,7 +36,7 @@
   "author": "David Zhao <david@davidzhao.com>",
   "license": "Apache-2.0",
   "dependencies": {
-    "@livekit/protocol": "1.13.0",
+    "@livekit/protocol": "1.15.0",
     "events": "^3.3.0",
     "loglevel": "^1.8.0",
     "sdp-transform": "^2.14.1",
@@ -46,8 +46,8 @@
     "webrtc-adapter": "^8.1.1"
   },
   "devDependencies": {
-    "@babel/core": "7.24.4",
-    "@babel/preset-env": "7.24.4",
+    "@babel/core": "7.24.5",
+    "@babel/preset-env": "7.24.5",
     "@bufbuild/protoc-gen-es": "^1.3.0",
     "@changesets/cli": "2.27.1",
     "@livekit/changesets-changelog-github": "^0.0.4",
@@ -73,15 +73,15 @@
     "gh-pages": "6.1.1",
     "jsdom": "^24.0.0",
     "prettier": "^3.0.0",
-    "rollup": "4.14.0",
+    "rollup": "4.17.2",
     "rollup-plugin-delete": "^2.0.0",
     "rollup-plugin-re": "1.0.7",
     "rollup-plugin-typescript2": "0.36.0",
     "size-limit": "^8.2.4",
-    "typedoc": "0.25.12",
+    "typedoc": "0.25.13",
     "typedoc-plugin-no-inherit": "1.4.0",
-    "typescript": "5.4.3",
-    "vite": "5.0.13",
+    "typescript": "5.4.5",
+    "vite": "5.2.10",
     "vitest": "^1.0.0"
   },
   "scripts": {

package/src/api/SignalClient.ts

@@ -6,6 +6,7 @@ import {
   DisconnectReason,
   JoinResponse,
   LeaveRequest,
+  LeaveRequest_Action,
   MuteTrackRequest,
   ParticipantInfo,
   Ping,
@@ -596,8 +597,9 @@ export class SignalClient {
     return this.sendRequest({
       case: 'leave',
       value: new LeaveRequest({
-        canReconnect: false,
         reason: DisconnectReason.CLIENT_INITIATED,
+        // server doesn't process this field, keeping it here to indicate the intent of a full disconnect
+        action: LeaveRequest_Action.DISCONNECT,
       }),
     });
   }

package/src/index.ts

@@ -44,7 +44,7 @@ export { facingModeFromDeviceLabel, facingModeFromLocalTrack } from './room/trac
 export * from './room/track/options';
 export * from './room/track/processor/types';
 export * from './room/track/types';
-export type { DataPublishOptions, SimulationScenario } from './room/types';
+export type { DataPublishOptions, SimulationScenario, TranscriptionSegment } from './room/types';
 export * from './version';
 export {
   ConnectionQuality,

package/src/room/RTCEngine.ts

@@ -9,6 +9,7 @@ import {
   DisconnectReason,
   type JoinResponse,
   type LeaveRequest,
+  LeaveRequest_Action,
   ParticipantInfo,
   ReconnectReason,
   type ReconnectResponse,
@@ -23,6 +24,7 @@ import {
   TrackInfo,
   type TrackPublishedResponse,
   TrackUnpublishedResponse,
+  Transcription,
   UpdateSubscription,
   UserPacket,
 } from '@livekit/protocol';
@@ -434,7 +436,9 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
       this.emit(EngineEvent.MediaTrackAdded, ev.track, ev.streams[0], ev.receiver);
     };
 
-    this.createDataChannels();
+    if (!supportOptionalDatachannel(joinResponse.serverInfo?.protocol)) {
+      this.createDataChannels();
+    }
   }
 
   private setupSignalClientCallbacks() {
@@ -503,16 +507,28 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
       this.handleDisconnect('signal', ReconnectReason.RR_SIGNAL_DISCONNECTED);
     };
 
-    this.client.onLeave = (leave?: LeaveRequest) => {
-      if (leave?.canReconnect) {
-        this.fullReconnectOnNext = true;
-        // reconnect immediately instead of waiting for next attempt
-        this.handleDisconnect(leaveReconnect);
-      } else {
-        this.emit(EngineEvent.Disconnected, leave?.reason);
-        this.close();
-      }
+    this.client.onLeave = (leave: LeaveRequest) => {
       this.log.debug('client leave request', { ...this.logContext, reason: leave?.reason });
+      if (leave.regions && this.regionUrlProvider) {
+        this.log.debug('updating regions', this.logContext);
+        this.regionUrlProvider.setServerReportedRegions(leave.regions);
+      }
+      switch (leave.action) {
+        case LeaveRequest_Action.DISCONNECT:
+          this.emit(EngineEvent.Disconnected, leave?.reason);
+          this.close();
+          break;
+        case LeaveRequest_Action.RECONNECT:
+          this.fullReconnectOnNext = true;
+          // reconnect immediately instead of waiting for next attempt
+          this.handleDisconnect(leaveReconnect);
+          break;
+        case LeaveRequest_Action.RESUME:
+          // reconnect immediately instead of waiting for next attempt
+          this.handleDisconnect(leaveReconnect);
+        default:
+          break;
+      }
     };
   }
 
@@ -634,6 +650,8 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
         this.emit(EngineEvent.ActiveSpeakersUpdate, dp.value.value.speakers);
       } else if (dp.value?.case === 'user') {
         this.emit(EngineEvent.DataPacketReceived, dp.value.value, dp.kind);
+      } else if (dp.value?.case === 'transcription') {
+        this.emit(EngineEvent.TranscriptionReceived, dp.value.value);
       }
     } finally {
       unlock();
@@ -1100,11 +1118,21 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
       throw new ConnectionError(`${transportName} connection not set`);
     }
 
+    let needNegotiation = false;
+    if (!subscriber && !this.dataChannelForKind(kind, subscriber)) {
+      this.createDataChannels();
+      needNegotiation = true;
+    }
+
     if (
+      !needNegotiation &&
       !subscriber &&
       !this.pcManager.publisher.isICEConnected &&
       this.pcManager.publisher.getICEConnectionState() !== 'checking'
     ) {
+      needNegotiation = true;
+    }
+    if (needNegotiation) {
       // start negotiation
       this.negotiate();
     }
@@ -1165,6 +1193,14 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
       }
 
       this.pcManager.requirePublisher();
+      // don't negotiate without any transceivers or data channel, it will generate sdp without ice frag then negotiate failed
+      if (
+        this.pcManager.publisher.getTransceivers().length == 0 &&
+        !this.lossyDC &&
+        !this.reliableDC
+      ) {
+        this.createDataChannels();
+      }
 
       const abortController = new AbortController();
 
@@ -1357,6 +1393,7 @@ export type EngineEventCallbacks = {
   ) => void;
   activeSpeakersUpdate: (speakers: Array<SpeakerInfo>) => void;
   dataPacketReceived: (userPacket: UserPacket, kind: DataPacket_Kind) => void;
+  transcriptionReceived: (transcription: Transcription) => void;
   transportsCreated: (publisher: PCTransport, subscriber: PCTransport) => void;
   /** @internal */
   trackSenderAdded: (track: Track, sender: RTCRtpSender) => void;
@@ -1374,3 +1411,7 @@ export type EngineEventCallbacks = {
   remoteMute: (trackSid: string, muted: boolean) => void;
   offline: () => void;
 };
+
+function supportOptionalDatachannel(protocol: number | undefined): boolean {
+  return protocol !== undefined && protocol > 13;
+}

package/src/room/RegionUrlProvider.ts

@@ -75,6 +75,11 @@ export class RegionUrlProvider {
       );
     }
   }
+
+  setServerReportedRegions(regions: RegionSettings) {
+    this.regionSettings = regions;
+    this.lastUpdateAt = Date.now();
+  }
 }
 
 function getCloudConfigUrl(serverUrl: URL) {

package/src/room/Room.ts

@@ -4,6 +4,7 @@ import {
   DisconnectReason,
   JoinResponse,
   LeaveRequest,
+  LeaveRequest_Action,
   ParticipantInfo,
   ParticipantInfo_State,
   ParticipantPermission,
@@ -18,6 +19,8 @@ import {
   TrackInfo,
   TrackSource,
   TrackType,
+  Transcription as TranscriptionModel,
+  TranscriptionSegment as TranscriptionSegmentModel,
   UserPacket,
   protoInt64,
 } from '@livekit/protocol';
@@ -61,11 +64,12 @@ import type { TrackPublication } from './track/TrackPublication';
 import type { TrackProcessor } from './track/processor/types';
 import type { AdaptiveStreamSettings } from './track/types';
 import { getNewAudioContext, sourceToKind } from './track/utils';
-import type { SimulationOptions, SimulationScenario } from './types';
+import type { SimulationOptions, SimulationScenario, TranscriptionSegment } from './types';
 import {
   Future,
   Mutex,
   createDummyVideoStreamTrack,
+  extractTranscriptionSegments,
   getEmptyAudioStreamTrack,
   isBrowserSupported,
   isCloud,
@@ -330,6 +334,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       })
       .on(EngineEvent.ActiveSpeakersUpdate, this.handleActiveSpeakersUpdate)
       .on(EngineEvent.DataPacketReceived, this.handleDataPacket)
+      .on(EngineEvent.TranscriptionReceived, this.handleTranscription)
       .on(EngineEvent.Resuming, () => {
         this.clearConnectionReconcile();
         this.isResuming = true;
@@ -855,7 +860,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
             onLeave(
               new LeaveRequest({
                 reason: DisconnectReason.CLIENT_INITIATED,
-                canReconnect: true,
+                action: LeaveRequest_Action.RECONNECT,
               }),
             );
           }
@@ -1471,6 +1476,23 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     participant?.emit(ParticipantEvent.DataReceived, userPacket.payload, kind);
   };
 
+  bufferedSegments: Map<string, TranscriptionSegmentModel> = new Map();
+
+  private handleTranscription = (transcription: TranscriptionModel) => {
+    // find the participant
+    const participant =
+      transcription.participantIdentity === this.localParticipant.identity
+        ? this.localParticipant
+        : this.remoteParticipants.get(transcription.participantIdentity);
+    const publication = participant?.trackPublications.get(transcription.trackId);
+
+    const segments = extractTranscriptionSegments(transcription);
+
+    publication?.emit(TrackEvent.TranscriptionReceived, segments);
+    participant?.emit(ParticipantEvent.TranscriptionReceived, segments, publication);
+    this.emit(RoomEvent.TranscriptionReceived, segments, participant, publication);
+  };
+
   private handleAudioPlaybackStarted = () => {
     if (this.canPlaybackAudio) {
       return;
@@ -2071,6 +2093,11 @@ export type RoomEventCallbacks = {
     kind?: DataPacket_Kind,
     topic?: string,
   ) => void;
+  transcriptionReceived: (
+    transcription: TranscriptionSegment[],
+    participant?: Participant,
+    publication?: TrackPublication,
+  ) => void;
   connectionQualityChanged: (quality: ConnectionQuality, participant: Participant) => void;
   mediaDevicesError: (error: Error) => void;
   trackStreamStateChanged: (

package/src/room/events.ts

@@ -197,6 +197,12 @@ export enum RoomEvent {
    */
   DataReceived = 'dataReceived',
 
+  /**
+   * Transcription received from a participant's track.
+   * @beta
+   */
+  TranscriptionReceived = 'transcriptionReceived',
+
   /**
    * Connection quality was changed for a Participant. It'll receive updates
    * from the local participant, as well as any [[RemoteParticipant]]s that we are
@@ -402,6 +408,12 @@ export enum ParticipantEvent {
    */
   DataReceived = 'dataReceived',
 
+  /**
+   * Transcription received from this participant as data source.
+   * @beta
+   */
+  TranscriptionReceived = 'transcriptionReceived',
+
   /**
    * Has speaking status changed for the current participant
    *
@@ -479,6 +491,7 @@ export enum EngineEvent {
   MediaTrackAdded = 'mediaTrackAdded',
   ActiveSpeakersUpdate = 'activeSpeakersUpdate',
   DataPacketReceived = 'dataPacketReceived',
+  TranscriptionReceived = 'transcriptionReceived',
   RTPVideoMapUpdate = 'rtpVideoMapUpdate',
   DCBufferStatusChanged = 'dcBufferStatusChanged',
   ParticipantUpdate = 'participantUpdate',
@@ -562,4 +575,14 @@ export enum TrackEvent {
    * @internal
    */
   AudioTrackFeatureUpdate = 'audioTrackFeatureUpdate',
+
+  /**
+   * @beta
+   */
+  TranscriptionReceived = 'transcriptionReceived',
+
+  /**
+   * @experimental
+   */
+  TimeSyncUpdate = 'timeSyncUpdate',
 }

package/src/room/participant/Participant.ts

@@ -16,7 +16,7 @@ import type RemoteTrack from '../track/RemoteTrack';
 import type RemoteTrackPublication from '../track/RemoteTrackPublication';
 import { Track } from '../track/Track';
 import type { TrackPublication } from '../track/TrackPublication';
-import type { LoggerOptions } from '../types';
+import type { LoggerOptions, TranscriptionSegment } from '../types';
 
 export enum ConnectionQuality {
   Excellent = 'excellent',
@@ -329,6 +329,10 @@ export type ParticipantEventCallbacks = {
   participantMetadataChanged: (prevMetadata: string | undefined, participant?: any) => void;
   participantNameChanged: (name: string) => void;
   dataReceived: (payload: Uint8Array, kind: DataPacket_Kind) => void;
+  transcriptionReceived: (
+    transcription: TranscriptionSegment[],
+    publication?: TrackPublication,
+  ) => void;
   isSpeakingChanged: (speaking: boolean) => void;
   connectionQualityChanged: (connectionQuality: ConnectionQuality) => void;
   trackStreamStateChanged: (

package/src/room/track/RemoteTrack.ts

@@ -77,7 +77,23 @@ export default abstract class RemoteTrack<
     if (!this.monitorInterval) {
       this.monitorInterval = setInterval(() => this.monitorReceiver(), monitorFrequency);
     }
+    this.registerTimeSyncUpdate();
   }
 
   protected abstract monitorReceiver(): void;
+
+  registerTimeSyncUpdate() {
+    const loop = () => {
+      this.timeSyncHandle = requestAnimationFrame(() => loop());
+      const sources = this.receiver?.getSynchronizationSources()[0];
+      if (sources) {
+        const { timestamp, rtpTimestamp } = sources;
+        if (rtpTimestamp && this.rtpTimestamp !== rtpTimestamp) {
+          this.emit(TrackEvent.TimeSyncUpdate, { timestamp, rtpTimestamp });
+          this.rtpTimestamp = rtpTimestamp;
+        }
+      }
+    };
+    loop();
+  }
 }

package/src/room/track/Track.ts

@@ -53,6 +53,9 @@ export abstract class Track<
    */
   streamState: Track.StreamState = Track.StreamState.Active;
 
+  /** @internal */
+  rtpTimestamp: number | undefined;
+
   protected _mediaStreamTrack: MediaStreamTrack;
 
   protected _mediaStreamID: string;
@@ -63,6 +66,8 @@ export abstract class Track<
 
   private loggerContextCb: LoggerOptions['loggerContextCb'];
 
+  protected timeSyncHandle: number | undefined;
+
   protected _currentBitrate: number = 0;
 
   protected monitorInterval?: ReturnType<typeof setInterval>;
@@ -255,6 +260,9 @@ export abstract class Track<
     if (this.monitorInterval) {
       clearInterval(this.monitorInterval);
     }
+    if (this.timeSyncHandle) {
+      cancelAnimationFrame(this.timeSyncHandle);
+    }
   }
 
   /** @internal */
@@ -517,4 +525,5 @@ export type TrackEventCallbacks = {
   upstreamResumed: (track: any) => void;
   trackProcessorUpdate: (processor?: TrackProcessor<Track.Kind, any>) => void;
   audioTrackFeatureUpdate: (track: any, feature: AudioTrackFeature, enabled: boolean) => void;
+  timeSyncUpdate: (update: { timestamp: number; rtpTimestamp: number }) => void;
 };

package/src/room/track/TrackPublication.ts

@@ -9,7 +9,7 @@ import { EventEmitter } from 'events';
 import type TypedEventEmitter from 'typed-emitter';
 import log, { LoggerNames, getLogger } from '../../logger';
 import { TrackEvent } from '../events';
-import type { LoggerOptions } from '../types';
+import type { LoggerOptions, TranscriptionSegment } from '../types';
 import LocalAudioTrack from './LocalAudioTrack';
 import LocalVideoTrack from './LocalVideoTrack';
 import RemoteAudioTrack from './RemoteAudioTrack';
@@ -174,4 +174,6 @@ export type PublicationEventCallbacks = {
     prevStatus: TrackPublication.SubscriptionStatus,
   ) => void;
   subscriptionFailed: (error: SubscriptionError) => void;
+  transcriptionReceived: (transcription: TranscriptionSegment[]) => void;
+  timeSyncUpdate: (timestamp: number) => void;
 };

package/src/room/types.ts

@@ -55,3 +55,12 @@ export type LoggerOptions = {
   loggerName?: string;
   loggerContextCb?: () => Record<string, unknown>;
 };
+
+export interface TranscriptionSegment {
+  id: string;
+  text: string;
+  language: string;
+  startTime: number;
+  endTime: number;
+  final: boolean;
+}

package/src/room/utils.ts

@@ -1,4 +1,4 @@
-import { ClientInfo, ClientInfo_SDK } from '@livekit/protocol';
+import { ClientInfo, ClientInfo_SDK, Transcription as TranscriptionModel } from '@livekit/protocol';
 import { getBrowser } from '../utils/browserParser';
 import { protocolVersion, version } from '../version';
 import CriticalTimers from './timers';
@@ -6,7 +6,7 @@ import type LocalAudioTrack from './track/LocalAudioTrack';
 import type RemoteAudioTrack from './track/RemoteAudioTrack';
 import { VideoCodec, videoCodecs } from './track/options';
 import { getNewAudioContext } from './track/utils';
-import type { LiveKitReactNativeInfo } from './types';
+import type { LiveKitReactNativeInfo, TranscriptionSegment } from './types';
 
 const separator = '|';
 export const ddExtensionURI =
@@ -527,3 +527,18 @@ export function toHttpUrl(url: string): string {
   }
   return url;
 }
+
+export function extractTranscriptionSegments(
+  transcription: TranscriptionModel,
+): TranscriptionSegment[] {
+  return transcription.segments.map(({ id, text, language, startTime, endTime, final }) => {
+    return {
+      id,
+      text,
+      startTime: Number.parseInt(startTime.toString()),
+      endTime: Number.parseInt(endTime.toString()),
+      final,
+      language,
+    };
+  });
+}

package/src/version.ts

@@ -1,4 +1,4 @@
 import { version as v } from '../package.json';
 
 export const version = v;
-export const protocolVersion = 12;
+export const protocolVersion = 13;