livekit-client 2.13.1 → 2.13.2

package/src/room/Room.ts

@@ -65,7 +65,7 @@ import { ConnectionError, ConnectionErrorReason, UnsupportedServer } from './err
 import { EngineEvent, ParticipantEvent, RoomEvent, TrackEvent } from './events';
 import LocalParticipant from './participant/LocalParticipant';
 import type Participant from './participant/Participant';
-import type { ConnectionQuality } from './participant/Participant';
+import { type ConnectionQuality, ParticipantKind } from './participant/Participant';
 import RemoteParticipant from './participant/RemoteParticipant';
 import { MAX_PAYLOAD_BYTES, RpcError, type RpcInvocationData, byteLength } from './rpc';
 import CriticalTimers from './timers';
@@ -1992,7 +1992,10 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     }
   };
 
-  private handleDeviceChange = async () => {
+  /**
+   * attempt to select the default devices if the previously selected devices are no longer available after a device change event
+   */
+  private async selectDefaultDevices() {
     const previousDevices = DeviceManager.getInstance().previousDevices;
     // check for available devices, but don't request permissions in order to avoid prompts for kinds that haven't been used before
     const availableDevices = await DeviceManager.getInstance().getDevices(undefined, false);
@@ -2055,7 +2058,13 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
         await this.switchActiveDevice(kind, devicesOfKind[0].deviceId);
       }
     }
+  }
 
+  private handleDeviceChange = async () => {
+    if (getBrowser()?.os !== 'iOS') {
+      // default devices are non deterministic on iOS, so we don't attempt to select them here
+      await this.selectDefaultDevices();
+    }
     this.emit(RoomEvent.MediaDevicesChanged);
   };
 
@@ -2247,6 +2256,9 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       })
       .on(ParticipantEvent.Active, () => {
         this.emitWhenConnected(RoomEvent.ParticipantActive, participant);
+        if (participant.kind === ParticipantKind.AGENT) {
+          this.localParticipant.setActiveAgent(participant);
+        }
       });
 
     // update info at the end after callbacks have been set up

package/src/room/defaults.ts

@@ -19,6 +19,7 @@ export const publishDefaults: TrackPublishDefaults = {
   stopMicTrackOnMute: false,
   videoCodec: defaultVideoCodec,
   backupCodec: true,
+  preConnectBuffer: false,
 } as const;
 
 export const audioDefaults: AudioCaptureOptions = {

package/src/room/events.ts

@@ -666,4 +666,9 @@ export enum TrackEvent {
    * @experimental
    */
   TimeSyncUpdate = 'timeSyncUpdate',
+
+  /**
+   * @internal
+   */
+  PreConnectBufferFlushed = 'preConnectBufferFlushed',
 }

package/src/room/participant/LocalParticipant.ts

@@ -1,6 +1,7 @@
 import { Mutex } from '@livekit/mutex';
 import {
   AddTrackRequest,
+  AudioTrackFeature,
   BackupCodecPolicy,
   ChatMessage as ChatMessageModel,
   Codec,
@@ -13,6 +14,7 @@ import {
   DataStream_TextHeader,
   DataStream_Trailer,
   Encryption_Type,
+  JoinResponse,
   ParticipantInfo,
   ParticipantPermission,
   RequestResponse,
@@ -103,6 +105,7 @@ import {
 import Participant from './Participant';
 import type { ParticipantTrackPermission } from './ParticipantTrackPermission';
 import { trackPermissionToProto } from './ParticipantTrackPermission';
+import type RemoteParticipant from './RemoteParticipant';
 import {
   computeTrackBackupEncodings,
   computeVideoEncodings,
@@ -146,6 +149,12 @@ export default class LocalParticipant extends Participant {
 
   private reconnectFuture?: Future<void>;
 
+  private signalConnectedFuture?: Future<void>;
+
+  private activeAgentFuture?: Future<RemoteParticipant>;
+
+  private firstActiveAgent?: RemoteParticipant;
+
   private rpcHandlers: Map<string, (data: RpcInvocationData) => Promise<string>>;
 
   private pendingSignalRequests: Map<
@@ -241,6 +250,7 @@ export default class LocalParticipant extends Participant {
 
     this.engine
       .on(EngineEvent.Connected, this.handleReconnected)
+      .on(EngineEvent.SignalConnected, this.handleSignalConnected)
       .on(EngineEvent.SignalRestarted, this.handleReconnected)
       .on(EngineEvent.SignalResumed, this.handleReconnected)
       .on(EngineEvent.Restarting, this.handleReconnecting)
@@ -270,6 +280,25 @@ export default class LocalParticipant extends Participant {
       this.reconnectFuture?.reject?.('Got disconnected during reconnection attempt');
       this.reconnectFuture = undefined;
     }
+    if (this.signalConnectedFuture) {
+      this.signalConnectedFuture.reject?.('Got disconnected without signal connected');
+      this.signalConnectedFuture = undefined;
+    }
+
+    this.activeAgentFuture?.reject?.('Got disconnected without active agent present');
+    this.activeAgentFuture = undefined;
+    this.firstActiveAgent = undefined;
+  };
+
+  private handleSignalConnected = (joinResponse: JoinResponse) => {
+    if (joinResponse.participant) {
+      this.updateInfo(joinResponse.participant);
+    }
+    if (!this.signalConnectedFuture) {
+      this.signalConnectedFuture = new Future<void>();
+    }
+
+    this.signalConnectedFuture.resolve?.();
   };
 
   private handleSignalRequestResponse = (response: RequestResponse) => {
@@ -523,6 +552,20 @@ export default class LocalParticipant extends Participant {
           this.pendingPublishing.delete(source);
           throw e;
         }
+
+        for (const localTrack of localTracks) {
+          if (
+            source === Track.Source.Microphone &&
+            isAudioTrack(localTrack) &&
+            publishOptions?.preConnectBuffer
+          ) {
+            this.log.info('starting preconnect buffer for microphone', {
+              ...this.logContext,
+            });
+            localTrack.startPreConnectBuffer();
+          }
+        }
+
         try {
           const publishPromises: Array<Promise<LocalTrackPublication>> = [];
           for (const localTrack of localTracks) {
@@ -849,16 +892,8 @@ export default class LocalParticipant extends Participant {
             ...this.logContext,
             track: getLogContextFromTrack(track),
           });
-          const onSignalConnected = async () => {
-            try {
-              const publication = await this.publish(track, opts, isStereo);
-              resolve(publication);
-            } catch (e) {
-              reject(e);
-            }
-          };
-          setTimeout(() => {
-            this.engine.off(EngineEvent.SignalConnected, onSignalConnected);
+
+          const timeout = setTimeout(() => {
             reject(
               new PublishTrackError(
                 'publishing rejected as engine not connected within timeout',
@@ -866,11 +901,10 @@ export default class LocalParticipant extends Participant {
               ),
             );
           }, 15_000);
-          this.engine.once(EngineEvent.SignalConnected, onSignalConnected);
-          this.engine.on(EngineEvent.Closing, () => {
-            this.engine.off(EngineEvent.SignalConnected, onSignalConnected);
-            reject(new PublishTrackError('publishing rejected as engine closed', 499));
-          });
+          await this.waitUntilEngineConnected();
+          clearTimeout(timeout);
+          const publication = await this.publish(track, opts, isStereo);
+          resolve(publication);
         } else {
           try {
             const publication = await this.publish(track, opts, isStereo);
@@ -894,6 +928,13 @@ export default class LocalParticipant extends Participant {
     }
   }
 
+  private waitUntilEngineConnected() {
+    if (!this.signalConnectedFuture) {
+      this.signalConnectedFuture = new Future<void>();
+    }
+    return this.signalConnectedFuture.promise;
+  }
+
   private hasPermissionsToPublish(track: LocalTrack): boolean {
     if (!this.permissions) {
       this.log.warn('no permissions present for publishing track', {
@@ -971,6 +1012,30 @@ export default class LocalParticipant extends Participant {
     track.on(TrackEvent.UpstreamResumed, this.onTrackUpstreamResumed);
     track.on(TrackEvent.AudioTrackFeatureUpdate, this.onTrackFeatureUpdate);
 
+    const audioFeatures: AudioTrackFeature[] = [];
+    const disableDtx = !(opts.dtx ?? true);
+
+    const settings = track.getSourceTrackSettings();
+
+    if (settings.autoGainControl) {
+      audioFeatures.push(AudioTrackFeature.TF_AUTO_GAIN_CONTROL);
+    }
+    if (settings.echoCancellation) {
+      audioFeatures.push(AudioTrackFeature.TF_ECHO_CANCELLATION);
+    }
+    if (settings.noiseSuppression) {
+      audioFeatures.push(AudioTrackFeature.TF_NOISE_SUPPRESSION);
+    }
+    if (settings.channelCount && settings.channelCount > 1) {
+      audioFeatures.push(AudioTrackFeature.TF_STEREO);
+    }
+    if (disableDtx) {
+      audioFeatures.push(AudioTrackFeature.TF_NO_DTX);
+    }
+    if (isLocalAudioTrack(track) && track.hasPreConnectBuffer) {
+      audioFeatures.push(AudioTrackFeature.TF_PRECONNECT_BUFFER);
+    }
+
     // create track publication from track
     const req = new AddTrackRequest({
       // get local track id for use during publishing
@@ -979,12 +1044,13 @@ export default class LocalParticipant extends Participant {
       type: Track.kindToProto(track.kind),
       muted: track.isMuted,
       source: Track.sourceToProto(track.source),
-      disableDtx: !(opts.dtx ?? true),
+      disableDtx,
       encryption: this.encryptionType,
       stereo: isStereo,
       disableRed: this.isE2EEEnabled || !(opts.red ?? true),
       stream: opts?.stream,
       backupCodecPolicy: opts?.backupCodecPolicy as BackupCodecPolicy,
+      audioFeatures,
     });
 
     // compute encodings and layers for video
@@ -1222,6 +1288,79 @@ export default class LocalParticipant extends Participant {
     this.addTrackPublication(publication);
     // send event for publication
     this.emit(ParticipantEvent.LocalTrackPublished, publication);
+
+    if (
+      isLocalAudioTrack(track) &&
+      ti.audioFeatures.includes(AudioTrackFeature.TF_PRECONNECT_BUFFER)
+    ) {
+      const stream = track.getPreConnectBuffer();
+      // TODO: we're registering the listener after negotiation, so there might be a race
+      this.on(ParticipantEvent.LocalTrackSubscribed, (pub) => {
+        if (pub.trackSid === ti.sid) {
+          if (!track.hasPreConnectBuffer) {
+            this.log.warn('subscribe event came to late, buffer already closed', this.logContext);
+            return;
+          }
+          this.log.debug('finished recording preconnect buffer', {
+            ...this.logContext,
+            ...getLogContextFromTrack(track),
+          });
+          track.stopPreConnectBuffer();
+        }
+      });
+
+      if (stream) {
+        const bufferStreamPromise = new Promise<void>(async (resolve, reject) => {
+          try {
+            this.log.debug('waiting for agent', {
+              ...this.logContext,
+              ...getLogContextFromTrack(track),
+            });
+            const agentActiveTimeout = setTimeout(() => {
+              reject(new Error('agent not active within 10 seconds'));
+            }, 10_000);
+            const agent = await this.waitUntilActiveAgentPresent();
+            clearTimeout(agentActiveTimeout);
+            this.log.debug('sending preconnect buffer', {
+              ...this.logContext,
+              ...getLogContextFromTrack(track),
+            });
+            const writer = await this.streamBytes({
+              name: 'preconnect-buffer',
+              mimeType: 'audio/opus',
+              topic: 'lk.agent.pre-connect-audio-buffer',
+              destinationIdentities: [agent.identity],
+              attributes: {
+                trackId: publication.trackSid,
+                sampleRate: String(settings.sampleRate ?? '48000'),
+                channels: String(settings.channelCount ?? '1'),
+              },
+            });
+            for await (const chunk of stream) {
+              await writer.write(chunk);
+            }
+            await writer.close();
+            resolve();
+          } catch (e) {
+            reject(e);
+          }
+        });
+        bufferStreamPromise
+          .then(() => {
+            this.log.debug('preconnect buffer sent successfully', {
+              ...this.logContext,
+              ...getLogContextFromTrack(track),
+            });
+          })
+          .catch((e) => {
+            this.log.error('error sending preconnect buffer', {
+              ...this.logContext,
+              ...getLogContextFromTrack(track),
+              error: e,
+            });
+          });
+      }
+    }
     return publication;
   }
 
@@ -2113,6 +2252,30 @@ export default class LocalParticipant extends Participant {
     );
   };
 
+  /** @internal */
+  setActiveAgent(agent: RemoteParticipant | undefined) {
+    this.firstActiveAgent = agent;
+    if (agent && !this.firstActiveAgent) {
+      this.firstActiveAgent = agent;
+    }
+    if (agent) {
+      this.activeAgentFuture?.resolve?.(agent);
+    } else {
+      this.activeAgentFuture?.reject?.('Agent disconnected');
+    }
+    this.activeAgentFuture = undefined;
+  }
+
+  private waitUntilActiveAgentPresent() {
+    if (this.firstActiveAgent) {
+      return Promise.resolve(this.firstActiveAgent);
+    }
+    if (!this.activeAgentFuture) {
+      this.activeAgentFuture = new Future<RemoteParticipant>();
+    }
+    return this.activeAgentFuture.promise;
+  }
+
   /** @internal */
   private onTrackUnmuted = (track: LocalTrack) => {
     this.onTrackMuted(track, track.isUpstreamPaused);

package/src/room/track/LocalTrack.ts

@@ -9,15 +9,19 @@ import { compareVersions, isMobile, sleep, unwrapConstraint } from '../utils';
 import { Track, attachToElement, detachTrack } from './Track';
 import type { VideoCodec } from './options';
 import type { TrackProcessor } from './processor/types';
+import { LocalTrackRecorder } from './record';
 import type { ReplaceTrackOptions } from './types';
 
-const defaultDimensionsTimeout = 1000;
+const DEFAULT_DIMENSIONS_TIMEOUT = 1000;
+const PRE_CONNECT_BUFFER_TIMEOUT = 10_000;
 
 export default abstract class LocalTrack<
   TrackKind extends Track.Kind = Track.Kind,
 > extends Track<TrackKind> {
   protected _sender?: RTCRtpSender;
 
+  private autoStopPreConnectBuffer: ReturnType<typeof setTimeout> | undefined;
+
   /** @internal */
   get sender(): RTCRtpSender | undefined {
     return this._sender;
@@ -35,6 +39,10 @@ export default abstract class LocalTrack<
     return this._constraints;
   }
 
+  get hasPreConnectBuffer() {
+    return !!this.localTrackRecorder;
+  }
+
   protected _constraints: MediaTrackConstraints;
 
   protected reacquireTrack: boolean;
@@ -55,6 +63,8 @@ export default abstract class LocalTrack<
 
   protected manuallyStopped: boolean = false;
 
+  protected localTrackRecorder: LocalTrackRecorder<typeof this> | undefined;
+
   private restartLock: Mutex;
 
   /**
@@ -203,7 +213,7 @@ export default abstract class LocalTrack<
     }
   }
 
-  async waitForDimensions(timeout = defaultDimensionsTimeout): Promise<Track.Dimensions> {
+  async waitForDimensions(timeout = DEFAULT_DIMENSIONS_TIMEOUT): Promise<Track.Dimensions> {
     if (this.kind === Track.Kind.Audio) {
       throw new Error('cannot get dimensions for audio tracks');
     }
@@ -585,5 +595,40 @@ export default abstract class LocalTrack<
     this.emit(TrackEvent.TrackProcessorUpdate);
   }
 
+  /** @internal */
+  startPreConnectBuffer(timeslice: number = 100) {
+    if (!this.localTrackRecorder) {
+      this.localTrackRecorder = new LocalTrackRecorder(this, {
+        mimeType: 'audio/webm;codecs=opus',
+      });
+    } else {
+      this.log.warn('preconnect buffer already started');
+      return;
+    }
+
+    this.localTrackRecorder.start(timeslice);
+    this.autoStopPreConnectBuffer = setTimeout(() => {
+      this.log.warn(
+        'preconnect buffer timed out, stopping recording automatically',
+        this.logContext,
+      );
+      this.stopPreConnectBuffer();
+    }, PRE_CONNECT_BUFFER_TIMEOUT);
+  }
+
+  /** @internal */
+  stopPreConnectBuffer() {
+    clearTimeout(this.autoStopPreConnectBuffer);
+    if (this.localTrackRecorder) {
+      this.localTrackRecorder.stop();
+      this.localTrackRecorder = undefined;
+    }
+  }
+
+  /** @internal */
+  getPreConnectBuffer() {
+    return this.localTrackRecorder?.byteStream;
+  }
+
   protected abstract monitorSender(): void;
 }

package/src/room/track/Track.ts

@@ -528,4 +528,5 @@ export type TrackEventCallbacks = {
   trackProcessorUpdate: (processor?: TrackProcessor<Track.Kind, any>) => void;
   audioTrackFeatureUpdate: (track: any, feature: AudioTrackFeature, enabled: boolean) => void;
   timeSyncUpdate: (update: { timestamp: number; rtpTimestamp: number }) => void;
+  preConnectBufferFlushed: (buffer: Uint8Array[]) => void;
 };

package/src/room/track/options.ts

@@ -119,6 +119,15 @@ export interface TrackPublishDefaults {
    * defaults to false
    */
   stopMicTrackOnMute?: boolean;
+
+  /**
+   * Enables preconnect buffer for a user's microphone track.
+   * This is useful for reducing perceived latency when the user starts to speak before the connection is established.
+   * Only works for agent use cases.
+   *
+   * Defaults to false.
+   */
+  preConnectBuffer?: boolean;
 }
 
 /**

package/src/room/track/record.ts

@@ -0,0 +1,51 @@
+import type LocalTrack from './LocalTrack';
+
+export class LocalTrackRecorder<T extends LocalTrack> extends MediaRecorder {
+  byteStream: ReadableStream<Uint8Array>;
+
+  constructor(track: T, options?: MediaRecorderOptions) {
+    super(new MediaStream([track.mediaStreamTrack]), options);
+
+    let dataListener: (event: BlobEvent) => void;
+
+    let streamController: ReadableStreamDefaultController<Uint8Array> | undefined;
+
+    const isClosed = () => streamController === undefined;
+
+    const onStop = () => {
+      this.removeEventListener('dataavailable', dataListener);
+      this.removeEventListener('stop', onStop);
+      this.removeEventListener('error', onError);
+      streamController?.close();
+      streamController = undefined;
+    };
+
+    const onError = (event: Event) => {
+      streamController?.error(event);
+      this.removeEventListener('dataavailable', dataListener);
+      this.removeEventListener('stop', onStop);
+      this.removeEventListener('error', onError);
+      streamController = undefined;
+    };
+
+    this.byteStream = new ReadableStream({
+      start: (controller) => {
+        streamController = controller;
+        dataListener = async (event: BlobEvent) => {
+          const arrayBuffer = await event.data.arrayBuffer();
+          if (isClosed()) {
+            return;
+          }
+          controller.enqueue(new Uint8Array(arrayBuffer));
+        };
+        this.addEventListener('dataavailable', dataListener);
+      },
+      cancel: () => {
+        onStop();
+      },
+    });
+
+    this.addEventListener('stop', onStop);
+    this.addEventListener('error', onError);
+  }
+}