@dxos/plugin-thread 0.8.4-main.5acf9ea → 0.8.4-main.66e292d

package/src/calls/media-manager.ts

@@ -3,11 +3,12 @@
 //
 
 import { DeferredTask, Event, sleep, synchronized } from '@dxos/async';
-import { cancelWithContext, type Context, Resource } from '@dxos/context';
+import { type Context, Resource, cancelWithContext } from '@dxos/context';
+import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 
 import { SpeakingMonitor } from './speaking-monitor';
-import { TrackNameCodec, type EncodedTrackName, type TrackObject } from './types';
+import { type EncodedTrackName, TrackNameCodec, type TrackObject } from './types';
 import {
   type CallsServiceConfig,
   CallsServicePeer,
@@ -48,19 +49,23 @@ const VIDEO_WIDTH = 1280;
 const VIDEO_HEIGHT = 720;
 const MAX_WEB_CAM_FRAMERATE = 24;
 const MAX_WEB_CAM_BITRATE = 120_0000;
-const RETRY_INTERVAL = 500;
+const RETRY_INTERVAL = 100;
 
 export type MediaManagerParams = {
   serviceConfig: CallsServiceConfig;
 };
 
-const USE_INAUDIBLE_AUDIO = false;
+const USE_INAUDIBLE_AUDIO = true;
 
 export class MediaManager extends Resource {
   public readonly stateUpdated = new Event<MediaState>();
-  private readonly _state: MediaState = { pulledVideoStreams: {}, pulledAudioTracks: {} };
-  private _speakingMonitor?: SpeakingMonitor = undefined;
 
+  private readonly _state: MediaState = {
+    pulledVideoStreams: {},
+    pulledAudioTracks: {},
+  };
+
+  private _speakingMonitor?: SpeakingMonitor = undefined;
   private _trackToReconcile: EncodedTrackName[] = [];
   private _blackCanvasStreamTrack?: MediaStreamTrack = undefined;
   private _inaudibleAudioStreamTrack?: MediaStreamTrack = undefined;
@@ -84,6 +89,7 @@ export class MediaManager extends Resource {
       width: VIDEO_WIDTH,
       height: VIDEO_HEIGHT,
     });
+
     this._state.videoTrack = this._blackCanvasStreamTrack;
     this._state.videoStream = new MediaStream();
     this._state.videoStream.addTrack(this._state.videoTrack);
@@ -136,6 +142,7 @@ export class MediaManager extends Resource {
     this._state.videoStream!.removeTrack(this._state.videoTrack!);
     this._state.videoTrack = await getUserMediaTrack('videoinput', { width: VIDEO_WIDTH, height: VIDEO_HEIGHT });
     this._state.videoStream!.addTrack(this._state.videoTrack);
+
     this._state.videoEnabled = true;
     this.stateUpdated.emit(this._state);
     this._pushTracksTask!.schedule();
@@ -225,7 +232,6 @@ export class MediaManager extends Resource {
     // Wait for cloudflare to process the track.
     await cancelWithContext(this._ctx, sleep(800));
     const trackNames = this._trackToReconcile;
-
     const tracksToPull = trackNames.filter(
       (name) => !this._state.pulledAudioTracks[name] && !this._state.pulledVideoStreams[name],
     );
@@ -248,7 +254,7 @@ export class MediaManager extends Resource {
     });
 
     // Pull new tracks.
-    await Promise.all(tracksToPull.map((name) => this._pullTrack(name)));
+    const pullResults = await Promise.all(tracksToPull.map((name) => this._pullTrack(name)));
 
     // Close old tracks.
     await Promise.all([...audioTracksToClose, ...videoStreamsToClose].map(([_, { ctx }]) => ctx.dispose()));
@@ -257,7 +263,14 @@ export class MediaManager extends Resource {
       currentAudioTracks: Object.keys(this._state.pulledAudioTracks),
       currentVideoStreams: Object.keys(this._state.pulledVideoStreams),
     });
+
     this.stateUpdated.emit(this._state);
+    if (pullResults.some(({ shouldRetry }) => shouldRetry)) {
+      await cancelWithContext(this._ctx, sleep(RETRY_INTERVAL));
+      log.info('retrying pull tracks', { tracksToPull });
+      invariant(this._pullTracksTask);
+      this._pullTracksTask.schedule();
+    }
   }
 
   private async _pushTracks(): Promise<void> {
@@ -266,38 +279,46 @@ export class MediaManager extends Resource {
     }
 
     let updated = false;
-    const [pushedVideoTrack, pushedAudioTrack, pushedScreenshareTrack] = await Promise.all([
-      this._pushTrack(this._state.videoTrack, this._state.pushedVideoTrack, [
+    const [pushVideoResult, pushAudioResult, pushScreenshareResult] = await Promise.all([
+      this._maybePushTrack(this._state.videoTrack, this._state.pushedVideoTrack, [
         { maxFramerate: MAX_WEB_CAM_FRAMERATE, maxBitrate: MAX_WEB_CAM_BITRATE },
       ]),
-      this._pushTrack(this._state.audioTrack, this._state.pushedAudioTrack, [{ networkPriority: 'high' }]),
-      this._pushTrack(this._state.screenshareTrack, this._state.pushedScreenshareTrack),
+      this._maybePushTrack(this._state.audioTrack, this._state.pushedAudioTrack, [{ networkPriority: 'high' }]),
+      this._maybePushTrack(this._state.screenshareTrack, this._state.pushedScreenshareTrack),
     ]);
 
-    if (pushedVideoTrack !== this._state.pushedVideoTrack) {
-      this._state.pushedVideoTrack = pushedVideoTrack;
+    if (pushVideoResult.track !== this._state.pushedVideoTrack) {
+      this._state.pushedVideoTrack = pushVideoResult.track;
       updated = true;
     }
-    if (pushedAudioTrack !== this._state.pushedAudioTrack) {
-      this._state.pushedAudioTrack = pushedAudioTrack;
+    if (pushAudioResult.track !== this._state.pushedAudioTrack) {
+      this._state.pushedAudioTrack = pushAudioResult.track;
       updated = true;
     }
-    if (pushedScreenshareTrack !== this._state.pushedScreenshareTrack) {
-      this._state.pushedScreenshareTrack = pushedScreenshareTrack;
+    if (pushScreenshareResult.track !== this._state.pushedScreenshareTrack) {
+      this._state.pushedScreenshareTrack = pushScreenshareResult.track;
       updated = true;
     }
-    if (!updated) {
-      return;
+
+    const shouldRetry = pushVideoResult.shouldRetry || pushAudioResult.shouldRetry || pushScreenshareResult.shouldRetry;
+    if (updated) {
+      this.stateUpdated.emit(this._state);
     }
 
-    this.stateUpdated.emit(this._state);
+    if (shouldRetry) {
+      await cancelWithContext(this._ctx, sleep(RETRY_INTERVAL));
+      log.info('retrying push tracks', { pushVideoResult, pushAudioResult, pushScreenshareResult });
+      invariant(this._pushTracksTask);
+      this._pushTracksTask.schedule();
+    }
   }
 
-  private async _pullTrack(name: EncodedTrackName): Promise<void> {
+  private async _pullTrack(name: EncodedTrackName): Promise<{ shouldRetry: boolean }> {
     const ctx = this._ctx.derive();
     try {
       const trackData = TrackNameCodec.decode(name);
-      const track = await this._state.peer!.pullTrack({ trackData, ctx });
+      // We need to set mid here to `undefined`, because mid is peer specific.
+      const track = await this._state.peer!.pullTrack({ trackData: { ...trackData, mid: undefined }, ctx });
       if (track?.readyState === 'ended') {
         throw new Error('Pulled track ended immediately');
       }
@@ -306,7 +327,7 @@ export class MediaManager extends Resource {
         case 'audio': {
           this._state.pulledAudioTracks[name] = { track, ctx };
           ctx.onDispose(() => delete this._state.pulledAudioTracks[name]);
-          break;
+          return { shouldRetry: false };
         }
         case 'video': {
           const mediaStream = new MediaStream();
@@ -317,33 +338,41 @@ export class MediaManager extends Resource {
             track.stop();
             delete this._state.pulledVideoStreams[name];
           });
-          break;
+          return { shouldRetry: false };
         }
         default:
           throw new Error(`Invalid track kind: ${track?.kind}`);
       }
     } catch (err) {
-      log.verbose('failed to pull track', { err, name });
+      log.info('failed to pull track', { err, name });
       void ctx.dispose();
-      await cancelWithContext(this._ctx, sleep(RETRY_INTERVAL));
-      log.verbose('retrying pull track', { name });
-      this._pullTracksTask!.schedule();
+      return { shouldRetry: true };
     }
   }
 
-  private async _pushTrack(
+  private async _maybePushTrack(
     track?: MediaStreamTrack,
     previousTrack?: TrackObject,
     encodings?: RTCRtpEncodingParameters[],
-  ): Promise<TrackObject | undefined> {
+  ): Promise<{ track?: TrackObject; shouldRetry?: boolean }> {
     if (!track && !previousTrack) {
-      return;
+      return { track: undefined };
     }
 
-    return this._state.peer!.pushTrack({
-      track: track ?? null,
-      previousTrack,
-      encodings,
-    });
+    const ctx = this._ctx.derive();
+    try {
+      return {
+        track: await this._state.peer!.pushTrack({
+          ctx,
+          track: track ?? null,
+          previousTrack,
+          encodings,
+        }),
+      };
+    } catch (err) {
+      log.info('failed to push track', { err, track, previousTrack, encodings });
+      void ctx.dispose();
+      return { shouldRetry: true };
+    }
   }
 }

package/src/calls/types.ts

@@ -2,11 +2,12 @@
 // Copyright 2025 DXOS.org
 //
 
+import { invariant } from '@dxos/invariant';
 import { buf } from '@dxos/protocols/buf';
 import {
   type ActivitySchema,
-  UserStateSchema,
   type UserState as UserStateProto,
+  UserStateSchema,
 } from '@dxos/protocols/buf/dxos/edge/calls_pb';
 
 /**
@@ -61,10 +62,13 @@ export type EncodedTrackName = string & { __brand: 'EncodedTrackName' };
 
 export const TrackNameCodec = {
   encode: (trackData: TrackObject): EncodedTrackName => {
-    return (trackData.sessionId + '/' + trackData.trackName) as EncodedTrackName;
+    invariant(trackData.sessionId);
+    invariant(trackData.trackName);
+    invariant(trackData.mid);
+    return (trackData.sessionId + '/' + trackData.trackName + '/' + trackData.mid) as EncodedTrackName;
   },
   decode: (name: EncodedTrackName): TrackObject => {
-    const [sessionId, trackName] = name.split('/');
-    return { sessionId, trackName, location: 'remote' };
+    const [sessionId, trackName, mid] = name.split('/');
+    return { sessionId, trackName, location: 'remote', mid };
   },
 };

package/src/calls/util/calls-service.ts

@@ -2,21 +2,22 @@
 // Copyright 2024 DXOS.org
 //
 
-import { PersistentLifecycle } from '@dxos/async';
+import { PersistentLifecycle, Trigger, scheduleTask } from '@dxos/async';
 import { type Context, Resource } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 
-import { HistoryCache } from './history-cache';
-import { BulkRequestDispatcher, FIFOScheduler } from './task-scheduling';
 import {
-  type SessionResponse,
   type ErrorResponse,
   type RenegotiationResponse,
+  type SessionResponse,
   type TrackObject,
   type TracksResponse,
 } from '../types';
 
+import { HistoryCache } from './history-cache';
+import { BulkRequestDispatcher, FIFOScheduler } from './task-scheduling';
+
 const NETWORK_TIMEOUT = 5_000;
 
 export type Session = {
@@ -32,12 +33,14 @@ export type CallsServiceConfig = {
 
 export type HistoryRecord =
   | {
+      endpoint: string;
       type: 'request';
       time: string;
       method: string;
       body: any;
     }
   | {
+      endpoint: string;
       type: 'response';
       time: string;
       status: number;
@@ -115,7 +118,7 @@ export class CallsServicePeer extends Resource {
     });
     peerConnection.addEventListener('connectionstatechange', () => {
       if (peerConnection.connectionState === 'failed' || peerConnection.connectionState === 'closed') {
-        log.warn('calls connection failed', { state: peerConnection.connectionState });
+        log.info('calls connection failed', { state: peerConnection.connectionState });
         void this._persistentLifecycle.scheduleRestart();
       }
     });
@@ -124,7 +127,7 @@ export class CallsServicePeer extends Resource {
     peerConnection.addEventListener('iceconnectionstatechange', () => {
       clearTimeout(iceTimeout);
       if (peerConnection.iceConnectionState === 'failed' || peerConnection.iceConnectionState === 'closed') {
-        log.warn('calls connection to ICEs failed', { state: peerConnection.iceConnectionState });
+        log.info('calls connection to ICEs failed', { state: peerConnection.iceConnectionState });
         void this._persistentLifecycle.scheduleRestart();
       } else if (peerConnection.iceConnectionState === 'disconnected') {
         // TODO(mykola): we should start to inspect the connection stats from here on for
@@ -137,7 +140,7 @@ export class CallsServicePeer extends Resource {
             return;
           }
 
-          log.warn('calls iceConnectionState timed out', { state: peerConnection.iceConnectionState, timeoutSeconds });
+          log.info('calls iceConnectionState timed out', { state: peerConnection.iceConnectionState, timeoutSeconds });
           void this._persistentLifecycle.scheduleRestart();
         }, timeoutSeconds * 1_000);
       }
@@ -154,6 +157,7 @@ export class CallsServicePeer extends Resource {
 
   private async _fetch<T extends ErrorResponse>(relativePath: string, requestInit?: RequestInit): Promise<T> {
     this.history.push({
+      endpoint: relativePath,
       type: 'request',
       time: new Date().toISOString(),
       method: requestInit?.method ?? 'GET',
@@ -169,6 +173,7 @@ export class CallsServicePeer extends Resource {
     try {
       const data = (await response.clone().json()) as T;
       this.history.push({
+        endpoint: relativePath,
         type: 'response',
         time: new Date().toISOString(),
         status: response.status,
@@ -182,6 +187,7 @@ export class CallsServicePeer extends Resource {
     } catch (error) {
       const text = await response.text();
       this.history.push({
+        endpoint: relativePath,
         type: 'response',
         time: new Date().toISOString(),
         status: response.status,
@@ -203,11 +209,11 @@ export class CallsServicePeer extends Resource {
     sessionId,
     trackName,
   }: {
+    ctx: Context;
     peerConnection: RTCPeerConnection;
     transceiver: RTCRtpTransceiver;
     sessionId: string;
     trackName: string;
-    ctx?: Context;
   }): Promise<TrackObject | void> {
     log.verbose('pushing track ', trackName);
     const pushedTrackPromise = this.pushTrackDispatcher
@@ -245,9 +251,13 @@ export class CallsServicePeer extends Resource {
           };
         }),
       )
-      .then(({ tracks }) => {
+      .then(async ({ tracks }) => {
         const trackData = tracks.find((t) => t.mid === transceiver.mid);
         if (trackData) {
+          // we wait for the transceiver to start sending data before we emit
+          // the track metadata to ensure that the track will be able to be
+          // pulled before making the metadata available to anyone else.
+          // await waitForTransceiverToSendData(ctx, transceiver);
           return {
             ...trackData,
             sessionId,
@@ -256,9 +266,6 @@ export class CallsServicePeer extends Resource {
         } else {
           log.error('missing track data');
         }
-      })
-      .catch((err) => {
-        log.catch(err);
       });
 
     const onDispose = async () => {
@@ -280,8 +287,8 @@ export class CallsServicePeer extends Resource {
     previousTrack,
   }: {
     track: MediaStreamTrack | null;
+    ctx: Context;
     encodings?: RTCRtpEncodingParameters[];
-    ctx?: Context;
     previousTrack?: TrackObject;
   }): Promise<TrackObject | undefined> {
     await this.waitUntilOpen();
@@ -404,7 +411,7 @@ export class CallsServicePeer extends Resource {
             })
             .catch((err) => log.catch(err));
         } else {
-          log.warn('missing track info', { trackData, availableTracks: Array.from(trackMap.keys()) });
+          log.info('missing track info', { trackData, availableTracks: Array.from(trackMap.keys()) });
         }
       });
 
@@ -528,3 +535,47 @@ const peerConnectionIsConnected = async (peerConnection: RTCPeerConnection) => {
     await connected;
   }
 };
+
+const INITIAL_DELAY = 50;
+const MAX_DELAY = 500;
+const MAX_RETRIES = 20;
+const WAIT_FOR_DATA_TIMEOUT = 10_000;
+
+const waitForTransceiverToSendData = async (ctx: Context, transceiver: RTCRtpTransceiver): Promise<void> => {
+  let delay = INITIAL_DELAY;
+  let checks = 0;
+  const waitForData = new Trigger();
+  ctx.onDispose(() => waitForData.throw(new Error('Transceiver was closed')));
+
+  const checkStats = async () => {
+    if (ctx.disposed) return;
+
+    try {
+      const stats = await transceiver.sender.getStats();
+      let dataFound = false;
+      stats.forEach((stat) => {
+        if (stat.type === 'outbound-rtp' && stat.bytesSent > 0) {
+          dataFound = true;
+        }
+      });
+
+      if (dataFound) {
+        waitForData.wake();
+        return;
+      }
+    } catch {
+      // Stats might not be available yet, continue checking
+    }
+    checks++;
+    if (checks > MAX_RETRIES) {
+      waitForData.throw(new Error('Transceiver did not send data'));
+      return;
+    }
+
+    delay = Math.min(delay * 2, MAX_DELAY); // Exponential backoff with max cap
+    scheduleTask(ctx, checkStats, delay);
+  };
+
+  void checkStats();
+  return waitForData.wait({ timeout: WAIT_FOR_DATA_TIMEOUT });
+};

package/src/calls/util/stub-media-streams.ts

@@ -3,8 +3,8 @@
 //
 //
 
-import { waitForCondition, scheduleTaskInterval } from '@dxos/async';
-import { cancelWithContext, type Context } from '@dxos/context';
+import { scheduleTaskInterval, waitForCondition } from '@dxos/async';
+import { type Context, cancelWithContext } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
 
 /**
@@ -30,7 +30,7 @@ export const createBlackCanvasStreamTrack = async ({
     canvasCtx.fillStyle = 'black';
     canvasCtx.fillRect(0, 0, canvas.width, canvas.height);
   };
-  scheduleTaskInterval(ctx, async () => drawFrame(), 1_000);
+  scheduleTaskInterval(ctx, async () => drawFrame(), 200);
 
   const track = canvas.captureStream().getVideoTracks()[0];
   drawFrame();
@@ -40,27 +40,84 @@ export const createBlackCanvasStreamTrack = async ({
   return track;
 };
 
+let audioContextStartedPreviously = false;
+const userGestureEvents = [
+  'click',
+  'contextmenu',
+  'auxclick',
+  'dblclick',
+  'mousedown',
+  'mouseup',
+  'pointerup',
+  'touchend',
+  'keydown',
+  'keyup',
+];
+
 /**
  * Creates an inaudible audio stream track.
  * We need this to have audio track available when audio is disabled.
  */
 export const createInaudibleAudioStreamTrack = async ({ ctx }: { ctx: Context }): Promise<MediaStreamTrack> => {
   const audioContext = new window.AudioContext();
+
   const oscillator = audioContext.createOscillator();
   oscillator.type = 'triangle';
-  // roughly sounds like a box fan
   oscillator.frequency.setValueAtTime(20, audioContext.currentTime);
 
   const gainNode = audioContext.createGain();
-  // even w/ gain at 0 some packets are sent
-  gainNode.gain.setValueAtTime(0.02, audioContext.currentTime);
+  gainNode.gain.setValueAtTime(0, audioContext.currentTime);
   oscillator.connect(gainNode);
 
   const destination = audioContext.createMediaStreamDestination();
   gainNode.connect(destination);
-  oscillator.start();
+
+  let oscillatorStarted = false;
+  const ensureOscillatorStarted = () => {
+    if (oscillatorStarted) return;
+    oscillator.start();
+    oscillatorStarted = true;
+  };
+
+  const stateChangeHandler = () => {
+    if (audioContext.state === 'running') {
+      audioContextStartedPreviously = true;
+      ensureOscillatorStarted();
+    }
+
+    if (audioContext.state === 'suspended' || (audioContext.state as string) === 'interrupted') {
+      resumeAudioContext();
+    }
+  };
+
+  audioContext.addEventListener('statechange', stateChangeHandler);
+
+  const resumeAudioContext = () => {
+    void audioContext.resume().then(() => {
+      cleanUpUserGestureListeners();
+    });
+  };
+
+  const cleanUpUserGestureListeners = () => {
+    userGestureEvents.forEach((gesture) => {
+      document.removeEventListener(gesture, resumeAudioContext, {
+        capture: true,
+      });
+    });
+  };
+
+  if (audioContextStartedPreviously) {
+    resumeAudioContext();
+  } else {
+    userGestureEvents.forEach((gesture) => {
+      document.addEventListener(gesture, resumeAudioContext, {
+        capture: true,
+      });
+    });
+  }
 
   ctx.onDispose(async () => {
+    cleanUpUserGestureListeners();
     oscillator.disconnect();
     oscillator.stop();
     await audioContext.close();