@dxos/plugin-thread 0.8.4-main.f9ba587 → 0.8.4-main.fcc0d83b33

package/src/calls/util/calls-service.ts

@@ -2,20 +2,20 @@
 // Copyright 2024 DXOS.org
 //
 
-import { PersistentLifecycle } from '@dxos/async';
+import { PersistentLifecycle, Trigger, scheduleTask } from '@dxos/async';
 import { type Context, Resource } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 
-import { HistoryCache } from './history-cache';
-import { BulkRequestDispatcher, FIFOScheduler } from './task-scheduling';
 import {
-  type SessionResponse,
   type ErrorResponse,
   type RenegotiationResponse,
+  type SessionResponse,
   type TrackObject,
   type TracksResponse,
 } from '../types';
+import { HistoryCache } from './history-cache';
+import { BulkRequestDispatcher, FIFOScheduler } from './task-scheduling';
 
 const NETWORK_TIMEOUT = 5_000;
 
@@ -26,18 +26,20 @@ export type Session = {
 
 export type CallsServiceConfig = {
   apiBase: string;
-  apiExtraParams?: string;
+  apiExtraProps?: string;
   iceServers?: RTCIceServer[];
 };
 
 export type HistoryRecord =
   | {
+      endpoint: string;
       type: 'request';
       time: string;
       method: string;
       body: any;
     }
   | {
+      endpoint: string;
       type: 'response';
       time: string;
       status: number;
@@ -115,7 +117,7 @@ export class CallsServicePeer extends Resource {
     });
     peerConnection.addEventListener('connectionstatechange', () => {
       if (peerConnection.connectionState === 'failed' || peerConnection.connectionState === 'closed') {
-        log.warn('calls connection failed', { state: peerConnection.connectionState });
+        log.info('calls connection failed', { state: peerConnection.connectionState });
         void this._persistentLifecycle.scheduleRestart();
       }
     });
@@ -124,7 +126,7 @@ export class CallsServicePeer extends Resource {
     peerConnection.addEventListener('iceconnectionstatechange', () => {
       clearTimeout(iceTimeout);
       if (peerConnection.iceConnectionState === 'failed' || peerConnection.iceConnectionState === 'closed') {
-        log.warn('calls connection to ICEs failed', { state: peerConnection.iceConnectionState });
+        log.info('calls connection to ICEs failed', { state: peerConnection.iceConnectionState });
         void this._persistentLifecycle.scheduleRestart();
       } else if (peerConnection.iceConnectionState === 'disconnected') {
         // TODO(mykola): we should start to inspect the connection stats from here on for
@@ -137,7 +139,7 @@ export class CallsServicePeer extends Resource {
             return;
           }
 
-          log.warn('calls iceConnectionState timed out', { state: peerConnection.iceConnectionState, timeoutSeconds });
+          log.info('calls iceConnectionState timed out', { state: peerConnection.iceConnectionState, timeoutSeconds });
           void this._persistentLifecycle.scheduleRestart();
         }, timeoutSeconds * 1_000);
       }
@@ -154,6 +156,7 @@ export class CallsServicePeer extends Resource {
 
   private async _fetch<T extends ErrorResponse>(relativePath: string, requestInit?: RequestInit): Promise<T> {
     this.history.push({
+      endpoint: relativePath,
       type: 'request',
       time: new Date().toISOString(),
       method: requestInit?.method ?? 'GET',
@@ -169,6 +172,7 @@ export class CallsServicePeer extends Resource {
     try {
       const data = (await response.clone().json()) as T;
       this.history.push({
+        endpoint: relativePath,
         type: 'response',
         time: new Date().toISOString(),
         status: response.status,
@@ -182,6 +186,7 @@ export class CallsServicePeer extends Resource {
     } catch (error) {
       const text = await response.text();
       this.history.push({
+        endpoint: relativePath,
         type: 'response',
         time: new Date().toISOString(),
         status: response.status,
@@ -203,11 +208,11 @@ export class CallsServicePeer extends Resource {
     sessionId,
     trackName,
   }: {
+    ctx: Context;
     peerConnection: RTCPeerConnection;
     transceiver: RTCRtpTransceiver;
     sessionId: string;
     trackName: string;
-    ctx?: Context;
   }): Promise<TrackObject | void> {
     log.verbose('pushing track ', trackName);
     const pushedTrackPromise = this.pushTrackDispatcher
@@ -245,9 +250,13 @@ export class CallsServicePeer extends Resource {
           };
         }),
       )
-      .then(({ tracks }) => {
+      .then(async ({ tracks }) => {
         const trackData = tracks.find((t) => t.mid === transceiver.mid);
         if (trackData) {
+          // we wait for the transceiver to start sending data before we emit
+          // the track metadata to ensure that the track will be able to be
+          // pulled before making the metadata available to anyone else.
+          // await waitForTransceiverToSendData(ctx, transceiver);
           return {
             ...trackData,
             sessionId,
@@ -256,9 +265,6 @@ export class CallsServicePeer extends Resource {
         } else {
           log.error('missing track data');
         }
-      })
-      .catch((err) => {
-        log.catch(err);
       });
 
     const onDispose = async () => {
@@ -280,8 +286,8 @@ export class CallsServicePeer extends Resource {
     previousTrack,
   }: {
     track: MediaStreamTrack | null;
+    ctx: Context;
     encodings?: RTCRtpEncodingParameters[];
-    ctx?: Context;
     previousTrack?: TrackObject;
   }): Promise<TrackObject | undefined> {
     await this.waitUntilOpen();
@@ -404,7 +410,7 @@ export class CallsServicePeer extends Resource {
             })
             .catch((err) => log.catch(err));
         } else {
-          log.warn('missing track info', { trackData, availableTracks: Array.from(trackMap.keys()) });
+          log.info('missing track info', { trackData, availableTracks: Array.from(trackMap.keys()) });
         }
       });
 
@@ -528,3 +534,49 @@ const peerConnectionIsConnected = async (peerConnection: RTCPeerConnection) => {
     await connected;
   }
 };
+
+const INITIAL_DELAY = 50;
+const MAX_DELAY = 500;
+const MAX_RETRIES = 20;
+const WAIT_FOR_DATA_TIMEOUT = 10_000;
+
+const waitForTransceiverToSendData = async (ctx: Context, transceiver: RTCRtpTransceiver): Promise<void> => {
+  let delay = INITIAL_DELAY;
+  let checks = 0;
+  const waitForData = new Trigger();
+  ctx.onDispose(() => waitForData.throw(new Error('Transceiver was closed')));
+
+  const checkStats = async () => {
+    if (ctx.disposed) {
+      return;
+    }
+
+    try {
+      const stats = await transceiver.sender.getStats();
+      let dataFound = false;
+      stats.forEach((stat) => {
+        if (stat.type === 'outbound-rtp' && stat.bytesSent > 0) {
+          dataFound = true;
+        }
+      });
+
+      if (dataFound) {
+        waitForData.wake();
+        return;
+      }
+    } catch {
+      // Stats might not be available yet, continue checking
+    }
+    checks++;
+    if (checks > MAX_RETRIES) {
+      waitForData.throw(new Error('Transceiver did not send data'));
+      return;
+    }
+
+    delay = Math.min(delay * 2, MAX_DELAY); // Exponential backoff with max cap
+    scheduleTask(ctx, checkStats, delay);
+  };
+
+  void checkStats();
+  return waitForData.wait({ timeout: WAIT_FOR_DATA_TIMEOUT });
+};

package/src/calls/util/stub-media-streams.ts

@@ -3,8 +3,8 @@
 //
 //
 
-import { waitForCondition, scheduleTaskInterval } from '@dxos/async';
-import { cancelWithContext, type Context } from '@dxos/context';
+import { scheduleTaskInterval, waitForCondition } from '@dxos/async';
+import { type Context, cancelWithContext } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
 
 /**
@@ -30,7 +30,7 @@ export const createBlackCanvasStreamTrack = async ({
     canvasCtx.fillStyle = 'black';
     canvasCtx.fillRect(0, 0, canvas.width, canvas.height);
   };
-  scheduleTaskInterval(ctx, async () => drawFrame(), 1_000);
+  scheduleTaskInterval(ctx, async () => drawFrame(), 200);
 
   const track = canvas.captureStream().getVideoTracks()[0];
   drawFrame();
@@ -40,27 +40,86 @@ export const createBlackCanvasStreamTrack = async ({
   return track;
 };
 
+let audioContextStartedPreviously = false;
+const userGestureEvents = [
+  'click',
+  'contextmenu',
+  'auxclick',
+  'dblclick',
+  'mousedown',
+  'mouseup',
+  'pointerup',
+  'touchend',
+  'keydown',
+  'keyup',
+];
+
 /**
  * Creates an inaudible audio stream track.
  * We need this to have audio track available when audio is disabled.
  */
 export const createInaudibleAudioStreamTrack = async ({ ctx }: { ctx: Context }): Promise<MediaStreamTrack> => {
   const audioContext = new window.AudioContext();
+
   const oscillator = audioContext.createOscillator();
   oscillator.type = 'triangle';
-  // roughly sounds like a box fan
   oscillator.frequency.setValueAtTime(20, audioContext.currentTime);
 
   const gainNode = audioContext.createGain();
-  // even w/ gain at 0 some packets are sent
-  gainNode.gain.setValueAtTime(0.02, audioContext.currentTime);
+  gainNode.gain.setValueAtTime(0, audioContext.currentTime);
   oscillator.connect(gainNode);
 
   const destination = audioContext.createMediaStreamDestination();
   gainNode.connect(destination);
-  oscillator.start();
+
+  let oscillatorStarted = false;
+  const ensureOscillatorStarted = () => {
+    if (oscillatorStarted) {
+      return;
+    }
+    oscillator.start();
+    oscillatorStarted = true;
+  };
+
+  const stateChangeHandler = () => {
+    if (audioContext.state === 'running') {
+      audioContextStartedPreviously = true;
+      ensureOscillatorStarted();
+    }
+
+    if (audioContext.state === 'suspended' || (audioContext.state as string) === 'interrupted') {
+      resumeAudioContext();
+    }
+  };
+
+  audioContext.addEventListener('statechange', stateChangeHandler);
+
+  const resumeAudioContext = () => {
+    void audioContext.resume().then(() => {
+      cleanUpUserGestureListeners();
+    });
+  };
+
+  const cleanUpUserGestureListeners = () => {
+    userGestureEvents.forEach((gesture) => {
+      document.removeEventListener(gesture, resumeAudioContext, {
+        capture: true,
+      });
+    });
+  };
+
+  if (audioContextStartedPreviously) {
+    resumeAudioContext();
+  } else {
+    userGestureEvents.forEach((gesture) => {
+      document.addEventListener(gesture, resumeAudioContext, {
+        capture: true,
+      });
+    });
+  }
 
   ctx.onDispose(async () => {
+    cleanUpUserGestureListeners();
     oscillator.disconnect();
     oscillator.stop();
     await audioContext.close();

package/src/calls/util/task-scheduling.ts

@@ -27,8 +27,8 @@ export class FIFOScheduler {
 }
 
 // TODO(mykola): Reconcile with @dxos/async task scheduling.
-export class BulkRequestDispatcher<RequestEntryParams, BulkResponse> {
-  #currentBatch: RequestEntryParams[];
+export class BulkRequestDispatcher<RequestEntryProps, BulkResponse> {
+  #currentBatch: RequestEntryProps[];
   #currentBulkResponse: Promise<BulkResponse> | null;
   #batchSizeLimit: number;
 
@@ -43,8 +43,8 @@ export class BulkRequestDispatcher<RequestEntryParams, BulkResponse> {
    * At the event loop iteration end, the accumulated entries will be dispatched as a bulk request.
    */
   doBulkRequest(
-    params: RequestEntryParams,
-    bulkRequestFunc: (bulkCopy: RequestEntryParams[]) => Promise<BulkResponse>,
+    params: RequestEntryProps,
+    bulkRequestFunc: (bulkCopy: RequestEntryProps[]) => Promise<BulkResponse>,
   ): Promise<BulkResponse> {
     if (this.#currentBatch.length >= this.#batchSizeLimit) {
       // If it reaches the batch size limit, we make another bulk request.

package/src/calls-service.stories.tsx

@@ -2,8 +2,7 @@
 // Copyright 2025 DXOS.org
 //
 
-import '@dxos-theme';
-
+import { type Meta, type StoryObj } from '@storybook/react-vite';
 import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 
 import { scheduleTask, sleep } from '@dxos/async';
@@ -13,14 +12,16 @@ import { log } from '@dxos/log';
 import { Config, useConfig } from '@dxos/react-client';
 import { withClientProvider } from '@dxos/react-client/testing';
 import { Button } from '@dxos/react-ui';
-import { Json } from '@dxos/react-ui-syntax-highlighter';
-import { withLayout, withTheme } from '@dxos/storybook-utils';
+import { JsonHighlighter } from '@dxos/react-ui-syntax-highlighter';
+import { withTheme } from '@dxos/react-ui/testing';
+
+const testVideo = new URL('../testing/video.mp4', import.meta.url).href;
+
+import { useBlackCanvasStreamTrack, useInaudibleAudioStreamTrack, useVideoStreamTrack } from '#hooks';
 
-import { type TrackObject, CallsServicePeer, CALLS_URL } from './calls';
-import { useInaudibleAudioStreamTrack, useVideoStreamTrack } from './hooks';
-// @ts-ignore
-import video from '../testing/video.mp4';
+import { CALLS_URL, CallsServicePeer, type TrackObject } from './calls';
 
+// TODO(burdon): THIS IS TOO COMPLEX FOR A TEST SETUP.
 const pushAndPullTrack = (mediaStreamTrack?: MediaStreamTrack) => {
   const config = useConfig();
   const callsConfig = {
@@ -35,16 +36,18 @@ const pushAndPullTrack = (mediaStreamTrack?: MediaStreamTrack) => {
   const [pulledTrack, setPulledTrack] = useState<MediaStreamTrack | undefined>(undefined);
   const hadRun = useRef(false);
 
+  const deps: any[] = [];
   // Push/pull video stream track to cloudflare.
   useEffect(() => {
     if (hadRun.current || !mediaStreamTrack || !peerPush || !peerPull) {
-      log.info('not running', { hadRun: hadRun.current, mediaStreamTrack, peerPush, peerPull });
       return;
     }
     hadRun.current = true;
     const ctx = Context.default();
+    log.info('setting up ctx', { ctx, mediaStreamTrack, peerPush, peerPull });
+    deps.push([ctx, mediaStreamTrack, peerPush, peerPull]);
     ctx.onDispose(() => {
-      log.info('disposing ctx', { ctx });
+      log.info('disposing ctx', {});
     });
     scheduleTask(
       ctx,
@@ -60,15 +63,12 @@ const pushAndPullTrack = (mediaStreamTrack?: MediaStreamTrack) => {
         // Push track to cloudflare.
         performance.mark('push:begin');
         log.info('pushing track', { track: mediaStreamTrack });
-        trackInfo.current = await peerPush.pushTrack({ track: mediaStreamTrack });
+        trackInfo.current = await peerPush.pushTrack({ track: mediaStreamTrack, ctx: Context.default() });
         performance.mark('push:end');
         const pushTime = performance.measure('push', 'push:begin', 'push:end').duration;
         log.info('successfully pushed track', { pushTime, trackInfo: trackInfo.current });
         setMetrics((prev) => ({ ...prev, 'time to push track [ms]': Math.round(pushTime) }));
 
-        // Wait for cloudflare to process the track.
-        await sleep(1000);
-
         // Pull track from cloudflare.
         performance.mark('pullTrack:begin');
         pullCtx.current = Context.default();
@@ -104,7 +104,10 @@ const pushAndPullTrack = (mediaStreamTrack?: MediaStreamTrack) => {
 
   const rePullTrack = useCallback(async () => {
     performance.mark('rePullVideo:begin');
-    invariant(pulledTrack);
+    if (!pulledTrack) {
+      log.info('no pulled track, skipping re-pull', { pulledTrack });
+      return;
+    }
     pulledTrack.stop();
     await pullCtx.current!.dispose();
     pullCtx.current = Context.default();
@@ -123,12 +126,57 @@ const pushAndPullTrack = (mediaStreamTrack?: MediaStreamTrack) => {
     pulledTrack,
     rePullTrack,
     metrics,
+    pushService: peerPush,
+    pullService: peerPull,
   };
 };
 
+type DefaultStoryProps = { source: string };
+
+const DefaultStory = ({ source }: DefaultStoryProps) => {
+  const pushVideoElement = useRef<HTMLVideoElement>(null);
+  const pullVideoElement = useRef<HTMLVideoElement>(null);
+  // Get video stream track.
+  const videoStreamTrack = useVideoStreamTrack(pushVideoElement, source);
+  const { pulledTrack, rePullTrack, metrics } = pushAndPullTrack(videoStreamTrack);
+  const hadRun = useRef(false);
+
+  // Push/pull video stream track to cloudflare.
+  useEffect(() => {
+    if (!pulledTrack || hadRun.current) {
+      return;
+    }
+    hadRun.current = true;
+    const ctx = Context.default();
+    scheduleTask(ctx, async () => {
+      pullVideoElement.current!.srcObject = new MediaStream([pulledTrack]);
+      await pullVideoElement.current!.play();
+    });
+
+    return () => {
+      void ctx.dispose();
+    };
+  }, [pulledTrack]);
+
+  return (
+    <div className='grid grid-rows-3 gap-4 items-center'>
+      <video ref={pushVideoElement} muted autoPlay loop />
+      <div className='flex flex-col gap-4'>
+        <video ref={pullVideoElement} muted />
+        <Button disabled={!pulledTrack} onClick={rePullTrack}>
+          Re-pull video
+        </Button>
+      </div>
+      <JsonHighlighter data={metrics} />
+    </div>
+  );
+};
+
 const meta = {
-  title: 'plugins/plugin-thread/calls-service',
+  title: 'plugins/plugin-thread/experimental/Calls',
+  render: DefaultStory,
   decorators: [
+    withTheme(),
     withClientProvider({
       config: new Config({
         runtime: {
@@ -138,77 +186,43 @@ const meta = {
         },
       }),
     }),
-    withTheme,
-    withLayout(),
   ],
   parameters: {
     layout: 'centered',
   },
-};
+} satisfies Meta<typeof DefaultStory>;
 
 export default meta;
 
-export const Default = {
-  render: ({ videoSrc }: { videoSrc: string }) => {
-    const pushVideoElement = useRef<HTMLVideoElement>(null);
-    const pullVideoElement = useRef<HTMLVideoElement>(null);
-    // Get video stream track.
-    const videoStreamTrack = useVideoStreamTrack(pushVideoElement, videoSrc);
-    const { pulledTrack, rePullTrack, metrics } = pushAndPullTrack(videoStreamTrack);
-    const hadRun = useRef(false);
-
-    // Push/pull video stream track to cloudflare.
-    useEffect(() => {
-      if (!pulledTrack || hadRun.current) {
-        return;
-      }
-      hadRun.current = true;
-      const ctx = Context.default();
-      scheduleTask(ctx, async () => {
-        pullVideoElement.current!.srcObject = new MediaStream([pulledTrack]);
-        await pullVideoElement.current!.play();
-      });
-
-      return () => {
-        void ctx.dispose();
-      };
-    }, [pulledTrack]);
+type Story = StoryObj<typeof meta>;
 
-    return (
-      <div className='grid grid-rows-3 gap-4 items-center'>
-        <video ref={pushVideoElement} muted autoPlay loop />
-        <div className='flex flex-col gap-4'>
-          <video ref={pullVideoElement} muted />
-          <Button
-            disabled={
-              !(
-                pullVideoElement?.current?.srcObject instanceof MediaStream &&
-                pullVideoElement.current.srcObject.getTracks()[0]
-              )
-            }
-            onClick={rePullTrack}
-          >
-            Re-pull video
-          </Button>
-        </div>
-        <Json data={metrics} />
-      </div>
-    );
-  },
+export const Default: Story = {
   args: {
-    videoSrc: video,
+    source: testVideo,
   },
 };
 
 export const InaudibleAudioStreamTrack = {
   render: () => {
     const audioStreamTrack = useInaudibleAudioStreamTrack();
-
     const { rePullTrack, metrics } = pushAndPullTrack(audioStreamTrack);
 
     return (
       <div className='flex flex-col gap-4 items-center'>
-        <Json data={metrics} />
+        <JsonHighlighter data={metrics} />
+        <Button onClick={rePullTrack}>Re-pull audio</Button>
+      </div>
+    );
+  },
+};
+
+export const BlackVideoStreamTrack = {
+  render: () => {
+    const videoStreamTrack = useBlackCanvasStreamTrack();
+    const { rePullTrack, metrics } = pushAndPullTrack(videoStreamTrack);
+    return (
+      <div className='flex flex-col gap-4 items-center'>
+        <JsonHighlighter data={metrics} />
         <Button onClick={rePullTrack}>Re-pull audio</Button>
       </div>
     );