@dxos/plugin-thread 0.8.3-staging.0fa589b → 0.8.4-main.03d5cd7b56

package/src/calls/util/calls-service.ts

@@ -2,20 +2,20 @@
 // Copyright 2024 DXOS.org
 //
 
-import { PersistentLifecycle } from '@dxos/async';
+import { PersistentLifecycle, Trigger, scheduleTask } from '@dxos/async';
 import { type Context, Resource } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 
-import { HistoryCache } from './history-cache';
-import { BulkRequestDispatcher, FIFOScheduler } from './task-scheduling';
 import {
-  type SessionResponse,
   type ErrorResponse,
   type RenegotiationResponse,
+  type SessionResponse,
   type TrackObject,
   type TracksResponse,
 } from '../types';
+import { HistoryCache } from './history-cache';
+import { BulkRequestDispatcher, FIFOScheduler } from './task-scheduling';
 
 const NETWORK_TIMEOUT = 5_000;
 
@@ -26,18 +26,20 @@ export type Session = {
 
 export type CallsServiceConfig = {
   apiBase: string;
-  apiExtraParams?: string;
+  apiExtraProps?: string;
   iceServers?: RTCIceServer[];
 };
 
 export type HistoryRecord =
   | {
+      endpoint: string;
       type: 'request';
       time: string;
       method: string;
       body: any;
     }
   | {
+      endpoint: string;
       type: 'response';
       time: string;
       status: number;
@@ -73,7 +75,10 @@ export class CallsServicePeer extends Resource {
   pullTrackDispatcher = new BulkRequestDispatcher<
     TrackObject,
     {
-      trackMap: Map<TrackObject, { resolvedTrack: Promise<MediaStreamTrack>; mid: string }>;
+      /**
+       * trackName -> { resolvedTrack, mid }
+       */
+      trackMap: Map<string, { resolvedTrack: Promise<MediaStreamTrack>; mid: string }>;
     }
   >(32);
 
@@ -112,7 +117,7 @@ export class CallsServicePeer extends Resource {
     });
     peerConnection.addEventListener('connectionstatechange', () => {
       if (peerConnection.connectionState === 'failed' || peerConnection.connectionState === 'closed') {
-        log.warn('calls connection failed', { state: peerConnection.connectionState });
+        log.info('calls connection failed', { state: peerConnection.connectionState });
         void this._persistentLifecycle.scheduleRestart();
       }
     });
@@ -121,7 +126,7 @@ export class CallsServicePeer extends Resource {
     peerConnection.addEventListener('iceconnectionstatechange', () => {
       clearTimeout(iceTimeout);
       if (peerConnection.iceConnectionState === 'failed' || peerConnection.iceConnectionState === 'closed') {
-        log.warn('calls connection to ICEs failed', { state: peerConnection.iceConnectionState });
+        log.info('calls connection to ICEs failed', { state: peerConnection.iceConnectionState });
         void this._persistentLifecycle.scheduleRestart();
       } else if (peerConnection.iceConnectionState === 'disconnected') {
         // TODO(mykola): we should start to inspect the connection stats from here on for
@@ -134,7 +139,7 @@ export class CallsServicePeer extends Resource {
             return;
           }
 
-          log.warn('calls iceConnectionState timed out', { state: peerConnection.iceConnectionState, timeoutSeconds });
+          log.info('calls iceConnectionState timed out', { state: peerConnection.iceConnectionState, timeoutSeconds });
           void this._persistentLifecycle.scheduleRestart();
         }, timeoutSeconds * 1_000);
       }
@@ -151,6 +156,7 @@ export class CallsServicePeer extends Resource {
 
   private async _fetch<T extends ErrorResponse>(relativePath: string, requestInit?: RequestInit): Promise<T> {
     this.history.push({
+      endpoint: relativePath,
       type: 'request',
       time: new Date().toISOString(),
       method: requestInit?.method ?? 'GET',
@@ -166,6 +172,7 @@ export class CallsServicePeer extends Resource {
     try {
       const data = (await response.clone().json()) as T;
       this.history.push({
+        endpoint: relativePath,
         type: 'response',
         time: new Date().toISOString(),
         status: response.status,
@@ -179,6 +186,7 @@ export class CallsServicePeer extends Resource {
     } catch (error) {
       const text = await response.text();
       this.history.push({
+        endpoint: relativePath,
         type: 'response',
         time: new Date().toISOString(),
         status: response.status,
@@ -200,11 +208,11 @@ export class CallsServicePeer extends Resource {
     sessionId,
     trackName,
   }: {
+    ctx: Context;
     peerConnection: RTCPeerConnection;
     transceiver: RTCRtpTransceiver;
     sessionId: string;
     trackName: string;
-    ctx?: Context;
   }): Promise<TrackObject | void> {
     log.verbose('pushing track ', trackName);
     const pushedTrackPromise = this.pushTrackDispatcher
@@ -242,9 +250,13 @@ export class CallsServicePeer extends Resource {
           };
         }),
       )
-      .then(({ tracks }) => {
+      .then(async ({ tracks }) => {
         const trackData = tracks.find((t) => t.mid === transceiver.mid);
         if (trackData) {
+          // we wait for the transceiver to start sending data before we emit
+          // the track metadata to ensure that the track will be able to be
+          // pulled before making the metadata available to anyone else.
+          // await waitForTransceiverToSendData(ctx, transceiver);
           return {
             ...trackData,
             sessionId,
@@ -253,9 +265,6 @@ export class CallsServicePeer extends Resource {
         } else {
           log.error('missing track data');
         }
-      })
-      .catch((err) => {
-        log.catch(err);
       });
 
     const onDispose = async () => {
@@ -277,8 +286,8 @@ export class CallsServicePeer extends Resource {
     previousTrack,
   }: {
     track: MediaStreamTrack | null;
+    ctx: Context;
     encodings?: RTCRtpEncodingParameters[];
-    ctx?: Context;
     previousTrack?: TrackObject;
   }): Promise<TrackObject | undefined> {
     await this.waitUntilOpen();
@@ -354,14 +363,14 @@ export class CallsServicePeer extends Resource {
             );
 
             if (pulledTrackData && pulledTrackData.mid) {
-              acc.set(track, {
+              acc.set(track.trackName!, {
                 mid: pulledTrackData.mid,
                 resolvedTrack: resolveTrack(peerConnection, (t) => t.mid === pulledTrackData.mid),
               });
             }
 
             return acc;
-          }, new Map<TrackObject, { resolvedTrack: Promise<MediaStreamTrack>; mid: string }>());
+          }, new Map<string, { resolvedTrack: Promise<MediaStreamTrack>; mid: string }>());
 
           if (newTrackResponse.requiresImmediateRenegotiation) {
             await peerConnection.setRemoteDescription(new RTCSessionDescription(newTrackResponse.sessionDescription));
@@ -392,7 +401,7 @@ export class CallsServicePeer extends Resource {
         }),
       )
       .then(({ trackMap }) => {
-        const trackInfo = trackMap.get(trackData);
+        const trackInfo = trackMap.get(trackData.trackName!);
         if (trackInfo) {
           return trackInfo.resolvedTrack
             .then((track) => {
@@ -401,7 +410,7 @@ export class CallsServicePeer extends Resource {
             })
             .catch((err) => log.catch(err));
         } else {
-          log.error('missing track info');
+          log.info('missing track info', { trackData, availableTracks: Array.from(trackMap.keys()) });
         }
       });
 
@@ -525,3 +534,49 @@ const peerConnectionIsConnected = async (peerConnection: RTCPeerConnection) => {
     await connected;
   }
 };
+
+const INITIAL_DELAY = 50;
+const MAX_DELAY = 500;
+const MAX_RETRIES = 20;
+const WAIT_FOR_DATA_TIMEOUT = 10_000;
+
+const waitForTransceiverToSendData = async (ctx: Context, transceiver: RTCRtpTransceiver): Promise<void> => {
+  let delay = INITIAL_DELAY;
+  let checks = 0;
+  const waitForData = new Trigger();
+  ctx.onDispose(() => waitForData.throw(new Error('Transceiver was closed')));
+
+  const checkStats = async () => {
+    if (ctx.disposed) {
+      return;
+    }
+
+    try {
+      const stats = await transceiver.sender.getStats();
+      let dataFound = false;
+      stats.forEach((stat) => {
+        if (stat.type === 'outbound-rtp' && stat.bytesSent > 0) {
+          dataFound = true;
+        }
+      });
+
+      if (dataFound) {
+        waitForData.wake();
+        return;
+      }
+    } catch {
+      // Stats might not be available yet, continue checking
+    }
+    checks++;
+    if (checks > MAX_RETRIES) {
+      waitForData.throw(new Error('Transceiver did not send data'));
+      return;
+    }
+
+    delay = Math.min(delay * 2, MAX_DELAY); // Exponential backoff with max cap
+    scheduleTask(ctx, checkStats, delay);
+  };
+
+  void checkStats();
+  return waitForData.wait({ timeout: WAIT_FOR_DATA_TIMEOUT });
+};

package/src/calls/util/stub-media-streams.ts

@@ -3,8 +3,8 @@
 //
 //
 
-import { waitForCondition, scheduleTaskInterval } from '@dxos/async';
-import { cancelWithContext, type Context } from '@dxos/context';
+import { scheduleTaskInterval, waitForCondition } from '@dxos/async';
+import { type Context, cancelWithContext } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
 
 /**
@@ -30,7 +30,7 @@ export const createBlackCanvasStreamTrack = async ({
     canvasCtx.fillStyle = 'black';
     canvasCtx.fillRect(0, 0, canvas.width, canvas.height);
   };
-  scheduleTaskInterval(ctx, async () => drawFrame(), 1_000);
+  scheduleTaskInterval(ctx, async () => drawFrame(), 200);
 
   const track = canvas.captureStream().getVideoTracks()[0];
   drawFrame();
@@ -40,30 +40,86 @@ export const createBlackCanvasStreamTrack = async ({
   return track;
 };
 
+let audioContextStartedPreviously = false;
+const userGestureEvents = [
+  'click',
+  'contextmenu',
+  'auxclick',
+  'dblclick',
+  'mousedown',
+  'mouseup',
+  'pointerup',
+  'touchend',
+  'keydown',
+  'keyup',
+];
+
 /**
  * Creates an inaudible audio stream track.
  * We need this to have audio track available when audio is disabled.
  */
 export const createInaudibleAudioStreamTrack = async ({ ctx }: { ctx: Context }): Promise<MediaStreamTrack> => {
   const audioContext = new window.AudioContext();
+
   const oscillator = audioContext.createOscillator();
   oscillator.type = 'triangle';
-
-  // set to 0.1 to make inaudible
-  oscillator.frequency.setValueAtTime(0.1, audioContext.currentTime);
+  oscillator.frequency.setValueAtTime(20, audioContext.currentTime);
 
   const gainNode = audioContext.createGain();
-  // even w/ gain at 0 some packets are sent
   gainNode.gain.setValueAtTime(0, audioContext.currentTime);
-
   oscillator.connect(gainNode);
 
   const destination = audioContext.createMediaStreamDestination();
   gainNode.connect(destination);
 
-  oscillator.start();
+  let oscillatorStarted = false;
+  const ensureOscillatorStarted = () => {
+    if (oscillatorStarted) {
+      return;
+    }
+    oscillator.start();
+    oscillatorStarted = true;
+  };
+
+  const stateChangeHandler = () => {
+    if (audioContext.state === 'running') {
+      audioContextStartedPreviously = true;
+      ensureOscillatorStarted();
+    }
+
+    if (audioContext.state === 'suspended' || (audioContext.state as string) === 'interrupted') {
+      resumeAudioContext();
+    }
+  };
+
+  audioContext.addEventListener('statechange', stateChangeHandler);
+
+  const resumeAudioContext = () => {
+    void audioContext.resume().then(() => {
+      cleanUpUserGestureListeners();
+    });
+  };
+
+  const cleanUpUserGestureListeners = () => {
+    userGestureEvents.forEach((gesture) => {
+      document.removeEventListener(gesture, resumeAudioContext, {
+        capture: true,
+      });
+    });
+  };
+
+  if (audioContextStartedPreviously) {
+    resumeAudioContext();
+  } else {
+    userGestureEvents.forEach((gesture) => {
+      document.addEventListener(gesture, resumeAudioContext, {
+        capture: true,
+      });
+    });
+  }
 
   ctx.onDispose(async () => {
+    cleanUpUserGestureListeners();
     oscillator.disconnect();
     oscillator.stop();
     await audioContext.close();

package/src/calls/util/task-scheduling.ts

@@ -27,8 +27,8 @@ export class FIFOScheduler {
 }
 
 // TODO(mykola): Reconcile with @dxos/async task scheduling.
-export class BulkRequestDispatcher<RequestEntryParams, BulkResponse> {
-  #currentBatch: RequestEntryParams[];
+export class BulkRequestDispatcher<RequestEntryProps, BulkResponse> {
+  #currentBatch: RequestEntryProps[];
   #currentBulkResponse: Promise<BulkResponse> | null;
   #batchSizeLimit: number;
 
@@ -43,8 +43,8 @@ export class BulkRequestDispatcher<RequestEntryParams, BulkResponse> {
    * At the event loop iteration end, the accumulated entries will be dispatched as a bulk request.
    */
   doBulkRequest(
-    params: RequestEntryParams,
-    bulkRequestFunc: (bulkCopy: RequestEntryParams[]) => Promise<BulkResponse>,
+    params: RequestEntryProps,
+    bulkRequestFunc: (bulkCopy: RequestEntryProps[]) => Promise<BulkResponse>,
   ): Promise<BulkResponse> {
     if (this.#currentBatch.length >= this.#batchSizeLimit) {
       // If it reaches the batch size limit, we make another bulk request.

package/src/calls-service.stories.tsx

@@ -0,0 +1,230 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type Meta, type StoryObj } from '@storybook/react-vite';
+import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+
+import { scheduleTask, sleep } from '@dxos/async';
+import { Context } from '@dxos/context';
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+import { Config, useConfig } from '@dxos/react-client';
+import { withClientProvider } from '@dxos/react-client/testing';
+import { Button } from '@dxos/react-ui';
+import { JsonHighlighter } from '@dxos/react-ui-syntax-highlighter';
+import { withTheme } from '@dxos/react-ui/testing';
+
+const testVideo = new URL('../testing/video.mp4', import.meta.url).href;
+
+import { useBlackCanvasStreamTrack, useInaudibleAudioStreamTrack, useVideoStreamTrack } from '#hooks';
+
+import { CALLS_URL, CallsServicePeer, type TrackObject } from './calls';
+
+// TODO(burdon): THIS IS TOO COMPLEX FOR A TEST SETUP.
+const pushAndPullTrack = (mediaStreamTrack?: MediaStreamTrack) => {
+  const config = useConfig();
+  const callsConfig = {
+    iceServers: config.get('runtime.services.ice'),
+    apiBase: `${CALLS_URL}/api/calls`,
+  };
+  const peerPush = useMemo(() => new CallsServicePeer(callsConfig), []);
+  const peerPull = useMemo(() => new CallsServicePeer(callsConfig), []);
+  const [metrics, setMetrics] = useState<Record<string, any>>({});
+  const trackInfo = useRef<TrackObject | undefined>(undefined);
+  const pullCtx = useRef<Context | undefined>(undefined);
+  const [pulledTrack, setPulledTrack] = useState<MediaStreamTrack | undefined>(undefined);
+  const hadRun = useRef(false);
+
+  const deps: any[] = [];
+  // Push/pull video stream track to cloudflare.
+  useEffect(() => {
+    if (hadRun.current || !mediaStreamTrack || !peerPush || !peerPull) {
+      return;
+    }
+    hadRun.current = true;
+    const ctx = Context.default();
+    log.info('setting up ctx', { ctx, mediaStreamTrack, peerPush, peerPull });
+    deps.push([ctx, mediaStreamTrack, peerPush, peerPull]);
+    ctx.onDispose(() => {
+      log.info('disposing ctx', {});
+    });
+    scheduleTask(
+      ctx,
+      async () => {
+        log.info('starting push/pull', { mediaStreamTrack });
+        await peerPush.open();
+        await peerPull.open();
+        ctx.onDispose(() => {
+          void peerPush.close();
+          void peerPull.close();
+        });
+
+        // Push track to cloudflare.
+        performance.mark('push:begin');
+        log.info('pushing track', { track: mediaStreamTrack });
+        trackInfo.current = await peerPush.pushTrack({ track: mediaStreamTrack, ctx: Context.default() });
+        performance.mark('push:end');
+        const pushTime = performance.measure('push', 'push:begin', 'push:end').duration;
+        log.info('successfully pushed track', { pushTime, trackInfo: trackInfo.current });
+        setMetrics((prev) => ({ ...prev, 'time to push track [ms]': Math.round(pushTime) }));
+
+        // Pull track from cloudflare.
+        performance.mark('pullTrack:begin');
+        pullCtx.current = Context.default();
+        let retries = 0;
+        let pulledTrack: MediaStreamTrack | undefined;
+        while (!pulledTrack && retries < 3) {
+          retries++;
+          log.info('pulling track', {
+            trackData: { ...trackInfo.current!, mid: undefined },
+            retries,
+            pulledTrack,
+          });
+          pulledTrack = await peerPull.pullTrack({
+            trackData: { ...trackInfo.current!, mid: undefined },
+            ctx: pullCtx.current,
+          });
+          await sleep(200);
+        }
+        invariant(pulledTrack);
+        setPulledTrack(pulledTrack);
+        performance.mark('pullTrack:end');
+        const pullTime = performance.measure('pullTrack', 'pullTrack:begin', 'pullTrack:end').duration;
+        setMetrics((prev) => ({ ...prev, 'time to pull track [ms]': Math.round(pullTime) }));
+        log.info('successfully pulled track', { pullTime, pulledTrack });
+      },
+      1000,
+    );
+
+    return () => {
+      void ctx.dispose();
+    };
+  }, [mediaStreamTrack, peerPush, peerPull]);
+
+  const rePullTrack = useCallback(async () => {
+    performance.mark('rePullVideo:begin');
+    if (!pulledTrack) {
+      log.info('no pulled track, skipping re-pull', { pulledTrack });
+      return;
+    }
+    pulledTrack.stop();
+    await pullCtx.current!.dispose();
+    pullCtx.current = Context.default();
+    const newPulledTrack = await peerPull.pullTrack({
+      trackData: { ...trackInfo.current!, mid: undefined },
+      ctx: pullCtx.current,
+    });
+    invariant(newPulledTrack);
+    setPulledTrack(newPulledTrack);
+    performance.mark('rePullVideo:end');
+    const rePullTime = performance.measure('rePullVideo', 'rePullVideo:begin', 'rePullVideo:end').duration;
+    setMetrics((prev) => ({ ...prev, 'time to re-pull video [ms]': Math.round(rePullTime) }));
+  }, [peerPull, trackInfo.current, pullCtx.current, pulledTrack]);
+
+  return {
+    pulledTrack,
+    rePullTrack,
+    metrics,
+    pushService: peerPush,
+    pullService: peerPull,
+  };
+};
+
+type DefaultStoryProps = { source: string };
+
+const DefaultStory = ({ source }: DefaultStoryProps) => {
+  const pushVideoElement = useRef<HTMLVideoElement>(null);
+  const pullVideoElement = useRef<HTMLVideoElement>(null);
+  // Get video stream track.
+  const videoStreamTrack = useVideoStreamTrack(pushVideoElement, source);
+  const { pulledTrack, rePullTrack, metrics } = pushAndPullTrack(videoStreamTrack);
+  const hadRun = useRef(false);
+
+  // Push/pull video stream track to cloudflare.
+  useEffect(() => {
+    if (!pulledTrack || hadRun.current) {
+      return;
+    }
+    hadRun.current = true;
+    const ctx = Context.default();
+    scheduleTask(ctx, async () => {
+      pullVideoElement.current!.srcObject = new MediaStream([pulledTrack]);
+      await pullVideoElement.current!.play();
+    });
+
+    return () => {
+      void ctx.dispose();
+    };
+  }, [pulledTrack]);
+
+  return (
+    <div className='grid grid-rows-3 gap-4 items-center'>
+      <video ref={pushVideoElement} muted autoPlay loop />
+      <div className='flex flex-col gap-4'>
+        <video ref={pullVideoElement} muted />
+        <Button disabled={!pulledTrack} onClick={rePullTrack}>
+          Re-pull video
+        </Button>
+      </div>
+      <JsonHighlighter data={metrics} />
+    </div>
+  );
+};
+
+const meta = {
+  title: 'plugins/plugin-thread/experimental/Calls',
+  render: DefaultStory,
+  decorators: [
+    withTheme(),
+    withClientProvider({
+      config: new Config({
+        runtime: {
+          services: {
+            iceProviders: [{ urls: 'https://edge.dxos.workers.dev/ice' }],
+          },
+        },
+      }),
+    }),
+  ],
+  parameters: {
+    layout: 'centered',
+  },
+} satisfies Meta<typeof DefaultStory>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const Default: Story = {
+  args: {
+    source: testVideo,
+  },
+};
+
+export const InaudibleAudioStreamTrack = {
+  render: () => {
+    const audioStreamTrack = useInaudibleAudioStreamTrack();
+    const { rePullTrack, metrics } = pushAndPullTrack(audioStreamTrack);
+
+    return (
+      <div className='flex flex-col gap-4 items-center'>
+        <JsonHighlighter data={metrics} />
+        <Button onClick={rePullTrack}>Re-pull audio</Button>
+      </div>
+    );
+  },
+};
+
+export const BlackVideoStreamTrack = {
+  render: () => {
+    const videoStreamTrack = useBlackCanvasStreamTrack();
+    const { rePullTrack, metrics } = pushAndPullTrack(videoStreamTrack);
+    return (
+      <div className='flex flex-col gap-4 items-center'>
+        <JsonHighlighter data={metrics} />
+        <Button onClick={rePullTrack}>Re-pull audio</Button>
+      </div>
+    );
+  },
+};