@dxos/plugin-thread 0.8.4-main.f5c0578 → 0.8.4-main.fcfe5033a5

package/src/calls/util/stub-media-streams.ts

@@ -30,7 +30,7 @@ export const createBlackCanvasStreamTrack = async ({
     canvasCtx.fillStyle = 'black';
     canvasCtx.fillRect(0, 0, canvas.width, canvas.height);
   };
-  scheduleTaskInterval(ctx, async () => drawFrame(), 1_000);
+  scheduleTaskInterval(ctx, async () => drawFrame(), 200);
 
   const track = canvas.captureStream().getVideoTracks()[0];
   drawFrame();
@@ -40,27 +40,84 @@ export const createBlackCanvasStreamTrack = async ({
   return track;
 };
 
+let audioContextStartedPreviously = false;
+const userGestureEvents = [
+  'click',
+  'contextmenu',
+  'auxclick',
+  'dblclick',
+  'mousedown',
+  'mouseup',
+  'pointerup',
+  'touchend',
+  'keydown',
+  'keyup',
+];
+
 /**
  * Creates an inaudible audio stream track.
  * We need this to have audio track available when audio is disabled.
  */
 export const createInaudibleAudioStreamTrack = async ({ ctx }: { ctx: Context }): Promise<MediaStreamTrack> => {
   const audioContext = new window.AudioContext();
+
   const oscillator = audioContext.createOscillator();
   oscillator.type = 'triangle';
-  // roughly sounds like a box fan
   oscillator.frequency.setValueAtTime(20, audioContext.currentTime);
 
   const gainNode = audioContext.createGain();
-  // even w/ gain at 0 some packets are sent
-  gainNode.gain.setValueAtTime(0.02, audioContext.currentTime);
+  gainNode.gain.setValueAtTime(0, audioContext.currentTime);
   oscillator.connect(gainNode);
 
   const destination = audioContext.createMediaStreamDestination();
   gainNode.connect(destination);
-  oscillator.start();
+
+  let oscillatorStarted = false;
+  const ensureOscillatorStarted = () => {
+    if (oscillatorStarted) return;
+    oscillator.start();
+    oscillatorStarted = true;
+  };
+
+  const stateChangeHandler = () => {
+    if (audioContext.state === 'running') {
+      audioContextStartedPreviously = true;
+      ensureOscillatorStarted();
+    }
+
+    if (audioContext.state === 'suspended' || (audioContext.state as string) === 'interrupted') {
+      resumeAudioContext();
+    }
+  };
+
+  audioContext.addEventListener('statechange', stateChangeHandler);
+
+  const resumeAudioContext = () => {
+    void audioContext.resume().then(() => {
+      cleanUpUserGestureListeners();
+    });
+  };
+
+  const cleanUpUserGestureListeners = () => {
+    userGestureEvents.forEach((gesture) => {
+      document.removeEventListener(gesture, resumeAudioContext, {
+        capture: true,
+      });
+    });
+  };
+
+  if (audioContextStartedPreviously) {
+    resumeAudioContext();
+  } else {
+    userGestureEvents.forEach((gesture) => {
+      document.addEventListener(gesture, resumeAudioContext, {
+        capture: true,
+      });
+    });
+  }
 
   ctx.onDispose(async () => {
+    cleanUpUserGestureListeners();
     oscillator.disconnect();
     oscillator.stop();
     await audioContext.close();

package/src/calls/util/task-scheduling.ts

@@ -27,8 +27,8 @@ export class FIFOScheduler {
 }
 
 // TODO(mykola): Reconcile with @dxos/async task scheduling.
-export class BulkRequestDispatcher<RequestEntryParams, BulkResponse> {
-  #currentBatch: RequestEntryParams[];
+export class BulkRequestDispatcher<RequestEntryProps, BulkResponse> {
+  #currentBatch: RequestEntryProps[];
   #currentBulkResponse: Promise<BulkResponse> | null;
   #batchSizeLimit: number;
 
@@ -43,8 +43,8 @@ export class BulkRequestDispatcher<RequestEntryParams, BulkResponse> {
    * At the event loop iteration end, the accumulated entries will be dispatched as a bulk request.
    */
   doBulkRequest(
-    params: RequestEntryParams,
-    bulkRequestFunc: (bulkCopy: RequestEntryParams[]) => Promise<BulkResponse>,
+    params: RequestEntryProps,
+    bulkRequestFunc: (bulkCopy: RequestEntryProps[]) => Promise<BulkResponse>,
   ): Promise<BulkResponse> {
     if (this.#currentBatch.length >= this.#batchSizeLimit) {
       // If it reaches the batch size limit, we make another bulk request.

package/src/calls-service.stories.tsx

@@ -2,8 +2,7 @@
 // Copyright 2025 DXOS.org
 //
 
-import '@dxos-theme';
-
+import { type Meta, type StoryObj } from '@storybook/react-vite';
 import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 
 import { scheduleTask, sleep } from '@dxos/async';
@@ -14,13 +13,15 @@ import { Config, useConfig } from '@dxos/react-client';
 import { withClientProvider } from '@dxos/react-client/testing';
 import { Button } from '@dxos/react-ui';
 import { Json } from '@dxos/react-ui-syntax-highlighter';
-import { withLayout, withTheme } from '@dxos/storybook-utils';
+import { withTheme } from '@dxos/react-ui/testing';
+
+const testVideo = new URL('../testing/video.mp4', import.meta.url).href;
 
-import video from '../testing/video.mp4?raw';
+import { useBlackCanvasStreamTrack, useInaudibleAudioStreamTrack, useVideoStreamTrack } from '#hooks';
 
 import { CALLS_URL, CallsServicePeer, type TrackObject } from './calls';
-import { useInaudibleAudioStreamTrack, useVideoStreamTrack } from './hooks';
 
+// TODO(burdon): THIS IS TOO COMPLEX FOR A TEST SETUP.
 const pushAndPullTrack = (mediaStreamTrack?: MediaStreamTrack) => {
   const config = useConfig();
   const callsConfig = {
@@ -35,16 +36,18 @@ const pushAndPullTrack = (mediaStreamTrack?: MediaStreamTrack) => {
   const [pulledTrack, setPulledTrack] = useState<MediaStreamTrack | undefined>(undefined);
   const hadRun = useRef(false);
 
+  const deps: any[] = [];
   // Push/pull video stream track to cloudflare.
   useEffect(() => {
     if (hadRun.current || !mediaStreamTrack || !peerPush || !peerPull) {
-      log.info('not running', { hadRun: hadRun.current, mediaStreamTrack, peerPush, peerPull });
       return;
     }
     hadRun.current = true;
     const ctx = Context.default();
+    log.info('setting up ctx', { ctx, mediaStreamTrack, peerPush, peerPull });
+    deps.push([ctx, mediaStreamTrack, peerPush, peerPull]);
     ctx.onDispose(() => {
-      log.info('disposing ctx', { ctx });
+      log.info('disposing ctx', {});
     });
     scheduleTask(
       ctx,
@@ -60,15 +63,12 @@ const pushAndPullTrack = (mediaStreamTrack?: MediaStreamTrack) => {
         // Push track to cloudflare.
         performance.mark('push:begin');
         log.info('pushing track', { track: mediaStreamTrack });
-        trackInfo.current = await peerPush.pushTrack({ track: mediaStreamTrack });
+        trackInfo.current = await peerPush.pushTrack({ track: mediaStreamTrack, ctx: Context.default() });
         performance.mark('push:end');
         const pushTime = performance.measure('push', 'push:begin', 'push:end').duration;
         log.info('successfully pushed track', { pushTime, trackInfo: trackInfo.current });
         setMetrics((prev) => ({ ...prev, 'time to push track [ms]': Math.round(pushTime) }));
 
-        // Wait for cloudflare to process the track.
-        await sleep(1000);
-
         // Pull track from cloudflare.
         performance.mark('pullTrack:begin');
         pullCtx.current = Context.default();
@@ -104,7 +104,10 @@ const pushAndPullTrack = (mediaStreamTrack?: MediaStreamTrack) => {
 
   const rePullTrack = useCallback(async () => {
     performance.mark('rePullVideo:begin');
-    invariant(pulledTrack);
+    if (!pulledTrack) {
+      log.info('no pulled track, skipping re-pull', { pulledTrack });
+      return;
+    }
     pulledTrack.stop();
     await pullCtx.current!.dispose();
     pullCtx.current = Context.default();
@@ -123,12 +126,57 @@ const pushAndPullTrack = (mediaStreamTrack?: MediaStreamTrack) => {
     pulledTrack,
     rePullTrack,
     metrics,
+    pushService: peerPush,
+    pullService: peerPull,
   };
 };
 
+type DefaultStoryProps = { source: string };
+
+const DefaultStory = ({ source }: DefaultStoryProps) => {
+  const pushVideoElement = useRef<HTMLVideoElement>(null);
+  const pullVideoElement = useRef<HTMLVideoElement>(null);
+  // Get video stream track.
+  const videoStreamTrack = useVideoStreamTrack(pushVideoElement, source);
+  const { pulledTrack, rePullTrack, metrics } = pushAndPullTrack(videoStreamTrack);
+  const hadRun = useRef(false);
+
+  // Push/pull video stream track to cloudflare.
+  useEffect(() => {
+    if (!pulledTrack || hadRun.current) {
+      return;
+    }
+    hadRun.current = true;
+    const ctx = Context.default();
+    scheduleTask(ctx, async () => {
+      pullVideoElement.current!.srcObject = new MediaStream([pulledTrack]);
+      await pullVideoElement.current!.play();
+    });
+
+    return () => {
+      void ctx.dispose();
+    };
+  }, [pulledTrack]);
+
+  return (
+    <div className='grid grid-rows-3 gap-4 items-center'>
+      <video ref={pushVideoElement} muted autoPlay loop />
+      <div className='flex flex-col gap-4'>
+        <video ref={pullVideoElement} muted />
+        <Button disabled={!pulledTrack} onClick={rePullTrack}>
+          Re-pull video
+        </Button>
+      </div>
+      <Json.Data data={metrics} />
+    </div>
+  );
+};
+
 const meta = {
-  title: 'plugins/plugin-thread/calls-service',
+  title: 'plugins/plugin-thread/experimental/Calls',
+  render: DefaultStory,
   decorators: [
+    withTheme(),
     withClientProvider({
       config: new Config({
         runtime: {
@@ -138,77 +186,43 @@ const meta = {
         },
       }),
     }),
-    withTheme,
-    withLayout(),
   ],
   parameters: {
     layout: 'centered',
   },
-};
+} satisfies Meta<typeof DefaultStory>;
 
 export default meta;
 
-export const Default = {
-  render: ({ videoSrc }: { videoSrc: string }) => {
-    const pushVideoElement = useRef<HTMLVideoElement>(null);
-    const pullVideoElement = useRef<HTMLVideoElement>(null);
-    // Get video stream track.
-    const videoStreamTrack = useVideoStreamTrack(pushVideoElement, videoSrc);
-    const { pulledTrack, rePullTrack, metrics } = pushAndPullTrack(videoStreamTrack);
-    const hadRun = useRef(false);
-
-    // Push/pull video stream track to cloudflare.
-    useEffect(() => {
-      if (!pulledTrack || hadRun.current) {
-        return;
-      }
-      hadRun.current = true;
-      const ctx = Context.default();
-      scheduleTask(ctx, async () => {
-        pullVideoElement.current!.srcObject = new MediaStream([pulledTrack]);
-        await pullVideoElement.current!.play();
-      });
-
-      return () => {
-        void ctx.dispose();
-      };
-    }, [pulledTrack]);
+type Story = StoryObj<typeof meta>;
 
-    return (
-      <div className='grid grid-rows-3 gap-4 items-center'>
-        <video ref={pushVideoElement} muted autoPlay loop />
-        <div className='flex flex-col gap-4'>
-          <video ref={pullVideoElement} muted />
-          <Button
-            disabled={
-              !(
-                pullVideoElement?.current?.srcObject instanceof MediaStream &&
-                pullVideoElement.current.srcObject.getTracks()[0]
-              )
-            }
-            onClick={rePullTrack}
-          >
-            Re-pull video
-          </Button>
-        </div>
-        <Json data={metrics} />
-      </div>
-    );
-  },
+export const Default: Story = {
   args: {
-    videoSrc: video,
+    source: testVideo,
   },
 };
 
 export const InaudibleAudioStreamTrack = {
   render: () => {
     const audioStreamTrack = useInaudibleAudioStreamTrack();
-
     const { rePullTrack, metrics } = pushAndPullTrack(audioStreamTrack);
 
     return (
       <div className='flex flex-col gap-4 items-center'>
-        <Json data={metrics} />
+        <Json.Data data={metrics} />
+        <Button onClick={rePullTrack}>Re-pull audio</Button>
+      </div>
+    );
+  },
+};
+
+export const BlackVideoStreamTrack = {
+  render: () => {
+    const videoStreamTrack = useBlackCanvasStreamTrack();
+    const { rePullTrack, metrics } = pushAndPullTrack(videoStreamTrack);
+    return (
+      <div className='flex flex-col gap-4 items-center'>
+        <Json.Data data={metrics} />
         <Button onClick={rePullTrack}>Re-pull audio</Button>
       </div>
     );