@dxos/plugin-thread 0.8.4-main.28f8d3d → 0.8.4-main.406dc2a

package/src/calls/util/calls-service.ts

@@ -2,7 +2,7 @@
 // Copyright 2024 DXOS.org
 //
 
-import { PersistentLifecycle } from '@dxos/async';
+import { PersistentLifecycle, Trigger, scheduleTask } from '@dxos/async';
 import { type Context, Resource } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
@@ -33,12 +33,14 @@ export type CallsServiceConfig = {
 
 export type HistoryRecord =
   | {
+      endpoint: string;
       type: 'request';
       time: string;
       method: string;
       body: any;
     }
   | {
+      endpoint: string;
       type: 'response';
       time: string;
       status: number;
@@ -116,7 +118,7 @@ export class CallsServicePeer extends Resource {
     });
     peerConnection.addEventListener('connectionstatechange', () => {
       if (peerConnection.connectionState === 'failed' || peerConnection.connectionState === 'closed') {
-        log.warn('calls connection failed', { state: peerConnection.connectionState });
+        log.info('calls connection failed', { state: peerConnection.connectionState });
         void this._persistentLifecycle.scheduleRestart();
       }
     });
@@ -125,7 +127,7 @@ export class CallsServicePeer extends Resource {
     peerConnection.addEventListener('iceconnectionstatechange', () => {
       clearTimeout(iceTimeout);
       if (peerConnection.iceConnectionState === 'failed' || peerConnection.iceConnectionState === 'closed') {
-        log.warn('calls connection to ICEs failed', { state: peerConnection.iceConnectionState });
+        log.info('calls connection to ICEs failed', { state: peerConnection.iceConnectionState });
         void this._persistentLifecycle.scheduleRestart();
       } else if (peerConnection.iceConnectionState === 'disconnected') {
         // TODO(mykola): we should start to inspect the connection stats from here on for
@@ -138,7 +140,7 @@ export class CallsServicePeer extends Resource {
             return;
           }
 
-          log.warn('calls iceConnectionState timed out', { state: peerConnection.iceConnectionState, timeoutSeconds });
+          log.info('calls iceConnectionState timed out', { state: peerConnection.iceConnectionState, timeoutSeconds });
           void this._persistentLifecycle.scheduleRestart();
         }, timeoutSeconds * 1_000);
       }
@@ -155,6 +157,7 @@ export class CallsServicePeer extends Resource {
 
   private async _fetch<T extends ErrorResponse>(relativePath: string, requestInit?: RequestInit): Promise<T> {
     this.history.push({
+      endpoint: relativePath,
       type: 'request',
       time: new Date().toISOString(),
       method: requestInit?.method ?? 'GET',
@@ -170,6 +173,7 @@ export class CallsServicePeer extends Resource {
     try {
       const data = (await response.clone().json()) as T;
       this.history.push({
+        endpoint: relativePath,
         type: 'response',
         time: new Date().toISOString(),
         status: response.status,
@@ -183,6 +187,7 @@ export class CallsServicePeer extends Resource {
     } catch (error) {
       const text = await response.text();
       this.history.push({
+        endpoint: relativePath,
         type: 'response',
         time: new Date().toISOString(),
         status: response.status,
@@ -204,11 +209,11 @@ export class CallsServicePeer extends Resource {
     sessionId,
     trackName,
   }: {
+    ctx: Context;
     peerConnection: RTCPeerConnection;
     transceiver: RTCRtpTransceiver;
     sessionId: string;
     trackName: string;
-    ctx?: Context;
   }): Promise<TrackObject | void> {
     log.verbose('pushing track ', trackName);
     const pushedTrackPromise = this.pushTrackDispatcher
@@ -246,9 +251,13 @@ export class CallsServicePeer extends Resource {
           };
         }),
       )
-      .then(({ tracks }) => {
+      .then(async ({ tracks }) => {
         const trackData = tracks.find((t) => t.mid === transceiver.mid);
         if (trackData) {
+          // we wait for the transceiver to start sending data before we emit
+          // the track metadata to ensure that the track will be able to be
+          // pulled before making the metadata available to anyone else.
+          // await waitForTransceiverToSendData(ctx, transceiver);
           return {
             ...trackData,
             sessionId,
@@ -257,9 +266,6 @@ export class CallsServicePeer extends Resource {
         } else {
           log.error('missing track data');
         }
-      })
-      .catch((err) => {
-        log.catch(err);
       });
 
     const onDispose = async () => {
@@ -281,8 +287,8 @@ export class CallsServicePeer extends Resource {
     previousTrack,
   }: {
     track: MediaStreamTrack | null;
+    ctx: Context;
     encodings?: RTCRtpEncodingParameters[];
-    ctx?: Context;
     previousTrack?: TrackObject;
   }): Promise<TrackObject | undefined> {
     await this.waitUntilOpen();
@@ -405,7 +411,7 @@ export class CallsServicePeer extends Resource {
             })
             .catch((err) => log.catch(err));
         } else {
-          log.warn('missing track info', { trackData, availableTracks: Array.from(trackMap.keys()) });
+          log.info('missing track info', { trackData, availableTracks: Array.from(trackMap.keys()) });
         }
       });
 
@@ -529,3 +535,47 @@ const peerConnectionIsConnected = async (peerConnection: RTCPeerConnection) => {
     await connected;
   }
 };
+
+const INITIAL_DELAY = 50;
+const MAX_DELAY = 500;
+const MAX_RETRIES = 20;
+const WAIT_FOR_DATA_TIMEOUT = 10_000;
+
+const waitForTransceiverToSendData = async (ctx: Context, transceiver: RTCRtpTransceiver): Promise<void> => {
+  let delay = INITIAL_DELAY;
+  let checks = 0;
+  const waitForData = new Trigger();
+  ctx.onDispose(() => waitForData.throw(new Error('Transceiver was closed')));
+
+  const checkStats = async () => {
+    if (ctx.disposed) return;
+
+    try {
+      const stats = await transceiver.sender.getStats();
+      let dataFound = false;
+      stats.forEach((stat) => {
+        if (stat.type === 'outbound-rtp' && stat.bytesSent > 0) {
+          dataFound = true;
+        }
+      });
+
+      if (dataFound) {
+        waitForData.wake();
+        return;
+      }
+    } catch {
+      // Stats might not be available yet, continue checking
+    }
+    checks++;
+    if (checks > MAX_RETRIES) {
+      waitForData.throw(new Error('Transceiver did not send data'));
+      return;
+    }
+
+    delay = Math.min(delay * 2, MAX_DELAY); // Exponential backoff with max cap
+    scheduleTask(ctx, checkStats, delay);
+  };
+
+  void checkStats();
+  return waitForData.wait({ timeout: WAIT_FOR_DATA_TIMEOUT });
+};

package/src/calls/util/stub-media-streams.ts

@@ -30,7 +30,7 @@ export const createBlackCanvasStreamTrack = async ({
     canvasCtx.fillStyle = 'black';
     canvasCtx.fillRect(0, 0, canvas.width, canvas.height);
   };
-  scheduleTaskInterval(ctx, async () => drawFrame(), 1_000);
+  scheduleTaskInterval(ctx, async () => drawFrame(), 200);
 
   const track = canvas.captureStream().getVideoTracks()[0];
   drawFrame();
@@ -40,6 +40,20 @@ export const createBlackCanvasStreamTrack = async ({
   return track;
 };
 
+let audioContextStartedPreviously = false;
+const userGestureEvents = [
+  'click',
+  'contextmenu',
+  'auxclick',
+  'dblclick',
+  'mousedown',
+  'mouseup',
+  'pointerup',
+  'touchend',
+  'keydown',
+  'keyup',
+];
+
 /**
  * Creates an inaudible audio stream track.
  * We need this to have audio track available when audio is disabled.
@@ -53,14 +67,57 @@ export const createInaudibleAudioStreamTrack = async ({ ctx }: { ctx: Context })
 
   const gainNode = audioContext.createGain();
   // even w/ gain at 0 some packets are sent
-  gainNode.gain.setValueAtTime(0.02, audioContext.currentTime);
+  gainNode.gain.setValueAtTime(0, audioContext.currentTime);
   oscillator.connect(gainNode);
 
   const destination = audioContext.createMediaStreamDestination();
   gainNode.connect(destination);
-  oscillator.start();
+
+  let oscillatorStarted = false;
+  const ensureOscillatorStarted = () => {
+    if (oscillatorStarted) return;
+    oscillator.start();
+    oscillatorStarted = true;
+  };
+
+  const stateChangeHandler = () => {
+    if (audioContext.state === 'running') {
+      audioContextStartedPreviously = true;
+      ensureOscillatorStarted();
+    }
+    if (audioContext.state === 'suspended' || (audioContext.state as string) === 'interrupted') {
+      resumeAudioContext();
+    }
+  };
+
+  audioContext.addEventListener('statechange', stateChangeHandler);
+
+  const resumeAudioContext = () => {
+    void audioContext.resume().then(() => {
+      cleanUpUserGestureListeners();
+    });
+  };
+
+  const cleanUpUserGestureListeners = () => {
+    userGestureEvents.forEach((gesture) => {
+      document.removeEventListener(gesture, resumeAudioContext, {
+        capture: true,
+      });
+    });
+  };
+
+  if (audioContextStartedPreviously) {
+    resumeAudioContext();
+  } else {
+    userGestureEvents.forEach((gesture) => {
+      document.addEventListener(gesture, resumeAudioContext, {
+        capture: true,
+      });
+    });
+  }
 
   ctx.onDispose(async () => {
+    cleanUpUserGestureListeners();
     oscillator.disconnect();
     oscillator.stop();
     await audioContext.close();

package/src/calls-service.stories.tsx

@@ -2,8 +2,7 @@
 // Copyright 2025 DXOS.org
 //
 
-import '@dxos-theme';
-
+import { type Meta, type StoryObj } from '@storybook/react-vite';
 import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 
 import { scheduleTask, sleep } from '@dxos/async';
@@ -13,14 +12,15 @@ import { log } from '@dxos/log';
 import { Config, useConfig } from '@dxos/react-client';
 import { withClientProvider } from '@dxos/react-client/testing';
 import { Button } from '@dxos/react-ui';
+import { withTheme } from '@dxos/react-ui/testing';
 import { Json } from '@dxos/react-ui-syntax-highlighter';
-import { withLayout, withTheme } from '@dxos/storybook-utils';
 
-import video from '../testing/video.mp4?raw';
+const testVideo = new URL('../testing/video.mp4', import.meta.url).href;
 
 import { CALLS_URL, CallsServicePeer, type TrackObject } from './calls';
-import { useInaudibleAudioStreamTrack, useVideoStreamTrack } from './hooks';
+import { useBlackCanvasStreamTrack, useInaudibleAudioStreamTrack, useVideoStreamTrack } from './hooks';
 
+// TODO(burdon): THIS IS TOO COMPLEX FOR A TEST SETUP.
 const pushAndPullTrack = (mediaStreamTrack?: MediaStreamTrack) => {
   const config = useConfig();
   const callsConfig = {
@@ -35,16 +35,18 @@ const pushAndPullTrack = (mediaStreamTrack?: MediaStreamTrack) => {
   const [pulledTrack, setPulledTrack] = useState<MediaStreamTrack | undefined>(undefined);
   const hadRun = useRef(false);
 
+  const deps: any[] = [];
   // Push/pull video stream track to cloudflare.
   useEffect(() => {
     if (hadRun.current || !mediaStreamTrack || !peerPush || !peerPull) {
-      log.info('not running', { hadRun: hadRun.current, mediaStreamTrack, peerPush, peerPull });
       return;
     }
     hadRun.current = true;
     const ctx = Context.default();
+    log.info('setting up ctx', { ctx, mediaStreamTrack, peerPush, peerPull });
+    deps.push([ctx, mediaStreamTrack, peerPush, peerPull]);
     ctx.onDispose(() => {
-      log.info('disposing ctx', { ctx });
+      log.info('disposing ctx', {});
     });
     scheduleTask(
       ctx,
@@ -60,15 +62,12 @@ const pushAndPullTrack = (mediaStreamTrack?: MediaStreamTrack) => {
         // Push track to cloudflare.
         performance.mark('push:begin');
         log.info('pushing track', { track: mediaStreamTrack });
-        trackInfo.current = await peerPush.pushTrack({ track: mediaStreamTrack });
+        trackInfo.current = await peerPush.pushTrack({ track: mediaStreamTrack, ctx: Context.default() });
         performance.mark('push:end');
         const pushTime = performance.measure('push', 'push:begin', 'push:end').duration;
         log.info('successfully pushed track', { pushTime, trackInfo: trackInfo.current });
         setMetrics((prev) => ({ ...prev, 'time to push track [ms]': Math.round(pushTime) }));
 
-        // Wait for cloudflare to process the track.
-        await sleep(1000);
-
         // Pull track from cloudflare.
         performance.mark('pullTrack:begin');
         pullCtx.current = Context.default();
@@ -104,7 +103,10 @@ const pushAndPullTrack = (mediaStreamTrack?: MediaStreamTrack) => {
 
   const rePullTrack = useCallback(async () => {
     performance.mark('rePullVideo:begin');
-    invariant(pulledTrack);
+    if (!pulledTrack) {
+      log.info('no pulled track, skipping re-pull', { pulledTrack });
+      return;
+    }
     pulledTrack.stop();
     await pullCtx.current!.dispose();
     pullCtx.current = Context.default();
@@ -123,12 +125,57 @@ const pushAndPullTrack = (mediaStreamTrack?: MediaStreamTrack) => {
     pulledTrack,
     rePullTrack,
     metrics,
+    pushService: peerPush,
+    pullService: peerPull,
   };
 };
 
+type StoryProps = { source: string };
+
+const DefaultStory = ({ source }: StoryProps) => {
+  const pushVideoElement = useRef<HTMLVideoElement>(null);
+  const pullVideoElement = useRef<HTMLVideoElement>(null);
+  // Get video stream track.
+  const videoStreamTrack = useVideoStreamTrack(pushVideoElement, source);
+  const { pulledTrack, rePullTrack, metrics } = pushAndPullTrack(videoStreamTrack);
+  const hadRun = useRef(false);
+
+  // Push/pull video stream track to cloudflare.
+  useEffect(() => {
+    if (!pulledTrack || hadRun.current) {
+      return;
+    }
+    hadRun.current = true;
+    const ctx = Context.default();
+    scheduleTask(ctx, async () => {
+      pullVideoElement.current!.srcObject = new MediaStream([pulledTrack]);
+      await pullVideoElement.current!.play();
+    });
+
+    return () => {
+      void ctx.dispose();
+    };
+  }, [pulledTrack]);
+
+  return (
+    <div className='grid grid-rows-3 gap-4 items-center'>
+      <video ref={pushVideoElement} muted autoPlay loop />
+      <div className='flex flex-col gap-4'>
+        <video ref={pullVideoElement} muted />
+        <Button disabled={!pulledTrack} onClick={rePullTrack}>
+          Re-pull video
+        </Button>
+      </div>
+      <Json data={metrics} />
+    </div>
+  );
+};
+
 const meta = {
   title: 'plugins/plugin-thread/calls-service',
+  render: DefaultStory,
   decorators: [
+    withTheme,
     withClientProvider({
       config: new Config({
         runtime: {
@@ -138,72 +185,25 @@ const meta = {
         },
       }),
     }),
-    withTheme,
-    withLayout(),
   ],
   parameters: {
     layout: 'centered',
   },
-};
+} satisfies Meta<typeof DefaultStory>;
 
 export default meta;
 
-export const Default = {
-  render: ({ videoSrc }: { videoSrc: string }) => {
-    const pushVideoElement = useRef<HTMLVideoElement>(null);
-    const pullVideoElement = useRef<HTMLVideoElement>(null);
-    // Get video stream track.
-    const videoStreamTrack = useVideoStreamTrack(pushVideoElement, videoSrc);
-    const { pulledTrack, rePullTrack, metrics } = pushAndPullTrack(videoStreamTrack);
-    const hadRun = useRef(false);
-
-    // Push/pull video stream track to cloudflare.
-    useEffect(() => {
-      if (!pulledTrack || hadRun.current) {
-        return;
-      }
-      hadRun.current = true;
-      const ctx = Context.default();
-      scheduleTask(ctx, async () => {
-        pullVideoElement.current!.srcObject = new MediaStream([pulledTrack]);
-        await pullVideoElement.current!.play();
-      });
-
-      return () => {
-        void ctx.dispose();
-      };
-    }, [pulledTrack]);
+type Story = StoryObj<typeof meta>;
 
-    return (
-      <div className='grid grid-rows-3 gap-4 items-center'>
-        <video ref={pushVideoElement} muted autoPlay loop />
-        <div className='flex flex-col gap-4'>
-          <video ref={pullVideoElement} muted />
-          <Button
-            disabled={
-              !(
-                pullVideoElement?.current?.srcObject instanceof MediaStream &&
-                pullVideoElement.current.srcObject.getTracks()[0]
-              )
-            }
-            onClick={rePullTrack}
-          >
-            Re-pull video
-          </Button>
-        </div>
-        <Json data={metrics} />
-      </div>
-    );
-  },
+export const Default: Story = {
   args: {
-    videoSrc: video,
+    source: testVideo,
   },
 };
 
 export const InaudibleAudioStreamTrack = {
   render: () => {
     const audioStreamTrack = useInaudibleAudioStreamTrack();
-
     const { rePullTrack, metrics } = pushAndPullTrack(audioStreamTrack);
 
     return (
@@ -214,3 +214,16 @@ export const InaudibleAudioStreamTrack = {
     );
   },
 };
+
+export const BlackVideoStreamTrack = {
+  render: () => {
+    const videoStreamTrack = useBlackCanvasStreamTrack();
+    const { rePullTrack, metrics } = pushAndPullTrack(videoStreamTrack);
+    return (
+      <div className='flex flex-col gap-4 items-center'>
+        <Json data={metrics} />
+        <Button onClick={rePullTrack}>Re-pull audio</Button>
+      </div>
+    );
+  },
+};

package/src/capabilities/app-graph-builder.ts

@@ -3,7 +3,8 @@
 //
 
 import { Rx } from '@effect-rx/rx-react';
-import { Option, pipe } from 'effect';
+import * as Function from 'effect/Function';
+import * as Option from 'effect/Option';
 
 import { Capabilities, type PluginContext, contributes, createIntent } from '@dxos/app-framework';
 import { Obj } from '@dxos/echo';
@@ -12,22 +13,24 @@ import { ATTENDABLE_PATH_SEPARATOR, DECK_COMPANION_TYPE, PLANK_COMPANION_TYPE }
 import { ROOT_ID, createExtension, rxFromSignal } from '@dxos/plugin-graph';
 import { fullyQualifiedId } from '@dxos/react-client/echo';
 
-import { THREAD_PLUGIN, meta } from '../meta';
+import { meta } from '../meta';
 import { ChannelType, ThreadAction } from '../types';
 import { getAnchor } from '../util';
 
 import { ThreadCapabilities } from './capabilities';
 
+// TODO(wittjosiah): Highlight active calls in L1.
+//  Track active meetings by subscribing to meetings query and polling the swarms of recent meetings in the space.
 export default (context: PluginContext) => {
   const resolve = (typename: string) =>
     context.getCapabilities(Capabilities.Metadata).find(({ id }) => id === typename)?.metadata ?? {};
 
   return contributes(Capabilities.AppGraphBuilder, [
     createExtension({
-      id: `${THREAD_PLUGIN}/active-call`,
+      id: `${meta.id}/active-call`,
       connector: (node) =>
         Rx.make((get) =>
-          pipe(
+          Function.pipe(
             get(node),
             Option.flatMap((node) => (node.id === ROOT_ID ? Option.some(node) : Option.none())),
             Option.map((node) => {
@@ -41,7 +44,7 @@ export default (context: PluginContext) => {
                           type: DECK_COMPANION_TYPE,
                           data: null,
                           properties: {
-                            label: ['call panel label', { ns: THREAD_PLUGIN }],
+                            label: ['call panel label', { ns: meta.id }],
                             icon: 'ph--video-conference--regular',
                             position: 'hoist',
                             disposition: 'hidden',
@@ -59,10 +62,10 @@ export default (context: PluginContext) => {
     // TODO(wittjosiah): The channel shouldn't become the companion during a call.
     //   Alternative: the call/meeting/thread should be a child node of the channel and that should be opened.
     createExtension({
-      id: `${THREAD_PLUGIN}/channel-chat-companion`,
+      id: `${meta.id}/channel-chat-companion`,
       connector: (node) => {
         return Rx.make((get) => {
-          return pipe(
+          return Function.pipe(
             get(node),
             Option.flatMap((node) => (Obj.instanceOf(ChannelType, node.data) ? Option.some(node.data) : Option.none())),
             Option.map((channel) => {
@@ -80,7 +83,7 @@ export default (context: PluginContext) => {
                   type: PLANK_COMPANION_TYPE,
                   data: 'chat',
                   properties: {
-                    label: ['channel companion label', { ns: THREAD_PLUGIN }],
+                    label: ['channel companion label', { ns: meta.id }],
                     icon: 'ph--hash--regular',
                     position: 'hoist',
                     disposition: 'hidden',
@@ -97,7 +100,7 @@ export default (context: PluginContext) => {
       id: `${meta.id}/comments-companion`,
       connector: (node) =>
         Rx.make((get) =>
-          pipe(
+          Function.pipe(
             get(node),
             Option.flatMap((node) => {
               if (!Obj.isObject(node.data) || Obj.instanceOf(ChannelType, node.data)) {
@@ -127,7 +130,7 @@ export default (context: PluginContext) => {
       id: `${meta.id}/comment-toolbar`,
       actions: (node) =>
         Rx.make((get) =>
-          pipe(
+          Function.pipe(
             get(node),
             Option.flatMap((node) => {
               if (!Obj.isObject(node.data) || Obj.instanceOf(ChannelType, node.data)) {

package/src/capabilities/capabilities.ts

@@ -6,11 +6,11 @@ import { defineCapability } from '@dxos/app-framework';
 import { type DeepReadonly } from '@dxos/util';
 
 import { type CallManager, type CallState, type MediaState } from '../calls';
-import { THREAD_PLUGIN } from '../meta';
+import { meta } from '../meta';
 import { type ChannelType, type ThreadState, type ViewState } from '../types';
 
 export namespace ThreadCapabilities {
-  export const CallManager = defineCapability<CallManager>(`${THREAD_PLUGIN}/capability/call-manager`);
+  export const CallManager = defineCapability<CallManager>(`${meta.id}/capability/call-manager`);
 
   // TODO(burdon): Better way to define specific extensions for meeting companions.
   // TODO(burdon): This brings in deps from ../calls; how should we manage/minimize explicit type exposure to other plugins?
@@ -21,13 +21,13 @@ export namespace ThreadCapabilities {
     onCallStateUpdated: (callState: CallState) => Promise<void>;
     onMediaStateUpdated: ([mediaState, isSpeaking]: [MediaState, boolean]) => Promise<void>;
   };
-  export const CallExtension = defineCapability<CallProperties>(`${THREAD_PLUGIN}/capability/call-extension`);
+  export const CallExtension = defineCapability<CallProperties>(`${meta.id}/capability/call-extension`);
 
   type GetViewState = (subjectId: string) => ViewState;
   export const State = defineCapability<{ state: DeepReadonly<ThreadState>; getViewState: GetViewState }>(
-    `${THREAD_PLUGIN}/capability/state`,
+    `${meta.id}/capability/state`,
   );
   export const MutableState = defineCapability<{ state: ThreadState; getViewState: GetViewState }>(
-    `${THREAD_PLUGIN}/capability/state`,
+    `${meta.id}/capability/state`,
   );
 }

package/src/capabilities/index.ts

@@ -12,5 +12,6 @@ export const ReactRoot = lazy(() => import('./react-root'));
 export const ReactSurface = lazy(() => import('./react-surface'));
 export const ThreadSettings = lazy(() => import('./settings'));
 export const ThreadState = lazy(() => import('./state'));
+export const Toolkit = lazy(() => import('./toolkit'));
 
 export * from './capabilities';