@dxos/plugin-thread 0.8.3 → 0.8.4-main.1f223c7

package/src/calls/util/calls-service.ts

@@ -2,21 +2,22 @@
 // Copyright 2024 DXOS.org
 //
 
-import { PersistentLifecycle } from '@dxos/async';
+import { PersistentLifecycle, Trigger, scheduleTask } from '@dxos/async';
 import { type Context, Resource } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 
-import { HistoryCache } from './history-cache';
-import { BulkRequestDispatcher, FIFOScheduler } from './task-scheduling';
 import {
-  type SessionResponse,
   type ErrorResponse,
   type RenegotiationResponse,
+  type SessionResponse,
   type TrackObject,
   type TracksResponse,
 } from '../types';
 
+import { HistoryCache } from './history-cache';
+import { BulkRequestDispatcher, FIFOScheduler } from './task-scheduling';
+
 const NETWORK_TIMEOUT = 5_000;
 
 export type Session = {
@@ -32,12 +33,14 @@ export type CallsServiceConfig = {
 
 export type HistoryRecord =
   | {
+      endpoint: string;
       type: 'request';
       time: string;
       method: string;
       body: any;
     }
   | {
+      endpoint: string;
       type: 'response';
       time: string;
       status: number;
@@ -73,7 +76,10 @@ export class CallsServicePeer extends Resource {
   pullTrackDispatcher = new BulkRequestDispatcher<
     TrackObject,
     {
-      trackMap: Map<TrackObject, { resolvedTrack: Promise<MediaStreamTrack>; mid: string }>;
+      /**
+       * trackName -> { resolvedTrack, mid }
+       */
+      trackMap: Map<string, { resolvedTrack: Promise<MediaStreamTrack>; mid: string }>;
     }
   >(32);
 
@@ -112,7 +118,7 @@ export class CallsServicePeer extends Resource {
     });
     peerConnection.addEventListener('connectionstatechange', () => {
       if (peerConnection.connectionState === 'failed' || peerConnection.connectionState === 'closed') {
-        log.warn('calls connection failed', { state: peerConnection.connectionState });
+        log.info('calls connection failed', { state: peerConnection.connectionState });
         void this._persistentLifecycle.scheduleRestart();
       }
     });
@@ -121,7 +127,7 @@ export class CallsServicePeer extends Resource {
     peerConnection.addEventListener('iceconnectionstatechange', () => {
       clearTimeout(iceTimeout);
       if (peerConnection.iceConnectionState === 'failed' || peerConnection.iceConnectionState === 'closed') {
-        log.warn('calls connection to ICEs failed', { state: peerConnection.iceConnectionState });
+        log.info('calls connection to ICEs failed', { state: peerConnection.iceConnectionState });
         void this._persistentLifecycle.scheduleRestart();
       } else if (peerConnection.iceConnectionState === 'disconnected') {
         // TODO(mykola): we should start to inspect the connection stats from here on for
@@ -134,7 +140,7 @@ export class CallsServicePeer extends Resource {
             return;
           }
 
-          log.warn('calls iceConnectionState timed out', { state: peerConnection.iceConnectionState, timeoutSeconds });
+          log.info('calls iceConnectionState timed out', { state: peerConnection.iceConnectionState, timeoutSeconds });
           void this._persistentLifecycle.scheduleRestart();
         }, timeoutSeconds * 1_000);
       }
@@ -151,6 +157,7 @@ export class CallsServicePeer extends Resource {
 
   private async _fetch<T extends ErrorResponse>(relativePath: string, requestInit?: RequestInit): Promise<T> {
     this.history.push({
+      endpoint: relativePath,
       type: 'request',
       time: new Date().toISOString(),
       method: requestInit?.method ?? 'GET',
@@ -166,6 +173,7 @@ export class CallsServicePeer extends Resource {
     try {
       const data = (await response.clone().json()) as T;
       this.history.push({
+        endpoint: relativePath,
         type: 'response',
         time: new Date().toISOString(),
         status: response.status,
@@ -179,6 +187,7 @@ export class CallsServicePeer extends Resource {
     } catch (error) {
       const text = await response.text();
       this.history.push({
+        endpoint: relativePath,
         type: 'response',
         time: new Date().toISOString(),
         status: response.status,
@@ -200,11 +209,11 @@ export class CallsServicePeer extends Resource {
     sessionId,
     trackName,
   }: {
+    ctx: Context;
     peerConnection: RTCPeerConnection;
     transceiver: RTCRtpTransceiver;
     sessionId: string;
     trackName: string;
-    ctx?: Context;
   }): Promise<TrackObject | void> {
     log.verbose('pushing track ', trackName);
     const pushedTrackPromise = this.pushTrackDispatcher
@@ -242,9 +251,13 @@ export class CallsServicePeer extends Resource {
           };
         }),
       )
-      .then(({ tracks }) => {
+      .then(async ({ tracks }) => {
         const trackData = tracks.find((t) => t.mid === transceiver.mid);
         if (trackData) {
+          // we wait for the transceiver to start sending data before we emit
+          // the track metadata to ensure that the track will be able to be
+          // pulled before making the metadata available to anyone else.
+          // await waitForTransceiverToSendData(ctx, transceiver);
           return {
             ...trackData,
             sessionId,
@@ -253,9 +266,6 @@ export class CallsServicePeer extends Resource {
         } else {
           log.error('missing track data');
         }
-      })
-      .catch((err) => {
-        log.catch(err);
       });
 
     const onDispose = async () => {
@@ -277,8 +287,8 @@ export class CallsServicePeer extends Resource {
     previousTrack,
   }: {
     track: MediaStreamTrack | null;
+    ctx: Context;
     encodings?: RTCRtpEncodingParameters[];
-    ctx?: Context;
     previousTrack?: TrackObject;
   }): Promise<TrackObject | undefined> {
     await this.waitUntilOpen();
@@ -354,14 +364,14 @@ export class CallsServicePeer extends Resource {
             );
 
             if (pulledTrackData && pulledTrackData.mid) {
-              acc.set(track, {
+              acc.set(track.trackName!, {
                 mid: pulledTrackData.mid,
                 resolvedTrack: resolveTrack(peerConnection, (t) => t.mid === pulledTrackData.mid),
               });
             }
 
             return acc;
-          }, new Map<TrackObject, { resolvedTrack: Promise<MediaStreamTrack>; mid: string }>());
+          }, new Map<string, { resolvedTrack: Promise<MediaStreamTrack>; mid: string }>());
 
           if (newTrackResponse.requiresImmediateRenegotiation) {
             await peerConnection.setRemoteDescription(new RTCSessionDescription(newTrackResponse.sessionDescription));
@@ -392,7 +402,7 @@ export class CallsServicePeer extends Resource {
         }),
       )
       .then(({ trackMap }) => {
-        const trackInfo = trackMap.get(trackData);
+        const trackInfo = trackMap.get(trackData.trackName!);
         if (trackInfo) {
           return trackInfo.resolvedTrack
             .then((track) => {
@@ -401,7 +411,7 @@ export class CallsServicePeer extends Resource {
             })
             .catch((err) => log.catch(err));
         } else {
-          log.error('missing track info');
+          log.info('missing track info', { trackData, availableTracks: Array.from(trackMap.keys()) });
         }
       });
 
@@ -525,3 +535,47 @@ const peerConnectionIsConnected = async (peerConnection: RTCPeerConnection) => {
     await connected;
   }
 };
+
+const INITIAL_DELAY = 50;
+const MAX_DELAY = 500;
+const MAX_RETRIES = 20;
+const WAIT_FOR_DATA_TIMEOUT = 10_000;
+
+const waitForTransceiverToSendData = async (ctx: Context, transceiver: RTCRtpTransceiver): Promise<void> => {
+  let delay = INITIAL_DELAY;
+  let checks = 0;
+  const waitForData = new Trigger();
+  ctx.onDispose(() => waitForData.throw(new Error('Transceiver was closed')));
+
+  const checkStats = async () => {
+    if (ctx.disposed) return;
+
+    try {
+      const stats = await transceiver.sender.getStats();
+      let dataFound = false;
+      stats.forEach((stat) => {
+        if (stat.type === 'outbound-rtp' && stat.bytesSent > 0) {
+          dataFound = true;
+        }
+      });
+
+      if (dataFound) {
+        waitForData.wake();
+        return;
+      }
+    } catch {
+      // Stats might not be available yet, continue checking
+    }
+    checks++;
+    if (checks > MAX_RETRIES) {
+      waitForData.throw(new Error('Transceiver did not send data'));
+      return;
+    }
+
+    delay = Math.min(delay * 2, MAX_DELAY); // Exponential backoff with max cap
+    scheduleTask(ctx, checkStats, delay);
+  };
+
+  void checkStats();
+  return waitForData.wait({ timeout: WAIT_FOR_DATA_TIMEOUT });
+};

package/src/calls/util/stub-media-streams.ts

@@ -3,8 +3,8 @@
 //
 //
 
-import { waitForCondition, scheduleTaskInterval } from '@dxos/async';
-import { cancelWithContext, type Context } from '@dxos/context';
+import { scheduleTaskInterval, waitForCondition } from '@dxos/async';
+import { type Context, cancelWithContext } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
 
 /**
@@ -30,7 +30,7 @@ export const createBlackCanvasStreamTrack = async ({
     canvasCtx.fillStyle = 'black';
     canvasCtx.fillRect(0, 0, canvas.width, canvas.height);
   };
-  scheduleTaskInterval(ctx, async () => drawFrame(), 1_000);
+  scheduleTaskInterval(ctx, async () => drawFrame(), 200);
 
   const track = canvas.captureStream().getVideoTracks()[0];
   drawFrame();
@@ -40,6 +40,20 @@ export const createBlackCanvasStreamTrack = async ({
   return track;
 };
 
+let audioContextStartedPreviously = false;
+const userGestureEvents = [
+  'click',
+  'contextmenu',
+  'auxclick',
+  'dblclick',
+  'mousedown',
+  'mouseup',
+  'pointerup',
+  'touchend',
+  'keydown',
+  'keyup',
+];
+
 /**
  * Creates an inaudible audio stream track.
  * We need this to have audio track available when audio is disabled.
@@ -48,22 +62,62 @@ export const createInaudibleAudioStreamTrack = async ({ ctx }: { ctx: Context })
   const audioContext = new window.AudioContext();
   const oscillator = audioContext.createOscillator();
   oscillator.type = 'triangle';
-
-  // set to 0.1 to make inaudible
-  oscillator.frequency.setValueAtTime(0.1, audioContext.currentTime);
+  // roughly sounds like a box fan
+  oscillator.frequency.setValueAtTime(20, audioContext.currentTime);
 
   const gainNode = audioContext.createGain();
   // even w/ gain at 0 some packets are sent
   gainNode.gain.setValueAtTime(0, audioContext.currentTime);
-
   oscillator.connect(gainNode);
 
   const destination = audioContext.createMediaStreamDestination();
   gainNode.connect(destination);
 
-  oscillator.start();
+  let oscillatorStarted = false;
+  const ensureOscillatorStarted = () => {
+    if (oscillatorStarted) return;
+    oscillator.start();
+    oscillatorStarted = true;
+  };
+
+  const stateChangeHandler = () => {
+    if (audioContext.state === 'running') {
+      audioContextStartedPreviously = true;
+      ensureOscillatorStarted();
+    }
+    if (audioContext.state === 'suspended' || (audioContext.state as string) === 'interrupted') {
+      resumeAudioContext();
+    }
+  };
+
+  audioContext.addEventListener('statechange', stateChangeHandler);
+
+  const resumeAudioContext = () => {
+    void audioContext.resume().then(() => {
+      cleanUpUserGestureListeners();
+    });
+  };
+
+  const cleanUpUserGestureListeners = () => {
+    userGestureEvents.forEach((gesture) => {
+      document.removeEventListener(gesture, resumeAudioContext, {
+        capture: true,
+      });
+    });
+  };
+
+  if (audioContextStartedPreviously) {
+    resumeAudioContext();
+  } else {
+    userGestureEvents.forEach((gesture) => {
+      document.addEventListener(gesture, resumeAudioContext, {
+        capture: true,
+      });
+    });
+  }
 
   ctx.onDispose(async () => {
+    cleanUpUserGestureListeners();
     oscillator.disconnect();
     oscillator.stop();
     await audioContext.close();

package/src/calls-service.stories.tsx

@@ -0,0 +1,232 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import '@dxos-theme';
+
+import { type Meta, type StoryObj } from '@storybook/react-vite';
+import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+
+import { scheduleTask, sleep } from '@dxos/async';
+import { Context } from '@dxos/context';
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+import { Config, useConfig } from '@dxos/react-client';
+import { withClientProvider } from '@dxos/react-client/testing';
+import { Button } from '@dxos/react-ui';
+import { Json } from '@dxos/react-ui-syntax-highlighter';
+import { withLayout, withTheme } from '@dxos/storybook-utils';
+
+const testVideo = new URL('../testing/video.mp4', import.meta.url).href;
+
+import { CALLS_URL, CallsServicePeer, type TrackObject } from './calls';
+import { useBlackCanvasStreamTrack, useInaudibleAudioStreamTrack, useVideoStreamTrack } from './hooks';
+
+// TODO(burdon): THIS IS TOO COMPLEX FOR A TEST SETUP.
+const pushAndPullTrack = (mediaStreamTrack?: MediaStreamTrack) => {
+  const config = useConfig();
+  const callsConfig = {
+    iceServers: config.get('runtime.services.ice'),
+    apiBase: `${CALLS_URL}/api/calls`,
+  };
+  const peerPush = useMemo(() => new CallsServicePeer(callsConfig), []);
+  const peerPull = useMemo(() => new CallsServicePeer(callsConfig), []);
+  const [metrics, setMetrics] = useState<Record<string, any>>({});
+  const trackInfo = useRef<TrackObject | undefined>(undefined);
+  const pullCtx = useRef<Context | undefined>(undefined);
+  const [pulledTrack, setPulledTrack] = useState<MediaStreamTrack | undefined>(undefined);
+  const hadRun = useRef(false);
+
+  const deps: any[] = [];
+  // Push/pull video stream track to cloudflare.
+  useEffect(() => {
+    if (hadRun.current || !mediaStreamTrack || !peerPush || !peerPull) {
+      return;
+    }
+    hadRun.current = true;
+    const ctx = Context.default();
+    log.info('setting up ctx', { ctx, mediaStreamTrack, peerPush, peerPull });
+    deps.push([ctx, mediaStreamTrack, peerPush, peerPull]);
+    ctx.onDispose(() => {
+      log.info('disposing ctx', {});
+    });
+    scheduleTask(
+      ctx,
+      async () => {
+        log.info('starting push/pull', { mediaStreamTrack });
+        await peerPush.open();
+        await peerPull.open();
+        ctx.onDispose(() => {
+          void peerPush.close();
+          void peerPull.close();
+        });
+
+        // Push track to cloudflare.
+        performance.mark('push:begin');
+        log.info('pushing track', { track: mediaStreamTrack });
+        trackInfo.current = await peerPush.pushTrack({ track: mediaStreamTrack, ctx: Context.default() });
+        performance.mark('push:end');
+        const pushTime = performance.measure('push', 'push:begin', 'push:end').duration;
+        log.info('successfully pushed track', { pushTime, trackInfo: trackInfo.current });
+        setMetrics((prev) => ({ ...prev, 'time to push track [ms]': Math.round(pushTime) }));
+
+        // Pull track from cloudflare.
+        performance.mark('pullTrack:begin');
+        pullCtx.current = Context.default();
+        let retries = 0;
+        let pulledTrack: MediaStreamTrack | undefined;
+        while (!pulledTrack && retries < 3) {
+          retries++;
+          log.info('pulling track', {
+            trackData: { ...trackInfo.current!, mid: undefined },
+            retries,
+            pulledTrack,
+          });
+          pulledTrack = await peerPull.pullTrack({
+            trackData: { ...trackInfo.current!, mid: undefined },
+            ctx: pullCtx.current,
+          });
+          await sleep(200);
+        }
+        invariant(pulledTrack);
+        setPulledTrack(pulledTrack);
+        performance.mark('pullTrack:end');
+        const pullTime = performance.measure('pullTrack', 'pullTrack:begin', 'pullTrack:end').duration;
+        setMetrics((prev) => ({ ...prev, 'time to pull track [ms]': Math.round(pullTime) }));
+        log.info('successfully pulled track', { pullTime, pulledTrack });
+      },
+      1000,
+    );
+
+    return () => {
+      void ctx.dispose();
+    };
+  }, [mediaStreamTrack, peerPush, peerPull]);
+
+  const rePullTrack = useCallback(async () => {
+    performance.mark('rePullVideo:begin');
+    if (!pulledTrack) {
+      log.info('no pulled track, skipping re-pull', { pulledTrack });
+      return;
+    }
+    pulledTrack.stop();
+    await pullCtx.current!.dispose();
+    pullCtx.current = Context.default();
+    const newPulledTrack = await peerPull.pullTrack({
+      trackData: { ...trackInfo.current!, mid: undefined },
+      ctx: pullCtx.current,
+    });
+    invariant(newPulledTrack);
+    setPulledTrack(newPulledTrack);
+    performance.mark('rePullVideo:end');
+    const rePullTime = performance.measure('rePullVideo', 'rePullVideo:begin', 'rePullVideo:end').duration;
+    setMetrics((prev) => ({ ...prev, 'time to re-pull video [ms]': Math.round(rePullTime) }));
+  }, [peerPull, trackInfo.current, pullCtx.current, pulledTrack]);
+
+  return {
+    pulledTrack,
+    rePullTrack,
+    metrics,
+    pushService: peerPush,
+    pullService: peerPull,
+  };
+};
+
+type StoryProps = { source: string };
+
+const DefaultStory = ({ source }: StoryProps) => {
+  const pushVideoElement = useRef<HTMLVideoElement>(null);
+  const pullVideoElement = useRef<HTMLVideoElement>(null);
+  // Get video stream track.
+  const videoStreamTrack = useVideoStreamTrack(pushVideoElement, source);
+  const { pulledTrack, rePullTrack, metrics } = pushAndPullTrack(videoStreamTrack);
+  const hadRun = useRef(false);
+
+  // Push/pull video stream track to cloudflare.
+  useEffect(() => {
+    if (!pulledTrack || hadRun.current) {
+      return;
+    }
+    hadRun.current = true;
+    const ctx = Context.default();
+    scheduleTask(ctx, async () => {
+      pullVideoElement.current!.srcObject = new MediaStream([pulledTrack]);
+      await pullVideoElement.current!.play();
+    });
+
+    return () => {
+      void ctx.dispose();
+    };
+  }, [pulledTrack]);
+
+  return (
+    <div className='grid grid-rows-3 gap-4 items-center'>
+      <video ref={pushVideoElement} muted autoPlay loop />
+      <div className='flex flex-col gap-4'>
+        <video ref={pullVideoElement} muted />
+        <Button disabled={!pulledTrack} onClick={rePullTrack}>
+          Re-pull video
+        </Button>
+      </div>
+      <Json data={metrics} />
+    </div>
+  );
+};
+
+const meta = {
+  title: 'plugins/plugin-thread/calls-service',
+  render: DefaultStory,
+  decorators: [
+    withClientProvider({
+      config: new Config({
+        runtime: {
+          services: {
+            iceProviders: [{ urls: 'https://edge.dxos.workers.dev/ice' }],
+          },
+        },
+      }),
+    }),
+    withTheme,
+    withLayout(),
+  ],
+  parameters: {
+    layout: 'centered',
+  },
+} satisfies Meta<typeof DefaultStory>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const Default: Story = {
+  args: {
+    source: testVideo,
+  },
+};
+
+export const InaudibleAudioStreamTrack = {
+  render: () => {
+    const audioStreamTrack = useInaudibleAudioStreamTrack();
+    const { rePullTrack, metrics } = pushAndPullTrack(audioStreamTrack);
+
+    return (
+      <div className='flex flex-col gap-4 items-center'>
+        <Json data={metrics} />
+        <Button onClick={rePullTrack}>Re-pull audio</Button>
+      </div>
+    );
+  },
+};
+
+export const BlackVideoStreamTrack = {
+  render: () => {
+    const videoStreamTrack = useBlackCanvasStreamTrack();
+    const { rePullTrack, metrics } = pushAndPullTrack(videoStreamTrack);
+    return (
+      <div className='flex flex-col gap-4 items-center'>
+        <Json data={metrics} />
+        <Button onClick={rePullTrack}>Re-pull audio</Button>
+      </div>
+    );
+  },
+};

package/src/capabilities/app-graph-builder.ts

@@ -5,18 +5,21 @@
 import { Rx } from '@effect-rx/rx-react';
 import { Option, pipe } from 'effect';
 
-import { Capabilities, contributes, createIntent, type PluginContext } from '@dxos/app-framework';
+import { Capabilities, type PluginContext, contributes, createIntent } from '@dxos/app-framework';
 import { Obj } from '@dxos/echo';
 import { AttentionCapabilities } from '@dxos/plugin-attention';
-import { PLANK_COMPANION_TYPE, ATTENDABLE_PATH_SEPARATOR, DECK_COMPANION_TYPE } from '@dxos/plugin-deck/types';
-import { createExtension, ROOT_ID, rxFromSignal } from '@dxos/plugin-graph';
+import { ATTENDABLE_PATH_SEPARATOR, DECK_COMPANION_TYPE, PLANK_COMPANION_TYPE } from '@dxos/plugin-deck/types';
+import { ROOT_ID, createExtension, rxFromSignal } from '@dxos/plugin-graph';
 import { fullyQualifiedId } from '@dxos/react-client/echo';
 
-import { ThreadCapabilities } from './capabilities';
-import { meta, THREAD_PLUGIN } from '../meta';
+import { THREAD_PLUGIN, meta } from '../meta';
 import { ChannelType, ThreadAction } from '../types';
 import { getAnchor } from '../util';
 
+import { ThreadCapabilities } from './capabilities';
+
+// TODO(wittjosiah): Highlight active calls in L1.
+//  Track active meetings by subscribing to meetings query and polling the swarms of recent meetings in the space.
 export default (context: PluginContext) => {
   const resolve = (typename: string) =>
     context.getCapabilities(Capabilities.Metadata).find(({ id }) => id === typename)?.metadata ?? {};

package/src/capabilities/call-manager.ts

@@ -2,12 +2,13 @@
 // Copyright 2025 DXOS.org
 //
 
-import { contributes, type PluginContext } from '@dxos/app-framework';
+import { type PluginContext, contributes } from '@dxos/app-framework';
 import { ClientCapabilities } from '@dxos/plugin-client';
 
-import { ThreadCapabilities } from './capabilities';
 import { CallManager } from '../calls';
 
+import { ThreadCapabilities } from './capabilities';
+
 export default async (context: PluginContext) => {
   const client = context.getCapability(ClientCapabilities.Client);
   const callManager = new CallManager(client);

package/src/capabilities/capabilities.ts

@@ -5,9 +5,9 @@
 import { defineCapability } from '@dxos/app-framework';
 import { type DeepReadonly } from '@dxos/util';
 
-import { type CallState, type MediaState, type CallManager } from '../calls';
+import { type CallManager, type CallState, type MediaState } from '../calls';
 import { THREAD_PLUGIN } from '../meta';
-import { type ViewState, type ThreadState, type ChannelType } from '../types';
+import { type ChannelType, type ThreadState, type ViewState } from '../types';
 
 export namespace ThreadCapabilities {
   export const CallManager = defineCapability<CallManager>(`${THREAD_PLUGIN}/capability/call-manager`);

package/src/capabilities/index.ts

@@ -12,5 +12,6 @@ export const ReactRoot = lazy(() => import('./react-root'));
 export const ReactSurface = lazy(() => import('./react-surface'));
 export const ThreadSettings = lazy(() => import('./settings'));
 export const ThreadState = lazy(() => import('./state'));
+export const Toolkit = lazy(() => import('./toolkit'));
 
 export * from './capabilities';