@dxos/plugin-thread 0.8.4-main.422d1c7879 → 0.8.4-main.43cb759274

package/src/calls/util/stub-media-streams.ts

@@ -1,127 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-//
-
-import { scheduleTaskInterval, waitForCondition } from '@dxos/async';
-import { type Context, cancelWithContext } from '@dxos/context';
-import { invariant } from '@dxos/invariant';
-
-/**
- * Creates a black canvas stream track.
- * We need this to to have black rectangle video that is live (have frames) when video feed is disabled.
- */
-export const createBlackCanvasStreamTrack = async ({
-  ctx,
-  width,
-  height,
-}: {
-  ctx: Context;
-  width: number;
-  height: number;
-}): Promise<MediaStreamTrack> => {
-  const canvas = document.createElement('canvas');
-  canvas.width = width;
-  canvas.height = height;
-
-  const canvasCtx = canvas.getContext('2d');
-  invariant(canvasCtx);
-  const drawFrame = () => {
-    canvasCtx.fillStyle = 'black';
-    canvasCtx.fillRect(0, 0, canvas.width, canvas.height);
-  };
-  scheduleTaskInterval(ctx, async () => drawFrame(), 200);
-
-  const track = canvas.captureStream().getVideoTracks()[0];
-  drawFrame();
-
-  await cancelWithContext(ctx, waitForCondition({ condition: () => track.readyState === 'live', timeout: 1_000 }));
-
-  return track;
-};
-
-let audioContextStartedPreviously = false;
-const userGestureEvents = [
-  'click',
-  'contextmenu',
-  'auxclick',
-  'dblclick',
-  'mousedown',
-  'mouseup',
-  'pointerup',
-  'touchend',
-  'keydown',
-  'keyup',
-];
-
-/**
- * Creates an inaudible audio stream track.
- * We need this to have audio track available when audio is disabled.
- */
-export const createInaudibleAudioStreamTrack = async ({ ctx }: { ctx: Context }): Promise<MediaStreamTrack> => {
-  const audioContext = new window.AudioContext();
-
-  const oscillator = audioContext.createOscillator();
-  oscillator.type = 'triangle';
-  oscillator.frequency.setValueAtTime(20, audioContext.currentTime);
-
-  const gainNode = audioContext.createGain();
-  gainNode.gain.setValueAtTime(0, audioContext.currentTime);
-  oscillator.connect(gainNode);
-
-  const destination = audioContext.createMediaStreamDestination();
-  gainNode.connect(destination);
-
-  let oscillatorStarted = false;
-  const ensureOscillatorStarted = () => {
-    if (oscillatorStarted) return;
-    oscillator.start();
-    oscillatorStarted = true;
-  };
-
-  const stateChangeHandler = () => {
-    if (audioContext.state === 'running') {
-      audioContextStartedPreviously = true;
-      ensureOscillatorStarted();
-    }
-
-    if (audioContext.state === 'suspended' || (audioContext.state as string) === 'interrupted') {
-      resumeAudioContext();
-    }
-  };
-
-  audioContext.addEventListener('statechange', stateChangeHandler);
-
-  const resumeAudioContext = () => {
-    void audioContext.resume().then(() => {
-      cleanUpUserGestureListeners();
-    });
-  };
-
-  const cleanUpUserGestureListeners = () => {
-    userGestureEvents.forEach((gesture) => {
-      document.removeEventListener(gesture, resumeAudioContext, {
-        capture: true,
-      });
-    });
-  };
-
-  if (audioContextStartedPreviously) {
-    resumeAudioContext();
-  } else {
-    userGestureEvents.forEach((gesture) => {
-      document.addEventListener(gesture, resumeAudioContext, {
-        capture: true,
-      });
-    });
-  }
-
-  ctx.onDispose(async () => {
-    cleanUpUserGestureListeners();
-    oscillator.disconnect();
-    oscillator.stop();
-    await audioContext.close();
-  });
-
-  return destination.stream.getAudioTracks()[0];
-};

package/src/calls/util/task-scheduling.ts

@@ -1,92 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-const DefaultBatchSizeLimit = 64;
-
-type Task<Value = unknown> = () => Promise<Value>;
-
-export class FIFOScheduler {
-  #schedulerChain: Promise<void>;
-
-  constructor() {
-    this.#schedulerChain = Promise.resolve();
-  }
-
-  schedule<T>(task: Task<T>): Promise<T> {
-    return new Promise((resolve, reject) => {
-      this.#schedulerChain = this.#schedulerChain.then(async () => {
-        try {
-          resolve(await task());
-        } catch (error: any) {
-          reject(error);
-        }
-      });
-    });
-  }
-}
-
-// TODO(mykola): Reconcile with @dxos/async task scheduling.
-export class BulkRequestDispatcher<RequestEntryProps, BulkResponse> {
-  #currentBatch: RequestEntryProps[];
-  #currentBulkResponse: Promise<BulkResponse> | null;
-  #batchSizeLimit: number;
-
-  constructor(batchSizeLimit: number = DefaultBatchSizeLimit) {
-    this.#currentBatch = [];
-    this.#currentBulkResponse = null;
-    this.#batchSizeLimit = batchSizeLimit;
-  }
-
-  /**
-   * Returns a bulk response promise.
-   * At the event loop iteration end, the accumulated entries will be dispatched as a bulk request.
-   */
-  doBulkRequest(
-    params: RequestEntryProps,
-    bulkRequestFunc: (bulkCopy: RequestEntryProps[]) => Promise<BulkResponse>,
-  ): Promise<BulkResponse> {
-    if (this.#currentBatch.length >= this.#batchSizeLimit) {
-      // If it reaches the batch size limit, we make another bulk request.
-      this.#currentBatch = [];
-      this.#currentBulkResponse = null;
-    }
-    this.#currentBatch.push(params);
-    if (this.#currentBulkResponse != null) {
-      return this.#currentBulkResponse;
-    }
-
-    // Save the current batch list reference in the function scope because this.#currentBatch could be reset
-    // if the batch limit is reached.
-    const batch = this.#currentBatch;
-    this.#currentBulkResponse = new Promise((resolve, reject) => {
-      //   script
-      //     |
-      //     V
-      // microtasks (Promise)
-      //     |
-      //     V
-      // macrotasks (setTimeout)
-      //
-      // macrotasks are run in the event loop iteration end, so we use that moment to make the bulkRequestFunc call.
-      setTimeout(() => {
-        // When the bulk request happens, the batch list and the response is reset to start another batch.
-        // Coming callers will wait for a new response promise.
-        this.#currentBulkResponse = null;
-
-        // We cut here to make the bulk request.
-        const batchCopy = batch.splice(0, batch.length);
-        bulkRequestFunc(batchCopy)
-          .then((result) => {
-            resolve(result);
-          })
-          .catch((err) => {
-            reject(err);
-          });
-      }, 0);
-    });
-
-    // This bulk response needs to be processed to extract the the result for the entry.
-    return this.#currentBulkResponse;
-  }
-}

package/src/calls-service.stories.tsx

@@ -1,230 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { type Meta, type StoryObj } from '@storybook/react-vite';
-import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
-
-import { scheduleTask, sleep } from '@dxos/async';
-import { Context } from '@dxos/context';
-import { invariant } from '@dxos/invariant';
-import { log } from '@dxos/log';
-import { Config, useConfig } from '@dxos/react-client';
-import { withClientProvider } from '@dxos/react-client/testing';
-import { Button } from '@dxos/react-ui';
-import { JsonHighlighter } from '@dxos/react-ui-syntax-highlighter';
-import { withTheme } from '@dxos/react-ui/testing';
-
-const testVideo = new URL('../testing/video.mp4', import.meta.url).href;
-
-import { useBlackCanvasStreamTrack, useInaudibleAudioStreamTrack, useVideoStreamTrack } from '#hooks';
-
-import { CALLS_URL, CallsServicePeer, type TrackObject } from './calls';
-
-// TODO(burdon): THIS IS TOO COMPLEX FOR A TEST SETUP.
-const pushAndPullTrack = (mediaStreamTrack?: MediaStreamTrack) => {
-  const config = useConfig();
-  const callsConfig = {
-    iceServers: config.get('runtime.services.ice'),
-    apiBase: `${CALLS_URL}/api/calls`,
-  };
-  const peerPush = useMemo(() => new CallsServicePeer(callsConfig), []);
-  const peerPull = useMemo(() => new CallsServicePeer(callsConfig), []);
-  const [metrics, setMetrics] = useState<Record<string, any>>({});
-  const trackInfo = useRef<TrackObject | undefined>(undefined);
-  const pullCtx = useRef<Context | undefined>(undefined);
-  const [pulledTrack, setPulledTrack] = useState<MediaStreamTrack | undefined>(undefined);
-  const hadRun = useRef(false);
-
-  const deps: any[] = [];
-  // Push/pull video stream track to cloudflare.
-  useEffect(() => {
-    if (hadRun.current || !mediaStreamTrack || !peerPush || !peerPull) {
-      return;
-    }
-    hadRun.current = true;
-    const ctx = Context.default();
-    log.info('setting up ctx', { ctx, mediaStreamTrack, peerPush, peerPull });
-    deps.push([ctx, mediaStreamTrack, peerPush, peerPull]);
-    ctx.onDispose(() => {
-      log.info('disposing ctx', {});
-    });
-    scheduleTask(
-      ctx,
-      async () => {
-        log.info('starting push/pull', { mediaStreamTrack });
-        await peerPush.open();
-        await peerPull.open();
-        ctx.onDispose(() => {
-          void peerPush.close();
-          void peerPull.close();
-        });
-
-        // Push track to cloudflare.
-        performance.mark('push:begin');
-        log.info('pushing track', { track: mediaStreamTrack });
-        trackInfo.current = await peerPush.pushTrack({ track: mediaStreamTrack, ctx: Context.default() });
-        performance.mark('push:end');
-        const pushTime = performance.measure('push', 'push:begin', 'push:end').duration;
-        log.info('successfully pushed track', { pushTime, trackInfo: trackInfo.current });
-        setMetrics((prev) => ({ ...prev, 'time to push track [ms]': Math.round(pushTime) }));
-
-        // Pull track from cloudflare.
-        performance.mark('pullTrack:begin');
-        pullCtx.current = Context.default();
-        let retries = 0;
-        let pulledTrack: MediaStreamTrack | undefined;
-        while (!pulledTrack && retries < 3) {
-          retries++;
-          log.info('pulling track', {
-            trackData: { ...trackInfo.current!, mid: undefined },
-            retries,
-            pulledTrack,
-          });
-          pulledTrack = await peerPull.pullTrack({
-            trackData: { ...trackInfo.current!, mid: undefined },
-            ctx: pullCtx.current,
-          });
-          await sleep(200);
-        }
-        invariant(pulledTrack);
-        setPulledTrack(pulledTrack);
-        performance.mark('pullTrack:end');
-        const pullTime = performance.measure('pullTrack', 'pullTrack:begin', 'pullTrack:end').duration;
-        setMetrics((prev) => ({ ...prev, 'time to pull track [ms]': Math.round(pullTime) }));
-        log.info('successfully pulled track', { pullTime, pulledTrack });
-      },
-      1000,
-    );
-
-    return () => {
-      void ctx.dispose();
-    };
-  }, [mediaStreamTrack, peerPush, peerPull]);
-
-  const rePullTrack = useCallback(async () => {
-    performance.mark('rePullVideo:begin');
-    if (!pulledTrack) {
-      log.info('no pulled track, skipping re-pull', { pulledTrack });
-      return;
-    }
-    pulledTrack.stop();
-    await pullCtx.current!.dispose();
-    pullCtx.current = Context.default();
-    const newPulledTrack = await peerPull.pullTrack({
-      trackData: { ...trackInfo.current!, mid: undefined },
-      ctx: pullCtx.current,
-    });
-    invariant(newPulledTrack);
-    setPulledTrack(newPulledTrack);
-    performance.mark('rePullVideo:end');
-    const rePullTime = performance.measure('rePullVideo', 'rePullVideo:begin', 'rePullVideo:end').duration;
-    setMetrics((prev) => ({ ...prev, 'time to re-pull video [ms]': Math.round(rePullTime) }));
-  }, [peerPull, trackInfo.current, pullCtx.current, pulledTrack]);
-
-  return {
-    pulledTrack,
-    rePullTrack,
-    metrics,
-    pushService: peerPush,
-    pullService: peerPull,
-  };
-};
-
-type DefaultStoryProps = { source: string };
-
-const DefaultStory = ({ source }: DefaultStoryProps) => {
-  const pushVideoElement = useRef<HTMLVideoElement>(null);
-  const pullVideoElement = useRef<HTMLVideoElement>(null);
-  // Get video stream track.
-  const videoStreamTrack = useVideoStreamTrack(pushVideoElement, source);
-  const { pulledTrack, rePullTrack, metrics } = pushAndPullTrack(videoStreamTrack);
-  const hadRun = useRef(false);
-
-  // Push/pull video stream track to cloudflare.
-  useEffect(() => {
-    if (!pulledTrack || hadRun.current) {
-      return;
-    }
-    hadRun.current = true;
-    const ctx = Context.default();
-    scheduleTask(ctx, async () => {
-      pullVideoElement.current!.srcObject = new MediaStream([pulledTrack]);
-      await pullVideoElement.current!.play();
-    });
-
-    return () => {
-      void ctx.dispose();
-    };
-  }, [pulledTrack]);
-
-  return (
-    <div className='grid grid-rows-3 gap-4 items-center'>
-      <video ref={pushVideoElement} muted autoPlay loop />
-      <div className='flex flex-col gap-4'>
-        <video ref={pullVideoElement} muted />
-        <Button disabled={!pulledTrack} onClick={rePullTrack}>
-          Re-pull video
-        </Button>
-      </div>
-      <JsonHighlighter data={metrics} />
-    </div>
-  );
-};
-
-const meta = {
-  title: 'plugins/plugin-thread/experimental/Calls',
-  render: DefaultStory,
-  decorators: [
-    withTheme(),
-    withClientProvider({
-      config: new Config({
-        runtime: {
-          services: {
-            iceProviders: [{ urls: 'https://edge.dxos.workers.dev/ice' }],
-          },
-        },
-      }),
-    }),
-  ],
-  parameters: {
-    layout: 'centered',
-  },
-} satisfies Meta<typeof DefaultStory>;
-
-export default meta;
-
-type Story = StoryObj<typeof meta>;
-
-export const Default: Story = {
-  args: {
-    source: testVideo,
-  },
-};
-
-export const InaudibleAudioStreamTrack = {
-  render: () => {
-    const audioStreamTrack = useInaudibleAudioStreamTrack();
-    const { rePullTrack, metrics } = pushAndPullTrack(audioStreamTrack);
-
-    return (
-      <div className='flex flex-col gap-4 items-center'>
-        <JsonHighlighter data={metrics} />
-        <Button onClick={rePullTrack}>Re-pull audio</Button>
-      </div>
-    );
-  },
-};
-
-export const BlackVideoStreamTrack = {
-  render: () => {
-    const videoStreamTrack = useBlackCanvasStreamTrack();
-    const { rePullTrack, metrics } = pushAndPullTrack(videoStreamTrack);
-    return (
-      <div className='flex flex-col gap-4 items-center'>
-        <JsonHighlighter data={metrics} />
-        <Button onClick={rePullTrack}>Re-pull audio</Button>
-      </div>
-    );
-  },
-};

package/src/capabilities/call-manager.ts

@@ -1,27 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import * as Effect from 'effect/Effect';
-
-import { Capabilities, Capability } from '@dxos/app-framework';
-import { ClientCapabilities } from '@dxos/plugin-client/types';
-
-import { ThreadCapabilities } from '#types';
-
-import { CallManager } from '../calls';
-
-export default Capability.makeModule(
-  Effect.fnUntraced(function* () {
-    const client = yield* Capability.get(ClientCapabilities.Client);
-    const registry = yield* Capability.get(Capabilities.AtomRegistry);
-    const callManager = new CallManager(client, registry);
-    yield* Effect.tryPromise(() => callManager.open());
-
-    return Capability.contributes(ThreadCapabilities.CallManager, callManager, () =>
-      Effect.sync(() => {
-        void callManager.close();
-      }),
-    );
-  }),
-);

package/src/capabilities/react-root.tsx

@@ -1,20 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import * as Effect from 'effect/Effect';
-import React from 'react';
-
-import { Capabilities, Capability } from '@dxos/app-framework';
-
-import { Call } from '#components';
-import { meta } from '#meta';
-
-export default Capability.makeModule(() =>
-  Effect.sync(() => {
-    return Capability.contributes(Capabilities.ReactRoot, {
-      id: meta.id,
-      root: () => <Call.Audio />,
-    });
-  }),
-);

package/src/components/Call/Call.tsx

@@ -1,93 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { useAtomValue } from '@effect-atom/atom-react';
-import React, { type FC, type PropsWithChildren } from 'react';
-
-import { useCapability } from '@dxos/app-framework/ui';
-import { type ThemedClassName } from '@dxos/react-ui';
-
-import { useDebugMode } from '#hooks';
-import { ThreadCapabilities } from '#types';
-
-import { AudioStream } from '../Media';
-import { ParticipantGrid } from '../Participant';
-import { Toolbar, type ToolbarProps } from './Toolbar';
-
-//
-// Root
-//
-
-type CallRootProps = PropsWithChildren<ThemedClassName>;
-
-const CallRoot: FC<CallRootProps> = ({ children }) => {
-  return <div className='relative dx-container flex flex-col'>{children}</div>;
-};
-
-CallRoot.displayName = 'CallRoot';
-
-//
-// Audio
-//
-
-type CallAudioProps = {};
-
-const CallAudio: FC<CallAudioProps> = () => {
-  const call = useCapability(ThreadCapabilities.CallManager);
-  const audioTracksToPlay = useAtomValue(call.audioTracksToPlayAtom);
-
-  return <AudioStream tracks={audioTracksToPlay} />;
-};
-
-CallAudio.displayName = 'CallAudio';
-
-//
-// Room
-//
-
-type CallGridProps = {
-  fullscreen?: boolean;
-};
-
-const CallGrid: FC<CallGridProps> = ({ fullscreen }) => {
-  const debug = useDebugMode();
-  const call = useCapability(ThreadCapabilities.CallManager);
-  const self = useAtomValue(call.selfAtom);
-  const users = useAtomValue(call.usersAtom);
-
-  return (
-    <div className='grid grow p-4 dark:bg-neutral-900'>
-      <ParticipantGrid self={self} users={users} debug={debug} fullscreen={fullscreen} />
-    </div>
-  );
-};
-
-CallGrid.displayName = 'CallGrid';
-
-//
-// Toolbar
-//
-
-type CallToolbarProps = Pick<ToolbarProps, 'channel' | 'onLeave'>;
-
-const CallToolbar: FC<CallToolbarProps> = (props) => {
-  return (
-    <div className='absolute bottom-0 left-0 right-0 flex justify-center'>
-      <Toolbar isInRoom {...props} />
-    </div>
-  );
-};
-
-CallToolbar.displayName = 'CallToolbar';
-
-//
-// Export
-//
-
-export const Call = {
-  Root: CallRoot,
-  Grid: CallGrid,
-  Audio: CallAudio,
-  Toolbar: CallToolbar,
-};