@dxos/plugin-thread 0.8.4-main.e8ec1fe → 0.8.4-main.effb148878

package/src/calls/util/stub-media-streams.ts

@@ -1,127 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-//
-
-import { scheduleTaskInterval, waitForCondition } from '@dxos/async';
-import { type Context, cancelWithContext } from '@dxos/context';
-import { invariant } from '@dxos/invariant';
-
-/**
- * Creates a black canvas stream track.
- * We need this to to have black rectangle video that is live (have frames) when video feed is disabled.
- */
-export const createBlackCanvasStreamTrack = async ({
-  ctx,
-  width,
-  height,
-}: {
-  ctx: Context;
-  width: number;
-  height: number;
-}): Promise<MediaStreamTrack> => {
-  const canvas = document.createElement('canvas');
-  canvas.width = width;
-  canvas.height = height;
-
-  const canvasCtx = canvas.getContext('2d');
-  invariant(canvasCtx);
-  const drawFrame = () => {
-    canvasCtx.fillStyle = 'black';
-    canvasCtx.fillRect(0, 0, canvas.width, canvas.height);
-  };
-  scheduleTaskInterval(ctx, async () => drawFrame(), 200);
-
-  const track = canvas.captureStream().getVideoTracks()[0];
-  drawFrame();
-
-  await cancelWithContext(ctx, waitForCondition({ condition: () => track.readyState === 'live', timeout: 1_000 }));
-
-  return track;
-};
-
-let audioContextStartedPreviously = false;
-const userGestureEvents = [
-  'click',
-  'contextmenu',
-  'auxclick',
-  'dblclick',
-  'mousedown',
-  'mouseup',
-  'pointerup',
-  'touchend',
-  'keydown',
-  'keyup',
-];
-
-/**
- * Creates an inaudible audio stream track.
- * We need this to have audio track available when audio is disabled.
- */
-export const createInaudibleAudioStreamTrack = async ({ ctx }: { ctx: Context }): Promise<MediaStreamTrack> => {
-  const audioContext = new window.AudioContext();
-
-  const oscillator = audioContext.createOscillator();
-  oscillator.type = 'triangle';
-  oscillator.frequency.setValueAtTime(20, audioContext.currentTime);
-
-  const gainNode = audioContext.createGain();
-  gainNode.gain.setValueAtTime(0, audioContext.currentTime);
-  oscillator.connect(gainNode);
-
-  const destination = audioContext.createMediaStreamDestination();
-  gainNode.connect(destination);
-
-  let oscillatorStarted = false;
-  const ensureOscillatorStarted = () => {
-    if (oscillatorStarted) return;
-    oscillator.start();
-    oscillatorStarted = true;
-  };
-
-  const stateChangeHandler = () => {
-    if (audioContext.state === 'running') {
-      audioContextStartedPreviously = true;
-      ensureOscillatorStarted();
-    }
-
-    if (audioContext.state === 'suspended' || (audioContext.state as string) === 'interrupted') {
-      resumeAudioContext();
-    }
-  };
-
-  audioContext.addEventListener('statechange', stateChangeHandler);
-
-  const resumeAudioContext = () => {
-    void audioContext.resume().then(() => {
-      cleanUpUserGestureListeners();
-    });
-  };
-
-  const cleanUpUserGestureListeners = () => {
-    userGestureEvents.forEach((gesture) => {
-      document.removeEventListener(gesture, resumeAudioContext, {
-        capture: true,
-      });
-    });
-  };
-
-  if (audioContextStartedPreviously) {
-    resumeAudioContext();
-  } else {
-    userGestureEvents.forEach((gesture) => {
-      document.addEventListener(gesture, resumeAudioContext, {
-        capture: true,
-      });
-    });
-  }
-
-  ctx.onDispose(async () => {
-    cleanUpUserGestureListeners();
-    oscillator.disconnect();
-    oscillator.stop();
-    await audioContext.close();
-  });
-
-  return destination.stream.getAudioTracks()[0];
-};

package/src/calls/util/task-scheduling.ts

@@ -1,92 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-const DefaultBatchSizeLimit = 64;
-
-type Task<Value = unknown> = () => Promise<Value>;
-
-export class FIFOScheduler {
-  #schedulerChain: Promise<void>;
-
-  constructor() {
-    this.#schedulerChain = Promise.resolve();
-  }
-
-  schedule<T>(task: Task<T>): Promise<T> {
-    return new Promise((resolve, reject) => {
-      this.#schedulerChain = this.#schedulerChain.then(async () => {
-        try {
-          resolve(await task());
-        } catch (error: any) {
-          reject(error);
-        }
-      });
-    });
-  }
-}
-
-// TODO(mykola): Reconcile with @dxos/async task scheduling.
-export class BulkRequestDispatcher<RequestEntryParams, BulkResponse> {
-  #currentBatch: RequestEntryParams[];
-  #currentBulkResponse: Promise<BulkResponse> | null;
-  #batchSizeLimit: number;
-
-  constructor(batchSizeLimit: number = DefaultBatchSizeLimit) {
-    this.#currentBatch = [];
-    this.#currentBulkResponse = null;
-    this.#batchSizeLimit = batchSizeLimit;
-  }
-
-  /**
-   * Returns a bulk response promise.
-   * At the event loop iteration end, the accumulated entries will be dispatched as a bulk request.
-   */
-  doBulkRequest(
-    params: RequestEntryParams,
-    bulkRequestFunc: (bulkCopy: RequestEntryParams[]) => Promise<BulkResponse>,
-  ): Promise<BulkResponse> {
-    if (this.#currentBatch.length >= this.#batchSizeLimit) {
-      // If it reaches the batch size limit, we make another bulk request.
-      this.#currentBatch = [];
-      this.#currentBulkResponse = null;
-    }
-    this.#currentBatch.push(params);
-    if (this.#currentBulkResponse != null) {
-      return this.#currentBulkResponse;
-    }
-
-    // Save the current batch list reference in the function scope because this.#currentBatch could be reset
-    // if the batch limit is reached.
-    const batch = this.#currentBatch;
-    this.#currentBulkResponse = new Promise((resolve, reject) => {
-      //   script
-      //     |
-      //     V
-      // microtasks (Promise)
-      //     |
-      //     V
-      // macrotasks (setTimeout)
-      //
-      // macrotasks are run in the event loop iteration end, so we use that moment to make the bulkRequestFunc call.
-      setTimeout(() => {
-        // When the bulk request happens, the batch list and the response is reset to start another batch.
-        // Coming callers will wait for a new response promise.
-        this.#currentBulkResponse = null;
-
-        // We cut here to make the bulk request.
-        const batchCopy = batch.splice(0, batch.length);
-        bulkRequestFunc(batchCopy)
-          .then((result) => {
-            resolve(result);
-          })
-          .catch((err) => {
-            reject(err);
-          });
-      }, 0);
-    });
-
-    // This bulk response needs to be processed to extract the the result for the entry.
-    return this.#currentBulkResponse;
-  }
-}

package/src/calls-service.stories.tsx

@@ -1,229 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { type Meta, type StoryObj } from '@storybook/react-vite';
-import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
-
-import { scheduleTask, sleep } from '@dxos/async';
-import { Context } from '@dxos/context';
-import { invariant } from '@dxos/invariant';
-import { log } from '@dxos/log';
-import { Config, useConfig } from '@dxos/react-client';
-import { withClientProvider } from '@dxos/react-client/testing';
-import { Button } from '@dxos/react-ui';
-import { withTheme } from '@dxos/react-ui/testing';
-import { Json } from '@dxos/react-ui-syntax-highlighter';
-
-const testVideo = new URL('../testing/video.mp4', import.meta.url).href;
-
-import { CALLS_URL, CallsServicePeer, type TrackObject } from './calls';
-import { useBlackCanvasStreamTrack, useInaudibleAudioStreamTrack, useVideoStreamTrack } from './hooks';
-
-// TODO(burdon): THIS IS TOO COMPLEX FOR A TEST SETUP.
-const pushAndPullTrack = (mediaStreamTrack?: MediaStreamTrack) => {
-  const config = useConfig();
-  const callsConfig = {
-    iceServers: config.get('runtime.services.ice'),
-    apiBase: `${CALLS_URL}/api/calls`,
-  };
-  const peerPush = useMemo(() => new CallsServicePeer(callsConfig), []);
-  const peerPull = useMemo(() => new CallsServicePeer(callsConfig), []);
-  const [metrics, setMetrics] = useState<Record<string, any>>({});
-  const trackInfo = useRef<TrackObject | undefined>(undefined);
-  const pullCtx = useRef<Context | undefined>(undefined);
-  const [pulledTrack, setPulledTrack] = useState<MediaStreamTrack | undefined>(undefined);
-  const hadRun = useRef(false);
-
-  const deps: any[] = [];
-  // Push/pull video stream track to cloudflare.
-  useEffect(() => {
-    if (hadRun.current || !mediaStreamTrack || !peerPush || !peerPull) {
-      return;
-    }
-    hadRun.current = true;
-    const ctx = Context.default();
-    log.info('setting up ctx', { ctx, mediaStreamTrack, peerPush, peerPull });
-    deps.push([ctx, mediaStreamTrack, peerPush, peerPull]);
-    ctx.onDispose(() => {
-      log.info('disposing ctx', {});
-    });
-    scheduleTask(
-      ctx,
-      async () => {
-        log.info('starting push/pull', { mediaStreamTrack });
-        await peerPush.open();
-        await peerPull.open();
-        ctx.onDispose(() => {
-          void peerPush.close();
-          void peerPull.close();
-        });
-
-        // Push track to cloudflare.
-        performance.mark('push:begin');
-        log.info('pushing track', { track: mediaStreamTrack });
-        trackInfo.current = await peerPush.pushTrack({ track: mediaStreamTrack, ctx: Context.default() });
-        performance.mark('push:end');
-        const pushTime = performance.measure('push', 'push:begin', 'push:end').duration;
-        log.info('successfully pushed track', { pushTime, trackInfo: trackInfo.current });
-        setMetrics((prev) => ({ ...prev, 'time to push track [ms]': Math.round(pushTime) }));
-
-        // Pull track from cloudflare.
-        performance.mark('pullTrack:begin');
-        pullCtx.current = Context.default();
-        let retries = 0;
-        let pulledTrack: MediaStreamTrack | undefined;
-        while (!pulledTrack && retries < 3) {
-          retries++;
-          log.info('pulling track', {
-            trackData: { ...trackInfo.current!, mid: undefined },
-            retries,
-            pulledTrack,
-          });
-          pulledTrack = await peerPull.pullTrack({
-            trackData: { ...trackInfo.current!, mid: undefined },
-            ctx: pullCtx.current,
-          });
-          await sleep(200);
-        }
-        invariant(pulledTrack);
-        setPulledTrack(pulledTrack);
-        performance.mark('pullTrack:end');
-        const pullTime = performance.measure('pullTrack', 'pullTrack:begin', 'pullTrack:end').duration;
-        setMetrics((prev) => ({ ...prev, 'time to pull track [ms]': Math.round(pullTime) }));
-        log.info('successfully pulled track', { pullTime, pulledTrack });
-      },
-      1000,
-    );
-
-    return () => {
-      void ctx.dispose();
-    };
-  }, [mediaStreamTrack, peerPush, peerPull]);
-
-  const rePullTrack = useCallback(async () => {
-    performance.mark('rePullVideo:begin');
-    if (!pulledTrack) {
-      log.info('no pulled track, skipping re-pull', { pulledTrack });
-      return;
-    }
-    pulledTrack.stop();
-    await pullCtx.current!.dispose();
-    pullCtx.current = Context.default();
-    const newPulledTrack = await peerPull.pullTrack({
-      trackData: { ...trackInfo.current!, mid: undefined },
-      ctx: pullCtx.current,
-    });
-    invariant(newPulledTrack);
-    setPulledTrack(newPulledTrack);
-    performance.mark('rePullVideo:end');
-    const rePullTime = performance.measure('rePullVideo', 'rePullVideo:begin', 'rePullVideo:end').duration;
-    setMetrics((prev) => ({ ...prev, 'time to re-pull video [ms]': Math.round(rePullTime) }));
-  }, [peerPull, trackInfo.current, pullCtx.current, pulledTrack]);
-
-  return {
-    pulledTrack,
-    rePullTrack,
-    metrics,
-    pushService: peerPush,
-    pullService: peerPull,
-  };
-};
-
-type StoryProps = { source: string };
-
-const DefaultStory = ({ source }: StoryProps) => {
-  const pushVideoElement = useRef<HTMLVideoElement>(null);
-  const pullVideoElement = useRef<HTMLVideoElement>(null);
-  // Get video stream track.
-  const videoStreamTrack = useVideoStreamTrack(pushVideoElement, source);
-  const { pulledTrack, rePullTrack, metrics } = pushAndPullTrack(videoStreamTrack);
-  const hadRun = useRef(false);
-
-  // Push/pull video stream track to cloudflare.
-  useEffect(() => {
-    if (!pulledTrack || hadRun.current) {
-      return;
-    }
-    hadRun.current = true;
-    const ctx = Context.default();
-    scheduleTask(ctx, async () => {
-      pullVideoElement.current!.srcObject = new MediaStream([pulledTrack]);
-      await pullVideoElement.current!.play();
-    });
-
-    return () => {
-      void ctx.dispose();
-    };
-  }, [pulledTrack]);
-
-  return (
-    <div className='grid grid-rows-3 gap-4 items-center'>
-      <video ref={pushVideoElement} muted autoPlay loop />
-      <div className='flex flex-col gap-4'>
-        <video ref={pullVideoElement} muted />
-        <Button disabled={!pulledTrack} onClick={rePullTrack}>
-          Re-pull video
-        </Button>
-      </div>
-      <Json data={metrics} />
-    </div>
-  );
-};
-
-const meta = {
-  title: 'plugins/plugin-thread/calls-service',
-  render: DefaultStory,
-  decorators: [
-    withTheme,
-    withClientProvider({
-      config: new Config({
-        runtime: {
-          services: {
-            iceProviders: [{ urls: 'https://edge.dxos.workers.dev/ice' }],
-          },
-        },
-      }),
-    }),
-  ],
-  parameters: {
-    layout: 'centered',
-  },
-} satisfies Meta<typeof DefaultStory>;
-
-export default meta;
-
-type Story = StoryObj<typeof meta>;
-
-export const Default: Story = {
-  args: {
-    source: testVideo,
-  },
-};
-
-export const InaudibleAudioStreamTrack = {
-  render: () => {
-    const audioStreamTrack = useInaudibleAudioStreamTrack();
-    const { rePullTrack, metrics } = pushAndPullTrack(audioStreamTrack);
-
-    return (
-      <div className='flex flex-col gap-4 items-center'>
-        <Json data={metrics} />
-        <Button onClick={rePullTrack}>Re-pull audio</Button>
-      </div>
-    );
-  },
-};
-
-export const BlackVideoStreamTrack = {
-  render: () => {
-    const videoStreamTrack = useBlackCanvasStreamTrack();
-    const { rePullTrack, metrics } = pushAndPullTrack(videoStreamTrack);
-    return (
-      <div className='flex flex-col gap-4 items-center'>
-        <Json data={metrics} />
-        <Button onClick={rePullTrack}>Re-pull audio</Button>
-      </div>
-    );
-  },
-};

package/src/capabilities/call-manager.ts

@@ -1,20 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { type PluginContext, contributes } from '@dxos/app-framework';
-import { ClientCapabilities } from '@dxos/plugin-client';
-
-import { CallManager } from '../calls';
-
-import { ThreadCapabilities } from './capabilities';
-
-export default async (context: PluginContext) => {
-  const client = context.getCapability(ClientCapabilities.Client);
-  const callManager = new CallManager(client);
-  await callManager.open();
-
-  return contributes(ThreadCapabilities.CallManager, callManager, () => {
-    void callManager.close();
-  });
-};

package/src/capabilities/capabilities.ts

@@ -1,33 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { defineCapability } from '@dxos/app-framework';
-import { type DeepReadonly } from '@dxos/util';
-
-import { type CallManager, type CallState, type MediaState } from '../calls';
-import { meta } from '../meta';
-import { type Channel, type ThreadState, type ViewState } from '../types';
-
-export namespace ThreadCapabilities {
-  export const CallManager = defineCapability<CallManager>(`${meta.id}/capability/call-manager`);
-
-  // TODO(burdon): Better way to define specific extensions for meeting companions.
-  // TODO(burdon): This brings in deps from ../calls; how should we manage/minimize explicit type exposure to other plugins?
-  // TODO(wittjosiah): These callbacks could be intents once we support broadcast.
-  export type CallProperties = {
-    onJoin: (state: { channel?: Channel.Channel; roomId?: string }) => Promise<void>;
-    onLeave: (roomId?: string) => Promise<void>;
-    onCallStateUpdated: (callState: CallState) => Promise<void>;
-    onMediaStateUpdated: ([mediaState, isSpeaking]: [MediaState, boolean]) => Promise<void>;
-  };
-  export const CallExtension = defineCapability<CallProperties>(`${meta.id}/capability/call-extension`);
-
-  type GetViewState = (subjectId: string) => ViewState;
-  export const State = defineCapability<{ state: DeepReadonly<ThreadState>; getViewState: GetViewState }>(
-    `${meta.id}/capability/state`,
-  );
-  export const MutableState = defineCapability<{ state: ThreadState; getViewState: GetViewState }>(
-    `${meta.id}/capability/state`,
-  );
-}