@dxos/plugin-thread 0.8.4-main.2e9d522 → 0.8.4-main.5ea62a8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/plugin-thread",
-  "version": "0.8.4-main.2e9d522",
+  "version": "0.8.4-main.5ea62a8",
   "description": "DXOS Threads Surface plugin",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -10,11 +10,13 @@
   "type": "module",
   "exports": {
     ".": {
+      "source": "./src/index.ts",
       "types": "./dist/types/src/index.d.ts",
       "browser": "./dist/lib/browser/index.mjs",
       "node": "./dist/lib/node-esm/index.mjs"
     },
     "./types": {
+      "source": "./src/types/index.ts",
       "types": "./dist/types/src/types/index.d.ts",
       "browser": "./dist/lib/browser/types/index.mjs",
       "node": "./dist/lib/node-esm/types/index.mjs"
@@ -36,6 +38,7 @@
     "@codemirror/language": "^6.10.3",
     "@codemirror/state": "^6.4.1",
     "@codemirror/view": "^6.34.1",
+    "@effect/ai": "0.25.2",
     "@peermetrics/webrtc-stats": "5.7.1",
     "@preact-signals/safe-react": "^0.9.0",
     "@preact/signals-core": "^1.9.0",
@@ -43,58 +46,59 @@
     "extendable-media-recorder": "^9.2.19",
     "extendable-media-recorder-wav-encoder": "^7.0.123",
     "md5-hex": "5.0.0",
-    "react-resize-detector": "^12.0.2",
+    "react-resize-detector": "^11.0.1",
     "wavefile": "^11.0.0",
-    "@dxos/app-framework": "0.8.4-main.2e9d522",
-    "@dxos/async": "0.8.4-main.2e9d522",
-    "@dxos/context": "0.8.4-main.2e9d522",
-    "@dxos/codec-protobuf": "0.8.4-main.2e9d522",
-    "@dxos/debug": "0.8.4-main.2e9d522",
-    "@dxos/devtools": "0.8.4-main.2e9d522",
-    "@dxos/echo": "0.8.4-main.2e9d522",
-    "@dxos/display-name": "0.8.4-main.2e9d522",
-    "@dxos/echo-schema": "0.8.4-main.2e9d522",
-    "@dxos/invariant": "0.8.4-main.2e9d522",
-    "@dxos/keys": "0.8.4-main.2e9d522",
-    "@dxos/live-object": "0.8.4-main.2e9d522",
-    "@dxos/local-storage": "0.8.4-main.2e9d522",
-    "@dxos/log": "0.8.4-main.2e9d522",
-    "@dxos/plugin-attention": "0.8.4-main.2e9d522",
-    "@dxos/plugin-client": "0.8.4-main.2e9d522",
-    "@dxos/plugin-deck": "0.8.4-main.2e9d522",
-    "@dxos/plugin-graph": "0.8.4-main.2e9d522",
-    "@dxos/plugin-markdown": "0.8.4-main.2e9d522",
-    "@dxos/plugin-observability": "0.8.4-main.2e9d522",
-    "@dxos/protocols": "0.8.4-main.2e9d522",
-    "@dxos/plugin-space": "0.8.4-main.2e9d522",
-    "@dxos/react-client": "0.8.4-main.2e9d522",
-    "@dxos/react-ui-attention": "0.8.4-main.2e9d522",
-    "@dxos/react-ui-dnd": "0.8.4-main.2e9d522",
-    "@dxos/react-ui-editor": "0.8.4-main.2e9d522",
-    "@dxos/react-ui-form": "0.8.4-main.2e9d522",
-    "@dxos/react-ui-menu": "0.8.4-main.2e9d522",
-    "@dxos/react-ui-sfx": "0.8.4-main.2e9d522",
-    "@dxos/react-ui-stack": "0.8.4-main.2e9d522",
-    "@dxos/react-ui-tabs": "0.8.4-main.2e9d522",
-    "@dxos/react-ui-syntax-highlighter": "0.8.4-main.2e9d522",
-    "@dxos/react-ui-thread": "0.8.4-main.2e9d522",
-    "@dxos/schema": "0.8.4-main.2e9d522",
-    "@dxos/util": "0.8.4-main.2e9d522"
+    "@dxos/app-framework": "0.8.4-main.5ea62a8",
+    "@dxos/assistant": "0.8.4-main.5ea62a8",
+    "@dxos/async": "0.8.4-main.5ea62a8",
+    "@dxos/debug": "0.8.4-main.5ea62a8",
+    "@dxos/devtools": "0.8.4-main.5ea62a8",
+    "@dxos/codec-protobuf": "0.8.4-main.5ea62a8",
+    "@dxos/display-name": "0.8.4-main.5ea62a8",
+    "@dxos/echo": "0.8.4-main.5ea62a8",
+    "@dxos/echo-schema": "0.8.4-main.5ea62a8",
+    "@dxos/invariant": "0.8.4-main.5ea62a8",
+    "@dxos/keys": "0.8.4-main.5ea62a8",
+    "@dxos/live-object": "0.8.4-main.5ea62a8",
+    "@dxos/context": "0.8.4-main.5ea62a8",
+    "@dxos/log": "0.8.4-main.5ea62a8",
+    "@dxos/local-storage": "0.8.4-main.5ea62a8",
+    "@dxos/plugin-attention": "0.8.4-main.5ea62a8",
+    "@dxos/plugin-deck": "0.8.4-main.5ea62a8",
+    "@dxos/plugin-graph": "0.8.4-main.5ea62a8",
+    "@dxos/plugin-markdown": "0.8.4-main.5ea62a8",
+    "@dxos/plugin-observability": "0.8.4-main.5ea62a8",
+    "@dxos/plugin-client": "0.8.4-main.5ea62a8",
+    "@dxos/plugin-space": "0.8.4-main.5ea62a8",
+    "@dxos/protocols": "0.8.4-main.5ea62a8",
+    "@dxos/react-client": "0.8.4-main.5ea62a8",
+    "@dxos/react-ui-dnd": "0.8.4-main.5ea62a8",
+    "@dxos/react-ui-editor": "0.8.4-main.5ea62a8",
+    "@dxos/react-ui-form": "0.8.4-main.5ea62a8",
+    "@dxos/react-ui-attention": "0.8.4-main.5ea62a8",
+    "@dxos/react-ui-menu": "0.8.4-main.5ea62a8",
+    "@dxos/react-ui-stack": "0.8.4-main.5ea62a8",
+    "@dxos/react-ui-sfx": "0.8.4-main.5ea62a8",
+    "@dxos/react-ui-syntax-highlighter": "0.8.4-main.5ea62a8",
+    "@dxos/react-ui-tabs": "0.8.4-main.5ea62a8",
+    "@dxos/schema": "0.8.4-main.5ea62a8",
+    "@dxos/util": "0.8.4-main.5ea62a8",
+    "@dxos/react-ui-thread": "0.8.4-main.5ea62a8"
   },
   "devDependencies": {
     "@effect-rx/rx-react": "0.38.0",
-    "@effect/platform": "0.89.0",
+    "@effect/platform": "0.90.2",
     "@types/react": "~18.2.0",
     "@types/react-dom": "~18.2.0",
-    "effect": "3.17.0",
+    "effect": "3.17.7",
     "react": "~18.2.0",
     "react-dom": "~18.2.0",
-    "vite": "5.4.7",
-    "@dxos/plugin-theme": "0.8.4-main.2e9d522",
-    "@dxos/random": "0.8.4-main.2e9d522",
-    "@dxos/react-ui": "0.8.4-main.2e9d522",
-    "@dxos/storybook-utils": "0.8.4-main.2e9d522",
-    "@dxos/react-ui-theme": "0.8.4-main.2e9d522"
+    "vite": "7.1.1",
+    "@dxos/plugin-theme": "0.8.4-main.5ea62a8",
+    "@dxos/random": "0.8.4-main.5ea62a8",
+    "@dxos/react-ui": "0.8.4-main.5ea62a8",
+    "@dxos/react-ui-theme": "0.8.4-main.5ea62a8",
+    "@dxos/storybook-utils": "0.8.4-main.5ea62a8"
   },
   "peerDependencies": {
     "@effect-rx/rx-react": "^0.34.1",
@@ -102,8 +106,8 @@
     "effect": "^3.13.3",
     "react": "~18.2.0",
     "react-dom": "~18.2.0",
-    "@dxos/react-ui": "0.8.4-main.2e9d522",
-    "@dxos/react-ui-theme": "0.8.4-main.2e9d522"
+    "@dxos/react-ui": "0.8.4-main.5ea62a8",
+    "@dxos/react-ui-theme": "0.8.4-main.5ea62a8"
   },
   "publishConfig": {
     "access": "public"

package/src/ThreadPlugin.tsx

@@ -2,7 +2,7 @@
 // Copyright 2023 DXOS.org
 //
 
-import { Capabilities, contributes, createIntent, defineModule, definePlugin, Events } from '@dxos/app-framework';
+import { Capabilities, Events, contributes, createIntent, defineModule, definePlugin } from '@dxos/app-framework';
 import { Ref } from '@dxos/echo';
 import { ClientCapabilities, ClientEvents } from '@dxos/plugin-client';
 import { MarkdownEvents } from '@dxos/plugin-markdown';
@@ -19,8 +19,9 @@ import {
   ReactRoot,
   ReactSurface,
   ThreadState,
+  Toolkit,
 } from './capabilities';
-import { meta, THREAD_ITEM } from './meta';
+import { THREAD_ITEM, meta } from './meta';
 import { translations } from './translations';
 import { ChannelType, ThreadAction, ThreadType } from './types';
 
@@ -152,4 +153,10 @@ export const ThreadPlugin = () =>
       activatesOn: Events.SetupAppGraph,
       activate: AppGraphBuilder,
     }),
+    defineModule({
+      id: `${meta.id}/module/toolkit`,
+      // TODO(wittjosiah): Shouldn't use the startup event.
+      activatesOn: Events.Startup,
+      activate: Toolkit,
+    }),
   ]);

package/src/calls/call-manager.ts

@@ -12,7 +12,7 @@ import { live } from '@dxos/live-object';
 import { type Tracks } from '@dxos/protocols/proto/dxos/edge/calls';
 import { isNonNullable } from '@dxos/util';
 
-import { CallSwarmSynchronizer, type CallState } from './call-swarm-synchronizer';
+import { type CallState, CallSwarmSynchronizer } from './call-swarm-synchronizer';
 import { MediaManager, type MediaState } from './media-manager';
 import { type ActivityState, CALLS_URL, type EncodedTrackName, TrackNameCodec, type UserState } from './types';
 

package/src/calls/call-swarm-synchronizer.ts

@@ -17,7 +17,7 @@ import { type Device, type NetworkService } from '@dxos/protocols/proto/dxos/cli
 import { ConnectionState, type SwarmResponse } from '@dxos/protocols/proto/dxos/edge/messenger';
 import { isNonNullable } from '@dxos/util';
 
-import { codec, type ActivityState, type UserState } from './types';
+import { type ActivityState, type UserState, codec } from './types';
 
 export type CallState = {
   /**

package/src/calls/media-manager.ts

@@ -3,11 +3,12 @@
 //
 
 import { DeferredTask, Event, sleep, synchronized } from '@dxos/async';
-import { cancelWithContext, type Context, Resource } from '@dxos/context';
+import { type Context, Resource, cancelWithContext } from '@dxos/context';
+import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 
 import { SpeakingMonitor } from './speaking-monitor';
-import { TrackNameCodec, type EncodedTrackName, type TrackObject } from './types';
+import { type EncodedTrackName, TrackNameCodec, type TrackObject } from './types';
 import {
   type CallsServiceConfig,
   CallsServicePeer,
@@ -48,13 +49,13 @@ const VIDEO_WIDTH = 1280;
 const VIDEO_HEIGHT = 720;
 const MAX_WEB_CAM_FRAMERATE = 24;
 const MAX_WEB_CAM_BITRATE = 120_0000;
-const RETRY_INTERVAL = 500;
+const RETRY_INTERVAL = 100;
 
 export type MediaManagerParams = {
   serviceConfig: CallsServiceConfig;
 };
 
-const USE_INAUDIBLE_AUDIO = false;
+const USE_INAUDIBLE_AUDIO = true;
 
 export class MediaManager extends Resource {
   public readonly stateUpdated = new Event<MediaState>();
@@ -248,7 +249,7 @@ export class MediaManager extends Resource {
     });
 
     // Pull new tracks.
-    await Promise.all(tracksToPull.map((name) => this._pullTrack(name)));
+    const pullResults = await Promise.all(tracksToPull.map((name) => this._pullTrack(name)));
 
     // Close old tracks.
     await Promise.all([...audioTracksToClose, ...videoStreamsToClose].map(([_, { ctx }]) => ctx.dispose()));
@@ -258,6 +259,13 @@ export class MediaManager extends Resource {
       currentVideoStreams: Object.keys(this._state.pulledVideoStreams),
     });
     this.stateUpdated.emit(this._state);
+
+    if (pullResults.some(({ shouldRetry }) => shouldRetry)) {
+      await cancelWithContext(this._ctx, sleep(RETRY_INTERVAL));
+      log.info('retrying pull tracks', { tracksToPull });
+      invariant(this._pullTracksTask);
+      this._pullTracksTask.schedule();
+    }
   }
 
   private async _pushTracks(): Promise<void> {
@@ -266,34 +274,42 @@ export class MediaManager extends Resource {
     }
 
     let updated = false;
-    const [pushedVideoTrack, pushedAudioTrack, pushedScreenshareTrack] = await Promise.all([
-      this._pushTrack(this._state.videoTrack, this._state.pushedVideoTrack, [
+    const [pushVideoResult, pushAudioResult, pushScreenshareResult] = await Promise.all([
+      this._maybePushTrack(this._state.videoTrack, this._state.pushedVideoTrack, [
         { maxFramerate: MAX_WEB_CAM_FRAMERATE, maxBitrate: MAX_WEB_CAM_BITRATE },
       ]),
-      this._pushTrack(this._state.audioTrack, this._state.pushedAudioTrack, [{ networkPriority: 'high' }]),
-      this._pushTrack(this._state.screenshareTrack, this._state.pushedScreenshareTrack),
+      this._maybePushTrack(this._state.audioTrack, this._state.pushedAudioTrack, [{ networkPriority: 'high' }]),
+      this._maybePushTrack(this._state.screenshareTrack, this._state.pushedScreenshareTrack),
     ]);
 
-    if (pushedVideoTrack !== this._state.pushedVideoTrack) {
-      this._state.pushedVideoTrack = pushedVideoTrack;
+    if (pushVideoResult.track !== this._state.pushedVideoTrack) {
+      this._state.pushedVideoTrack = pushVideoResult.track;
       updated = true;
     }
-    if (pushedAudioTrack !== this._state.pushedAudioTrack) {
-      this._state.pushedAudioTrack = pushedAudioTrack;
+    if (pushAudioResult.track !== this._state.pushedAudioTrack) {
+      this._state.pushedAudioTrack = pushAudioResult.track;
       updated = true;
     }
-    if (pushedScreenshareTrack !== this._state.pushedScreenshareTrack) {
-      this._state.pushedScreenshareTrack = pushedScreenshareTrack;
+    if (pushScreenshareResult.track !== this._state.pushedScreenshareTrack) {
+      this._state.pushedScreenshareTrack = pushScreenshareResult.track;
       updated = true;
     }
-    if (!updated) {
-      return;
+
+    const shouldRetry = pushVideoResult.shouldRetry || pushAudioResult.shouldRetry || pushScreenshareResult.shouldRetry;
+
+    if (updated) {
+      this.stateUpdated.emit(this._state);
     }
 
-    this.stateUpdated.emit(this._state);
+    if (shouldRetry) {
+      await cancelWithContext(this._ctx, sleep(RETRY_INTERVAL));
+      log.info('retrying push tracks', { pushVideoResult, pushAudioResult, pushScreenshareResult });
+      invariant(this._pushTracksTask);
+      this._pushTracksTask.schedule();
+    }
   }
 
-  private async _pullTrack(name: EncodedTrackName): Promise<void> {
+  private async _pullTrack(name: EncodedTrackName): Promise<{ shouldRetry: boolean }> {
     const ctx = this._ctx.derive();
     try {
       const trackData = TrackNameCodec.decode(name);
@@ -306,7 +322,7 @@ export class MediaManager extends Resource {
         case 'audio': {
           this._state.pulledAudioTracks[name] = { track, ctx };
           ctx.onDispose(() => delete this._state.pulledAudioTracks[name]);
-          break;
+          return { shouldRetry: false };
         }
         case 'video': {
           const mediaStream = new MediaStream();
@@ -317,33 +333,42 @@ export class MediaManager extends Resource {
             track.stop();
             delete this._state.pulledVideoStreams[name];
           });
-          break;
+          return { shouldRetry: false };
         }
         default:
           throw new Error(`Invalid track kind: ${track?.kind}`);
       }
     } catch (err) {
-      log.verbose('failed to pull track', { err, name });
+      log.info('failed to pull track', { err, name });
       void ctx.dispose();
-      await cancelWithContext(this._ctx, sleep(RETRY_INTERVAL));
-      log.verbose('retrying pull track', { name });
-      this._pullTracksTask!.schedule();
+      return { shouldRetry: true };
     }
   }
 
-  private async _pushTrack(
+  private async _maybePushTrack(
     track?: MediaStreamTrack,
     previousTrack?: TrackObject,
     encodings?: RTCRtpEncodingParameters[],
-  ): Promise<TrackObject | undefined> {
+  ): Promise<{ track?: TrackObject; shouldRetry?: boolean }> {
     if (!track && !previousTrack) {
-      return;
+      return { track: undefined };
     }
 
-    return this._state.peer!.pushTrack({
-      track: track ?? null,
-      previousTrack,
-      encodings,
-    });
+    const ctx = this._ctx.derive();
+
+    try {
+      return {
+        track: await this._state.peer!.pushTrack({
+          ctx,
+          track: track ?? null,
+          previousTrack,
+          encodings,
+        }),
+      };
+    } catch (err) {
+      log.info('failed to push track', { err, track, previousTrack, encodings });
+      void ctx.dispose();
+      return { shouldRetry: true };
+    }
   }
 }

package/src/calls/types.ts

@@ -5,8 +5,8 @@
 import { buf } from '@dxos/protocols/buf';
 import {
   type ActivitySchema,
-  UserStateSchema,
   type UserState as UserStateProto,
+  UserStateSchema,
 } from '@dxos/protocols/buf/dxos/edge/calls_pb';
 
 /**

package/src/calls/util/calls-service.ts

@@ -2,21 +2,22 @@
 // Copyright 2024 DXOS.org
 //
 
-import { PersistentLifecycle } from '@dxos/async';
+import { PersistentLifecycle, Trigger, scheduleTask } from '@dxos/async';
 import { type Context, Resource } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 
-import { HistoryCache } from './history-cache';
-import { BulkRequestDispatcher, FIFOScheduler } from './task-scheduling';
 import {
-  type SessionResponse,
   type ErrorResponse,
   type RenegotiationResponse,
+  type SessionResponse,
   type TrackObject,
   type TracksResponse,
 } from '../types';
 
+import { HistoryCache } from './history-cache';
+import { BulkRequestDispatcher, FIFOScheduler } from './task-scheduling';
+
 const NETWORK_TIMEOUT = 5_000;
 
 export type Session = {
@@ -203,11 +204,11 @@ export class CallsServicePeer extends Resource {
     sessionId,
     trackName,
   }: {
+    ctx: Context;
     peerConnection: RTCPeerConnection;
     transceiver: RTCRtpTransceiver;
     sessionId: string;
     trackName: string;
-    ctx?: Context;
   }): Promise<TrackObject | void> {
     log.verbose('pushing track ', trackName);
     const pushedTrackPromise = this.pushTrackDispatcher
@@ -245,9 +246,13 @@ export class CallsServicePeer extends Resource {
           };
         }),
       )
-      .then(({ tracks }) => {
+      .then(async ({ tracks }) => {
         const trackData = tracks.find((t) => t.mid === transceiver.mid);
         if (trackData) {
+          // we wait for the transceiver to start sending data before we emit
+          // the track metadata to ensure that the track will be able to be
+          // pulled before making the metadata available to anyone else.
+          await waitForTransceiverToSendData(ctx, transceiver);
           return {
             ...trackData,
             sessionId,
@@ -256,9 +261,6 @@ export class CallsServicePeer extends Resource {
         } else {
           log.error('missing track data');
         }
-      })
-      .catch((err) => {
-        log.catch(err);
       });
 
     const onDispose = async () => {
@@ -280,8 +282,8 @@ export class CallsServicePeer extends Resource {
     previousTrack,
   }: {
     track: MediaStreamTrack | null;
+    ctx: Context;
     encodings?: RTCRtpEncodingParameters[];
-    ctx?: Context;
     previousTrack?: TrackObject;
   }): Promise<TrackObject | undefined> {
     await this.waitUntilOpen();
@@ -528,3 +530,47 @@ const peerConnectionIsConnected = async (peerConnection: RTCPeerConnection) => {
     await connected;
   }
 };
+
+const INITIAL_DELAY = 50;
+const MAX_DELAY = 500;
+const MAX_RETRIES = 50;
+const WAIT_FOR_DATA_TIMEOUT = 20_000;
+
+const waitForTransceiverToSendData = async (ctx: Context, transceiver: RTCRtpTransceiver): Promise<void> => {
+  let delay = INITIAL_DELAY;
+  let checks = 0;
+  const waitForData = new Trigger();
+  ctx.onDispose(() => waitForData.throw(new Error('Transceiver was closed')));
+
+  const checkStats = async () => {
+    if (ctx.disposed) return;
+
+    try {
+      const stats = await transceiver.sender.getStats();
+      let dataFound = false;
+      stats.forEach((stat) => {
+        if (stat.type === 'outbound-rtp' && stat.bytesSent > 0) {
+          dataFound = true;
+        }
+      });
+
+      if (dataFound) {
+        waitForData.wake();
+        return;
+      }
+    } catch {
+      // Stats might not be available yet, continue checking
+    }
+    checks++;
+    if (checks > MAX_RETRIES) {
+      waitForData.throw(new Error('Transceiver did not send data'));
+      return;
+    }
+
+    delay = Math.min(delay * 2, MAX_DELAY); // Exponential backoff with max cap
+    scheduleTask(ctx, checkStats, delay);
+  };
+
+  void checkStats();
+  return waitForData.wait({ timeout: WAIT_FOR_DATA_TIMEOUT });
+};

package/src/calls/util/stub-media-streams.ts

@@ -3,8 +3,8 @@
 //
 //
 
-import { waitForCondition, scheduleTaskInterval } from '@dxos/async';
-import { cancelWithContext, type Context } from '@dxos/context';
+import { scheduleTaskInterval, waitForCondition } from '@dxos/async';
+import { type Context, cancelWithContext } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
 
 /**
@@ -30,7 +30,7 @@ export const createBlackCanvasStreamTrack = async ({
     canvasCtx.fillStyle = 'black';
     canvasCtx.fillRect(0, 0, canvas.width, canvas.height);
   };
-  scheduleTaskInterval(ctx, async () => drawFrame(), 1_000);
+  scheduleTaskInterval(ctx, async () => drawFrame(), 200);
 
   const track = canvas.captureStream().getVideoTracks()[0];
   drawFrame();
@@ -52,8 +52,7 @@ export const createInaudibleAudioStreamTrack = async ({ ctx }: { ctx: Context })
   oscillator.frequency.setValueAtTime(20, audioContext.currentTime);
 
   const gainNode = audioContext.createGain();
-  // even w/ gain at 0 some packets are sent
-  gainNode.gain.setValueAtTime(0.02, audioContext.currentTime);
+  gainNode.gain.setValueAtTime(0.01, audioContext.currentTime);
   oscillator.connect(gainNode);
 
   const destination = audioContext.createMediaStreamDestination();