@dxos/plugin-thread 0.8.3 → 0.8.4-main.1f223c7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/plugin-thread",
-  "version": "0.8.3",
+  "version": "0.8.4-main.1f223c7",
   "description": "DXOS Threads Surface plugin",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -10,11 +10,13 @@
   "type": "module",
   "exports": {
     ".": {
+      "source": "./src/index.ts",
       "types": "./dist/types/src/index.d.ts",
       "browser": "./dist/lib/browser/index.mjs",
       "node": "./dist/lib/node-esm/index.mjs"
     },
     "./types": {
+      "source": "./src/types/index.ts",
       "types": "./dist/types/src/types/index.d.ts",
       "browser": "./dist/lib/browser/types/index.mjs",
       "node": "./dist/lib/node-esm/types/index.mjs"
@@ -33,9 +35,10 @@
     "src"
   ],
   "dependencies": {
-    "@codemirror/language": "^6.10.3",
-    "@codemirror/state": "^6.4.1",
-    "@codemirror/view": "^6.34.1",
+    "@codemirror/language": "^6.11.3",
+    "@codemirror/state": "^6.5.2",
+    "@codemirror/view": "^6.38.2",
+    "@effect/ai": "0.25.2",
     "@peermetrics/webrtc-stats": "5.7.1",
     "@preact-signals/safe-react": "^0.9.0",
     "@preact/signals-core": "^1.9.0",
@@ -43,68 +46,68 @@
     "extendable-media-recorder": "^9.2.19",
     "extendable-media-recorder-wav-encoder": "^7.0.123",
     "md5-hex": "5.0.0",
-    "react-resize-detector": "^12.0.2",
+    "react-resize-detector": "^11.0.1",
     "wavefile": "^11.0.0",
-    "@dxos/app-framework": "0.8.3",
-    "@dxos/codec-protobuf": "0.8.3",
-    "@dxos/async": "0.8.3",
-    "@dxos/context": "0.8.3",
-    "@dxos/devtools": "0.8.3",
-    "@dxos/echo": "0.8.3",
-    "@dxos/echo-schema": "0.8.3",
-    "@dxos/keys": "0.8.3",
-    "@dxos/display-name": "0.8.3",
-    "@dxos/invariant": "0.8.3",
-    "@dxos/local-storage": "0.8.3",
-    "@dxos/live-object": "0.8.3",
-    "@dxos/log": "0.8.3",
-    "@dxos/plugin-attention": "0.8.3",
-    "@dxos/plugin-client": "0.8.3",
-    "@dxos/plugin-deck": "0.8.3",
-    "@dxos/plugin-graph": "0.8.3",
-    "@dxos/plugin-observability": "0.8.3",
-    "@dxos/plugin-space": "0.8.3",
-    "@dxos/plugin-markdown": "0.8.3",
-    "@dxos/protocols": "0.8.3",
-    "@dxos/react-client": "0.8.3",
-    "@dxos/react-ui-attention": "0.8.3",
-    "@dxos/react-ui-editor": "0.8.3",
-    "@dxos/react-ui-form": "0.8.3",
-    "@dxos/react-ui-dnd": "0.8.3",
-    "@dxos/react-ui-menu": "0.8.3",
-    "@dxos/react-ui-syntax-highlighter": "0.8.3",
-    "@dxos/react-ui-sfx": "0.8.3",
-    "@dxos/react-ui-stack": "0.8.3",
-    "@dxos/react-ui-tabs": "0.8.3",
-    "@dxos/schema": "0.8.3",
-    "@dxos/util": "0.8.3",
-    "@dxos/react-ui-thread": "0.8.3"
+    "@dxos/app-framework": "0.8.4-main.1f223c7",
+    "@dxos/async": "0.8.4-main.1f223c7",
+    "@dxos/context": "0.8.4-main.1f223c7",
+    "@dxos/codec-protobuf": "0.8.4-main.1f223c7",
+    "@dxos/debug": "0.8.4-main.1f223c7",
+    "@dxos/assistant": "0.8.4-main.1f223c7",
+    "@dxos/display-name": "0.8.4-main.1f223c7",
+    "@dxos/echo": "0.8.4-main.1f223c7",
+    "@dxos/devtools": "0.8.4-main.1f223c7",
+    "@dxos/invariant": "0.8.4-main.1f223c7",
+    "@dxos/echo-schema": "0.8.4-main.1f223c7",
+    "@dxos/keys": "0.8.4-main.1f223c7",
+    "@dxos/live-object": "0.8.4-main.1f223c7",
+    "@dxos/local-storage": "0.8.4-main.1f223c7",
+    "@dxos/log": "0.8.4-main.1f223c7",
+    "@dxos/plugin-attention": "0.8.4-main.1f223c7",
+    "@dxos/plugin-client": "0.8.4-main.1f223c7",
+    "@dxos/plugin-deck": "0.8.4-main.1f223c7",
+    "@dxos/plugin-markdown": "0.8.4-main.1f223c7",
+    "@dxos/plugin-space": "0.8.4-main.1f223c7",
+    "@dxos/plugin-observability": "0.8.4-main.1f223c7",
+    "@dxos/protocols": "0.8.4-main.1f223c7",
+    "@dxos/plugin-graph": "0.8.4-main.1f223c7",
+    "@dxos/react-client": "0.8.4-main.1f223c7",
+    "@dxos/react-ui-attention": "0.8.4-main.1f223c7",
+    "@dxos/react-ui-dnd": "0.8.4-main.1f223c7",
+    "@dxos/react-ui-editor": "0.8.4-main.1f223c7",
+    "@dxos/react-ui-form": "0.8.4-main.1f223c7",
+    "@dxos/react-ui-menu": "0.8.4-main.1f223c7",
+    "@dxos/react-ui-sfx": "0.8.4-main.1f223c7",
+    "@dxos/react-ui-stack": "0.8.4-main.1f223c7",
+    "@dxos/react-ui-syntax-highlighter": "0.8.4-main.1f223c7",
+    "@dxos/react-ui-tabs": "0.8.4-main.1f223c7",
+    "@dxos/react-ui-thread": "0.8.4-main.1f223c7",
+    "@dxos/util": "0.8.4-main.1f223c7",
+    "@dxos/schema": "0.8.4-main.1f223c7"
   },
   "devDependencies": {
-    "@effect-rx/rx-react": "^0.34.1",
-    "@effect/platform": "0.80.12",
-    "@phosphor-icons/react": "^2.1.5",
+    "@effect-rx/rx-react": "0.38.0",
+    "@effect/platform": "0.90.2",
     "@types/react": "~18.2.0",
     "@types/react-dom": "~18.2.0",
-    "effect": "3.14.21",
+    "effect": "3.17.7",
     "react": "~18.2.0",
     "react-dom": "~18.2.0",
-    "vite": "5.4.7",
-    "@dxos/random": "0.8.3",
-    "@dxos/plugin-theme": "0.8.3",
-    "@dxos/react-ui": "0.8.3",
-    "@dxos/storybook-utils": "0.8.3",
-    "@dxos/react-ui-theme": "0.8.3"
+    "vite": "7.1.1",
+    "@dxos/react-ui": "0.8.4-main.1f223c7",
+    "@dxos/random": "0.8.4-main.1f223c7",
+    "@dxos/react-ui-theme": "0.8.4-main.1f223c7",
+    "@dxos/plugin-theme": "0.8.4-main.1f223c7",
+    "@dxos/storybook-utils": "0.8.4-main.1f223c7"
   },
   "peerDependencies": {
     "@effect-rx/rx-react": "^0.34.1",
-    "@effect/platform": "0.80.12",
-    "@phosphor-icons/react": "^2.1.5",
+    "@effect/platform": "^0.80.12",
     "effect": "^3.13.3",
     "react": "~18.2.0",
     "react-dom": "~18.2.0",
-    "@dxos/react-ui": "0.8.3",
-    "@dxos/react-ui-theme": "0.8.3"
+    "@dxos/react-ui": "0.8.4-main.1f223c7",
+    "@dxos/react-ui-theme": "0.8.4-main.1f223c7"
   },
   "publishConfig": {
     "access": "public"

package/src/ThreadPlugin.tsx

@@ -2,14 +2,12 @@
 // Copyright 2023 DXOS.org
 //
 
-import { Effect } from 'effect';
-
-import { Capabilities, contributes, createIntent, defineModule, definePlugin, Events } from '@dxos/app-framework';
-import { Ref, Type } from '@dxos/echo';
+import { Capabilities, Events, contributes, createIntent, defineModule, definePlugin } from '@dxos/app-framework';
+import { Ref } from '@dxos/echo';
 import { ClientCapabilities, ClientEvents } from '@dxos/plugin-client';
 import { MarkdownEvents } from '@dxos/plugin-markdown';
 import { SpaceCapabilities, SpaceEvents } from '@dxos/plugin-space';
-import { CollectionAction, defineObjectForm } from '@dxos/plugin-space/types';
+import { defineObjectForm } from '@dxos/plugin-space/types';
 import { translations as threadTranslations } from '@dxos/react-ui-thread';
 import { AnchoredTo, DataType } from '@dxos/schema';
 
@@ -21,9 +19,10 @@ import {
   ReactRoot,
   ReactSurface,
   ThreadState,
+  Toolkit,
 } from './capabilities';
-import { meta, THREAD_ITEM } from './meta';
-import translations from './translations';
+import { THREAD_ITEM, meta } from './meta';
+import { translations } from './translations';
 import { ChannelType, ThreadAction, ThreadType } from './types';
 
 // TODO(Zan): Every instance of `cursor` should be replaced with `anchor`.
@@ -124,23 +123,10 @@ export const ThreadPlugin = () =>
     defineModule({
       id: `${meta.id}/module/on-space-created`,
       activatesOn: SpaceEvents.SpaceCreated,
-      activate: (context) => {
-        const { dispatch } = context.getCapability(Capabilities.IntentDispatcher);
-        return contributes(SpaceCapabilities.OnSpaceCreated, async ({ space, rootCollection }) => {
-          const program = Effect.gen(function* () {
-            const { object: collection } = yield* dispatch(
-              createIntent(CollectionAction.CreateQueryCollection, { typename: Type.getTypename(ChannelType) }),
-            );
-            rootCollection.objects.push(Ref.make(collection));
-
-            const { object: channel } = yield* dispatch(
-              createIntent(ThreadAction.CreateChannel, { name: 'General', spaceId: space.id }),
-            );
-            space.db.add(channel);
-          });
-          await Effect.runPromise(program);
-        });
-      },
+      activate: () =>
+        contributes(SpaceCapabilities.OnSpaceCreated, ({ space, rootCollection }) =>
+          createIntent(ThreadAction.OnSpaceCreated, { space, rootCollection }),
+        ),
     }),
     defineModule({
       id: `${meta.id}/module/markdown`,
@@ -167,4 +153,10 @@ export const ThreadPlugin = () =>
       activatesOn: Events.SetupAppGraph,
       activate: AppGraphBuilder,
     }),
+    defineModule({
+      id: `${meta.id}/module/toolkit`,
+      // TODO(wittjosiah): Shouldn't use the startup event.
+      activatesOn: Events.Startup,
+      activate: Toolkit,
+    }),
   ]);

package/src/calls/call-manager.ts

@@ -12,7 +12,7 @@ import { live } from '@dxos/live-object';
 import { type Tracks } from '@dxos/protocols/proto/dxos/edge/calls';
 import { isNonNullable } from '@dxos/util';
 
-import { CallSwarmSynchronizer, type CallState } from './call-swarm-synchronizer';
+import { type CallState, CallSwarmSynchronizer } from './call-swarm-synchronizer';
 import { MediaManager, type MediaState } from './media-manager';
 import { type ActivityState, CALLS_URL, type EncodedTrackName, TrackNameCodec, type UserState } from './types';
 
@@ -88,8 +88,12 @@ export class CallManager extends Resource {
   }
 
   /** @reactive */
-  get pulledAudioTracks() {
-    return Object.values(this._state.media.pulledAudioTracks).map((track) => track.track);
+  get audioTracksToPlay(): MediaStreamTrack[] {
+    return (this._state.call.users ?? [])
+      .map((user) => (user.tracks?.audioEnabled ? user.tracks?.audio : undefined))
+      .filter(isNonNullable)
+      .map((track) => this._state.media.pulledAudioTracks[track as EncodedTrackName]?.track)
+      .filter(isNonNullable);
   }
 
   /** @reactive */
@@ -206,7 +210,7 @@ export class CallManager extends Resource {
 
   private _onCallStateUpdated(state: CallState): void {
     const tracksToPull = state.users
-      ?.filter((user) => user.joined && user.id !== state.self?.id)
+      ?.filter((user) => user.joined && user.id !== state.self!.id)
       ?.flatMap((user) => [user.tracks?.video, user.tracks?.audio, user.tracks?.screenshare])
       .filter(isNonNullable);
     this._mediaManager._schedulePullTracks(tracksToPull as EncodedTrackName[]);

package/src/calls/call-swarm-synchronizer.ts

@@ -17,7 +17,7 @@ import { type Device, type NetworkService } from '@dxos/protocols/proto/dxos/cli
 import { ConnectionState, type SwarmResponse } from '@dxos/protocols/proto/dxos/edge/messenger';
 import { isNonNullable } from '@dxos/util';
 
-import { codec, type ActivityState, type UserState } from './types';
+import { type ActivityState, type UserState, codec } from './types';
 
 export type CallState = {
   /**

package/src/calls/media-manager.ts

@@ -3,11 +3,12 @@
 //
 
 import { DeferredTask, Event, sleep, synchronized } from '@dxos/async';
-import { cancelWithContext, type Context, Resource } from '@dxos/context';
+import { type Context, Resource, cancelWithContext } from '@dxos/context';
+import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 
 import { SpeakingMonitor } from './speaking-monitor';
-import { TrackNameCodec, type EncodedTrackName, type TrackObject } from './types';
+import { type EncodedTrackName, TrackNameCodec, type TrackObject } from './types';
 import {
   type CallsServiceConfig,
   CallsServicePeer,
@@ -48,12 +49,14 @@ const VIDEO_WIDTH = 1280;
 const VIDEO_HEIGHT = 720;
 const MAX_WEB_CAM_FRAMERATE = 24;
 const MAX_WEB_CAM_BITRATE = 120_0000;
-const RETRY_INTERVAL = 500;
+const RETRY_INTERVAL = 100;
 
 export type MediaManagerParams = {
   serviceConfig: CallsServiceConfig;
 };
 
+const USE_INAUDIBLE_AUDIO = true;
+
 export class MediaManager extends Resource {
   public readonly stateUpdated = new Event<MediaState>();
   private readonly _state: MediaState = { pulledVideoStreams: {}, pulledAudioTracks: {} };
@@ -86,8 +89,10 @@ export class MediaManager extends Resource {
     this._state.videoStream = new MediaStream();
     this._state.videoStream.addTrack(this._state.videoTrack);
 
-    this._inaudibleAudioStreamTrack = await createInaudibleAudioStreamTrack({ ctx: this._ctx });
-    this._state.audioTrack = this._inaudibleAudioStreamTrack;
+    if (USE_INAUDIBLE_AUDIO) {
+      this._inaudibleAudioStreamTrack = await createInaudibleAudioStreamTrack({ ctx: this._ctx });
+      this._state.audioTrack = this._inaudibleAudioStreamTrack;
+    }
 
     this._pushTracksTask = new DeferredTask(this._ctx, async () => {
       await this._pushTracks();
@@ -170,11 +175,14 @@ export class MediaManager extends Resource {
   async turnAudioOff(): Promise<void> {
     void this._speakingMonitor?.close();
     this._speakingMonitor = undefined;
-    if (this._state.audioTrack !== this._inaudibleAudioStreamTrack) {
+    this._state.audioEnabled = false;
+    if (USE_INAUDIBLE_AUDIO && this._state.audioTrack !== this._inaudibleAudioStreamTrack) {
       this._state.audioTrack?.stop();
       this._state.audioTrack = this._inaudibleAudioStreamTrack;
+    } else {
+      this._state.audioTrack?.stop();
+      this._state.audioTrack = undefined;
     }
-    this._state.audioEnabled = false;
     this.stateUpdated.emit(this._state);
     this._pushTracksTask!.schedule();
   }
@@ -241,7 +249,7 @@ export class MediaManager extends Resource {
     });
 
     // Pull new tracks.
-    await Promise.all(tracksToPull.map((name) => this._pullTrack(name)));
+    const pullResults = await Promise.all(tracksToPull.map((name) => this._pullTrack(name)));
 
     // Close old tracks.
     await Promise.all([...audioTracksToClose, ...videoStreamsToClose].map(([_, { ctx }]) => ctx.dispose()));
@@ -251,6 +259,13 @@ export class MediaManager extends Resource {
       currentVideoStreams: Object.keys(this._state.pulledVideoStreams),
     });
     this.stateUpdated.emit(this._state);
+
+    if (pullResults.some(({ shouldRetry }) => shouldRetry)) {
+      await cancelWithContext(this._ctx, sleep(RETRY_INTERVAL));
+      log.info('retrying pull tracks', { tracksToPull });
+      invariant(this._pullTracksTask);
+      this._pullTracksTask.schedule();
+    }
   }
 
   private async _pushTracks(): Promise<void> {
@@ -259,38 +274,47 @@ export class MediaManager extends Resource {
     }
 
     let updated = false;
-    const [pushedVideoTrack, pushedAudioTrack, pushedScreenshareTrack] = await Promise.all([
-      this._pushTrack(this._state.videoTrack, this._state.pushedVideoTrack, [
+    const [pushVideoResult, pushAudioResult, pushScreenshareResult] = await Promise.all([
+      this._maybePushTrack(this._state.videoTrack, this._state.pushedVideoTrack, [
         { maxFramerate: MAX_WEB_CAM_FRAMERATE, maxBitrate: MAX_WEB_CAM_BITRATE },
       ]),
-      this._pushTrack(this._state.audioTrack, this._state.pushedAudioTrack, [{ networkPriority: 'high' }]),
-      this._pushTrack(this._state.screenshareTrack, this._state.pushedScreenshareTrack),
+      this._maybePushTrack(this._state.audioTrack, this._state.pushedAudioTrack, [{ networkPriority: 'high' }]),
+      this._maybePushTrack(this._state.screenshareTrack, this._state.pushedScreenshareTrack),
     ]);
 
-    if (pushedVideoTrack !== this._state.pushedVideoTrack) {
-      this._state.pushedVideoTrack = pushedVideoTrack;
+    if (pushVideoResult.track !== this._state.pushedVideoTrack) {
+      this._state.pushedVideoTrack = pushVideoResult.track;
       updated = true;
     }
-    if (pushedAudioTrack !== this._state.pushedAudioTrack) {
-      this._state.pushedAudioTrack = pushedAudioTrack;
+    if (pushAudioResult.track !== this._state.pushedAudioTrack) {
+      this._state.pushedAudioTrack = pushAudioResult.track;
       updated = true;
     }
-    if (pushedScreenshareTrack !== this._state.pushedScreenshareTrack) {
-      this._state.pushedScreenshareTrack = pushedScreenshareTrack;
+    if (pushScreenshareResult.track !== this._state.pushedScreenshareTrack) {
+      this._state.pushedScreenshareTrack = pushScreenshareResult.track;
       updated = true;
     }
-    if (!updated) {
-      return;
+
+    const shouldRetry = pushVideoResult.shouldRetry || pushAudioResult.shouldRetry || pushScreenshareResult.shouldRetry;
+
+    if (updated) {
+      this.stateUpdated.emit(this._state);
     }
 
-    this.stateUpdated.emit(this._state);
+    if (shouldRetry) {
+      await cancelWithContext(this._ctx, sleep(RETRY_INTERVAL));
+      log.info('retrying push tracks', { pushVideoResult, pushAudioResult, pushScreenshareResult });
+      invariant(this._pushTracksTask);
+      this._pushTracksTask.schedule();
+    }
   }
 
-  private async _pullTrack(name: EncodedTrackName): Promise<void> {
+  private async _pullTrack(name: EncodedTrackName): Promise<{ shouldRetry: boolean }> {
     const ctx = this._ctx.derive();
     try {
       const trackData = TrackNameCodec.decode(name);
-      const track = await this._state.peer!.pullTrack({ trackData, ctx });
+      // We need to set mid here to `undefined`, because mid is peer specific.
+      const track = await this._state.peer!.pullTrack({ trackData: { ...trackData, mid: undefined }, ctx });
       if (track?.readyState === 'ended') {
         throw new Error('Pulled track ended immediately');
       }
@@ -299,7 +323,7 @@ export class MediaManager extends Resource {
         case 'audio': {
           this._state.pulledAudioTracks[name] = { track, ctx };
           ctx.onDispose(() => delete this._state.pulledAudioTracks[name]);
-          break;
+          return { shouldRetry: false };
         }
         case 'video': {
           const mediaStream = new MediaStream();
@@ -310,33 +334,42 @@ export class MediaManager extends Resource {
             track.stop();
             delete this._state.pulledVideoStreams[name];
           });
-          break;
+          return { shouldRetry: false };
         }
         default:
           throw new Error(`Invalid track kind: ${track?.kind}`);
       }
     } catch (err) {
-      log.verbose('failed to pull track', { err, name });
+      log.info('failed to pull track', { err, name });
       void ctx.dispose();
-      await cancelWithContext(this._ctx, sleep(RETRY_INTERVAL));
-      log.verbose('retrying pull track', { name });
-      this._pullTracksTask!.schedule();
+      return { shouldRetry: true };
     }
   }
 
-  private async _pushTrack(
+  private async _maybePushTrack(
     track?: MediaStreamTrack,
     previousTrack?: TrackObject,
     encodings?: RTCRtpEncodingParameters[],
-  ): Promise<TrackObject | undefined> {
+  ): Promise<{ track?: TrackObject; shouldRetry?: boolean }> {
     if (!track && !previousTrack) {
-      return;
+      return { track: undefined };
     }
 
-    return this._state.peer!.pushTrack({
-      track: track ?? null,
-      previousTrack,
-      encodings,
-    });
+    const ctx = this._ctx.derive();
+
+    try {
+      return {
+        track: await this._state.peer!.pushTrack({
+          ctx,
+          track: track ?? null,
+          previousTrack,
+          encodings,
+        }),
+      };
+    } catch (err) {
+      log.info('failed to push track', { err, track, previousTrack, encodings });
+      void ctx.dispose();
+      return { shouldRetry: true };
+    }
   }
 }

package/src/calls/types.ts

@@ -2,11 +2,12 @@
 // Copyright 2025 DXOS.org
 //
 
+import { invariant } from '@dxos/invariant';
 import { buf } from '@dxos/protocols/buf';
 import {
   type ActivitySchema,
-  UserStateSchema,
   type UserState as UserStateProto,
+  UserStateSchema,
 } from '@dxos/protocols/buf/dxos/edge/calls_pb';
 
 /**
@@ -61,10 +62,13 @@ export type EncodedTrackName = string & { __brand: 'EncodedTrackName' };
 
 export const TrackNameCodec = {
   encode: (trackData: TrackObject): EncodedTrackName => {
-    return (trackData.sessionId + '/' + trackData.trackName) as EncodedTrackName;
+    invariant(trackData.sessionId);
+    invariant(trackData.trackName);
+    invariant(trackData.mid);
+    return (trackData.sessionId + '/' + trackData.trackName + '/' + trackData.mid) as EncodedTrackName;
   },
   decode: (name: EncodedTrackName): TrackObject => {
-    const [sessionId, trackName] = name.split('/');
-    return { sessionId, trackName, location: 'remote' };
+    const [sessionId, trackName, mid] = name.split('/');
+    return { sessionId, trackName, location: 'remote', mid };
   },
 };