@dxos/plugin-thread 0.8.2-staging.7ac8446 → 0.8.3-main.672df60

package/src/calls/speaking-monitor.ts

@@ -0,0 +1,85 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { DeferredTask, Event, sleep } from '@dxos/async';
+import { Resource } from '@dxos/context';
+
+import { monitorAudioLevel } from './util';
+
+const NOT_SPEAKING_AFTER = 500; // [ms]
+const START_SPEAKING_THRESHOLD = 0.05;
+const CONTINUE_SPEAKING_THRESHOLD = 0.02;
+const MIN_EVENT_INTERVAL = 100; // [ms]
+
+export class SpeakingMonitor extends Resource {
+  /**
+   * Event that is emitted when the user starts or stops speaking.
+   * It is debounced to prevent rapid state changes.
+   */
+  public readonly speakingChanged = new Event<boolean>();
+
+  /**
+   * Real state of the user speaking.
+   */
+  private _isSpeaking = false;
+  private _lastEventTime = 0;
+  private _deferredUpdate?: DeferredTask | undefined;
+
+  constructor(private readonly _mediaStreamTrack: MediaStreamTrack) {
+    super();
+  }
+
+  get isSpeaking(): boolean {
+    return this._isSpeaking;
+  }
+
+  protected override async _open(): Promise<void> {
+    this._deferredUpdate = new DeferredTask(this._ctx, async () => this._emitSpeakingChanged());
+
+    let timeout: NodeJS.Timeout | undefined;
+    const cleanup = monitorAudioLevel({
+      mediaStreamTrack: this._mediaStreamTrack,
+      onMeasure: (vol) => {
+        // Once the user has been determined to be speaking, we want
+        // to lower the threshold because speech patterns don't always
+        // kick up above 0.05.
+        const audioLevelAboveThreshold =
+          vol > (this._isSpeaking ? CONTINUE_SPEAKING_THRESHOLD : START_SPEAKING_THRESHOLD);
+        if (audioLevelAboveThreshold) {
+          // User is still speaking, clear timeout & reset.
+          clearTimeout(timeout);
+          timeout = undefined;
+          if (!this._isSpeaking) {
+            this._isSpeaking = true;
+            this._deferredUpdate?.schedule();
+          }
+        } else if (timeout === undefined) {
+          // User is not speaking and timeout is not set.
+          timeout = setTimeout(() => {
+            this._isSpeaking = false;
+            timeout = undefined;
+            this._deferredUpdate?.schedule();
+          }, NOT_SPEAKING_AFTER);
+        }
+      },
+    });
+    this._ctx.onDispose(() => {
+      cleanup();
+      clearTimeout(timeout);
+    });
+  }
+
+  /**
+   * Emits the speaking changed event with debouncing based on timing.
+   */
+  private async _emitSpeakingChanged(): Promise<void> {
+    const timeSinceLastEvent = Date.now() - this._lastEventTime;
+    const timeToNextEvent = MIN_EVENT_INTERVAL - timeSinceLastEvent;
+    if (timeToNextEvent > 0) {
+      await sleep(timeToNextEvent);
+    }
+    this._lastEventTime = Date.now();
+    this.speakingChanged.emit(this._isSpeaking);
+  }
+}

package/src/calls/types.ts

@@ -0,0 +1,70 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { buf } from '@dxos/protocols/buf';
+import {
+  type ActivitySchema,
+  UserStateSchema,
+  type UserState as UserStateProto,
+} from '@dxos/protocols/buf/dxos/edge/calls_pb';
+
+/**
+ * Endpoint to the calls service.
+ */
+export const CALLS_URL = 'https://calls-service.dxos.workers.dev';
+
+export type UserState = buf.MessageInitShape<typeof UserStateSchema>;
+
+export type ActivityState = buf.MessageInitShape<typeof ActivitySchema>;
+
+export const codec = {
+  encode: (message: UserState): Uint8Array => {
+    return buf.toBinary(UserStateSchema, buf.create(UserStateSchema, message));
+  },
+  decode: (message: Uint8Array): UserStateProto => {
+    return buf.fromBinary(UserStateSchema, message);
+  },
+};
+
+export type TrackObject = {
+  location?: 'local' | 'remote';
+  trackName?: string;
+  sessionId?: string;
+  mid?: string | null;
+};
+
+export type SessionDescription = {
+  type: 'offer' | 'answer';
+  sdp: string;
+};
+
+export interface ErrorResponse {
+  errorCode?: string;
+  errorDescription?: string;
+}
+
+export interface SessionResponse extends ErrorResponse {
+  sessionId: string;
+  sessionDescription: SessionDescription;
+}
+
+export interface TracksResponse extends ErrorResponse {
+  sessionDescription: SessionDescription;
+  requiresImmediateRenegotiation: boolean;
+  tracks?: (TrackObject & ErrorResponse)[];
+}
+
+export interface RenegotiationResponse extends ErrorResponse {}
+
+export type EncodedTrackName = string & { __brand: 'EncodedTrackName' };
+
+export const TrackNameCodec = {
+  encode: (trackData: TrackObject): EncodedTrackName => {
+    return (trackData.sessionId + '/' + trackData.trackName) as EncodedTrackName;
+  },
+  decode: (name: EncodedTrackName): TrackObject => {
+    const [sessionId, trackName] = name.split('/');
+    return { sessionId, trackName, location: 'remote' };
+  },
+};

package/src/calls/util/audio.ts

@@ -0,0 +1,58 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export const monitorAudioLevel = ({
+  mediaStreamTrack,
+  intervalMs = 1000 / 10, // 12 fps.
+  onMeasure,
+}: {
+  mediaStreamTrack: MediaStreamTrack;
+  intervalMs?: number;
+  onMeasure: (value: number) => void;
+}) => {
+  let timeout = -1;
+  let interval = -1;
+  const audioContext = new AudioContext();
+  const stream = new MediaStream();
+  stream.addTrack(mediaStreamTrack);
+  const mediaStreamAudioSourceNode = audioContext.createMediaStreamSource(stream);
+  const analyserNode = audioContext.createAnalyser();
+  mediaStreamAudioSourceNode.connect(analyserNode);
+  // Since we just need a rough approximation and will be measuring
+  // frequently, we want to drop this down from the default of 2048.
+  analyserNode.fftSize = 32;
+
+  const pcmData = new Float32Array(analyserNode.fftSize);
+  let peak = 0;
+
+  interval = window.setInterval(() => {
+    onMeasure(peak);
+    peak = 0;
+  }, intervalMs);
+
+  const tick = () => {
+    timeout = window.setTimeout(() => {
+      analyserNode.getFloatTimeDomainData(pcmData);
+      let sumSquares = 0.0;
+      for (const amplitude of pcmData) {
+        sumSquares += amplitude * amplitude;
+      }
+      const current = Math.sqrt(sumSquares / pcmData.length);
+      if (current > peak) {
+        peak = current;
+      }
+      tick();
+    }, intervalMs);
+  };
+  tick();
+
+  return () => {
+    mediaStreamAudioSourceNode.disconnect();
+    analyserNode.disconnect();
+    void audioContext.close();
+    clearInterval(interval);
+    clearTimeout(timeout);
+    stream.removeTrack(mediaStreamTrack);
+  };
+};