@tribe-nest/media-client 0.2.1 → 0.4.0

package/src/core/state.ts

@@ -29,6 +29,21 @@ export type ProducerEntry = {
   source?: string;
 };
 
+/** One entry of the node's speaker ranking, loudest first. */
+export type SpeakerInfo = {
+  identity: string;
+  producerId: string;
+  /** Quantized by the node to 5 dB steps, so most ticks are value-identical. */
+  volumeDb: number;
+};
+
+/** A hand currently up, oldest first: the order is the queue. */
+export type RaisedHand = {
+  identity: string;
+  /** Node clock, for ordering only. */
+  raisedAt: number;
+};
+
 export type RoomState = {
   phase: "idle" | "joined" | "closed";
   /** Our own identity, as the node reported it. */
@@ -44,6 +59,21 @@ export type RoomState = {
   producers: readonly ProducerEntry[];
   /** The set the NODE decided we should consume, in its order. */
   activeSpeakers: readonly string[];
+  /**
+   * The same set split by kind, because the two halves are treated
+   * differently: audio is never narrowed by a viewport, video may be. Derived
+   * from producer kinds when an older node sends no split, with unknown ids
+   * kept on the AUDIO side so nothing a node told us to take is ever narrowed
+   * away by our own ignorance.
+   */
+  activeAudio: readonly string[];
+  activeVideo: readonly string[];
+  /** The node's ranking with levels, loudest first. Meters and layouts. */
+  speakers: readonly SpeakerInfo[];
+  /** Whether a consume outside the set would actually be refused. */
+  activeSetEnforced: boolean;
+  /** Hands currently up, oldest first. Replaced wholesale by `joined`. */
+  raisedHands: readonly RaisedHand[];
   /** True while any `kind: "egress"` participant is present. A consent flag. */
   recording: boolean;
   /** The latest rule pushed for us, if one ever was. */
@@ -60,6 +90,11 @@ export const initialRoomState: RoomState = {
   peers: [],
   producers: [],
   activeSpeakers: [],
+  activeAudio: [],
+  activeVideo: [],
+  speakers: [],
+  activeSetEnforced: false,
+  raisedHands: [],
   recording: false,
   subscribeRule: null,
   draining: null,
@@ -94,7 +129,13 @@ export function reduceRoomState(state: RoomState, frame: EventFrame): RoomState
         room: frame.room,
         peers: frame.peers,
         producers,
-        activeSpeakers: pruneSpeakers(state.activeSpeakers, producers),
+        ...pruneActiveSets(state, producers),
+        // Carried like the set itself: the flag describes the set, and the
+        // room clears both on disconnect.
+        activeSetEnforced: state.activeSetEnforced,
+        // Wholesale, like everything else in the snapshot: a hand lowered
+        // while we were away must not survive the reconnect.
+        raisedHands: sortHands(frame.raisedHands ?? []),
         recording: frame.recording,
         subscribeRule: null,
         draining: null,
@@ -118,11 +159,15 @@ export function reduceRoomState(state: RoomState, frame: EventFrame): RoomState
       // guaranteed when a peer's socket simply dies, and a producer left behind
       // is a tile that never clears.
       const producers = state.producers.filter((p) => p.identity !== frame.identity);
+      const hands = state.raisedHands.filter((h) => h.identity !== frame.identity);
       return {
         ...state,
         peers: state.peers.filter((p) => p.identity !== frame.identity),
         producers,
-        activeSpeakers: pruneSpeakers(state.activeSpeakers, producers),
+        ...pruneActiveSets(state, producers),
+        // Their hand goes with them, whether or not the node's own
+        // `handChanged` for the departure ever arrives.
+        raisedHands: hands.length === state.raisedHands.length ? state.raisedHands : hands,
       };
     }
 
@@ -146,7 +191,7 @@ export function reduceRoomState(state: RoomState, frame: EventFrame): RoomState
     case "producerClosed": {
       if (!state.producers.some((p) => p.producerId === frame.producerId)) return state;
       const producers = state.producers.filter((p) => p.producerId !== frame.producerId);
-      return { ...state, producers, activeSpeakers: pruneSpeakers(state.activeSpeakers, producers) };
+      return { ...state, producers, ...pruneActiveSets(state, producers) };
     }
 
     case "producerPaused": {
@@ -159,11 +204,44 @@ export function reduceRoomState(state: RoomState, frame: EventFrame): RoomState
       return changed ? { ...state, producers } : state;
     }
 
-    case "activeSpeakers":
+    case "activeSpeakers": {
       // Stored as sent, INCLUDING ids we have not seen a `producerAppeared` for
       // yet. The node decides the active set; dropping an id because our view
       // is a frame behind would silently discard a stream we were told to take.
-      return { ...state, activeSpeakers: frame.producerIds };
+      const split = splitActiveSet(frame, state.producers);
+      const speakers = frame.speakers ?? [];
+      const enforced = frame.enforced ?? false;
+      // Value-equal frames return the SAME reference. The node ticks every
+      // 400ms and quantizes levels for exactly this comparison; without it a
+      // grid of tiles re-renders on every tick of a steady conversation.
+      if (
+        sameStrings(state.activeSpeakers, frame.producerIds) &&
+        sameStrings(state.activeAudio, split.audio) &&
+        sameStrings(state.activeVideo, split.video) &&
+        state.activeSetEnforced === enforced &&
+        sameSpeakers(state.speakers, speakers)
+      ) {
+        return state;
+      }
+      return {
+        ...state,
+        activeSpeakers: frame.producerIds,
+        activeAudio: split.audio,
+        activeVideo: split.video,
+        speakers,
+        activeSetEnforced: enforced,
+      };
+    }
+
+    case "handChanged": {
+      const without = state.raisedHands.filter((h) => h.identity !== frame.identity);
+      if (frame.raisedAt === null) {
+        return without.length === state.raisedHands.length ? state : { ...state, raisedHands: without };
+      }
+      const existing = state.raisedHands.find((h) => h.identity === frame.identity);
+      if (existing && existing.raisedAt === frame.raisedAt) return state;
+      return { ...state, raisedHands: sortHands([...without, { identity: frame.identity, raisedAt: frame.raisedAt }]) };
+    }
 
     case "recordingChanged":
       return state.recording === frame.recording ? state : { ...state, recording: frame.recording };
@@ -174,7 +252,17 @@ export function reduceRoomState(state: RoomState, frame: EventFrame): RoomState
     case "roomClosed":
       // Producers and speakers are gone for good; peers are kept so a caller can
       // still say who was in the room on the "call ended" screen.
-      return { ...state, phase: "closed", closedReason: frame.reason, producers: [], activeSpeakers: [] };
+      return {
+        ...state,
+        phase: "closed",
+        closedReason: frame.reason,
+        producers: [],
+        activeSpeakers: [],
+        activeAudio: [],
+        activeVideo: [],
+        speakers: [],
+        raisedHands: [],
+      };
 
     case "subscribeRuleChanged":
       /**
@@ -194,10 +282,7 @@ export function reduceRoomState(state: RoomState, frame: EventFrame): RoomState
       return {
         ...state,
         subscribeRule: frame.subscribe,
-        activeSpeakers: pruneSpeakers(
-          state.activeSpeakers,
-          visibleUnder(frame.subscribe, state.identity, state.producers),
-        ),
+        ...pruneActiveSets(state, visibleUnder(frame.subscribe, state.identity, state.producers)),
       };
 
     default:
@@ -255,6 +340,64 @@ function pruneSpeakers(speakers: readonly string[], producers: readonly Producer
   return kept.length === speakers.length ? speakers : kept;
 }
 
+/**
+ * Every active-set field pruned against `producers` in one move, each array
+ * keeping its reference when nothing fell out. Used everywhere a producer can
+ * disappear, so no field can be pruned in one place and forgotten in another.
+ */
+function pruneActiveSets(
+  state: Pick<RoomState, "activeSpeakers" | "activeAudio" | "activeVideo" | "speakers">,
+  producers: readonly ProducerEntry[],
+): Pick<RoomState, "activeSpeakers" | "activeAudio" | "activeVideo" | "speakers"> {
+  const live = new Set(producers.map((p) => p.producerId));
+  const keptSpeakers = state.speakers.filter((s) => live.has(s.producerId));
+  return {
+    activeSpeakers: pruneSpeakers(state.activeSpeakers, producers),
+    activeAudio: pruneSpeakers(state.activeAudio, producers),
+    activeVideo: pruneSpeakers(state.activeVideo, producers),
+    speakers: keptSpeakers.length === state.speakers.length ? state.speakers : keptSpeakers,
+  };
+}
+
+/**
+ * The audio/video split of an active-speakers frame.
+ *
+ * The frame's own split wins. An older node sends none; the fallback derives
+ * it from the producer kinds we know, and an id we know NOTHING about lands on
+ * the AUDIO side deliberately: audio is the side a viewport never narrows, so
+ * our own ignorance can delay a subscription but never suppress one.
+ */
+function splitActiveSet(
+  frame: { producerIds: readonly string[]; audio?: readonly string[]; video?: readonly string[] },
+  producers: readonly ProducerEntry[],
+): { audio: readonly string[]; video: readonly string[] } {
+  if (frame.audio || frame.video) {
+    return { audio: frame.audio ?? [], video: frame.video ?? [] };
+  }
+  const kinds = new Map(producers.map((p) => [p.producerId, p.kind]));
+  return {
+    audio: frame.producerIds.filter((id) => kinds.get(id) !== "video"),
+    video: frame.producerIds.filter((id) => kinds.get(id) === "video"),
+  };
+}
+
+function sameStrings(a: readonly string[], b: readonly string[]): boolean {
+  return a.length === b.length && a.every((value, index) => value === b[index]);
+}
+
+function sameSpeakers(a: readonly SpeakerInfo[], b: readonly SpeakerInfo[]): boolean {
+  return (
+    a.length === b.length &&
+    a.every(
+      (s, i) => s.identity === b[i]!.identity && s.producerId === b[i]!.producerId && s.volumeDb === b[i]!.volumeDb,
+    )
+  );
+}
+
+function sortHands(hands: readonly RaisedHand[]): readonly RaisedHand[] {
+  return [...hands].sort((a, b) => a.raisedAt - b.raisedAt || a.identity.localeCompare(b.identity));
+}
+
 /**
  * Asks the CONTRACT's decision function rather than reading the rule here.
  *

package/src/index.ts

@@ -34,8 +34,10 @@ export {
   type DisconnectCause,
   type MediaCoreCredentials,
   type ProducerEntry,
+  type RaisedHand,
   type RoomState,
   type SignalLogLevel,
+  type SpeakerInfo,
 } from "./core";
 
 /**

package/src/react/index.tsx

@@ -15,6 +15,8 @@ import type { MediaGrants, Peer } from "@tribe-nest/media-protocol";
 import { createBrowserDevice } from "../room/browserDevice";
 import {
   MediaRoom,
+  type BroadcastEvent,
+  type CallDiagnostics,
   type ConnectionState,
   type LocalPublication,
   type LocalPublicationSource,
@@ -228,6 +230,184 @@ export function useRecording(): boolean {
   return useRoomState().recording;
 }
 
+/**
+ * The node's speaker ranking with quantized levels, loudest first.
+ *
+ * Read this from the SMALLEST component that renders it (a meter inside a
+ * name badge), not from the grid: the list moves on the observer's tick, and
+ * a grid subscribed to it re-renders every tile a few times a second.
+ */
+export function useSpeakers(): RoomState["speakers"] {
+  return useRoomState().speakers;
+}
+
+/** Hands currently up, oldest first: the order IS the queue. */
+export function useRaisedHands(): RoomState["raisedHands"] {
+  return useRoomState().raisedHands;
+}
+
+/** Whether a consume outside the active set would actually be refused. */
+export function useActiveSetEnforced(): boolean {
+  return useRoomState().activeSetEnforced;
+}
+
+/**
+ * Incoming ephemeral broadcasts (reactions, courtesy lower-hands), as events.
+ *
+ * The handler lives in a ref, so an inline arrow does not resubscribe per
+ * render. Unknown `type` values are delivered; ignore what you do not know.
+ */
+export function useBroadcast(handler: (event: BroadcastEvent) => void): void {
+  const { room } = useRoomContext();
+  const handlerRef = useRef(handler);
+  handlerRef.current = handler;
+  useEffect(() => {
+    if (!room) return undefined;
+    return room.onBroadcast((event) => handlerRef.current(event));
+  }, [room]);
+}
+
+/**
+ * Poll the call's diagnostics while `enabled`: per tile, the layer asked for,
+ * the layer the node is giving, and the decoded picture; plus the node's
+ * congestion estimate and this side's publish health. Null until the first
+ * answer, and null again when disabled. One request a second against the
+ * node while open, which is why it is gated rather than always on.
+ */
+export function useCallDiagnostics(enabled: boolean, intervalMs = 1000): CallDiagnostics | null {
+  const { room } = useRoomContext();
+  const [diagnostics, setDiagnostics] = useState<CallDiagnostics | null>(null);
+  useEffect(() => {
+    if (!enabled || !room) {
+      setDiagnostics(null);
+      return undefined;
+    }
+    let cancelled = false;
+    const tick = () =>
+      void room
+        .getDiagnostics()
+        .then((d) => {
+          if (!cancelled) setDiagnostics(d);
+        })
+        .catch(() => undefined);
+    tick();
+    const timer = setInterval(tick, intervalMs);
+    return () => {
+      cancelled = true;
+      clearInterval(timer);
+    };
+  }, [enabled, room, intervalMs]);
+  return diagnostics;
+}
+
+export type { CallDiagnostics, TileDiagnostics, NodeDiagnostics } from "../room/room";
+
+export type TileViewportControls = {
+  /**
+   * A ref-callback factory: `<video ref={observeTile(producerId)} />`.
+   * Registers the element as rendering that producer; passing a null element
+   * (unmount) unregisters it.
+   */
+  observeTile: (producerId: string) => (element: HTMLElement | null) => void;
+};
+
+/**
+ * Report what the UI is actually rendering, so the room narrows its video
+ * subscriptions to it (audio never narrows) and picks simulcast layers from
+ * the real tile sizes.
+ *
+ * One shared IntersectionObserver and ResizeObserver for every tile, plus
+ * `visibilitychange` for the backgrounded tab. Where `IntersectionObserver`
+ * does not exist (jsdom, SSR) this never calls `setViewport`, which IS the
+ * legacy subscribe-everything behavior.
+ */
+export function useTileViewport(): TileViewportControls {
+  const { room } = useRoomContext();
+
+  const internals = useMemo(() => {
+    if (typeof IntersectionObserver === "undefined" || typeof ResizeObserver === "undefined") return null;
+    const byElement = new Map<Element, { producerId: string; visible: boolean; widthPx: number }>();
+    const report = () => {
+      if (!room) return;
+      if (typeof document !== "undefined" && document.visibilityState === "hidden") {
+        room.setViewport([]);
+        return;
+      }
+      const entries: { producerId: string; widthPx: number }[] = [];
+      for (const tile of byElement.values()) {
+        if (tile.visible) entries.push({ producerId: tile.producerId, widthPx: tile.widthPx });
+      }
+      room.setViewport(entries);
+    };
+    const intersection = new IntersectionObserver(
+      (observed) => {
+        for (const entry of observed) {
+          const tile = byElement.get(entry.target);
+          if (!tile) continue;
+          tile.visible = entry.isIntersecting;
+          if (entry.isIntersecting) tile.widthPx = Math.round(entry.boundingClientRect.width) || tile.widthPx;
+        }
+        report();
+      },
+      // A sliver of a tile at a page edge is not "rendering it".
+      { threshold: 0.05 },
+    );
+    const resize = new ResizeObserver((observed) => {
+      for (const entry of observed) {
+        const tile = byElement.get(entry.target);
+        if (!tile) continue;
+        tile.widthPx = Math.round(entry.contentRect.width) || tile.widthPx;
+      }
+      report();
+    });
+    return { byElement, report, intersection, resize };
+  }, [room]);
+
+  useEffect(() => {
+    if (!internals || !room) return undefined;
+    const onVisibility = () => internals.report();
+    document.addEventListener("visibilitychange", onVisibility);
+    return () => {
+      document.removeEventListener("visibilitychange", onVisibility);
+      internals.intersection.disconnect();
+      internals.resize.disconnect();
+      internals.byElement.clear();
+      // The tiles are gone with the component that rendered them; the room
+      // goes back to following the whole set rather than an empty viewport.
+      room.clearViewport();
+    };
+  }, [internals, room]);
+
+  const observeTile = useCallback(
+    (producerId: string) => (element: HTMLElement | null) => {
+      if (!internals) return;
+      if (element === null) {
+        for (const [el, tile] of internals.byElement) {
+          if (tile.producerId !== producerId) continue;
+          internals.intersection.unobserve(el);
+          internals.resize.unobserve(el);
+          internals.byElement.delete(el);
+        }
+        internals.report();
+        return;
+      }
+      internals.byElement.set(element, {
+        producerId,
+        // Visible until the observer's first callback says otherwise: the
+        // pessimistic default would pause every tile for a beat on mount.
+        visible: true,
+        widthPx: Math.round(element.getBoundingClientRect().width) || 320,
+      });
+      internals.intersection.observe(element);
+      internals.resize.observe(element);
+      internals.report();
+    },
+    [internals],
+  );
+
+  return { observeTile };
+}
+
 /**
  * One remote track, and a ref callback that attaches it.
  *
@@ -324,10 +504,24 @@ export function useRemoteTrack(producerId: string): {
 
 export type LocalSource = LocalPublicationSource;
 
+export type PublishDeviceOptions = {
+  /** Capture from this device rather than the browser's default. */
+  deviceId?: string;
+  /** Publish three spatial layers (VP8 only). For group calls whose viewers
+   * pick a layer per tile; wrong for a 1:1 where one encoding carries. */
+  simulcast?: boolean;
+};
+
 export type LocalMediaControls = {
-  publishCamera: () => Promise<void>;
-  publishMicrophone: () => Promise<void>;
+  publishCamera: (options?: PublishDeviceOptions) => Promise<void>;
+  publishMicrophone: (options?: PublishDeviceOptions) => Promise<void>;
   publishScreen: () => Promise<void>;
+  /**
+   * Swap the capture behind a live publication to another device, keeping the
+   * producer: remote tiles do not blink, and Safari is not re-prompted. No-op
+   * for a source that is not published.
+   */
+  switchDevice: (source: "camera" | "microphone", deviceId: string) => Promise<void>;
   unpublish: (source: LocalSource) => Promise<void>;
   /**
    * Mute or unmute a source that is already published, keeping its capture.
@@ -376,6 +570,24 @@ const selectLostSources = (room: MediaRoom): readonly LocalSource[] => room.lost
  * BROWSER concern with a UI consequence - a denied prompt is something a person
  * has to be told about - and the room has no way to say so.
  */
+/**
+ * The camera ask: 720p wanted, nothing REQUIRED.
+ *
+ * Without constraints the browser answers `getUserMedia({ video: true })`
+ * with 640x480 - the health panel caught a whole call whose "full" layer
+ * was SD 4:3 because of exactly this line. `ideal` rather than `exact`/`min`
+ * on purpose: a laptop whose camera tops out lower must still join the call,
+ * not be refused with OverconstrainedError over a resolution.
+ */
+function cameraConstraints(deviceId?: string): MediaTrackConstraints {
+  return {
+    width: { ideal: 1280 },
+    height: { ideal: 720 },
+    frameRate: { ideal: 30 },
+    ...(deviceId ? { deviceId: { exact: deviceId } } : {}),
+  };
+}
+
 export function useLocalMedia(): LocalMediaControls {
   const { room } = useRoomContext();
   const [error, setError] = useState<Error | undefined>(undefined);
@@ -444,7 +656,7 @@ export function useLocalMedia(): LocalMediaControls {
    * published).
    */
   const publish = useCallback(
-    async (source: LocalSource) =>
+    async (source: LocalSource, options?: PublishDeviceOptions) =>
       exclusively(source, async () => {
         if (!room) return;
         setError(undefined);
@@ -454,11 +666,15 @@ export function useLocalMedia(): LocalMediaControls {
           const stream =
             source === "screen"
               ? await navigator.mediaDevices.getDisplayMedia({ video: true })
-              : await navigator.mediaDevices.getUserMedia(source === "camera" ? { video: true } : { audio: true });
+              : await navigator.mediaDevices.getUserMedia(
+                  source === "camera"
+                    ? { video: cameraConstraints(options?.deviceId) }
+                    : { audio: options?.deviceId ? { deviceId: { exact: options.deviceId } } : true },
+                );
           captured = stream.getTracks();
           const track = captured[0];
           if (!track) throw new Error(`no ${source} track was captured`);
-          await room.publish(track, source);
+          await room.publish(track, source, options?.simulcast && source === "camera" ? { simulcast: true } : {});
           published = track;
         } catch (err) {
           // Surfaced rather than thrown: a denied permission prompt is an
@@ -473,6 +689,38 @@ export function useLocalMedia(): LocalMediaControls {
     [room, exclusively],
   );
 
+  // Same lock as publish: a switch pressed twice must not open two captures,
+  // and a switch racing a publish must not replace a track that is not there
+  // yet. The new capture is owned here until `replaceTrack` takes it.
+  const switchDevice = useCallback(
+    async (source: "camera" | "microphone", deviceId: string) =>
+      exclusively(source, async () => {
+        if (!room) return;
+        const publication = room.localPublications.find((p) => p.source === source);
+        if (!publication) return;
+        setError(undefined);
+        let captured: MediaStreamTrack[] = [];
+        let handedOver: MediaStreamTrack | undefined;
+        try {
+          const stream = await navigator.mediaDevices.getUserMedia(
+            source === "camera" ? { video: cameraConstraints(deviceId) } : { audio: { deviceId: { exact: deviceId } } },
+          );
+          captured = stream.getTracks();
+          const track = captured[0];
+          if (!track) throw new Error(`no ${source} track was captured`);
+          await room.replaceTrack(publication.producerId, track);
+          handedOver = track;
+        } catch (err) {
+          setError(err instanceof Error ? err : new Error(String(err)));
+        } finally {
+          for (const track of captured) {
+            if (track !== handedOver) track.stop();
+          }
+        }
+      }),
+    [room, exclusively],
+  );
+
   // Under the same lock as `publish`, because the pair is what a toggle
   // alternates between: a stop pressed while a start is still capturing would
   // otherwise find no publication to close and leave the capture that lands a
@@ -502,9 +750,10 @@ export function useLocalMedia(): LocalMediaControls {
   );
 
   return {
-    publishCamera: useCallback(() => publish("camera"), [publish]),
-    publishMicrophone: useCallback(() => publish("microphone"), [publish]),
+    publishCamera: useCallback((options?: PublishDeviceOptions) => publish("camera", options), [publish]),
+    publishMicrophone: useCallback((options?: PublishDeviceOptions) => publish("microphone", options), [publish]),
     publishScreen: useCallback(() => publish("screen"), [publish]),
+    switchDevice,
     unpublish,
     setPaused,
     isCameraEnabled: has("camera"),
@@ -529,6 +778,11 @@ const EMPTY_STATE: RoomState = {
   peers: [],
   producers: [],
   activeSpeakers: [],
+  activeAudio: [],
+  activeVideo: [],
+  speakers: [],
+  activeSetEnforced: false,
+  raisedHands: [],
   recording: false,
   subscribeRule: null,
   draining: null,