@memberjunction/ai-realtime-client 6.2.0-edge.2 → 6.2.0-edge.4

package/dist/media/mediaStage.d.ts

@@ -0,0 +1,99 @@
+/**
+ * @fileoverview MEDIA STAGE: a headless layout model. Participants, placeable surfaces and the user's moves go
+ * in; a layout comes out: one stage, a stack of picture-in-picture tiles, tabs, hidden surfaces, the other
+ * participants (for a filmstrip or grid), and a screen-share split. It knows nothing about Angular, the DOM,
+ * LiveKit or the realtime client, so the realtime overlay and the meeting room lay out the same way.
+ *
+ * Placement rules:
+ * - A surface goes where the user last moved it, if that placement is allowed for it; otherwise to its default.
+ * - **One stage.** The surface the user moved to the stage most recently wins it. A surface it displaces goes
+ *   to its own default placement, or to a PiP when its default is also the stage. With no move, the first
+ *   surface whose default is the stage keeps it, so a new surface never takes the stage on its own.
+ * - **PiPs stack** in one corner, newest first: the latest move first, then surfaces by when they appeared.
+ * - When no surface holds the stage, the spotlight participant does: a pin, then the active speaker, then a
+ *   speaking participant, then the agent, then the first other participant, then the user.
+ * - A participant already shown through a surface on the stage or in a PiP (the avatar, the user's camera) is not
+ *   repeated among the others, and one shown in a PiP does not take the spotlight: the next in line does.
+ *
+ * A host that lets the user move participants' tiles passes each as a surface ({@link ParticipantTileSurface}), as the
+ * meeting room does; a host with channel surfaces passes those, as the realtime call does.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+import type { MediaParticipant, MediaPlacement, MediaPlacementMove, MediaSurface } from './model.js';
+/** What the stage is laid out from. */
+export interface MediaStageInput {
+    /** Everyone in the session. The local user has `Role: 'self'`. */
+    Participants: readonly MediaParticipant[];
+    /** Identities the platform reports as speaking, most recent first (a meeting room's active-speaker list). */
+    ActiveSpeakers?: readonly string[];
+    /** Surfaces that can be placed, in the order they appeared. */
+    Surfaces?: readonly MediaSurface[];
+    /** The user's moves, oldest first. Only each surface's latest move counts, so a host may keep just those, in order. */
+    Moves?: readonly MediaPlacementMove[];
+    /** A participant the user pinned to the stage. */
+    PinnedIdentity?: string | null;
+    /** Whether the user's own tile is among the other participants. Defaults to true. */
+    ShowSelfView?: boolean;
+}
+/** One tile: a placed surface, or a participant. */
+export type MediaTile = {
+    Kind: 'surface';
+    Surface: MediaSurface;
+} | {
+    Kind: 'participant';
+    Participant: MediaParticipant;
+};
+/** Where everything goes. */
+export interface MediaStageLayout {
+    /** What fills the stage: a surface placed there, else the spotlight participant; null for an empty session. */
+    Stage: MediaTile | null;
+    /** Picture-in-picture surfaces, newest first. */
+    Pips: readonly MediaSurface[];
+    /** Surfaces shown as tabs, in the order they appeared. */
+    Tabs: readonly MediaSurface[];
+    /** Surfaces the user hid, so the UI can offer them back. */
+    Hidden: readonly MediaSurface[];
+    /** Participants not already on screen (on the stage or in a PiP), for a filmstrip or grid; the user first when shown. */
+    Others: readonly MediaParticipant[];
+    /** The screen being shared and the current speaker, for a split view; null when nobody is sharing. */
+    Split: {
+        Sharer: MediaParticipant;
+        Speaker: MediaParticipant | null;
+    } | null;
+}
+/** The key a participant's tile is moved under, so it never collides with a channel's surface key. */
+export declare function ParticipantTileKey(identity: string): string;
+/**
+ * A participant's tile as a surface the user can move: among the others (`tab`, a filmstrip or grid) until moved,
+ * showing the participant's camera, within the places the host offers (every placement when it names none).
+ */
+export declare function ParticipantTileSurface(participant: MediaParticipant, allowed?: readonly MediaPlacement[]): MediaSurface;
+/** Who fills the stage: the participant a surface on it shows (a tile moved there, an avatar), else the spotlight participant. */
+export declare function StageParticipantIdentity(stage: MediaTile | null): string | null;
+/** Lays out the stage. Pure: the same input always gives the same layout. */
+export declare function LayoutMediaStage(input: MediaStageInput): MediaStageLayout;
+/** Each surface's placement after the user's moves, with the one-stage and PiP-order rules applied. */
+export declare function ResolveSurfacePlacements(surfaces: readonly MediaSurface[], moves: readonly MediaPlacementMove[]): {
+    Stage: MediaSurface | null;
+    Pips: MediaSurface[];
+    Tabs: MediaSurface[];
+    Hidden: MediaSurface[];
+};
+/**
+ * The spotlight participant: a pin, then the active speaker, then a speaking participant, then the agent, then
+ * the first other participant, then the user.
+ */
+export declare function SelectSpotlight(participants: readonly MediaParticipant[], activeSpeakers: readonly string[], pinnedIdentity: string | null): MediaParticipant | null;
+/** The participants to show in a filmstrip or grid: the user first, when shown. */
+export declare function SelectDisplayParticipants(participants: readonly MediaParticipant[], showSelfView: boolean): MediaParticipant[];
+/** The first participant sharing a screen, if any. */
+export declare function SelectScreenSharer(participants: readonly MediaParticipant[]): MediaParticipant | null;
+/** The speaker pane beside a shared screen: the active speaker other than the sharer, then the agent, then the first other non-sharer, then the user. */
+export declare function SelectSplitSpeaker(participants: readonly MediaParticipant[], activeSpeakers: readonly string[]): MediaParticipant | null;
+/**
+ * Where a surface goes when it leaves the stage, displaced by another or sent off it: its default, or the first allowed of
+ * PiP, tab, hidden.
+ */
+export declare function PlacementOffStage(surface: MediaSurface): MediaPlacement;
+//# sourceMappingURL=mediaStage.d.ts.map

package/dist/media/mediaStage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mediaStage.d.ts","sourceRoot":"","sources":["../../src/media/mediaStage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,cAAc,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAGlG,uCAAuC;AACvC,MAAM,WAAW,eAAe;IAC5B,kEAAkE;IAClE,YAAY,EAAE,SAAS,gBAAgB,EAAE,CAAC;IAC1C,6GAA6G;IAC7G,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IACnC,uHAAuH;IACvH,KAAK,CAAC,EAAE,SAAS,kBAAkB,EAAE,CAAC;IACtC,kDAAkD;IAClD,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,qFAAqF;IACrF,YAAY,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,oDAAoD;AACpD,MAAM,MAAM,SAAS,GAAG;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,OAAO,EAAE,YAAY,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,WAAW,EAAE,gBAAgB,CAAA;CAAE,CAAC;AAE5H,6BAA6B;AAC7B,MAAM,WAAW,gBAAgB;IAC7B,+GAA+G;IAC/G,KAAK,EAAE,SAAS,GAAG,IAAI,CAAC;IACxB,iDAAiD;IACjD,IAAI,EAAE,SAAS,YAAY,EAAE,CAAC;IAC9B,0DAA0D;IAC1D,IAAI,EAAE,SAAS,YAAY,EAAE,CAAC;IAC9B,4DAA4D;IAC5D,MAAM,EAAE,SAAS,YAAY,EAAE,CAAC;IAChC,yHAAyH;IACzH,MAAM,EAAE,SAAS,gBAAgB,EAAE,CAAC;IACpC,sGAAsG;IACtG,KAAK,EAAE;QAAE,MAAM,EAAE,gBAAgB,CAAC;QAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAC;CAChF;AAED,sGAAsG;AACtG,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAE3D;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,WAAW,EAAE,gBAAgB,EAAE,OAAO,CAAC,EAAE,SAAS,cAAc,EAAE,GAAG,YAAY,CAQvH;AAED,kIAAkI;AAClI,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,SAAS,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAK/E;AAED,6EAA6E;AAC7E,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,eAAe,GAAG,gBAAgB,CAkBzE;AAED,uGAAuG;AACvG,wBAAgB,wBAAwB,CACpC,QAAQ,EAAE,SAAS,YAAY,EAAE,EACjC,KAAK,EAAE,SAAS,kBAAkB,EAAE,GACrC;IAAE,KAAK,EAAE,YAAY,GAAG,IAAI,CAAC;IAAC,IAAI,EAAE,YAAY,EAAE,CAAC;IAAC,IAAI,EAAE,YAAY,EAAE,CAAC;IAAC,MAAM,EAAE,YAAY,EAAE,CAAA;CAAE,CAoBpG;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAC3B,YAAY,EAAE,SAAS,gBAAgB,EAAE,EACzC,cAAc,EAAE,SAAS,MAAM,EAAE,EACjC,cAAc,EAAE,MAAM,GAAG,IAAI,GAC9B,gBAAgB,GAAG,IAAI,CAWzB;AAED,mFAAmF;AACnF,wBAAgB,yBAAyB,CAAC,YAAY,EAAE,SAAS,gBAAgB,EAAE,EAAE,YAAY,EAAE,OAAO,GAAG,gBAAgB,EAAE,CAI9H;AAED,sDAAsD;AACtD,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,SAAS,gBAAgB,EAAE,GAAG,gBAAgB,GAAG,IAAI,CAErG;AAED,yJAAyJ;AACzJ,wBAAgB,kBAAkB,CAAC,YAAY,EAAE,SAAS,gBAAgB,EAAE,EAAE,cAAc,EAAE,SAAS,MAAM,EAAE,GAAG,gBAAgB,GAAG,IAAI,CAQxI;AAuCD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,YAAY,GAAG,cAAc,CAKvE"}

package/dist/media/mediaStage.js

@@ -0,0 +1,171 @@
+/**
+ * @fileoverview MEDIA STAGE: a headless layout model. Participants, placeable surfaces and the user's moves go
+ * in; a layout comes out: one stage, a stack of picture-in-picture tiles, tabs, hidden surfaces, the other
+ * participants (for a filmstrip or grid), and a screen-share split. It knows nothing about Angular, the DOM,
+ * LiveKit or the realtime client, so the realtime overlay and the meeting room lay out the same way.
+ *
+ * Placement rules:
+ * - A surface goes where the user last moved it, if that placement is allowed for it; otherwise to its default.
+ * - **One stage.** The surface the user moved to the stage most recently wins it. A surface it displaces goes
+ *   to its own default placement, or to a PiP when its default is also the stage. With no move, the first
+ *   surface whose default is the stage keeps it, so a new surface never takes the stage on its own.
+ * - **PiPs stack** in one corner, newest first: the latest move first, then surfaces by when they appeared.
+ * - When no surface holds the stage, the spotlight participant does: a pin, then the active speaker, then a
+ *   speaking participant, then the agent, then the first other participant, then the user.
+ * - A participant already shown through a surface on the stage or in a PiP (the avatar, the user's camera) is not
+ *   repeated among the others, and one shown in a PiP does not take the spotlight: the next in line does.
+ *
+ * A host that lets the user move participants' tiles passes each as a surface ({@link ParticipantTileSurface}), as the
+ * meeting room does; a host with channel surfaces passes those, as the realtime call does.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+import { MEDIA_PLACEMENTS } from './model.js';
+/** The key a participant's tile is moved under, so it never collides with a channel's surface key. */
+export function ParticipantTileKey(identity) {
+    return `participant:${identity}`;
+}
+/**
+ * A participant's tile as a surface the user can move: among the others (`tab`, a filmstrip or grid) until moved,
+ * showing the participant's camera, within the places the host offers (every placement when it names none).
+ */
+export function ParticipantTileSurface(participant, allowed) {
+    return {
+        Key: ParticipantTileKey(participant.Identity),
+        Label: participant.DisplayName,
+        DefaultPlacement: 'tab',
+        ...(allowed ? { AllowedPlacements: allowed } : {}),
+        Video: { ParticipantIdentity: participant.Identity, Kind: 'camera' },
+    };
+}
+/** Who fills the stage: the participant a surface on it shows (a tile moved there, an avatar), else the spotlight participant. */
+export function StageParticipantIdentity(stage) {
+    if (stage?.Kind === 'surface') {
+        return stage.Surface.Video?.ParticipantIdentity ?? null;
+    }
+    return stage?.Participant.Identity ?? null;
+}
+/** Lays out the stage. Pure: the same input always gives the same layout. */
+export function LayoutMediaStage(input) {
+    const placed = ResolveSurfacePlacements(input.Surfaces ?? [], input.Moves ?? []);
+    const inPips = new Set(placed.Pips.flatMap((s) => (s.Video ? [s.Video.ParticipantIdentity] : [])));
+    const candidates = input.Participants.filter((p) => !inPips.has(p.Identity));
+    const spotlight = placed.Stage ? null : SelectSpotlight(candidates, input.ActiveSpeakers ?? [], input.PinnedIdentity ?? null);
+    const onScreen = new Set([placed.Stage, ...placed.Pips].flatMap((s) => (s?.Video ? [s.Video.ParticipantIdentity] : [])));
+    if (spotlight) {
+        onScreen.add(spotlight.Identity);
+    }
+    const shown = SelectDisplayParticipants(input.Participants, input.ShowSelfView ?? true);
+    return {
+        Stage: placed.Stage ? { Kind: 'surface', Surface: placed.Stage } : spotlight ? { Kind: 'participant', Participant: spotlight } : null,
+        Pips: placed.Pips,
+        Tabs: placed.Tabs,
+        Hidden: placed.Hidden,
+        Others: shown.filter((p) => !onScreen.has(p.Identity)),
+        Split: selectSplit(input.Participants, input.ActiveSpeakers ?? []),
+    };
+}
+/** Each surface's placement after the user's moves, with the one-stage and PiP-order rules applied. */
+export function ResolveSurfacePlacements(surfaces, moves) {
+    const latestMove = latestMoves(surfaces, moves);
+    const wanted = new Map(surfaces.map((s) => [s.Key, latestMove.get(s.Key)?.Placement ?? s.DefaultPlacement]));
+    const stage = pickStage(surfaces, wanted, latestMove);
+    const result = { Stage: stage, Pips: [], Tabs: [], Hidden: [] };
+    for (const surface of surfaces) {
+        if (surface === stage) {
+            continue;
+        }
+        const placement = wanted.get(surface.Key) === 'stage' ? PlacementOffStage(surface) : (wanted.get(surface.Key) ?? surface.DefaultPlacement);
+        if (placement === 'pip') {
+            result.Pips.push(surface);
+        }
+        else if (placement === 'tab') {
+            result.Tabs.push(surface);
+        }
+        else if (placement === 'hidden') {
+            result.Hidden.push(surface);
+        }
+    }
+    result.Pips.sort((a, b) => recency(b, surfaces, latestMove) - recency(a, surfaces, latestMove));
+    return result;
+}
+/**
+ * The spotlight participant: a pin, then the active speaker, then a speaking participant, then the agent, then
+ * the first other participant, then the user.
+ */
+export function SelectSpotlight(participants, activeSpeakers, pinnedIdentity) {
+    const pinned = pinnedIdentity ? participants.find((p) => p.Identity === pinnedIdentity) : undefined;
+    if (pinned) {
+        return pinned;
+    }
+    const others = participants.filter((p) => p.Role !== 'self');
+    const speakerId = activeSpeakers.find((id) => others.some((p) => p.Identity === id));
+    // The platform's active-speaker list can miss a server-published agent; a participant's own speaking flag
+    // catches it.
+    const speaking = (speakerId ? others.find((p) => p.Identity === speakerId) : undefined) ?? others.find((p) => p.IsSpeaking);
+    return speaking ?? others.find((p) => p.Role === 'agent') ?? others[0] ?? participants.find((p) => p.Role === 'self') ?? null;
+}
+/** The participants to show in a filmstrip or grid: the user first, when shown. */
+export function SelectDisplayParticipants(participants, showSelfView) {
+    const self = participants.filter((p) => p.Role === 'self');
+    const others = participants.filter((p) => p.Role !== 'self');
+    return showSelfView ? [...self, ...others] : others;
+}
+/** The first participant sharing a screen, if any. */
+export function SelectScreenSharer(participants) {
+    return participants.find((p) => p.Video.screen !== undefined) ?? null;
+}
+/** The speaker pane beside a shared screen: the active speaker other than the sharer, then the agent, then the first other non-sharer, then the user. */
+export function SelectSplitSpeaker(participants, activeSpeakers) {
+    const sharerId = SelectScreenSharer(participants)?.Identity;
+    const speakerId = activeSpeakers.find((id) => id !== sharerId && participants.some((p) => p.Identity === id));
+    const speaking = (speakerId ? participants.find((p) => p.Identity === speakerId) : undefined) ??
+        participants.find((p) => p.IsSpeaking && p.Identity !== sharerId);
+    const others = participants.filter((p) => p.Role !== 'self');
+    return speaking ?? others.find((p) => p.Role === 'agent') ?? others.find((p) => p.Identity !== sharerId) ?? participants.find((p) => p.Role === 'self') ?? null;
+}
+function selectSplit(participants, activeSpeakers) {
+    const sharer = SelectScreenSharer(participants);
+    return sharer ? { Sharer: sharer, Speaker: SelectSplitSpeaker(participants, activeSpeakers) } : null;
+}
+/** Whether a placement is allowed for a surface. */
+function allows(surface, placement) {
+    return !surface.AllowedPlacements || surface.AllowedPlacements.includes(placement);
+}
+/** Each live surface's latest allowed move, with its position in the move list (its recency). */
+function latestMoves(surfaces, moves) {
+    const byKey = new Map(surfaces.map((s) => [s.Key, s]));
+    const latest = new Map();
+    moves.forEach((move, order) => {
+        const surface = byKey.get(move.SurfaceKey);
+        if (surface && allows(surface, move.Placement)) {
+            latest.set(move.SurfaceKey, { Placement: move.Placement, Order: order });
+        }
+    });
+    return latest;
+}
+/** The stage holder: the latest move to the stage, else the first surface whose default is the stage. */
+function pickStage(surfaces, wanted, latestMove) {
+    const contenders = surfaces.filter((s) => wanted.get(s.Key) === 'stage');
+    const moved = contenders.filter((s) => latestMove.get(s.Key)?.Placement === 'stage');
+    if (moved.length > 0) {
+        return moved.reduce((a, b) => ((latestMove.get(b.Key)?.Order ?? -1) > (latestMove.get(a.Key)?.Order ?? -1) ? b : a));
+    }
+    return contenders[0] ?? null;
+}
+/**
+ * Where a surface goes when it leaves the stage, displaced by another or sent off it: its default, or the first allowed of
+ * PiP, tab, hidden.
+ */
+export function PlacementOffStage(surface) {
+    if (surface.DefaultPlacement !== 'stage' && allows(surface, surface.DefaultPlacement)) {
+        return surface.DefaultPlacement;
+    }
+    return MEDIA_PLACEMENTS.slice(1).find((p) => allows(surface, p)) ?? 'hidden';
+}
+/** Higher is newer: a move outranks appearance order, and a later move outranks an earlier one. */
+function recency(surface, surfaces, latestMove) {
+    const move = latestMove.get(surface.Key);
+    return move ? surfaces.length + move.Order : surfaces.indexOf(surface);
+}
+//# sourceMappingURL=mediaStage.js.map

package/dist/media/mediaStage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mediaStage.js","sourceRoot":"","sources":["../../src/media/mediaStage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAGH,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAqC3C,sGAAsG;AACtG,MAAM,UAAU,kBAAkB,CAAC,QAAgB;IAC/C,OAAO,eAAe,QAAQ,EAAE,CAAC;AACrC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,WAA6B,EAAE,OAAmC;IACrG,OAAO;QACH,GAAG,EAAE,kBAAkB,CAAC,WAAW,CAAC,QAAQ,CAAC;QAC7C,KAAK,EAAE,WAAW,CAAC,WAAW;QAC9B,gBAAgB,EAAE,KAAK;QACvB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClD,KAAK,EAAE,EAAE,mBAAmB,EAAE,WAAW,CAAC,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE;KACvE,CAAC;AACN,CAAC;AAED,kIAAkI;AAClI,MAAM,UAAU,wBAAwB,CAAC,KAAuB;IAC5D,IAAI,KAAK,EAAE,IAAI,KAAK,SAAS,EAAE,CAAC;QAC5B,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,mBAAmB,IAAI,IAAI,CAAC;IAC5D,CAAC;IACD,OAAO,KAAK,EAAE,WAAW,CAAC,QAAQ,IAAI,IAAI,CAAC;AAC/C,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,gBAAgB,CAAC,KAAsB;IACnD,MAAM,MAAM,GAAG,wBAAwB,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,EAAE,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IACjF,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACnG,MAAM,UAAU,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC7E,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,UAAU,EAAE,KAAK,CAAC,cAAc,IAAI,EAAE,EAAE,KAAK,CAAC,cAAc,IAAI,IAAI,CAAC,CAAC;IAC9H,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACzH,IAAI,SAAS,EAAE,CAAC;QACZ,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC;IACD,MAAM,KAAK,GAAG,yBAAyB,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC;IACxF,OAAO;QACH,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI;QACrI,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACtD,KAAK,EAAE,WAAW,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC;KACrE,CAAC;AACN,CAAC;AAED,uGAAuG;AACvG,MAAM,UAAU,wBAAwB,CACpC,QAAiC,EACjC,KAAoC;IAEpC,MAAM,UAAU,GAAG,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,SAAS,IAAI,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;IAC7G,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,EAAoB,EAAE,IAAI,EAAE,EAAoB,EAAE,MAAM,EAAE,EAAoB,EAAE,CAAC;IACtH,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC7B,IAAI,OAAO,KAAK,KAAK,EAAE,CAAC;YACpB,SAAS;QACb,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC3I,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;YACtB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9B,CAAC;aAAM,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9B,CAAC;aAAM,IAAI,SAAS,KAAK,QAAQ,EAAE,CAAC;YAChC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAChC,CAAC;IACL,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,CAAC,GAAG,OAAO,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC;IAChG,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC3B,YAAyC,EACzC,cAAiC,EACjC,cAA6B;IAE7B,MAAM,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACpG,IAAI,MAAM,EAAE,CAAC;QACT,OAAO,MAAM,CAAC;IAClB,CAAC;IACD,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;IAC7D,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC;IACrF,0GAA0G;IAC1G,cAAc;IACd,MAAM,QAAQ,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAC5H,OAAO,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC;AAClI,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,yBAAyB,CAAC,YAAyC,EAAE,YAAqB;IACtG,MAAM,IAAI,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;IAC3D,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;IAC7D,OAAO,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;AACxD,CAAC;AAED,sDAAsD;AACtD,MAAM,UAAU,kBAAkB,CAAC,YAAyC;IACxE,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,IAAI,IAAI,CAAC;AAC1E,CAAC;AAED,yJAAyJ;AACzJ,MAAM,UAAU,kBAAkB,CAAC,YAAyC,EAAE,cAAiC;IAC3G,MAAM,QAAQ,GAAG,kBAAkB,CAAC,YAAY,CAAC,EAAE,QAAQ,CAAC;IAC5D,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,KAAK,QAAQ,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC;IAC9G,MAAM,QAAQ,GACV,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC5E,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC;IACtE,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;IAC7D,OAAO,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC;AACpK,CAAC;AAED,SAAS,WAAW,CAAC,YAAyC,EAAE,cAAiC;IAC7F,MAAM,MAAM,GAAG,kBAAkB,CAAC,YAAY,CAAC,CAAC;IAChD,OAAO,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,kBAAkB,CAAC,YAAY,EAAE,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACzG,CAAC;AAED,oDAAoD;AACpD,SAAS,MAAM,CAAC,OAAqB,EAAE,SAAyB;IAC5D,OAAO,CAAC,OAAO,CAAC,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;AACvF,CAAC;AAED,iGAAiG;AACjG,SAAS,WAAW,CAAC,QAAiC,EAAE,KAAoC;IACxF,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwD,CAAC;IAC/E,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC1B,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC3C,IAAI,OAAO,IAAI,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7C,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QAC7E,CAAC;IACL,CAAC,CAAC,CAAC;IACH,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,yGAAyG;AACzG,SAAS,SAAS,CACd,QAAiC,EACjC,MAAmC,EACnC,UAAqE;IAErE,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,CAAC;IACzE,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,SAAS,KAAK,OAAO,CAAC,CAAC;IACrF,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnB,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzH,CAAC;IACD,OAAO,UAAU,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;AACjC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,OAAqB;IACnD,IAAI,OAAO,CAAC,gBAAgB,KAAK,OAAO,IAAI,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACpF,OAAO,OAAO,CAAC,gBAAgB,CAAC;IACpC,CAAC;IACD,OAAO,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC;AACjF,CAAC;AAED,mGAAmG;AACnG,SAAS,OAAO,CAAC,OAAqB,EAAE,QAAiC,EAAE,UAAqE;IAC5I,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACzC,OAAO,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;AAC3E,CAAC"}

package/dist/media/model.d.ts

@@ -0,0 +1,154 @@
+/**
+ * @fileoverview Shared types for the `/media` entry point: the local devices and capture state that the
+ * media controllers and the UI exchange.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+/** A kind of local device the user captures from. */
+export type LocalMediaKind = 'camera' | 'microphone';
+/** What a device does: a camera or a microphone the user captures from, or a speaker that plays sound. */
+export type MediaDeviceKind = LocalMediaKind | 'speaker';
+/** One device. */
+export interface MediaDevice {
+    /** The browser's id for the device, stable for this site until the user clears its data. */
+    DeviceID: string;
+    Kind: MediaDeviceKind;
+    /** The device's name. Browsers leave it empty until the user has allowed this site that kind of device. */
+    Label: string;
+    /** Devices on the same physical product (a webcam's camera and microphone) share a group id. */
+    GroupID: string;
+}
+/** Why a local device could not be used. */
+export type LocalMediaFailure = 
+/** The user, the operating system or the page's permissions policy refused access. */
+'denied'
+/** There is no device of that kind, or not the one asked for. */
+ | 'not-found'
+/** The device exists, but another application is using it. */
+ | 'in-use'
+/** The browser cannot capture at all (no `getUserMedia`). */
+ | 'unsupported'
+/** Anything else; see the message. */
+ | 'error';
+/** One kind of local capture. */
+export interface LocalTrackState {
+    Status: 'off' | 'starting' | 'on' | 'failed';
+    /** The device in use, while `on`. */
+    DeviceID?: string;
+    /** That device's name, while `on`. */
+    Label?: string;
+    /** Why capture failed, while `failed`. */
+    Failure?: LocalMediaFailure;
+    /** The browser's message for the failure. */
+    Message?: string;
+}
+/** Everything captured locally, as the `LocalMediaController` reports it. */
+export interface LocalMediaState {
+    Camera: LocalTrackState;
+    Microphone: LocalTrackState;
+    /** The input devices the browser lists now. */
+    Devices: MediaDevice[];
+}
+/** The outcome of starting or switching a local device. It always resolves; it never rejects. */
+export type LocalMediaResult = {
+    Status: 'started';
+    Stream: MediaStream;
+} | {
+    Status: 'failed';
+    Reason: LocalMediaFailure;
+    Message: string;
+};
+/**
+ * A source of video to show: a live `MediaStream` (a camera, a shared screen, a WebRTC track), or a player that
+ * must own the `<video>` element (MSE or WebCodecs playout of encoded video, such as an avatar). Renderers show
+ * either with `AttachVideoSource`.
+ */
+export type MediaVideoSource = 
+/** A live stream, shown muted: its audio is played elsewhere. */
+{
+    Kind: 'stream';
+    Stream: MediaStream;
+}
+/** A player that takes over the element and returns a function that releases it. */
+ | {
+    Kind: 'element';
+    Attach(element: HTMLVideoElement): () => void;
+};
+/**
+ * What a participant's video shows. Open vocabulary. These are on-screen kinds; the video source arbiter's
+ * `VideoSourceKind` is a different list, of what the model is shown.
+ */
+export type MediaSourceKind = 'camera' | 'screen' | 'avatar' | (string & {});
+/** Where a surface appears: the main stage, a picture-in-picture tile, a tab, or nowhere. */
+export type MediaPlacement = 'stage' | 'pip' | 'tab' | 'hidden';
+/** Every placement, in the order a displaced surface falls back through them. */
+export declare const MEDIA_PLACEMENTS: readonly MediaPlacement[];
+/** One person or agent in a session, as the media stage and its tiles see them. */
+export interface MediaParticipant {
+    /** Stable identity within the session. */
+    Identity: string;
+    DisplayName: string;
+    /** `'self'` is the local user. */
+    Role: 'self' | 'agent' | 'host' | 'participant';
+    /** Whether the participant is speaking now. */
+    IsSpeaking: boolean;
+    /** The participant's videos, by kind: a camera, a shared screen, an avatar. */
+    Video: Partial<Record<MediaSourceKind, MediaVideoSource>>;
+    /** Which video a tile shows when there are several. Defaults to a shared screen, then the camera, then the avatar. */
+    PreferredVideo?: MediaSourceKind;
+    /** Whether the participant's microphone is off. */
+    IsMuted?: boolean;
+    /** The connection's quality, when the platform reports one. */
+    ConnectionQuality?: MediaConnectionQuality;
+    /** Reads the participant's current audio level, 0..1, for a meter. Called on every animation frame. */
+    GetAudioLevel?: () => number;
+    /**
+     * Whether an agent can see this participant's camera and shared screen: they allowed it, and an agent is there to
+     * watch. A tile shows "Agent can see". Absent means no.
+     */
+    AgentCanSee?: boolean;
+}
+/** A participant's connection quality. */
+export type MediaConnectionQuality = 'excellent' | 'good' | 'poor' | 'lost' | 'unknown';
+/** The agent's conversational state, for an indicator. */
+export type MediaAgentState = 'idle' | 'listening' | 'thinking' | 'speaking';
+/** The state of a session's connection, for an overlay. */
+export type MediaConnectionStatus = 'idle' | 'connecting' | 'connected' | 'reconnecting' | 'disconnected' | 'error';
+/** Why a session ended. */
+export type MediaDisconnectReason = 'client-initiated' | 'server-shutdown' | 'participant-removed' | 'room-deleted' | 'connection-lost' | 'duplicate-identity' | 'unknown';
+/** A device the user picked in a device menu. */
+export interface MediaDeviceSelection {
+    Kind: MediaDeviceKind;
+    DeviceID: string;
+}
+/**
+ * Something that can be placed: a channel's surface (a whiteboard, a remote browser) or a participant's video
+ * (the avatar, the user's camera, a shared screen).
+ */
+export interface MediaSurface {
+    /** Stable key, usually the channel key. The user's moves are recorded against it. */
+    Key: string;
+    Label: string;
+    /** Where it goes unless the user moved it. */
+    DefaultPlacement: MediaPlacement;
+    /** Where the user may move it. Defaults to every placement. */
+    AllowedPlacements?: readonly MediaPlacement[];
+    /** The participant video it shows, when it is one; absent for a component surface such as a whiteboard. */
+    Video?: {
+        ParticipantIdentity: string;
+        Kind: MediaSourceKind;
+    };
+}
+/** One move the user made: a surface to a placement. */
+export interface MediaPlacementMove {
+    SurfaceKey: string;
+    Placement: MediaPlacement;
+}
+/** A picture-in-picture box as fractions of the stage (0 to 1), so it survives a resized window: what a host saves. */
+export interface MediaPipRect {
+    X: number;
+    Y: number;
+    W: number;
+    H: number;
+}
+//# sourceMappingURL=model.d.ts.map

package/dist/media/model.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../../src/media/model.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,qDAAqD;AACrD,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,YAAY,CAAC;AAErD,0GAA0G;AAC1G,MAAM,MAAM,eAAe,GAAG,cAAc,GAAG,SAAS,CAAC;AAEzD,kBAAkB;AAClB,MAAM,WAAW,WAAW;IACxB,4FAA4F;IAC5F,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,eAAe,CAAC;IACtB,2GAA2G;IAC3G,KAAK,EAAE,MAAM,CAAC;IACd,gGAAgG;IAChG,OAAO,EAAE,MAAM,CAAC;CACnB;AAED,4CAA4C;AAC5C,MAAM,MAAM,iBAAiB;AACzB,sFAAsF;AACpF,QAAQ;AACV,iEAAiE;GAC/D,WAAW;AACb,8DAA8D;GAC5D,QAAQ;AACV,6DAA6D;GAC3D,aAAa;AACf,sCAAsC;GACpC,OAAO,CAAC;AAEd,iCAAiC;AACjC,MAAM,WAAW,eAAe;IAC5B,MAAM,EAAE,KAAK,GAAG,UAAU,GAAG,IAAI,GAAG,QAAQ,CAAC;IAC7C,qCAAqC;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sCAAsC;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,OAAO,CAAC,EAAE,iBAAiB,CAAC;IAC5B,6CAA6C;IAC7C,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,6EAA6E;AAC7E,MAAM,WAAW,eAAe;IAC5B,MAAM,EAAE,eAAe,CAAC;IACxB,UAAU,EAAE,eAAe,CAAC;IAC5B,+CAA+C;IAC/C,OAAO,EAAE,WAAW,EAAE,CAAC;CAC1B;AAED,iGAAiG;AACjG,MAAM,MAAM,gBAAgB,GACtB;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,WAAW,CAAA;CAAE,GAC1C;IAAE,MAAM,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,iBAAiB,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvE;;;;GAIG;AACH,MAAM,MAAM,gBAAgB;AACxB,iEAAiE;AAC/D;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,WAAW,CAAA;CAAE;AACzC,oFAAoF;GAClF;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,MAAM,CAAC,OAAO,EAAE,gBAAgB,GAAG,MAAM,IAAI,CAAA;CAAE,CAAC;AAEzE;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;AAE7E,6FAA6F;AAC7F,MAAM,MAAM,cAAc,GAAG,OAAO,GAAG,KAAK,GAAG,KAAK,GAAG,QAAQ,CAAC;AAEhE,iFAAiF;AACjF,eAAO,MAAM,gBAAgB,EAAE,SAAS,cAAc,EAAsC,CAAC;AAE7F,mFAAmF;AACnF,MAAM,WAAW,gBAAgB;IAC7B,0CAA0C;IAC1C,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,kCAAkC;IAClC,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,GAAG,aAAa,CAAC;IAChD,+CAA+C;IAC/C,UAAU,EAAE,OAAO,CAAC;IACpB,+EAA+E;IAC/E,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,eAAe,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAC1D,sHAAsH;IACtH,cAAc,CAAC,EAAE,eAAe,CAAC;IACjC,mDAAmD;IACnD,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,+DAA+D;IAC/D,iBAAiB,CAAC,EAAE,sBAAsB,CAAC;IAC3C,uGAAuG;IACvG,aAAa,CAAC,EAAE,MAAM,MAAM,CAAC;IAC7B;;;OAGG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,0CAA0C;AAC1C,MAAM,MAAM,sBAAsB,GAAG,WAAW,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;AAExF,0DAA0D;AAC1D,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,WAAW,GAAG,UAAU,GAAG,UAAU,CAAC;AAE7E,2DAA2D;AAC3D,MAAM,MAAM,qBAAqB,GAAG,MAAM,GAAG,YAAY,GAAG,WAAW,GAAG,cAAc,GAAG,cAAc,GAAG,OAAO,CAAC;AAEpH,2BAA2B;AAC3B,MAAM,MAAM,qBAAqB,GAC3B,kBAAkB,GAClB,iBAAiB,GACjB,qBAAqB,GACrB,cAAc,GACd,iBAAiB,GACjB,oBAAoB,GACpB,SAAS,CAAC;AAEhB,iDAAiD;AACjD,MAAM,WAAW,oBAAoB;IACjC,IAAI,EAAE,eAAe,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IACzB,qFAAqF;IACrF,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,8CAA8C;IAC9C,gBAAgB,EAAE,cAAc,CAAC;IACjC,+DAA+D;IAC/D,iBAAiB,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAC9C,2GAA2G;IAC3G,KAAK,CAAC,EAAE;QAAE,mBAAmB,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,eAAe,CAAA;KAAE,CAAC;CAClE;AAED,wDAAwD;AACxD,MAAM,WAAW,kBAAkB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,cAAc,CAAC;CAC7B;AAED,uHAAuH;AACvH,MAAM,WAAW,YAAY;IACzB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;CACb"}

package/dist/media/model.js

@@ -0,0 +1,9 @@
+/**
+ * @fileoverview Shared types for the `/media` entry point: the local devices and capture state that the
+ * media controllers and the UI exchange.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+/** Every placement, in the order a displaced surface falls back through them. */
+export const MEDIA_PLACEMENTS = ['stage', 'pip', 'tab', 'hidden'];
+//# sourceMappingURL=model.js.map

package/dist/media/model.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"model.js","sourceRoot":"","sources":["../../src/media/model.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AA8EH,iFAAiF;AACjF,MAAM,CAAC,MAAM,gBAAgB,GAA8B,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC"}

package/dist/media/playbackClock.d.ts

@@ -0,0 +1,20 @@
+/**
+ * @fileoverview PLAYBACK CLOCK: where the voice is, so a video player can show each frame when the voice reaches it and
+ * the face moves with the voice.
+ *
+ * The voice is the master clock, as in the meeting bot: a driver that times both its PCM and its video frames on one media
+ * timeline gives the player its PCM playback (`RealtimePcmPlayback`) as `VideoPlayoutOptions.Clock`. The chunk and image
+ * decoders then show a frame when the clock reaches its `PresentationTimeMs`. MSE needs no clock: an MP4 that carries the
+ * voice plays voice and face together on its own timestamps.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+/** Where the voice is: the clock a video player shows frames against, so the face moves with the voice. */
+export interface IPlaybackClock {
+    /**
+     * The media time, in milliseconds on the stream's timeline, of the audio heard now; `null` while no timed audio plays
+     * (nothing is queued, or the audio queued carries no media times).
+     */
+    readonly CurrentTimeMs: number | null;
+}
+//# sourceMappingURL=playbackClock.d.ts.map

package/dist/media/playbackClock.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"playbackClock.d.ts","sourceRoot":"","sources":["../../src/media/playbackClock.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,2GAA2G;AAC3G,MAAM,WAAW,cAAc;IAC3B;;;OAGG;IACH,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CACzC"}

package/dist/media/playbackClock.js

@@ -0,0 +1,13 @@
+/**
+ * @fileoverview PLAYBACK CLOCK: where the voice is, so a video player can show each frame when the voice reaches it and
+ * the face moves with the voice.
+ *
+ * The voice is the master clock, as in the meeting bot: a driver that times both its PCM and its video frames on one media
+ * timeline gives the player its PCM playback (`RealtimePcmPlayback`) as `VideoPlayoutOptions.Clock`. The chunk and image
+ * decoders then show a frame when the clock reaches its `PresentationTimeMs`. MSE needs no clock: an MP4 that carries the
+ * voice plays voice and face together on its own timestamps.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+export {};
+//# sourceMappingURL=playbackClock.js.map

package/dist/media/playbackClock.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"playbackClock.js","sourceRoot":"","sources":["../../src/media/playbackClock.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG"}

package/dist/media/videoFrameDecoder.d.ts

@@ -0,0 +1,86 @@
+/**
+ * @fileoverview VIDEO FRAME DECODERS: what plays one kind of `RealtimeVideoFrame` into a `<video>` element for
+ * {@link VideoPlayout}, and how a decoder is offered to the player.
+ *
+ * The player keeps one decoder at a time. For each frame it picks, from the {@link VideoFrameDecoderRegistry}, the first
+ * registration of the frame's kind that can play its MIME type, and hands the decoder a {@link VideoFrameDecoderContext}:
+ * the player's settings and where to report. A decoder plays into the host's element, whatever it decodes with, so the
+ * host shows every kind of video the same way.
+ *
+ * Built in: `'mse-fmp4'` (fragmented MP4 through Media Source Extensions), `'webcodecs'` (encoded chunks through a
+ * WebCodecs `VideoDecoder`) and `'image'` (still images through `createImageBitmap`).
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+import type { RealtimeVideoFrame, RealtimeVideoFrameKind } from '@memberjunction/ai';
+import type { IPlaybackClock } from './playbackClock.js';
+import type { VideoPlayoutProblem } from './videoPlayout.js';
+/** What a decoder plays with and reports through. The player gives one to each decoder it creates. */
+export interface VideoFrameDecoderContext {
+    /**
+     * The player's configured type (`VideoPlayoutOptions.MimeType`), with its codecs: what a decoder that opens by type
+     * (MSE) opens with until a frame says more.
+     */
+    readonly MimeType: string;
+    /** Seconds of played media a buffering decoder keeps behind the playhead (`VideoPlayoutOptions.BackBufferSeconds`). */
+    readonly BackBufferSeconds: number;
+    /** Whether the element plays the media's own audio (an avatar MP4's voice). The player keeps it current. */
+    readonly CarriesVoice: boolean;
+    /**
+     * The element hook from `VideoPlayoutOptions.OnElementAttached`: the voice's route into Web Audio. A decoder whose media
+     * carries audio calls it with each element it takes over, before playback starts.
+     */
+    OnElementAttached?(element: HTMLVideoElement): void;
+    /**
+     * The voice's playback clock (`VideoPlayoutOptions.Clock`), when the player has one: a decoder that paces its own
+     * frames shows each when the clock reaches its `PresentationTimeMs`. A decoder whose media carries the voice ignores it.
+     */
+    readonly Clock?: IPlaybackClock;
+    /** Reports a problem. The player passes each kind on once. */
+    Report(problem: VideoPlayoutProblem, message: string): void;
+    /**
+     * The decoder can't go on (its browser decoder closed for good, or the browser can't decode the stream's codec). The
+     * player disposes it and picks the next decoder that can play the frame's type; out of decoders, frames of that type are
+     * dropped.
+     */
+    Failed(message: string): void;
+}
+/** Plays one kind of frame into a `<video>` element. The player creates one at a time and moves it between elements. */
+export interface IVideoFrameDecoder {
+    /** Takes the element over and shows its video there, keeping the frames it holds and the last one shown. */
+    Attach(element: HTMLVideoElement): void;
+    /** Lets the element go (the tile moved or closed). The decoder keeps what it holds for the next element. */
+    Detach(): void;
+    /** Takes one frame of its kind, in the order they arrived. */
+    Append(frame: RealtimeVideoFrame): void;
+    /** The turn's last frame has arrived: everything that came plays to the end, then the last frame holds. */
+    EndOfTurn(): void;
+    /** Barge-in: drops what isn't shown yet. The last frame shown stays on screen. */
+    Flush(): void;
+    /** Whether the decoder's own audio is audibly playing (an MP4 that carries the voice); `false` for decoders without audio. */
+    readonly IsPlaying: boolean;
+    /**
+     * How many of the frames it took are still to play: waiting, being decoded or appended, or buffered ahead of the
+     * playhead. A frame it dropped (a barge-in's flush, an overflow, a decode error) or played past is not counted.
+     */
+    readonly FramesAhead: number;
+    /** Stops, releases the element and frees what it holds. */
+    Dispose(): void;
+}
+/** One decoder the registry can create. */
+export interface VideoFrameDecoderRegistration {
+    /** Its name, unique in the registry: registering the same name again replaces it. Built in: `'mse-fmp4'`, `'webcodecs'`, `'image'`. */
+    Name: string;
+    /** The kind of frame it plays. */
+    Kind: RealtimeVideoFrameKind;
+    /** Among the decoders that can play a type, a higher priority is tried first. The built-ins have 0. */
+    Priority: number;
+    /**
+     * Whether it can play frames of this MIME type in this browser, answered at once. A decoder whose exact answer needs an
+     * asynchronous check answers from what it can see now, and calls `Failed` later if the check says no.
+     */
+    CanPlay(mimeType: string): boolean;
+    /** A new decoder, for one player. */
+    Create(context: VideoFrameDecoderContext): IVideoFrameDecoder;
+}
+//# sourceMappingURL=videoFrameDecoder.d.ts.map

package/dist/media/videoFrameDecoder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"videoFrameDecoder.d.ts","sourceRoot":"","sources":["../../src/media/videoFrameDecoder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,KAAK,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AACrF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAE1D,sGAAsG;AACtG,MAAM,WAAW,wBAAwB;IACrC;;;OAGG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,uHAAuH;IACvH,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,4GAA4G;IAC5G,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC;IAC/B;;;OAGG;IACH,iBAAiB,CAAC,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACpD;;;OAGG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC;IAChC,8DAA8D;IAC9D,MAAM,CAAC,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5D;;;;OAIG;IACH,MAAM,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACjC;AAED,wHAAwH;AACxH,MAAM,WAAW,kBAAkB;IAC/B,4GAA4G;IAC5G,MAAM,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACxC,4GAA4G;IAC5G,MAAM,IAAI,IAAI,CAAC;IACf,8DAA8D;IAC9D,MAAM,CAAC,KAAK,EAAE,kBAAkB,GAAG,IAAI,CAAC;IACxC,2GAA2G;IAC3G,SAAS,IAAI,IAAI,CAAC;IAClB,kFAAkF;IAClF,KAAK,IAAI,IAAI,CAAC;IACd,8HAA8H;IAC9H,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B;;;OAGG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,2DAA2D;IAC3D,OAAO,IAAI,IAAI,CAAC;CACnB;AAED,2CAA2C;AAC3C,MAAM,WAAW,6BAA6B;IAC1C,uIAAuI;IACvI,IAAI,EAAE,MAAM,CAAC;IACb,kCAAkC;IAClC,IAAI,EAAE,sBAAsB,CAAC;IAC7B,uGAAuG;IACvG,QAAQ,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IACnC,qCAAqC;IACrC,MAAM,CAAC,OAAO,EAAE,wBAAwB,GAAG,kBAAkB,CAAC;CACjE"}

package/dist/media/videoFrameDecoder.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=videoFrameDecoder.js.map

package/dist/media/videoFrameDecoder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"videoFrameDecoder.js","sourceRoot":"","sources":["../../src/media/videoFrameDecoder.ts"],"names":[],"mappings":""}