@memberjunction/realtime-runtime 6.2.0-edge.3 → 6.2.0-edge.5

package/dist/session/realtime-captures.d.ts

@@ -0,0 +1,236 @@
+/**
+ * @fileoverview The camera and the screen share of one realtime session: what the user chooses to show the agent
+ * besides the channels' own surfaces.
+ *
+ * A capture starts only when the user asks (a click in the host's UI) and in this order:
+ * 0. the session's policy must allow it ({@link RealtimeCapturesOptions.Admit}: the runtime admits a capture only while
+ *    the channel that fronts it is in the session and lets the agent see pixels);
+ * 1. the model must be able to take it: an inbound video track is already live, or one is added now
+ *    (`AddTrack`; a session never requests video before anyone shares, so it is not billed or limited for it);
+ * 2. the device or surface is opened: the camera through the host's `ILocalMediaController`, a screen, window or
+ *    tab through the host's `RequestDisplayCapture`;
+ * 3. frames are sampled at the rate the track negotiated, by the host's sampler or the DOM default, and it becomes a
+ *    source of the session's video source arbiter, which decides what the model sees and tells it when that changes.
+ *    A sampler that cannot start fails the capture as `unsupported`: the capture does not report on, and the model is
+ *    never told it can see it.
+ *
+ * With the camera check on ({@link RealtimeCapturesOptions.CameraCheck}), the first camera start waits between 2 and 3:
+ * the camera is open for the user to look at, and becomes a source only once the user confirms
+ * ({@link RealtimeCaptures.ConfirmCamera}).
+ *
+ * Stopping undoes all three. The track is removed again only when this class added it, no capture is left, and the
+ * arbiter has no other source: a video track the session was minted with (a channel's) stays, and so does one a channel
+ * started sending on while a capture was on. A camera that goes away (unplugged) or a share the user ends from the
+ * browser's own bar stops the capture the same way.
+ *
+ * @module @memberjunction/realtime-runtime
+ */
+import { type Observable } from 'rxjs';
+import { type BaseRealtimeClient, type CapturedDisplaySurface, type DisplayCaptureOptions, type FrameSamplerOptions, type ILocalMediaController, type MediaDevice } from '@memberjunction/ai-realtime-client';
+import type { IRealtimeFrameSampler, IRealtimeMediaHost } from '../hosts/IRealtimeMediaHost.js';
+/** What the user can show the agent besides a channel's surface. */
+export type RealtimeCaptureKind = 'camera' | 'screen';
+/** Why a capture did not start. */
+export type RealtimeCaptureFailure = 
+/** There is no live session to show it to. */
+'no-session'
+/** The model takes no (more) video, or the host cannot open this kind of capture. */
+ | 'unsupported'
+/** The user, the operating system or the page's permissions policy refused. */
+ | 'denied'
+/** There is no camera, or not the one asked for. */
+ | 'not-found'
+/** The camera exists, but another application is using it. */
+ | 'in-use'
+/** The user closed the browser's share picker without choosing. */
+ | 'cancelled'
+/** The call's policy does not allow it: the channel that fronts it is not in the call, or may not show the agent video. */
+ | 'policy'
+/** Anything else; see the message. */
+ | 'error';
+/** One capture. */
+export interface RealtimeCaptureState {
+    Status: 'off' | 'starting' | 'on' | 'failed';
+    /**
+     * The camera is open for the user to check, and the agent sees nothing until {@link RealtimeCaptures.ConfirmCamera}.
+     * Set only on a starting camera.
+     */
+    Checking?: boolean;
+    /**
+     * The live stream while on, or while the user checks the camera: the camera (show it mirrored) or the shared surface
+     * (show it as is).
+     */
+    Stream?: MediaStream;
+    /** What a screen share shows, while on. */
+    Surface?: CapturedDisplaySurface;
+    /**
+     * The shared panel's name, while a share of one panel of the page is on and the host named it
+     * (`DisplayCaptureOptions.PanelLabel`). The agent sees the share as "<name> (shared panel)".
+     */
+    PanelLabel?: string;
+    /** The camera in use, while the camera is open (being checked or on). */
+    DeviceID?: string;
+    /**
+     * The cameras the browser lists, while the camera is open: what {@link RealtimeCaptures.SwitchCamera} can move it to.
+     * A browser gives their names only once the user has allowed the camera.
+     */
+    Devices?: MediaDevice[];
+    /** Why the last start failed, while failed. */
+    Failure?: RealtimeCaptureFailure;
+    /** What to tell the user about the failure. */
+    Message?: string;
+}
+/** Both captures. */
+export interface RealtimeCaptureStates {
+    Camera: RealtimeCaptureState;
+    Screen: RealtimeCaptureState;
+}
+/** Which captures a call offers the user: those it could start now. */
+export interface RealtimeCaptureOffers {
+    Camera: boolean;
+    Screen: boolean;
+}
+/** Neither capture offered: what a host shows outside a call. */
+export declare const REALTIME_CAPTURE_OFFERS_NONE: RealtimeCaptureOffers;
+/**
+ * Creates the sampler for one capture, as {@link IRealtimeMediaHost.CreateFrameSampler} does: the runtime passes the
+ * host's; the default draws through the DOM; a test passes one that needs none.
+ */
+export type RealtimeFrameSamplerFactory = (stream: MediaStream, options: FrameSamplerOptions) => IRealtimeFrameSampler;
+/**
+ * The session's policy for one capture: refused with a reason for the user, or admitted with the channel it belongs to
+ * and whether the agent may see it yet.
+ */
+export type RealtimeCaptureAdmission = {
+    Admitted: false;
+    Message: string;
+} | {
+    Admitted: true;
+    /** The channel that fronts the capture: the user's "agent can see" choice for it is kept under this key. */
+    ChannelKey?: string;
+    /** Whether the agent may see it now. When not, it runs for the user only until {@link RealtimeCaptures.SetVisibleToAgent}. */
+    VisibleToAgent: boolean;
+};
+export interface RealtimeCapturesOptions {
+    /** The session's realtime client: the arbiter's sink and the owner of the video track. */
+    Client: BaseRealtimeClient;
+    /** The host's camera-and-microphone controller; without one there is no camera. */
+    LocalMedia: ILocalMediaController | null;
+    /** The host, for screen sharing (`RequestDisplayCapture`); without it there is no screen share. */
+    Host: IRealtimeMediaHost;
+    /**
+     * Creates each capture's frame sampler: the host's ({@link IRealtimeMediaHost.CreateFrameSampler}), which the runtime
+     * passes here. Defaults to a DOM {@link FrameSampler}, which cannot start without a DOM.
+     */
+    CreateSampler?: RealtimeFrameSamplerFactory;
+    /** Asked before each start, before anything is added or opened. Default: admitted, visible, with no channel. */
+    Admit?: (kind: RealtimeCaptureKind) => RealtimeCaptureAdmission;
+    /**
+     * Hold the first camera start for the user's check: the camera opens, and reaches the agent only after
+     * {@link RealtimeCaptures.ConfirmCamera}. Later starts skip the check. Default: off.
+     */
+    CameraCheck?: boolean;
+}
+/** Both captures off: what a host shows outside a session. */
+export declare const REALTIME_CAPTURES_OFF: RealtimeCaptureStates;
+/**
+ * The ids the camera and the screen share go by on the session's video source arbiter, and so on the runtime's
+ * `VideoSources$`; prefixed so they never meet a channel's (`Key#instance`). A host or channel finds a capture's source
+ * by its id there, to tell whether the model is being sent its frames (`Enabled` and `Active`).
+ */
+export declare const REALTIME_CAPTURE_SOURCE_IDS: Readonly<Record<RealtimeCaptureKind, string>>;
+/** The camera and the screen share of one session. Created once the session is connected; disposed at its end. */
+export declare class RealtimeCaptures {
+    private readonly options;
+    private readonly states;
+    private readonly live;
+    /** Bumped by every Stop, so a start that resolves after it knows it was cancelled. */
+    private readonly generation;
+    /** Whether this class added the session's inbound video track, and so may remove it. */
+    private addedVideoTrack;
+    /** Whether the user has confirmed the camera check; later camera starts skip it. */
+    private cameraChecked;
+    private disposed;
+    /** How each capture is shown to the agent: set at its start, kept up to date by {@link SetVisibleToAgent}. */
+    private readonly showing;
+    constructor(options: RealtimeCapturesOptions);
+    /** Both captures now. */
+    get States(): RealtimeCaptureStates;
+    /** Both captures, now and on every change. Completes on {@link Dispose}. */
+    get States$(): Observable<RealtimeCaptureStates>;
+    /**
+     * Starts a capture and shows it to the agent. Already starting or on, it returns where it is. Always resolves;
+     * a failure is a state, with a message for the user. A camera that waits for the user's check resolves still
+     * starting, with `Checking` set and the stream to preview.
+     *
+     * @param kind The camera or a screen share.
+     * @param options The camera's device id, or what the share picker offers first (and a panel to share).
+     */
+    Start(kind: 'camera', options?: {
+        DeviceID?: string;
+    }): Promise<RealtimeCaptureState>;
+    Start(kind: 'screen', options?: DisplayCaptureOptions): Promise<RealtimeCaptureState>;
+    /**
+     * The user checked the camera and turned it on: it is shown to the agent from now on, and later camera starts skip the
+     * check. Returns the camera's state; unless the camera is waiting for its check, nothing changes. When frames cannot be
+     * taken from the camera, it fails as `unsupported` and the camera is released.
+     */
+    ConfirmCamera(): RealtimeCaptureState;
+    /**
+     * Moves the open camera (being checked or on) to another of its {@link RealtimeCaptureState.Devices}. The stream stays
+     * the same, so the preview and the agent's frames carry on from the new camera. When the new camera cannot open, the
+     * host's controller goes back to the one in use; when that fails too, the camera stops. Resolves with the camera's
+     * state; nothing changes while the camera is not open.
+     */
+    SwitchCamera(deviceId: string): Promise<RealtimeCaptureState>;
+    /**
+     * Lets the agent see a capture, or hides it from the agent while it keeps running for the user (the policy of the
+     * channel that fronts it changed). Applies to a capture that is still starting too. Quiet: the channel tells the model.
+     */
+    SetVisibleToAgent(kind: RealtimeCaptureKind, visible: boolean): void;
+    /** Stops a capture: no more frames, out of the arbiter, the device or share released. Safe when it is off. */
+    Stop(kind: RealtimeCaptureKind): void;
+    /** Stops both captures and completes {@link States$}. */
+    Dispose(): void;
+    private stateOf;
+    private setState;
+    private fail;
+    /**
+     * Fails a capture whose device or share opened but whose frames cannot be taken for the agent (its sampler could not
+     * start): lets go of what opened, reports `unsupported`, and gives the track back.
+     */
+    private failToShow;
+    /** The camera in use and the cameras to choose from, as the host's controller reports them now. */
+    private cameraDevices;
+    /**
+     * Keeps the open camera's device and list current as the controller reports them: a switch, a camera plugged in or
+     * removed, names that appear once the user allows the camera. While a switch is under way the device in use stays.
+     */
+    private followCamera;
+    /**
+     * Makes sure an inbound video track is live: the one the session already has, or one added now.
+     *
+     * @returns Why the model cannot take video, or `null` when it can.
+     */
+    private ensureVideoTrack;
+    /**
+     * Removes the video track this class added, once no capture is on or starting and the arbiter has no other source.
+     * A channel that started sending on the track while a capture was on (one opened mid-call before it had a track of its
+     * own) is such a source: removing the track would leave it sending nothing, right after the model was told it can see
+     * that channel. The track then stays for the session, as a channel's own does.
+     */
+    private releaseVideoTrack;
+    /**
+     * Samples the capture at the negotiated rate and registers it with the arbiter under `label`, as its channel's and as
+     * visible as the policy allows. The sampler starts first: one that cannot start (no DOM, and no sampler from the host)
+     * leaves nothing registered, so the model is never told it can see a capture that would send it nothing.
+     *
+     * @returns The running sampler, or `null` when sampling could not start.
+     */
+    private showToAgent;
+    /** Opens the camera through the host's controller, and stops the capture if the camera goes away. */
+    private openCamera;
+    /** Asks the host for a screen, window, tab or panel, and stops the capture when the share ends. */
+    private openScreen;
+}
+//# sourceMappingURL=realtime-captures.d.ts.map

package/dist/session/realtime-captures.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"realtime-captures.d.ts","sourceRoot":"","sources":["../../src/session/realtime-captures.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EAAmB,KAAK,UAAU,EAAqB,MAAM,MAAM,CAAC;AAE3E,OAAO,EAGH,KAAK,kBAAkB,EACvB,KAAK,sBAAsB,EAE3B,KAAK,qBAAqB,EAC1B,KAAK,mBAAmB,EACxB,KAAK,qBAAqB,EAE1B,KAAK,WAAW,EACnB,MAAM,oCAAoC,CAAC;AAC5C,OAAO,KAAK,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAC;AAE7F,oEAAoE;AACpE,MAAM,MAAM,mBAAmB,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAEtD,mCAAmC;AACnC,MAAM,MAAM,sBAAsB;AAC9B,8CAA8C;AAC5C,YAAY;AACd,qFAAqF;GACnF,aAAa;AACf,+EAA+E;GAC7E,QAAQ;AACV,oDAAoD;GAClD,WAAW;AACb,8DAA8D;GAC5D,QAAQ;AACV,mEAAmE;GACjE,WAAW;AACb,2HAA2H;GACzH,QAAQ;AACV,sCAAsC;GACpC,OAAO,CAAC;AAEd,mBAAmB;AACnB,MAAM,WAAW,oBAAoB;IACjC,MAAM,EAAE,KAAK,GAAG,UAAU,GAAG,IAAI,GAAG,QAAQ,CAAC;IAC7C;;;OAGG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;OAGG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,2CAA2C;IAC3C,OAAO,CAAC,EAAE,sBAAsB,CAAC;IACjC;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IACxB,+CAA+C;IAC/C,OAAO,CAAC,EAAE,sBAAsB,CAAC;IACjC,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAqB;AACrB,MAAM,WAAW,qBAAqB;IAClC,MAAM,EAAE,oBAAoB,CAAC;IAC7B,MAAM,EAAE,oBAAoB,CAAC;CAChC;AAED,uEAAuE;AACvE,MAAM,WAAW,qBAAqB;IAClC,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;CACnB;AAED,iEAAiE;AACjE,eAAO,MAAM,4BAA4B,EAAE,qBAAuE,CAAC;AAEnH;;;GAGG;AACH,MAAM,MAAM,2BAA2B,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,mBAAmB,KAAK,qBAAqB,CAAC;AAEvH;;;GAGG;AACH,MAAM,MAAM,wBAAwB,GAC9B;IAAE,QAAQ,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACpC;IACI,QAAQ,EAAE,IAAI,CAAC;IACf,4GAA4G;IAC5G,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,8HAA8H;IAC9H,cAAc,EAAE,OAAO,CAAC;CAC3B,CAAC;AAER,MAAM,WAAW,uBAAuB;IACpC,0FAA0F;IAC1F,MAAM,EAAE,kBAAkB,CAAC;IAC3B,mFAAmF;IACnF,UAAU,EAAE,qBAAqB,GAAG,IAAI,CAAC;IACzC,mGAAmG;IACnG,IAAI,EAAE,kBAAkB,CAAC;IACzB;;;OAGG;IACH,aAAa,CAAC,EAAE,2BAA2B,CAAC;IAC5C,gHAAgH;IAChH,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,mBAAmB,KAAK,wBAAwB,CAAC;IAChE;;;OAGG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACzB;AAOD,8DAA8D;AAC9D,eAAO,MAAM,qBAAqB,EAAE,qBAAmE,CAAC;AAExG;;;;GAIG;AACH,eAAO,MAAM,2BAA2B,EAAE,QAAQ,CAAC,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAGpF,CAAC;AAyCH,kHAAkH;AAClH,qBAAa,gBAAgB;IAgBb,OAAO,CAAC,QAAQ,CAAC,OAAO;IAfpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqE;IAC5F,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAmF;IACxG,sFAAsF;IACtF,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAiE;IAC5F,wFAAwF;IACxF,OAAO,CAAC,eAAe,CAAS;IAChC,oFAAoF;IACpF,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAS;IACzB,8GAA8G;IAC9G,OAAO,CAAC,QAAQ,CAAC,OAAO,CAGtB;gBAE2B,OAAO,EAAE,uBAAuB;IAE7D,yBAAyB;IACzB,IAAW,MAAM,IAAI,qBAAqB,CAEzC;IAED,4EAA4E;IAC5E,IAAW,OAAO,IAAI,UAAU,CAAC,qBAAqB,CAAC,CAEtD;IAED;;;;;;;OAOG;IACI,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC;IACrF,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAiD5F;;;;OAIG;IACI,aAAa,IAAI,oBAAoB;IAe5C;;;;;OAKG;IACU,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAQ1E;;;OAGG;IACI,iBAAiB,CAAC,IAAI,EAAE,mBAAmB,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI;IAO3E,8GAA8G;IACvG,IAAI,CAAC,IAAI,EAAE,mBAAmB,GAAG,IAAI;IAe5C,yDAAyD;IAClD,OAAO,IAAI,IAAI;IAUtB,OAAO,CAAC,OAAO;IAIf,OAAO,CAAC,QAAQ;IAOhB,OAAO,CAAC,IAAI;IAIZ;;;OAGG;IACH,OAAO,CAAC,UAAU;IAQlB,mGAAmG;IACnG,OAAO,CAAC,aAAa;IAIrB;;;OAGG;IACH,OAAO,CAAC,YAAY;IAQpB;;;;OAIG;IACH,OAAO,CAAC,gBAAgB;IAaxB;;;;;OAKG;IACH,OAAO,CAAC,iBAAiB;IASzB;;;;;;OAMG;IACH,OAAO,CAAC,WAAW;IAiBnB,qGAAqG;YACvF,UAAU;IA0BxB,mGAAmG;YACrF,UAAU;CAyB3B"}

package/dist/session/realtime-captures.js

@@ -0,0 +1,383 @@
+/**
+ * @fileoverview The camera and the screen share of one realtime session: what the user chooses to show the agent
+ * besides the channels' own surfaces.
+ *
+ * A capture starts only when the user asks (a click in the host's UI) and in this order:
+ * 0. the session's policy must allow it ({@link RealtimeCapturesOptions.Admit}: the runtime admits a capture only while
+ *    the channel that fronts it is in the session and lets the agent see pixels);
+ * 1. the model must be able to take it: an inbound video track is already live, or one is added now
+ *    (`AddTrack`; a session never requests video before anyone shares, so it is not billed or limited for it);
+ * 2. the device or surface is opened: the camera through the host's `ILocalMediaController`, a screen, window or
+ *    tab through the host's `RequestDisplayCapture`;
+ * 3. frames are sampled at the rate the track negotiated, by the host's sampler or the DOM default, and it becomes a
+ *    source of the session's video source arbiter, which decides what the model sees and tells it when that changes.
+ *    A sampler that cannot start fails the capture as `unsupported`: the capture does not report on, and the model is
+ *    never told it can see it.
+ *
+ * With the camera check on ({@link RealtimeCapturesOptions.CameraCheck}), the first camera start waits between 2 and 3:
+ * the camera is open for the user to look at, and becomes a source only once the user confirms
+ * ({@link RealtimeCaptures.ConfirmCamera}).
+ *
+ * Stopping undoes all three. The track is removed again only when this class added it, no capture is left, and the
+ * arbiter has no other source: a video track the session was minted with (a channel's) stays, and so does one a channel
+ * started sending on while a capture was on. A camera that goes away (unplugged) or a share the user ends from the
+ * browser's own bar stops the capture the same way.
+ *
+ * @module @memberjunction/realtime-runtime
+ */
+import { BehaviorSubject } from 'rxjs';
+import { FrameSampler, VideoSourceArbiter, } from '@memberjunction/ai-realtime-client';
+/** Neither capture offered: what a host shows outside a call. */
+export const REALTIME_CAPTURE_OFFERS_NONE = Object.freeze({ Camera: false, Screen: false });
+const OFF = { Status: 'off' };
+/** What a capture gets when the session sets no policy. */
+const ADMITTED = { Admitted: true, VisibleToAgent: true };
+/** Both captures off: what a host shows outside a session. */
+export const REALTIME_CAPTURES_OFF = Object.freeze({ Camera: OFF, Screen: OFF });
+/**
+ * The ids the camera and the screen share go by on the session's video source arbiter, and so on the runtime's
+ * `VideoSources$`; prefixed so they never meet a channel's (`Key#instance`). A host or channel finds a capture's source
+ * by its id there, to tell whether the model is being sent its frames (`Enabled` and `Active`).
+ */
+export const REALTIME_CAPTURE_SOURCE_IDS = Object.freeze({
+    camera: 'capture:camera',
+    screen: 'capture:screen',
+});
+/** The names the model is told it is looking at, and the "agent can see" list shows. A shared panel goes by its own. */
+const LABELS = { camera: 'Camera', screen: 'Shared screen' };
+/** What the user is told when frames cannot be taken from a capture for the agent: its sampler could not start. */
+const CANNOT_SAMPLE = {
+    camera: 'This app cannot show the agent your camera.',
+    screen: 'This app cannot show the agent your screen.',
+};
+/** What the model and the "agent can see" list call a capture: a named shared panel by its name, as "Whiteboard (shared panel)". */
+function sourceLabel(kind, panelLabel) {
+    return panelLabel ? `${panelLabel} (shared panel)` : LABELS[kind];
+}
+/**
+ * The inbound video track a capture needs. No rate: the model's own ceiling applies. The user's click is the
+ * consent the track asks for.
+ */
+const CAPTURE_VIDEO_TRACK = {
+    Modality: 'video',
+    Direction: 'inbound',
+    Encoding: 'image/jpeg',
+    UsageBasis: ['tokens', 'frames'],
+    RequiresConsent: true,
+};
+/** The camera and the screen share of one session. Created once the session is connected; disposed at its end. */
+export class RealtimeCaptures {
+    constructor(options) {
+        this.options = options;
+        this.states = new BehaviorSubject(REALTIME_CAPTURES_OFF);
+        this.live = { camera: null, screen: null };
+        /** Bumped by every Stop, so a start that resolves after it knows it was cancelled. */
+        this.generation = { camera: 0, screen: 0 };
+        /** Whether this class added the session's inbound video track, and so may remove it. */
+        this.addedVideoTrack = false;
+        /** Whether the user has confirmed the camera check; later camera starts skip it. */
+        this.cameraChecked = false;
+        this.disposed = false;
+        /** How each capture is shown to the agent: set at its start, kept up to date by {@link SetVisibleToAgent}. */
+        this.showing = {
+            camera: { VisibleToAgent: true },
+            screen: { VisibleToAgent: true },
+        };
+    }
+    /** Both captures now. */
+    get States() {
+        return this.states.value;
+    }
+    /** Both captures, now and on every change. Completes on {@link Dispose}. */
+    get States$() {
+        return this.states.asObservable();
+    }
+    async Start(kind, options) {
+        const current = this.stateOf(kind);
+        if (this.disposed || current.Status === 'starting' || current.Status === 'on') {
+            return current;
+        }
+        const admission = this.options.Admit?.(kind) ?? ADMITTED;
+        if (!admission.Admitted) {
+            return this.fail(kind, 'policy', admission.Message);
+        }
+        this.showing[kind] = { ChannelKey: admission.ChannelKey, VisibleToAgent: admission.VisibleToAgent };
+        const generation = ++this.generation[kind];
+        this.setState(kind, { Status: 'starting' });
+        const refusal = this.ensureVideoTrack();
+        if (refusal) {
+            return this.fail(kind, 'unsupported', refusal);
+        }
+        const acquired = kind === 'camera' ? await this.openCamera(options) : await this.openScreen(options);
+        if (generation !== this.generation[kind]) {
+            // Stopped (or disposed) while opening: let go of what just opened.
+            if (acquired.Status === 'started') {
+                acquired.Release();
+            }
+            return this.stateOf(kind);
+        }
+        if (acquired.Status === 'failed') {
+            // Failed first, so this capture no longer counts as starting when the track is given back.
+            const failed = this.fail(kind, acquired.Failure, acquired.Message);
+            this.releaseVideoTrack();
+            return failed;
+        }
+        if (kind === 'camera' && this.options.CameraCheck && !this.cameraChecked) {
+            this.live.camera = { Stream: acquired.Stream, Sampler: null, Release: acquired.Release };
+            return this.setState('camera', { Status: 'starting', Checking: true, Stream: acquired.Stream, ...this.cameraDevices() });
+        }
+        const sampler = this.showToAgent(kind, acquired.Stream, sourceLabel(kind, acquired.PanelLabel));
+        if (!sampler) {
+            return this.failToShow(kind, acquired.Release);
+        }
+        this.live[kind] = { Stream: acquired.Stream, Sampler: sampler, Release: acquired.Release };
+        return this.setState(kind, {
+            Status: 'on',
+            Stream: acquired.Stream,
+            ...(acquired.Surface ? { Surface: acquired.Surface } : {}),
+            ...(acquired.PanelLabel ? { PanelLabel: acquired.PanelLabel } : {}),
+            ...(kind === 'camera' ? this.cameraDevices() : {}),
+        });
+    }
+    /**
+     * The user checked the camera and turned it on: it is shown to the agent from now on, and later camera starts skip the
+     * check. Returns the camera's state; unless the camera is waiting for its check, nothing changes. When frames cannot be
+     * taken from the camera, it fails as `unsupported` and the camera is released.
+     */
+    ConfirmCamera() {
+        const live = this.live.camera;
+        if (!live || !this.States.Camera.Checking) {
+            return this.States.Camera;
+        }
+        const sampler = this.showToAgent('camera', live.Stream, LABELS.camera);
+        if (!sampler) {
+            this.live.camera = null;
+            return this.failToShow('camera', live.Release);
+        }
+        this.cameraChecked = true;
+        live.Sampler = sampler;
+        return this.setState('camera', { Status: 'on', Stream: live.Stream, ...this.cameraDevices() });
+    }
+    /**
+     * Moves the open camera (being checked or on) to another of its {@link RealtimeCaptureState.Devices}. The stream stays
+     * the same, so the preview and the agent's frames carry on from the new camera. When the new camera cannot open, the
+     * host's controller goes back to the one in use; when that fails too, the camera stops. Resolves with the camera's
+     * state; nothing changes while the camera is not open.
+     */
+    async SwitchCamera(deviceId) {
+        const controller = this.options.LocalMedia;
+        if (this.live.camera && controller) {
+            await controller.SwitchDevice('camera', deviceId);
+        }
+        return this.States.Camera;
+    }
+    /**
+     * Lets the agent see a capture, or hides it from the agent while it keeps running for the user (the policy of the
+     * channel that fronts it changed). Applies to a capture that is still starting too. Quiet: the channel tells the model.
+     */
+    SetVisibleToAgent(kind, visible) {
+        this.showing[kind] = { ...this.showing[kind], VisibleToAgent: visible };
+        if (this.live[kind]) {
+            VideoSourceArbiter.ForSink(this.options.Client).SetSourceEnabled(REALTIME_CAPTURE_SOURCE_IDS[kind], visible, false);
+        }
+    }
+    /** Stops a capture: no more frames, out of the arbiter, the device or share released. Safe when it is off. */
+    Stop(kind) {
+        this.generation[kind]++;
+        const live = this.live[kind];
+        this.live[kind] = null;
+        if (live) {
+            live.Sampler?.Stop();
+            VideoSourceArbiter.ForSink(this.options.Client).UnregisterSource(REALTIME_CAPTURE_SOURCE_IDS[kind]);
+            live.Release();
+        }
+        if (this.stateOf(kind).Status !== 'off') {
+            this.setState(kind, OFF);
+        }
+        this.releaseVideoTrack();
+    }
+    /** Stops both captures and completes {@link States$}. */
+    Dispose() {
+        if (this.disposed) {
+            return;
+        }
+        this.Stop('camera');
+        this.Stop('screen');
+        this.disposed = true;
+        this.states.complete();
+    }
+    stateOf(kind) {
+        return kind === 'camera' ? this.states.value.Camera : this.states.value.Screen;
+    }
+    setState(kind, state) {
+        if (!this.disposed) {
+            this.states.next(kind === 'camera' ? { ...this.states.value, Camera: state } : { ...this.states.value, Screen: state });
+        }
+        return state;
+    }
+    fail(kind, failure, message) {
+        return this.setState(kind, { Status: 'failed', Failure: failure, Message: message });
+    }
+    /**
+     * Fails a capture whose device or share opened but whose frames cannot be taken for the agent (its sampler could not
+     * start): lets go of what opened, reports `unsupported`, and gives the track back.
+     */
+    failToShow(kind, release) {
+        release();
+        // Failed first, so this capture no longer counts as starting when the track is given back.
+        const failed = this.fail(kind, 'unsupported', CANNOT_SAMPLE[kind]);
+        this.releaseVideoTrack();
+        return failed;
+    }
+    /** The camera in use and the cameras to choose from, as the host's controller reports them now. */
+    cameraDevices() {
+        return cameraDevicesOf(this.options.LocalMedia?.State);
+    }
+    /**
+     * Keeps the open camera's device and list current as the controller reports them: a switch, a camera plugged in or
+     * removed, names that appear once the user allows the camera. While a switch is under way the device in use stays.
+     */
+    followCamera(media) {
+        const current = this.States.Camera;
+        const next = cameraDevicesOf(media, current.DeviceID);
+        if (next.DeviceID !== current.DeviceID || !sameDevices(next.Devices, current.Devices ?? [])) {
+            this.setState('camera', { ...current, ...next });
+        }
+    }
+    /**
+     * Makes sure an inbound video track is live: the one the session already has, or one added now.
+     *
+     * @returns Why the model cannot take video, or `null` when it can.
+     */
+    ensureVideoTrack() {
+        const client = this.options.Client;
+        if (client.IsTrackEstablished('video', 'inbound')) {
+            return null;
+        }
+        const track = client.AddTrack(CAPTURE_VIDEO_TRACK);
+        if (!track || track.State !== 'live') {
+            return track?.Reason ?? 'This model cannot see video.';
+        }
+        this.addedVideoTrack = true;
+        return null;
+    }
+    /**
+     * Removes the video track this class added, once no capture is on or starting and the arbiter has no other source.
+     * A channel that started sending on the track while a capture was on (one opened mid-call before it had a track of its
+     * own) is such a source: removing the track would leave it sending nothing, right after the model was told it can see
+     * that channel. The track then stays for the session, as a channel's own does.
+     */
+    releaseVideoTrack() {
+        const busy = [this.states.value.Camera, this.states.value.Screen].some((s) => s.Status === 'on' || s.Status === 'starting');
+        if (!this.addedVideoTrack || busy || VideoSourceArbiter.ForSink(this.options.Client).GetSources().length > 0) {
+            return;
+        }
+        this.addedVideoTrack = false;
+        this.options.Client.RemoveTrack(CAPTURE_VIDEO_TRACK);
+    }
+    /**
+     * Samples the capture at the negotiated rate and registers it with the arbiter under `label`, as its channel's and as
+     * visible as the policy allows. The sampler starts first: one that cannot start (no DOM, and no sampler from the host)
+     * leaves nothing registered, so the model is never told it can see a capture that would send it nothing.
+     *
+     * @returns The running sampler, or `null` when sampling could not start.
+     */
+    showToAgent(kind, stream, label) {
+        const arbiter = VideoSourceArbiter.ForSink(this.options.Client);
+        const sourceId = REALTIME_CAPTURE_SOURCE_IDS[kind];
+        const create = this.options.CreateSampler ?? createDomSampler;
+        const sampler = create(stream, {
+            Rate: this.options.Client.InboundVideoRate ?? 1,
+            OnFrame: (frame) => arbiter.PushFrame(sourceId, frame.Data, frame.MimeType),
+        });
+        if (!sampler.Start()) {
+            sampler.Stop();
+            return null;
+        }
+        const { ChannelKey, VisibleToAgent } = this.showing[kind];
+        arbiter.RegisterSource({ SourceID: sourceId, Label: label, Kind: kind, Enabled: VisibleToAgent, ...(ChannelKey ? { ChannelKey } : {}) });
+        return sampler;
+    }
+    /** Opens the camera through the host's controller, and stops the capture if the camera goes away. */
+    async openCamera(options) {
+        const controller = this.options.LocalMedia;
+        if (!controller) {
+            return { Status: 'failed', Failure: 'unsupported', Message: 'This app cannot open a camera.' };
+        }
+        const result = await controller.Start('camera', options?.DeviceID);
+        if (result.Status === 'failed') {
+            return { Status: 'failed', Failure: result.Reason, Message: result.Message };
+        }
+        const watch = controller.State$.subscribe((state) => {
+            if (state.Camera.Status === 'off' || state.Camera.Status === 'failed') {
+                this.Stop('camera');
+            }
+            else if (this.live.camera) {
+                this.followCamera(state);
+            }
+        });
+        return {
+            Status: 'started',
+            Stream: result.Stream,
+            Release: () => {
+                watch.unsubscribe();
+                controller.Stop('camera');
+            },
+        };
+    }
+    /** Asks the host for a screen, window, tab or panel, and stops the capture when the share ends. */
+    async openScreen(options) {
+        const host = this.options.Host;
+        if (!host.RequestDisplayCapture) {
+            return { Status: 'failed', Failure: 'unsupported', Message: 'This app cannot share a screen.' };
+        }
+        const result = await host.RequestDisplayCapture(options);
+        if (result.Status === 'cancelled') {
+            return { Status: 'failed', Failure: 'cancelled', Message: 'Sharing was cancelled.' };
+        }
+        if (result.Status === 'failed') {
+            return { Status: 'failed', Failure: screenFailure(result.Reason), Message: result.Message };
+        }
+        const capture = result.Capture;
+        const stopWatching = capture.OnEnded(() => this.Stop('screen'));
+        return {
+            Status: 'started',
+            Stream: capture.Stream,
+            Surface: capture.Surface,
+            ...(capture.PanelLabel ? { PanelLabel: capture.PanelLabel } : {}),
+            Release: () => {
+                stopWatching();
+                capture.Stop();
+            },
+        };
+    }
+}
+/**
+ * The camera in use and the cameras among the devices, as a controller reports them. The camera in use is known while
+ * the controller reports it on; otherwise it is `inUse`, the one the capture already names.
+ */
+function cameraDevicesOf(media, inUse) {
+    const deviceId = (media?.Camera.Status === 'on' ? media.Camera.DeviceID : undefined) ?? inUse;
+    return { ...(deviceId ? { DeviceID: deviceId } : {}), Devices: media?.Devices.filter((d) => d.Kind === 'camera') ?? [] };
+}
+/** Whether two lists name the same devices in the same order. */
+function sameDevices(a, b) {
+    return a.length === b.length && a.every((d, i) => d.DeviceID === b[i].DeviceID && d.Label === b[i].Label);
+}
+/** The default sampler: a DOM {@link FrameSampler}, whose `Start()` returns `false` without a DOM. */
+function createDomSampler(stream, options) {
+    return new FrameSampler(stream, options);
+}
+/** A share failure as a capture failure: a panel the browser cannot narrow to is unsupported; a wrong one an error. */
+function screenFailure(reason) {
+    switch (reason) {
+        case 'unsupported':
+        case 'panel-unsupported':
+            return 'unsupported';
+        case 'denied':
+            return 'denied';
+        default:
+            return 'error';
+    }
+}
+//# sourceMappingURL=realtime-captures.js.map

package/dist/session/realtime-captures.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"realtime-captures.js","sourceRoot":"","sources":["../../src/session/realtime-captures.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EAAE,eAAe,EAAsC,MAAM,MAAM,CAAC;AAE3E,OAAO,EACH,YAAY,EACZ,kBAAkB,GASrB,MAAM,oCAAoC,CAAC;AAsE5C,iEAAiE;AACjE,MAAM,CAAC,MAAM,4BAA4B,GAA0B,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;AA2CnH,MAAM,GAAG,GAAyB,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAEpD,2DAA2D;AAC3D,MAAM,QAAQ,GAA6B,EAAE,QAAQ,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC;AAEpF,8DAA8D;AAC9D,MAAM,CAAC,MAAM,qBAAqB,GAA0B,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;AAExG;;;;GAIG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAkD,MAAM,CAAC,MAAM,CAAC;IACpG,MAAM,EAAE,gBAAgB;IACxB,MAAM,EAAE,gBAAgB;CAC3B,CAAC,CAAC;AAEH,wHAAwH;AACxH,MAAM,MAAM,GAAwC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;AAElG,mHAAmH;AACnH,MAAM,aAAa,GAAwC;IACvD,MAAM,EAAE,6CAA6C;IACrD,MAAM,EAAE,6CAA6C;CACxD,CAAC;AAEF,oIAAoI;AACpI,SAAS,WAAW,CAAC,IAAyB,EAAE,UAA8B;IAC1E,OAAO,UAAU,CAAC,CAAC,CAAC,GAAG,UAAU,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AACtE,CAAC;AAED;;;GAGG;AACH,MAAM,mBAAmB,GAA4B;IACjD,QAAQ,EAAE,OAAO;IACjB,SAAS,EAAE,SAAS;IACpB,QAAQ,EAAE,YAAY;IACtB,UAAU,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAChC,eAAe,EAAE,IAAI;CACxB,CAAC;AAeF,kHAAkH;AAClH,MAAM,OAAO,gBAAgB;IAgBzB,YAA6B,OAAgC;QAAhC,YAAO,GAAP,OAAO,CAAyB;QAf5C,WAAM,GAAG,IAAI,eAAe,CAAwB,qBAAqB,CAAC,CAAC;QAC3E,SAAI,GAAoD,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACxG,sFAAsF;QACrE,eAAU,GAAwC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;QAC5F,wFAAwF;QAChF,oBAAe,GAAG,KAAK,CAAC;QAChC,oFAAoF;QAC5E,kBAAa,GAAG,KAAK,CAAC;QACtB,aAAQ,GAAG,KAAK,CAAC;QACzB,8GAA8G;QAC7F,YAAO,GAAkF;YACtG,MAAM,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE;YAChC,MAAM,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE;SACnC,CAAC;IAE8D,CAAC;IAEjE,yBAAyB;IACzB,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;IAC7B,CAAC;IAED,4EAA4E;IAC5E,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;IACtC,CAAC;IAYM,KAAK,CAAC,KAAK,CAAC,IAAyB,EAAE,OAAuD;QACjG,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,MAAM,KAAK,UAAU,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;YAC5E,OAAO,OAAO,CAAC;QACnB,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC;QACzD,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC;QACxD,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,SAAS,CAAC,UAAU,EAAE,cAAc,EAAE,SAAS,CAAC,cAAc,EAAE,CAAC;QACpG,MAAM,UAAU,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxC,IAAI,OAAO,EAAE,CAAC;YACV,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,aAAa,EAAE,OAAO,CAAC,CAAC;QACnD,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,OAA4C,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,OAA4C,CAAC,CAAC;QAC/K,IAAI,UAAU,KAAK,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACvC,mEAAmE;YACnE,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAChC,QAAQ,CAAC,OAAO,EAAE,CAAC;YACvB,CAAC;YACD,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/B,2FAA2F;YAC3F,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;YACnE,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACzB,OAAO,MAAM,CAAC;QAClB,CAAC;QACD,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACvE,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC;YACzF,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;QAC7H,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,IAAI,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC;QAChG,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;QACnD,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC;QAC3F,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;YACvB,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1D,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,GAAG,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrD,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACI,aAAa;QAChB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACxC,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAC9B,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QACvE,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YACxB,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QACnD,CAAC;QACD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;IACnG,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,YAAY,CAAC,QAAgB;QACtC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC;QAC3C,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;YACjC,MAAM,UAAU,CAAC,YAAY,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC9B,CAAC;IAED;;;OAGG;IACI,iBAAiB,CAAC,IAAyB,EAAE,OAAgB;QAChE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC;QACxE,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAClB,kBAAkB,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,gBAAgB,CAAC,2BAA2B,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QACxH,CAAC;IACL,CAAC;IAED,8GAA8G;IACvG,IAAI,CAAC,IAAyB;QACjC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;QACvB,IAAI,IAAI,EAAE,CAAC;YACP,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;YACrB,kBAAkB,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,gBAAgB,CAAC,2BAA2B,CAAC,IAAI,CAAC,CAAC,CAAC;YACpG,IAAI,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;YACtC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC7B,CAAC;QACD,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC7B,CAAC;IAED,yDAAyD;IAClD,OAAO;QACV,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,OAAO;QACX,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;IAC3B,CAAC;IAEO,OAAO,CAAC,IAAyB;QACrC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;IACnF,CAAC;IAEO,QAAQ,CAAC,IAAyB,EAAE,KAA2B;QACnE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACjB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAC5H,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,IAAI,CAAC,IAAyB,EAAE,OAA+B,EAAE,OAAe;QACpF,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IACzF,CAAC;IAED;;;OAGG;IACK,UAAU,CAAC,IAAyB,EAAE,OAAmB;QAC7D,OAAO,EAAE,CAAC;QACV,2FAA2F;QAC3F,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,aAAa,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;QACnE,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACzB,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,mGAAmG;IAC3F,aAAa;QACjB,OAAO,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAC3D,CAAC;IAED;;;OAGG;IACK,YAAY,CAAC,KAAsB;QACvC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QACnC,MAAM,IAAI,GAAG,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QACtD,IAAI,IAAI,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,CAAC;YAC1F,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,GAAG,OAAO,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;QACrD,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,gBAAgB;QACpB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QACnC,IAAI,MAAM,CAAC,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;YAChD,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;QACnD,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;YACnC,OAAO,KAAK,EAAE,MAAM,IAAI,8BAA8B,CAAC;QAC3D,CAAC;QACD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QAC5B,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;;;;OAKG;IACK,iBAAiB;QACrB,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC;QAC5H,IAAI,CAAC,IAAI,CAAC,eAAe,IAAI,IAAI,IAAI,kBAAkB,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3G,OAAO;QACX,CAAC;QACD,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,mBAAmB,CAAC,CAAC;IACzD,CAAC;IAED;;;;;;OAMG;IACK,WAAW,CAAC,IAAyB,EAAE,MAAmB,EAAE,KAAa;QAC7E,MAAM,OAAO,GAAG,kBAAkB,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAChE,MAAM,QAAQ,GAAG,2BAA2B,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,IAAI,gBAAgB,CAAC;QAC9D,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE;YAC3B,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,gBAAgB,IAAI,CAAC;YAC/C,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC;SAC9E,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,MAAM,EAAE,UAAU,EAAE,cAAc,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC1D,OAAO,CAAC,cAAc,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACzI,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,qGAAqG;IAC7F,KAAK,CAAC,UAAU,CAAC,OAA0C;QAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC;QAC3C,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC;QACnG,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,QAAQ,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QACnE,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC7B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;QACjF,CAAC;QACD,MAAM,KAAK,GAAiB,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE;YAC9D,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBACpE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxB,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC1B,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;QACL,CAAC,CAAC,CAAC;QACH,OAAO;YACH,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,OAAO,EAAE,GAAG,EAAE;gBACV,KAAK,CAAC,WAAW,EAAE,CAAC;gBACpB,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC9B,CAAC;SACJ,CAAC;IACN,CAAC;IAED,mGAAmG;IAC3F,KAAK,CAAC,UAAU,CAAC,OAA0C;QAC/D,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC9B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,iCAAiC,EAAE,CAAC;QACpG,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAChC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,wBAAwB,EAAE,CAAC;QACzF,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC7B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;QAChG,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC/B,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;QAChE,OAAO;YACH,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,OAAO,EAAE,GAAG,EAAE;gBACV,YAAY,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,EAAE,CAAC;YACnB,CAAC;SACJ,CAAC;IACN,CAAC;CACJ;AAKD;;;GAGG;AACH,SAAS,eAAe,CAAC,KAAkC,EAAE,KAAc;IACvE,MAAM,QAAQ,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC;IAC9F,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;AAC7H,CAAC;AAED,iEAAiE;AACjE,SAAS,WAAW,CAAC,CAAyB,EAAE,CAAyB;IACrE,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAC9G,CAAC;AAED,sGAAsG;AACtG,SAAS,gBAAgB,CAAC,MAAmB,EAAE,OAA4B;IACvE,OAAO,IAAI,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAC7C,CAAC;AAED,uHAAuH;AACvH,SAAS,aAAa,CAAC,MAA6B;IAChD,QAAQ,MAAM,EAAE,CAAC;QACb,KAAK,aAAa,CAAC;QACnB,KAAK,mBAAmB;YACpB,OAAO,aAAa,CAAC;QACzB,KAAK,QAAQ;YACT,OAAO,QAAQ,CAAC;QACpB;YACI,OAAO,OAAO,CAAC;IACvB,CAAC;AACL,CAAC"}