@memberjunction/realtime-runtime 6.2.0-edge.2 → 6.2.0-edge.4

package/dist/channels/base-realtime-channel-client.d.ts

@@ -1,8 +1,15 @@
-import type { Observable } from 'rxjs';
+import { Observable } from 'rxjs';
 import { IMetadataProvider } from '@memberjunction/core';
-import { JSONValue, RealtimeToolDefinition, RealtimeTrackDescriptor, RealtimeTrackDirection } from '@memberjunction/ai';
-import type { BaseRealtimeClient } from '@memberjunction/ai-realtime-client';
-import type { AppContextSnapshot } from '@memberjunction/ai-core-plus';
+import { JSONObject, JSONValue, RealtimeToolDefinition, RealtimeTrackDescriptor, RealtimeTrackDirection } from '@memberjunction/ai';
+import { ChannelInboundVideoBridge, type BaseRealtimeClient, type IChannelFrameProvider, type MediaPlacement, type MediaVideoSource, type VideoSourceState } from '@memberjunction/ai-realtime-client';
+import { type AppContextSnapshot, type RealtimeChannelActor, type RealtimeChannelDescriptor, type RealtimeChannelExposure, type RealtimeChannelVerb } from '@memberjunction/ai-core-plus';
+import type { RealtimeChannelEvent, RealtimeChannelOutput, RealtimeChannelVerbResult, RealtimeContextActionRequest, RealtimeContextActionResult } from './channel-contract-types.js';
+import type { RealtimeSessionStreamEvent } from '../session/session-event-hub.js';
+import { type ChannelPerceptionOptions } from './channel-perception.js';
+import { type ChannelSurfacePlacement } from './channel-surface-placement.js';
+import type { RealtimeCaptureKind, RealtimeCaptureState, RealtimeCaptureStates } from '../session/realtime-captures.js';
+import type { RealtimeConnectionState } from '../session/RealtimeSessionRuntime.js';
+import type { ParsedDelegationArtifact } from '../session/delegation-result-parser.js';
 /**
  * A UI framework's component-class reference, as far as this runtime is concerned.
  *
@@ -63,7 +70,7 @@ export interface RealtimeChannelContext {
     RequestSave(stateJson: string): void;
     /**
      * Requests (or releases) the FOCUS layout for this channel's surface: the overlay
-     * collapses the main call column so the surface owns the screen, with a compact floating
+     * collapses the main call column and fills its stage with the surface, with a compact floating
      * call pill keeping mute / thread / end reachable. Any channel may request it; the host
      * tracks which channel holds focus and routes the pill's "exit" back to it via
      * {@link BaseRealtimeChannelClient.RequestFocusExit}.
@@ -140,10 +147,47 @@ export interface RealtimeChannelContext {
         ErrorMessage?: string;
     }>;
     /**
-     * OPTIONAL — sends a visual frame into the live session's inbound video track
-     * (e.g. from Whiteboard or Remote Browser video bridges). No-op when the session
-     * has not established an inbound video track or is not live.
+     * OPTIONAL — runs a CHANNEL-ADDRESSED `ContextTool` call: `{ action, params, target: { channel, instance? } }`.
+     * The headless `ClientContextChannel`'s proxy routes a call here when the model named a target; the
+     * host runtime resolves the channel, validates `params` against the verb's declared schema (a failure
+     * comes back as a structured, model-recoverable error — never a half-applied mutation), enforces
+     * `InvokableBy`, and opens an `on-demand` channel when the action is `'open'`. Never throws.
      *
+     * Absent on hosts that do not scope channels; the proxy then reports that channel addressing is
+     * unavailable instead of guessing.
+     *
+     * @param request The addressed call.
+     * @returns A structured outcome the proxy serializes back to the model.
+     */
+    DispatchContextAction?(request: RealtimeContextActionRequest): Promise<RealtimeContextActionResult>;
+    /**
+     * The resolved per-channel configuration for THIS channel: the host's defaults beneath the agent's
+     * and the app's `channels.config[<key>]` (see `ResolveRealtimeChannelScope`). Empty (`{}`) when none
+     * was configured. A channel reads its policy knobs from here (e.g. an identity channel's allowed
+     * email domains); it never reads the cascade itself.
+     *
+     * Treat it as untrusted, validated input: it is JSON authored by an operator, not typed code.
+     */
+    ChannelConfig?: Readonly<JSONObject>;
+    /**
+     * OPTIONAL — the events the SERVER publishes to this session (identity verification today; app-defined
+     * types too), hot with no replay. A channel that reacts to them (an identity channel learning that the
+     * user verified) subscribes in {@link BaseRealtimeChannelClient.OnInitialize} and unsubscribes at
+     * {@link BaseRealtimeChannelClient.Dispose}. Absent on hosts whose session has no event transport.
+     */
+    SessionEvents$?: Observable<RealtimeSessionStreamEvent>;
+    /**
+     * OPTIONAL — sends a visual frame into the live session's inbound video track, as this channel's source at the
+     * session's `VideoSourceArbiter` (the id its own frame bridge uses, {@link ChannelVideoSourceID}). The frame reaches
+     * the model only while the arbiter picks this channel, and the model is told when that changes; nothing is sent while
+     * the channel's exposure is below `pixels`. No-op when the session has not established an inbound video track or is
+     * not live.
+     *
+     * @deprecated Use a `ChannelInboundVideoBridge` (`@memberjunction/ai-realtime-client`), or
+     *   {@link BaseRealtimeChannelClient.EnableVisualPerception}, which runs one for the channel: the bridge lists the
+     *   source before its first frame, removes it when stopped, and says whether each frame was sent. The context's
+     *   {@link Client} still bypasses the arbiter: a frame sent with its own `SendVideoFrame` overrides the source the
+     *   arbiter picked, and the model is not told.
      * @param base64Image The image data (base64-encoded JPEG/PNG).
      * @param mimeType The image MIME type (defaults to 'image/jpeg').
      */
@@ -153,9 +197,48 @@ export interface RealtimeChannelContext {
      */
     IsTrackEstablished?(modality: string, direction: RealtimeTrackDirection): boolean;
     /**
-     * OPTIONAL — the underlying {@link BaseRealtimeClient} driving the media and transport planes.
+     * OPTIONAL — the underlying {@link BaseRealtimeClient} driving the media and transport planes. Send video through a
+     * `ChannelInboundVideoBridge` over it, not through its own `SendVideoFrame`, which bypasses the session's
+     * `VideoSourceArbiter`.
      */
     Client?: BaseRealtimeClient | null;
+    /**
+     * OPTIONAL — the session's camera and screen share, now and on every change (the runtime's `Captures$`). A channel that
+     * fronts a capture ({@link BaseRealtimeChannelClient.CaptureKind}) follows it here.
+     */
+    Captures$?: Observable<RealtimeCaptureStates>;
+    /**
+     * OPTIONAL — starts the camera or a screen share for the user's click on the channel's surface: the runtime's
+     * `StartCamera` or `StartScreenShare`, under the same policy. Resolves with the capture's state; a failure is a state,
+     * never a throw.
+     */
+    StartCapture?(kind: RealtimeCaptureKind): Promise<RealtimeCaptureState>;
+    /** OPTIONAL — stops the camera or the screen share (the runtime's `StopCamera` / `StopScreenShare`). */
+    StopCapture?(kind: RealtimeCaptureKind): void;
+    /**
+     * OPTIONAL — the session's video sources, now and on every change (the runtime's `VideoSources$`): each source the
+     * video source arbiter knows, whether it is on (`Enabled`) and whether the model is being sent its frames (`Active`).
+     * On a model that takes one video stream, a capture that is on can still be one the model is not sent. A channel that
+     * fronts a capture finds its source by `REALTIME_CAPTURE_SOURCE_IDS`, and says the agent sees the capture only while
+     * that source is both.
+     */
+    VideoSources$?: Observable<readonly VideoSourceState[]>;
+    /**
+     * OPTIONAL — the agent's video while the model sends it, `null` otherwise (the runtime's `AgentVideo$`). A channel that
+     * shows the agent, one that sinks outbound video ({@link BaseRealtimeChannelClient.GetSunkTracks}), follows it here.
+     */
+    AgentVideo$?: Observable<MediaVideoSource | null>;
+    /**
+     * OPTIONAL — the call's state, now and on every change (the runtime's `ConnectionState$`): connecting, listening,
+     * speaking, thinking, an error, or closed. A channel that shows the agent follows the agent's turn here.
+     */
+    ConnectionState$?: Observable<RealtimeConnectionState>;
+    /**
+     * OPTIONAL — whether the call is resuming on a new provider connection mid-call, now and on every change (the runtime's
+     * `Resuming$`). The agent's video can stop for a few seconds while it does, so a channel that shows the agent holds the
+     * last frame through the gap rather than showing that the video has stalled.
+     */
+    Resuming$?: Observable<boolean>;
 }
 /**
  * The first-run INTRO content for an interactive channel — the concise "what is this surface
@@ -210,10 +293,13 @@ export interface ChannelOnboardingDetails {
  *
  * ### Lifecycle — ONE INSTANCE PER SESSION (not a singleton)
  * `ClassFactory.CreateInstance` → {@link Initialize}(ctx) → zero or more
- * {@link BindSurface}/{@link UnbindSurface} cycles (the surface pane is created/destroyed
- * with the overlay's tab panel, e.g. collapse/expand) → {@link Dispose} at teardown.
+ * {@link BindSurface}/{@link UnbindSurface} cycles (the Angular overlay creates the surface the
+ * first time it is shown and keeps it until the channel leaves the session; another host may
+ * recreate it), each telling the plugin when its surface goes in and out of sight
+ * ({@link OnSurfaceVisibilityChange}) and where it is placed ({@link OnSurfacePlacementChange})
+ * → {@link Dispose} at teardown.
  * {@link ApplyAgentTool} MUST work with NO surface bound (apply to the state engine
- * directly; skip the UI garnish) — tool calls can arrive while the panel is collapsed.
+ * directly; skip the UI garnish) — tool calls can arrive before the surface is first shown.
  *
  * @typeParam TSurface The plugin's Angular surface component type. The host only ever
  *   sees the default (`object`) — the typed parameter exists so concrete plugins get a
@@ -235,12 +321,16 @@ export declare abstract class BaseRealtimeChannelClient<TSurface extends object
      * The shared name prefix of every tool this channel exposes (e.g. `'Whiteboard_'`).
      * The host registers ONE local-execution route per plugin: tool calls whose name starts
      * with this prefix go to {@link ApplyAgentTool} instead of the server relay.
+     *
+     * Default `''` — a v2 channel with NO native tools (it is reached only through the `ContextTool`
+     * proxy). The host registers no route for an empty prefix: an empty prefix would match *every*
+     * tool name and hijack the server relay.
      */
-    abstract get ToolNamePrefix(): string;
-    /** Label for the channel's tab on the overlay's surface panel (e.g. `'Whiteboard'`). */
-    abstract get TabTitle(): string;
-    /** Font Awesome icon class for the channel's tab (e.g. `'fa-solid fa-chalkboard'`). */
-    abstract get TabIcon(): string;
+    get ToolNamePrefix(): string;
+    /** Label for the channel's tab on the overlay's surface panel (e.g. `'Whiteboard'`). Defaults to {@link ChannelName}. */
+    get TabTitle(): string;
+    /** Font Awesome icon class for the channel's tab (e.g. `'fa-solid fa-chalkboard'`). Defaults to a generic puzzle-piece. */
+    get TabIcon(): string;
     /**
      * OPTIONAL accent color for the channel's tab (a CSS color string, e.g. an `hsl()` /
      * token). When a plugin supplies one, the overlay paints the tab's dot + active underline
@@ -253,18 +343,26 @@ export declare abstract class BaseRealtimeChannelClient<TSurface extends object
      * The channel's CLIENT-EXECUTED tool declarations, aggregated by the session service
      * into the `clientTools` set declared to the realtime model at session mint. The server
      * only DECLARES these — execution stays in the browser via {@link ApplyAgentTool}.
+     *
+     * Default `[]` — a v2 channel whose verbs are reached through the proxy only.
      */
-    abstract GetToolDefinitions(): RealtimeToolDefinition[];
+    GetToolDefinitions(): RealtimeToolDefinition[];
     /**
      * Executes ONE agent tool call locally (the ACTION direction) and returns the result
      * JSON string fed back to the model as the `tool_response`. Called for every tool whose
      * name starts with {@link ToolNamePrefix}. Must work both WITH a bound surface (apply +
-     * UI garnish) and WITHOUT one (apply to the state engine directly — the tab pane may not
-     * exist, e.g. the surface panel is collapsed). Should not throw: return a
+     * UI garnish) and WITHOUT one (apply to the state engine directly — the surface may not
+     * exist yet, e.g. it has not been shown). Should not throw: return a
      * `{ success: false, error }` payload so the model can narrate the failure (the host
      * additionally wraps anything thrown).
+     *
+     * **This is the LEGACY entry point.** Channels written against the v2 contract override
+     * {@link ApplyVerb} instead and inherit this method: the default maps the tool name back to its verb,
+     * calls {@link ApplyVerb} as the agent, and serializes the structured result. Channels written
+     * against v1 override THIS and inherit {@link ApplyVerb}, which adapts it. A channel must override
+     * at least one of the two — overriding neither is reported as a failure result, never a recursion.
      */
-    abstract ApplyAgentTool(toolName: string, argsJson: string): string | Promise<string>;
+    ApplyAgentTool(toolName: string, argsJson: string): string | Promise<string>;
     /**
      * The Angular component the overlay creates dynamically as this channel's tab pane, or `null`
      * for a **server-only** channel that renders no MJ surface (its surface, if any, lives on the
@@ -305,16 +403,57 @@ export declare abstract class BaseRealtimeChannelClient<TSurface extends object
      * The plugin — which knows its own component type — sets inputs (state engine, agent
      * name, …) and subscribes outputs here, wiring perception/garnish flows back through
      * {@link Context}. May be called again with a NEW instance after an
-     * {@link UnbindSurface} (the pane is destroyed/recreated with the tab panel).
+     * {@link UnbindSurface} (the host recreated the surface).
      */
-    abstract BindSurface(instance: TSurface): void;
+    BindSurface(_instance: TSurface): void;
     /**
-     * Called by the host when the surface component is being destroyed (tab panel
-     * collapsed / overlay torn down). Drop the instance reference and unsubscribe any
+     * Called by the host when the surface component is being destroyed (the channel left
+     * the session / the overlay was torn down). Drop the instance reference and unsubscribe any
      * output subscriptions — after this, {@link ApplyAgentTool} runs in its no-surface
      * mode. Default: no-op.
      */
     UnbindSurface(): void;
+    /**
+     * Called by the host when the bound surface comes into sight or goes out of it (the panel collapsed or hidden,
+     * another tab active, the call minimized). The surface stays bound and keeps its state; a channel can pause work
+     * nobody sees, such as a screenshot poll, and resume it when the surface is shown again. The host calls it right
+     * after {@link BindSurface} with the surface's current visibility, then on every change, and not after
+     * {@link UnbindSurface}. Default: no-op, so a surface keeps running out of sight.
+     */
+    OnSurfaceVisibilityChange(_visible: boolean): void;
+    /**
+     * Called by the host when the bound surface moves: to the stage (it fills the call), to its tab, or out of sight
+     * because the user hid it. A channel can adapt its own chrome, such as a "Move to stage" button that has nothing to
+     * do once the surface is on the stage. Called right after {@link BindSurface} with the current placement, then on
+     * every move, and not after {@link UnbindSurface}. Default: no-op.
+     */
+    OnSurfacePlacementChange(_placement: MediaPlacement): void;
+    private surfacePlacement;
+    /**
+     * Where this channel's surface shows when a call starts, and where the user may move it: from the channel's registry
+     * row (`UIConfig.Placement` and `UIConfig.AllowedPlacements`). A channel without a row, or whose row says nothing,
+     * starts on its tab and can go anywhere.
+     */
+    get SurfacePlacement(): ChannelSurfacePlacement;
+    /**
+     * Sets {@link SurfacePlacement}. The runtime calls it with the registry row's placement when it builds the channel, before
+     * the channel mounts, as it applies exposure policy with {@link ApplyExposure}.
+     */
+    ApplySurfacePlacement(placement: ChannelSurfacePlacement): void;
+    /**
+     * Whether the user may share this channel's surface on its own: a host that offers it lists the surface, named by
+     * {@link TabTitle} with {@link TabIcon}, under "This panel" in its Share menu while the surface is on screen, and the
+     * share then shows only that surface. Default `false`, so a channel opts in with this getter. A surface that shows the
+     * agent its own call (the camera, the screen share, the agent's video) never should.
+     */
+    get SurfaceShareable(): boolean;
+    /**
+     * The runtime capture this channel fronts (`'camera'` or `'screen'`), or `null`. Such a channel is the capture's policy:
+     * the runtime starts the capture only while the channel is in the session and the session's policy lets the agent see
+     * pixels through it, and the user's "agent can see" choice for the channel decides whether the capture's frames reach
+     * the model. Default `null`.
+     */
+    get CaptureKind(): RealtimeCaptureKind | null;
     /**
      * Binds the host context and invokes the {@link OnInitialize} hook. Called exactly once
      * per session, right after ClassFactory instantiation and before any tool call or
@@ -329,11 +468,21 @@ export declare abstract class BaseRealtimeChannelClient<TSurface extends object
     protected OnInitialize(): void;
     /**
      * Subclass hook invoked once the realtime session is connected and live (the client driver
-     * is created, connected, and media tracks negotiated). Channels that establish media bridges
-     * (e.g. video streaming) can start them here when `Context.Client` is available.
-     * Default: no-op.
+     * is created, connected, and media tracks negotiated), or when the channel is opened in a live
+     * session. Channels that establish media bridges (e.g. video streaming) can start them here when
+     * `Context.Client` is available.
+     * Default: a channel with visual perception adds the inbound video track it sources when the call
+     * has none (it was opened mid-call) and brings its frame bridge up; any other channel does nothing.
+     * An override that keeps visual perception calls `super.OnSessionStarted()`.
      */
     OnSessionStarted(): void;
+    /**
+     * Adds the inbound video track this channel sources ({@link GetSourcedTracks}) when the call is live without one. A
+     * channel open when the call connected had its track negotiated then, so this does nothing for it. A channel opened
+     * mid-call (`on-demand`) was not part of that negotiation: without its own track it could only send on a track a camera
+     * or screen share added, which goes when that capture stops. Only on a model that takes inbound video.
+     */
+    private addVisualTrackMidCall;
     /**
      * Max time {@link ResolveAgentSessionId} waits for the session id to bind before giving up, and the
      * poll interval it re-checks on. Protected so tests can shrink the wait; production keeps the
@@ -389,8 +538,326 @@ export declare abstract class BaseRealtimeChannelClient<TSurface extends object
     /**
      * Media tracks this client channel can SINK — samples flowing from the model OUT.
      * Default `[]`.
+     *
+     * The session requests them when it connects, so the driver establishes the ones the model supports. A channel that
+     * sinks outbound video shows the agent's video: it follows {@link RealtimeChannelContext.AgentVideo$}, and counts as
+     * used once the video arrives, so the host shows its surface.
      */
     GetSunkTracks(): readonly RealtimeTrackDescriptor[];
+    /**
+     * Whether this channel shows the agent's video: it sinks outbound video ({@link GetSunkTracks}). The runtime marks it
+     * as used when the video arrives, and a host presents its surface as the agent (the call overlay puts it in the agent's
+     * place in the call rather than in the focus layout).
+     */
+    get ShowsAgentVideo(): boolean;
+    /**
+     * The channel's stable primary-instance id. A single-instance channel has exactly one; a
+     * {@link RealtimeChannelDescriptor.MultiInstance} channel owns further instance ids itself and
+     * routes by the `instanceId` argument of {@link ApplyVerb}. Appears in every structured note
+     * (`[channel:<Key>#<instance>] …`) and every {@link RealtimeChannelEvent}.
+     */
+    get InstanceId(): string;
+    /**
+     * The channel's SELF-DESCRIPTION — what it is, what it holds, what it accepts, how it behaves in a
+     * session. This is the contract an agent reads to operate a channel it has never seen, and what the
+     * runtime reads to scope, advertise and validate. Pure data; called often, so keep it cheap.
+     *
+     * Default: a descriptor SYNTHESIZED from the v1 members (tools → verbs, intro → instructions),
+     * stamped with the legacy contract version — so a channel that implements nothing new is still
+     * describable and works exactly as before. A v2 channel overrides this with an authored descriptor.
+     * (Build verbs from {@link GetToolDefinitions} with `BuildToolBackedVerbs` when they are the
+     * native tools, so a subclass that adds a tool does not leave the descriptor lying.)
+     */
+    GetDescriptor(): RealtimeChannelDescriptor;
+    /**
+     * A snapshot of the channel's things (the descriptor's nouns), keyed by noun name. Perception notes
+     * are computed as deltas between successive snapshots, so this must be a pure read.
+     *
+     * Default: the parsed {@link SerializeState} when that is a JSON object, else `{}` — which makes a
+     * v1 channel's state of record its (single, opaque) snapshot with no code change.
+     */
+    GetState(): JSONObject;
+    /**
+     * Checks the current {@link GetState} snapshot against the descriptor's noun schemas and returns
+     * every violation (empty when valid). Useful in a channel's own tests; the runtime also runs it on
+     * every perception note when {@link StrictStateChecks} is on.
+     */
+    ValidateState(): string[];
+    /**
+     * Whether state snapshots are validated against the descriptor's noun schemas as notes are built —
+     * a development aid that surfaces a channel whose state drifted from its own declaration. On by
+     * default only when a Node-style environment reports a non-production `NODE_ENV` (tests, tooling);
+     * a browser production build never pays for it. Hosts may set it explicitly.
+     */
+    static StrictStateChecks: boolean;
+    /**
+     * Runs one verb of this channel — the contract-v2 entry point, reached through the `ContextTool`
+     * proxy (and, via {@link ApplyAgentTool}, by native tool calls).
+     *
+     * The runtime has ALREADY resolved the verb against the descriptor, enforced `InvokableBy`, and
+     * validated `args` against the verb's parameter schema before calling this, so an override can
+     * trust its inputs' shape. It must work with no surface bound, and should not throw — return a
+     * failure result so the model can recover.
+     *
+     * Default: adapts the legacy {@link ApplyAgentTool} (the verb's native tool name, args as JSON),
+     * so every v1 channel is reachable through the proxy with no change. A v2 channel overrides this.
+     *
+     * @param verb The verb name (as declared in the descriptor).
+     * @param args The validated parameters.
+     * @param actor Who is acting: the agent (a model call) or the user (their own interaction).
+     * @param instanceId Which instance of a multi-instance channel; `undefined` for the primary instance.
+     */
+    ApplyVerb(verb: string, args: JSONObject, actor: RealtimeChannelActor, instanceId?: string): RealtimeChannelVerbResult | Promise<RealtimeChannelVerbResult>;
+    /**
+     * Maps a native tool name back to its verb: the descriptor's `NativeToolName` match first, then the
+     * name minus {@link ToolNamePrefix}. Protected so a channel with an unusual naming scheme can
+     * override it.
+     */
+    protected ResolveVerbForTool(toolName: string): string;
+    /** Maps a verb to the native tool name it is reachable as (inverse of {@link ResolveVerbForTool}). */
+    protected ResolveNativeToolName(verb: string): string;
+    /**
+     * The descriptor verb a native tool call runs, or `undefined` when the tool is not one of the channel's verbs
+     * (the runtime uses it to apply {@link RefuseVerbForExposure} on the native-tool route as well as the `ContextTool` one).
+     *
+     * @param toolName The native tool name the model called.
+     */
+    FindVerbForNativeTool(toolName: string): RealtimeChannelVerb | undefined;
+    /**
+     * Whether the agent may be given this verb's result at the channel's current exposure. A verb that declares
+     * `ReturnsChannelData` is refused whole when exposure is below that level, because its result would hand the agent
+     * what the user or a zero-data-retention policy held back from it. Policy, not redaction: the result is never filtered.
+     *
+     * Applies to the AGENT (the dispatcher and the native-tool route call it); a user acting through the surface is not restricted.
+     *
+     * @param verb The verb the agent is calling.
+     * @returns The sentence to give the agent, or `null` when the call may proceed.
+     */
+    RefuseVerbForExposure(verb: Pick<RealtimeChannelVerb, 'Name' | 'ReturnsChannelData'>): string | null;
+    /** Why a channel that overrides neither entry point cannot run anything. */
+    private implementsNeitherMessage;
+    private readonly channelEventsSubject;
+    private readonly channelOutputSubject;
+    /**
+     * Everything this channel streams out, as typed events: state changes, discrete user actions, frames
+     * pushed, opened/completed. Hosts subscribe (the embeddable element re-emits these as DOM events);
+     * the model is told about a change through a coalesced note instead — see {@link RecordChange}.
+     * Completes at {@link Dispose}.
+     */
+    readonly Events$: Observable<RealtimeChannelEvent>;
+    /** What the channel handed back when it completed (see {@link Complete}). Completes at {@link Dispose}. */
+    readonly Output$: Observable<RealtimeChannelOutput>;
+    /**
+     * Opens the channel — mounts it for use and seeds it with `inputs`. Called by the runtime when the
+     * agent (or user) opens an `on-demand` channel, after the channel has been initialized.
+     *
+     * Validates `inputs` against the descriptor's `Inputs` schema (a violation is a structured
+     * `'invalid_params'` failure, never a half-opened channel), runs the {@link OnOpen} hook, then tells
+     * the model the channel is open and what it currently holds (`opened` note with a state snapshot) —
+     * and takes that snapshot as the baseline later changes are described against.
+     *
+     * @param inputs What the agent asked to seed the channel with.
+     */
+    Open(inputs?: JSONObject): Promise<RealtimeChannelVerbResult>;
+    /**
+     * Subclass hook for {@link Open}: apply the (already validated) seed `inputs`. Return a failure
+     * result to refuse the open; return nothing for success. Default: nothing to do.
+     */
+    protected OnOpen(_inputs: JSONObject): void | RealtimeChannelVerbResult | Promise<void | RealtimeChannelVerbResult>;
+    /**
+     * The channel finished and is handing something back (a submitted form, a chosen option, a
+     * finished game). Emits on {@link Output$} and {@link Events$} and tells the model with a
+     * `completed` note. The output is validated against the descriptor's `Output` schema when
+     * {@link StrictStateChecks} is on; a mismatch is logged, never swallowed into silence or thrown.
+     *
+     * @param output What the channel hands back.
+     * @param instanceId The instance that completed, for a multi-instance channel (default: the primary instance).
+     */
+    Complete(output: JSONObject, instanceId?: string): void;
+    /** Monotonic id of the latest state change; tags state events and the frames that describe them. */
+    private channelChangeSeq;
+    /** Re-entrancy guard for the default {@link ApplyVerb} ↔ {@link ApplyAgentTool} bridge. */
+    private bridgingVerbAndNativeTool;
+    private channelPerception;
+    /** Dev-check messages already logged, so a drifting state warns once, not on every note. */
+    private readonly warnedStateIssueKeys;
+    /** Tuning for the perception coalescer (debounce window, max note size). Override to change it. */
+    protected PerceptionOptions: ChannelPerceptionOptions;
+    /** The id of the latest state change (0 before any). */
+    protected get CurrentChangeId(): number;
+    /**
+     * Records that the channel's state changed. Call this on EVERY mutation, whoever caused it.
+     *
+     * It (1) assigns the change a monotonic id, (2) emits a `state_changed` event on {@link Events$}
+     * immediately (observers see every change), and (3) — unless `Perceive` is `false` — schedules a
+     * COALESCED structured note to the model: a burst of N changes inside the debounce window produces
+     * ONE `[channel:<Key>#<instance>] state_changed {"changeId":…,"changes":N,"delta":{…}}` note carrying
+     * what differs from what the model was last told (the first note is a full snapshot). Notes that
+     * would be too large degrade to a list of changed paths.
+     *
+     * @param options `Author` — who made the change (carried on the event); `Perceive` — whether the
+     *   model should be told (default `true`; a channel that already feeds the model its own perception
+     *   — like the Whiteboard's scene deltas — passes `false`).
+     * @returns The change's id.
+     */
+    protected RecordChange(options?: {
+        Author?: RealtimeChannelActor | 'system';
+        Perceive?: boolean;
+    }): number;
+    /**
+     * Emits a typed event on {@link Events$} (NOT to the model — use {@link RecordChange} for state the
+     * model should perceive, or send an explicit note for a discrete event it must hear about).
+     *
+     * @param name The event name (declare it in the descriptor's `Events`).
+     * @param payload The event payload.
+     * @param changeId The change id the event relates to, when it relates to one.
+     * @param instanceId The instance the event is about, for a multi-instance channel (default: the primary instance).
+     */
+    protected EmitChannelEvent(name: string, payload?: JSONObject, changeId?: number, instanceId?: string): void;
+    /** Sends any pending coalesced perception note now (e.g. before a surface is torn down). */
+    protected FlushPerception(): void;
+    /** The perception coalescer, created on first use. */
+    private ensurePerception;
+    /** {@link GetState}, with the development-time schema check applied when {@link StrictStateChecks} is on. */
+    private getStateChecked;
+    /** Logs a state problem once per channel instance — these checks run on every perception note. */
+    private warnStateIssueOnce;
+    /**
+     * Whether this channel wants to be offered the artifacts a delegated run just produced (the agent asked
+     * Skip or Sage to build something, and it came back as artifacts). A channel that hosts artifacts says yes
+     * for the ones it can show — and for an artifact it ALREADY shows, so a newer version replaces it in place.
+     *
+     * Asked of every channel in the session, mounted or merely advertised. For an advertised channel it is
+     * asked before the channel has been initialized, so it must not depend on {@link Context}; the channel's
+     * resolved configuration is passed in for exactly that reason. Saying yes for an advertised channel
+     * mounts it, so only answer yes when you will do something with the artifacts.
+     *
+     * Default: `false` — a channel that does not host artifacts is never bothered.
+     *
+     * @param artifacts What the delegated run produced.
+     * @param config This channel's resolved configuration (host defaults beneath agent/app config).
+     */
+    AcceptsDelegationArtifacts(_artifacts: readonly ParsedDelegationArtifact[], _config: JSONObject): boolean;
+    /**
+     * A delegated run produced artifacts this channel said it wants ({@link AcceptsDelegationArtifacts}). The
+     * channel is mounted and initialized by now. Never throw into the runtime: handle your own failures and
+     * report them (the runtime logs anything that escapes, but cannot recover the work).
+     *
+     * Default: nothing.
+     *
+     * @param artifacts What the delegated run produced.
+     */
+    OnDelegationArtifacts(_artifacts: readonly ParsedDelegationArtifact[]): void | Promise<void>;
+    /** The exposure the SERVER's policy allows (agent cap, zero data retention); `undefined` until the runtime applies one. */
+    private policyExposure;
+    /** The user's own choice for this channel; `undefined` when they made none. */
+    private userExposure;
+    /** Why exposure is below the channel's ceiling, as sentences the agent can be told. */
+    private exposureReasons;
+    /**
+     * How much of this channel the model may perceive without asking: the lowest of the channel's own ceiling
+     * (`GetDescriptor().MaxExposure`), the server policy and the user's choice. A channel nobody has applied a
+     * policy to simply has its ceiling, which is what every channel had before exposure policy existed.
+     *
+     * - `'none'`: no state notes and no frames.
+     * - `'state'`: structured state notes; no frames.
+     * - `'pixels'`: state notes and frames.
+     *
+     * It governs what flows to the model UNPROMPTED (perception notes, the contents of `opened` / `completed`
+     * notes, frames). It does not REDACT what a verb's result contains: a verb is all-or-nothing. A verb whose
+     * result would show the model something above this level declares `ReturnsChannelData` on the verb, and
+     * when this level is below it the verb is refused whole (`exposure_restricted`) and listed to the model as
+     * unavailable; see {@link RefuseVerbForExposure}. A verb that does not declare it returns what the channel
+     * says it returns, so a channel whose verbs reveal sensitive data must either declare it or lower its own
+     * `MaxExposure`.
+     */
+    get Exposure(): RealtimeChannelExposure;
+    /** Why {@link Exposure} is below the channel's ceiling; empty when it is not. */
+    get ExposureReasons(): readonly string[];
+    /**
+     * Whether the server policy (not the user) leaves room for pixels. Deliberately does not consult the
+     * descriptor: a synthesized descriptor derives its `MaxExposure` from {@link GetSourcedTracks}, which asks
+     * this, so reading it here would recurse. A channel that enabled visual perception has declared pixels.
+     */
+    private policyAllowsPixels;
+    /**
+     * Applies exposure policy: the server's decision and the user's own choice. The runtime calls this when
+     * it mounts the channel and again whenever the user changes their choice.
+     *
+     * Withdrawing exposure takes effect immediately: a pending perception note is dropped, a pending frame
+     * is dropped, and the channel's video source stops being forwarded. Restoring it re-baselines (the
+     * model's picture of the channel is stale, so the next note is a full snapshot) and sends a fresh frame.
+     * When the call is live the model is told about the change, with the reason, so it never assumes it
+     * can still see what it can't.
+     *
+     * @param settings `Policy` — the server-decided exposure; `User` — the user's choice (omit for none);
+     *   `Reasons` — why the policy lowered it, from the server's limits.
+     */
+    ApplyExposure(settings: ChannelExposureSettings): void;
+    /** Makes a change in exposure real: stops what is no longer allowed, restarts what is again. */
+    private applyExposureEffects;
+    /** Emits the change on {@link Events$} and, when the call is live, tells the model what it can now perceive and why. */
+    private announceExposureChange;
+    /** The frame source the visual pump captures from, once {@link EnableVisualPerception} ran. */
+    private visualFrameProvider;
+    private visualPerceptionOptions;
+    private visualFramePump;
+    /**
+     * The shared frame bridge that carries frames to the model's inbound video track. Deliberately NOT named
+     * `videoBridge`: channels that predate the pump (the Remote Browser, app channels) declare their own
+     * private `videoBridge`, and a second private of the same name in the base class would stop them compiling.
+     */
+    protected VisualVideoBridge: ChannelInboundVideoBridge | null;
+    /** Stops refreshing the model's picture when the bridge's source is sent again; set while the bridge exists. */
+    private stopRefreshOnSourceActivated;
+    /** Default frame cadence when the track negotiated none (1 fps — the Gemini Live ceiling). */
+    private static readonly DEFAULT_VISUAL_CADENCE_MS;
+    /** Floor for the negotiated cadence (4 fps) so a fast model never makes a channel spin. */
+    private static readonly MIN_VISUAL_CADENCE_MS;
+    /**
+     * Opts this channel into VISUAL perception: its surface can be shown to a video-capable model as
+     * frames. Call it from {@link OnInitialize} with the channel's frame source.
+     *
+     * The channel then needs only to say *when* its picture changed — {@link NotifyVisualChange} after a
+     * user edit, {@link ConfirmVisualChange} after a successful agent edit — and the base class does the
+     * rest: change-driven (no idle heartbeat), paced to the negotiated cadence, a trailing settle so the
+     * model ends on the final state of a burst, dedupe of identical frames, and a single confirmation
+     * frame (with a "do not narrate" note) after an agent edit. Every frame is tagged with the change id
+     * of the state it was captured at (`frame_pushed` events), so state and pixels can be cross-checked.
+     *
+     * Also makes {@link GetSourcedTracks} report the inbound video track, which is what gets it
+     * negotiated. Idempotent: calling it again replaces the provider and options.
+     *
+     * @param provider Renders the channel's current picture as a base64 JPEG (or `null`).
+     * @param options See {@link VisualPerceptionOptions}.
+     */
+    protected EnableVisualPerception(provider: IChannelFrameProvider, options?: VisualPerceptionOptions): void;
+    /**
+     * The channel's picture changed because of a USER action (or a scene replacement): push a frame,
+     * respecting the cadence. Resolves when the decision has been carried out; never rejects. A no-op
+     * until {@link EnableVisualPerception} ran and the session has an inbound video track.
+     */
+    protected NotifyVisualChange(): Promise<void>;
+    /**
+     * An AGENT action succeeded and changed the channel's picture: push exactly one confirmation frame
+     * now and tell the model not to narrate its own change. Call only on SUCCESS — confirming a failed
+     * action would assert an edit landed that did not. A no-op until {@link EnableVisualPerception} ran.
+     */
+    protected ConfirmVisualChange(): Promise<void>;
+    /**
+     * Returns the shared frame bridge, creating it if needed and (only when the channel opted into the
+     * legacy poller) starting its fixed-rate poll once the video track is up. `null` when no context is bound.
+     */
+    protected EnsureVideoBridge(): ChannelInboundVideoBridge | null;
+    /** Builds the pump, wiring it to this channel's context, bridge and event stream. */
+    private createVisualPump;
+    /** The "do not narrate your own change" note sent with a confirmation frame. */
+    private confirmationNote;
+    /**
+     * The frame cadence in ms: the rate the inbound video track negotiated (`1000 / Rate`), clamped to
+     * a 250ms floor; 1000ms when nothing was negotiated.
+     */
+    private getNegotiatedVisualCadenceMs;
     /**
      * Tears the plugin down at session end: release the surface binding, unsubscribe
      * state-engine subscriptions, then drop the context. Subclasses overriding this MUST
@@ -399,4 +866,44 @@ export declare abstract class BaseRealtimeChannelClient<TSurface extends object
      */
     Dispose(): void;
 }
+/**
+ * The id of a channel's video source at the session's `VideoSourceArbiter`: `<channel key>#<instance id>`. The channel's
+ * own frame bridge and the runtime's {@link RealtimeChannelContext.SendVideoFrame} register under it, so a channel is
+ * one source however it sends, and the "agent can see" control lists it once.
+ *
+ * @param channelKey The channel's descriptor key.
+ * @param instanceId The channel instance.
+ */
+export declare function ChannelVideoSourceID(channelKey: string, instanceId: string): string;
+/**
+ * Settings for {@link BaseRealtimeChannelClient.ApplyExposure}.
+ */
+export interface ChannelExposureSettings {
+    /** The exposure the server's policy allows (agent cap and zero-data-retention requirement). Omit when no server policy applies. */
+    Policy?: RealtimeChannelExposure;
+    /** The user's own choice for this channel. Omit when they made none. */
+    User?: RealtimeChannelExposure;
+    /** Why the policy lowered exposure, as sentences the agent can be told. */
+    Reasons?: readonly string[];
+}
+/**
+ * Options for {@link BaseRealtimeChannelClient.EnableVisualPerception}.
+ */
+export interface VisualPerceptionOptions {
+    /**
+     * The note sent after a confirmation frame — a literal string, or a function of the change id.
+     * Default: a structured `frame_confirmed` note carrying a do-not-narrate instruction. The Whiteboard
+     * passes its original wording to stay byte-for-byte what it was.
+     */
+    ConfirmationNote?: string | ((changeId: number) => string);
+    /**
+     * Also start the shared bridge's fixed-rate poller (1 fps) once the video track is up.
+     *
+     * Default `false`: the pump is purely change-driven. This exists ONLY so the Whiteboard keeps the
+     * behavior it had before the pump was lifted: it has always started the bridge's poller (which
+     * re-sends the board every second regardless of change) despite its own comments and design
+     * stating it is heartbeat-free. Nothing new should set this.
+     */
+    StartBridgePoller?: boolean;
+}
 //# sourceMappingURL=base-realtime-channel-client.d.ts.map