@memberjunction/realtime-runtime 6.2.0-edge.3 → 6.2.0-edge.4

package/dist/channels/base-realtime-channel-client.js

@@ -1,3 +1,13 @@
+import { Subject } from 'rxjs';
+import { IsPlainObject } from '@memberjunction/global';
+import { CHANNEL_INBOUND_VIDEO_TRACK } from '@memberjunction/ai';
+import { ChannelInboundVideoBridge } from '@memberjunction/ai-realtime-client';
+import { CompareExposure, DescribeExposureLimit, DescribeWithheldVerb, IsVerbWithheld, MinExposure, WithheldVerbs, ValidateJsonAgainstSchemaSubset, } from '@memberjunction/ai-core-plus';
+import { SynthesizeChannelDescriptor } from './channel-descriptor-synthesis.js';
+import { ChannelPerceptionCoalescer, DEFAULT_CHANNEL_PERCEPTION_OPTIONS } from './channel-perception.js';
+import { FormatChannelNote } from './channel-state-delta.js';
+import { VisualPerceptionPump } from './channel-visual-pump.js';
+import { DEFAULT_CHANNEL_SURFACE_PLACEMENT } from './channel-surface-placement.js';
 /**
  * Base class for CLIENT-SIDE interactive-channel plugins (per
  * `plans/ai-agent-sessions.md` → "Interactive Channels" / "Pluggable Channel Interfaces").
@@ -32,10 +42,13 @@
  *
  * ### Lifecycle — ONE INSTANCE PER SESSION (not a singleton)
  * `ClassFactory.CreateInstance` → {@link Initialize}(ctx) → zero or more
- * {@link BindSurface}/{@link UnbindSurface} cycles (the surface pane is created/destroyed
- * with the overlay's tab panel, e.g. collapse/expand) → {@link Dispose} at teardown.
+ * {@link BindSurface}/{@link UnbindSurface} cycles (the Angular overlay creates the surface the
+ * first time it is shown and keeps it until the channel leaves the session; another host may
+ * recreate it), each telling the plugin when its surface goes in and out of sight
+ * ({@link OnSurfaceVisibilityChange}) and where it is placed ({@link OnSurfacePlacementChange})
+ * → {@link Dispose} at teardown.
  * {@link ApplyAgentTool} MUST work with NO surface bound (apply to the state engine
- * directly; skip the UI garnish) — tool calls can arrive while the panel is collapsed.
+ * directly; skip the UI garnish) — tool calls can arrive before the surface is first shown.
  *
  * @typeParam TSurface The plugin's Angular surface component type. The host only ever
  *   sees the default (`object`) — the typed parameter exists so concrete plugins get a
@@ -48,6 +61,7 @@ export class BaseRealtimeChannelClient {
          * `null` outside that window — guard with `?.` in any code that can run early/late.
          */
         this.Context = null;
+        this.surfacePlacement = DEFAULT_CHANNEL_SURFACE_PLACEMENT;
         /**
          * Max time {@link ResolveAgentSessionId} waits for the session id to bind before giving up, and the
          * poll interval it re-checks on. Protected so tests can shrink the wait; production keeps the
@@ -55,6 +69,62 @@ export class BaseRealtimeChannelClient {
          */
         this.SessionIdWaitTimeoutMs = 8000;
         this.SessionIdWaitIntervalMs = 200;
+        this.channelEventsSubject = new Subject();
+        this.channelOutputSubject = new Subject();
+        /**
+         * Everything this channel streams out, as typed events: state changes, discrete user actions, frames
+         * pushed, opened/completed. Hosts subscribe (the embeddable element re-emits these as DOM events);
+         * the model is told about a change through a coalesced note instead — see {@link RecordChange}.
+         * Completes at {@link Dispose}.
+         */
+        this.Events$ = this.channelEventsSubject.asObservable();
+        /** What the channel handed back when it completed (see {@link Complete}). Completes at {@link Dispose}. */
+        this.Output$ = this.channelOutputSubject.asObservable();
+        // ── Contract v2: change tracking + structured perception ───────────────────
+        /** Monotonic id of the latest state change; tags state events and the frames that describe them. */
+        this.channelChangeSeq = 0;
+        /** Re-entrancy guard for the default {@link ApplyVerb} ↔ {@link ApplyAgentTool} bridge. */
+        this.bridgingVerbAndNativeTool = false;
+        this.channelPerception = null;
+        /** Dev-check messages already logged, so a drifting state warns once, not on every note. */
+        this.warnedStateIssueKeys = new Set();
+        /** Tuning for the perception coalescer (debounce window, max note size). Override to change it. */
+        this.PerceptionOptions = DEFAULT_CHANNEL_PERCEPTION_OPTIONS;
+        /** Why exposure is below the channel's ceiling, as sentences the agent can be told. */
+        this.exposureReasons = [];
+        // ── Contract v2: visual perception (the change-driven frame pump) ──────────
+        /** The frame source the visual pump captures from, once {@link EnableVisualPerception} ran. */
+        this.visualFrameProvider = null;
+        this.visualPerceptionOptions = {};
+        this.visualFramePump = null;
+        /**
+         * The shared frame bridge that carries frames to the model's inbound video track. Deliberately NOT named
+         * `videoBridge`: channels that predate the pump (the Remote Browser, app channels) declare their own
+         * private `videoBridge`, and a second private of the same name in the base class would stop them compiling.
+         */
+        this.VisualVideoBridge = null;
+        /** Stops refreshing the model's picture when the bridge's source is sent again; set while the bridge exists. */
+        this.stopRefreshOnSourceActivated = null;
+    }
+    /**
+     * The shared name prefix of every tool this channel exposes (e.g. `'Whiteboard_'`).
+     * The host registers ONE local-execution route per plugin: tool calls whose name starts
+     * with this prefix go to {@link ApplyAgentTool} instead of the server relay.
+     *
+     * Default `''` — a v2 channel with NO native tools (it is reached only through the `ContextTool`
+     * proxy). The host registers no route for an empty prefix: an empty prefix would match *every*
+     * tool name and hijack the server relay.
+     */
+    get ToolNamePrefix() {
+        return '';
+    }
+    /** Label for the channel's tab on the overlay's surface panel (e.g. `'Whiteboard'`). Defaults to {@link ChannelName}. */
+    get TabTitle() {
+        return this.ChannelName;
+    }
+    /** Font Awesome icon class for the channel's tab (e.g. `'fa-solid fa-chalkboard'`). Defaults to a generic puzzle-piece. */
+    get TabIcon() {
+        return 'fa-solid fa-puzzle-piece';
     }
     /**
      * OPTIONAL accent color for the channel's tab (a CSS color string, e.g. an `hsl()` /
@@ -66,6 +136,45 @@ export class BaseRealtimeChannelClient {
     get TabColor() {
         return null;
     }
+    /**
+     * The channel's CLIENT-EXECUTED tool declarations, aggregated by the session service
+     * into the `clientTools` set declared to the realtime model at session mint. The server
+     * only DECLARES these — execution stays in the browser via {@link ApplyAgentTool}.
+     *
+     * Default `[]` — a v2 channel whose verbs are reached through the proxy only.
+     */
+    GetToolDefinitions() {
+        return [];
+    }
+    /**
+     * Executes ONE agent tool call locally (the ACTION direction) and returns the result
+     * JSON string fed back to the model as the `tool_response`. Called for every tool whose
+     * name starts with {@link ToolNamePrefix}. Must work both WITH a bound surface (apply +
+     * UI garnish) and WITHOUT one (apply to the state engine directly — the surface may not
+     * exist yet, e.g. it has not been shown). Should not throw: return a
+     * `{ success: false, error }` payload so the model can narrate the failure (the host
+     * additionally wraps anything thrown).
+     *
+     * **This is the LEGACY entry point.** Channels written against the v2 contract override
+     * {@link ApplyVerb} instead and inherit this method: the default maps the tool name back to its verb,
+     * calls {@link ApplyVerb} as the agent, and serializes the structured result. Channels written
+     * against v1 override THIS and inherit {@link ApplyVerb}, which adapts it. A channel must override
+     * at least one of the two — overriding neither is reported as a failure result, never a recursion.
+     */
+    ApplyAgentTool(toolName, argsJson) {
+        if (this.bridgingVerbAndNativeTool) {
+            return JSON.stringify({ success: false, error: this.implementsNeitherMessage() });
+        }
+        this.bridgingVerbAndNativeTool = true;
+        let result;
+        try {
+            result = this.ApplyVerb(this.ResolveVerbForTool(toolName), parseToolArguments(argsJson), 'agent');
+        }
+        finally {
+            this.bridgingVerbAndNativeTool = false;
+        }
+        return result instanceof Promise ? result.then(serializeVerbResult) : serializeVerbResult(result);
+    }
     /**
      * The Angular component the overlay creates dynamically as this channel's tab pane, or `null`
      * for a **server-only** channel that renders no MJ surface (its surface, if any, lives on the
@@ -107,14 +216,77 @@ export class BaseRealtimeChannelClient {
         return null;
     }
     /**
-     * Called by the host when the surface component is being destroyed (tab panel
-     * collapsed / overlay torn down). Drop the instance reference and unsubscribe any
+     * Called by the host right after it created the surface component (and BEFORE the
+     * component's first change detection, so inputs set here are visible in its `ngOnInit`).
+     * The plugin — which knows its own component type — sets inputs (state engine, agent
+     * name, …) and subscribes outputs here, wiring perception/garnish flows back through
+     * {@link Context}. May be called again with a NEW instance after an
+     * {@link UnbindSurface} (the host recreated the surface).
+     */
+    BindSurface(_instance) {
+        // default: a channel with no surface has nothing to bind
+    }
+    /**
+     * Called by the host when the surface component is being destroyed (the channel left
+     * the session / the overlay was torn down). Drop the instance reference and unsubscribe any
      * output subscriptions — after this, {@link ApplyAgentTool} runs in its no-surface
      * mode. Default: no-op.
      */
     UnbindSurface() {
         // default: nothing to release
     }
+    /**
+     * Called by the host when the bound surface comes into sight or goes out of it (the panel collapsed or hidden,
+     * another tab active, the call minimized). The surface stays bound and keeps its state; a channel can pause work
+     * nobody sees, such as a screenshot poll, and resume it when the surface is shown again. The host calls it right
+     * after {@link BindSurface} with the surface's current visibility, then on every change, and not after
+     * {@link UnbindSurface}. Default: no-op, so a surface keeps running out of sight.
+     */
+    OnSurfaceVisibilityChange(_visible) {
+        // default: nothing to pause
+    }
+    /**
+     * Called by the host when the bound surface moves: to the stage (it fills the call), to its tab, or out of sight
+     * because the user hid it. A channel can adapt its own chrome, such as a "Move to stage" button that has nothing to
+     * do once the surface is on the stage. Called right after {@link BindSurface} with the current placement, then on
+     * every move, and not after {@link UnbindSurface}. Default: no-op.
+     */
+    OnSurfacePlacementChange(_placement) {
+        // default: a surface looks the same wherever it is placed
+    }
+    /**
+     * Where this channel's surface shows when a call starts, and where the user may move it: from the channel's registry
+     * row (`UIConfig.Placement` and `UIConfig.AllowedPlacements`). A channel without a row, or whose row says nothing,
+     * starts on its tab and can go anywhere.
+     */
+    get SurfacePlacement() {
+        return this.surfacePlacement;
+    }
+    /**
+     * Sets {@link SurfacePlacement}. The runtime calls it with the registry row's placement when it builds the channel, before
+     * the channel mounts, as it applies exposure policy with {@link ApplyExposure}.
+     */
+    ApplySurfacePlacement(placement) {
+        this.surfacePlacement = placement;
+    }
+    /**
+     * Whether the user may share this channel's surface on its own: a host that offers it lists the surface, named by
+     * {@link TabTitle} with {@link TabIcon}, under "This panel" in its Share menu while the surface is on screen, and the
+     * share then shows only that surface. Default `false`, so a channel opts in with this getter. A surface that shows the
+     * agent its own call (the camera, the screen share, the agent's video) never should.
+     */
+    get SurfaceShareable() {
+        return false;
+    }
+    /**
+     * The runtime capture this channel fronts (`'camera'` or `'screen'`), or `null`. Such a channel is the capture's policy:
+     * the runtime starts the capture only while the channel is in the session and the session's policy lets the agent see
+     * pixels through it, and the user's "agent can see" choice for the channel decides whether the capture's frames reach
+     * the model. Default `null`.
+     */
+    get CaptureKind() {
+        return null;
+    }
     /**
      * Binds the host context and invokes the {@link OnInitialize} hook. Called exactly once
      * per session, right after ClassFactory instantiation and before any tool call or
@@ -134,12 +306,37 @@ export class BaseRealtimeChannelClient {
     }
     /**
      * Subclass hook invoked once the realtime session is connected and live (the client driver
-     * is created, connected, and media tracks negotiated). Channels that establish media bridges
-     * (e.g. video streaming) can start them here when `Context.Client` is available.
-     * Default: no-op.
+     * is created, connected, and media tracks negotiated), or when the channel is opened in a live
+     * session. Channels that establish media bridges (e.g. video streaming) can start them here when
+     * `Context.Client` is available.
+     * Default: a channel with visual perception adds the inbound video track it sources when the call
+     * has none (it was opened mid-call) and brings its frame bridge up; any other channel does nothing.
+     * An override that keeps visual perception calls `super.OnSessionStarted()`.
      */
     OnSessionStarted() {
-        // default: nothing to do
+        // A channel with visual perception brings its frame bridge up now: the video track is only
+        // negotiated once the session connects, so this is the first moment it can be established.
+        if (this.visualFrameProvider) {
+            this.addVisualTrackMidCall();
+            this.EnsureVideoBridge();
+        }
+    }
+    /**
+     * Adds the inbound video track this channel sources ({@link GetSourcedTracks}) when the call is live without one. A
+     * channel open when the call connected had its track negotiated then, so this does nothing for it. A channel opened
+     * mid-call (`on-demand`) was not part of that negotiation: without its own track it could only send on a track a camera
+     * or screen share added, which goes when that capture stops. Only on a model that takes inbound video.
+     */
+    addVisualTrackMidCall() {
+        const client = this.Context?.Client;
+        if (!client?.SupportsInboundVideo || client.IsTrackEstablished('video', 'inbound')) {
+            return;
+        }
+        for (const track of this.GetSourcedTracks()) {
+            if (track.Direction === 'inbound' && String(track.Modality).trim().toLowerCase() === 'video') {
+                client.AddTrack(track);
+            }
+        }
     }
     /**
      * Resolves the live {@link RealtimeChannelContext.AgentSessionID}, briefly WAITING for it when it
@@ -210,15 +407,600 @@ export class BaseRealtimeChannelClient {
      * Default `[]`.
      */
     GetSourcedTracks() {
-        return [];
+        // A channel that enabled visual perception sources the inbound video track the pump writes to — unless the
+        // agent's policy has already ruled pixels out (the user's own toggle does not: a user who turns the agent's
+        // view back on mid-call needs the track to exist). Not requesting video is also what keeps a session out of
+        // the shorter session limits video carries.
+        return this.visualFrameProvider && this.policyAllowsPixels() ? [CHANNEL_INBOUND_VIDEO_TRACK] : [];
     }
     /**
      * Media tracks this client channel can SINK — samples flowing from the model OUT.
      * Default `[]`.
+     *
+     * The session requests them when it connects, so the driver establishes the ones the model supports. A channel that
+     * sinks outbound video shows the agent's video: it follows {@link RealtimeChannelContext.AgentVideo$}, and counts as
+     * used once the video arrives, so the host shows its surface.
      */
     GetSunkTracks() {
         return [];
     }
+    /**
+     * Whether this channel shows the agent's video: it sinks outbound video ({@link GetSunkTracks}). The runtime marks it
+     * as used when the video arrives, and a host presents its surface as the agent (the call overlay puts it in the agent's
+     * place in the call rather than in the focus layout).
+     */
+    get ShowsAgentVideo() {
+        return this.GetSunkTracks().some((t) => t.Direction === 'outbound' && String(t.Modality).trim().toLowerCase() === 'video');
+    }
+    // ── Contract v2: descriptor, state, verbs, events, open/complete ───────────
+    /**
+     * The channel's stable primary-instance id. A single-instance channel has exactly one; a
+     * {@link RealtimeChannelDescriptor.MultiInstance} channel owns further instance ids itself and
+     * routes by the `instanceId` argument of {@link ApplyVerb}. Appears in every structured note
+     * (`[channel:<Key>#<instance>] …`) and every {@link RealtimeChannelEvent}.
+     */
+    get InstanceId() {
+        return PRIMARY_CHANNEL_INSTANCE_ID;
+    }
+    /**
+     * The channel's SELF-DESCRIPTION — what it is, what it holds, what it accepts, how it behaves in a
+     * session. This is the contract an agent reads to operate a channel it has never seen, and what the
+     * runtime reads to scope, advertise and validate. Pure data; called often, so keep it cheap.
+     *
+     * Default: a descriptor SYNTHESIZED from the v1 members (tools → verbs, intro → instructions),
+     * stamped with the legacy contract version — so a channel that implements nothing new is still
+     * describable and works exactly as before. A v2 channel overrides this with an authored descriptor.
+     * (Build verbs from {@link GetToolDefinitions} with `BuildToolBackedVerbs` when they are the
+     * native tools, so a subclass that adds a tool does not leave the descriptor lying.)
+     */
+    GetDescriptor() {
+        return SynthesizeChannelDescriptor(this);
+    }
+    /**
+     * A snapshot of the channel's things (the descriptor's nouns), keyed by noun name. Perception notes
+     * are computed as deltas between successive snapshots, so this must be a pure read.
+     *
+     * Default: the parsed {@link SerializeState} when that is a JSON object, else `{}` — which makes a
+     * v1 channel's state of record its (single, opaque) snapshot with no code change.
+     */
+    GetState() {
+        const serialized = this.SerializeState();
+        if (serialized === null) {
+            return {};
+        }
+        try {
+            const parsed = JSON.parse(serialized);
+            // JSON.parse output is JSON by construction, so narrowing it to JSONObject is not a type lie.
+            return IsPlainObject(parsed) ? parsed : { state: parsed };
+        }
+        catch (error) {
+            // A channel whose state of record is not JSON has no structured snapshot to offer; say so once
+            // (this runs on every perception note) rather than silently presenting it as empty.
+            this.warnStateIssueOnce('unparseable-serialized-state', `SerializeState() did not return JSON, so GetState() reports an empty snapshot: ${error instanceof Error ? error.message : String(error)}`);
+            return {};
+        }
+    }
+    /**
+     * Checks the current {@link GetState} snapshot against the descriptor's noun schemas and returns
+     * every violation (empty when valid). Useful in a channel's own tests; the runtime also runs it on
+     * every perception note when {@link StrictStateChecks} is on.
+     */
+    ValidateState() {
+        const state = this.GetState();
+        const issues = [];
+        for (const noun of this.GetDescriptor().Nouns) {
+            if (noun.Name in state) {
+                issues.push(...ValidateJsonAgainstSchemaSubset(state[noun.Name], noun.Schema, `$.${noun.Name}`));
+            }
+        }
+        return issues;
+    }
+    /**
+     * Whether state snapshots are validated against the descriptor's noun schemas as notes are built —
+     * a development aid that surfaces a channel whose state drifted from its own declaration. On by
+     * default only when a Node-style environment reports a non-production `NODE_ENV` (tests, tooling);
+     * a browser production build never pays for it. Hosts may set it explicitly.
+     */
+    static { this.StrictStateChecks = isDevelopmentEnvironment(); }
+    /**
+     * Runs one verb of this channel — the contract-v2 entry point, reached through the `ContextTool`
+     * proxy (and, via {@link ApplyAgentTool}, by native tool calls).
+     *
+     * The runtime has ALREADY resolved the verb against the descriptor, enforced `InvokableBy`, and
+     * validated `args` against the verb's parameter schema before calling this, so an override can
+     * trust its inputs' shape. It must work with no surface bound, and should not throw — return a
+     * failure result so the model can recover.
+     *
+     * Default: adapts the legacy {@link ApplyAgentTool} (the verb's native tool name, args as JSON),
+     * so every v1 channel is reachable through the proxy with no change. A v2 channel overrides this.
+     *
+     * @param verb The verb name (as declared in the descriptor).
+     * @param args The validated parameters.
+     * @param actor Who is acting: the agent (a model call) or the user (their own interaction).
+     * @param instanceId Which instance of a multi-instance channel; `undefined` for the primary instance.
+     */
+    ApplyVerb(verb, args, actor, instanceId) {
+        if (this.bridgingVerbAndNativeTool) {
+            return { Success: false, ErrorCode: 'verb_failed', Error: this.implementsNeitherMessage() };
+        }
+        this.bridgingVerbAndNativeTool = true;
+        let outcome;
+        try {
+            outcome = this.ApplyAgentTool(this.ResolveNativeToolName(verb), JSON.stringify(args));
+        }
+        finally {
+            this.bridgingVerbAndNativeTool = false;
+        }
+        return outcome instanceof Promise ? outcome.then(parseLegacyVerbResult) : parseLegacyVerbResult(outcome);
+    }
+    /**
+     * Maps a native tool name back to its verb: the descriptor's `NativeToolName` match first, then the
+     * name minus {@link ToolNamePrefix}. Protected so a channel with an unusual naming scheme can
+     * override it.
+     */
+    ResolveVerbForTool(toolName) {
+        const declared = this.GetDescriptor().Verbs.find((v) => v.NativeToolName === toolName);
+        if (declared) {
+            return declared.Name;
+        }
+        const prefix = this.ToolNamePrefix;
+        return prefix.length > 0 && toolName.length > prefix.length && toolName.startsWith(prefix) ? toolName.slice(prefix.length) : toolName;
+    }
+    /** Maps a verb to the native tool name it is reachable as (inverse of {@link ResolveVerbForTool}). */
+    ResolveNativeToolName(verb) {
+        const wanted = verb.trim().toLowerCase();
+        const declared = this.GetDescriptor().Verbs.find((v) => v.Name.toLowerCase() === wanted);
+        if (declared?.NativeToolName) {
+            return declared.NativeToolName;
+        }
+        const prefix = this.ToolNamePrefix;
+        return prefix.length > 0 && !verb.startsWith(prefix) ? `${prefix}${verb}` : verb;
+    }
+    /**
+     * The descriptor verb a native tool call runs, or `undefined` when the tool is not one of the channel's verbs
+     * (the runtime uses it to apply {@link RefuseVerbForExposure} on the native-tool route as well as the `ContextTool` one).
+     *
+     * @param toolName The native tool name the model called.
+     */
+    FindVerbForNativeTool(toolName) {
+        const wanted = this.ResolveVerbForTool(toolName).toLowerCase();
+        return this.GetDescriptor().Verbs.find((v) => v.Name.toLowerCase() === wanted);
+    }
+    /**
+     * Whether the agent may be given this verb's result at the channel's current exposure. A verb that declares
+     * `ReturnsChannelData` is refused whole when exposure is below that level, because its result would hand the agent
+     * what the user or a zero-data-retention policy held back from it. Policy, not redaction: the result is never filtered.
+     *
+     * Applies to the AGENT (the dispatcher and the native-tool route call it); a user acting through the surface is not restricted.
+     *
+     * @param verb The verb the agent is calling.
+     * @returns The sentence to give the agent, or `null` when the call may proceed.
+     */
+    RefuseVerbForExposure(verb) {
+        const effective = this.Exposure;
+        return IsVerbWithheld(verb, effective) ? DescribeWithheldVerb(verb.Name, this.GetDescriptor().DisplayName, effective, this.exposureReasons) : null;
+    }
+    /** Why a channel that overrides neither entry point cannot run anything. */
+    implementsNeitherMessage() {
+        return `Channel '${this.ChannelName}' implements neither ApplyVerb nor ApplyAgentTool.`;
+    }
+    /**
+     * Opens the channel — mounts it for use and seeds it with `inputs`. Called by the runtime when the
+     * agent (or user) opens an `on-demand` channel, after the channel has been initialized.
+     *
+     * Validates `inputs` against the descriptor's `Inputs` schema (a violation is a structured
+     * `'invalid_params'` failure, never a half-opened channel), runs the {@link OnOpen} hook, then tells
+     * the model the channel is open and what it currently holds (`opened` note with a state snapshot) —
+     * and takes that snapshot as the baseline later changes are described against.
+     *
+     * @param inputs What the agent asked to seed the channel with.
+     */
+    async Open(inputs = {}) {
+        const descriptor = this.GetDescriptor();
+        if (descriptor.Inputs) {
+            const issues = ValidateJsonAgainstSchemaSubset(inputs, descriptor.Inputs);
+            if (issues.length > 0) {
+                return {
+                    Success: false,
+                    ErrorCode: 'invalid_params',
+                    Error: `The inputs for opening ${descriptor.Key} are invalid.`,
+                    Details: issues
+                };
+            }
+        }
+        const opened = await this.OnOpen(inputs);
+        if (opened && !opened.Success) {
+            return opened;
+        }
+        // A multi-instance channel names the instance it just created; every other channel has one id.
+        const instanceId = opened?.Instance ?? this.InstanceId;
+        this.EmitChannelEvent('opened', { inputs }, undefined, instanceId);
+        if (this.Exposure === 'none') {
+            // The agent opened it and may use it, but exposure policy keeps what it holds from the model.
+            this.Context?.SendContextNote(FormatChannelNote(descriptor.Key, instanceId, 'opened', { exposure: 'none' }));
+        }
+        else {
+            const state = this.GetState();
+            this.ensurePerception().SetBaseline(state);
+            this.Context?.SendContextNote(FormatChannelNote(descriptor.Key, instanceId, 'opened', { state }));
+        }
+        const extra = opened && IsPlainObject(opened.Result) ? opened.Result : {};
+        return { Success: true, Result: { opened: true, channel: descriptor.Key, instance: instanceId, ...extra } };
+    }
+    /**
+     * Subclass hook for {@link Open}: apply the (already validated) seed `inputs`. Return a failure
+     * result to refuse the open; return nothing for success. Default: nothing to do.
+     */
+    OnOpen(_inputs) {
+        // default: a channel with no seed data has nothing to apply
+    }
+    /**
+     * The channel finished and is handing something back (a submitted form, a chosen option, a
+     * finished game). Emits on {@link Output$} and {@link Events$} and tells the model with a
+     * `completed` note. The output is validated against the descriptor's `Output` schema when
+     * {@link StrictStateChecks} is on; a mismatch is logged, never swallowed into silence or thrown.
+     *
+     * @param output What the channel hands back.
+     * @param instanceId The instance that completed, for a multi-instance channel (default: the primary instance).
+     */
+    Complete(output, instanceId = this.InstanceId) {
+        const descriptor = this.GetDescriptor();
+        if (descriptor.Output && BaseRealtimeChannelClient.StrictStateChecks) {
+            const issues = ValidateJsonAgainstSchemaSubset(output, descriptor.Output, '$');
+            if (issues.length > 0) {
+                console.warn(`[RealtimeChannel:${descriptor.Key}] Complete() output does not match the descriptor's Output schema: ${issues.join('; ')}`);
+            }
+        }
+        this.channelOutputSubject.next({ Channel: descriptor.Key, Instance: instanceId, Output: output, OccurredAt: Date.now() });
+        this.EmitChannelEvent('completed', output, undefined, instanceId);
+        // The output is what the channel holds (a submitted form, a chosen option): state exposure decides
+        // whether the model is told it, or only that the channel finished.
+        this.Context?.SendContextNote(FormatChannelNote(descriptor.Key, instanceId, 'completed', this.Exposure === 'none' ? { exposure: 'none' } : output));
+    }
+    /** The id of the latest state change (0 before any). */
+    get CurrentChangeId() {
+        return this.channelChangeSeq;
+    }
+    /**
+     * Records that the channel's state changed. Call this on EVERY mutation, whoever caused it.
+     *
+     * It (1) assigns the change a monotonic id, (2) emits a `state_changed` event on {@link Events$}
+     * immediately (observers see every change), and (3) — unless `Perceive` is `false` — schedules a
+     * COALESCED structured note to the model: a burst of N changes inside the debounce window produces
+     * ONE `[channel:<Key>#<instance>] state_changed {"changeId":…,"changes":N,"delta":{…}}` note carrying
+     * what differs from what the model was last told (the first note is a full snapshot). Notes that
+     * would be too large degrade to a list of changed paths.
+     *
+     * @param options `Author` — who made the change (carried on the event); `Perceive` — whether the
+     *   model should be told (default `true`; a channel that already feeds the model its own perception
+     *   — like the Whiteboard's scene deltas — passes `false`).
+     * @returns The change's id.
+     */
+    RecordChange(options = {}) {
+        const id = ++this.channelChangeSeq;
+        this.EmitChannelEvent('state_changed', options.Author ? { author: options.Author } : {}, id);
+        // Observers of Events$ see every change; the MODEL is told only what exposure policy lets it perceive.
+        if (options.Perceive !== false && this.Exposure !== 'none') {
+            this.ensurePerception().Record(id);
+        }
+        return id;
+    }
+    /**
+     * Emits a typed event on {@link Events$} (NOT to the model — use {@link RecordChange} for state the
+     * model should perceive, or send an explicit note for a discrete event it must hear about).
+     *
+     * @param name The event name (declare it in the descriptor's `Events`).
+     * @param payload The event payload.
+     * @param changeId The change id the event relates to, when it relates to one.
+     * @param instanceId The instance the event is about, for a multi-instance channel (default: the primary instance).
+     */
+    EmitChannelEvent(name, payload = {}, changeId, instanceId = this.InstanceId) {
+        const event = {
+            Channel: this.ChannelName,
+            Instance: instanceId,
+            Name: name,
+            Payload: payload,
+            OccurredAt: Date.now()
+        };
+        if (changeId !== undefined) {
+            event.ChangeId = changeId;
+        }
+        this.channelEventsSubject.next(event);
+    }
+    /** Sends any pending coalesced perception note now (e.g. before a surface is torn down). */
+    FlushPerception() {
+        this.channelPerception?.Flush();
+    }
+    /** The perception coalescer, created on first use. */
+    ensurePerception() {
+        if (!this.channelPerception) {
+            const channel = this;
+            this.channelPerception = new ChannelPerceptionCoalescer({
+                get ChannelKey() {
+                    return channel.ChannelName;
+                },
+                get InstanceId() {
+                    return channel.InstanceId;
+                },
+                GetState: () => this.getStateChecked(),
+                SendNote: (text) => this.Context?.SendContextNote(text),
+                OnError: (error) => console.error(`[RealtimeChannel:${this.ChannelName}] Perception note failed:`, error)
+            }, this.PerceptionOptions);
+        }
+        return this.channelPerception;
+    }
+    /** {@link GetState}, with the development-time schema check applied when {@link StrictStateChecks} is on. */
+    getStateChecked() {
+        const state = this.GetState();
+        if (BaseRealtimeChannelClient.StrictStateChecks) {
+            for (const issue of this.ValidateState()) {
+                this.warnStateIssueOnce(issue, `State does not match its descriptor: ${issue}`);
+            }
+        }
+        return state;
+    }
+    /** Logs a state problem once per channel instance — these checks run on every perception note. */
+    warnStateIssueOnce(key, message) {
+        if (this.warnedStateIssueKeys.has(key)) {
+            return;
+        }
+        this.warnedStateIssueKeys.add(key);
+        console.warn(`[RealtimeChannel:${this.ChannelName}] ${message}`);
+    }
+    // ── Contract v2: artifacts from delegated runs ─────────────────────────────
+    /**
+     * Whether this channel wants to be offered the artifacts a delegated run just produced (the agent asked
+     * Skip or Sage to build something, and it came back as artifacts). A channel that hosts artifacts says yes
+     * for the ones it can show — and for an artifact it ALREADY shows, so a newer version replaces it in place.
+     *
+     * Asked of every channel in the session, mounted or merely advertised. For an advertised channel it is
+     * asked before the channel has been initialized, so it must not depend on {@link Context}; the channel's
+     * resolved configuration is passed in for exactly that reason. Saying yes for an advertised channel
+     * mounts it, so only answer yes when you will do something with the artifacts.
+     *
+     * Default: `false` — a channel that does not host artifacts is never bothered.
+     *
+     * @param artifacts What the delegated run produced.
+     * @param config This channel's resolved configuration (host defaults beneath agent/app config).
+     */
+    AcceptsDelegationArtifacts(_artifacts, _config) {
+        return false;
+    }
+    /**
+     * A delegated run produced artifacts this channel said it wants ({@link AcceptsDelegationArtifacts}). The
+     * channel is mounted and initialized by now. Never throw into the runtime: handle your own failures and
+     * report them (the runtime logs anything that escapes, but cannot recover the work).
+     *
+     * Default: nothing.
+     *
+     * @param artifacts What the delegated run produced.
+     */
+    OnDelegationArtifacts(_artifacts) {
+        // default: a channel that does not host artifacts ignores them
+    }
+    /**
+     * How much of this channel the model may perceive without asking: the lowest of the channel's own ceiling
+     * (`GetDescriptor().MaxExposure`), the server policy and the user's choice. A channel nobody has applied a
+     * policy to simply has its ceiling, which is what every channel had before exposure policy existed.
+     *
+     * - `'none'`: no state notes and no frames.
+     * - `'state'`: structured state notes; no frames.
+     * - `'pixels'`: state notes and frames.
+     *
+     * It governs what flows to the model UNPROMPTED (perception notes, the contents of `opened` / `completed`
+     * notes, frames). It does not REDACT what a verb's result contains: a verb is all-or-nothing. A verb whose
+     * result would show the model something above this level declares `ReturnsChannelData` on the verb, and
+     * when this level is below it the verb is refused whole (`exposure_restricted`) and listed to the model as
+     * unavailable; see {@link RefuseVerbForExposure}. A verb that does not declare it returns what the channel
+     * says it returns, so a channel whose verbs reveal sensitive data must either declare it or lower its own
+     * `MaxExposure`.
+     */
+    get Exposure() {
+        return MinExposure(this.GetDescriptor().MaxExposure, this.policyExposure, this.userExposure);
+    }
+    /** Why {@link Exposure} is below the channel's ceiling; empty when it is not. */
+    get ExposureReasons() {
+        return this.exposureReasons;
+    }
+    /**
+     * Whether the server policy (not the user) leaves room for pixels. Deliberately does not consult the
+     * descriptor: a synthesized descriptor derives its `MaxExposure` from {@link GetSourcedTracks}, which asks
+     * this, so reading it here would recurse. A channel that enabled visual perception has declared pixels.
+     */
+    policyAllowsPixels() {
+        return this.policyExposure === undefined || this.policyExposure === 'pixels';
+    }
+    /**
+     * Applies exposure policy: the server's decision and the user's own choice. The runtime calls this when
+     * it mounts the channel and again whenever the user changes their choice.
+     *
+     * Withdrawing exposure takes effect immediately: a pending perception note is dropped, a pending frame
+     * is dropped, and the channel's video source stops being forwarded. Restoring it re-baselines (the
+     * model's picture of the channel is stale, so the next note is a full snapshot) and sends a fresh frame.
+     * When the call is live the model is told about the change, with the reason, so it never assumes it
+     * can still see what it can't.
+     *
+     * @param settings `Policy` — the server-decided exposure; `User` — the user's choice (omit for none);
+     *   `Reasons` — why the policy lowered it, from the server's limits.
+     */
+    ApplyExposure(settings) {
+        const previous = this.Exposure;
+        this.policyExposure = settings.Policy;
+        this.userExposure = settings.User;
+        this.exposureReasons = [...(settings.Reasons ?? [])];
+        const next = this.Exposure;
+        if (previous === next) {
+            return;
+        }
+        this.applyExposureEffects(previous, next);
+        this.announceExposureChange(previous, next);
+    }
+    /** Makes a change in exposure real: stops what is no longer allowed, restarts what is again. */
+    applyExposureEffects(previous, next) {
+        const hadState = CompareExposure(previous, 'state') >= 0;
+        const hasState = CompareExposure(next, 'state') >= 0;
+        if (hadState && !hasState) {
+            this.channelPerception?.CancelPending();
+        }
+        else if (!hadState && hasState) {
+            // The model's picture of this channel went stale while it could not see it.
+            this.channelPerception?.ResetBaseline();
+            if (this.Context?.Client) {
+                this.ensurePerception().Record(this.channelChangeSeq);
+            }
+        }
+        const hadPixels = previous === 'pixels';
+        const hasPixels = next === 'pixels';
+        this.VisualVideoBridge?.SetSourceEnabled?.(hasPixels, false);
+        if (hadPixels && !hasPixels) {
+            this.visualFramePump?.CancelPending();
+        }
+        else if (!hadPixels && hasPixels) {
+            void this.NotifyVisualChange();
+        }
+    }
+    /** Emits the change on {@link Events$} and, when the call is live, tells the model what it can now perceive and why. */
+    announceExposureChange(previous, next) {
+        const ceiling = this.GetDescriptor().MaxExposure;
+        const limit = DescribeExposureLimit(next, ceiling, this.exposureReasons);
+        const payload = { exposure: next, was: previous };
+        if (limit) {
+            payload['limit'] = limit;
+        }
+        const unavailable = WithheldVerbs(this.GetDescriptor().Verbs, next).map((v) => v.Name);
+        if (unavailable.length > 0) {
+            payload['unavailableActions'] = unavailable;
+        }
+        this.EmitChannelEvent('exposure_changed', payload);
+        if (this.Context?.Client) {
+            this.Context.SendContextNote(FormatChannelNote(this.ChannelName, this.InstanceId, 'exposure_changed', payload));
+        }
+    }
+    /** Default frame cadence when the track negotiated none (1 fps — the Gemini Live ceiling). */
+    static { this.DEFAULT_VISUAL_CADENCE_MS = 1000; }
+    /** Floor for the negotiated cadence (4 fps) so a fast model never makes a channel spin. */
+    static { this.MIN_VISUAL_CADENCE_MS = 250; }
+    /**
+     * Opts this channel into VISUAL perception: its surface can be shown to a video-capable model as
+     * frames. Call it from {@link OnInitialize} with the channel's frame source.
+     *
+     * The channel then needs only to say *when* its picture changed — {@link NotifyVisualChange} after a
+     * user edit, {@link ConfirmVisualChange} after a successful agent edit — and the base class does the
+     * rest: change-driven (no idle heartbeat), paced to the negotiated cadence, a trailing settle so the
+     * model ends on the final state of a burst, dedupe of identical frames, and a single confirmation
+     * frame (with a "do not narrate" note) after an agent edit. Every frame is tagged with the change id
+     * of the state it was captured at (`frame_pushed` events), so state and pixels can be cross-checked.
+     *
+     * Also makes {@link GetSourcedTracks} report the inbound video track, which is what gets it
+     * negotiated. Idempotent: calling it again replaces the provider and options.
+     *
+     * @param provider Renders the channel's current picture as a base64 JPEG (or `null`).
+     * @param options See {@link VisualPerceptionOptions}.
+     */
+    EnableVisualPerception(provider, options = {}) {
+        this.visualFrameProvider = provider;
+        this.visualPerceptionOptions = options;
+        if (this.visualFramePump) {
+            this.visualFramePump.CancelPending();
+        }
+        else {
+            this.visualFramePump = this.createVisualPump(provider);
+        }
+        this.EnsureVideoBridge();
+    }
+    /**
+     * The channel's picture changed because of a USER action (or a scene replacement): push a frame,
+     * respecting the cadence. Resolves when the decision has been carried out; never rejects. A no-op
+     * until {@link EnableVisualPerception} ran and the session has an inbound video track.
+     */
+    NotifyVisualChange() {
+        return this.visualFramePump ? this.visualFramePump.OnChange() : Promise.resolve();
+    }
+    /**
+     * An AGENT action succeeded and changed the channel's picture: push exactly one confirmation frame
+     * now and tell the model not to narrate its own change. Call only on SUCCESS — confirming a failed
+     * action would assert an edit landed that did not. A no-op until {@link EnableVisualPerception} ran.
+     */
+    ConfirmVisualChange() {
+        return this.visualFramePump ? this.visualFramePump.Confirm() : Promise.resolve();
+    }
+    /**
+     * Returns the shared frame bridge, creating it if needed and (only when the channel opted into the
+     * legacy poller) starting its fixed-rate poll once the video track is up. `null` when no context is bound.
+     */
+    EnsureVideoBridge() {
+        if (!this.VisualVideoBridge && this.Context && this.visualFrameProvider) {
+            // The channel registers as a SOURCE with the session's video arbiter (the single writer of inbound
+            // video) rather than writing to the model itself, so when several sources are live the arbiter decides
+            // which one the model sees and tells it. The id and label are how the "agent can see" UI names it.
+            const descriptor = this.GetDescriptor();
+            this.VisualVideoBridge = new ChannelInboundVideoBridge(() => this.Context?.Client, this.visualFrameProvider, {
+                SourceID: ChannelVideoSourceID(descriptor.Key, this.InstanceId),
+                Label: descriptor.DisplayName,
+                Kind: 'surface',
+                ChannelKey: descriptor.Key,
+            });
+            // The pump sends only on change, and a frame the arbiter refused (another source had the model's only stream, or
+            // this one was off) never reached the model: when the arbiter sends this source again, send the current picture.
+            this.stopRefreshOnSourceActivated = this.VisualVideoBridge.OnSourceActivated(() => {
+                void this.visualFramePump?.Refresh();
+            });
+            if (this.Exposure !== 'pixels') {
+                this.VisualVideoBridge.SetSourceEnabled?.(false, false);
+            }
+        }
+        // List the source with the arbiter as soon as the video track is up, so the user can see and switch it
+        // before it has sent a frame (a source switched off never sends one, and would otherwise be unlistable).
+        // (Optional call: channels' tests install minimal bridge stubs that predate source registration, as with Start below.)
+        this.VisualVideoBridge?.Register?.();
+        if (this.visualPerceptionOptions.StartBridgePoller &&
+            this.VisualVideoBridge &&
+            !this.VisualVideoBridge.IsActive &&
+            this.Context?.Client?.IsTrackEstablished('video', 'inbound')) {
+            this.VisualVideoBridge.Start?.();
+        }
+        return this.VisualVideoBridge;
+    }
+    /** Builds the pump, wiring it to this channel's context, bridge and event stream. */
+    createVisualPump(provider) {
+        return new VisualPerceptionPump({
+            GetSink: () => this.EnsureVideoBridge(),
+            IsInboundVideoEstablished: () => this.Context?.Client?.IsTrackEstablished('video', 'inbound') ?? false,
+            IsPermitted: () => this.Exposure === 'pixels',
+            GetCadenceMs: () => this.getNegotiatedVisualCadenceMs(),
+            CaptureFrame: async () => provider.GetLatestFrame(),
+            GetChangeId: () => this.channelChangeSeq,
+            OnFramePushed: (frame, reason, changeId) => this.EmitChannelEvent('frame_pushed', { reason, bytes: frame.length }, changeId),
+            SendConfirmationNote: (changeId) => this.Context?.SendContextNote(this.confirmationNote(changeId)),
+            OnError: (context, error) => console.error(`[RealtimeChannel:${this.ChannelName}] Error in the visual ${context} path:`, error)
+        });
+    }
+    /** The "do not narrate your own change" note sent with a confirmation frame. */
+    confirmationNote(changeId) {
+        const configured = this.visualPerceptionOptions.ConfirmationNote;
+        if (typeof configured === 'string') {
+            return configured;
+        }
+        if (typeof configured === 'function') {
+            return configured(changeId);
+        }
+        return FormatChannelNote(this.ChannelName, this.InstanceId, 'frame_confirmed', {
+            changeId,
+            guidance: 'visual confirmation of your action (background — do NOT narrate or announce your own change; continue naturally)'
+        });
+    }
+    /**
+     * The frame cadence in ms: the rate the inbound video track negotiated (`1000 / Rate`), clamped to
+     * a 250ms floor; 1000ms when nothing was negotiated.
+     */
+    getNegotiatedVisualCadenceMs() {
+        const tracks = this.Context?.Client?.EstablishedTracks;
+        const rate = tracks?.find((t) => t.Descriptor.Modality === 'video' && t.Descriptor.Direction === 'inbound')?.Descriptor.Rate;
+        if (typeof rate === 'number' && rate > 0) {
+            return Math.max(BaseRealtimeChannelClient.MIN_VISUAL_CADENCE_MS, Math.floor(1000 / rate));
+        }
+        return BaseRealtimeChannelClient.DEFAULT_VISUAL_CADENCE_MS;
+    }
     /**
      * Tears the plugin down at session end: release the surface binding, unsubscribe
      * state-engine subscriptions, then drop the context. Subclasses overriding this MUST
@@ -227,7 +1009,79 @@ export class BaseRealtimeChannelClient {
      */
     Dispose() {
         this.UnbindSurface();
+        this.channelPerception?.Dispose();
+        this.channelPerception = null;
+        this.visualFramePump?.Dispose();
+        this.visualFramePump = null;
+        this.stopRefreshOnSourceActivated?.();
+        this.stopRefreshOnSourceActivated = null;
+        this.VisualVideoBridge?.Stop();
+        this.VisualVideoBridge = null;
+        this.channelEventsSubject.complete();
+        this.channelOutputSubject.complete();
         this.Context = null;
     }
 }
+/** The primary instance id of a single-instance channel. */
+const PRIMARY_CHANNEL_INSTANCE_ID = '1';
+/**
+ * The id of a channel's video source at the session's `VideoSourceArbiter`: `<channel key>#<instance id>`. The channel's
+ * own frame bridge and the runtime's {@link RealtimeChannelContext.SendVideoFrame} register under it, so a channel is
+ * one source however it sends, and the "agent can see" control lists it once.
+ *
+ * @param channelKey The channel's descriptor key.
+ * @param instanceId The channel instance.
+ */
+export function ChannelVideoSourceID(channelKey, instanceId) {
+    return `${channelKey}#${instanceId}`;
+}
+/** True when a Node-style environment reports a non-production `NODE_ENV` (never in a plain browser). */
+function isDevelopmentEnvironment() {
+    return typeof process !== 'undefined' && !!process.env && process.env['NODE_ENV'] !== undefined && process.env['NODE_ENV'] !== 'production';
+}
+/** Tolerantly parses native tool arguments into a plain object (`{}` for anything else). */
+function parseToolArguments(argsJson) {
+    try {
+        const parsed = argsJson ? JSON.parse(argsJson) : {};
+        // JSON.parse output is JSON by construction, so narrowing it to JSONObject is not a type lie.
+        return IsPlainObject(parsed) ? parsed : {};
+    }
+    catch {
+        return {};
+    }
+}
+/** Serializes a structured verb result for the native tool path (the model reads this JSON). */
+function serializeVerbResult(result) {
+    if (result.Success) {
+        return JSON.stringify(result.Result === undefined ? { success: true } : { success: true, result: result.Result });
+    }
+    const failure = { success: false, error: result.Error ?? 'The action could not be performed.' };
+    if (result.ErrorCode) {
+        failure['errorCode'] = result.ErrorCode;
+    }
+    if (result.Details) {
+        failure['details'] = result.Details;
+    }
+    return JSON.stringify(failure);
+}
+/**
+ * Adapts a legacy tool result string to a structured {@link RealtimeChannelVerbResult}: a JSON object
+ * with `success: false` is a failure (its `error`/`output`/`message` becomes the message); any other
+ * JSON is the result; a non-JSON string is the result as text.
+ */
+function parseLegacyVerbResult(resultJson) {
+    let parsed;
+    try {
+        parsed = JSON.parse(resultJson);
+    }
+    catch {
+        return { Success: true, Result: resultJson };
+    }
+    if (IsPlainObject(parsed) && parsed['success'] === false) {
+        const message = [parsed['error'], parsed['output'], parsed['message']].find((m) => typeof m === 'string' && m.length > 0);
+        return { Success: false, ErrorCode: 'verb_failed', Error: message ?? 'The action could not be performed.', Result: parsed };
+    }
+    // JSON.parse output is JSON by construction, so narrowing it to JSONValue is not a type lie.
+    return { Success: true, Result: parsed };
+}
 //# sourceMappingURL=base-realtime-channel-client.js.map