@memberjunction/server 6.2.0-edge.5 → 6.2.0-edge.7

package/dist/resolvers/RemoteBrowserActionResolver.js

@@ -30,7 +30,7 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
  */
 import { Resolver, Mutation, Query, Arg, Ctx, Float, Int, ObjectType, Field, PubSub, PubSubEngine } from 'type-graphql';
 import { LogError } from '@memberjunction/core';
-import { UUIDsEqual } from '@memberjunction/global';
+import { NormalizeUUID } from '@memberjunction/global';
 import { NormalizeInstanceKey, RemoteBrowserEngine } from '@memberjunction/remote-browser-server';
 import { BeginBrowserGoalStep, FinalizeBrowserGoalStep, ExtractCoAgentRunID } from '../agentSessions/remoteBrowserGoalEngine.js';
 import { RemoteBrowserGoalRegistry } from '../agentSessions/remoteBrowserGoalRegistry.js';
@@ -39,11 +39,15 @@ import { randomUUID } from 'node:crypto';
 import { RemoteBrowserCapabilityNotSupportedError, } from '@memberjunction/remote-browser-base';
 import { ResolverBase } from '../generic/ResolverBase.js';
 import { GetReadWriteProvider } from '../util.js';
+import { LoadOwnedRealtimeSession, RealtimeSessionNotFoundError, RealtimeSessionOwnershipError, } from '../realtimeSessions/ownedRealtimeSession.js';
+import { AuthorizeSessionAccess, ResolveSessionScopeIdentity } from '../realtimeSessions/sessionAccess.js';
+import { ConversationBrowserLedger, RemoteBrowserRefusedError } from '../agentSessions/conversationBrowsers.js';
+import { configInfo } from '../config.js';
 import { AIEngine } from '@memberjunction/aiengine';
 import { AIPromptRunner } from '@memberjunction/ai-prompts';
 import { AIPromptParams } from '@memberjunction/ai-core-plus';
-/** Entity name — centralised so the `MJ:`-prefix convention is applied in exactly one place. */
-const SESSION_ENTITY = 'MJ: AI Agent Sessions';
+/** Entity name for turn-based text conversations that scope a live Remote Browser session. */
+const CONVERSATION_ENTITY = 'MJ: Conversations';
 /** Entity name for the agent whose `TypeConfiguration` carries the remote-browser provider preference. */
 const AGENT_ENTITY = 'MJ: AI Agents';
 /**
@@ -55,7 +59,7 @@ const AGENT_ENTITY = 'MJ: AI Agents';
 const VISUAL_INTERPRETER_PROMPT_NAME = 'Remote Browser Visual Interpreter';
 /**
  * Result of {@link RemoteBrowserActionResolver.ExecuteRemoteBrowserAction} — the outcome of one browser
- * action plus the resulting URL the client narrates / displays.
+ * action plus the resulting URL and optional structured DOM observation / filled-field list.
  */
 let RemoteBrowserActionResult = class RemoteBrowserActionResult {
 };
@@ -71,6 +75,14 @@ __decorate([
     Field(() => String, { nullable: true }),
     __metadata("design:type", String)
 ], RemoteBrowserActionResult.prototype, "Detail", void 0);
+__decorate([
+    Field(() => String, { nullable: true }),
+    __metadata("design:type", String)
+], RemoteBrowserActionResult.prototype, "ObservationJson", void 0);
+__decorate([
+    Field(() => String, { nullable: true }),
+    __metadata("design:type", String)
+], RemoteBrowserActionResult.prototype, "FilledJson", void 0);
 RemoteBrowserActionResult = __decorate([
     ObjectType()
 ], RemoteBrowserActionResult);
@@ -250,7 +262,7 @@ export { RemoteBrowserInterpretation };
 const MAX_STREAM_ENTRY_AGE_MS = 4 * 60 * 60 * 1000; // 4 hours — generous vs. SessionJanitor's 15-minute idle threshold
 let RemoteBrowserActionResolver = class RemoteBrowserActionResolver extends ResolverBase {
     constructor() {
-        super(...arguments);
+        super();
         /**
          * Surfaces whose live CDP screencast this resolver has already started, keyed by
          * {@link RemoteBrowserActionResolver.streamKey} (agent session **plus** instance) so a re-issued
@@ -274,6 +286,15 @@ let RemoteBrowserActionResolver = class RemoteBrowserActionResolver extends Reso
          * {@link RemoteBrowserActionResolver.startedScreencasts} for why the value is a timestamp.
          */
         this.startedAudioStreams = new Map();
+        /**
+         * The push session each surface's frames and audio go to, by stream key. A reloaded tab starts again
+         * from a new push session while the browser keeps streaming; it takes the frames over here instead of
+         * being told "already streaming" while they still go to the old session. An entry whose streams have
+         * stopped is dropped by {@link sweepStreamEntries}.
+         */
+        this.streamTargets = new Map();
+        // The idle reaper ends a conversation browser the way EndRemoteBrowserSession does.
+        ConversationBrowserLedger.Instance.EndBrowser = (conversationID) => this.endBrowser(conversationID);
     }
     /**
      * Drops any {@link startedScreencasts} / {@link startedAudioStreams} entry older than
@@ -293,6 +314,12 @@ let RemoteBrowserActionResolver = class RemoteBrowserActionResolver extends Reso
                 this.startedAudioStreams.delete(key);
             }
         }
+        // A target outlives its streams only until here (a stop, a failed start, a voice session's end).
+        for (const key of this.streamTargets.keys()) {
+            if (!this.startedScreencasts.has(key) && !this.startedAudioStreams.has(key)) {
+                this.streamTargets.delete(key);
+            }
+        }
     }
     /**
      * Execute ONE browser action relayed from the client-direct realtime session, returning the outcome +
@@ -310,18 +337,24 @@ let RemoteBrowserActionResolver = class RemoteBrowserActionResolver extends Reso
      * @param kind The action kind (`'navigate' | 'click' | 'type' | 'key' | 'scroll' | 'back' | 'forward' | 'wait'`).
      * @returns The action result (success + resulting URL + detail).
      */
-    async ExecuteRemoteBrowserAction(agentSessionID, kind, { userPayload, providers }, url, selector, x, y, text, key, deltaX, deltaY, ms, instanceKey) {
+    async ExecuteRemoteBrowserAction(agentSessionID, kind, { userPayload, providers }, url, selector, x, y, text, key, deltaX, deltaY, ms, instanceKey, ref, valuesJson) {
         const { contextUser, provider } = this.requireUserAndProvider(userPayload, providers);
-        const session = await this.loadOwnedSession(agentSessionID, contextUser, provider);
-        const action = this.buildAction({ kind, url, selector, x, y, text, key, deltaX, deltaY, ms });
+        const session = await this.loadOwnedSession(agentSessionID, contextUser, provider, instanceKey);
+        const action = this.buildAction({ kind, url, selector, ref, x, y, text, key, deltaX, deltaY, ms, valuesJson });
         if (!action) {
             return { Success: false, Detail: `Unknown or incomplete remote-browser action '${kind}'.` };
         }
         const providerName = await this.resolveProviderName(session, contextUser, provider);
         try {
-            const liveSession = await RemoteBrowserEngine.Instance.StartSessionForAgentSession(agentSessionID, contextUser, providerName, instanceKey);
+            const liveSession = await this.startBrowser(agentSessionID, session, contextUser, providerName, instanceKey);
             const result = await liveSession.ExecuteAction(action);
-            return { Success: result.Success, CurrentUrl: result.CurrentUrl, Detail: result.Detail };
+            return {
+                Success: result.Success,
+                CurrentUrl: result.CurrentUrl,
+                Detail: result.Detail,
+                ...(result.Observation ? { ObservationJson: JSON.stringify(result.Observation) } : {}),
+                ...(result.Filled ? { FilledJson: JSON.stringify(result.Filled) } : {}),
+            };
         }
         catch (err) {
             // A dead handle is not a failure to report, it is a mapping to delete (#3598) — and this is the
@@ -347,6 +380,8 @@ let RemoteBrowserActionResolver = class RemoteBrowserActionResolver extends Reso
                         Detail: retried.Success
                             ? retried.Detail
                             : `The browser had closed and was replaced (it is now on a blank page). ${retried.Detail ?? ''}`.trim(),
+                        ...(retried.Observation ? { ObservationJson: JSON.stringify(retried.Observation) } : {}),
+                        ...(retried.Filled ? { FilledJson: JSON.stringify(retried.Filled) } : {}),
                     };
                 }
                 catch (retryError) {
@@ -376,7 +411,11 @@ let RemoteBrowserActionResolver = class RemoteBrowserActionResolver extends Reso
      */
     async ExecuteRemoteBrowserGoal(agentSessionID, goal, { userPayload, providers }, startUrl, maxSteps, preferredStrategy, instanceKey) {
         const { contextUser, provider } = this.requireUserAndProvider(userPayload, providers);
-        const session = await this.loadOwnedSession(agentSessionID, contextUser, provider);
+        const session = await this.loadOwnedSession(agentSessionID, contextUser, provider, instanceKey);
+        if (session.IsConversation) {
+            // A goal loop clicks and navigates on its own, past the click guard the browser_* tools go through.
+            return { Success: false, Status: 'Error', Detail: 'Goal runs are not available in a text conversation; use the browser_* tools.' };
+        }
         const providerName = await this.resolveProviderName(session, contextUser, provider);
         // Observability: nest this goal's many prompt runs under ONE "Browser goal" step on the realtime
         // co-agent run (when the session has one). Best-effort — a null step just means the goal runs unlinked.
@@ -444,7 +483,7 @@ let RemoteBrowserActionResolver = class RemoteBrowserActionResolver extends Reso
      */
     async RemoteBrowserSnapshot(agentSessionID, { userPayload, providers }, instanceKey) {
         const { contextUser, provider } = this.requireUserAndProvider(userPayload, providers);
-        await this.loadOwnedSession(agentSessionID, contextUser, provider);
+        await this.loadOwnedSession(agentSessionID, contextUser, provider, instanceKey);
         const liveSession = RemoteBrowserEngine.Instance.GetSessionForAgentSession(agentSessionID, instanceKey);
         if (!liveSession) {
             return {};
@@ -517,7 +556,7 @@ let RemoteBrowserActionResolver = class RemoteBrowserActionResolver extends Reso
      */
     async InterpretRemoteBrowserPage(agentSessionID, { userPayload, providers }, query, instanceKey) {
         const { contextUser, provider } = this.requireUserAndProvider(userPayload, providers);
-        await this.loadOwnedSession(agentSessionID, contextUser, provider);
+        await this.loadOwnedSession(agentSessionID, contextUser, provider, instanceKey);
         const liveSession = RemoteBrowserEngine.Instance.GetSessionForAgentSession(agentSessionID, instanceKey);
         if (!liveSession) {
             return { Description: undefined, Elements: [], Detail: 'no live browser' };
@@ -551,18 +590,19 @@ let RemoteBrowserActionResolver = class RemoteBrowserActionResolver extends Reso
      */
     async StartRemoteBrowserScreencast(agentSessionID, { userPayload, providers }, pubSub, instanceKey) {
         const { contextUser, provider } = this.requireUserAndProvider(userPayload, providers);
-        const session = await this.loadOwnedSession(agentSessionID, contextUser, provider);
+        const session = await this.loadOwnedSession(agentSessionID, contextUser, provider, instanceKey);
         // Idempotent PER SURFACE: a re-bind must not stack a second screencast on the one live browser,
         // but a SECOND surface binding for the first time is not a re-bind (#3531).
         this.sweepStreamEntries();
         const streamKey = this.streamKey(agentSessionID, instanceKey);
+        this.streamTargets.set(streamKey, userPayload);
         if (this.startedScreencasts.has(streamKey)) {
             return { Streaming: true };
         }
         const providerName = await this.resolveProviderName(session, contextUser, provider);
         try {
-            const liveSession = await RemoteBrowserEngine.Instance.StartSessionForAgentSession(agentSessionID, contextUser, providerName, instanceKey);
-            await liveSession.StartScreencast((frame) => this.publishFrame(pubSub, userPayload, agentSessionID, instanceKey, frame, liveSession.GetCurrentUrl()));
+            const liveSession = await this.startBrowser(agentSessionID, session, contextUser, providerName, instanceKey);
+            await liveSession.StartScreencast((frame) => this.publishFrame(pubSub, this.streamTargets.get(streamKey) ?? userPayload, agentSessionID, instanceKey, frame, liveSession.GetCurrentUrl()));
             this.startedScreencasts.set(streamKey, Date.now());
             return { Streaming: true };
         }
@@ -571,6 +611,9 @@ let RemoteBrowserActionResolver = class RemoteBrowserActionResolver extends Reso
                 // Backend can't stream — the client keeps polling. Not an error condition.
                 return { Streaming: false };
             }
+            if (err instanceof RemoteBrowserRefusedError) {
+                throw err; // the client must stop and say so, not poll for a browser that will never start
+            }
             const message = err instanceof Error ? err.message : String(err);
             LogError(`StartRemoteBrowserScreencast failed (provider='${providerName}'): ${message}`);
             return { Streaming: false };
@@ -586,7 +629,7 @@ let RemoteBrowserActionResolver = class RemoteBrowserActionResolver extends Reso
      */
     async StopRemoteBrowserScreencast(agentSessionID, { userPayload, providers }, instanceKey) {
         const { contextUser, provider } = this.requireUserAndProvider(userPayload, providers);
-        await this.loadOwnedSession(agentSessionID, contextUser, provider);
+        await this.loadOwnedSession(agentSessionID, contextUser, provider, instanceKey);
         this.startedScreencasts.delete(this.streamKey(agentSessionID, instanceKey));
         const liveSession = RemoteBrowserEngine.Instance.GetSessionForAgentSession(agentSessionID, instanceKey);
         if (liveSession) {
@@ -601,6 +644,33 @@ let RemoteBrowserActionResolver = class RemoteBrowserActionResolver extends Reso
         }
         return true;
     }
+    /**
+     * Ends the browser the caller's own session or conversation started, and forgets its stream
+     * bookkeeping. Ownership-gated. Idempotent: ending a scope that holds no browser resolves `false`.
+     * A text chat calls this when its browser drawer closes, the conversation changes, or the chat is
+     * destroyed, so a closed drawer never leaves a Chromium process (or a billed hosted browser) running.
+     *
+     * @param agentSessionID The `AIAgentSession` id or `Conversation` id.
+     * @returns `true` when a live browser was ended.
+     */
+    async EndRemoteBrowserSession(agentSessionID, { userPayload, providers }, instanceKey) {
+        const { contextUser, provider } = this.requireUserAndProvider(userPayload, providers);
+        await this.loadOwnedSession(agentSessionID, contextUser, provider, instanceKey);
+        return this.endBrowser(agentSessionID, instanceKey);
+    }
+    /**
+     * Ends a scope's browser and everything kept about it: the ledger slot (released first, so an action
+     * landing during teardown is counted again), the stream entries (so the next open streams the next
+     * browser) and the browser itself. The idle reaper ends conversation browsers through here too.
+     */
+    async endBrowser(agentSessionID, instanceKey) {
+        ConversationBrowserLedger.Instance.Release(agentSessionID);
+        const streamKey = this.streamKey(agentSessionID, instanceKey);
+        this.startedScreencasts.delete(streamKey);
+        this.startedAudioStreams.delete(streamKey);
+        this.streamTargets.delete(streamKey);
+        return RemoteBrowserEngine.Instance.EndSessionForAgentSession(agentSessionID, instanceKey);
+    }
     /**
      * Starts streaming the session browser's TAB AUDIO and PUSHES each encoded chunk to the calling user's
      * push-status topic — so a co-agent demoing a video/audio site is HEARD, not just seen. Ownership-gated.
@@ -616,17 +686,18 @@ let RemoteBrowserActionResolver = class RemoteBrowserActionResolver extends Reso
      */
     async StartRemoteBrowserAudioStream(agentSessionID, { userPayload, providers }, pubSub, instanceKey) {
         const { contextUser, provider } = this.requireUserAndProvider(userPayload, providers);
-        const session = await this.loadOwnedSession(agentSessionID, contextUser, provider);
+        const session = await this.loadOwnedSession(agentSessionID, contextUser, provider, instanceKey);
         // Idempotent PER SURFACE — same reasoning as the screencast map (#3531).
         this.sweepStreamEntries();
         const streamKey = this.streamKey(agentSessionID, instanceKey);
+        this.streamTargets.set(streamKey, userPayload);
         if (this.startedAudioStreams.has(streamKey)) {
             return { Streaming: true };
         }
         const providerName = await this.resolveProviderName(session, contextUser, provider);
         try {
-            const liveSession = await RemoteBrowserEngine.Instance.StartSessionForAgentSession(agentSessionID, contextUser, providerName, instanceKey);
-            await liveSession.StartAudioStream((chunk) => this.publishAudioChunk(pubSub, userPayload, agentSessionID, instanceKey, chunk));
+            const liveSession = await this.startBrowser(agentSessionID, session, contextUser, providerName, instanceKey);
+            await liveSession.StartAudioStream((chunk) => this.publishAudioChunk(pubSub, this.streamTargets.get(streamKey) ?? userPayload, agentSessionID, instanceKey, chunk));
             this.startedAudioStreams.set(streamKey, Date.now());
             return { Streaming: true };
         }
@@ -635,6 +706,9 @@ let RemoteBrowserActionResolver = class RemoteBrowserActionResolver extends Reso
                 // Backend can't capture audio — the client plays no audio. Not an error condition.
                 return { Streaming: false };
             }
+            if (err instanceof RemoteBrowserRefusedError) {
+                throw err;
+            }
             const message = err instanceof Error ? err.message : String(err);
             LogError(`StartRemoteBrowserAudioStream failed (provider='${providerName}'): ${message}`);
             return { Streaming: false };
@@ -650,7 +724,7 @@ let RemoteBrowserActionResolver = class RemoteBrowserActionResolver extends Reso
      */
     async StopRemoteBrowserAudioStream(agentSessionID, { userPayload, providers }, instanceKey) {
         const { contextUser, provider } = this.requireUserAndProvider(userPayload, providers);
-        await this.loadOwnedSession(agentSessionID, contextUser, provider);
+        await this.loadOwnedSession(agentSessionID, contextUser, provider, instanceKey);
         this.startedAudioStreams.delete(this.streamKey(agentSessionID, instanceKey));
         const liveSession = RemoteBrowserEngine.Instance.GetSessionForAgentSession(agentSessionID, instanceKey);
         if (liveSession) {
@@ -691,7 +765,7 @@ let RemoteBrowserActionResolver = class RemoteBrowserActionResolver extends Reso
      */
     async RelayRemoteBrowserHumanInput(agentSessionID, kind, { userPayload, providers }, x, y, button, key, text, deltaX, deltaY, modifiers, instanceKey) {
         const { contextUser, provider } = this.requireUserAndProvider(userPayload, providers);
-        await this.loadOwnedSession(agentSessionID, contextUser, provider);
+        await this.loadOwnedSession(agentSessionID, contextUser, provider, instanceKey);
         const liveSession = RemoteBrowserEngine.Instance.GetSessionForAgentSession(agentSessionID, instanceKey);
         if (!liveSession) {
             return false;
@@ -730,7 +804,7 @@ let RemoteBrowserActionResolver = class RemoteBrowserActionResolver extends Reso
      */
     async GetRemoteBrowserSelection(agentSessionID, { userPayload, providers }, instanceKey) {
         const { contextUser, provider } = this.requireUserAndProvider(userPayload, providers);
-        await this.loadOwnedSession(agentSessionID, contextUser, provider);
+        await this.loadOwnedSession(agentSessionID, contextUser, provider, instanceKey);
         const liveSession = RemoteBrowserEngine.Instance.GetSessionForAgentSession(agentSessionID, instanceKey);
         if (!liveSession) {
             return { Text: '' };
@@ -939,21 +1013,83 @@ let RemoteBrowserActionResolver = class RemoteBrowserActionResolver extends Reso
         return { contextUser, provider: GetReadWriteProvider(providers) };
     }
     /**
-     * Loads the agent session and enforces inbound ownership: the session's `UserID` must equal the
-     * caller's. Throws when the session is missing or owned by another user.
+     * Loads the owning scope and enforces inbound ownership: an `MJ: AI Agent Sessions` row (a realtime
+     * call, via `LoadOwnedRealtimeSession`) or, failing that, an `MJ: Conversations` row (a text chat's
+     * browser), both under `AuthorizeSessionAccess`. Throws when neither exists or the caller may not use it.
      *
-     * @param agentSessionID The `AIAgentSession` id.
-     * @returns The loaded, owned session entity.
+     * @param agentSessionID The `AIAgentSession` id or `Conversation` id.
+     * @returns The owned scope.
+     */
+    async loadOwnedSession(agentSessionID, contextUser, provider, instanceKey) {
+        try {
+            return await LoadOwnedRealtimeSession(agentSessionID, contextUser, provider);
+        }
+        catch (err) {
+            if (err instanceof RealtimeSessionOwnershipError) {
+                throw new RemoteBrowserRefusedError(`Not authorized: remote-browser session ${agentSessionID} is not owned by you.`);
+            }
+            if (!(err instanceof RealtimeSessionNotFoundError)) {
+                throw err;
+            }
+        }
+        // Not a realtime session: a text chat scopes its browser to a conversation, which this server must opt
+        // into (each browser is a server Chromium or a billed hosted session), under the same access rule
+        // (owner, and for an anonymous widget guest its scope id), loaded as the caller so its RLS applies too.
+        const settings = this.ConversationBrowserSettings();
+        if (!settings.enabled) {
+            throw new RemoteBrowserRefusedError(`Remote-browser session ${agentSessionID} not found. Conversation browsers are not enabled on this server (realtime.conversationBrowser.enabled).`);
+        }
+        if (contextUser.IsMagicLinkAnonymous && !settings.allowAnonymous) {
+            throw new RemoteBrowserRefusedError(`Not authorized: anonymous users cannot open a conversation browser on this server.`);
+        }
+        // One browser per conversation: an instance key would key a second one past the cap, the network
+        // policy and the reaper (the client never sends one).
+        if (instanceKey?.trim()) {
+            throw new RemoteBrowserRefusedError('A conversation browser has no instances; instanceKey is not accepted.');
+        }
+        const conversation = await provider.GetEntityObject(CONVERSATION_ENTITY, contextUser);
+        if (!(await conversation.Load(agentSessionID))) {
+            throw new RemoteBrowserRefusedError(`Remote-browser session ${agentSessionID} not found.`);
+        }
+        const access = AuthorizeSessionAccess({ SessionUserID: conversation.UserID, ConversationExternalID: conversation.ExternalID, ConversationReadable: true }, contextUser);
+        if (!access.Allowed) {
+            throw new RemoteBrowserRefusedError(`Not authorized: remote-browser session ${agentSessionID} is not owned by you.`);
+        }
+        // Each use puts off the browser's idle end.
+        ConversationBrowserLedger.Instance.Touch(agentSessionID);
+        return { ID: conversation.ID, AgentID: null, Config_: null, IsConversation: true };
+    }
+    /** This server's conversation-browser settings (a seam for tests). */
+    ConversationBrowserSettings() {
+        return configInfo.realtime.conversationBrowser;
+    }
+    /**
+     * Starts (or reuses) the scope's browser. A conversation's browser first counts against its owner's
+     * cap, and gives the slot back when it fails to start.
      */
-    async loadOwnedSession(agentSessionID, contextUser, provider) {
-        const session = await provider.GetEntityObject(SESSION_ENTITY, contextUser);
-        if (!(await session.Load(agentSessionID))) {
-            throw new Error(`Remote-browser session ${agentSessionID} not found.`);
+    async startBrowser(agentSessionID, scope, contextUser, providerName, instanceKey) {
+        if (scope.IsConversation) {
+            const settings = this.ConversationBrowserSettings();
+            // ponytail: an action that joins a launch a concurrent End then tears down keeps this slot until the
+            // reaper frees it (idleMinutes); release it on that teardown if users hit maxPerUser this way.
+            // An anonymous guest's cap is its own: every guest is the one shared Anonymous user.
+            const guestScope = contextUser.IsMagicLinkAnonymous ? ResolveSessionScopeIdentity(contextUser) : undefined;
+            const owner = { Key: guestScope ? `scope:${guestScope}` : `user:${NormalizeUUID(contextUser.ID)}`, Anonymous: Boolean(contextUser.IsMagicLinkAnonymous) };
+            ConversationBrowserLedger.Instance.Claim(agentSessionID, owner, settings);
+            // Set once, before the browser launches; the engine re-applies it to any replacement.
+            if (!RemoteBrowserEngine.Instance.GetSessionForAgentSession(agentSessionID, instanceKey)) {
+                await RemoteBrowserEngine.Instance.RestrictPrivateNetworkForAgentSession(agentSessionID, settings.allowedPrivateHosts);
+            }
         }
-        if (!UUIDsEqual(session.UserID, contextUser.ID)) {
-            throw new Error(`Not authorized: remote-browser session ${agentSessionID} is not owned by you.`);
+        try {
+            return await RemoteBrowserEngine.Instance.StartSessionForAgentSession(agentSessionID, contextUser, providerName, instanceKey);
+        }
+        catch (err) {
+            if (scope.IsConversation && !RemoteBrowserEngine.Instance.GetSessionForAgentSession(agentSessionID, instanceKey)) {
+                ConversationBrowserLedger.Instance.Release(agentSessionID);
+            }
+            throw err;
         }
-        return session;
     }
     /**
      * Resolves the remote-browser backend name for the session from the session's agent's
@@ -961,10 +1097,13 @@ let RemoteBrowserActionResolver = class RemoteBrowserActionResolver extends Reso
      * config, or unparseable JSON all yield `undefined`, letting the engine fall back to the single Active
      * provider.
      *
-     * @param session The owned session entity (supplies the agent id).
+     * @param session The owned scope (supplies the agent id; a conversation has none).
      * @returns The configured backend name, or `undefined` to let the engine auto-select.
      */
     async resolveProviderName(session, contextUser, provider) {
+        if (!session.AgentID) {
+            return undefined;
+        }
         // The session's agent IS the co-agent (the realtime voice agent), and the interactive channels
         // (Remote Browser, Whiteboard) are the CO-AGENT's abilities — so the remoteBrowser backend config
         // lives on the co-agent's TypeConfiguration, not on the target agent it voices.
@@ -1014,8 +1153,15 @@ let RemoteBrowserActionResolver = class RemoteBrowserActionResolver extends Reso
             case 'navigate':
                 return input.url ? { Kind: 'navigate', Url: input.url } : null;
             case 'click':
-                if (input.selector || (typeof input.x === 'number' && typeof input.y === 'number')) {
-                    return { Kind: 'click', Selector: input.selector, X: input.x, Y: input.y };
+                if (input.selector || input.ref || input.text || (typeof input.x === 'number' && typeof input.y === 'number')) {
+                    return {
+                        Kind: 'click',
+                        Selector: input.selector,
+                        Ref: input.ref,
+                        Text: input.text,
+                        X: input.x,
+                        Y: input.y,
+                    };
                 }
                 return null;
             case 'type':
@@ -1036,10 +1182,37 @@ let RemoteBrowserActionResolver = class RemoteBrowserActionResolver extends Reso
                     return { Kind: 'wait', Ms: input.ms, Selector: input.selector };
                 }
                 return null;
+            case 'inspect':
+                return { Kind: 'inspect' };
+            case 'getPageText':
+                return { Kind: 'getPageText' };
+            case 'fill': {
+                const values = this.parseFillValues(input.valuesJson);
+                return values.length > 0 ? { Kind: 'fill', Values: values } : null;
+            }
             default:
                 return null;
         }
     }
+    /**
+     * Parses the `valuesJson` payload for a `fill` action: a JSON object mapping a field ref (`f1`) or
+     * label to its value (`string`, or `boolean` for a checkbox). Anything else yields no values.
+     */
+    parseFillValues(valuesJson) {
+        let parsed;
+        try {
+            parsed = JSON.parse(valuesJson ?? '');
+        }
+        catch {
+            return [];
+        }
+        if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+            return [];
+        }
+        return Object.entries(parsed)
+            .filter(([target, value]) => target.trim().length > 0 && value !== undefined && value !== null)
+            .map(([target, value]) => ({ Target: target.trim(), Value: typeof value === 'boolean' ? value : String(value) }));
+    }
     /**
      * Builds a strongly-typed {@link RemoteBrowserHumanInput} from the relayed `kind` + fields, validating
      * each kind's required field(s). Returns `null` for an unknown kind or a kind missing its required
@@ -1138,8 +1311,10 @@ __decorate([
     __param(10, Arg('deltaY', () => Float, { nullable: true })),
     __param(11, Arg('ms', () => Float, { nullable: true })),
     __param(12, Arg('instanceKey', () => String, { nullable: true })),
+    __param(13, Arg('ref', () => String, { nullable: true })),
+    __param(14, Arg('valuesJson', () => String, { nullable: true })),
     __metadata("design:type", Function),
-    __metadata("design:paramtypes", [String, String, Object, String, String, Number, Number, String, String, Number, Number, Number, String]),
+    __metadata("design:paramtypes", [String, String, Object, String, String, Number, Number, String, String, Number, Number, Number, String, String, String]),
     __metadata("design:returntype", Promise)
 ], RemoteBrowserActionResolver.prototype, "ExecuteRemoteBrowserAction", null);
 __decorate([
@@ -1202,6 +1377,15 @@ __decorate([
     __metadata("design:paramtypes", [String, Object, String]),
     __metadata("design:returntype", Promise)
 ], RemoteBrowserActionResolver.prototype, "StopRemoteBrowserScreencast", null);
+__decorate([
+    Mutation(() => Boolean),
+    __param(0, Arg('agentSessionID', () => String)),
+    __param(1, Ctx()),
+    __param(2, Arg('instanceKey', () => String, { nullable: true })),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [String, Object, String]),
+    __metadata("design:returntype", Promise)
+], RemoteBrowserActionResolver.prototype, "EndRemoteBrowserSession", null);
 __decorate([
     Mutation(() => RemoteBrowserAudioStreamResult),
     __param(0, Arg('agentSessionID', () => String)),
@@ -1249,7 +1433,8 @@ __decorate([
     __metadata("design:returntype", Promise)
 ], RemoteBrowserActionResolver.prototype, "GetRemoteBrowserSelection", null);
 RemoteBrowserActionResolver = __decorate([
-    Resolver()
+    Resolver(),
+    __metadata("design:paramtypes", [])
 ], RemoteBrowserActionResolver);
 export { RemoteBrowserActionResolver };
 //# sourceMappingURL=RemoteBrowserActionResolver.js.map