@memberjunction/server 5.41.0 → 5.42.0

package/src/resolvers/RemoteBrowserActionResolver.ts

@@ -16,13 +16,22 @@
  *
  * @module @memberjunction/server
  */
-import { Resolver, Mutation, Query, Arg, Ctx, Float, ObjectType, Field, PubSub, PubSubEngine } from 'type-graphql';
+import { Resolver, Mutation, Query, Arg, Ctx, Float, Int, ObjectType, Field, PubSub, PubSubEngine } from 'type-graphql';
 import { AppContext, UserPayload } from '../types.js';
 import { UserInfo, IMetadataProvider, LogError } from '@memberjunction/core';
 import { UUIDsEqual } from '@memberjunction/global';
 import { MJAIAgentSessionEntity, MJAIAgentEntity } from '@memberjunction/core-entities';
 import { RemoteBrowserEngine } from '@memberjunction/remote-browser-server';
-import { IRemoteBrowserSession, RemoteBrowserAction, RemoteBrowserAudioChunk, RemoteBrowserHumanInput, RemoteBrowserModifierKey, RemoteBrowserCapabilityNotSupportedError } from '@memberjunction/remote-browser-base';
+import { beginBrowserGoalStep, finalizeBrowserGoalStep, extractCoAgentRunID } from '../agentSessions/remoteBrowserGoalEngine.js';
+import { RemoteBrowserGoalRegistry } from '../agentSessions/remoteBrowserGoalRegistry.js';
+import { randomUUID } from 'node:crypto';
+import {
+  RemoteBrowserAction,
+  RemoteBrowserAudioChunk,
+  RemoteBrowserHumanInput,
+  RemoteBrowserModifierKey,
+  RemoteBrowserCapabilityNotSupportedError,
+} from '@memberjunction/remote-browser-base';
 import { ResolverBase } from '../generic/ResolverBase.js';
 import { GetReadWriteProvider } from '../util.js';
 import { PUSH_STATUS_UPDATES_TOPIC } from '../generic/PushStatusResolver.js';
@@ -72,6 +81,45 @@ export class RemoteBrowserActionResult {
   Detail?: string;
 }
 
+/**
+ * Result of {@link RemoteBrowserActionResolver.ExecuteRemoteBrowserGoal} — the outcome of an autonomous,
+ * goal-driven browser run (computer-use loop or backend native AI).
+ */
+@ObjectType()
+export class RemoteBrowserGoalResultType {
+  /** Whether the goal was achieved. */
+  @Field(() => Boolean)
+  Success: boolean;
+
+  /**
+   * Handle for the async goal run. `ExecuteRemoteBrowserGoal` STARTS the goal and returns this with
+   * `Status: 'Running'`; the client polls {@link RemoteBrowserActionResolver.GetRemoteBrowserGoalResult}
+   * with it until the run reports a terminal status.
+   */
+  @Field(() => String, { nullable: true })
+  GoalRunID?: string;
+
+  /** Which control strategy executed the goal (`ComputerUse` / `NativeAI`). */
+  @Field(() => String, { nullable: true })
+  Strategy?: string;
+
+  /** The page URL when the run ended, when known. */
+  @Field(() => String, { nullable: true })
+  CurrentUrl?: string;
+
+  /** Terminal status label (`Completed` / `MaxStepsReached` / `Impossible` / `Error` / …). */
+  @Field(() => String, { nullable: true })
+  Status?: string;
+
+  /** Number of perceive-act steps executed (computer-use strategy). */
+  @Field(() => Int, { nullable: true })
+  StepCount?: number;
+
+  /** Human-readable detail (judge feedback / error message). */
+  @Field(() => String, { nullable: true })
+  Detail?: string;
+}
+
 /**
  * Result of {@link RemoteBrowserActionResolver.RemoteBrowserSnapshot} — the current viewport screenshot +
  * URL for the client's live view. Both fields are null when the session holds no live browser.
@@ -113,6 +161,18 @@ export class RemoteBrowserAudioStreamResult {
   Streaming: boolean;
 }
 
+/**
+ * Result of {@link RemoteBrowserActionResolver.GetRemoteBrowserSelection} — the remote page's current text
+ * selection, the copy-out half of human clipboard support. `Text` is `''` when nothing is selected, no live
+ * browser exists, or the backend can't read the selection (the client then writes nothing to the clipboard).
+ */
+@ObjectType()
+export class RemoteBrowserSelection {
+  /** The remote page's current selection text, or `''` when nothing is selected / unavailable. */
+  @Field(() => String)
+  Text: string;
+}
+
 /**
  * One UI element the visual interpreter localized in the screenshot — a label plus the pixel centroid the
  * voice agent can feed straight into `browser_Click(x, y)`. Coordinates are in the SCREENSHOT's own pixel
@@ -241,6 +301,93 @@ export class RemoteBrowserActionResolver extends ResolverBase {
     }
   }
 
+  /**
+   * Execute a high-level GOAL against the session's browser — the agent sets an intent ("log in and open
+   * the latest invoice") and the resolved control strategy (computer-use loop or backend native AI) plans
+   * + executes it autonomously, instead of relaying granular actions. Ownership-gated; lazily starts the
+   * browser. Returns the terminal outcome (this is a request/response mutation — for live progress
+   * narration in a server-bridged realtime session the broker calls {@link RemoteBrowserEngine.AchieveGoal}
+   * in-process with an `OnProgress` callback).
+   *
+   * @param agentSessionID The `AIAgentSession` id the browser is bound to.
+   * @param goal The natural-language goal.
+   * @returns The goal outcome (success, strategy, status, step count, url, detail).
+   */
+  @Mutation(() => RemoteBrowserGoalResultType)
+  async ExecuteRemoteBrowserGoal(
+    @Arg('agentSessionID', () => String) agentSessionID: string,
+    @Arg('goal', () => String) goal: string,
+    @Ctx() { userPayload, providers }: AppContext,
+    @Arg('startUrl', () => String, { nullable: true }) startUrl?: string,
+    @Arg('maxSteps', () => Int, { nullable: true }) maxSteps?: number,
+    @Arg('preferredStrategy', () => String, { nullable: true }) preferredStrategy?: string,
+  ): Promise<RemoteBrowserGoalResultType> {
+    const { contextUser, provider } = this.requireUserAndProvider(userPayload, providers);
+    const session = await this.loadOwnedSession(agentSessionID, contextUser, provider);
+    const providerName = await this.resolveProviderName(session, contextUser, provider);
+    // Observability: nest this goal's many prompt runs under ONE "Browser goal" step on the realtime
+    // co-agent run (when the session has one). Best-effort — a null step just means the goal runs unlinked.
+    const coAgentRunID = extractCoAgentRunID(session.Config_);
+    const goalStep = await beginBrowserGoalStep(provider, contextUser, coAgentRunID, goal);
+
+    // ASYNC START: a goal loop can run for minutes; do NOT hold this request open for it (browser
+    // fetch / proxy / janitor timeouts would kill the request while the loop runs on, and the agent
+    // would get "no response from the server" despite a successful run). Register the run, kick the
+    // loop off WITHOUT awaiting, and return a GoalRunID the client polls via GetRemoteBrowserGoalResult.
+    const goalRunID = randomUUID();
+    RemoteBrowserGoalRegistry.Instance.Begin(agentSessionID, goalRunID);
+    void RemoteBrowserEngine.Instance.AchieveGoal(agentSessionID, goal, {
+      ContextUser: contextUser,
+      ProviderName: providerName,
+      StartUrl: startUrl,
+      MaxSteps: maxSteps,
+      PreferredStrategy: preferredStrategy === 'NativeAI' || preferredStrategy === 'ComputerUse' ? preferredStrategy : undefined,
+      AgentRunID: coAgentRunID,
+      AgentRunStepID: goalStep?.ID,
+    })
+      .then(async (result) => {
+        await finalizeBrowserGoalStep(goalStep, result);
+        RemoteBrowserGoalRegistry.Instance.Complete(agentSessionID, goalRunID, result);
+      })
+      .catch(async (err) => {
+        const message = err instanceof Error ? err.message : String(err);
+        LogError(`ExecuteRemoteBrowserGoal failed (provider='${providerName}'): ${message}`);
+        const failure = { Success: false, Status: 'Error', Detail: `Remote browser error (${providerName}): ${message}` };
+        await finalizeBrowserGoalStep(goalStep, failure);
+        RemoteBrowserGoalRegistry.Instance.Complete(agentSessionID, goalRunID, failure);
+      });
+
+    return { Success: true, Status: 'Running', GoalRunID: goalRunID, Detail: 'Goal started.' };
+  }
+
+  /**
+   * Poll the outcome of a goal STARTED by {@link RemoteBrowserActionResolver.ExecuteRemoteBrowserGoal}.
+   * Returns `Status: 'Running'` while the loop is in flight, then the terminal outcome (success,
+   * strategy, status, step count, url, detail) once it finishes. Ownership-gated. A `Status: 'Unknown'`
+   * result means the run id is unrecognized — it expired (results are retained briefly) or never existed.
+   *
+   * @param agentSessionID The `AIAgentSession` id the goal runs against.
+   * @param goalRunID The handle returned by `ExecuteRemoteBrowserGoal`.
+   * @returns The current/terminal goal outcome.
+   */
+  @Query(() => RemoteBrowserGoalResultType)
+  async GetRemoteBrowserGoalResult(
+    @Arg('agentSessionID', () => String) agentSessionID: string,
+    @Arg('goalRunID', () => String) goalRunID: string,
+    @Ctx() { userPayload, providers }: AppContext,
+  ): Promise<RemoteBrowserGoalResultType> {
+    const { contextUser, provider } = this.requireUserAndProvider(userPayload, providers);
+    await this.loadOwnedSession(agentSessionID, contextUser, provider);
+    const record = RemoteBrowserGoalRegistry.Instance.Get(agentSessionID, goalRunID);
+    if (!record) {
+      return { Success: false, Status: 'Unknown', GoalRunID: goalRunID, Detail: 'No such goal run (it may have expired).' };
+    }
+    if (record.Status === 'Running' || !record.Outcome) {
+      return { Success: false, Status: 'Running', GoalRunID: goalRunID };
+    }
+    return { ...record.Outcome, GoalRunID: goalRunID };
+  }
+
   /**
    * Return the current viewport screenshot + URL for the session's live browser — the client's live view.
    * Ownership-gated. When the session holds no live browser (never started, or already torn down), the
@@ -481,7 +628,8 @@ export class RemoteBrowserActionResolver extends ResolverBase {
    * agent⇄human floor, so we just relay. Finer floor / `AgentOnly` gating is a follow-up.
    *
    * @param agentSessionID The `AIAgentSession` id the browser is bound to.
-   * @param kind The input kind (`'pointer-move' | 'pointer-click' | 'pointer-down' | 'pointer-up' | 'key' | 'scroll'`).
+   * @param kind The input kind (`'pointer-move' | 'pointer-click' | 'pointer-down' | 'pointer-up' | 'key' | 'text' | 'scroll'`).
+   * @param text The pasted text (the `'text'` paste-in kind only) — inserted into the page's focused element.
    * @param modifiers Optional comma-separated modifier keys held during the input (`'Shift'`, `'Control'`,
    *   `'Alt'`, `'Meta'`) — carries Shift-click selection and Ctrl/Cmd+key chords faithfully.
    * @returns `true` when the input was routed, else `false`.
@@ -495,6 +643,7 @@ export class RemoteBrowserActionResolver extends ResolverBase {
     @Arg('y', () => Float, { nullable: true }) y?: number,
     @Arg('button', () => String, { nullable: true }) button?: string,
     @Arg('key', () => String, { nullable: true }) key?: string,
+    @Arg('text', () => String, { nullable: true }) text?: string,
     @Arg('deltaX', () => Float, { nullable: true }) deltaX?: number,
     @Arg('deltaY', () => Float, { nullable: true }) deltaY?: number,
     @Arg('modifiers', () => String, { nullable: true }) modifiers?: string,
@@ -507,7 +656,7 @@ export class RemoteBrowserActionResolver extends ResolverBase {
       return false;
     }
 
-    const input = this.buildHumanInput({ kind, x, y, button, key, deltaX, deltaY, modifiers });
+    const input = this.buildHumanInput({ kind, x, y, button, key, text, deltaX, deltaY, modifiers });
     if (!input) {
       return false;
     }
@@ -526,6 +675,46 @@ export class RemoteBrowserActionResolver extends ResolverBase {
     }
   }
 
+  /**
+   * Returns the remote page's CURRENT text selection — the copy-out half of human clipboard support. The
+   * viewer captures a local `copy` / Cmd+C, calls this to read what the human selected on the live page, and
+   * writes the result to the LOCAL clipboard (sidestepping the isolated remote clipboard, the mirror of the
+   * `'text'` paste-in path). Ownership-gated.
+   *
+   * Gracefully best-effort, never throws past the ownership gate (mirrors {@link RemoteBrowserSnapshot}):
+   * - No live browser for the session → `{ Text: '' }`.
+   * - Backend lacks `HumanTakeover` ({@link RemoteBrowserCapabilityNotSupportedError}) → `{ Text: '' }`.
+   * - Nothing selected / any other read failure → `{ Text: '' }` (logged).
+   *
+   * @param agentSessionID The `AIAgentSession` id the browser is bound to.
+   * @returns The selection text, or `{ Text: '' }` when none is readable.
+   */
+  @Query(() => RemoteBrowserSelection)
+  async GetRemoteBrowserSelection(
+    @Arg('agentSessionID', () => String) agentSessionID: string,
+    @Ctx() { userPayload, providers }: AppContext,
+  ): Promise<RemoteBrowserSelection> {
+    const { contextUser, provider } = this.requireUserAndProvider(userPayload, providers);
+    await this.loadOwnedSession(agentSessionID, contextUser, provider);
+
+    const liveSession = RemoteBrowserEngine.Instance.GetSessionForAgentSession(agentSessionID);
+    if (!liveSession) {
+      return { Text: '' };
+    }
+
+    try {
+      return { Text: await liveSession.GetSelectionText() };
+    } catch (err) {
+      if (err instanceof RemoteBrowserCapabilityNotSupportedError) {
+        // Backend can't read the selection — copy-out is simply unavailable. Not an error condition.
+        return { Text: '' };
+      }
+      const message = err instanceof Error ? err.message : String(err);
+      LogError(`GetRemoteBrowserSelection failed for session ${agentSessionID}: ${message}`);
+      return { Text: '' };
+    }
+  }
+
   // ----- internals -------------------------------------------------------------------------
 
   /**
@@ -569,12 +758,7 @@ export class RemoteBrowserActionResolver extends ResolverBase {
    * @param agentSessionID The `AIAgentSession` id the chunk belongs to.
    * @param chunk The encoded audio chunk.
    */
-  private publishAudioChunk(
-    pubSub: PubSubEngine,
-    userPayload: UserPayload,
-    agentSessionID: string,
-    chunk: RemoteBrowserAudioChunk,
-  ): void {
+  private publishAudioChunk(pubSub: PubSubEngine, userPayload: UserPayload, agentSessionID: string, chunk: RemoteBrowserAudioChunk): void {
     pubSub.publish(PUSH_STATUS_UPDATES_TOPIC, {
       message: JSON.stringify({
         resolver: 'RemoteBrowserActionResolver',
@@ -844,15 +1028,25 @@ export class RemoteBrowserActionResolver extends ResolverBase {
   /**
    * Builds a strongly-typed {@link RemoteBrowserHumanInput} from the relayed `kind` + fields, validating
    * each kind's required field(s). Returns `null` for an unknown kind or a kind missing its required
-   * field(s): pointer-move/click/down/up need finite `x`,`y`; key needs a non-empty `key`; scroll needs
-   * finite `x`,`y`,`deltaX`,`deltaY`. The `button` is clamped to the allowed union
-   * (`'left' | 'middle' | 'right'`), defaulting unknown/absent values to `'left'`. Held `modifiers`
+   * field(s): pointer-move/click/down/up need finite `x`,`y`; key needs a non-empty `key`; text needs a
+   * non-empty `text`; scroll needs finite `x`,`y`,`deltaX`,`deltaY`. The `button` is clamped to the allowed
+   * union (`'left' | 'middle' | 'right'`), defaulting unknown/absent values to `'left'`. Held `modifiers`
    * (e.g. Shift-click, Ctrl/Cmd+key) ride on pointer clicks/presses and key presses.
    *
    * @param input The relayed input kind + all optional fields.
    * @returns The built human input, or `null` when the kind is unknown / incomplete.
    */
-  private buildHumanInput(input: { kind: string; x?: number; y?: number; button?: string; key?: string; deltaX?: number; deltaY?: number; modifiers?: string }): RemoteBrowserHumanInput | null {
+  private buildHumanInput(input: {
+    kind: string;
+    x?: number;
+    y?: number;
+    button?: string;
+    key?: string;
+    text?: string;
+    deltaX?: number;
+    deltaY?: number;
+    modifiers?: string;
+  }): RemoteBrowserHumanInput | null {
     const modifiers = this.parseModifiers(input.modifiers);
     const hasXy = Number.isFinite(input.x) && Number.isFinite(input.y);
     switch (input.kind) {
@@ -860,18 +1054,39 @@ export class RemoteBrowserActionResolver extends ResolverBase {
         return hasXy ? { Kind: 'pointer-move', X: input.x as number, Y: input.y as number } : null;
       case 'pointer-click':
         return hasXy
-          ? { Kind: 'pointer-click', X: input.x as number, Y: input.y as number, Button: this.clampButton(input.button), ...(modifiers.length ? { Modifiers: modifiers } : {}) }
+          ? {
+              Kind: 'pointer-click',
+              X: input.x as number,
+              Y: input.y as number,
+              Button: this.clampButton(input.button),
+              ...(modifiers.length ? { Modifiers: modifiers } : {}),
+            }
           : null;
       case 'pointer-down':
         return hasXy
-          ? { Kind: 'pointer-down', X: input.x as number, Y: input.y as number, Button: this.clampButton(input.button), ...(modifiers.length ? { Modifiers: modifiers } : {}) }
+          ? {
+              Kind: 'pointer-down',
+              X: input.x as number,
+              Y: input.y as number,
+              Button: this.clampButton(input.button),
+              ...(modifiers.length ? { Modifiers: modifiers } : {}),
+            }
           : null;
       case 'pointer-up':
         return hasXy
-          ? { Kind: 'pointer-up', X: input.x as number, Y: input.y as number, Button: this.clampButton(input.button), ...(modifiers.length ? { Modifiers: modifiers } : {}) }
+          ? {
+              Kind: 'pointer-up',
+              X: input.x as number,
+              Y: input.y as number,
+              Button: this.clampButton(input.button),
+              ...(modifiers.length ? { Modifiers: modifiers } : {}),
+            }
           : null;
       case 'key':
         return input.key && input.key.length > 0 ? { Kind: 'key', Key: input.key, ...(modifiers.length ? { Modifiers: modifiers } : {}) } : null;
+      case 'text':
+        // Human paste — insert the relayed clipboard text into the focused element. Empty text is a no-op.
+        return typeof input.text === 'string' && input.text.length > 0 ? { Kind: 'text', Text: input.text } : null;
       case 'scroll':
         return hasXy && Number.isFinite(input.deltaX) && Number.isFinite(input.deltaY)
           ? { Kind: 'scroll', X: input.x as number, Y: input.y as number, DeltaX: input.deltaX as number, DeltaY: input.deltaY as number }

package/src/resolvers/RunAIAgentResolver.ts

@@ -329,15 +329,7 @@ export class RunAIAgentResolver extends ResolverBase {
                 console.error('❌ No agent run available for streaming callback');
                 return;
             }
-            
-            console.log('💬 Publishing streaming content:', {
-                content: chunk.content.substring(0, 50) + '...',
-                isComplete: chunk.isComplete,
-                stepType: chunk.stepType,
-                sessionId,
-                agentRunId: agentRun.ID
-            });
-            
+
             // Publish streaming content with the full serialized agent run
             const streamMsg: AgentExecutionStreamMessage = {
                 sessionId,

package/src/rest/setupRESTEndpoints.ts

@@ -1,5 +1,6 @@
 import express from 'express';
 import BodyParser from 'body-parser';
+import { LogStatusEx } from '@memberjunction/core';
 import { RESTEndpointHandler } from './RESTEndpointHandler.js';
 
 export const ___REST_API_BASE_PATH = '/api/v1';
@@ -63,9 +64,10 @@ export function setupRESTEndpoints(
     // Merge with default options
     const config = { ...DEFAULT_REST_API_OPTIONS, ...options };
 
-    // Skip setup if REST API is disabled
+    // Skip setup if REST API is disabled. REST on/off state is surfaced in the startup
+    // summary `Auth` line at standard level, so this detail is verbose-only.
     if (!config.enabled) {
-        console.log('REST API endpoints are disabled');
+        LogStatusEx({ message: 'REST API endpoints are disabled', verboseOnly: true });
         return;
     }
 
@@ -87,5 +89,5 @@ export function setupRESTEndpoints(
         app.use(basePath, BodyParser.json({ limit: '50mb' }), restHandler.getRouter());
     }
 
-    console.log(`REST API endpoints have been set up at ${basePath}`);
+    LogStatusEx({ message: `REST API endpoints have been set up at ${basePath}`, verboseOnly: true });
 }