@mobileai/react-native 0.9.26 → 0.9.28

package/lib/module/config/endpoints.js

@@ -10,8 +10,29 @@
  * to route telemetry through your own backend without touching this file.
  */
 
-const MOBILEAI_BASE = process.env.EXPO_PUBLIC_MOBILEAI_BASE_URL || process.env.NEXT_PUBLIC_MOBILEAI_BASE_URL || 'https://mobileai.cloud';
+import { Platform } from 'react-native';
+function resolveMobileAIBase() {
+  const configuredBase = process.env.EXPO_PUBLIC_MOBILEAI_BASE_URL || process.env.NEXT_PUBLIC_MOBILEAI_BASE_URL || 'https://mobileai.cloud';
+
+  // Android emulators cannot reach the host machine via localhost/127.0.0.1.
+  // Translate those hostnames to 10.0.2.2 so the Expo example can talk to the
+  // local dashboard/backend without affecting iOS.
+  if (Platform.OS === 'android') {
+    return configuredBase.replace(/^http:\/\/(localhost|127\.0\.0\.1)(?=[:/]|$)/, 'http://10.0.2.2');
+  }
+  return configuredBase;
+}
+const MOBILEAI_BASE = resolveMobileAIBase();
+function toWebSocketBase(url) {
+  if (url.startsWith('https://')) return `wss://${url.slice('https://'.length)}`;
+  if (url.startsWith('http://')) return `ws://${url.slice('http://'.length)}`;
+  return url;
+}
 export const ENDPOINTS = {
+  /** Hosted MobileAI text proxy — used by default when analyticsKey is set */
+  hostedTextProxy: `${MOBILEAI_BASE}/api/v1/hosted-proxy/text`,
+  /** Hosted MobileAI voice proxy — used by default when analyticsKey is set */
+  hostedVoiceProxy: `${toWebSocketBase(MOBILEAI_BASE)}/ws/hosted-proxy/voice`,
   /** Telemetry event ingest — receives batched SDK events */
   telemetryIngest: `${MOBILEAI_BASE}/api/v1/events`,
   /** Feature flag sync — fetches remote flags for this analyticsKey */

package/lib/module/core/AgentRuntime.js

@@ -15,10 +15,12 @@ import { logger } from "../utils/logger.js";
 import { walkFiberTree } from "./FiberTreeWalker.js";
 import { dehydrateScreen } from "./ScreenDehydrator.js";
 import { buildSystemPrompt, buildKnowledgeOnlyPrompt } from "./systemPrompt.js";
+import { buildVerificationAction, createVerificationSnapshot, OutcomeVerifier } from "./OutcomeVerifier.js";
 import { KnowledgeBaseService } from "../services/KnowledgeBaseService.js";
 import { installAlertInterceptor, uninstallAlertInterceptor } from "./NativeAlertInterceptor.js";
 import { createTapTool, createLongPressTool, createTypeTool, createScrollTool, createSliderTool, createPickerTool, createDatePickerTool, createKeyboardTool, createGuideTool, createSimplifyTool, createRestoreTool } from "../tools/index.js";
 import { actionRegistry } from "./ActionRegistry.js";
+import { createProvider } from "../providers/ProviderFactory.js";
 const DEFAULT_MAX_STEPS = 25;
 function generateTraceId() {
   return `trace_${Date.now()}_${Math.random().toString(36).slice(2, 10)}`;
@@ -28,7 +30,6 @@ const APPROVAL_REJECTED_TOKEN = '__APPROVAL_REJECTED__';
 const APPROVAL_ALREADY_DONE_TOKEN = '__APPROVAL_ALREADY_DONE__';
 const USER_ALREADY_COMPLETED_MESSAGE = '✅ It looks like you already completed that step yourself. Great — let me know if you want help with anything else.';
 const ACTION_NOT_APPROVED_MESSAGE = "Okay — I won't do that. If you'd like, I can help with something else instead.";
-
 // ─── Agent Runtime ─────────────────────────────────────────────
 
 export class AgentRuntime {
@@ -40,6 +41,10 @@ export class AgentRuntime {
   knowledgeService = null;
   lastDehydratedRoot = null;
   currentTraceId = null;
+  currentUserGoal = '';
+  verifierProvider = null;
+  outcomeVerifier = null;
+  pendingCriticalVerification = null;
 
   // ─── Task-scoped error suppression ──────────────────────────
   // Installed once at execute() start, removed after grace period.
@@ -51,15 +56,71 @@ export class AgentRuntime {
   originalReportErrorsAsExceptions = undefined;
 
   // ─── App-action approval gate ────────────────────────────────
-  // Tracks whether the support consent flow (ask_user + request_app_action=true)
-  // has been issued and whether the user has explicitly approved it via button tap.
-  // Only UI-altering tools are gated; informational tools (done, query_knowledge) are not.
-  appActionApproved = false; // true only after __APPROVAL_GRANTED__ received
-  // Tools that physically alter the app — must be gated by appAction approval
-  static APP_ACTION_TOOLS = new Set(['tap', 'type', 'scroll', 'navigate', 'long_press', 'slider', 'picker', 'date_picker', 'keyboard']);
+  // Copilot uses a workflow-scoped approval model:
+  // - none: routine UI actions are blocked
+  // - workflow: routine UI actions are allowed for the current task
+  // Final irreversible commits are still protected separately by prompt rules
+  // and aiConfirm-based confirmation checks.
+  appActionApprovalScope = 'none';
+  appActionApprovalSource = 'none';
+  // Tools that physically alter the app — must be gated by workflow approval
+  static APP_ACTION_TOOLS = new Set(['tap', 'type', 'scroll', 'navigate', 'long_press', 'adjust_slider', 'select_picker', 'set_date', 'dismiss_keyboard']);
   getConfig() {
     return this.config;
   }
+  resetAppActionApproval(reason) {
+    this.appActionApprovalScope = 'none';
+    this.appActionApprovalSource = 'none';
+    logger.info('AgentRuntime', `🔒 Workflow approval cleared (${reason})`);
+  }
+  grantWorkflowApproval(source, reason) {
+    this.appActionApprovalScope = 'workflow';
+    this.appActionApprovalSource = source;
+    logger.info('AgentRuntime', `✅ Workflow approval granted via ${source} (${reason})`);
+  }
+  hasWorkflowApproval() {
+    return this.appActionApprovalScope === 'workflow' && this.appActionApprovalSource !== 'none';
+  }
+  debugLogChunked(label, text, chunkSize = 1600) {
+    if (!text) {
+      logger.debug('AgentRuntime', `${label}: (empty)`);
+      return;
+    }
+    logger.debug('AgentRuntime', `${label} (length=${text.length})`);
+    for (let start = 0; start < text.length; start += chunkSize) {
+      const end = Math.min(start + chunkSize, text.length);
+      const chunkIndex = Math.floor(start / chunkSize) + 1;
+      const chunkCount = Math.ceil(text.length / chunkSize);
+      logger.debug('AgentRuntime', `${label} [chunk ${chunkIndex}/${chunkCount}]`, text.slice(start, end));
+    }
+  }
+  formatInteractiveForDebug(element) {
+    const props = element.props || {};
+    const stateParts = [];
+    if (props.accessibilityRole) stateParts.push(`role=${String(props.accessibilityRole)}`);
+    if (props.value !== undefined && typeof props.value !== 'function') stateParts.push(`value=${String(props.value)}`);
+    if (props.checked !== undefined && typeof props.checked !== 'function') stateParts.push(`checked=${String(props.checked)}`);
+    if (props.selected !== undefined && typeof props.selected !== 'function') stateParts.push(`selected=${String(props.selected)}`);
+    if (props.enabled !== undefined && typeof props.enabled !== 'function') stateParts.push(`enabled=${String(props.enabled)}`);
+    if (props.disabled === true) stateParts.push('disabled=true');
+    if (element.aiPriority) stateParts.push(`aiPriority=${element.aiPriority}`);
+    if (element.zoneId) stateParts.push(`zoneId=${element.zoneId}`);
+    if (element.requiresConfirmation) stateParts.push('requiresConfirmation=true');
+    const summary = `[${element.index}] <${element.type}> "${element.label}"`;
+    return stateParts.length > 0 ? `${summary} | ${stateParts.join(' | ')}` : summary;
+  }
+  debugScreenSnapshot(screenName, elements, rawElementsText, transformedScreenContent, contextMessage) {
+    const interactiveSummary = elements.length > 0 ? elements.map(element => this.formatInteractiveForDebug(element)).join('\n') : '(no interactive elements)';
+    logger.debug('AgentRuntime', `Screen snapshot for "${screenName}" | interactiveCount=${elements.length}`);
+    this.debugLogChunked('Interactive inventory', interactiveSummary);
+    this.debugLogChunked('Raw dehydrated elementsText', rawElementsText);
+    if (transformedScreenContent !== rawElementsText) {
+      this.debugLogChunked('Transformed screen content', transformedScreenContent);
+    }
+    if (contextMessage) {
+      this.debugLogChunked('Full provider context message', contextMessage);
+    }
+  }
   constructor(provider, config, rootRef, navRef) {
     this.provider = provider;
     this.config = config;
@@ -92,6 +153,77 @@ export class AgentRuntime {
       }
     }
   }
+  getVerifier() {
+    if (this.config.verifier?.enabled === false) {
+      return null;
+    }
+    if (!this.outcomeVerifier) {
+      const verifierConfig = this.config.verifier;
+      if (verifierConfig?.provider || verifierConfig?.model || verifierConfig?.proxyUrl || verifierConfig?.proxyHeaders) {
+        this.verifierProvider = createProvider(verifierConfig.provider || this.config.provider || 'gemini', this.config.apiKey, verifierConfig.model || this.config.model, verifierConfig.proxyUrl || this.config.proxyUrl, verifierConfig.proxyHeaders || this.config.proxyHeaders);
+      } else {
+        this.verifierProvider = this.provider;
+      }
+      this.outcomeVerifier = new OutcomeVerifier(this.verifierProvider, this.config);
+    }
+    return this.outcomeVerifier;
+  }
+  createCurrentVerificationSnapshot(screenName, screenContent, elements, screenshot) {
+    return createVerificationSnapshot(screenName, screenContent, elements, screenshot);
+  }
+  async updateCriticalVerification(screenName, screenContent, elements, screenshot, stepIndex) {
+    if (!this.pendingCriticalVerification) return;
+    const verifier = this.getVerifier();
+    if (!verifier) {
+      this.pendingCriticalVerification = null;
+      return;
+    }
+    const postAction = this.createCurrentVerificationSnapshot(screenName, screenContent, elements, screenshot);
+    this.pendingCriticalVerification.followupSteps += 1;
+    const result = await verifier.verify({
+      goal: this.pendingCriticalVerification.goal,
+      action: this.pendingCriticalVerification.action,
+      preAction: this.pendingCriticalVerification.preAction,
+      postAction
+    });
+    this.emitTrace('critical_action_verified', {
+      action: this.pendingCriticalVerification.action.toolName,
+      label: this.pendingCriticalVerification.action.label,
+      status: result.status,
+      failureKind: result.failureKind,
+      evidence: result.evidence,
+      source: result.source,
+      followupSteps: this.pendingCriticalVerification.followupSteps
+    }, stepIndex);
+    if (result.status === 'success') {
+      this.pendingCriticalVerification = null;
+      return;
+    }
+    if (result.status === 'error') {
+      this.observations.push(`Outcome verifier: The previous action "${this.pendingCriticalVerification.action.label}" did NOT complete successfully. ${result.evidence} Treat this as a ${result.failureKind} failure, do not claim success, and either recover or explain the issue clearly.`);
+      return;
+    }
+    const maxFollowupSteps = verifier.getMaxFollowupSteps();
+    const ageNote = this.pendingCriticalVerification.followupSteps >= maxFollowupSteps ? ` This critical action is still unverified after ${this.pendingCriticalVerification.followupSteps} follow-up checks.` : '';
+    this.observations.push(`Outcome verifier: The previous action "${this.pendingCriticalVerification.action.label}" is still unverified. ${result.evidence}${ageNote} Before calling done(success=true), keep checking for success or error evidence on the current screen.`);
+  }
+  maybeStartCriticalVerification(toolName, args, preAction) {
+    const verifier = this.getVerifier();
+    if (!verifier) return;
+    const action = buildVerificationAction(toolName, args, preAction.elements, this.getToolStatusLabel(toolName, args));
+    if (!verifier.isCriticalAction(action)) {
+      return;
+    }
+    this.pendingCriticalVerification = {
+      goal: this.currentUserGoal,
+      action,
+      preAction,
+      followupSteps: 0
+    };
+  }
+  shouldBlockSuccessCompletion() {
+    return this.pendingCriticalVerification !== null;
+  }
 
   // ─── Tool Registration ─────────────────────────────────────
 
@@ -242,7 +374,7 @@ export class AgentRuntime {
     // ask_user — ask for clarification
     this.tools.set('ask_user', {
       name: 'ask_user',
-      description: 'Communicate with the user. Use this to ask questions, request permission for app actions, OR answer a question the user asked.',
+      description: 'Communicate with the user. Use this to ask questions, request explicit permission for app actions, answer a direct user question, or collect missing low-risk workflow data that can authorize routine in-flow steps.',
       parameters: {
         question: {
           type: 'string',
@@ -253,6 +385,11 @@ export class AgentRuntime {
           type: 'boolean',
           description: 'Set to true when requesting permission to take an action in the app (navigate, tap, investigate). Shows explicit approval buttons to the user.',
           required: true
+        },
+        grants_workflow_approval: {
+          type: 'boolean',
+          description: 'Optional. Set to true only when asking for missing low-risk input or a low-risk selection that you will directly apply in the current action workflow. If the user answers, their answer authorizes routine in-flow actions like typing/selecting/toggling, but NOT irreversible final commits or support investigations.',
+          required: false
         }
       },
       execute: async args => {
@@ -261,12 +398,16 @@ export class AgentRuntime {
         if (typeof cleanQuestion === 'string') {
           cleanQuestion = cleanQuestion.replace(/\[\d+\]/g, '').replace(/  +/g, ' ').trim();
         }
-        const kind = args.request_app_action ? 'approval' : 'freeform';
-
-        // Mark that the support approval flow has been initiated
-        if (args.request_app_action) {
-          this.appActionApproved = false; // reset until user taps Allow
-          logger.info('AgentRuntime', '🔒 App action gate: approval requested, UI tools now BLOCKED until granted');
+        const wantsExplicitAppApproval = args.request_app_action === true;
+        const grantsWorkflowApproval = args.grants_workflow_approval === true;
+        const kind = wantsExplicitAppApproval ? 'approval' : 'freeform';
+
+        // Mark that an explicit approval checkpoint is now pending.
+        if (wantsExplicitAppApproval) {
+          this.resetAppActionApproval('explicit approval requested');
+          logger.info('AgentRuntime', '🔒 App action gate: explicit approval requested, UI tools now BLOCKED until granted');
+        } else if (grantsWorkflowApproval) {
+          logger.info('AgentRuntime', '📝 ask_user will grant workflow approval if the user answers with routine action data');
         }
         logger.info('AgentRuntime', `❓ ask_user emitted (kind=${kind}): "${cleanQuestion}"`);
         if (this.config.onAskUser) {
@@ -281,13 +422,14 @@ export class AgentRuntime {
 
           // Resolve approval gate based on button response
           if (answer === '__APPROVAL_GRANTED__') {
-            this.appActionApproved = true;
-            logger.info('AgentRuntime', '✅ App action gate: APPROVED — UI tools unblocked');
+            this.grantWorkflowApproval('explicit_button', 'user tapped Allow');
           } else if (answer === '__APPROVAL_REJECTED__') {
-            this.appActionApproved = false;
+            this.resetAppActionApproval('explicit approval rejected');
             logger.info('AgentRuntime', '🚫 App action gate: REJECTED — UI tools remain blocked');
+          } else if (grantsWorkflowApproval && typeof answer === 'string' && answer.trim().length > 0) {
+            this.grantWorkflowApproval('user_input', 'user supplied requested workflow data');
           }
-          // Any other text answer (conversational interruption) leaves appActionApproved as-is
+          // Any other text answer leaves workflow approval unchanged.
 
           return `User answered: ${answer}`;
         }
@@ -820,8 +962,8 @@ ${screen.elementsText}
       // Mandate explicit ask_user approval for all UI-altering tools ONLY if we are in
       // copilot mode AND the host app has provided an onAskUser callback.
       // If the model tries to use a UI tool without explicitly getting approval, we block it.
-      if (this.config.interactionMode !== 'autopilot' && this.config.onAskUser && AgentRuntime.APP_ACTION_TOOLS.has(toolName) && !this.appActionApproved) {
-        const blockedMsg = `🚫 APP ACTION BLOCKED: You are attempting to use "${toolName}" but have not yet received explicit user approval. You MUST first call ask_user(request_app_action=true) and wait for the user to explicitly tap 'Allow' before executing ANY UI actions (including navigate, tap, scroll, etc).`;
+      if (this.config.interactionMode !== 'autopilot' && this.config.onAskUser && AgentRuntime.APP_ACTION_TOOLS.has(toolName) && !this.hasWorkflowApproval()) {
+        const blockedMsg = `🚫 APP ACTION BLOCKED: You are attempting to use "${toolName}" without workflow approval. Before routine UI actions, either (1) call ask_user(request_app_action=true) and wait for the user to tap 'Allow', or (2) if you are collecting missing low-risk input/selection for the current action workflow, call ask_user(grants_workflow_approval=true) so the user's answer authorizes routine in-flow actions. Never use option (2) for support investigations or irreversible final commits.`;
         logger.warn('AgentRuntime', blockedMsg);
         this.emitTrace('app_action_gate_blocked', {
           tool: toolName,
@@ -1263,8 +1405,12 @@ ${screen.elementsText}
     this.currentTraceId = generateTraceId();
     this.observations = [];
     this.lastScreenName = '';
-    // Reset app-action approval gate for each new task
-    this.appActionApproved = false;
+    this.pendingCriticalVerification = null;
+    this.outcomeVerifier = null;
+    this.verifierProvider = null;
+    this.currentUserGoal = userMessage;
+    // Reset workflow approval for each new task
+    this.resetAppActionApproval('new task');
     const maxSteps = this.config.maxSteps || DEFAULT_MAX_STEPS;
     const stepDelay = this.config.stepDelay ?? 300;
 
@@ -1282,6 +1428,7 @@ ${screen.elementsText}
       contextualMessage = `(Note: You just asked the user: "${this.lastAskUserQuestion}")\n\nUser replied: ${userMessage}`;
       this.lastAskUserQuestion = null; // Consume the question
     }
+    this.currentUserGoal = contextualMessage;
     logger.info('AgentRuntime', `Starting execution: "${contextualMessage}"`);
 
     // Lifecycle: onBeforeTask
@@ -1422,19 +1569,21 @@ ${screen.elementsText}
 
         // 4. Assemble structured user prompt
         const contextMessage = this.assembleUserPrompt(step, maxSteps, contextualMessage, screenName, screenContent, chatHistory);
+        this.debugScreenSnapshot(screen.screenName, screen.elements, screen.elementsText, screenContent, contextMessage);
 
         // 4.5. Capture screenshot for Gemini vision (optional)
         const screenshot = await this.captureScreenshot();
+        await this.updateCriticalVerification(screenName, screenContent, screen.elements, screenshot, step);
 
         // 5. Send to AI provider
         this.config.onStatusUpdate?.('Thinking...');
         const hasKnowledge = !!this.knowledgeService;
         const isCopilot = this.config.interactionMode !== 'autopilot';
-        const systemPrompt = buildSystemPrompt('en', hasKnowledge, isCopilot);
+        const systemPrompt = buildSystemPrompt('en', hasKnowledge, isCopilot, this.config.supportStyle);
         const tools = this.buildToolsForProvider();
         logger.info('AgentRuntime', `Sending to AI with ${tools.length} tools...`);
         logger.debug('AgentRuntime', 'System prompt length:', systemPrompt.length);
-        logger.debug('AgentRuntime', 'User context message:', contextMessage.substring(0, 300));
+        logger.debug('AgentRuntime', 'User context preview:', contextMessage.substring(0, 300));
         const response = await this.provider.generateContent(systemPrompt, contextMessage, tools, this.history, screenshot);
         this.emitTrace('provider_response', {
           text: response.text,
@@ -1495,6 +1644,13 @@ ${screen.elementsText}
 
         // 6. Process tool calls
         if (!response.toolCalls || response.toolCalls.length === 0) {
+          if (this.shouldBlockSuccessCompletion()) {
+            this.emitTrace('task_completion_blocked_needs_verification', {
+              responseText: response.text,
+              pendingVerification: this.pendingCriticalVerification
+            }, step);
+            continue;
+          }
           logger.warn('AgentRuntime', 'No tool calls in response. Text:', response.text);
           this.emitTrace('task_completed_without_tool', {
             responseText: response.text
@@ -1539,6 +1695,7 @@ ${screen.elementsText}
         // Prefer the human-readable plan over the raw tool status if available to avoid double statuses
         const statusDisplay = reasoning.plan || statusLabel;
         this.config.onStatusUpdate?.(statusDisplay);
+        const preActionSnapshot = this.createCurrentVerificationSnapshot(screenName, screenContent, screen.elements, screenshot);
 
         // Find and execute the tool
         const tool = this.tools.get(toolCall.name) || this.buildToolsForProvider().find(t => t.name === toolCall.name);
@@ -1558,6 +1715,11 @@ ${screen.elementsText}
           args: toolCall.args,
           output
         }, step);
+        if (output.startsWith('✅')) {
+          this.maybeStartCriticalVerification(toolCall.name, toolCall.args, preActionSnapshot);
+        } else if (toolCall.name !== 'done') {
+          this.pendingCriticalVerification = null;
+        }
         if (output === APPROVAL_ALREADY_DONE_TOKEN) {
           const result = {
             success: true,
@@ -1586,6 +1748,12 @@ ${screen.elementsText}
 
         // Check if done
         if (toolCall.name === 'done') {
+          if (toolCall.args.success !== false && this.shouldBlockSuccessCompletion()) {
+            this.emitTrace('done_blocked_needs_verification', {
+              pendingVerification: this.pendingCriticalVerification
+            }, step);
+            continue;
+          }
           const result = {
             success: toolCall.args.success !== false,
             message: toolCall.args.text || toolCall.args.message || output || reasoning.plan || (toolCall.args.success === false ? 'Action stopped.' : 'Action completed.'),