@mobileai/react-native 0.9.26 → 0.9.27

package/lib/module/components/AIAgent.js

@@ -400,7 +400,7 @@ export function AIAgent({
     if (knowledgeBase) return knowledgeBase;
     if (!analyticsKey) return undefined;
     return createMobileAIKnowledgeRetriever({
-      publishableKey: analyticsKey,
+      analyticsKey: analyticsKey,
       baseUrl: analyticsProxyUrl ?? ENDPOINTS.escalation,
       headers: analyticsProxyHeaders
     });
@@ -1143,7 +1143,29 @@ export function AIAgent({
   useEffect(() => {
     logger.info('AIAgent', `⚙️ Runtime config recomputed: mode=${mode} interactionMode=${interactionMode || 'copilot(default)'} onAskUser=${mode !== 'voice'} mergedTools=${Object.keys(mergedCustomTools).join(', ') || '(none)'}`);
   }, [mode, interactionMode, mergedCustomTools]);
-  const provider = useMemo(() => createProvider(providerName, apiKey, model, proxyUrl, proxyHeaders), [providerName, apiKey, model, proxyUrl, proxyHeaders]);
+  const effectiveProxyHeaders = useMemo(() => {
+    if (!analyticsKey) return proxyHeaders;
+    const isAuthMissing = !proxyHeaders || !Object.keys(proxyHeaders).some(k => k.toLowerCase() === 'authorization');
+    if (isAuthMissing) {
+      return {
+        ...proxyHeaders,
+        Authorization: `Bearer ${analyticsKey}`
+      };
+    }
+    return proxyHeaders;
+  }, [proxyHeaders, analyticsKey]);
+  const effectiveVoiceProxyHeaders = useMemo(() => {
+    if (!analyticsKey) return voiceProxyHeaders;
+    const isAuthMissing = !voiceProxyHeaders || !Object.keys(voiceProxyHeaders).some(k => k.toLowerCase() === 'authorization');
+    if (isAuthMissing) {
+      return {
+        ...voiceProxyHeaders,
+        Authorization: `Bearer ${analyticsKey}`
+      };
+    }
+    return voiceProxyHeaders;
+  }, [voiceProxyHeaders, analyticsKey]);
+  const provider = useMemo(() => createProvider(providerName, apiKey, model, proxyUrl, effectiveProxyHeaders), [providerName, apiKey, model, proxyUrl, effectiveProxyHeaders]);
   const runtime = useMemo(() => new AgentRuntime(provider, config, rootViewRef.current, navRef),
   // eslint-disable-next-line react-hooks/exhaustive-deps
   [provider, config]);
@@ -1348,7 +1370,7 @@ export function AIAgent({
       voiceServiceRef.current = new VoiceService({
         apiKey,
         proxyUrl: voiceProxyUrl || proxyUrl,
-        proxyHeaders: voiceProxyHeaders || proxyHeaders,
+        proxyHeaders: effectiveVoiceProxyHeaders || effectiveProxyHeaders,
         systemPrompt: voicePrompt,
         tools: runtimeTools,
         language: 'en'

package/lib/module/components/AgentChatBar.js

@@ -788,7 +788,7 @@ export function AgentChatBar({
           style: styles.approvalPanel,
           children: [/*#__PURE__*/_jsx(Text, {
             style: styles.approvalHint,
-            children: "The AI agent is requesting permission to perform this action. Tap \"Do it\" to approve, or \"Don\u2019t do it\" to cancel."
+            children: "The AI agent is requesting permission to perform this action. Tap \"Allow\" to approve, or \"Don\u2019t Allow\" to cancel."
           }), /*#__PURE__*/_jsxs(View, {
             style: styles.approvalActions,
             children: [/*#__PURE__*/_jsx(Pressable, {
@@ -796,14 +796,14 @@ export function AgentChatBar({
               onPress: () => onPendingApprovalAction('reject'),
               children: /*#__PURE__*/_jsx(Text, {
                 style: [styles.approvalActionText, styles.approvalActionSecondaryText],
-                children: "Don\u2019t do it"
+                children: "Don\u2019t Allow"
               })
             }), /*#__PURE__*/_jsx(Pressable, {
               style: [styles.approvalActionBtn, styles.approvalActionPrimary],
               onPress: () => onPendingApprovalAction('approve'),
               children: /*#__PURE__*/_jsx(Text, {
                 style: [styles.approvalActionText, styles.approvalActionPrimaryText],
-                children: "Do it"
+                children: "Allow"
               })
             })]
           })]

package/lib/module/core/AgentRuntime.js

@@ -28,7 +28,6 @@ const APPROVAL_REJECTED_TOKEN = '__APPROVAL_REJECTED__';
 const APPROVAL_ALREADY_DONE_TOKEN = '__APPROVAL_ALREADY_DONE__';
 const USER_ALREADY_COMPLETED_MESSAGE = '✅ It looks like you already completed that step yourself. Great — let me know if you want help with anything else.';
 const ACTION_NOT_APPROVED_MESSAGE = "Okay — I won't do that. If you'd like, I can help with something else instead.";
-
 // ─── Agent Runtime ─────────────────────────────────────────────
 
 export class AgentRuntime {
@@ -51,15 +50,71 @@ export class AgentRuntime {
   originalReportErrorsAsExceptions = undefined;
 
   // ─── App-action approval gate ────────────────────────────────
-  // Tracks whether the support consent flow (ask_user + request_app_action=true)
-  // has been issued and whether the user has explicitly approved it via button tap.
-  // Only UI-altering tools are gated; informational tools (done, query_knowledge) are not.
-  appActionApproved = false; // true only after __APPROVAL_GRANTED__ received
-  // Tools that physically alter the app — must be gated by appAction approval
-  static APP_ACTION_TOOLS = new Set(['tap', 'type', 'scroll', 'navigate', 'long_press', 'slider', 'picker', 'date_picker', 'keyboard']);
+  // Copilot uses a workflow-scoped approval model:
+  // - none: routine UI actions are blocked
+  // - workflow: routine UI actions are allowed for the current task
+  // Final irreversible commits are still protected separately by prompt rules
+  // and aiConfirm-based confirmation checks.
+  appActionApprovalScope = 'none';
+  appActionApprovalSource = 'none';
+  // Tools that physically alter the app — must be gated by workflow approval
+  static APP_ACTION_TOOLS = new Set(['tap', 'type', 'scroll', 'navigate', 'long_press', 'adjust_slider', 'select_picker', 'set_date', 'dismiss_keyboard']);
   getConfig() {
     return this.config;
   }
+  resetAppActionApproval(reason) {
+    this.appActionApprovalScope = 'none';
+    this.appActionApprovalSource = 'none';
+    logger.info('AgentRuntime', `🔒 Workflow approval cleared (${reason})`);
+  }
+  grantWorkflowApproval(source, reason) {
+    this.appActionApprovalScope = 'workflow';
+    this.appActionApprovalSource = source;
+    logger.info('AgentRuntime', `✅ Workflow approval granted via ${source} (${reason})`);
+  }
+  hasWorkflowApproval() {
+    return this.appActionApprovalScope === 'workflow' && this.appActionApprovalSource !== 'none';
+  }
+  debugLogChunked(label, text, chunkSize = 1600) {
+    if (!text) {
+      logger.debug('AgentRuntime', `${label}: (empty)`);
+      return;
+    }
+    logger.debug('AgentRuntime', `${label} (length=${text.length})`);
+    for (let start = 0; start < text.length; start += chunkSize) {
+      const end = Math.min(start + chunkSize, text.length);
+      const chunkIndex = Math.floor(start / chunkSize) + 1;
+      const chunkCount = Math.ceil(text.length / chunkSize);
+      logger.debug('AgentRuntime', `${label} [chunk ${chunkIndex}/${chunkCount}]`, text.slice(start, end));
+    }
+  }
+  formatInteractiveForDebug(element) {
+    const props = element.props || {};
+    const stateParts = [];
+    if (props.accessibilityRole) stateParts.push(`role=${String(props.accessibilityRole)}`);
+    if (props.value !== undefined && typeof props.value !== 'function') stateParts.push(`value=${String(props.value)}`);
+    if (props.checked !== undefined && typeof props.checked !== 'function') stateParts.push(`checked=${String(props.checked)}`);
+    if (props.selected !== undefined && typeof props.selected !== 'function') stateParts.push(`selected=${String(props.selected)}`);
+    if (props.enabled !== undefined && typeof props.enabled !== 'function') stateParts.push(`enabled=${String(props.enabled)}`);
+    if (props.disabled === true) stateParts.push('disabled=true');
+    if (element.aiPriority) stateParts.push(`aiPriority=${element.aiPriority}`);
+    if (element.zoneId) stateParts.push(`zoneId=${element.zoneId}`);
+    if (element.requiresConfirmation) stateParts.push('requiresConfirmation=true');
+    const summary = `[${element.index}] <${element.type}> "${element.label}"`;
+    return stateParts.length > 0 ? `${summary} | ${stateParts.join(' | ')}` : summary;
+  }
+  debugScreenSnapshot(screenName, elements, rawElementsText, transformedScreenContent, contextMessage) {
+    const interactiveSummary = elements.length > 0 ? elements.map(element => this.formatInteractiveForDebug(element)).join('\n') : '(no interactive elements)';
+    logger.debug('AgentRuntime', `Screen snapshot for "${screenName}" | interactiveCount=${elements.length}`);
+    this.debugLogChunked('Interactive inventory', interactiveSummary);
+    this.debugLogChunked('Raw dehydrated elementsText', rawElementsText);
+    if (transformedScreenContent !== rawElementsText) {
+      this.debugLogChunked('Transformed screen content', transformedScreenContent);
+    }
+    if (contextMessage) {
+      this.debugLogChunked('Full provider context message', contextMessage);
+    }
+  }
   constructor(provider, config, rootRef, navRef) {
     this.provider = provider;
     this.config = config;
@@ -242,7 +297,7 @@ export class AgentRuntime {
     // ask_user — ask for clarification
     this.tools.set('ask_user', {
       name: 'ask_user',
-      description: 'Communicate with the user. Use this to ask questions, request permission for app actions, OR answer a question the user asked.',
+      description: 'Communicate with the user. Use this to ask questions, request explicit permission for app actions, answer a direct user question, or collect missing low-risk workflow data that can authorize routine in-flow steps.',
       parameters: {
         question: {
           type: 'string',
@@ -253,6 +308,11 @@ export class AgentRuntime {
           type: 'boolean',
           description: 'Set to true when requesting permission to take an action in the app (navigate, tap, investigate). Shows explicit approval buttons to the user.',
           required: true
+        },
+        grants_workflow_approval: {
+          type: 'boolean',
+          description: 'Optional. Set to true only when asking for missing low-risk input or a low-risk selection that you will directly apply in the current action workflow. If the user answers, their answer authorizes routine in-flow actions like typing/selecting/toggling, but NOT irreversible final commits or support investigations.',
+          required: false
         }
       },
       execute: async args => {
@@ -261,12 +321,16 @@ export class AgentRuntime {
         if (typeof cleanQuestion === 'string') {
           cleanQuestion = cleanQuestion.replace(/\[\d+\]/g, '').replace(/  +/g, ' ').trim();
         }
-        const kind = args.request_app_action ? 'approval' : 'freeform';
-
-        // Mark that the support approval flow has been initiated
-        if (args.request_app_action) {
-          this.appActionApproved = false; // reset until user taps Allow
-          logger.info('AgentRuntime', '🔒 App action gate: approval requested, UI tools now BLOCKED until granted');
+        const wantsExplicitAppApproval = args.request_app_action === true;
+        const grantsWorkflowApproval = args.grants_workflow_approval === true;
+        const kind = wantsExplicitAppApproval ? 'approval' : 'freeform';
+
+        // Mark that an explicit approval checkpoint is now pending.
+        if (wantsExplicitAppApproval) {
+          this.resetAppActionApproval('explicit approval requested');
+          logger.info('AgentRuntime', '🔒 App action gate: explicit approval requested, UI tools now BLOCKED until granted');
+        } else if (grantsWorkflowApproval) {
+          logger.info('AgentRuntime', '📝 ask_user will grant workflow approval if the user answers with routine action data');
         }
         logger.info('AgentRuntime', `❓ ask_user emitted (kind=${kind}): "${cleanQuestion}"`);
         if (this.config.onAskUser) {
@@ -281,13 +345,14 @@ export class AgentRuntime {
 
           // Resolve approval gate based on button response
           if (answer === '__APPROVAL_GRANTED__') {
-            this.appActionApproved = true;
-            logger.info('AgentRuntime', '✅ App action gate: APPROVED — UI tools unblocked');
+            this.grantWorkflowApproval('explicit_button', 'user tapped Allow');
           } else if (answer === '__APPROVAL_REJECTED__') {
-            this.appActionApproved = false;
+            this.resetAppActionApproval('explicit approval rejected');
             logger.info('AgentRuntime', '🚫 App action gate: REJECTED — UI tools remain blocked');
+          } else if (grantsWorkflowApproval && typeof answer === 'string' && answer.trim().length > 0) {
+            this.grantWorkflowApproval('user_input', 'user supplied requested workflow data');
           }
-          // Any other text answer (conversational interruption) leaves appActionApproved as-is
+          // Any other text answer leaves workflow approval unchanged.
 
           return `User answered: ${answer}`;
         }
@@ -820,8 +885,8 @@ ${screen.elementsText}
       // Mandate explicit ask_user approval for all UI-altering tools ONLY if we are in
       // copilot mode AND the host app has provided an onAskUser callback.
       // If the model tries to use a UI tool without explicitly getting approval, we block it.
-      if (this.config.interactionMode !== 'autopilot' && this.config.onAskUser && AgentRuntime.APP_ACTION_TOOLS.has(toolName) && !this.appActionApproved) {
-        const blockedMsg = `🚫 APP ACTION BLOCKED: You are attempting to use "${toolName}" but have not yet received explicit user approval. You MUST first call ask_user(request_app_action=true) and wait for the user to explicitly tap 'Allow' before executing ANY UI actions (including navigate, tap, scroll, etc).`;
+      if (this.config.interactionMode !== 'autopilot' && this.config.onAskUser && AgentRuntime.APP_ACTION_TOOLS.has(toolName) && !this.hasWorkflowApproval()) {
+        const blockedMsg = `🚫 APP ACTION BLOCKED: You are attempting to use "${toolName}" without workflow approval. Before routine UI actions, either (1) call ask_user(request_app_action=true) and wait for the user to tap 'Allow', or (2) if you are collecting missing low-risk input/selection for the current action workflow, call ask_user(grants_workflow_approval=true) so the user's answer authorizes routine in-flow actions. Never use option (2) for support investigations or irreversible final commits.`;
         logger.warn('AgentRuntime', blockedMsg);
         this.emitTrace('app_action_gate_blocked', {
           tool: toolName,
@@ -1263,8 +1328,8 @@ ${screen.elementsText}
     this.currentTraceId = generateTraceId();
     this.observations = [];
     this.lastScreenName = '';
-    // Reset app-action approval gate for each new task
-    this.appActionApproved = false;
+    // Reset workflow approval for each new task
+    this.resetAppActionApproval('new task');
     const maxSteps = this.config.maxSteps || DEFAULT_MAX_STEPS;
     const stepDelay = this.config.stepDelay ?? 300;
 
@@ -1422,6 +1487,7 @@ ${screen.elementsText}
 
         // 4. Assemble structured user prompt
         const contextMessage = this.assembleUserPrompt(step, maxSteps, contextualMessage, screenName, screenContent, chatHistory);
+        this.debugScreenSnapshot(screen.screenName, screen.elements, screen.elementsText, screenContent, contextMessage);
 
         // 4.5. Capture screenshot for Gemini vision (optional)
         const screenshot = await this.captureScreenshot();
@@ -1434,7 +1500,7 @@ ${screen.elementsText}
         const tools = this.buildToolsForProvider();
         logger.info('AgentRuntime', `Sending to AI with ${tools.length} tools...`);
         logger.debug('AgentRuntime', 'System prompt length:', systemPrompt.length);
-        logger.debug('AgentRuntime', 'User context message:', contextMessage.substring(0, 300));
+        logger.debug('AgentRuntime', 'User context preview:', contextMessage.substring(0, 300));
         const response = await this.provider.generateContent(systemPrompt, contextMessage, tools, this.history, screenshot);
         this.emitTrace('provider_response', {
           text: response.text,