@mobileai/react-native 0.9.16 → 0.9.18

package/src/tools/keyboardTool.ts

@@ -1,30 +0,0 @@
-/**
- * Keyboard Tool — Dismiss the on-screen keyboard.
- *
- * Pattern from:
- * - Maestro: hideKeyboard
- * - Detox: closeSoftKeyboard
- *
- * Uses React Native's Keyboard.dismiss() API.
- * This is a global action — no element index needed.
- */
-
-import { Keyboard } from 'react-native';
-import type { AgentTool } from './types';
-
-export function createKeyboardTool(): AgentTool {
-  return {
-    name: 'dismiss_keyboard',
-    description: 'Dismiss the on-screen keyboard. Use after typing into a text input when the keyboard is blocking other elements.',
-    parameters: {},
-    execute: async () => {
-      try {
-        Keyboard.dismiss();
-        await new Promise(resolve => setTimeout(resolve, 300));
-        return '✅ Keyboard dismissed.';
-      } catch (error: any) {
-        return `❌ Error dismissing keyboard: ${error.message}`;
-      }
-    },
-  };
-}

package/src/tools/longPressTool.ts

@@ -1,61 +0,0 @@
-/**
- * Long Press Tool — Extended press interaction via onLongPress.
- *
- * Pattern from:
- * - Detox: longPress(point, duration)
- * - Maestro: longPress(point)
- *
- * In JS fiber level: calls onLongPress prop (available on Pressable/TouchableOpacity).
- * Bubbles up fiber tree if direct element doesn't have onLongPress.
- */
-
-import { walkFiberTree } from '../core/FiberTreeWalker';
-import type { AgentTool, ToolContext } from './types';
-
-export function createLongPressTool(context: ToolContext): AgentTool {
-  return {
-    name: 'long_press',
-    description: 'Long-press an interactive element by its index. Use for actions that require a longer touch, such as showing context menus, reordering items, or triggering secondary actions.',
-    parameters: {
-      index: { type: 'number', description: 'The index of the element to long-press', required: true },
-    },
-    execute: async (args) => {
-      const { interactives: elements } = walkFiberTree(context.getRootRef(), context.getWalkConfig());
-      const element = elements.find(el => el.index === args.index);
-      if (!element) {
-        return `❌ Element with index ${args.index} not found. Available indexes: ${elements.map(e => e.index).join(', ')}`;
-      }
-
-      // Strategy 1: Direct onLongPress
-      if (element.props.onLongPress && typeof element.props.onLongPress === 'function') {
-        try {
-          element.props.onLongPress();
-          await new Promise(resolve => setTimeout(resolve, 500));
-          return `✅ Long-pressed [${args.index}] "${element.label}"`;
-        } catch (error: any) {
-          return `❌ Error long-pressing [${args.index}]: ${error.message}`;
-        }
-      }
-
-      // Strategy 2: Bubble up fiber tree
-      let fiber = element.fiberNode?.return;
-      let bubbleDepth = 0;
-      while (fiber && bubbleDepth < 5) {
-        const parentProps = fiber.memoizedProps || {};
-        if (parentProps.onLongPress && typeof parentProps.onLongPress === 'function') {
-          try {
-            parentProps.onLongPress();
-            await new Promise(resolve => setTimeout(resolve, 500));
-            return `✅ Long-pressed parent of [${args.index}] "${element.label}"`;
-          } catch (error: any) {
-            return `❌ Error long-pressing parent of [${args.index}]: ${error.message}`;
-          }
-        }
-        fiber = fiber.return;
-        bubbleDepth++;
-      }
-
-      return `❌ Element [${args.index}] "${element.label}" has no long-press handler. Try using tap instead.`;
-    },
-  };
-}

package/src/tools/pickerTool.ts

@@ -1,115 +0,0 @@
-/**
- * Picker Tool — Select values from picker/dropdown components.
- *
- * Pattern from Detox: setColumnToValue(column, value)
- *
- * In JS: calls onValueChange(itemValue, itemIndex).
- * Reads available options from props.children (Picker.Item) or props.items array.
- */
-
-import { walkFiberTree } from '../core/FiberTreeWalker';
-import type { AgentTool, ToolContext } from './types';
-
-/**
- * Extract available options from a picker element's props.
- * Handles multiple picker libraries:
- * - RN Picker: children are Picker.Item with { label, value } props
- * - RNPickerSelect: items prop is array of { label, value }
- * - DropDownPicker: items prop is array of { label, value }
- */
-function extractPickerOptions(element: any): Array<{ label: string; value: any }> {
-  const props = element.props || {};
-  const options: Array<{ label: string; value: any }> = [];
-
-  // Pattern 1: items prop (RNPickerSelect, DropDownPicker)
-  if (Array.isArray(props.items)) {
-    for (const item of props.items) {
-      if (item && item.label !== undefined) {
-        options.push({ label: String(item.label), value: item.value });
-      }
-    }
-    return options;
-  }
-
-  // Pattern 2: options prop (some custom pickers)
-  if (Array.isArray(props.options)) {
-    for (const item of props.options) {
-      if (typeof item === 'string') {
-        options.push({ label: item, value: item });
-      } else if (item && item.label !== undefined) {
-        options.push({ label: String(item.label), value: item.value });
-      }
-    }
-    return options;
-  }
-
-  // Pattern 3: Fiber children (RN Picker with Picker.Item children)
-  const fiberNode = element.fiberNode;
-  if (fiberNode?.child) {
-    let child = fiberNode.child;
-    while (child) {
-      const childProps = child.memoizedProps || {};
-      if (childProps.label !== undefined && childProps.value !== undefined) {
-        options.push({ label: String(childProps.label), value: childProps.value });
-      }
-      child = child.sibling;
-    }
-  }
-
-  return options;
-}
-
-export function createPickerTool(context: ToolContext): AgentTool {
-  return {
-    name: 'select_picker',
-    description: 'Select a value from a picker/dropdown by its index. Provide the exact value string to select.',
-    parameters: {
-      index: { type: 'number', description: 'The index of the picker element', required: true },
-      value: { type: 'string', description: 'The value to select (must match an available option)', required: true },
-    },
-    execute: async (args) => {
-      const { interactives: elements } = walkFiberTree(context.getRootRef(), context.getWalkConfig());
-      const element = elements.find(el => el.index === args.index);
-      if (!element) {
-        return `❌ Element with index ${args.index} not found.`;
-      }
-
-      const onValueChange = element.props.onValueChange;
-      if (!onValueChange || typeof onValueChange !== 'function') {
-        return `❌ Element [${args.index}] "${element.label}" is not a picker (no onValueChange handler).`;
-      }
-
-      // Find available options
-      const options = extractPickerOptions(element);
-
-      if (options.length > 0) {
-        // Find matching option (case-insensitive)
-        const match = options.find(
-          opt => String(opt.value) === args.value ||
-                 opt.label.toLowerCase() === args.value.toLowerCase()
-        );
-        if (!match) {
-          const available = options.map(o => `"${o.label}" (${o.value})`).join(', ');
-          return `❌ Value "${args.value}" not found in picker. Available: ${available}`;
-        }
-        try {
-          const matchIndex = options.indexOf(match);
-          onValueChange(match.value, matchIndex);
-          await new Promise(resolve => setTimeout(resolve, 500));
-          return `✅ Selected "${match.label}" in picker [${args.index}] "${element.label}"`;
-        } catch (error: any) {
-          return `❌ Error selecting picker value: ${error.message}`;
-        }
-      }
-
-      // No options found — try direct value pass
-      try {
-        onValueChange(args.value, 0);
-        await new Promise(resolve => setTimeout(resolve, 500));
-        return `✅ Set picker [${args.index}] "${element.label}" to "${args.value}"`;
-      } catch (error: any) {
-        return `❌ Error setting picker value: ${error.message}`;
-      }
-    },
-  };
-}

package/src/tools/restoreTool.ts

@@ -1,33 +0,0 @@
-import type { ToolDefinition } from '../core/types';
-import { globalZoneRegistry } from '../core/ZoneRegistry';
-import { logger } from '../utils/logger';
-
-export function createRestoreTool(): ToolDefinition {
-  return {
-    name: 'restore_zone',
-    description: 'Restore a specific Zone to its default state, reversing any previous simplify_zone or inject_card operations.',
-    parameters: {
-      zoneId: { 
-        type: 'string', 
-        description: 'The ID of the AIZone to restore',
-        required: true,
-      }
-    },
-    execute: async (args) => {
-      const zoneId = String(args.zoneId);
-
-      const zone = globalZoneRegistry.get(zoneId) as any;
-      if (!zone) {
-        return `❌ Cannot restore zone "${zoneId}": Zone does not exist.`;
-      }
-
-      if (zone._controller) {
-        zone._controller.restore();
-        logger.info('RestoreTool', `Restored zone: ${zoneId}`);
-        return `✅ Successfully restored zone "${zoneId}" to its default state.`;
-      }
-
-      return `❌ Cannot restore zone "${zoneId}": Controller not attached.`;
-    },
-  };
-}

package/src/tools/scrollTool.ts

@@ -1,156 +0,0 @@
-/**
- * Scroll Tool — Direction-based scrolling with safety checks.
- *
- * Patterns implemented:
- * - Detox: ScrollHelper.perform(direction, amountInDP) — relative scroll
- * - Detox: scrollableProbe.atScrollingEdge() — pre-scroll edge detection
- * - Detox: ScrollToIndexAction.getConstraints() — PagerView rejection
- * - Appium: mobileScrollGesture returns boolean (canScrollMore)
- * - Maestro: hierarchy diff post-scroll (no-op detection)
- */
-
-import { findScrollableContainers } from '../core/FiberTreeWalker';
-import type { AgentTool, ToolContext } from './types';
-
-export function createScrollTool(context: ToolContext): AgentTool {
-  return {
-    name: 'scroll',
-    description: 'Scroll the current screen to reveal more content. Use when you need to see items that are not yet visible, e.g. in lazy-loaded lists, long forms, or paginated content. If the screen has multiple scrollable areas, specify containerIndex to target a specific one.',
-    parameters: {
-      direction: {
-        type: 'string',
-        description: "Scroll direction: 'down' or 'up'",
-        required: true,
-        enum: ['down', 'up'],
-      },
-      amount: {
-        type: 'string',
-        description: "How far to scroll: 'page' (default, ~one screenful), 'toEnd' (jump to bottom), or 'toStart' (jump to top)",
-        required: false,
-        enum: ['page', 'toEnd', 'toStart'],
-      },
-      containerIndex: {
-        type: 'number',
-        description: 'Index of the scrollable container to scroll (0-based). Use when the screen has multiple scrollable areas. Default: 0 (the main/first scrollable area).',
-        required: false,
-      },
-    },
-    execute: async (args) => {
-      const screenName = context.getCurrentScreenName();
-      const containers = findScrollableContainers(context.getRootRef(), screenName);
-
-      if (containers.length === 0) {
-        return `❌ No scrollable container found on screen "${screenName}". Content may still be loading — wait and try again.`;
-      }
-
-      const targetIndex = args.containerIndex ?? 0;
-      const container = containers[targetIndex];
-      if (!container) {
-        const available = containers
-          .filter(c => !c.isPagerLike)
-          .map(c => `[${c.index}] ${c.label} (${c.componentName})`)
-          .join(', ');
-        return `❌ Container index ${targetIndex} not found. Available scrollable containers on "${screenName}": ${available}`;
-      }
-
-      // PagerView/TabView Rejection (Detox: getConstraints() rejects ViewPager)
-      if (container.isPagerLike) {
-        const scrollableAlts = containers
-          .filter(c => !c.isPagerLike)
-          .map(c => `[${c.index}] ${c.label} (${c.componentName})`)
-          .join(', ');
-        const suggestion = scrollableAlts
-          ? `Try scrolling a content container instead: ${scrollableAlts}`
-          : 'Use tap on the tab labels to switch tabs instead of scrolling.';
-        return `⚠️ Container [${targetIndex}] "${container.label}" is a ${container.componentName} (tab/page container). ` +
-          `Scrolling pager containers directly causes native crashes. ${suggestion}`;
-      }
-
-      const direction: string = args.direction || 'down';
-      const amount: string = args.amount || 'page';
-      const scrollRef = container.stateNode;
-
-      // Edge Detection (Detox: scrollableProbe.atScrollingEdge())
-      if (amount === 'page' && typeof scrollRef.scrollToOffset === 'function') {
-        const metrics = scrollRef._scrollMetrics;
-        if (metrics) {
-          const { offset, contentLength, visibleLength } = metrics;
-          const isAtBottom = offset + visibleLength >= contentLength - 2;
-          const isAtTop = offset <= 0;
-
-          if (direction === 'down' && isAtBottom) {
-            return `⚠️ Already at the bottom of ${container.label} (${container.componentName}). No more content to scroll to. All items are visible.`;
-          }
-          if (direction === 'up' && isAtTop) {
-            return `⚠️ Already at the top of ${container.label} (${container.componentName}). Cannot scroll further up.`;
-          }
-        }
-      }
-
-      // Track position for no-op detection (Maestro: hierarchy diff)
-      const offsetBefore = typeof scrollRef.scrollToOffset === 'function'
-        ? scrollRef._scrollMetrics?.offset ?? 0
-        : 0;
-
-      try {
-        if (amount === 'toEnd') {
-          if (typeof scrollRef.scrollToEnd === 'function') {
-            scrollRef.scrollToEnd({ animated: true });
-          } else if (typeof scrollRef.scrollTo === 'function') {
-            scrollRef.scrollTo({ y: 999999, animated: true });
-          }
-        } else if (amount === 'toStart') {
-          if (typeof scrollRef.scrollTo === 'function') {
-            scrollRef.scrollTo({ y: 0, animated: true });
-          } else if (typeof scrollRef.scrollToOffset === 'function') {
-            scrollRef.scrollToOffset({ offset: 0, animated: true });
-          }
-        } else {
-          // Direction-based relative scroll (Detox: ScrollHelper, Appium: percent)
-          const FALLBACK_PAGE_HEIGHT = 800;
-
-          if (typeof scrollRef.scrollToOffset === 'function') {
-            const metrics = scrollRef._scrollMetrics;
-            const currentOffset = metrics?.offset ?? 0;
-            const visibleLength = metrics?.visibleLength ?? FALLBACK_PAGE_HEIGHT;
-            const scrollAmount = Math.round(visibleLength * 0.8);
-
-            const newOffset = direction === 'down'
-              ? currentOffset + scrollAmount
-              : Math.max(0, currentOffset - scrollAmount);
-
-            scrollRef.scrollToOffset({ offset: newOffset, animated: true });
-          } else if (typeof scrollRef.scrollTo === 'function') {
-            const currentY = scrollRef._nativeRef?.contentOffset?.y ?? 0;
-            const scrollAmount = FALLBACK_PAGE_HEIGHT;
-
-            scrollRef.scrollTo({
-              y: direction === 'down'
-                ? currentY + scrollAmount
-                : Math.max(0, currentY - scrollAmount),
-              animated: true,
-            });
-          }
-        }
-
-        // Wait for scroll animation + onEndReached + lazy load
-        await new Promise(resolve => setTimeout(resolve, 1500));
-
-        // Post-scroll no-op detection
-        const offsetAfter = typeof scrollRef.scrollToOffset === 'function'
-          ? scrollRef._scrollMetrics?.offset ?? 0
-          : -1;
-
-        if (offsetAfter >= 0 && Math.abs(offsetAfter - offsetBefore) < 2) {
-          const edge = direction === 'down' ? 'bottom' : 'top';
-          return `⚠️ Scroll had no effect — already at the ${edge} of ${container.label}. All visible content has been loaded.`;
-        }
-
-        const amountLabel = amount === 'toEnd' ? 'to end' : amount === 'toStart' ? 'to start' : `${direction} one page`;
-        return `✅ Scrolled ${amountLabel} in ${container.label} (${container.componentName}). Check the updated screen content for newly loaded items.`;
-      } catch (error: any) {
-        return `❌ Scroll failed: ${error.message}`;
-      }
-    },
-  };
-}

package/src/tools/simplifyTool.ts

@@ -1,33 +0,0 @@
-import type { ToolDefinition } from '../core/types';
-import { globalZoneRegistry } from '../core/ZoneRegistry';
-import { logger } from '../utils/logger';
-
-export function createSimplifyTool(): ToolDefinition {
-  return {
-    name: 'simplify_zone',
-    description: 'Simplify the UI in a specific Zone by telling it to hide elements marked as aiPriority="low". Use this when a screen zone looks cluttered and you want to reduce cognitive load for the user.',
-    parameters: {
-      zoneId: { 
-        type: 'string', 
-        description: 'The ID of the AIZone you want to simplify (e.g. from the screen layout context)',
-        required: true,
-      }
-    },
-    execute: async (args) => {
-      const zoneId = String(args.zoneId);
-
-      if (!globalZoneRegistry.isActionAllowed(zoneId, 'simplify')) {
-        return `❌ Cannot simplify zone "${zoneId}": Zone does not exist or allowSimplify is false.`;
-      }
-
-      const zone = globalZoneRegistry.get(zoneId) as any;
-      if (zone && zone._controller) {
-        zone._controller.simplify();
-        logger.info('SimplifyTool', `Simplified zone: ${zoneId}`);
-        return `✅ Successfully requested simplification for zone "${zoneId}".`;
-      }
-
-      return `❌ Cannot simplify zone "${zoneId}": Controller not attached. Is the zone currently rendered on screen?`;
-    },
-  };
-}

package/src/tools/sliderTool.ts

@@ -1,65 +0,0 @@
-/**
- * Slider Tool — Adjust slider/seekbar values.
- *
- * Pattern from Detox:
- * - adjustSliderToPosition(normalizedPosition) — 0.0 to 1.0
- * - UISlider+DetoxUtils: value = normalized * (max - min) + min
- *
- * In JS: calls onValueChange(actualValue) then onSlidingComplete(actualValue).
- * Reads min/max from props (minimumValue/maximumValue).
- */
-
-import { walkFiberTree } from '../core/FiberTreeWalker';
-import type { AgentTool, ToolContext } from './types';
-
-export function createSliderTool(context: ToolContext): AgentTool {
-  return {
-    name: 'adjust_slider',
-    description: 'Adjust a slider to a specific position. Use for sliders, seek bars, and range selectors. Value is normalized 0.0 (minimum) to 1.0 (maximum).',
-    parameters: {
-      index: { type: 'number', description: 'The index of the slider element', required: true },
-      value: { type: 'number', description: 'Target position from 0.0 (min) to 1.0 (max)', required: true },
-    },
-    execute: async (args) => {
-      const { interactives: elements } = walkFiberTree(context.getRootRef(), context.getWalkConfig());
-      const element = elements.find(el => el.index === args.index);
-      if (!element) {
-        return `❌ Element with index ${args.index} not found.`;
-      }
-
-      // Validate normalized value
-      const normalizedValue = Number(args.value);
-      if (!Number.isFinite(normalizedValue) || normalizedValue < 0 || normalizedValue > 1) {
-        return `❌ Slider value must be between 0.0 and 1.0, got ${args.value}`;
-      }
-
-      // Find onValueChange (Slider's primary callback)
-      const onValueChange = element.props.onValueChange;
-      if (!onValueChange || typeof onValueChange !== 'function') {
-        return `❌ Element [${args.index}] "${element.label}" is not a slider (no onValueChange handler).`;
-      }
-
-      // Calculate actual value from normalized position
-      // Pattern from Detox UISlider+DetoxUtils:
-      // actualValue = normalized * (max - min) + min
-      const min = element.props.minimumValue ?? 0;
-      const max = element.props.maximumValue ?? 1;
-      const actualValue = normalizedValue * (max - min) + min;
-
-      try {
-        // Call onValueChange (continuous feedback)
-        onValueChange(actualValue);
-
-        // Call onSlidingComplete if available (final value callback)
-        if (element.props.onSlidingComplete && typeof element.props.onSlidingComplete === 'function') {
-          element.props.onSlidingComplete(actualValue);
-        }
-
-        await new Promise(resolve => setTimeout(resolve, 500));
-        return `✅ Adjusted slider [${args.index}] "${element.label}" to ${Math.round(normalizedValue * 100)}% (value: ${actualValue.toFixed(2)})`;
-      } catch (error: any) {
-        return `❌ Error adjusting slider: ${error.message}`;
-      }
-    },
-  };
-}

package/src/tools/tapTool.ts

@@ -1,93 +0,0 @@
-/**
- * Tap Tool — Universal element interaction via onPress.
- *
- * Strategies (in priority order):
- * 1. Switch → onValueChange (toggle)
- * 2. Direct onPress on element
- * 3. Bubble up fiber tree to find parent onPress (max 5 levels)
- *
- * Includes Maestro-style tap verification:
- * - Captures element count + screen name before tap
- * - Compares after tap to detect dead taps
- */
-
-import { walkFiberTree } from '../core/FiberTreeWalker';
-import type { AgentTool, ToolContext } from './types';
-
-export function createTapTool(context: ToolContext): AgentTool {
-  return {
-    name: 'tap',
-    description: 'Tap an interactive element by its index. Works universally on buttons, switches, and custom components.',
-    parameters: {
-      index: { type: 'number', description: 'The index of the element to tap', required: true },
-    },
-    execute: async (args) => {
-      const walkResult = walkFiberTree(context.getRootRef(), context.getWalkConfig());
-      const elements = walkResult.interactives;
-      const element = elements.find(el => el.index === args.index);
-      if (!element) {
-        return `❌ Element with index ${args.index} not found. Available indexes: ${elements.map(e => e.index).join(', ')}`;
-      }
-
-      // Pre-tap snapshot for change detection (Pattern from Maestro: hierarchyBasedTap)
-      const elementCountBefore = elements.length;
-      const screenBefore = context.getCurrentScreenName();
-
-      // Strategy 1: Switch → onValueChange
-      if (element.type === 'switch' && element.props.onValueChange) {
-        try {
-          element.props.onValueChange(!element.props.value);
-          await new Promise(resolve => setTimeout(resolve, 500));
-          return `✅ Toggled [${args.index}] "${element.label}" to ${!element.props.value}`;
-        } catch (error: any) {
-          return `❌ Error toggling [${args.index}]: ${error.message}`;
-        }
-      }
-
-      // Strategy 2: Direct onPress
-      if (element.props.onPress) {
-        try {
-          element.props.onPress();
-          await new Promise(resolve => setTimeout(resolve, 500));
-
-          // Post-tap observation (Maestro pattern: compare hierarchy after tap)
-          const postWalk = walkFiberTree(context.getRootRef(), context.getWalkConfig());
-          const screenAfter = context.getCurrentScreenName();
-          const elementCountAfter = postWalk.interactives.length;
-
-          if (screenAfter !== screenBefore) {
-            return `✅ Tapped [${args.index}] "${element.label}" → navigated to "${screenAfter}"`;
-          }
-          if (Math.abs(elementCountAfter - elementCountBefore) > 0) {
-            return `✅ Tapped [${args.index}] "${element.label}" → screen updated (${elementCountBefore} → ${elementCountAfter} elements)`;
-          }
-          // Many valid taps (add-to-cart, favorites, API calls) update state
-          // without changing visible UI elements. Report success and move on.
-          return `✅ Tapped [${args.index}] "${element.label}" → action executed successfully. Proceed to your next step.`;
-        } catch (error: any) {
-          return `❌ Error tapping [${args.index}]: ${error.message}`;
-        }
-      }
-
-      // Strategy 3: Bubble up fiber tree (like RNTL's findEventHandler)
-      let fiber = element.fiberNode?.return;
-      let bubbleDepth = 0;
-      while (fiber && bubbleDepth < 5) {
-        const parentProps = fiber.memoizedProps || {};
-        if (parentProps.onPress && typeof parentProps.onPress === 'function') {
-          try {
-            parentProps.onPress();
-            await new Promise(resolve => setTimeout(resolve, 500));
-            return `✅ Tapped parent of [${args.index}] "${element.label}"`;
-          } catch (error: any) {
-            return `❌ Error tapping parent of [${args.index}]: ${error.message}`;
-          }
-        }
-        fiber = fiber.return;
-        bubbleDepth++;
-      }
-
-      return `❌ Element [${args.index}] "${element.label}" has no tap handler (no onPress or onValueChange found).`;
-    },
-  };
-}