mcp-maestro-mobile-ai 1.3.1 → 1.6.0

package/src/mcp-server/utils/yamlTemplate.js

@@ -3,8 +3,18 @@
  *
  * Provides comprehensive, GENERIC instructions for AI to generate valid Maestro YAML.
  * Works for ANY test scenario - login, forms, navigation, search, etc.
+ *
+ * Includes Known Issues & Solutions Registry for handling edge cases
+ * discovered during test execution across Flutter, React Native, and Native apps.
  */
 
+import {
+  getInteractionPatternHints,
+  getYamlGenerationRules,
+  getBestPractices,
+  PatternCategory,
+} from "./knownIssues.js";
+
 /**
  * COMPREHENSIVE YAML generation instructions for AI
  * These are GENERIC rules that apply to ALL test scenarios
@@ -210,6 +220,195 @@ appId: {APP_ID}
 
 ---
 
+### 7.1. FLUTTER DROPDOWN PATTERNS (⚠️ CRITICAL FOR FLUTTER APPS)
+
+Flutter dropdowns are rendered as overlays and require special handling:
+
+#### Method 1: Tap and Select by Visible Text (Preferred)
+\`\`\`yaml
+# Step 1: Tap on the dropdown to open it
+- tapOn: "Current Selected Value"    # e.g., "Australia"
+
+# Step 2: Wait for dropdown overlay to appear
+- waitForAnimationToEnd
+
+# Step 3: Tap on the desired option by its exact text
+- tapOn: "Desired Option Text"       # e.g., "United Kingdom"
+\`\`\`
+
+#### Method 2: Using text: selector for partial matching
+\`\`\`yaml
+# Open dropdown
+- tapOn:
+    text: "Australia"
+- waitForAnimationToEnd
+
+# Select option using partial text match
+- tapOn:
+    text: "United Kingdom"
+\`\`\`
+
+#### Method 3: Scroll within dropdown to find option
+\`\`\`yaml
+# Open dropdown
+- tapOn: "Australia"
+- waitForAnimationToEnd
+
+# If option is not visible, scroll within the dropdown
+- scroll:
+    direction: DOWN
+
+# Then tap on the option
+- tapOn: "United Kingdom"
+\`\`\`
+
+#### Method 4: Using index for dropdown items (when text fails)
+\`\`\`yaml
+# Open dropdown
+- tapOn: "Australia"
+- waitForAnimationToEnd
+
+# Select by index (0-based) - e.g., index 1 for second item
+- tapOn:
+    index: 1
+\`\`\`
+
+#### Method 5: Using containsText for flexible matching
+\`\`\`yaml
+# Open dropdown
+- tapOn:
+    containsText: "Austral"
+- waitForAnimationToEnd
+
+# Select using contains
+- tapOn:
+    containsText: "United King"
+\`\`\`
+
+#### Method 6: Using optional property for robust selection
+\`\`\`yaml
+# Open dropdown and handle if already open
+- tapOn:
+    text: "Australia"
+    optional: true
+- waitForAnimationToEnd
+
+# Try multiple selectors for the option
+- runFlow:
+    when:
+      visible: "United Kingdom"
+    commands:
+      - tapOn: "United Kingdom"
+
+# Fallback: try by index if text fails
+- runFlow:
+    when:
+      notVisible: "United Kingdom"  
+    commands:
+      - tapOn:
+          index: 1
+\`\`\`
+
+#### Method 7: Using assertVisible before tap (for debugging)
+\`\`\`yaml
+# Open dropdown
+- tapOn: "Australia"
+- waitForAnimationToEnd
+
+# Verify the dropdown opened and options are visible
+- assertVisible: "United Kingdom"
+
+# Now tap
+- tapOn: "United Kingdom"
+\`\`\`
+
+#### Method 8: Complete Login with Dropdown Example
+\`\`\`yaml
+appId: com.example.app
+---
+# Login with region selection
+- clearState
+- launchApp
+
+# Enter credentials (always tap before input!)
+- tapOn: "Username"
+- inputText: "myuser"
+- tapOn: "Password"  
+- inputText: "mypassword"
+
+# Hide keyboard before interacting with dropdown
+- hideKeyboard
+
+# Select region from dropdown
+- tapOn: "Australia"           # Current selection - opens dropdown
+- waitForAnimationToEnd        # Wait for overlay
+- tapOn: "United Kingdom"      # Desired option
+
+# Submit form
+- tapOn: "SIGN IN"
+
+# Verify success
+- assertVisible: "Welcome"
+\`\`\`
+
+#### Flutter Dropdown Tips:
+1. ⚠️ **ALWAYS hideKeyboard** before tapping a dropdown - keyboard may cover it!
+2. ⚠️ **ALWAYS waitForAnimationToEnd** after opening dropdown
+3. ⚠️ Use **exact text** of the dropdown options when possible
+4. ⚠️ If options are scrollable, add **scroll** command before selecting
+5. ⚠️ Some Flutter dropdowns need a **pressKey: escape** to close if tapped outside
+6. ⚠️ Use **assertVisible** before tapOn to verify dropdown is open
+
+#### Flutter Accessibility Issue - When Text Matching Fails:
+⚠️ **CRITICAL**: Flutter apps often expose elements only via \`accessibilityText\`, not \`text\`.
+When Maestro cannot find elements by text, use **point coordinates**:
+
+\`\`\`yaml
+# Step 1: Use maestro hierarchy command to get element bounds
+# Example: bounds = "[53,1238][1028,1386]"
+# Calculate center: X = (53+1028)/2 = 540, Y = (1238+1386)/2 = 1312
+
+# Step 2: Tap by coordinates
+- tapOn:
+    point: 540,1312
+
+# For dropdown options, get bounds from hierarchy and tap centers:
+- tapOn:
+    point: 540,1438   # Center of "United Kingdom" option
+\`\`\`
+
+**How to get coordinates:**
+1. Run \`maestro hierarchy\` to dump UI tree
+2. Find element by \`accessibilityText\` in the JSON
+3. Extract \`bounds\`: "[left,top][right,bottom]"
+4. Calculate center: X = (left+right)/2, Y = (top+bottom)/2
+5. Use \`tapOn: point: X,Y\`
+
+---
+
+### 7.2. PICKER AND DATE PICKER PATTERNS (Mobile)
+
+\`\`\`yaml
+# Date Picker
+- tapOn: "Date Field"
+- waitForAnimationToEnd
+- scroll:
+    direction: DOWN
+    element: "year picker"
+- tapOn: "2024"
+- tapOn: "OK"
+
+# Time Picker  
+- tapOn: "Time Field"
+- waitForAnimationToEnd
+- tapOn: "10"   # Hour
+- tapOn: "30"   # Minutes
+- tapOn: "AM"   # Period
+- tapOn: "OK"
+\`\`\`
+
+---
+
 ### 8. ❌ NEVER DO THESE
 
 1. ❌ **NEVER** use \`inputText\` without \`tapOn\` first
@@ -218,6 +417,8 @@ appId: {APP_ID}
 4. ❌ **NEVER** use coordinates (x,y) unless absolutely necessary
 5. ❌ **NEVER** use hardcoded waits (\`sleep\`) - use \`extendedWaitUntil\`
 6. ❌ **NEVER** forget to hide keyboard after text input if it blocks elements
+7. ❌ **NEVER** tap dropdown options without \`waitForAnimationToEnd\` after opening dropdown
+8. ❌ **NEVER** try to select dropdown option while keyboard is visible
 
 ---
 
@@ -229,6 +430,9 @@ appId: {APP_ID}
 4. ✅ **ALWAYS** add \`assertVisible\` to verify expected results
 5. ✅ **ALWAYS** use waits for elements that may take time to load
 6. ✅ **ALWAYS** hide keyboard if it might block buttons: \`- hideKeyboard\`
+7. ✅ **ALWAYS** use \`hideKeyboard\` BEFORE interacting with dropdowns
+8. ✅ **ALWAYS** use \`waitForAnimationToEnd\` AFTER opening a dropdown/picker
+9. ✅ **ALWAYS** tap the CURRENT VALUE to open dropdown, then tap the DESIRED VALUE
 
 ---
 
@@ -304,6 +508,12 @@ export function getYamlGenerationContext(appId, appContext = null) {
     }
   }
 
+  // Add generic interaction patterns for reliable tests
+  context += getInteractionPatternHints();
+
+  // Add best practices
+  context += getBestPractices();
+
   return context;
 }
 
@@ -402,6 +612,86 @@ appId: {APP_ID}
 - scroll  # If logout is below fold
 - tapOn: "{logout_button}"
 - assertVisible: "{login_screen_element}"
+`,
+
+  dropdown: `# Dropdown/Picker Selection Pattern
+appId: {APP_ID}
+---
+- clearState
+- launchApp
+
+# IMPORTANT: Hide keyboard first if any text field was focused
+- hideKeyboard
+
+# Step 1: Tap on dropdown (tap the CURRENT selected value)
+- tapOn: "{current_selected_value}"
+
+# Step 2: ALWAYS wait for dropdown overlay to appear
+- waitForAnimationToEnd
+
+# Step 3: If options might be scrollable, scroll to find the option
+# - scroll:
+#     direction: DOWN
+
+# Step 4: Tap on the desired option
+- tapOn: "{desired_option_value}"
+
+# Step 5: Verify selection (optional)
+- assertVisible: "{desired_option_value}"
+`,
+
+  flutter_login_with_dropdown: `# Flutter Login with Dropdown Pattern (e.g., Region Selection)
+appId: {APP_ID}
+---
+# Login with dropdown selection (e.g., region/country picker)
+- clearState
+- launchApp
+
+# Enter username
+- tapOn: "{username_field}"
+- inputText: "{username}"
+
+# Enter password
+- tapOn: "{password_field}"
+- inputText: "{password}"
+
+# CRITICAL: Hide keyboard before dropdown interaction!
+- hideKeyboard
+
+# Select from dropdown
+- tapOn: "{current_dropdown_value}"    # e.g., "Australia"
+- waitForAnimationToEnd                 # Wait for dropdown overlay
+- tapOn: "{desired_dropdown_value}"     # e.g., "United Kingdom"
+
+# Submit
+- tapOn: "{submit_button}"
+
+# Verify success
+- assertVisible: "{success_indicator}"
+`,
+
+  flutter_multi_dropdown: `# Flutter Multiple Dropdowns Pattern
+appId: {APP_ID}
+---
+- clearState
+- launchApp
+
+# First dropdown
+- hideKeyboard
+- tapOn: "{dropdown_1_current_value}"
+- waitForAnimationToEnd
+- tapOn: "{dropdown_1_desired_value}"
+
+# Wait for first dropdown to close
+- waitForAnimationToEnd
+
+# Second dropdown
+- tapOn: "{dropdown_2_current_value}"
+- waitForAnimationToEnd
+- tapOn: "{dropdown_2_desired_value}"
+
+# Continue with form...
+- tapOn: "{submit_button}"
 `,
 };
 
@@ -550,10 +840,23 @@ This ensures consistent YAML generation next time!
 `;
 }
 
+// Re-export interaction pattern utilities for direct access
+export {
+  getInteractionPatternHints,
+  getYamlGenerationRules,
+  getBestPractices,
+  PatternCategory,
+} from "./knownIssues.js";
+
 export default {
   YAML_GENERATION_INSTRUCTIONS,
   getYamlGenerationContext,
   TEST_PATTERNS,
   validateYamlStructure,
   getScreenAnalysisInstructions,
+  // Interaction pattern utilities
+  getInteractionPatternHints,
+  getYamlGenerationRules,
+  getBestPractices,
+  PatternCategory,
 };