mcp-maestro-mobile-ai 1.1.0 → 1.3.0

package/src/mcp-server/tools/contextTools.js

@@ -14,6 +14,13 @@ import {
   clearAppContext,
   listAppContexts,
 } from "../utils/appContext.js";
+import {
+  YAML_GENERATION_INSTRUCTIONS,
+  getYamlGenerationContext,
+  TEST_PATTERNS,
+  validateYamlStructure,
+  getScreenAnalysisInstructions,
+} from "../utils/yamlTemplate.js";
 
 /**
  * Format result as MCP response
@@ -180,6 +187,118 @@ export async function listContexts() {
   return formatResponse(result);
 }
 
+/**
+ * Get YAML generation instructions
+ * AI MUST call this before generating any Maestro YAML
+ */
+export async function getYamlInstructions(appId) {
+  if (!appId) {
+    // Return generic instructions without app context
+    return formatResponse({
+      success: true,
+      instructions: YAML_GENERATION_INSTRUCTIONS,
+      patterns: TEST_PATTERNS,
+      message: "Use these instructions when generating Maestro YAML. Always follow the tapOn → inputText pattern for text input.",
+    });
+  }
+
+  // Get app-specific context
+  const appContext = await loadAppContext(appId);
+  const instructions = getYamlGenerationContext(appId, appContext);
+
+  return formatResponse({
+    success: true,
+    appId,
+    instructions,
+    patterns: TEST_PATTERNS,
+    hasAppContext: Object.keys(appContext.elements || {}).length > 0,
+    message: "IMPORTANT: Follow these instructions EXACTLY when generating YAML. Always use tapOn before inputText.",
+  });
+}
+
+/**
+ * Validate YAML before running
+ * Checks for common issues like missing tapOn before inputText
+ */
+export async function validateYamlBeforeRun(yamlContent) {
+  if (!yamlContent) {
+    return formatResponse({
+      success: false,
+      error: "yamlContent is required",
+    });
+  }
+
+  const validation = validateYamlStructure(yamlContent);
+
+  if (!validation.valid) {
+    return formatResponse({
+      success: false,
+      valid: false,
+      issues: validation.issues,
+      message: "YAML has structural issues that may cause test failures. Please fix before running.",
+      fixedYaml: null, // Could auto-fix in future
+    });
+  }
+
+  return formatResponse({
+    success: true,
+    valid: true,
+    message: "YAML structure is valid.",
+  });
+}
+
+/**
+ * Get a test pattern template
+ */
+export async function getTestPattern(patternName) {
+  const patterns = {
+    login: TEST_PATTERNS.login,
+    search: TEST_PATTERNS.search,
+    navigation: TEST_PATTERNS.navigation,
+    form: TEST_PATTERNS.form,
+    list: TEST_PATTERNS.list,
+    settings: TEST_PATTERNS.settings,
+    logout: TEST_PATTERNS.logout,
+  };
+
+  const pattern = patterns[patternName?.toLowerCase()];
+
+  if (!pattern) {
+    return formatResponse({
+      success: false,
+      error: `Unknown pattern: ${patternName}`,
+      availablePatterns: Object.keys(patterns),
+      hint: "Use: login, form, search, navigation, list, settings, or logout",
+    });
+  }
+
+  return formatResponse({
+    success: true,
+    pattern: patternName,
+    template: pattern,
+    message: "Replace placeholders in {} with actual values. REMEMBER: Always use tapOn before inputText!",
+  });
+}
+
+/**
+ * Get screen analysis instructions
+ * Helps AI understand how to gather UI element information
+ */
+export async function getScreenAnalysis() {
+  const instructions = getScreenAnalysisInstructions();
+  
+  return formatResponse({
+    success: true,
+    instructions,
+    message: "Use these instructions to gather UI element information before generating YAML.",
+    requiredInfo: [
+      "Field labels/placeholders for text inputs",
+      "Button text for actions",
+      "Success/error indicators to verify results",
+    ],
+  });
+}
+
 export default {
   registerAppElements,
   registerAppScreen,
@@ -190,5 +309,9 @@ export default {
   getAppContext,
   clearContext,
   listContexts,
+  getYamlInstructions,
+  validateYamlBeforeRun,
+  getTestPattern,
+  getScreenAnalysis,
 };
 

package/src/mcp-server/tools/runTools.js

@@ -4,12 +4,13 @@
  */
 
 import fs from 'fs/promises';
-import path from 'path';
 import { fileURLToPath } from 'url';
 import { dirname, join } from 'path';
 import { logger } from '../utils/logger.js';
 import { runMaestroFlow, checkDeviceConnection, checkAppInstalled, getConfig } from '../utils/maestro.js';
 import { validateMaestroYaml } from './validateTools.js';
+import { validateYamlStructure } from '../utils/yamlTemplate.js';
+import { generateReport, getReportsDir, listReports } from '../utils/reportGenerator.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -47,7 +48,49 @@ export async function runTest(yamlContent, testName, options = {}) {
       };
     }
 
-    // Validate YAML first
+    // STEP 1: Validate YAML structure (catches common AI generation errors)
+    const structureValidation = validateYamlStructure(yamlContent);
+    
+    if (!structureValidation.valid) {
+      logger.warn(`YAML structure validation failed for: ${testName}`, { errors: structureValidation.errors });
+      
+      return {
+        content: [
+          {
+            type: 'text',
+            text: JSON.stringify({
+              success: false,
+              name: testName,
+              error: 'YAML STRUCTURE ERROR - Please fix before running',
+              structureErrors: structureValidation.errors,
+              warnings: structureValidation.warnings,
+              summary: structureValidation.summary,
+              instructions: `
+⚠️ YOUR YAML HAS CRITICAL ISSUES. Please follow these rules:
+
+1. ALWAYS use tapOn BEFORE inputText:
+   CORRECT:
+   - tapOn: "Username"
+   - inputText: "value"
+   
+   WRONG (text goes to wrong field!):
+   - inputText: "value"
+
+2. ALWAYS start with:
+   - clearState
+   - launchApp
+
+3. ALWAYS include appId at the top.
+
+Please regenerate the YAML following these rules.
+`,
+            }),
+          },
+        ],
+      };
+    }
+
+    // STEP 2: Validate YAML syntax
     const validation = await validateMaestroYaml(yamlContent);
     const validationResult = JSON.parse(validation.content[0].text);
 
@@ -59,9 +102,9 @@ export async function runTest(yamlContent, testName, options = {}) {
             text: JSON.stringify({
               success: false,
               name: testName,
-              error: 'YAML validation failed',
+              error: 'YAML syntax validation failed',
               validationErrors: validationResult.errors,
-              hint: 'Fix the YAML errors and try again. Common issues: missing appId, invalid syntax.',
+              hint: 'Fix the YAML syntax errors and try again.',
             }),
           },
         ],
@@ -351,7 +394,187 @@ async function autoCleanupResults() {
   }
 }
 
+/**
+ * Generate a report from test results
+ */
+export async function generateTestReport(results, metadata = {}) {
+  try {
+    const reportResult = await generateReport(results, metadata);
+    
+    return {
+      content: [
+        {
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            reportId: reportResult.reportId,
+            htmlPath: reportResult.htmlPath,
+            jsonPath: reportResult.jsonPath,
+            summary: reportResult.summary,
+            message: `Report generated successfully! Open the HTML file to view: ${reportResult.htmlPath}`,
+            reportsDir: getReportsDir(),
+          }),
+        },
+      ],
+    };
+  } catch (error) {
+    logger.error('Report generation error', { error: error.message });
+    return {
+      content: [
+        {
+          type: 'text',
+          text: JSON.stringify({
+            success: false,
+            error: error.message,
+          }),
+        },
+      ],
+    };
+  }
+}
+
+/**
+ * List all generated reports
+ */
+export async function listTestReports() {
+  try {
+    const reports = await listReports();
+    
+    return {
+      content: [
+        {
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            reportsDir: getReportsDir(),
+            reports,
+            count: reports.length,
+          }),
+        },
+      ],
+    };
+  } catch (error) {
+    logger.error('List reports error', { error: error.message });
+    return {
+      content: [
+        {
+          type: 'text',
+          text: JSON.stringify({
+            success: false,
+            error: error.message,
+          }),
+        },
+      ],
+    };
+  }
+}
+
+/**
+ * Run test suite and generate report
+ */
+export async function runTestSuiteWithReport(tests, options = {}) {
+  try {
+    const results = [];
+    const startTime = Date.now();
+
+    for (let i = 0; i < tests.length; i++) {
+      const test = tests[i];
+      logger.info(`Running test ${i + 1}/${tests.length}: ${test.name}`);
+
+      // Validate structure first
+      const structureValidation = validateYamlStructure(test.yaml);
+      if (!structureValidation.valid) {
+        results.push({
+          name: test.name,
+          success: false,
+          error: 'YAML structure error: ' + structureValidation.errors.map(e => e.issue).join(', '),
+          structureErrors: structureValidation.errors,
+        });
+        continue;
+      }
+
+      // Validate syntax
+      const syntaxValidation = await validateMaestroYaml(test.yaml);
+      const syntaxResult = JSON.parse(syntaxValidation.content[0].text);
+      if (!syntaxResult.valid) {
+        results.push({
+          name: test.name,
+          success: false,
+          error: 'YAML syntax error',
+          syntaxErrors: syntaxResult.errors,
+        });
+        continue;
+      }
+
+      // Run the test
+      const testResult = await runMaestroFlow(test.yaml, test.name, {
+        retries: options.retries,
+      });
+      
+      results.push(testResult);
+    }
+
+    const totalDuration = ((Date.now() - startTime) / 1000).toFixed(2);
+
+    // Generate report
+    const reportResult = await generateReport(results, {
+      promptFile: options.promptFile || 'Manual Test Suite',
+      appId: options.appId || process.env.APP_ID || 'unknown',
+      totalDuration,
+    });
+
+    const passed = results.filter(r => r.success).length;
+    const failed = results.length - passed;
+
+    return {
+      content: [
+        {
+          type: 'text',
+          text: JSON.stringify({
+            success: failed === 0,
+            totalDuration: `${totalDuration}s`,
+            summary: {
+              total: results.length,
+              passed,
+              failed,
+              passRate: reportResult.summary.passRate,
+            },
+            report: {
+              reportId: reportResult.reportId,
+              htmlPath: reportResult.htmlPath,
+              jsonPath: reportResult.jsonPath,
+              message: `📊 Report generated: ${reportResult.htmlPath}`,
+            },
+            tests: results.map(r => ({
+              name: r.name,
+              success: r.success,
+              duration: r.duration,
+              error: r.error || null,
+            })),
+          }),
+        },
+      ],
+    };
+  } catch (error) {
+    logger.error('Test suite with report error', { error: error.message });
+    return {
+      content: [
+        {
+          type: 'text',
+          text: JSON.stringify({
+            success: false,
+            error: error.message,
+          }),
+        },
+      ],
+    };
+  }
+}
+
 export default {
   runTest,
   runTestSuite,
+  generateTestReport,
+  listTestReports,
+  runTestSuiteWithReport,
 };