mcp-maestro-mobile-ai 1.1.1 → 1.3.0

package/src/mcp-server/tools/runTools.js

@@ -4,12 +4,13 @@
  */
 
 import fs from 'fs/promises';
-import path from 'path';
 import { fileURLToPath } from 'url';
 import { dirname, join } from 'path';
 import { logger } from '../utils/logger.js';
 import { runMaestroFlow, checkDeviceConnection, checkAppInstalled, getConfig } from '../utils/maestro.js';
 import { validateMaestroYaml } from './validateTools.js';
+import { validateYamlStructure } from '../utils/yamlTemplate.js';
+import { generateReport, getReportsDir, listReports } from '../utils/reportGenerator.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -47,7 +48,49 @@ export async function runTest(yamlContent, testName, options = {}) {
       };
     }
 
-    // Validate YAML first
+    // STEP 1: Validate YAML structure (catches common AI generation errors)
+    const structureValidation = validateYamlStructure(yamlContent);
+    
+    if (!structureValidation.valid) {
+      logger.warn(`YAML structure validation failed for: ${testName}`, { errors: structureValidation.errors });
+      
+      return {
+        content: [
+          {
+            type: 'text',
+            text: JSON.stringify({
+              success: false,
+              name: testName,
+              error: 'YAML STRUCTURE ERROR - Please fix before running',
+              structureErrors: structureValidation.errors,
+              warnings: structureValidation.warnings,
+              summary: structureValidation.summary,
+              instructions: `
+⚠️ YOUR YAML HAS CRITICAL ISSUES. Please follow these rules:
+
+1. ALWAYS use tapOn BEFORE inputText:
+   CORRECT:
+   - tapOn: "Username"
+   - inputText: "value"
+   
+   WRONG (text goes to wrong field!):
+   - inputText: "value"
+
+2. ALWAYS start with:
+   - clearState
+   - launchApp
+
+3. ALWAYS include appId at the top.
+
+Please regenerate the YAML following these rules.
+`,
+            }),
+          },
+        ],
+      };
+    }
+
+    // STEP 2: Validate YAML syntax
     const validation = await validateMaestroYaml(yamlContent);
     const validationResult = JSON.parse(validation.content[0].text);
 
@@ -59,9 +102,9 @@ export async function runTest(yamlContent, testName, options = {}) {
             text: JSON.stringify({
               success: false,
               name: testName,
-              error: 'YAML validation failed',
+              error: 'YAML syntax validation failed',
               validationErrors: validationResult.errors,
-              hint: 'Fix the YAML errors and try again. Common issues: missing appId, invalid syntax.',
+              hint: 'Fix the YAML syntax errors and try again.',
             }),
           },
         ],
@@ -351,7 +394,187 @@ async function autoCleanupResults() {
   }
 }
 
+/**
+ * Generate a report from test results
+ */
+export async function generateTestReport(results, metadata = {}) {
+  try {
+    const reportResult = await generateReport(results, metadata);
+    
+    return {
+      content: [
+        {
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            reportId: reportResult.reportId,
+            htmlPath: reportResult.htmlPath,
+            jsonPath: reportResult.jsonPath,
+            summary: reportResult.summary,
+            message: `Report generated successfully! Open the HTML file to view: ${reportResult.htmlPath}`,
+            reportsDir: getReportsDir(),
+          }),
+        },
+      ],
+    };
+  } catch (error) {
+    logger.error('Report generation error', { error: error.message });
+    return {
+      content: [
+        {
+          type: 'text',
+          text: JSON.stringify({
+            success: false,
+            error: error.message,
+          }),
+        },
+      ],
+    };
+  }
+}
+
+/**
+ * List all generated reports
+ */
+export async function listTestReports() {
+  try {
+    const reports = await listReports();
+    
+    return {
+      content: [
+        {
+          type: 'text',
+          text: JSON.stringify({
+            success: true,
+            reportsDir: getReportsDir(),
+            reports,
+            count: reports.length,
+          }),
+        },
+      ],
+    };
+  } catch (error) {
+    logger.error('List reports error', { error: error.message });
+    return {
+      content: [
+        {
+          type: 'text',
+          text: JSON.stringify({
+            success: false,
+            error: error.message,
+          }),
+        },
+      ],
+    };
+  }
+}
+
+/**
+ * Run test suite and generate report
+ */
+export async function runTestSuiteWithReport(tests, options = {}) {
+  try {
+    const results = [];
+    const startTime = Date.now();
+
+    for (let i = 0; i < tests.length; i++) {
+      const test = tests[i];
+      logger.info(`Running test ${i + 1}/${tests.length}: ${test.name}`);
+
+      // Validate structure first
+      const structureValidation = validateYamlStructure(test.yaml);
+      if (!structureValidation.valid) {
+        results.push({
+          name: test.name,
+          success: false,
+          error: 'YAML structure error: ' + structureValidation.errors.map(e => e.issue).join(', '),
+          structureErrors: structureValidation.errors,
+        });
+        continue;
+      }
+
+      // Validate syntax
+      const syntaxValidation = await validateMaestroYaml(test.yaml);
+      const syntaxResult = JSON.parse(syntaxValidation.content[0].text);
+      if (!syntaxResult.valid) {
+        results.push({
+          name: test.name,
+          success: false,
+          error: 'YAML syntax error',
+          syntaxErrors: syntaxResult.errors,
+        });
+        continue;
+      }
+
+      // Run the test
+      const testResult = await runMaestroFlow(test.yaml, test.name, {
+        retries: options.retries,
+      });
+      
+      results.push(testResult);
+    }
+
+    const totalDuration = ((Date.now() - startTime) / 1000).toFixed(2);
+
+    // Generate report
+    const reportResult = await generateReport(results, {
+      promptFile: options.promptFile || 'Manual Test Suite',
+      appId: options.appId || process.env.APP_ID || 'unknown',
+      totalDuration,
+    });
+
+    const passed = results.filter(r => r.success).length;
+    const failed = results.length - passed;
+
+    return {
+      content: [
+        {
+          type: 'text',
+          text: JSON.stringify({
+            success: failed === 0,
+            totalDuration: `${totalDuration}s`,
+            summary: {
+              total: results.length,
+              passed,
+              failed,
+              passRate: reportResult.summary.passRate,
+            },
+            report: {
+              reportId: reportResult.reportId,
+              htmlPath: reportResult.htmlPath,
+              jsonPath: reportResult.jsonPath,
+              message: `📊 Report generated: ${reportResult.htmlPath}`,
+            },
+            tests: results.map(r => ({
+              name: r.name,
+              success: r.success,
+              duration: r.duration,
+              error: r.error || null,
+            })),
+          }),
+        },
+      ],
+    };
+  } catch (error) {
+    logger.error('Test suite with report error', { error: error.message });
+    return {
+      content: [
+        {
+          type: 'text',
+          text: JSON.stringify({
+            success: false,
+            error: error.message,
+          }),
+        },
+      ],
+    };
+  }
+}
+
 export default {
   runTest,
   runTestSuite,
+  generateTestReport,
+  listTestReports,
+  runTestSuiteWithReport,
 };

package/src/mcp-server/utils/reportGenerator.js

@@ -0,0 +1,455 @@
+/**
+ * Report Generator
+ * 
+ * Generates HTML and JSON reports from test results
+ */
+
+import fs from 'fs/promises';
+import path from 'path';
+import os from 'os';
+
+// Report storage location
+const USER_HOME = os.homedir();
+const REPORTS_DIR = path.join(USER_HOME, '.maestro-mcp', 'reports');
+
+/**
+ * Ensure reports directory exists
+ */
+async function ensureReportsDir() {
+  await fs.mkdir(REPORTS_DIR, { recursive: true });
+}
+
+/**
+ * Generate a unique report ID
+ */
+function generateReportId() {
+  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+  return `report-${timestamp}`;
+}
+
+/**
+ * Generate JSON report
+ */
+export async function generateJsonReport(results, metadata = {}) {
+  await ensureReportsDir();
+  
+  const reportId = generateReportId();
+  const report = {
+    reportId,
+    generatedAt: new Date().toISOString(),
+    metadata: {
+      promptFile: metadata.promptFile || 'unknown',
+      appId: metadata.appId || 'unknown',
+      totalTests: results.length,
+      ...metadata,
+    },
+    summary: {
+      total: results.length,
+      passed: results.filter(r => r.success).length,
+      failed: results.filter(r => !r.success).length,
+      passRate: results.length > 0 
+        ? Math.round((results.filter(r => r.success).length / results.length) * 100) 
+        : 0,
+    },
+    tests: results.map((result, index) => ({
+      index: index + 1,
+      name: result.name,
+      success: result.success,
+      duration: result.duration || 'N/A',
+      attempts: result.attempts || 1,
+      error: result.error || null,
+      screenshot: result.screenshot || null,
+    })),
+  };
+
+  const jsonPath = path.join(REPORTS_DIR, `${reportId}.json`);
+  await fs.writeFile(jsonPath, JSON.stringify(report, null, 2), 'utf8');
+
+  return {
+    reportId,
+    jsonPath,
+    report,
+  };
+}
+
+/**
+ * Generate HTML report
+ */
+export async function generateHtmlReport(results, metadata = {}) {
+  await ensureReportsDir();
+  
+  const reportId = generateReportId();
+  const timestamp = new Date().toLocaleString();
+  
+  const total = results.length;
+  const passed = results.filter(r => r.success).length;
+  const failed = total - passed;
+  const passRate = total > 0 ? Math.round((passed / total) * 100) : 0;
+
+  // Generate test rows
+  const testRows = results.map((result, index) => {
+    const statusClass = result.success ? 'passed' : 'failed';
+    const statusIcon = result.success ? '✅' : '❌';
+    const errorSection = result.error 
+      ? `<div class="error-details">${escapeHtml(result.error)}</div>` 
+      : '';
+    
+    return `
+      <tr class="${statusClass}">
+        <td>${index + 1}</td>
+        <td>${escapeHtml(result.name)}</td>
+        <td class="status-${statusClass}">${statusIcon} ${result.success ? 'PASSED' : 'FAILED'}</td>
+        <td>${result.duration || 'N/A'}</td>
+        <td>${result.attempts || 1}</td>
+        <td>${errorSection}</td>
+      </tr>
+    `;
+  }).join('\n');
+
+  const html = `<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Test Report - ${reportId}</title>
+  <style>
+    :root {
+      --bg-color: #0d1117;
+      --card-bg: #161b22;
+      --text-color: #c9d1d9;
+      --text-muted: #8b949e;
+      --border-color: #30363d;
+      --success-color: #238636;
+      --success-bg: #1a4721;
+      --error-color: #f85149;
+      --error-bg: #4a1d1d;
+      --primary-color: #58a6ff;
+      --header-bg: #21262d;
+    }
+    
+    * {
+      box-sizing: border-box;
+      margin: 0;
+      padding: 0;
+    }
+    
+    body {
+      font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, sans-serif;
+      background-color: var(--bg-color);
+      color: var(--text-color);
+      line-height: 1.6;
+      padding: 20px;
+    }
+    
+    .container {
+      max-width: 1200px;
+      margin: 0 auto;
+    }
+    
+    header {
+      background: var(--card-bg);
+      border: 1px solid var(--border-color);
+      border-radius: 12px;
+      padding: 24px;
+      margin-bottom: 24px;
+    }
+    
+    h1 {
+      font-size: 24px;
+      font-weight: 600;
+      margin-bottom: 8px;
+      color: var(--primary-color);
+    }
+    
+    .meta {
+      color: var(--text-muted);
+      font-size: 14px;
+    }
+    
+    .summary-cards {
+      display: grid;
+      grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
+      gap: 16px;
+      margin-bottom: 24px;
+    }
+    
+    .card {
+      background: var(--card-bg);
+      border: 1px solid var(--border-color);
+      border-radius: 12px;
+      padding: 20px;
+      text-align: center;
+    }
+    
+    .card-value {
+      font-size: 36px;
+      font-weight: 700;
+      margin-bottom: 4px;
+    }
+    
+    .card-label {
+      color: var(--text-muted);
+      font-size: 14px;
+      text-transform: uppercase;
+      letter-spacing: 0.5px;
+    }
+    
+    .card-passed .card-value { color: var(--success-color); }
+    .card-failed .card-value { color: var(--error-color); }
+    .card-rate .card-value { color: var(--primary-color); }
+    
+    .results-table {
+      background: var(--card-bg);
+      border: 1px solid var(--border-color);
+      border-radius: 12px;
+      overflow: hidden;
+    }
+    
+    table {
+      width: 100%;
+      border-collapse: collapse;
+    }
+    
+    th {
+      background: var(--header-bg);
+      padding: 14px 16px;
+      text-align: left;
+      font-weight: 600;
+      font-size: 13px;
+      text-transform: uppercase;
+      letter-spacing: 0.5px;
+      color: var(--text-muted);
+      border-bottom: 1px solid var(--border-color);
+    }
+    
+    td {
+      padding: 14px 16px;
+      border-bottom: 1px solid var(--border-color);
+      font-size: 14px;
+    }
+    
+    tr:last-child td {
+      border-bottom: none;
+    }
+    
+    tr.passed {
+      background: var(--success-bg);
+    }
+    
+    tr.failed {
+      background: var(--error-bg);
+    }
+    
+    .status-passed {
+      color: var(--success-color);
+      font-weight: 600;
+    }
+    
+    .status-failed {
+      color: var(--error-color);
+      font-weight: 600;
+    }
+    
+    .error-details {
+      background: rgba(0,0,0,0.3);
+      padding: 8px 12px;
+      border-radius: 6px;
+      font-size: 12px;
+      font-family: 'Monaco', 'Menlo', monospace;
+      color: var(--error-color);
+      margin-top: 8px;
+      max-width: 400px;
+      overflow: auto;
+    }
+    
+    footer {
+      margin-top: 24px;
+      text-align: center;
+      color: var(--text-muted);
+      font-size: 12px;
+    }
+    
+    .progress-bar {
+      height: 8px;
+      background: var(--border-color);
+      border-radius: 4px;
+      overflow: hidden;
+      margin-top: 10px;
+    }
+    
+    .progress-fill {
+      height: 100%;
+      background: linear-gradient(90deg, var(--success-color), #2ea043);
+      transition: width 0.3s ease;
+    }
+  </style>
+</head>
+<body>
+  <div class="container">
+    <header>
+      <h1>🧪 MCP Maestro Test Report</h1>
+      <div class="meta">
+        <p><strong>Report ID:</strong> ${reportId}</p>
+        <p><strong>Generated:</strong> ${timestamp}</p>
+        <p><strong>Prompt File:</strong> ${escapeHtml(metadata.promptFile || 'N/A')}</p>
+        <p><strong>App ID:</strong> ${escapeHtml(metadata.appId || 'N/A')}</p>
+      </div>
+    </header>
+    
+    <div class="summary-cards">
+      <div class="card">
+        <div class="card-value">${total}</div>
+        <div class="card-label">Total Tests</div>
+      </div>
+      <div class="card card-passed">
+        <div class="card-value">${passed}</div>
+        <div class="card-label">Passed</div>
+      </div>
+      <div class="card card-failed">
+        <div class="card-value">${failed}</div>
+        <div class="card-label">Failed</div>
+      </div>
+      <div class="card card-rate">
+        <div class="card-value">${passRate}%</div>
+        <div class="card-label">Pass Rate</div>
+        <div class="progress-bar">
+          <div class="progress-fill" style="width: ${passRate}%"></div>
+        </div>
+      </div>
+    </div>
+    
+    <div class="results-table">
+      <table>
+        <thead>
+          <tr>
+            <th>#</th>
+            <th>Test Name</th>
+            <th>Status</th>
+            <th>Duration</th>
+            <th>Attempts</th>
+            <th>Details</th>
+          </tr>
+        </thead>
+        <tbody>
+          ${testRows}
+        </tbody>
+      </table>
+    </div>
+    
+    <footer>
+      <p>Generated by MCP Maestro Mobile AI</p>
+      <p>Report Location: ${REPORTS_DIR}</p>
+    </footer>
+  </div>
+</body>
+</html>`;
+
+  const htmlPath = path.join(REPORTS_DIR, `${reportId}.html`);
+  await fs.writeFile(htmlPath, html, 'utf8');
+
+  return {
+    reportId,
+    htmlPath,
+    summary: {
+      total,
+      passed,
+      failed,
+      passRate,
+    },
+  };
+}
+
+/**
+ * Generate both JSON and HTML reports
+ */
+export async function generateReport(results, metadata = {}) {
+  const jsonResult = await generateJsonReport(results, metadata);
+  const htmlResult = await generateHtmlReport(results, metadata);
+
+  return {
+    reportId: jsonResult.reportId,
+    jsonPath: jsonResult.jsonPath,
+    htmlPath: htmlResult.htmlPath,
+    summary: htmlResult.summary,
+    report: jsonResult.report,
+  };
+}
+
+/**
+ * Get reports directory path
+ */
+export function getReportsDir() {
+  return REPORTS_DIR;
+}
+
+/**
+ * List all generated reports
+ */
+export async function listReports() {
+  await ensureReportsDir();
+  
+  try {
+    const files = await fs.readdir(REPORTS_DIR);
+    const htmlReports = files.filter(f => f.endsWith('.html'));
+    
+    const reports = [];
+    for (const file of htmlReports) {
+      const reportId = file.replace('.html', '');
+      const jsonPath = path.join(REPORTS_DIR, `${reportId}.json`);
+      const htmlPath = path.join(REPORTS_DIR, file);
+      
+      try {
+        const jsonContent = await fs.readFile(jsonPath, 'utf8');
+        const data = JSON.parse(jsonContent);
+        reports.push({
+          reportId,
+          generatedAt: data.generatedAt,
+          promptFile: data.metadata?.promptFile,
+          summary: data.summary,
+          htmlPath,
+          jsonPath,
+        });
+      } catch {
+        // JSON file might not exist
+        reports.push({
+          reportId,
+          htmlPath,
+        });
+      }
+    }
+
+    // Sort by date (newest first)
+    reports.sort((a, b) => {
+      if (a.generatedAt && b.generatedAt) {
+        return new Date(b.generatedAt) - new Date(a.generatedAt);
+      }
+      return 0;
+    });
+
+    return reports;
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Escape HTML special characters
+ */
+function escapeHtml(text) {
+  if (!text) return '';
+  return String(text)
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&#039;');
+}
+
+export default {
+  generateJsonReport,
+  generateHtmlReport,
+  generateReport,
+  getReportsDir,
+  listReports,
+};
+