@appliqation/automation-sdk 2.1.0

package/src/reporters/playwright/AppliqationReporter.js

@@ -0,0 +1,654 @@
+const AppliqationClient = require('../../AppliqationClient');
+const UuidExtractor = require('./helpers/UuidExtractor');
+const DeviceOsDetector = require('./helpers/DeviceOsDetector');
+const PayloadBuilder = require('../../utils/PayloadBuilder');
+const logger = require('../../utils/logger');
+
+/**
+ * Appliqation Reporter for Playwright
+ * Automatically reports test results to Appliqation platform
+ *
+ * @example
+ * // playwright.config.js
+ * import { AppliqationReporter } from '@appliqation/automation-sdk/playwright';
+ *
+ * export default {
+ *   reporter: [
+ *     ['list'],
+ *     [AppliqationReporter, {
+ *       baseUrl: 'https://your-instance.appliqation.com',
+ *       apiKey: process.env.APPLIQATION_API_KEY,
+ *       projectKey: process.env.APPLIQATION_PROJECT_KEY,
+ *       scenarioId: 123,
+ *       environment: 'Production'
+ *     }]
+ *   ]
+ * };
+ */
+class AppliqationReporter {
+  /**
+   * Create Appliqation reporter
+   * @param {Object} config - Reporter configuration
+   * @param {string} config.baseUrl - Appliqation instance URL
+   * @param {string} config.apiKey - API key
+   * @param {string} config.projectKey - Project key
+   * @param {number} config.scenarioId - Scenario ID
+   * @param {number} [config.testSetId] - Test Set ID (alternative to scenarioId)
+   * @param {string} [config.environment='Local'] - Environment name
+   * @param {boolean} [config.autoCreateRun=true] - Auto-create run matrix
+   * @param {boolean} [config.logOrphans=true] - Log orphan tests
+   * @param {boolean} [config.batchSubmit=true] - Use batch submission
+   * @param {number} [config.batchSize=50] - Batch size for submissions
+   * @param {string} [config.logLevel='info'] - Log level
+   */
+  constructor(config) {
+    this.config = {
+      autoCreateRun: true,
+      logOrphans: true,
+      batchSubmit: true,
+      batchSize: 50,
+      logLevel: 'info',
+      ...config
+    };
+
+    // Validate configuration
+    this.validateConfig();
+
+    // Initialize SDK client
+    this.client = new AppliqationClient({
+      baseUrl: this.config.baseUrl,
+      apiKey: this.config.apiKey,
+      projectKey: this.config.projectKey,
+      rejectUnauthorized: this.config.rejectUnauthorized, // Pass through SSL configuration
+      options: {
+        logLevel: this.config.logLevel,
+        logOrphans: this.config.logOrphans
+      }
+    });
+
+    // State tracking
+    this.runsByProject = new Map(); // Map<projectKey, runInfo>
+    this.resultsByRun = new Map(); // Map<runId, results[]>
+    this.orphansByRun = new Map(); // Map<runId, orphans[]>
+    this.browserVersionDetected = new Map(); // Map<projectKey, boolean> - Track if version detected
+    this.totalTests = 0;
+    this.passedTests = 0;
+    this.failedTests = 0;
+    this.skippedTests = 0;
+    this.orphanTests = 0;
+
+    logger.info('Appliqation Playwright Reporter initialized', {
+      baseUrl: this.config.baseUrl,
+      scenarioId: this.config.scenarioId,
+      environment: this.config.environment
+    });
+  }
+
+  /**
+   * Called once before running tests
+   * @param {Object} config - Playwright config
+   * @param {Object} suite - Root test suite
+   */
+  async onBegin(config, suite) {
+    logger.info('Test run starting...');
+
+    if (!this.config.autoCreateRun) {
+      logger.info('Auto-create run disabled. Skipping run matrix creation.');
+      return;
+    }
+
+    // Filter out projects that have custom restrictive testMatch patterns in the original config
+    // (e.g., API project with /.*\.api\.spec\.js/)
+    // Note: When you run "npx playwright test file.spec.js", Playwright adds testMatch to ALL projects
+    // So we need to check if testMatch is a CUSTOM pattern (contains ".api" or similar)
+    const browserProjects = config.projects.filter(project => {
+      // Check if testMatch is a custom API-specific pattern
+      if (project.testMatch) {
+        const testMatchStr = project.testMatch.toString();
+        // Exclude API testing projects (testMatch contains ".api")
+        if (testMatchStr.includes('.api')) {
+          logger.debug(`Excluding API project "${project.name}" from matrix creation`);
+          return false;
+        }
+      }
+      return true;
+    });
+
+    logger.debug(`Creating matrices for ${browserProjects.length} browser projects (filtered from ${config.projects.length} total)`);
+
+    // Auto-detect browser versions for projects that don't have metadata.browserVersion configured
+    await this.injectBrowserVersions(browserProjects);
+
+    // Get unique device/OS/browser matrix configurations from browser projects only
+    const matrixConfigs = DeviceOsDetector.getMatrixConfigurations(browserProjects);
+
+    logger.info(`Creating ${matrixConfigs.length} run matrices for device/OS combinations...`);
+    logger.debug(`Matrix configurations:`, matrixConfigs);
+
+    // Create runs for each device/OS combination with ALL browsers
+    for (const matrixConfig of matrixConfigs) {
+      const projectKey = `${matrixConfig.device}-${matrixConfig.os}`;
+
+      // Skip if already created (shouldn't happen but safety check)
+      if (this.runsByProject.has(projectKey)) continue;
+
+      try {
+        const runOptions = {
+          scenarioId: this.config.scenarioId,
+          testSetId: this.config.testSetId,
+          environment: this.config.environment,
+          browsers: matrixConfig.browsers,  // ALL browsers for this device/OS
+          device: matrixConfig.device,
+          os: matrixConfig.os,
+          title: this.config.title || process.env.APPLIQATION_RUN_TITLE || `Automation Run - ${new Date().toISOString()}`
+        };
+
+        const run = await this.client.createRun(runOptions);
+
+        const runInfo = {
+          ...run,
+          device: matrixConfig.device,
+          os: matrixConfig.os,
+          browsers: matrixConfig.browsers
+        };
+
+        this.runsByProject.set(projectKey, runInfo);
+        this.resultsByRun.set(run.runId, []);
+        this.orphansByRun.set(run.runId, []);
+
+        logger.info(`Run matrix created for ${projectKey}`, {
+          runId: run.runId,
+          browsers: matrixConfig.browsers
+        });
+      } catch (error) {
+        // Check if this is an API validation error with details
+        if (error.details) {
+          const details = error.details;
+
+          // Log structured validation error
+          logger.error(`API validation error for ${projectKey}`, {
+            error_code: details.error_code,
+            message: details.message,
+            status: error.statusCode,
+            details: details
+          });
+
+          // Display formatted validation error to user
+          console.error('\n' + '═'.repeat(80));
+          console.error('❌ ENVIRONMENT VALIDATION ERROR');
+          console.error('═'.repeat(80));
+          console.error(`Project ID: ${details.project_id || this.config.projectKey}`);
+          console.error(`Device/OS: ${projectKey}`);
+          console.error(`Error: ${details.message || error.message}`);
+
+          if (details.error_code === 'NO_ENVIRONMENTS_CONFIGURED') {
+            console.error('\n⚠️  ACTION REQUIRED:');
+            console.error(`   ${details.action_required || 'Configure testing environments in project settings'}`);
+            if (details.help_url) {
+              console.error(`   URL: ${details.help_url}`);
+            }
+          } else if (details.error_code === 'INVALID_ENVIRONMENT') {
+            console.error(`\nProvided Environment: "${details.provided_environment}"`);
+            if (details.valid_environments && details.valid_environments.length > 0) {
+              console.error('\n✓ Valid Environments:');
+              details.valid_environments.forEach(env => {
+                console.error(`   - ${env}`);
+              });
+            }
+            if (details.hint) {
+              console.error(`\n💡 Hint: ${details.hint}`);
+            }
+          }
+
+          console.error('═'.repeat(80) + '\n');
+
+          // Throw error to fail the test run visibly
+          throw new Error(`Environment validation failed: ${details.message || error.message}`);
+        } else {
+          // Generic error without validation details
+          logger.error(`Failed to create run matrix for ${projectKey}`, {
+            error: error.message,
+            stack: error.stack
+          });
+          console.error(`⚠️  WARNING: Run matrix creation failed for ${projectKey}: ${error.message}`);
+
+          // Throw error to fail the test run
+          throw error;
+        }
+      }
+    }
+  }
+
+  /**
+   * Called after a test completes
+   * @param {Object} test - Test object
+   * @param {Object} result - Test result
+   */
+  async onTestEnd(test, result) {
+    this.totalTests++;
+
+    try {
+      // Extract UUID from test - check result.annotations first since test.info().annotations.push() adds to result
+      logger.debug('Extracting UUID for test', {
+        title: test.title,
+        resultAnnotations: result.annotations || [],
+        testAnnotations: test.annotations || []
+      });
+      const uuid = UuidExtractor.extractFromAnnotations(result.annotations || []) || UuidExtractor.extractFromTest(test);
+      logger.debug('UUID extraction result', { title: test.title, uuid });
+
+      // Get device info from project
+      // Note: project() is a method, not a property
+      const project = test.parent?.project?.() || test.parent?.project;
+
+      // Browser version auto-detection happens during onBegin via getBrowserVersionFromPlaywright()
+      // No need to extract browser instance here - version is already in project metadata
+      const deviceInfo = DeviceOsDetector.getDeviceInfo(project, null);
+      const projectKey = `${deviceInfo.device}-${deviceInfo.os}`;
+
+      // Get run info for this device/OS combination (pre-created in onBegin)
+      const runInfo = this.runsByProject.get(projectKey);
+
+      if (!runInfo) {
+        logger.warn('No run matrix found for project', { projectKey });
+        return;
+      }
+
+      if (!uuid) {
+        // Orphan test - no UUID found
+        this.orphanTests++;
+        this.trackOrphan(runInfo.runId, test, result, deviceInfo.browser);
+
+        logger.warn('Test without UUID', {
+          title: test.title,
+          file: test.location?.file,
+          browser: deviceInfo.browser
+        });
+
+        return;
+      }
+
+      // Create result object
+      const testResult = {
+        runId: runInfo.runId, // Add runId for batch submission
+        uuid,
+        status: this.mapStatus(result.status),
+        browser: deviceInfo.browser,
+        device: deviceInfo.device,
+        os: deviceInfo.os,
+        comment: this.buildComment(test, result),
+        duration: result.duration,
+        timestamp: Math.floor(Date.now() / 1000)
+      };
+
+      // Track result for batch submission
+      this.trackResult(runInfo.runId, testResult);
+
+      // Update counters
+      if (testResult.status === 'passed') {
+        this.passedTests++;
+      } else if (testResult.status === 'failed') {
+        this.failedTests++;
+      } else if (testResult.status === 'skipped') {
+        this.skippedTests++;
+      }
+
+      // Submit immediately if batch mode disabled
+      if (!this.config.batchSubmit) {
+        await this.client.submitResult(runInfo.runId, testResult);
+      }
+    } catch (error) {
+      logger.error('Error processing test result', {
+        error: error.message,
+        test: test.title
+      });
+    }
+  }
+
+  /**
+   * Called once after all tests complete
+   */
+  async onEnd(result) {
+    logger.info('Test run complete. Submitting results...');
+
+    try {
+      // Submit batched results
+      if (this.config.batchSubmit) {
+        await this.submitBatchedResults();
+      }
+
+      // Submit orphan tests
+      await this.submitOrphanTests();
+
+      // Print summary
+      this.printSummary();
+    } catch (error) {
+      logger.error('Error in onEnd', {
+        error: error.message
+      });
+    }
+  }
+
+  /**
+   * Submit batched results for all runs
+   * @private
+   */
+  async submitBatchedResults() {
+    for (const [runId, results] of this.resultsByRun.entries()) {
+      if (results.length === 0) continue;
+
+      try {
+        logger.info(`Submitting ${results.length} results for run ${runId}...`);
+
+        // Find run info for this runId to get metadata
+        let runInfo = null;
+        for (const [projectKey, info] of this.runsByProject.entries()) {
+          if (info.runId === runId) {
+            runInfo = info;
+            break;
+          }
+        }
+
+        // Build run metadata for legacy endpoint
+        const runMetadata = {
+          nid: runInfo?.metadata?.nid || 0,
+          run_timestamp: runInfo?.timestamp || Math.floor(Date.now() / 1000),
+          environment: runInfo?.metadata?.environment || this.config.environment || 'Local'
+        };
+
+        const summary = await this.client.submitBatch(results, {
+          batchSize: this.config.batchSize,
+          runMetadata: runMetadata,
+          onProgress: (progress) => {
+            logger.debug('Batch progress', progress);
+          }
+        });
+
+        logger.info(`Results submitted for run ${runId}`, {
+          success: summary.success,
+          failed: summary.failed,
+          total: summary.total
+        });
+      } catch (error) {
+        logger.error(`Failed to submit results for run ${runId}`, {
+          error: error.message
+        });
+      }
+    }
+  }
+
+  /**
+   * Submit orphan tests for all runs
+   * @private
+   */
+  async submitOrphanTests() {
+    if (!this.config.logOrphans) return;
+
+    for (const [runId, orphans] of this.orphansByRun.entries()) {
+      if (orphans.length === 0) continue;
+
+      try {
+        await this.client.logOrphanTests(runId, orphans);
+
+        logger.info(`Orphan tests logged for run ${runId}`, {
+          count: orphans.length
+        });
+      } catch (error) {
+        logger.error(`Failed to log orphan tests for run ${runId}`, {
+          error: error.message
+        });
+      }
+    }
+  }
+
+  /**
+   * Track result for batch submission
+   * @private
+   */
+  trackResult(runId, result) {
+    if (!this.resultsByRun.has(runId)) {
+      this.resultsByRun.set(runId, []);
+    }
+    this.resultsByRun.get(runId).push(result);
+  }
+
+  /**
+   * Track orphan test
+   * @private
+   */
+  trackOrphan(runId, test, result, browser) {
+    if (!this.orphansByRun.has(runId)) {
+      this.orphansByRun.set(runId, []);
+    }
+
+    const orphanEntry = UuidExtractor.createOrphanEntry(test, result, browser);
+    this.orphansByRun.get(runId).push(orphanEntry);
+  }
+
+  /**
+   * Map Playwright status to Appliqation status
+   * @private
+   */
+  mapStatus(status) {
+    const statusMap = {
+      'passed': 'passed',
+      'failed': 'failed',
+      'timedOut': 'failed',
+      'skipped': 'skipped',
+      'interrupted': 'skipped'
+    };
+
+    return statusMap[status] || 'failed';
+  }
+
+  /**
+   * Build comment from test result
+   * @private
+   */
+  buildComment(test, result) {
+    const parts = [];
+
+    if (result.duration) {
+      parts.push(`Duration: ${(result.duration / 1000).toFixed(2)}s`);
+    }
+
+    if (result.error) {
+      const errorMsg = result.error.message || result.error.toString();
+      parts.push(`Error: ${errorMsg.substring(0, 500)}`);
+    }
+
+    if (result.retry > 0) {
+      parts.push(`Retry: ${result.retry}`);
+    }
+
+    return parts.join(' | ');
+  }
+
+  /**
+   * Print summary report
+   * @private
+   */
+  printSummary() {
+    console.log('\n╔═══════════════════════════════════════════════════════════╗');
+    console.log('║         Appliqation Test Results Summary                 ║');
+    console.log('╠═══════════════════════════════════════════════════════════╣');
+    console.log(`║  Total Tests:       ${this.totalTests.toString().padStart(5)}                               ║`);
+    console.log(`║  Passed:            ${this.passedTests.toString().padStart(5)}                               ║`);
+    console.log(`║  Failed:            ${this.failedTests.toString().padStart(5)}                               ║`);
+    console.log(`║  Skipped:           ${this.skippedTests.toString().padStart(5)}                               ║`);
+    console.log(`║  Orphan (No UUID):  ${this.orphanTests.toString().padStart(5)}                               ║`);
+    console.log('╠═══════════════════════════════════════════════════════════╣');
+    console.log(`║  Run Matrices Created: ${this.runsByProject.size}                               ║`);
+
+    for (const [projectKey, runInfo] of this.runsByProject.entries()) {
+      console.log(`║    ${projectKey}: ${runInfo.runId}            ║`);
+    }
+
+    console.log('╚═══════════════════════════════════════════════════════════╝\n');
+
+    if (this.orphanTests > 0) {
+      console.log('⚠️  Warning: Some tests are missing UUID annotations!');
+      console.log('   Add UUIDs to map tests to Appliqation test cases:');
+      console.log('   test(\'My Test\', { tag: \'@uuid:123-xxx-...\' }, async ({ page }) => { ... });\n');
+    }
+  }
+
+  /**
+   * Extract project names that are actually running from the test suite
+   * @private
+   * @param {Object} suite - Root test suite
+   * @returns {Array<string>} Array of project names
+   */
+  getRunningProjectNames(suite) {
+    const projectNames = new Set();
+
+    const extractProjectNames = (node) => {
+      // Check if this node has a project
+      if (node.project && typeof node.project === 'function') {
+        const project = node.project();
+        if (project && project.name) {
+          projectNames.add(project.name);
+        }
+      }
+
+      // Recursively check suites
+      if (node.suites && Array.isArray(node.suites)) {
+        for (const childSuite of node.suites) {
+          extractProjectNames(childSuite);
+        }
+      }
+    };
+
+    extractProjectNames(suite);
+    return Array.from(projectNames);
+  }
+
+  /**
+   * Auto-detect and inject browser versions for projects missing browserVersion metadata
+   * @private
+   * @param {Array} projects - Array of Playwright project configurations
+   */
+  async injectBrowserVersions(projects) {
+    // Use playwright package directly (not @playwright/test) to avoid circular dependency
+    let playwright = null;
+    try {
+      playwright = require('playwright');
+    } catch (e) {
+      // If playwright is not installed, try playwright-core
+      try {
+        playwright = require('playwright-core');
+      } catch (e2) {
+        logger.debug('Could not load playwright package for browser version detection');
+        return;
+      }
+    }
+
+    const { chromium, firefox, webkit } = playwright;
+
+    for (const project of projects) {
+      // Skip if browserVersion is already configured
+      if (project.use?.metadata?.browserVersion) {
+        logger.debug(`Project "${project.name}" already has browserVersion configured`, {
+          version: project.use.metadata.browserVersion
+        });
+        continue;
+      }
+
+      const browserName = project.use?.browserName;
+      if (!browserName) continue;
+
+      try {
+        let browser = null;
+        let version = null;
+
+        // Launch browser to detect version
+        switch (browserName.toLowerCase()) {
+          case 'chromium':
+          case 'chrome':
+            browser = await chromium.launch();
+            version = browser.version();
+            await browser.close();
+            break;
+
+          case 'firefox':
+            browser = await firefox.launch();
+            version = browser.version();
+            await browser.close();
+            break;
+
+          case 'webkit':
+          case 'safari':
+            browser = await webkit.launch();
+            version = browser.version();
+            await browser.close();
+            break;
+        }
+
+        if (version) {
+          // Extract major version (e.g., "140.0.6778.44" -> "140")
+          const majorVersion = version.match(/^(\d+)/)?.[1];
+
+          if (majorVersion) {
+            // Inject browserVersion into project metadata
+            if (!project.use.metadata) {
+              project.use.metadata = {};
+            }
+            project.use.metadata.browserVersion = majorVersion;
+
+            logger.info(`Auto-detected browser version for project "${project.name}"`, {
+              browser: browserName,
+              version: majorVersion,
+              fullVersion: version
+            });
+          }
+        }
+      } catch (error) {
+        // Silently fail - version detection is optional
+        logger.debug(`Could not auto-detect browser version for project "${project.name}"`, {
+          error: error.message
+        });
+      }
+    }
+  }
+
+  /**
+   * Validate reporter configuration
+   * @private
+   */
+  validateConfig() {
+    if (!this.config.baseUrl) {
+      throw new Error('baseUrl is required');
+    }
+
+    if (!this.config.apiKey) {
+      throw new Error('apiKey is required');
+    }
+
+    if (!this.config.projectKey) {
+      throw new Error('projectKey is required');
+    }
+
+    // Note: scenarioId and testSetId are now optional
+    // If neither is provided, will default to 0 for generic automation runs
+  }
+
+  /**
+   * Called when the reporter starts
+   * @param {Object} runnerSuite - Root suite
+   * @param {Object} config - Playwright config
+   */
+  onStdOut(chunk, test, result) {
+    // Optional: capture stdout if needed
+  }
+
+  /**
+   * Called when the reporter starts
+   * @param {Object} runnerSuite - Root suite
+   * @param {Object} config - Playwright config
+   */
+  onStdErr(chunk, test, result) {
+    // Optional: capture stderr if needed
+  }
+}
+
+module.exports = AppliqationReporter;