@appliqation/automation-sdk 2.1.10 → 2.2.0

package/src/reporters/playwright/AppliqationReporter.js

@@ -1,3 +1,4 @@
+const path = require('path');
 const AppliqationClient = require('../../AppliqationClient');
 const UuidExtractor = require('./helpers/UuidExtractor');
 const DeviceOsDetector = require('./helpers/DeviceOsDetector');
@@ -5,6 +6,51 @@ const PayloadBuilder = require('../../utils/PayloadBuilder');
 const logger = require('../../utils/logger');
 const { DEFAULT_APPLIQATION_BASE_URL } = require('../../constants');
 
+/**
+ * Check if Appliqation reporting should be enabled
+ * Checks both environment variable and CLI flag
+ * @returns {boolean} True if reporting should be enabled
+ */
+function isAppqEnabled() {
+  // Check environment variable first (easier to use)
+  const envEnabled = process.env.APPQ_ENABLE === '1' ||
+                     process.env.APPQ_ENABLE === 'true' ||
+                     process.env.APPLIQATION_ENABLE === '1' ||
+                     process.env.APPLIQATION_ENABLE === 'true';
+
+  if (envEnabled) {
+    return true;
+  }
+
+  // Check CLI flag (requires -- separator: npx playwright test -- --appq)
+  const argv = process.argv || [];
+  return argv.includes('--appq');
+}
+
+/**
+ * Extract run title from CLI arguments
+ * Supports: --appq_run_title="My Title" or --appq_run_title=MyTitle
+ * @returns {string|null} Run title from CLI args, or null if not found
+ */
+function getRunTitleFromCli() {
+  const argv = process.argv || [];
+
+  // Look for --appq_run_title=value or --appq_run_title="value"
+  for (const arg of argv) {
+    if (arg.startsWith('--appq_run_title=')) {
+      // Extract value after the = sign
+      let value = arg.substring('--appq_run_title='.length);
+
+      // Remove surrounding quotes if present
+      value = value.replace(/^["']|["']$/g, '');
+
+      return value || null;
+    }
+  }
+
+  return null;
+}
+
 /**
  * Appliqation Reporter for Playwright
  * Automatically reports test results to Appliqation platform
@@ -66,15 +112,26 @@ class AppliqationReporter {
     // Note: No default fallback - validation will catch missing value
 
     // Apply title fallback if not provided
-    // Priority: 1) config.title, 2) process.env.APPLIQATION_RUN_TITLE, 3) timestamp-based default
+    // Priority: 1) config.title, 2) CLI --appq_run_title, 3) process.env.APPLIQATION_RUN_TITLE, 4) timestamp-based default
     if (!this.config.title) {
-      this.config.title = process.env.APPLIQATION_RUN_TITLE || `Automation Run - ${new Date().toISOString()}`;
+      const cliTitle = getRunTitleFromCli();
+      this.config.title = cliTitle || process.env.APPLIQATION_RUN_TITLE || `Automation Run - ${new Date().toISOString()}`;
     }
 
-    // Validate configuration
-    this.validateConfig();
+    // Check if --appq flag is present (opt-in approach)
+    this.appqEnabled = isAppqEnabled();
+
+    if (this.appqEnabled) {
+      // Validate configuration only if appq is enabled
+      this.validateConfig();
+      logger.info('Appliqation reporting ENABLED');
+      console.log('✅ Appliqation reporting enabled: Results will be sent to Appliqation portal');
+    } else {
+      logger.info('Appliqation reporting DISABLED');
+      console.log('ℹ️  Appliqation reporting disabled: Set APPQ_ENABLE=1 or add -- --appq flag to enable');
+    }
 
-    // Initialize SDK client
+    // Initialize SDK client (needed for UUID validation even if disabled)
     this.client = new AppliqationClient({
       baseUrl: this.config.baseUrl,
       apiKey: this.config.apiKey,
@@ -90,12 +147,22 @@ class AppliqationReporter {
     this.runsByProject = new Map(); // Map<projectKey, runInfo>
     this.resultsByRun = new Map(); // Map<runId, results[]>
     this.orphansByRun = new Map(); // Map<runId, orphans[]>
+    this.submittedUuidsByRun = new Map(); // Map<runId, Map<uuid, {file, title, browser}>>
+    this.submittedUuidsGlobal = new Map(); // Map<uuid, {file, title, browser, runId}>
     this.browserVersionDetected = new Map(); // Map<projectKey, boolean> - Track if version detected
     this.totalTests = 0;
     this.passedTests = 0;
     this.failedTests = 0;
     this.skippedTests = 0;
     this.orphanTests = 0;
+    this.duplicateTests = 0; // Track duplicate UUID submissions
+    this.duplicateDetails = []; // Store details of each duplicate for summary
+    this.backendRejectedTests = 0; // Track tests rejected by backend validation (e.g., project mismatch)
+
+    // Execution tracking for summary file generation
+    this.executionStartTime = null;
+    this.executionEndTime = null;
+    this.playwrightOutputDir = null;
 
     // Show baseUrl only in DEBUG mode
     logger.info('Appliqation Playwright Reporter initialized', {
@@ -113,8 +180,23 @@ class AppliqationReporter {
    * @param {Object} suite - Root test suite
    */
   async onBegin(config, suite) {
+    // Capture execution start time for summary file
+    this.executionStartTime = new Date();
+
+    // Capture Playwright output directory
+    // Priority: 1) config._internal.outputDir (actual test-results path)
+    //           2) config.rootDir (project root)
+    //           3) process.cwd() (current working directory)
+    this.playwrightOutputDir = config._internal?.outputDir || config.rootDir || process.cwd();
+
     logger.info('Test run starting...');
 
+    // Skip run creation if --appq flag not present
+    if (!this.appqEnabled) {
+      logger.info('Appliqation reporting disabled. Skipping run matrix creation.');
+      return;
+    }
+
     if (!this.config.autoCreateRun) {
       logger.info('Auto-create run disabled. Skipping run matrix creation.');
       return;
@@ -309,7 +391,7 @@ class AppliqationReporter {
    * @param {Object} result - Test result
    */
   async onTestEnd(test, result) {
-    this.totalTests++;
+    // Note: Don't increment totalTests here - it's incremented later for non-duplicates/non-orphans only
 
     try {
       // Extract UUID from test - check result.annotations first since test.info().annotations.push() adds to result
@@ -357,6 +439,73 @@ class AppliqationReporter {
         return;
       }
 
+      // Duplicate detection (global across all runs + per run)
+      const trackingKey = runInfo.runId;
+      const submittedUuids = this.submittedUuidsByRun.get(trackingKey) || new Map();
+
+      const existingGlobal = this.submittedUuidsGlobal.get(uuid);
+
+      // Check global map first to catch duplicates across files/runs/browsers
+      if (existingGlobal) {
+        const firstFile = path.basename(existingGlobal.file);
+        const currentFile = path.basename(test.location?.file || 'unknown');
+
+        this.duplicateTests++;
+        // Note: Don't increment skippedTests - duplicates have their own category
+        this.duplicateDetails.push({
+          uuid,
+          firstFile,
+          firstTitle: existingGlobal.title,
+          duplicateFile: currentFile,
+          duplicateTitle: test.title
+        });
+
+        logger.warn(`Duplicate UUID detected (global): ${uuid}`);
+        logger.warn(`  First: ${firstFile} -> "${existingGlobal.title}"`);
+        logger.warn(`  Dup:   ${currentFile} -> "${test.title}"`);
+        return; // Skip duplicate submission
+      }
+
+      // Check per-run map to allow future per-run reporting if needed
+      if (submittedUuids.has(uuid)) {
+        const firstOccurrence = submittedUuids.get(uuid);
+        const firstFile = path.basename(firstOccurrence.file);
+        const currentFile = path.basename(test.location?.file || 'unknown');
+
+        this.duplicateTests++;
+        // Note: Don't increment skippedTests - duplicates have their own category
+        this.duplicateDetails.push({
+          uuid,
+          firstFile,
+          firstTitle: firstOccurrence.title,
+          duplicateFile: currentFile,
+          duplicateTitle: test.title
+        });
+
+        logger.warn(`Duplicate UUID detected: ${uuid}`);
+        logger.warn(`  First: ${firstFile} -> "${firstOccurrence.title}"`);
+        logger.warn(`  Dup:   ${currentFile} -> "${test.title}"`);
+        return; // Skip duplicate submission
+      }
+
+      // Store UUID occurrence
+      submittedUuids.set(uuid, {
+        file: test.location?.file || 'unknown',
+        title: test.title,
+        browser: deviceInfo.browser
+      });
+      this.submittedUuidsByRun.set(trackingKey, submittedUuids);
+      // Also track globally to catch duplicates across files/runs/browsers
+      this.submittedUuidsGlobal.set(uuid, {
+        file: test.location?.file || 'unknown',
+        title: test.title,
+        browser: deviceInfo.browser,
+        runId: runInfo.runId
+      });
+
+      // Count only tests that will be submitted (non-duplicate, non-orphan)
+      this.totalTests++;
+
       // Create result object
       const testResult = {
         runId: runInfo.runId, // Add runId for batch submission
@@ -398,19 +547,29 @@ class AppliqationReporter {
    * Called once after all tests complete
    */
   async onEnd(result) {
-    logger.info('Test run complete. Submitting results...');
+    logger.info('Test run complete.');
 
     try {
-      // Submit batched results
-      if (this.config.batchSubmit) {
-        await this.submitBatchedResults();
-      }
+      // Skip submission if --appq flag not present
+      if (this.appqEnabled) {
+        logger.info('Submitting results to Appliqation...');
 
-      // Submit orphan tests
-      await this.submitOrphanTests();
+        // Submit batched results
+        if (this.config.batchSubmit) {
+          await this.submitBatchedResults();
+        }
+
+        // Submit orphan tests
+        await this.submitOrphanTests();
+      } else {
+        logger.info('Appliqation reporting disabled. Skipping result submission.');
+      }
 
-      // Print summary
+      // Print summary (always show, even if not submitted)
       this.printSummary();
+
+      // Write summary to file (always write, even if not submitted)
+      await this.writeSummaryToFile();
     } catch (error) {
       logger.error('Error in onEnd', {
         error: error.message
@@ -453,6 +612,9 @@ class AppliqationReporter {
           }
         });
 
+        // Track backend validation rejections
+        this.backendRejectedTests += summary.failed || 0;
+
         logger.info(`Results submitted for run ${runId}`, {
           success: summary.success,
           failed: summary.failed,
@@ -558,23 +720,60 @@ class AppliqationReporter {
    * @private
    */
   printSummary() {
+    const testsSubmitted = this.totalTests;
+    const testsAccepted = testsSubmitted - this.backendRejectedTests;
+
     console.log('\n╔═══════════════════════════════════════════════════════════╗');
     console.log('║         Appliqation Test Results Summary                 ║');
     console.log('╠═══════════════════════════════════════════════════════════╣');
-    console.log(`║  Total Tests:       ${this.totalTests.toString().padStart(5)}                               ║`);
-    console.log(`║  Passed:            ${this.passedTests.toString().padStart(5)}                               ║`);
-    console.log(`║  Failed:            ${this.failedTests.toString().padStart(5)}                               ║`);
-    console.log(`║  Skipped:           ${this.skippedTests.toString().padStart(5)}                               ║`);
-    console.log(`║  Orphan (No UUID):  ${this.orphanTests.toString().padStart(5)}                               ║`);
+
+    // Show reporting status
+    if (!this.appqEnabled) {
+      console.log('║  ⚠️  Appliqation Reporting: DISABLED                      ║');
+      console.log('║      (Set APPQ_ENABLE=1 or add -- --appq flag)            ║');
+      console.log('╠═══════════════════════════════════════════════════════════╣');
+    } else {
+      console.log('║  Submitted to Backend:                                    ║');
+      console.log(`║    Total Submitted: ${testsSubmitted.toString().padStart(5)}                               ║`);
+      console.log(`║    ✅ Accepted:     ${testsAccepted.toString().padStart(5)}                               ║`);
+      console.log(`║    ❌ Rejected:     ${this.backendRejectedTests.toString().padStart(5)}                               ║`);
+      console.log('║                                                           ║');
+    }
+    console.log('║                                                           ║');
+    console.log('║  Test Execution Results (Playwright):                    ║');
+    console.log(`║    Passed:          ${this.passedTests.toString().padStart(5)}                               ║`);
+    console.log(`║    Failed:          ${this.failedTests.toString().padStart(5)}                               ║`);
+    console.log(`║    Skipped:         ${this.skippedTests.toString().padStart(5)}                               ║`);
+    console.log('║                                                           ║');
+    console.log('║  Not Submitted:                                           ║');
+    console.log(`║    Orphan (No UUID):${this.orphanTests.toString().padStart(5)}                               ║`);
+    console.log(`║    Duplicates:      ${this.duplicateTests.toString().padStart(5)}                               ║`);
     console.log('╠═══════════════════════════════════════════════════════════╣');
-    console.log(`║  Run Matrices Created: ${this.runsByProject.size}                               ║`);
 
-    for (const [projectKey, runInfo] of this.runsByProject.entries()) {
-      console.log(`║    ${projectKey}: ${runInfo.runId}            ║`);
+    // Only show run matrices if reporting is enabled
+    if (this.appqEnabled && this.runsByProject.size > 0) {
+      console.log(`║  Run Matrices Created: ${this.runsByProject.size}                               ║`);
+
+      for (const [projectKey, runInfo] of this.runsByProject.entries()) {
+        console.log(`║    ${projectKey}: ${runInfo.runId}            ║`);
+      }
     }
 
     console.log('╚═══════════════════════════════════════════════════════════╝\n');
 
+    if (this.duplicateTests > 0) {
+      console.log('⚠️  Warning: Duplicate UUID(s) detected!');
+      console.log('');
+      this.duplicateDetails.forEach((dup, index) => {
+        console.log(`${index + 1}. UUID: ${dup.uuid}`);
+        console.log(`   First occurrence: ${dup.firstFile} - "${dup.firstTitle}"`);
+        console.log(`   Duplicate found:  ${dup.duplicateFile} - "${dup.duplicateTitle}"`);
+        console.log('');
+      });
+      console.log(`Total: ${this.duplicateTests} duplicate test(s) skipped.`);
+      console.log('Each test should have a unique UUID within a browser.\n');
+    }
+
     if (this.orphanTests > 0) {
       console.log('⚠️  Warning: Some tests are missing UUID annotations!');
       console.log('   Add UUIDs to map tests to Appliqation test cases:');
@@ -582,6 +781,209 @@ class AppliqationReporter {
     }
   }
 
+  /**
+   * Write execution summary to file
+   * Creates timestamped txt file in AppQ_Execution_Summary folder
+   * @private
+   */
+  async writeSummaryToFile() {
+    const fs = require('fs').promises;
+    const path = require('path');
+
+    try {
+      this.executionEndTime = new Date();
+      const duration = this.executionEndTime - this.executionStartTime;
+
+      // Build output directory path
+      // If playwrightOutputDir already contains 'test-results' (from config._internal.outputDir),
+      // don't add it again. Otherwise, add 'test-results' to the path.
+      const baseDir = this.playwrightOutputDir;
+      const needsTestResults = !baseDir.includes('test-results');
+
+      const summaryDir = needsTestResults
+        ? path.join(baseDir, 'test-results', 'AppQ_Execution_Summary')
+        : path.join(baseDir, 'AppQ_Execution_Summary');
+
+      // Create directory if it doesn't exist
+      await fs.mkdir(summaryDir, { recursive: true });
+
+      // Build filename with run title and timestamp
+      const runTitle = this.config.title || 'execution_summary';
+      const timestamp = this.formatDateTimeForFilename(this.executionStartTime);
+      const filename = `${runTitle}_${timestamp}.txt`;
+      const filepath = path.join(summaryDir, filename);
+
+      // Build comprehensive summary content
+      const summaryContent = this.buildSummaryContent(duration);
+
+      // Write to file
+      await fs.writeFile(filepath, summaryContent, 'utf8');
+
+      logger.info(`Execution summary saved to: ${filepath}`);
+      console.log(`\n📄 Execution summary saved: ${filename}`);
+
+    } catch (error) {
+      logger.error('Failed to write summary file', { error: error.message });
+      // Don't throw - file writing failure should not break test run
+    }
+  }
+
+  /**
+   * Format date/time for filename: 2025-01-21_14-30-45
+   * @private
+   */
+  formatDateTimeForFilename(date) {
+    const year = date.getFullYear();
+    const month = String(date.getMonth() + 1).padStart(2, '0');
+    const day = String(date.getDate()).padStart(2, '0');
+    const hours = String(date.getHours()).padStart(2, '0');
+    const minutes = String(date.getMinutes()).padStart(2, '0');
+    const seconds = String(date.getSeconds()).padStart(2, '0');
+
+    return `${year}-${month}-${day}_${hours}-${minutes}-${seconds}`;
+  }
+
+  /**
+   * Build comprehensive summary content for file
+   * @private
+   */
+  buildSummaryContent(duration) {
+    const lines = [];
+
+    // Header
+    lines.push('═══════════════════════════════════════════════════════════');
+    lines.push('          APPLIQATION TEST EXECUTION SUMMARY');
+    lines.push('═══════════════════════════════════════════════════════════');
+    lines.push('');
+
+    // Execution Metadata
+    lines.push('EXECUTION METADATA:');
+    lines.push('─────────────────────────────────────────────────────────');
+    lines.push(`Start Time:     ${this.executionStartTime.toISOString()}`);
+    lines.push(`End Time:       ${this.executionEndTime.toISOString()}`);
+    lines.push(`Duration:       ${this.formatDuration(duration)}`);
+    lines.push(`Run Title:      ${this.config.title || 'N/A'}`);
+    lines.push(`Appq Reporting: ${this.appqEnabled ? 'ENABLED' : 'DISABLED (Set APPQ_ENABLE=1 or add -- --appq flag)'}`);
+    lines.push('');
+
+    // Test Results Summary (ASCII Table from printSummary)
+    const testsSubmitted = this.totalTests;
+    const testsAccepted = testsSubmitted - this.backendRejectedTests;
+
+    lines.push('╔═══════════════════════════════════════════════════════════╗');
+    lines.push('║         Appliqation Test Results Summary                 ║');
+    lines.push('╠═══════════════════════════════════════════════════════════╣');
+
+    // Show submission status
+    if (!this.appqEnabled) {
+      lines.push('║  ⚠️  Appliqation Reporting: DISABLED                      ║');
+      lines.push('║      (Set APPQ_ENABLE=1 or add -- --appq flag)            ║');
+      lines.push('╠═══════════════════════════════════════════════════════════╣');
+    } else {
+      lines.push('║  Submitted to Backend:                                    ║');
+      lines.push(`║    Total Submitted: ${testsSubmitted.toString().padStart(5)}                               ║`);
+      lines.push(`║    ✅ Accepted:     ${testsAccepted.toString().padStart(5)}                               ║`);
+      lines.push(`║    ❌ Rejected:     ${this.backendRejectedTests.toString().padStart(5)}                               ║`);
+      lines.push('║                                                           ║');
+    }
+    lines.push('║  Test Execution Results (Playwright):                    ║');
+    lines.push(`║    Passed:          ${this.passedTests.toString().padStart(5)}                               ║`);
+    lines.push(`║    Failed:          ${this.failedTests.toString().padStart(5)}                               ║`);
+    lines.push(`║    Skipped:         ${this.skippedTests.toString().padStart(5)}                               ║`);
+    lines.push('║                                                           ║');
+    lines.push('║  Not Submitted:                                           ║');
+    lines.push(`║    Orphan (No UUID):${this.orphanTests.toString().padStart(5)}                               ║`);
+    lines.push(`║    Duplicates:      ${this.duplicateTests.toString().padStart(5)}                               ║`);
+    lines.push('╠═══════════════════════════════════════════════════════════╣');
+
+    // Run IDs (only show if reporting is enabled)
+    if (this.appqEnabled && this.runsByProject.size > 0) {
+      lines.push(`║  Run Matrices Created: ${this.runsByProject.size.toString().padStart(5)}                           ║`);
+      for (const [projectKey, runInfo] of this.runsByProject) {
+        const displayKey = projectKey.length > 30 ? projectKey.substring(0, 27) + '...' : projectKey;
+        const paddedKey = displayKey.padEnd(30);
+        lines.push(`║    ${paddedKey}: ${runInfo.runId.padEnd(20)} ║`);
+      }
+    }
+
+    lines.push('╚═══════════════════════════════════════════════════════════╝');
+    lines.push('');
+
+    // Detailed Errors Section
+    if (this.duplicateTests > 0 || this.orphanTests > 0 || this.backendRejectedTests > 0) {
+      lines.push('');
+      lines.push('DETAILED ERRORS & WARNINGS:');
+      lines.push('═══════════════════════════════════════════════════════════');
+      lines.push('');
+    }
+
+    // Duplicate UUIDs Details
+    if (this.duplicateTests > 0 && this.duplicateDetails.length > 0) {
+      lines.push('DUPLICATE UUIDs DETECTED:');
+      lines.push('─────────────────────────────────────────────────────────');
+      this.duplicateDetails.forEach((dup, idx) => {
+        lines.push(`${idx + 1}. UUID: ${dup.uuid}`);
+        lines.push(`   First occurrence: ${dup.firstFile}`);
+        lines.push(`                     "${dup.firstTitle}"`);
+        lines.push(`   Duplicate found:  ${dup.duplicateFile}`);
+        lines.push(`                     "${dup.duplicateTitle}"`);
+        lines.push('');
+      });
+    }
+
+    // Orphan Tests Warning
+    if (this.orphanTests > 0) {
+      lines.push('ORPHAN TESTS (Missing UUID Annotations):');
+      lines.push('─────────────────────────────────────────────────────────');
+      lines.push(`Total orphan tests: ${this.orphanTests}`);
+      lines.push('');
+      lines.push('These tests are missing UUID annotations. Add UUIDs to map tests');
+      lines.push('to Appliqation test cases:');
+      lines.push('  test(\'My Test\', { tag: \'@uuid:123-xxx-...\' }, async ({ page }) => {');
+      lines.push('    // test code');
+      lines.push('  });');
+      lines.push('');
+    }
+
+    // Backend Rejections
+    if (this.backendRejectedTests > 0) {
+      lines.push('BACKEND VALIDATION REJECTIONS:');
+      lines.push('─────────────────────────────────────────────────────────');
+      lines.push(`Total rejected: ${this.backendRejectedTests}`);
+      lines.push('');
+      lines.push('These test cases were rejected due to project ownership validation.');
+      lines.push('The test case UUID does not belong to the project associated with');
+      lines.push('your API key. Verify the correct project and test case mappings.');
+      lines.push('');
+    }
+
+    // Footer
+    lines.push('═══════════════════════════════════════════════════════════');
+    lines.push(`Generated by Appliqation SDK v${require('../../../package.json').version}`);
+    lines.push(`Timestamp: ${new Date().toISOString()}`);
+    lines.push('═══════════════════════════════════════════════════════════');
+
+    return lines.join('\n');
+  }
+
+  /**
+   * Format duration in human-readable format
+   * @private
+   */
+  formatDuration(ms) {
+    const seconds = Math.floor(ms / 1000);
+    const minutes = Math.floor(seconds / 60);
+    const hours = Math.floor(minutes / 60);
+
+    if (hours > 0) {
+      return `${hours}h ${minutes % 60}m ${seconds % 60}s`;
+    } else if (minutes > 0) {
+      return `${minutes}m ${seconds % 60}s`;
+    } else {
+      return `${seconds}s`;
+    }
+  }
+
   /**
    * Extract project names that are actually running from the test suite
    * @private