@appliqation/automation-sdk 2.7.0 → 2.8.1

package/src/reporters/playwright/AppliqationReporter.js

@@ -1,5 +1,6 @@
 const path = require('path');
 const AppliqationClient = require('../../AppliqationClient');
+const ScopeValidator = require('../../services/ScopeValidator');
 const UuidExtractor = require('./helpers/UuidExtractor');
 const DeviceOsDetector = require('./helpers/DeviceOsDetector');
 const PayloadBuilder = require('../../utils/PayloadBuilder');
@@ -176,6 +177,21 @@ class AppliqationReporter {
     this.executionStartTime = null;
     this.executionEndTime = null;
     this.playwrightOutputDir = null;
+
+    // Playwright does not reliably await a reporter's onTestEnd() return
+    // value before calling onEnd() — only onEnd()'s own promise is
+    // guaranteed to be awaited. onTestEnd() can itself wait (via
+    // waitForRunMatrix) for a still-in-flight run creation, so without
+    // this tracking, onEnd() can start reading resultsByRun/orphansByRun
+    // before onTestEnd has finished populating them, silently submitting
+    // nothing. Every onTestEnd() invocation pushes its own promise here;
+    // onEnd() awaits them all first.
+    this.pendingTestEndPromises = [];
+
+    // C5 — scope validation state
+    this.scopeFilterEnabled = false;  // true when strategy === 'filter'
+    this.inScopeUuids = null;          // Set<string>, null = no filter
+    this.droppedOutOfScopeCount = 0;
   }
 
   /**
@@ -211,6 +227,37 @@ class AppliqationReporter {
         }
       }
 
+      // Extract every Appliqation-mapped UUID from the suite once, then
+      // share with scope validation (C5) and run pre-population (C4/E3).
+      // Single walk over the test tree, single source of truth.
+      const allSuiteUuids = ScopeValidator.prototype.extractUuidsFromSuite.call(
+        { /* no scope needed for extraction */ }, suite
+      );
+
+      // C5 — pre-execution scope validation.
+      // If the user set scenarioId or testSetId, verify every test about to
+      // run actually belongs to that scope. Mismatch → apply the configured
+      // resolution strategy (cancel / adhoc / filter). This is the SDK's
+      // primary defence against silent data corruption: without it, a
+      // tag-based selection that spans scenarios writes to the wrong run.
+      const scopeOutcome = await this._runScopeValidation(suite, allSuiteUuids);
+      if (scopeOutcome.cancelled) {
+        // User chose cancel (or default in CI on mismatch). Skip run
+        // creation and result submission entirely; tests still execute
+        // locally so the user keeps their own Playwright output.
+        this.appqEnabled = false;
+        return;
+      }
+
+      // C4 / E3 — UUIDs to pre-populate on the run document. If the filter
+      // strategy applied, only the in-scope subset is sent; otherwise the
+      // full extracted set. Backend stores these in data[] so the UI grid
+      // and orphan detector know which TCs belong to the run before any
+      // result lands.
+      this.preExecutionUuids = this.scopeFilterEnabled && this.inScopeUuids
+        ? Array.from(this.inScopeUuids)
+        : allSuiteUuids;
+
       if (!this.config.autoCreateRun) {
         logger.info('Auto-create run disabled. Skipping run matrix creation.');
         return;
@@ -276,7 +323,11 @@ class AppliqationReporter {
             browsers: matrixConfig.browsers,
             device: matrixConfig.device,
             os: matrixConfig.os,
-            title: this.config.title
+            title: this.config.title,
+            // C4 / E3 — pre-populate the run's data[] array with the
+            // UUIDs about to execute. Same set across all matrix configs
+            // (Playwright reruns the same tests per browser project).
+            uuids: this.preExecutionUuids
           };
 
           const run = await this.client.createRun(runOptions);
@@ -364,14 +415,36 @@ class AppliqationReporter {
   }
 
   /**
-   * Called after a test completes — non-blocking
+   * Called after a test completes.
+   *
+   * Playwright does not reliably await this method's returned promise
+   * before calling onEnd() — so the actual work (which can wait on a
+   * still-in-flight run creation via waitForRunMatrix) runs in
+   * _processTestEnd(), and this wrapper only tracks that promise in
+   * pendingTestEndPromises so onEnd() can await it explicitly.
    */
-  async onTestEnd(test, result) {
+  onTestEnd(test, result) {
     if (!this.appqEnabled) return;
+    const promise = this._processTestEnd(test, result);
+    this.pendingTestEndPromises.push(promise);
+    return promise;
+  }
 
+  /** @private */
+  async _processTestEnd(test, result) {
     try {
       const uuid = UuidExtractor.extractFromAnnotations(result.annotations || []) || UuidExtractor.extractFromTest(test);
 
+      // C5 filter strategy: scope validation in onBegin flagged this test
+      // as out-of-scope for the configured scenarioId/testSetId. The test
+      // still executed (the customer has their local report), but its
+      // verdict must not land in the wrong run.
+      if (this.scopeFilterEnabled && uuid && this.inScopeUuids && !this.inScopeUuids.has(uuid)) {
+        this.droppedOutOfScopeCount++;
+        logger.debug('Dropping out-of-scope result (C5 filter strategy)', { uuid });
+        return;
+      }
+
       const project = test.parent?.project?.() || test.parent?.project;
       const deviceInfo = DeviceOsDetector.getDeviceInfo(project, null);
       const projectKey = `${deviceInfo.device}-${deviceInfo.os}`;
@@ -405,7 +478,15 @@ class AppliqationReporter {
         return;
       }
 
-      // Duplicate detection
+      // Duplicate detection — user-facing warning only, NOT a data-integrity
+      // guard. The backend's upsert key is (run_id, uuid, parent_uuid,
+      // browser), so re-submitting the same UUID for the same browser is
+      // idempotent at the DB level (see appq PR #532's automation.attempt
+      // $inc semantics). This tracking exists purely so the end-of-run
+      // summary can point out authoring mistakes (two tests annotated with
+      // the same UUID), and it's per-reporter-instance — cross-shard
+      // duplicates from `--shard` are not caught here, and don't need to
+      // be, because the backend's upsert still handles them correctly.
       const trackingKey = `${runInfo.runId}:${deviceInfo.browser}`;
       const submittedUuids = this.submittedUuidsByRun.get(trackingKey) || new Map();
 
@@ -462,9 +543,9 @@ class AppliqationReporter {
 
       this.trackResult(runInfo.runId, testResult);
 
-      if (testResult.status === 'Pass') this.passedTests++;
-      else if (testResult.status === 'Fail') this.failedTests++;
-      else if (testResult.status === 'Skipped') this.skippedTests++;
+      if (testResult.status === 'passed') this.passedTests++;
+      else if (testResult.status === 'failed') this.failedTests++;
+      else if (testResult.status === 'skipped') this.skippedTests++;
 
       if (!this.config.batchSubmit) {
         await this.client.submitResult(runInfo.runId, testResult);
@@ -565,6 +646,14 @@ class AppliqationReporter {
     logger.info('Test run complete.');
 
     try {
+      // See onTestEnd()'s doc comment — Playwright doesn't reliably await
+      // per-test reporter hooks, so results/orphans tracked by still-
+      // pending onTestEnd() calls (e.g. waiting on a slow run creation)
+      // would otherwise be silently missing from what we're about to read.
+      if (this.pendingTestEndPromises.length > 0) {
+        await Promise.allSettled(this.pendingTestEndPromises);
+      }
+
       const orphanDeletedRuns = await this.handleOrphanOnlyRuns();
 
       if (this.appqEnabled) {
@@ -660,20 +749,33 @@ class AppliqationReporter {
 
   /** @private */
   mapStatus(status) {
+    // Playwright's `interrupted` means the entire test run was aborted
+    // (Ctrl+C, CI timeout, OOM). Mapping it to 'skipped' would mask
+    // failures and inflate the skipped counter — surface as failed instead
+    // so the run summary honestly reflects an aborted execution.
+    //
+    // Values MUST match PayloadBuilder.validateAutomationResultPayload's
+    // ALLOWED_STATUSES ({passed, failed, skipped}, lowercase past-tense —
+    // mirrors the backend's InputValidator). This reporter's own mapped
+    // value is what ultimately reaches that validator (via trackResult →
+    // submitBatch → PayloadBuilder.buildResultPayload → normalizeStatus,
+    // which passes an already-recognized lowercase value straight through),
+    // so drifting from that vocabulary here silently fails every submission.
     const statusMap = {
-      'passed': 'Pass',
-      'failed': 'Fail',
-      'timedOut': 'Fail',
-      'skipped': 'Skipped',
-      'interrupted': 'Skipped'
+      'passed': 'passed',
+      'failed': 'failed',
+      'timedOut': 'failed',
+      'skipped': 'skipped',
+      'interrupted': 'failed'
     };
-    return statusMap[status] || 'Fail';
+    return statusMap[status] || 'failed';
   }
 
   /** @private */
   buildComment(test, result) {
     const parts = [];
     if (result.duration) parts.push(`Duration: ${(result.duration / 1000).toFixed(2)}s`);
+    if (result.status === 'interrupted') parts.push('Test run interrupted');
     if (result.error) {
       const errorMsg = result.error.message || result.error.toString();
       parts.push(`Error: ${errorMsg.substring(0, 500)}`);
@@ -868,6 +970,82 @@ class AppliqationReporter {
     return Array.from(projectNames);
   }
 
+  /**
+   * Run C5 scope validation against the Playwright suite. Mutates
+   * `this.config` if the user chooses the 'adhoc' strategy (clears
+   * scenarioId/testSetId so subsequent run creation is ad-hoc).
+   * Sets `this.scopeFilterEnabled` and `this.inScopeUuids` for the
+   * 'filter' strategy so `onTestEnd` can drop out-of-scope results.
+   *
+   * @private
+   * @param {Object} suite - Playwright Suite from onBegin
+   * @returns {Promise<{cancelled: boolean}>}
+   */
+  async _runScopeValidation(suite, preExtractedUuids) {
+    const validator = new ScopeValidator({
+      scenarioId: this.config.scenarioId,
+      testSetId: this.config.testSetId,
+      strategy: this.config.onScopeMismatch,
+      fetchTestSetScope: this._fetchTestSetScope.bind(this)
+    });
+
+    if (!validator.isScoped()) {
+      return { cancelled: false };
+    }
+
+    const uuids = preExtractedUuids || validator.extractUuidsFromSuite(suite);
+    const validation = await validator.validate(uuids);
+    if (!validation.hasMismatch) {
+      return { cancelled: false };
+    }
+
+    const outcome = await validator.resolve(validation);
+    // Surface the resolution message at WARN so it's visible at default
+    // log level — this is operational signal, not noise.
+    logger.warn(outcome.message);
+
+    if (outcome.action === ScopeValidator.STRATEGY.CANCEL) {
+      // Set non-zero exit code so CI fails. Don't throw — we still want
+      // Playwright to finish executing tests locally for the user's logs.
+      process.exitCode = 1;
+      return { cancelled: true };
+    }
+
+    if (outcome.overrideToAdhoc) {
+      // Drop the configured scope so createRun produces an ad-hoc run.
+      delete this.config.scenarioId;
+      delete this.config.testSetId;
+      return { cancelled: false };
+    }
+
+    if (outcome.action === ScopeValidator.STRATEGY.FILTER) {
+      this.scopeFilterEnabled = true;
+      this.inScopeUuids = new Set(outcome.submitUuids);
+    }
+
+    return { cancelled: false };
+  }
+
+  /**
+   * Resolve a testSet's UUID membership. Requires a backend endpoint
+   * (GET /api/automation/testset/{id}/scope) that returns the UUID list.
+   * Until that endpoint ships, this throws — ScopeValidator catches and
+   * gracefully degrades to skip testset validation with a warning.
+   *
+   * @private
+   * @param {number} testSetId
+   * @returns {Promise<Set<string>>}
+   */
+  async _fetchTestSetScope(testSetId) {
+    if (!this.client || typeof this.client.fetchTestSetScope !== 'function') {
+      throw new Error(
+        'TestSet scope endpoint not yet available in AppliqationClient — '
+        + 'testset scope validation will be skipped'
+      );
+    }
+    return this.client.fetchTestSetScope(testSetId);
+  }
+
   /**
    * Auto-detect browser versions
    * @private

package/src/services/OrphanTestService.js

@@ -44,10 +44,29 @@ class OrphanTestService {
 
       return response;
     } catch (error) {
-      logger.error('Failed to log orphan tests', {
+      // S5 — as of appq_automation_api.routing.yml, /api/automation/
+      // orphan-tests is not a registered route. Backend implementation
+      // is pending. Until it lands, a 404 here is expected — demote to
+      // debug so it doesn't spam customer CI logs. Orphans are still
+      // tracked locally and shown in the reporter's end-of-run summary.
+      const status = error.status
+        || error.response?.status
+        || error.details?.status
+        || null;
+
+      if (status === 404) {
+        logger.debug('Orphan tests endpoint not available on backend; skipping network log', {
+          runId,
+          count: orphanTests.length
+        });
+        return { success: false, notImplemented: true, count: orphanTests.length };
+      }
+
+      logger.warn('Failed to log orphan tests', {
         error: error.message,
         runId,
-        count: orphanTests.length
+        count: orphanTests.length,
+        status
       });
       throw new Error(`Failed to log orphan tests: ${error.message}`);
     }

package/src/services/ResultService.js

@@ -1,5 +1,6 @@
 const logger = require('../utils/logger');
 const { normalizeBrowser } = require('../utils/RunDataNormalizer');
+const PayloadBuilder = require('../utils/PayloadBuilder');
 
 class ResultService {
   constructor(httpClient, taggingService = null, config = { options: {} }) {
@@ -27,6 +28,16 @@ class ResultService {
 
       const payload = this.buildAutomationResultPayload(runId, result, runMetadata);
 
+      // S8 — final shape check before the wire. Catches schema drift
+      // between SDK assembly and backend expectations at the client
+      // instead of failing at the server with a generic 400.
+      const validation = PayloadBuilder.validateAutomationResultPayload(payload);
+      if (!validation.valid) {
+        throw new Error(
+          `Result payload failed pre-submission validation: ${validation.errors.join('; ')}`
+        );
+      }
+
       logger.debug('Submitting single result', {
         runId,
         uuid: result.uuid,
@@ -36,16 +47,12 @@ class ResultService {
       // Use automation endpoint to enable project validation
       const response = await this.http.post('/api/automation/result/submit', payload);
 
-      // DEBUG: Show response with context
-      console.log('\n🔍 ========== SUBMIT RESULT RESPONSE ==========');
-      console.log('UUID:', result.uuid);
-      if (response.data && response.data.debug_context) {
-        console.log('Project ID (from API):', response.data.debug_context.project_id);
-        console.log('User ID (from API):', response.data.debug_context.uid);
-      }
-      console.log('Status:', response.status);
-      console.log('Success:', response.success);
-      console.log('===========================================\n');
+      logger.debug('Result submitted', {
+        uuid: result.uuid,
+        status: response.status,
+        success: response.success,
+        projectId: response.data?.debug_context?.project_id
+      });
 
       // Check if request was successful
       if (!response.success) {
@@ -132,30 +139,51 @@ class ResultService {
           results: batch.map(r => this.buildAutomationResultPayload(runId, r, runMetadata))
         };
 
-        logger.debug('Batch payload', { payload: JSON.stringify(payload, null, 2) });
-
-        // Show UUIDs being submitted (test execution status shown, NOT backend validation status)
-        console.log('\n📤 Submitting batch with UUIDs:');
+        // S8 — validate every result in the batch before the wire.
+        // A malformed entry is filtered out and moved to failures rather
+        // than tainting the whole batch (backend would 400 the whole
+        // batch otherwise). Track per-result so downstream reporting
+        // still knows which UUIDs never made it.
+        const validResults = [];
         payload.results.forEach((r, idx) => {
-          console.log(`  ${idx + 1}. ${r.test_case_uuid}`);
+          const v = PayloadBuilder.validateAutomationResultPayload(r);
+          if (v.valid) {
+            validResults.push(r);
+          } else {
+            logger.warn('Skipping malformed result at client-side validation', {
+              uuid: r.test_case_uuid,
+              errors: v.errors
+            });
+            failures.push(batch[idx]);
+          }
+        });
+        if (validResults.length === 0) {
+          logger.warn('Every result in batch failed pre-submission validation; skipping HTTP call', {
+            runId, batchIndex: i + 1
+          });
+          continue;
+        }
+        payload.results = validResults;
+
+        logger.debug('Submitting batch', {
+          runId,
+          batchIndex: i + 1,
+          totalBatches: batches.length,
+          uuids: payload.results.map(r => r.test_case_uuid)
         });
 
         try {
           const response = await this.http.post('/api/automation/result/batch', payload);
 
-          // DEBUG: Show batch response with context
-          console.log('\n🔍 ========== BATCH SUBMIT RESPONSE ==========');
-          console.log('Run ID:', runId);
-          console.log('Batch:', `${i + 1}/${batches.length}`);
-          console.log('Results in batch:', batch.length);
-          if (response.data && response.data.debug_context) {
-            console.log('Project ID (from API):', response.data.debug_context.project_id);
-            console.log('User ID (from API):', response.data.debug_context.uid);
-          } else {
-            console.log('⚠️  No debug_context in response');
-          }
-          console.log('Status:', response.status);
-          console.log('Success:', response.success);
+          logger.debug('Batch submitted', {
+            runId,
+            batchIndex: i + 1,
+            totalBatches: batches.length,
+            resultsInBatch: batch.length,
+            status: response.status,
+            success: response.success,
+            projectId: response.data?.debug_context?.project_id
+          });
 
           // Check for validation failures - response has nested data object
           const failedResults = response.data?.data?.results?.failed || response.data?.results?.failed || response.data?.failed || [];
@@ -163,24 +191,20 @@ class ResultService {
           const submittedCount = typeof submittedResults === 'number' ? submittedResults : (Array.isArray(submittedResults) ? submittedResults.length : 0);
 
           if (failedResults.length > 0) {
-            console.log('\n❌ ========== VALIDATION FAILURES ==========');
-            console.log(`   ✅ ACCEPTED: ${submittedCount} result(s)`);
-            console.log(`   ❌ REJECTED: ${failedResults.length} result(s)\n`);
-            failedResults.forEach((fail, idx) => {
-              const uuid = fail.test_case_uuid || fail.uuid;
-              const error = fail.error || fail.message;
-              const code = fail.code;
-
-              console.log(`   ${idx + 1}. UUID: ${uuid}`);
-              console.log(`      ❌ Reason: ${error}`);
-              if (code) {
-                console.log(`      🚫 HTTP Code: ${code}`);
-              }
-              console.log('');
+            // Validation failures are operational signal — surface at WARN, not DEBUG.
+            // Logger sanitization redacts any sensitive fields in the payload.
+            logger.warn(`Backend rejected ${failedResults.length} of ${batch.length} results`, {
+              runId,
+              batchIndex: i + 1,
+              accepted: submittedCount,
+              rejected: failedResults.length,
+              rejections: failedResults.map(f => ({
+                uuid: f.test_case_uuid || f.uuid,
+                reason: f.error || f.message,
+                code: f.code
+              }))
             });
-            console.log('===========================================');
           }
-          console.log('============================================\n');
 
           if (!response.success) {
             throw new Error(response.error || 'Batch submission failed');
@@ -348,12 +372,19 @@ class ResultService {
       failed: 'failed',
       skipped: 'skipped',
       timedOut: 'failed',
-      interrupted: 'skipped'
+      // Aborted run, not a skip — see AppliqationReporter.mapStatus.
+      interrupted: 'failed'
     };
 
     return {
       run_id: runId || result.runId,
       test_case_uuid: result.uuid,
+      // Routes the backend to submitAutomationResult() — writes to the
+      // automation.* subdocument only, never overwrites a human verdict on
+      // top-level status/comment/verdict_source. Post PR #532 the backend
+      // routes API-key traffic to this path unconditionally; sending the
+      // field explicitly makes intent unambiguous and is forward-safe.
+      source: 'automation',
       status: statusMap[result.status] || result.status,
       duration: result.duration || 0,
       error_message: result.error || result.comment || '',

package/src/services/RunMatrixService.js

@@ -53,9 +53,17 @@ class RunMatrixService {
       const browsers = options.browsers || ['Chrome'];
       const normalizedBrowsers = browsers.map(browser => normalizeBrowser(browser));
 
+      // S2 — backend (AutomationApiController::createRun) uppercases env
+      // names on storage and matches case-insensitively on validation.
+      // Uppercase locally so the run-creation log and the stored doc
+      // agree, and so any downstream filter that matches on the case the
+      // SDK sent doesn't quietly mismatch.
+      const envRaw = options.environment || 'Local';
+      const environment = typeof envRaw === 'string' ? envRaw.toUpperCase() : envRaw;
+
       const payload = {
         project_key: projectKeyValue,
-        environment: options.environment || 'Local',
+        environment,
         browsers: normalizedBrowsers,
         device: normalizeDevice(options.device || this.detectDevice()),
         os: normalizeOS(options.os || this.detectOS()),
@@ -75,6 +83,18 @@ class RunMatrixService {
         payload.type = options.type || 'automation';
       }
 
+      // C4 / E3 — pre-populate data[] with the UUIDs about to be executed.
+      // Backend stores this on the run document so the UI grid knows which
+      // TCs belong to the run before any result lands. Without it, ad-hoc
+      // runs show 0 expected tests and the orphan detector flags every
+      // valid test as orphan.
+      //
+      // PR #532 (appq) changed data[] from objects ({uuid: '...'}) to
+      // plain strings — match that format.
+      if (Array.isArray(options.uuids) && options.uuids.length > 0) {
+        payload.data = options.uuids.map((u) => String(u));
+      }
+
       logger.info('Creating run matrix...', {
         type: payload.type,
         id: payload.scenario_id || payload.testset_id,