@appliqation/automation-sdk 2.5.1 → 2.8.0

package/src/services/ResultService.js

@@ -1,5 +1,6 @@
 const logger = require('../utils/logger');
 const { normalizeBrowser } = require('../utils/RunDataNormalizer');
+const PayloadBuilder = require('../utils/PayloadBuilder');
 
 class ResultService {
   constructor(httpClient, taggingService = null, config = { options: {} }) {
@@ -27,6 +28,16 @@ class ResultService {
 
       const payload = this.buildAutomationResultPayload(runId, result, runMetadata);
 
+      // S8 — final shape check before the wire. Catches schema drift
+      // between SDK assembly and backend expectations at the client
+      // instead of failing at the server with a generic 400.
+      const validation = PayloadBuilder.validateAutomationResultPayload(payload);
+      if (!validation.valid) {
+        throw new Error(
+          `Result payload failed pre-submission validation: ${validation.errors.join('; ')}`
+        );
+      }
+
       logger.debug('Submitting single result', {
         runId,
         uuid: result.uuid,
@@ -36,16 +47,12 @@ class ResultService {
       // Use automation endpoint to enable project validation
       const response = await this.http.post('/api/automation/result/submit', payload);
 
-      // DEBUG: Show response with context
-      console.log('\n🔍 ========== SUBMIT RESULT RESPONSE ==========');
-      console.log('UUID:', result.uuid);
-      if (response.data && response.data.debug_context) {
-        console.log('Project ID (from API):', response.data.debug_context.project_id);
-        console.log('User ID (from API):', response.data.debug_context.uid);
-      }
-      console.log('Status:', response.status);
-      console.log('Success:', response.success);
-      console.log('===========================================\n');
+      logger.debug('Result submitted', {
+        uuid: result.uuid,
+        status: response.status,
+        success: response.success,
+        projectId: response.data?.debug_context?.project_id
+      });
 
       // Check if request was successful
       if (!response.success) {
@@ -132,30 +139,51 @@ class ResultService {
           results: batch.map(r => this.buildAutomationResultPayload(runId, r, runMetadata))
         };
 
-        logger.debug('Batch payload', { payload: JSON.stringify(payload, null, 2) });
-
-        // Show UUIDs being submitted (test execution status shown, NOT backend validation status)
-        console.log('\n📤 Submitting batch with UUIDs:');
+        // S8 — validate every result in the batch before the wire.
+        // A malformed entry is filtered out and moved to failures rather
+        // than tainting the whole batch (backend would 400 the whole
+        // batch otherwise). Track per-result so downstream reporting
+        // still knows which UUIDs never made it.
+        const validResults = [];
         payload.results.forEach((r, idx) => {
-          console.log(`  ${idx + 1}. ${r.test_case_uuid}`);
+          const v = PayloadBuilder.validateAutomationResultPayload(r);
+          if (v.valid) {
+            validResults.push(r);
+          } else {
+            logger.warn('Skipping malformed result at client-side validation', {
+              uuid: r.test_case_uuid,
+              errors: v.errors
+            });
+            failures.push(batch[idx]);
+          }
+        });
+        if (validResults.length === 0) {
+          logger.warn('Every result in batch failed pre-submission validation; skipping HTTP call', {
+            runId, batchIndex: i + 1
+          });
+          continue;
+        }
+        payload.results = validResults;
+
+        logger.debug('Submitting batch', {
+          runId,
+          batchIndex: i + 1,
+          totalBatches: batches.length,
+          uuids: payload.results.map(r => r.test_case_uuid)
         });
 
         try {
           const response = await this.http.post('/api/automation/result/batch', payload);
 
-          // DEBUG: Show batch response with context
-          console.log('\n🔍 ========== BATCH SUBMIT RESPONSE ==========');
-          console.log('Run ID:', runId);
-          console.log('Batch:', `${i + 1}/${batches.length}`);
-          console.log('Results in batch:', batch.length);
-          if (response.data && response.data.debug_context) {
-            console.log('Project ID (from API):', response.data.debug_context.project_id);
-            console.log('User ID (from API):', response.data.debug_context.uid);
-          } else {
-            console.log('⚠️  No debug_context in response');
-          }
-          console.log('Status:', response.status);
-          console.log('Success:', response.success);
+          logger.debug('Batch submitted', {
+            runId,
+            batchIndex: i + 1,
+            totalBatches: batches.length,
+            resultsInBatch: batch.length,
+            status: response.status,
+            success: response.success,
+            projectId: response.data?.debug_context?.project_id
+          });
 
           // Check for validation failures - response has nested data object
           const failedResults = response.data?.data?.results?.failed || response.data?.results?.failed || response.data?.failed || [];
@@ -163,24 +191,20 @@ class ResultService {
           const submittedCount = typeof submittedResults === 'number' ? submittedResults : (Array.isArray(submittedResults) ? submittedResults.length : 0);
 
           if (failedResults.length > 0) {
-            console.log('\n❌ ========== VALIDATION FAILURES ==========');
-            console.log(`   ✅ ACCEPTED: ${submittedCount} result(s)`);
-            console.log(`   ❌ REJECTED: ${failedResults.length} result(s)\n`);
-            failedResults.forEach((fail, idx) => {
-              const uuid = fail.test_case_uuid || fail.uuid;
-              const error = fail.error || fail.message;
-              const code = fail.code;
-
-              console.log(`   ${idx + 1}. UUID: ${uuid}`);
-              console.log(`      ❌ Reason: ${error}`);
-              if (code) {
-                console.log(`      🚫 HTTP Code: ${code}`);
-              }
-              console.log('');
+            // Validation failures are operational signal — surface at WARN, not DEBUG.
+            // Logger sanitization redacts any sensitive fields in the payload.
+            logger.warn(`Backend rejected ${failedResults.length} of ${batch.length} results`, {
+              runId,
+              batchIndex: i + 1,
+              accepted: submittedCount,
+              rejected: failedResults.length,
+              rejections: failedResults.map(f => ({
+                uuid: f.test_case_uuid || f.uuid,
+                reason: f.error || f.message,
+                code: f.code
+              }))
             });
-            console.log('===========================================');
           }
-          console.log('============================================\n');
 
           if (!response.success) {
             throw new Error(response.error || 'Batch submission failed');
@@ -348,12 +372,19 @@ class ResultService {
       failed: 'failed',
       skipped: 'skipped',
       timedOut: 'failed',
-      interrupted: 'skipped'
+      // Aborted run, not a skip — see AppliqationReporter.mapStatus.
+      interrupted: 'failed'
     };
 
     return {
       run_id: runId || result.runId,
       test_case_uuid: result.uuid,
+      // Routes the backend to submitAutomationResult() — writes to the
+      // automation.* subdocument only, never overwrites a human verdict on
+      // top-level status/comment/verdict_source. Post PR #532 the backend
+      // routes API-key traffic to this path unconditionally; sending the
+      // field explicitly makes intent unambiguous and is forward-safe.
+      source: 'automation',
       status: statusMap[result.status] || result.status,
       duration: result.duration || 0,
       error_message: result.error || result.comment || '',

package/src/services/RunMatrixService.js

@@ -53,9 +53,17 @@ class RunMatrixService {
       const browsers = options.browsers || ['Chrome'];
       const normalizedBrowsers = browsers.map(browser => normalizeBrowser(browser));
 
+      // S2 — backend (AutomationApiController::createRun) uppercases env
+      // names on storage and matches case-insensitively on validation.
+      // Uppercase locally so the run-creation log and the stored doc
+      // agree, and so any downstream filter that matches on the case the
+      // SDK sent doesn't quietly mismatch.
+      const envRaw = options.environment || 'Local';
+      const environment = typeof envRaw === 'string' ? envRaw.toUpperCase() : envRaw;
+
       const payload = {
         project_key: projectKeyValue,
-        environment: options.environment || 'Local',
+        environment,
         browsers: normalizedBrowsers,
         device: normalizeDevice(options.device || this.detectDevice()),
         os: normalizeOS(options.os || this.detectOS()),
@@ -75,6 +83,18 @@ class RunMatrixService {
         payload.type = options.type || 'automation';
       }
 
+      // C4 / E3 — pre-populate data[] with the UUIDs about to be executed.
+      // Backend stores this on the run document so the UI grid knows which
+      // TCs belong to the run before any result lands. Without it, ad-hoc
+      // runs show 0 expected tests and the orphan detector flags every
+      // valid test as orphan.
+      //
+      // PR #532 (appq) changed data[] from objects ({uuid: '...'}) to
+      // plain strings — match that format.
+      if (Array.isArray(options.uuids) && options.uuids.length > 0) {
+        payload.data = options.uuids.map((u) => String(u));
+      }
+
       logger.info('Creating run matrix...', {
         type: payload.type,
         id: payload.scenario_id || payload.testset_id,

package/src/services/ScopeValidator.js

@@ -0,0 +1,314 @@
+const readline = require('readline');
+const UuidValidator = require('../utils/UuidValidator');
+const UuidExtractor = require('../reporters/playwright/helpers/UuidExtractor');
+const logger = require('../utils/logger');
+
+const STRATEGY = {
+  CANCEL: 'cancel',
+  ADHOC: 'adhoc',
+  FILTER: 'filter'
+};
+
+const VALID_STRATEGIES = Object.values(STRATEGY);
+
+/**
+ * Validates that the UUIDs about to be executed match the user's configured
+ * scenarioId or testSetId scope, and applies a resolution strategy when they
+ * don't.
+ *
+ * Closes finding C5 in the SDK audit — without this, the SDK silently
+ * submits cross-scenario results to a single-scenario run, corrupting the
+ * Appliqation data store.
+ *
+ * Three resolution strategies, selected via `config.onScopeMismatch`:
+ *   'cancel' (default in CI)  — fail fast, exit 1, submit nothing
+ *   'adhoc'                   — override config, create an ad-hoc run
+ *   'filter'                  — drop out-of-scope results, submit the rest
+ *
+ * In a TTY (no `CI=true` and stdin is a TTY), the user is prompted
+ * interactively before any test runs. In CI, the configured strategy is
+ * used; if unset, 'cancel' is the fail-safe default.
+ */
+class ScopeValidator {
+  /**
+   * @param {Object} options
+   * @param {number} [options.scenarioId]
+   * @param {number} [options.testSetId]
+   * @param {'cancel'|'adhoc'|'filter'} [options.strategy]
+   * @param {Function} [options.fetchTestSetScope] async (testSetId) -> Set<uuid>
+   */
+  constructor(options = {}) {
+    this.scenarioId = options.scenarioId || null;
+    this.testSetId = options.testSetId || null;
+    this.strategy = this._resolveStrategy(options.strategy);
+    this.fetchTestSetScope = options.fetchTestSetScope || null;
+  }
+
+  _resolveStrategy(explicit) {
+    const envValue = process.env.APPLIQATION_ON_SCOPE_MISMATCH;
+    const raw = (explicit || envValue || STRATEGY.CANCEL).toLowerCase();
+    if (!VALID_STRATEGIES.includes(raw)) {
+      logger.warn(`Unknown onScopeMismatch strategy "${raw}", falling back to "cancel"`);
+      return STRATEGY.CANCEL;
+    }
+    return raw;
+  }
+
+  /**
+   * @returns {boolean} True if scenarioId or testSetId is set — i.e., the
+   *   user has declared a scope and we need to validate against it.
+   */
+  isScoped() {
+    return Boolean(this.scenarioId || this.testSetId);
+  }
+
+  /**
+   * Extract every Appliqation UUID from a Playwright suite tree.
+   *
+   * @param {Object} suite - Playwright Suite from onBegin
+   * @returns {string[]} Unique UUIDs, in test-discovery order
+   */
+  extractUuidsFromSuite(suite) {
+    if (!suite || typeof suite.allTests !== 'function') return [];
+
+    const seen = new Set();
+    const ordered = [];
+    for (const test of suite.allTests()) {
+      const uuid = UuidExtractor.extractFromAnnotations(test.annotations || [])
+        || UuidExtractor.extractFromTest(test);
+      if (uuid && !seen.has(uuid)) {
+        seen.add(uuid);
+        ordered.push(uuid);
+      }
+    }
+    return ordered;
+  }
+
+  /**
+   * Validate the set of UUIDs against the configured scope.
+   *
+   * @param {string[]} uuids
+   * @returns {Promise<{
+   *   hasMismatch: boolean,
+   *   inScope: string[],
+   *   outOfScope: Array<{uuid: string, actualScenarioId?: number}>,
+   *   reason: 'scenario' | 'testset' | null
+   * }>}
+   */
+  async validate(uuids) {
+    if (!this.isScoped()) {
+      return { hasMismatch: false, inScope: uuids, outOfScope: [], reason: null };
+    }
+    if (!Array.isArray(uuids) || uuids.length === 0) {
+      return { hasMismatch: false, inScope: [], outOfScope: [], reason: null };
+    }
+
+    if (this.scenarioId) {
+      return this._validateScenario(uuids);
+    }
+    return this._validateTestSet(uuids);
+  }
+
+  _validateScenario(uuids) {
+    const inScope = [];
+    const outOfScope = [];
+
+    for (const uuid of uuids) {
+      const nid = UuidValidator.extractNid(uuid);
+      if (nid === null) {
+        // Malformed UUID — let the regular submission path reject it.
+        // Don't fail the scope check on bad data we didn't author.
+        inScope.push(uuid);
+        continue;
+      }
+      if (nid === Number(this.scenarioId)) {
+        inScope.push(uuid);
+      } else {
+        outOfScope.push({ uuid, actualScenarioId: nid });
+      }
+    }
+
+    return {
+      hasMismatch: outOfScope.length > 0,
+      inScope,
+      outOfScope,
+      reason: 'scenario'
+    };
+  }
+
+  async _validateTestSet(uuids) {
+    if (!this.fetchTestSetScope) {
+      // No way to fetch testset membership — fail open with a warning
+      // rather than blocking the run on a missing dependency.
+      logger.warn(
+        'testSetId is set but no testset-scope fetcher was provided; '
+        + 'skipping scope validation. Pass `fetchTestSetScope` to enable.'
+      );
+      return { hasMismatch: false, inScope: uuids, outOfScope: [], reason: null };
+    }
+
+    let scope;
+    try {
+      scope = await this.fetchTestSetScope(this.testSetId);
+    } catch (error) {
+      logger.warn('Failed to fetch testset scope; skipping validation', {
+        testSetId: this.testSetId,
+        error: error.message
+      });
+      return { hasMismatch: false, inScope: uuids, outOfScope: [], reason: null };
+    }
+
+    const scopeSet = scope instanceof Set ? scope : new Set(scope || []);
+    const inScope = [];
+    const outOfScope = [];
+
+    for (const uuid of uuids) {
+      if (scopeSet.has(uuid)) {
+        inScope.push(uuid);
+      } else {
+        outOfScope.push({ uuid });
+      }
+    }
+
+    return {
+      hasMismatch: outOfScope.length > 0,
+      inScope,
+      outOfScope,
+      reason: 'testset'
+    };
+  }
+
+  /**
+   * Resolve a scope mismatch by applying the chosen strategy. In a TTY,
+   * the user is prompted interactively before any strategy is applied.
+   *
+   * @param {Object} validation - result of `validate()`
+   * @returns {Promise<{
+   *   action: 'cancel' | 'adhoc' | 'filter',
+   *   submitUuids: string[],
+   *   droppedUuids: string[],
+   *   overrideToAdhoc: boolean,
+   *   message: string
+   * }>}
+   */
+  async resolve(validation) {
+    if (!validation.hasMismatch) {
+      return {
+        action: 'pass',
+        submitUuids: validation.inScope,
+        droppedUuids: [],
+        overrideToAdhoc: false,
+        message: 'Scope check passed.'
+      };
+    }
+
+    const strategy = this._isInteractive()
+      ? await this._promptInteractive(validation)
+      : this.strategy;
+
+    return this._applyStrategy(strategy, validation);
+  }
+
+  _isInteractive() {
+    if (process.env.CI === 'true' || process.env.CI === '1') return false;
+    return Boolean(process.stdin && process.stdin.isTTY);
+  }
+
+  async _promptInteractive(validation) {
+    const summary = this._formatMismatchSummary(validation);
+    // Use process.stdout (not logger) because this is a synchronous user
+    // prompt that must appear regardless of log level.
+    process.stdout.write('\n');
+    process.stdout.write(summary);
+    process.stdout.write('\n');
+    process.stdout.write('Choose how to proceed:\n');
+    process.stdout.write('  [1] cancel  — fail fast, submit nothing (safest)\n');
+    process.stdout.write('  [2] adhoc   — create an ad-hoc run and submit everything\n');
+    process.stdout.write('  [3] filter  — submit only in-scope results, drop the rest\n');
+
+    const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+    const ask = (q) => new Promise((res) => rl.question(q, (a) => res(a)));
+    try {
+      const answer = (await ask('Selection [1/2/3]: ')).trim();
+      const map = { '1': STRATEGY.CANCEL, '2': STRATEGY.ADHOC, '3': STRATEGY.FILTER };
+      return map[answer] || STRATEGY.CANCEL;
+    } finally {
+      rl.close();
+    }
+  }
+
+  _applyStrategy(strategy, validation) {
+    const droppedUuids = validation.outOfScope.map((o) => o.uuid);
+    const summary = this._formatMismatchSummary(validation);
+
+    if (strategy === STRATEGY.CANCEL) {
+      return {
+        action: STRATEGY.CANCEL,
+        submitUuids: [],
+        droppedUuids: [...validation.inScope, ...droppedUuids],
+        overrideToAdhoc: false,
+        message:
+          `${summary}\n`
+          + 'Strategy: cancel. No results will be submitted to Appliqation.\n'
+          + 'Set onScopeMismatch (or APPLIQATION_ON_SCOPE_MISMATCH) to '
+          + '"adhoc" or "filter" to change this behaviour.'
+      };
+    }
+
+    if (strategy === STRATEGY.ADHOC) {
+      return {
+        action: STRATEGY.ADHOC,
+        submitUuids: [...validation.inScope, ...droppedUuids],
+        droppedUuids: [],
+        overrideToAdhoc: true,
+        message:
+          `${summary}\n`
+          + 'Strategy: adhoc. The configured scenarioId/testSetId is being '
+          + 'overridden — an ad-hoc run will be created instead and all '
+          + 'results will be submitted to it.'
+      };
+    }
+
+    // FILTER
+    return {
+      action: STRATEGY.FILTER,
+      submitUuids: validation.inScope,
+      droppedUuids,
+      overrideToAdhoc: false,
+      message:
+        `${summary}\n`
+        + `Strategy: filter. ${validation.inScope.length} in-scope result(s) `
+        + `will be submitted; ${droppedUuids.length} out-of-scope result(s) `
+        + 'will execute locally but their verdicts will NOT be uploaded.'
+    };
+  }
+
+  _formatMismatchSummary(validation) {
+    const target = validation.reason === 'scenario'
+      ? `scenarioId ${this.scenarioId}`
+      : `testSetId ${this.testSetId}`;
+
+    const examples = validation.outOfScope.slice(0, 5).map((o) => {
+      if (o.actualScenarioId !== undefined) {
+        return `  - ${o.uuid} (belongs to scenario ${o.actualScenarioId})`;
+      }
+      return `  - ${o.uuid} (not in testset)`;
+    }).join('\n');
+
+    const more = validation.outOfScope.length > 5
+      ? `\n  … and ${validation.outOfScope.length - 5} more`
+      : '';
+
+    return (
+      `[Appliqation] Scope mismatch detected.\n`
+      + `Configured: ${target}\n`
+      + `Out-of-scope test(s): ${validation.outOfScope.length} of ${validation.inScope.length + validation.outOfScope.length}\n`
+      + examples
+      + more
+    );
+  }
+}
+
+ScopeValidator.STRATEGY = STRATEGY;
+
+module.exports = ScopeValidator;

package/src/services/TaggingService.js

@@ -1,4 +1,5 @@
 const logger = require('../utils/logger');
+const { AUTO_TAG_NAME } = require('../constants');
 
 /**
  * Service for auto-tagging test cases after successful runs
@@ -8,12 +9,11 @@ class TaggingService {
     this.http = httpClient;
     this.config = config;
 
-    // Configuration resolution: Runtime options > Env vars > Defaults
+    // Whether to auto-tag is configurable; the tag name itself is not —
+    // see AUTO_TAG_NAME's doc comment.
     this.enabled = this.config?.options?.autoTag !== false &&
                    process.env.APPLIQATION_AUTO_TAG_ENABLED !== 'false';
-    this.tagName = this.config?.options?.autoTagName ||
-                   process.env.APPLIQATION_AUTO_TAG_NAME ||
-                   'Appq_automated';
+    this.tagName = AUTO_TAG_NAME;
     this.batchSize = this.config?.options?.autoTagBatchSize || 50;
     this.retries = this.config?.options?.autoTagRetries || 2;
   }
@@ -42,7 +42,7 @@ class TaggingService {
     const unique = Array.from(new Set(uuids.filter(Boolean)));
 
     try {
-      // Build query string: uuids=123-xxx,124-yyy&tag=Appq_automated
+      // Build query string: uuids=123-xxx,124-yyy&tag=Appq_Auto
       const uuidsParam = unique.join(',');
       const url = `/api/automation/testcases/tags/check?uuids=${encodeURIComponent(uuidsParam)}&tag=${encodeURIComponent(tag)}`;
       const response = await this.http.get(url);