@appliqation/automation-sdk 2.7.0 → 2.8.0

package/src/services/ScopeValidator.js

@@ -0,0 +1,314 @@
+const readline = require('readline');
+const UuidValidator = require('../utils/UuidValidator');
+const UuidExtractor = require('../reporters/playwright/helpers/UuidExtractor');
+const logger = require('../utils/logger');
+
+const STRATEGY = {
+  CANCEL: 'cancel',
+  ADHOC: 'adhoc',
+  FILTER: 'filter'
+};
+
+const VALID_STRATEGIES = Object.values(STRATEGY);
+
+/**
+ * Validates that the UUIDs about to be executed match the user's configured
+ * scenarioId or testSetId scope, and applies a resolution strategy when they
+ * don't.
+ *
+ * Closes finding C5 in the SDK audit — without this, the SDK silently
+ * submits cross-scenario results to a single-scenario run, corrupting the
+ * Appliqation data store.
+ *
+ * Three resolution strategies, selected via `config.onScopeMismatch`:
+ *   'cancel' (default in CI)  — fail fast, exit 1, submit nothing
+ *   'adhoc'                   — override config, create an ad-hoc run
+ *   'filter'                  — drop out-of-scope results, submit the rest
+ *
+ * In a TTY (no `CI=true` and stdin is a TTY), the user is prompted
+ * interactively before any test runs. In CI, the configured strategy is
+ * used; if unset, 'cancel' is the fail-safe default.
+ */
+class ScopeValidator {
+  /**
+   * @param {Object} options
+   * @param {number} [options.scenarioId]
+   * @param {number} [options.testSetId]
+   * @param {'cancel'|'adhoc'|'filter'} [options.strategy]
+   * @param {Function} [options.fetchTestSetScope] async (testSetId) -> Set<uuid>
+   */
+  constructor(options = {}) {
+    this.scenarioId = options.scenarioId || null;
+    this.testSetId = options.testSetId || null;
+    this.strategy = this._resolveStrategy(options.strategy);
+    this.fetchTestSetScope = options.fetchTestSetScope || null;
+  }
+
+  _resolveStrategy(explicit) {
+    const envValue = process.env.APPLIQATION_ON_SCOPE_MISMATCH;
+    const raw = (explicit || envValue || STRATEGY.CANCEL).toLowerCase();
+    if (!VALID_STRATEGIES.includes(raw)) {
+      logger.warn(`Unknown onScopeMismatch strategy "${raw}", falling back to "cancel"`);
+      return STRATEGY.CANCEL;
+    }
+    return raw;
+  }
+
+  /**
+   * @returns {boolean} True if scenarioId or testSetId is set — i.e., the
+   *   user has declared a scope and we need to validate against it.
+   */
+  isScoped() {
+    return Boolean(this.scenarioId || this.testSetId);
+  }
+
+  /**
+   * Extract every Appliqation UUID from a Playwright suite tree.
+   *
+   * @param {Object} suite - Playwright Suite from onBegin
+   * @returns {string[]} Unique UUIDs, in test-discovery order
+   */
+  extractUuidsFromSuite(suite) {
+    if (!suite || typeof suite.allTests !== 'function') return [];
+
+    const seen = new Set();
+    const ordered = [];
+    for (const test of suite.allTests()) {
+      const uuid = UuidExtractor.extractFromAnnotations(test.annotations || [])
+        || UuidExtractor.extractFromTest(test);
+      if (uuid && !seen.has(uuid)) {
+        seen.add(uuid);
+        ordered.push(uuid);
+      }
+    }
+    return ordered;
+  }
+
+  /**
+   * Validate the set of UUIDs against the configured scope.
+   *
+   * @param {string[]} uuids
+   * @returns {Promise<{
+   *   hasMismatch: boolean,
+   *   inScope: string[],
+   *   outOfScope: Array<{uuid: string, actualScenarioId?: number}>,
+   *   reason: 'scenario' | 'testset' | null
+   * }>}
+   */
+  async validate(uuids) {
+    if (!this.isScoped()) {
+      return { hasMismatch: false, inScope: uuids, outOfScope: [], reason: null };
+    }
+    if (!Array.isArray(uuids) || uuids.length === 0) {
+      return { hasMismatch: false, inScope: [], outOfScope: [], reason: null };
+    }
+
+    if (this.scenarioId) {
+      return this._validateScenario(uuids);
+    }
+    return this._validateTestSet(uuids);
+  }
+
+  _validateScenario(uuids) {
+    const inScope = [];
+    const outOfScope = [];
+
+    for (const uuid of uuids) {
+      const nid = UuidValidator.extractNid(uuid);
+      if (nid === null) {
+        // Malformed UUID — let the regular submission path reject it.
+        // Don't fail the scope check on bad data we didn't author.
+        inScope.push(uuid);
+        continue;
+      }
+      if (nid === Number(this.scenarioId)) {
+        inScope.push(uuid);
+      } else {
+        outOfScope.push({ uuid, actualScenarioId: nid });
+      }
+    }
+
+    return {
+      hasMismatch: outOfScope.length > 0,
+      inScope,
+      outOfScope,
+      reason: 'scenario'
+    };
+  }
+
+  async _validateTestSet(uuids) {
+    if (!this.fetchTestSetScope) {
+      // No way to fetch testset membership — fail open with a warning
+      // rather than blocking the run on a missing dependency.
+      logger.warn(
+        'testSetId is set but no testset-scope fetcher was provided; '
+        + 'skipping scope validation. Pass `fetchTestSetScope` to enable.'
+      );
+      return { hasMismatch: false, inScope: uuids, outOfScope: [], reason: null };
+    }
+
+    let scope;
+    try {
+      scope = await this.fetchTestSetScope(this.testSetId);
+    } catch (error) {
+      logger.warn('Failed to fetch testset scope; skipping validation', {
+        testSetId: this.testSetId,
+        error: error.message
+      });
+      return { hasMismatch: false, inScope: uuids, outOfScope: [], reason: null };
+    }
+
+    const scopeSet = scope instanceof Set ? scope : new Set(scope || []);
+    const inScope = [];
+    const outOfScope = [];
+
+    for (const uuid of uuids) {
+      if (scopeSet.has(uuid)) {
+        inScope.push(uuid);
+      } else {
+        outOfScope.push({ uuid });
+      }
+    }
+
+    return {
+      hasMismatch: outOfScope.length > 0,
+      inScope,
+      outOfScope,
+      reason: 'testset'
+    };
+  }
+
+  /**
+   * Resolve a scope mismatch by applying the chosen strategy. In a TTY,
+   * the user is prompted interactively before any strategy is applied.
+   *
+   * @param {Object} validation - result of `validate()`
+   * @returns {Promise<{
+   *   action: 'cancel' | 'adhoc' | 'filter',
+   *   submitUuids: string[],
+   *   droppedUuids: string[],
+   *   overrideToAdhoc: boolean,
+   *   message: string
+   * }>}
+   */
+  async resolve(validation) {
+    if (!validation.hasMismatch) {
+      return {
+        action: 'pass',
+        submitUuids: validation.inScope,
+        droppedUuids: [],
+        overrideToAdhoc: false,
+        message: 'Scope check passed.'
+      };
+    }
+
+    const strategy = this._isInteractive()
+      ? await this._promptInteractive(validation)
+      : this.strategy;
+
+    return this._applyStrategy(strategy, validation);
+  }
+
+  _isInteractive() {
+    if (process.env.CI === 'true' || process.env.CI === '1') return false;
+    return Boolean(process.stdin && process.stdin.isTTY);
+  }
+
+  async _promptInteractive(validation) {
+    const summary = this._formatMismatchSummary(validation);
+    // Use process.stdout (not logger) because this is a synchronous user
+    // prompt that must appear regardless of log level.
+    process.stdout.write('\n');
+    process.stdout.write(summary);
+    process.stdout.write('\n');
+    process.stdout.write('Choose how to proceed:\n');
+    process.stdout.write('  [1] cancel  — fail fast, submit nothing (safest)\n');
+    process.stdout.write('  [2] adhoc   — create an ad-hoc run and submit everything\n');
+    process.stdout.write('  [3] filter  — submit only in-scope results, drop the rest\n');
+
+    const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+    const ask = (q) => new Promise((res) => rl.question(q, (a) => res(a)));
+    try {
+      const answer = (await ask('Selection [1/2/3]: ')).trim();
+      const map = { '1': STRATEGY.CANCEL, '2': STRATEGY.ADHOC, '3': STRATEGY.FILTER };
+      return map[answer] || STRATEGY.CANCEL;
+    } finally {
+      rl.close();
+    }
+  }
+
+  _applyStrategy(strategy, validation) {
+    const droppedUuids = validation.outOfScope.map((o) => o.uuid);
+    const summary = this._formatMismatchSummary(validation);
+
+    if (strategy === STRATEGY.CANCEL) {
+      return {
+        action: STRATEGY.CANCEL,
+        submitUuids: [],
+        droppedUuids: [...validation.inScope, ...droppedUuids],
+        overrideToAdhoc: false,
+        message:
+          `${summary}\n`
+          + 'Strategy: cancel. No results will be submitted to Appliqation.\n'
+          + 'Set onScopeMismatch (or APPLIQATION_ON_SCOPE_MISMATCH) to '
+          + '"adhoc" or "filter" to change this behaviour.'
+      };
+    }
+
+    if (strategy === STRATEGY.ADHOC) {
+      return {
+        action: STRATEGY.ADHOC,
+        submitUuids: [...validation.inScope, ...droppedUuids],
+        droppedUuids: [],
+        overrideToAdhoc: true,
+        message:
+          `${summary}\n`
+          + 'Strategy: adhoc. The configured scenarioId/testSetId is being '
+          + 'overridden — an ad-hoc run will be created instead and all '
+          + 'results will be submitted to it.'
+      };
+    }
+
+    // FILTER
+    return {
+      action: STRATEGY.FILTER,
+      submitUuids: validation.inScope,
+      droppedUuids,
+      overrideToAdhoc: false,
+      message:
+        `${summary}\n`
+        + `Strategy: filter. ${validation.inScope.length} in-scope result(s) `
+        + `will be submitted; ${droppedUuids.length} out-of-scope result(s) `
+        + 'will execute locally but their verdicts will NOT be uploaded.'
+    };
+  }
+
+  _formatMismatchSummary(validation) {
+    const target = validation.reason === 'scenario'
+      ? `scenarioId ${this.scenarioId}`
+      : `testSetId ${this.testSetId}`;
+
+    const examples = validation.outOfScope.slice(0, 5).map((o) => {
+      if (o.actualScenarioId !== undefined) {
+        return `  - ${o.uuid} (belongs to scenario ${o.actualScenarioId})`;
+      }
+      return `  - ${o.uuid} (not in testset)`;
+    }).join('\n');
+
+    const more = validation.outOfScope.length > 5
+      ? `\n  … and ${validation.outOfScope.length - 5} more`
+      : '';
+
+    return (
+      `[Appliqation] Scope mismatch detected.\n`
+      + `Configured: ${target}\n`
+      + `Out-of-scope test(s): ${validation.outOfScope.length} of ${validation.inScope.length + validation.outOfScope.length}\n`
+      + examples
+      + more
+    );
+  }
+}
+
+ScopeValidator.STRATEGY = STRATEGY;
+
+module.exports = ScopeValidator;

package/src/services/TaggingService.js

@@ -1,4 +1,5 @@
 const logger = require('../utils/logger');
+const { AUTO_TAG_NAME } = require('../constants');
 
 /**
  * Service for auto-tagging test cases after successful runs
@@ -8,12 +9,11 @@ class TaggingService {
     this.http = httpClient;
     this.config = config;
 
-    // Configuration resolution: Runtime options > Env vars > Defaults
+    // Whether to auto-tag is configurable; the tag name itself is not —
+    // see AUTO_TAG_NAME's doc comment.
     this.enabled = this.config?.options?.autoTag !== false &&
                    process.env.APPLIQATION_AUTO_TAG_ENABLED !== 'false';
-    this.tagName = this.config?.options?.autoTagName ||
-                   process.env.APPLIQATION_AUTO_TAG_NAME ||
-                   'Appq_automated';
+    this.tagName = AUTO_TAG_NAME;
     this.batchSize = this.config?.options?.autoTagBatchSize || 50;
     this.retries = this.config?.options?.autoTagRetries || 2;
   }
@@ -42,7 +42,7 @@ class TaggingService {
     const unique = Array.from(new Set(uuids.filter(Boolean)));
 
     try {
-      // Build query string: uuids=123-xxx,124-yyy&tag=Appq_automated
+      // Build query string: uuids=123-xxx,124-yyy&tag=Appq_Auto
       const uuidsParam = unique.join(',');
       const url = `/api/automation/testcases/tags/check?uuids=${encodeURIComponent(uuidsParam)}&tag=${encodeURIComponent(tag)}`;
       const response = await this.http.get(url);

package/src/utils/PayloadBuilder.js

@@ -114,36 +114,19 @@ class PayloadBuilder {
   static normalizeStatus(status) {
     if (!status) return 'unknown';
 
+    // Playwright-only status map. Cypress/Jest/generic variants were
+    // removed alongside the Cypress and Jest reporters — D1/D2/D3.
     const statusMap = {
-      // Standard statuses
       'passed': 'passed',
       'failed': 'failed',
       'skipped': 'skipped',
       'pending': 'pending',
-
-      // Playwright variants
       'expected': 'passed',
       'unexpected': 'failed',
       'flaky': 'flaky',
       'timedOut': 'failed',
-      'interrupted': 'skipped',
-
-      // Cypress variants
-      'pass': 'passed',
-      'fail': 'failed',
-
-      // Jest variants
-      'success': 'passed',
-      'error': 'failed',
-      'disabled': 'skipped',
-
-      // Generic variants
-      'ok': 'passed',
-      'success': 'passed',
-      'failure': 'failed',
-      'error': 'failed',
-      'skip': 'skipped',
-      'todo': 'pending'
+      // Aborted run, not a skip — see AppliqationReporter.mapStatus.
+      'interrupted': 'failed'
     };
 
     const normalized = statusMap[status.toLowerCase()];
@@ -212,6 +195,114 @@ class PayloadBuilder {
     };
   }
 
+  /**
+   * S8 — Pre-submission payload validation for the automation result
+   * endpoint. Runs AFTER buildAutomationResultPayload has assembled the
+   * final wire payload, immediately before the HTTP POST. Enforces the
+   * exact shape the backend's InputValidator + AutomationResultManager
+   * accept, so shape drift (wrong field name, missing required field,
+   * bad type) is caught at the client instead of failing at the server
+   * with a generic 400 mid-run.
+   *
+   * Defence-in-depth on top of H2 (which validates user-supplied CONFIG
+   * at startup) — S8 validates the assembled OUTGOING payload.
+   *
+   * Backend constraints (mirror of appq_mongo InputValidator +
+   * AutomationApiController::submitResultBatch):
+   *   run_id            required, matches /^[a-zA-Z0-9_:-]+$/ max 100
+   *   test_case_uuid    required, matches UuidValidator's regex
+   *   status            required, one of {passed, failed, skipped}
+   *   source            'automation' when SDK-submitted (post C1)
+   *   browser           required, matches /^[a-zA-Z0-9 .\-_(),\/]+$/ max 255
+   *   environment       matches /^[a-zA-Z0-9._-]+$/ max 255
+   *   duration          integer >= 0 when present
+   *   timestamp         positive integer when present
+   *
+   * @param {Object} payload - Fully assembled result payload
+   * @returns {{ valid: boolean, errors: string[] }}
+   */
+  static validateAutomationResultPayload(payload) {
+    const errors = [];
+
+    if (!payload || typeof payload !== 'object') {
+      return { valid: false, errors: ['payload must be an object'] };
+    }
+
+    // run_id
+    if (typeof payload.run_id !== 'string' || payload.run_id.length === 0) {
+      errors.push('run_id must be a non-empty string');
+    } else if (payload.run_id.length > 100) {
+      errors.push(`run_id exceeds 100 chars (got ${payload.run_id.length})`);
+    } else if (!/^[a-zA-Z0-9_:-]+$/.test(payload.run_id)) {
+      errors.push(`run_id contains invalid characters: "${payload.run_id}"`);
+    }
+
+    // test_case_uuid — reuse UuidValidator for the canonical check
+    if (!payload.test_case_uuid) {
+      errors.push('test_case_uuid is required');
+    } else {
+      // Require nid-prefixed uuid-v4 shape. Duplicated here to avoid a
+      // circular require between PayloadBuilder and UuidValidator.
+      const uuidPattern = /^\d+-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i;
+      if (!uuidPattern.test(payload.test_case_uuid)) {
+        errors.push(`test_case_uuid does not match {nid}-{uuid-v4} shape: "${payload.test_case_uuid}"`);
+      }
+    }
+
+    // status
+    const ALLOWED_STATUSES = new Set(['passed', 'failed', 'skipped']);
+    if (!payload.status) {
+      errors.push('status is required');
+    } else if (!ALLOWED_STATUSES.has(String(payload.status).toLowerCase())) {
+      errors.push(`status must be one of {passed, failed, skipped} (got "${payload.status}")`);
+    }
+
+    // source — must be 'automation' when set (C1 contract)
+    if (payload.source !== undefined && payload.source !== 'automation') {
+      errors.push(`source must be "automation" if provided (got "${payload.source}")`);
+    }
+
+    // browser
+    if (payload.browser !== undefined) {
+      if (typeof payload.browser !== 'string' || payload.browser.length === 0) {
+        errors.push('browser must be a non-empty string when provided');
+      } else if (payload.browser.length > 255) {
+        errors.push(`browser exceeds 255 chars (got ${payload.browser.length})`);
+      } else if (!/^[a-zA-Z0-9 .\-_(),\/]+$/.test(payload.browser)) {
+        errors.push(`browser contains invalid characters: "${payload.browser}"`);
+      }
+    }
+
+    // environment
+    if (payload.environment !== undefined && payload.environment !== '') {
+      if (typeof payload.environment !== 'string') {
+        errors.push(`environment must be a string (got ${typeof payload.environment})`);
+      } else if (payload.environment.length > 255) {
+        errors.push(`environment exceeds 255 chars (got ${payload.environment.length})`);
+      } else if (!/^[a-zA-Z0-9._-]+$/.test(payload.environment)) {
+        errors.push(`environment contains invalid characters: "${payload.environment}"`);
+      }
+    }
+
+    // duration
+    if (payload.duration !== undefined && payload.duration !== null) {
+      const d = payload.duration;
+      if (typeof d !== 'number' || !Number.isFinite(d) || d < 0 || !Number.isInteger(d)) {
+        errors.push(`duration must be a non-negative integer (got ${d})`);
+      }
+    }
+
+    // timestamp
+    if (payload.timestamp !== undefined && payload.timestamp !== null) {
+      const t = payload.timestamp;
+      if (typeof t !== 'number' || !Number.isFinite(t) || t <= 0 || !Number.isInteger(t)) {
+        errors.push(`timestamp must be a positive integer (got ${t})`);
+      }
+    }
+
+    return { valid: errors.length === 0, errors };
+  }
+
   /**
    * Validate run creation options
    * @param {Object} options - Run options

package/src/utils/logger.js

@@ -5,7 +5,8 @@ const LOG_LEVELS = {
   DEBUG: 3
 };
 
-// Sensitive data patterns to sanitize
+// Regex-based sanitization catches value-shaped secrets inside string
+// payloads (raw JWTs in error messages, axios request dumps, etc).
 const SENSITIVE_PATTERNS = [
   // API Keys
   { pattern: /(appq_\w+_[a-f0-9]{32})/gi, replacement: 'appq_***_REDACTED' },
@@ -30,6 +31,51 @@ const SENSITIVE_PATTERNS = [
   { pattern: /("session"\s*:\s*")([^"]+)(")/gi, replacement: '$1***REDACTED***$3' }
 ];
 
+// H5 — field-name-based sanitization catches custom-named fields the regex
+// list will miss: `x-secret`, `myAuthToken`, `dbPassword`, `serviceKey`, etc.
+// Applied recursively before the regex pass. Errs on the side of over-
+// redaction: any object key matching this pattern has its value replaced
+// regardless of nesting depth.
+const SENSITIVE_KEY_PATTERN = /token|secret|key|password|credential|auth/i;
+
+function redactByFieldName(value, seen) {
+  if (value === null || typeof value !== 'object') return value;
+  if (!seen) seen = new WeakSet();
+  if (seen.has(value)) return '[Circular]';
+  seen.add(value);
+
+  if (Array.isArray(value)) {
+    return value.map((item) => redactByFieldName(item, seen));
+  }
+
+  const result = {};
+  for (const k of Object.keys(value)) {
+    const v = value[k];
+    if (SENSITIVE_KEY_PATTERN.test(k)) {
+      if (v === null || v === undefined) {
+        // Preserve null/undefined so consumers can detect "field present
+        // but empty" without leaking real values.
+        result[k] = v;
+      } else if (typeof v === 'object') {
+        // Recurse into nested objects/arrays. A sensitive container like
+        // `credentials: { user, password }` keeps its structure but its
+        // leaf values get redacted by the inner pass, so debug output
+        // still tells you which fields existed.
+        result[k] = redactByFieldName(v, seen);
+      } else if (typeof v === 'string' && /^Bearer\s+/i.test(v)) {
+        // Preserve scheme prefix for Authorization-style headers so the
+        // log reader can tell what kind of token was redacted.
+        result[k] = v.replace(/^(Bearer\s+).*$/i, '$1***REDACTED***');
+      } else {
+        result[k] = '***REDACTED***';
+      }
+    } else {
+      result[k] = redactByFieldName(v, seen);
+    }
+  }
+  return result;
+}
+
 class Logger {
   constructor(level = 'INFO') {
     this.level = LOG_LEVELS[level.toUpperCase()] || LOG_LEVELS.INFO;
@@ -55,17 +101,25 @@ class Logger {
       return sanitized;
     }
 
-    // Handle objects and arrays by converting to JSON and back
+    // Handle objects and arrays: redact by field name first (catches
+    // custom-named credential fields), then apply the regex pass to any
+    // remaining string values for value-shaped secrets.
     if (typeof data === 'object') {
       try {
-        let jsonString = JSON.stringify(data);
+        const redacted = redactByFieldName(data);
+        let jsonString = JSON.stringify(redacted);
         SENSITIVE_PATTERNS.forEach(({ pattern, replacement }) => {
           jsonString = jsonString.replace(pattern, replacement);
         });
         return JSON.parse(jsonString);
       } catch (error) {
-        // If JSON parsing fails, return original data
-        return data;
+        // If JSON serialization fails, return the field-redacted form
+        // rather than the raw data — never leak unfiltered objects.
+        try {
+          return redactByFieldName(data);
+        } catch (e) {
+          return data;
+        }
       }
     }