@appliqation/automation-sdk 2.5.1 → 2.8.0

package/src/index.d.ts

@@ -1,8 +1,20 @@
 /**
  * TypeScript definitions for @appliqation/automation-sdk
- * @version 2.1.0
+ * @version 2.8.0
  */
 
+/**
+ * Scope-mismatch resolution strategy (audit finding C5).
+ *
+ * When `scenarioId` or `testSetId` is set in config and the executed
+ * suite contains UUIDs outside that scope, the SDK applies one of
+ * these strategies before submitting results.
+ *   'cancel' — fail fast, submit nothing, exit 1 (default in CI)
+ *   'adhoc'  — override config, create an ad-hoc run, submit everything
+ *   'filter' — submit only in-scope results, drop the rest locally
+ */
+export type ScopeMismatchStrategy = 'cancel' | 'adhoc' | 'filter';
+
 // ============================================================================
 // Core Types
 // ============================================================================
@@ -49,10 +61,16 @@ export interface AppliqationConfig {
   /** Project key */
   projectKey?: string;
 
-  /** Username for legacy CSRF authentication */
+  /**
+   * @deprecated Legacy CSRF authentication. Slated for removal in a
+   * future major version (audit finding D4). Use `apiKey` instead.
+   */
   username?: string;
 
-  /** Password for legacy CSRF authentication */
+  /**
+   * @deprecated Legacy CSRF authentication. Slated for removal in a
+   * future major version (audit finding D4). Use `apiKey` instead.
+   */
   password?: string;
 
   /** Scenario ID (optional, 0 for generic automation runs) */
@@ -88,9 +106,25 @@ export interface AppliqationConfig {
   /** Batch size for result submission */
   batchSize?: number;
 
-  /** SSL certificate verification (default: true) */
+  /**
+   * SSL certificate verification (default: true).
+   *
+   * H3 — setting this to `false` in a production-like environment
+   * (`CI=true`, `NODE_ENV=production`/`prod`/`staging`, or any well-known
+   * CI env var) will throw `ConfigurationError` at client construction.
+   * Explicit override via `APPLIQATION_INSECURE=allow` for the rare case
+   * where a customer genuinely needs to bypass TLS in CI.
+   */
   rejectUnauthorized?: boolean;
 
+  /**
+   * C5 — scope-mismatch resolution strategy. Applied when the executed
+   * suite contains UUIDs outside the configured `scenarioId`/`testSetId`.
+   * Also settable via `APPLIQATION_ON_SCOPE_MISMATCH` env var.
+   * Defaults to 'cancel' in CI (fail safe), interactive prompt in TTY.
+   */
+  onScopeMismatch?: ScopeMismatchStrategy;
+
   /** Additional options */
   options?: {
     /** Request timeout in milliseconds (default: 30000) */
@@ -232,9 +266,16 @@ export class AppliqationClient {
   constructor(config: AppliqationConfig);
 
   /**
-   * Create a new automation run
+   * Create a new automation run.
+   *
+   * The optional `uuids` field (C4 / E3) pre-populates the run
+   * document's `data[]` array so the UI grid and orphan detector know
+   * which TCs belong to the run before any result lands. Pass plain
+   * string UUIDs matching PR appq/#532's schema (not `{uuid}` objects).
    */
-  createRun(config?: Partial<AppliqationConfig>): Promise<RunMatrixResponse>;
+  createRun(
+    config?: Partial<AppliqationConfig> & { uuids?: string[] }
+  ): Promise<RunMatrixResponse>;
 
   /**
    * Submit a single test result
@@ -299,6 +340,61 @@ export class PayloadBuilder {
     status: TestStatus,
     options?: { comment?: string; attachments?: string[]; metadata?: Record<string, any> }
   ): TestResult;
+
+  /**
+   * S8 — validate a fully assembled automation result payload against
+   * backend shape constraints (InputValidator regexes + status enum).
+   * Runs client-side immediately before submission so schema drift is
+   * caught with a precise field-path error instead of a generic 400.
+   */
+  static validateAutomationResultPayload(payload: {
+    run_id?: unknown;
+    test_case_uuid?: unknown;
+    status?: unknown;
+    source?: unknown;
+    browser?: unknown;
+    environment?: unknown;
+    duration?: unknown;
+    timestamp?: unknown;
+    [key: string]: unknown;
+  }): { valid: boolean; errors: string[] };
+}
+
+/**
+ * C5 — Pre-execution scope validator. Verifies UUIDs about to execute
+ * match the configured scenarioId/testSetId; applies cancel/adhoc/filter
+ * strategy on mismatch. Wired automatically by AppliqationReporter in
+ * `onBegin`; exposed for programmatic use.
+ */
+export class ScopeValidator {
+  constructor(options: {
+    scenarioId?: number | string;
+    testSetId?: number | string;
+    strategy?: ScopeMismatchStrategy;
+    fetchTestSetScope?: (testSetId: number | string) => Promise<Set<string> | string[]>;
+  });
+
+  static readonly STRATEGY: {
+    CANCEL: 'cancel';
+    ADHOC: 'adhoc';
+    FILTER: 'filter';
+  };
+
+  isScoped(): boolean;
+  extractUuidsFromSuite(suite: unknown): string[];
+  validate(uuids: string[]): Promise<{
+    hasMismatch: boolean;
+    inScope: string[];
+    outOfScope: Array<{ uuid: string; actualScenarioId?: number }>;
+    reason: 'scenario' | 'testset' | null;
+  }>;
+  resolve(validation: unknown): Promise<{
+    action: 'cancel' | 'adhoc' | 'filter' | 'pass';
+    submitUuids: string[];
+    droppedUuids: string[];
+    overrideToAdhoc: boolean;
+    message: string;
+  }>;
 }
 
 export const logger: {
@@ -331,3 +427,60 @@ export const logger: {
  * ```
  */
 export function mapAppqUuid(testInfo: any, uuid: string): void;
+
+/**
+ * Returns the canonical Playwright `storageState` file path for a given
+ * Appliqation project + role. The same path resolves regardless of
+ * execution context, so the same generated/authored test runs unchanged
+ * in:
+ *   - Appliqation's executor (LoginHelper writes the file at this path)
+ *   - Customer CI / local (the `appq-auth-setup` CLI writes the file)
+ *
+ * Does NOT perform login itself — only computes the path. Pair with
+ * `npx appq-auth-setup --project-id X --role Y` (or Appliqation's
+ * executor) to populate the file before tests run.
+ *
+ * @example
+ * ```typescript
+ * import { test } from '@playwright/test';
+ * import { mapAppqUuid, setupAuth } from '@appliqation/automation-sdk/utils';
+ *
+ * test.use({ storageState: setupAuth({ project_id: 126, role: 'default' }) });
+ *
+ * test('manager dashboard loads', async ({ page }, testInfo) => {
+ *   mapAppqUuid(testInfo, '1141-...');
+ *   await page.goto('/dashboard'); // already authenticated
+ * });
+ * ```
+ */
+export function setupAuth(options: { project_id: number | string; role?: string }): string;
+
+/**
+ * Computes the canonical storageState file path. Same as setupAuth, but
+ * usable by tooling that needs the path without going through Playwright
+ * (e.g. the appq-auth-setup CLI writes to this path).
+ */
+export function authStatePath(options: { project_id: number | string; role?: string }): string;
+
+/**
+ * Base directory for storageState files. Defaults to `~/.appq-auth/`;
+ * overridable via APPQ_AUTH_STATE_DIR env (Appliqation's executor sets
+ * this per-run for tenant isolation).
+ */
+export function authStateDir(): string;
+
+/**
+ * The canonical env-var names the CLI looks up for credentials, given
+ * a project_id + role. Customers populate these from the `.env` template
+ * downloaded from Appliqation's project settings.
+ */
+export function envVarNames(options: { project_id: number | string; role?: string }): {
+  username: string;
+  password: string;
+};
+
+/**
+ * Normalize a role name to lowercase with only alphanumerics, underscore,
+ * and hyphen. Mirrors the role-name validation in the Appliqation UI.
+ */
+export function sanitizeRole(role: string): string;

package/src/login/index.d.ts

@@ -0,0 +1,77 @@
+/**
+ * TypeScript definitions for @appliqation/automation-sdk/login.
+ *
+ * Customers writing `tests/automan/auth/login.ts` (default path;
+ * configurable in Project Settings) import these
+ * types for autocomplete + compile-time checking on the login flow
+ * function shape. Same file gets imported (untyped, dynamically) by
+ * the Appliqation executor's LoginHelper and the appq-auth-setup CLI.
+ */
+
+import type { Page } from '@playwright/test';
+
+/**
+ * Per-call context passed to the customer's login function. Sourced
+ * differently in each runtime:
+ *
+ *   - Appliqation executor: username + password from AWS Secrets
+ *     Manager (read-only IAM); baseURL from the project's env
+ *     configuration; role from snapshot.auth_role.
+ *   - Customer CI (`npx appq-auth-setup`): username + password from
+ *     APPQ_PROJECT_<id>_<ROLE>_USERNAME/PASSWORD env vars; baseURL
+ *     from APPLIQATION_SUT_BASE_URL env var; role from --role flag.
+ */
+export interface LoginContext {
+  /** Username / email for the configured role. */
+  username: string;
+  /** Password for the configured role. */
+  password: string;
+  /**
+   * Role name as configured in Project Settings. Lets the customer
+   * branch login flow per role (e.g. an admin's login may include a
+   * "Switch to admin view" step that other roles skip).
+   */
+  role: string;
+  /**
+   * SUT base URL for the env this run targets. Use `await
+   * page.goto('/login')` — Playwright resolves the relative path
+   * against the context's baseURL, which is set from this value.
+   * Provided here for cases where the customer needs the full URL
+   * (e.g. cross-origin OAuth redirects).
+   */
+  baseURL: string;
+}
+
+/**
+ * Customer's login function. Should leave the page in an
+ * authenticated state — typically by navigating to the login URL,
+ * filling credentials, submitting, and waiting for a post-login URL
+ * or visible-element indicator. The caller (LoginHelper or
+ * appq-auth-setup) captures the resulting storageState immediately
+ * after this resolves.
+ *
+ * Throw on failure. The caller will surface the error verbatim in
+ * the executor's run audit / the CLI's exit code.
+ */
+export type LoginFunction = (
+  page: Page,
+  ctx: LoginContext,
+) => Promise<void>;
+
+/**
+ * Identity helper for type inference. Wrap your login function with
+ * this so your IDE provides autocomplete on the LoginContext fields:
+ *
+ * ```typescript
+ * import { defineLogin } from '@appliqation/automation-sdk/login';
+ *
+ * export default defineLogin(async (page, { username, password }) => {
+ *   await page.goto('/login');
+ *   await page.getByLabel('Email').fill(username);
+ *   await page.getByLabel('Password').fill(password);
+ *   await page.getByRole('button', { name: 'Sign in' }).click();
+ *   await page.waitForURL('**\/dashboard');
+ * });
+ * ```
+ */
+export function defineLogin<F extends LoginFunction>(fn: F): F;

package/src/login/index.js

@@ -0,0 +1,49 @@
+/**
+ * @appliqation/automation-sdk/login — types + helpers for customer-
+ * supplied login flows.
+ *
+ * Customers define their SUT's login flow as a Playwright function in
+ * their own repo (canonical path: `tests/automan/auth/login.ts`, default;
+ * configurable in Project Settings).
+ * The function is imported by:
+ *   1. Appliqation's executor (LoginHelper) — pulled from MongoDB
+ *      after a GitHub-webhook upsert into automan_canonical_scripts
+ *      with script_type:'login', dynamic-imported per role per cache
+ *      miss.
+ *   2. Customer CI (`npx appq-auth-setup`) — dynamic-imported from
+ *      disk, run with role-specific creds from env vars, output a
+ *      Playwright storageState file all tests load via
+ *      test.use({ storageState: setupAuth({ ... }) }).
+ *
+ * Same source file, two readers. Both run the customer's own code.
+ *
+ * Why offline GitOps and not in-app upload/edit:
+ *   - Customer's IDE + lint + PR review + git history come for free
+ *   - No in-app editor to build, no sandboxing concern beyond what we
+ *     already have for test scripts
+ *   - The login function is just Playwright code — it should live next
+ *     to the Playwright tests
+ *
+ * The selector-fill model in the legacy auth_config schema (login_url
+ * + 3 CSS selectors + success_indicator) only handled the simplest
+ * form-login case. SSO / MFA / OAuth / multi-step / captcha all
+ * require custom code anyway. Customer login.ts handles everything.
+ */
+
+/**
+ * Identity helper that gives type inference + auto-complete for
+ * customers writing their login.ts. Mirrors Playwright's
+ * `defineConfig` pattern — pure function, no runtime side-effects.
+ *
+ * @template {LoginFunction} F
+ * @param {F} fn
+ * @returns {F}
+ */
+function defineLogin(fn) {
+  if (typeof fn !== 'function') {
+    throw new TypeError('defineLogin: argument must be a function');
+  }
+  return fn;
+}
+
+module.exports = { defineLogin };

package/src/playwright/fixture.js

@@ -9,7 +9,10 @@ try {
 }
 
 const JwtBrowserAuth = require('./JwtBrowserAuth');
-require('dotenv').config();
+// L3 — dotenv removed from the fixture. In enterprise CI, env vars are
+// injected by the platform (GitHub Actions, Jenkins, GitLab); a stray
+// committed .env file would otherwise silently override them. Env var
+// loading is the customer's responsibility in playwright.config.ts.
 const { DEFAULT_APPLIQATION_BASE_URL } = require('../constants');
 
 /**

package/src/reporters/playwright/AppliqationReporter.js

@@ -1,5 +1,6 @@
 const path = require('path');
 const AppliqationClient = require('../../AppliqationClient');
+const ScopeValidator = require('../../services/ScopeValidator');
 const UuidExtractor = require('./helpers/UuidExtractor');
 const DeviceOsDetector = require('./helpers/DeviceOsDetector');
 const PayloadBuilder = require('../../utils/PayloadBuilder');
@@ -176,6 +177,11 @@ class AppliqationReporter {
     this.executionStartTime = null;
     this.executionEndTime = null;
     this.playwrightOutputDir = null;
+
+    // C5 — scope validation state
+    this.scopeFilterEnabled = false;  // true when strategy === 'filter'
+    this.inScopeUuids = null;          // Set<string>, null = no filter
+    this.droppedOutOfScopeCount = 0;
   }
 
   /**
@@ -211,6 +217,37 @@ class AppliqationReporter {
         }
       }
 
+      // Extract every Appliqation-mapped UUID from the suite once, then
+      // share with scope validation (C5) and run pre-population (C4/E3).
+      // Single walk over the test tree, single source of truth.
+      const allSuiteUuids = ScopeValidator.prototype.extractUuidsFromSuite.call(
+        { /* no scope needed for extraction */ }, suite
+      );
+
+      // C5 — pre-execution scope validation.
+      // If the user set scenarioId or testSetId, verify every test about to
+      // run actually belongs to that scope. Mismatch → apply the configured
+      // resolution strategy (cancel / adhoc / filter). This is the SDK's
+      // primary defence against silent data corruption: without it, a
+      // tag-based selection that spans scenarios writes to the wrong run.
+      const scopeOutcome = await this._runScopeValidation(suite, allSuiteUuids);
+      if (scopeOutcome.cancelled) {
+        // User chose cancel (or default in CI on mismatch). Skip run
+        // creation and result submission entirely; tests still execute
+        // locally so the user keeps their own Playwright output.
+        this.appqEnabled = false;
+        return;
+      }
+
+      // C4 / E3 — UUIDs to pre-populate on the run document. If the filter
+      // strategy applied, only the in-scope subset is sent; otherwise the
+      // full extracted set. Backend stores these in data[] so the UI grid
+      // and orphan detector know which TCs belong to the run before any
+      // result lands.
+      this.preExecutionUuids = this.scopeFilterEnabled && this.inScopeUuids
+        ? Array.from(this.inScopeUuids)
+        : allSuiteUuids;
+
       if (!this.config.autoCreateRun) {
         logger.info('Auto-create run disabled. Skipping run matrix creation.');
         return;
@@ -276,7 +313,11 @@ class AppliqationReporter {
             browsers: matrixConfig.browsers,
             device: matrixConfig.device,
             os: matrixConfig.os,
-            title: this.config.title
+            title: this.config.title,
+            // C4 / E3 — pre-populate the run's data[] array with the
+            // UUIDs about to execute. Same set across all matrix configs
+            // (Playwright reruns the same tests per browser project).
+            uuids: this.preExecutionUuids
           };
 
           const run = await this.client.createRun(runOptions);
@@ -372,6 +413,16 @@ class AppliqationReporter {
     try {
       const uuid = UuidExtractor.extractFromAnnotations(result.annotations || []) || UuidExtractor.extractFromTest(test);
 
+      // C5 filter strategy: scope validation in onBegin flagged this test
+      // as out-of-scope for the configured scenarioId/testSetId. The test
+      // still executed (the customer has their local report), but its
+      // verdict must not land in the wrong run.
+      if (this.scopeFilterEnabled && uuid && this.inScopeUuids && !this.inScopeUuids.has(uuid)) {
+        this.droppedOutOfScopeCount++;
+        logger.debug('Dropping out-of-scope result (C5 filter strategy)', { uuid });
+        return;
+      }
+
       const project = test.parent?.project?.() || test.parent?.project;
       const deviceInfo = DeviceOsDetector.getDeviceInfo(project, null);
       const projectKey = `${deviceInfo.device}-${deviceInfo.os}`;
@@ -405,7 +456,15 @@ class AppliqationReporter {
         return;
       }
 
-      // Duplicate detection
+      // Duplicate detection — user-facing warning only, NOT a data-integrity
+      // guard. The backend's upsert key is (run_id, uuid, parent_uuid,
+      // browser), so re-submitting the same UUID for the same browser is
+      // idempotent at the DB level (see appq PR #532's automation.attempt
+      // $inc semantics). This tracking exists purely so the end-of-run
+      // summary can point out authoring mistakes (two tests annotated with
+      // the same UUID), and it's per-reporter-instance — cross-shard
+      // duplicates from `--shard` are not caught here, and don't need to
+      // be, because the backend's upsert still handles them correctly.
       const trackingKey = `${runInfo.runId}:${deviceInfo.browser}`;
       const submittedUuids = this.submittedUuidsByRun.get(trackingKey) || new Map();
 
@@ -462,9 +521,9 @@ class AppliqationReporter {
 
       this.trackResult(runInfo.runId, testResult);
 
-      if (testResult.status === 'Pass') this.passedTests++;
-      else if (testResult.status === 'Fail') this.failedTests++;
-      else if (testResult.status === 'Skipped') this.skippedTests++;
+      if (testResult.status === 'passed') this.passedTests++;
+      else if (testResult.status === 'failed') this.failedTests++;
+      else if (testResult.status === 'skipped') this.skippedTests++;
 
       if (!this.config.batchSubmit) {
         await this.client.submitResult(runInfo.runId, testResult);
@@ -660,20 +719,33 @@ class AppliqationReporter {
 
   /** @private */
   mapStatus(status) {
+    // Playwright's `interrupted` means the entire test run was aborted
+    // (Ctrl+C, CI timeout, OOM). Mapping it to 'skipped' would mask
+    // failures and inflate the skipped counter — surface as failed instead
+    // so the run summary honestly reflects an aborted execution.
+    //
+    // Values MUST match PayloadBuilder.validateAutomationResultPayload's
+    // ALLOWED_STATUSES ({passed, failed, skipped}, lowercase past-tense —
+    // mirrors the backend's InputValidator). This reporter's own mapped
+    // value is what ultimately reaches that validator (via trackResult →
+    // submitBatch → PayloadBuilder.buildResultPayload → normalizeStatus,
+    // which passes an already-recognized lowercase value straight through),
+    // so drifting from that vocabulary here silently fails every submission.
     const statusMap = {
-      'passed': 'Pass',
-      'failed': 'Fail',
-      'timedOut': 'Fail',
-      'skipped': 'Skipped',
-      'interrupted': 'Skipped'
+      'passed': 'passed',
+      'failed': 'failed',
+      'timedOut': 'failed',
+      'skipped': 'skipped',
+      'interrupted': 'failed'
     };
-    return statusMap[status] || 'Fail';
+    return statusMap[status] || 'failed';
   }
 
   /** @private */
   buildComment(test, result) {
     const parts = [];
     if (result.duration) parts.push(`Duration: ${(result.duration / 1000).toFixed(2)}s`);
+    if (result.status === 'interrupted') parts.push('Test run interrupted');
     if (result.error) {
       const errorMsg = result.error.message || result.error.toString();
       parts.push(`Error: ${errorMsg.substring(0, 500)}`);
@@ -868,6 +940,82 @@ class AppliqationReporter {
     return Array.from(projectNames);
   }
 
+  /**
+   * Run C5 scope validation against the Playwright suite. Mutates
+   * `this.config` if the user chooses the 'adhoc' strategy (clears
+   * scenarioId/testSetId so subsequent run creation is ad-hoc).
+   * Sets `this.scopeFilterEnabled` and `this.inScopeUuids` for the
+   * 'filter' strategy so `onTestEnd` can drop out-of-scope results.
+   *
+   * @private
+   * @param {Object} suite - Playwright Suite from onBegin
+   * @returns {Promise<{cancelled: boolean}>}
+   */
+  async _runScopeValidation(suite, preExtractedUuids) {
+    const validator = new ScopeValidator({
+      scenarioId: this.config.scenarioId,
+      testSetId: this.config.testSetId,
+      strategy: this.config.onScopeMismatch,
+      fetchTestSetScope: this._fetchTestSetScope.bind(this)
+    });
+
+    if (!validator.isScoped()) {
+      return { cancelled: false };
+    }
+
+    const uuids = preExtractedUuids || validator.extractUuidsFromSuite(suite);
+    const validation = await validator.validate(uuids);
+    if (!validation.hasMismatch) {
+      return { cancelled: false };
+    }
+
+    const outcome = await validator.resolve(validation);
+    // Surface the resolution message at WARN so it's visible at default
+    // log level — this is operational signal, not noise.
+    logger.warn(outcome.message);
+
+    if (outcome.action === ScopeValidator.STRATEGY.CANCEL) {
+      // Set non-zero exit code so CI fails. Don't throw — we still want
+      // Playwright to finish executing tests locally for the user's logs.
+      process.exitCode = 1;
+      return { cancelled: true };
+    }
+
+    if (outcome.overrideToAdhoc) {
+      // Drop the configured scope so createRun produces an ad-hoc run.
+      delete this.config.scenarioId;
+      delete this.config.testSetId;
+      return { cancelled: false };
+    }
+
+    if (outcome.action === ScopeValidator.STRATEGY.FILTER) {
+      this.scopeFilterEnabled = true;
+      this.inScopeUuids = new Set(outcome.submitUuids);
+    }
+
+    return { cancelled: false };
+  }
+
+  /**
+   * Resolve a testSet's UUID membership. Requires a backend endpoint
+   * (GET /api/automation/testset/{id}/scope) that returns the UUID list.
+   * Until that endpoint ships, this throws — ScopeValidator catches and
+   * gracefully degrades to skip testset validation with a warning.
+   *
+   * @private
+   * @param {number} testSetId
+   * @returns {Promise<Set<string>>}
+   */
+  async _fetchTestSetScope(testSetId) {
+    if (!this.client || typeof this.client.fetchTestSetScope !== 'function') {
+      throw new Error(
+        'TestSet scope endpoint not yet available in AppliqationClient — '
+        + 'testset scope validation will be skipped'
+      );
+    }
+    return this.client.fetchTestSetScope(testSetId);
+  }
+
   /**
    * Auto-detect browser versions
    * @private

package/src/services/OrphanTestService.js

@@ -44,10 +44,29 @@ class OrphanTestService {
 
       return response;
     } catch (error) {
-      logger.error('Failed to log orphan tests', {
+      // S5 — as of appq_automation_api.routing.yml, /api/automation/
+      // orphan-tests is not a registered route. Backend implementation
+      // is pending. Until it lands, a 404 here is expected — demote to
+      // debug so it doesn't spam customer CI logs. Orphans are still
+      // tracked locally and shown in the reporter's end-of-run summary.
+      const status = error.status
+        || error.response?.status
+        || error.details?.status
+        || null;
+
+      if (status === 404) {
+        logger.debug('Orphan tests endpoint not available on backend; skipping network log', {
+          runId,
+          count: orphanTests.length
+        });
+        return { success: false, notImplemented: true, count: orphanTests.length };
+      }
+
+      logger.warn('Failed to log orphan tests', {
         error: error.message,
         runId,
-        count: orphanTests.length
+        count: orphanTests.length,
+        status
       });
       throw new Error(`Failed to log orphan tests: ${error.message}`);
     }