@appliqation/automation-sdk 2.8.0 → 2.9.0

package/README.md

@@ -125,23 +125,30 @@ Both forms accept the usual Playwright flags (`--project`, `--headed`,
 
 ## Scoping runs: Scenario or Test Set
 
-Add `scenarioId` to link a run to one scenario, or `testSetId` to link it
-to a Test Set — the grouping you'd use for a smoke, sanity, or regression
-suite that spans multiple scenarios. They're mutually exclusive;
-`scenarioId` wins if both are set.
+Link a run to one scenario (`scenarioId`) or to a Test Set (`testSetId`) —
+the grouping you'd use for a smoke, sanity, or regression suite that spans
+multiple scenarios. They're mutually exclusive; `scenarioId` wins if both
+are set. No config changes needed — set it per-invocation:
+
+```bash
+# CLI flag
+npx playwright test -- --appq --appq-scenario-id=1154
+npx playwright test -- --appq --appq-testset-id=2489
+
+# Env var — same effect
+APPLIQATION_SCENARIO_ID=1154 APPQ_ENABLED=1 npx playwright test
+APPLIQATION_TEST_SET_ID=2489 APPQ_ENABLED=1 npx playwright test
+```
+
+Or hardcode a default in `playwright.config.js` if a project always runs
+against the same scope — CLI flag and env var still override it per-run:
 
 ```javascript
-const appliqationConfig = {
+[AppliqationReporter, {
   apiKey: process.env.APPLIQATION_API_KEY,
   projectKey: process.env.APPLIQATION_PROJECT_KEY,
-
-  scenarioId: parseInt(process.env.APPLIQATION_SCENARIO_ID) || undefined,
-  // testSetId: parseInt(process.env.MY_SMOKE_TEST_SET_ID) || undefined,
-
-  // How to resolve a run whose executed tests fall outside the configured
-  // scope — 'cancel' (default, fail-safe), 'adhoc', or 'filter'.
-  onScopeMismatch: process.env.APPLIQATION_ON_SCOPE_MISMATCH,
-};
+  testSetId: 2489,
+}]
 ```
 
 Before the suite runs, the reporter checks every UUID it's about to execute
@@ -156,7 +163,9 @@ write results into the wrong run:
 | `filter` | Submits only the in-scope results; out-of-scope tests still execute but aren't uploaded. |
 
 Outside CI, with an interactive terminal, you're prompted to choose one of
-the three before the run starts.
+the three before the run starts. Set the strategy the same way as the
+scope itself — `--appq-on-scope-mismatch=filter` or
+`APPLIQATION_ON_SCOPE_MISMATCH=filter`.
 
 ## Authenticated apps (gated SUTs)
 
@@ -315,24 +324,33 @@ CircleCI, Jenkins, or any other runner.
 
 ### Environment variables
 
-| Variable | Required | Description |
-|---|---|---|
-| `APPLIQATION_API_KEY` | Yes | API key from Project Settings → API Keys |
-| `APPLIQATION_PROJECT_KEY` | Yes | Project key from Project Settings |
-| `TEST_APP_URL` | Recommended | Your own convention for Playwright's `baseURL` — read it in your `playwright.config.js`, the SDK doesn't read it directly |
-| `APPLIQATION_ENVIRONMENT` | No | Environment label for the run (e.g. `Staging`) — also settable via `--appq-env` |
-| `APPLIQATION_RUN_TITLE` | No | Custom run title — also settable via `--appq_run_title` |
-| `APPLIQATION_RUN_ID` | No | Reuse an existing run instead of creating one (TDD iteration mode) |
-| `APPQ_ENABLED` | No | `1`/`true` enables reporting — alternative to the `--appq` CLI flag |
-| `APPLIQATION_ON_SCOPE_MISMATCH` | No | `cancel` (default) / `adhoc` / `filter` — see [Scoping runs](#scoping-runs-scenario-or-test-set) |
-| `APPLIQATION_AUTO_TAG_ENABLED` | No | `false` disables auto-tagging (default: enabled). The tag name itself (`Appq_Auto`) is fixed, not configurable — see [Auto-tagging](#auto-tagging-test-cases) |
-| `APPLIQATION_INSECURE` | No | Set to `allow` to permit `rejectUnauthorized: false` in CI/production — see [Troubleshooting](#tls-certificate-errors) |
-| `LOG_LEVEL` | No | `ERROR` / `WARN` / `INFO` (default) / `DEBUG` |
-
-`scenarioId`/`testSetId` and orphan-run handling (below) are reporter
-constructor options, not environment variables — read your own env var of
-choice into them in `playwright.config.js`, the same way the examples above
-do for `scenarioId`.
+Every row below (except `APPLIQATION_API_KEY`/`APPLIQATION_PROJECT_KEY`,
+which have no CLI equivalent — secrets don't belong in shell history or
+`ps` output) also has a matching `--appq-*` CLI flag, at
+**CLI flag > env var > `playwright.config.js` value > built-in default**
+precedence, applied consistently across every one of them:
+
+| Variable | CLI flag | Required | Description |
+|---|---|---|---|
+| `APPLIQATION_API_KEY` | — | Yes | API key from Project Settings → API Keys |
+| `APPLIQATION_PROJECT_KEY` | — | Yes | Project key from Project Settings |
+| `TEST_APP_URL` | — | Recommended | Your own convention for Playwright's `baseURL` — read it in your `playwright.config.js`, the SDK doesn't read it directly |
+| `APPLIQATION_ENVIRONMENT` | `--appq-env` | No | Environment label for the run (e.g. `Staging`) |
+| `APPLIQATION_RUN_TITLE` | `--appq-title` / `--appq_run_title` | No | Custom run title |
+| `APPLIQATION_RUN_ID` | `--appq-run-id` / `--appq_run_id` | No | Reuse an existing run instead of creating one (TDD iteration mode) |
+| `APPQ_ENABLED` | `--appq` | No | `1`/`true` enables reporting |
+| `APPLIQATION_SCENARIO_ID` | `--appq-scenario-id` / `--appq_scenario_id` | No | Link the run to one scenario — see [Scoping runs](#scoping-runs-scenario-or-test-set) |
+| `APPLIQATION_TEST_SET_ID` | `--appq-testset-id` / `--appq_testset_id` | No | Link the run to a Test Set (scenario wins if both are set) |
+| `APPLIQATION_ON_SCOPE_MISMATCH` | `--appq-on-scope-mismatch` / `--appq_on_scope_mismatch` | No | `cancel` (default) / `adhoc` / `filter` |
+| `LOG_LEVEL` | `--appq-log-level` / `--appq_log_level` | No | `ERROR` / `WARN` / `INFO` (default) / `DEBUG` |
+| `APPLIQATION_AUTO_TAG_ENABLED` | — | No | `false` disables auto-tagging (default: enabled). The tag name itself (`Appq_Auto`) is fixed, not configurable — see [Auto-tagging](#auto-tagging-test-cases) |
+| `APPLIQATION_INSECURE` | — | No | Set to `allow` to permit `rejectUnauthorized: false` in CI/production — see [Troubleshooting](#tls-certificate-errors) |
+
+`autoCreateRun`, `batchSubmit`, `batchSize`, `logOrphans`,
+`deleteOrphanOnlyRuns`, `failOnOrphanOnlyRuns`, and `rejectUnauthorized`
+are pipeline *policy*, not per-run identity — deliberately config-only,
+not CLI/env-settable, so a stray shell variable can't silently change how
+a project's runs behave.
 
 ### Reporter options (`playwright.config.js`)
 
@@ -340,16 +358,15 @@ do for `scenarioId`.
 ['@appliqation/automation-sdk/playwright/reporter', {
   apiKey: process.env.APPLIQATION_API_KEY,
   projectKey: process.env.APPLIQATION_PROJECT_KEY,
-  environment: process.env.APPLIQATION_ENVIRONMENT,
 
-  scenarioId: parseInt(process.env.APPLIQATION_SCENARIO_ID) || undefined,
-  onScopeMismatch: process.env.APPLIQATION_ON_SCOPE_MISMATCH,
+  // Everything below has a CLI flag + env var too (table above) — set
+  // here only if a project should default to a fixed value.
+  environment: 'Local',
 
   autoCreateRun: true,     // default: true
   batchSubmit: true,       // default: true
   batchSize: 50,           // results per batch
   logOrphans: true,        // log tests with no mapped UUID
-  logLevel: 'INFO',
 
   deleteOrphanOnlyRuns: true,  // delete a run if every test in it was orphaned
   failOnOrphanOnlyRuns: true,  // exit 1 in that case, so CI catches it

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@appliqation/automation-sdk",
-  "version": "2.8.0",
+  "version": "2.9.0",
   "description": "Appliqation Automation SDK for Playwright — API-key auth, scope-validated result reporting, portable storageState (setupAuth), and customer-defined login flows (defineLogin) for gated apps",
   "main": "src/index.js",
   "types": "src/index.d.ts",

package/src/index.d.ts

@@ -1,6 +1,6 @@
 /**
  * TypeScript definitions for @appliqation/automation-sdk
- * @version 2.8.0
+ * @version 2.9.0
  */
 
 /**
@@ -73,10 +73,19 @@ export interface AppliqationConfig {
    */
   password?: string;
 
-  /** Scenario ID (optional, 0 for generic automation runs) */
+  /**
+   * Scenario ID (optional, 0 for generic automation runs). Also settable
+   * via `--appq-scenario-id`/`--appq_scenario_id` CLI flag or
+   * `APPLIQATION_SCENARIO_ID` env var — CLI > env var > this config value.
+   */
   scenarioId?: number;
 
-  /** Test set ID (alternative to scenarioId) */
+  /**
+   * Test set ID (alternative to scenarioId — scenarioId wins if both are
+   * set). Also settable via `--appq-testset-id`/`--appq_testset_id` CLI
+   * flag or `APPLIQATION_TEST_SET_ID` env var — CLI > env var > this
+   * config value.
+   */
   testSetId?: number;
 
   /** Environment name (e.g., 'Local', 'Development', 'Staging', 'Production') */

package/src/reporters/playwright/AppliqationReporter.js

@@ -58,6 +58,74 @@ function getRunIdFromCli() {
   return null;
 }
 
+/**
+ * Extract scenario ID from CLI arguments.
+ * Supports: --appq-scenario-id=1154 or --appq_scenario_id=1154
+ * @returns {string|null}
+ */
+function getScenarioIdFromCli() {
+  const argv = process.argv || [];
+  for (const arg of argv) {
+    if (arg.startsWith('--appq-scenario-id=') || arg.startsWith('--appq_scenario_id=')) {
+      let value = arg.substring(arg.indexOf('=') + 1);
+      value = value.replace(/^["']|["']$/g, '');
+      return value || null;
+    }
+  }
+  return null;
+}
+
+/**
+ * Extract test set ID from CLI arguments.
+ * Supports: --appq-testset-id=2489 or --appq_testset_id=2489
+ * @returns {string|null}
+ */
+function getTestSetIdFromCli() {
+  const argv = process.argv || [];
+  for (const arg of argv) {
+    if (arg.startsWith('--appq-testset-id=') || arg.startsWith('--appq_testset_id=')) {
+      let value = arg.substring(arg.indexOf('=') + 1);
+      value = value.replace(/^["']|["']$/g, '');
+      return value || null;
+    }
+  }
+  return null;
+}
+
+/**
+ * Extract scope-mismatch strategy from CLI arguments.
+ * Supports: --appq-on-scope-mismatch=filter or --appq_on_scope_mismatch=filter
+ * @returns {string|null}
+ */
+function getOnScopeMismatchFromCli() {
+  const argv = process.argv || [];
+  for (const arg of argv) {
+    if (arg.startsWith('--appq-on-scope-mismatch=') || arg.startsWith('--appq_on_scope_mismatch=')) {
+      let value = arg.substring(arg.indexOf('=') + 1);
+      value = value.replace(/^["']|["']$/g, '');
+      return value || null;
+    }
+  }
+  return null;
+}
+
+/**
+ * Extract log level from CLI arguments.
+ * Supports: --appq-log-level=DEBUG or --appq_log_level=DEBUG
+ * @returns {string|null}
+ */
+function getLogLevelFromCli() {
+  const argv = process.argv || [];
+  for (const arg of argv) {
+    if (arg.startsWith('--appq-log-level=') || arg.startsWith('--appq_log_level=')) {
+      let value = arg.substring(arg.indexOf('=') + 1);
+      value = value.replace(/^["']|["']$/g, '');
+      return value || null;
+    }
+  }
+  return null;
+}
+
 /**
  * Appliqation Reporter for Playwright
  *
@@ -72,6 +140,25 @@ function getRunIdFromCli() {
  * ]
  *
  * // Run: npx playwright test -- --appq --appq-env=Stage
+ *
+ * // Scenario/test-set scoping — CLI flag > env var > config, same
+ * // precedence as environment/title/runId above. scenarioId wins if
+ * // both are set.
+ * // Run: npx playwright test -- --appq --appq-scenario-id=1154
+ * // Run: npx playwright test -- --appq --appq-testset-id=2489
+ * // Or:  APPLIQATION_SCENARIO_ID=1154 npx playwright test -- --appq
+ * // Or:  APPLIQATION_TEST_SET_ID=2489 npx playwright test -- --appq
+ *
+ * // Scope-mismatch strategy and log level — same CLI > env var > config
+ * // precedence throughout.
+ * // Run: npx playwright test -- --appq --appq-on-scope-mismatch=filter
+ * // Run: npx playwright test -- --appq --appq-log-level=DEBUG
+ * // Or:  APPLIQATION_ON_SCOPE_MISMATCH=filter npx playwright test -- --appq
+ * // Or:  LOG_LEVEL=DEBUG npx playwright test -- --appq
+ *
+ * // apiKey/projectKey/baseUrl are env-var + config only, deliberately no
+ * // CLI flag: secrets don't belong in shell history/process listings,
+ * // and baseUrl is internal/dev-only (see docs/SETUP.md).
  */
 class AppliqationReporter {
   constructor(config = {}) {
@@ -85,7 +172,6 @@ class AppliqationReporter {
       logOrphans: true,
       batchSubmit: true,
       batchSize: 50,
-      logLevel: 'info',
       deleteOrphanOnlyRuns: config.deleteOrphanOnlyRuns !== false,
       failOnOrphanOnlyRuns: config.failOnOrphanOnlyRuns !== false,
       ...config
@@ -108,6 +194,39 @@ class AppliqationReporter {
     // Check for existing run ID (TDD iteration mode)
     this.existingRunId = this.config.runId || getRunIdFromCli() || process.env.APPLIQATION_RUN_ID || null;
 
+    // Resolve scenarioId: CLI --appq-scenario-id > env var > config
+    if (!this.config.scenarioId) {
+      const cliScenarioId = getScenarioIdFromCli();
+      this.config.scenarioId = parseInt(cliScenarioId, 10)
+        || parseInt(process.env.APPLIQATION_SCENARIO_ID, 10)
+        || undefined;
+    }
+
+    // Resolve testSetId: CLI --appq-testset-id > env var > config
+    if (!this.config.testSetId) {
+      const cliTestSetId = getTestSetIdFromCli();
+      this.config.testSetId = parseInt(cliTestSetId, 10)
+        || parseInt(process.env.APPLIQATION_TEST_SET_ID, 10)
+        || undefined;
+    }
+
+    // Resolve onScopeMismatch: CLI --appq-on-scope-mismatch > config.
+    // The env var (APPLIQATION_ON_SCOPE_MISMATCH) is read as a further
+    // fallback inside ScopeValidator itself — not duplicated here.
+    if (!this.config.onScopeMismatch) {
+      this.config.onScopeMismatch = getOnScopeMismatchFromCli() || undefined;
+    }
+
+    // Resolve logLevel: CLI --appq-log-level > env var > config > default.
+    // Must happen before logger.setLevel() is called below (and again in
+    // the AppliqationClient constructor) — LOG_LEVEL is otherwise read
+    // once at logger module load and then silently overwritten by
+    // whatever this resolves to.
+    if (!this.config.logLevel) {
+      const cliLogLevel = getLogLevelFromCli();
+      this.config.logLevel = cliLogLevel || process.env.LOG_LEVEL || 'info';
+    }
+
     // If disabled, stop here — true no-op
     if (!this.appqEnabled) {
       this.client = null;
@@ -178,6 +297,16 @@ class AppliqationReporter {
     this.executionEndTime = null;
     this.playwrightOutputDir = null;
 
+    // Playwright does not reliably await a reporter's onTestEnd() return
+    // value before calling onEnd() — only onEnd()'s own promise is
+    // guaranteed to be awaited. onTestEnd() can itself wait (via
+    // waitForRunMatrix) for a still-in-flight run creation, so without
+    // this tracking, onEnd() can start reading resultsByRun/orphansByRun
+    // before onTestEnd has finished populating them, silently submitting
+    // nothing. Every onTestEnd() invocation pushes its own promise here;
+    // onEnd() awaits them all first.
+    this.pendingTestEndPromises = [];
+
     // C5 — scope validation state
     this.scopeFilterEnabled = false;  // true when strategy === 'filter'
     this.inScopeUuids = null;          // Set<string>, null = no filter
@@ -405,11 +534,23 @@ class AppliqationReporter {
   }
 
   /**
-   * Called after a test completes — non-blocking
+   * Called after a test completes.
+   *
+   * Playwright does not reliably await this method's returned promise
+   * before calling onEnd() — so the actual work (which can wait on a
+   * still-in-flight run creation via waitForRunMatrix) runs in
+   * _processTestEnd(), and this wrapper only tracks that promise in
+   * pendingTestEndPromises so onEnd() can await it explicitly.
    */
-  async onTestEnd(test, result) {
+  onTestEnd(test, result) {
     if (!this.appqEnabled) return;
+    const promise = this._processTestEnd(test, result);
+    this.pendingTestEndPromises.push(promise);
+    return promise;
+  }
 
+  /** @private */
+  async _processTestEnd(test, result) {
     try {
       const uuid = UuidExtractor.extractFromAnnotations(result.annotations || []) || UuidExtractor.extractFromTest(test);
 
@@ -624,6 +765,14 @@ class AppliqationReporter {
     logger.info('Test run complete.');
 
     try {
+      // See onTestEnd()'s doc comment — Playwright doesn't reliably await
+      // per-test reporter hooks, so results/orphans tracked by still-
+      // pending onTestEnd() calls (e.g. waiting on a slow run creation)
+      // would otherwise be silently missing from what we're about to read.
+      if (this.pendingTestEndPromises.length > 0) {
+        await Promise.allSettled(this.pendingTestEndPromises);
+      }
+
       const orphanDeletedRuns = await this.handleOrphanOnlyRuns();
 
       if (this.appqEnabled) {

package/src/reporters/playwright/index.d.ts

@@ -1,6 +1,6 @@
 /**
  * TypeScript definitions for @appliqation/automation-sdk/playwright
- * @version 2.1.0
+ * @version 2.9.0
  */
 
 import type {
@@ -22,7 +22,10 @@ import type { AppliqationConfig } from '../../index';
  * Configuration for the Appliqation Playwright Reporter
  */
 export interface AppliqationReporterConfig extends AppliqationConfig {
-  /** Log level for reporter (default: 'INFO') */
+  /**
+   * Log level for reporter (default: 'info'). CLI `--appq-log-level` >
+   * `LOG_LEVEL` env var > this config value > default.
+   */
   logLevel?: 'ERROR' | 'WARN' | 'INFO' | 'DEBUG';
 
   /** Log orphan tests without UUID mappings (default: true) */
@@ -45,16 +48,24 @@ export interface AppliqationReporterConfig extends AppliqationConfig {
  *   reporter: [
  *     ['list'],
  *     [AppliqationReporter, {
- *       baseUrl: process.env.APPLIQATION_BASE_URL,
+ *       // baseUrl intentionally omitted — internal/dev-only, defaults to
+ *       // the real Appliqation instance. Don't set it.
  *       apiKey: process.env.APPLIQATION_API_KEY,
  *       projectKey: process.env.APPLIQATION_PROJECT_KEY,
- *       scenarioId: 0,
  *       environment: 'Local',
- *       logLevel: 'INFO'
  *     }]
  *   ]
  * };
  * ```
+ *
+ * @remarks
+ * `scenarioId`, `testSetId`, `onScopeMismatch`, and `logLevel` can all be
+ * set without touching this config object at all — CLI flag > env var >
+ * config value, in that order:
+ * - `--appq-scenario-id=1154` / `APPLIQATION_SCENARIO_ID=1154`
+ * - `--appq-testset-id=2489` / `APPLIQATION_TEST_SET_ID=2489`
+ * - `--appq-on-scope-mismatch=filter` / `APPLIQATION_ON_SCOPE_MISMATCH=filter`
+ * - `--appq-log-level=DEBUG` / `LOG_LEVEL=DEBUG`
  */
 export class AppliqationReporter implements Reporter {
   /**