@koenvanbelle/cypress-soft-assertions 2.3.2 → 2.4.3

package/README.md

@@ -130,6 +130,23 @@ describe('Test Suite', () => {
 });
 ```
 
+### `soft_it.expectFailure`
+
+Use `soft_it.expectFailure()` for browser behavior specs that intentionally trigger
+soft failures but should still finish green overall.
+
+```typescript
+soft_it.expectFailure('captures a known soft failure without failing the spec', () => {
+  cy.get('#title').should('have.text', 'Wrong Title');
+  cy.get('#status').should('have.text', 'Ready');
+});
+```
+
+This mode still verifies that a final `SoftAssertionError` was produced. If the
+test completes without any soft failure, the test fails.
+
+`soft_it.expectFailure.only(...)` is also available.
+
 ## How it works
 
 1. `soft_it()` patches Chai's assertion mechanism for the duration of the test.
@@ -158,7 +175,7 @@ SOFT ASSERTION FAILURES (3 failed):
 - **Retriable assertions are retried**: `.should()` and `.and()` assertions are retried by Cypress before being captured as soft failures. Assertions that eventually pass are not reported.
 - **Non-assertion errors still stop execution**: Network errors, visit timeouts, and other command errors are captured as soft failures but may prevent subsequent commands from running.
 - **State resets between tests**: Each `soft_it()` test starts with a clean slate — failures from one test never leak into another.
-- **Timeout-aware retries**: The plugin respects Cypress's `defaultCommandTimeout` and per-command `{ timeout }` overrides. Retriable assertions (`.should()`, `.and()`) are retried for up to 75% of the effective timeout before being captured as soft failures. For slow applications, increase the timeout to give assertions more time to pass:
+- **Timeout-aware retries**: The plugin respects Cypress's `defaultCommandTimeout` and per-command `{ timeout }` overrides. Retriable assertions (`.should()`, `.and()`) are retried until just before the effective timeout expires, using `max(timeout - 100ms, timeout * 0.9)` as the swallow threshold. For slow applications, increase the timeout to give assertions more time to pass:
   ```typescript
   // Global: set in cypress.config.ts
   e2e: { defaultCommandTimeout: 10000 }
@@ -249,36 +266,6 @@ soft_it('works with automatic translation checks', () => {
 - Both plugins register hooks/events, but they can run together safely with the setup above.
 - Keep translation-checker in non-invasive mode (`failOnError: false` in auto mode) so functional assertions remain the source of test pass/fail behavior.
 
-## Force-Fail Hook (strict mode)
-
-If your project (or another plugin) mutates test state in hooks (for example in `test:after:run`) and soft-failed tests appear as passed, enable strict mode per test:
-
-```typescript
-soft_it.strict('fails hard when soft assertions are captured', () => {
-  cy.visit('/');
-  cy.get('#title').should('have.text', 'Wrong Title');
-  cy.get('#status').should('have.text', 'Ready');
-});
-```
-
-You can also use `soft_it.strict.only(...)` and `soft_it.strict.skip(...)`.
-
-To force strict mode for all soft tests in a run, use env flags:
-
-```bash
-cypress run --env softAssertForceFail=true
-```
-
-You can also use:
-
-```bash
-cypress run --env SOFT_ASSERT_FORCE_FAIL=true
-```
-
-When strict mode is enabled (per test or via env), the plugin throws the final `SoftAssertionError` from `afterEach` on the last attempt, preventing downstream hook-based recovery from turning the test green.
-
-Note: strict mode may abort remaining tests in the same suite after the first forced soft failure (standard Cypress hook-failure behavior).
-
 ## License
 
 MIT

package/dist/index.d.ts

@@ -40,14 +40,12 @@ declare global {
          */
         function only(title: string, fn: Mocha.Func | Mocha.AsyncFunc): Mocha.Test;
         /**
-         * Run this soft test in strict mode (force unrecoverable failure)
+         * Run a soft test that is expected to finish with a SoftAssertionError.
          */
-        function strict(title: string, fn: Mocha.Func | Mocha.AsyncFunc): Mocha.Test;
-        namespace strict {
-            /** Run only this strict soft test */
+        function expectFailure(title: string, fn: Mocha.Func | Mocha.AsyncFunc): Mocha.Test;
+        namespace expectFailure {
+            /** Run only this expected-failure soft test */
             function only(title: string, fn: Mocha.Func | Mocha.AsyncFunc): Mocha.Test;
-            /** Skip this strict soft test */
-            function skip(title: string, fn: Mocha.Func | Mocha.AsyncFunc): void;
         }
         /**
          * Skip this test (like it.skip)

package/dist/index.js

@@ -20,7 +20,7 @@ let softAssertionErrors = [];
 let retryAssertionFailures = new Map();
 let retryFirstSeen = new Map();
 let isInSoftTest = false;
-let forceFailCurrentSoftTest = false;
+let expectSoftFailureCurrentSoftTest = false;
 let activeFailHandler = null;
 let originalChaiAssert = null;
 // ---------------------------------------------------------------------------
@@ -42,26 +42,6 @@ function getEffectiveTimeout() {
         return 4000;
     }
 }
-function isTruthy(value) {
-    if (typeof value === 'boolean')
-        return value;
-    if (typeof value === 'number')
-        return value !== 0;
-    const normalized = String(value !== null && value !== void 0 ? value : '').trim().toLowerCase();
-    return normalized === '1' || normalized === 'true' || normalized === 'yes' || normalized === 'on';
-}
-function shouldForceFailSoftAssertions() {
-    var _a;
-    try {
-        const env = (_a = Cypress.env) === null || _a === void 0 ? void 0 : _a.bind(Cypress);
-        if (!env)
-            return false;
-        return isTruthy(env('softAssertForceFail')) || isTruthy(env('SOFT_ASSERT_FORCE_FAIL'));
-    }
-    catch (_b) {
-        return false;
-    }
-}
 // ---------------------------------------------------------------------------
 // Internal helpers (Cypress-dependent — not unit-testable in isolation)
 // ---------------------------------------------------------------------------
@@ -181,6 +161,12 @@ function setupSoftAssertions() {
             // Final aggregated error must propagate to fail the test.
             if (String((error === null || error === void 0 ? void 0 : error.name) || '') === 'SoftAssertionError')
                 throw error;
+            const runnable = cy.state('runnable');
+            const retryStatusInfo = getRetryStatusInfo(runnable);
+            if (retryStatusInfo.shouldAttemptsContinue) {
+                abortSoftTest();
+                throw error;
+            }
             // Check if this is an assertion error that was already handled by
             // patchedAssertionAssert (swallowed after timeout). In that case
             // the fail handler should NOT fire. But for non-assertion command
@@ -229,28 +215,70 @@ function buildSoftAssertionError() {
 }
 function finalizeSoftTest() {
     isInSoftTest = false;
-    forceFailCurrentSoftTest = false;
+    expectSoftFailureCurrentSoftTest = false;
     restoreAssertions();
     return buildSoftAssertionError();
 }
 function abortSoftTest() {
     isInSoftTest = false;
-    forceFailCurrentSoftTest = false;
+    expectSoftFailureCurrentSoftTest = false;
     restoreAssertions();
     retryAssertionFailures.clear();
     retryFirstSeen.clear();
 }
+function getRetryStatusInfo(test) {
+    var _a;
+    const currentRetry = typeof (test === null || test === void 0 ? void 0 : test.currentRetry) === 'function'
+        ? test.currentRetry()
+        : typeof Cypress.currentRetry === 'number'
+            ? Cypress.currentRetry
+            : 0;
+    const maxRetries = typeof (test === null || test === void 0 ? void 0 : test.retries) === 'function'
+        ? test.retries()
+        : typeof (test === null || test === void 0 ? void 0 : test._retries) === 'number'
+            ? test._retries
+            : typeof Cypress.getTestRetries === 'function'
+                ? (_a = Cypress.getTestRetries()) !== null && _a !== void 0 ? _a : 0
+                : 0;
+    return {
+        attempts: currentRetry + 1,
+        shouldAttemptsContinue: currentRetry < maxRetries,
+    };
+}
+function finalizeSoftTestInQueue(expectsSoftFailure) {
+    return cy.then(() => {
+        if (!isInSoftTest)
+            return;
+        const finalError = finalizeSoftTest();
+        if (expectsSoftFailure) {
+            if (!finalError) {
+                throw new Error('Expected SoftAssertionError but soft_it.expectFailure test completed without one.');
+            }
+            return;
+        }
+        if (finalError) {
+            throw finalError;
+        }
+    });
+}
 function createSoftIt(baseIt, options) {
     return function (title, fn) {
         return baseIt(title, function () {
             isInSoftTest = true;
-            forceFailCurrentSoftTest = Boolean(options === null || options === void 0 ? void 0 : options.strict) || shouldForceFailSoftAssertions();
+            expectSoftFailureCurrentSoftTest = Boolean(options === null || options === void 0 ? void 0 : options.expectFailure);
             softAssertionErrors = [];
             retryAssertionFailures.clear();
             retryFirstSeen.clear();
             setupSoftAssertions();
             try {
-                return fn.call(this);
+                const result = fn.call(this);
+                if (result && typeof result.then === 'function' && !Cypress.isCy(result)) {
+                    return Cypress.Promise.resolve(result).then(() => {
+                        finalizeSoftTestInQueue(expectSoftFailureCurrentSoftTest);
+                    });
+                }
+                finalizeSoftTestInQueue(expectSoftFailureCurrentSoftTest);
+                return result;
             }
             catch (error) {
                 abortSoftTest();
@@ -282,57 +310,54 @@ globalThis.soft_it = createSoftIt(it);
  */
 globalThis.soft_it.only = createSoftIt(it.only);
 /**
- * soft_it.strict - Run this soft test in strict mode.
- *
- * Strict mode throws the final SoftAssertionError from afterEach on the
- * last attempt, making the failure unrecoverable by downstream hooks.
+ * soft_it.expectFailure - Run a soft test that is expected to aggregate
+ * into a final SoftAssertionError without failing the enclosing behavior spec.
  */
-globalThis.soft_it.strict = createSoftIt(it, { strict: true });
+globalThis.soft_it.expectFailure = createSoftIt(it, { expectFailure: true });
 /**
- * soft_it.strict.only - Run only this strict soft test
+ * soft_it.expectFailure.only - Run only this expected-failure soft test.
  */
-globalThis.soft_it.strict.only = createSoftIt(it.only, { strict: true });
+globalThis.soft_it.expectFailure.only = createSoftIt(it.only, { expectFailure: true });
 /**
  * soft_it.skip - Skip this soft test
  */
 globalThis.soft_it.skip = it.skip;
-/**
- * soft_it.strict.skip - Skip this strict soft test
- */
-globalThis.soft_it.strict.skip = it.skip;
 // Global afterEach hook: finalize soft assertions after each test.
 // Registered at the root level so it applies to all suites.
 // For non-soft tests (isInSoftTest === false), this is a no-op.
 afterEach(function () {
-    var _a, _b, _c;
     if (!isInSoftTest)
         return;
+    const expectsSoftFailure = expectSoftFailureCurrentSoftTest;
     const finalError = finalizeSoftTest();
+    if (expectsSoftFailure) {
+        if (!finalError) {
+            throw new Error('Expected SoftAssertionError but soft_it.expectFailure test completed without one.');
+        }
+        return;
+    }
     if (finalError) {
         const test = this.currentTest;
-        const currentRetry = (_a = test === null || test === void 0 ? void 0 : test._currentRetry) !== null && _a !== void 0 ? _a : 0;
-        const maxRetries = (_b = test === null || test === void 0 ? void 0 : test._retries) !== null && _b !== void 0 ? _b : 0;
-        if (currentRetry < maxRetries) {
-            // Intermediate retry attempt — throw so Cypress triggers the next retry.
-            // Cypress's retry machinery intercepts hook failures during non-final
-            // attempts and does NOT abort the suite, so this is safe here.
-            throw finalError;
-        }
-        // Last (or only) attempt — use runner.fail() to mark the TEST as failed
-        // rather than the hook. Throwing from afterEach on the final attempt
-        // would skip remaining tests in the suite.
-        if (forceFailCurrentSoftTest) {
-            // Strict mode: force an unrecoverable hook failure.
-            // This guarantees a failed result even when other plugins mutate
-            // test state in test:after:run.
-            throw finalError;
-        }
-        const runner = (_c = Cypress.mocha) === null || _c === void 0 ? void 0 : _c.getRunner();
-        if (runner && test) {
-            runner.fail(test, finalError);
-        }
-        else {
-            throw finalError;
+        if (test) {
+            const retryStatusInfo = getRetryStatusInfo(test);
+            if (retryStatusInfo.shouldAttemptsContinue) {
+                throw finalError;
+            }
+            test.err = finalError;
+            test._cypressTestStatusInfo = {
+                outerStatus: 'failed',
+                shouldAttemptsContinue: retryStatusInfo.shouldAttemptsContinue,
+                attempts: retryStatusInfo.attempts,
+                strategy: 'detect-flake-and-pass-on-threshold',
+            };
+            const prevAttempts = Array.isArray(test.prevAttempts) ? test.prevAttempts : [];
+            if (!prevAttempts.some((attempt) => (attempt === null || attempt === void 0 ? void 0 : attempt.state) === 'failed' && (attempt === null || attempt === void 0 ? void 0 : attempt.err))) {
+                prevAttempts.unshift({
+                    state: 'failed',
+                    err: finalError,
+                });
+            }
+            test.prevAttempts = prevAttempts;
         }
     }
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@koenvanbelle/cypress-soft-assertions",
-  "version": "2.3.2",
+  "version": "2.4.3",
   "description": "A Cypress plugin that provides soft_it() for soft assertions - all assertions continue on failure and are reported together",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",