kempo-testing-framework 1.4.14 → 1.5.0

package/README.md

@@ -195,6 +195,21 @@ export default {
 ```
 
 
+### How `pass` and `fail` Behave
+
+- `pass(message)` marks the test as passed and `fail(message)` marks it as failed. A test that throws also fails.
+- **A failure is final.** Once `fail()` has been called, a later `pass()` is ignored, and the ignored call is written to the test's logs so the contradiction is visible.
+- `fail()` does not stop the test. Stop it with `return fail(...)` from the test function itself, or by throwing.
+
+Be careful with `return fail(...)` inside a nested callback such as a helper, a promise executor or a `.then`. It only leaves that callback, so the test carries on to whatever follows. Throw an error there instead, which fails the test and stops it:
+
+```javascript
+await withTempDir(async (dir) => {
+  if(!(await exists(dir))) throw new Error('directory was not created');
+});
+pass('directory created');
+```
+
 ## Running Tests
 
 ### CLI (Command-Line Interface)

package/llms.txt

@@ -28,7 +28,8 @@ export default {
   'test name': ({ pass, fail, log }) => {
     // log(msg)  — emit a log message
     // pass(msg) — mark test passed
-    // fail(msg) — mark test failed
+    // fail(msg) — mark test failed. Final: a later pass() is ignored. Does not stop the test;
+    //             return it from the test body, or throw, to stop (a throw also fails the test)
     pass('ok');
   },
   'async test': async ({ pass, fail, log }) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kempo-testing-framework",
-  "version": "1.4.14",
+  "version": "1.5.0",
   "description": "The Kempo Testing Framework is a simple testing framework built on the principle that code intended to be ran in the browser should be tested in the browser, code intended to be ran in Node should be tested in Node, and code intended to be ran in both should be tested in both. No mocking, no complexity—just testing.",
   "main": "index.js",
   "bin": {

package/src/runTests.js

@@ -63,6 +63,15 @@ export default async ({
         tests[name]({
           log,
           pass: message => {
+            /*
+              A failure is final. `return fail()` from inside a nested callback only leaves that callback,
+              so the test carries on to its own trailing pass(); letting that overwrite the failure reported
+              success for a test that had failed. The call is still logged so the contradiction is visible.
+            */
+            if(result.passed === false){
+              log(`pass() ignored because the test already failed: ${message}`, 'log', 2);
+              return;
+            }
             result.passed = true;
             log(message, 'pass', 2); // was 3; make visible at NORMAL
           },

package/tests/src/runTests.node-test.js

@@ -11,6 +11,8 @@ const mkSuite = () => ({
   }
 });
 
+const runOne = async (test) => (await runTests({ default: { subject: test } }, false, 0)).tests.subject;
+
 export default {
   'runs all tests and records logs': async ({ pass, fail, log }) => {
     try {
@@ -52,5 +54,65 @@ export default {
     } catch (e) {
       fail(e.stack || String(e));
     }
+  },
+  'a later pass() cannot overwrite an earlier fail()': async ({ pass, fail }) => {
+    const result = await runOne(({ pass, fail }) => {
+      fail('the check broke');
+      pass('but then it passed');
+    });
+    if(result.passed !== false) return fail(`Expected the test to fail, got passed=${result.passed}`);
+    if(result.logs.some(l => l.type === 'pass')) return fail('An ignored pass() must not be logged as a pass');
+    if(!result.logs.some(l => l.type === 'fail' && l.message === 'the check broke')) return fail('The failure message was not recorded');
+    if(!result.logs.some(l => l.message.includes('pass() ignored') && l.message.includes('but then it passed'))) return fail('The ignored pass() was not logged');
+    pass('fail() is final');
+  },
+  'return fail() from a nested callback is not masked by a trailing pass()': async ({ pass, fail }) => {
+    /*
+      The real-world shape: a helper takes a callback, the callback does `return fail(...)`, which only
+      leaves the callback, and the test then reaches its own pass().
+    */
+    const result = await runOne(async ({ pass, fail }) => {
+      await (async () => { return fail('inner check failed'); })();
+      pass('reached the end');
+    });
+    if(result.passed !== false) return fail(`A failure inside a nested callback was reported as passed=${result.passed}`);
+    pass('nested fail() is not masked');
+  },
+  'pass() followed by fail() is a failure': async ({ pass, fail }) => {
+    const result = await runOne(({ pass, fail }) => {
+      pass('looked fine');
+      fail('then it broke');
+    });
+    if(result.passed !== false) return fail(`Expected the test to fail, got passed=${result.passed}`);
+    pass('fail() after pass() fails the test');
+  },
+  'an exception after pass() is a failure': async ({ pass, fail }) => {
+    const result = await runOne(({ pass }) => {
+      pass('looked fine');
+      throw new Error('then it threw');
+    });
+    if(result.passed !== false) return fail(`Expected the test to fail, got passed=${result.passed}`);
+    pass('a throw after pass() fails the test');
+  },
+  'a test that only passes still passes, including repeated pass() calls': async ({ pass, fail }) => {
+    const result = await runOne(({ pass, log }) => {
+      log('working');
+      pass('first');
+      pass('second');
+    });
+    if(result.passed !== true) return fail(`Expected the test to pass, got passed=${result.passed}`);
+    if(result.logs.filter(l => l.type === 'pass').length !== 2) return fail('Both pass() calls should be logged as passes');
+    pass('unaffected');
+  },
+  'a failing test in a suite does not affect its neighbours': async ({ pass, fail }) => {
+    const res = await runTests({
+      default: {
+        'bad': ({ pass, fail }) => { fail('broke'); pass('masked'); },
+        'good': ({ pass }) => { pass('fine'); }
+      }
+    }, false, 0);
+    if(res.tests.bad.passed !== false) return fail('bad should fail');
+    if(res.tests.good.passed !== true) return fail('good should still pass');
+    pass('failures are isolated per test');
   }
 };