mocha-distributed 0.9.6 → 0.9.8

package/README.md

@@ -173,7 +173,7 @@ Keep in mind that:
 You might have a look at list-tests-from-redis.js for an example on how to
 query redis and list all tests.
 
-### Mark tests to run serially
+## Run tests serially
 
 If you'd like some of your tests to run serially you can use a magic string with
 this framework.

package/example/suite-7-duplicated-titles.js

@@ -0,0 +1,25 @@
+const util = require('./util.js');
+const { expect } = require('chai');
+
+// require('mocha-distributed')
+require('../index.js');
+
+describe ('suite-7', async function () {
+  this.timeout (10*1000);
+  this.retries(3);
+
+  // this test is created to make sure that
+  for (let i = 0; i < 10; i++) {
+    it ('test-7.1-duplicated-title', async function() {
+      const retryCount = this.test.currentRetry();
+      const id = `${i}`.padStart(2, '0');
+      console.log (`[id=${id}] test-7.1-duplicated-title (retry ${retryCount})`);
+      await util.sleep(0.1 + 0.5*Math.random());
+
+      // make this test flacky to make sure that it is retried
+      if (retryCount < 2) {
+        expect (false).to.be.true;
+      }
+    })
+  }
+})

package/index.js

@@ -34,6 +34,16 @@ let g_redis = null;
 
 let g_capture = { stdout: null, stderr: null };
 
+// Cache errors from intermediate retry attempts. Mocha only sets test.err via
+// the reporter on the final EVENT_TEST_FAIL; for non-final retries it emits
+// EVENT_TEST_RETRY instead and never stores the error on the test object.
+const g_retryErrors = new Map();
+
+// Map to keep track of the number of times a test key full path has been seen,
+// to add a suffix and parallelize duplicated test titles in multiple runners
+const g_duplicateTestKeyFullPathCount = new Map();
+let g_lastTestKeyFullPath = null;
+
 // -----------------------------------------------------------------------------
 // getTestPath
 //
@@ -108,6 +118,11 @@ function captureStream(stream) {
 // Initialize redis once before the tests
 // -----------------------------------------------------------------------------
 exports.mochaGlobalSetup = async function () {
+  // `this` is the Mocha Runner — store errors from non-final retry attempts
+  // so afterEach can record them (Mocha never sets test.err for those).
+  this.on('retry', (test, err) => {
+    g_retryErrors.set(test.fullTitle(), err);
+  });
   if (g_mochaVerbose) {
     const redisNoCredentials = g_redisAddress.replace(
       /\/\/[^@]*@/,
@@ -160,9 +175,26 @@ exports.mochaGlobalTeardown = async function () {
 exports.mochaHooks = {
   beforeEach: async function () {
     const testPath = getTestPath(this.currentTest);
-    const testKeyFullPath = `${g_testExecutionId}:${getSerialGranularity(testPath.join(":"))}`;
+    let testKeyFullPath = `${g_testExecutionId}:${getSerialGranularity(testPath.join(":"))}`;
     const testKeySuite = `${g_testExecutionId}:${getSerialGranularity(testPath[0])}`;
 
+    // if this is the first attempt, we need to put a suffix to be able to
+    // parallelize duplicates in multiple runners
+    if ((this.currentTest._currentRetry || 0) === 0) {
+      g_duplicateTestKeyFullPathCount.set(
+        testKeyFullPath,
+        (g_duplicateTestKeyFullPathCount.get(testKeyFullPath) || 0) + 1
+      );
+      testKeyFullPath += `:dup-${g_duplicateTestKeyFullPathCount.get(testKeyFullPath)}`;
+      g_lastTestKeyFullPath = testKeyFullPath;
+    }
+    else {
+      // ensure we use the same key for retries as the original attempt, otherwise
+      // we will screw up the distribution of tests; we use the last generated key
+      // since retries are executed sequentially
+      testKeyFullPath = g_lastTestKeyFullPath;
+    }
+
     const testKey =
       g_granularity === GRANULARITY.TEST ? testKeyFullPath : testKeySuite;
 
@@ -183,7 +215,7 @@ exports.mochaHooks = {
     }
   },
 
-  afterEach(done) {
+  afterEach: async function () {
     const SKIPPED = "pending";
     const FAILED = "failed";
     const PASSED = "passed";
@@ -227,7 +259,10 @@ exports.mochaHooks = {
       // Error objects cannot be properly serialized with stringify, thus
       // we need to use this hack to make it look like a normal object.
       // Hopefully this should work as well with other sort of objects
-      const err = this.currentTest.err || null;
+      const err = this.currentTest.err
+        || g_retryErrors.get(this.currentTest.fullTitle())
+        || null;
+      g_retryErrors.delete(this.currentTest.fullTitle());
       const errObj = JSON.parse(
         JSON.stringify(err, Object.getOwnPropertyNames(err || {}))
       );
@@ -252,16 +287,16 @@ exports.mochaHooks = {
       };
 
       // save results as single line on purpose
-      const key = `${g_testExecutionId}:test_result`;
-      g_redis.rPush(key, JSON.stringify(testResult));
-      g_redis.expire(key, g_expirationTime);
-
-      // increment passed_count/failed_count & set expiry time
+      const resultKey = `${g_testExecutionId}:test_result`;
       const countKey = `${g_testExecutionId}:${stateFixed}_count`;
-      g_redis.incr(countKey);
-      g_redis.expire(countKey, g_expirationTime);
-    }
 
-    done();
+      await g_redis
+        .multi()
+        .rPush(resultKey, JSON.stringify(testResult))
+        .expire(resultKey, g_expirationTime)
+        .incr(countKey)
+        .expire(countKey, g_expirationTime)
+        .exec();
+    }
   },
 };

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "mocha-distributed",
-  "version": "0.9.6",
+  "version": "0.9.8",
   "description": "Run multiple mocha suites and tests in parallel, from different processes and different machines. Results available on a redis database.",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "mocha test/test-retry.js"
   },
   "keywords": [
     "mocha",

package/test/test-retry.js

@@ -0,0 +1,142 @@
+// -----------------------------------------------------------------------------
+// test/test-retry.js
+//
+// Mocha test suite verifying that mocha-distributed records err, stdout and
+// stderr on ALL retry attempts, not just the last one.
+//
+// Run with: mocha test/test-retry.js
+// -----------------------------------------------------------------------------
+'use strict';
+
+const assert = require('assert');
+const Mocha  = require('mocha/lib/mocha');
+const Suite  = require('mocha/lib/suite');
+const Test   = require('mocha/lib/test');
+
+// -----------------------------------------------------------------------------
+// Mock redis helpers
+// -----------------------------------------------------------------------------
+const redisResolved = require.resolve('redis');
+
+const mockMulti = (writtenResults) => () => {
+  const cmds = [];
+  const chain = {
+    set:    (...a) => { cmds.push(['set',    ...a]); return chain; },
+    get:    (...a) => { cmds.push(['get',    ...a]); return chain; },
+    rPush:  (...a) => {
+      cmds.push(['rPush', ...a]);
+      writtenResults.push(JSON.parse(a[1]));
+      return chain;
+    },
+    expire: (...a) => { cmds.push(['expire', ...a]); return chain; },
+    incr:   (...a) => { cmds.push(['incr',   ...a]); return chain; },
+    exec: async () => {
+      // beforeEach pipeline: SET NX + GET — this runner always wins ownership
+      if (cmds[0] && cmds[0][0] === 'set') {
+        return [null, process.env.MOCHA_DISTRIBUTED_RUNNER_ID];
+      }
+      // afterEach pipeline: rPush + expire + incr + expire
+      return [1, 1, 1, 1];
+    }
+  };
+  return chain;
+};
+
+function injectMockRedis(writtenResults) {
+  require.cache[redisResolved] = {
+    id:       redisResolved,
+    filename: redisResolved,
+    loaded:   true,
+    exports:  {
+      createClient: () => ({
+        on:      () => {},
+        connect: async () => {},
+        quit:    async () => {},
+        multi:   mockMulti(writtenResults),
+      })
+    },
+  };
+}
+
+function restoreRedis() {
+  delete require.cache[redisResolved];
+}
+
+function loadFreshLib() {
+  const libPath = require.resolve('../index.js');
+  delete require.cache[libPath];
+  return require('../index.js');
+}
+
+// -----------------------------------------------------------------------------
+// Tests
+// -----------------------------------------------------------------------------
+describe('mocha-distributed', function () {
+
+  describe('retry attempt recording', function () {
+    let writtenResults;
+    let lib;
+
+    before(function () {
+      writtenResults = [];
+      injectMockRedis(writtenResults);
+
+      process.env.MOCHA_DISTRIBUTED              = 'redis://mock';
+      process.env.MOCHA_DISTRIBUTED_EXECUTION_ID = 'test-exec-retry';
+      process.env.MOCHA_DISTRIBUTED_RUNNER_ID    = 'runner-test';
+
+      lib = loadFreshLib();
+    });
+
+    after(function () {
+      restoreRedis();
+      delete require.cache[require.resolve('../index.js')];
+    });
+
+    it('records err, stdout and stderr on every retry attempt', async function () {
+      this.timeout(10000);
+
+      // Build the inner mocha instance with a flaky test that fails on
+      // attempts 0 and 1, and passes on attempt 2
+      const m = new Mocha({ reporter: 'min' });
+      m.rootHooks(lib.mochaHooks);
+      m.globalSetup([lib.mochaGlobalSetup]);
+      m.globalTeardown([lib.mochaGlobalTeardown]);
+
+      const suite = Suite.create(m.suite, 'retry-suite');
+      suite.retries(2);
+
+      let attempt = 0;
+      suite.addTest(new Test('flaky-test', function () {
+        attempt++;
+        console.log('stdout from attempt ' + attempt);
+        console.error('stderr from attempt ' + attempt);
+        if (attempt < 3) throw new Error('intentional failure on attempt ' + attempt);
+      }));
+
+      await new Promise(resolve => m.run(resolve));
+
+      // Sort by retryAttempt for stable assertions
+      const results = writtenResults.slice().sort((a, b) => a.retryAttempt - b.retryAttempt);
+
+      assert.strictEqual(results.length, 3, '3 results written to Redis (one per attempt)');
+
+      // Attempts 0 and 1: failed, err populated, stdout and stderr present
+      for (const i of [0, 1]) {
+        const r = results[i];
+        assert.strictEqual(r.retryAttempt, i,             `attempt ${i}: retryAttempt`);
+        assert.strictEqual(r.state, 'failed',             `attempt ${i}: state`);
+        assert.ok(r.err && r.err.message,                 `attempt ${i}: err.message should be set`);
+        assert.ok(r.stdout.includes(`attempt ${i + 1}`), `attempt ${i}: stdout`);
+        assert.ok(r.stderr.includes(`attempt ${i + 1}`), `attempt ${i}: stderr`);
+      }
+
+      // Attempt 2: passed, stdout and stderr present
+      const r2 = results[2];
+      assert.strictEqual(r2.retryAttempt, 2,    'attempt 2: retryAttempt');
+      assert.strictEqual(r2.state, 'passed',     'attempt 2: state');
+      assert.ok(r2.stdout.includes('attempt 3'), 'attempt 2: stdout');
+      assert.ok(r2.stderr.includes('attempt 3'), 'attempt 2: stderr');
+    });
+  });
+});

package/test-example-nworkers.sh

@@ -5,10 +5,10 @@ export MOCHA_DISTRIBUTED="redis://127.0.0.1"
 npm install > /dev/null 2>&1
 
 N=$1
-COMMAND="mocha --require ./index.js example/**/*.js"
+COMMAND="npx mocha --require ./index.js example/**/*.js"
 
 # cleanup tmp files
-rm tmp-*
+rm -f tmp-*
 
 echo "Spawning $N commands in parallel with Execution ID: $MOCHA_DISTRIBUTED_EXECUTION_ID"