mocha-distributed 0.8.1 → 0.9.3

package/index.js

@@ -1,49 +1,218 @@
 // -----------------------------------------------------------------------------
-// index.js
+// Copyright (c) 2018 Pau Sanchez
 //
-// Copyright(c) 2019 Pau Sanchez - MIT License
+// MIT Licensed
 // -----------------------------------------------------------------------------
+const redis = require("redis");
+const crypto = require("crypto");
 
-// Three scenarios for MOCHA_DISTRIBUTED:
-//   - master (MOCHA_DISTRIBUTED = 'master[:PORT]')
-//     Runs the server as master node, waiting for connections and waiting
-//     others to run the tests. For simplicity, this node won't execute
-//     any tests, but gather the result of all of them.
+const GRANULARITY = {
+  TEST: "test",
+  SUITE: "suite",
+};
+
+// Initialize variables from environment
+const g_redisAddress = process.env.MOCHA_DISTRIBUTED || "";
+const g_testExecutionId = process.env.MOCHA_DISTRIBUTED_EXECUTION_ID || "";
+const g_expirationTime =
+  process.env.MOCHA_DISTRIBUTED_EXPIRATION_TIME || `${24 * 3600}`;
+
+// Generate a unique random id for this runner (with almost 100% certainty
+// to be different on any machine/environment).
+const _randomRunnerBuf = Buffer.alloc(16);
+const _randomRunnerId = crypto.randomFillSync(_randomRunnerBuf).toString("hex");
+const g_runnerId = process.env.MOCHA_DISTRIBUTED_RUNNER_ID || _randomRunnerId;
+let g_granularity =
+  process.env.MOCHA_DISTRIBUTED_GRANULARITY || GRANULARITY.TEST;
+const g_mochaVerbose = process.env.MOCHA_DISTRIBUTED_VERBOSE === "true";
+
+if (g_granularity !== GRANULARITY.TEST) {
+  g_granularity = GRANULARITY.SUITE;
+}
+
+let g_redis = null;
+
+let g_capture = { stdout: null, stderr: null };
+
+// -----------------------------------------------------------------------------
+// getTestPath
 //
-//   - runner (MOCHA_DISTRIBUTED='<master-address>[:PORT]'  IP/DNS of the master node)
-//     IP/DNS of the master node (a.k.a runner), this node will get a unique
-//     name from the master, and will ask, before each suite or orphaned test
-//     if it needs to run it or not
+// Returns an array with the test suites and test name from a test context
+// as found in the hooks
 //
-//   - Empty or not defined
-//     Runs mocha normally, we don't redefine any variable at all.
-//     Everything will run locally, if spawned on many machines or processes
-//     tests will run on all of them as if this module did not exist.
-const master = require('./master.js');
-const runner = require('./runner.js');
-
-// Initialize mode & port from environment variable MOCHA_DISTRIBUTED
-const DEFAULT_PORT = 12421;
-let mode = (process.env.MOCHA_DISTRIBUTED || '').toLowerCase();
-let port = DEFAULT_PORT;
-
-if (mode.indexOf (':') >= 0) {
-  const splitted = mode.split(':');
-  mode = splitted[0];
-  port = parseInt (splitted[1], 10);
-}
+// Example:
+//
+//    >>> getTestPath(ctxt)
+//
+// -----------------------------------------------------------------------------
+function getTestPath(testContext) {
+  const path = [testContext.title];
 
-// let's get the party started
-if (!mode) {
-  // run as normal mocha
-}
-else if (mode === 'master') {
-  master (port);
+  while (!testContext.root && testContext.parent) {
+    testContext = testContext.parent;
+
+    if (testContext && !testContext.root) {
+      path.push(testContext.title);
+    }
+  }
+
+  return path.reverse();
 }
-else {
-  const masterAddress = mode;
-  runner (masterAddress, port);
+
+// -----------------------------------------------------------------------------
+// captureStream
+// -----------------------------------------------------------------------------
+function captureStream(stream) {
+  var oldWrite = stream.write;
+  var buf = [];
+
+  stream.write = function(chunk, encoding, callback){
+    buf.push (chunk.toString()); // chunk is a String or Buffer
+    oldWrite.apply(stream, arguments);
+  }
+
+  return {
+    unhook(){
+     stream.write = oldWrite;
+    },
+    captured(){
+      return buf;
+    }
+  };
 }
 
-// remove from cache, so it is always reinitialized
-delete require.cache[require.resolve('./index.js')];
+// -----------------------------------------------------------------------------
+// Initialize redis once before the tests
+// -----------------------------------------------------------------------------
+exports.mochaGlobalSetup = async function () {
+  if (g_mochaVerbose) {
+    const redisNoCredentials = g_redisAddress.replace(
+      /\/\/[^@]*@/,
+      "//***:***@"
+    );
+    console.log("---------------------------------------------------");
+    console.log(" Mocha Distributed");
+    console.log("   - Runner Id                :", g_runnerId);
+    console.log("   - Redis Address            :", redisNoCredentials);
+    console.log("   - Execution Id             :", g_testExecutionId);
+    console.log("   - Data Expiration Time     :", g_expirationTime);
+    console.log("   - Test Parallel Granularity:", g_granularity);
+    console.log("---------------------------------------------------");
+  }
+
+  if (!g_redisAddress || !g_testExecutionId) {
+    console.log (g_redisAddress, g_testExecutionId)
+    console.error(
+      "You need to set at least the following environment variables:\n" +
+        "  - MOCHA_DISTRIBUTED\n" +
+        "  - MOCHA_DISTRIBUTED_EXECUTION_ID\n"
+    );
+    process.exit(-1);
+  }
+
+  g_redis = redis.createClient({ url: g_redisAddress });
+  g_redis.on("error", (err) => {
+    console.log("Redis Client Error", err);
+    console.log("Closing application!");
+    process.exit(-1);
+  });
+  await g_redis.connect();
+};
+
+// -----------------------------------------------------------------------------
+// Quit from redis
+// -----------------------------------------------------------------------------
+exports.mochaGlobalTeardown = async function () {
+  if (g_redis) {
+    await g_redis.quit();
+  }
+};
+
+// -----------------------------------------------------------------------------
+// Hook tests
+//
+// Please note that we run skip before each test if the ownership of it has
+// already been defined by another runner.
+// -----------------------------------------------------------------------------
+exports.mochaHooks = {
+  beforeEach: async function () {
+    const testPath = getTestPath(this.currentTest);
+    const testKeyFullPath = `${g_testExecutionId}:${testPath.join(":")}`;
+    const testKeySuite = `${g_testExecutionId}:${testPath[0]}`;
+
+    const testKey =
+      g_granularity === GRANULARITY.TEST ? testKeyFullPath : testKeySuite;
+
+    // Atomically set/get the runner id associated to this test. Only the first
+    // runner to get there will set the value to its own runner id.
+    const [_, assignedRunnerId] = await g_redis
+      .multi()
+      .set(testKey, g_runnerId, { EX: g_expirationTime, NX: true })
+      .get(testKey)
+      .exec();
+
+    if (assignedRunnerId !== g_runnerId) {
+      this.currentTest.title += " (skipped by mocha_distributted)";
+      this.skip();
+    }
+    else {
+      g_capture.stdout = captureStream(process.stdout);
+      g_capture.stderr = captureStream(process.stderr);
+    }
+  },
+  afterEach(done) {
+    const SKIPPED = "pending";
+    const FAILED = "failed";
+    const PASSED = "passed";
+
+    let capturedStdout = '';
+    let capturedStderr = '';
+    if (g_capture.stdout) {
+      const stdoutArray = g_capture.stdout.captured();
+      capturedStdout = stdoutArray.join('');
+      capturedStdout = capturedStdout.replace(/\s*\u001b\[3[12]m[^\n]*\n$/g, '');
+      g_capture.stdout.unhook();
+      g_capture.stdout = null;
+    }
+
+    if (g_capture.stderr) {
+      capturedStderr = g_capture.stderr.captured().join('');
+      g_capture.stderr.unhook();
+      g_capture.stderr = null;
+    }
+
+    // Save all data in redis in a way it can be retrieved and aggregated
+    // easily for all test by an external reporter
+    if (this.currentTest.state !== SKIPPED) {
+      const stateFixed = this.currentTest.state || (this.currentTest.timedOut ? FAILED : PASSED)
+      const testResult = {
+        id: getTestPath(this.currentTest),
+        type: this.currentTest.type,
+        title: this.currentTest.title,
+        timedOut: this.currentTest.timedOut,
+        duration: this.currentTest.duration,
+        startTime: Date.now() - (this.currentTest.duration || 0),
+        endTime: Date.now(),
+        file: this.currentTest.file,
+        state: stateFixed,
+        failed: stateFixed === FAILED,
+        speed: this.currentTest.speed,
+        err: this.currentTest.err || null,
+        stdout: capturedStdout,
+        stderr: capturedStderr
+      };
+
+      // save results as single line on purpose
+      const key = `${g_testExecutionId}:test_result`;
+      g_redis.rPush(key, JSON.stringify(testResult));
+      g_redis.expire(key, g_expirationTime);
+
+      // increment passed_count/failed_count & set expiry time
+      const countKey = `${g_testExecutionId}:${stateFixed}_count`
+      g_redis.incr(countKey);
+      g_redis.expire(countKey, g_expirationTime);
+    }
+
+    done();
+  },
+};

package/list-tests-from-redis.js

@@ -0,0 +1,85 @@
+// -----------------------------------------------------------------------------
+// Copyright (c) 2018 Pau Sanchez
+//
+// MIT Licensed
+// -----------------------------------------------------------------------------
+
+const redis = require("redis");
+
+async function getTestResults(redisHost = 'localhost', redisPort = 6379) {
+  const allResults = []
+  try {
+    const redisClient = redis.createClient({
+      url: `redis://${redisHost}:${redisPort}/`
+    });
+
+    await redisClient.connect()
+
+    // get all potential test results keys
+    const keyResults = await redisClient.keys('*:test_result')
+
+    // we can get all in parallel
+    const parallelResults = []
+    for(const key of keyResults) {
+      const rawResultList = await redisClient.lRange(key, 0, -1)
+
+      let passed = 0
+      let failed = 0
+      let aggregatedDurationMs = 0
+      let firstStartTime = null
+      let lastEndTime = null
+      const jsonResultList = []
+      for (const rawResult of rawResultList) {
+        const jsonResult = JSON.parse(rawResult)
+        jsonResultList.push(jsonResult)
+
+        // compute some extra stuff
+        if (firstStartTime === null || jsonResult.startTime < firstStartTime) {
+          firstStartTime = jsonResult.startTime
+        }
+
+        if (lastEndTime === null || jsonResult.endTime > lastEndTime) {
+          lastEndTime = jsonResult.endTime
+        }
+
+        passed += (jsonResult.state === "passed")
+        failed += (jsonResult.state === "failed")
+
+        aggregatedDurationMs += (jsonResult.duration || 0)
+      }
+
+      // sort all jsonResultList by the full test route, because they will
+      // be visualized better
+      jsonResultList.sort((a, b) => (a.id.join('/') > b.id.join('/')) ? 1 : -1)
+
+      allResults.push({
+        key: key,
+        start_time : firstStartTime || 0,
+        end_time : lastEndTime || 0,
+        aggregated_duration: aggregatedDurationMs,
+        real_duration: (firstStartTime && lastEndTime) ? lastEndTime - firstStartTime : aggregatedDurationMs,
+        tests_passed : passed,
+        tests_failed : failed,
+        test_results : jsonResultList
+      })
+    }
+
+    // sort results by startTime, most recent first
+    allResults.sort((a, b) => (a.start_time < b.start_time) ? 1 : -1)
+
+    await redisClient.quit()
+    return allResults
+  }
+  catch(e) {
+    console.error ("Test Error: ", e)
+    return false
+  }
+}
+
+
+// -----------------------------------------------------------------------------
+// Print results as JSON output
+// -----------------------------------------------------------------------------
+getTestResults().then( (results) => {
+  console.log (JSON.stringify(results, null, 2))
+})

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "mocha-distributed",
-  "version": "0.8.1",
-  "description": "Run multiple mocha suites and tests in parallel, from different processes or different machines",
+  "version": "0.9.3",
+  "description": "Run multiple mocha suites and tests in parallel, from different processes and different machines. Results available on a redis database.",
   "main": "index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
@@ -14,6 +14,7 @@
     "integration tests",
     "pipelines",
     "chai",
+    "expect",
     "kubernetes",
     "docker",
     "mocha-parallel-tests",
@@ -27,10 +28,10 @@
   "author": "Pau Sanchez",
   "license": "MIT",
   "dependencies": {
-    "axios": "^0.18.0"
+    "redis": "^4.0.2"
   },
   "devDependencies": {
     "chai": "^4.2.0",
-    "mocha": "^5.2.0"
+    "mocha": "^9.1.4"
   }
 }

package/constants.js

@@ -1,33 +0,0 @@
-// -----------------------------------------------------------------------------
-// constants.js
-//
-// Copyright(c) 2019 Pau Sanchez - MIT License
-// -----------------------------------------------------------------------------
-
-const RUNNER_ID_PREFIX = 'runner-';
-
-const TEST_STATUS_RUNNING = 'running';
-const TEST_STATUS_SUCCESS = 'success';
-const TEST_STATUS_FAILED  = 'failed';
-
-const TEST_PATH_SEPARATOR = '>>>';
-
-const EVENT_FINISHED = 'finished';
-
-const ERRORS = {
-  INVALID_REQUEST : 'INVALID_REQUEST',
-  INVALID_TEST_ID : 'INVALID_TEST_ID',
-  INVALID_REQUEST_ACTION : 'INVALID_REQUEST_ACTION',
-  INVALID_RUNNER_OWNERSHIP : 'INVALID_RUNNER_OWNERSHIP',
-  ALREADY_RUNNING : 'ALREADY_RUNNING',
-};
-
-module.exports = {
-  RUNNER_ID_PREFIX,
-  TEST_STATUS_RUNNING,
-  TEST_STATUS_SUCCESS,
-  TEST_STATUS_FAILED,
-  TEST_PATH_SEPARATOR,
-  EVENT_FINISHED,
-  ERRORS
-};

package/master-mocha-bindings.js

@@ -1,159 +0,0 @@
-// -----------------------------------------------------------------------------
-// runner-mocha-bindings.js
-//
-// NOTE: mocha first does a first pass executing all describe/it/... and then
-//       when it has gathered all information, it runs the functions associated
-//       to each. What we do is we hook our method in the middle and call
-//       or not, the original test method.
-//
-// Copyright(c) 2019 Pau Sanchez - MIT License
-// -----------------------------------------------------------------------------
-const constants = require('./constants.js');
-
-let g_mochaMethods = {
-  describe   : global.describe || null,
-  it         : global.it || null,
-  before     : global.before || null,
-  beforeEach : global.beforeEach || null,
-  after      : global.after || null,
-  afterEach  : global.afterEach || null
-};
-
-// contains the actual test path such as suite.title > suite.title > it.title
-let g_testPath = [];
-let g_testResults = new Map();
-let g_testEventEmitter = null;
-
-// -----------------------------------------------------------------------------
-// setEventEmitter
-//
-// To communicate with the mocha bindings
-// -----------------------------------------------------------------------------
-function setEventEmitter (testEventEmitter) {
-  g_testEventEmitter = testEventEmitter;
-}
-
-// -----------------------------------------------------------------------------
-// describe
-//
-// Custom version to describe a test, with same signature
-// -----------------------------------------------------------------------------
-async function describe (title, fn) {
-  g_testPath.push (title);
-
-  // TODO: handle timeouts!!
-  const result = g_mochaMethods.describe (title, fn);
-  g_testPath.pop();
-
-  return result;
-}
-
-// TODO:
-// describe.skip = async function (title, fn) { }
-// describe.once = async function (title, fn) { }
-
-// -----------------------------------------------------------------------------
-// it
-// -----------------------------------------------------------------------------
-async function it (title, fn) {
-  // store current path for when the 'it' function executes
-  g_testPath.push (title);
-  const testPath = g_testPath.slice(0);
-  const testId   = testPath.join(constants.TEST_PATH_SEPARATOR);
-  const listenerId = constants.EVENT_FINISHED + ':' + testId;
-  g_testPath.pop();
-
-  // TODO: manage timeouts
-
-  // Two listeners are created, once beforehand, and other in runtime.
-  //
-  // Runtime one will wait the master to receive the status... but since the
-  // master node runs serially, it can also happen that another runner executes
-  // the tests before the master can initialize the second event listener, and
-  // we use this one to capture the result beforehand.
-  g_testEventEmitter.once (
-    listenerId,
-    function (test) {
-      g_testResults.set (test.id, test);
-    }
-  );
-
-  // define our own function hook to wait until the test finishes
-  return g_mochaMethods.it (title, async function () {
-    await new Promise(function (resolve, reject) {
-      // helper function to avoid repeating the code twice
-      function helperProcessTestResult(test) {
-        if (test.status === constants.TEST_STATUS_FAILED) {
-          reject (test.error);
-          return;
-        }
-
-        resolve();
-      }
-
-      if (g_testResults.has(testId)) {
-        const test = g_testResults.get (testId);
-        helperProcessTestResult (test);
-        return;
-      }
-
-      // first event-listener has not triggered, so let's install another
-      // event listener to resolve as quickly as possible
-      g_testEventEmitter.once (listenerId, helperProcessTestResult);
-    });
-  });
-}
-
-// -----------------------------------------------------------------------------
-// execHookMochaMethod
-//
-// Method used accross all mocha hooks (before, beforeEach, after, afterEach)
-// in order to query the master server if it needs to be executed or not.
-// -----------------------------------------------------------------------------
-function execHookMochaMethod (title, orgMochaMethod, fn) {
-  g_testPath.push (title);
-  const testPath = g_testPath.slice(0);
-  g_testPath.pop();
-
-  return orgMochaMethod (async function () {
-    // TODO: wait
-  });
-}
-
-// -----------------------------------------------------------------------------
-// before
-// -----------------------------------------------------------------------------
-async function before (fn) {
-  return execHookMochaMethod (':before', g_mochaMethods.before, fn);
-}
-
-// -----------------------------------------------------------------------------
-// beforeEach
-// -----------------------------------------------------------------------------
-async function beforeEach (fn) {
-  return execHookMochaMethod (':beforeEach', g_mochaMethods.beforeEach, fn);
-}
-
-// -----------------------------------------------------------------------------
-// after
-// -----------------------------------------------------------------------------
-async function after (fn) {
-  return execHookMochaMethod (':after', g_mochaMethods.after, fn);
-}
-
-// -----------------------------------------------------------------------------
-// afterEach
-// -----------------------------------------------------------------------------
-async function afterEach (fn) {
-  return execHookMochaMethod (':afterEach', g_mochaMethods.after, fn);
-}
-
-module.exports = {
-  setEventEmitter,
-  describe,
-  it,
-  before,
-  beforeEach,
-  after,
-  afterEach
-};