pa11y-ci-reporter-runner 0.7.1 → 1.0.0

package/README.md

@@ -1,6 +1,6 @@
 # Pa11y CI Reporter Runner
 
-Pa11y CI Reporter Runner is designed to facilitate testing of [Pa11y CI reporters](https://github.com/pa11y/pa11y-ci#write-a-custom-reporter). Given a Pa11y CI JSON results file and optional configuration it simulates the Pa11y CI calls to the reporter, including proper transformation of results and configuration data. Functionally, it's an emulation of the Pa11y CI side of the reporter interface with finer control over execution.
+Pa11y CI Reporter Runner is designed to facilitate testing of [Pa11y CI reporters](https://github.com/pa11y/pa11y-ci#write-a-custom-reporter). Given a Pa11y CI JSON results file and optional configuration it simulates the Pa11y CI calls to the reporter, including proper transformation of results and configuration data. Functionally, it's an emulation of the Pa11y CI side of the reporter interface with explicit control over execution of each step.
 
 ## Installation
 
@@ -42,15 +42,26 @@ flowchart LR;
 
 When calling reporter functions, the runner transforms the Pa11y CI results and configuration data to provide the appropriate arguments. For example, the `results` function is called with the results as returned from Pa11y (slightly different than those returned from Pa11y CI) and the consolidated configuration for the analysis of that URL.
 
+The runner state can be obtained via the following runner functions:
+
+- `getCurrentState()`: The current state of the runner.
+- `getNextState()`: The next state of the runner (i.e. the state that will be obtained by calling the `runNext()` function).
+
+Both functions return an object with the following properties:
+
+- `state`: The current runner state (any state value shown above)
+- `url`: The current URL for any state with an applicable URL (`beginUrl` and `urlResults`), otherwise `undefined`.
+
 ### Runner Execution
 
-The reporter runner has four control functions:
+The reporter runner has five control functions:
 
 - `runAll()`: Simulates Pa11y CI running the analysis from the provided JSON results file from the current state through the end, calling all associated reporter functions.
 - `runNext()`: Simulates Pa11y CI running through the next state from the provided JSON results file, calling the associated reporter function as noted above.
 - `runUntil(targetState, targetUrl)`: Simulates Pa11y CI running the analysis from the provided JSON results file from the current state through the specified state/URL, calling the associated reporter functions as noted above. An error will be thrown if the end of the results are reached and the target was not found. This function takes the following arguments:
-  - `targetState`: The target state of the runner (any state above except `init`).
+  - `targetState`: The target state of the runner (any state above except `init`, or any valid value of the `RunnerStates` enum).
   - `targetUrl`: An optional target URL. If no URL is specified, the runner will stop at the first instance of the target state.
+- `runUntilNext(targetState, targetUrl)`: Provides the same functionality as `runUntil`, but execution ends at the state prior to the target state/URL, so that the target will execute if `runNext()` is subsequently called.
 - `reset()`: Resets the runner to the `init` state. This can be sent from any state.
 
 These command are all asynchronous and must be completed before another is sent, otherwise an error will be thrown. In addition, once a run has been completed and the runner is in the `afterAll` state it must be `reset` before accepting any run command.
@@ -66,36 +77,40 @@ const resultsFileName = "pa11yci-results.json";
 const reporterName = "../test-reporter.js";
 const reporterOptions = { "isSomething": true };
 const config = {
-  defaults: {
-    timeout: 30000,
-  },
-  urls: [
-    "http://localhost:8080/page1-with-errors.html",
-    "http://localhost:8080/page1-no-errors.html",
-    {
-      url: "https://pa11y.org/timed-out.html",
-      timeout: 50,
+    defaults: {
+        timeout: 30000,
     },
-  ],
+    urls: [
+        "http://localhost:8080/page1-with-errors.html",
+        "http://localhost:8080/page1-no-errors.html",
+        {
+            url: "https://pa11y.org/timed-out.html",
+            timeout: 50,
+        },
+    ],
 };
 
 test('test all reporter functions', async () => {
-  const runner = createRunner(resultsFileName, reporterName, reporterOptions, config);
+    const runner = createRunner(resultsFileName, reporterName, reporterOptions, config);
 
-  await runner.runAll();
+    await runner.runAll();
 
-  // Test reporter results
+    // Test reporter results
 });
 
 test('test reporter at urlResults state', async () => {
-  const runner = createRunner(resultsFileName, reporterName, reporterOptions, config);
-
-  await runner.runUntil(RunnerStates.beginUrl, 'http://localhost:8080/page1-no-errors.html');
-  // The runner is now in the beginUrl state for http://localhost:8080/page1-no-errors.html
-  await runner.runNext();
-  // The runner is now in the urlResults state for http://localhost:8080/page1-no-errors.html
-
-  // Test reporter results
+    const runner = createRunner(resultsFileName, reporterName, reporterOptions, config);
+
+    await runner.runUntil(RunnerStates.beginUrl, 'http://localhost:8080/page1-no-errors.html');
+    let currentState = runner.getCurrentState();
+    // { state: "beginUrl", url: "http://localhost:8080/page1-no-errors.html" }
+    const nextState = runner.getNextState();
+    // { state: "urlResults", url: "http://localhost:8080/page1-no-errors.html" }
+    await runner.runNext();
+    currentState = runner.getCurrentState();
+    // { state: "urlResults", url: "http://localhost:8080/page1-no-errors.html" }
+
+    // Test reporter results
 });
 ```
 

package/index.js

@@ -117,6 +117,7 @@ const createRunner = (resultsFileName, reporterName, options = {}, config = {})
     /**
      * Implements the runner beforeAll event, calling reporter.beforeAll.
      *
+     * @async
      * @private
      */
     const beforeAll = async () => {
@@ -126,6 +127,7 @@ const createRunner = (resultsFileName, reporterName, options = {}, config = {})
     /**
      * Implements the runner beginUrl event, calling reporter.begin.
      *
+     * @async
      * @private
      * @param {string} url The url being analyzed.
      */
@@ -137,6 +139,7 @@ const createRunner = (resultsFileName, reporterName, options = {}, config = {})
      * Implements the runner urlResults event, calling reporter.results or
      * reporter.error as appropriate based on the results.
      *
+     * @async
      * @private
      * @param {string} url The url being analyzed.
      */
@@ -151,6 +154,7 @@ const createRunner = (resultsFileName, reporterName, options = {}, config = {})
     /**
      * Implements the runner afterAll event, calling reporter.afterAll.
      *
+     * @async
      * @private
      */
     const afterAll = async () => {
@@ -172,6 +176,7 @@ const createRunner = (resultsFileName, reporterName, options = {}, config = {})
     /**
      * Resets the runner to the initial state.
      *
+     * @async
      * @public
      * @instance
      */
@@ -182,6 +187,7 @@ const createRunner = (resultsFileName, reporterName, options = {}, config = {})
     /**
      * Executes the entire Pa11y CI sequence, calling all reporter functions.
      *
+     * @async
      * @public
      * @instance
      */
@@ -193,6 +199,7 @@ const createRunner = (resultsFileName, reporterName, options = {}, config = {})
      * Executes the next event in the Pa11y CI sequence, calling the
      * appropriate reporter function.
      *
+     * @async
      * @public
      * @instance
      */
@@ -202,11 +209,12 @@ const createRunner = (resultsFileName, reporterName, options = {}, config = {})
 
     /**
      * Executes the entire Pa11y CI sequence, calling all reporter functions,
-     * until the specified state and optional URL are reached. If no URL is provided
+     * until the specified current state and optional URL are reached. If a URL is not
      * specified, the run completes on the first occurrence of the target state.
      *
+     * @async
      * @public
-     * @interface
+     * @instance
      * @param {string} targetState The target state to run to.
      * @param {string} [targetUrl] The target URL to run to.
      */
@@ -214,11 +222,49 @@ const createRunner = (resultsFileName, reporterName, options = {}, config = {})
         await service.runUntil(targetState, targetUrl);
     };
 
+    /**
+     * Executes the entire Pa11y CI sequence, calling all reporter functions,
+     * until the specified next state and optional URL are reached. If a URL is not
+     * specified, the run completes on the first occurrence of the target state.
+     *
+     * @async
+     * @public
+     * @instance
+     * @param {string} targetState The target state to run to.
+     * @param {string} [targetUrl] The target URL to run to.
+     */
+    const runUntilNext = async (targetState, targetUrl) => {
+        await service.runUntilNext(targetState, targetUrl);
+    };
+
+    /**
+     * Get the current state (state, url).
+     *
+     * @public
+     * @instance
+     * @returns {object} The current state.
+     */
+    // eslint-disable-next-line prefer-destructuring -- required for jsdoc
+    const getCurrentState = service.getCurrentState;
+
+    /**
+     * Get the next state (state, url).
+     *
+     * @public
+     * @instance
+     * @returns {object} The next state.
+     */
+    // eslint-disable-next-line prefer-destructuring -- required for jsdoc
+    const getNextState = service.getNextState;
+
     return {
+        getCurrentState,
+        getNextState,
         reset,
         runAll,
         runNext,
-        runUntil
+        runUntil,
+        runUntilNext
     };
 };
 

package/lib/pa11yci-machine.js

@@ -1,7 +1,17 @@
 'use strict';
 
+
+/**
+ * State machine for pa11yci.
+ *
+ * @module pa11yci-machine
+ */
+
 const { createMachine, assign } = require('xstate');
 
+/**
+ * State machine for pa11yci.
+ */
 const pa11yciMachine = createMachine(
     {
         id: 'pa11yci-runner',

package/lib/pa11yci-service.js

@@ -1,11 +1,43 @@
+/* eslint-disable max-lines */
 'use strict';
 
-const { interpret } = require('xstate');
-const machine = require('./pa11yci-machine');
+/**
+ * Service that interprets pa11yci-machine.
+ *
+ * @module pa11yci-service
+ */
 
+const machine = require('./pa11yci-machine');
 const RunnerStates = require('./runner-states');
+
 const finalState = RunnerStates.afterAll;
 
+/**
+ * Enum for machine events.
+ *
+ * @enum {string}
+ * @private
+ * @readonly
+ * @static
+ */
+const MachineEvents = Object.freeze({
+    NEXT: 'NEXT',
+    RESET: 'RESET'
+});
+
+/**
+ * Enum for runner service states.
+ *
+ * @enum {string}
+ * @private
+ * @readonly
+ * @static
+ */
+const StateTypes = Object.freeze({
+    current: 'current',
+    next: 'next'
+});
+
 /**
  * Gets the initial pa11yci-runner state machine context given an array of URLs.
  *
@@ -19,19 +51,29 @@ const getInitialContext = urls => ({
 });
 
 /**
- * Checks the state to determine whether it has an associated URL.
+ * Checks the runner state to determine whether it has an associated URL.
  *
  * @private
- * @param   {RunnerStates} state The state to check.
- * @returns {boolean}            True if the state has an associated url.
+ * @param   {string}  state The state to check.
+ * @returns {boolean}       True if the state has an associated url.
  */
 const hasUrl = state => state === RunnerStates.beginUrl || state === RunnerStates.urlResults;
 
+/**
+ * Gets the URL for the given machine state.
+ *
+ * @private
+ * @param   {object} machineState The machine state to check against.
+ * @returns {string}              The current URL for the machine state.
+ */
+const getUrlForState = machineState => hasUrl(machineState.value)
+    ? machineState.context.urls[machineState.context.urlIndex] : undefined;
+
 /**
  * Validates that the given state is a valid RunnerStates value. Throws if not.
  *
  * @private
- * @param {string} state The state to validate.
+ * @param   {string} state The state to validate.
  */
 const validateRunnerState = (state) => {
     if (!Object.keys(RunnerStates).includes(state)) {
@@ -40,21 +82,34 @@ const validateRunnerState = (state) => {
 };
 
 /**
- * Checks if the context matches the targetState and either no
+ * Checks if the machine state matches the targetState and either no
  * targetUrl was specified or the context matches the targetUrl.
  *
  * @private
- * @param   {object}  context     The current state machine context.
- * @param   {string}  targetState The target runner state.
- * @param   {string}  [targetUrl] The target URL.
- * @returns {boolean}             True if the context matches the target, otherwise false.
+ * @param   {object}        machineState The machine state to check against.
+ * @param   {RunnerStates}  targetState  The target runner state.
+ * @param   {string}        [targetUrl]  The target URL.
+ * @returns {boolean}                    True if the context matches the target, otherwise false.
  */
-const isAtTarget = (context, targetState, targetUrl) => {
-    return context.state === targetState && (!targetUrl || context.url === targetUrl);
+const isAtTarget = (machineState, targetState, targetUrl) => {
+    return machineState.value === targetState &&
+        (!targetUrl || getUrlForState(machineState) === targetUrl);
 };
 
 /**
- * Factory function that return a pa11yci-runner service.
+ * Gets a summary of the machine state (state, url).
+ *
+ * @private
+ * @param   {object} machineState The machine state.
+ * @returns {object}              The machine state summary.
+ */
+const getStateSummary = machineState => ({
+    state: machineState.value,
+    url: getUrlForState(machineState)
+});
+
+/**
+ * Factory function that returns a pa11yci-runner service.
  *
  * @public
  * @static
@@ -65,35 +120,16 @@ const isAtTarget = (context, targetState, targetUrl) => {
 // eslint-disable-next-line max-lines-per-function
 const serviceFactory = (urls, actions) => {
     let pendingCommand;
-    const currentContext = {
-        state: undefined,
-        url: undefined
-    };
 
-    // Create service from pa11yci-machine with context, setup
-    // transition event handler, and start service
-    const service = interpret(machine.withContext(getInitialContext(urls)));
-    service.onTransition(async (state) => {
-        // Save the current state and url for use in manual transitions.
-        // The state machine only increments the url index, so use that
-        // to get the url from the original urls array.
-        currentContext.state = state.value;
-        currentContext.url = hasUrl(currentContext.state)
-            ? urls[state.context.urlIndex] : undefined;
-
-        if (pendingCommand) {
-            // Do not take any action in init state
-            if (currentContext.state !== 'init') {
-                await actions[currentContext.state](currentContext.url);
-            }
-            pendingCommand.resolve();
-            pendingCommand = undefined;
-        }
-    });
-    service.start();
+    // Implement custom xstate interpreter, which allows for tracking for current
+    // and the next state, which is required for some control functions.
+    const pa11yMachine = machine.withContext(getInitialContext(urls));
+    let currentState = pa11yMachine.initialState;
+    // machine.transition is a pure function, so only retrieves the state
+    let nextState = machine.transition(currentState, MachineEvents.NEXT);
 
     /**
-     * Validate that a command is allowed in the given state. Throws if invalid.
+     * Validates that a command is allowed in the given state. Throws if invalid.
      *
      * @private
      */
@@ -101,80 +137,163 @@ const serviceFactory = (urls, actions) => {
         if (pendingCommand) {
             throw new Error('runner cannot accept a command while another command is pending, await previous command');
         }
-        if (currentContext.state === finalState) {
+        if (currentState.value === finalState) {
             throw new Error(`runner must be reset before executing any other functions from the ${finalState} state`);
         }
     };
 
     /**
-     * Sends the specified event to the pa11yci-runner service. Resolves when
-     * the next state has been reached and the reporter event raised.
+     * Sends the specified event to the pa11yci-machine and executes
+     * the applicable action for that state.
      *
+     * @async
      * @private
-     * @param   {string}        event The event name to be sent.
-     * @returns {Promise<void>}       Promise that indicates a pending state change.
+     * @param   {MachineEvents} event The event name to be sent.
      */
-    const sendEvent = (event) => {
-        return new Promise((resolve, reject) => {
-            pendingCommand = { resolve, reject };
-            service.send({ type: event });
-        });
+    const sendEvent = async (event) => {
+        try {
+            // Track pending command to block other commands
+            pendingCommand = true;
+
+            // Send event to the machine and executes the action for the
+            // current state (except in init, which has no action)
+            currentState = machine.transition(currentState, event);
+            if (currentState.value !== 'init') {
+                await actions[currentState.value](getUrlForState(currentState));
+            }
+
+            // Check next state and save for reference (machine.transition
+            // is a pure function, so only retrieves the state)
+            if (currentState.value !== finalState) {
+                nextState = machine.transition(currentState, MachineEvents.NEXT);
+            }
+        }
+        finally {
+            // Ensure pending command is reset in all cases, including on error
+            pendingCommand = false;
+        }
     };
 
     /**
      * Resets the service to the init state.
      *
+     * @async
      * @public
      * @instance
      */
     const reset = async () => {
-        await sendEvent('RESET');
+        await sendEvent(MachineEvents.RESET);
     };
 
     /**
      * Executes the next event in the Pa11y CI sequence, calling the
      * appropriate reporter function.
      *
+     * @async
      * @public
      * @instance
      */
     const runNext = async () => {
         validateCommandAllowed();
 
-        await sendEvent('NEXT');
+        await sendEvent(MachineEvents.NEXT);
     };
 
     /**
-     * Executes the entire Pa11y CI sequence, calling all reporter functions,
-     * until the specified state and optional URL are reached. If no URL is provided
-     * specified, the run completes on the first occurrence of the target state.
+     * Gets the state for the given state type (current or next).
      *
-     * @public
-     * @interface
-     * @param {string} targetState The target state to run to.
-     * @param {string} [targetUrl] The target URL to run to.
+     * @private
+     * @param   {StateTypes} stateType The state type.
+     * @returns {object}               The state object for teh given type.
      */
-    const runUntil = async (targetState, targetUrl) => {
+    const getState = stateType => stateType === StateTypes.current ? currentState : nextState;
+
+    /**
+     * Common function for executing runUntil and runUntilNext using the
+     * specified state type to check for completion. Executes the entire
+     * Pa11y CI sequence, calling all reporter functions, until the
+     * specified state and optional URL are reached. If a URL is not specified,
+     * the run completes on the first occurrence of the target state.
+     *
+     * @async
+     * @private
+     * @param   {StateTypes}   stateType   The state type.
+     * @param   {RunnerStates} targetState The target state to run to.
+     * @param   {string}       [targetUrl] The target URL to run to.
+     */
+    const runUntilInternal = async (stateType, targetState, targetUrl) => {
         validateCommandAllowed();
         validateRunnerState(targetState);
 
-        while (!isAtTarget(currentContext, targetState, targetUrl)
-            && currentContext.state !== finalState) {
-            await sendEvent('NEXT');
+        // Run until target or final state is achieved
+        while (!isAtTarget(getState(stateType), targetState, targetUrl)
+            && (getState(stateType)).value !== finalState) {
+            await sendEvent(MachineEvents.NEXT);
         }
 
         // If the finalState is reached and not at target then it is
         // not in the results and throw to indicate the command failed.
-        if (!isAtTarget(currentContext, targetState, targetUrl)) {
+        if (!isAtTarget(getState(stateType), targetState, targetUrl)) {
             const urlString = targetUrl ? ` for targetUrl "${targetUrl}"` : '';
             throw new Error(`targetState "${targetState}"${urlString} was not found`);
         }
     };
 
+    /**
+     * Executes the entire Pa11y CI sequence, calling all reporter functions,
+     * until the specified current state and optional URL are reached. If a URL is not
+     * specified, the run completes on the first occurrence of the target state.
+     *
+     * @async
+     * @public
+     * @instance
+     * @param    {RunnerStates} targetState The target state to run to.
+     * @param    {string}       [targetUrl] The target URL to run to.
+     */
+    const runUntil = async (targetState, targetUrl) => {
+        await runUntilInternal(StateTypes.current, targetState, targetUrl);
+    };
+
+    /**
+     * Executes the entire Pa11y CI sequence, calling all reporter functions,
+     * until the specified next state and optional URL are reached. If a URL is not
+     * specified, the run completes on the first occurrence of the target state.
+     *
+     * @async
+     * @public
+     * @instance
+     * @param    {RunnerStates} targetState The target state to run to.
+     * @param    {string}       [targetUrl] The target URL to run to.
+     */
+    const runUntilNext = async (targetState, targetUrl) => {
+        await runUntilInternal(StateTypes.next, targetState, targetUrl);
+    };
+
+    /**
+     * Get the current state (state, url).
+     *
+     * @public
+     * @instance
+     * @returns  {object} The current state.
+     */
+    const getCurrentState = () => getStateSummary(currentState);
+
+    /**
+     * Get the next state (state, url).
+     *
+     * @public
+     * @instance
+     * @returns  {object} The next state.
+     */
+    const getNextState = () => getStateSummary(nextState);
+
     return {
+        getCurrentState,
+        getNextState,
         reset,
         runNext,
-        runUntil
+        runUntil,
+        runUntilNext
     };
 };
 

package/lib/runner-states.js

@@ -1,6 +1,8 @@
 'use strict';
 
 /**
+ * Valid states for the runner.
+ *
  * @module runner-states
  */
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pa11y-ci-reporter-runner",
-  "version": "0.7.1",
+  "version": "1.0.0",
   "description": "Pa11y CI Reporter Runner is designed to facilitate testing of Pa11y CI reporters. Given a Pa11y CI JSON results file and optional configuration it simulates the Pa11y CI calls to the reporter.",
   "main": "index.js",
   "scripts": {
@@ -36,7 +36,7 @@
   "homepage": "https://gitlab.com/gitlab-ci-utils/pa11y-ci-reporter-runner#readme",
   "devDependencies": {
     "@aarongoldenthal/eslint-config-standard": "^12.0.2",
-    "eslint": "^8.10.0",
+    "eslint": "^8.11.0",
     "jest": "^27.5.1",
     "jest-junit": "^13.0.0",
     "markdownlint-cli": "^0.31.1",
@@ -44,6 +44,6 @@
   },
   "dependencies": {
     "lodash": "^4.17.21",
-    "xstate": "^4.30.5"
+    "xstate": "^4.30.6"
   }
 }