pa11y-ci-reporter-runner 0.5.0

package/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2022 Aaron Goldenthal
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,59 @@
+# Pa11y CI Reporter Runner
+
+Pa11y CI Reporter Runner is designed to facilitate testing of [Pa11y CI reporters](https://github.com/pa11y/pa11y-ci#write-a-custom-reporter). Given a Pa11y CI JSON results file and optional configuration it simulates the Pa11y CI calls to the reporter, including proper tranformation of results and configuration data. Functionally, it's a mock of the Pa11y CI side of the reporter interface.
+
+## Installation
+
+Install Pa11y CI Reporter Runner via [npm](https://www.npmjs.com/package/pa11y-ci-reporter-runner).
+
+```sh
+npm install pa11y-ci-reporter-runner
+```
+
+## Usage
+
+Pa11y CI Reporter Runner exports a factory function that creates a reporter runner. This function has four arguments:
+
+- `resultsFileName`: Path to a Pa11y CI JSON results file.
+- `reporterName`: Name of the reporter to execute. Can be a dependency (e.g. `pa11y-ci-reporter-html`) or a path to a reporter file.
+- `options`: Optional [reporter options](https://github.com/pa11y/pa11y-ci#reporter-options).
+- `config`: Optional Pa11y CI configuration file that produces the Pa11y CI JSON results.
+
+The reporter runner currently has one function:
+
+- `runAll`: Simulates Pa11y CI running the complete analysis from the provided JSON results file, calling all associated reporter functions (`beforeAll`, `begin` and `results`/`error` for each URL, `afterAll`).
+
+A complete example is provided below:
+
+```js
+const createRunner = require("pa11y-ci-reporter-runner");
+
+const resultsFileName = "pa11yci-results.json";
+const reporterName = "../test-reporter.js";
+const reporterOptions = { "isSomething": true };
+const config = {
+  defaults: {
+    timeout: 30000,
+  },
+  urls: [
+    "http://localhost:8080/page1-with-errors.html",
+    "http://localhost:8080/page1-no-errors.html",
+    {
+      url: "https://pa11y.org/timed-out.html",
+      timeout: 50,
+    },
+  ],
+};
+
+test('test reporter', async () => {
+  const runner = createRunner(resultsFileName, reporterName, reporterOptions, config);
+
+  await runner.runAll();
+
+  // Test reporter results
+});
+```
+
+## Limitations
+
+When passing config to `results`, `error`, and `afterAll`, Pa11y CI Reporter Runner includes the same properties as Pa11y CI except the `browser` property (with the `puppeteer` `browser` object used by Pa11y CI). If the `browser` object is needed, testing should be done with Pa11y CI to ensure proper `browser` configuration.

package/index.js

@@ -0,0 +1,125 @@
+'use strict';
+
+/**
+ * Pa11y CI Reporter Runner allows a pa11y-ci reporter to be run
+ * from a JSON results file without using pa11y-ci.
+ *
+ * @module pa11y-ci-reporter-runner
+ */
+
+const fs = require('fs');
+const formatter = require('./lib/formatter');
+const reporterBuilder = require('./lib/reporterBuilder');
+const createConfig = require('./lib/config');
+
+const loadPa11yciResults = fileName => {
+    if (typeof fileName !== 'string') {
+        throw new TypeError('fileName must be a string');
+    }
+
+    try {
+        const results = JSON.parse(fs.readFileSync(fileName, 'utf8'));
+        return formatter.convertJsonToResultsObject(results);
+    }
+    catch (err) {
+        throw new Error(`Error loading results file - ${err.message}`);
+    }
+};
+
+const isError = results => results.length === 1 && results[0] instanceof Error;
+
+/**
+ * Pa11y CI configuration allows config.urls entries to be either url strings
+ * or object with url and other configuration, so this return a list of just
+ * the url strings.
+ *
+ * @private
+ * @static
+ * @param   {object[]} values The urls array from configuration.
+ * @returns {string[]}        Array of URL strings.
+ */
+const getUrlList = values => {
+    const result = [];
+    for (const value of values) {
+        if (typeof value === 'string') {
+            result.push(value);
+        }
+        else if (typeof value.url === 'string') {
+            result.push(value.url);
+        }
+        else {
+            throw new TypeError('invalid url element');
+        }
+    }
+    return result;
+};
+
+/**
+ * Compares URLs in Pa1y CI results and configuration files to ensure
+ * consistency (i.e. The same URLs are in both lists, although they
+ * may not be in the same order). URLs in config are only checked if
+ * provided. If specified and inconsistent, throws error.
+ *
+ * @private
+ * @static
+ * @param   {object} results Pa11y CI JSON results file.
+ * @param   {object} config  Pa11y CI configuration.
+ */
+const validateUrls = (results, config) => {
+    // Valid if no urls specified in config
+    if (!config.urls) {
+        return;
+    }
+    const resultUrls = Object.keys(results.results);
+    const configUrls = getUrlList(config.urls);
+    if (resultUrls.length !== configUrls.length ||
+        JSON.stringify(resultUrls.sort()) !== JSON.stringify(configUrls.sort())) {
+        throw new TypeError('config.urls is specified and does not match results');
+    }
+};
+
+/**
+ * Factory to create a pa11y-ci reporter runner that can execute
+ * a reporter with the specified pa11y-ci JSON results file.
+ *
+ * @public
+ * @static
+ * @param   {string} resultsFileName Pa11y CI JSON results file.
+ * @param   {string} reporterName    Name of the reporter to execute (module or path).
+ * @param   {object} options         The reporter options.
+ * @param   {object} config          The Pa11y CI configuration.
+ * @returns {object}                 A Pa11y CI reporter runner.
+ */
+const createRunner = (resultsFileName, reporterName, options = {}, config = {}) => {
+    const pa11yciResults = loadPa11yciResults(resultsFileName);
+
+    validateUrls(pa11yciResults, config);
+
+    const pa11yciConfig = createConfig(config);
+    const reporter = reporterBuilder.buildReporter(reporterName, options, pa11yciConfig.defaults);
+
+    const runAll = async () => {
+        await reporter.beforeAll(config.urls || Object.keys(pa11yciResults.results));
+
+        for (const url of Object.keys(pa11yciResults.results)) {
+            await reporter.begin(url);
+
+            const urlResults = pa11yciResults.results[url];
+            const urlConfig = pa11yciConfig.getConfigForUrl(url);
+            if (isError(urlResults)) {
+                await reporter.error(urlResults[0], url, urlConfig);
+            }
+            else {
+                await reporter.results(formatter.getPa11yResultsFromPa11yCiResults(url, pa11yciResults), urlConfig);
+            }
+        }
+
+        await reporter.afterAll(pa11yciResults, pa11yciConfig.defaults);
+    };
+
+    return {
+        runAll
+    };
+};
+
+module.exports = createRunner;

package/lib/config.js

@@ -0,0 +1,54 @@
+'use strict';
+
+/**
+ * Manages Pa11y CI configuration.
+ *
+ * @module config
+ */
+
+const { defaultsDeep, omit } = require('lodash');
+
+const defaultConfig = require('./default-config');
+
+const normalizeConfig = (config) => {
+    // Remove log and reporters to be consistent with pa11y-ci
+    const configDefaults = omit(config.defaults || {}, ['log', 'reporters']);
+    return {
+        defaults: defaultsDeep({}, configDefaults, defaultConfig),
+        urls: config.urls || []
+    };
+};
+
+/**
+ * Factory function that returns a config object for managing PA11y CI
+ * configuration including defaults and determining the consolidated
+ * configuration for each URL.
+ *
+ * @static
+ * @param   {object} config The Pa11y CI configuration.
+ * @returns {object}        Config object.
+ */
+const configFactory = (config) => {
+    const { defaults, urls } = normalizeConfig(config);
+
+    const getConfigForUrl = (url) => {
+        if (!urls || urls.length === 0) {
+            return defaults;
+        }
+
+        // Config URLs are validated against results, if specified, so will find a result.
+        const result = urls.find(urlObject => urlObject === url || urlObject.url === url);
+        if (typeof result === 'string') {
+            return defaults;
+        }
+
+        return defaultsDeep({}, result, defaults);
+    };
+
+    return {
+        defaults,
+        getConfigForUrl
+    };
+};
+
+module.exports = configFactory;

package/lib/default-config.js

@@ -0,0 +1,17 @@
+'use strict';
+
+/**
+ * Default pa11y-ci configuration.
+ *
+ * @module default-config
+ */
+
+const defaultColumnWidth = 80;
+
+const defaultConfig = {
+    wrapWidth: process.stdout.columns || defaultColumnWidth,
+    concurrency: 1,
+    useIncognitoBrowserContext: true
+};
+
+module.exports = defaultConfig;

package/lib/formatter.js

@@ -0,0 +1,66 @@
+'use strict';
+
+/**
+ * Reformats pa11y-ci JSON data.
+ *
+ * @module formatter
+ */
+
+/**
+ * Converts Pa11y CI JSON output to an equivalent Pa11y CI object,
+ * allowing JSON result files to be used for reporter interface
+ * testing. Error messages are converted to Error objects.
+ *
+ * @static
+ * @param   {object} jsonResults Pa11y CI JSON results.
+ * @returns {object}             The equivalent Pa11y CI object.
+ */
+const convertJsonToResultsObject = jsonResults => {
+    const results = {
+        total: jsonResults.total,
+        passes: jsonResults.passes,
+        errors: jsonResults.errors,
+        results: {}
+    };
+
+    for (const url of Object.keys(jsonResults.results)) {
+        const issues = jsonResults.results[url];
+        let formattedIssues;
+        if (issues.length === 1 && Object.keys(issues[0]).length === 1 && issues[0].message) {
+            formattedIssues = [new Error(issues[0].message)];
+        }
+        else {
+            formattedIssues = issues;
+        }
+        results.results[url] = formattedIssues;
+    }
+
+    return results;
+};
+
+/**
+ * Checks the given Pa11y CI results for the given URL, and if found
+ * returns a Pa11y results object of the format sent to the reporter
+ * results event. Throws if the url is not found in the results.
+ *
+ * @static
+ * @param   {string} url     The URL to find results for,.
+ * @param   {object} results Pa11y CI results object.
+ * @returns {object}         The Pa11y results object for the URL.
+ */
+const getPa11yResultsFromPa11yCiResults = (url, results) => {
+    const issues = results.results[url];
+    if (!issues) {
+        throw new Error(`Results for url "${url} not found`);
+    }
+
+    return {
+        documentTitle: `Title for ${url}`,
+        pageUrl: url,
+        issues
+    };
+};
+
+module.exports.convertJsonToResultsObject = convertJsonToResultsObject;
+module.exports.getPa11yResultsFromPa11yCiResults = getPa11yResultsFromPa11yCiResults;
+

package/lib/reporterBuilder.js

@@ -0,0 +1,52 @@
+'use strict';
+
+const path = require('path');
+
+const reporterMethods = ['beforeAll', 'begin', 'results', 'error', 'afterAll'];
+const noop = () => {};
+
+const loadReporter = reporterName => {
+    try {
+        // eslint-disable-next-line node/global-require
+        return require(reporterName);
+    }
+    catch {
+        // eslint-disable-next-line node/global-require
+        return require(path.resolve(process.cwd(), reporterName));
+    }
+};
+
+/**
+ * Creates a reporter object for the specified path, following the logic
+ * used by pa11y-ci for reporter generation. Unlike pa11y-ci, ensures a
+ * function exists for each reporter event to allow manual execution.
+ *
+ * @param   {string} reporterName    Name of the reporter to execute
+ *                                   (module or path).
+ * @param   {object} options         The reporter options.
+ * @param   {object} config          The Pa11y CI configuration.
+ * @returns {object}                 The reporter object.
+ */
+const buildReporter = (reporterName, options, config) => {
+    if (typeof reporterName !== 'string') {
+        throw new TypeError('reporterName must be a string');
+    }
+
+    try {
+        let reporter = loadReporter(reporterName);
+        if (typeof reporter === 'function') {
+            reporter = reporter(options, config);
+        }
+        reporterMethods.forEach(method => {
+            if (typeof reporter[method] !== 'function') {
+                reporter[method] = noop;
+            }
+        });
+        return reporter;
+    }
+    catch (err) {
+        throw new Error(`Error loading reporter: ${err.message}`);
+    }
+};
+
+module.exports.buildReporter = buildReporter;

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "pa11y-ci-reporter-runner",
+  "version": "0.5.0",
+  "description": "Pa11y CI Reporter Runner is designed to facilitate testing of Pa11y CI reporters. Given a Pa11y CI JSON results file and optional configuration it simulates the Pa11y CI calls to the reporter.",
+  "main": "index.js",
+  "scripts": {
+    "test": "jest --ci",
+    "lint-js": "eslint .",
+    "lint-md": "markdownlint **/*.md --ignore node_modules --ignore Archive",
+    "lint": "npm run lint-js && npm run lint-md",
+    "push": "npm run lint && npm audit --audit-level=high && npm test"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://gitlab.com/gitlab-ci-utils/pa11y-ci-reporter-runner.git"
+  },
+  "keywords": [
+    "pa11y-ci",
+    "pa11y",
+    "reporter",
+    "test",
+    "testing"
+  ],
+  "author": "Aaron Goldenthal <npm@aarongoldenthal.com>",
+  "license": "MIT",
+  "engines": {
+    "node": "^12.20.0 || ^14.15.0 || >=16.0.0"
+  },
+  "files": [
+    "index.js",
+    "lib/"
+  ],
+  "bugs": {
+    "url": "https://gitlab.com/gitlab-ci-utils/pa11y-ci-reporter-runner/issues"
+  },
+  "homepage": "https://gitlab.com/gitlab-ci-utils/pa11y-ci-reporter-runner#readme",
+  "devDependencies": {
+    "@aarongoldenthal/eslint-config-standard": "^11.0.0",
+    "eslint": "^8.8.0",
+    "jest": "^27.4.7",
+    "jest-junit": "^13.0.0",
+    "markdownlint-cli": "^0.30.0",
+    "pa11y-ci-reporter-cli-summary": "^1.0.1"
+  },
+  "dependencies": {
+    "lodash": "^4.17.21"
+  }
+}