@suffolklitlab/alkiln 4.5.0 → 4.6.0

package/.github/workflows/test_tests.yml

@@ -5,9 +5,11 @@ on:
   workflow_dispatch:
     inputs:
       extra_languages:
+        required: False
         description: 'Optional. A list of comma separated language names visible on buttons or links that change the language of the interview. Overrides the EXTRA_LANGUAGES repo secret.'
         default: ''
       tags:
+        required: False
         description: 'Optional. Use a "tag expression" specify which tagged tests to run (https://cucumber.io/docs/cucumber/api/#tag-expressions)'
         default: ''
 
@@ -78,6 +80,13 @@ jobs:
         with:
           name: report
           path: ./**/*_report_*
+      - name: upload a11y reports
+        uses: actions/upload-artifact@v2
+        # They will upload even if a previous step has failed
+        if: ${{ always() }}
+        with:
+          name: a11y report
+          path: ./**/aXe_failure_*
       - name: upload debug_log artifact
         uses: actions/upload-artifact@v2
         if: ${{ always() }}

package/CHANGELOG.md

@@ -40,7 +40,23 @@ Format:
 -->
 <!-- ## [Unreleased] -->
 
+## [4.6.0] - 2022-06-10
+### Added
+- Steps to test web accessibility either on individual page or for all subsequent pages.
+
+## [4.5.2] - 2022-06-10
+### Fixed
+- (internal) Expected in report is reset properly
+  - if a test used "the Scenario report should include ...", then all tests that were
+    run after that would pass even if the test itself failed but wasn't expected to fail
+
+## [4.5.1] - 2022-06-10
+### Fixed
+- Corrected some schema errors in the `action.yml` (the `run` key was specified twice, should only have been once, and `required: False` is needed
+  for optional inputs)
+
 ## [4.5.0] - 2022-05-23
+### Added
 - Steps that tap tabs (made using the `tabbed_templates_html` function from ALToolbox) and other arbitrary elements on the page.
 
 ## [4.4.0] - 2022-05-23

package/action.yml

@@ -61,8 +61,9 @@ runs:
         echo "MAX_SECONDS_FOR_SETUP=${{ inputs.MAX_SECONDS_FOR_SETUP }}" >> $GITHUB_ENV
       shell: bash
     - name: "ALKiln: confirm info"
-      run: echo -e "\nRepo is $REPO_URL\nBranch ref is $BRANCH_PATH\nManually added languages are $EXTRA_LANGUAGES\nAll other data is secret\n"
-      run: echo -e "\nALKiln version is $ALKILN_VERSION\nRepo is $REPO_URL\nBranch ref is $BRANCH_PATH\nMAX_SECONDS_FOR_SETUP is $MAX_SECONDS_FOR_SETUP\nManually added languages are $EXTRA_LANGUAGES\n"
+      run: |
+        echo -e "\nRepo is $REPO_URL\nBranch ref is $BRANCH_PATH\nManually added languages are $EXTRA_LANGUAGES\nAll other data is secret\n"
+        echo -e "\nALKiln version is $ALKILN_VERSION\nRepo is $REPO_URL\nBranch ref is $BRANCH_PATH\nMAX_SECONDS_FOR_SETUP is $MAX_SECONDS_FOR_SETUP\nManually added languages are $EXTRA_LANGUAGES\n"
       shell: bash
 
     # This doesn't use an already established package-lock.json and that
@@ -146,6 +147,13 @@ runs:
       with:
         name: report
         path: ./**/*_report_*
+    - name: "ALKiln: Upload Accessibility failures, if any, to action summary artifacts"
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v2
+      # They will upload even if a previous step has failed
+      with:
+        name: a11y report
+        path: ./**/aXe_failure*
 
 # ##################################
 # # A developer's workflow should look similar to the below

package/lib/cucumber_8_shim.js

@@ -1,7 +1,7 @@
 module.exports = async function wrapPromiseWithTimeout (
   promise,
   timeoutInMilliseconds,
-  timeoutMessage
+  timeoutMessage=''
 ) {
   /** Race between a timeout and another function and clean up
   *    async afterwards.

package/lib/scope.js

@@ -2,12 +2,15 @@ const { expect } = require('chai');
 const cheerio = require('cheerio');
 const path = require('path');
 const fs = require('fs');
+const { AxePuppeteer } = require('@axe-core/puppeteer');
 const safe_filename = require("sanitize-filename");
 
 // Ours
 const session_vars = require('./utils/session_vars');
 const get_base_interview_url = require(`./docassemble/get_base_interview_url`);
 const time = require('./utils/time');
+const log = require('./utils/log');
+const { waitForTimeout } = require('./utils/time');
 
 
 let ordinal_to_integer = {
@@ -25,7 +28,7 @@ let escape_regex = function( to_escape ) {
 let get_original_row = function (row) {
   /* `row` might be a row, a source row, or an original row. */
 
-  // Until we flatten row data a litte, we need to find the `orignal` row, which has the flags
+  // Until we flatten row data a little, we need to find the `original` row, which has the flags
   let source_row = row.source || row;
   let original = source_row.original || source_row;
   return original;
@@ -56,11 +59,11 @@ module.exports = {
     * 
     * The report has a key for each scenario and each of those
     * keys should correspond to a list containing the relevant data. */
-    if ( session_vars.get_debug() ) { console.log(`Report:\ntype: ${ type }, value: ${ value }`) }
+    if ( session_vars.get_debug() ) { console.log(`Report:\ntype: ${ type }, value: ${ value }\n`) }
     let scenario = await scope.makeSureReportPartsExist( scope );
     scenario.lines.push({ type, value });
 
-    fs.appendFileSync('debug_log.txt', `Report:\ntype: ${ type }, value: ${ value }`);
+    log.add_to_debug_log(`Report:\ntype: ${ type }, value: ${ value }\n`);
     return scenario;
     
   },  // Ends scope.addToReport()
@@ -475,6 +478,44 @@ module.exports = {
     await scope.tapElement( scope, elem, tab_id );
   },
 
+  checkForA11y: async function (scope ) {
+    /** Return if there were any violations and the file containing aXe output with more details of the checks */
+    let { id } = await scope.examinePageID( scope, 'none to match' );
+    if ( scope.page_id !== id ) {
+      await scope.addToReport(scope, { type: `page_id`, value: `${ id }`, });
+      scope.page_id = id;
+    }
+    const axe = new AxePuppeteer(scope.page);
+    axe.options({
+      runOnly: ["wcag2a", "wcag2aa", "wcag21a", "wcag21aa"],
+      resultTypes: ['violations', 'incomplete']
+    });
+    let axe_result = await axe.analyze();
+    axe_result.page_id = scope.page_id;
+    axe_result.passes_ids = axe_result.passes.map(pass_obj => pass_obj.id);
+    axe_result.inapplicable_ids = axe_result.inapplicable.map(inapp_obj => inapp_obj.id);
+    axe_result.passes = [];
+    axe_result.inapplicable = [];
+
+    let axe_filename = null;
+    const passed = axe_result.violations.length == 0;
+    if (!passed) {
+      axe_filename = await scope.getSafeScenarioFilename( scope, {prefix: `aXe_failure_${ scope.page_id }`}) + '.json';
+      fs.writeFileSync( axe_filename, JSON.stringify( axe_result, null, 2 ));
+      let msg = `Found potential accessibility issues on the ${ scope.page_id } screen. Details in ${ axe_filename }.`;
+      await scope.addToReport( scope, {
+        type: `error`,
+        value: msg
+      });
+    } else {
+      await scope.addToReport(scope, { type: `info`, value: `Accessibility on ${ scope.page_id } passed!`})
+    }
+    return {
+      passed: axe_result.violations.length == 0,
+      axe_filename: axe_filename
+    };
+  },
+
   tapElementBySelector: async function ( scope, selector, waitForTimeout ) {
     let elem = await scope.page.$(`${selector}`);
     let error_msg = `Couldn't find "${selector}" on the page, is there a typo?`;
@@ -1641,9 +1682,17 @@ module.exports = {
       //   await scope.throwPageError( scope, { id: id });  
       // }
 
+      did_navigate = true;
       process.stdout.write(`\x1b[36m${ '>' }\x1b[0m`);  // continued successfully
     }  // ends if !did_navigate && ensure_navigation
 
+    if ( did_navigate && scope.check_all_for_a11y ) {
+      let { passed } = await scope.checkForA11y(scope);
+      if (!passed) {
+        scope.passed_all_for_a11y = false;
+      }
+    }
+
     // TODO: Since we now continue only once no matter what, we can put
     // the error in the caller of the function, though we may want
     // to pass the symbol to print as well.

package/lib/steps.js

@@ -18,11 +18,13 @@ const { v4: uuidv4 } = require('uuid');
 
 // Ours
 const scope = require('./scope');
+const log = require('./utils/log');
 const session_vars = require('./utils/session_vars');
 /* Of Note:
 - We're using `*=` because sometimes da text has funny characters in it that are hard to anticipate
 */
 
+
 require("dotenv").config();  // Where did this go?
 
 /* TODO:
@@ -68,11 +70,13 @@ BeforeAll(async () => {
   scope.timeout = 30 * 1000;
   scope.showif_timeout = 600;
   scope.report = new Map();
-  fs.writeFileSync('debug_log.txt', "");
+  fs.writeFileSync(log.debug_log_file, "");
 });
 
 Before(async (scenario) => {
   scope.scenario_id = uuidv4();
+  scope.expected_status = null;
+  scope.expected_in_report = null;
 
   await scope.addReportHeading(scope, {scenario});
   scope.base_filename = await scope.getSafeScenarioBaseFilename(scope, {scenario});
@@ -86,6 +90,8 @@ Before(async (scenario) => {
   scope.disable_error_screenshot = false;
 
   scope.page_id = null;
+  scope.check_all_for_a11y = false;
+  scope.passed_all_for_a11y = true;
 
   // Reset default timeout
   scope.timeout = 30 * 1000;
@@ -728,6 +734,19 @@ When(/I tap the "([^"]+)" element and wait ([0-9]+) second(?:s)?/i, { timeout: -
   )
 });
 
+When(/I check the page for (?:accessibility|a11y) issues/i, {timeout: -1}, async () => {
+  let { passed, axe_filename } = await wrapPromiseWithTimeout(
+    scope.checkForA11y(scope),
+    scope.timeout
+  )
+  let msg = `Found potential accessibility issues on the ${ scope.page_id } screen. Details in ${ axe_filename }.`;
+  expect( passed, msg ).to.be.true;
+});
+
+When(/I check all pages for (?:accessibility|a11y) issues/i, {timeout: -1}, async () => {
+  scope.check_all_for_a11y = true;
+  scope.passed_all_for_a11y = true;
+});
 
 //#####################################
 // UI element interaction
@@ -891,6 +910,11 @@ Given(/the final Scenario status should be "(passed|failed)"/i, async ( expected
 //#####################################
 
 After(async function(scenario) {
+  // The accessibility failures don't happen til the end, so the rest of the test can continue
+  if ( scope.check_all_for_a11y && !scope.passed_all_for_a11y) {
+    // The fact that the scenario failed has already been noted earlier in the report
+    scenario.result.status = `FAILED`;
+  }
 
   // Record status before internal report tests can interfere with values
   await scope.setReportScenarioStatus( scope, { status: scenario.result.status });
@@ -930,11 +954,10 @@ After(async function(scenario) {
     scope.expected_status = null;  // reset for next Scenario
     expect( scenario.result.status ).to.equal( expected );
   }
-  if ( scope.expected_in_report ) {
+  if ( scope.expected_in_report && scope.expected_in_report.length > 0) {
     let expected = scope.expected_in_report;
     // Reset report values no matter what so they don't mess up future scenarios
-    // Maybe this should be in `Before`, but it would be so out of context there...
-    scope.expected_in_report = [];
+    scope.expected_in_report = null;
     let all_were_included = await scope.reportIncludesAllExpected( scope, { expected });
     if ( all_were_included ) { scenario.result.status = `PASSED`; }
   }

package/lib/utils/clean_reports.js

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 
 const fs = require('fs');
+const log = require('./log');
 
 // This is hardcoded information from `cucumber.js`. Once we move to cucumber 8, we should
 // change that the cucumber.json and actually parse the info from it. 
@@ -13,8 +14,7 @@ fs.readFile(summary_file_name, 'utf8', (err, data) => {
   // Replace terminal color codes, if present
   // https://gist.github.com/fnky/458719343aabd01cfb17a3a4f7296797#colors--graphics-mode
   data = data.replace(/\x1b\[[0-9][0-9]?[0-9]?m/g, '');
-  const debug_log_file = 'debug_log.txt';
-  fs.appendFile(debug_log_file, data, err => {
+  fs.appendFile(log.debug_log_file, data, err => {
     if (err) {
       console.error(err);
       return;

package/lib/utils/log.js

@@ -1,3 +1,4 @@
+const fs = require('fs');
 const session_vars = require( './session_vars' );
 
 let log = {};
@@ -49,3 +50,9 @@ log.debug = function ({ type='', pre='', data='', post='' }) {
     if ( post ) { console.log( post ); }
   }
 };
+
+log.add_to_debug_log = function(value) {
+  fs.appendFileSync(log.debug_log_file, value);
+};
+
+log.debug_log_file = 'debug_log.txt'

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@suffolklitlab/alkiln",
-  "version": "4.5.0",
+  "version": "4.6.0",
   "description": "Integrated automated end-to-end testing with docassemble, puppeteer, and cucumber.",
   "main": "lib/index.js",
   "scripts": {
     "test": "npm run unit && npm run langs_setup && npm run cucumber -- $@",
     "cucumber_base": "run_cucumber(){ ./node_modules/.bin/cucumber-js \"$@\"; status=$?; ./lib/utils/clean_reports.js; return $status; }; run_cucumber",
     "cucumber": "run_base(){ npm run cucumber_base -- \"$@\" tests/features/*.feature; }; run_base",
+    "cucumber_external": "run_cucumber(){ tmp_path=`pwd`; ( new_path=\"$1\"; shift; cd \"$new_path\"; npm run --prefix \"${tmp_path}\" cucumber_base -- \"$@\" \"$new_path\"/docassemble/*/data/sources/*.feature;) }; run_cucumber",
     "langs": "npm run langs_setup && ./node_modules/.bin/cucumber-js --require ./lib/index.js tests/features/*_lang*.feature",
     "langs_setup": "./lib/utils/langs.js 'tests/features/*.feature'",
     "langs_local": "npm run langs_setup && npm run local",
@@ -34,6 +35,7 @@
   },
   "homepage": "https://suffolklitlab.org/docassemble-AssemblyLine-documentation/docs/automated_integrated_testing/",
   "dependencies": {
+    "@axe-core/puppeteer": "4.4.2",
     "@cucumber/cucumber": "7.3.2",
     "axios": "0.24.0",
     "chai": "4.2.0",

package/tests/features/interactive_steps.feature

@@ -41,6 +41,11 @@ Scenario: I set text-type values
   And I set the var "showif_dropdown" to "showif_dropdown_1"
   When I tap to continue
   # Next page
+  Then the question id should be "object checkboxes"
+  And I set the var "object_checkboxes_test['obj_chkbx_opt_1']" to "true" 
+  And I set the var "object_dropdown" to "obj_opt_2"
+  When I tap to continue
+  # Next page
   Then the question id should be "buttons yesnomaybe"
   When I set the var "buttons_yesnomaybe" to "None"
   # Next page
@@ -114,7 +119,7 @@ Scenario: tap elements
   And I tap the "#special_event" element and wait 1 second
   Then I see the phrase "Portishead"
 
-@i4 @tap-elements @tabs
+@i4 @tap-elements @tabs @error
 Scenario: tap element with an error
   Given the final Scenario status should be "failed"
   And the Scenario report should include:

package/tests/features/observation_steps.feature

@@ -46,6 +46,8 @@ Scenario: I check navigation
   Then I arrive at the next page
   Then I get to "screen features" with this data:
     | var | value | trigger |
+    | object_checkboxes_test["obj_chkbx_opt_1"] | True | |
+    | object_dropdown | obj_opt_2 | |
     | button_continue | True |  |
     | buttons_other | button_2 |  |
     | buttons_yesnomaybe | True |  |
@@ -121,8 +123,31 @@ Scenario: I get the page's JSON
   Given I start the interview at "all_tests"
   Then I get the page's JSON variables and values
 
-@fast @o9
+@fast @o11
 Scenario: I take a screenshot
   Given I start the interview at "all_tests"
   Then I take a screenshot
   Then I take a screenshot named "some-screenshot"
+
+@slow @o12 @accessibility @a11y
+Scenario: I check the pages for accessibility
+  Given I start the interview at "all_tests"
+  And I check all pages for accessibility issues
+  And I tap to continue
+  Then I get to "screen features" with this data:
+    | var | value | trigger |
+    | double_quote_dict["double_quote_key"]['dq_two'] | true |  |
+    | checkboxes_other['checkbox_other_opt_1'] | true |  |
+    | dropdown_test | dropdown_opt_2 | |
+    | radio_yesno | False | false |
+    | radio_other | radio_other_opt_3 | |
+    | object_checkboxes_test["obj_chkbx_opt_1"] | True | |
+    | object_dropdown | obj_opt_2 | |
+    | object_select_test | obj_chkbx_opt_2 | |
+    | single_quote_dict['single_quote_key']['sq_two'] | true |  |
+    | text_input | Regular text input field value | |
+    | textarea | Multiline text\narea value | |
+    | date_input | today | |
+    | button_continue | True |  |
+    | buttons_other | button_2 |  |
+    | buttons_yesnomaybe | True |  |

package/tests/features/reports.feature

@@ -160,6 +160,8 @@ Scenario: Fail with link with given text does not lead to correct url
   Then I arrive at the next page
   Then I get to "screen features" with this data:
     | var | value | trigger |
+    | object_checkboxes_test["obj_chkbx_opt_1"] | true | |
+    | object_dropdown | obj_opt_2 | |
     | button_continue | True |  |
     | buttons_other | button_2 |  |
     | buttons_yesnomaybe | True |  |
@@ -188,6 +190,8 @@ Scenario: Fail with link unexpectedly opens in same window
   Then I arrive at the next page
   Then I get to "screen features" with this data:
     | var | value | trigger |
+    | object_checkboxes_test["obj_chkbx_opt_1"] | true | |
+    | object_dropdown | obj_opt_2 | |
     | button_continue | True |  |
     | buttons_other | button_2 |  |
     | buttons_yesnomaybe | True |  |
@@ -216,6 +220,8 @@ Scenario: Fail with link unexpectedly opens in a new window
   Then I arrive at the next page
   Then I get to "screen features" with this data:
     | var | value | trigger |
+    | object_checkboxes_test["obj_chkbx_opt_1"] | true | |
+    | object_dropdown | obj_opt_2 | |
     | button_continue | True |  |
     | buttons_other | button_2 |  |
     | buttons_yesnomaybe | True |  |
@@ -472,6 +478,11 @@ Scenario: Report still shows page id when I tap to continue without setting any
   # Next page
   Then the question id should be "showifs"
   When I tap to continue
+  # Next page
+  Then the question id should be "object checkboxes"
+  And I set the var "object_checkboxes_test['obj_chkbx_opt_1']" to "true" 
+  And I set the var "object_dropdown" to "obj_opt_2"
+  When I tap to continue
   # Next page (showifs ID SHOULD BE SHOWN IN REPORT)
   Then the question id should be "buttons yesnomaybe"
 
@@ -517,6 +528,8 @@ Scenario: Report lists unused table rows
     | extra_5 | extra 5 |  |
   Then I get to "screen features" with this data:
     | var | value | trigger |
+    | object_checkboxes_test["obj_chkbx_opt_1"] | true | |
+    | object_dropdown | obj_opt_2 | |
     | extra_6 | extra 6 |  |
     | extra_7 | extra 7 |  |
     | button_continue | True |  |

package/tests/features/story_tables.feature

@@ -47,6 +47,8 @@ Covers story table tests for:
     | checkboxes_yesno | True |  |
     | direct_standard_fields | True |  |
     | dropdown_test | dropdown_opt_2 |  |
+    | object_checkboxes_test["obj_chkbx_opt_1"] | true | |
+    | object_dropdown | obj_opt_2 | |
     | radio_other | radio_other_opt_3 |  |
     | radio_yesno | False |  |
     | screen_features | True |  |