@jrpool/kilotest 24.0.4

package/issues/index.js

@@ -0,0 +1,173 @@
+/*
+  index.js
+  Answers the targets question.
+*/
+
+// IMPORTS
+
+const {sendAlert} = require('../alerts');
+const {
+  annotateReport,
+  getReport,
+  getToolNamesString,
+  getLogs,
+  getWCAGLink,
+  getWeightName,
+  objectSort,
+  ruleIDs
+} = require('../util');
+const {issues} = require('testilo/procs/score/tic');
+const fs = require('fs/promises');
+const path = require('path');
+
+// FUNCTIONS
+
+// Gets summary data on the issues reported in a set of reports.
+const getIssuesSummary = async logs => {
+  // Initialize data for a summary.
+  const issuesData = {};
+  // For each log of a report to be inspected:
+  for (const log of logs) {
+    const {annotated, jobName} = log;
+    const [timeStamp, jobID] = jobName.split('-');
+    // If the corresponding report is not yet annotated:
+    if (! annotated) {
+      // Annotate it and mark it as annotated in the log.
+      await annotateReport(ruleIDs, timeStamp, jobID);
+    }
+    // Get the corresponding report.
+    const report = await getReport(timeStamp, jobID);
+    // For each act in it:
+    report.acts.forEach(act => {
+      // If it is a test act:
+      if (act.type === 'test') {
+        const {result, which} = act;
+        const instances = result?.standardResult?.instances ?? [];
+        // For each of its standard instances:
+        instances.forEach(instance => {
+          const {count, issueID} = instance;
+          // If the instance has a non-ignorable issue ID:
+          if (issueID && issueID !== 'ignorable') {
+            issuesData[issueID] ??= {
+              count: 0,
+              reporters: new Set()
+            };
+            // Increment the data with the count and reporter of the instance.
+            issuesData[issueID].count += count ?? 1;
+            issuesData[issueID].reporters.add(which);
+          }
+        });
+      }
+    });
+  }
+  // Initialize the summary.
+  const summary = {
+    totalCount: 0,
+    issues: []
+  };
+  // For each issue:
+  Object.entries(issuesData).forEach(([issueID, data]) => {
+    const {count, reporters} = data;
+    // If the issue is still classified:
+    if (issues[issueID]) {
+      // Increment the report violation count by the issue violation count.
+      summary.totalCount += count;
+      // Add the issue data and an initilized percentage to the summary.
+      summary.issues.push({
+        issueID,
+        weight: issues[issueID].weight,
+        count,
+        percentage: 0,
+        reporters: getToolNamesString(reporters)
+      });
+    }
+    // Otherwise, i.e. if it is no longer classified:
+    else {
+      // Report this.
+      console.log(`ERROR: Annotations obsolete for issue ${issueID}; reannotate`);
+      // Notify a manager.
+      sendAlert('Annotations obsolete', `Annotations for issue ${issueID} obsolete; reannotate.`);
+    }
+  });
+  // For each summarized issue:
+  summary.issues.forEach(issue => {
+    // Add its percentage to its entry.
+    issue.percentage = Math.round(100 * (issue.count / summary.totalCount));
+  });
+  // Sort the issues in descending count order.
+  objectSort(summary.issues, 'count', 'numericDown');
+  // Sort the issues in descending priority order.
+  objectSort(summary.issues, 'weight', 'numericDown');
+  // Return the summary.
+  return summary;
+};
+// Adds parameters to a query for the answer page.
+const populateQuery = async query => {
+  // Get the logs of the latest reports on the tested targets.
+  const targetLogs = (await getLogs()).filter(log => ! log.superseded);
+  // Get summary data on the issues.
+  const issuesSummary = await getIssuesSummary(targetLogs);
+  // Initialize the lines.
+  const lines = [];
+  const margin = ' '.repeat(6);
+  // For each weight:
+  [4, 3, 2, 1].forEach(weight => {
+    // Add a heading to the lines.
+    lines.push(`${margin}<h2>${getWeightName(weight)} priority</h2>`);
+    const reportedIssues = issuesSummary.issues;
+    lines.push(`${margin}<ul>`);
+    let existsIssue = false;
+    // For each reported issue:
+    reportedIssues.forEach(reportedIssue => {
+      const {issueID, percentage, reporters} = reportedIssue;
+      // If it has the weight and its percentage is at least 2:
+      if (reportedIssue.weight === weight && percentage >= 2) {
+        existsIssue = true;
+        // Get the data on it from the issue classification.
+        const issue = issues[issueID];
+        const {summary, wcag, why} = issue;
+        const wcagLink = `<a href="${getWCAGLink(wcag)}">${wcag}</a>`;
+        // Add a description of it to the lines.
+        lines.push(`${margin}  <li>${summary}`);
+        lines.push(`${margin}    <ul>`);
+        lines.push(`${margin}      <li>Why it matters: ${why}`);
+        lines.push(`${margin}      <li>Related WCAG standard: ${wcagLink}`);
+        lines.push(`${margin}      <li>Share of violations: ${percentage}%</li>`);
+        lines.push(`${margin}      <li>Violations reported by ${reporters}</li>`);
+        lines.push(`${margin}    </ul>`);
+        lines.push(`${margin}    <ul class="nav">`);
+        const linkText = 'What rules belong to this issue?';
+        const label = `What rules belong to the <q>${summary}</q> issue?`;
+        const href = `/rules.html/${issueID}`;
+        lines.push(
+          `${margin}      <li><a href="${href}" aria-label="${label}">${linkText}</a></li>`
+        );
+        lines.push(`${margin}    </ul>`);
+        lines.push(`${margin}  </li>`);
+      }
+    });
+    lines.push(`${margin}</ul>`);
+    if (! existsIssue) {
+      lines.push(`${margin}<p>No issues with this priority.</p>`);
+    }
+  });
+  // Add the lines to the query.
+  query.issues = lines.join('\n');
+};
+// Returns a page answering the issues question.
+exports.answer = async () => {
+  const query = {};
+  // Create a query to replace placeholders.
+  await populateQuery(query);
+  // Get the template.
+  let answerPage = await fs.readFile(path.join(__dirname, 'index.html'), 'utf8');
+  // Replace its placeholders.
+  Object.keys(query).forEach(param => {
+    answerPage = answerPage.replace(new RegExp(`__${param}__`, 'g'), query[param]);
+  });
+  // Return the populated page.
+  return {
+    status: 'ok',
+    answerPage
+  };
+};

package/job.json

@@ -0,0 +1,100 @@
+{
+  "id": "",
+  "what": "Kilotest job",
+  "strict": false,
+  "standard": "only",
+  "observe": false,
+  "device": {
+    "id": "default",
+    "windowOptions": {
+      "reducedMotion": "no-preference"
+    }
+  },
+  "browserID": "chromium",
+  "creationTimeStamp": "",
+  "executionTimeStamp": "",
+  "target": {
+    "what": "",
+    "url": ""
+  },
+  "sources": {
+    "application": "Kilotest"
+  },
+  "acts": [
+    {
+      "type": "test",
+      "launch": {},
+      "which": "alfa",
+      "what": "Alfa"
+    },
+    {
+      "type": "test",
+      "launch": {},
+      "which": "aslint",
+      "what": "ASLint"
+    },
+    {
+      "type": "test",
+      "launch": {},
+      "which": "axe",
+      "what": "Axe by Deque",
+      "detailLevel": 2
+    },
+    {
+      "type": "test",
+      "launch": {},
+      "which": "ed11y",
+      "what": "Editoria11y by Princeton University"
+    },
+    {
+      "type": "test",
+      "launch": {},
+      "which": "htmlcs",
+      "what": "HTML CodeSniffer by Squiz Labs"
+    },
+    {
+      "type": "test",
+      "launch": {},
+      "which": "ibm",
+      "what": "Accessibility Checker by IBM",
+      "withItems": true,
+      "withNewContent": false
+    },
+    {
+      "type": "test",
+      "launch": {},
+      "which": "nuVal",
+      "what": "Nu Html Checker API by World Wide Web Consortium",
+      "withSource": false
+    },
+    {
+      "type": "test",
+      "launch": {},
+      "which": "nuVnu",
+      "what": "Nu Html Checker by World Wide Web Consortium",
+      "withSource": false
+    },
+    {
+      "type": "test",
+      "launch": {},
+      "which": "qualWeb",
+      "what": "QualWeb by University of Lisbon",
+      "withNewContent": false
+    },
+    {
+      "type": "test",
+      "launch": {},
+      "which": "testaro",
+      "what": "Testaro by CVS Health",
+      "withItems": true,
+      "stopOnFail": false
+    },
+    {
+      "type": "test",
+      "launch": {},
+      "which": "wave",
+      "what": "WAVE by Utah State University",
+      "reportType": 4
+    }
+  ]
+}

package/manage/index.html

@@ -0,0 +1,32 @@
+<!DOCTYPE html>
+<html lang="en-US">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="publisher" content="Jonathan Robert Pool">
+    <meta name="creator" content="Jonathan Robert Pool">
+    <meta name="keywords" content="report,accessibility,a11y">
+    <title>Management | Kilotest</title>
+    <link rel="icon" href="/favicon.ico">
+    <link rel="stylesheet" href="/style.css">
+  </head>
+  <body>
+    <main>
+      <h1><a href="/">Kilotest</a> management</h1>
+      <p>Kilotest managers can:</p>
+      <ul class="nav">
+        <li><a href="/recActionForm.html">Approve or reject a test recommendation</a></li>
+        <li><a href="/reannotateForm.html">
+          Reannotate reports to incorporate newly classified and reclassified rules
+        </a></li>
+        <li><a href="/reportsPruneForm.html">Delete superseded reports</a></li>
+        <li><a href="/reportsRewindForm.html">Delete latest superseding reports</a></li>
+        <li><a href="/reportsExpungeForm.html">Delete sole reports</a></li>
+        <li><a href="/reportHideForm.html">Classify a report as experimental</a></li>
+        <li><a href="/reportUnhideForm.html">Declassify a report as experimental</a></li>
+        <li><a href="/ai0BalanceForm.html">Record the current balance of AI service 0</a></li>
+        <li><a href="/wcagRenewForm.html">Renew the map of WCAG numeric to text identifiers</a></li>
+      </ul>
+    </main>
+  </body>
+</html>

package/manage/index.js

@@ -0,0 +1,22 @@
+/*
+  index.js
+  Answers the managers question.
+*/
+
+// IMPORTS
+
+const fs = require('fs/promises');
+const path = require('path');
+
+// FUNCTIONS
+
+// Returns the answer page.
+exports.answer = async () => {
+  // Get the answer page.
+  let answerPage = await fs.readFile(path.join(__dirname, 'index.html'), 'utf8');
+  // Return it.
+  return {
+    status: 'ok',
+    answerPage
+  };
+};

package/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@jrpool/kilotest",
+  "version": "24.0.4",
+  "description": "An ensemble testing service with a focus on accessibility",
+  "main": "index.js",
+  "scripts": {
+    "start": "node index",
+    "lint": "npx eslint",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/jrpool/kilotest"
+  },
+  "keywords": [
+    "accessibility",
+    "a11y",
+    "testing"
+  ],
+  "author": "Jonathan Robert Pool <pool@jpdev.pro>",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/jrpool/kilotest/issues"
+  },
+  "homepage": "https://github.com/jrpool/kilotest",
+  "dependencies": {
+    "dotenv": "*",
+    "testilo": "*"
+  },
+  "devDependencies": {
+    "@eslint/css": "^1.0.0",
+    "@eslint/js": "^10.0.1",
+    "@eslint/json": "^1.1.0",
+    "@eslint/markdown": "^7.5.1",
+    "eslint": "^10.0.3",
+    "globals": "^17.4.0"
+  }
+}

package/pm2.config.js

@@ -0,0 +1,15 @@
+module.exports = {
+  apps: [{
+    name: 'kilotest',
+    script: 'index.js',
+    instances: 1,
+    autorestart: true,
+    watch: false,
+    max_memory_restart: '500M',
+    env: {
+      NODE_ENV: 'production',
+      BASE_PATH: '/',
+      DEMO_SSE_DELAY_MS: '100'
+    }
+  }]
+};

package/reannotate/index.html

@@ -0,0 +1,19 @@
+<!DOCTYPE html>
+<html lang="en-US">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="publisher" content="Jonathan Robert Pool">
+    <meta name="creator" content="Jonathan Robert Pool">
+    <meta name="keywords" content="report,accessibility,a11y">
+    <title>Reannotation order | Kilotest</title>
+    <link rel="icon" href="/favicon.ico">
+    <link rel="stylesheet" href="/style.css">
+  </head>
+  <body>
+    <main>
+      <h1><a href="/">Kilotest</a>: Reannotation order</h1>
+      <p>The latest report on each target has been reannotated.</p>
+    </main>
+  </body>
+</html>

package/reannotate/index.js

@@ -0,0 +1,39 @@
+/*
+  index.js
+  Implements a reannotation order, i.e. an order to update the issue IDs of the standard instances of the latest reports on all tested targets.
+*/
+
+// IMPORTS
+
+const {annotateReport, getLogs, ruleIDs} = require('../util');
+const fs = require('fs/promises');
+const path = require('path');
+
+// FUNCTIONS
+
+// Implements a reannotation order and returns an acknowledgement page.
+exports.answer = async authCode => {
+  // If the authorization code is valid:
+  if (authCode === process.env.AUTH_CODE) {
+    // Get the logs of the latest reports per target.
+    const targetsData = (await getLogs()).filter(log => ! log.superseded);
+    // For each report:
+    for (const targetData of targetsData) {
+      const [timeStamp, jobID] = targetData.jobName.split('-');
+      // Reannotate it.
+      await annotateReport(ruleIDs, timeStamp, jobID);
+    }
+    // Get the answer page.
+    let answerPage = await fs.readFile(path.join(__dirname, 'index.html'), 'utf8');
+    // Return it.
+    return {
+      status: 'ok',
+      answerPage
+    };
+  }
+  // Otherwise, i.e. if the authorization code is invalid, return an error page.
+  return {
+    status: 'error',
+    error: 'Invalid authorization code'
+  };
+};

package/reannotateForm/index.html

@@ -0,0 +1,29 @@
+<!DOCTYPE html>
+<html lang="en-US">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="publisher" content="Jonathan Robert Pool">
+    <meta name="creator" content="Jonathan Robert Pool">
+    <meta name="keywords" content="report,accessibility,a11y">
+    <title>Reannotation order | Kilotest</title>
+    <link rel="icon" href="/favicon.ico">
+    <link rel="stylesheet" href="/style.css">
+  </head>
+  <body>
+    <main>
+      <h1><a href="/">Kilotest</a>: Reannotation order</h1>
+      <p>These are the unclassified rules whose violations have been reported. Only the most recent report on each target is inspected.</p>
+      <h2>Still unclassified</h2>
+      <ul>
+        __stillUnclassified__
+      </ul>
+      <h2>Reclassified</h2>
+      <ul>
+        __reClassified__
+      </ul>
+      <p>__how__</p>
+      __reannotateForm__
+    </main>
+  </body>
+</html>

package/reannotateForm/index.js

@@ -0,0 +1,114 @@
+/*
+  index.js
+  Discloses unclassified rules.
+*/
+
+// IMPORTS
+
+const {getIssue, getReport, getLogs, ruleIDs} = require('../util');
+const fs = require('fs/promises');
+const path = require('path');
+
+// FUNCTIONS
+
+// Adds parameters to a query for the answer page.
+const populateQuery = async query => {
+  const targetLogs = (await getLogs()).filter(log => ! log.superseded);
+  const stillUnclassified = {};
+  const reClassified = {};
+  // For each target:
+  for (const targetLog of targetLogs) {
+    const {jobName} = targetLog;
+    // Get the latest report on it.
+    const report = await getReport(... jobName.split('-'));
+    const {acts} = report;
+    // For each act in the report:
+    acts.forEach(act => {
+      const {result, type, which} = act;
+      // If it is a test act with standard instances:
+      if (type === 'test' && result?.standardResult?.instances?.length) {
+        // For each standard instance:
+        result.standardResult.instances.forEach(instance => {
+          const {ruleID} = instance;
+          // Get the issue ID of the rule, or null if none.
+          const issueID = getIssue(ruleIDs, which, ruleID);
+          // If the issue ID of the instance differs from that of the rule:
+          if ((instance.issueID || null) !== issueID) {
+            // Add the rule and the report to the rules with changed issue IDs.
+            reClassified[which] ??= {};
+            reClassified[which][ruleID] ??= new Set();
+            reClassified[which][ruleID].add(jobName);
+          }
+          // Otherwise, if the instance and the rule both have no issue ID:
+          else if (! issueID){
+            // Add the rule and the report to the rules that are still unclassified.
+            stillUnclassified[which] ??= {};
+            stillUnclassified[which][ruleID] ??= new Set();
+            stillUnclassified[which][ruleID].add(jobName);
+          }
+        });
+      }
+    });
+  };
+  const stillUnclassifiedLines = [];
+  const reClassifiedLines = [];
+  const margin = ' '.repeat(6);
+  // For each tool reporting any violations of still unclassified rules:
+  Object.keys(stillUnclassified).forEach(toolID => {
+    // For each such rule:
+    Object.keys(stillUnclassified[toolID]).forEach(ruleID => {
+      const reportIDs = Array.from(stillUnclassified[toolID][ruleID]);
+      // Add a line to the lines on the rule.
+      stillUnclassifiedLines.push(
+        `${margin}<li>${toolID}: ${ruleID} (${reportIDs.join(', ')})</li>`
+      );
+    });
+  });
+  // Add the lines to the query.
+  query.stillUnclassified = stillUnclassifiedLines.join('\n');
+  // For each tool reporting any discrepancies in rule classification:
+  Object.keys(reClassified).forEach(toolID => {
+    // For each such rule:
+    Object.keys(reClassified[toolID]).forEach(ruleID => {
+      const reportIDs = Array.from(reClassified[toolID][ruleID]);
+      // Add a line to the lines on the rule.
+      reClassifiedLines.push(
+        `${margin}<li>${toolID}: ${ruleID} (${reportIDs.join(', ')})</li>`
+      );
+    });
+  });
+  // Add the lines to the query.
+  query.reClassified = reClassifiedLines.join('\n');
+  if (reClassifiedLines.length) {
+    query.how = 'Each <q>reclassified</q> rule indicates that report annotations are out of date. To update them, submit your authorization code.';
+    const formLines = [];
+    formLines.push(`${margin}<form action="/reannotate.html" method="post">`);
+    formLines.push(
+      `${margin}  <p><label>Authorization code: <input size="3" minLength="3" maxlength="3" name="authCode" required></label></p>`
+    );
+    formLines.push(`${margin}  <p><button type="submit">Reannotate</button></p>`);
+    formLines.push(`${margin}</form>`);
+    query.reannotateForm = formLines.join('\n');
+  }
+  else {
+    query.how = 'No violated rules have been classified or reclassified after being reported, so reannotation of the reports is not necessary.';
+    query.reannotateForm = '';
+  }
+};
+// Returns a page disclosing newly classified rules and a form to reannotate reports.
+exports.answer = async () => {
+  const query = {};
+  // Create a query to replace placeholders.
+  await populateQuery(query);
+  // Get the template.
+  let answerPage = await fs.readFile(path.join(__dirname, 'index.html'), 'utf8');
+  // Replace its placeholders.
+  Object.keys(query).forEach(param => {
+    answerPage = answerPage.replace(new RegExp(`__${param}__`, 'g'), query[param]);
+  });
+  // Return the populated page.
+  return {
+    status: 'ok',
+    answerPage
+  };
+};

package/recActionForm/index.html

@@ -0,0 +1,33 @@
+<!DOCTYPE html>
+<html lang="en-US">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="publisher" content="Jonathan Robert Pool">
+    <meta name="creator" content="Jonathan Robert Pool">
+    <meta name="keywords" content="report,accessibility,a11y">
+    <title>Recommendation approval or rejection | Kilotest</title>
+    <link rel="icon" href="/favicon.ico">
+    <link rel="stylesheet" href="/style.css">
+  </head>
+  <body>
+    <main>
+      <h1><a href="/">Kilotest</a>: Recommendation approval or rejection</h1>
+      <p>To approve a test or retest recommendation, check its recommended page name.</p>
+      <p>To approve testing a page that is not yet recommended, first <a href="targets.html">recommend it</a>.</p>
+      <p>To reject all recommendations for a page, check the radio button of its URL.</p>
+      <form action="/recAction.html" method="post">
+        <fieldset>
+          <legend>Recommendations</legend>
+          __recs__
+          <p>__noRecs__</p>
+        </fieldset>
+        <p><label>
+          Authorization code:
+          <input size="3" minLength="3" maxlength="3" name="authCode" required__disabled__>
+        </label></p>
+        <p><button type="submit"__disabled__>Submit</button></p>
+      </form>
+    </main>
+  </body>
+</html>

package/recActionForm/index.js

@@ -0,0 +1,49 @@
+/*
+  index.js
+  Serves a form for ordering a recommended test.
+*/
+
+// IMPORTS
+
+const {getRecs} = require('../util');
+const fs = require('fs/promises');
+const path = require('path');
+
+// FUNCTIONS
+
+// Returns a test order form.
+exports.answer = async () => {
+  const recs = await getRecs();
+  const urls = Object.keys(recs);
+  const margin = ' '.repeat(12);
+  const lines = [];
+  // For each page with any recommendations:
+  urls.forEach(url => {
+    // Add a radio button and its URL to the lines.
+    lines.push(`${margin}<h2><input type="radio" name="target" value="${url}" required> ${url}</h2>`);
+    // Get the recommended target names for the page.
+    const targetNames = new Set(recs[url].map(rec => rec.what));
+    // For each recommended target name:
+    targetNames.forEach(what => {
+      const radio = `<input type="radio" name="target" value="${url}\t${what}" required>`;
+      // Add a radio button and the recommended page name to the lines.
+      lines.push(`${margin}    <p>${radio} ${what}</p>`);
+    });
+  });
+  const query = {
+    recs: lines.join('\n'),
+    noRecs: urls.length ? '' : 'No recommendations exist now.',
+    disabled: urls.length ? '' : ' disabled'
+  };
+  // Get the order form template.
+  let answerPage = await fs.readFile(path.join(__dirname, 'index.html'), 'utf8');
+  // Replace its placeholders.
+  Object.keys(query).forEach(param => {
+    answerPage = answerPage.replace(new RegExp(`__${param}__`, 'g'), query[param]);
+  });
+  // Return the populated page.
+  return {
+    status: 'ok',
+    answerPage
+  };
+};