@jrpool/kilotest 57.3.0 → 58.2.4

package/web/listIssues/index.ts

@@ -8,147 +8,132 @@
 import {
   getReport,
   getTestActInstances,
-  htmlSafe,
-  isUsableReport,
+  isReportError,
   objectSort,
-  populateTemplate,
-  ruleEngines
+  populateTemplate
 } from '../../util.ts';
+import type {UsableReport} from '../../util.ts';
 import {
-  getPageData,
+  getIssueFactsLines,
   getPageDataStrings,
+  getPageFactsLines,
+  getEngineIDs,
   getEngineNamesString,
-  getWCAGLink,
   getWeightName
 } from '../util.ts';
+import type {ResultsSummary} from '../util.ts';
 import {issues as issueSpecs} from 'testaro-issues';
 
 // FUNCTIONS
 
 // Returns data on the issues reported by a report.
-const getIssuesData = async (timeStamp: string, jobID: string) => {
-  // Get the report.
-  const report = await getReport(timeStamp, jobID);
-  // If it exists and is valid:
-  if (isUsableReport(report)) {
-    // Initialize the temporary data.
-    const temp = {
-      issues: {} as Record<string, any>,
-      reporters: new Set<string>(),
-      violators: new Set<string>()
-    };
-    // Initialize the final data.
-    const final: Record<string, any> = {
-      reporters: [],
-      reporterList: '',
-      reporterCount: 0,
-      violatorCount: 0,
-      preventions: report.jobData.preventions,
-      issues: {
-        4: [],
-        3: [],
-        2: [],
-        1: []
-      },
-      issueCount: 0
-    };
-    // For each violating standard instance of each test act:
-    getTestActInstances(report, {violationsOnly: true}).forEach(({act, instance}) => {
-      const {catalogIndex, issueID} = instance;
-      const which = act.which!;
-      // If it identifies a non-ignorable issue:
-      if (issueID && issueID !== 'ignorable') {
-        const issueClassification = issueSpecs[issueID];
-        // If the issue has a current weighted classification:
-        if (issueClassification && [1, 2, 3, 4].includes(issueClassification.weight)) {
-          const {summary, wcag, weight, why} = issueClassification;
-          // Initialize the temporary data on the issue if necessary.
-          temp.issues[issueID] ??= {
-            issueID,
-            summary,
-            wcag,
-            why,
-            weight,
-            reporters: new Set(),
-            reporterList: '',
-            violators: new Set()
-          };
-          // Ensure the rule engine is in the temporary data.
-          temp.issues[issueID].reporters.add(which);
-          temp.reporters.add(which);
-          // If the instance has a catalog index:
-          if (catalogIndex) {
-            // Ensure the violator is in the temporary data.
-            temp.issues[issueID].violators.add(catalogIndex);
-            temp.violators.add(String(catalogIndex));
-          }
+const getIssuesData = (report: UsableReport): ResultsSummary & Record<string, any> => {
+  // Initialize the temporary data.
+  const temp = {
+    issues: {} as Record<string, any>,
+    reporters: new Set<string>(),
+    violators: new Set<string>()
+  };
+  // Initialize the final data.
+  const final: ResultsSummary & Record<string, any> = {
+    reporters: [],
+    reporterList: '',
+    reporterCount: 0,
+    engineCount: 0,
+    testedEngineCount: 0,
+    violationCount: 0,
+    violatorCount: 0,
+    issues: {
+      4: [],
+      3: [],
+      2: [],
+      1: []
+    },
+    issueCount: 0
+  };
+  // For each violating standard instance of each test act:
+  getTestActInstances(report, {violationsOnly: true}).forEach(({act, instance}) => {
+    const {catalogIndex, issueID} = instance;
+    const which = act.which!;
+    // If it identifies a non-ignorable issue:
+    if (issueID && issueID !== 'ignorable') {
+      const issueClassification = issueSpecs[issueID];
+      // If the issue has a current weighted classification:
+      if (issueClassification && [1, 2, 3, 4].includes(issueClassification.weight)) {
+        const {summary, wcag, weight, why} = issueClassification;
+        // Initialize the temporary data on the issue if necessary.
+        temp.issues[issueID] ??= {
+          issueID,
+          summary,
+          wcag,
+          why,
+          weight,
+          reporters: new Set(),
+          reporterList: '',
+          violationCount: 0,
+          violators: new Set()
+        };
+        // Ensure the rule engine is in the temporary data.
+        temp.issues[issueID].reporters.add(which);
+        temp.reporters.add(which);
+        // Increment the violation counts.
+        temp.issues[issueID].violationCount++;
+        final.violationCount++;
+        // If the instance has a catalog index:
+        if (catalogIndex) {
+          // Ensure the violator is in the temporary data.
+          temp.issues[issueID].violators.add(catalogIndex);
+          temp.violators.add(String(catalogIndex));
         }
       }
-    });
-    // Finish populating the final data.
-    final.reporterList = getEngineNamesString(temp.reporters);
-    final.reporterCount = temp.reporters.size;
-    final.violatorCount = temp.violators.size;
-    Object.values(temp.issues).forEach(issue => {
-      const {issueID, summary, wcag, why, weight} = issue;
-      const finalIssue: Record<string, any> = {
-        issueID,
-        summary,
-        wcag,
-        why,
-        weight
-      };
-      finalIssue.reporterList = getEngineNamesString(issue.reporters);
-      finalIssue.reporterCount = issue.reporters.size;
-      finalIssue.violatorCount = issue.violators.size;
-      final.issues[issue.weight].push(finalIssue);
-    });
-    final.issueCount = Object.keys(temp.issues).length;
-    // For each weight:
-    [4, 3, 2, 1].forEach(weight => {
-      // Sort its issues in the final data alphabetically by reporter names.
-      objectSort(final.issues[weight], 'reporterList', 'alpha');
-      // Sort the issues again in descending reporter-count order, making this the primary order.
-      objectSort(final.issues[weight], 'reporterCount', 'numericDown');
-    });
-    // Return the data.
-    return final;
-  }
-  // Otherwise, i.e. if it is invalid or does not exist, return this.
-  return {error: 'Report missing or invalid.'};
-};
-// Get page and issues data from a report.
-const getData = async (timeStamp: string, jobID: string) => {
-  const pageData = await getPageData(timeStamp, jobID);
-  const issuesData: any = await getIssuesData(timeStamp, jobID);
-  const pageError = ('error' in pageData ? pageData.error : '');
-  const issuesError = issuesData.error || '';
-  const errors = [pageError, issuesError].filter(Boolean).join('; ');
-  // If the data of either type are missing or invalid:
-  if (errors) {
-    // Return this.
-    return {error: errors};
-  }
-  // Otherwise, return the data.
-  return {
-    pageData,
-    issuesData
-  };
+    }
+  });
+  // Finish populating the final data.
+  const {calledIDs, preventedIDs} = getEngineIDs(report);
+  final.engineCount = calledIDs.length;
+  final.testedEngineCount = calledIDs.length - preventedIDs.length;
+  final.reporterList = getEngineNamesString(temp.reporters);
+  final.reporterCount = temp.reporters.size;
+  final.violatorCount = temp.violators.size;
+  Object.values(temp.issues).forEach(issue => {
+    const {issueID, summary, wcag, why, weight} = issue;
+    const finalIssue: Record<string, any> = {
+      issueID,
+      summary,
+      wcag,
+      why,
+      weight
+    };
+    finalIssue.reporterList = getEngineNamesString(issue.reporters);
+    finalIssue.reporterCount = issue.reporters.size;
+    finalIssue.violationCount = issue.violationCount;
+    finalIssue.violatorCount = issue.violators.size;
+    final.issues[issue.weight].push(finalIssue);
+  });
+  final.issueCount = Object.keys(temp.issues).length;
+  // For each weight:
+  [4, 3, 2, 1].forEach(weight => {
+    // Sort its issues in the final data alphabetically by reporter names.
+    objectSort(final.issues[weight], 'reporterList', 'alpha');
+    // Sort the issues again in descending reporter-count order, making this the primary order.
+    objectSort(final.issues[weight], 'reporterCount', 'numericDown');
+  });
+  // Return the data.
+  return final;
 };
 // Adds parameters to a query for the answer page.
 const populateQuery = async (timeStamp: string, jobID: string, query: Record<string, any>) => {
-  // Get data on the target and its issues according to the report.
-  const data: any = await getData(timeStamp, jobID);
-  const {pageData, issuesData} = data;
+  // Get the report.
+  const report = await getReport(timeStamp, jobID);
   // If this failed:
-  if (data.error) {
+  if (isReportError(report)) {
     // Populate the query with the reason.
-    query.error = data.error;
+    query.error = report.error;
     // Stop populating the query.
     return;
   }
   // Otherwise, i.e. if it succeeded, get fact descriptions for the target.
-  const pageInfo = await getPageDataStrings(timeStamp, jobID, pageData);
+  const pageInfo = getPageDataStrings(report);
   // If this failed:
   if (pageInfo.error !== undefined) {
     // Populate the query with the reason.
@@ -156,45 +141,24 @@ const populateQuery = async (timeStamp: string, jobID: string, query: Record<str
     // Stop populating the query.
     return;
   }
-  const {testInfo, urlLink, description} = pageInfo;
-  // Otherwise, i.e. if it succeeded, add target data to the query.
+  // Otherwise, i.e. if it succeeded, get data on the issues.
+  const issuesData = getIssuesData(report);
+  const {testInfo, description} = pageInfo;
+  // Add target data to the query.
   query.target = description;
-  query.urlLink = urlLink;
   query.testInfo = testInfo;
-  const {
-    reporterList,
-    reporterCount,
-    violatorCount,
-    issueCount,
-    preventions,
-    issues
-  } = issuesData;
-  // Initialize strings for the prevention notices query property.
-  const preventionStrings: string[] = [];
   const margin = ' '.repeat(6);
-  Object.keys(preventions).forEach(preventedEngineID => {
-    const [engineName, engineSponsor] = ruleEngines[preventedEngineID] ?? [preventedEngineID, 'unknown sponsor'];
-    const engineNameString = `${engineName} (${engineSponsor})`;
-    const causeString = htmlSafe(preventions[preventedEngineID]);
-    const preventionString = `${margin}<li>Page not testable by ${engineNameString}: ${causeString}</li>`;
-    preventionStrings.push(preventionString);
-  });
-  // Add prevention notices to the query.
-  query.preventions = preventionStrings.join('\n');
+  // Add the page-facts list to the query.
+  query.pageFacts = getPageFactsLines(pageInfo, issuesData, margin).join('\n');
+  const {issues} = issuesData;
   // Add report data to the query.
   query.timeStamp = timeStamp;
   query.jobID = jobID;
-  // Add reporter information to the query.
-  query.reporterCount = reporterCount === 1 ? '1 rule engine' : `${reporterCount} rule engines`;
-  query.reporters = reporterList;
   // Add a summary of the issues to the query.
-  query.issueCount = issueCount === 1 ? '1 issue was' : `${issueCount} issues were`;
   query.highestCount = issues[4].length;
   query.highCount = issues[3].length;
   query.lowCount = issues[2].length;
   query.lowestCount = issues[1].length;
-  // Add a violator count to the query.
-  query.violatorCount = violatorCount === 1 ? '1 violator was' : `${violatorCount} violators were`;
   // For each weight:
   [4, 3, 2, 1].forEach(weight => {
     const weightName = getWeightName(weight);
@@ -208,35 +172,15 @@ const populateQuery = async (timeStamp: string, jobID: string, query: Record<str
         const weightIssueCount = weightIssues.length;
         // Add the issue count to the query.
         query[`${weightName}Count`] = weightIssueCount;
-        const {
-          issueID,
-          reporterCount,
-          reporterList,
-          summary,
-          violatorCount,
-          wcag,
-          why
-        } = issueData;
-        const wcagLink = `<a href="${getWCAGLink(wcag)}">${wcag}</a>`;
+        const {issueID, summary} = issueData;
         // Add the start of a list item to the lines.
         detailsLines.push(`${margin}  <li>`);
         // Add a heading summarizing the issue to the lines.
         detailsLines.push(`${margin}    <h5>${summary}</h5>`);
-        // Add the start of a fact list about the issue to the lines.
-        detailsLines.push(`${margin}    <ul class="pseudoTopLevel">`);
         // Add the issue facts to the lines.
-        detailsLines.push(`${margin}      <li>Why it matters: ${why}`);
-        detailsLines.push(`${margin}      <li>Related WCAG standard: ${wcagLink}`);
-        const reporterCountString = reporterCount === 1 ? '1 rule engine' : `${reporterCount} rule engines`;
         detailsLines.push(
-          `${margin}      <li>Reported by ${reporterCountString} (${reporterList})</li>`
+          ...getIssueFactsLines(report, issueID, `${margin}    `, 'pseudoTopLevel')
         );
-        const violatorCountString = violatorCount === 1
-        ? '1 violator was'
-        : `${violatorCount} violators were`;
-        detailsLines.push(`${margin}      <li>${violatorCountString} reported</li>`);
-        // Add the end of the fact list to the lines.
-        detailsLines.push(`${margin}    </ul>`);
         // Add the start of a link list to the lines.
         detailsLines.push(`${margin}    <ul class="nav">`);
         const whereQuestionString = 'Where was the issue found?';

package/web/listIssues/listIssues.test.ts

@@ -18,7 +18,7 @@ let pageDataStringsOverride: any = null;
 mock.module('../util.ts', {
   exports: {
     ...realWebUtil,
-    getPageDataStrings: async (...args: any[]) => {
+    getPageDataStrings: (...args: any[]) => {
       if (pageDataStringsOverride !== null) {
         return pageDataStringsOverride;
       }
@@ -61,6 +61,18 @@ test('listIssues includes the page description in the HTML for the mixed report'
   assert.ok(result.answerPage.includes('Mixed Outcomes Page'));
 });
 
+test('listIssues interpolates the page description into the AI prompt block quote', async () => {
+  const result = await answer('260101T0000/mix');
+  const html = parse(result.answerPage);
+  const blockQuote = html.querySelector('details blockquote');
+  assert.ok(blockQuote);
+  assert.ok(
+    blockQuote.textContent.startsWith('Kilotest reports defects on the Mixed Outcomes Page web page.')
+  );
+  // No placeholder may remain uninterpolated.
+  assert.ok(!blockQuote.textContent.includes('__target__'));
+});
+
 test('listIssues includes links to listViolators for issues in the mixed report', async () => {
   const result = await answer('260101T0000/mix');
   const html = parse(result.answerPage);
@@ -93,14 +105,23 @@ test('listIssues for the empty report shows no issue links', async () => {
 });
 
 
-test('listIssues includes prevention notices for the prevented report', async () => {
+test('listIssues counts the prevented rule engine as called but not able to test', async () => {
   const result = await answer('260101T0006/prv');
   assert.equal(result.status, 'ok');
-  assert.ok(result.answerPage.includes('Page not testable by'));
-  assert.ok(result.answerPage.includes('page timed out'));
+  assert.ok(result.answerPage.includes('<li>Called: 2</li>'));
+  assert.ok(result.answerPage.includes('<li>Were able to test: 1</li>'));
+  assert.ok(result.answerPage.includes('<li>Reported any rule violations: 1</li>'));
+});
+
+test('listIssues counts a rule engine that reported no violations as called and able to test', async () => {
+  const result = await answer('260101T0005/emp');
+  assert.equal(result.status, 'ok');
+  assert.ok(result.answerPage.includes('<li>Called: 1</li>'));
+  assert.ok(result.answerPage.includes('<li>Were able to test: 1</li>'));
+  assert.ok(result.answerPage.includes('<li>Reported any rule violations: 0</li>'));
 });
 
-test('listIssues falls back to the engine ID for an unknown prevented engine', async () => {
+test('listIssues counts an unknown prevented engine as called but not able to test', async () => {
   const fs = await import('node:fs/promises');
   const prvJSON = await fs.readFile(path.join(realUtil.reportsPath(), '260101T0006-prv.json'), 'utf8');
   const report = JSON.parse(prvJSON);
@@ -110,14 +131,15 @@ test('listIssues falls back to the engine ID for an unknown prevented engine', a
   try {
     const result = await answer('260103T0000/unk');
     assert.equal(result.status, 'ok');
-    assert.ok(result.answerPage.includes('unknownEngine (unknown sponsor)'));
+    assert.ok(result.answerPage.includes('<li>Called: 3</li>'));
+    assert.ok(result.answerPage.includes('<li>Were able to test: 1</li>'));
   }
   finally {
     await fs.unlink(reportPath);
   }
 });
 
-test('listIssues returns an error when getPageDataStrings fails after getData succeeds', async () => {
+test('listIssues returns an error when getPageDataStrings fails after getReport succeeds', async () => {
   pageDataStringsOverride = {error: 'Page data strings error'};
   try {
     const result = await answer('260101T0000/mix');
@@ -145,11 +167,12 @@ test('listIssues returns an error when report facts are not obtained', async ()
   }
 });
 
-test('listIssues shows plural violator count for an issue with multiple violators', async () => {
-  // The mul report has linkNoText with 3 violators.
+test('listIssues shows violation and violator counts for an issue with multiple violators', async () => {
+  // The mul report has linkNoText with 4 violations by 3 violators.
   const result = await answer('260101T0008/mul');
   assert.equal(result.status, 'ok');
-  assert.ok(result.answerPage.includes('3 violators were'));
+  assert.ok(result.answerPage.includes('<li>Violations: 4</li>'));
+  assert.ok(result.answerPage.includes('<li>Violators: 3</li>'));
 });
 
 test('listIssues handles acts with no standardResult instances', async () => {

package/web/listReports/index.ts

@@ -6,19 +6,20 @@
 // IMPORTS
 
 import {
-  getAgoDays,
   getJobNames,
   getObject,
-  getTestRequests,
+  getReport,
   getReportExtracts,
+  getTestRequests,
+  isReportError,
   jobsPath,
   objectSort,
   populateTemplate
 } from '../../util.ts';
 import {
-  getCountString,
   getMultiReportWhats,
   getPageDataStrings,
+  getPageFactsLines,
   getReportData
 } from '../util.ts';
 import path from 'node:path';
@@ -96,68 +97,35 @@ const populateQuery = async (query: Record<string, any>) => {
   query.some = (reportCount || jobFileNames.queue.length || jobFileNames.claimed.length)
   ? 'another'
   : 'a';
-  const multiReportWhats = await getMultiReportWhats();
+  const multiReportWhats = await getMultiReportWhats(reportExtracts);
   // Sort them primarily by page description and secondarily by completion time.
   let sortedExtracts = objectSort(reportExtracts, 'reportTime', 'alpha');
   sortedExtracts = objectSort(sortedExtracts, 'description', 'alpha');
   // For each report:
   for (const extract of sortedExtracts) {
     const {jobID, timeStamp, url, description, superseded} = extract;
-    // Get data about it.
-    const reportData = await getReportData(timeStamp, jobID);
+    // Get the report.
+    const report = await getReport(timeStamp, jobID);
     // If this failed:
-    if (reportData.error !== undefined) {
-      console.error(reportData.error);
+    if (isReportError(report)) {
+      console.error(report.error);
       // Populate the query with the reason.
-      query.error = reportData.error;
+      query.error = report.error;
       // Stop populating the query.
       return;
     }
-    const {
-      issueCount,
-      preventedEngineCount,
-      preventedEngineNames,
-      reporterNames,
-      reporterCount,
-      violationCount,
-      violatorCount
-    } = reportData;
-    // Otherwise, i.e. if it succeeded, add lines about the report.
+    // Otherwise, i.e. if it succeeded, get data about the report.
+    const reportData = getReportData(report);
+    const pageDataStrings = getPageDataStrings(report);
+    const {issueCount} = reportData;
+    const {testInfo} = pageDataStrings;
+    // Add lines about the report.
     lines.tested.push(`${margin}<details>`);
-    const daysAgo = getAgoDays(timeStamp);
-    const pageDataStrings = await getPageDataStrings(timeStamp, jobID, {description, url, daysAgo});
-    const {urlLink, testInfo}: any = pageDataStrings;
     const testText = multiReportWhats.includes(description) ? ` (${testInfo.toLowerCase()})` : '';
     lines.tested.push(`${margin}  <summary>${description}${testText}</summary>`);
-    lines.tested.push(`${margin}  <ul>`);
-    // Add the URL of the target to the lines.
-    lines.tested.push(`${margin}    <li>URL: ${urlLink}</li>`);
-    // Add facts about the report to the lines.
-    lines.tested.push(`${margin}    <li>${testInfo}</li>`);
-    // If the page prevented any rule engine from performing its tests:
-    if (preventedEngineCount) {
-      // Add this to the lines.
-      const engineCountString = getCountString(preventedEngineCount, 'rule engine', 'rule engines');
-      lines.tested.push(
-        `${margin}    <li>Page not testable by ${engineCountString} (${preventedEngineNames.join(' + ')})</li>`,
-      );
-    }
-    // Add facts about the test results to the lines.
-    let reporterString = `${getCountString(reporterCount, 'rule engine', 'rule engines')} reported issues`;
-    if (reporterCount) {
-      const reporterNamesString = reporterNames.join(' + ');
-      reporterString = `${reporterString} (${reporterNamesString})`;
-    }
-    lines.tested.push(`${margin}    <li>${reporterString}</li>`);
-    lines.tested.push(`${margin}    <li>Summary of results:`);
-    lines.tested.push(`${margin}      <ul>`);
-    lines.tested.push(`${margin}        <li>Violations: ${violationCount}</li>`);
-    lines.tested.push(`${margin}        <li>Violators: ${violatorCount}</li>`);
-    lines.tested.push(`${margin}        <li>Issues: ${issueCount}</li>`);
-    lines.tested.push(`${margin}      </ul>`);
-    lines.tested.push(`${margin}    </li>`);
-    lines.tested.push(`${margin}  </ul>`);
-    lines.tested.push(`${margin}<ul class="nav">`);
+    // Add the page facts to the lines.
+    lines.tested.push(...getPageFactsLines(pageDataStrings, reportData, `${margin}  `));
+    lines.tested.push(`${margin}  <ul class="nav">`);
     // If any issues were reported:
     if (issueCount) {
       // Add a link to issue details to the lines.

package/web/listReports/listReports.test.ts

@@ -231,7 +231,7 @@ const getReportDetails = (html: any, description: string, hrefFragment?: string)
   return matches[0];
 };
 
-// Returns the rule-engine line, the prevention line, and the 3 summary amounts of a details element.
+// Returns the 3 rule-engine amounts and the 3 summary amounts of a details element.
 const getResultFacts = (details: any) => {
   const items = details.querySelectorAll('li');
   const itemTexts: string[] = items.map((li: any) => li.text.trim());
@@ -240,13 +240,17 @@ const getResultFacts = (details: any) => {
     assert.ok(text, `${label} item exists`);
     return Number(text.slice(label.length + 2));
   };
-  // The summary item is the parent of the 3 amount items.
+  // The summary item is the parent of the rule-engine item and the 3 amount items.
   const summaryItem = items.find((li: any) => li.text.trim().startsWith('Summary of results:'));
   assert.ok(summaryItem, 'Summary of results item exists');
-  const amountLabels = summaryItem.querySelectorAll('ul > li').map((li: any) => li.text.trim().split(':')[0]);
+  const summaryList = summaryItem.querySelector('ul');
+  const amountLabels = summaryList.childNodes
+  .filter((node: any) => node.tagName === 'LI')
+  .map((li: any) => li.text.trim().split(':')[0]);
   return {
-    reporterLine: itemTexts.find(t => t.endsWith('reported issues') || t.includes('reported issues (')),
-    preventionLine: itemTexts.find(t => t.startsWith('Page not testable by')),
+    called: getAmount('Called'),
+    tested: getAmount('Were able to test'),
+    reporters: getAmount('Reported any rule violations'),
     amountLabels,
     violations: getAmount('Violations'),
     violators: getAmount('Violators'),
@@ -255,37 +259,42 @@ const getResultFacts = (details: any) => {
 };
 
 // Expected facts, hand-computed from the fixtures.
-const expectedFacts: [string, string | undefined, string, number, number, number][] = [
-  // Description, issue-link fragment, reporter line, violations, violators, issues.
-  ['Mixed Outcomes Page', '260101T0000/mix', '2 rule engines reported issues (Alfa + Axe)', 3, 2, 2],
-  ['Mixed Outcomes Page', '260202T0000/new', '1 rule engine reported issues (Axe)', 1, 1, 1],
-  ['All CantTell Page', undefined, '0 rule engines reported issues', 0, 0, 0],
-  ['No Outcomes Page', undefined, '1 rule engine reported issues (Accessibility Checker)', 1, 1, 1],
-  ['Empty Results Page', undefined, '0 rule engines reported issues', 0, 0, 0],
-  ['Prevented Page', undefined, '1 rule engine reported issues (Axe)', 1, 1, 1],
-  ['Multi Violator Page', undefined, '2 rule engines reported issues (Alfa + Axe)', 4, 3, 1],
-  ['Branch Coverage Page', undefined, '1 rule engine reported issues (Axe)', 4, 4, 1]
+const expectedFacts: [string, string | undefined, number, number, number, number, number, number][] = [
+  // Description, issue-link fragment, engines called, engines able to test, reporting engines, violations, violators, issues.
+  ['Mixed Outcomes Page', '260101T0000/mix', 2, 2, 2, 3, 2, 2],
+  ['Mixed Outcomes Page', '260202T0000/new', 1, 1, 1, 1, 1, 1],
+  ['All CantTell Page', undefined, 1, 1, 0, 0, 0, 0],
+  ['No Outcomes Page', undefined, 1, 1, 1, 1, 1, 1],
+  ['Empty Results Page', undefined, 1, 1, 0, 0, 0, 0],
+  ['Prevented Page', undefined, 2, 1, 1, 1, 1, 1],
+  ['Multi Violator Page', undefined, 2, 2, 2, 4, 3, 1],
+  ['Branch Coverage Page', undefined, 3, 3, 1, 4, 4, 1]
 ];
 
-for (const [description, fragment, reporterLine, violations, violators, issues] of expectedFacts) {
-  test(`listReports gives correct rule-engine count and names and summary amounts for ${description}${fragment ? ` (${fragment})` : ''}`, async () => {
+for (const [description, fragment, called, tested, reporters, violations, violators, issues] of expectedFacts) {
+  test(`listReports gives correct rule-engine and summary amounts for ${description}${fragment ? ` (${fragment})` : ''}`, async () => {
     const result: any = await answer();
     const html = parse(result.answerPage);
     const facts = getResultFacts(getReportDetails(html, description, fragment));
-    assert.equal(facts.reporterLine, reporterLine);
-    assert.deepEqual(facts.amountLabels, ['Violations', 'Violators', 'Issues']);
+    assert.equal(facts.called, called);
+    assert.equal(facts.tested, tested);
+    assert.equal(facts.reporters, reporters);
+    assert.deepEqual(facts.amountLabels, ['Rule engines', 'Violations', 'Violators', 'Issues']);
     assert.equal(facts.violations, violations);
     assert.equal(facts.violators, violators);
     assert.equal(facts.issues, issues);
   });
 }
 
-test('listReports reports the prevented rule engine only for the report with a prevention', async () => {
+test('listReports counts a prevented rule engine as called but not able to test', async () => {
   const result: any = await answer();
   const html = parse(result.answerPage);
   const prevented = getResultFacts(getReportDetails(html, 'Prevented Page'));
-  assert.equal(prevented.preventionLine, 'Page not testable by 1 rule engine (Alfa)');
+  assert.equal(prevented.called - prevented.tested, 1);
   const others = html.querySelectorAll('details').filter((d: any) => !d.text.includes('Prevented Page'));
   assert.equal(others.length, 7);
-  assert.ok(others.every((d: any) => !d.text.includes('Page not testable by')));
+  assert.ok(others.every((d: any) => {
+    const facts = getResultFacts(d);
+    return facts.called === facts.tested;
+  }));
 });