@jrpool/kilotest 38.1.0 → 39.0.1

package/.markdownlint-cli2.jsonc

@@ -0,0 +1,12 @@
+{
+  "globs": [
+    "**/*.md"
+  ],
+  "ignores": [
+    "docs/IDEAS.md",
+    "node_modules/**"
+  ],
+  "config": {
+    "MD013": false
+  }
+}

package/.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "eslint.rules.customizations": []
+}

package/CLAUDE.md

@@ -0,0 +1,5 @@
+# Cascade Rules
+
+- Ask mode: If the current mode is Ask, answer questions and provide code, diffs, or edit suggestions in code blocks or as plain text. Do not invite the user to switch to Normal or Code mode or any mode that permits writing files.
+- Comply with the current convention to omit line breaks in long lines in `.md` files and in code comments.
+- In conversation responses (not files), when a Markdown table cell would contain a line longer than 120 characters divided by the number of columns, insert `<br>` tags to keep displayed line lengths shorter than that maximum, since raw newlines are not valid inside table cells. If this rule makes any table cumbersome, use a non-tabular format, such as a list, instead.

package/README.md

@@ -1,10 +1,10 @@
 # Kilotest
 
-An ensemble testing service with a focus on accessibility
+An ensemble testing and reporting service for front-end web quality
 
 ## Features
 
-This application uses an ensemble of 10 rule engines to test public web pages for front-end quality (i.e. accessibility, usability, and conformity to standards).
+This application uses an ensemble of 10 rule engines to test public web pages for front-end quality (i.e. accessibility, usability, and standards conformity).
 
 The testing paradigm employed by Kilotest is discussed in these papers:
 
@@ -12,7 +12,9 @@ The testing paradigm employed by Kilotest is discussed in these papers:
 - [Testaro: Efficient Ensemble Testing for Web Accessibility](https://arxiv.org/abs/2309.10167)
 - [Accessibility Metatesting: Comparing Nine Testing Tools](https://arxiv.org/abs/2304.07591)
 
-Kilotest acts as a server with human and agent users and with one or more testing agents that obtain jobs from Kilotest and do the actual testing. Those agents are instances of the [Testaro](https://www.npmjs.com/package/testaro) package.
+Kilotest acts as a web server for human users, an MCP server for AI platforms, and an API for programmatic access.
+
+Testing for Kilotest is performed by one or more testing agents that obtain jobs from Kilotest, perform them, and send reports of the results back to Kilotest. Those agents are instances of the [Testaro](https://www.npmjs.com/package/testaro) package.
 
 An active production instance of Kilotest may require multiple testing agents to handle the load, because testing one web page typically takes about 3 minutes and agents test only one page at a time.
 
@@ -45,4 +47,4 @@ See the `SERVICE.md` file for instructions on how to make Kilotest a service.
 
 ## Using Kilotest as an AI tool
 
-See the `AI-TOOL.md` file for information about the configuration of Kilotest as an AI tool.
+See the `AI-TOOLS.md` file for information about the configuration of Kilotest as a collection of AI tools.

package/ai0BalanceForm/index.html

@@ -21,7 +21,7 @@
           <input type="number" size="5" min="0" max="99.99" step="0.01" name="newBalance" required>
         </label></p>
         <p><label>
-          Authorization code: <input size="3" minLength="3" maxlength="3" name="authCode" required>
+          Authorization code: <input size="3" minlength="3" maxlength="3" name="authCode" required>
         </label></p>
         <p><button type="submit">Submit</button></p>
       </form>

package/alerts.js

@@ -39,7 +39,9 @@ exports.sendAlert = (subject, body) => new Promise(resolve => {
       }
     }, res => {
       let data = '';
-      res.on('data', chunk => { data += chunk; });
+      res.on('data', chunk => {
+        data += chunk;
+      });
       res.on('end', () => {
         if (res.statusCode >= 200 && res.statusCode < 300) {
           console.log(`Alert sent (${subject})`);

package/api/getReport.js

@@ -0,0 +1,66 @@
+/*
+  getReport.js
+  Returns one report.
+*/
+
+// IMPORTS
+
+const {
+  getResponseMetadata,
+  getToolsFacts,
+  thisHost
+} = require('./util');
+const {getReport, getReportStats} = require('../util');
+
+// FUNCTIONS
+
+// Returns the response body.
+exports.response = async args => {
+  const [timeStamp = '', jobID = ''] = args;
+  // Initialize the response content.
+  const responseContent = {
+    'size of the report in bytes': null,
+    'full report': null
+  };
+  // Get the report size.
+  const reportStats = await getReportStats(timeStamp, jobID);
+  // If this failed:
+  if (!reportStats) {
+    // Add this to the response content.
+    responseContent['size of the report in bytes'] = 'Error: The report could not be accessed for an unknown reason.';
+  }
+  // Otherwise, i.e. if it succeeded:
+  else {
+    const {reportSize} = reportStats;
+    // Add the size to the response content.
+    responseContent['size of the report in bytes'] = reportSize;
+    // Get the report (which may be only an error message).
+    const report = await getReport(timeStamp, jobID);
+    // Add it to the response content.
+    responseContent['full report'] = report;
+  }
+  // Create a response body.
+  const body = {
+    'tool collection': getToolsFacts(),
+    'tool name': 'getReport',
+    'this request': {
+      description: 'Provide one full report in JSON format. The timeStamp and jobID parameters identify the report that I want details about. Those parameters were in the response to my earlier listIssues request.',
+      method: 'GET',
+      URL: `${thisHost}/api/getReport/${timeStamp}/${jobID}`,
+      'closest ancestor request': {
+        'tool name': 'listIssues',
+        description: 'Provide details about one report, including basics about the issues reported in it.',
+        method: 'GET',
+        URL: `${thisHost}/api/listIssues/${timeStamp}/${jobID}`
+      }
+    },
+    'URLs of similar requests for web users': {
+      'this request': `${thisHost}/fullReport.json/${timeStamp}/${jobID}`,
+      'closest ancestor request': `${thisHost}/reportIssues.html/${timeStamp}/${jobID}`
+    },
+    'response metadata': getResponseMetadata(),
+    'response content': responseContent
+  };
+  // Return it.
+  return body;
+};

package/api/listDiagnoses.js

@@ -0,0 +1,157 @@
+/*
+  listDiagnoses.js
+  Returns details about one violator of one issue in one report and facts about all its diagnoses.
+*/
+
+// IMPORTS
+
+const {
+  getIssueClassification,
+  getReportBasics,
+  getResponseMetadata,
+  getToolsFacts,
+  thisHost
+} = require('./util');
+const {getReport} = require('../util');
+
+// FUNCTIONS
+
+// Returns the response body.
+exports.response = async args => {
+  const [catalogIndex = '', issueID = '', timeStamp = '', jobID = ''] = args;
+  // Initialize the response content.
+  const responseContent = {
+    'basics about the report': null,
+    'basics about the issue': null,
+    'basics about the element': null,
+    'details about the element': null,
+    'diagnoses of how the element exhibited the issue': null
+  };
+  // Get the report.
+  const report = await getReport(timeStamp, jobID);
+  // If this failed:
+  if (report.error) {
+    // Add this to the response content.
+    responseContent['basics about the report'] = report;
+  }
+  // Otherwise, i.e. if it succeeded:
+  else {
+    // Get the basics about the report (which may be only an error message).
+    const reportBasics = await getReportBasics(timeStamp, jobID);
+    // Add them to the response content.
+    responseContent['basics about the report'] = reportBasics;
+  }
+  // Get the classification of the issue.
+  const issueClassification = issueID ? getIssueClassification(issueID) : null;
+  // If the issue is ignorable or not fully classified:
+  if (!issueClassification) {
+    // Add this to the response content.
+    responseContent['basics about the issue'] = {
+      'error': 'No information about the specified issue is available'
+    };
+    responseContent['diagnoses of how the element exhibited the issue'] = {
+      'error': 'No diagnoses of how the element exhibited the issue is available'
+    };
+  }
+  // Otherwise, i.e. if it is non-ignorable and fully classified:
+  else {
+    const {summary, wcag, weight, why} = issueClassification;
+    // Initialize the basics about it.
+    const issueBasics = {
+      'identifier': issueID,
+      summary,
+      'impact on a user': why,
+      'related WCAG standard': {
+        'layer': (wcag.length > 4 ? 'success criterion' : 'guideline'),
+        'identifier': wcag
+      },
+      priority: ['lowest', 'low', 'high', 'highest'][weight - 1]
+    };
+    // Add them to the response content.
+    responseContent['basics about the issue'] = issueBasics;
+  }
+  // If the report and its catalog exist:
+  if (report.catalog) {
+    const catalogItem = report.catalog[catalogIndex];
+    // If the violator is not in it:
+    if (!catalogItem) {
+      // Add this to the response content.
+      responseContent['basics about the element'] = {
+        'error': 'No information about the specified element is available'
+      };
+    }
+    // Otherwise, i.e. if the violator is in the catalog:
+    else {
+      // Get basics about the violator.
+      const violatorBasics = {
+        identifier: catalogIndex,
+        'tag name': catalogItem.tagName || null,
+        'inner text': catalogItem.text ?? null
+      };
+      // Add them to the response content.
+      responseContent['basics about the element'] = violatorBasics;
+      // Get details about the violator.
+      const violatorDetails = {
+        'start tag': catalogItem.startTag ?? null,
+        'XPath': catalogItem.pathID ?? null,
+        'x, y, width, and height of bounding box': catalogItem.boxID ?? null
+      };
+      // Add them to the response content.
+      responseContent['details about the element'] = violatorDetails;
+      // If the issue is non-ignorable and fully classified:
+      if (issueClassification) {
+        // Initialize data about the diagnoses of the violation of the issue.
+        const diagnoses = [];
+        // For each act in the report:
+        report.acts.forEach(act => {
+          const {result, type, which} = act;
+          const instances = result?.standardResult?.instances || [];
+          // If the act is a test act with a specified rule engine:
+          if (type === 'test' && which) {
+            // For each standard instance of the act:
+            instances.forEach(instance => {
+              const {count, ordinalSeverity, ruleID, what} = instance;
+              // If the instance has the issue ID and the catalog index of the violator:
+              if (instance.issueID === issueID && instance.catalogIndex === catalogIndex) {
+                // Get the diagnosis.
+                const diagnosis = {
+                  'identifier of the violated rule': ruleID !== what ? ruleID : null,
+                  'description of the violation': what,
+                  'severity of the violation on a 0-to-3 scale': ordinalSeverity,
+                  'count of violations of the rule by the element': count ?? 1
+                }
+                diagnoses.push(diagnosis);
+              }
+            });
+          }
+        });
+        // Add the diagnoses to the response content.
+        responseContent['diagnoses of how the element exhibited the issue'] = diagnoses;
+      }
+    }
+  }
+  // Create a response body.
+  const body = {
+    'tool collection': getToolsFacts(),
+    'tool name': 'listDiagnoses',
+    'this request': {
+      description: 'Provide details about one element reported as exhibiting one issue in one report, including the diagnoses provided by rule engines about how the element exhibited the issue. The catalogIndex, issueID, timeStamp, and jobID parameters identify the element, issue, and report that I want details about. Those parameters were in the response to my earlier listViolators request.',
+      method: 'GET',
+      URL: `${thisHost}/api/listDiagnoses/${catalogIndex}/${issueID}/${timeStamp}/${jobID}`,
+      'closest ancestor request': {
+        'tool name': 'listViolators',
+        description: 'Provide details about one issue in one report, including basics about the elements of the tested page that were reported as exhibiting the issue.',
+        method: 'GET',
+        URL: `${thisHost}/api/listViolators/${issueID}/${timeStamp}/${jobID}`
+      }
+    },
+    'URLs of similar requests for web users': {
+      'this request': `${thisHost}/diagnoses.html/${issueID}/${timeStamp}/${jobID}/${catalogIndex}`,
+      'closest ancestor request': `${thisHost}/reportIssue.html/${issueID}/${timeStamp}/${jobID}`
+    },
+    'response metadata': getResponseMetadata(),
+    'response content': responseContent
+  }
+  // Return it.
+  return body;
+};

package/api/listIssues.js

@@ -0,0 +1,225 @@
+/*
+  listIssues.js
+  Returns details about one report and basics about all the issues in it.
+*/
+
+// IMPORTS
+
+const {
+  getIssueClassification,
+  getReportBasics,
+  getResponseMetadata,
+  getRuleEngineFacts,
+  getRuleEnginesFacts,
+  getToolsFacts,
+  thisHost
+} = require('./util');
+const {getReport, getReportStats, objectSort} = require('../util');
+
+// FUNCTIONS
+
+// Returns the response body.
+exports.response = async args => {
+  const [timeStamp = '', jobID = ''] = args;
+  // Initialize the response content.
+  const responseContent = {
+    'basics about the report': null,
+    'details about the report': null,
+    'how to request that the page be retested': null,
+    'how a web user can request that the page be retested': null,
+    'how to get the full report in JSON': null,
+    'how a web user can get the full report in JSON': null,
+    'basics about all issues reported in the report': null
+  };
+  // Get the report.
+  const report = await getReport(timeStamp, jobID);
+  // If this failed:
+  if (report.error) {
+    // Add this to the response content.
+    responseContent['basics about the report'] = report;
+  }
+  // Otherwise, i.e. if it succeeded:
+  else {
+    // Get the basics about the report (which may be only an error message).
+    const reportBasics = await getReportBasics(timeStamp, jobID);
+    // Add them to the response content.
+    responseContent['basics about the report'] = reportBasics;
+    // If the basics about the report were obtained:
+    if (!reportBasics.error) {
+      const {
+        strict = null,
+        standard = null,
+        device = {id: 'default'},
+        browserID = null
+      } = report;
+      // Get details about the job definition.
+      const jobDefinitionDetails = {
+        'whether the job prohibited redirection': strict,
+        'whether the native results of rule engines are reported': ['also', 'no'].includes(standard),
+        'whether standardized results are reported': ['also', 'only'].includes(standard),
+        'device emulated by the job': device,
+        'browser type used by the job': browserID
+      };
+      // Initialize data about the test results.
+      const ruleEngineIDs = new Set();
+      const reporterIDs = new Set();
+      const violatorIndexes = new Set();
+      const issuesData = {};
+      // For each act in the report:
+      report.acts.forEach(act => {
+        const {result, type, which} = act;
+        // If the act is a test act with a specified rule engine:
+        if (type === 'test' && which) {
+          // Ensure its rule engine is in the results data.
+          ruleEngineIDs.add(which);
+          const instances = result?.standardResult?.instances ?? [];
+          // For each standard instance of the act:
+          instances.forEach(instance => {
+            const {catalogIndex, issueID} = instance;
+            // Get the classification of its issue.
+            const issueClassification = issueID ? getIssueClassification(issueID) : null;
+            // If the instance has a non-ignorable and fully classified issue:
+            if (issueClassification) {
+              const {summary, weight, why} = issueClassification;
+              // Ensure the rule-engine ID is in the reporters data.
+              reporterIDs.add(which);
+              // If the instance has a catalog index:
+              if (catalogIndex) {
+                // Ensure the index of the violator is in the results data.
+                violatorIndexes.add(catalogIndex);
+              }
+              // Ensure the data about the issue are in the results data.
+              issuesData[issueID] ??= {
+                id: issueID,
+                summary,
+                weight,
+                why,
+                reporterIDs: new Set()
+              };
+              // Ensure the reporter ID is in the data about the issue.
+              issuesData[issueID].reporterIDs.add(which);
+            }
+          });
+        }
+      });
+      const {preventions} = report.jobData;
+      // Get the details about the rule engines that could not test the page.
+      const preventionFacts = Object.entries(preventions ?? {}).map(([ruleEngineID, reason]) => ({
+        'name': getRuleEngineFacts(ruleEngineID).name,
+        'reason for failure': reason
+      }));
+      const sortedPreventionFacts = objectSort(preventionFacts ?? [], 'name', 'alpha');
+      // Initialize the counts of issues by weight.
+      const weightCounts = [0, 0, 0, 0];
+      // For each issue:
+      Object.values(issuesData).forEach(issueData => {
+        // Increment the count of issues with its weight.
+        weightCounts[issueData.weight - 1]++;
+      });
+      // Get details about the test results.
+      const resultDetails = {
+        'rule engines that tried to test the page': getRuleEnginesFacts(ruleEngineIDs),
+        'rule engines that could not test the page': sortedPreventionFacts,
+        'names of rule engines that reported rule violations': getRuleEnginesFacts(reporterIDs)
+        .map(facts => facts.name),
+        'counts of issues by priority': {
+          'highest': weightCounts[3],
+          'high': weightCounts[2],
+          'low': weightCounts[1],
+          'lowest': weightCounts[0]
+        },
+        'number of elements reported as violators': violatorIndexes.size
+      };
+      // Get the size of the report.
+      const reportStats = await getReportStats(timeStamp, jobID);
+      // If this succeeded:
+      if (reportStats) {
+        const {reportSize} = reportStats;
+        // Add instructions for getting the full report to the response content.
+        responseContent['how to get the full report in JSON'] = {
+          'size of the report in bytes': reportSize,
+          method: 'GET',
+          URL: `${thisHost}/api/getReport/${timeStamp}/${jobID}`
+        };
+        // Add instructions for web users to get the full report to the response content.
+        responseContent['how a web user can get the full report in JSON'] = {
+          URL: `${thisHost}/fullReport.json/${timeStamp}/${jobID}`
+        };
+      }
+      // Add the details about the job and test results to the response content.
+      responseContent['details about the report'] = {
+        'job definition': jobDefinitionDetails,
+        'test results': resultDetails
+      };
+      // If the report has been superseded:
+      if (reportBasics['whether a later report about the same page exists']) {
+        // Add information about requesting a retest to the response content.
+        responseContent['how to request that the page be retested'] = {
+          notice: 'A later report about the same page already exists. The listIssues output about that report includes instructions for requesting a retest.'
+        };
+      }
+      // Otherwise, i.e. if the report has not been superseded:
+      else {
+        // Add instructions for a retest request to the response content.
+        responseContent['how to request that the page be retested'] = {
+          method: 'POST',
+          URL: `${thisHost}/api/requestRetest/${timeStamp}/${jobID}`,
+          'request body': {
+            reason: '20- to 100-character reason why the page should be retested'
+          },
+          'how to check whether the request has been fulfilled':
+          'use the listReports tool to determine whether a report about the page has become available (typical wait time: 1 hour to 1 day)'
+        };
+        // Add instructions for a web user to request a retest.
+        responseContent['how a web user can request that the page be retested'] = {
+          URL: `${thisHost}/retestRecForm.html/${timeStamp}/${jobID}`
+        };
+      }
+      // Sort the data about issues by summary.
+      const sortedIssuesData = objectSort(Object.values(issuesData), 'summary', 'alpha');
+      // Get the basics about the issues.
+      const issuesBasics = sortedIssuesData.map(issueData => {
+        const {id, summary, weight, why, reporterIDs} = issueData;
+        return {
+          identifier: id,
+          summary,
+          priority: ['lowest', 'low', 'high', 'highest'][weight - 1],
+          'impact on a user': why,
+          'rule engines with any violations belonging to the issue': getRuleEnginesFacts(reporterIDs)
+          .map(ruleEnginesFact => ruleEnginesFact.name),
+          'how to get details about the issue': {
+            method: 'GET',
+            URL: `${thisHost}/api/listViolators/${id}/${timeStamp}/${jobID}`
+          },
+          'web users can get details about the issue at': `${thisHost}/reportIssue.html/${id}/${timeStamp}/${jobID}`
+        };
+      });
+      // Add the basics about the issues to the response content.
+      responseContent['basics about all issues reported in the report'] = issuesBasics;
+    }
+  }
+  // Create a response body.
+  const body = {
+    'tool collection': getToolsFacts(),
+    'tool name': 'listIssues',
+    'this request': {
+      description: 'Provide details about one report, including basics about the issues reported in it. The timeStamp and jobID parameters identify the report that I want details about. Those parameters were in the response to my earlier listReports request.',
+      method: 'GET',
+      URL: `${thisHost}/api/listIssues/${timeStamp}/${jobID}`,
+      'closest ancestor request': {
+        'tool name': 'listReports',
+        description: 'Provide basics about all available reports.',
+        method: 'GET',
+        URL: `${thisHost}/api/listReports`,
+      }
+    },
+    'URLs of similar requests for web users': {
+      'this request': `${thisHost}/reportIssues.html/${timeStamp}/${jobID}`,
+      'closest ancestor request': `${thisHost}/targets.html`
+    },
+    'response metadata': getResponseMetadata(),
+    'response content': responseContent
+  };
+  // Return it.
+  return body;
+};

package/api/listReports.js

@@ -0,0 +1,87 @@
+/*
+  listReports.js
+  Returns basics about all the available reports.
+*/
+
+// IMPORTS
+
+const {getReportBasics, getResponseMetadata, getToolsFacts, thisHost} = require('./util');
+const {getReportExtracts} = require('../util');
+
+// FUNCTIONS
+
+// Returns the response body.
+exports.response = async () => {
+  // Initialize the response content.
+  const responseContent = {
+    'basics about all available reports': null,
+    'how to request that a page with no report be tested': null,
+    'how a web user can request that the page be tested': null
+  };
+  // Initialize an array of basics about the reports.
+  const reportsBasics = [];
+  // Get extracts of all available reports.
+  const reportExtracts = await getReportExtracts();
+  // For each report:
+  for (const extract of reportExtracts) {
+    const {error, jobID, timeStamp} = extract;
+    // Get the basics about it (which may be only an error message).
+    const reportBasics = await getReportBasics(timeStamp, jobID);
+    // If this succeeded:
+    if (!error) {
+      // Add instructions for getting details to the basics.
+      reportBasics['how to get details about the report'] = {
+        method: 'GET',
+        URL: `${thisHost}/api/listIssues/${timeStamp}/${jobID}`
+      };
+      reportBasics['web users can get details about the report at'] = `${thisHost}/reportIssues.html/${timeStamp}/${jobID}`;
+      // Add the basics to the array.
+      reportsBasics.push(reportBasics);
+    }
+  }
+  // Sort the array by page description and secondarily by completion recency.
+  reportsBasics.sort((a, b) => {
+    if (a['tested web page'].description !== b['tested web page'].description) {
+      return a['tested web page']
+      .description
+      .localeCompare(b['tested web page'].description, 'en', {sensitivity: 'base'});
+    }
+    return a['completion date and time'].localeCompare(b['completion date and time']);
+  });
+  // Add the sorted basics about the reports to the response content.
+  responseContent['basics about all available reports'] = reportsBasics;
+  // Add instructions for requesting a test to the response content.
+  responseContent['how to request that a page with no report be tested'] = {
+    method: 'POST',
+    URL: `${thisHost}/api/requestTest`,
+    'request body': {
+      description: '10- to 100-character description of the page conforming to the naming convention used in this list of reports',
+      URL: '12- to 300-character URL of the page, including the https:// scheme and any query',
+      reason: '20- to 100-character reason why the page should be tested'
+    },
+    'how to check whether the request has been fulfilled': 'use this listReports tool to determine whether a report about the page has become available (typical wait time: 1 hour to 1 day)'
+  };
+  // Add instructions for a web user to request a test to the response content.
+  responseContent['how a web user can request that the page be tested'] = {
+    URL: `${thisHost}/testRecForm.html`
+  };
+  // Create a response body.
+  const body = {
+    'tool collection': getToolsFacts(),
+    'tool name': 'listReports',
+    'this request': {
+      description: 'Provide basics about all available reports.',
+      method: 'GET',
+      URL: `${thisHost}/api/listReports`,
+      'closest ancestor request': null
+    },
+    'URLs of similar requests for web users': {
+      'this request': `${thisHost}/targets.html`,
+      'closest ancestor request': null
+    },
+    'response metadata': getResponseMetadata(),
+    'response content': responseContent
+  };
+  // Return it.
+  return body;
+};